frx/eway
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

add export as yml_catalog

Browse Source
This commit is contained in:
Aleksandr Trushkin
2024-01-26 19:34:47 +03:00
parent a1e767217b
commit a0b36ba83d
7 changed files with 371 additions and 57 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
.gitignore vendored
View File

@ -2,3 +2,4 @@
*.zst
database
bin
*.xml

0
cmd/converter/components/di.go → cmd/cli/components/di.go
View File

278
cmd/converter/main.go → cmd/cli/main.go
View File

@ -6,17 +6,21 @@ import (
"crypto/rand"
"encoding/hex"
"encoding/json"
"encoding/xml"
"errors"
"fmt"
"io"
"os"
"os/signal"
"strconv"
"time"
"git.loyso.art/frx/eway/cmd/converter/components"
"git.loyso.art/frx/eway/cmd/cli/components"
"git.loyso.art/frx/eway/internal/encoding/fbs"
"git.loyso.art/frx/eway/internal/entity"
"git.loyso.art/frx/eway/internal/export"
"github.com/brianvoe/gofakeit/v6"
"github.com/rodaine/table"
"github.com/rs/zerolog"
"github.com/urfave/cli"
@ -88,45 +92,8 @@ func setupCLI(ctx context.Context) *cli.App {
app.After = releaseDI
app.Commands = cli.Commands{
newImportCmd(ctx),
newExportCmd(ctx),
newViewCmd(ctx),
cli.Command{
Name: "test-fbs",
Usage: "a simple check for tbs",
Action: cli.ActionFunc(func(c *cli.Context) error {
gooditem := entity.GoodsItem{
Articul: "some-sku",
Photo: "/photo/path.jpg",
Name: "some-name",
Description: "bad-desc",
Category: "",
Type: "some-type",
Producer: "my-producer",
Pack: 123,
Step: 10,
Price: 12.34,
TariffPrice: 43.21,
Cart: 1998,
Stock: 444,
}
data := fbs.MakeDomainGoodItemFinished(gooditem)
datahexed := hex.EncodeToString(data)
println(datahexed)
got, err := fbs.ParseGoodsItem(data)
if err != nil {
return fmt.Errorf("parsing: %w", err)
}
if got != gooditem {
gotStr := fmt.Sprintf("%v", got)
hasStr := fmt.Sprintf("%v", gooditem)
println(gotStr, "\n", hasStr)
}
return nil
}),
},
}
return app
@ -142,6 +109,75 @@ func newImportCmd(ctx context.Context) cli.Command {
}
}
func newImportFromFileCmd(ctx context.Context) cli.Command {
return cli.Command{
Name: "fromfile",
Usage: "imports from file into db",
Flags: []cli.Flag{
&cli.StringFlag{
Name: "src",
Value: "",
Usage: "source of the data.",
Required: true,
},
},
Action: decorateAction(ctx, importFromFileAction),
}
}
func newExportCmd(ctx context.Context) cli.Command {
return cli.Command{
Name: "export",
Usage: "category for exporting stored data",
Subcommands: cli.Commands{
newExportYMLCatalogCmd(ctx),
},
}
}
func newExportYMLCatalogCmd(ctx context.Context) cli.Command {
return cli.Command{
Name: "yml_catalog",
Usage: "export data into as yml_catalog in xml format",
Flags: []cli.Flag{
&cli.StringFlag{
Name: "dst",
Usage: "destination path",
Required: true,
Value: "yml_catalog.xml",
TakesFile: true,
},
&cli.IntFlag{
Name: "limit",
Usage: "limits output offers to required value",
},
&cli.BoolFlag{
Name: "pretty",
Usage: "prettify output",
},
},
Action: decorateAction(ctx, exportYMLCatalogAction),
}
}
func newTestCmd(ctx context.Context) cli.Command {
return cli.Command{
Name: "test",
Usage: "various commands for testing",
Subcommands: cli.Commands{
newTestFBSCmd(ctx),
},
}
}
func newTestFBSCmd(ctx context.Context) cli.Command {
return cli.Command{
Name: "fbs",
Usage: "serialize and deserialize gooditem entity",
Action: decorateAction(ctx, testFBSAction),
}
}
func newViewCmd(ctx context.Context) cli.Command {
return cli.Command{
Name: "view",
@ -224,22 +260,6 @@ func newViewCategoriesListCmd(ctx context.Context) cli.Command {
}
}
func newImportFromFileCmd(ctx context.Context) cli.Command {
return cli.Command{
Name: "fromfile",
Usage: "imports from file into db",
Flags: []cli.Flag{
&cli.StringFlag{
Name: "src",
Value: "",
Usage: "source of the data.",
Required: true,
},
},
Action: decorateAction(ctx, importFromFileAction),
}
}
func viewItemsGetAction(ctx context.Context, c *cli.Context) error {
r, err := components.GetRepository()
if err != nil {
@ -483,3 +503,151 @@ func decorateAction(ctx context.Context, a action) cli.ActionFunc {
return a(ctx, c)
}
}
func exportYMLCatalogAction(ctx context.Context, c *cli.Context) error {
path := c.String("dst")
limit := c.Int("limit")
pretty := c.Bool("pretty")
log, err := components.GetLogger()
if err != nil {
return fmt.Errorf("getting logger: %w", err)
}
r, err := components.GetRepository()
if err != nil {
return fmt.Errorf("getting repository: %w", err)
}
categories, err := r.Category().List(ctx)
if err != nil {
return fmt.Errorf("listing categories: %w", err)
}
shop := export.Shop{
Currencies: []export.Currency{{
ID: "RUR",
Rate: 1,
}},
Categories: make([]export.Category, 0, len(categories)),
}
categoryByNameIdx := make(map[string]int64, len(categories))
for _, category := range categories {
categoryByNameIdx[category.Name] = category.ID
shop.Categories = append(shop.Categories, export.Category{
ID: category.ID,
Name: category.Name,
})
}
itemsIter, err := r.GoodsItem().ListIter(ctx, 1)
if err != nil {
return fmt.Errorf("getting items iterator: %w", err)
}
for item := range itemsIter {
offer := goodsItemAsOffer(item, categoryByNameIdx)
shop.Offers = append(shop.Offers, offer)
}
if err = ctx.Err(); err != nil {
return err
}
f, err := os.Create(path)
if err != nil {
return fmt.Errorf("creating file: %w", err)
}
defer func() {
errClose := f.Close()
if err == nil {
err = errClose
return
}
log.Err(errClose).Msg("file closed or not")
}()
if limit > 0 {
shop.Offers = shop.Offers[:limit]
}
container := export.YmlContainer{
YmlCatalog: export.YmlCatalog{
Shop: shop,
Date: time.Now(),
},
}
_, err = f.Write([]byte(xml.Header))
if err != nil {
return fmt.Errorf("writing header: %w", err)
}
_, err = f.Write([]byte("<!DOCTYPE yml_catalog SYSTEM \"shops.dtd\">\n"))
enc := xml.NewEncoder(f)
if pretty {
enc.Indent("", " ")
}
return enc.Encode(container)
}
func goodsItemAsOffer(in entity.GoodsItem, categoryIDByName map[string]int64) (out export.Offer) {
const defaultType = "vendor.model"
const defaultCurrency = "RUR"
const defaultAvailable = true
const quantityParamName = "Количество на складе «Москва»"
categoryID := categoryIDByName[in.Type]
out = export.Offer{
ID: in.Cart,
Price: int(in.TariffPrice),
CategoryID: categoryID,
PictureURLs: []string{
in.Photo,
},
Model: in.Name,
Vendor: in.Producer,
TypePrefix: in.Name,
Description: in.Description,
ManufacturerWarrany: true,
Params: []export.Param{
{
Name: quantityParamName,
Value: strconv.Itoa(in.Stock),
},
},
Type: defaultType,
CurrencyID: defaultCurrency,
Available: defaultAvailable,
}
return out
}
func testFBSAction(ctx context.Context, c *cli.Context) error {
var gooditem entity.GoodsItem
err := gofakeit.Struct(&gooditem)
if err != nil {
return fmt.Errorf("faking struct: %w", err)
}
data := fbs.MakeDomainGoodItemFinished(gooditem)
datahexed := hex.EncodeToString(data)
println(datahexed)
got, err := fbs.ParseGoodsItem(data)
if err != nil {
return fmt.Errorf("parsing: %w", err)
}
if got != gooditem {
gotStr := fmt.Sprintf("%v", got)
hasStr := fmt.Sprintf("%v", gooditem)
println(gotStr, "\n", hasStr)
}
return nil
}

5
go.mod
View File

@ -3,17 +3,19 @@ module git.loyso.art/frx/eway
go 1.21.4
require (
github.com/BurntSushi/toml v1.3.2
github.com/brianvoe/gofakeit/v6 v6.28.0
github.com/dgraph-io/badger/v4 v4.2.0
github.com/dgraph-io/ristretto v0.1.1
github.com/go-resty/resty/v2 v2.10.0
github.com/google/flatbuffers v23.5.26+incompatible
github.com/rodaine/table v1.1.1
github.com/rs/zerolog v1.31.0
github.com/samber/do v1.6.0
github.com/urfave/cli v1.22.14
)
require (
github.com/BurntSushi/toml v1.3.2 // indirect
github.com/cespare/xxhash/v2 v2.2.0 // indirect
github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect
github.com/dustin/go-humanize v1.0.0 // indirect
@ -27,7 +29,6 @@ require (
github.com/mattn/go-isatty v0.0.19 // indirect
github.com/pkg/errors v0.9.1 // indirect
github.com/russross/blackfriday/v2 v2.1.0 // indirect
github.com/samber/do v1.6.0 // indirect
go.opencensus.io v0.22.5 // indirect
golang.org/x/net v0.17.0 // indirect
golang.org/x/sys v0.13.0 // indirect

2
go.sum
View File

@ -2,6 +2,8 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMT
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
github.com/BurntSushi/toml v1.3.2 h1:o7IhLm0Msx3BaB+n3Ag7L8EVlByGnpq14C4YWiu/gL8=
github.com/BurntSushi/toml v1.3.2/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ=
github.com/brianvoe/gofakeit/v6 v6.28.0 h1:Xib46XXuQfmlLS2EXRuJpqcw8St6qSZz75OUo0tgAW4=
github.com/brianvoe/gofakeit/v6 v6.28.0/go.mod h1:Xj58BMSnFqcn/fAQeSK+/PLtC5kSb7FJIq4JyGa8vEs=
github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44=
github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=

61
internal/export/itemsmarket.go Normal file
View File

@ -0,0 +1,61 @@
package export
import "time"
type Param struct {
Name string `xml:"name,attr"`
Value string `xml:",chardata"`
}
type Offer struct {
ID int64 `xml:"id,attr"`
Type string `xml:"type,attr"`
Available bool `xml:"available,attr"`
URL string `xml:"url"`
Price int `xml:"price"`
CurrencyID string `xml:"currencyId"`
CategoryID int64 `xml:"categoryId"`
PictureURLs []string `xml:"picture"`
Vendor string `xml:"vendor"`
Model string `xml:"model"`
VendorCode int `xml:"vendorCode"`
TypePrefix string `xml:"typePrefix"`
Description string `xml:"description"`
ManufacturerWarrany bool `xml:"manufacturer_warranty"`
Params []Param `xml:"param"`
}
type Currency struct {
ID string `xml:"id,attr"` // RUR only
Rate int64 `xml:"rate,attr"` // 1?
}
type Category struct {
ID int64 `xml:"id,attr"`
ParentID int64 `xml:"parent_id,attr,omiempty"`
Name string `xml:",chardata"`
}
type Shop struct {
Name string `xml:"name"` // r
Company string `xml:"company"` // r
URL string `xml:"url"` // r
Platform string `xml:"platform"`
Version string `xml:"version"`
Currencies []Currency `xml:"currencies"` // r RUR only
Categories []Category `xml:"categories>category"` // r
Offers []Offer `xml:"offer"` // r
}
type YmlContainer struct {
XMLName struct{} `xml:"yml_catalog"`
YmlCatalog
}
type YmlCatalog struct {
Date time.Time `xml:"date,attr"`
Shop Shop `xml:"shop"`
}

81
internal/export/itemsmarket_test.go Normal file
View File

@ -0,0 +1,81 @@
package export
import (
"encoding/xml"
"os"
"testing"
"github.com/brianvoe/gofakeit/v6"
)
func TestYMLSerialize(t *testing.T) {
faker := gofakeit.New(0)
categories := make([]Category, faker.Rand.Intn(4))
knownCategory := map[int]struct{}{}
categoryIDs := make([]int, 0, 10)
for i := range categories {
categories[i].ID = faker.Rand.Int()
categories[i].Name = faker.HipsterWord()
categories[i].ParentID = faker.Rand.Int()
if _, ok := knownCategory[categories[i].ID]; ok {
continue
}
knownCategory[categories[i].ID] = struct{}{}
categoryIDs = append(categoryIDs, categories[i].ID)
}
offers := make([]Offer, faker.Rand.Intn(5)+1)
for i := range offers {
offer := &offers[i]
offer.ID = faker.Int64()
offer.Type = "vendor.model"
offer.Available = true
offer.URL = faker.URL()
offer.Price = int(faker.Price(10, 1000))
offer.CurrencyID = "RUR"
offer.CategoryID = categoryIDs[faker.Rand.Intn(len(categoryIDs))]
for i := 0; i < faker.Rand.Intn(3); i++ {
offer.PictureURLs = append(offer.PictureURLs, faker.ImageURL(128, 128))
}
offer.Vendor = faker.Company()
offer.Model = faker.CarModel()
offer.VendorCode = faker.Rand.Int()
offer.TypePrefix = faker.ProductName()
offer.Description = faker.Sentence(12)
offer.ManufacturerWarrany = true
for i := 0; i < faker.Rand.Intn(8); i++ {
offer.Params = append(offer.Params, Param{
Name: faker.AdjectiveProper(),
Value: faker.Digit(),
})
}
}
var catalog YmlCatalog
catalog.Shop = Shop{
Name: faker.ProductName(),
URL: faker.URL(),
Company: faker.Company(),
Platform: "BSM/Yandex/Market",
Version: faker.AppVersion(),
Currencies: []Currency{{
ID: "RUR",
Rate: 1,
}},
Categories: categories,
Offers: offers,
}
catalog.Date = faker.Date()
container := YmlContainer{
YmlCatalog: catalog,
}
enc := xml.NewEncoder(os.Stdout)
enc.Indent("", " ")
_ = enc.Encode(container)
println()
t.FailNow()
}