From a0b36ba83d6b56eba3b8159a5a8ed63efdc8c735 Mon Sep 17 00:00:00 2001 From: Aleksandr Trushkin Date: Fri, 26 Jan 2024 19:34:47 +0300 Subject: [PATCH] add export as yml_catalog --- .gitignore | 1 + cmd/{converter => cli}/components/di.go | 0 cmd/{converter => cli}/main.go | 278 +++++++++++++++++++----- go.mod | 5 +- go.sum | 2 + internal/export/itemsmarket.go | 61 ++++++ internal/export/itemsmarket_test.go | 81 +++++++ 7 files changed, 371 insertions(+), 57 deletions(-) rename cmd/{converter => cli}/components/di.go (100%) rename cmd/{converter => cli}/main.go (67%) create mode 100644 internal/export/itemsmarket.go create mode 100644 internal/export/itemsmarket_test.go diff --git a/.gitignore b/.gitignore index 13dc1d7..3d9ddc7 100644 --- a/.gitignore +++ b/.gitignore @@ -2,3 +2,4 @@ *.zst database bin +*.xml diff --git a/cmd/converter/components/di.go b/cmd/cli/components/di.go similarity index 100% rename from cmd/converter/components/di.go rename to cmd/cli/components/di.go diff --git a/cmd/converter/main.go b/cmd/cli/main.go similarity index 67% rename from cmd/converter/main.go rename to cmd/cli/main.go index 0683418..4426675 100644 --- a/cmd/converter/main.go +++ b/cmd/cli/main.go @@ -6,17 +6,21 @@ import ( "crypto/rand" "encoding/hex" "encoding/json" + "encoding/xml" "errors" "fmt" "io" "os" "os/signal" + "strconv" "time" - "git.loyso.art/frx/eway/cmd/converter/components" + "git.loyso.art/frx/eway/cmd/cli/components" "git.loyso.art/frx/eway/internal/encoding/fbs" "git.loyso.art/frx/eway/internal/entity" + "git.loyso.art/frx/eway/internal/export" + "github.com/brianvoe/gofakeit/v6" "github.com/rodaine/table" "github.com/rs/zerolog" "github.com/urfave/cli" @@ -88,45 +92,8 @@ func setupCLI(ctx context.Context) *cli.App { app.After = releaseDI app.Commands = cli.Commands{ newImportCmd(ctx), + newExportCmd(ctx), newViewCmd(ctx), - cli.Command{ - Name: "test-fbs", - Usage: "a simple check for tbs", - Action: cli.ActionFunc(func(c *cli.Context) error { - gooditem := entity.GoodsItem{ - Articul: "some-sku", - Photo: "/photo/path.jpg", - Name: "some-name", - Description: "bad-desc", - Category: "", - Type: "some-type", - Producer: "my-producer", - Pack: 123, - Step: 10, - Price: 12.34, - TariffPrice: 43.21, - Cart: 1998, - Stock: 444, - } - - data := fbs.MakeDomainGoodItemFinished(gooditem) - datahexed := hex.EncodeToString(data) - println(datahexed) - - got, err := fbs.ParseGoodsItem(data) - if err != nil { - return fmt.Errorf("parsing: %w", err) - } - - if got != gooditem { - gotStr := fmt.Sprintf("%v", got) - hasStr := fmt.Sprintf("%v", gooditem) - println(gotStr, "\n", hasStr) - } - - return nil - }), - }, } return app @@ -142,6 +109,75 @@ func newImportCmd(ctx context.Context) cli.Command { } } +func newImportFromFileCmd(ctx context.Context) cli.Command { + return cli.Command{ + Name: "fromfile", + Usage: "imports from file into db", + Flags: []cli.Flag{ + &cli.StringFlag{ + Name: "src", + Value: "", + Usage: "source of the data.", + Required: true, + }, + }, + Action: decorateAction(ctx, importFromFileAction), + } +} + +func newExportCmd(ctx context.Context) cli.Command { + return cli.Command{ + Name: "export", + Usage: "category for exporting stored data", + Subcommands: cli.Commands{ + newExportYMLCatalogCmd(ctx), + }, + } +} + +func newExportYMLCatalogCmd(ctx context.Context) cli.Command { + return cli.Command{ + Name: "yml_catalog", + Usage: "export data into as yml_catalog in xml format", + Flags: []cli.Flag{ + &cli.StringFlag{ + Name: "dst", + Usage: "destination path", + Required: true, + Value: "yml_catalog.xml", + TakesFile: true, + }, + &cli.IntFlag{ + Name: "limit", + Usage: "limits output offers to required value", + }, + &cli.BoolFlag{ + Name: "pretty", + Usage: "prettify output", + }, + }, + Action: decorateAction(ctx, exportYMLCatalogAction), + } +} + +func newTestCmd(ctx context.Context) cli.Command { + return cli.Command{ + Name: "test", + Usage: "various commands for testing", + Subcommands: cli.Commands{ + newTestFBSCmd(ctx), + }, + } +} + +func newTestFBSCmd(ctx context.Context) cli.Command { + return cli.Command{ + Name: "fbs", + Usage: "serialize and deserialize gooditem entity", + Action: decorateAction(ctx, testFBSAction), + } +} + func newViewCmd(ctx context.Context) cli.Command { return cli.Command{ Name: "view", @@ -224,22 +260,6 @@ func newViewCategoriesListCmd(ctx context.Context) cli.Command { } } -func newImportFromFileCmd(ctx context.Context) cli.Command { - return cli.Command{ - Name: "fromfile", - Usage: "imports from file into db", - Flags: []cli.Flag{ - &cli.StringFlag{ - Name: "src", - Value: "", - Usage: "source of the data.", - Required: true, - }, - }, - Action: decorateAction(ctx, importFromFileAction), - } -} - func viewItemsGetAction(ctx context.Context, c *cli.Context) error { r, err := components.GetRepository() if err != nil { @@ -483,3 +503,151 @@ func decorateAction(ctx context.Context, a action) cli.ActionFunc { return a(ctx, c) } } + +func exportYMLCatalogAction(ctx context.Context, c *cli.Context) error { + path := c.String("dst") + limit := c.Int("limit") + pretty := c.Bool("pretty") + + log, err := components.GetLogger() + if err != nil { + return fmt.Errorf("getting logger: %w", err) + } + + r, err := components.GetRepository() + if err != nil { + return fmt.Errorf("getting repository: %w", err) + } + + categories, err := r.Category().List(ctx) + if err != nil { + return fmt.Errorf("listing categories: %w", err) + } + + shop := export.Shop{ + Currencies: []export.Currency{{ + ID: "RUR", + Rate: 1, + }}, + Categories: make([]export.Category, 0, len(categories)), + } + + categoryByNameIdx := make(map[string]int64, len(categories)) + for _, category := range categories { + categoryByNameIdx[category.Name] = category.ID + shop.Categories = append(shop.Categories, export.Category{ + ID: category.ID, + Name: category.Name, + }) + } + + itemsIter, err := r.GoodsItem().ListIter(ctx, 1) + if err != nil { + return fmt.Errorf("getting items iterator: %w", err) + } + + for item := range itemsIter { + offer := goodsItemAsOffer(item, categoryByNameIdx) + shop.Offers = append(shop.Offers, offer) + } + if err = ctx.Err(); err != nil { + return err + } + + f, err := os.Create(path) + if err != nil { + return fmt.Errorf("creating file: %w", err) + } + defer func() { + errClose := f.Close() + if err == nil { + err = errClose + return + } + + log.Err(errClose).Msg("file closed or not") + }() + + if limit > 0 { + shop.Offers = shop.Offers[:limit] + } + + container := export.YmlContainer{ + YmlCatalog: export.YmlCatalog{ + Shop: shop, + Date: time.Now(), + }, + } + + _, err = f.Write([]byte(xml.Header)) + if err != nil { + return fmt.Errorf("writing header: %w", err) + } + _, err = f.Write([]byte("\n")) + enc := xml.NewEncoder(f) + if pretty { + enc.Indent("", " ") + } + return enc.Encode(container) +} + +func goodsItemAsOffer(in entity.GoodsItem, categoryIDByName map[string]int64) (out export.Offer) { + const defaultType = "vendor.model" + const defaultCurrency = "RUR" + const defaultAvailable = true + const quantityParamName = "Количество на складе «Москва»" + + categoryID := categoryIDByName[in.Type] + + out = export.Offer{ + ID: in.Cart, + Price: int(in.TariffPrice), + CategoryID: categoryID, + PictureURLs: []string{ + in.Photo, + }, + + Model: in.Name, + Vendor: in.Producer, + TypePrefix: in.Name, + Description: in.Description, + ManufacturerWarrany: true, + Params: []export.Param{ + { + Name: quantityParamName, + Value: strconv.Itoa(in.Stock), + }, + }, + + Type: defaultType, + CurrencyID: defaultCurrency, + Available: defaultAvailable, + } + + return out +} + +func testFBSAction(ctx context.Context, c *cli.Context) error { + var gooditem entity.GoodsItem + err := gofakeit.Struct(&gooditem) + if err != nil { + return fmt.Errorf("faking struct: %w", err) + } + + data := fbs.MakeDomainGoodItemFinished(gooditem) + datahexed := hex.EncodeToString(data) + println(datahexed) + + got, err := fbs.ParseGoodsItem(data) + if err != nil { + return fmt.Errorf("parsing: %w", err) + } + + if got != gooditem { + gotStr := fmt.Sprintf("%v", got) + hasStr := fmt.Sprintf("%v", gooditem) + println(gotStr, "\n", hasStr) + } + + return nil +} diff --git a/go.mod b/go.mod index 140b150..a15fc44 100644 --- a/go.mod +++ b/go.mod @@ -3,17 +3,19 @@ module git.loyso.art/frx/eway go 1.21.4 require ( + github.com/BurntSushi/toml v1.3.2 + github.com/brianvoe/gofakeit/v6 v6.28.0 github.com/dgraph-io/badger/v4 v4.2.0 github.com/dgraph-io/ristretto v0.1.1 github.com/go-resty/resty/v2 v2.10.0 github.com/google/flatbuffers v23.5.26+incompatible github.com/rodaine/table v1.1.1 github.com/rs/zerolog v1.31.0 + github.com/samber/do v1.6.0 github.com/urfave/cli v1.22.14 ) require ( - github.com/BurntSushi/toml v1.3.2 // indirect github.com/cespare/xxhash/v2 v2.2.0 // indirect github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect github.com/dustin/go-humanize v1.0.0 // indirect @@ -27,7 +29,6 @@ require ( github.com/mattn/go-isatty v0.0.19 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/russross/blackfriday/v2 v2.1.0 // indirect - github.com/samber/do v1.6.0 // indirect go.opencensus.io v0.22.5 // indirect golang.org/x/net v0.17.0 // indirect golang.org/x/sys v0.13.0 // indirect diff --git a/go.sum b/go.sum index 0cbea73..f766c36 100644 --- a/go.sum +++ b/go.sum @@ -2,6 +2,8 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMT github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/toml v1.3.2 h1:o7IhLm0Msx3BaB+n3Ag7L8EVlByGnpq14C4YWiu/gL8= github.com/BurntSushi/toml v1.3.2/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= +github.com/brianvoe/gofakeit/v6 v6.28.0 h1:Xib46XXuQfmlLS2EXRuJpqcw8St6qSZz75OUo0tgAW4= +github.com/brianvoe/gofakeit/v6 v6.28.0/go.mod h1:Xj58BMSnFqcn/fAQeSK+/PLtC5kSb7FJIq4JyGa8vEs= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= diff --git a/internal/export/itemsmarket.go b/internal/export/itemsmarket.go new file mode 100644 index 0000000..e6245c2 --- /dev/null +++ b/internal/export/itemsmarket.go @@ -0,0 +1,61 @@ +package export + +import "time" + +type Param struct { + Name string `xml:"name,attr"` + Value string `xml:",chardata"` +} + +type Offer struct { + ID int64 `xml:"id,attr"` + Type string `xml:"type,attr"` + Available bool `xml:"available,attr"` + + URL string `xml:"url"` + Price int `xml:"price"` + CurrencyID string `xml:"currencyId"` + CategoryID int64 `xml:"categoryId"` + PictureURLs []string `xml:"picture"` + Vendor string `xml:"vendor"` + Model string `xml:"model"` + VendorCode int `xml:"vendorCode"` + TypePrefix string `xml:"typePrefix"` + Description string `xml:"description"` + ManufacturerWarrany bool `xml:"manufacturer_warranty"` + Params []Param `xml:"param"` +} + +type Currency struct { + ID string `xml:"id,attr"` // RUR only + Rate int64 `xml:"rate,attr"` // 1? +} + +type Category struct { + ID int64 `xml:"id,attr"` + ParentID int64 `xml:"parent_id,attr,omiempty"` + Name string `xml:",chardata"` +} + +type Shop struct { + Name string `xml:"name"` // r + Company string `xml:"company"` // r + URL string `xml:"url"` // r + Platform string `xml:"platform"` + Version string `xml:"version"` + Currencies []Currency `xml:"currencies"` // r RUR only + Categories []Category `xml:"categories>category"` // r + + Offers []Offer `xml:"offer"` // r +} + +type YmlContainer struct { + XMLName struct{} `xml:"yml_catalog"` + + YmlCatalog +} + +type YmlCatalog struct { + Date time.Time `xml:"date,attr"` + Shop Shop `xml:"shop"` +} diff --git a/internal/export/itemsmarket_test.go b/internal/export/itemsmarket_test.go new file mode 100644 index 0000000..e709599 --- /dev/null +++ b/internal/export/itemsmarket_test.go @@ -0,0 +1,81 @@ +package export + +import ( + "encoding/xml" + "os" + "testing" + + "github.com/brianvoe/gofakeit/v6" +) + +func TestYMLSerialize(t *testing.T) { + faker := gofakeit.New(0) + + categories := make([]Category, faker.Rand.Intn(4)) + knownCategory := map[int]struct{}{} + categoryIDs := make([]int, 0, 10) + for i := range categories { + categories[i].ID = faker.Rand.Int() + categories[i].Name = faker.HipsterWord() + categories[i].ParentID = faker.Rand.Int() + + if _, ok := knownCategory[categories[i].ID]; ok { + continue + } + + knownCategory[categories[i].ID] = struct{}{} + categoryIDs = append(categoryIDs, categories[i].ID) + } + + offers := make([]Offer, faker.Rand.Intn(5)+1) + for i := range offers { + offer := &offers[i] + offer.ID = faker.Int64() + offer.Type = "vendor.model" + offer.Available = true + offer.URL = faker.URL() + offer.Price = int(faker.Price(10, 1000)) + offer.CurrencyID = "RUR" + offer.CategoryID = categoryIDs[faker.Rand.Intn(len(categoryIDs))] + for i := 0; i < faker.Rand.Intn(3); i++ { + offer.PictureURLs = append(offer.PictureURLs, faker.ImageURL(128, 128)) + } + offer.Vendor = faker.Company() + offer.Model = faker.CarModel() + offer.VendorCode = faker.Rand.Int() + offer.TypePrefix = faker.ProductName() + offer.Description = faker.Sentence(12) + offer.ManufacturerWarrany = true + for i := 0; i < faker.Rand.Intn(8); i++ { + offer.Params = append(offer.Params, Param{ + Name: faker.AdjectiveProper(), + Value: faker.Digit(), + }) + } + } + + var catalog YmlCatalog + catalog.Shop = Shop{ + Name: faker.ProductName(), + URL: faker.URL(), + Company: faker.Company(), + Platform: "BSM/Yandex/Market", + Version: faker.AppVersion(), + Currencies: []Currency{{ + ID: "RUR", + Rate: 1, + }}, + Categories: categories, + Offers: offers, + } + catalog.Date = faker.Date() + + container := YmlContainer{ + YmlCatalog: catalog, + } + enc := xml.NewEncoder(os.Stdout) + enc.Indent("", " ") + _ = enc.Encode(container) + println() + t.FailNow() +}