package main import ( "bufio" "context" "crypto/rand" "encoding/hex" "encoding/json" "encoding/xml" "errors" "fmt" "io" "math/big" "os" "os/signal" "strconv" "time" "git.loyso.art/frx/eway/cmd/cli/components" "git.loyso.art/frx/eway/internal/encoding/fbs" "git.loyso.art/frx/eway/internal/entity" "git.loyso.art/frx/eway/internal/export" "git.loyso.art/frx/eway/internal/interconnect/eway" "github.com/brianvoe/gofakeit/v6" "github.com/rodaine/table" "github.com/rs/zerolog" "github.com/urfave/cli/v3" ) func main() { ctx, cancel := signal.NotifyContext(context.Background(), os.Interrupt) defer func() { cancel() }() err := runcli(ctx) if err != nil { fmt.Fprintf(os.Stderr, "unable to handle app: %v", err) os.Exit(1) } os.Exit(0) } func runcli(ctx context.Context) (err error) { app := setupCLI() return app.Run(ctx, os.Args) } func setupDI() cli.BeforeFunc { return func(ctx context.Context, cmd *cli.Command) error { cfgpath := cmd.String("config") err := components.SetupDI(ctx, cfgpath) if err != nil { return fmt.Errorf("setting up di: %w", err) } return nil } } func releaseDI() cli.AfterFunc { return func(ctx context.Context, c *cli.Command) error { log, err := components.GetLogger() if err != nil { return fmt.Errorf("getting logger: %w", err) } start := time.Now() defer func() { since := time.Since(start) if err == nil { return } log.Err(err).Dur("elapsed", since).Msg("shutdown finished") }() return components.Shutdown() } } func setupCLI() *cli.Command { app := &cli.Command{ Name: "cli", Description: "a cli for running eway logic", Flags: []cli.Flag{ &cli.StringFlag{ Name: "config", Usage: "path to config in TOML format", Value: "config.toml", TakesFile: true, }, }, Before: setupDI(), After: releaseDI(), Commands: []*cli.Command{ newParseCmd(), newImportCmd(), newExportCmd(), newViewCmd(), }, } return app } func newParseCmd() *cli.Command { return &cli.Command{ Name: "parse", Usage: "category for parsing items from various sources", Commands: []*cli.Command{ newParseEwayCmd(), }, } } func newParseEwayCmd() *cli.Command { return &cli.Command{ Name: "eway", Usage: "parse all available eway goods", Action: decorateAction(parseEwayAction), } } func newImportCmd() *cli.Command { return &cli.Command{ Name: "import", Usage: "category for importing data from sources", Commands: []*cli.Command{ newImportFromFileCmd(), }, } } func newImportFromFileCmd() *cli.Command { return &cli.Command{ Name: "file", Usage: "imports from file into db", Flags: []cli.Flag{ &cli.StringFlag{ Name: "src", Value: "", Usage: "source of the data.", Required: true, }, }, Action: decorateAction(importFromFileAction), } } func newExportCmd() *cli.Command { return &cli.Command{ Name: "export", Usage: "category for exporting stored data", Commands: []*cli.Command{ newExportYMLCatalogCmd(), }, } } func newExportYMLCatalogCmd() *cli.Command { return &cli.Command{ Name: "yml_catalog", Usage: "export data into as yml_catalog in xml format", Flags: []cli.Flag{ &cli.StringFlag{ Name: "out", Usage: "destination path", Value: "yml_catalog.xml", TakesFile: true, }, &cli.IntFlag{ Name: "limit", Usage: "limits output offers to required value", }, &cli.BoolFlag{ Name: "pretty", Usage: "prettify output", }, }, Action: decorateAction(exportYMLCatalogAction), } } func newTestCmd(ctx context.Context) *cli.Command { return &cli.Command{ Name: "test", Usage: "various commands for testing", Commands: []*cli.Command{ newTestFBSCmd(), }, } } func newTestFBSCmd() *cli.Command { return &cli.Command{ Name: "fbs", Usage: "serialize and deserialize gooditem entity", Action: decorateAction(testFBSAction), } } func newViewCmd() *cli.Command { return &cli.Command{ Name: "view", Usage: "Set of commands to view the data inside db", Commands: []*cli.Command{ newViewCategoriesCmd(), newViewItemsCmd(), }, } } func newViewCategoriesCmd() *cli.Command { return &cli.Command{ Name: "categories", Usage: "Set of commands to work with categories", Commands: []*cli.Command{ newViewCategoriesListCmd(), }, } } func newViewItemsCmd() *cli.Command { return &cli.Command{ Name: "items", Usage: "Set of command to work with items", Commands: []*cli.Command{ newViewItemsGetCmd(), newViewItemsCountCmd(), }, } } func newViewItemsCountCmd() *cli.Command { return &cli.Command{ Name: "count", Usage: "iterates over collection and counts number of items", Action: decorateAction(viewItemsCountAction), } } func newViewItemsGetCmd() *cli.Command { return &cli.Command{ Name: "get", Usage: "gets goods item by its id", Flags: []cli.Flag{ &cli.StringFlag{ Name: "id", Usage: "id of the goods item. Either id or cart-id should be set", }, &cli.IntFlag{ Name: "cart-id", Usage: "cart-id of the item. Either cart-id or id should be set", }, }, Action: decorateAction(viewItemsGetAction), } } func newViewCategoriesListCmd() *cli.Command { return &cli.Command{ Name: "list", Usage: "lists stored categories stored in database", Flags: []cli.Flag{ &cli.IntFlag{ Name: "limit", Usage: "limits output to selected items", Value: 20, }, &cli.IntFlag{ Name: "page", Usage: "in case of limit, selects page", Value: 0, }, &cli.BoolFlag{ Name: "with-total", Usage: "prints total count of categories", }, }, Action: decorateAction(viewCategoriesListAction), } } func viewItemsGetAction(ctx context.Context, c *cli.Command) error { r, err := components.GetRepository() if err != nil { return fmt.Errorf("getting repository: %w", err) } id := c.String("id") cartID := c.Int("cart-id") if id == "" && cartID == 0 { return cli.Exit("oneof: id or cart-id should be set", 1) } else if id != "" && cartID != 0 { return cli.Exit("oneof: id or cart-id should be set", 1) } var item entity.GoodsItem if id != "" { item, err = r.GoodsItem().Get(ctx, id) } else { item, err = r.GoodsItem().GetByCart(ctx, cartID) } if err != nil { return fmt.Errorf("getting item: %w", err) } enc := json.NewEncoder(os.Stdout) enc.SetIndent("", " ") err = enc.Encode(item) if err != nil { return fmt.Errorf("encoding item: %w", err) } return nil } func viewItemsCountAction(ctx context.Context, c *cli.Command) error { r, err := components.GetRepository() if err != nil { return fmt.Errorf("getting repository: %w", err) } itemChan, err := r.GoodsItem().ListIter(ctx, 10) if err != nil { if !errors.Is(err, entity.ErrNotImplemented) { return fmt.Errorf("getting list iter: %w", err) } } var count int if err == nil { var done bool for !done { select { case _, ok := <-itemChan: if !ok { done = true continue } count++ case <-ctx.Done(): return ctx.Err() } } } items, err := r.GoodsItem().List(ctx) if err != nil { return fmt.Errorf("getting list: %w", err) } zerolog.Ctx(ctx).Info().Int("count", count).Int("list_count", len(items)).Msg("read all items") return nil } func viewCategoriesListAction(ctx context.Context, c *cli.Command) error { r, err := components.GetRepository() if err != nil { return fmt.Errorf("getting repository: %w", err) } categories, err := r.Category().List(ctx) if err != nil { return fmt.Errorf("listing categories: %w", err) } limit := int(c.Int("limit")) page := int(c.Int("page")) total := len(categories) if page == 0 { page = 1 } if limit > 0 { offset := (page - 1) * limit if offset > len(categories) { offset = len(categories) - 1 } limit = offset + limit if limit > len(categories) { limit = len(categories) } categories = categories[offset:limit] } tbl := table.New("ID", "Name") for _, category := range categories { if category.ID == 0 && category.Name == "" { continue } tbl.AddRow(category.ID, category.Name) } tbl.Print() if c.Bool("with-total") { zerolog.Ctx(ctx).Info().Int("count", total).Msg("total categories stats") } return nil } func importFromFileAction(ctx context.Context, c *cli.Command) error { const maxBatch = 2000 r, err := components.GetRepository() if err != nil { return fmt.Errorf("getting repository: %w", err) } filesrc := c.String("src") log := zerolog.Ctx(ctx) log.Debug().Str("filepath", filesrc).Msg("importing data from file") productsFile, err := os.Open(filesrc) if err != nil { return fmt.Errorf("opening file: %w", err) } defer func() { errClose := productsFile.Close() if errClose != nil { log.Warn().Err(errClose).Msg("unable to close file") } }() failedItems, err := os.Create("failed.json") if err != nil { log.Warn().Err(err).Msg("unable to open file for failed results") failedItems = os.Stdout } defer func() { if failedItems == os.Stdout { return } errClose := failedItems.Close() log.Err(errClose).Msg("closing file") }() var ( goodsItem entity.GoodsItem goodsItems []entity.GoodsItem failedToInsert int ) seenCategories := make(map[string]struct{}) categories, err := r.Category().List(ctx) if err != nil { return fmt.Errorf("listing categories: %w", err) } for _, category := range categories { seenCategories[category.Name] = struct{}{} } bfile := bufio.NewReader(productsFile) for { line, _, err := bfile.ReadLine() if err != nil { if errors.Is(err, io.EOF) { break } return fmt.Errorf("reading line: %w", err) } err = json.Unmarshal(line, &goodsItem) if err != nil { log.Warn().Err(err).Str("line", string(line)).Msg("unable to unmarshal line into item") _, _ = failedItems.Write(line) _, _ = failedItems.Write([]byte{'\n'}) failedToInsert++ continue } goodsItems = append(goodsItems, goodsItem) if _, ok := seenCategories[goodsItem.Type]; ok { continue } if goodsItem.Type == "" { log.Warn().Msg("bad item without proper type") _ = json.NewEncoder(failedItems).Encode(goodsItem) continue } _, err = r.Category().Create(ctx, goodsItem.Type) if err != nil { return fmt.Errorf("unable to create new category: %w", err) } log.Debug().Any("category", goodsItem.Type).Msg("inserted new category") seenCategories[goodsItem.Type] = struct{}{} } log.Debug().Int("count", len(goodsItems)).Int("failed", failedToInsert).Msg("preparing to upload") start := time.Now() batchSize := int(maxBatch) for i := 0; i < len(goodsItems); i += batchSize { to := i + batchSize if to > len(goodsItems) { to = len(goodsItems) } _, err = r.GoodsItem().UpsertMany(ctx, goodsItems[i:to]...) if err != nil { return fmt.Errorf("upserting items: %w", err) } log.Debug().Int("count", to-i).Msg("inserted batch") } log.Debug().Dur("elapsed", time.Since(start)).Msg("upload finished") return nil } type action func(ctx context.Context, c *cli.Command) error func decorateAction(a action) cli.ActionFunc { return func(ctx context.Context, c *cli.Command) error { var data [3]byte _, _ = rand.Read(data[:]) reqid := hex.EncodeToString(data[:]) log, err := components.GetLogger() if err != nil { return fmt.Errorf("getting logger: %w", err) } log = log.With().Str("reqid", reqid).Logger() ctx = log.WithContext(ctx) start := time.Now() defer func() { log.Info().Dur("elapsed", time.Since(start)).Msg("command completed") }() log.Info().Msg("command execution started") return a(ctx, c) } } func exportYMLCatalogAction(ctx context.Context, c *cli.Command) error { path := c.String("dst") limit := c.Int("limit") pretty := c.Bool("pretty") log, err := components.GetLogger() if err != nil { return fmt.Errorf("getting logger: %w", err) } r, err := components.GetRepository() if err != nil { return fmt.Errorf("getting repository: %w", err) } categories, err := r.Category().List(ctx) if err != nil { return fmt.Errorf("listing categories: %w", err) } shop := export.Shop{ Currencies: []export.Currency{{ ID: "RUR", Rate: 1, }}, Categories: make([]export.Category, 0, len(categories)), } categoryByNameIdx := make(map[string]int64, len(categories)) for _, category := range categories { categoryByNameIdx[category.Name] = category.ID shop.Categories = append(shop.Categories, export.Category{ ID: category.ID, Name: category.Name, }) } itemsIter, err := r.GoodsItem().ListIter(ctx, 1) if err != nil { return fmt.Errorf("getting items iterator: %w", err) } for item := range itemsIter { offer := goodsItemAsOffer(item, categoryByNameIdx) shop.Offers = append(shop.Offers, offer) } if err = ctx.Err(); err != nil { return err } f, err := os.Create(path) if err != nil { return fmt.Errorf("creating file: %w", err) } defer func() { errClose := f.Close() if err == nil { err = errClose return } log.Err(errClose).Msg("file closed or not") }() if limit > 0 { shop.Offers = shop.Offers[:limit] } container := export.YmlContainer{ YmlCatalog: export.YmlCatalog{ Shop: shop, Date: time.Now(), }, } _, err = f.Write([]byte(xml.Header)) if err != nil { return fmt.Errorf("writing header: %w", err) } _, err = f.Write([]byte("\n")) enc := xml.NewEncoder(f) if pretty { enc.Indent("", " ") } return enc.Encode(container) } func parseEwayAction(ctx context.Context, c *cli.Command) error { client, err := components.GetEwayClient() if err != nil { return fmt.Errorf("getting eway client: %w", err) } repository, err := components.GetRepository() if err != nil { return fmt.Errorf("getting repository: %w", err) } logger, err := components.GetLogger() if err != nil { return fmt.Errorf("getting logger: %w", err) } const batchSize = 100 var i int var start int goodsItems := make([]entity.GoodsItem, 0, batchSize) productIDs := make([]int, 0, batchSize) knownCategories := make(map[string]struct{}) err = entity.IterWithErr(repository.Category().List(ctx)).Do(func(c entity.Category) error { knownCategories[c.Name] = struct{}{} return nil }) if err != nil { return fmt.Errorf("filling known categories: %w", err) } startFrom := time.Now() for { select { case <-ctx.Done(): return ctx.Err() default: } items, total, err := client.GetGoodsNew(ctx, eway.GetGoodsNewParams{ Draw: i, Start: start, Length: batchSize, SearchInStocks: true, RemmantsAtleast: 5, }) if err != nil { return fmt.Errorf("getting next goods batch: %w", err) } productIDs = productIDs[:0] for _, item := range items { productIDs = append(productIDs, int(item.Cart)) } remnants, err := client.GetGoodsRemnants(ctx, productIDs) if err != nil { return fmt.Errorf("getting goods remnants: %w", err) } goodsItems = goodsItems[:0] for _, item := range items { goodsItem, err := entity.MakeGoodsItem(item, remnants) if err != nil { logger.Warn().Err(err).Any("item", item).Msg("unable to make goods item") continue } goodsItems = append(goodsItems, goodsItem) if goodsItem.Type == "" { continue } if _, ok := knownCategories[goodsItem.Type]; ok { continue } category, err := repository.Category().Create(ctx, goodsItem.Type) if err != nil { return fmt.Errorf("creating category: %w", err) } logger.Debug(). Str("name", category.Name). Int64("id", category.ID). Msg("created new category") knownCategories[goodsItem.Type] = struct{}{} } _, err = repository.GoodsItem().UpsertMany(ctx, goodsItems...) if err != nil { return fmt.Errorf("upserting items: %w", err) } progressFloat := float64(start) / float64(total) progress := big.NewFloat(progressFloat).Text('f', 3) elapsed := time.Since(startFrom).Seconds() var left int if progressFloat != 0 { left = int(((1 - progressFloat) / progressFloat) * elapsed) } logger.Debug(). Int("from", start). Int("to", start+batchSize). Int("total", total). Str("progress", progress). Int("seconds_left", left). Msg("handled next batch items") if len(items) < batchSize { break } start += batchSize i++ } return nil } func goodsItemAsOffer(in entity.GoodsItem, categoryIDByName map[string]int64) (out export.Offer) { const defaultType = "vendor.model" const defaultCurrency = "RUR" const defaultAvailable = true const quantityParamName = "Количество на складе «Москва»" const basePictureURL = "https://eway.elevel.ru" imgurl := func(path string) string { return basePictureURL + path } categoryID := categoryIDByName[in.Type] out = export.Offer{ ID: in.Cart, VendorCode: in.Articul, Price: int(in.TariffPrice), CategoryID: categoryID, PictureURLs: []string{ imgurl(in.Photo), }, Model: in.Name, Vendor: in.Producer, TypePrefix: in.Name, Description: in.Description, ManufacturerWarrany: true, Params: []export.Param{ { Name: quantityParamName, Value: strconv.Itoa(in.Stock), }, }, Type: defaultType, CurrencyID: defaultCurrency, Available: defaultAvailable, } return out } func testFBSAction(ctx context.Context, c *cli.Command) error { var gooditem entity.GoodsItem err := gofakeit.Struct(&gooditem) if err != nil { return fmt.Errorf("faking struct: %w", err) } data := fbs.MakeDomainGoodItemFinished(gooditem) datahexed := hex.EncodeToString(data) println(datahexed) got, err := fbs.ParseGoodsItem(data) if err != nil { return fmt.Errorf("parsing: %w", err) } if got != gooditem { gotStr := fmt.Sprintf("%v", got) hasStr := fmt.Sprintf("%v", gooditem) println(gotStr, "\n", hasStr) } return nil }