package main import ( "bufio" "context" "crypto/rand" "encoding/hex" "encoding/json" "errors" "fmt" "io" "math/big" "os" "os/signal" "syscall" "time" rooteway "git.loyso.art/frx/eway" "git.loyso.art/frx/eway/cmd/cli/commands" "git.loyso.art/frx/eway/cmd/cli/components" "git.loyso.art/frx/eway/internal/crypto" "git.loyso.art/frx/eway/internal/entity" "git.loyso.art/frx/eway/internal/interconnect/eway" "github.com/rodaine/table" "github.com/rs/zerolog" "github.com/urfave/cli/v3" ) const ( defaultItemType = "Электрика" ) type empty entity.Empty func main() { ctx, cancel := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) defer func() { cancel() }() err := runcli(ctx) if err != nil { fmt.Fprintf(os.Stderr, "unable to handle app: %v\n", err) os.Exit(1) } os.Exit(0) } func runcli(ctx context.Context) (err error) { app := setupCLI() return app.Run(ctx, os.Args) } func setupDI() cli.BeforeFunc { return func(ctx context.Context, cmd *cli.Command) error { if out := cmd.String("output"); out != "" { var err error cmd.Writer, err = os.Create(out) if err != nil { return fmt.Errorf("setting writer: %w", err) } } cfgpath := cmd.String("config") if cfgpath == "" { return errors.New("no config path provided") } debugLevel := cmd.Bool("verbose") jsonFormat := cmd.Bool("json") err := components.SetupDI(ctx, cfgpath, debugLevel, jsonFormat) if err != nil { return fmt.Errorf("setting up di: %w", err) } return nil } } func releaseDI() cli.AfterFunc { return func(ctx context.Context, c *cli.Command) error { if f, ok := c.Writer.(*os.File); ok { err := f.Close() if err != nil { println("unable to close output file:", err.Error()) } } return components.Shutdown() } } func setupCLI() *cli.Command { app := &cli.Command{ Name: "ewaycli", Description: "a cli for running eway logic", Version: fmt.Sprintf("%s (%s) %s", rooteway.Version(), rooteway.Commit(), rooteway.BuildTime()), Flags: []cli.Flag{ &cli.StringFlag{ Name: "config", Usage: "path to config in TOML format", Value: "config.toml", Sources: cli.NewValueSourceChain( cli.EnvVar("EWAY_CONFIG"), ), TakesFile: true, }, &cli.BoolFlag{ Name: "verbose", Aliases: []string{"d"}, Usage: "enables verbose logging", }, &cli.BoolFlag{ Name: "json", Usage: "enables json log format", Persistent: true, }, &cli.StringFlag{ Name: "output", Aliases: []string{"o"}, Usage: "Defines output for commands", TakesFile: true, }, }, Before: setupDI(), After: releaseDI(), Commands: []*cli.Command{ commands.CategoriesCommandTree(), commands.ItemsCommandTree(), commands.ExportCommandTree(), newCryptoCmd(), newParseCmd(), newImportCmd(), }, } return app } func newCryptoCmd() *cli.Command { return &cli.Command{ Name: "crypto", Usage: "methods for encrypt/decrypt various things", Commands: []*cli.Command{ newCryptoEncyptCmd(), newCryptoDecryptCmd(), }, } } func newCryptoEncyptCmd() *cli.Command { return &cli.Command{ Name: "encrypt", Usage: "encypt incoming text", Flags: []cli.Flag{ &cli.StringFlag{ Name: "text", Aliases: []string{"t"}, Required: true, }, }, Action: cryptoDeEncryptAction(true), } } func newParseCmd() *cli.Command { return &cli.Command{ Name: "parse", Usage: "category for parsing items from various sources", Commands: []*cli.Command{ newParseEwayCmd(), }, } } func newParseEwayCmd() *cli.Command { return &cli.Command{ Name: "eway", Usage: "parse all available eway goods", Commands: []*cli.Command{ newParseEwayGetCmd(), newParseEwayListCmd(), newParseEwayDumpCmd(), }, } } func newParseEwayGetCmd() *cli.Command { return &cli.Command{ Name: "get", Usage: "loads information about the product by parsing product's html page", Flags: []cli.Flag{ &cli.IntFlag{ Name: "cart", Aliases: []string{"c"}, Usage: "cart of the product", Required: true, }, }, Action: decorateAction(parseEwayGetAction), } } func newParseEwayListCmd() *cli.Command { return &cli.Command{ Name: "list", Usage: "parse all available eway goods", Flags: []cli.Flag{ &cli.IntFlag{ Name: "page", Usage: "choose page to load", Value: 1, }, &cli.IntFlag{ Name: "limit", Usage: "limits output", Value: 100, }, &cli.IntFlag{ Name: "min-stock", Usage: "filters by minimum available items in stock", }, &cli.BoolFlag{ Name: "pretty", Usage: "pretty prints output", }, }, Action: decorateAction(parseEwayListAction), } } func newParseEwayDumpCmd() *cli.Command { return &cli.Command{ Name: "dump", Usage: "dumps content of eway catalog inside db", Action: decorateAction(parseEwayDumpAction), } } func newImportCmd() *cli.Command { return &cli.Command{ Name: "import", Usage: "category for importing data from sources", Commands: []*cli.Command{ newImportFromFileCmd(), }, } } func newImportFromFileCmd() *cli.Command { return &cli.Command{ Name: "file", Usage: "imports from file into db", Flags: []cli.Flag{ &cli.StringFlag{ Name: "src", Value: "", Usage: "source of the data.", Required: true, }, }, Action: decorateAction(importFromFileAction), } } func importFromFileAction(ctx context.Context, c *cli.Command) error { const maxBatch = 2000 r, err := components.GetRepository() if err != nil { return fmt.Errorf("getting repository: %w", err) } filesrc := c.String("src") log := zerolog.Ctx(ctx) log.Debug().Str("filepath", filesrc).Msg("importing data from file") productsFile, err := os.Open(filesrc) if err != nil { return fmt.Errorf("opening file: %w", err) } defer func() { errClose := productsFile.Close() if errClose != nil { log.Warn().Err(errClose).Msg("unable to close file") } }() failedItems, err := os.Create("failed.json") if err != nil { log.Warn().Err(err).Msg("unable to open file for failed results") failedItems = os.Stdout } defer func() { if failedItems == os.Stdout { return } errClose := failedItems.Close() log.Err(errClose).Msg("closing file") }() var ( goodsItem entity.GoodsItem goodsItems []entity.GoodsItem failedToInsert int ) seenCategories := make(map[string]empty) categories, err := r.Category().List(ctx) if err != nil { return fmt.Errorf("listing categories: %w", err) } for _, category := range categories { seenCategories[category.Name] = empty{} } bfile := bufio.NewReader(productsFile) for { line, _, err := bfile.ReadLine() if err != nil { if errors.Is(err, io.EOF) { break } return fmt.Errorf("reading line: %w", err) } err = json.Unmarshal(line, &goodsItem) if err != nil { log.Warn().Err(err).Str("line", string(line)).Msg("unable to unmarshal line into item") _, _ = failedItems.Write(line) _, _ = failedItems.Write([]byte{'\n'}) failedToInsert++ continue } goodsItems = append(goodsItems, goodsItem) if _, ok := seenCategories[goodsItem.Type]; ok { continue } if goodsItem.Type == "" { log.Warn().Msg("bad item without proper type") _ = json.NewEncoder(failedItems).Encode(goodsItem) continue } _, err = r.Category().Create(ctx, goodsItem.Type) if err != nil { return fmt.Errorf("unable to create new category: %w", err) } log.Debug().Any("category", goodsItem.Type).Msg("inserted new category") seenCategories[goodsItem.Type] = empty{} } log.Debug().Int("count", len(goodsItems)).Int("failed", failedToInsert).Msg("preparing to upload") start := time.Now() batchSize := int(maxBatch) for i := 0; i < len(goodsItems); i += batchSize { to := i + batchSize if to > len(goodsItems) { to = len(goodsItems) } _, err = r.GoodsItem().UpsertMany(ctx, goodsItems[i:to]...) if err != nil { return fmt.Errorf("upserting items: %w", err) } log.Debug().Int("count", to-i).Msg("inserted batch") } log.Debug().Dur("elapsed", time.Since(start)).Msg("upload finished") return nil } type action func(ctx context.Context, c *cli.Command) error func decorateAction(a action) cli.ActionFunc { return func(ctx context.Context, c *cli.Command) error { var data [3]byte _, _ = rand.Read(data[:]) reqid := hex.EncodeToString(data[:]) log, err := components.GetLogger() if err != nil { return fmt.Errorf("getting logger: %w", err) } log = log.With().Str("reqid", reqid).Logger() rctx := log.WithContext(ctx) start := time.Now() defer func() { log.Info().Float64("elapsed", time.Since(start).Seconds()).Msg("command completed") }() log.Info().Msg("command execution started") return a(rctx, c) } } func parseEwayGetAction(ctx context.Context, cmd *cli.Command) error { cartID := cmd.Int("cart") client, err := components.GetEwayClient() if err != nil { return fmt.Errorf("getting eway client: %w", err) } pi, err := client.GetProductInfo(ctx, cartID) if err != nil { return fmt.Errorf("getting product info: %w", err) } enc := json.NewEncoder(cmd.Writer) enc.SetIndent("", " ") err = enc.Encode(pi) if err != nil { return fmt.Errorf("encoding data: %w", err) } return nil } func parseEwayListAction(ctx context.Context, cmd *cli.Command) error { page := cmd.Int("page") limit := cmd.Int("limit") atLeast := cmd.Int("min-stock") searchInStocks := atLeast > 0 client, err := components.GetEwayClient() if err != nil { return fmt.Errorf("getting eway client: %w", err) } log, err := components.GetLogger() if err != nil { return fmt.Errorf("getting logger: %w", err) } start := (page - 1) * limit items, total, err := client.GetGoodsNew(ctx, eway.GetGoodsNewParams{ Draw: 1, Start: int(start), Length: int(limit), SearchInStocks: searchInStocks, RemmantsAtleast: int(atLeast), }) if err != nil { return fmt.Errorf("getting new goods: %w", err) } productIDs := make([]int, 0, len(items)) for _, item := range items { productIDs = append(productIDs, int(item.Cart)) } remnants, err := client.GetGoodsRemnants(ctx, productIDs) if err != nil { return fmt.Errorf("getting remnants: %w", err) } goodsItems := make([]entity.GoodsItem, 0, len(items)) for _, item := range items { if ctx.Err() != nil { break } pi, err := client.GetProductInfo(ctx, int64(item.Cart)) if err != nil { log.Warn().Err(err).Msg("unable to get product info") } outGood, err := entity.MakeGoodsItem(item, remnants, pi) if err != nil { return fmt.Errorf("making goods item: %w", err) } goodsItems = append(goodsItems, outGood) } var stats = struct { Handled int `json:"handled"` Loaded int `json:"loaded"` Total int `json:"total"` }{ Handled: len(goodsItems), Loaded: len(items), Total: total, } if cmd.Bool("json") { enc := json.NewEncoder(cmd.Writer) if cmd.Bool("pretty") { enc.SetIndent("", " ") _ = enc.Encode(goodsItems) } enc.SetIndent("", "") _ = enc.Encode(stats) } else { tbl := table. New("sku", "category", "cart", "stock", "price", "parameters"). WithWriter(cmd.Writer) for _, outGood := range goodsItems { parameters, _ := json.MarshalIndent(outGood.Parameters, "", " ") tbl.AddRow( outGood.Articul, outGood.Type, outGood.Cart, outGood.Stock, outGood.Price, string(parameters), ) } tbl.Print() table. New("handled", "loaded", "total"). WithWriter(cmd.Writer). AddRow(stats.Handled, stats.Loaded, stats.Total). Print() } return nil } func parseEwayDumpAction(ctx context.Context, cmd *cli.Command) error { client, err := components.GetEwayClient() if err != nil { return fmt.Errorf("getting eway client: %w", err) } repository, err := components.GetRepository() if err != nil { return fmt.Errorf("getting repository: %w", err) } logger, err := components.GetLogger() if err != nil { return fmt.Errorf("getting logger: %w", err) } const batchSize = 100 var i int var start int seenItems, err := entity.IterIntoMap[string, entity.GoodsItem](repository.GoodsItem().List(ctx)).Map(func(gi entity.GoodsItem) (string, error) { return gi.Articul, nil }) if err != nil { return fmt.Errorf("making seen items map: %w", err) } goodsItems := make([]entity.GoodsItem, 0, batchSize) productIDs := make([]int, 0, batchSize) knownCategories := make(map[string]empty) err = entity.IterWithErr(repository.Category().List(ctx)).Do(func(c entity.Category) error { knownCategories[c.Name] = empty{} return nil }) if err != nil { return fmt.Errorf("filling known categories: %w", err) } itemsUpdated := make(map[string]empty, len(seenItems)) stats := struct { fetchedInfo int handledAll int cachedInfo int skippedItem int }{} dimensionDispatcher := makeDefaultDimensionDispatcher() startFrom := time.Now() for { select { case <-ctx.Done(): return ctx.Err() default: } items, total, err := client.GetGoodsNew(ctx, eway.GetGoodsNewParams{ Draw: i, Start: start, Length: batchSize, SearchInStocks: true, RemmantsAtleast: 5, }) if err != nil { logger.Warn().Err(err).Msg("unable to get items from catalog") continue } productIDs = productIDs[:0] for _, item := range items { productIDs = append(productIDs, int(item.Cart)) } remnants, err := client.GetGoodsRemnants(ctx, productIDs) if err != nil { logger.Warn().Err(err).Msg("unable to get goods remnants") return fmt.Errorf("getting goods remnants: %w", err) } goodsItems = goodsItems[:0] for _, item := range items { var pi entity.GoodsItemInfo seenItem := seenItems[item.SKU] if time.Since(seenItem.CreatedAt) < time.Hour*24 { logger.Debug().Str("sku", item.SKU).Msg("skipping item because it's too fresh") stats.skippedItem++ itemsUpdated[item.SKU] = empty{} continue } if len(seenItem.Parameters) != 0 && len(seenItem.PhotoURLs) != 0 { pi.Parameters = seenItem.Parameters pi.PhotoURLs = seenItem.PhotoURLs stats.cachedInfo++ } else { pi, err = client.GetProductInfo(ctx, int64(item.Cart)) if err != nil { logger.Warn().Err(err).Msg("unable to get product info, skipping") continue } stats.fetchedInfo++ } goodsItem, err := entity.MakeGoodsItem(item, remnants, pi) if err != nil { logger.Err(err).Any("item", item).Msg("unable to make goods item") continue } for key, value := range goodsItem.Parameters { dimensionDispatcher.dispatch(ctx, key, value, &goodsItem.Sizes) } itemsUpdated[goodsItem.Articul] = empty{} stats.handledAll++ goodsItems = append(goodsItems, goodsItem) if goodsItem.Type == "" { logger.Warn().Int64("cart_id", goodsItem.Cart).Msg("found item without category, setting default type") goodsItem.Type = defaultItemType } if _, ok := knownCategories[goodsItem.Type]; ok { continue } category, err := repository.Category().Create(ctx, goodsItem.Type) if err != nil { return fmt.Errorf("creating category: %w", err) } logger.Info(). Str("name", category.Name). Int64("id", category.ID). Msg("created new category") knownCategories[goodsItem.Type] = empty{} } _, err = repository.GoodsItem().UpsertMany(ctx, goodsItems...) if err != nil { return fmt.Errorf("upserting items: %w", err) } progressFloat := float64(start) / float64(total) progress := big.NewFloat(progressFloat).Text('f', 3) elapsed := time.Since(startFrom).Seconds() var left int if progressFloat != 0 { left = int(((1 - progressFloat) / progressFloat) * elapsed) } logger.Info(). Int("from", start). Int("to", start+batchSize). Int("total", total). Str("progress", progress). Int("seconds_left", left). Msg("handled next batch items") if len(items) < batchSize { break } start += batchSize i++ } for k := range itemsUpdated { delete(seenItems, k) } logger.Info(). Int("handled", stats.handledAll). Int("cached", stats.cachedInfo). Int("fetched", stats.fetchedInfo). Int("skipped", stats.skippedItem). Int("to_delete", len(seenItems)). Msg("processed items") for k := range seenItems { _, err := repository.GoodsItem().Delete(ctx, k) if err != nil { if errors.Is(err, entity.ErrNotFound) { continue } logger.Warn().Err(err).Str("sku", k).Msg("unable to delete item") continue } logger.Info().Str("sku", k).Msg("deleted item") } return nil } func cryptoDeEncryptAction(encrypt bool) cli.ActionFunc { return func(ctx context.Context, c *cli.Command) (err error) { value := c.String("text") var out string if encrypt { out, err = crypto.Encrypt(value) } else { out, err = crypto.Decrypt(value) } if err != nil { return err } _, err = c.Writer.Write([]byte(out)) if err != nil { return err } _, err = c.Writer.Write([]byte{'\n'}) return err } }