format another way

assets/gooditem.fbs

@@ -15,6 +15,7 @@ table GoodItem {
   cart:long;
   stock:short;
   parameters:string;
+  created_at:long;
 }
 
 table GoodItems {

cmd/cli/components/di.go

@@ -39,7 +39,7 @@ func GetLogger() (zerolog.Logger, error) {
 	return do.Invoke[zerolog.Logger](diInjector)
 }
 
-func SetupDI(ctx context.Context, cfgpath string) error {
+func SetupDI(ctx context.Context, cfgpath string, verbose bool, logAsJSON bool) error {
 	cfg, err := parseSettings(cfgpath)
 	if err != nil {
 		// if no settings provided allow cli to run without them.
@@ -57,9 +57,23 @@ func SetupDI(ctx context.Context, cfgpath string) error {
 			wr.TimeFormat = time.RFC3339
 		}
 
-		log := zerolog.New(zerolog.NewConsoleWriter(tsSet)).With().Timestamp().Str("app", "converter").Logger()
+		var writer io.Writer = zerolog.NewConsoleWriter(tsSet)
+		if logAsJSON {
+			writer = os.Stdout
+		}
 
-		return log, nil
+		log := zerolog.
+			New(writer).
+			With().
+			Timestamp().
+			Str("app", "converter").
+			Logger()
+		if verbose {
+
+			return log.Level(zerolog.DebugLevel), nil
+		}
+
+		return log.Level(zerolog.InfoLevel), nil
 	})
 
 	do.Provide[eway.Client](diInjector, func(i *do.Injector) (eway.Client, error) {

cmd/cli/main.go

@@ -17,32 +17,32 @@ import (
 	"os"
 	"os/signal"
 	"strconv"
+	"strings"
+	"syscall"
 	"time"
 
 	rooteway "git.loyso.art/frx/eway"
 	"git.loyso.art/frx/eway/cmd/cli/components"
 	"git.loyso.art/frx/eway/internal/crypto"
-	"git.loyso.art/frx/eway/internal/encoding/fbs"
 	"git.loyso.art/frx/eway/internal/entity"
 	"git.loyso.art/frx/eway/internal/export"
 	"git.loyso.art/frx/eway/internal/interconnect/eway"
+	"git.loyso.art/frx/eway/internal/matcher"
 
-	"github.com/brianvoe/gofakeit/v6"
 	"github.com/rodaine/table"
 	"github.com/rs/zerolog"
 	"github.com/urfave/cli/v3"
 )
 
 func main() {
-	ctx, cancel := signal.NotifyContext(context.Background(), os.Interrupt)
+	ctx, cancel := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
 	defer func() {
 		cancel()
 	}()
-
 	err := runcli(ctx)
 
 	if err != nil {
-		fmt.Fprintf(os.Stderr, "unable to handle app: %v", err)
+		fmt.Fprintf(os.Stderr, "unable to handle app: %v\n", err)
 		os.Exit(1)
 	}
 
@@ -56,9 +56,19 @@ func runcli(ctx context.Context) (err error) {
 
 func setupDI() cli.BeforeFunc {
 	return func(ctx context.Context, cmd *cli.Command) error {
-		cfgpath := cmd.String("config")
+		if out := cmd.String("output"); out != "" {
+			var err error
+			cmd.Writer, err = os.Create(out)
+			if err != nil {
+				return fmt.Errorf("setting writer: %w", err)
+			}
+		}
 
-		err := components.SetupDI(ctx, cfgpath)
+		cfgpath := cmd.String("config")
+		debugLevel := cmd.Bool("verbose")
+		jsonFormat := cmd.Bool("json")
+
+		err := components.SetupDI(ctx, cfgpath, debugLevel, jsonFormat)
 		if err != nil {
 			return fmt.Errorf("setting up di: %w", err)
 		}
@@ -69,13 +79,20 @@ func setupDI() cli.BeforeFunc {
 
 func releaseDI() cli.AfterFunc {
 	return func(ctx context.Context, c *cli.Command) error {
+		if f, ok := c.Writer.(*os.File); ok {
+			err := f.Close()
+			if err != nil {
+				println("unable to close output file:", err.Error())
+			}
+		}
+
 		return components.Shutdown()
 	}
 }
 
 func setupCLI() *cli.Command {
 	app := &cli.Command{
-		Name:        "cli",
+		Name:        "ewaycli",
 		Description: "a cli for running eway logic",
 		Version:     fmt.Sprintf("%s (%s) %s", rooteway.Version(), rooteway.Commit(), rooteway.BuildTime()),
 		Flags: []cli.Flag{
@@ -85,6 +102,22 @@ func setupCLI() *cli.Command {
 				Value:     "config.toml",
 				TakesFile: true,
 			},
+			&cli.BoolFlag{
+				Name:    "verbose",
+				Aliases: []string{"d"},
+				Usage:   "enables verbose logging",
+			},
+			&cli.BoolFlag{
+				Name:       "json",
+				Usage:      "enables json log format",
+				Persistent: true,
+			},
+			&cli.StringFlag{
+				Name:      "output",
+				Aliases:   []string{"o"},
+				Usage:     "Defines output for commands",
+				TakesFile: true,
+			},
 		},
 
 		Before: setupDI(),
@@ -103,12 +136,6 @@ func setupCLI() *cli.Command {
 	return app
 }
 
-func newVersionCmd() *cli.Command {
-	return &cli.Command{
-		Name: "version",
-	}
-}
-
 func newCryptoCmd() *cli.Command {
 	return &cli.Command{
 		Name:  "crypto",
@@ -209,9 +236,10 @@ func newParseEwayGetCmd() *cli.Command {
 		Usage: "loads information about the product by parsing product's html page",
 		Flags: []cli.Flag{
 			&cli.IntFlag{
-				Name:    "cart",
-				Aliases: []string{"c"},
-				Usage:   "cart of the product",
+				Name:     "cart",
+				Aliases:  []string{"c"},
+				Usage:    "cart of the product",
+				Required: true,
 			},
 		},
 		Action: decorateAction(parseEwayGetAction),
@@ -237,6 +265,10 @@ func newParseEwayListCmd() *cli.Command {
 				Name:  "min-stock",
 				Usage: "filters by minimum available items in stock",
 			},
+			&cli.BoolFlag{
+				Name:  "pretty",
+				Usage: "pretty prints output",
+			},
 		},
 		Action: decorateAction(parseEwayListAction),
 	}
@@ -311,24 +343,6 @@ func newExportYMLCatalogCmd() *cli.Command {
 	}
 }
 
-func newTestCmd(ctx context.Context) *cli.Command {
-	return &cli.Command{
-		Name:  "test",
-		Usage: "various commands for testing",
-		Commands: []*cli.Command{
-			newTestFBSCmd(),
-		},
-	}
-}
-
-func newTestFBSCmd() *cli.Command {
-	return &cli.Command{
-		Name:   "fbs",
-		Usage:  "serialize and deserialize gooditem entity",
-		Action: decorateAction(testFBSAction),
-	}
-}
-
 func newViewCmd() *cli.Command {
 	return &cli.Command{
 		Name:  "view",
@@ -357,10 +371,44 @@ func newViewItemsCmd() *cli.Command {
 		Commands: []*cli.Command{
 			newViewItemsGetCmd(),
 			newViewItemsCountCmd(),
+			newViewItemsUniqueParams(),
+			newViewItemsParamsKnownValues(),
 		},
 	}
 }
 
+func newViewItemsUniqueParams() *cli.Command {
+	return &cli.Command{
+		Name:  "unique-params",
+		Usage: "Show all stored unique param values",
+		Description: "This command iterates over each item and collect keys of params in a dict and then" +
+			" print it to the output. It's useful to find all unique parameters",
+		Action: decorateAction(viewItemsUniqueParamsAction),
+	}
+}
+
+func newViewItemsParamsKnownValues() *cli.Command {
+	return &cli.Command{
+		Name:  "params-values",
+		Usage: "Show all values of requested parameters",
+		Flags: []cli.Flag{
+			&cli.BoolFlag{
+				Name:  "case-insensitive",
+				Usage: "Ignores cases of keys",
+			},
+			&cli.StringSliceFlag{
+				Name:  "regex",
+				Usage: "Registers regex to match",
+			},
+			&cli.BoolFlag{
+				Name:  "keys-only",
+				Usage: "prints only keys",
+			},
+		},
+		Action: decorateAction(viewItemsParamsKnownValuesAction),
+	}
+}
+
 func newViewItemsCountCmd() *cli.Command {
 	return &cli.Command{
 		Name:   "count",
@@ -446,6 +494,155 @@ func viewItemsGetAction(ctx context.Context, c *cli.Command) error {
 	return nil
 }
 
+func viewItemsUniqueParamsAction(ctx context.Context, c *cli.Command) error {
+	repository, err := components.GetRepository()
+	if err != nil {
+		return fmt.Errorf("getting repository: %w", err)
+	}
+
+	knownParams := map[string]struct{}{}
+	iter, err := repository.GoodsItem().ListIter(ctx, 1)
+	if err != nil {
+		return fmt.Errorf("getting list iter: %w", err)
+	}
+	for item := range iter {
+		for k := range item.Parameters {
+			knownParams[k] = struct{}{}
+		}
+	}
+
+	bw := bufio.NewWriter(c.Writer)
+	for paramName := range knownParams {
+		_, err = bw.WriteString(paramName + "\n")
+		if err != nil {
+			return fmt.Errorf("unable to write: %w", err)
+		}
+	}
+
+	return bw.Flush()
+}
+
+type chanIter[T any] struct {
+	in   <-chan T
+	err  error
+	next T
+}
+
+func (i *chanIter[T]) Next() (ok bool) {
+	if i.err != nil {
+		return false
+	}
+
+	i.next, ok = <-i.in
+	if !ok {
+		i.err = errors.New("channel closed")
+	}
+
+	return ok
+}
+
+func (i *chanIter[T]) Get() T {
+	return i.next
+}
+
+func (i *chanIter[T]) Err() error {
+	return i.err
+}
+
+func (i *chanIter[T]) Close() {
+	for range i.in {
+	}
+}
+
+func getItemsIter(ctx context.Context, r entity.GoodsItemRepository) *chanIter[entity.GoodsItem] {
+	in, err := r.ListIter(ctx, 3)
+
+	return &chanIter[entity.GoodsItem]{
+		in:  in,
+		err: err,
+	}
+}
+
+func viewItemsParamsKnownValuesAction(ctx context.Context, c *cli.Command) error {
+	repository, err := components.GetRepository()
+	if err != nil {
+		return fmt.Errorf("getting repository: %w", err)
+	}
+	log, err := components.GetLogger()
+	if err != nil {
+		return fmt.Errorf("getting logger: %w", err)
+	}
+
+	params := c.Args().Slice()
+	opts := make([]matcher.RadixOpt, 0, 1)
+	if c.Bool("case-insensitive") {
+		opts = append(opts, matcher.RadixCaseInsensitive())
+	}
+
+	m := matcher.NewRadix(opts...)
+	for _, param := range params {
+		log.Debug().Str("param", param).Msg("registering param")
+		m.Register(param)
+	}
+	for _, regexp := range c.StringSlice("regex") {
+		log.Debug().Str("regexp", regexp).Msg("registering regexp")
+		m.RegisterRegexp(regexp)
+	}
+
+	requestedValues := make(map[string]map[string]struct{}, len(params))
+	requestedValuesByPattern := make(map[string]map[string]struct{}, len(params))
+	iter := getItemsIter(ctx, repository.GoodsItem())
+	for iter.Next() {
+		item := iter.Get()
+		for k, v := range item.Parameters {
+			matchedPattern := m.MatchByPattern(k)
+			if matchedPattern == "" {
+				continue
+			}
+
+			values, ok := requestedValues[k]
+			if !ok {
+				values = make(map[string]struct{})
+			}
+			values[v] = struct{}{}
+			requestedValues[k] = values
+
+			values, ok = requestedValuesByPattern[matchedPattern]
+			if !ok {
+				values = map[string]struct{}{}
+			}
+			values[v] = struct{}{}
+			requestedValuesByPattern[matchedPattern] = values
+		}
+	}
+
+	bw := bufio.NewWriter(c.Writer)
+	_, _ = bw.WriteString("Matches:\n")
+
+	if c.Bool("keys-only") {
+		for k := range requestedValues {
+			_, _ = bw.WriteString(k + "\n")
+		}
+	} else {
+		for k, v := range requestedValues {
+			_, _ = bw.WriteString(k + ": ")
+			values := make([]string, 0, len(v))
+			for item := range v {
+				values = append(values, strconv.Quote(item))
+			}
+			valuesStr := "[" + strings.Join(values, ",") + "]\n"
+			_, _ = bw.WriteString(valuesStr)
+		}
+	}
+
+	_, _ = bw.WriteString("\nPatterns:\n")
+	for _, pattern := range m.Patterns() {
+		_, _ = bw.WriteString(pattern + "\n")
+	}
+
+	return bw.Flush()
+}
+
 func viewItemsCountAction(ctx context.Context, c *cli.Command) error {
 	r, err := components.GetRepository()
 	if err != nil {
@@ -803,7 +1000,7 @@ func parseEwayListAction(ctx context.Context, cmd *cli.Command) error {
 		return fmt.Errorf("getting logger: %w", err)
 	}
 
-	start := page * limit
+	start := (page - 1) * limit
 
 	items, total, err := client.GetGoodsNew(ctx, eway.GetGoodsNewParams{
 		Draw:            1,
@@ -826,8 +1023,12 @@ func parseEwayListAction(ctx context.Context, cmd *cli.Command) error {
 		return fmt.Errorf("getting remnants: %w", err)
 	}
 
-	tbl := table.New("sku", "category", "cart", "stock", "price", "parameters")
+	goodsItems := make([]entity.GoodsItem, 0, len(items))
 	for _, item := range items {
+		if ctx.Err() != nil {
+			break
+		}
+
 		pi, err := client.GetProductInfo(ctx, int64(item.Cart))
 		if err != nil {
 			log.Warn().Err(err).Msg("unable to get product info")
@@ -837,21 +1038,55 @@ func parseEwayListAction(ctx context.Context, cmd *cli.Command) error {
 			return fmt.Errorf("making goods item: %w", err)
 		}
 
-		parameters, _ := json.MarshalIndent(outGood.Parameters, "", " ")
+		goodsItems = append(goodsItems, outGood)
 
-		tbl.AddRow(
-			outGood.Articul,
-			outGood.Type,
-			outGood.Cart,
-			outGood.Stock,
-			outGood.Price,
-			string(parameters),
-		)
 	}
 
-	tbl.Print()
+	var stats = struct {
+		Handled int `json:"handled"`
+		Loaded  int `json:"loaded"`
+		Total   int `json:"total"`
+	}{
+		Handled: len(goodsItems),
+		Loaded:  len(items),
+		Total:   total,
+	}
+	if cmd.Bool("json") {
+		enc := json.NewEncoder(cmd.Writer)
+		if cmd.Bool("pretty") {
+			enc.SetIndent("", " ")
+			_ = enc.Encode(goodsItems)
+		}
+
+		enc.SetIndent("", "")
+		_ = enc.Encode(stats)
+	} else {
+		tbl := table.
+			New("sku", "category", "cart", "stock", "price", "parameters").
+			WithWriter(cmd.Writer)
+
+		for _, outGood := range goodsItems {
+			parameters, _ := json.MarshalIndent(outGood.Parameters, "", " ")
+
+			tbl.AddRow(
+				outGood.Articul,
+				outGood.Type,
+				outGood.Cart,
+				outGood.Stock,
+				outGood.Price,
+				string(parameters),
+			)
+		}
+
+		tbl.Print()
+
+		table.
+			New("handled", "loaded", "total").
+			WithWriter(cmd.Writer).
+			AddRow(stats.Handled, stats.Loaded, stats.Total).
+			Print()
+	}
 
-	println("total:", total)
 	return nil
 }
 
@@ -899,6 +1134,7 @@ func parseEwayDumpAction(ctx context.Context, cmd *cli.Command) error {
 		fetchedInfo int
 		handledAll  int
 		cachedInfo  int
+		skippedItem int
 	}{}
 
 	startFrom := time.Now()
@@ -934,6 +1170,14 @@ func parseEwayDumpAction(ctx context.Context, cmd *cli.Command) error {
 		for _, item := range items {
 			var pi entity.GoodsItemInfo
 			seenItem := seenItems[item.SKU]
+			if time.Since(seenItem.CreatedAt) < time.Hour*24 {
+				logger.Debug().Str("sku", item.SKU).Msg("skipping item because it's too fresh")
+				stats.skippedItem++
+				itemsUpdated[item.SKU] = struct{}{}
+
+				continue
+			}
+
 			if len(seenItem.Parameters) != 0 && len(seenItem.PhotoURLs) != 0 {
 				pi.Parameters = seenItem.Parameters
 				pi.PhotoURLs = seenItem.PhotoURLs
@@ -1015,6 +1259,7 @@ func parseEwayDumpAction(ctx context.Context, cmd *cli.Command) error {
 		Int("handled", stats.handledAll).
 		Int("cached", stats.cachedInfo).
 		Int("fetched", stats.fetchedInfo).
+		Int("skipped", stats.skippedItem).
 		Int("to_delete", len(seenItems)).
 		Msg("processed items")
 
@@ -1081,31 +1326,6 @@ func goodsItemAsOffer(in entity.GoodsItem, categoryIDByName map[string]int64) (o
 	return out
 }
 
-func testFBSAction(ctx context.Context, c *cli.Command) error {
-	var gooditem entity.GoodsItem
-	err := gofakeit.Struct(&gooditem)
-	if err != nil {
-		return fmt.Errorf("faking struct: %w", err)
-	}
-
-	data := fbs.MakeDomainGoodItemFinished(gooditem)
-	datahexed := hex.EncodeToString(data)
-	println(datahexed)
-
-	got, err := fbs.ParseGoodsItem(data)
-	if err != nil {
-		return fmt.Errorf("parsing: %w", err)
-	}
-
-	if got.Articul != gooditem.Articul {
-		gotStr := fmt.Sprintf("%v", got)
-		hasStr := fmt.Sprintf("%v", gooditem)
-		println(gotStr, "\n", hasStr)
-	}
-
-	return nil
-}
-
 func appYMLExporterAction(ctx context.Context, cmd *cli.Command) error {
 	port := cmd.Int("port")
 	src := cmd.String("src")
@@ -1185,10 +1405,6 @@ func appYMLExporterAction(ctx context.Context, cmd *cli.Command) error {
 	return nil
 }
 
-var (
-	someDumbKey = []byte("9530e001b619e8e98a889055f06821bb")
-)
-
 func cryptoDeEncryptAction(encrypt bool) cli.ActionFunc {
 	return func(ctx context.Context, c *cli.Command) (err error) {
 		value := c.String("text")

config.toml

@@ -13,3 +13,4 @@ _session_id = "19b98ed56cc144f47e040e68dbcd8481"
 _session_user = "1490"
 owner_id = "26476"
 debug = false
+workers_pool = 3

go.mod

@@ -4,6 +4,7 @@ go 1.21.4
 
 require (
 	github.com/BurntSushi/toml v1.3.2
+	github.com/PuerkitoBio/goquery v1.8.1
 	github.com/brianvoe/gofakeit/v6 v6.28.0
 	github.com/dgraph-io/badger/v4 v4.2.0
 	github.com/dgraph-io/ristretto v0.1.1
@@ -16,32 +17,21 @@ require (
 )
 
 require (
-	github.com/PuerkitoBio/goquery v1.8.1 // indirect
 	github.com/andybalholm/cascadia v1.3.1 // indirect
-	github.com/antchfx/htmlquery v1.3.0 // indirect
-	github.com/antchfx/xmlquery v1.3.18 // indirect
-	github.com/antchfx/xpath v1.2.4 // indirect
 	github.com/cespare/xxhash/v2 v2.2.0 // indirect
 	github.com/dustin/go-humanize v1.0.0 // indirect
-	github.com/gobwas/glob v0.2.3 // indirect
-	github.com/gocolly/colly v1.2.0 // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect
 	github.com/golang/glog v1.0.0 // indirect
 	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
 	github.com/golang/protobuf v1.5.2 // indirect
 	github.com/golang/snappy v0.0.3 // indirect
-	github.com/kennygrant/sanitize v1.2.4 // indirect
 	github.com/klauspost/compress v1.12.3 // indirect
 	github.com/mattn/go-colorable v0.1.13 // indirect
 	github.com/mattn/go-isatty v0.0.19 // indirect
 	github.com/pkg/errors v0.9.1 // indirect
-	github.com/saintfish/chardet v0.0.0-20230101081208-5e3ef4b5456d // indirect
-	github.com/temoto/robotstxt v1.1.2 // indirect
 	github.com/xrash/smetrics v0.0.0-20231213231151-1d8dd44e695e // indirect
 	go.opencensus.io v0.22.5 // indirect
 	golang.org/x/net v0.17.0 // indirect
 	golang.org/x/sys v0.13.0 // indirect
-	golang.org/x/text v0.13.0 // indirect
-	google.golang.org/appengine v1.4.0 // indirect
 	google.golang.org/protobuf v1.28.1 // indirect
 )

go.sum

@@ -6,13 +6,6 @@ github.com/PuerkitoBio/goquery v1.8.1 h1:uQxhNlArOIdbrH1tr0UXwdVFgDcZDrZVdcpygAc
 github.com/PuerkitoBio/goquery v1.8.1/go.mod h1:Q8ICL1kNUJ2sXGoAhPGUdYDJvgQgHzJsnnd3H7Ho5jQ=
 github.com/andybalholm/cascadia v1.3.1 h1:nhxRkql1kdYCc8Snf7D5/D3spOX+dBgjA6u8x004T2c=
 github.com/andybalholm/cascadia v1.3.1/go.mod h1:R4bJ1UQfqADjvDa4P6HZHLh/3OxWWEqc0Sk8XGwHqvA=
-github.com/antchfx/htmlquery v1.3.0 h1:5I5yNFOVI+egyia5F2s/5Do2nFWxJz41Tr3DyfKD25E=
-github.com/antchfx/htmlquery v1.3.0/go.mod h1:zKPDVTMhfOmcwxheXUsx4rKJy8KEY/PU6eXr/2SebQ8=
-github.com/antchfx/xmlquery v1.3.18 h1:FSQ3wMuphnPPGJOFhvc+cRQ2CT/rUj4cyQXkJcjOwz0=
-github.com/antchfx/xmlquery v1.3.18/go.mod h1:Afkq4JIeXut75taLSuI31ISJ/zeq+3jG7TunF7noreA=
-github.com/antchfx/xpath v1.2.3/go.mod h1:i54GszH55fYfBmoZXapTHN8T8tkcHfRgLyVwwqzXNcs=
-github.com/antchfx/xpath v1.2.4 h1:dW1HB/JxKvGtJ9WyVGJ0sIoEcqftV3SqIstujI+B9XY=
-github.com/antchfx/xpath v1.2.4/go.mod h1:i54GszH55fYfBmoZXapTHN8T8tkcHfRgLyVwwqzXNcs=
 github.com/brianvoe/gofakeit/v6 v6.28.0 h1:Xib46XXuQfmlLS2EXRuJpqcw8St6qSZz75OUo0tgAW4=
 github.com/brianvoe/gofakeit/v6 v6.28.0/go.mod h1:Xj58BMSnFqcn/fAQeSK+/PLtC5kSb7FJIq4JyGa8vEs=
 github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
@@ -33,17 +26,12 @@ github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4
 github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
 github.com/go-resty/resty/v2 v2.10.0 h1:Qla4W/+TMmv0fOeeRqzEpXPLfTUnR5HZ1+lGs+CkiCo=
 github.com/go-resty/resty/v2 v2.10.0/go.mod h1:iiP/OpA0CkcL3IGt1O0+/SIItFUbkkyw5BGXiVdTu+A=
-github.com/gobwas/glob v0.2.3 h1:A4xDbljILXROh+kObIiy5kIaPYD8e96x1tgBhUI5J+Y=
-github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJAkT8=
-github.com/gocolly/colly v1.2.0 h1:qRz9YAn8FIH0qzgNUw+HT9UN7wm1oF9OBAilwEWpyrI=
-github.com/gocolly/colly v1.2.0/go.mod h1:Hof5T3ZswNVsOHYmba1u03W65HDWgpV5HifSuueE0EA=
 github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
 github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=
 github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=
 github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
 github.com/golang/glog v1.0.0 h1:nfP3RFugxnNRyKgeWd4oI1nYvXpxrx8ck8ZrcizshdQ=
 github.com/golang/glog v1.0.0/go.mod h1:EWib/APOK0SL3dFbYqvxE3UYd8E6s1ouQ7iEp/0LWV4=
-github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6 h1:ZgQEtGgCBiWRM39fZuwSd1LwSqqSW0hOdXCYYDX0R3I=
 github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
 github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE=
 github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
@@ -61,8 +49,6 @@ github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMyw
 github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
 github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
-github.com/kennygrant/sanitize v1.2.4 h1:gN25/otpP5vAsO2djbMhF/LQX6R7+O1TB4yv8NzpJ3o=
-github.com/kennygrant/sanitize v1.2.4/go.mod h1:LGsjYYtgxbetdg5owWB2mpgUL6e2nfw2eObZ0u0qvak=
 github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
 github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
 github.com/klauspost/compress v1.12.3 h1:G5AfA94pHPysR56qqrkO2pxEexdDzrpFJ6yt/VqWxVU=
@@ -85,21 +71,16 @@ github.com/rodaine/table v1.1.1/go.mod h1:iqTRptjn+EVcrVBYtNMlJ2wrJZa3MpULUmcXFp
 github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg=
 github.com/rs/zerolog v1.31.0 h1:FcTR3NnLWW+NnTwwhFWiJSZr4ECLpqCm6QsEnyvbV4A=
 github.com/rs/zerolog v1.31.0/go.mod h1:/7mN4D5sKwJLZQ2b/znpjC3/GQWY/xaDXUM0kKWRHss=
-github.com/saintfish/chardet v0.0.0-20230101081208-5e3ef4b5456d h1:hrujxIzL1woJ7AwssoOcM/tq5JjjG2yYOc8odClEiXA=
-github.com/saintfish/chardet v0.0.0-20230101081208-5e3ef4b5456d/go.mod h1:uugorj2VCxiV1x+LzaIdVa9b4S4qGAcH6cbhh4qVxOU=
 github.com/samber/do v1.6.0 h1:Jy/N++BXINDB6lAx5wBlbpHlUdl0FKpLWgGEV9YWqaU=
 github.com/samber/do v1.6.0/go.mod h1:DWqBvumy8dyb2vEnYZE7D7zaVEB64J45B0NjTlY/M4k=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
 github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
-github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
 github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
 github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
 github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk=
 github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
-github.com/temoto/robotstxt v1.1.2 h1:W2pOjSJ6SWvldyEuiFXNxz3xZ8aiWX5LbfDiOFd7Fxg=
-github.com/temoto/robotstxt v1.1.2/go.mod h1:+1AmkuG3IYkh1kv0d2qEB9Le88ehNO0zwOr3ujewlOo=
 github.com/urfave/cli/v3 v3.0.0-alpha8 h1:H+qxFPoCkGzdF8KUMs2fEOZl5io/1QySgUiGfar8occ=
 github.com/urfave/cli/v3 v3.0.0-alpha8/go.mod h1:0kK/RUFHyh+yIKSfWxwheGndfnrvYSmYFVeKCh03ZUc=
 github.com/xrash/smetrics v0.0.0-20231213231151-1d8dd44e695e h1:+SOyEddqYF09QP7vr7CgJ1eti3pY9Fn3LHO1M1r/0sI=
@@ -133,7 +114,6 @@ golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwY
 golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
 golang.org/x/net v0.0.0-20210916014120-12bc252f5db8/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
-golang.org/x/net v0.5.0/go.mod h1:DivGGAXEgPSlEBzxGzZI+ZLohi+xUj054jfeKui00ws=
 golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
 golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
 golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg=
@@ -160,7 +140,6 @@ golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBc
 golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20221010170243-090e33056c14/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.4.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
@@ -169,7 +148,6 @@ golang.org/x/sys v0.13.0 h1:Af8nKPmuFypiUBjVoU9V20FiaFXOcuZI21p0ycVYYGE=
 golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
-golang.org/x/term v0.4.0/go.mod h1:9P2UbLfCdcvo3p/nzKvsmas4TnlujnuoV9hGgYzW1lQ=
 golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
 golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo=
 golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U=
@@ -177,10 +155,8 @@ golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
-golang.org/x/text v0.6.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
 golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
 golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
-golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k=
 golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
 golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4=
 golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
@@ -198,7 +174,6 @@ golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8T
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
-google.golang.org/appengine v1.4.0 h1:/wp5JvzpHIxhs/dumFmF7BXTf3Z+dd4uXta4kVyO508=
 google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
 google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
 google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=

internal/config/eway.go

@@ -7,4 +7,5 @@ type Eway struct {
 	SessionUser string `toml:"session_user"`
 	OwnerID     string `toml:"owner_id"`
 	Debug       bool   `toml:"debug"`
+	WorkersPool int    `toml:"workers_pool"`
 }

internal/encoding/fbs/GoodItem.go

@@ -177,8 +177,20 @@ func (rcv *GoodItem) Parameters() []byte {
 	return nil
 }
 
+func (rcv *GoodItem) CreatedAt() int64 {
+	o := flatbuffers.UOffsetT(rcv._tab.Offset(32))
+	if o != 0 {
+		return rcv._tab.GetInt64(o + rcv._tab.Pos)
+	}
+	return 0
+}
+
+func (rcv *GoodItem) MutateCreatedAt(n int64) bool {
+	return rcv._tab.MutateInt64Slot(32, n)
+}
+
 func GoodItemStart(builder *flatbuffers.Builder) {
-	builder.StartObject(14)
+	builder.StartObject(15)
 }
 func GoodItemAddSku(builder *flatbuffers.Builder, sku flatbuffers.UOffsetT) {
 	builder.PrependUOffsetTSlot(0, flatbuffers.UOffsetT(sku), 0)
@@ -222,6 +234,9 @@ func GoodItemAddStock(builder *flatbuffers.Builder, stock int16) {
 func GoodItemAddParameters(builder *flatbuffers.Builder, parameters flatbuffers.UOffsetT) {
 	builder.PrependUOffsetTSlot(13, flatbuffers.UOffsetT(parameters), 0)
 }
+func GoodItemAddCreatedAt(builder *flatbuffers.Builder, createdAt int64) {
+	builder.PrependInt64Slot(14, createdAt, 0)
+}
 func GoodItemEnd(builder *flatbuffers.Builder) flatbuffers.UOffsetT {
 	return builder.EndObject()
 }

internal/encoding/fbs/helpers.go

@@ -6,6 +6,7 @@ import (
 	"fmt"
 	"strings"
 	"sync"
+	"time"
 
 	"git.loyso.art/frx/eway/internal/entity"
 
@@ -98,6 +99,7 @@ func makeDomainGoodItem(builder *flatbuffers.Builder, in entity.GoodsItem) flatb
 	GoodItemAddCart(builder, int64(in.Cart))
 	GoodItemAddStock(builder, int16(in.Stock))
 	GoodItemAddParameters(builder, parameters)
+	GoodItemAddCreatedAt(builder, in.CreatedAt.Unix())
 
 	return GoodItemEnd(builder)
 }
@@ -134,6 +136,11 @@ func ParseGoodsItem(data []byte) (item entity.GoodsItem, err error) {
 		}
 	}
 
+	createdAt := itemFBS.CreatedAt()
+	if createdAt > 0 {
+		item.CreatedAt = time.Unix(createdAt, 0)
+	}
+
 	return item, nil
 }
 

internal/entity/gooditem.go

@@ -4,6 +4,7 @@ import (
 	"fmt"
 	"strconv"
 	"strings"
+	"time"
 	"unicode"
 )
 
@@ -22,6 +23,7 @@ type GoodsItem struct {
 	Cart        int64             `json:"cart"`
 	Stock       int               `json:"stock"`
 	Parameters  map[string]string `json:"parameters"`
+	CreatedAt   time.Time         `json:"created_at"`
 }
 
 type GoodsItemRaw struct {

internal/interconnect/eway/client.go

@@ -11,14 +11,15 @@ import (
 	"reflect"
 	"strconv"
 	"strings"
+	"sync"
 	"time"
 
 	"git.loyso.art/frx/eway/internal/config"
 	"git.loyso.art/frx/eway/internal/crypto"
 	"git.loyso.art/frx/eway/internal/entity"
 
+	"github.com/PuerkitoBio/goquery"
 	"github.com/go-resty/resty/v2"
-	"github.com/gocolly/colly"
 	"github.com/rs/zerolog"
 )
 
@@ -35,33 +36,50 @@ type client struct {
 	http *resty.Client
 	log  zerolog.Logger
 
-	ownerID string
+	htmlParseSema     chan struct{}
+	releaseSemaDelay  time.Duration
+	getProductInfoBus chan getProductInfoRequest
+	ownerID           string
+	workersPool       int
+	workerswg         *sync.WaitGroup
 }
 
 type Config config.Eway
 
-func New(ctx context.Context, cfg Config, log zerolog.Logger) (client, error) {
+func New(ctx context.Context, cfg Config, log zerolog.Logger) (*client, error) {
 	httpclient := resty.New().
 		SetDebug(cfg.Debug).
 		SetBaseURL("https://eway.elevel.ru/api")
 
 	c := client{
-		http: httpclient,
-		log:  log.With().Str("client", "eway").Logger(),
+		http:              httpclient,
+		log:               log.With().Str("client", "eway").Logger(),
+		htmlParseSema:     make(chan struct{}, 2),
+		getProductInfoBus: make(chan getProductInfoRequest),
+		releaseSemaDelay:  time.Second / 2,
+		workerswg:         &sync.WaitGroup{},
+	}
+
+	for i := 0; i < cfg.WorkersPool; i++ {
+		c.workerswg.Add(1)
+		go func() {
+			defer c.workerswg.Done()
+			c.productInfoWorker(ctx, c.getProductInfoBus)
+		}()
 	}
 
 	if cfg.SessionID == "" || cfg.SessionUser == "" {
 		if cfg.Login == "" || cfg.Password == "" {
-			return client{}, entity.SimpleError("no auth method provided")
+			return nil, entity.SimpleError("no auth method provided")
 		}
 
 		decryptedPassword, err := crypto.Decrypt(cfg.Password)
 		if err != nil {
-			return client{}, fmt.Errorf("decrypting password: %w", err)
+			return nil, fmt.Errorf("decrypting password: %w", err)
 		}
 		err = c.login(ctx, cfg.Login, decryptedPassword)
 		if err != nil {
-			return client{}, err
+			return nil, err
 		}
 
 		log.Info().Msg("login successful")
@@ -83,10 +101,10 @@ func New(ctx context.Context, cfg Config, log zerolog.Logger) (client, error) {
 
 		c.http.SetCookies(cookies)
 	} else {
-		return client{}, entity.SimpleError("bad configuration: either session_id and session_user should be set or login and password")
+		return nil, entity.SimpleError("bad configuration: either session_id and session_user should be set or login and password")
 	}
 
-	return c, nil
+	return &c, nil
 }
 
 type GetGoodsNewParams struct {
@@ -106,8 +124,6 @@ type getGoodsNewResponse struct {
 	Replacement     bool    `json:"replacement"`
 }
 
-type goodRemnant [4]int
-
 func parseGoodItem(items []any) (out entity.GoodsItemRaw) {
 	valueOf := reflect.ValueOf(&out).Elem()
 	typeOf := valueOf.Type()
@@ -162,7 +178,7 @@ func mapResponseByOrder(response getGoodsNewResponse) (items []entity.GoodsItemR
 	return items
 }
 
-func (c client) GetGoodsRemnants(
+func (c *client) GetGoodsRemnants(
 	ctx context.Context,
 	productIDs []int,
 ) (out entity.MappedGoodsRemnants, err error) {
@@ -175,12 +191,12 @@ func (c client) GetGoodsRemnants(
 		productsStr = append(productsStr, strconv.Itoa(sku))
 	}
 
-	resp, err := c.http.R().
+	req := c.http.R().
 		SetFormData(map[string]string{
 			"products": strings.Join(productsStr, ","),
 		}).
-		SetDoNotParseResponse(true).
-		Post("/goods_remnants")
+		SetDoNotParseResponse(true)
+	resp, err := c.do(ctx, "GetGoodsRemnants", req, resty.MethodPost, "/goods_remnants")
 	if err != nil {
 		return nil, fmt.Errorf("getting goods new: %w", err)
 	}
@@ -209,27 +225,41 @@ func (c client) GetGoodsRemnants(
 	return out, nil
 }
 
-func (c client) GetGoodsNew(
+func (c *client) GetGoodsNew(
 	ctx context.Context,
 	params GetGoodsNewParams,
 ) (items []entity.GoodsItemRaw, total int, err error) {
 	var response getGoodsNewResponse
-	resp, err := c.http.R().
-		SetFormData(map[string]string{
-			"draw":             strconv.Itoa(params.Draw),
-			"start":            strconv.Itoa(params.Start),
-			"length":           strconv.Itoa(params.Length),
-			"order[0][column]": "14",
-			"order[0][dir]":    "desc",
-			"search[value]":    "",
-			"search[regex]":    "false",
-			"search_in_stocks": "on",
-			"remnants_atleast": "5",
-		}).
+	formData := map[string]string{
+		"draw":             strconv.Itoa(params.Draw),
+		"start":            strconv.Itoa(params.Start),
+		"length":           strconv.Itoa(params.Length),
+		"order[0][column]": "14",
+		"order[0][dir]":    "desc",
+		"search[value]":    "",
+		"search[regex]":    "false",
+	}
+	if params.SearchInStocks {
+		stocksNum := strconv.Itoa(params.RemmantsAtleast)
+		formData["search_in_stocks"] = "on"
+		formData["remnants_atleast"] = stocksNum
+	}
+
+	c.log.Debug().
+		Int("remnants", params.RemmantsAtleast).
+		Bool("search_in_stocks", params.SearchInStocks).
+		Int("draw", params.Draw).
+		Int("start", params.Start).
+		Int("length", params.Length).
+		Msg("sending request")
+
+	req := c.http.R().
+		SetFormData(formData).
 		SetQueryParam("category_id", "0").
 		SetQueryParam("own", c.ownerID). // user id?
-		SetDoNotParseResponse(true).
-		Post("/goods_new")
+		SetDoNotParseResponse(true)
+
+	resp, err := c.do(ctx, "GetGoodsNew", req, resty.MethodPost, "/goods_new")
 	if err != nil {
 		return nil, -1, fmt.Errorf("getting goods new: %w", err)
 	}
@@ -239,7 +269,6 @@ func (c client) GetGoodsNew(
 			c.log.Error().Err(err).Msg("unable to close body")
 		}
 	}()
-
 	if resp.IsError() {
 		return nil, -1, errors.New("request was not successful")
 	}
@@ -252,13 +281,15 @@ func (c client) GetGoodsNew(
 	return mapResponseByOrder(response), response.RecordsTotal, nil
 }
 
-func (c client) login(ctx context.Context, user, pass string) error {
-	resp, err := c.http.R().
+func (c *client) login(ctx context.Context, user, pass string) error {
+	req := c.http.R().
 		SetDoNotParseResponse(true).
 		SetFormData(map[string]string{
 			"username": user,
 			"password": pass,
-		}).Post("https://eway.elevel.ru/")
+		})
+
+	resp, err := c.do(ctx, "login", req, resty.MethodPost, "https://eway.elevel.ru/")
 	if err != nil {
 		return fmt.Errorf("sending request: %w", err)
 	}
@@ -272,63 +303,159 @@ func (c client) login(ctx context.Context, user, pass string) error {
 	return nil
 }
 
-type ProductInfo struct {
-	ImageLinks []string
-	Parameters map[string]string
+func (c *client) do(ctx context.Context, name string, req *resty.Request, method string, url string) (resp *resty.Response, err error) {
+	resp, err = req.
+		EnableTrace().
+		Execute(method, url)
+
+	traceInfo := resp.Request.TraceInfo()
+	c.log.Debug().
+		Str("name", name).
+		Str("path", url).
+		Str("method", method).
+		Float64("elapsed", traceInfo.TotalTime.Seconds()).
+		Float64("response_time", traceInfo.ResponseTime.Seconds()).
+		Int("attempt", traceInfo.RequestAttempt).
+		Bool("success", resp.IsSuccess()).
+		Msg("request processed")
+
+	if err != nil {
+		return nil, fmt.Errorf("executing request: %w", err)
+	}
+
+	return resp, err
 }
 
-type parameterSelector struct {
-	Name  string `selector:"div"`
-	Value string `selector:"div.text-right"`
+func (c *client) GetProductInfo(ctx context.Context, cart int64) (pi entity.GoodsItemInfo, err error) {
+	if c.workersPool == 0 {
+		return c.getProductInfo(ctx, cart)
+	}
+
+	responseBus := make(chan taskResult[entity.GoodsItemInfo], 1)
+	c.getProductInfoBus <- getProductInfoRequest{
+		cartID:   cart,
+		response: responseBus,
+	}
+
+	select {
+	case response := <-responseBus:
+		return response.value, response.err
+	case <-ctx.Done():
+		return pi, ctx.Err()
+	}
 }
 
-func (c client) GetProductInfo(ctx context.Context, cart int64) (pi entity.GoodsItemInfo, err error) {
-	collector := colly.NewCollector(
-		colly.AllowedDomains("eway.elevel.ru"),
-		colly.AllowURLRevisit(),
-	)
+func (c *client) getProductInfo(ctx context.Context, cartID int64) (pi entity.GoodsItemInfo, err error) {
+	reqpath := "https://eway.elevel.ru/product/" + strconv.Itoa(int(cartID)) + "/"
+
+	req := c.http.R().SetDoNotParseResponse(true).AddRetryCondition(func(r *resty.Response, err error) bool {
+		if r.Request.Attempt > 3 {
+			return false
+		}
+		return strings.Contains(err.Error(), "pipe")
+	})
+
+	c.log.Debug().Msg("using go query")
 
 	pi.Parameters = map[string]string{}
-
-	start := time.Now()
+	resp, err := c.do(ctx, "getProductInfo", req, resty.MethodGet, reqpath)
+	if err != nil {
+		return pi, fmt.Errorf("getting product info: %w", err)
+	}
 	defer func() {
-		elapsed := time.Since(start).Seconds()
-		c.log.Info().Float64("elapsed", elapsed).Msg("request processed")
+		errClose := resp.RawBody().Close()
+		if errClose == nil {
+			return
+		}
+
+		if err == nil {
+			err = errClose
+			return
+		}
+
+		c.log.Warn().Err(errClose).Msg("unable to close body")
 	}()
+	if resp.IsError() {
+		return pi, errors.New("request was not successful")
+	}
 
-	collector.OnHTML("body > div.page-container > div.page-content > div.content-wrapper > div.content > div.row > div.col-md-4 > div > div > div:nth-child(6)", func(e *colly.HTMLElement) {
-		e.ForEach("div.display-flex", func(i int, h *colly.HTMLElement) {
-			var s parameterSelector
-			err = h.Unmarshal(&s)
-			if err != nil {
-				c.log.Warn().Err(err).Msg("unable to unmarshal")
-				return
-			}
+	doc, err := goquery.NewDocumentFromReader(resp.RawBody())
+	if err != nil {
+		return pi, fmt.Errorf("makind new document: %w", err)
+	}
 
-			if s.Name == "" || s.Value == "" {
-				c.log.Warn().Msg("got empty key or value, skipping")
-				return
-			}
+	cleanText := func(t string) string {
+		return strings.TrimSuffix(strings.TrimSpace(t), ":")
+	}
 
-			pi.Parameters[s.Name] = s.Value
+	const parametersSelector = "body > div.page-container > div.page-content > div.content-wrapper > div.content > div.row > div.col-md-4 > div > div > div:nth-child(6)"
+	const parametersInnerNode = "div.display-flex"
+	doc.
+		Find(parametersSelector).
+		Find(parametersInnerNode).
+		Each(func(i int, s *goquery.Selection) {
+			name := cleanText(s.Find("div").Eq(0).Text())
+			value := cleanText(s.Find("div.text-right").Text())
+			pi.Parameters[name] = value
 		})
-	})
-	collector.OnHTML("div.gallery_panel", func(h *colly.HTMLElement) {
-		h.ForEach("div.gallery_thumbnail > img", func(i int, h *colly.HTMLElement) {
-			imageURL := h.Attr("src")
 
-			if imageURL == "" {
+	const galleryPanelSelector = "div.gallery_panel"
+	const galleryImageSelector = "div.gallery_thumbnail > img"
+	doc.
+		Find(galleryPanelSelector).
+		Find(galleryImageSelector).
+		Each(func(i int, s *goquery.Selection) {
+			imageURL, ok := s.Attr("src")
+			if !ok || len(imageURL) == 0 {
 				return
 			}
 
 			pi.PhotoURLs = append(pi.PhotoURLs, imageURL)
 		})
-	})
-
-	err = collector.Visit("https://eway.elevel.ru/product/" + strconv.Itoa(int(cart)) + "/")
-	if err != nil {
-		return pi, fmt.Errorf("visiting site: %w", err)
-	}
 
 	return pi, nil
 }
+
+type getProductInfoRequest struct {
+	cartID   int64
+	response chan taskResult[entity.GoodsItemInfo]
+}
+
+type taskResult[T any] struct {
+	value T
+	err   error
+}
+
+func (c *client) productInfoWorker(
+	ctx context.Context,
+	in <-chan getProductInfoRequest,
+) {
+	var req getProductInfoRequest
+	var ok bool
+	for {
+		select {
+		case <-ctx.Done():
+			return
+		case req, ok = <-in:
+			if !ok {
+				return
+			}
+		}
+
+		func() {
+			c.htmlParseSema <- struct{}{}
+			defer func() {
+				go func() {
+					time.Sleep(c.releaseSemaDelay)
+					<-c.htmlParseSema
+				}()
+			}()
+
+			pi, err := c.getProductInfo(ctx, req.cartID)
+			req.response <- taskResult[entity.GoodsItemInfo]{
+				value: pi,
+				err:   err,
+			}
+		}()
+	}
+}

internal/matcher/radix.go

@@ -0,0 +1,171 @@
+package matcher
+
+import (
+	"regexp"
+	"strings"
+)
+
+const radixWildcard = '*'
+
+type radixNode struct {
+	exact bool
+	next  map[rune]*radixNode
+}
+
+type radixMatcher struct {
+	root *radixNode
+
+	caseInsensitive bool
+	saved           []string
+	regexps         []*regexp.Regexp
+}
+
+type RadixOpt func(m *radixMatcher)
+
+func RadixCaseInsensitive() RadixOpt {
+	return func(m *radixMatcher) {
+		m.caseInsensitive = true
+	}
+}
+
+func NewRadix(opts ...RadixOpt) *radixMatcher {
+	m := &radixMatcher{
+		root: &radixNode{
+			next: make(map[rune]*radixNode),
+		},
+	}
+
+	for _, opt := range opts {
+		opt(m)
+	}
+
+	return m
+}
+
+func (r *radixMatcher) MatchByPattern(value string) (pattern string) {
+	originValue := value
+	if r.caseInsensitive {
+		value = strings.ToLower(value)
+	}
+
+	node := r.root
+	lastIdx := len([]rune(value)) - 1
+
+	var sb strings.Builder
+	for i, v := range value {
+		var ok bool
+		if _, ok := node.next[radixWildcard]; ok {
+			_, _ = sb.WriteRune(radixWildcard)
+			return sb.String()
+		}
+
+		node, ok = node.next[v]
+		if !ok {
+			return r.findByRegexp(originValue)
+		}
+
+		sb.WriteRune(v)
+
+		if i != lastIdx {
+			continue
+		}
+
+		if !node.exact {
+			return r.findByRegexp(value)
+		}
+
+		return sb.String()
+	}
+
+	return r.findByRegexp(originValue)
+}
+
+func (r *radixMatcher) Match(value string) bool {
+	originValue := value
+	if r.caseInsensitive {
+		value = strings.ToLower(value)
+	}
+
+	node := r.root
+	lastIdx := len([]rune(value)) - 1
+	for i, v := range value {
+		var ok bool
+		if _, ok = node.next[radixWildcard]; ok {
+			return true
+		}
+
+		node, ok = node.next[v]
+		if !ok {
+			return r.findByRegexp(originValue) != ""
+		}
+
+		if i == lastIdx {
+			return node.exact
+		}
+	}
+
+	return r.findByRegexp(originValue) != ""
+}
+
+func (r *radixMatcher) findByRegexp(value string) (regexpPattern string) {
+	for _, rx := range r.regexps {
+		if rx.MatchString(value) {
+			return rx.String()
+		}
+	}
+
+	return ""
+}
+
+func (r *radixMatcher) RegisterRegexp(regexpPattern string) {
+	pattern := regexp.MustCompile(regexpPattern)
+	r.regexps = append(r.regexps, pattern)
+}
+
+func (r *radixMatcher) Register(pattern string) {
+	if len(pattern) == 0 {
+		panic("unable to handle empty pattern")
+	}
+
+	if r.caseInsensitive {
+		pattern = strings.ToLower(pattern)
+	}
+
+	r.saved = append(r.saved, pattern)
+
+	node := r.root
+	lastIdx := len([]rune(pattern)) - 1
+	for i, v := range pattern {
+		nextNode, ok := node.next[v]
+		if !ok {
+			nextNode = &radixNode{
+				next: make(map[rune]*radixNode),
+			}
+			node.next[v] = nextNode
+		}
+		node = nextNode
+
+		if v == '*' {
+			return
+		}
+
+		if i != lastIdx {
+			continue
+		}
+
+		node.exact = true
+	}
+}
+
+func (r *radixMatcher) Patterns() []string {
+	ownIdx := len(r.saved)
+	out := make([]string, len(r.saved)+len(r.regexps))
+	copy(out, r.saved[:ownIdx])
+
+	for i := 0; i < len(r.regexps); i++ {
+		idx := i + ownIdx
+		out[idx] = r.regexps[i].String()
+	}
+
+	return out
+}

internal/matcher/radix_test.go

@@ -0,0 +1,118 @@
+package matcher_test
+
+import (
+	"testing"
+
+	"git.loyso.art/frx/eway/internal/matcher"
+)
+
+func TestRadixMatcherWithPattern(t *testing.T) {
+	m := matcher.NewRadix()
+	m.Register("aloha")
+	m.Register("hawaii")
+	m.Register("te*")
+
+	var tt = []struct {
+		name    string
+		in      string
+		pattern string
+	}{{
+		name:    "should match exact",
+		in:      "aloha",
+		pattern: "aloha",
+	}, {
+		name:    "should match exact 2",
+		in:      "hawaii",
+		pattern: "hawaii",
+	}, {
+		name:    "should match pattern",
+		in:      "test",
+		pattern: "te*",
+	}, {
+		name: "should not match",
+		in:   "alohe",
+	}, {
+		name: "should not match 2",
+		in:   "whoa",
+	}}
+
+	for _, tc := range tt {
+		tc := tc
+		t.Run(tc.name, func(t *testing.T) {
+			pattern := m.MatchByPattern(tc.in)
+			if pattern != tc.pattern {
+				t.Errorf("expected %s got %s", tc.pattern, pattern)
+			}
+		})
+	}
+}
+
+func TestRadixMatcher(t *testing.T) {
+	m := matcher.NewRadix()
+	m.Register("aloha")
+	m.Register("hawaii")
+	m.Register("te*")
+
+	var tt = []struct {
+		name  string
+		in    string
+		match bool
+	}{{
+		name:  "should match exact",
+		in:    "aloha",
+		match: true,
+	}, {
+		name:  "should match exact 2",
+		in:    "hawaii",
+		match: true,
+	}, {
+		name:  "should match pattern",
+		in:    "test",
+		match: true,
+	}, {
+		name: "should not match",
+		in:   "alohe",
+	}, {
+		name: "should not match 2",
+		in:   "whoa",
+	}}
+
+	for _, tc := range tt {
+		tc := tc
+		t.Run(tc.name, func(t *testing.T) {
+			match := m.Match(tc.in)
+			if match != tc.match {
+				t.Errorf("expected %t got %t", tc.match, match)
+			}
+		})
+	}
+}
+
+func TestRadixMatcherWildcard(t *testing.T) {
+	m := matcher.NewRadix()
+	m.Register("*")
+
+	var tt = []struct {
+		name  string
+		in    string
+		match bool
+	}{{
+		name:  "should match exact",
+		in:    "aloha",
+		match: true,
+	}, {
+		name:  "should match exact 2",
+		in:    "hawaii",
+		match: true,
+	}}
+
+	for _, tc := range tt {
+		tc := tc
+		t.Run(tc.name, func(t *testing.T) {
+			match := m.Match(tc.in)
+			if match != tc.match {
+				t.Errorf("expected %t got %t", tc.match, match)
+			}
+		})
+	}
+}

internal/storage/badger/db.go

@@ -35,10 +35,11 @@ func Open(ctx context.Context, path string, debug bool, log zerolog.Logger) (*ba
 		log: log.With().Str("db", "badger").Logger(),
 	}
 
-	level := badger.INFO
+	level := badger.WARNING
 	if debug {
 		level = badger.DEBUG
 	}
+
 	opts := badger.DefaultOptions(path).
 		WithLogger(bl).
 		WithLoggingLevel(level).

internal/storage/badger/goodsitem.go

@@ -7,6 +7,7 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
+	"time"
 	"unsafe"
 
 	"git.loyso.art/frx/eway/internal/encoding/fbs"
@@ -248,18 +249,20 @@ func (c *goodsItemClient) upsertByBatch(ctx context.Context, items []entity.Good
 	batch := c.db.NewWriteBatch()
 	defer batch.Cancel()
 
+	createdAt := time.Now()
 	err := func() error {
 		for _, item := range items {
 			if ctx.Err() != nil {
 				return ctx.Err()
 			}
 
-			key := c.prefixedStr(item.Articul)
+			item.CreatedAt = createdAt
 			value, err := c.s.Serialize(item)
 			if err != nil {
 				return fmt.Errorf("serializing item: %w", err)
 			}
 
+			key := c.prefixedStr(item.Articul)
 			idxValue := make([]byte, len(key))
 			copy(idxValue, key)