update log creation file

.gitea/workflows/demo.yaml

@@ -0,0 +1,17 @@
+name: Gitea Actions Demo
+run-name: ${{ gitea.actor }} is testing out Gitea Actions 🚀
+on: [push]
+
+jobs:
+  Explore-Gitea-Actions:
+    runs-on: linux-arm64
+    steps:
+      - run: echo "The job was automatically triggered by a ${{ gitea.event_name }} event."
+      - run: echo "This job is now running on a ${{ runner.os }} server hosted by Gitea!"
+      - run: echo "The name of your branch is ${{ gitea.ref }} and your repository is ${{ gitea.repository }}."
+      - run: echo "The ${{ gitea.repository }} repository has been cloned to the runner."
+      - run: echo "The workflow is now ready to test your code on the runner."
+      - name: List files in the repository
+        run: |
+          ls ${{ gitea.workspace }}
+      - run: echo "This job's status is ${{ job.status }}."

.gitignore

@@ -1,2 +1,7 @@
 *.json
 *.zst
+database
+bin
+*.xml
+Makefile
+config.toml

assets/gooditem.fbs

@@ -1,5 +1,11 @@
 namespace internal.encoding.fbs;
 
+struct Dimensions {
+  width:float;
+  height:float;
+  length:float;
+}
+
 table GoodItem {
   sku:string;
   photo:string;
@@ -14,6 +20,9 @@ table GoodItem {
   tariff:float;
   cart:long;
   stock:short;
+  parameters:string;
+  created_at:long;
+  sizes:Dimensions;
 }
 
 table GoodItems {

buildinfo.go

@@ -0,0 +1,36 @@
+package eway
+
+import (
+	"sync"
+	"time"
+)
+
+var (
+	version      string = "v0.0.0"
+	commit       string = "0000000"
+	buildTimeStr string
+
+	buildTime time.Time
+
+	parseOnce sync.Once
+)
+
+func Version() string {
+	return version
+}
+
+func Commit() string {
+	return commit
+}
+
+func BuildTime() time.Time {
+	parseOnce.Do(func() {
+		if buildTimeStr == "" {
+			return
+		}
+
+		buildTime, _ = time.Parse(buildTimeStr, time.RFC3339)
+	})
+
+	return buildTime
+}

cmd/cli/commands/categories.go

@@ -0,0 +1,101 @@
+package commands
+
+import (
+	"context"
+	"fmt"
+
+	"git.loyso.art/frx/eway/cmd/cli/components"
+
+	"github.com/rodaine/table"
+	"github.com/rs/zerolog"
+	"github.com/urfave/cli/v3"
+)
+
+func CategoriesCommandTree() *cli.Command {
+	var h categoriesHandlers
+
+	return &cli.Command{
+		Name:  "categories",
+		Usage: "Interact with stored categories",
+		Commands: []*cli.Command{
+			newCategoriesListCommand(h),
+		},
+	}
+}
+
+func newCategoriesListCommand(h categoriesHandlers) *cli.Command {
+	cmd := cli.Command{
+		Name:  "list",
+		Usage: "List categories, stories in db",
+		Flags: []cli.Flag{
+			&cli.IntFlag{
+				Name:  "limit",
+				Usage: "limits output to selected items",
+				Value: 20,
+			},
+			&cli.IntFlag{
+				Name:  "page",
+				Usage: "in case of limit, selects page",
+				Value: 0,
+			},
+			&cli.BoolFlag{
+				Name:  "with-total",
+				Usage: "prints total count of categories",
+			},
+		},
+	}
+
+	return cmdWithAction(cmd, h.List)
+}
+
+type categoriesHandlers struct{}
+
+func (categoriesHandlers) List(ctx context.Context, c *cli.Command) error {
+	r, err := components.GetRepository()
+	if err != nil {
+		return fmt.Errorf("getting repository: %w", err)
+	}
+
+	categories, err := r.Category().List(ctx)
+	if err != nil {
+		return fmt.Errorf("listing categories: %w", err)
+	}
+
+	limit := int(c.Int("limit"))
+	page := int(c.Int("page"))
+	total := len(categories)
+
+	if page == 0 {
+		page = 1
+	}
+
+	if limit > 0 {
+		offset := (page - 1) * limit
+		if offset > len(categories) {
+			offset = len(categories) - 1
+		}
+
+		limit = offset + limit
+		if limit > len(categories) {
+			limit = len(categories)
+		}
+
+		categories = categories[offset:limit]
+	}
+
+	tbl := table.New("ID", "Name")
+	for _, category := range categories {
+		if category.ID == 0 && category.Name == "" {
+			continue
+		}
+		tbl.AddRow(category.ID, category.Name)
+	}
+
+	tbl.Print()
+
+	if c.Bool("with-total") {
+		zerolog.Ctx(ctx).Info().Int("count", total).Msg("total categories stats")
+	}
+
+	return nil
+}

cmd/cli/commands/channeliter.go

@@ -0,0 +1,55 @@
+package commands
+
+import (
+	"context"
+	"errors"
+
+	"git.loyso.art/frx/eway/internal/entity"
+)
+
+type chanIter[T any] struct {
+	in   <-chan T
+	err  error
+	next T
+}
+
+var errChannelClosed = errors.New("channel closed")
+
+func (i *chanIter[T]) Next() (ok bool) {
+	if i.err != nil {
+		return false
+	}
+
+	i.next, ok = <-i.in
+	if !ok {
+		i.err = errChannelClosed
+	}
+
+	return ok
+}
+
+func (i *chanIter[T]) Get() T {
+	return i.next
+}
+
+func (i *chanIter[T]) Err() error {
+	if errors.Is(i.err, errChannelClosed) {
+		return nil
+	}
+
+	return i.err
+}
+
+func (i *chanIter[T]) Close() {
+	for range i.in {
+	}
+}
+
+func getItemsIter(ctx context.Context, r entity.GoodsItemRepository) *chanIter[entity.GoodsItem] {
+	in, err := r.ListIter(ctx, 3)
+
+	return &chanIter[entity.GoodsItem]{
+		in:  in,
+		err: err,
+	}
+}

cmd/cli/commands/dimensiondispatcher.go

@@ -0,0 +1,72 @@
+package commands
+
+import (
+	"context"
+	"strings"
+
+	"git.loyso.art/frx/eway/internal/dimension"
+	"git.loyso.art/frx/eway/internal/entity"
+
+	"github.com/rs/zerolog"
+)
+
+type dimensionDispatcher struct {
+	m *dimension.Matcher
+}
+
+func pickFirst[T, V any](t T, v V) T {
+	return t
+}
+
+func (d dimensionDispatcher) isDimensionParam(value string) bool {
+	return pickFirst(d.m.Match(value)) != dimension.MatchResultMiss
+}
+
+func (d dimensionDispatcher) applyDimensionValue(ctx context.Context, key, value string, in *entity.GoodsItemSize) (updated bool) {
+	matchResult, priority := d.m.Match(key)
+	if matchResult == dimension.MatchResultMiss {
+		return false
+	}
+
+	log := zerolog.Ctx(ctx).With().Str("key", key).Str("value", value).Logger()
+
+	if strings.Contains(value, "/") {
+		dimensionValues := strings.Split(value, "/")
+		for _, dv := range dimensionValues {
+			updated = updated || d.applyDimensionValue(ctx, key, dv, in)
+		}
+	} else {
+		out, err := entity.ParseDimention(value, entity.DimensionLocalRU)
+		if err != nil {
+			log.Warn().Err(err).Msg("unable to parse key, skipping")
+			return false
+		}
+
+		out = out.AdjustTo(entity.DimensionKindCentimeter)
+
+		switch matchResult {
+		case dimension.MatchResultHeight:
+			if !priority && in.Height.Value != 0 {
+				return false
+			}
+
+			in.Height = out
+		case dimension.MatchResultLength:
+			if !priority && in.Length.Value != 0 {
+				return false
+			}
+
+			in.Length = out
+		case dimension.MatchResultWidth:
+			if !priority && in.Width.Value != 0 {
+				return false
+			}
+
+			in.Width = out
+		case dimension.MatchResultDepth:
+			in.UnmatchedDepth = out
+		}
+	}
+
+	return true
+}

cmd/cli/commands/exports.go

@@ -0,0 +1,353 @@
+package commands
+
+import (
+	"context"
+	"encoding/xml"
+	"fmt"
+	"io"
+	"os"
+	"strconv"
+	"strings"
+	"time"
+
+	"git.loyso.art/frx/eway/cmd/cli/components"
+	"git.loyso.art/frx/eway/internal/entity"
+	"git.loyso.art/frx/eway/internal/export"
+
+	"github.com/rs/zerolog"
+	"github.com/urfave/cli/v3"
+)
+
+func ExportCommandTree() *cli.Command {
+	var h exportHandlers
+	return &cli.Command{
+		Name:  "export",
+		Usage: "Provide actions to export data from the database",
+		Commands: []*cli.Command{
+			newExportYMLCatalogCommand(h),
+		},
+	}
+}
+
+func newExportYMLCatalogCommand(h exportHandlers) *cli.Command {
+	cmd := cli.Command{
+		Name:  "yml-catalog",
+		Usage: "Export data as yml_catalog",
+		Flags: []cli.Flag{
+			&cli.StringFlag{
+				Name:      "out",
+				Aliases:   []string{"o"},
+				Usage:     "Output to file or stdout/stderr",
+				Value:     "yml_catalog.xml",
+				TakesFile: true,
+			},
+			&cli.IntFlag{
+				Name:    "limit",
+				Aliases: []string{"l"},
+				Usage:   "limits amount of items to save",
+			},
+			&cli.BoolFlag{
+				Name:    "pretty",
+				Aliases: []string{"p"},
+				Usage:   "pretty-prints output",
+			},
+		},
+	}
+
+	return cmdWithAction(cmd, h.YMLCatalog)
+}
+
+type exportHandlers struct{}
+
+func (h exportHandlers) YMLCatalog(ctx context.Context, cmd *cli.Command) error {
+	const defaultCurrency = "RUR"
+
+	log := zerolog.Ctx(ctx)
+
+	path := cmd.String("out")
+	limit := cmd.Int("limit")
+	pretty := cmd.Bool("pretty")
+
+	r, err := components.GetRepository()
+	if err != nil {
+		return fmt.Errorf("getting repository: %w", err)
+	}
+
+	log.Info().
+		Str("path", path).
+		Int64("limit", limit).
+		Bool("pretty", pretty).
+		Msg("executing with parameters")
+
+	categories, err := r.Category().List(ctx)
+	if err != nil {
+		return fmt.Errorf("listing categories: %w", err)
+	}
+
+	shop := export.Shop{
+		Currencies: []export.Currency{{
+			ID:   defaultCurrency,
+			Rate: 1,
+		}},
+		Categories: make([]export.Category, 0, len(categories)),
+	}
+
+	categoryByNameIdx := make(map[string]int64, len(categories))
+	for _, category := range categories {
+		categoryByNameIdx[category.Name] = category.ID
+		shop.Categories = append(shop.Categories, export.Category{
+			ID:   category.ID,
+			Name: category.Name,
+		})
+	}
+
+	matcher, err := components.GetDimensionMatcher()
+	if err != nil {
+		return fmt.Errorf("getting dimension matcher: %w", err)
+	}
+
+	dimensionDispatcher := dimensionDispatcher{m: matcher}
+
+	log.Info().Any("patterns", matcher.GetRegisteredPatterns()).Msg("configured patterns")
+
+	const (
+		reasonNoSize        = "no_size"
+		reasonTooLarge      = "too_large"
+		reasonNoDescription = "no_description"
+		reasonNoPhoto       = "no_photo"
+	)
+
+	skippedByReasons := map[string]int{
+		reasonNoSize:        0,
+		reasonTooLarge:      0,
+		reasonNoDescription: 0,
+		reasonNoPhoto:       0,
+	}
+
+	addToSkip := func(condition bool, name string, log zerolog.Logger) bool {
+		if !condition {
+			return false
+		}
+
+		log.Debug().Str("reason", name).Msg("skipping item")
+		skippedByReasons[name]++
+
+		return condition
+	}
+	iter := getItemsIter(ctx, r.GoodsItem())
+	for iter.Next() {
+		const maximumAllowedSizes = 160
+
+		item := iter.Get()
+		sublog := log.With().Int64("cart", item.Cart).Logger()
+
+		switch {
+		case addToSkip(item.Description == "", reasonNoDescription, sublog):
+			continue
+		case addToSkip(!item.Sizes.AllSizesSet(), reasonNoSize, sublog):
+			continue
+		case addToSkip(item.Sizes.GetSum(entity.DimensionKindCentimeter) > maximumAllowedSizes, reasonTooLarge, sublog):
+			continue
+		}
+
+		offer := h.goodsItemAsOffer(item, categoryByNameIdx, dimensionDispatcher, sublog)
+
+		if addToSkip(len(offer.PictureURLs) == 0, reasonNoPhoto, sublog) {
+			continue
+		}
+
+		shop.Offers = append(shop.Offers, offer)
+	}
+	if err = iter.Err(); err != nil {
+		return fmt.Errorf("iterating over items: %w", err)
+	}
+
+	var f io.WriteCloser
+	switch path {
+	case "stdout":
+		f = os.Stdout
+	case "stderr":
+		f = os.Stderr
+	default:
+		f, err = os.Create(path)
+		if err != nil {
+			return fmt.Errorf("creating file: %w", err)
+		}
+		defer func() {
+			errClose := f.Close()
+			if err == nil {
+				err = errClose
+				return
+			}
+
+			log.Err(errClose).Msg("file closed or not")
+		}()
+	}
+
+	if limit > 0 {
+		shop.Offers = shop.Offers[:limit]
+	}
+
+	container := export.YmlContainer{
+		YmlCatalog: export.YmlCatalog{
+			Shop: shop,
+			Date: time.Now(),
+		},
+	}
+
+	_, err = f.Write([]byte(xml.Header))
+	if err != nil {
+		return fmt.Errorf("writing header: %w", err)
+	}
+	_, err = f.Write([]byte("<!DOCTYPE yml_catalog SYSTEM \"shops.dtd\">\n"))
+	enc := xml.NewEncoder(f)
+	if pretty {
+		enc.Indent("", " ")
+	}
+
+	log.Info().Int("processed", len(shop.Offers)).Any("skipped", skippedByReasons).Msg("completed")
+
+	return enc.Encode(container)
+}
+
+func (h exportHandlers) goodsItemAsOffer(in entity.GoodsItem, categoryIDByName map[string]int64, d dimensionDispatcher, log zerolog.Logger) (out export.Offer) {
+	const defaultType = "vendor.model"
+	const defaultCurrency = "RUR"
+	const defaultAvailable = true
+	const quantityParamName = "Количество на складе «Москва»"
+	const basePictureURL = "https://eway.elevel.ru"
+
+	imgurl := func(path string) string {
+		return basePictureURL + path
+	}
+
+	categoryID := categoryIDByName[in.Type]
+
+	pictureURLs := make([]string, 0, len(in.PhotoURLs))
+	for _, url := range in.PhotoURLs {
+		if url == "" {
+			continue
+		}
+
+		outurl := imgurl(url)
+		if outurl == basePictureURL {
+			continue
+		}
+
+		pictureURLs = append(pictureURLs, imgurl(url))
+	}
+
+	params := make([]export.Param, 0, len(in.Parameters))
+	for k, v := range in.Parameters {
+		if k == "" || v == "" {
+			continue
+		}
+
+		if d.isDimensionParam(k) {
+			continue
+		}
+
+		params = append(params, export.Param{
+			Name:  k,
+			Value: v,
+		})
+	}
+	params = append(params, export.Param{
+		Name:  quantityParamName,
+		Value: strconv.Itoa(in.Stock),
+	})
+
+	dimensions := h.formatSizeAsDimensions(in.Sizes, log)
+	out = export.Offer{
+		ID:          in.Cart,
+		VendorCode:  in.Articul,
+		Price:       int(in.TariffPrice),
+		Model:       in.Name,
+		Vendor:      in.Producer,
+		TypePrefix:  in.Name,
+		Description: in.Description,
+		Dimensions:  dimensions,
+
+		CategoryID:  categoryID,
+		PictureURLs: pictureURLs,
+		Params:      params,
+
+		Type:                defaultType,
+		CurrencyID:          defaultCurrency,
+		Available:           defaultAvailable,
+		ManufacturerWarrany: true,
+	}
+
+	return out
+}
+
+// check cart id 126584
+func (exportHandlers) formatSizeAsDimensions(size entity.GoodsItemSize, log zerolog.Logger) string {
+	const delimeter = "/"
+	makeFloat := func(d entity.Dimension) string {
+		value := d.AdjustTo(entity.DimensionKindCentimeter).Value
+		value = float64(int(value*100)) / 100.0
+
+		return strconv.FormatFloat(value, 'f', 1, 64)
+	}
+
+	knownSizes := make([]entity.Dimension, 0, 3)
+
+	dimensions := []entity.Dimension{
+		size.Length,
+		size.Width,
+		size.Height,
+	}
+	for i, d := range dimensions {
+		if d.IsZero() {
+			continue
+		}
+
+		knownSizes = append(knownSizes, dimensions[i])
+	}
+
+	l := makeFloat(size.Length)
+	w := makeFloat(size.Width)
+	h := makeFloat(size.Height)
+	switch len(knownSizes) {
+	case 3:
+		// go on
+	case 2:
+		var side string
+		unknownDefaultSize := makeFloat(entity.NewCentimeterDimensionOrEmpty(30))
+		switch {
+		case size.Length.IsZero():
+			side = "length"
+			l = unknownDefaultSize
+		case size.Width.IsZero():
+			side = "width"
+			w = unknownDefaultSize
+		case size.Height.IsZero():
+			side = "height"
+			h = unknownDefaultSize
+		}
+		log.Debug().Str("size", side).Msg("setting to default value")
+	case 1:
+		var side string
+		unknownDefaultSize := makeFloat(entity.NewCentimeterDimensionOrEmpty(30))
+		switch {
+		case !size.Length.IsZero():
+			side = "width"
+			w = unknownDefaultSize
+		case !size.Width.IsZero():
+			side = "length"
+			l = unknownDefaultSize
+		case !size.Height.IsZero():
+			return ""
+		}
+		log.Debug().Str("size", side).Msg("setting to default value")
+	default:
+		return ""
+	}
+
+	// Output should be the following format:
+	// length/width/height in centimeters
+	return strings.Join([]string{
+		l, w, h,
+	}, delimeter)
+}

cmd/cli/commands/helper.go

@@ -0,0 +1,55 @@
+package commands
+
+import (
+	"context"
+	"crypto/rand"
+	"encoding/hex"
+	"fmt"
+	"time"
+
+	"git.loyso.art/frx/eway/cmd/cli/components"
+	"git.loyso.art/frx/eway/internal/entity"
+
+	"github.com/urfave/cli/v3"
+)
+
+type empty entity.Empty
+
+type action func(ctx context.Context, c *cli.Command) error
+
+func decorateAction(a action) cli.ActionFunc {
+	return func(ctx context.Context, c *cli.Command) error {
+		var data [3]byte
+		_, _ = rand.Read(data[:])
+		reqid := hex.EncodeToString(data[:])
+
+		log, err := components.GetLogger()
+		if err != nil {
+			return fmt.Errorf("getting logger: %w", err)
+		}
+
+		log = log.With().Str("reqid", reqid).Logger()
+		rctx := log.WithContext(ctx)
+
+		start := time.Now()
+		defer func() {
+			log.Info().Float64("elapsed", time.Since(start).Seconds()).Msg("command completed")
+		}()
+
+		log.Info().Msg("command execution started")
+		return a(rctx, c)
+	}
+}
+
+func cmdWithAction(cmd cli.Command, a action) *cli.Command {
+	if a == nil {
+		a = notImplementedAction
+	}
+
+	cmd.Action = decorateAction(a)
+	return &cmd
+}
+
+func notImplementedAction(_ context.Context, _ *cli.Command) error {
+	return entity.ErrNotImplemented
+}

cmd/cli/commands/items.go

@@ -0,0 +1,385 @@
+package commands
+
+import (
+	"bufio"
+	"context"
+	"encoding/json"
+	"fmt"
+	"os"
+	"strconv"
+	"strings"
+
+	"git.loyso.art/frx/eway/cmd/cli/components"
+	"git.loyso.art/frx/eway/internal/entity"
+	"git.loyso.art/frx/eway/internal/matcher"
+
+	"github.com/rs/zerolog"
+	"github.com/urfave/cli/v3"
+)
+
+func ItemsCommandTree() *cli.Command {
+	var h itemsHandlers
+	return &cli.Command{
+		Name:  "items",
+		Usage: "Interact with items stored inside db",
+		Commands: []*cli.Command{
+			newItemsGetCommand(h),
+			newItemsCountCommand(h),
+			newItemsUniqueParamsCommand(h),
+			newItemsAggregateParametersCommand(h),
+			newItemsFixSizesCommand(h),
+		},
+	}
+}
+
+func newItemsGetCommand(h itemsHandlers) *cli.Command {
+	cmd := cli.Command{
+		Name:  "get",
+		Usage: "gets goods item by its id",
+		Flags: []cli.Flag{
+			&cli.StringFlag{
+				Name:  "id",
+				Usage: "id of the goods item. Either id or cart-id should be set",
+			},
+			&cli.IntFlag{
+				Name:  "cart-id",
+				Usage: "cart-id of the item. Either cart-id or id should be set",
+			},
+		},
+	}
+
+	return cmdWithAction(cmd, h.Get)
+}
+
+func newItemsCountCommand(h itemsHandlers) *cli.Command {
+	cmd := cli.Command{
+		Name:  "count",
+		Usage: "iterates over collection and counts number of items",
+		Flags: []cli.Flag{
+			&cli.StringSliceFlag{
+				Name:  "param-key-match",
+				Usage: "filters by parameters with AND logic",
+			},
+		},
+	}
+
+	return cmdWithAction(cmd, h.Count)
+}
+
+func newItemsUniqueParamsCommand(h itemsHandlers) *cli.Command {
+	cmd := cli.Command{
+		Name:  "unique-params",
+		Usage: "Show all stored unique param values",
+		Description: "This command iterates over each item and collect keys of params in a dict and then" +
+			" print it to the output. It's useful to find all unique parameters",
+	}
+
+	return cmdWithAction(cmd, h.UniqueParameters)
+}
+
+func newItemsAggregateParametersCommand(h itemsHandlers) *cli.Command {
+	cmd := cli.Command{
+		Name:  "aggregate-parameters",
+		Usage: "Show all values of requested parameters",
+		Flags: []cli.Flag{
+			&cli.BoolFlag{
+				Name:  "case-insensitive",
+				Usage: "Ignores cases of keys",
+			},
+			&cli.StringSliceFlag{
+				Name:  "regex",
+				Usage: "Registers regex to match",
+			},
+			&cli.BoolFlag{
+				Name:  "keys-only",
+				Usage: "prints only keys",
+			},
+		},
+	}
+
+	return cmdWithAction(cmd, h.AggregateParameters)
+}
+
+func newItemsFixSizesCommand(h itemsHandlers) *cli.Command {
+	cmd := cli.Command{
+		Name:  "fix-sizes",
+		Usage: "Iterates over params and sets sizes from parameters",
+	}
+
+	return cmdWithAction(cmd, h.FixSizes)
+}
+
+type itemsHandlers struct{}
+
+func (itemsHandlers) Get(ctx context.Context, cmd *cli.Command) error {
+	r, err := components.GetRepository()
+	if err != nil {
+		return fmt.Errorf("getting repository: %w", err)
+	}
+
+	id := cmd.String("id")
+	cartID := cmd.Int("cart-id")
+	if id == "" && cartID == 0 {
+		return cli.Exit("oneof: id or cart-id should be set", 1)
+	} else if id != "" && cartID != 0 {
+		return cli.Exit("oneof: id or cart-id should be set", 1)
+	}
+
+	var item entity.GoodsItem
+	if id != "" {
+		item, err = r.GoodsItem().Get(ctx, id)
+	} else {
+		item, err = r.GoodsItem().GetByCart(ctx, cartID)
+	}
+	if err != nil {
+		return fmt.Errorf("getting item: %w", err)
+	}
+
+	enc := json.NewEncoder(os.Stdout)
+	enc.SetIndent("", " ")
+
+	err = enc.Encode(item)
+	if err != nil {
+		return fmt.Errorf("encoding item: %w", err)
+	}
+
+	return nil
+}
+
+func (itemsHandlers) Count(ctx context.Context, cmd *cli.Command) error {
+	r, err := components.GetRepository()
+	if err != nil {
+		return fmt.Errorf("getting repository: %w", err)
+	}
+
+	filters := cmd.StringSlice("param-key-match")
+	m := matcher.NewRadix()
+	patternMapped := make(map[string]empty, len(filters))
+	if len(filters) == 0 {
+		m.Register("*")
+	} else {
+		for _, f := range filters {
+			m.Register(f)
+		}
+		for _, pattern := range m.Patterns() {
+			patternMapped[pattern] = empty{}
+		}
+	}
+
+	var count int
+	items, err := r.GoodsItem().List(ctx)
+	if err != nil {
+		return fmt.Errorf("getting items: %w", err)
+	}
+	for _, item := range items {
+		seenPatterns := map[string]empty{}
+
+		for k := range item.Parameters {
+			pattern := m.MatchByPattern(k)
+			if pattern == "" {
+				continue
+			}
+			if _, ok := seenPatterns[pattern]; ok {
+				continue
+			}
+			seenPatterns[pattern] = empty{}
+		}
+
+		if len(seenPatterns) == len(patternMapped) {
+			count++
+		}
+	}
+
+	return nil
+}
+
+func (itemsHandlers) UniqueParameters(ctx context.Context, cmd *cli.Command) error {
+	repository, err := components.GetRepository()
+	if err != nil {
+		return fmt.Errorf("getting repository: %w", err)
+	}
+
+	knownParams := map[string]empty{}
+	iter, err := repository.GoodsItem().ListIter(ctx, 1)
+	if err != nil {
+		return fmt.Errorf("getting list iter: %w", err)
+	}
+	for item := range iter {
+		for k := range item.Parameters {
+			knownParams[k] = empty{}
+		}
+	}
+
+	bw := bufio.NewWriter(cmd.Writer)
+	for paramName := range knownParams {
+		_, err = bw.WriteString(paramName + "\n")
+		if err != nil {
+			return fmt.Errorf("unable to write: %w", err)
+		}
+	}
+
+	return bw.Flush()
+}
+
+func (itemsHandlers) AggregateParameters(ctx context.Context, cmd *cli.Command) error {
+	repository, err := components.GetRepository()
+	if err != nil {
+		return fmt.Errorf("getting repository: %w", err)
+	}
+	log, err := components.GetLogger()
+	if err != nil {
+		return fmt.Errorf("getting logger: %w", err)
+	}
+
+	params := cmd.Args().Slice()
+	opts := make([]matcher.RadixOpt, 0, 1)
+	if cmd.Bool("case-insensitive") {
+		opts = append(opts, matcher.RadixCaseInsensitive())
+	}
+
+	m := matcher.NewRadix(opts...)
+	for _, param := range params {
+		log.Debug().Str("param", param).Msg("registering param")
+		m.Register(param)
+	}
+	for _, regexp := range cmd.StringSlice("regex") {
+		log.Debug().Str("regexp", regexp).Msg("registering regexp")
+		m.RegisterRegexp(regexp)
+	}
+
+	requestedValues := make(map[string]map[string]empty, len(params))
+	requestedValuesByPattern := make(map[string]map[string]empty, len(params))
+	iter := getItemsIter(ctx, repository.GoodsItem())
+	for iter.Next() {
+		item := iter.Get()
+		for k, v := range item.Parameters {
+			matchedPattern := m.MatchByPattern(k)
+			if matchedPattern == "" {
+				continue
+			}
+
+			values, ok := requestedValues[k]
+			if !ok {
+				values = make(map[string]empty)
+			}
+			values[v] = empty{}
+			requestedValues[k] = values
+
+			values, ok = requestedValuesByPattern[matchedPattern]
+			if !ok {
+				values = map[string]empty{}
+			}
+			values[v] = empty{}
+			requestedValuesByPattern[matchedPattern] = values
+		}
+	}
+
+	bw := bufio.NewWriter(cmd.Writer)
+	_, _ = bw.WriteString("Matches:\n")
+
+	if cmd.Bool("keys-only") {
+		for k := range requestedValues {
+			_, _ = bw.WriteString(k + "\n")
+		}
+	} else {
+		for k, v := range requestedValues {
+			_, _ = bw.WriteString(k + ": ")
+			values := make([]string, 0, len(v))
+			for item := range v {
+				values = append(values, strconv.Quote(item))
+			}
+			valuesStr := "[" + strings.Join(values, ",") + "]\n"
+			_, _ = bw.WriteString(valuesStr)
+		}
+	}
+
+	_, _ = bw.WriteString("\nPatterns:\n")
+	for _, pattern := range m.Patterns() {
+		_, _ = bw.WriteString(pattern + "\n")
+	}
+
+	return bw.Flush()
+}
+
+func (itemsHandlers) FixSizes(ctx context.Context, cmd *cli.Command) error {
+	repository, err := components.GetRepository()
+	if err != nil {
+		return fmt.Errorf("getting repository: %w", err)
+	}
+
+	matcher, err := components.GetDimensionMatcher()
+	if err != nil {
+		return fmt.Errorf("getting dimension matcher: %w", err)
+	}
+
+	dimensionDispatcher := dimensionDispatcher{m: matcher}
+
+	log := zerolog.Ctx(ctx)
+
+	toUpdate := make([]entity.GoodsItem, 0, 20_000)
+	bus := getItemsIter(ctx, repository.GoodsItem())
+	for bus.Next() {
+		item := bus.Get()
+
+		var valueBeenUpdated bool
+		for key, value := range item.Parameters {
+			trimmedValue := strings.TrimSpace(value)
+			trimmedKey := strings.TrimSpace(key)
+
+			if trimmedKey != key || trimmedValue != value {
+				log.Warn().
+					Str("old_key", key).
+					Str("new_key", trimmedKey).
+					Str("old_value", value).
+					Str("new_value", trimmedValue).
+					Msg("found mismatch")
+
+				delete(item.Parameters, key)
+
+				key = trimmedKey
+				value = trimmedValue
+
+				item.Parameters[key] = value
+				valueBeenUpdated = true
+			}
+
+			updateValue := strings.HasSuffix(key, ":")
+			if updateValue {
+				key = strings.TrimSuffix(key, ":")
+				item.Parameters[key] = value
+				delete(item.Parameters, key+":")
+				valueBeenUpdated = true
+			}
+
+			if dimensionDispatcher.applyDimensionValue(ctx, key, value, &item.Sizes) {
+				valueBeenUpdated = true
+				log.Debug().Str("key", key).Any("sizes", item.Sizes).Msg("been updated")
+			}
+		}
+
+		var updated bool
+		oldSizes := item.Sizes
+		item.Sizes, updated = entity.FixupSizes(oldSizes)
+		if updated {
+			log.Info().Int64("cart", item.Cart).Any("old", oldSizes).Any("new", item.Sizes).Msg("sizes been fixed")
+		}
+
+		valueBeenUpdated = valueBeenUpdated || updated
+
+		if valueBeenUpdated {
+			toUpdate = append(toUpdate, item)
+		}
+	}
+	if bus.Err() != nil {
+		return fmt.Errorf("iterating: %w", bus.Err())
+	}
+
+	_, err = repository.GoodsItem().UpsertMany(ctx, toUpdate...)
+	if err != nil {
+		return fmt.Errorf("updating items: %w", err)
+	}
+
+	log.Info().Int("count", len(toUpdate)).Msg("updated items")
+
+	return nil
+}

cmd/cli/components/di.go

@@ -0,0 +1,221 @@
+package components
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"io"
+	"os"
+	"time"
+
+	"git.loyso.art/frx/eway/internal/config"
+	"git.loyso.art/frx/eway/internal/dimension"
+	"git.loyso.art/frx/eway/internal/interconnect/eway"
+	"git.loyso.art/frx/eway/internal/storage"
+	xbadger "git.loyso.art/frx/eway/internal/storage/badger"
+
+	"github.com/BurntSushi/toml"
+	"github.com/dgraph-io/badger/v4"
+	"github.com/rs/zerolog"
+	"github.com/samber/do"
+)
+
+// Yeah, singleton is not good UNLESS you're really lazy
+var diInjector *do.Injector
+
+func GetEwayClient() (eway.Client, error) {
+	return do.Invoke[eway.Client](diInjector)
+}
+
+func GetRepository() (storage.Repository, error) {
+	adapter, err := do.Invoke[*storageRepositoryAdapter](diInjector)
+	if err != nil {
+		return nil, err
+	}
+
+	return adapter.entity, nil
+}
+
+func GetLogger() (zerolog.Logger, error) {
+	log, err := do.Invoke[*loggerAdapter](diInjector)
+	if err != nil {
+		return zerolog.Nop(), err
+	}
+
+	return log.entity.log, nil
+}
+
+func GetDimensionMatcher() (*dimension.Matcher, error) {
+	return do.Invoke[*dimension.Matcher](diInjector)
+}
+
+func SetupDI(ctx context.Context, cfgpath string, verbose bool, logAsJSON bool) error {
+	cfg, err := parseSettings(cfgpath)
+	if err != nil {
+		// if no settings provided allow cli to run without them.
+		if errors.Is(err, os.ErrNotExist) {
+			return nil
+		}
+
+		return err
+	}
+
+	diInjector = do.New()
+
+	do.Provide(diInjector, func(i *do.Injector) (*loggerAdapter, error) {
+		tsSet := func(wr *zerolog.ConsoleWriter) {
+			wr.TimeFormat = time.RFC3339
+		}
+
+		var outfile *os.File
+		var output io.Writer
+		switch cfg.Log.Output {
+		case "", "stdout":
+			output = os.Stdout
+		case "stderr":
+			output = os.Stderr
+		default:
+			outfile, err = os.OpenFile(cfg.Log.Output, os.O_RDWR|os.O_CREATE|os.O_APPEND, 0666)
+			if err != nil {
+				return nil, fmt.Errorf("creating file for logging: %w", err)
+			}
+
+			output = zerolog.SyncWriter(outfile)
+		}
+
+		var writer io.Writer
+		if logAsJSON {
+			writer = output
+		} else {
+			writer = zerolog.NewConsoleWriter(tsSet, func(w *zerolog.ConsoleWriter) {
+				w.Out = output
+			})
+		}
+
+		log := zerolog.
+			New(writer).
+			With().
+			Timestamp().
+			Str("app", "converter").
+			Logger()
+
+		if verbose {
+			log = log.Level(zerolog.DebugLevel)
+		} else {
+			log = log.Level(zerolog.InfoLevel)
+		}
+
+		out := &logger{
+			log:            log,
+			underlyingFile: outfile,
+		}
+		return &loggerAdapter{
+			entity: out,
+		}, nil
+	})
+
+	do.Provide[eway.Client](diInjector, func(i *do.Injector) (eway.Client, error) {
+		log, err := GetLogger()
+		if err != nil {
+			return nil, fmt.Errorf("getting logger: %w", err)
+		}
+
+		client, err := eway.New(ctx, eway.Config(cfg.Eway), log)
+		if err != nil {
+			return nil, fmt.Errorf("making new eway client: %w", err)
+		}
+
+		return client, nil
+	})
+
+	do.Provide[*badgerDBAdapter](diInjector, func(i *do.Injector) (*badgerDBAdapter, error) {
+		db, err := xbadger.Open(ctx, cfg.Badger.Dir, cfg.Badger.Debug, zerolog.Nop())
+		if err != nil {
+			return nil, fmt.Errorf("getting db: %w", err)
+		}
+
+		out := &badgerDBAdapter{entity: db}
+		return out, nil
+	})
+
+	do.Provide[*storageRepositoryAdapter](diInjector, func(i *do.Injector) (*storageRepositoryAdapter, error) {
+		db, err := getDB()
+		if err != nil {
+			return nil, err
+		}
+
+		client, err := xbadger.NewClient(db)
+		if err != nil {
+			return nil, fmt.Errorf("getting badger client: %w", err)
+		}
+
+		out := &storageRepositoryAdapter{entity: client}
+		return out, nil
+	})
+
+	do.Provide[*dimension.Matcher](diInjector, func(i *do.Injector) (*dimension.Matcher, error) {
+		matcher := dimension.New(cfg.DimensionMatcher)
+
+		return matcher, nil
+	})
+
+	return nil
+}
+
+func Shutdown() error {
+	if diInjector == nil {
+		return nil
+	}
+
+	return diInjector.Shutdown()
+}
+
+func getDB() (*badger.DB, error) {
+	adapter, err := do.Invoke[*badgerDBAdapter](diInjector)
+	if err != nil {
+		return nil, err
+	}
+
+	return adapter.entity, nil
+}
+
+type settings struct {
+	Badger           config.Badger           `toml:"badger"`
+	Log              config.Log              `toml:"log"`
+	Eway             config.Eway             `toml:"eway"`
+	DimensionMatcher config.DimensionMatcher `toml:"dimension_matcher"`
+}
+
+func parseSettings(cfgpath string) (cfg settings, err error) {
+	_, err = toml.DecodeFile(cfgpath, &cfg)
+	if err != nil {
+		return cfg, fmt.Errorf("parsing file: %w", err)
+	}
+
+	return cfg, nil
+}
+
+type logger struct {
+	log            zerolog.Logger
+	underlyingFile *os.File
+}
+
+func (l *logger) Close() error {
+	if l.underlyingFile == nil {
+		return nil
+	}
+
+	return l.underlyingFile.Close()
+}
+
+type entityCloserAdapter[T io.Closer] struct {
+	entity T
+}
+
+func (a entityCloserAdapter[T]) Shutdown() error {
+	return a.entity.Close()
+}
+
+type storageRepositoryAdapter entityCloserAdapter[storage.Repository]
+type badgerDBAdapter entityCloserAdapter[*badger.DB]
+type loggerAdapter entityCloserAdapter[*logger]

cmd/cli/dimensiondispatcher.go

@@ -0,0 +1,78 @@
+package main
+
+import (
+	"context"
+	"strings"
+
+	"git.loyso.art/frx/eway/internal/entity"
+	"git.loyso.art/frx/eway/internal/matcher"
+	"github.com/rs/zerolog"
+)
+
+type dimensionDispatcher struct {
+	heigth matcher.Unit
+	width  matcher.Unit
+	length matcher.Unit
+}
+
+func (d dimensionDispatcher) isDimensionParam(value string) bool {
+	return d.heigth.Match(value) || d.width.Match(value) || d.length.Match(value)
+}
+
+func (d dimensionDispatcher) dispatch(ctx context.Context, key, value string, in *entity.GoodsItemSize) (updated bool) {
+	if !d.isDimensionParam(key) {
+		return false
+	}
+
+	log := zerolog.Ctx(ctx).With().Str("key", key).Str("value", value).Logger()
+	if strings.Contains(value, "/") {
+		dimensionValues := strings.Split(value, "/")
+		for _, dv := range dimensionValues {
+			updated = updated || d.dispatch(ctx, key, dv, in)
+		}
+	} else {
+		out, err := entity.ParseDimention(value, entity.DimensionLocalRU)
+		if err != nil {
+			log.Warn().Err(err).Msg("unable to parse key, skipping")
+			return false
+		}
+
+		out = out.AdjustTo(entity.DimensionKindCentimeter)
+
+		updated = true
+		switch {
+		case d.heigth.Match(key):
+			in.Height = out
+		case d.width.Match(key):
+			in.Width = out
+		case d.length.Match(key):
+			in.Length = out
+		default:
+			log.Error().Str("key", key).Msg("unable to find proper matcher")
+			updated = false
+		}
+	}
+
+	return updated
+}
+
+func makeDefaultDimensionDispatcher() dimensionDispatcher {
+	h := matcher.NewRadix(matcher.RadixCaseInsensitive())
+	h.Register("Высота")
+	h.Register("Высота/*")
+
+	w := matcher.NewRadix(matcher.RadixCaseInsensitive())
+	w.Register("Ширина")
+	w.Register("Ширина/*")
+
+	l := matcher.NewRadix(matcher.RadixCaseInsensitive())
+	l.Register("Длина")
+	l.Register("Длина/*")
+	l.Register("Общ. длина")
+
+	return dimensionDispatcher{
+		heigth: h,
+		width:  w,
+		length: l,
+	}
+}

cmd/cli/main.go

@@ -0,0 +1,766 @@
+package main
+
+import (
+	"bufio"
+	"context"
+	"crypto/rand"
+	"encoding/hex"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"math/big"
+	"os"
+	"os/signal"
+	"syscall"
+	"time"
+
+	rooteway "git.loyso.art/frx/eway"
+	"git.loyso.art/frx/eway/cmd/cli/commands"
+	"git.loyso.art/frx/eway/cmd/cli/components"
+	"git.loyso.art/frx/eway/internal/crypto"
+	"git.loyso.art/frx/eway/internal/entity"
+	"git.loyso.art/frx/eway/internal/interconnect/eway"
+
+	"github.com/rodaine/table"
+	"github.com/rs/zerolog"
+	"github.com/urfave/cli/v3"
+)
+
+const (
+	defaultItemType = "Электрика"
+)
+
+type empty entity.Empty
+
+func main() {
+	ctx, cancel := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
+	defer func() {
+		cancel()
+	}()
+	err := runcli(ctx)
+
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "unable to handle app: %v\n", err)
+		os.Exit(1)
+	}
+
+	os.Exit(0)
+}
+
+func runcli(ctx context.Context) (err error) {
+	app := setupCLI()
+	return app.Run(ctx, os.Args)
+}
+
+func setupDI() cli.BeforeFunc {
+	return func(ctx context.Context, cmd *cli.Command) error {
+		if out := cmd.String("output"); out != "" {
+			var err error
+			cmd.Writer, err = os.Create(out)
+			if err != nil {
+				return fmt.Errorf("setting writer: %w", err)
+			}
+		}
+
+		cfgpath := cmd.String("config")
+		if cfgpath == "" {
+			return errors.New("no config path provided")
+		}
+
+		debugLevel := cmd.Bool("verbose")
+		jsonFormat := cmd.Bool("json")
+
+		err := components.SetupDI(ctx, cfgpath, debugLevel, jsonFormat)
+		if err != nil {
+			return fmt.Errorf("setting up di: %w", err)
+		}
+
+		return nil
+	}
+}
+
+func releaseDI() cli.AfterFunc {
+	return func(ctx context.Context, c *cli.Command) error {
+		if f, ok := c.Writer.(*os.File); ok {
+			err := f.Close()
+			if err != nil {
+				println("unable to close output file:", err.Error())
+			}
+		}
+
+		return components.Shutdown()
+	}
+}
+
+func setupCLI() *cli.Command {
+	app := &cli.Command{
+		Name:        "ewaycli",
+		Description: "a cli for running eway logic",
+		Version:     fmt.Sprintf("%s (%s) %s", rooteway.Version(), rooteway.Commit(), rooteway.BuildTime()),
+		Flags: []cli.Flag{
+			&cli.StringFlag{
+				Name:  "config",
+				Usage: "path to config in TOML format",
+				Value: "config.toml",
+				Sources: cli.NewValueSourceChain(
+					cli.EnvVar("EWAY_CONFIG"),
+				),
+				TakesFile: true,
+			},
+			&cli.BoolFlag{
+				Name:    "verbose",
+				Aliases: []string{"d"},
+				Usage:   "enables verbose logging",
+			},
+			&cli.BoolFlag{
+				Name:       "json",
+				Usage:      "enables json log format",
+				Persistent: true,
+			},
+			&cli.StringFlag{
+				Name:      "output",
+				Aliases:   []string{"o"},
+				Usage:     "Defines output for commands",
+				TakesFile: true,
+			},
+		},
+
+		Before: setupDI(),
+		After:  releaseDI(),
+
+		Commands: []*cli.Command{
+			commands.CategoriesCommandTree(),
+			commands.ItemsCommandTree(),
+			commands.ExportCommandTree(),
+
+			newCryptoCmd(),
+			newParseCmd(),
+			newImportCmd(),
+		},
+	}
+
+	return app
+}
+
+func newCryptoCmd() *cli.Command {
+	return &cli.Command{
+		Name:  "crypto",
+		Usage: "methods for encrypt/decrypt various things",
+		Commands: []*cli.Command{
+			newCryptoEncyptCmd(),
+			newCryptoDecryptCmd(),
+		},
+	}
+}
+
+func newCryptoEncyptCmd() *cli.Command {
+	return &cli.Command{
+		Name:  "encrypt",
+		Usage: "encypt incoming text",
+		Flags: []cli.Flag{
+			&cli.StringFlag{
+				Name:     "text",
+				Aliases:  []string{"t"},
+				Required: true,
+			},
+		},
+		Action: cryptoDeEncryptAction(true),
+	}
+}
+
+func newParseCmd() *cli.Command {
+	return &cli.Command{
+		Name:  "parse",
+		Usage: "category for parsing items from various sources",
+		Commands: []*cli.Command{
+			newParseEwayCmd(),
+		},
+	}
+}
+
+func newParseEwayCmd() *cli.Command {
+	return &cli.Command{
+		Name:  "eway",
+		Usage: "parse all available eway goods",
+		Commands: []*cli.Command{
+			newParseEwayGetCmd(),
+			newParseEwayListCmd(),
+			newParseEwayDumpCmd(),
+		},
+	}
+}
+
+func newParseEwayGetCmd() *cli.Command {
+	return &cli.Command{
+		Name:  "get",
+		Usage: "loads information about the product by parsing product's html page",
+		Flags: []cli.Flag{
+			&cli.IntFlag{
+				Name:     "cart",
+				Aliases:  []string{"c"},
+				Usage:    "cart of the product",
+				Required: true,
+			},
+		},
+		Action: decorateAction(parseEwayGetAction),
+	}
+}
+
+func newParseEwayListCmd() *cli.Command {
+	return &cli.Command{
+		Name:  "list",
+		Usage: "parse all available eway goods",
+		Flags: []cli.Flag{
+			&cli.IntFlag{
+				Name:  "page",
+				Usage: "choose page to load",
+				Value: 1,
+			},
+			&cli.IntFlag{
+				Name:  "limit",
+				Usage: "limits output",
+				Value: 100,
+			},
+			&cli.IntFlag{
+				Name:  "min-stock",
+				Usage: "filters by minimum available items in stock",
+			},
+			&cli.BoolFlag{
+				Name:  "pretty",
+				Usage: "pretty prints output",
+			},
+		},
+		Action: decorateAction(parseEwayListAction),
+	}
+}
+
+func newParseEwayDumpCmd() *cli.Command {
+	return &cli.Command{
+		Name:   "dump",
+		Usage:  "dumps content of eway catalog inside db",
+		Action: decorateAction(parseEwayDumpAction),
+	}
+}
+
+func newImportCmd() *cli.Command {
+	return &cli.Command{
+		Name:  "import",
+		Usage: "category for importing data from sources",
+		Commands: []*cli.Command{
+			newImportFromFileCmd(),
+		},
+	}
+}
+
+func newImportFromFileCmd() *cli.Command {
+	return &cli.Command{
+		Name:  "file",
+		Usage: "imports from file into db",
+		Flags: []cli.Flag{
+			&cli.StringFlag{
+				Name:     "src",
+				Value:    "",
+				Usage:    "source of the data.",
+				Required: true,
+			},
+		},
+		Action: decorateAction(importFromFileAction),
+	}
+}
+
+func importFromFileAction(ctx context.Context, c *cli.Command) error {
+	const maxBatch = 2000
+
+	r, err := components.GetRepository()
+	if err != nil {
+		return fmt.Errorf("getting repository: %w", err)
+	}
+
+	filesrc := c.String("src")
+	log := zerolog.Ctx(ctx)
+
+	log.Debug().Str("filepath", filesrc).Msg("importing data from file")
+
+	productsFile, err := os.Open(filesrc)
+	if err != nil {
+		return fmt.Errorf("opening file: %w", err)
+	}
+
+	defer func() {
+		errClose := productsFile.Close()
+		if errClose != nil {
+			log.Warn().Err(errClose).Msg("unable to close file")
+		}
+	}()
+
+	failedItems, err := os.Create("failed.json")
+	if err != nil {
+		log.Warn().Err(err).Msg("unable to open file for failed results")
+		failedItems = os.Stdout
+	}
+	defer func() {
+		if failedItems == os.Stdout {
+			return
+		}
+
+		errClose := failedItems.Close()
+		log.Err(errClose).Msg("closing file")
+	}()
+
+	var (
+		goodsItem      entity.GoodsItem
+		goodsItems     []entity.GoodsItem
+		failedToInsert int
+	)
+
+	seenCategories := make(map[string]empty)
+	categories, err := r.Category().List(ctx)
+	if err != nil {
+		return fmt.Errorf("listing categories: %w", err)
+	}
+
+	for _, category := range categories {
+		seenCategories[category.Name] = empty{}
+	}
+
+	bfile := bufio.NewReader(productsFile)
+	for {
+		line, _, err := bfile.ReadLine()
+		if err != nil {
+			if errors.Is(err, io.EOF) {
+				break
+
+			}
+			return fmt.Errorf("reading line: %w", err)
+		}
+
+		err = json.Unmarshal(line, &goodsItem)
+		if err != nil {
+			log.Warn().Err(err).Str("line", string(line)).Msg("unable to unmarshal line into item")
+			_, _ = failedItems.Write(line)
+			_, _ = failedItems.Write([]byte{'\n'})
+			failedToInsert++
+			continue
+		}
+
+		goodsItems = append(goodsItems, goodsItem)
+
+		if _, ok := seenCategories[goodsItem.Type]; ok {
+			continue
+		}
+
+		if goodsItem.Type == "" {
+			log.Warn().Msg("bad item without proper type")
+			_ = json.NewEncoder(failedItems).Encode(goodsItem)
+			continue
+		}
+
+		_, err = r.Category().Create(ctx, goodsItem.Type)
+		if err != nil {
+			return fmt.Errorf("unable to create new category: %w", err)
+		}
+		log.Debug().Any("category", goodsItem.Type).Msg("inserted new category")
+		seenCategories[goodsItem.Type] = empty{}
+	}
+
+	log.Debug().Int("count", len(goodsItems)).Int("failed", failedToInsert).Msg("preparing to upload")
+
+	start := time.Now()
+	batchSize := int(maxBatch)
+	for i := 0; i < len(goodsItems); i += batchSize {
+		to := i + batchSize
+		if to > len(goodsItems) {
+			to = len(goodsItems)
+		}
+
+		_, err = r.GoodsItem().UpsertMany(ctx, goodsItems[i:to]...)
+		if err != nil {
+			return fmt.Errorf("upserting items: %w", err)
+		}
+		log.Debug().Int("count", to-i).Msg("inserted batch")
+	}
+	log.Debug().Dur("elapsed", time.Since(start)).Msg("upload finished")
+
+	return nil
+}
+
+type action func(ctx context.Context, c *cli.Command) error
+
+func decorateAction(a action) cli.ActionFunc {
+	return func(ctx context.Context, c *cli.Command) error {
+		var data [3]byte
+		_, _ = rand.Read(data[:])
+		reqid := hex.EncodeToString(data[:])
+
+		log, err := components.GetLogger()
+		if err != nil {
+			return fmt.Errorf("getting logger: %w", err)
+		}
+
+		log = log.With().Str("reqid", reqid).Logger()
+		rctx := log.WithContext(ctx)
+
+		start := time.Now()
+		defer func() {
+			log.Info().Float64("elapsed", time.Since(start).Seconds()).Msg("command completed")
+		}()
+
+		log.Info().Msg("command execution started")
+		return a(rctx, c)
+	}
+}
+
+func parseEwayGetAction(ctx context.Context, cmd *cli.Command) error {
+	cartID := cmd.Int("cart")
+
+	client, err := components.GetEwayClient()
+	if err != nil {
+		return fmt.Errorf("getting eway client: %w", err)
+	}
+
+	pi, err := client.GetProductInfo(ctx, cartID)
+	if err != nil {
+		return fmt.Errorf("getting product info: %w", err)
+	}
+
+	enc := json.NewEncoder(cmd.Writer)
+	enc.SetIndent("", " ")
+	err = enc.Encode(pi)
+	if err != nil {
+		return fmt.Errorf("encoding data: %w", err)
+	}
+
+	return nil
+}
+
+func parseEwayListAction(ctx context.Context, cmd *cli.Command) error {
+	page := cmd.Int("page")
+	limit := cmd.Int("limit")
+	atLeast := cmd.Int("min-stock")
+	searchInStocks := atLeast > 0
+
+	client, err := components.GetEwayClient()
+	if err != nil {
+		return fmt.Errorf("getting eway client: %w", err)
+	}
+
+	log, err := components.GetLogger()
+	if err != nil {
+		return fmt.Errorf("getting logger: %w", err)
+	}
+
+	start := (page - 1) * limit
+
+	items, total, err := client.GetGoodsNew(ctx, eway.GetGoodsNewParams{
+		Draw:            1,
+		Start:           int(start),
+		Length:          int(limit),
+		SearchInStocks:  searchInStocks,
+		RemmantsAtleast: int(atLeast),
+	})
+	if err != nil {
+		return fmt.Errorf("getting new goods: %w", err)
+	}
+
+	productIDs := make([]int, 0, len(items))
+	for _, item := range items {
+		productIDs = append(productIDs, int(item.Cart))
+	}
+
+	remnants, err := client.GetGoodsRemnants(ctx, productIDs)
+	if err != nil {
+		return fmt.Errorf("getting remnants: %w", err)
+	}
+
+	goodsItems := make([]entity.GoodsItem, 0, len(items))
+	for _, item := range items {
+		if ctx.Err() != nil {
+			break
+		}
+
+		pi, err := client.GetProductInfo(ctx, int64(item.Cart))
+		if err != nil {
+			log.Warn().Err(err).Msg("unable to get product info")
+		}
+		outGood, err := entity.MakeGoodsItem(item, remnants, pi)
+		if err != nil {
+			return fmt.Errorf("making goods item: %w", err)
+		}
+
+		goodsItems = append(goodsItems, outGood)
+
+	}
+
+	var stats = struct {
+		Handled int `json:"handled"`
+		Loaded  int `json:"loaded"`
+		Total   int `json:"total"`
+	}{
+		Handled: len(goodsItems),
+		Loaded:  len(items),
+		Total:   total,
+	}
+	if cmd.Bool("json") {
+		enc := json.NewEncoder(cmd.Writer)
+		if cmd.Bool("pretty") {
+			enc.SetIndent("", " ")
+			_ = enc.Encode(goodsItems)
+		}
+
+		enc.SetIndent("", "")
+		_ = enc.Encode(stats)
+	} else {
+		tbl := table.
+			New("sku", "category", "cart", "stock", "price", "parameters").
+			WithWriter(cmd.Writer)
+
+		for _, outGood := range goodsItems {
+			parameters, _ := json.MarshalIndent(outGood.Parameters, "", " ")
+
+			tbl.AddRow(
+				outGood.Articul,
+				outGood.Type,
+				outGood.Cart,
+				outGood.Stock,
+				outGood.Price,
+				string(parameters),
+			)
+		}
+
+		tbl.Print()
+
+		table.
+			New("handled", "loaded", "total").
+			WithWriter(cmd.Writer).
+			AddRow(stats.Handled, stats.Loaded, stats.Total).
+			Print()
+	}
+
+	return nil
+}
+
+func parseEwayDumpAction(ctx context.Context, cmd *cli.Command) error {
+	client, err := components.GetEwayClient()
+	if err != nil {
+		return fmt.Errorf("getting eway client: %w", err)
+	}
+
+	repository, err := components.GetRepository()
+	if err != nil {
+		return fmt.Errorf("getting repository: %w", err)
+	}
+
+	logger, err := components.GetLogger()
+	if err != nil {
+		return fmt.Errorf("getting logger: %w", err)
+	}
+
+	const batchSize = 100
+	var i int
+	var start int
+
+	seenItems, err := entity.IterIntoMap[string, entity.GoodsItem](repository.GoodsItem().List(ctx)).Map(func(gi entity.GoodsItem) (string, error) {
+		return gi.Articul, nil
+	})
+	if err != nil {
+		return fmt.Errorf("making seen items map: %w", err)
+	}
+
+	goodsItems := make([]entity.GoodsItem, 0, batchSize)
+	productIDs := make([]int, 0, batchSize)
+
+	knownCategories := make(map[string]empty)
+	err = entity.IterWithErr(repository.Category().List(ctx)).Do(func(c entity.Category) error {
+		knownCategories[c.Name] = empty{}
+		return nil
+	})
+	if err != nil {
+		return fmt.Errorf("filling known categories: %w", err)
+	}
+
+	itemsUpdated := make(map[string]empty, len(seenItems))
+	stats := struct {
+		fetchedInfo int
+		handledAll  int
+		cachedInfo  int
+		skippedItem int
+	}{}
+
+	dimensionDispatcher := makeDefaultDimensionDispatcher()
+
+	startFrom := time.Now()
+	for {
+		select {
+		case <-ctx.Done():
+			return ctx.Err()
+		default:
+		}
+
+		items, total, err := client.GetGoodsNew(ctx, eway.GetGoodsNewParams{
+			Draw:            i,
+			Start:           start,
+			Length:          batchSize,
+			SearchInStocks:  true,
+			RemmantsAtleast: 5,
+		})
+		if err != nil {
+			logger.Warn().Err(err).Msg("unable to get items from catalog")
+
+			continue
+		}
+
+		productIDs = productIDs[:0]
+		for _, item := range items {
+			productIDs = append(productIDs, int(item.Cart))
+		}
+
+		remnants, err := client.GetGoodsRemnants(ctx, productIDs)
+		if err != nil {
+			logger.Warn().Err(err).Msg("unable to get goods remnants")
+			return fmt.Errorf("getting goods remnants: %w", err)
+		}
+
+		goodsItems = goodsItems[:0]
+		for _, item := range items {
+			var pi entity.GoodsItemInfo
+			seenItem := seenItems[item.SKU]
+			if time.Since(seenItem.CreatedAt) < time.Hour*24 {
+				logger.Debug().Str("sku", item.SKU).Msg("skipping item because it's too fresh")
+
+				stats.skippedItem++
+				itemsUpdated[item.SKU] = empty{}
+
+				continue
+			}
+
+			if len(seenItem.Parameters) != 0 && len(seenItem.PhotoURLs) != 0 {
+				pi.Parameters = seenItem.Parameters
+				pi.PhotoURLs = seenItem.PhotoURLs
+				stats.cachedInfo++
+			} else {
+				pi, err = client.GetProductInfo(ctx, int64(item.Cart))
+				if err != nil {
+					logger.Warn().Err(err).Msg("unable to get product info, skipping")
+					continue
+				}
+				stats.fetchedInfo++
+			}
+
+			goodsItem, err := entity.MakeGoodsItem(item, remnants, pi)
+			if err != nil {
+				logger.Err(err).Any("item", item).Msg("unable to make goods item")
+
+				continue
+			}
+
+			for key, value := range goodsItem.Parameters {
+				dimensionDispatcher.dispatch(ctx, key, value, &goodsItem.Sizes)
+			}
+
+			itemsUpdated[goodsItem.Articul] = empty{}
+			stats.handledAll++
+
+			goodsItems = append(goodsItems, goodsItem)
+			if goodsItem.Type == "" {
+				logger.Warn().Int64("cart_id", goodsItem.Cart).Msg("found item without category, setting default type")
+				goodsItem.Type = defaultItemType
+			}
+
+			if _, ok := knownCategories[goodsItem.Type]; ok {
+				continue
+			}
+
+			category, err := repository.Category().Create(ctx, goodsItem.Type)
+			if err != nil {
+				return fmt.Errorf("creating category: %w", err)
+			}
+
+			logger.Info().
+				Str("name", category.Name).
+				Int64("id", category.ID).
+				Msg("created new category")
+
+			knownCategories[goodsItem.Type] = empty{}
+		}
+
+		_, err = repository.GoodsItem().UpsertMany(ctx, goodsItems...)
+		if err != nil {
+			return fmt.Errorf("upserting items: %w", err)
+		}
+
+		progressFloat := float64(start) / float64(total)
+		progress := big.NewFloat(progressFloat).Text('f', 3)
+		elapsed := time.Since(startFrom).Seconds()
+
+		var left int
+		if progressFloat != 0 {
+			left = int(((1 - progressFloat) / progressFloat) * elapsed)
+		}
+
+		logger.Info().
+			Int("from", start).
+			Int("to", start+batchSize).
+			Int("total", total).
+			Str("progress", progress).
+			Int("seconds_left", left).
+			Msg("handled next batch items")
+
+		if len(items) < batchSize {
+			break
+		}
+
+		start += batchSize
+		i++
+	}
+
+	for k := range itemsUpdated {
+		delete(seenItems, k)
+	}
+
+	logger.Info().
+		Int("handled", stats.handledAll).
+		Int("cached", stats.cachedInfo).
+		Int("fetched", stats.fetchedInfo).
+		Int("skipped", stats.skippedItem).
+		Int("to_delete", len(seenItems)).
+		Msg("processed items")
+
+	for k := range seenItems {
+		_, err := repository.GoodsItem().Delete(ctx, k)
+		if err != nil {
+			if errors.Is(err, entity.ErrNotFound) {
+				continue
+			}
+			logger.Warn().Err(err).Str("sku", k).Msg("unable to delete item")
+
+			continue
+		}
+
+		logger.Info().Str("sku", k).Msg("deleted item")
+	}
+
+	return nil
+}
+
+func cryptoDeEncryptAction(encrypt bool) cli.ActionFunc {
+	return func(ctx context.Context, c *cli.Command) (err error) {
+		value := c.String("text")
+		var out string
+		if encrypt {
+			out, err = crypto.Encrypt(value)
+		} else {
+			out, err = crypto.Decrypt(value)
+		}
+		if err != nil {
+			return err
+		}
+
+		_, err = c.Writer.Write([]byte(out))
+		if err != nil {
+			return err
+		}
+		_, err = c.Writer.Write([]byte{'\n'})
+		return err
+	}
+}

cmd/cli/main_encoff.go

@@ -0,0 +1,27 @@
+//go:build !encon
+// +build !encon
+
+package main
+
+import (
+	"context"
+
+	"github.com/urfave/cli/v3"
+)
+
+func newCryptoDecryptCmd() *cli.Command {
+	return &cli.Command{
+		Name:  "decrypt",
+		Usage: "decrypt incoming text",
+		Flags: []cli.Flag{
+			&cli.StringFlag{
+				Name:     "text",
+				Aliases:  []string{"t"},
+				Required: true,
+			},
+		},
+		Action: func(context.Context, *cli.Command) error {
+			return cli.Exit("decryption is turned off", -1)
+		},
+	}
+}

cmd/cli/main_encon.go

@@ -0,0 +1,21 @@
+//go:build encon
+// +build encon
+
+package main
+
+import "github.com/urfave/cli/v3"
+
+func newCryptoDecryptCmd() *cli.Command {
+	return &cli.Command{
+		Name:  "decrypt",
+		Usage: "decrypt incoming text",
+		Flags: []cli.Flag{
+			&cli.StringFlag{
+				Name:     "text",
+				Aliases:  []string{"t"},
+				Required: true,
+			},
+		},
+		Action: cryptoDeEncryptAction(false),
+	}
+}

cmd/converter/main.go

@@ -1,444 +0,0 @@
-package main
-
-import (
-	"bufio"
-	"context"
-	"crypto/rand"
-	"encoding/hex"
-	"encoding/json"
-	"errors"
-	"fmt"
-	"io"
-	"os"
-	"os/signal"
-	"sync"
-	"sync/atomic"
-	"time"
-
-	"git.loyso.art/frx/eway/internal/config"
-	"git.loyso.art/frx/eway/internal/entity"
-	"git.loyso.art/frx/eway/internal/storage"
-	xbadger "git.loyso.art/frx/eway/internal/storage/badger"
-
-	badger "github.com/dgraph-io/badger/v4"
-	"github.com/dgraph-io/badger/v4/pb"
-	"github.com/rodaine/table"
-	"github.com/rs/zerolog"
-	"github.com/urfave/cli"
-)
-
-type appSettings struct {
-	Badger config.Badger
-}
-
-func main() {
-	ctx, cancel := signal.NotifyContext(context.Background(), os.Interrupt)
-	defer func() {
-		cancel()
-	}()
-
-	var corewg sync.WaitGroup
-	err := runcli(ctx, &corewg)
-
-	if err != nil {
-		fmt.Fprintf(os.Stderr, "unable to handle app: %v", err)
-
-		corewg.Wait()
-		os.Exit(1)
-	}
-
-	corewg.Wait()
-	os.Exit(0)
-}
-
-func runcli(ctx context.Context, wg *sync.WaitGroup) (err error) {
-	tsSet := func(wr *zerolog.ConsoleWriter) {
-		wr.TimeFormat = time.RFC3339
-	}
-	log := zerolog.New(zerolog.NewConsoleWriter(tsSet)).Level(zerolog.DebugLevel).With().Timestamp().Str("app", "converter").Logger()
-
-	defer func() {
-		log.Info().Err(err).Msg("app finished")
-	}()
-
-	ctx = log.WithContext(ctx)
-
-	log.Info().Msg("making badger")
-
-	db, err := xbadger.Open(ctx, "badger/", log)
-	if err != nil {
-		return fmt.Errorf("opening badger: %w", err)
-	}
-
-	go func() {
-		var item atomic.Uint64
-		err = db.Subscribe(
-			ctx,
-			func(kvlist *badger.KVList) error {
-				kvs := kvlist.GetKv()
-				for _, kv := range kvs {
-					count := item.Add(1)
-					log.Debug().Bytes("key", kv.GetKey()).Uint64("count", count).Msg("inspecting")
-				}
-				return nil
-			},
-			[]pb.Match{
-				{
-					Prefix: []byte("!!category!!"),
-				},
-				{
-					Prefix: []byte("!!goodsitem!!"),
-				},
-			},
-		)
-		log.Err(err).Msg("subscribing")
-
-	}()
-
-	maxBatch := db.MaxBatchCount()
-	log.Info().Int("max_batch", int(maxBatch)).Msg("max batch settings")
-	client, err := xbadger.NewClient(db)
-	if err != nil {
-		return fmt.Errorf("making new client: %w", err)
-	}
-
-	wg.Add(1)
-	defer func() {
-		defer wg.Done()
-
-		println("closing client")
-		errClose := client.Close()
-		if errClose != nil {
-			log.Warn().Err(errClose).Msg("unable to close client")
-		}
-
-		println("flushing db")
-		errSync := db.Sync()
-		if errSync != nil {
-			log.Warn().Err(errSync).Msg("unable to sync db")
-		}
-
-		// time.Sleep(time.Second * 5)
-
-		println("closing db")
-		errClose = db.Close()
-		if errClose != nil {
-			log.Warn().Err(errClose).Msg("unable to close db")
-		}
-	}()
-
-	app := setupCLI(ctx, client, maxBatch)
-
-	return app.Run(os.Args)
-}
-
-func setupCLI(ctx context.Context, r storage.Repository, maxBatch int64) *cli.App {
-	app := cli.NewApp()
-
-	app.Commands = cli.Commands{
-		newImportCmd(ctx, r, maxBatch),
-		newViewCmd(ctx, r),
-	}
-
-	return app
-}
-
-func newImportCmd(ctx context.Context, r storage.Repository, maxBatch int64) cli.Command {
-	return cli.Command{
-		Name:  "import",
-		Usage: "category for importing data from sources",
-		Flags: []cli.Flag{
-			// &cli.StringFlag{
-			// 	Name:      "config",
-			// 	Usage:     "path to config",
-			// 	Value:     "config.json",
-			// 	TakesFile: true,
-			// },
-			&cli.BoolFlag{
-				Name:  "verbose",
-				Usage: "set logger to debug mode",
-			},
-		},
-		Before: func(c *cli.Context) error {
-			if c.Bool("verbose") {
-				zerolog.SetGlobalLevel(zerolog.DebugLevel)
-			}
-			return nil
-		},
-		Subcommands: cli.Commands{
-			newImportFromFileCmd(ctx, r, maxBatch),
-		},
-	}
-}
-
-func newViewCmd(ctx context.Context, r storage.Repository) cli.Command {
-	return cli.Command{
-		Name:  "view",
-		Usage: "Set of commands to view the data inside db",
-		Subcommands: []cli.Command{
-			newViewCategoriesCmd(ctx, r),
-			newViewItemsCmd(ctx, r),
-		},
-	}
-}
-
-func newViewCategoriesCmd(ctx context.Context, r storage.Repository) cli.Command {
-	return cli.Command{
-		Name:  "categories",
-		Usage: "Set of commands to work with categories",
-		Subcommands: []cli.Command{
-			newViewCategoriesListCmd(ctx, r),
-		},
-	}
-}
-
-func newViewItemsCmd(ctx context.Context, r storage.Repository) cli.Command {
-	return cli.Command{
-		Name:  "items",
-		Usage: "Set of command to work with items",
-		Subcommands: cli.Commands{
-			newViewItemsCountCmd(ctx, r),
-		},
-	}
-}
-
-func newViewItemsCountCmd(ctx context.Context, r storage.Repository) cli.Command {
-	return cli.Command{
-		Name:   "count",
-		Usage:  "iterates over collection and counts number of items",
-		Action: viewItemsCount(ctx, r),
-	}
-}
-
-func newViewCategoriesListCmd(ctx context.Context, r storage.Repository) cli.Command {
-	return cli.Command{
-		Name:  "list",
-		Usage: "lists stored categories stored in database",
-		Flags: []cli.Flag{
-			&cli.IntFlag{
-				Name:  "limit",
-				Usage: "limits output to selected items",
-				Value: 20,
-			},
-			&cli.IntFlag{
-				Name:  "page",
-				Usage: "in case of limit, selects page",
-				Value: 0,
-			},
-			&cli.BoolFlag{
-				Name:  "with-total",
-				Usage: "prints total count of categories",
-			},
-		},
-		Action: viewCategoriesListAction(ctx, r),
-	}
-}
-
-func viewItemsCount(ctx context.Context, r storage.Repository) cli.ActionFunc {
-	f := func(c *cli.Context) error {
-		// itemChan, err := r.GoodsItem().ListIter(ctx, 10)
-		// if err != nil {
-		// 	return fmt.Errorf("getting list iter: %w", err)
-		// }
-		//
-		var count int
-		// for range itemChan {
-		// 	count++
-		// }
-		//
-		items, err := r.GoodsItem().List(ctx)
-		if err != nil {
-			return fmt.Errorf("getting list: %w", err)
-		}
-
-		zerolog.Ctx(ctx).Info().Int("count", count).Int("list_count", len(items)).Msg("read all items")
-		return nil
-	}
-
-	return wrapActionFunc(ctx, f)
-}
-
-func viewCategoriesListAction(ctx context.Context, r storage.Repository) cli.ActionFunc {
-	f := func(c *cli.Context) error {
-		categories, err := r.Category().List(ctx)
-		if err != nil {
-			return fmt.Errorf("listing categories: %w", err)
-		}
-
-		limit := c.Int("limit")
-		page := c.Int("page")
-		total := len(categories)
-
-		if page == 0 {
-			page = 1
-		}
-
-		if limit > 0 {
-			offset := (page - 1) * limit
-			if offset > len(categories) {
-				offset = len(categories) - 1
-			}
-
-			limit = offset + limit
-			if limit > len(categories) {
-				limit = len(categories)
-			}
-
-			categories = categories[offset:limit]
-		}
-
-		tbl := table.New("ID", "Name")
-		for _, category := range categories {
-			if category.ID == 0 && category.Name == "" {
-				continue
-			}
-			tbl.AddRow(category.ID, category.Name)
-		}
-
-		tbl.Print()
-
-		if c.Bool("with-total") {
-			zerolog.Ctx(ctx).Info().Int("count", total).Msg("total categories stats")
-		}
-
-		return nil
-	}
-
-	return wrapActionFunc(ctx, f)
-}
-
-func newImportFromFileCmd(ctx context.Context, r storage.Repository, maxBatch int64) cli.Command {
-	return cli.Command{
-		Name:  "fromfile",
-		Usage: "imports from file into db",
-		Flags: []cli.Flag{
-			&cli.StringFlag{
-				Name:     "src",
-				Value:    "",
-				Usage:    "source of the data.",
-				Required: true,
-			},
-		},
-		Action: handleConvert(ctx, r, maxBatch),
-	}
-}
-
-func handleConvert(ctx context.Context, r storage.Repository, maxBatch int64) cli.ActionFunc {
-	f := func(c *cli.Context) error {
-		filesrc := c.String("src")
-		log := zerolog.Ctx(ctx)
-
-		log.Debug().Str("filepath", filesrc).Msg("importing data from file")
-
-		productsFile, err := os.Open(filesrc)
-		if err != nil {
-			return fmt.Errorf("opening file: %w", err)
-		}
-
-		defer func() {
-			errClose := productsFile.Close()
-			if errClose != nil {
-				log.Warn().Err(errClose).Msg("unable to close file")
-			}
-		}()
-
-		var (
-			goodsItem entity.GoodsItem
-			// goodsItems     []entity.GoodsItem
-			failedToInsert int
-		)
-
-		seenCategories := make(map[string]struct{})
-		categories, err := r.Category().List(ctx)
-		if err != nil {
-			return fmt.Errorf("listing categories: %w", err)
-		}
-
-		for _, category := range categories {
-			seenCategories[category.Name] = struct{}{}
-		}
-
-		bfile := bufio.NewReader(productsFile)
-		for {
-			line, _, err := bfile.ReadLine()
-			if err != nil {
-				if errors.Is(err, io.EOF) {
-					break
-
-				}
-				return fmt.Errorf("reading line: %w", err)
-			}
-
-			err = json.Unmarshal(line, &goodsItem)
-			if err != nil {
-				// log.Warn().Err(err).Str("line", string(line)).Msg("unable to unmarshal line into item")
-				failedToInsert++
-				continue
-			}
-
-			_, err = r.GoodsItem().UpsertMany(ctx, goodsItem)
-			if err != nil {
-				return fmt.Errorf("unable to upsert new item: %w", err)
-			}
-
-			// goodsItems = append(goodsItems, goodsItem)
-
-			if _, ok := seenCategories[goodsItem.Type]; ok {
-				continue
-			}
-
-			_, err = r.Category().Create(ctx, goodsItem.Type)
-			if err != nil {
-				return fmt.Errorf("unable to create new category: %w", err)
-			}
-			log.Debug().Any("category", goodsItem.Type).Msg("inserted new category")
-			seenCategories[goodsItem.Type] = struct{}{}
-		}
-
-		// log.Debug().Int("count", len(goodsItems)).Int("failed", failedToInsert).Msg("preparing to upload")
-		//
-		// start := time.Now()
-		// batchSize := int(maxBatch)
-		// for i := 0; i < len(goodsItems); i += batchSize {
-		// 	to := i + batchSize
-		// 	if to > len(goodsItems) {
-		// 		to = len(goodsItems)
-		// 	}
-		//
-		// 	_, err = r.GoodsItem().UpsertMany(ctx, goodsItems[i:to]...)
-		// 	if err != nil {
-		// 		return fmt.Errorf("upserting items: %w", err)
-		// 	}
-		// 	log.Debug().Int("count", to-i).Msg("inserted batch")
-		// 	time.Sleep(time.Second)
-		// }
-		// log.Debug().Dur("elapsed", time.Since(start)).Msg("upload finished")
-		//
-
-		time.Sleep(time.Second * 30)
-
-		return nil
-	}
-
-	time.Sleep(time.Second * 10)
-
-	return wrapActionFunc(ctx, f)
-}
-
-func wrapActionFunc(ctx context.Context, next cli.ActionFunc) cli.ActionFunc {
-	return func(c *cli.Context) error {
-		var data [3]byte
-		_, _ = rand.Read(data[:])
-		reqid := hex.EncodeToString(data[:])
-
-		log := zerolog.Ctx(ctx).With().Str("reqid", reqid).Logger()
-		ctx = log.WithContext(ctx)
-		start := time.Now()
-		defer func() {
-			log.Info().Dur("elapsed", time.Since(start)).Msg("command completed")
-		}()
-
-		log.Info().Msg("command execution started")
-		return next(c)
-	}
-}

go.mod

@@ -3,32 +3,40 @@ module git.loyso.art/frx/eway
 go 1.21.4
 
 require (
+	github.com/BurntSushi/toml v1.3.2
+	github.com/PuerkitoBio/goquery v1.8.1
+	github.com/brianvoe/gofakeit/v6 v6.28.0
+	github.com/dgraph-io/badger/v4 v4.2.0
+	github.com/dgraph-io/ristretto v0.1.1
+	github.com/go-resty/resty/v2 v2.10.0
+	github.com/google/flatbuffers v23.5.26+incompatible
+	github.com/rodaine/table v1.1.1
+	github.com/rs/zerolog v1.31.0
+	github.com/samber/do v1.6.0
+	github.com/stretchr/testify v1.8.4
+	github.com/urfave/cli/v3 v3.0.0-alpha8
+)
+
+require (
+	github.com/andybalholm/cascadia v1.3.1 // indirect
 	github.com/cespare/xxhash/v2 v2.2.0 // indirect
-	github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect
-	github.com/dgraph-io/badger/v4 v4.2.0 // indirect
-	github.com/dgraph-io/ristretto v0.1.1 // indirect
+	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/dustin/go-humanize v1.0.0 // indirect
-	github.com/go-resty/resty/v2 v2.10.0 // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect
 	github.com/golang/glog v1.0.0 // indirect
-	github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6 // indirect
+	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
 	github.com/golang/protobuf v1.5.2 // indirect
 	github.com/golang/snappy v0.0.3 // indirect
-	github.com/google/flatbuffers v23.5.26+incompatible // indirect
-	github.com/karlseguin/ccache/v3 v3.0.5 // indirect
-	github.com/kataras/tablewriter v0.0.0-20180708051242-e063d29b7c23 // indirect
 	github.com/klauspost/compress v1.12.3 // indirect
+	github.com/kr/text v0.2.0 // indirect
 	github.com/mattn/go-colorable v0.1.13 // indirect
 	github.com/mattn/go-isatty v0.0.19 // indirect
-	github.com/mattn/go-runewidth v0.0.15 // indirect
 	github.com/pkg/errors v0.9.1 // indirect
-	github.com/rivo/uniseg v0.2.0 // indirect
-	github.com/rodaine/table v1.1.1 // indirect
-	github.com/rs/zerolog v1.31.0 // indirect
-	github.com/russross/blackfriday/v2 v2.1.0 // indirect
-	github.com/urfave/cli v1.22.14 // indirect
+	github.com/pmezard/go-difflib v1.0.0 // indirect
+	github.com/xrash/smetrics v0.0.0-20231213231151-1d8dd44e695e // indirect
 	go.opencensus.io v0.22.5 // indirect
 	golang.org/x/net v0.17.0 // indirect
 	golang.org/x/sys v0.13.0 // indirect
 	google.golang.org/protobuf v1.28.1 // indirect
+	gopkg.in/yaml.v3 v3.0.1 // indirect
 )

go.sum

@@ -1,19 +1,27 @@
 cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
+github.com/BurntSushi/toml v1.3.2 h1:o7IhLm0Msx3BaB+n3Ag7L8EVlByGnpq14C4YWiu/gL8=
 github.com/BurntSushi/toml v1.3.2/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ=
+github.com/PuerkitoBio/goquery v1.8.1 h1:uQxhNlArOIdbrH1tr0UXwdVFgDcZDrZVdcpygAcwmWM=
+github.com/PuerkitoBio/goquery v1.8.1/go.mod h1:Q8ICL1kNUJ2sXGoAhPGUdYDJvgQgHzJsnnd3H7Ho5jQ=
+github.com/andybalholm/cascadia v1.3.1 h1:nhxRkql1kdYCc8Snf7D5/D3spOX+dBgjA6u8x004T2c=
+github.com/andybalholm/cascadia v1.3.1/go.mod h1:R4bJ1UQfqADjvDa4P6HZHLh/3OxWWEqc0Sk8XGwHqvA=
+github.com/brianvoe/gofakeit/v6 v6.28.0 h1:Xib46XXuQfmlLS2EXRuJpqcw8St6qSZz75OUo0tgAW4=
+github.com/brianvoe/gofakeit/v6 v6.28.0/go.mod h1:Xj58BMSnFqcn/fAQeSK+/PLtC5kSb7FJIq4JyGa8vEs=
 github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
 github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44=
 github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
 github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
 github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc=
-github.com/cpuguy83/go-md2man/v2 v2.0.2 h1:p1EgwI/C7NhT0JmVkwCD2ZBK8j4aeHQX2pMHHBfMQ6w=
-github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
+github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/dgraph-io/badger/v4 v4.2.0 h1:kJrlajbXXL9DFTNuhhu9yCx7JJa4qpYWxtE8BzuWsEs=
 github.com/dgraph-io/badger/v4 v4.2.0/go.mod h1:qfCqhPoWDFJRx1gp5QwwyGo8xk1lbHUxvK9nK0OGAak=
 github.com/dgraph-io/ristretto v0.1.1 h1:6CWw5tJNgpegArSHpNHJKldNeq03FQCwYvfMVWajOK8=
 github.com/dgraph-io/ristretto v0.1.1/go.mod h1:S1GPSBCYCIhmVNfcth17y2zZtQT6wzkzgwUve0VDWWA=
+github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2 h1:tdlZCpZ/P9DhczCTSixgIKmwPv6+wP5DGjqLYw5SUiA=
 github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw=
 github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo=
 github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
@@ -25,8 +33,9 @@ github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69
 github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
 github.com/golang/glog v1.0.0 h1:nfP3RFugxnNRyKgeWd4oI1nYvXpxrx8ck8ZrcizshdQ=
 github.com/golang/glog v1.0.0/go.mod h1:EWib/APOK0SL3dFbYqvxE3UYd8E6s1ouQ7iEp/0LWV4=
-github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6 h1:ZgQEtGgCBiWRM39fZuwSd1LwSqqSW0hOdXCYYDX0R3I=
 github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
+github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE=
+github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
 github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
 github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
@@ -39,15 +48,16 @@ github.com/google/flatbuffers v23.5.26+incompatible h1:M9dgRyhJemaM4Sw8+66GHBu8i
 github.com/google/flatbuffers v23.5.26+incompatible/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8=
 github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
 github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
 github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
-github.com/karlseguin/ccache/v3 v3.0.5 h1:hFX25+fxzNjsRlREYsoGNa2LoVEw5mPF8wkWq/UnevQ=
-github.com/karlseguin/ccache/v3 v3.0.5/go.mod h1:qxC372+Qn+IBj8Pe3KvGjHPj0sWwEF7AeZVhsNPZ6uY=
-github.com/kataras/tablewriter v0.0.0-20180708051242-e063d29b7c23 h1:M8exrBzuhWcU6aoHJlHWPe4qFjVKzkMGRal78f5jRRU=
-github.com/kataras/tablewriter v0.0.0-20180708051242-e063d29b7c23/go.mod h1:kBSna6b0/RzsOcOZf515vAXwSsXYusl2U7SA0XP09yI=
 github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
 github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
 github.com/klauspost/compress v1.12.3 h1:G5AfA94pHPysR56qqrkO2pxEexdDzrpFJ6yt/VqWxVU=
 github.com/klauspost/compress v1.12.3/go.mod h1:8dP1Hq4DHOhN9w426knH3Rhby4rFm6D8eO+e+Dq5Gzg=
+github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
+github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
+github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
+github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
 github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA=
 github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg=
 github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
@@ -57,6 +67,7 @@ github.com/mattn/go-runewidth v0.0.15 h1:UNAjwbU9l54TA3KzvqLGxwWjHmMgBUVhBiTjelZ
 github.com/mattn/go-runewidth v0.0.15/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=
 github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
 github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/rivo/uniseg v0.2.0 h1:S1pD9weZBuJdFmowNwbpi7BJ8TNftyUImj/0WQi72jY=
 github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
@@ -65,17 +76,20 @@ github.com/rodaine/table v1.1.1/go.mod h1:iqTRptjn+EVcrVBYtNMlJ2wrJZa3MpULUmcXFp
 github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg=
 github.com/rs/zerolog v1.31.0 h1:FcTR3NnLWW+NnTwwhFWiJSZr4ECLpqCm6QsEnyvbV4A=
 github.com/rs/zerolog v1.31.0/go.mod h1:/7mN4D5sKwJLZQ2b/znpjC3/GQWY/xaDXUM0kKWRHss=
-github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk=
-github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
+github.com/samber/do v1.6.0 h1:Jy/N++BXINDB6lAx5wBlbpHlUdl0FKpLWgGEV9YWqaU=
+github.com/samber/do v1.6.0/go.mod h1:DWqBvumy8dyb2vEnYZE7D7zaVEB64J45B0NjTlY/M4k=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
 github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
 github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
 github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
+github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk=
 github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
-github.com/urfave/cli v1.22.14 h1:ebbhrRiGK2i4naQJr+1Xj92HXZCrK7MsyTS/ob3HnAk=
-github.com/urfave/cli v1.22.14/go.mod h1:X0eDS6pD6Exaclxm99NJ3FiCDRED7vIHpx2mDOHLvkA=
+github.com/urfave/cli/v3 v3.0.0-alpha8 h1:H+qxFPoCkGzdF8KUMs2fEOZl5io/1QySgUiGfar8occ=
+github.com/urfave/cli/v3 v3.0.0-alpha8/go.mod h1:0kK/RUFHyh+yIKSfWxwheGndfnrvYSmYFVeKCh03ZUc=
+github.com/xrash/smetrics v0.0.0-20231213231151-1d8dd44e695e h1:+SOyEddqYF09QP7vr7CgJ1eti3pY9Fn3LHO1M1r/0sI=
+github.com/xrash/smetrics v0.0.0-20231213231151-1d8dd44e695e/go.mod h1:N3UwUGtsrSj3ccvlPHLoLsHnpR27oXr4ZE984MbSER8=
 github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
@@ -103,8 +117,10 @@ golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLL
 golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
 golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
+golang.org/x/net v0.0.0-20210916014120-12bc252f5db8/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
 golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
+golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
 golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg=
 golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM=
 golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE=
@@ -123,6 +139,7 @@ golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
@@ -141,10 +158,12 @@ golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo=
 golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
 golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
 golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
 golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
+golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4=
 golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
@@ -170,8 +189,10 @@ google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQ
 google.golang.org/protobuf v1.28.1 h1:d0NfwRgPtno5B1Wa6L2DAG+KivqkdutMf1UhdNx175w=
 google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo=
+gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
-gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
 gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=

internal/config/badger.go

@@ -1,5 +1,7 @@
 package config
 
 type Badger struct {
-	Path string `json:"path"`
+	Debug    bool    `toml:"debug"`
+	Dir      string  `toml:"dir"`
+	ValueDir *string `toml:"value_dir"`
 }

internal/config/eway.go

@@ -0,0 +1,11 @@
+package config
+
+type Eway struct {
+	Login       string `toml:"login"`
+	Password    string `toml:"password"`
+	SessionID   string `toml:"session_id"`
+	SessionUser string `toml:"session_user"`
+	OwnerID     string `toml:"owner_id"`
+	Debug       bool   `toml:"debug"`
+	WorkersPool int    `toml:"workers_pool"`
+}

internal/config/log.go

@@ -0,0 +1,56 @@
+package config
+
+import (
+	"strings"
+
+	"git.loyso.art/frx/eway/internal/entity"
+)
+
+type LogLevel uint8
+
+const (
+	LogLevelDebug LogLevel = iota
+	LogLevelInfo
+	LogLevelWarn
+)
+
+func (l *LogLevel) UnmarshalText(data []byte) (err error) {
+	switch strings.ToLower(string(data)) {
+	case "debug":
+		*l = LogLevelDebug
+	case "info":
+		*l = LogLevelInfo
+	case "warn":
+		*l = LogLevelWarn
+	default:
+		return entity.SimpleError("unsupported level " + string(data))
+	}
+
+	return nil
+}
+
+type LogFormat uint8
+
+const (
+	LogFormatText LogFormat = iota
+	LogFormatJSON
+)
+
+func (l *LogFormat) UnmarshalText(data []byte) (err error) {
+	switch strings.ToLower(string(data)) {
+	case "text":
+		*l = LogFormatText
+	case "info":
+		*l = LogFormatJSON
+	default:
+		return entity.SimpleError("unsupported format " + string(data))
+	}
+
+	return nil
+}
+
+type Log struct {
+	Level  string `json:"level" toml:"level"`
+	Format string `json:"format" toml:"format"`
+	Output string `json:"output" toml:"output"`
+}

internal/config/matcher.go

@@ -0,0 +1,42 @@
+package config
+
+import (
+	"errors"
+	"strings"
+)
+
+type MatcherPredicateType uint8
+
+const (
+	MatcherPredicateTypeUnknown MatcherPredicateType = iota
+	MatcherPredicateTypePattern
+	MatcherPredicateTypeRegexp
+)
+
+func (t *MatcherPredicateType) UnmarshalText(data []byte) error {
+	switch dataStr := strings.ToLower(string(data)); dataStr {
+	case "":
+		*t = MatcherPredicateTypeUnknown
+	case "pattern":
+		*t = MatcherPredicateTypePattern
+	case "regexp":
+		*t = MatcherPredicateTypeRegexp
+	default:
+		return errors.New("unsupported type " + dataStr)
+	}
+
+	return nil
+}
+
+type MatcherPredicate struct {
+	Value string               `toml:"value"`
+	Type  MatcherPredicateType `toml:"type"`
+}
+
+type DimensionMatcher struct {
+	CaseInsensitive bool `toml:"case_insensitive"`
+
+	Length []MatcherPredicate `toml:"length"`
+	Width  []MatcherPredicate `toml:"width"`
+	Height []MatcherPredicate `toml:"height"`
+}

internal/crypto/cipher.go

@@ -0,0 +1,32 @@
+package crypto
+
+import (
+	"crypto/aes"
+	"crypto/cipher"
+	"crypto/rand"
+	"encoding/hex"
+	"fmt"
+)
+
+var (
+	someDumbKey = []byte("9530e001b619e8e98a889055f06821bb")
+)
+
+func Encrypt(plaintext string) (hexed string, err error) {
+	aes, err := aes.NewCipher(someDumbKey)
+	if err != nil {
+		return "", fmt.Errorf("making new cipher: %w", err)
+	}
+	gcm, err := cipher.NewGCM(aes)
+	if err != nil {
+		return "", fmt.Errorf("making new gcm: %w", err)
+	}
+
+	nonce := make([]byte, gcm.NonceSize())
+	_, err = rand.Read(nonce)
+	if err != nil {
+		return "", fmt.Errorf("generating nonce: %w", err)
+	}
+	outvalue := gcm.Seal(nonce, nonce, []byte(plaintext), nil)
+	return hex.EncodeToString(outvalue), nil
+}

internal/crypto/decrypt.go

@@ -0,0 +1,32 @@
+package crypto
+
+import (
+	"crypto/aes"
+	"crypto/cipher"
+	"encoding/hex"
+	"fmt"
+)
+
+func Decrypt(hexedcipher string) (plaintext string, err error) {
+	aes, err := aes.NewCipher(someDumbKey)
+	if err != nil {
+		return "", fmt.Errorf("making new cipher: %w", err)
+	}
+	gcm, err := cipher.NewGCM(aes)
+	if err != nil {
+		return "", fmt.Errorf("making new gcm: %w", err)
+	}
+
+	nonceSize := gcm.NonceSize()
+	valueDecoded, err := hex.DecodeString(hexedcipher)
+	if err != nil {
+		return "", fmt.Errorf("decoding hexed value: %w", err)
+	}
+	nonce, cipherText := valueDecoded[:nonceSize], valueDecoded[nonceSize:]
+	plaintextRaw, err := gcm.Open(nil, []byte(nonce), []byte(cipherText), nil)
+	if err != nil {
+		return "", fmt.Errorf("opening: %w", err)
+	}
+
+	return string(plaintextRaw), nil
+}

internal/dimension/matcher.go

@@ -0,0 +1,96 @@
+package dimension
+
+import (
+	"git.loyso.art/frx/eway/internal/config"
+	"git.loyso.art/frx/eway/internal/matcher"
+)
+
+type MatchResult uint8
+
+const (
+	MatchResultMiss MatchResult = iota
+	MatchResultLength
+	MatchResultHeight
+	MatchResultWidth
+	MatchResultDepth
+)
+
+type Matcher struct {
+	length matcher.Unit
+	width  matcher.Unit
+	height matcher.Unit
+}
+
+func New(cfg config.DimensionMatcher) *Matcher {
+	return &Matcher{
+		length: makeMatcherByConig(cfg.CaseInsensitive, cfg.Length...),
+		width:  makeMatcherByConig(cfg.CaseInsensitive, cfg.Width...),
+		height: makeMatcherByConig(cfg.CaseInsensitive, cfg.Height...),
+	}
+}
+
+func makeMatcherByConig(insensitive bool, cfgs ...config.MatcherPredicate) matcher.Unit {
+	opts := make([]matcher.RadixOpt, 0, 1)
+	if insensitive {
+		opts = append(opts, matcher.RadixCaseInsensitive())
+	}
+	m := matcher.NewRadix(opts...)
+
+	for _, cfg := range cfgs {
+		switch cfg.Type {
+		case config.MatcherPredicateTypePattern:
+			m.Register(cfg.Value)
+		case config.MatcherPredicateTypeRegexp:
+			m.RegisterRegexp(cfg.Value)
+		default:
+			panic("unsupported matcher type")
+		}
+	}
+
+	return m
+}
+
+func (m *Matcher) Match(value string) (r MatchResult, priority bool) {
+	switch {
+	case value == "Высота":
+		priority = true
+		fallthrough
+	case m.height.Match(value):
+		return MatchResultHeight, priority
+	case value == "Глубина":
+		priority = true
+		return MatchResultDepth, priority
+	case value == "Длина":
+		priority = true
+		fallthrough
+	case m.length.Match(value):
+		return MatchResultLength, priority
+	case value == "Ширина":
+		priority = true
+		fallthrough
+	case m.width.Match(value):
+		return MatchResultWidth, priority
+	}
+
+	return MatchResultMiss, false
+}
+
+func (m *Matcher) GetRegisteredPatterns() map[string][]string {
+	out := map[string][]string{
+		"length": nil,
+		"width":  nil,
+		"height": nil,
+	}
+
+	if m.height != nil {
+		out["height"] = m.height.Patterns()
+	}
+	if m.width != nil {
+		out["width"] = m.width.Patterns()
+	}
+	if m.length != nil {
+		out["length"] = m.length.Patterns()
+	}
+
+	return out
+}

internal/encoding/fbs/Dimensions.go

@@ -0,0 +1,49 @@
+// Code generated by the FlatBuffers compiler. DO NOT EDIT.
+
+package fbs
+
+import (
+	flatbuffers "github.com/google/flatbuffers/go"
+)
+
+type Dimensions struct {
+	_tab flatbuffers.Struct
+}
+
+func (rcv *Dimensions) Init(buf []byte, i flatbuffers.UOffsetT) {
+	rcv._tab.Bytes = buf
+	rcv._tab.Pos = i
+}
+
+func (rcv *Dimensions) Table() flatbuffers.Table {
+	return rcv._tab.Table
+}
+
+func (rcv *Dimensions) Width() float32 {
+	return rcv._tab.GetFloat32(rcv._tab.Pos + flatbuffers.UOffsetT(0))
+}
+func (rcv *Dimensions) MutateWidth(n float32) bool {
+	return rcv._tab.MutateFloat32(rcv._tab.Pos+flatbuffers.UOffsetT(0), n)
+}
+
+func (rcv *Dimensions) Height() float32 {
+	return rcv._tab.GetFloat32(rcv._tab.Pos + flatbuffers.UOffsetT(4))
+}
+func (rcv *Dimensions) MutateHeight(n float32) bool {
+	return rcv._tab.MutateFloat32(rcv._tab.Pos+flatbuffers.UOffsetT(4), n)
+}
+
+func (rcv *Dimensions) Length() float32 {
+	return rcv._tab.GetFloat32(rcv._tab.Pos + flatbuffers.UOffsetT(8))
+}
+func (rcv *Dimensions) MutateLength(n float32) bool {
+	return rcv._tab.MutateFloat32(rcv._tab.Pos+flatbuffers.UOffsetT(8), n)
+}
+
+func CreateDimensions(builder *flatbuffers.Builder, width float32, height float32, length float32) flatbuffers.UOffsetT {
+	builder.Prep(4, 12)
+	builder.PrependFloat32(length)
+	builder.PrependFloat32(height)
+	builder.PrependFloat32(width)
+	return builder.Offset()
+}

internal/encoding/fbs/GoodItem.go

@@ -169,8 +169,41 @@ func (rcv *GoodItem) MutateStock(n int16) bool {
 	return rcv._tab.MutateInt16Slot(28, n)
 }
 
+func (rcv *GoodItem) Parameters() []byte {
+	o := flatbuffers.UOffsetT(rcv._tab.Offset(30))
+	if o != 0 {
+		return rcv._tab.ByteVector(o + rcv._tab.Pos)
+	}
+	return nil
+}
+
+func (rcv *GoodItem) CreatedAt() int64 {
+	o := flatbuffers.UOffsetT(rcv._tab.Offset(32))
+	if o != 0 {
+		return rcv._tab.GetInt64(o + rcv._tab.Pos)
+	}
+	return 0
+}
+
+func (rcv *GoodItem) MutateCreatedAt(n int64) bool {
+	return rcv._tab.MutateInt64Slot(32, n)
+}
+
+func (rcv *GoodItem) Sizes(obj *Dimensions) *Dimensions {
+	o := flatbuffers.UOffsetT(rcv._tab.Offset(34))
+	if o != 0 {
+		x := o + rcv._tab.Pos
+		if obj == nil {
+			obj = new(Dimensions)
+		}
+		obj.Init(rcv._tab.Bytes, x)
+		return obj
+	}
+	return nil
+}
+
 func GoodItemStart(builder *flatbuffers.Builder) {
-	builder.StartObject(13)
+	builder.StartObject(16)
 }
 func GoodItemAddSku(builder *flatbuffers.Builder, sku flatbuffers.UOffsetT) {
 	builder.PrependUOffsetTSlot(0, flatbuffers.UOffsetT(sku), 0)
@@ -211,6 +244,15 @@ func GoodItemAddCart(builder *flatbuffers.Builder, cart int64) {
 func GoodItemAddStock(builder *flatbuffers.Builder, stock int16) {
 	builder.PrependInt16Slot(12, stock, 0)
 }
+func GoodItemAddParameters(builder *flatbuffers.Builder, parameters flatbuffers.UOffsetT) {
+	builder.PrependUOffsetTSlot(13, flatbuffers.UOffsetT(parameters), 0)
+}
+func GoodItemAddCreatedAt(builder *flatbuffers.Builder, createdAt int64) {
+	builder.PrependInt64Slot(14, createdAt, 0)
+}
+func GoodItemAddSizes(builder *flatbuffers.Builder, sizes flatbuffers.UOffsetT) {
+	builder.PrependStructSlot(15, flatbuffers.UOffsetT(sizes), 0)
+}
 func GoodItemEnd(builder *flatbuffers.Builder) flatbuffers.UOffsetT {
 	return builder.EndObject()
 }

internal/encoding/fbs/helpers.go

@@ -1,7 +1,12 @@
 package fbs
 
 import (
+	"encoding/base64"
+	"encoding/json"
+	"fmt"
+	"strings"
 	"sync"
+	"time"
 
 	"git.loyso.art/frx/eway/internal/entity"
 
@@ -21,8 +26,8 @@ func getBuilder() *flatbuffers.Builder {
 }
 
 func putBuilder(builder *flatbuffers.Builder) {
-	builder.Reset()
-	builderPool.Put(builder)
+	// builder.Reset()
+	// builderPool.Put(builder)
 }
 
 func MakeDomainGoodItems(in ...entity.GoodsItem) []byte {
@@ -60,16 +65,30 @@ func MakeDomainGoodItemFinished(in entity.GoodsItem) []byte {
 
 func makeDomainGoodItem(builder *flatbuffers.Builder, in entity.GoodsItem) flatbuffers.UOffsetT {
 	sku := builder.CreateString(in.Articul)
-	photo := builder.CreateString(in.Photo)
+	photo := builder.CreateString(strings.Join(in.PhotoURLs, ";"))
 	name := builder.CreateString(in.Name)
-	desc := builder.CreateString(in.Description)
+
+	descBase64 := base64.RawStdEncoding.EncodeToString([]byte(in.Description))
+	desc := builder.CreateString(descBase64)
+
 	var cat flatbuffers.UOffsetT
 	if in.Category != "" {
 		cat = builder.CreateString(in.Category)
 	}
 	t := builder.CreateString(in.Type)
+
+	parametersData, _ := json.Marshal(in.Parameters)
+	parameters := builder.CreateByteString(parametersData)
+
 	producer := builder.CreateString(in.Producer)
 
+	var w, h, l float32
+	if in.Sizes != (entity.GoodsItemSize{}) {
+		w = float32(in.Sizes.Width.AdjustTo(entity.DimensionKindCentimeter).Value)
+		h = float32(in.Sizes.Height.AdjustTo(entity.DimensionKindCentimeter).Value)
+		l = float32(in.Sizes.Length.AdjustTo(entity.DimensionKindCentimeter).Value)
+	}
+
 	GoodItemStart(builder)
 	GoodItemAddSku(builder, sku)
 	GoodItemAddPhoto(builder, photo)
@@ -86,16 +105,28 @@ func makeDomainGoodItem(builder *flatbuffers.Builder, in entity.GoodsItem) flatb
 	GoodItemAddTariff(builder, float32(in.TariffPrice))
 	GoodItemAddCart(builder, int64(in.Cart))
 	GoodItemAddStock(builder, int16(in.Stock))
+	GoodItemAddParameters(builder, parameters)
+	GoodItemAddCreatedAt(builder, in.CreatedAt.Unix())
+	GoodItemAddSizes(builder, CreateDimensions(builder, w, h, l))
 
 	return GoodItemEnd(builder)
 }
 
-func ParseGoodsItem(data []byte) (item entity.GoodsItem) {
+func ParseGoodsItem(data []byte) (item entity.GoodsItem, err error) {
 	itemFBS := GetRootAsGoodItem(data, 0)
 	item.Articul = string(itemFBS.Sku())
-	item.Photo = string(itemFBS.Photo())
+	photoURLs := string(itemFBS.Photo())
+	if len(photoURLs) > 0 {
+		item.PhotoURLs = strings.Split(photoURLs, ";")
+	}
 	item.Name = string(itemFBS.Name())
-	item.Description = string(itemFBS.Description())
+
+	description, err := base64.RawStdEncoding.DecodeString(string(itemFBS.Description()))
+	if err != nil {
+		return item, fmt.Errorf("decoding description from base64: %w", err)
+	}
+
+	item.Description = string(description)
 	if value := itemFBS.Category(); value != nil {
 		item.Category = string(value)
 	}
@@ -107,8 +138,35 @@ func ParseGoodsItem(data []byte) (item entity.GoodsItem) {
 	item.TariffPrice = float64(itemFBS.Tariff())
 	item.Cart = itemFBS.Cart()
 	item.Stock = int(itemFBS.Stock())
+	item.Parameters = map[string]string{}
+	parameters := itemFBS.Parameters()
+	if len(parameters) > 0 {
+		err = json.Unmarshal(itemFBS.Parameters(), &item.Parameters)
+		if err != nil {
+			return item, fmt.Errorf("unmarshalling data: %w", err)
+		}
+	}
 
-	return item
+	var w, h, l float64
+	sizes := itemFBS.Sizes(nil)
+	if sizes != nil {
+		w = float64(sizes.Width())
+		h = float64(sizes.Height())
+		l = float64(sizes.Length())
+	}
+
+	item.Sizes = entity.GoodsItemSize{
+		Width:  entity.NewCentimeterDimensionOrEmpty(w),
+		Height: entity.NewCentimeterDimensionOrEmpty(h),
+		Length: entity.NewCentimeterDimensionOrEmpty(l),
+	}
+
+	createdAt := itemFBS.CreatedAt()
+	if createdAt > 0 {
+		item.CreatedAt = time.Unix(createdAt, 0)
+	}
+
+	return item, nil
 }
 
 func ParseCategory(data []byte) (category entity.Category) {

internal/entity/dimension.go

@@ -0,0 +1,154 @@
+package entity
+
+import (
+	"fmt"
+	"strconv"
+	"strings"
+)
+
+var DefaultLocale = DimensionLocalRU
+
+type DimensionLocale uint8
+
+const (
+	DimensionLocalUnspecified DimensionLocale = iota
+	DimensionLocalRU
+
+	dimensionLocalEnd
+)
+
+type DimensionKind uint8
+
+func (k DimensionKind) GetPos() float64 {
+	switch k {
+	case DimensionKindMilimeter:
+		return 1
+	case DimensionKindCentimeter:
+		return 10
+	case DimensionKindMeter:
+		return 1000
+	default:
+		return 0
+	}
+}
+
+func (k DimensionKind) String() string {
+	m := getLocaleKindToStringMap()[DefaultLocale]
+	return m[k]
+}
+
+const (
+	DimensionKindUnspecified DimensionKind = iota
+	DimensionKindMilimeter
+	DimensionKindCentimeter
+	DimensionKindMeter
+)
+
+type Dimension struct {
+	Value float64
+	Kind  DimensionKind
+}
+
+func (d Dimension) IsZero() bool {
+	return d.Value == 0
+}
+
+func (d Dimension) MarshalText() ([]byte, error) {
+	if d.Value == 0 && d.Kind == DimensionKindUnspecified {
+		return nil, nil
+	}
+
+	value := strconv.FormatFloat(d.Value, 'f', 4, 64) + " " + d.Kind.String()
+	return []byte(value), nil
+}
+
+func (d *Dimension) UnmarshalText(data []byte) (err error) {
+	*d, err = ParseDimention(string(data), DefaultLocale)
+	return err
+}
+
+func (d Dimension) AdjustTo(kind DimensionKind) Dimension {
+	from := d.Kind.GetPos()
+	to := kind.GetPos()
+
+	switch {
+	case from < to:
+		mult := to / from
+		return Dimension{
+			Kind:  kind,
+			Value: d.Value / float64(mult),
+		}
+	case from > to:
+		mult := from / to
+		return Dimension{
+			Kind:  kind,
+			Value: d.Value * float64(mult),
+		}
+	}
+
+	return d
+}
+
+func ParseDimention(value string, locale DimensionLocale) (Dimension, error) {
+	switch locale {
+	case DimensionLocalRU:
+	default:
+		return Dimension{}, SimpleError("unknown locale for parse")
+	}
+
+	dimensionStrToKind := getLocaleToKindMap()[locale]
+	lastSpaceIdx := strings.LastIndex(value, " ")
+	if lastSpaceIdx == -1 {
+		return Dimension{}, SimpleError("expected 2 values after split for value " + value)
+	}
+
+	var splitted [2]string
+	splitted[0] = strings.ReplaceAll(value[:lastSpaceIdx], " ", "")
+	splitted[1] = value[lastSpaceIdx+1:]
+
+	var out Dimension
+	var ok bool
+	out.Kind, ok = dimensionStrToKind[splitted[1]]
+	if !ok {
+		return Dimension{}, SimpleError("dimension map not found for kind " + splitted[1])
+	}
+
+	var err error
+	out.Value, err = strconv.ParseFloat(splitted[0], 64)
+	if err != nil {
+		return Dimension{}, fmt.Errorf("parsing value: %w", err)
+	}
+
+	return out, nil
+}
+
+func NewCentimeterDimensionOrEmpty(value float64) Dimension {
+	if value == 0 {
+		return Dimension{}
+	}
+
+	return Dimension{
+		Value: value,
+		Kind:  DimensionKindCentimeter,
+	}
+}
+
+func getLocaleToKindMap() map[DimensionLocale]map[string]DimensionKind {
+	return map[DimensionLocale]map[string]DimensionKind{
+		DimensionLocalRU: {
+			"мм": DimensionKindMilimeter,
+			"см": DimensionKindCentimeter,
+			"м":  DimensionKindMeter,
+		},
+	}
+}
+
+func getLocaleKindToStringMap() map[DimensionLocale]map[DimensionKind]string {
+	return map[DimensionLocale]map[DimensionKind]string{
+		DimensionLocalRU: {
+			DimensionKindMilimeter:  "мм",
+			DimensionKindCentimeter: "см",
+			DimensionKindMeter:      "м",
+		},
+	}
+}

internal/entity/dimension_inner_test.go

@@ -0,0 +1,31 @@
+package entity
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestLocaleMap(t *testing.T) {
+	kindToStr := getLocaleKindToStringMap()
+	strToKind := getLocaleToKindMap()
+
+	assert := assert.New(t)
+	for locale := DimensionLocalUnspecified + 1; locale < dimensionLocalEnd; locale++ {
+		localeKinds, ok := kindToStr[locale]
+		assert.True(ok)
+		localeStrs, ok := strToKind[locale]
+		assert.True(ok)
+
+		assert.Equal(len(localeKinds), len(localeStrs))
+
+		for kindKey, kindValue := range localeKinds {
+			strKey := kindValue
+			strValue, ok := localeStrs[strKey]
+			assert.True(ok)
+
+			assert.Equal(kindKey, strValue)
+			assert.Equal(strKey, kindValue)
+		}
+	}
+}

internal/entity/dimension_test.go

@@ -0,0 +1,72 @@
+package entity_test
+
+import (
+	"errors"
+	"testing"
+
+	"git.loyso.art/frx/eway/internal/entity"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestDimension_AdjustTo(t *testing.T) {
+	// Test adjusting from smaller dimension to larger one
+	d := entity.Dimension{Value: 5.0, Kind: entity.DimensionKindCentimeter}
+	expected := entity.Dimension{Value: 0.05, Kind: entity.DimensionKindMeter}
+	actual := d.AdjustTo(entity.DimensionKindMeter)
+
+	assert.EqualValues(t, expected.Value, actual.Value)
+	assert.Equal(t, expected.Kind, actual.Kind)
+
+	// Test adjusting from larger dimension to smaller one
+	d = entity.Dimension{Value: 0.05, Kind: entity.DimensionKindMeter}
+	expected = entity.Dimension{Value: 50.0, Kind: entity.DimensionKindMilimeter}
+	actual = d.AdjustTo(entity.DimensionKindMilimeter)
+
+	assert.EqualValues(t, expected.Value, actual.Value)
+	assert.Equal(t, expected.Kind, actual.Kind)
+}
+
+func TestParseDimension_Success(t *testing.T) {
+	// Test parsing a valid dimension string with RU locale
+	input := "10 см"
+	expected := entity.Dimension{Value: 10.0, Kind: entity.DimensionKindCentimeter}
+
+	actual, err := entity.ParseDimention(input, entity.DimensionLocalRU)
+	require.NoError(t, err)
+
+	assert.EqualValues(t, expected.Value, actual.Value)
+	assert.Equal(t, expected.Kind, actual.Kind)
+}
+
+func TestParseDimensionComplex_Success(t *testing.T) {
+	// Test parsing a valid dimension string with RU locale
+	input := "10 256.20 см"
+	expected := entity.Dimension{Value: 10256.20, Kind: entity.DimensionKindCentimeter}
+
+	actual, err := entity.ParseDimention(input, entity.DimensionLocalRU)
+	require.NoError(t, err)
+
+	assert.EqualValues(t, expected.Value, actual.Value)
+	assert.Equal(t, expected.Kind, actual.Kind)
+}
+
+func TestParseDimension_InvalidInputFormat(t *testing.T) {
+	// Test parsing an invalid dimension string with RU locale
+	input := "invalid value 2"
+	expectedErr := errors.New("expected 2 values after split for value invalid value 2")
+
+	_, err := entity.ParseDimention(input, entity.DimensionLocalRU)
+	assert.Error(t, err)
+	assert.EqualError(t, err, expectedErr.Error())
+}
+
+func TestParseDimension_InvalidLocale(t *testing.T) {
+	// Test parsing a dimension string with an unsupported locale
+	input := "10 мм"
+	expectedErr := errors.New("unknown locale for parse")
+
+	_, err := entity.ParseDimention(input, 3) // An invalid locale value is used here for demonstration purposes
+	assert.EqualError(t, err, expectedErr.Error())
+}

internal/entity/empty.go

@@ -0,0 +1,3 @@
+package entity
+
+type Empty struct{}

internal/entity/error.go

@@ -7,5 +7,6 @@ func (err SimpleError) Error() string {
 }
 
 const (
-	ErrNotFound SimpleError = "not found"
+	ErrNotFound       SimpleError = "not found"
+	ErrNotImplemented SimpleError = "not implemented"
 )

internal/entity/gooditem.go

@@ -4,23 +4,101 @@ import (
 	"fmt"
 	"strconv"
 	"strings"
+	"time"
 	"unicode"
 )
 
+type GoodsItemSize struct {
+	Width  Dimension
+	Height Dimension
+	Length Dimension
+
+	UnmatchedDepth Dimension
+}
+
+func FixupSizes(s GoodsItemSize) (GoodsItemSize, bool) {
+	// Nothing to substitute
+	if s.UnmatchedDepth.IsZero() {
+		return s, false
+	}
+
+	var count int
+	for _, d := range []Dimension{
+		s.Width,
+		s.Height,
+		s.Length,
+	} {
+		if d.IsZero() {
+			count++
+		}
+	}
+
+	// Can only replace one dimension
+	if count != 1 {
+		return s, false
+	}
+
+	switch {
+	case s.Width.IsZero():
+		s.Width = s.UnmatchedDepth
+	case s.Height.IsZero():
+		s.Height = s.UnmatchedDepth
+	case s.Length.IsZero():
+		s.Length = s.UnmatchedDepth
+	}
+
+	s.UnmatchedDepth = Dimension{}
+
+	return s, true
+}
+
+func (s GoodsItemSize) AllSizesSet() bool {
+	var count int
+	for _, d := range []Dimension{
+		s.Width,
+		s.Height,
+		s.Length,
+		s.UnmatchedDepth,
+	} {
+		if d.IsZero() {
+			count++
+		}
+	}
+
+	return count >= 3
+}
+
+func (s GoodsItemSize) GetSum(kind DimensionKind) float64 {
+	var value float64
+	sum := func(ds ...Dimension) {
+		for _, d := range ds {
+			value += d.AdjustTo(kind).Value
+
+		}
+	}
+
+	sum(s.Height, s.Length, s.Length)
+
+	return value
+}
+
 type GoodsItem struct {
-	Articul     string  `json:"sku"`
-	Photo       string  `json:"photo"`
-	Name        string  `json:"name"`
-	Description string  `json:"description"`
-	Category    string  `json:"category"`
-	Type        string  `json:"type"`
-	Producer    string  `json:"producer"`
-	Pack        int     `json:"pack"`
-	Step        int     `json:"step"`
-	Price       float64 `json:"price"`
-	TariffPrice float64 `json:"tariff_price"`
-	Cart        int64   `json:"cart"`
-	Stock       int     `json:"stock"`
+	Articul     string            `json:"sku"`
+	PhotoURLs   []string          `json:"photo"`
+	Name        string            `json:"name"`
+	Description string            `json:"description"`
+	Category    string            `json:"category"`
+	Type        string            `json:"type"`
+	Producer    string            `json:"producer"`
+	Pack        int               `json:"pack"`
+	Step        int               `json:"step"`
+	Price       float64           `json:"price"`
+	TariffPrice float64           `json:"tariff_price"`
+	Cart        int64             `json:"cart"`
+	Stock       int               `json:"stock"`
+	Sizes       GoodsItemSize     `json:"sizes"`
+	Parameters  map[string]string `json:"parameters"`
+	CreatedAt   time.Time         `json:"created_at"`
 }
 
 type GoodsItemRaw struct {
@@ -42,21 +120,18 @@ type GoodsItemRaw struct {
 	Other         string
 }
 
+type GoodsItemInfo struct {
+	Parameters map[string]string
+	PhotoURLs  []string
+}
+
 type MappedGoodsRemnants map[int]GoodsRemnant
 type GoodsRemnant [4]int32
 
-func ExtractProductIDs(items []GoodsItem) (out []int) {
-	out = make([]int, 0, len(items))
-	for _, item := range items {
-		out = append(out, int(item.Cart))
-	}
-
-	return out
-}
-
 func MakeGoodsItem(
 	gi GoodsItemRaw,
 	remnants MappedGoodsRemnants,
+	info GoodsItemInfo,
 ) (out GoodsItem, err error) {
 	var name, desc string
 	var pack, step int
@@ -104,9 +179,13 @@ func MakeGoodsItem(
 		return out, fmt.Errorf("getting step count (%s): %w", gi.Step, err)
 	}
 
+	photoURLs := info.PhotoURLs
+	if len(photoURLs) > 7 {
+		photoURLs = info.PhotoURLs[:7]
+	}
+
 	return GoodsItem{
 		Articul:     gi.SKU,
-		Photo:       gi.Photo,
 		Name:        name,
 		Description: desc,
 		Category:    gi.Category,
@@ -118,5 +197,7 @@ func MakeGoodsItem(
 		TariffPrice: tariffPrice,
 		Cart:        int64(gi.Cart),
 		Stock:       int(remnants[int(gi.Cart)][0]),
+		PhotoURLs:   photoURLs,
+		Parameters:  info.Parameters,
 	}, nil
 }

internal/entity/iter.go

@@ -0,0 +1,59 @@
+package entity
+
+func IterIntoMap[K comparable, V any](v []V, err error) iterIntoMap[K, V] {
+	bi := IterWithErr(v, err)
+
+	return iterIntoMap[K, V]{
+		baseIter: bi,
+	}
+}
+
+type iterIntoMap[K comparable, V any] struct {
+	baseIter[V]
+}
+
+func (i iterIntoMap[K, V]) Map(f func(V) (K, error)) (map[K]V, error) {
+	if i.err != nil {
+		return nil, i.err
+	}
+
+	out := make(map[K]V, len(i.items))
+	for _, item := range i.items {
+		var key K
+		key, i.err = f(item)
+		if i.err != nil {
+			return nil, i.err
+		}
+
+		out[key] = item
+	}
+
+	return out, nil
+}
+
+func IterWithErr[T any](t []T, err error) baseIter[T] {
+	return baseIter[T]{
+		items: t,
+		err:   err,
+	}
+}
+
+type baseIter[T any] struct {
+	items []T
+	err   error
+}
+
+func (iter baseIter[T]) Do(f func(T) error) error {
+	if iter.err != nil {
+		return iter.err
+	}
+
+	for _, item := range iter.items {
+		err := f(item)
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}

internal/entity/repository.go

@@ -9,6 +9,7 @@ type GoodsItemRepository interface {
 	GetByCart(context.Context, int64) (GoodsItem, error)
 
 	UpsertMany(context.Context, ...GoodsItem) ([]GoodsItem, error)
+	Delete(context.Context, string) (GoodsItem, error)
 }
 
 type CategoryRepository interface {

internal/entity/task.go

@@ -0,0 +1,23 @@
+package entity
+
+import (
+	"context"
+	"time"
+)
+
+type PublishParams struct {
+	Body      []byte
+	ExpiresAt time.Time
+}
+
+type MessageQueue interface {
+	Publish(context.Context, PublishParams) (Task, error)
+	Consume(context.Context) (Task, error)
+}
+
+type Task struct {
+	ID        uint64
+	CreatedAt time.Time
+	ExpiresAt *time.Time
+	Body      []byte
+}

internal/export/itemsmarket.go

@@ -0,0 +1,62 @@
+package export
+
+import "time"
+
+type Param struct {
+	Name  string `xml:"name,attr"`
+	Value string `xml:",chardata"`
+}
+
+type Offer struct {
+	ID        int64  `xml:"id,attr"`
+	Type      string `xml:"type,attr"`
+	Available bool   `xml:"available,attr"`
+
+	URL                 string   `xml:"url"`
+	Price               int      `xml:"price"`
+	CurrencyID          string   `xml:"currencyId"`
+	CategoryID          int64    `xml:"categoryId"`
+	PictureURLs         []string `xml:"picture"`
+	Vendor              string   `xml:"vendor"`
+	Model               string   `xml:"model"`
+	VendorCode          string   `xml:"vendorCode"`
+	TypePrefix          string   `xml:"typePrefix"`
+	Description         string   `xml:"description"`
+	ManufacturerWarrany bool     `xml:"manufacturer_warranty"`
+	Dimensions          string   `xml:"dimensions"`
+	Params              []Param  `xml:"param,omitempty"`
+}
+
+type Currency struct {
+	ID   string `xml:"id,attr"`   // RUR only
+	Rate int64  `xml:"rate,attr"` // 1?
+}
+
+type Category struct {
+	ID       int64  `xml:"id,attr"`
+	ParentID int64  `xml:"parent_id,attr,omitempty"`
+	Name     string `xml:",chardata"`
+}
+
+type Shop struct {
+	Name       string     `xml:"name"`    // r
+	Company    string     `xml:"company"` // r
+	URL        string     `xml:"url"`     // r
+	Platform   string     `xml:"platform"`
+	Version    string     `xml:"version"`
+	Currencies []Currency `xml:"currencies"`          // r RUR only
+	Categories []Category `xml:"categories>category"` // r
+
+	Offers []Offer `xml:"offer"` // r
+}
+
+type YmlContainer struct {
+	XMLName struct{} `xml:"yml_catalog"`
+
+	YmlCatalog
+}
+
+type YmlCatalog struct {
+	Date time.Time `xml:"date,attr"`
+	Shop Shop      `xml:"shop"`
+}

internal/export/itemsmarket_test.go

@@ -0,0 +1,82 @@
+package export
+
+import (
+	"encoding/xml"
+	"os"
+	"testing"
+	"time"
+
+	"github.com/brianvoe/gofakeit/v6"
+)
+
+func TestYMLSerialize(t *testing.T) {
+	faker := gofakeit.New(0)
+
+	categories := make([]Category, faker.Rand.Intn(4))
+	knownCategory := map[int64]struct{}{}
+	categoryIDs := make([]int64, 0, 10)
+	for i := range categories {
+		categories[i].ID = faker.Int64()
+		categories[i].Name = faker.HipsterWord()
+		categories[i].ParentID = faker.Int64()
+
+		if _, ok := knownCategory[categories[i].ID]; ok {
+			continue
+		}
+
+		knownCategory[categories[i].ID] = struct{}{}
+		categoryIDs = append(categoryIDs, categories[i].ID)
+	}
+
+	offers := make([]Offer, faker.Rand.Intn(5)+1)
+	for i := range offers {
+		offer := &offers[i]
+		offer.ID = faker.Int64()
+		offer.Type = "vendor.model"
+		offer.Available = true
+		offer.URL = faker.URL()
+		offer.Price = int(faker.Price(10, 1000))
+		offer.CurrencyID = "RUR"
+		offer.CategoryID = categoryIDs[faker.Rand.Intn(len(categoryIDs))]
+		for i := 0; i < faker.Rand.Intn(3); i++ {
+			offer.PictureURLs = append(offer.PictureURLs, faker.ImageURL(128, 128))
+		}
+		offer.Vendor = faker.Company()
+		offer.Model = faker.CarModel()
+		offer.VendorCode = faker.DigitN(8)
+		offer.TypePrefix = faker.ProductName()
+		offer.Description = faker.Sentence(12)
+		offer.ManufacturerWarrany = true
+		for i := 0; i < faker.Rand.Intn(8); i++ {
+			offer.Params = append(offer.Params, Param{
+				Name:  faker.AdjectiveProper(),
+				Value: faker.Digit(),
+			})
+		}
+	}
+
+	var catalog YmlCatalog
+	catalog.Shop = Shop{
+		Name:     faker.ProductName(),
+		URL:      faker.URL(),
+		Company:  faker.Company(),
+		Platform: "BSM/Yandex/Market",
+		Version:  faker.AppVersion(),
+		Currencies: []Currency{{
+			ID:   "RUR",
+			Rate: 1,
+		}},
+		Categories: categories,
+		Offers:     offers,
+	}
+	catalog.Date = faker.Date().Truncate(time.Second)
+
+	container := YmlContainer{
+		YmlCatalog: catalog,
+	}
+	enc := xml.NewEncoder(os.Stdout)
+	enc.Indent("", " ")
+	_ = enc.Encode(container)
+	println()
+	t.FailNow()
+}

internal/interconnect/eway/client.go

@@ -11,59 +11,104 @@ import (
 	"reflect"
 	"strconv"
 	"strings"
+	"sync"
+	"time"
 
+	"git.loyso.art/frx/eway/internal/config"
+	"git.loyso.art/frx/eway/internal/crypto"
 	"git.loyso.art/frx/eway/internal/entity"
 
+	"github.com/PuerkitoBio/goquery"
 	"github.com/go-resty/resty/v2"
 	"github.com/rs/zerolog"
 )
 
+type Client interface {
+	GetGoodsRemnants(context.Context, []int) (entity.MappedGoodsRemnants, error)
+	GetGoodsNew(
+		context.Context,
+		GetGoodsNewParams,
+	) (items []entity.GoodsItemRaw, total int, err error)
+	GetProductInfo(context.Context, int64) (entity.GoodsItemInfo, error)
+}
+
 type client struct {
 	http *resty.Client
 	log  zerolog.Logger
+
+	htmlParseSema     chan struct{}
+	releaseSemaDelay  time.Duration
+	getProductInfoBus chan getProductInfoRequest
+	ownerID           string
+	workersPool       int
+	workerswg         *sync.WaitGroup
 }
 
-func NewClientWithSession(sessionid, sessionuser string, log zerolog.Logger) client {
-	cookies := []*http.Cookie{
-		{
-			Name:     "session_id",
-			Value:    sessionid,
-			Domain:   "eway.elevel.ru",
-			HttpOnly: true,
-		},
-		{
-			Name:     "session_user",
-			Value:    sessionuser,
-			Domain:   "eway.elevel.ru",
-			HttpOnly: true,
-		},
-		{
-			Name:     "contract",
-			Value:    "6101",
-			Domain:   "eway.elevel.ru",
-			HttpOnly: true,
-		},
-	}
+type Config config.Eway
 
+func New(ctx context.Context, cfg Config, log zerolog.Logger) (*client, error) {
 	httpclient := resty.New().
-		SetDebug(false).
-		SetCookies(cookies).
+		SetDebug(cfg.Debug).
 		SetBaseURL("https://eway.elevel.ru/api")
 
-	return client{
-		http: httpclient,
-		log:  log.With().Str("client", "eway").Logger(),
+	c := client{
+		http:              httpclient,
+		log:               log.With().Str("client", "eway").Logger(),
+		htmlParseSema:     make(chan struct{}, 2),
+		getProductInfoBus: make(chan getProductInfoRequest),
+		releaseSemaDelay:  time.Second / 2,
+		workerswg:         &sync.WaitGroup{},
 	}
-}
 
-type getGoodsNewOrder struct {
-	Column int
-	Dir    string
+	for i := 0; i < cfg.WorkersPool; i++ {
+		c.workerswg.Add(1)
+		go func() {
+			defer c.workerswg.Done()
+			c.productInfoWorker(ctx, c.getProductInfoBus)
+		}()
+	}
+
+	if cfg.SessionID == "" || cfg.SessionUser == "" {
+		if cfg.Login == "" || cfg.Password == "" {
+			return nil, entity.SimpleError("no auth method provided")
+		}
+
+		decryptedPassword, err := crypto.Decrypt(cfg.Password)
+		if err != nil {
+			return nil, fmt.Errorf("decrypting password: %w", err)
+		}
+		err = c.login(ctx, cfg.Login, decryptedPassword)
+		if err != nil {
+			return nil, err
+		}
+
+		log.Info().Msg("login successful")
+	} else if cfg.SessionID != "" && cfg.SessionUser != "" {
+		cookies := []*http.Cookie{
+			{
+				Name:     "session_id",
+				Value:    cfg.SessionID,
+				Domain:   "eway.elevel.ru",
+				HttpOnly: true,
+			},
+			{
+				Name:     "session_user",
+				Value:    cfg.SessionUser,
+				Domain:   "eway.elevel.ru",
+				HttpOnly: true,
+			},
+		}
+
+		c.http.SetCookies(cookies)
+	} else {
+		return nil, entity.SimpleError("bad configuration: either session_id and session_user should be set or login and password")
+	}
+
+	return &c, nil
 }
 
 type GetGoodsNewParams struct {
 	Draw  int
-	Order getGoodsNewOrder
 	Start int
 	// 100 is max
 	Length          int
@@ -79,8 +124,6 @@ type getGoodsNewResponse struct {
 	Replacement     bool    `json:"replacement"`
 }
 
-type goodRemnant [4]int
-
 func parseGoodItem(items []any) (out entity.GoodsItemRaw) {
 	valueOf := reflect.ValueOf(&out).Elem()
 	typeOf := valueOf.Type()
@@ -135,7 +178,7 @@ func mapResponseByOrder(response getGoodsNewResponse) (items []entity.GoodsItemR
 	return items
 }
 
-func (c client) GetGoodsRemnants(
+func (c *client) GetGoodsRemnants(
 	ctx context.Context,
 	productIDs []int,
 ) (out entity.MappedGoodsRemnants, err error) {
@@ -148,12 +191,12 @@ func (c client) GetGoodsRemnants(
 		productsStr = append(productsStr, strconv.Itoa(sku))
 	}
 
-	resp, err := c.http.R().
+	req := c.http.R().
 		SetFormData(map[string]string{
 			"products": strings.Join(productsStr, ","),
 		}).
-		SetDoNotParseResponse(true).
-		Post("/goods_remnants")
+		SetDoNotParseResponse(true)
+	resp, err := c.do(ctx, "GetGoodsRemnants", req, resty.MethodPost, "/goods_remnants")
 	if err != nil {
 		return nil, fmt.Errorf("getting goods new: %w", err)
 	}
@@ -173,8 +216,6 @@ func (c client) GetGoodsRemnants(
 		return nil, fmt.Errorf("reading raw body: %w", err)
 	}
 
-	c.log.Debug().RawJSON("response", data).Msg("body prepared")
-
 	out = make(entity.MappedGoodsRemnants, len(productIDs))
 	err = json.NewDecoder(bytes.NewReader(data)).Decode(&out)
 	if err != nil {
@@ -184,27 +225,41 @@ func (c client) GetGoodsRemnants(
 	return out, nil
 }
 
-func (c client) GetGoodsNew(
+func (c *client) GetGoodsNew(
 	ctx context.Context,
 	params GetGoodsNewParams,
 ) (items []entity.GoodsItemRaw, total int, err error) {
 	var response getGoodsNewResponse
-	resp, err := c.http.R().
-		SetFormData(map[string]string{
-			"draw":             strconv.Itoa(params.Draw),
-			"start":            strconv.Itoa(params.Start),
-			"length":           strconv.Itoa(params.Length),
-			"order[0][column]": "14",
-			"order[0][dir]":    "desc",
-			"search[value]":    "",
-			"search[regex]":    "false",
-			"search_in_stocks": "on",
-			"remnants_atleast": "5",
-		}).
+	formData := map[string]string{
+		"draw":             strconv.Itoa(params.Draw),
+		"start":            strconv.Itoa(params.Start),
+		"length":           strconv.Itoa(params.Length),
+		"order[0][column]": "14",
+		"order[0][dir]":    "desc",
+		"search[value]":    "",
+		"search[regex]":    "false",
+	}
+	if params.SearchInStocks {
+		stocksNum := strconv.Itoa(params.RemmantsAtleast)
+		formData["search_in_stocks"] = "on"
+		formData["remnants_atleast"] = stocksNum
+	}
+
+	c.log.Debug().
+		Int("remnants", params.RemmantsAtleast).
+		Bool("search_in_stocks", params.SearchInStocks).
+		Int("draw", params.Draw).
+		Int("start", params.Start).
+		Int("length", params.Length).
+		Msg("sending request")
+
+	req := c.http.R().
+		SetFormData(formData).
 		SetQueryParam("category_id", "0").
-		SetQueryParam("own", "26476"). // user id?
-		SetDoNotParseResponse(true).
-		Post("/goods_new")
+		SetQueryParam("own", c.ownerID). // user id?
+		SetDoNotParseResponse(true)
+
+	resp, err := c.do(ctx, "GetGoodsNew", req, resty.MethodPost, "/goods_new")
 	if err != nil {
 		return nil, -1, fmt.Errorf("getting goods new: %w", err)
 	}
@@ -214,7 +269,6 @@ func (c client) GetGoodsNew(
 			c.log.Error().Err(err).Msg("unable to close body")
 		}
 	}()
-
 	if resp.IsError() {
 		return nil, -1, errors.New("request was not successful")
 	}
@@ -226,3 +280,183 @@ func (c client) GetGoodsNew(
 
 	return mapResponseByOrder(response), response.RecordsTotal, nil
 }
+
+func (c *client) login(ctx context.Context, user, pass string) error {
+	req := c.http.R().
+		SetDoNotParseResponse(true).
+		SetFormData(map[string]string{
+			"username": user,
+			"password": pass,
+		})
+
+	resp, err := c.do(ctx, "login", req, resty.MethodPost, "https://eway.elevel.ru/")
+	if err != nil {
+		return fmt.Errorf("sending request: %w", err)
+	}
+
+	if resp.IsError() {
+		zerolog.Ctx(ctx).Warn().Int("code", resp.StatusCode()).Msg("bad response")
+
+		return entity.SimpleError("request was not successful")
+	}
+
+	return nil
+}
+
+func (c *client) do(ctx context.Context, name string, req *resty.Request, method string, url string) (resp *resty.Response, err error) {
+	resp, err = req.
+		EnableTrace().
+		Execute(method, url)
+
+	traceInfo := resp.Request.TraceInfo()
+	c.log.Debug().
+		Str("name", name).
+		Str("path", url).
+		Str("method", method).
+		Float64("elapsed", traceInfo.TotalTime.Seconds()).
+		Float64("response_time", traceInfo.ResponseTime.Seconds()).
+		Int("attempt", traceInfo.RequestAttempt).
+		Bool("success", resp.IsSuccess()).
+		Msg("request processed")
+
+	if err != nil {
+		return nil, fmt.Errorf("executing request: %w", err)
+	}
+
+	return resp, err
+}
+
+func (c *client) GetProductInfo(ctx context.Context, cart int64) (pi entity.GoodsItemInfo, err error) {
+	if c.workersPool == 0 {
+		return c.getProductInfo(ctx, cart)
+	}
+
+	responseBus := make(chan taskResult[entity.GoodsItemInfo], 1)
+	c.getProductInfoBus <- getProductInfoRequest{
+		cartID:   cart,
+		response: responseBus,
+	}
+
+	select {
+	case response := <-responseBus:
+		return response.value, response.err
+	case <-ctx.Done():
+		return pi, ctx.Err()
+	}
+}
+
+func (c *client) getProductInfo(ctx context.Context, cartID int64) (pi entity.GoodsItemInfo, err error) {
+	reqpath := "https://eway.elevel.ru/product/" + strconv.Itoa(int(cartID)) + "/"
+
+	req := c.http.R().SetDoNotParseResponse(true).AddRetryCondition(func(r *resty.Response, err error) bool {
+		if r.Request.Attempt > 3 {
+			return false
+		}
+		return strings.Contains(err.Error(), "pipe")
+	})
+
+	pi.Parameters = map[string]string{}
+	resp, err := c.do(ctx, "getProductInfo", req, resty.MethodGet, reqpath)
+	if err != nil {
+		return pi, fmt.Errorf("getting product info: %w", err)
+	}
+	defer func() {
+		errClose := resp.RawBody().Close()
+		if errClose == nil {
+			return
+		}
+
+		if err == nil {
+			err = errClose
+			return
+		}
+
+		c.log.Warn().Err(errClose).Msg("unable to close body")
+	}()
+	if resp.IsError() {
+		return pi, errors.New("request was not successful")
+	}
+
+	doc, err := goquery.NewDocumentFromReader(resp.RawBody())
+	if err != nil {
+		return pi, fmt.Errorf("makind new document: %w", err)
+	}
+
+	cleanText := func(t string) string {
+		return strings.TrimSpace(strings.TrimSuffix(strings.TrimSpace(t), ":"))
+	}
+
+	const parametersSelector = "body > div.page-container > div.page-content > div.content-wrapper > div.content > div.row > div.col-md-4 > div > div > div:nth-child(6)"
+	const parametersInnerNode = "div.display-flex"
+	doc.
+		Find(parametersSelector).
+		Find(parametersInnerNode).
+		Each(func(i int, s *goquery.Selection) {
+			name := cleanText(s.Find("div").Eq(0).Text())
+			value := cleanText(s.Find("div.text-right").Text())
+			pi.Parameters[name] = value
+		})
+
+	const galleryPanelSelector = "div.gallery_panel"
+	const galleryImageSelector = "div.gallery_thumbnail > img"
+	doc.
+		Find(galleryPanelSelector).
+		Find(galleryImageSelector).
+		Each(func(i int, s *goquery.Selection) {
+			imageURL, ok := s.Attr("data-src")
+			if !ok {
+				imageURL, ok = s.Attr("src")
+			}
+			if !ok || len(imageURL) == 0 {
+				return
+			}
+
+			pi.PhotoURLs = append(pi.PhotoURLs, imageURL)
+		})
+
+	return pi, nil
+}
+
+type getProductInfoRequest struct {
+	cartID   int64
+	response chan taskResult[entity.GoodsItemInfo]
+}
+
+type taskResult[T any] struct {
+	value T
+	err   error
+}
+
+func (c *client) productInfoWorker(
+	ctx context.Context,
+	in <-chan getProductInfoRequest,
+) {
+	var req getProductInfoRequest
+	var ok bool
+	for {
+		select {
+		case <-ctx.Done():
+			return
+		case req, ok = <-in:
+			if !ok {
+				return
+			}
+		}
+
+		func() {
+			c.htmlParseSema <- struct{}{}
+			defer func() {
+				go func() {
+					time.Sleep(c.releaseSemaDelay)
+					<-c.htmlParseSema
+				}()
+			}()
+
+			pi, err := c.getProductInfo(ctx, req.cartID)
+			req.response <- taskResult[entity.GoodsItemInfo]{
+				value: pi,
+				err:   err,
+			}
+		}()
+	}
+}

internal/matcher/matcher.go

@@ -0,0 +1,13 @@
+package matcher
+
+type Unit interface {
+	MatchByPattern(value string) (pattern string)
+	Match(value string) bool
+
+	// Move this to init stage because some of matchers
+	// might not be applicable to it
+	RegisterRegexp(regexpPattern string)
+	Register(pattern string)
+
+	Patterns() []string
+}

internal/matcher/radix.go

@@ -0,0 +1,171 @@
+package matcher
+
+import (
+	"regexp"
+	"strings"
+)
+
+const radixWildcard = '*'
+
+type radixNode struct {
+	exact bool
+	next  map[rune]*radixNode
+}
+
+type radixMatcher struct {
+	root *radixNode
+
+	caseInsensitive bool
+	saved           []string
+	regexps         []*regexp.Regexp
+}
+
+type RadixOpt func(m *radixMatcher)
+
+func RadixCaseInsensitive() RadixOpt {
+	return func(m *radixMatcher) {
+		m.caseInsensitive = true
+	}
+}
+
+func NewRadix(opts ...RadixOpt) *radixMatcher {
+	m := &radixMatcher{
+		root: &radixNode{
+			next: make(map[rune]*radixNode),
+		},
+	}
+
+	for _, opt := range opts {
+		opt(m)
+	}
+
+	return m
+}
+
+func (r *radixMatcher) MatchByPattern(value string) (pattern string) {
+	originValue := value
+	if r.caseInsensitive {
+		value = strings.ToLower(value)
+	}
+
+	node := r.root
+	lastIdx := len([]rune(value)) - 1
+
+	var sb strings.Builder
+	for i, v := range value {
+		var ok bool
+		if _, ok := node.next[radixWildcard]; ok {
+			_, _ = sb.WriteRune(radixWildcard)
+			return sb.String()
+		}
+
+		node, ok = node.next[v]
+		if !ok {
+			return r.findByRegexp(originValue)
+		}
+
+		sb.WriteRune(v)
+
+		if i != lastIdx {
+			continue
+		}
+
+		if !node.exact {
+			return r.findByRegexp(originValue)
+		}
+
+		return sb.String()
+	}
+
+	return r.findByRegexp(originValue)
+}
+
+func (r *radixMatcher) Match(value string) bool {
+	originValue := value
+	if r.caseInsensitive {
+		value = strings.ToLower(value)
+	}
+
+	node := r.root
+	lastIdx := len([]rune(value)) - 1
+	for i, v := range value {
+		var ok bool
+		if _, ok = node.next[radixWildcard]; ok {
+			return true
+		}
+
+		node, ok = node.next[v]
+		if !ok {
+			return r.findByRegexp(originValue) != ""
+		}
+
+		if i == lastIdx {
+			return node.exact
+		}
+	}
+
+	return r.findByRegexp(originValue) != ""
+}
+
+func (r *radixMatcher) findByRegexp(value string) (regexpPattern string) {
+	for _, rx := range r.regexps {
+		if rx.MatchString(value) {
+			return rx.String()
+		}
+	}
+
+	return ""
+}
+
+func (r *radixMatcher) RegisterRegexp(regexpPattern string) {
+	pattern := regexp.MustCompile(regexpPattern)
+	r.regexps = append(r.regexps, pattern)
+}
+
+func (r *radixMatcher) Register(pattern string) {
+	if len(pattern) == 0 {
+		panic("unable to handle empty pattern")
+	}
+
+	if r.caseInsensitive {
+		pattern = strings.ToLower(pattern)
+	}
+
+	r.saved = append(r.saved, pattern)
+
+	node := r.root
+	lastIdx := len([]rune(pattern)) - 1
+	for i, v := range pattern {
+		nextNode, ok := node.next[v]
+		if !ok {
+			nextNode = &radixNode{
+				next: make(map[rune]*radixNode),
+			}
+			node.next[v] = nextNode
+		}
+		node = nextNode
+
+		if v == '*' {
+			return
+		}
+
+		if i != lastIdx {
+			continue
+		}
+
+		node.exact = true
+	}
+}
+
+func (r *radixMatcher) Patterns() []string {
+	ownIdx := len(r.saved)
+	out := make([]string, len(r.saved)+len(r.regexps))
+	copy(out, r.saved[:ownIdx])
+
+	for i := 0; i < len(r.regexps); i++ {
+		idx := i + ownIdx
+		out[idx] = r.regexps[i].String()
+	}
+
+	return out
+}

internal/matcher/radix_test.go

@@ -0,0 +1,125 @@
+package matcher_test
+
+import (
+	"testing"
+
+	"git.loyso.art/frx/eway/internal/matcher"
+)
+
+func TestRadixMatcherWithPattern(t *testing.T) {
+	m := matcher.NewRadix()
+	m.Register("aloha")
+	m.Register("hawaii")
+	m.Register("te*")
+	m.Register("Ширина")
+
+	var tt = []struct {
+		name    string
+		in      string
+		pattern string
+	}{{
+		name:    "should match exact",
+		in:      "aloha",
+		pattern: "aloha",
+	}, {
+		name:    "should match exact 2",
+		in:      "hawaii",
+		pattern: "hawaii",
+	}, {
+		name:    "should match pattern",
+		in:      "test",
+		pattern: "te*",
+	}, {
+		name: "should not match",
+		in:   "alohe",
+	}, {
+		name: "should not match 2",
+		in:   "whoa",
+	}, {
+		name: "should not match 3",
+		in:   "alohaya",
+	}, {
+		name: "should match exact 3",
+		in:   "Ширина",
+	}}
+
+	for _, tc := range tt {
+		tc := tc
+		t.Run(tc.name, func(t *testing.T) {
+			pattern := m.MatchByPattern(tc.in)
+			if pattern != tc.pattern {
+				t.Errorf("expected %s got %s", tc.pattern, pattern)
+			}
+		})
+	}
+}
+
+func TestRadixMatcher(t *testing.T) {
+	m := matcher.NewRadix()
+	m.Register("aloha")
+	m.Register("hawaii")
+	m.Register("te*")
+
+	var tt = []struct {
+		name  string
+		in    string
+		match bool
+	}{{
+		name:  "should match exact",
+		in:    "aloha",
+		match: true,
+	}, {
+		name:  "should match exact 2",
+		in:    "hawaii",
+		match: true,
+	}, {
+		name:  "should match pattern",
+		in:    "test",
+		match: true,
+	}, {
+		name: "should not match",
+		in:   "alohe",
+	}, {
+		name: "should not match 2",
+		in:   "whoa",
+	}}
+
+	for _, tc := range tt {
+		tc := tc
+		t.Run(tc.name, func(t *testing.T) {
+			match := m.Match(tc.in)
+			if match != tc.match {
+				t.Errorf("expected %t got %t", tc.match, match)
+			}
+		})
+	}
+}
+
+func TestRadixMatcherWildcard(t *testing.T) {
+	m := matcher.NewRadix()
+	m.Register("*")
+
+	var tt = []struct {
+		name  string
+		in    string
+		match bool
+	}{{
+		name:  "should match exact",
+		in:    "aloha",
+		match: true,
+	}, {
+		name:  "should match exact 2",
+		in:    "hawaii",
+		match: true,
+	}}
+
+	for _, tc := range tt {
+		tc := tc
+		t.Run(tc.name, func(t *testing.T) {
+			match := m.Match(tc.in)
+			if match != tc.match {
+				t.Errorf("expected %t got %t", tc.match, match)
+			}
+		})
+	}
+}

internal/storage/badger/category.go

@@ -10,7 +10,6 @@ import (
 	"git.loyso.art/frx/eway/internal/entity"
 
 	badger "github.com/dgraph-io/badger/v4"
-	"github.com/rs/zerolog"
 )
 
 type categoryClient struct {
@@ -105,24 +104,14 @@ func (c categoryClient) Get(ctx context.Context, id int64) (out entity.Category,
 
 // Create new category inside DB. It also applies new id to it.
 func (c categoryClient) Create(ctx context.Context, name string) (out entity.Category, err error) {
-	seqGen, err := c.db.GetSequence(categorySequenceIDKey, 1)
-	if err != nil {
-		return out, fmt.Errorf("getting sequence for categories: %w", err)
-	}
-	defer func() {
-		errRelese := seqGen.Release()
-		if errRelese != nil {
-			zerolog.Ctx(ctx).Warn().Err(err).Msg("unable to release seq")
-		}
-	}()
-
-	nextid, err := seqGen.Next()
+	nextid, err := c.seqGen.Next()
 	if err != nil {
 		return out, fmt.Errorf("getting next id: %w", err)
 	}
 
 	out = entity.Category{
-		ID:   int64(nextid),
+		// Because first value from sequence generator is 0
+		ID:   int64(nextid + 1),
 		Name: name,
 	}
 

internal/storage/badger/client.go

@@ -1,47 +1,118 @@
 package badger
 
 import (
+	"fmt"
+	"time"
+	"unsafe"
+
 	"git.loyso.art/frx/eway/internal/entity"
 	badger "github.com/dgraph-io/badger/v4"
 )
 
 var (
-	categorySequenceIDKey = []byte("cat:")
+	categorySequenceIDKey = []byte("!!cat_seq!!")
+	queueSequenceIDKey    = []byte("!!que_seq!!")
 )
 
 type client struct {
-	db *badger.DB
-
-	// nextCategoryIDSeq *badger.Sequence
+	db                *badger.DB
+	nextCategoryIDSeq *badger.Sequence
+	nextQueueIDSeq    *badger.Sequence
 }
 
 func NewClient(db *badger.DB) (*client, error) {
-	// categorySeqGen, err := db.GetSequence(categorySequenceIDKey, 10)
-	// if err != nil {
-	// 	return nil, fmt.Errorf("getting sequence for categories: %w", err)
-	// }
-	//
+	categorySeqGen, err := db.GetSequence(categorySequenceIDKey, 10)
+	if err != nil {
+		return nil, fmt.Errorf("getting sequence for categories: %w", err)
+	}
+	queueSeqGen, err := db.GetSequence(queueSequenceIDKey, 10)
+	if err != nil {
+		return nil, fmt.Errorf("getting sequence for queues: %w", err)
+	}
+
 	return &client{
-		db: db,
-		// nextCategoryIDSeq: categorySeqGen,
+		db:                db,
+		nextCategoryIDSeq: categorySeqGen,
+		nextQueueIDSeq:    queueSeqGen,
 	}, nil
 }
 
 // Close closes the underlying sequences in the client. Should be called right before
 // underlying *badger.DB closed.
 func (c *client) Close() error {
-	// err := c.nextCategoryIDSeq.Release()
-	// if err != nil {
-	// 	return fmt.Errorf("releasing next_category_sequence: %w", err)
-	// }
+	err := c.nextCategoryIDSeq.Release()
+	if err != nil {
+		return fmt.Errorf("releasing next_category_sequence: %w", err)
+	}
 
 	return nil
 }
 
 func (c *client) Category() entity.CategoryRepository {
-	return newCategoryClient(c.db, nil)
+	return newCategoryClient(c.db, c.nextCategoryIDSeq)
 }
 
 func (c *client) GoodsItem() entity.GoodsItemRepository {
-	return newGoodsItemClient(c.db)
+	return newGoodsItemClient(c.db, useJSON)
+}
+
+func (c *client) QueueClient() entity.MessageQueue {
+	nc := c.Table("queues")
+	return newQueueClient(nc, c.nextQueueIDSeq)
+}
+
+func (c *client) Table(name string) namedClient {
+	tableBytes := unsafe.Slice(unsafe.StringData("!!"+name+"!!"), len(name)+4)
+	return namedClient{
+		table: tableBytes,
+		db:    c.db,
+	}
+}
+
+type namedClient struct {
+	table []byte
+	db    *badger.DB
+}
+
+type putOpt func(*badger.Entry)
+
+func withTTL(duration time.Duration) putOpt {
+	return func(e *badger.Entry) {
+		e.WithTTL(duration)
+	}
+}
+
+func (c *namedClient) Put(key, value []byte, opts ...putOpt) error {
+	return c.db.Update(func(txn *badger.Txn) error {
+		tableKey := c.makeKey(key)
+		entry := badger.NewEntry(tableKey, value)
+		for _, opt := range opts {
+			opt(entry)
+		}
+
+		return txn.SetEntry(entry)
+	})
+}
+
+func (c *namedClient) Get(key []byte) ([]byte, error) {
+	var out []byte
+	err := c.db.View(func(txn *badger.Txn) error {
+		item, err := txn.Get(c.makeKey(key))
+		if err != nil {
+			return err
+		}
+
+		out = make([]byte, item.ValueSize())
+		out, err = item.ValueCopy(out)
+		return err
+	})
+	if err != nil {
+		return nil, err
+	}
+
+	return out, nil
+}
+
+func (c *namedClient) makeKey(key []byte) (out []byte) {
+	return append(c.table, key...)
 }

internal/storage/badger/db.go

@@ -30,21 +30,22 @@ func (za zerologAdapter) fmt(event *zerolog.Event, format string, args ...any) {
 	event.Msgf(strings.TrimSuffix(format, "\n"), args...)
 }
 
-func Open(ctx context.Context, path string, log zerolog.Logger) (*badger.DB, error) {
+func Open(ctx context.Context, path string, debug bool, log zerolog.Logger) (*badger.DB, error) {
 	bl := zerologAdapter{
 		log: log.With().Str("db", "badger").Logger(),
 	}
+
+	level := badger.WARNING
+	if debug {
+		level = badger.DEBUG
+	}
+
 	opts := badger.DefaultOptions(path).
 		WithLogger(bl).
-		WithLoggingLevel(badger.INFO).
+		WithLoggingLevel(level).
 		WithValueLogFileSize(4 << 20).
 		WithDir(path).
 		WithValueDir(path)
-		// WithMaxLevels(4).
-		// WithMemTableSize(8 << 20).
-		// WithMetricsEnabled(true).
-		// WithCompactL0OnClose(true).
-		// WithBlockCacheSize(8 << 20)
 
 	db, err := badger.Open(opts)
 	if err != nil {

internal/storage/badger/goodsitem.go

@@ -1,29 +1,42 @@
 package badger
 
 import (
+	"bytes"
 	"context"
 	"encoding/binary"
+	"encoding/json"
 	"errors"
 	"fmt"
-	"runtime"
+	"time"
 	"unsafe"
 
 	"git.loyso.art/frx/eway/internal/encoding/fbs"
 	"git.loyso.art/frx/eway/internal/entity"
 
 	badger "github.com/dgraph-io/badger/v4"
-	"github.com/dgraph-io/badger/v4/pb"
 	"github.com/dgraph-io/ristretto/z"
 	"github.com/rs/zerolog"
 )
 
+const useJSON = false
+
 type goodsItemClient struct {
 	db *badger.DB
+
+	s itemSerializer[entity.GoodsItem]
 }
 
-func newGoodsItemClient(db *badger.DB) *goodsItemClient {
+func newGoodsItemClient(db *badger.DB, serializeAsJSON bool) *goodsItemClient {
+	var s itemSerializer[entity.GoodsItem]
+	if serializeAsJSON {
+		s = goodsItemJSONSerializer{}
+	} else {
+		s = goodsItemFlatbufSerializer{}
+	}
+
 	return &goodsItemClient{
 		db: db,
+		s:  s,
 	}
 }
 
@@ -40,7 +53,7 @@ func (c *goodsItemClient) prefixedStr(key string) []byte {
 	return c.prefixed(keyBytes)
 }
 
-func (c *goodsItemClient) prefixedIDByCartStr(key int64) []byte {
+func (c *goodsItemClient) prefixedIDByCartInt64(key int64) []byte {
 	var keyBytes [8]byte
 	binary.BigEndian.PutUint64(keyBytes[:], uint64(key))
 	return c.prefixedIDByCart(keyBytes[:])
@@ -65,7 +78,13 @@ func (c *goodsItemClient) ListIter(
 		}
 
 		for _, kv := range list.GetKv() {
-			bus <- fbs.ParseGoodsItem(kv.GetValue())
+			var gooditem entity.GoodsItem
+			gooditem, err = c.s.Deserialize(kv.GetValue())
+			if err != nil {
+				return fmt.Errorf("deserializing item: %w", err)
+			}
+
+			bus <- gooditem
 		}
 
 		return nil
@@ -74,7 +93,7 @@ func (c *goodsItemClient) ListIter(
 	go func(ctx context.Context) {
 		defer close(bus)
 
-		err := stream.Orchestrate(context.Background())
+		err := stream.Orchestrate(ctx)
 		if err != nil {
 			zerolog.Ctx(ctx).Warn().Err(err).Msg("unable to orchestrate")
 		}
@@ -86,6 +105,10 @@ func (c *goodsItemClient) ListIter(
 func (c *goodsItemClient) List(
 	ctx context.Context,
 ) (out []entity.GoodsItem, err error) {
+	if ctx.Err() != nil {
+		return nil, ctx.Err()
+	}
+
 	err = c.db.View(func(txn *badger.Txn) error {
 		opts := badger.DefaultIteratorOptions
 		opts.PrefetchValues = true
@@ -94,10 +117,21 @@ func (c *goodsItemClient) List(
 		defer iter.Close()
 
 		prefix := c.prefix()
+		var cursor int
 		for iter.Seek(prefix); iter.ValidForPrefix(prefix); iter.Next() {
+			if ctx.Err() != nil {
+				return ctx.Err()
+			}
+
+			cursor++
 			current := iter.Item()
 			err = current.Value(func(val []byte) error {
-				goodsItem := fbs.ParseGoodsItem(val)
+				var goodsItem entity.GoodsItem
+				goodsItem, err = c.s.Deserialize(val)
+				if err != nil {
+					return fmt.Errorf("deserializing: %w", err)
+				}
+
 				out = append(out, goodsItem)
 
 				return nil
@@ -120,6 +154,10 @@ func (c *goodsItemClient) Get(
 	ctx context.Context,
 	sku string,
 ) (out entity.GoodsItem, err error) {
+	if ctx.Err() != nil {
+		return out, ctx.Err()
+	}
+
 	err = c.db.View(func(txn *badger.Txn) error {
 		key := unsafe.Slice(unsafe.StringData(sku), len(sku))
 		out, err = c.getBySKU(key, txn)
@@ -137,21 +175,26 @@ func (c *goodsItemClient) Get(
 }
 
 func (c *goodsItemClient) GetByCart(ctx context.Context, id int64) (out entity.GoodsItem, err error) {
-	err = c.db.View(func(txn *badger.Txn) error {
-		var idByte [8]byte
-		binary.BigEndian.PutUint64(idByte[:], uint64(id))
+	if ctx.Err() != nil {
+		return out, ctx.Err()
+	}
 
-		item, err := txn.Get(c.prefixedIDByCart(idByte[:]))
+	err = c.db.View(func(txn *badger.Txn) error {
+		idxKey := c.prefixedIDByCartInt64(id)
+		skuByCartIDItem, err := txn.Get(idxKey)
 		if err != nil {
 			return fmt.Errorf("getting key: %w", err)
 		}
 
-		sku := make([]byte, item.ValueSize())
-		sku, err = item.ValueCopy(sku)
+		sku := make([]byte, skuByCartIDItem.ValueSize())
+		sku, err = skuByCartIDItem.ValueCopy(sku)
 		if err != nil {
 			return fmt.Errorf("getting value of idx: %w", err)
 		}
 
+		// well, yeah, that's kind of dumb to trim prefix here and
+		// and prefix later, but who cares.
+		sku = bytes.TrimPrefix(sku, c.prefix())
 		out, err = c.getBySKU(sku, txn)
 		return err
 	})
@@ -167,105 +210,82 @@ func (c *goodsItemClient) GetByCart(ctx context.Context, id int64) (out entity.G
 }
 
 func (c *goodsItemClient) UpsertMany(ctx context.Context, items ...entity.GoodsItem) ([]entity.GoodsItem, error) {
-	return items, c.upsertByOne(ctx, items)
+	return items, c.upsertByBatch(ctx, items)
 }
 
-func (c *goodsItemClient) upsertByOne(ctx context.Context, items []entity.GoodsItem) error {
-	return c.db.Update(func(txn *badger.Txn) error {
-		for _, item := range items {
-			key := c.prefixedStr(item.Articul)
-			value := fbs.MakeDomainGoodItemFinished(item)
-			valueIdx := make([]byte, len(key))
-			copy(valueIdx, key)
+func (c *goodsItemClient) Delete(ctx context.Context, sku string) (out entity.GoodsItem, err error) {
+	if ctx.Err() != nil {
+		return out, ctx.Err()
+	}
 
-			err := txn.Set(key, value)
-			if err != nil {
-				return err
-			}
+	err = c.db.Update(func(txn *badger.Txn) error {
+		skuKey := c.prefixedStr(sku)
+		out, err = c.getBySKU(skuKey, txn)
+		if err != nil {
+			return err
+		}
 
-			err = txn.Set(c.prefixedIDByCartStr(item.Cart), valueIdx)
-			if err != nil {
-				return err
-			}
+		err = txn.Delete(skuKey)
+		if err != nil {
+			return fmt.Errorf("deleting key: %w", err)
+		}
+
+		cartID := c.prefixedIDByCartInt64(out.Cart)
+		err = txn.Delete(cartID)
+		if err != nil {
+			return fmt.Errorf("deleting index key: %w", err)
 		}
 
 		return nil
 	})
-}
-
-func (c *goodsItemClient) upsertByStream(ctx context.Context, items []entity.GoodsItem) error {
-	stream := c.db.NewStreamWriter()
-	defer stream.Cancel()
-
-	err := stream.Prepare()
 	if err != nil {
-		return fmt.Errorf("preparing stream: %w", err)
+		return entity.GoodsItem{}, err
 	}
 
-	buf := z.NewBuffer(len(items), "sometag")
-	for _, item := range items {
-		key := c.prefixedStr(item.Articul)
-		keyIdx := c.prefixedIDByCartStr(item.Cart)
-		value := fbs.MakeDomainGoodItemFinished(item)
-
-		itemKV := &pb.KV{Key: key, Value: value}
-		itemKVIdx := &pb.KV{Key: keyIdx, Value: key}
-
-		badger.KVToBuffer(itemKV, buf)
-		badger.KVToBuffer(itemKVIdx, buf)
-	}
-
-	err = stream.Write(buf)
-	if err != nil {
-		return fmt.Errorf("writing buf: %w", err)
-	}
-
-	err = stream.Flush()
-	if err != nil {
-		return fmt.Errorf("flushing changes: %w", err)
-	}
-
-	return nil
+	return out, nil
 }
 
 func (c *goodsItemClient) upsertByBatch(ctx context.Context, items []entity.GoodsItem) error {
 	batch := c.db.NewWriteBatch()
-	defer func() {
-		println("closing batch")
-		batch.Cancel()
+	defer batch.Cancel()
+
+	createdAt := time.Now()
+	err := func() error {
+		for _, item := range items {
+			if ctx.Err() != nil {
+				return ctx.Err()
+			}
+
+			item.CreatedAt = createdAt
+			value, err := c.s.Serialize(item)
+			if err != nil {
+				return fmt.Errorf("serializing item: %w", err)
+			}
+
+			key := c.prefixedStr(item.Articul)
+			idxValue := make([]byte, len(key))
+			copy(idxValue, key)
+
+			coreEntry := badger.NewEntry(key, value)
+			if err := batch.SetEntry(coreEntry); err != nil {
+				return fmt.Errorf("setting core entry: %w", err)
+			}
+
+			idxKey := c.prefixedIDByCartInt64(item.Cart)
+			idxEntry := badger.NewEntry(idxKey, idxValue)
+			if err := batch.SetEntry(idxEntry); err != nil {
+				return fmt.Errorf("setting index entry: %w", err)
+			}
+		}
+
+		return nil
 	}()
-
-	log := zerolog.Ctx(ctx)
-	for _, item := range items {
-		select {
-		case <-ctx.Done():
-			break
-		default:
-		}
-		key := c.prefixedStr(item.Articul)
-		value := fbs.MakeDomainGoodItemFinished(item)
-		idxValue := make([]byte, len(key))
-		copy(idxValue, key)
-		coreEntry := badger.NewEntry(key, value)
-		if err := batch.SetEntry(coreEntry); err != nil {
-			log.Warn().Err(err).Msg("unable to set item, breaking")
-			break
-		}
-
-		idxKey := c.prefixedIDByCartStr(item.Cart)
-		idxEntry := badger.NewEntry(idxKey, idxValue)
-		if err := batch.SetEntry(idxEntry); err != nil {
-			log.Warn().Err(err).Msg("unable to set idx, breaking")
-			break
-		}
-		runtime.Gosched()
+	if err != nil && !errors.Is(err, context.Canceled) {
+		return err
 	}
 
-	println("flushing")
-	err := batch.Flush()
-	runtime.Gosched()
+	err = batch.Flush()
 	if err != nil {
-		println("flush err", err.Error())
 		return fmt.Errorf("flushing changes: %w", err)
 	}
 
@@ -279,8 +299,8 @@ func (c *goodsItemClient) getBySKU(sku []byte, txn *badger.Txn) (out entity.Good
 	}
 
 	err = item.Value(func(val []byte) error {
-		out = fbs.ParseGoodsItem(val)
-		return nil
+		out, err = c.s.Deserialize(val)
+		return err
 	})
 	if err != nil {
 		return out, fmt.Errorf("reading value: %w", err)
@@ -288,3 +308,39 @@ func (c *goodsItemClient) getBySKU(sku []byte, txn *badger.Txn) (out entity.Good
 
 	return out, nil
 }
+
+type itemSerializer[T any] interface {
+	Serialize(T) ([]byte, error)
+	Deserialize([]byte) (T, error)
+}
+
+type goodsItemJSONSerializer struct{}
+
+func (goodsItemJSONSerializer) Serialize(in entity.GoodsItem) ([]byte, error) {
+	data, err := json.Marshal(in)
+	if err != nil {
+		return nil, fmt.Errorf("marshalling data: %w", err)
+	}
+
+	return data, nil
+}
+
+func (goodsItemJSONSerializer) Deserialize(data []byte) (in entity.GoodsItem, err error) {
+	err = json.Unmarshal(data, &in)
+	return in, err
+}
+
+type goodsItemFlatbufSerializer struct{}
+
+func (goodsItemFlatbufSerializer) Serialize(in entity.GoodsItem) ([]byte, error) {
+	out := fbs.MakeDomainGoodItemFinished(in)
+	return out, nil
+}
+
+func (goodsItemFlatbufSerializer) Deserialize(data []byte) (out entity.GoodsItem, err error) {
+	out, err = fbs.ParseGoodsItem(data)
+	if err != nil {
+		return entity.GoodsItem{}, err
+	}
+	return out, nil
+}

internal/storage/badger/queue.go

@@ -0,0 +1,130 @@
+package badger
+
+import (
+	"bytes"
+	"context"
+	"encoding/binary"
+	"errors"
+	"fmt"
+	"time"
+
+	"git.loyso.art/frx/eway/internal/entity"
+
+	"github.com/dgraph-io/badger/v4"
+)
+
+type queueClient struct {
+	namedClient
+
+	seqGen *badger.Sequence
+}
+
+func newQueueClient(nc namedClient, seqGen *badger.Sequence) queueClient {
+	return queueClient{
+		namedClient: nc,
+		seqGen:      seqGen,
+	}
+}
+
+type taskDB struct {
+	createdAt int64
+	body      []byte
+}
+
+func (t taskDB) asBinary() []byte {
+	buf := make([]byte, 8+len(t.body))
+	binary.BigEndian.PutUint64(buf[:8], uint64(t.createdAt))
+	copy(buf[8:], t.body)
+
+	return buf
+}
+
+func (t *taskDB) fromBinary(data []byte) error {
+	if len(data) < 8 {
+		return errors.New("bad data")
+	}
+
+	t.createdAt = int64(binary.BigEndian.Uint64(data[:8]))
+	if len(data) == 8 {
+		return nil
+	}
+
+	t.body = make([]byte, len(t.body)-8)
+	copy(t.body, data[8:])
+
+	return nil
+}
+
+func (c queueClient) Publish(ctx context.Context, params entity.PublishParams) (task entity.Task, err error) {
+	task.ID, err = c.seqGen.Next()
+	if err != nil {
+		return task, fmt.Errorf("generating id: %w", err)
+	}
+
+	task.CreatedAt = time.Now()
+	tdb := taskDB{
+		createdAt: task.CreatedAt.Unix(),
+		body:      params.Body,
+	}
+
+	var keyData [8]byte
+	binary.BigEndian.PutUint64(keyData[:], task.ID)
+	opts := make([]putOpt, 0, 1)
+	if !params.ExpiresAt.IsZero() {
+		duration := time.Until(params.ExpiresAt)
+		opts = append(opts, withTTL(duration))
+	}
+	err = c.Put(keyData[:], tdb.asBinary(), opts...)
+	if err != nil {
+		return entity.Task{}, fmt.Errorf("saving data: %w", err)
+	}
+
+	return task, nil
+}
+
+func (c queueClient) Consume(ctx context.Context) (task entity.Task, err error) {
+	err = c.db.View(func(txn *badger.Txn) error {
+		iterOpts := badger.IteratorOptions{
+			PrefetchSize:   1,
+			PrefetchValues: true,
+			Reverse:        false,
+			AllVersions:    false,
+			Prefix:         c.table,
+		}
+		iter := txn.NewIterator(iterOpts)
+		defer iter.Close()
+
+		iter.Seek(c.table)
+		if !iter.ValidForPrefix(c.table) {
+			return entity.ErrNotFound
+		}
+
+		item := iter.Item()
+		keyData := item.KeyCopy(nil)
+		valueData, err := item.ValueCopy(nil)
+		if err != nil {
+			return fmt.Errorf("getting value: %w", err)
+		}
+
+		var tdb taskDB
+		err = tdb.fromBinary(valueData)
+		if err != nil {
+			return err
+		}
+
+		task.ID = binary.BigEndian.Uint64(bytes.TrimPrefix(keyData, c.table))
+		task.CreatedAt = time.Unix(tdb.createdAt, 0)
+		task.Body = tdb.body
+		if expiresAt := item.ExpiresAt(); expiresAt > 0 {
+			t := time.Now().Add(time.Second * time.Duration(expiresAt))
+			task.ExpiresAt = &t
+		}
+
+		return nil
+	})
+	if err != nil {
+		return entity.Task{}, err
+	}
+
+	return task, nil
+}

internal/storage/storage.go

@@ -1,8 +1,14 @@
 package storage
 
-import "git.loyso.art/frx/eway/internal/entity"
+import (
+	"io"
+
+	"git.loyso.art/frx/eway/internal/entity"
+)
 
 type Repository interface {
+	io.Closer
+
 	Category() entity.CategoryRepository
 	GoodsItem() entity.GoodsItemRepository
 }