minor rework and filter dimensions

This commit is contained in:
Aleksandr Trushkin
2024-02-13 21:19:39 +03:00
parent 23a4f007fc
commit 8f26b8ba6d
13 changed files with 969 additions and 780 deletions

View File

@ -0,0 +1,101 @@
package commands
import (
"context"
"fmt"
"git.loyso.art/frx/eway/cmd/cli/components"
"github.com/rodaine/table"
"github.com/rs/zerolog"
"github.com/urfave/cli/v3"
)
func CategoriesCommandTree() *cli.Command {
var h categoriesHandlers
return &cli.Command{
Name: "categories",
Usage: "Interact with stored categories",
Commands: []*cli.Command{
newCategoriesListCommand(h),
},
}
}
func newCategoriesListCommand(h categoriesHandlers) *cli.Command {
cmd := cli.Command{
Name: "list",
Usage: "List categories, stories in db",
Flags: []cli.Flag{
&cli.IntFlag{
Name: "limit",
Usage: "limits output to selected items",
Value: 20,
},
&cli.IntFlag{
Name: "page",
Usage: "in case of limit, selects page",
Value: 0,
},
&cli.BoolFlag{
Name: "with-total",
Usage: "prints total count of categories",
},
},
}
return cmdWithAction(cmd, h.List)
}
type categoriesHandlers struct{}
func (categoriesHandlers) List(ctx context.Context, c *cli.Command) error {
r, err := components.GetRepository()
if err != nil {
return fmt.Errorf("getting repository: %w", err)
}
categories, err := r.Category().List(ctx)
if err != nil {
return fmt.Errorf("listing categories: %w", err)
}
limit := int(c.Int("limit"))
page := int(c.Int("page"))
total := len(categories)
if page == 0 {
page = 1
}
if limit > 0 {
offset := (page - 1) * limit
if offset > len(categories) {
offset = len(categories) - 1
}
limit = offset + limit
if limit > len(categories) {
limit = len(categories)
}
categories = categories[offset:limit]
}
tbl := table.New("ID", "Name")
for _, category := range categories {
if category.ID == 0 && category.Name == "" {
continue
}
tbl.AddRow(category.ID, category.Name)
}
tbl.Print()
if c.Bool("with-total") {
zerolog.Ctx(ctx).Info().Int("count", total).Msg("total categories stats")
}
return nil
}

View File

@ -0,0 +1,55 @@
package commands
import (
"context"
"errors"
"git.loyso.art/frx/eway/internal/entity"
)
type chanIter[T any] struct {
in <-chan T
err error
next T
}
var errChannelClosed = errors.New("channel closed")
func (i *chanIter[T]) Next() (ok bool) {
if i.err != nil {
return false
}
i.next, ok = <-i.in
if !ok {
i.err = errChannelClosed
}
return ok
}
func (i *chanIter[T]) Get() T {
return i.next
}
func (i *chanIter[T]) Err() error {
if errors.Is(i.err, errChannelClosed) {
return nil
}
return i.err
}
func (i *chanIter[T]) Close() {
for range i.in {
}
}
func getItemsIter(ctx context.Context, r entity.GoodsItemRepository) *chanIter[entity.GoodsItem] {
in, err := r.ListIter(ctx, 3)
return &chanIter[entity.GoodsItem]{
in: in,
err: err,
}
}

View File

@ -0,0 +1,78 @@
package commands
import (
"context"
"strings"
"git.loyso.art/frx/eway/internal/entity"
"git.loyso.art/frx/eway/internal/matcher"
"github.com/rs/zerolog"
)
type dimensionDispatcher struct {
heigth matcher.Unit
width matcher.Unit
length matcher.Unit
}
func (d dimensionDispatcher) isDimensionParam(value string) bool {
return d.heigth.Match(value) || d.width.Match(value) || d.length.Match(value)
}
func (d dimensionDispatcher) dispatch(ctx context.Context, key, value string, in *entity.GoodsItemSize) (updated bool) {
if !d.isDimensionParam(key) {
return false
}
log := zerolog.Ctx(ctx).With().Str("key", key).Str("value", value).Logger()
if strings.Contains(value, "/") {
dimensionValues := strings.Split(value, "/")
for _, dv := range dimensionValues {
updated = updated || d.dispatch(ctx, key, dv, in)
}
} else {
out, err := entity.ParseDimention(value, entity.DimensionLocalRU)
if err != nil {
log.Warn().Err(err).Msg("unable to parse key, skipping")
return false
}
out = out.AdjustTo(entity.DimensionKindCentimeter)
updated = true
switch {
case d.heigth.Match(key):
in.Height = out
case d.width.Match(key):
in.Width = out
case d.length.Match(key):
in.Length = out
default:
log.Error().Str("key", key).Msg("unable to find proper matcher")
updated = false
}
}
return updated
}
func makeDefaultDimensionDispatcher() dimensionDispatcher {
h := matcher.NewRadix(matcher.RadixCaseInsensitive())
h.Register("Высота")
h.Register("Высота/*")
w := matcher.NewRadix(matcher.RadixCaseInsensitive())
w.Register("Ширина")
w.Register("Ширина/*")
l := matcher.NewRadix(matcher.RadixCaseInsensitive())
l.Register("Длина")
l.Register("Длина/*")
l.Register("Общ. длина")
return dimensionDispatcher{
heigth: h,
width: w,
length: l,
}
}

259
cmd/cli/commands/exports.go Normal file
View File

@ -0,0 +1,259 @@
package commands
import (
"context"
"encoding/xml"
"fmt"
"os"
"strconv"
"strings"
"time"
"git.loyso.art/frx/eway/cmd/cli/components"
"git.loyso.art/frx/eway/internal/entity"
"git.loyso.art/frx/eway/internal/export"
"github.com/rs/zerolog"
"github.com/urfave/cli/v3"
)
func ExportCommandTree() *cli.Command {
var h exportHandlers
return &cli.Command{
Name: "export",
Usage: "Provide actions to export data from the database",
Commands: []*cli.Command{
newExportYMLCatalogCommand(h),
},
}
}
func newExportYMLCatalogCommand(h exportHandlers) *cli.Command {
cmd := cli.Command{
Name: "yml-catalog",
Usage: "Export data as yml_catalog",
}
return cmdWithAction(cmd, h.YMLCatalog)
}
type exportHandlers struct{}
func (h exportHandlers) YMLCatalog(ctx context.Context, cmd *cli.Command) error {
const defaultCurrency = "RUR"
log := zerolog.Ctx(ctx)
path := cmd.String("out")
limit := cmd.Int("limit")
pretty := cmd.Bool("pretty")
r, err := components.GetRepository()
if err != nil {
return fmt.Errorf("getting repository: %w", err)
}
log.Info().
Str("path", path).
Int64("limit", limit).
Bool("pretty", pretty).
Msg("executing with parameters")
categories, err := r.Category().List(ctx)
if err != nil {
return fmt.Errorf("listing categories: %w", err)
}
shop := export.Shop{
Currencies: []export.Currency{{
ID: defaultCurrency,
Rate: 1,
}},
Categories: make([]export.Category, 0, len(categories)),
}
categoryByNameIdx := make(map[string]int64, len(categories))
for _, category := range categories {
categoryByNameIdx[category.Name] = category.ID
shop.Categories = append(shop.Categories, export.Category{
ID: category.ID,
Name: category.Name,
})
}
matcher := makeDefaultDimensionDispatcher()
var skipped int
iter := getItemsIter(ctx, r.GoodsItem())
for iter.Next() {
item := iter.Get()
sublog := log.With().Int64("cart", item.Cart).Logger()
if item.Sizes == (entity.GoodsItemSize{}) {
sublog.Warn().Str("sku", item.Articul).Int64("cart", item.Cart).Msg("skipping item because it does not have size")
skipped++
continue
}
offer := h.goodsItemAsOffer(iter.Get(), categoryByNameIdx, matcher, sublog)
shop.Offers = append(shop.Offers, offer)
}
if err = iter.Err(); err != nil {
return fmt.Errorf("iterating over items: %w", err)
}
f, err := os.Create(path)
if err != nil {
return fmt.Errorf("creating file: %w", err)
}
defer func() {
errClose := f.Close()
if err == nil {
err = errClose
return
}
log.Err(errClose).Msg("file closed or not")
}()
if limit > 0 {
shop.Offers = shop.Offers[:limit]
}
container := export.YmlContainer{
YmlCatalog: export.YmlCatalog{
Shop: shop,
Date: time.Now(),
},
}
_, err = f.Write([]byte(xml.Header))
if err != nil {
return fmt.Errorf("writing header: %w", err)
}
_, err = f.Write([]byte("<!DOCTYPE yml_catalog SYSTEM \"shops.dtd\">\n"))
enc := xml.NewEncoder(f)
if pretty {
enc.Indent("", " ")
}
log.Info().Int("processed", len(shop.Offers)).Int("skipped", skipped).Msg("completed")
return enc.Encode(container)
}
func (h exportHandlers) goodsItemAsOffer(in entity.GoodsItem, categoryIDByName map[string]int64, d dimensionDispatcher, log zerolog.Logger) (out export.Offer) {
const defaultType = "vendor.model"
const defaultCurrency = "RUR"
const defaultAvailable = true
const quantityParamName = "Количество на складе «Москва»"
const basePictureURL = "https://eway.elevel.ru"
imgurl := func(path string) string {
return basePictureURL + path
}
categoryID := categoryIDByName[in.Type]
pictureURLs := make([]string, 0, len(in.PhotoURLs))
for _, url := range in.PhotoURLs {
pictureURLs = append(pictureURLs, imgurl(url))
}
params := make([]export.Param, len(in.Parameters))
for k, v := range in.Parameters {
if k == "" || v == "" {
continue
}
if d.isDimensionParam(k) {
continue
}
params = append(params, export.Param{
Name: k,
Value: v,
})
}
params = append(params, export.Param{
Name: quantityParamName,
Value: strconv.Itoa(in.Stock),
})
dimensions := h.formatSizeAsDimensions(in.Sizes, log)
out = export.Offer{
ID: in.Cart,
VendorCode: in.Articul,
Price: int(in.TariffPrice),
Model: in.Name,
Vendor: in.Producer,
TypePrefix: in.Name,
Description: in.Description,
Dimensions: dimensions,
CategoryID: categoryID,
PictureURLs: pictureURLs,
Params: params,
Type: defaultType,
CurrencyID: defaultCurrency,
Available: defaultAvailable,
ManufacturerWarrany: true,
}
return out
}
func (exportHandlers) formatSizeAsDimensions(size entity.GoodsItemSize, log zerolog.Logger) string {
const delimeter = "/"
makeFloat := func(d entity.Dimension) string {
value := size.Length.AdjustTo(entity.DimensionKindCentimeter).Value
value = float64(int(value*1000)) / 1000.0
return strconv.FormatFloat(value, 'f', 8, 64)
}
knownSizes := make([]entity.Dimension, 0, 3)
for _, d := range []entity.Dimension{
size.Length,
size.Width,
size.Height,
} {
if d.IsZero() {
continue
}
knownSizes = append(knownSizes, d)
}
l := makeFloat(size.Length)
w := makeFloat(size.Width)
h := makeFloat(size.Height)
switch len(knownSizes) {
case 3:
// go on
case 2:
var side string
unknownDefaultSize := makeFloat(entity.NewCentimeterDimensionOrEmpty(30))
switch {
case size.Length.IsZero():
side = "length"
l = unknownDefaultSize
case size.Width.IsZero():
side = "width"
w = unknownDefaultSize
case size.Height.IsZero():
side = "height"
h = unknownDefaultSize
}
log.Warn().Str("size", side).Msg("setting to default value")
default:
return ""
}
// Output should be the following format:
// length/width/height in centimeters
return strings.Join([]string{
l, w, h,
}, delimeter)
}

View File

@ -0,0 +1,55 @@
package commands
import (
"context"
"crypto/rand"
"encoding/hex"
"fmt"
"time"
"git.loyso.art/frx/eway/cmd/cli/components"
"git.loyso.art/frx/eway/internal/entity"
"github.com/urfave/cli/v3"
)
type empty entity.Empty
type action func(ctx context.Context, c *cli.Command) error
func decorateAction(a action) cli.ActionFunc {
return func(ctx context.Context, c *cli.Command) error {
var data [3]byte
_, _ = rand.Read(data[:])
reqid := hex.EncodeToString(data[:])
log, err := components.GetLogger()
if err != nil {
return fmt.Errorf("getting logger: %w", err)
}
log = log.With().Str("reqid", reqid).Logger()
rctx := log.WithContext(ctx)
start := time.Now()
defer func() {
log.Info().Float64("elapsed", time.Since(start).Seconds()).Msg("command completed")
}()
log.Info().Msg("command execution started")
return a(rctx, c)
}
}
func cmdWithAction(cmd cli.Command, a action) *cli.Command {
if a == nil {
a = notImplementedAction
}
cmd.Action = decorateAction(a)
return &cmd
}
func notImplementedAction(_ context.Context, _ *cli.Command) error {
return entity.ErrNotImplemented
}

373
cmd/cli/commands/items.go Normal file
View File

@ -0,0 +1,373 @@
package commands
import (
"bufio"
"context"
"encoding/json"
"fmt"
"os"
"strconv"
"strings"
"git.loyso.art/frx/eway/cmd/cli/components"
"git.loyso.art/frx/eway/internal/entity"
"git.loyso.art/frx/eway/internal/matcher"
"github.com/rs/zerolog"
"github.com/urfave/cli/v3"
)
func ItemsCommandTree() *cli.Command {
var h itemsHandlers
return &cli.Command{
Name: "items",
Usage: "Interact with items stored inside db",
Commands: []*cli.Command{
newItemsGetCommand(h),
newItemsCountCommand(h),
newItemsUniqueParamsCommand(h),
newItemsAggregateParametersCommand(h),
newItemsFillSizesCommand(h),
},
}
}
func newItemsGetCommand(h itemsHandlers) *cli.Command {
cmd := cli.Command{
Name: "get",
Usage: "gets goods item by its id",
Flags: []cli.Flag{
&cli.StringFlag{
Name: "id",
Usage: "id of the goods item. Either id or cart-id should be set",
},
&cli.IntFlag{
Name: "cart-id",
Usage: "cart-id of the item. Either cart-id or id should be set",
},
},
}
return cmdWithAction(cmd, h.Get)
}
func newItemsCountCommand(h itemsHandlers) *cli.Command {
cmd := cli.Command{
Name: "count",
Usage: "iterates over collection and counts number of items",
Flags: []cli.Flag{
&cli.StringSliceFlag{
Name: "param-key-match",
Usage: "filters by parameters with AND logic",
},
},
}
return cmdWithAction(cmd, h.Count)
}
func newItemsUniqueParamsCommand(h itemsHandlers) *cli.Command {
cmd := cli.Command{
Name: "unique-params",
Usage: "Show all stored unique param values",
Description: "This command iterates over each item and collect keys of params in a dict and then" +
" print it to the output. It's useful to find all unique parameters",
}
return cmdWithAction(cmd, h.UniqueParameters)
}
func newItemsAggregateParametersCommand(h itemsHandlers) *cli.Command {
cmd := cli.Command{
Name: "aggregate-parameters",
Usage: "Show all values of requested parameters",
Flags: []cli.Flag{
&cli.BoolFlag{
Name: "case-insensitive",
Usage: "Ignores cases of keys",
},
&cli.StringSliceFlag{
Name: "regex",
Usage: "Registers regex to match",
},
&cli.BoolFlag{
Name: "keys-only",
Usage: "prints only keys",
},
},
}
return cmdWithAction(cmd, h.AggregateParameters)
}
func newItemsFillSizesCommand(h itemsHandlers) *cli.Command {
cmd := cli.Command{
Name: "fix-sizes",
Usage: "Iterates over params and sets sizes from parameters",
}
return cmdWithAction(cmd, h.FillSizes)
}
type itemsHandlers struct{}
func (itemsHandlers) Get(ctx context.Context, cmd *cli.Command) error {
r, err := components.GetRepository()
if err != nil {
return fmt.Errorf("getting repository: %w", err)
}
id := cmd.String("id")
cartID := cmd.Int("cart-id")
if id == "" && cartID == 0 {
return cli.Exit("oneof: id or cart-id should be set", 1)
} else if id != "" && cartID != 0 {
return cli.Exit("oneof: id or cart-id should be set", 1)
}
var item entity.GoodsItem
if id != "" {
item, err = r.GoodsItem().Get(ctx, id)
} else {
item, err = r.GoodsItem().GetByCart(ctx, cartID)
}
if err != nil {
return fmt.Errorf("getting item: %w", err)
}
enc := json.NewEncoder(os.Stdout)
enc.SetIndent("", " ")
err = enc.Encode(item)
if err != nil {
return fmt.Errorf("encoding item: %w", err)
}
return nil
}
func (itemsHandlers) Count(ctx context.Context, cmd *cli.Command) error {
r, err := components.GetRepository()
if err != nil {
return fmt.Errorf("getting repository: %w", err)
}
filters := cmd.StringSlice("param-key-match")
m := matcher.NewRadix()
patternMapped := make(map[string]empty, len(filters))
if len(filters) == 0 {
m.Register("*")
} else {
for _, f := range filters {
m.Register(f)
}
for _, pattern := range m.Patterns() {
patternMapped[pattern] = empty{}
}
}
var count int
items, err := r.GoodsItem().List(ctx)
if err != nil {
return fmt.Errorf("getting items: %w", err)
}
for _, item := range items {
seenPatterns := map[string]empty{}
for k := range item.Parameters {
pattern := m.MatchByPattern(k)
if pattern == "" {
continue
}
if _, ok := seenPatterns[pattern]; ok {
continue
}
seenPatterns[pattern] = empty{}
}
if len(seenPatterns) == len(patternMapped) {
println("Item matched", item.Articul, item.Cart)
count++
}
}
println(count)
return nil
}
func (itemsHandlers) UniqueParameters(ctx context.Context, cmd *cli.Command) error {
repository, err := components.GetRepository()
if err != nil {
return fmt.Errorf("getting repository: %w", err)
}
knownParams := map[string]empty{}
iter, err := repository.GoodsItem().ListIter(ctx, 1)
if err != nil {
return fmt.Errorf("getting list iter: %w", err)
}
for item := range iter {
for k := range item.Parameters {
knownParams[k] = empty{}
}
}
bw := bufio.NewWriter(cmd.Writer)
for paramName := range knownParams {
_, err = bw.WriteString(paramName + "\n")
if err != nil {
return fmt.Errorf("unable to write: %w", err)
}
}
return bw.Flush()
}
func (itemsHandlers) AggregateParameters(ctx context.Context, cmd *cli.Command) error {
repository, err := components.GetRepository()
if err != nil {
return fmt.Errorf("getting repository: %w", err)
}
log, err := components.GetLogger()
if err != nil {
return fmt.Errorf("getting logger: %w", err)
}
params := cmd.Args().Slice()
opts := make([]matcher.RadixOpt, 0, 1)
if cmd.Bool("case-insensitive") {
opts = append(opts, matcher.RadixCaseInsensitive())
}
m := matcher.NewRadix(opts...)
for _, param := range params {
log.Debug().Str("param", param).Msg("registering param")
m.Register(param)
}
for _, regexp := range cmd.StringSlice("regex") {
log.Debug().Str("regexp", regexp).Msg("registering regexp")
m.RegisterRegexp(regexp)
}
requestedValues := make(map[string]map[string]empty, len(params))
requestedValuesByPattern := make(map[string]map[string]empty, len(params))
iter := getItemsIter(ctx, repository.GoodsItem())
for iter.Next() {
item := iter.Get()
for k, v := range item.Parameters {
matchedPattern := m.MatchByPattern(k)
if matchedPattern == "" {
continue
}
values, ok := requestedValues[k]
if !ok {
values = make(map[string]empty)
}
values[v] = empty{}
requestedValues[k] = values
values, ok = requestedValuesByPattern[matchedPattern]
if !ok {
values = map[string]empty{}
}
values[v] = empty{}
requestedValuesByPattern[matchedPattern] = values
}
}
bw := bufio.NewWriter(cmd.Writer)
_, _ = bw.WriteString("Matches:\n")
if cmd.Bool("keys-only") {
for k := range requestedValues {
_, _ = bw.WriteString(k + "\n")
}
} else {
for k, v := range requestedValues {
_, _ = bw.WriteString(k + ": ")
values := make([]string, 0, len(v))
for item := range v {
values = append(values, strconv.Quote(item))
}
valuesStr := "[" + strings.Join(values, ",") + "]\n"
_, _ = bw.WriteString(valuesStr)
}
}
_, _ = bw.WriteString("\nPatterns:\n")
for _, pattern := range m.Patterns() {
_, _ = bw.WriteString(pattern + "\n")
}
return bw.Flush()
}
func (itemsHandlers) FillSizes(ctx context.Context, cmd *cli.Command) error {
repository, err := components.GetRepository()
if err != nil {
return fmt.Errorf("getting repository: %w", err)
}
dimensionDispatcher := makeDefaultDimensionDispatcher()
log := zerolog.Ctx(ctx)
toUpdate := make([]entity.GoodsItem, 0, 20_000)
bus := getItemsIter(ctx, repository.GoodsItem())
for bus.Next() {
item := bus.Get()
var valueBeenUpdated bool
for key, value := range item.Parameters {
trimmedValue := strings.TrimSpace(value)
trimmedKey := strings.TrimSpace(key)
if trimmedKey != key || trimmedValue != value {
log.Warn().
Str("old_key", key).
Str("new_key", trimmedKey).
Str("old_value", value).
Str("new_value", trimmedValue).
Msg("found mismatch")
delete(item.Parameters, key)
key = trimmedKey
value = trimmedValue
item.Parameters[key] = value
valueBeenUpdated = true
}
updateValue := strings.HasSuffix(key, ":")
if updateValue {
key = strings.TrimSuffix(key, ":")
item.Parameters[key] = value
delete(item.Parameters, key+":")
valueBeenUpdated = true
}
if dimensionDispatcher.dispatch(ctx, key, value, &item.Sizes) {
valueBeenUpdated = true
log.Debug().Str("key", key).Any("sizes", item.Sizes).Msg("been updated")
}
}
if valueBeenUpdated {
toUpdate = append(toUpdate, item)
}
}
if bus.Err() != nil {
return fmt.Errorf("iterating: %w", bus.Err())
}
_, err = repository.GoodsItem().UpsertMany(ctx, toUpdate...)
if err != nil {
return fmt.Errorf("updating items: %w", err)
}
log.Info().Int("count", len(toUpdate)).Msg("updated items")
return nil
}