frx/eway
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

refactor logging and filters

Browse Source
This commit is contained in:
Aleksandr Trushkin
2024-02-19 18:11:59 +03:00
parent 288245f6f0
commit 4563a3bede
13 changed files with 364 additions and 92 deletions
Download Patch File Download Diff File Expand all files Collapse all files

52
cmd/cli/commands/dimensiondispatcher.go
View File

@ -4,27 +4,28 @@ import (
"context"
"strings"
"git.loyso.art/frx/eway/internal/dimension"
"git.loyso.art/frx/eway/internal/entity"
"git.loyso.art/frx/eway/internal/matcher"
"github.com/rs/zerolog"
)
type dimensionDispatcher struct {
heigth matcher.Unit
width matcher.Unit
length matcher.Unit
m *dimension.Matcher
}
func (d dimensionDispatcher) isDimensionParam(value string) bool {
return d.heigth.Match(value) || d.width.Match(value) || d.length.Match(value)
return d.m.Match(value) != dimension.MatchResultMiss
}
func (d dimensionDispatcher) dispatch(ctx context.Context, key, value string, in *entity.GoodsItemSize) (updated bool) {
if !d.isDimensionParam(key) {
matchResult := d.m.Match(key)
if matchResult == dimension.MatchResultMiss {
return false
}
log := zerolog.Ctx(ctx).With().Str("key", key).Str("value", value).Logger()
if strings.Contains(value, "/") {
dimensionValues := strings.Split(value, "/")
for _, dv := range dimensionValues {
@ -39,40 +40,17 @@ func (d dimensionDispatcher) dispatch(ctx context.Context, key, value string, in
out = out.AdjustTo(entity.DimensionKindCentimeter)
updated = true
switch {
case d.heigth.Match(key):
switch matchResult {
case dimension.MatchResultHeight:
in.Height = out
case d.width.Match(key):
in.Width = out
case d.length.Match(key):
case dimension.MatchResultLength:
in.Length = out
default:
log.Error().Str("key", key).Msg("unable to find proper matcher")
updated = false
case dimension.MatchResultWidth:
in.Width = out
case dimension.MatchResultDepth:
in.UnmatchedDepth = out
}
}
return updated
}
func makeDefaultDimensionDispatcher() dimensionDispatcher {
h := matcher.NewRadix(matcher.RadixCaseInsensitive())
h.Register("Высота")
h.Register("Высота/*")
w := matcher.NewRadix(matcher.RadixCaseInsensitive())
w.Register("Ширина")
w.Register("Ширина/*")
l := matcher.NewRadix(matcher.RadixCaseInsensitive())
l.Register("Длина")
l.Register("Длина/*")
l.Register("Общ. длина")
return dimensionDispatcher{
heigth: h,
width: w,
length: l,
}
return true
}

142
cmd/cli/commands/exports.go
View File

@ -4,6 +4,7 @@ import (
"context"
"encoding/xml"
"fmt"
"io"
"os"
"strconv"
"strings"
@ -32,6 +33,25 @@ func newExportYMLCatalogCommand(h exportHandlers) *cli.Command {
cmd := cli.Command{
Name: "yml-catalog",
Usage: "Export data as yml_catalog",
Flags: []cli.Flag{
&cli.StringFlag{
Name: "out",
Aliases: []string{"o"},
Usage: "Output to file or stdout/stderr",
Value: "yml_catalog.xml",
TakesFile: true,
},
&cli.IntFlag{
Name: "limit",
Aliases: []string{"l"},
Usage: "limits amount of items to save",
},
&cli.BoolFlag{
Name: "pretty",
Aliases: []string{"p"},
Usage: "pretty-prints output",
},
},
}
return cmdWithAction(cmd, h.YMLCatalog)
@ -81,56 +101,88 @@ func (h exportHandlers) YMLCatalog(ctx context.Context, cmd *cli.Command) error
})
}
matcher := makeDefaultDimensionDispatcher()
matcher, err := components.GetDimensionMatcher()
if err != nil {
return fmt.Errorf("getting dimension matcher: %w", err)
}
dimensionDispatcher := dimensionDispatcher{m: matcher}
log.Info().Any("patterns", matcher.GetRegisteredPatterns()).Msg("configured patterns")
const (
reasonNoSize = "no_size"
reasonTooLarge = "too_large"
reasonNoSize = "no_size"
reasonTooLarge = "too_large"
reasonNoDescription = "no_description"
reasonNoPhoto = "no_photo"
)
skippedByReasons := map[string]int{
reasonNoSize: 0,
reasonTooLarge: 0,
reasonNoSize: 0,
reasonTooLarge: 0,
reasonNoDescription: 0,
reasonNoPhoto: 0,
}
addToSkip := func(condition bool, name string, log zerolog.Logger) bool {
if !condition {
return false
}
log.Debug().Str("reason", name).Msg("skipping item")
skippedByReasons[name]++
return condition
}
iter := getItemsIter(ctx, r.GoodsItem())
for iter.Next() {
const maximumAllowedSizes = 160
item := iter.Get()
sublog := log.With().Int64("cart", item.Cart).Logger()
if item.Sizes == (entity.GoodsItemSize{}) {
sublog.Warn().Str("reason", reasonNoSize).Msg("skipping item")
skippedByReasons[reasonNoSize]++
switch {
case addToSkip(item.Description == "", reasonNoDescription, sublog):
continue
case addToSkip(item.Sizes == (entity.GoodsItemSize{}), reasonNoSize, sublog):
continue
case addToSkip(item.Sizes.GetSum(entity.DimensionKindCentimeter) > maximumAllowedSizes, reasonTooLarge, sublog):
continue
}
const maximumAllowedSizes = 160
if sum := item.Sizes.GetSum(entity.DimensionKindCentimeter); sum > maximumAllowedSizes {
sublog.Warn().Str("reason", reasonTooLarge).Msg("skipping item")
skippedByReasons[reasonTooLarge]++
offer := h.goodsItemAsOffer(item, categoryByNameIdx, dimensionDispatcher, sublog)
if addToSkip(len(offer.PictureURLs) == 0, reasonNoPhoto, sublog) {
continue
}
offer := h.goodsItemAsOffer(iter.Get(), categoryByNameIdx, matcher, sublog)
shop.Offers = append(shop.Offers, offer)
}
if err = iter.Err(); err != nil {
return fmt.Errorf("iterating over items: %w", err)
}
f, err := os.Create(path)
if err != nil {
return fmt.Errorf("creating file: %w", err)
}
defer func() {
errClose := f.Close()
if err == nil {
err = errClose
return
var f io.WriteCloser
switch path {
case "stdout":
f = os.Stdout
case "stderr":
f = os.Stderr
default:
f, err = os.Create(path)
if err != nil {
return fmt.Errorf("creating file: %w", err)
}
defer func() {
errClose := f.Close()
if err == nil {
err = errClose
return
}
log.Err(errClose).Msg("file closed or not")
}()
log.Err(errClose).Msg("file closed or not")
}()
}
if limit > 0 {
shop.Offers = shop.Offers[:limit]
@ -173,9 +225,19 @@ func (h exportHandlers) goodsItemAsOffer(in entity.GoodsItem, categoryIDByName m
pictureURLs := make([]string, 0, len(in.PhotoURLs))
for _, url := range in.PhotoURLs {
if url == "" {
continue
}
outurl := imgurl(url)
if outurl == basePictureURL {
continue
}
pictureURLs = append(pictureURLs, imgurl(url))
}
params := make([]export.Param, len(in.Parameters))
params := make([]export.Param, 0, len(in.Parameters))
for k, v := range in.Parameters {
if k == "" || v == "" {
continue
@ -219,27 +281,29 @@ func (h exportHandlers) goodsItemAsOffer(in entity.GoodsItem, categoryIDByName m
return out
}
// check cart id 126584
func (exportHandlers) formatSizeAsDimensions(size entity.GoodsItemSize, log zerolog.Logger) string {
const delimeter = "/"
makeFloat := func(d entity.Dimension) string {
value := size.Length.AdjustTo(entity.DimensionKindCentimeter).Value
value = float64(int(value*1000)) / 1000.0
value := d.AdjustTo(entity.DimensionKindCentimeter).Value
value = float64(int(value*100)) / 100.0
return strconv.FormatFloat(value, 'f', 8, 64)
return strconv.FormatFloat(value, 'f', 1, 64)
}
knownSizes := make([]entity.Dimension, 0, 3)
for _, d := range []entity.Dimension{
dimensions := []entity.Dimension{
size.Length,
size.Width,
size.Height,
} {
}
for i, d := range dimensions {
if d.IsZero() {
continue
}
knownSizes = append(knownSizes, d)
knownSizes = append(knownSizes, dimensions[i])
}
l := makeFloat(size.Length)
@ -262,7 +326,21 @@ func (exportHandlers) formatSizeAsDimensions(size entity.GoodsItemSize, log zero
side = "height"
h = unknownDefaultSize
}
log.Warn().Str("size", side).Msg("setting to default value")
log.Debug().Str("size", side).Msg("setting to default value")
case 1:
var side string
unknownDefaultSize := makeFloat(entity.NewCentimeterDimensionOrEmpty(30))
switch {
case !size.Length.IsZero():
side = "width"
w = unknownDefaultSize
case !size.Width.IsZero():
side = "length"
l = unknownDefaultSize
case !size.Height.IsZero():
return ""
}
log.Debug().Str("size", side).Msg("setting to default value")
default:
return ""
}

28
cmd/cli/commands/items.go
View File

@ -27,7 +27,7 @@ func ItemsCommandTree() *cli.Command {
newItemsCountCommand(h),
newItemsUniqueParamsCommand(h),
newItemsAggregateParametersCommand(h),
newItemsFillSizesCommand(h),
newItemsFixSizesCommand(h),
},
}
}
@ -100,13 +100,13 @@ func newItemsAggregateParametersCommand(h itemsHandlers) *cli.Command {
return cmdWithAction(cmd, h.AggregateParameters)
}
func newItemsFillSizesCommand(h itemsHandlers) *cli.Command {
func newItemsFixSizesCommand(h itemsHandlers) *cli.Command {
cmd := cli.Command{
Name: "fix-sizes",
Usage: "Iterates over params and sets sizes from parameters",
}
return cmdWithAction(cmd, h.FillSizes)
return cmdWithAction(cmd, h.FixSizes)
}
type itemsHandlers struct{}
@ -186,13 +186,10 @@ func (itemsHandlers) Count(ctx context.Context, cmd *cli.Command) error {
}
if len(seenPatterns) == len(patternMapped) {
println("Item matched", item.Articul, item.Cart)
count++
}
}
println(count)
return nil
}
@ -304,13 +301,19 @@ func (itemsHandlers) AggregateParameters(ctx context.Context, cmd *cli.Command)
return bw.Flush()
}
func (itemsHandlers) FillSizes(ctx context.Context, cmd *cli.Command) error {
func (itemsHandlers) FixSizes(ctx context.Context, cmd *cli.Command) error {
repository, err := components.GetRepository()
if err != nil {
return fmt.Errorf("getting repository: %w", err)
}
dimensionDispatcher := makeDefaultDimensionDispatcher()
matcher, err := components.GetDimensionMatcher()
if err != nil {
return fmt.Errorf("getting dimension matcher: %w", err)
}
dimensionDispatcher := dimensionDispatcher{m: matcher}
log := zerolog.Ctx(ctx)
toUpdate := make([]entity.GoodsItem, 0, 20_000)
@ -354,6 +357,15 @@ func (itemsHandlers) FillSizes(ctx context.Context, cmd *cli.Command) error {
}
}
var updated bool
oldSizes := item.Sizes
item.Sizes, updated = entity.FixupSizes(oldSizes)
if updated {
log.Info().Int64("cart", item.Cart).Any("old", oldSizes).Any("new", item.Sizes).Msg("sizes been fixed")
}
valueBeenUpdated = valueBeenUpdated || updated
if valueBeenUpdated {
toUpdate = append(toUpdate, item)
}