frx/eway
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

refactor logging and filters

Browse Source
This commit is contained in:
Aleksandr Trushkin
2024-02-19 18:11:59 +03:00
parent 288245f6f0
commit 4563a3bede
13 changed files with 364 additions and 92 deletions
Download Patch File Download Diff File Expand all files Collapse all files

52
cmd/cli/commands/dimensiondispatcher.go
View File

@ -4,27 +4,28 @@ import (
"context"
"strings"
"git.loyso.art/frx/eway/internal/dimension"
"git.loyso.art/frx/eway/internal/entity"
"git.loyso.art/frx/eway/internal/matcher"
"github.com/rs/zerolog"
)
type dimensionDispatcher struct {
heigth matcher.Unit
width matcher.Unit
length matcher.Unit
m *dimension.Matcher
}
func (d dimensionDispatcher) isDimensionParam(value string) bool {
return d.heigth.Match(value) || d.width.Match(value) || d.length.Match(value)
return d.m.Match(value) != dimension.MatchResultMiss
}
func (d dimensionDispatcher) dispatch(ctx context.Context, key, value string, in *entity.GoodsItemSize) (updated bool) {
if !d.isDimensionParam(key) {
matchResult := d.m.Match(key)
if matchResult == dimension.MatchResultMiss {
return false
}
log := zerolog.Ctx(ctx).With().Str("key", key).Str("value", value).Logger()
if strings.Contains(value, "/") {
dimensionValues := strings.Split(value, "/")
for _, dv := range dimensionValues {
@ -39,40 +40,17 @@ func (d dimensionDispatcher) dispatch(ctx context.Context, key, value string, in
out = out.AdjustTo(entity.DimensionKindCentimeter)
updated = true
switch {
case d.heigth.Match(key):
switch matchResult {
case dimension.MatchResultHeight:
in.Height = out
case d.width.Match(key):
in.Width = out
case d.length.Match(key):
case dimension.MatchResultLength:
in.Length = out
default:
log.Error().Str("key", key).Msg("unable to find proper matcher")
updated = false
case dimension.MatchResultWidth:
in.Width = out
case dimension.MatchResultDepth:
in.UnmatchedDepth = out
}
}
return updated
}
func makeDefaultDimensionDispatcher() dimensionDispatcher {
h := matcher.NewRadix(matcher.RadixCaseInsensitive())
h.Register("Высота")
h.Register("Высота/*")
w := matcher.NewRadix(matcher.RadixCaseInsensitive())
w.Register("Ширина")
w.Register("Ширина/*")
l := matcher.NewRadix(matcher.RadixCaseInsensitive())
l.Register("Длина")
l.Register("Длина/*")
l.Register("Общ. длина")
return dimensionDispatcher{
heigth: h,
width: w,
length: l,
}
return true
}

142
cmd/cli/commands/exports.go
View File

@ -4,6 +4,7 @@ import (
"context"
"encoding/xml"
"fmt"
"io"
"os"
"strconv"
"strings"
@ -32,6 +33,25 @@ func newExportYMLCatalogCommand(h exportHandlers) *cli.Command {
cmd := cli.Command{
Name: "yml-catalog",
Usage: "Export data as yml_catalog",
Flags: []cli.Flag{
&cli.StringFlag{
Name: "out",
Aliases: []string{"o"},
Usage: "Output to file or stdout/stderr",
Value: "yml_catalog.xml",
TakesFile: true,
},
&cli.IntFlag{
Name: "limit",
Aliases: []string{"l"},
Usage: "limits amount of items to save",
},
&cli.BoolFlag{
Name: "pretty",
Aliases: []string{"p"},
Usage: "pretty-prints output",
},
},
}
return cmdWithAction(cmd, h.YMLCatalog)
@ -81,56 +101,88 @@ func (h exportHandlers) YMLCatalog(ctx context.Context, cmd *cli.Command) error
})
}
matcher := makeDefaultDimensionDispatcher()
matcher, err := components.GetDimensionMatcher()
if err != nil {
return fmt.Errorf("getting dimension matcher: %w", err)
}
dimensionDispatcher := dimensionDispatcher{m: matcher}
log.Info().Any("patterns", matcher.GetRegisteredPatterns()).Msg("configured patterns")
const (
reasonNoSize = "no_size"
reasonTooLarge = "too_large"
reasonNoSize = "no_size"
reasonTooLarge = "too_large"
reasonNoDescription = "no_description"
reasonNoPhoto = "no_photo"
)
skippedByReasons := map[string]int{
reasonNoSize: 0,
reasonTooLarge: 0,
reasonNoSize: 0,
reasonTooLarge: 0,
reasonNoDescription: 0,
reasonNoPhoto: 0,
}
addToSkip := func(condition bool, name string, log zerolog.Logger) bool {
if !condition {
return false
}
log.Debug().Str("reason", name).Msg("skipping item")
skippedByReasons[name]++
return condition
}
iter := getItemsIter(ctx, r.GoodsItem())
for iter.Next() {
const maximumAllowedSizes = 160
item := iter.Get()
sublog := log.With().Int64("cart", item.Cart).Logger()
if item.Sizes == (entity.GoodsItemSize{}) {
sublog.Warn().Str("reason", reasonNoSize).Msg("skipping item")
skippedByReasons[reasonNoSize]++
switch {
case addToSkip(item.Description == "", reasonNoDescription, sublog):
continue
case addToSkip(item.Sizes == (entity.GoodsItemSize{}), reasonNoSize, sublog):
continue
case addToSkip(item.Sizes.GetSum(entity.DimensionKindCentimeter) > maximumAllowedSizes, reasonTooLarge, sublog):
continue
}
const maximumAllowedSizes = 160
if sum := item.Sizes.GetSum(entity.DimensionKindCentimeter); sum > maximumAllowedSizes {
sublog.Warn().Str("reason", reasonTooLarge).Msg("skipping item")
skippedByReasons[reasonTooLarge]++
offer := h.goodsItemAsOffer(item, categoryByNameIdx, dimensionDispatcher, sublog)
if addToSkip(len(offer.PictureURLs) == 0, reasonNoPhoto, sublog) {
continue
}
offer := h.goodsItemAsOffer(iter.Get(), categoryByNameIdx, matcher, sublog)
shop.Offers = append(shop.Offers, offer)
}
if err = iter.Err(); err != nil {
return fmt.Errorf("iterating over items: %w", err)
}
f, err := os.Create(path)
if err != nil {
return fmt.Errorf("creating file: %w", err)
}
defer func() {
errClose := f.Close()
if err == nil {
err = errClose
return
var f io.WriteCloser
switch path {
case "stdout":
f = os.Stdout
case "stderr":
f = os.Stderr
default:
f, err = os.Create(path)
if err != nil {
return fmt.Errorf("creating file: %w", err)
}
defer func() {
errClose := f.Close()
if err == nil {
err = errClose
return
}
log.Err(errClose).Msg("file closed or not")
}()
log.Err(errClose).Msg("file closed or not")
}()
}
if limit > 0 {
shop.Offers = shop.Offers[:limit]
@ -173,9 +225,19 @@ func (h exportHandlers) goodsItemAsOffer(in entity.GoodsItem, categoryIDByName m
pictureURLs := make([]string, 0, len(in.PhotoURLs))
for _, url := range in.PhotoURLs {
if url == "" {
continue
}
outurl := imgurl(url)
if outurl == basePictureURL {
continue
}
pictureURLs = append(pictureURLs, imgurl(url))
}
params := make([]export.Param, len(in.Parameters))
params := make([]export.Param, 0, len(in.Parameters))
for k, v := range in.Parameters {
if k == "" || v == "" {
continue
@ -219,27 +281,29 @@ func (h exportHandlers) goodsItemAsOffer(in entity.GoodsItem, categoryIDByName m
return out
}
// check cart id 126584
func (exportHandlers) formatSizeAsDimensions(size entity.GoodsItemSize, log zerolog.Logger) string {
const delimeter = "/"
makeFloat := func(d entity.Dimension) string {
value := size.Length.AdjustTo(entity.DimensionKindCentimeter).Value
value = float64(int(value*1000)) / 1000.0
value := d.AdjustTo(entity.DimensionKindCentimeter).Value
value = float64(int(value*100)) / 100.0
return strconv.FormatFloat(value, 'f', 8, 64)
return strconv.FormatFloat(value, 'f', 1, 64)
}
knownSizes := make([]entity.Dimension, 0, 3)
for _, d := range []entity.Dimension{
dimensions := []entity.Dimension{
size.Length,
size.Width,
size.Height,
} {
}
for i, d := range dimensions {
if d.IsZero() {
continue
}
knownSizes = append(knownSizes, d)
knownSizes = append(knownSizes, dimensions[i])
}
l := makeFloat(size.Length)
@ -262,7 +326,21 @@ func (exportHandlers) formatSizeAsDimensions(size entity.GoodsItemSize, log zero
side = "height"
h = unknownDefaultSize
}
log.Warn().Str("size", side).Msg("setting to default value")
log.Debug().Str("size", side).Msg("setting to default value")
case 1:
var side string
unknownDefaultSize := makeFloat(entity.NewCentimeterDimensionOrEmpty(30))
switch {
case !size.Length.IsZero():
side = "width"
w = unknownDefaultSize
case !size.Width.IsZero():
side = "length"
l = unknownDefaultSize
case !size.Height.IsZero():
return ""
}
log.Debug().Str("size", side).Msg("setting to default value")
default:
return ""
}

28
cmd/cli/commands/items.go
View File

@ -27,7 +27,7 @@ func ItemsCommandTree() *cli.Command {
newItemsCountCommand(h),
newItemsUniqueParamsCommand(h),
newItemsAggregateParametersCommand(h),
newItemsFillSizesCommand(h),
newItemsFixSizesCommand(h),
},
}
}
@ -100,13 +100,13 @@ func newItemsAggregateParametersCommand(h itemsHandlers) *cli.Command {
return cmdWithAction(cmd, h.AggregateParameters)
}
func newItemsFillSizesCommand(h itemsHandlers) *cli.Command {
func newItemsFixSizesCommand(h itemsHandlers) *cli.Command {
cmd := cli.Command{
Name: "fix-sizes",
Usage: "Iterates over params and sets sizes from parameters",
}
return cmdWithAction(cmd, h.FillSizes)
return cmdWithAction(cmd, h.FixSizes)
}
type itemsHandlers struct{}
@ -186,13 +186,10 @@ func (itemsHandlers) Count(ctx context.Context, cmd *cli.Command) error {
}
if len(seenPatterns) == len(patternMapped) {
println("Item matched", item.Articul, item.Cart)
count++
}
}
println(count)
return nil
}
@ -304,13 +301,19 @@ func (itemsHandlers) AggregateParameters(ctx context.Context, cmd *cli.Command)
return bw.Flush()
}
func (itemsHandlers) FillSizes(ctx context.Context, cmd *cli.Command) error {
func (itemsHandlers) FixSizes(ctx context.Context, cmd *cli.Command) error {
repository, err := components.GetRepository()
if err != nil {
return fmt.Errorf("getting repository: %w", err)
}
dimensionDispatcher := makeDefaultDimensionDispatcher()
matcher, err := components.GetDimensionMatcher()
if err != nil {
return fmt.Errorf("getting dimension matcher: %w", err)
}
dimensionDispatcher := dimensionDispatcher{m: matcher}
log := zerolog.Ctx(ctx)
toUpdate := make([]entity.GoodsItem, 0, 20_000)
@ -354,6 +357,15 @@ func (itemsHandlers) FillSizes(ctx context.Context, cmd *cli.Command) error {
}
}
var updated bool
oldSizes := item.Sizes
item.Sizes, updated = entity.FixupSizes(oldSizes)
if updated {
log.Info().Int64("cart", item.Cart).Any("old", oldSizes).Any("new", item.Sizes).Msg("sizes been fixed")
}
valueBeenUpdated = valueBeenUpdated || updated
if valueBeenUpdated {
toUpdate = append(toUpdate, item)
}

18
cmd/cli/components/di.go
View File

@ -9,6 +9,7 @@ import (
"time"
"git.loyso.art/frx/eway/internal/config"
"git.loyso.art/frx/eway/internal/dimension"
"git.loyso.art/frx/eway/internal/interconnect/eway"
"git.loyso.art/frx/eway/internal/storage"
xbadger "git.loyso.art/frx/eway/internal/storage/badger"
@ -39,6 +40,10 @@ func GetLogger() (zerolog.Logger, error) {
return do.Invoke[zerolog.Logger](diInjector)
}
func GetDimensionMatcher() (*dimension.Matcher, error) {
return do.Invoke[*dimension.Matcher](diInjector)
}
func SetupDI(ctx context.Context, cfgpath string, verbose bool, logAsJSON bool) error {
cfg, err := parseSettings(cfgpath)
if err != nil {
@ -115,6 +120,12 @@ func SetupDI(ctx context.Context, cfgpath string, verbose bool, logAsJSON bool)
return out, nil
})
do.Provide[*dimension.Matcher](diInjector, func(i *do.Injector) (*dimension.Matcher, error) {
matcher := dimension.New(cfg.DimensionMatcher)
return matcher, nil
})
return nil
}
@ -136,9 +147,10 @@ func getDB() (*badger.DB, error) {
}
type settings struct {
Badger config.Badger `toml:"badger"`
Log config.Log `toml:"log"`
Eway config.Eway `toml:"eway"`
Badger config.Badger `toml:"badger"`
Log config.Log `toml:"log"`
Eway config.Eway `toml:"eway"`
DimensionMatcher config.DimensionMatcher `toml:"dimension_matcher"`
}
func parseSettings(cfgpath string) (cfg settings, err error) {

29
cmd/cli/main.go
View File

@ -27,6 +27,10 @@ import (
"github.com/urfave/cli/v3"
)
const (
defaultItemType = "Электрика"
)
type empty entity.Empty
func main() {
@ -594,7 +598,9 @@ func parseEwayDumpAction(ctx context.Context, cmd *cli.Command) error {
RemmantsAtleast: 5,
})
if err != nil {
return fmt.Errorf("getting next goods batch: %w", err)
logger.Warn().Err(err).Msg("unable to get items from catalog")
continue
}
productIDs = productIDs[:0]
@ -604,6 +610,7 @@ func parseEwayDumpAction(ctx context.Context, cmd *cli.Command) error {
remnants, err := client.GetGoodsRemnants(ctx, productIDs)
if err != nil {
logger.Warn().Err(err).Msg("unable to get goods remnants")
return fmt.Errorf("getting goods remnants: %w", err)
}
@ -613,6 +620,7 @@ func parseEwayDumpAction(ctx context.Context, cmd *cli.Command) error {
seenItem := seenItems[item.SKU]
if time.Since(seenItem.CreatedAt) < time.Hour*24 {
logger.Debug().Str("sku", item.SKU).Msg("skipping item because it's too fresh")
stats.skippedItem++
itemsUpdated[item.SKU] = empty{}
@ -626,14 +634,16 @@ func parseEwayDumpAction(ctx context.Context, cmd *cli.Command) error {
} else {
pi, err = client.GetProductInfo(ctx, int64(item.Cart))
if err != nil {
return fmt.Errorf("getting product info: %w", err)
logger.Warn().Err(err).Msg("unable to get product info, skipping")
continue
}
stats.fetchedInfo++
}
goodsItem, err := entity.MakeGoodsItem(item, remnants, pi)
if err != nil {
logger.Warn().Err(err).Any("item", item).Msg("unable to make goods item")
logger.Err(err).Any("item", item).Msg("unable to make goods item")
continue
}
@ -646,7 +656,8 @@ func parseEwayDumpAction(ctx context.Context, cmd *cli.Command) error {
goodsItems = append(goodsItems, goodsItem)
if goodsItem.Type == "" {
continue
logger.Warn().Int64("cart_id", goodsItem.Cart).Msg("found item without category, setting default type")
goodsItem.Type = defaultItemType
}
if _, ok := knownCategories[goodsItem.Type]; ok {
@ -658,7 +669,7 @@ func parseEwayDumpAction(ctx context.Context, cmd *cli.Command) error {
return fmt.Errorf("creating category: %w", err)
}
logger.Debug().
logger.Info().
Str("name", category.Name).
Int64("id", category.ID).
Msg("created new category")
@ -673,8 +684,8 @@ func parseEwayDumpAction(ctx context.Context, cmd *cli.Command) error {
progressFloat := float64(start) / float64(total)
progress := big.NewFloat(progressFloat).Text('f', 3)
elapsed := time.Since(startFrom).Seconds()
var left int
if progressFloat != 0 {
left = int(((1 - progressFloat) / progressFloat) * elapsed)
@ -711,11 +722,15 @@ func parseEwayDumpAction(ctx context.Context, cmd *cli.Command) error {
for k := range seenItems {
_, err := repository.GoodsItem().Delete(ctx, k)
if err != nil {
if errors.Is(err, entity.ErrNotFound) {
continue
}
logger.Warn().Err(err).Str("sku", k).Msg("unable to delete item")
continue
}
logger.Debug().Str("sku", k).Msg("deleted item")
logger.Info().Str("sku", k).Msg("deleted item")
}
return nil

42
internal/config/matcher.go Normal file
View File

@ -0,0 +1,42 @@
package config
import (
"errors"
"strings"
)
type MatcherPredicateType uint8
const (
MatcherPredicateTypeUnknown MatcherPredicateType = iota
MatcherPredicateTypePattern
MatcherPredicateTypeRegexp
)
func (t *MatcherPredicateType) UnmarshalText(data []byte) error {
switch dataStr := strings.ToLower(string(data)); dataStr {
case "":
*t = MatcherPredicateTypeUnknown
case "pattern":
*t = MatcherPredicateTypePattern
case "regexp":
*t = MatcherPredicateTypeRegexp
default:
return errors.New("unsupported type " + dataStr)
}
return nil
}
type MatcherPredicate struct {
Value string `toml:"value"`
Type MatcherPredicateType `toml:"type"`
}
type DimensionMatcher struct {
CaseInsensitive bool `toml:"case_insensitive"`
Length []MatcherPredicate `toml:"length"`
Width []MatcherPredicate `toml:"width"`
Height []MatcherPredicate `toml:"height"`
}

92
internal/dimension/matcher.go Normal file
View File

@ -0,0 +1,92 @@
package dimension
import (
"git.loyso.art/frx/eway/internal/config"
"git.loyso.art/frx/eway/internal/matcher"
)
type MatchResult uint8
const (
MatchResultMiss MatchResult = iota
MatchResultLength
MatchResultHeight
MatchResultWidth
MatchResultDepth
)
type Matcher struct {
length matcher.Unit
width matcher.Unit
height matcher.Unit
}
func New(cfg config.DimensionMatcher) *Matcher {
return &Matcher{
length: makeMatcherByConig(cfg.CaseInsensitive, cfg.Length...),
width: makeMatcherByConig(cfg.CaseInsensitive, cfg.Width...),
height: makeMatcherByConig(cfg.CaseInsensitive, cfg.Height...),
}
}
func makeMatcherByConig(insensitive bool, cfgs ...config.MatcherPredicate) matcher.Unit {
opts := make([]matcher.RadixOpt, 0, 1)
if insensitive {
opts = append(opts, matcher.RadixCaseInsensitive())
}
m := matcher.NewRadix(opts...)
for _, cfg := range cfgs {
switch cfg.Type {
case config.MatcherPredicateTypePattern:
m.Register(cfg.Value)
case config.MatcherPredicateTypeRegexp:
m.RegisterRegexp(cfg.Value)
default:
panic("unsupported matcher type")
}
}
return m
}
func (m *Matcher) Match(value string) MatchResult {
switch {
case value == "Высота":
fallthrough
case m.height.Match(value):
return MatchResultHeight
case value == "Глубина":
return MatchResultDepth
case value == "Длина":
fallthrough
case m.length.Match(value):
return MatchResultLength
case value == "Ширина":
fallthrough
case m.width.Match(value):
return MatchResultWidth
}
return MatchResultMiss
}
func (m *Matcher) GetRegisteredPatterns() map[string][]string {
out := map[string][]string{
"length": nil,
"width": nil,
"height": nil,
}
if m.height != nil {
out["height"] = m.height.Patterns()
}
if m.width != nil {
out["width"] = m.width.Patterns()
}
if m.length != nil {
out["length"] = m.length.Patterns()
}
return out
}

5
internal/encoding/fbs/helpers.go
View File

@ -115,7 +115,10 @@ func makeDomainGoodItem(builder *flatbuffers.Builder, in entity.GoodsItem) flatb
func ParseGoodsItem(data []byte) (item entity.GoodsItem, err error) {
itemFBS := GetRootAsGoodItem(data, 0)
item.Articul = string(itemFBS.Sku())
item.PhotoURLs = strings.Split(string(itemFBS.Photo()), ";")
photoURLs := string(itemFBS.Photo())
if len(photoURLs) > 0 {
item.PhotoURLs = strings.Split(photoURLs, ";")
}
item.Name = string(itemFBS.Name())
description, err := base64.RawStdEncoding.DecodeString(string(itemFBS.Description()))

38
internal/entity/gooditem.go
View File

@ -12,6 +12,44 @@ type GoodsItemSize struct {
Width Dimension
Height Dimension
Length Dimension
UnmatchedDepth Dimension
}
func FixupSizes(s GoodsItemSize) (GoodsItemSize, bool) {
// Nothing to substitute
if s.UnmatchedDepth.IsZero() {
return s, false
}
var count int
for _, d := range []Dimension{
s.Width,
s.Height,
s.Length,
} {
if d.IsZero() {
count++
}
}
// Can only replace one dimension
if count != 1 {
return s, false
}
switch {
case s.Width.IsZero():
s.Width = s.UnmatchedDepth
case s.Height.IsZero():
s.Height = s.UnmatchedDepth
case s.Length.IsZero():
s.Length = s.UnmatchedDepth
}
s.UnmatchedDepth = Dimension{}
return s, true
}
func (s GoodsItemSize) GetSum(kind DimensionKind) float64 {

2
internal/export/itemsmarket.go
View File

@ -24,7 +24,7 @@ type Offer struct {
Description string `xml:"description"`
ManufacturerWarrany bool `xml:"manufacturer_warranty"`
Dimensions string `xml:"dimensions"`
Params []Param `xml:"param"`
Params []Param `xml:"param,omitempty"`
}
type Currency struct {

2
internal/interconnect/eway/client.go
View File

@ -355,8 +355,6 @@ func (c *client) getProductInfo(ctx context.Context, cartID int64) (pi entity.Go
return strings.Contains(err.Error(), "pipe")
})
c.log.Debug().Msg("using go query")
pi.Parameters = map[string]string{}
resp, err := c.do(ctx, "getProductInfo", req, resty.MethodGet, reqpath)
if err != nil {

2
internal/matcher/radix.go
View File

@ -71,7 +71,7 @@ func (r *radixMatcher) MatchByPattern(value string) (pattern string) {
}
if !node.exact {
return r.findByRegexp(value)
return r.findByRegexp(originValue)
}
return sb.String()

4
internal/matcher/radix_test.go
View File

@ -11,6 +11,7 @@ func TestRadixMatcherWithPattern(t *testing.T) {
m.Register("aloha")
m.Register("hawaii")
m.Register("te*")
m.Register("Ширина")
var tt = []struct {
name string
@ -37,6 +38,9 @@ func TestRadixMatcherWithPattern(t *testing.T) {
}, {
name: "should not match 3",
in: "alohaya",
}, {
name: "should match exact 3",
in: "Ширина",
}}
for _, tc := range tt {