refactor logging and filters
This commit is contained in:
@ -4,27 +4,28 @@ import (
|
|||||||
"context"
|
"context"
|
||||||
"strings"
|
"strings"
|
||||||
|
|
||||||
|
"git.loyso.art/frx/eway/internal/dimension"
|
||||||
"git.loyso.art/frx/eway/internal/entity"
|
"git.loyso.art/frx/eway/internal/entity"
|
||||||
"git.loyso.art/frx/eway/internal/matcher"
|
|
||||||
"github.com/rs/zerolog"
|
"github.com/rs/zerolog"
|
||||||
)
|
)
|
||||||
|
|
||||||
type dimensionDispatcher struct {
|
type dimensionDispatcher struct {
|
||||||
heigth matcher.Unit
|
m *dimension.Matcher
|
||||||
width matcher.Unit
|
|
||||||
length matcher.Unit
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func (d dimensionDispatcher) isDimensionParam(value string) bool {
|
func (d dimensionDispatcher) isDimensionParam(value string) bool {
|
||||||
return d.heigth.Match(value) || d.width.Match(value) || d.length.Match(value)
|
return d.m.Match(value) != dimension.MatchResultMiss
|
||||||
}
|
}
|
||||||
|
|
||||||
func (d dimensionDispatcher) dispatch(ctx context.Context, key, value string, in *entity.GoodsItemSize) (updated bool) {
|
func (d dimensionDispatcher) dispatch(ctx context.Context, key, value string, in *entity.GoodsItemSize) (updated bool) {
|
||||||
if !d.isDimensionParam(key) {
|
matchResult := d.m.Match(key)
|
||||||
|
if matchResult == dimension.MatchResultMiss {
|
||||||
return false
|
return false
|
||||||
}
|
}
|
||||||
|
|
||||||
log := zerolog.Ctx(ctx).With().Str("key", key).Str("value", value).Logger()
|
log := zerolog.Ctx(ctx).With().Str("key", key).Str("value", value).Logger()
|
||||||
|
|
||||||
if strings.Contains(value, "/") {
|
if strings.Contains(value, "/") {
|
||||||
dimensionValues := strings.Split(value, "/")
|
dimensionValues := strings.Split(value, "/")
|
||||||
for _, dv := range dimensionValues {
|
for _, dv := range dimensionValues {
|
||||||
@ -39,40 +40,17 @@ func (d dimensionDispatcher) dispatch(ctx context.Context, key, value string, in
|
|||||||
|
|
||||||
out = out.AdjustTo(entity.DimensionKindCentimeter)
|
out = out.AdjustTo(entity.DimensionKindCentimeter)
|
||||||
|
|
||||||
updated = true
|
switch matchResult {
|
||||||
switch {
|
case dimension.MatchResultHeight:
|
||||||
case d.heigth.Match(key):
|
|
||||||
in.Height = out
|
in.Height = out
|
||||||
case d.width.Match(key):
|
case dimension.MatchResultLength:
|
||||||
in.Width = out
|
|
||||||
case d.length.Match(key):
|
|
||||||
in.Length = out
|
in.Length = out
|
||||||
default:
|
case dimension.MatchResultWidth:
|
||||||
log.Error().Str("key", key).Msg("unable to find proper matcher")
|
in.Width = out
|
||||||
updated = false
|
case dimension.MatchResultDepth:
|
||||||
|
in.UnmatchedDepth = out
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
return updated
|
return true
|
||||||
}
|
|
||||||
|
|
||||||
func makeDefaultDimensionDispatcher() dimensionDispatcher {
|
|
||||||
h := matcher.NewRadix(matcher.RadixCaseInsensitive())
|
|
||||||
h.Register("Высота")
|
|
||||||
h.Register("Высота/*")
|
|
||||||
|
|
||||||
w := matcher.NewRadix(matcher.RadixCaseInsensitive())
|
|
||||||
w.Register("Ширина")
|
|
||||||
w.Register("Ширина/*")
|
|
||||||
|
|
||||||
l := matcher.NewRadix(matcher.RadixCaseInsensitive())
|
|
||||||
l.Register("Длина")
|
|
||||||
l.Register("Длина/*")
|
|
||||||
l.Register("Общ. длина")
|
|
||||||
|
|
||||||
return dimensionDispatcher{
|
|
||||||
heigth: h,
|
|
||||||
width: w,
|
|
||||||
length: l,
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|||||||
@ -4,6 +4,7 @@ import (
|
|||||||
"context"
|
"context"
|
||||||
"encoding/xml"
|
"encoding/xml"
|
||||||
"fmt"
|
"fmt"
|
||||||
|
"io"
|
||||||
"os"
|
"os"
|
||||||
"strconv"
|
"strconv"
|
||||||
"strings"
|
"strings"
|
||||||
@ -32,6 +33,25 @@ func newExportYMLCatalogCommand(h exportHandlers) *cli.Command {
|
|||||||
cmd := cli.Command{
|
cmd := cli.Command{
|
||||||
Name: "yml-catalog",
|
Name: "yml-catalog",
|
||||||
Usage: "Export data as yml_catalog",
|
Usage: "Export data as yml_catalog",
|
||||||
|
Flags: []cli.Flag{
|
||||||
|
&cli.StringFlag{
|
||||||
|
Name: "out",
|
||||||
|
Aliases: []string{"o"},
|
||||||
|
Usage: "Output to file or stdout/stderr",
|
||||||
|
Value: "yml_catalog.xml",
|
||||||
|
TakesFile: true,
|
||||||
|
},
|
||||||
|
&cli.IntFlag{
|
||||||
|
Name: "limit",
|
||||||
|
Aliases: []string{"l"},
|
||||||
|
Usage: "limits amount of items to save",
|
||||||
|
},
|
||||||
|
&cli.BoolFlag{
|
||||||
|
Name: "pretty",
|
||||||
|
Aliases: []string{"p"},
|
||||||
|
Usage: "pretty-prints output",
|
||||||
|
},
|
||||||
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
return cmdWithAction(cmd, h.YMLCatalog)
|
return cmdWithAction(cmd, h.YMLCatalog)
|
||||||
@ -81,44 +101,75 @@ func (h exportHandlers) YMLCatalog(ctx context.Context, cmd *cli.Command) error
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
matcher := makeDefaultDimensionDispatcher()
|
matcher, err := components.GetDimensionMatcher()
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("getting dimension matcher: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
dimensionDispatcher := dimensionDispatcher{m: matcher}
|
||||||
|
|
||||||
|
log.Info().Any("patterns", matcher.GetRegisteredPatterns()).Msg("configured patterns")
|
||||||
|
|
||||||
const (
|
const (
|
||||||
reasonNoSize = "no_size"
|
reasonNoSize = "no_size"
|
||||||
reasonTooLarge = "too_large"
|
reasonTooLarge = "too_large"
|
||||||
|
reasonNoDescription = "no_description"
|
||||||
|
reasonNoPhoto = "no_photo"
|
||||||
)
|
)
|
||||||
|
|
||||||
skippedByReasons := map[string]int{
|
skippedByReasons := map[string]int{
|
||||||
reasonNoSize: 0,
|
reasonNoSize: 0,
|
||||||
reasonTooLarge: 0,
|
reasonTooLarge: 0,
|
||||||
|
reasonNoDescription: 0,
|
||||||
|
reasonNoPhoto: 0,
|
||||||
|
}
|
||||||
|
|
||||||
|
addToSkip := func(condition bool, name string, log zerolog.Logger) bool {
|
||||||
|
if !condition {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
log.Debug().Str("reason", name).Msg("skipping item")
|
||||||
|
skippedByReasons[name]++
|
||||||
|
|
||||||
|
return condition
|
||||||
}
|
}
|
||||||
iter := getItemsIter(ctx, r.GoodsItem())
|
iter := getItemsIter(ctx, r.GoodsItem())
|
||||||
for iter.Next() {
|
for iter.Next() {
|
||||||
|
const maximumAllowedSizes = 160
|
||||||
|
|
||||||
item := iter.Get()
|
item := iter.Get()
|
||||||
sublog := log.With().Int64("cart", item.Cart).Logger()
|
sublog := log.With().Int64("cart", item.Cart).Logger()
|
||||||
if item.Sizes == (entity.GoodsItemSize{}) {
|
|
||||||
sublog.Warn().Str("reason", reasonNoSize).Msg("skipping item")
|
|
||||||
skippedByReasons[reasonNoSize]++
|
|
||||||
|
|
||||||
|
switch {
|
||||||
|
case addToSkip(item.Description == "", reasonNoDescription, sublog):
|
||||||
|
continue
|
||||||
|
case addToSkip(item.Sizes == (entity.GoodsItemSize{}), reasonNoSize, sublog):
|
||||||
|
continue
|
||||||
|
case addToSkip(item.Sizes.GetSum(entity.DimensionKindCentimeter) > maximumAllowedSizes, reasonTooLarge, sublog):
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
|
||||||
const maximumAllowedSizes = 160
|
offer := h.goodsItemAsOffer(item, categoryByNameIdx, dimensionDispatcher, sublog)
|
||||||
if sum := item.Sizes.GetSum(entity.DimensionKindCentimeter); sum > maximumAllowedSizes {
|
|
||||||
sublog.Warn().Str("reason", reasonTooLarge).Msg("skipping item")
|
|
||||||
skippedByReasons[reasonTooLarge]++
|
|
||||||
|
|
||||||
|
if addToSkip(len(offer.PictureURLs) == 0, reasonNoPhoto, sublog) {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
|
||||||
offer := h.goodsItemAsOffer(iter.Get(), categoryByNameIdx, matcher, sublog)
|
|
||||||
shop.Offers = append(shop.Offers, offer)
|
shop.Offers = append(shop.Offers, offer)
|
||||||
}
|
}
|
||||||
if err = iter.Err(); err != nil {
|
if err = iter.Err(); err != nil {
|
||||||
return fmt.Errorf("iterating over items: %w", err)
|
return fmt.Errorf("iterating over items: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
f, err := os.Create(path)
|
var f io.WriteCloser
|
||||||
|
switch path {
|
||||||
|
case "stdout":
|
||||||
|
f = os.Stdout
|
||||||
|
case "stderr":
|
||||||
|
f = os.Stderr
|
||||||
|
default:
|
||||||
|
f, err = os.Create(path)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("creating file: %w", err)
|
return fmt.Errorf("creating file: %w", err)
|
||||||
}
|
}
|
||||||
@ -131,6 +182,7 @@ func (h exportHandlers) YMLCatalog(ctx context.Context, cmd *cli.Command) error
|
|||||||
|
|
||||||
log.Err(errClose).Msg("file closed or not")
|
log.Err(errClose).Msg("file closed or not")
|
||||||
}()
|
}()
|
||||||
|
}
|
||||||
|
|
||||||
if limit > 0 {
|
if limit > 0 {
|
||||||
shop.Offers = shop.Offers[:limit]
|
shop.Offers = shop.Offers[:limit]
|
||||||
@ -173,9 +225,19 @@ func (h exportHandlers) goodsItemAsOffer(in entity.GoodsItem, categoryIDByName m
|
|||||||
|
|
||||||
pictureURLs := make([]string, 0, len(in.PhotoURLs))
|
pictureURLs := make([]string, 0, len(in.PhotoURLs))
|
||||||
for _, url := range in.PhotoURLs {
|
for _, url := range in.PhotoURLs {
|
||||||
|
if url == "" {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
outurl := imgurl(url)
|
||||||
|
if outurl == basePictureURL {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
pictureURLs = append(pictureURLs, imgurl(url))
|
pictureURLs = append(pictureURLs, imgurl(url))
|
||||||
}
|
}
|
||||||
params := make([]export.Param, len(in.Parameters))
|
|
||||||
|
params := make([]export.Param, 0, len(in.Parameters))
|
||||||
for k, v := range in.Parameters {
|
for k, v := range in.Parameters {
|
||||||
if k == "" || v == "" {
|
if k == "" || v == "" {
|
||||||
continue
|
continue
|
||||||
@ -219,27 +281,29 @@ func (h exportHandlers) goodsItemAsOffer(in entity.GoodsItem, categoryIDByName m
|
|||||||
return out
|
return out
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// check cart id 126584
|
||||||
func (exportHandlers) formatSizeAsDimensions(size entity.GoodsItemSize, log zerolog.Logger) string {
|
func (exportHandlers) formatSizeAsDimensions(size entity.GoodsItemSize, log zerolog.Logger) string {
|
||||||
const delimeter = "/"
|
const delimeter = "/"
|
||||||
makeFloat := func(d entity.Dimension) string {
|
makeFloat := func(d entity.Dimension) string {
|
||||||
value := size.Length.AdjustTo(entity.DimensionKindCentimeter).Value
|
value := d.AdjustTo(entity.DimensionKindCentimeter).Value
|
||||||
value = float64(int(value*1000)) / 1000.0
|
value = float64(int(value*100)) / 100.0
|
||||||
|
|
||||||
return strconv.FormatFloat(value, 'f', 8, 64)
|
return strconv.FormatFloat(value, 'f', 1, 64)
|
||||||
}
|
}
|
||||||
|
|
||||||
knownSizes := make([]entity.Dimension, 0, 3)
|
knownSizes := make([]entity.Dimension, 0, 3)
|
||||||
|
|
||||||
for _, d := range []entity.Dimension{
|
dimensions := []entity.Dimension{
|
||||||
size.Length,
|
size.Length,
|
||||||
size.Width,
|
size.Width,
|
||||||
size.Height,
|
size.Height,
|
||||||
} {
|
}
|
||||||
|
for i, d := range dimensions {
|
||||||
if d.IsZero() {
|
if d.IsZero() {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
|
||||||
knownSizes = append(knownSizes, d)
|
knownSizes = append(knownSizes, dimensions[i])
|
||||||
}
|
}
|
||||||
|
|
||||||
l := makeFloat(size.Length)
|
l := makeFloat(size.Length)
|
||||||
@ -262,7 +326,21 @@ func (exportHandlers) formatSizeAsDimensions(size entity.GoodsItemSize, log zero
|
|||||||
side = "height"
|
side = "height"
|
||||||
h = unknownDefaultSize
|
h = unknownDefaultSize
|
||||||
}
|
}
|
||||||
log.Warn().Str("size", side).Msg("setting to default value")
|
log.Debug().Str("size", side).Msg("setting to default value")
|
||||||
|
case 1:
|
||||||
|
var side string
|
||||||
|
unknownDefaultSize := makeFloat(entity.NewCentimeterDimensionOrEmpty(30))
|
||||||
|
switch {
|
||||||
|
case !size.Length.IsZero():
|
||||||
|
side = "width"
|
||||||
|
w = unknownDefaultSize
|
||||||
|
case !size.Width.IsZero():
|
||||||
|
side = "length"
|
||||||
|
l = unknownDefaultSize
|
||||||
|
case !size.Height.IsZero():
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
log.Debug().Str("size", side).Msg("setting to default value")
|
||||||
default:
|
default:
|
||||||
return ""
|
return ""
|
||||||
}
|
}
|
||||||
|
|||||||
@ -27,7 +27,7 @@ func ItemsCommandTree() *cli.Command {
|
|||||||
newItemsCountCommand(h),
|
newItemsCountCommand(h),
|
||||||
newItemsUniqueParamsCommand(h),
|
newItemsUniqueParamsCommand(h),
|
||||||
newItemsAggregateParametersCommand(h),
|
newItemsAggregateParametersCommand(h),
|
||||||
newItemsFillSizesCommand(h),
|
newItemsFixSizesCommand(h),
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@ -100,13 +100,13 @@ func newItemsAggregateParametersCommand(h itemsHandlers) *cli.Command {
|
|||||||
return cmdWithAction(cmd, h.AggregateParameters)
|
return cmdWithAction(cmd, h.AggregateParameters)
|
||||||
}
|
}
|
||||||
|
|
||||||
func newItemsFillSizesCommand(h itemsHandlers) *cli.Command {
|
func newItemsFixSizesCommand(h itemsHandlers) *cli.Command {
|
||||||
cmd := cli.Command{
|
cmd := cli.Command{
|
||||||
Name: "fix-sizes",
|
Name: "fix-sizes",
|
||||||
Usage: "Iterates over params and sets sizes from parameters",
|
Usage: "Iterates over params and sets sizes from parameters",
|
||||||
}
|
}
|
||||||
|
|
||||||
return cmdWithAction(cmd, h.FillSizes)
|
return cmdWithAction(cmd, h.FixSizes)
|
||||||
}
|
}
|
||||||
|
|
||||||
type itemsHandlers struct{}
|
type itemsHandlers struct{}
|
||||||
@ -186,13 +186,10 @@ func (itemsHandlers) Count(ctx context.Context, cmd *cli.Command) error {
|
|||||||
}
|
}
|
||||||
|
|
||||||
if len(seenPatterns) == len(patternMapped) {
|
if len(seenPatterns) == len(patternMapped) {
|
||||||
println("Item matched", item.Articul, item.Cart)
|
|
||||||
count++
|
count++
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
println(count)
|
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -304,13 +301,19 @@ func (itemsHandlers) AggregateParameters(ctx context.Context, cmd *cli.Command)
|
|||||||
return bw.Flush()
|
return bw.Flush()
|
||||||
}
|
}
|
||||||
|
|
||||||
func (itemsHandlers) FillSizes(ctx context.Context, cmd *cli.Command) error {
|
func (itemsHandlers) FixSizes(ctx context.Context, cmd *cli.Command) error {
|
||||||
repository, err := components.GetRepository()
|
repository, err := components.GetRepository()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("getting repository: %w", err)
|
return fmt.Errorf("getting repository: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
dimensionDispatcher := makeDefaultDimensionDispatcher()
|
matcher, err := components.GetDimensionMatcher()
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("getting dimension matcher: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
dimensionDispatcher := dimensionDispatcher{m: matcher}
|
||||||
|
|
||||||
log := zerolog.Ctx(ctx)
|
log := zerolog.Ctx(ctx)
|
||||||
|
|
||||||
toUpdate := make([]entity.GoodsItem, 0, 20_000)
|
toUpdate := make([]entity.GoodsItem, 0, 20_000)
|
||||||
@ -354,6 +357,15 @@ func (itemsHandlers) FillSizes(ctx context.Context, cmd *cli.Command) error {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
var updated bool
|
||||||
|
oldSizes := item.Sizes
|
||||||
|
item.Sizes, updated = entity.FixupSizes(oldSizes)
|
||||||
|
if updated {
|
||||||
|
log.Info().Int64("cart", item.Cart).Any("old", oldSizes).Any("new", item.Sizes).Msg("sizes been fixed")
|
||||||
|
}
|
||||||
|
|
||||||
|
valueBeenUpdated = valueBeenUpdated || updated
|
||||||
|
|
||||||
if valueBeenUpdated {
|
if valueBeenUpdated {
|
||||||
toUpdate = append(toUpdate, item)
|
toUpdate = append(toUpdate, item)
|
||||||
}
|
}
|
||||||
|
|||||||
@ -9,6 +9,7 @@ import (
|
|||||||
"time"
|
"time"
|
||||||
|
|
||||||
"git.loyso.art/frx/eway/internal/config"
|
"git.loyso.art/frx/eway/internal/config"
|
||||||
|
"git.loyso.art/frx/eway/internal/dimension"
|
||||||
"git.loyso.art/frx/eway/internal/interconnect/eway"
|
"git.loyso.art/frx/eway/internal/interconnect/eway"
|
||||||
"git.loyso.art/frx/eway/internal/storage"
|
"git.loyso.art/frx/eway/internal/storage"
|
||||||
xbadger "git.loyso.art/frx/eway/internal/storage/badger"
|
xbadger "git.loyso.art/frx/eway/internal/storage/badger"
|
||||||
@ -39,6 +40,10 @@ func GetLogger() (zerolog.Logger, error) {
|
|||||||
return do.Invoke[zerolog.Logger](diInjector)
|
return do.Invoke[zerolog.Logger](diInjector)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func GetDimensionMatcher() (*dimension.Matcher, error) {
|
||||||
|
return do.Invoke[*dimension.Matcher](diInjector)
|
||||||
|
}
|
||||||
|
|
||||||
func SetupDI(ctx context.Context, cfgpath string, verbose bool, logAsJSON bool) error {
|
func SetupDI(ctx context.Context, cfgpath string, verbose bool, logAsJSON bool) error {
|
||||||
cfg, err := parseSettings(cfgpath)
|
cfg, err := parseSettings(cfgpath)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@ -115,6 +120,12 @@ func SetupDI(ctx context.Context, cfgpath string, verbose bool, logAsJSON bool)
|
|||||||
return out, nil
|
return out, nil
|
||||||
})
|
})
|
||||||
|
|
||||||
|
do.Provide[*dimension.Matcher](diInjector, func(i *do.Injector) (*dimension.Matcher, error) {
|
||||||
|
matcher := dimension.New(cfg.DimensionMatcher)
|
||||||
|
|
||||||
|
return matcher, nil
|
||||||
|
})
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -139,6 +150,7 @@ type settings struct {
|
|||||||
Badger config.Badger `toml:"badger"`
|
Badger config.Badger `toml:"badger"`
|
||||||
Log config.Log `toml:"log"`
|
Log config.Log `toml:"log"`
|
||||||
Eway config.Eway `toml:"eway"`
|
Eway config.Eway `toml:"eway"`
|
||||||
|
DimensionMatcher config.DimensionMatcher `toml:"dimension_matcher"`
|
||||||
}
|
}
|
||||||
|
|
||||||
func parseSettings(cfgpath string) (cfg settings, err error) {
|
func parseSettings(cfgpath string) (cfg settings, err error) {
|
||||||
|
|||||||
@ -27,6 +27,10 @@ import (
|
|||||||
"github.com/urfave/cli/v3"
|
"github.com/urfave/cli/v3"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
const (
|
||||||
|
defaultItemType = "Электрика"
|
||||||
|
)
|
||||||
|
|
||||||
type empty entity.Empty
|
type empty entity.Empty
|
||||||
|
|
||||||
func main() {
|
func main() {
|
||||||
@ -594,7 +598,9 @@ func parseEwayDumpAction(ctx context.Context, cmd *cli.Command) error {
|
|||||||
RemmantsAtleast: 5,
|
RemmantsAtleast: 5,
|
||||||
})
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("getting next goods batch: %w", err)
|
logger.Warn().Err(err).Msg("unable to get items from catalog")
|
||||||
|
|
||||||
|
continue
|
||||||
}
|
}
|
||||||
|
|
||||||
productIDs = productIDs[:0]
|
productIDs = productIDs[:0]
|
||||||
@ -604,6 +610,7 @@ func parseEwayDumpAction(ctx context.Context, cmd *cli.Command) error {
|
|||||||
|
|
||||||
remnants, err := client.GetGoodsRemnants(ctx, productIDs)
|
remnants, err := client.GetGoodsRemnants(ctx, productIDs)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
logger.Warn().Err(err).Msg("unable to get goods remnants")
|
||||||
return fmt.Errorf("getting goods remnants: %w", err)
|
return fmt.Errorf("getting goods remnants: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -613,6 +620,7 @@ func parseEwayDumpAction(ctx context.Context, cmd *cli.Command) error {
|
|||||||
seenItem := seenItems[item.SKU]
|
seenItem := seenItems[item.SKU]
|
||||||
if time.Since(seenItem.CreatedAt) < time.Hour*24 {
|
if time.Since(seenItem.CreatedAt) < time.Hour*24 {
|
||||||
logger.Debug().Str("sku", item.SKU).Msg("skipping item because it's too fresh")
|
logger.Debug().Str("sku", item.SKU).Msg("skipping item because it's too fresh")
|
||||||
|
|
||||||
stats.skippedItem++
|
stats.skippedItem++
|
||||||
itemsUpdated[item.SKU] = empty{}
|
itemsUpdated[item.SKU] = empty{}
|
||||||
|
|
||||||
@ -626,14 +634,16 @@ func parseEwayDumpAction(ctx context.Context, cmd *cli.Command) error {
|
|||||||
} else {
|
} else {
|
||||||
pi, err = client.GetProductInfo(ctx, int64(item.Cart))
|
pi, err = client.GetProductInfo(ctx, int64(item.Cart))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("getting product info: %w", err)
|
logger.Warn().Err(err).Msg("unable to get product info, skipping")
|
||||||
|
continue
|
||||||
}
|
}
|
||||||
stats.fetchedInfo++
|
stats.fetchedInfo++
|
||||||
}
|
}
|
||||||
|
|
||||||
goodsItem, err := entity.MakeGoodsItem(item, remnants, pi)
|
goodsItem, err := entity.MakeGoodsItem(item, remnants, pi)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
logger.Warn().Err(err).Any("item", item).Msg("unable to make goods item")
|
logger.Err(err).Any("item", item).Msg("unable to make goods item")
|
||||||
|
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -646,7 +656,8 @@ func parseEwayDumpAction(ctx context.Context, cmd *cli.Command) error {
|
|||||||
|
|
||||||
goodsItems = append(goodsItems, goodsItem)
|
goodsItems = append(goodsItems, goodsItem)
|
||||||
if goodsItem.Type == "" {
|
if goodsItem.Type == "" {
|
||||||
continue
|
logger.Warn().Int64("cart_id", goodsItem.Cart).Msg("found item without category, setting default type")
|
||||||
|
goodsItem.Type = defaultItemType
|
||||||
}
|
}
|
||||||
|
|
||||||
if _, ok := knownCategories[goodsItem.Type]; ok {
|
if _, ok := knownCategories[goodsItem.Type]; ok {
|
||||||
@ -658,7 +669,7 @@ func parseEwayDumpAction(ctx context.Context, cmd *cli.Command) error {
|
|||||||
return fmt.Errorf("creating category: %w", err)
|
return fmt.Errorf("creating category: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
logger.Debug().
|
logger.Info().
|
||||||
Str("name", category.Name).
|
Str("name", category.Name).
|
||||||
Int64("id", category.ID).
|
Int64("id", category.ID).
|
||||||
Msg("created new category")
|
Msg("created new category")
|
||||||
@ -673,8 +684,8 @@ func parseEwayDumpAction(ctx context.Context, cmd *cli.Command) error {
|
|||||||
|
|
||||||
progressFloat := float64(start) / float64(total)
|
progressFloat := float64(start) / float64(total)
|
||||||
progress := big.NewFloat(progressFloat).Text('f', 3)
|
progress := big.NewFloat(progressFloat).Text('f', 3)
|
||||||
|
|
||||||
elapsed := time.Since(startFrom).Seconds()
|
elapsed := time.Since(startFrom).Seconds()
|
||||||
|
|
||||||
var left int
|
var left int
|
||||||
if progressFloat != 0 {
|
if progressFloat != 0 {
|
||||||
left = int(((1 - progressFloat) / progressFloat) * elapsed)
|
left = int(((1 - progressFloat) / progressFloat) * elapsed)
|
||||||
@ -711,11 +722,15 @@ func parseEwayDumpAction(ctx context.Context, cmd *cli.Command) error {
|
|||||||
for k := range seenItems {
|
for k := range seenItems {
|
||||||
_, err := repository.GoodsItem().Delete(ctx, k)
|
_, err := repository.GoodsItem().Delete(ctx, k)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
if errors.Is(err, entity.ErrNotFound) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
logger.Warn().Err(err).Str("sku", k).Msg("unable to delete item")
|
logger.Warn().Err(err).Str("sku", k).Msg("unable to delete item")
|
||||||
|
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
|
||||||
logger.Debug().Str("sku", k).Msg("deleted item")
|
logger.Info().Str("sku", k).Msg("deleted item")
|
||||||
}
|
}
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
|
|||||||
42
internal/config/matcher.go
Normal file
42
internal/config/matcher.go
Normal file
@ -0,0 +1,42 @@
|
|||||||
|
package config
|
||||||
|
|
||||||
|
import (
|
||||||
|
"errors"
|
||||||
|
"strings"
|
||||||
|
)
|
||||||
|
|
||||||
|
type MatcherPredicateType uint8
|
||||||
|
|
||||||
|
const (
|
||||||
|
MatcherPredicateTypeUnknown MatcherPredicateType = iota
|
||||||
|
MatcherPredicateTypePattern
|
||||||
|
MatcherPredicateTypeRegexp
|
||||||
|
)
|
||||||
|
|
||||||
|
func (t *MatcherPredicateType) UnmarshalText(data []byte) error {
|
||||||
|
switch dataStr := strings.ToLower(string(data)); dataStr {
|
||||||
|
case "":
|
||||||
|
*t = MatcherPredicateTypeUnknown
|
||||||
|
case "pattern":
|
||||||
|
*t = MatcherPredicateTypePattern
|
||||||
|
case "regexp":
|
||||||
|
*t = MatcherPredicateTypeRegexp
|
||||||
|
default:
|
||||||
|
return errors.New("unsupported type " + dataStr)
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
type MatcherPredicate struct {
|
||||||
|
Value string `toml:"value"`
|
||||||
|
Type MatcherPredicateType `toml:"type"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type DimensionMatcher struct {
|
||||||
|
CaseInsensitive bool `toml:"case_insensitive"`
|
||||||
|
|
||||||
|
Length []MatcherPredicate `toml:"length"`
|
||||||
|
Width []MatcherPredicate `toml:"width"`
|
||||||
|
Height []MatcherPredicate `toml:"height"`
|
||||||
|
}
|
||||||
92
internal/dimension/matcher.go
Normal file
92
internal/dimension/matcher.go
Normal file
@ -0,0 +1,92 @@
|
|||||||
|
package dimension
|
||||||
|
|
||||||
|
import (
|
||||||
|
"git.loyso.art/frx/eway/internal/config"
|
||||||
|
"git.loyso.art/frx/eway/internal/matcher"
|
||||||
|
)
|
||||||
|
|
||||||
|
type MatchResult uint8
|
||||||
|
|
||||||
|
const (
|
||||||
|
MatchResultMiss MatchResult = iota
|
||||||
|
MatchResultLength
|
||||||
|
MatchResultHeight
|
||||||
|
MatchResultWidth
|
||||||
|
MatchResultDepth
|
||||||
|
)
|
||||||
|
|
||||||
|
type Matcher struct {
|
||||||
|
length matcher.Unit
|
||||||
|
width matcher.Unit
|
||||||
|
height matcher.Unit
|
||||||
|
}
|
||||||
|
|
||||||
|
func New(cfg config.DimensionMatcher) *Matcher {
|
||||||
|
return &Matcher{
|
||||||
|
length: makeMatcherByConig(cfg.CaseInsensitive, cfg.Length...),
|
||||||
|
width: makeMatcherByConig(cfg.CaseInsensitive, cfg.Width...),
|
||||||
|
height: makeMatcherByConig(cfg.CaseInsensitive, cfg.Height...),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func makeMatcherByConig(insensitive bool, cfgs ...config.MatcherPredicate) matcher.Unit {
|
||||||
|
opts := make([]matcher.RadixOpt, 0, 1)
|
||||||
|
if insensitive {
|
||||||
|
opts = append(opts, matcher.RadixCaseInsensitive())
|
||||||
|
}
|
||||||
|
m := matcher.NewRadix(opts...)
|
||||||
|
|
||||||
|
for _, cfg := range cfgs {
|
||||||
|
switch cfg.Type {
|
||||||
|
case config.MatcherPredicateTypePattern:
|
||||||
|
m.Register(cfg.Value)
|
||||||
|
case config.MatcherPredicateTypeRegexp:
|
||||||
|
m.RegisterRegexp(cfg.Value)
|
||||||
|
default:
|
||||||
|
panic("unsupported matcher type")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return m
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m *Matcher) Match(value string) MatchResult {
|
||||||
|
switch {
|
||||||
|
case value == "Высота":
|
||||||
|
fallthrough
|
||||||
|
case m.height.Match(value):
|
||||||
|
return MatchResultHeight
|
||||||
|
case value == "Глубина":
|
||||||
|
return MatchResultDepth
|
||||||
|
case value == "Длина":
|
||||||
|
fallthrough
|
||||||
|
case m.length.Match(value):
|
||||||
|
return MatchResultLength
|
||||||
|
case value == "Ширина":
|
||||||
|
fallthrough
|
||||||
|
case m.width.Match(value):
|
||||||
|
return MatchResultWidth
|
||||||
|
}
|
||||||
|
|
||||||
|
return MatchResultMiss
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m *Matcher) GetRegisteredPatterns() map[string][]string {
|
||||||
|
out := map[string][]string{
|
||||||
|
"length": nil,
|
||||||
|
"width": nil,
|
||||||
|
"height": nil,
|
||||||
|
}
|
||||||
|
|
||||||
|
if m.height != nil {
|
||||||
|
out["height"] = m.height.Patterns()
|
||||||
|
}
|
||||||
|
if m.width != nil {
|
||||||
|
out["width"] = m.width.Patterns()
|
||||||
|
}
|
||||||
|
if m.length != nil {
|
||||||
|
out["length"] = m.length.Patterns()
|
||||||
|
}
|
||||||
|
|
||||||
|
return out
|
||||||
|
}
|
||||||
@ -115,7 +115,10 @@ func makeDomainGoodItem(builder *flatbuffers.Builder, in entity.GoodsItem) flatb
|
|||||||
func ParseGoodsItem(data []byte) (item entity.GoodsItem, err error) {
|
func ParseGoodsItem(data []byte) (item entity.GoodsItem, err error) {
|
||||||
itemFBS := GetRootAsGoodItem(data, 0)
|
itemFBS := GetRootAsGoodItem(data, 0)
|
||||||
item.Articul = string(itemFBS.Sku())
|
item.Articul = string(itemFBS.Sku())
|
||||||
item.PhotoURLs = strings.Split(string(itemFBS.Photo()), ";")
|
photoURLs := string(itemFBS.Photo())
|
||||||
|
if len(photoURLs) > 0 {
|
||||||
|
item.PhotoURLs = strings.Split(photoURLs, ";")
|
||||||
|
}
|
||||||
item.Name = string(itemFBS.Name())
|
item.Name = string(itemFBS.Name())
|
||||||
|
|
||||||
description, err := base64.RawStdEncoding.DecodeString(string(itemFBS.Description()))
|
description, err := base64.RawStdEncoding.DecodeString(string(itemFBS.Description()))
|
||||||
|
|||||||
@ -12,6 +12,44 @@ type GoodsItemSize struct {
|
|||||||
Width Dimension
|
Width Dimension
|
||||||
Height Dimension
|
Height Dimension
|
||||||
Length Dimension
|
Length Dimension
|
||||||
|
|
||||||
|
UnmatchedDepth Dimension
|
||||||
|
}
|
||||||
|
|
||||||
|
func FixupSizes(s GoodsItemSize) (GoodsItemSize, bool) {
|
||||||
|
// Nothing to substitute
|
||||||
|
if s.UnmatchedDepth.IsZero() {
|
||||||
|
return s, false
|
||||||
|
}
|
||||||
|
|
||||||
|
var count int
|
||||||
|
for _, d := range []Dimension{
|
||||||
|
s.Width,
|
||||||
|
s.Height,
|
||||||
|
s.Length,
|
||||||
|
} {
|
||||||
|
if d.IsZero() {
|
||||||
|
count++
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Can only replace one dimension
|
||||||
|
if count != 1 {
|
||||||
|
return s, false
|
||||||
|
}
|
||||||
|
|
||||||
|
switch {
|
||||||
|
case s.Width.IsZero():
|
||||||
|
s.Width = s.UnmatchedDepth
|
||||||
|
case s.Height.IsZero():
|
||||||
|
s.Height = s.UnmatchedDepth
|
||||||
|
case s.Length.IsZero():
|
||||||
|
s.Length = s.UnmatchedDepth
|
||||||
|
}
|
||||||
|
|
||||||
|
s.UnmatchedDepth = Dimension{}
|
||||||
|
|
||||||
|
return s, true
|
||||||
}
|
}
|
||||||
|
|
||||||
func (s GoodsItemSize) GetSum(kind DimensionKind) float64 {
|
func (s GoodsItemSize) GetSum(kind DimensionKind) float64 {
|
||||||
|
|||||||
@ -24,7 +24,7 @@ type Offer struct {
|
|||||||
Description string `xml:"description"`
|
Description string `xml:"description"`
|
||||||
ManufacturerWarrany bool `xml:"manufacturer_warranty"`
|
ManufacturerWarrany bool `xml:"manufacturer_warranty"`
|
||||||
Dimensions string `xml:"dimensions"`
|
Dimensions string `xml:"dimensions"`
|
||||||
Params []Param `xml:"param"`
|
Params []Param `xml:"param,omitempty"`
|
||||||
}
|
}
|
||||||
|
|
||||||
type Currency struct {
|
type Currency struct {
|
||||||
|
|||||||
@ -355,8 +355,6 @@ func (c *client) getProductInfo(ctx context.Context, cartID int64) (pi entity.Go
|
|||||||
return strings.Contains(err.Error(), "pipe")
|
return strings.Contains(err.Error(), "pipe")
|
||||||
})
|
})
|
||||||
|
|
||||||
c.log.Debug().Msg("using go query")
|
|
||||||
|
|
||||||
pi.Parameters = map[string]string{}
|
pi.Parameters = map[string]string{}
|
||||||
resp, err := c.do(ctx, "getProductInfo", req, resty.MethodGet, reqpath)
|
resp, err := c.do(ctx, "getProductInfo", req, resty.MethodGet, reqpath)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
|||||||
@ -71,7 +71,7 @@ func (r *radixMatcher) MatchByPattern(value string) (pattern string) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
if !node.exact {
|
if !node.exact {
|
||||||
return r.findByRegexp(value)
|
return r.findByRegexp(originValue)
|
||||||
}
|
}
|
||||||
|
|
||||||
return sb.String()
|
return sb.String()
|
||||||
|
|||||||
@ -11,6 +11,7 @@ func TestRadixMatcherWithPattern(t *testing.T) {
|
|||||||
m.Register("aloha")
|
m.Register("aloha")
|
||||||
m.Register("hawaii")
|
m.Register("hawaii")
|
||||||
m.Register("te*")
|
m.Register("te*")
|
||||||
|
m.Register("Ширина")
|
||||||
|
|
||||||
var tt = []struct {
|
var tt = []struct {
|
||||||
name string
|
name string
|
||||||
@ -37,6 +38,9 @@ func TestRadixMatcherWithPattern(t *testing.T) {
|
|||||||
}, {
|
}, {
|
||||||
name: "should not match 3",
|
name: "should not match 3",
|
||||||
in: "alohaya",
|
in: "alohaya",
|
||||||
|
}, {
|
||||||
|
name: "should match exact 3",
|
||||||
|
in: "Ширина",
|
||||||
}}
|
}}
|
||||||
|
|
||||||
for _, tc := range tt {
|
for _, tc := range tt {
|
||||||
|
|||||||
Reference in New Issue
Block a user