From be6aec73df7131180208465584cca7c5d065fb24 Mon Sep 17 00:00:00 2001 From: Aleksandr Trushkin Date: Sat, 3 Feb 2024 20:28:33 +0300 Subject: [PATCH] support parsing item page info --- assets/gooditem.fbs | 1 + cmd/cli/main.go | 105 ++++++++++++++---- go.mod | 20 +++- go.sum | 45 ++++++-- internal/encoding/fbs/GoodItem.go | 13 ++- .../fbs/{GoodsItem.go => GoodItems.go} | 0 internal/encoding/fbs/helpers.go | 16 ++- internal/entity/gooditem.go | 41 ++++--- internal/interconnect/eway/client.go | 65 ++++++++++- internal/storage/badger/queue.go | 17 +-- 10 files changed, 262 insertions(+), 61 deletions(-) rename internal/encoding/fbs/{GoodsItem.go => GoodItems.go} (100%) diff --git a/assets/gooditem.fbs b/assets/gooditem.fbs index ab29fec..d33dc4b 100644 --- a/assets/gooditem.fbs +++ b/assets/gooditem.fbs @@ -14,6 +14,7 @@ table GoodItem { tariff:float; cart:long; stock:short; + parameters:string; } table GoodItems { diff --git a/cmd/cli/main.go b/cmd/cli/main.go index c1144b5..71372c9 100644 --- a/cmd/cli/main.go +++ b/cmd/cli/main.go @@ -197,6 +197,7 @@ func newParseEwayCmd() *cli.Command { Usage: "parse all available eway goods", Commands: []*cli.Command{ newParseEwayGetCmd(), + newParseEwayListCmd(), newParseEwayDumpCmd(), }, } @@ -205,6 +206,21 @@ func newParseEwayCmd() *cli.Command { func newParseEwayGetCmd() *cli.Command { return &cli.Command{ Name: "get", + Usage: "loads information about the product by parsing product's html page", + Flags: []cli.Flag{ + &cli.IntFlag{ + Name: "cart", + Aliases: []string{"c"}, + Usage: "cart of the product", + }, + }, + Action: decorateAction(parseEwayGetAction), + } +} + +func newParseEwayListCmd() *cli.Command { + return &cli.Command{ + Name: "list", Usage: "parse all available eway goods", Flags: []cli.Flag{ &cli.IntFlag{ @@ -222,7 +238,7 @@ func newParseEwayGetCmd() *cli.Command { Usage: "filters by minimum available items in stock", }, }, - Action: decorateAction(parseEwayGetAction), + Action: decorateAction(parseEwayListAction), } } @@ -653,7 +669,7 @@ func decorateAction(a action) cli.ActionFunc { start := time.Now() defer func() { - log.Info().Dur("elapsed", time.Since(start)).Msg("command completed") + log.Info().Float64("elapsed", time.Since(start).Seconds()).Msg("command completed") }() log.Info().Msg("command execution started") @@ -749,6 +765,29 @@ func exportYMLCatalogAction(ctx context.Context, c *cli.Command) error { } func parseEwayGetAction(ctx context.Context, cmd *cli.Command) error { + cartID := cmd.Int("cart") + + client, err := components.GetEwayClient() + if err != nil { + return fmt.Errorf("getting eway client: %w", err) + } + + pi, err := client.GetProductInfo(ctx, cartID) + if err != nil { + return fmt.Errorf("getting product info: %w", err) + } + + enc := json.NewEncoder(cmd.Writer) + enc.SetIndent("", " ") + err = enc.Encode(pi) + if err != nil { + return fmt.Errorf("encoding data: %w", err) + } + + return nil +} + +func parseEwayListAction(ctx context.Context, cmd *cli.Command) error { page := cmd.Int("page") limit := cmd.Int("limit") atLeast := cmd.Int("min-stock") @@ -759,6 +798,11 @@ func parseEwayGetAction(ctx context.Context, cmd *cli.Command) error { return fmt.Errorf("getting eway client: %w", err) } + log, err := components.GetLogger() + if err != nil { + return fmt.Errorf("getting logger: %w", err) + } + start := page * limit items, total, err := client.GetGoodsNew(ctx, eway.GetGoodsNewParams{ @@ -782,19 +826,26 @@ func parseEwayGetAction(ctx context.Context, cmd *cli.Command) error { return fmt.Errorf("getting remnants: %w", err) } - tbl := table.New("sku", "category", "cart", "stock", "price") + tbl := table.New("sku", "category", "cart", "stock", "price", "parameters") for _, item := range items { - outGood, err := entity.MakeGoodsItem(item, remnants) + pi, err := client.GetProductInfo(ctx, int64(item.Cart)) + if err != nil { + log.Warn().Err(err).Msg("unable to get product info") + } + outGood, err := entity.MakeGoodsItem(item, remnants, pi) if err != nil { return fmt.Errorf("making goods item: %w", err) } + parameters, _ := json.MarshalIndent(outGood.Parameters, "", " ") + tbl.AddRow( outGood.Articul, outGood.Type, outGood.Cart, outGood.Stock, outGood.Price, + string(parameters), ) } @@ -867,7 +918,12 @@ func parseEwayDumpAction(ctx context.Context, cmd *cli.Command) error { goodsItems = goodsItems[:0] for _, item := range items { - goodsItem, err := entity.MakeGoodsItem(item, remnants) + pi, err := client.GetProductInfo(ctx, int64(item.Cart)) + if err != nil { + return fmt.Errorf("getting product info: %w", err) + } + + goodsItem, err := entity.MakeGoodsItem(item, remnants, pi) if err != nil { logger.Warn().Err(err).Any("item", item).Msg("unable to make goods item") continue @@ -942,26 +998,34 @@ func goodsItemAsOffer(in entity.GoodsItem, categoryIDByName map[string]int64) (o categoryID := categoryIDByName[in.Type] + pictureURLs := make([]string, 0, len(in.PhotoURLs)) + for _, url := range in.PhotoURLs { + pictureURLs = append(pictureURLs, imgurl(url)) + } + params := make([]export.Param, len(in.Parameters)) + for k, v := range in.Parameters { + params = append(params, export.Param{ + Name: k, + Value: v, + }) + } + params = append(params, export.Param{ + Name: quantityParamName, + Value: strconv.Itoa(in.Stock), + }) out = export.Offer{ - ID: in.Cart, - VendorCode: in.Articul, - Price: int(in.TariffPrice), - CategoryID: categoryID, - PictureURLs: []string{ - imgurl(in.Photo), - }, + ID: in.Cart, + VendorCode: in.Articul, + Price: int(in.TariffPrice), + CategoryID: categoryID, + PictureURLs: pictureURLs, Model: in.Name, Vendor: in.Producer, TypePrefix: in.Name, Description: in.Description, ManufacturerWarrany: true, - Params: []export.Param{ - { - Name: quantityParamName, - Value: strconv.Itoa(in.Stock), - }, - }, + Params: params, Type: defaultType, CurrencyID: defaultCurrency, @@ -987,7 +1051,7 @@ func testFBSAction(ctx context.Context, c *cli.Command) error { return fmt.Errorf("parsing: %w", err) } - if got != gooditem { + if got.Articul != gooditem.Articul { gotStr := fmt.Sprintf("%v", got) hasStr := fmt.Sprintf("%v", gooditem) println(gotStr, "\n", hasStr) @@ -1093,6 +1157,9 @@ func cryptoDeEncryptAction(encrypt bool) cli.ActionFunc { } _, err = c.Writer.Write([]byte(out)) + if err != nil { + return err + } _, err = c.Writer.Write([]byte{'\n'}) return err } diff --git a/go.mod b/go.mod index d3691f4..850e854 100644 --- a/go.mod +++ b/go.mod @@ -12,28 +12,36 @@ require ( github.com/rodaine/table v1.1.1 github.com/rs/zerolog v1.31.0 github.com/samber/do v1.6.0 - github.com/urfave/cli v1.22.14 + github.com/urfave/cli/v3 v3.0.0-alpha8 ) require ( + github.com/PuerkitoBio/goquery v1.8.1 // indirect + github.com/andybalholm/cascadia v1.3.1 // indirect + github.com/antchfx/htmlquery v1.3.0 // indirect + github.com/antchfx/xmlquery v1.3.18 // indirect + github.com/antchfx/xpath v1.2.4 // indirect github.com/cespare/xxhash/v2 v2.2.0 // indirect - github.com/cpuguy83/go-md2man/v2 v2.0.3 // indirect github.com/dustin/go-humanize v1.0.0 // indirect + github.com/gobwas/glob v0.2.3 // indirect + github.com/gocolly/colly v1.2.0 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/golang/glog v1.0.0 // indirect - github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6 // indirect + github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/golang/protobuf v1.5.2 // indirect github.com/golang/snappy v0.0.3 // indirect + github.com/kennygrant/sanitize v1.2.4 // indirect github.com/klauspost/compress v1.12.3 // indirect github.com/mattn/go-colorable v0.1.13 // indirect github.com/mattn/go-isatty v0.0.19 // indirect github.com/pkg/errors v0.9.1 // indirect - github.com/russross/blackfriday/v2 v2.1.0 // indirect - github.com/urfave/cli/v2 v2.27.1 // indirect - github.com/urfave/cli/v3 v3.0.0-alpha8 // indirect + github.com/saintfish/chardet v0.0.0-20230101081208-5e3ef4b5456d // indirect + github.com/temoto/robotstxt v1.1.2 // indirect github.com/xrash/smetrics v0.0.0-20231213231151-1d8dd44e695e // indirect go.opencensus.io v0.22.5 // indirect golang.org/x/net v0.17.0 // indirect golang.org/x/sys v0.13.0 // indirect + golang.org/x/text v0.13.0 // indirect + google.golang.org/appengine v1.4.0 // indirect google.golang.org/protobuf v1.28.1 // indirect ) diff --git a/go.sum b/go.sum index 2f4ab96..05d9235 100644 --- a/go.sum +++ b/go.sum @@ -2,6 +2,17 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMT github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/toml v1.3.2 h1:o7IhLm0Msx3BaB+n3Ag7L8EVlByGnpq14C4YWiu/gL8= github.com/BurntSushi/toml v1.3.2/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= +github.com/PuerkitoBio/goquery v1.8.1 h1:uQxhNlArOIdbrH1tr0UXwdVFgDcZDrZVdcpygAcwmWM= +github.com/PuerkitoBio/goquery v1.8.1/go.mod h1:Q8ICL1kNUJ2sXGoAhPGUdYDJvgQgHzJsnnd3H7Ho5jQ= +github.com/andybalholm/cascadia v1.3.1 h1:nhxRkql1kdYCc8Snf7D5/D3spOX+dBgjA6u8x004T2c= +github.com/andybalholm/cascadia v1.3.1/go.mod h1:R4bJ1UQfqADjvDa4P6HZHLh/3OxWWEqc0Sk8XGwHqvA= +github.com/antchfx/htmlquery v1.3.0 h1:5I5yNFOVI+egyia5F2s/5Do2nFWxJz41Tr3DyfKD25E= +github.com/antchfx/htmlquery v1.3.0/go.mod h1:zKPDVTMhfOmcwxheXUsx4rKJy8KEY/PU6eXr/2SebQ8= +github.com/antchfx/xmlquery v1.3.18 h1:FSQ3wMuphnPPGJOFhvc+cRQ2CT/rUj4cyQXkJcjOwz0= +github.com/antchfx/xmlquery v1.3.18/go.mod h1:Afkq4JIeXut75taLSuI31ISJ/zeq+3jG7TunF7noreA= +github.com/antchfx/xpath v1.2.3/go.mod h1:i54GszH55fYfBmoZXapTHN8T8tkcHfRgLyVwwqzXNcs= +github.com/antchfx/xpath v1.2.4 h1:dW1HB/JxKvGtJ9WyVGJ0sIoEcqftV3SqIstujI+B9XY= +github.com/antchfx/xpath v1.2.4/go.mod h1:i54GszH55fYfBmoZXapTHN8T8tkcHfRgLyVwwqzXNcs= github.com/brianvoe/gofakeit/v6 v6.28.0 h1:Xib46XXuQfmlLS2EXRuJpqcw8St6qSZz75OUo0tgAW4= github.com/brianvoe/gofakeit/v6 v6.28.0/go.mod h1:Xj58BMSnFqcn/fAQeSK+/PLtC5kSb7FJIq4JyGa8vEs= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= @@ -9,10 +20,6 @@ github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= -github.com/cpuguy83/go-md2man/v2 v2.0.2 h1:p1EgwI/C7NhT0JmVkwCD2ZBK8j4aeHQX2pMHHBfMQ6w= -github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= -github.com/cpuguy83/go-md2man/v2 v2.0.3 h1:qMCsGGgs+MAzDFyp9LpAe1Lqy/fY/qCovCm0qnXZOBM= -github.com/cpuguy83/go-md2man/v2 v2.0.3/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -26,6 +33,10 @@ github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4 github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/go-resty/resty/v2 v2.10.0 h1:Qla4W/+TMmv0fOeeRqzEpXPLfTUnR5HZ1+lGs+CkiCo= github.com/go-resty/resty/v2 v2.10.0/go.mod h1:iiP/OpA0CkcL3IGt1O0+/SIItFUbkkyw5BGXiVdTu+A= +github.com/gobwas/glob v0.2.3 h1:A4xDbljILXROh+kObIiy5kIaPYD8e96x1tgBhUI5J+Y= +github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJAkT8= +github.com/gocolly/colly v1.2.0 h1:qRz9YAn8FIH0qzgNUw+HT9UN7wm1oF9OBAilwEWpyrI= +github.com/gocolly/colly v1.2.0/go.mod h1:Hof5T3ZswNVsOHYmba1u03W65HDWgpV5HifSuueE0EA= github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= @@ -34,6 +45,8 @@ github.com/golang/glog v1.0.0 h1:nfP3RFugxnNRyKgeWd4oI1nYvXpxrx8ck8ZrcizshdQ= github.com/golang/glog v1.0.0/go.mod h1:EWib/APOK0SL3dFbYqvxE3UYd8E6s1ouQ7iEp/0LWV4= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6 h1:ZgQEtGgCBiWRM39fZuwSd1LwSqqSW0hOdXCYYDX0R3I= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= +github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= @@ -48,6 +61,8 @@ github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMyw github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/kennygrant/sanitize v1.2.4 h1:gN25/otpP5vAsO2djbMhF/LQX6R7+O1TB4yv8NzpJ3o= +github.com/kennygrant/sanitize v1.2.4/go.mod h1:LGsjYYtgxbetdg5owWB2mpgUL6e2nfw2eObZ0u0qvak= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/compress v1.12.3 h1:G5AfA94pHPysR56qqrkO2pxEexdDzrpFJ6yt/VqWxVU= @@ -70,22 +85,21 @@ github.com/rodaine/table v1.1.1/go.mod h1:iqTRptjn+EVcrVBYtNMlJ2wrJZa3MpULUmcXFp github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= github.com/rs/zerolog v1.31.0 h1:FcTR3NnLWW+NnTwwhFWiJSZr4ECLpqCm6QsEnyvbV4A= github.com/rs/zerolog v1.31.0/go.mod h1:/7mN4D5sKwJLZQ2b/znpjC3/GQWY/xaDXUM0kKWRHss= -github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= -github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/saintfish/chardet v0.0.0-20230101081208-5e3ef4b5456d h1:hrujxIzL1woJ7AwssoOcM/tq5JjjG2yYOc8odClEiXA= +github.com/saintfish/chardet v0.0.0-20230101081208-5e3ef4b5456d/go.mod h1:uugorj2VCxiV1x+LzaIdVa9b4S4qGAcH6cbhh4qVxOU= github.com/samber/do v1.6.0 h1:Jy/N++BXINDB6lAx5wBlbpHlUdl0FKpLWgGEV9YWqaU= github.com/samber/do v1.6.0/go.mod h1:DWqBvumy8dyb2vEnYZE7D7zaVEB64J45B0NjTlY/M4k= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= -github.com/urfave/cli v1.22.14 h1:ebbhrRiGK2i4naQJr+1Xj92HXZCrK7MsyTS/ob3HnAk= -github.com/urfave/cli v1.22.14/go.mod h1:X0eDS6pD6Exaclxm99NJ3FiCDRED7vIHpx2mDOHLvkA= -github.com/urfave/cli/v2 v2.27.1 h1:8xSQ6szndafKVRmfyeUMxkNUJQMjL1F2zmsZ+qHpfho= -github.com/urfave/cli/v2 v2.27.1/go.mod h1:8qnjx1vcq5s2/wpsqoZFndg2CE5tNFyrTvS6SinrnYQ= +github.com/temoto/robotstxt v1.1.2 h1:W2pOjSJ6SWvldyEuiFXNxz3xZ8aiWX5LbfDiOFd7Fxg= +github.com/temoto/robotstxt v1.1.2/go.mod h1:+1AmkuG3IYkh1kv0d2qEB9Le88ehNO0zwOr3ujewlOo= github.com/urfave/cli/v3 v3.0.0-alpha8 h1:H+qxFPoCkGzdF8KUMs2fEOZl5io/1QySgUiGfar8occ= github.com/urfave/cli/v3 v3.0.0-alpha8/go.mod h1:0kK/RUFHyh+yIKSfWxwheGndfnrvYSmYFVeKCh03ZUc= github.com/xrash/smetrics v0.0.0-20231213231151-1d8dd44e695e h1:+SOyEddqYF09QP7vr7CgJ1eti3pY9Fn3LHO1M1r/0sI= @@ -117,8 +131,11 @@ golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210916014120-12bc252f5db8/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.5.0/go.mod h1:DivGGAXEgPSlEBzxGzZI+ZLohi+xUj054jfeKui00ws= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= +golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM= golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= @@ -137,11 +154,13 @@ golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20221010170243-090e33056c14/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.4.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -150,14 +169,18 @@ golang.org/x/sys v0.13.0 h1:Af8nKPmuFypiUBjVoU9V20FiaFXOcuZI21p0ycVYYGE= golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.4.0/go.mod h1:9P2UbLfCdcvo3p/nzKvsmas4TnlujnuoV9hGgYzW1lQ= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.6.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= +golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k= golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4= golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -175,6 +198,7 @@ golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8T golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= +google.golang.org/appengine v1.4.0 h1:/wp5JvzpHIxhs/dumFmF7BXTf3Z+dd4uXta4kVyO508= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= @@ -186,7 +210,6 @@ google.golang.org/protobuf v1.28.1 h1:d0NfwRgPtno5B1Wa6L2DAG+KivqkdutMf1UhdNx175 google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/internal/encoding/fbs/GoodItem.go b/internal/encoding/fbs/GoodItem.go index e7ee1f9..bc53e72 100644 --- a/internal/encoding/fbs/GoodItem.go +++ b/internal/encoding/fbs/GoodItem.go @@ -169,8 +169,16 @@ func (rcv *GoodItem) MutateStock(n int16) bool { return rcv._tab.MutateInt16Slot(28, n) } +func (rcv *GoodItem) Parameters() []byte { + o := flatbuffers.UOffsetT(rcv._tab.Offset(30)) + if o != 0 { + return rcv._tab.ByteVector(o + rcv._tab.Pos) + } + return nil +} + func GoodItemStart(builder *flatbuffers.Builder) { - builder.StartObject(13) + builder.StartObject(14) } func GoodItemAddSku(builder *flatbuffers.Builder, sku flatbuffers.UOffsetT) { builder.PrependUOffsetTSlot(0, flatbuffers.UOffsetT(sku), 0) @@ -211,6 +219,9 @@ func GoodItemAddCart(builder *flatbuffers.Builder, cart int64) { func GoodItemAddStock(builder *flatbuffers.Builder, stock int16) { builder.PrependInt16Slot(12, stock, 0) } +func GoodItemAddParameters(builder *flatbuffers.Builder, parameters flatbuffers.UOffsetT) { + builder.PrependUOffsetTSlot(13, flatbuffers.UOffsetT(parameters), 0) +} func GoodItemEnd(builder *flatbuffers.Builder) flatbuffers.UOffsetT { return builder.EndObject() } diff --git a/internal/encoding/fbs/GoodsItem.go b/internal/encoding/fbs/GoodItems.go similarity index 100% rename from internal/encoding/fbs/GoodsItem.go rename to internal/encoding/fbs/GoodItems.go diff --git a/internal/encoding/fbs/helpers.go b/internal/encoding/fbs/helpers.go index ed77f6e..4ecbea8 100644 --- a/internal/encoding/fbs/helpers.go +++ b/internal/encoding/fbs/helpers.go @@ -2,7 +2,9 @@ package fbs import ( "encoding/base64" + "encoding/json" "fmt" + "strings" "sync" "git.loyso.art/frx/eway/internal/entity" @@ -62,7 +64,7 @@ func MakeDomainGoodItemFinished(in entity.GoodsItem) []byte { func makeDomainGoodItem(builder *flatbuffers.Builder, in entity.GoodsItem) flatbuffers.UOffsetT { sku := builder.CreateString(in.Articul) - photo := builder.CreateString(in.Photo) + photo := builder.CreateString(strings.Join(in.PhotoURLs, ";")) name := builder.CreateString(in.Name) descBase64 := base64.RawStdEncoding.EncodeToString([]byte(in.Description)) @@ -73,6 +75,10 @@ func makeDomainGoodItem(builder *flatbuffers.Builder, in entity.GoodsItem) flatb cat = builder.CreateString(in.Category) } t := builder.CreateString(in.Type) + + parametersData, _ := json.Marshal(in.Parameters) + parameters := builder.CreateByteString(parametersData) + producer := builder.CreateString(in.Producer) GoodItemStart(builder) @@ -91,6 +97,7 @@ func makeDomainGoodItem(builder *flatbuffers.Builder, in entity.GoodsItem) flatb GoodItemAddTariff(builder, float32(in.TariffPrice)) GoodItemAddCart(builder, int64(in.Cart)) GoodItemAddStock(builder, int16(in.Stock)) + GoodItemAddParameters(builder, parameters) return GoodItemEnd(builder) } @@ -98,7 +105,7 @@ func makeDomainGoodItem(builder *flatbuffers.Builder, in entity.GoodsItem) flatb func ParseGoodsItem(data []byte) (item entity.GoodsItem, err error) { itemFBS := GetRootAsGoodItem(data, 0) item.Articul = string(itemFBS.Sku()) - item.Photo = string(itemFBS.Photo()) + item.PhotoURLs = strings.Split(string(itemFBS.Photo()), ";") item.Name = string(itemFBS.Name()) description, err := base64.RawStdEncoding.DecodeString(string(itemFBS.Description())) @@ -118,6 +125,11 @@ func ParseGoodsItem(data []byte) (item entity.GoodsItem, err error) { item.TariffPrice = float64(itemFBS.Tariff()) item.Cart = itemFBS.Cart() item.Stock = int(itemFBS.Stock()) + item.Parameters = map[string]string{} + err = json.Unmarshal(itemFBS.Parameters(), &item.Parameters) + if err != nil { + return item, fmt.Errorf("unmarshalling data: %w", err) + } return item, nil } diff --git a/internal/entity/gooditem.go b/internal/entity/gooditem.go index 235ae42..ebe7ec4 100644 --- a/internal/entity/gooditem.go +++ b/internal/entity/gooditem.go @@ -8,19 +8,20 @@ import ( ) type GoodsItem struct { - Articul string `json:"sku"` - Photo string `json:"photo"` - Name string `json:"name"` - Description string `json:"description"` - Category string `json:"category"` - Type string `json:"type"` - Producer string `json:"producer"` - Pack int `json:"pack"` - Step int `json:"step"` - Price float64 `json:"price"` - TariffPrice float64 `json:"tariff_price"` - Cart int64 `json:"cart"` - Stock int `json:"stock"` + Articul string `json:"sku"` + PhotoURLs []string `json:"photo"` + Name string `json:"name"` + Description string `json:"description"` + Category string `json:"category"` + Type string `json:"type"` + Producer string `json:"producer"` + Pack int `json:"pack"` + Step int `json:"step"` + Price float64 `json:"price"` + TariffPrice float64 `json:"tariff_price"` + Cart int64 `json:"cart"` + Stock int `json:"stock"` + Parameters map[string]string `json:"parameters"` } type GoodsItemRaw struct { @@ -42,12 +43,18 @@ type GoodsItemRaw struct { Other string } +type GoodsItemInfo struct { + Parameters map[string]string + PhotoURLs []string +} + type MappedGoodsRemnants map[int]GoodsRemnant type GoodsRemnant [4]int32 func MakeGoodsItem( gi GoodsItemRaw, remnants MappedGoodsRemnants, + info GoodsItemInfo, ) (out GoodsItem, err error) { var name, desc string var pack, step int @@ -95,9 +102,13 @@ func MakeGoodsItem( return out, fmt.Errorf("getting step count (%s): %w", gi.Step, err) } + photoURLs := info.PhotoURLs + if len(photoURLs) > 7 { + photoURLs = info.PhotoURLs[:7] + } + return GoodsItem{ Articul: gi.SKU, - Photo: gi.Photo, Name: name, Description: desc, Category: gi.Category, @@ -109,5 +120,7 @@ func MakeGoodsItem( TariffPrice: tariffPrice, Cart: int64(gi.Cart), Stock: int(remnants[int(gi.Cart)][0]), + PhotoURLs: photoURLs, + Parameters: info.Parameters, }, nil } diff --git a/internal/interconnect/eway/client.go b/internal/interconnect/eway/client.go index b4ac69a..ef28957 100644 --- a/internal/interconnect/eway/client.go +++ b/internal/interconnect/eway/client.go @@ -11,12 +11,14 @@ import ( "reflect" "strconv" "strings" + "time" "git.loyso.art/frx/eway/internal/config" "git.loyso.art/frx/eway/internal/crypto" "git.loyso.art/frx/eway/internal/entity" "github.com/go-resty/resty/v2" + "github.com/gocolly/colly" "github.com/rs/zerolog" ) @@ -26,6 +28,7 @@ type Client interface { context.Context, GetGoodsNewParams, ) (items []entity.GoodsItemRaw, total int, err error) + GetProductInfo(context.Context, int64) (entity.GoodsItemInfo, error) } type client struct { @@ -40,7 +43,6 @@ type Config config.Eway func New(ctx context.Context, cfg Config, log zerolog.Logger) (client, error) { httpclient := resty.New(). SetDebug(cfg.Debug). - // SetCookies(cookies). SetBaseURL("https://eway.elevel.ru/api") c := client{ @@ -269,3 +271,64 @@ func (c client) login(ctx context.Context, user, pass string) error { return nil } + +type ProductInfo struct { + ImageLinks []string + Parameters map[string]string +} + +type parameterSelector struct { + Name string `selector:"div"` + Value string `selector:"div.text-right"` +} + +func (c client) GetProductInfo(ctx context.Context, cart int64) (pi entity.GoodsItemInfo, err error) { + collector := colly.NewCollector( + colly.AllowedDomains("eway.elevel.ru"), + colly.AllowURLRevisit(), + ) + + pi.Parameters = map[string]string{} + + start := time.Now() + defer func() { + elapsed := time.Since(start).Seconds() + c.log.Info().Float64("elapsed", elapsed).Msg("request processed") + }() + + collector.OnHTML("body > div.page-container > div.page-content > div.content-wrapper > div.content > div.row > div.col-md-4 > div > div > div:nth-child(6)", func(e *colly.HTMLElement) { + e.ForEach("div.display-flex", func(i int, h *colly.HTMLElement) { + var s parameterSelector + err = h.Unmarshal(&s) + if err != nil { + c.log.Warn().Err(err).Msg("unable to unmarshal") + return + } + + if s.Name == "" || s.Value == "" { + c.log.Warn().Msg("got empty key or value, skipping") + return + } + + pi.Parameters[s.Name] = s.Value + }) + }) + collector.OnHTML("div.gallery_panel", func(h *colly.HTMLElement) { + h.ForEach("div.gallery_thumbnail > img", func(i int, h *colly.HTMLElement) { + imageURL := h.Attr("src") + + if imageURL == "" { + return + } + + pi.PhotoURLs = append(pi.PhotoURLs, imageURL) + }) + }) + + err = collector.Visit("https://eway.elevel.ru/product/" + strconv.Itoa(int(cart)) + "/") + if err != nil { + return pi, fmt.Errorf("visiting site: %w", err) + } + + return pi, nil +} diff --git a/internal/storage/badger/queue.go b/internal/storage/badger/queue.go index 9482ce0..90e30f8 100644 --- a/internal/storage/badger/queue.go +++ b/internal/storage/badger/queue.go @@ -9,6 +9,7 @@ import ( "time" "git.loyso.art/frx/eway/internal/entity" + "github.com/dgraph-io/badger/v4" ) @@ -70,10 +71,10 @@ func (c queueClient) Publish(ctx context.Context, params entity.PublishParams) ( binary.BigEndian.PutUint64(keyData[:], task.ID) opts := make([]putOpt, 0, 1) if !params.ExpiresAt.IsZero() { - duration := params.ExpiresAt.Sub(time.Now()) + duration := time.Until(params.ExpiresAt) opts = append(opts, withTTL(duration)) } - err = c.namedClient.Put(keyData[:], tdb.asBinary(), opts...) + err = c.Put(keyData[:], tdb.asBinary(), opts...) if err != nil { return entity.Task{}, fmt.Errorf("saving data: %w", err) } @@ -83,11 +84,13 @@ func (c queueClient) Publish(ctx context.Context, params entity.PublishParams) ( func (c queueClient) Consume(ctx context.Context) (task entity.Task, err error) { err = c.db.View(func(txn *badger.Txn) error { - iterOpts := badger.DefaultIteratorOptions - iterOpts.PrefetchSize = 1 - iterOpts.PrefetchValues = true - iterOpts.Prefix = c.table - iterOpts.Reverse = false + iterOpts := badger.IteratorOptions{ + PrefetchSize: 1, + PrefetchValues: true, + Reverse: false, + AllVersions: false, + Prefix: c.table, + } iter := txn.NewIterator(iterOpts) defer iter.Close()