diff --git a/assets/gooditem.fbs b/assets/gooditem.fbs index d33dc4b..1bf2310 100644 --- a/assets/gooditem.fbs +++ b/assets/gooditem.fbs @@ -15,6 +15,7 @@ table GoodItem { cart:long; stock:short; parameters:string; + created_at:long; } table GoodItems { diff --git a/cmd/cli/main.go b/cmd/cli/main.go index eb86dcd..4698fa9 100644 --- a/cmd/cli/main.go +++ b/cmd/cli/main.go @@ -899,6 +899,7 @@ func parseEwayDumpAction(ctx context.Context, cmd *cli.Command) error { fetchedInfo int handledAll int cachedInfo int + skippedItem int }{} startFrom := time.Now() @@ -934,6 +935,14 @@ func parseEwayDumpAction(ctx context.Context, cmd *cli.Command) error { for _, item := range items { var pi entity.GoodsItemInfo seenItem := seenItems[item.SKU] + if time.Since(seenItem.CreatedAt) < time.Hour*24 { + logger.Debug().Str("sku", item.SKU).Msg("skipping item because it's too fresh") + stats.skippedItem++ + itemsUpdated[item.SKU] = struct{}{} + + continue + } + if len(seenItem.Parameters) != 0 && len(seenItem.PhotoURLs) != 0 { pi.Parameters = seenItem.Parameters pi.PhotoURLs = seenItem.PhotoURLs @@ -1015,6 +1024,7 @@ func parseEwayDumpAction(ctx context.Context, cmd *cli.Command) error { Int("handled", stats.handledAll). Int("cached", stats.cachedInfo). Int("fetched", stats.fetchedInfo). + Int("skipped", stats.skippedItem). Int("to_delete", len(seenItems)). Msg("processed items") diff --git a/internal/encoding/fbs/GoodItem.go b/internal/encoding/fbs/GoodItem.go index bc53e72..2682cb5 100644 --- a/internal/encoding/fbs/GoodItem.go +++ b/internal/encoding/fbs/GoodItem.go @@ -177,8 +177,20 @@ func (rcv *GoodItem) Parameters() []byte { return nil } +func (rcv *GoodItem) CreatedAt() int64 { + o := flatbuffers.UOffsetT(rcv._tab.Offset(32)) + if o != 0 { + return rcv._tab.GetInt64(o + rcv._tab.Pos) + } + return 0 +} + +func (rcv *GoodItem) MutateCreatedAt(n int64) bool { + return rcv._tab.MutateInt64Slot(32, n) +} + func GoodItemStart(builder *flatbuffers.Builder) { - builder.StartObject(14) + builder.StartObject(15) } func GoodItemAddSku(builder *flatbuffers.Builder, sku flatbuffers.UOffsetT) { builder.PrependUOffsetTSlot(0, flatbuffers.UOffsetT(sku), 0) @@ -222,6 +234,9 @@ func GoodItemAddStock(builder *flatbuffers.Builder, stock int16) { func GoodItemAddParameters(builder *flatbuffers.Builder, parameters flatbuffers.UOffsetT) { builder.PrependUOffsetTSlot(13, flatbuffers.UOffsetT(parameters), 0) } +func GoodItemAddCreatedAt(builder *flatbuffers.Builder, createdAt int64) { + builder.PrependInt64Slot(14, createdAt, 0) +} func GoodItemEnd(builder *flatbuffers.Builder) flatbuffers.UOffsetT { return builder.EndObject() } diff --git a/internal/encoding/fbs/helpers.go b/internal/encoding/fbs/helpers.go index c4d40a7..0cdd110 100644 --- a/internal/encoding/fbs/helpers.go +++ b/internal/encoding/fbs/helpers.go @@ -6,6 +6,7 @@ import ( "fmt" "strings" "sync" + "time" "git.loyso.art/frx/eway/internal/entity" @@ -98,6 +99,7 @@ func makeDomainGoodItem(builder *flatbuffers.Builder, in entity.GoodsItem) flatb GoodItemAddCart(builder, int64(in.Cart)) GoodItemAddStock(builder, int16(in.Stock)) GoodItemAddParameters(builder, parameters) + GoodItemAddCreatedAt(builder, in.CreatedAt.Unix()) return GoodItemEnd(builder) } @@ -134,6 +136,11 @@ func ParseGoodsItem(data []byte) (item entity.GoodsItem, err error) { } } + createdAt := itemFBS.CreatedAt() + if createdAt > 0 { + item.CreatedAt = time.Unix(createdAt, 0) + } + return item, nil } diff --git a/internal/entity/gooditem.go b/internal/entity/gooditem.go index ebe7ec4..ef7f5e7 100644 --- a/internal/entity/gooditem.go +++ b/internal/entity/gooditem.go @@ -4,6 +4,7 @@ import ( "fmt" "strconv" "strings" + "time" "unicode" ) @@ -22,6 +23,7 @@ type GoodsItem struct { Cart int64 `json:"cart"` Stock int `json:"stock"` Parameters map[string]string `json:"parameters"` + CreatedAt time.Time `json:"created_at"` } type GoodsItemRaw struct { diff --git a/internal/interconnect/eway/client.go b/internal/interconnect/eway/client.go index ef28957..50fdbb4 100644 --- a/internal/interconnect/eway/client.go +++ b/internal/interconnect/eway/client.go @@ -325,7 +325,18 @@ func (c client) GetProductInfo(ctx context.Context, cart int64) (pi entity.Goods }) }) - err = collector.Visit("https://eway.elevel.ru/product/" + strconv.Itoa(int(cart)) + "/") + for i := 0; i < 3; i++ { + err = collector.Visit("https://eway.elevel.ru/product/" + strconv.Itoa(int(cart)) + "/") + if err != nil { + c.log.Warn().Err(err).Msg("unable to visit site, retrying...") + select { + case <-time.After(time.Second * 2): + continue + case <-ctx.Done(): + return pi, ctx.Err() + } + } + } if err != nil { return pi, fmt.Errorf("visiting site: %w", err) } diff --git a/internal/storage/badger/goodsitem.go b/internal/storage/badger/goodsitem.go index 872b969..7e2697d 100644 --- a/internal/storage/badger/goodsitem.go +++ b/internal/storage/badger/goodsitem.go @@ -248,18 +248,20 @@ func (c *goodsItemClient) upsertByBatch(ctx context.Context, items []entity.Good batch := c.db.NewWriteBatch() defer batch.Cancel() + createdAt := time.Now() err := func() error { for _, item := range items { if ctx.Err() != nil { return ctx.Err() } - key := c.prefixedStr(item.Articul) + item.CreatedAt = createdAt value, err := c.s.Serialize(item) if err != nil { return fmt.Errorf("serializing item: %w", err) } + key := c.prefixedStr(item.Articul) idxValue := make([]byte, len(key)) copy(idxValue, key)