allow to skip item by created

assets/gooditem.fbs

@@ -15,6 +15,7 @@ table GoodItem {
   cart:long;
   stock:short;
   parameters:string;
+  created_at:long;
 }
 
 table GoodItems {

cmd/cli/main.go

@@ -899,6 +899,7 @@ func parseEwayDumpAction(ctx context.Context, cmd *cli.Command) error {
 		fetchedInfo int
 		handledAll  int
 		cachedInfo  int
+		skippedItem int
 	}{}
 
 	startFrom := time.Now()
@@ -934,6 +935,14 @@ func parseEwayDumpAction(ctx context.Context, cmd *cli.Command) error {
 		for _, item := range items {
 			var pi entity.GoodsItemInfo
 			seenItem := seenItems[item.SKU]
+			if time.Since(seenItem.CreatedAt) < time.Hour*24 {
+				logger.Debug().Str("sku", item.SKU).Msg("skipping item because it's too fresh")
+				stats.skippedItem++
+				itemsUpdated[item.SKU] = struct{}{}
+
+				continue
+			}
+
 			if len(seenItem.Parameters) != 0 && len(seenItem.PhotoURLs) != 0 {
 				pi.Parameters = seenItem.Parameters
 				pi.PhotoURLs = seenItem.PhotoURLs
@@ -1015,6 +1024,7 @@ func parseEwayDumpAction(ctx context.Context, cmd *cli.Command) error {
 		Int("handled", stats.handledAll).
 		Int("cached", stats.cachedInfo).
 		Int("fetched", stats.fetchedInfo).
+		Int("skipped", stats.skippedItem).
 		Int("to_delete", len(seenItems)).
 		Msg("processed items")
 

internal/encoding/fbs/GoodItem.go

@@ -177,8 +177,20 @@ func (rcv *GoodItem) Parameters() []byte {
 	return nil
 }
 
+func (rcv *GoodItem) CreatedAt() int64 {
+	o := flatbuffers.UOffsetT(rcv._tab.Offset(32))
+	if o != 0 {
+		return rcv._tab.GetInt64(o + rcv._tab.Pos)
+	}
+	return 0
+}
+
+func (rcv *GoodItem) MutateCreatedAt(n int64) bool {
+	return rcv._tab.MutateInt64Slot(32, n)
+}
+
 func GoodItemStart(builder *flatbuffers.Builder) {
-	builder.StartObject(14)
+	builder.StartObject(15)
 }
 func GoodItemAddSku(builder *flatbuffers.Builder, sku flatbuffers.UOffsetT) {
 	builder.PrependUOffsetTSlot(0, flatbuffers.UOffsetT(sku), 0)
@@ -222,6 +234,9 @@ func GoodItemAddStock(builder *flatbuffers.Builder, stock int16) {
 func GoodItemAddParameters(builder *flatbuffers.Builder, parameters flatbuffers.UOffsetT) {
 	builder.PrependUOffsetTSlot(13, flatbuffers.UOffsetT(parameters), 0)
 }
+func GoodItemAddCreatedAt(builder *flatbuffers.Builder, createdAt int64) {
+	builder.PrependInt64Slot(14, createdAt, 0)
+}
 func GoodItemEnd(builder *flatbuffers.Builder) flatbuffers.UOffsetT {
 	return builder.EndObject()
 }

internal/encoding/fbs/helpers.go

@@ -6,6 +6,7 @@ import (
 	"fmt"
 	"strings"
 	"sync"
+	"time"
 
 	"git.loyso.art/frx/eway/internal/entity"
 
@@ -98,6 +99,7 @@ func makeDomainGoodItem(builder *flatbuffers.Builder, in entity.GoodsItem) flatb
 	GoodItemAddCart(builder, int64(in.Cart))
 	GoodItemAddStock(builder, int16(in.Stock))
 	GoodItemAddParameters(builder, parameters)
+	GoodItemAddCreatedAt(builder, in.CreatedAt.Unix())
 
 	return GoodItemEnd(builder)
 }
@@ -134,6 +136,11 @@ func ParseGoodsItem(data []byte) (item entity.GoodsItem, err error) {
 		}
 	}
 
+	createdAt := itemFBS.CreatedAt()
+	if createdAt > 0 {
+		item.CreatedAt = time.Unix(createdAt, 0)
+	}
+
 	return item, nil
 }
 

internal/entity/gooditem.go

@@ -4,6 +4,7 @@ import (
 	"fmt"
 	"strconv"
 	"strings"
+	"time"
 	"unicode"
 )
 
@@ -22,6 +23,7 @@ type GoodsItem struct {
 	Cart        int64             `json:"cart"`
 	Stock       int               `json:"stock"`
 	Parameters  map[string]string `json:"parameters"`
+	CreatedAt   time.Time         `json:"created_at"`
 }
 
 type GoodsItemRaw struct {

internal/interconnect/eway/client.go

@@ -325,7 +325,18 @@ func (c client) GetProductInfo(ctx context.Context, cart int64) (pi entity.Goods
 		})
 	})
 
-	err = collector.Visit("https://eway.elevel.ru/product/" + strconv.Itoa(int(cart)) + "/")
+	for i := 0; i < 3; i++ {
+		err = collector.Visit("https://eway.elevel.ru/product/" + strconv.Itoa(int(cart)) + "/")
+		if err != nil {
+			c.log.Warn().Err(err).Msg("unable to visit site, retrying...")
+			select {
+			case <-time.After(time.Second * 2):
+				continue
+			case <-ctx.Done():
+				return pi, ctx.Err()
+			}
+		}
+	}
 	if err != nil {
 		return pi, fmt.Errorf("visiting site: %w", err)
 	}

internal/storage/badger/goodsitem.go

@@ -248,18 +248,20 @@ func (c *goodsItemClient) upsertByBatch(ctx context.Context, items []entity.Good
 	batch := c.db.NewWriteBatch()
 	defer batch.Cancel()
 
+	createdAt := time.Now()
 	err := func() error {
 		for _, item := range items {
 			if ctx.Err() != nil {
 				return ctx.Err()
 			}
 
-			key := c.prefixedStr(item.Articul)
+			item.CreatedAt = createdAt
 			value, err := c.s.Serialize(item)
 			if err != nil {
 				return fmt.Errorf("serializing item: %w", err)
 			}
 
+			key := c.prefixedStr(item.Articul)
 			idxValue := make([]byte, len(key))
 			copy(idxValue, key)