actualize items list in db

cmd/cli/main.go

@@ -875,10 +875,17 @@ func parseEwayDumpAction(ctx context.Context, cmd *cli.Command) error {
 	var i int
 	var start int
 
+	seenItems, err := entity.IterIntoMap[string, entity.GoodsItem](repository.GoodsItem().List(ctx)).Map(func(gi entity.GoodsItem) (string, error) {
+		return gi.Articul, nil
+	})
+	if err != nil {
+		return fmt.Errorf("making seen items map: %w", err)
+	}
+
 	goodsItems := make([]entity.GoodsItem, 0, batchSize)
 	productIDs := make([]int, 0, batchSize)
-	knownCategories := make(map[string]struct{})
 
+	knownCategories := make(map[string]struct{})
 	err = entity.IterWithErr(repository.Category().List(ctx)).Do(func(c entity.Category) error {
 		knownCategories[c.Name] = struct{}{}
 		return nil
@@ -887,6 +894,8 @@ func parseEwayDumpAction(ctx context.Context, cmd *cli.Command) error {
 		return fmt.Errorf("filling known categories: %w", err)
 	}
 
+	itemsUpdated := make(map[string]struct{}, len(seenItems))
+
 	startFrom := time.Now()
 	for {
 		select {
@@ -918,9 +927,15 @@ func parseEwayDumpAction(ctx context.Context, cmd *cli.Command) error {
 
 		goodsItems = goodsItems[:0]
 		for _, item := range items {
-			pi, err := client.GetProductInfo(ctx, int64(item.Cart))
+			var pi entity.GoodsItemInfo
-			if err != nil {
+			if seenItem, ok := seenItems[item.SKU]; ok {
-				return fmt.Errorf("getting product info: %w", err)
+				pi.Parameters = seenItem.Parameters
+				pi.PhotoURLs = seenItem.PhotoURLs
+			} else {
+				pi, err = client.GetProductInfo(ctx, int64(item.Cart))
+				if err != nil {
+					return fmt.Errorf("getting product info: %w", err)
+				}
 			}
 
 			goodsItem, err := entity.MakeGoodsItem(item, remnants, pi)
@@ -929,8 +944,9 @@ func parseEwayDumpAction(ctx context.Context, cmd *cli.Command) error {
 				continue
 			}
 
-			goodsItems = append(goodsItems, goodsItem)
+			itemsUpdated[goodsItem.Articul] = struct{}{}
 
+			goodsItems = append(goodsItems, goodsItem)
 			if goodsItem.Type == "" {
 				continue
 			}
@@ -982,6 +998,13 @@ func parseEwayDumpAction(ctx context.Context, cmd *cli.Command) error {
 		i++
 	}
 
+	for k := range itemsUpdated {
+		delete(seenItems, k)
+	}
+	for k := range seenItems {
+		repository.GoodsItem().Delete(ctx, k)
+	}
+
 	return nil
 }
 

internal/entity/iter.go

@@ -1,18 +1,49 @@
 package entity
 
-func IterWithErr[T any](t []T, err error) iterWithErr[T] {
+func IterIntoMap[K comparable, V any](v []V, err error) iterIntoMap[K, V] {
-	return iterWithErr[T]{
+	bi := IterWithErr(v, err)
+
+	return iterIntoMap[K, V]{
+		baseIter: bi,
+	}
+}
+
+type iterIntoMap[K comparable, V any] struct {
+	baseIter[V]
+}
+
+func (i iterIntoMap[K, V]) Map(f func(V) (K, error)) (map[K]V, error) {
+	if i.err != nil {
+		return nil, i.err
+	}
+
+	out := make(map[K]V, len(i.items))
+	for _, item := range i.items {
+		var key K
+		key, i.err = f(item)
+		if i.err != nil {
+			return nil, i.err
+		}
+
+		out[key] = item
+	}
+
+	return out, nil
+}
+
+func IterWithErr[T any](t []T, err error) baseIter[T] {
+	return baseIter[T]{
 		items: t,
 		err:   err,
 	}
 }
 
-type iterWithErr[T any] struct {
+type baseIter[T any] struct {
 	items []T
 	err   error
 }
 
-func (iter iterWithErr[T]) Do(f func(T) error) error {
+func (iter baseIter[T]) Do(f func(T) error) error {
 	if iter.err != nil {
 		return iter.err
 	}

internal/entity/repository.go

@@ -9,6 +9,7 @@ type GoodsItemRepository interface {
 	GetByCart(context.Context, int64) (GoodsItem, error)
 
 	UpsertMany(context.Context, ...GoodsItem) ([]GoodsItem, error)
+	Delete(context.Context, string) (GoodsItem, error)
 }
 
 type CategoryRepository interface {

internal/storage/badger/goodsitem.go

@@ -21,11 +21,20 @@ const useJSON = false
 
 type goodsItemClient struct {
 	db *badger.DB
+
+	s itemSerializer[entity.GoodsItem]
 }
 
-func newGoodsItemClient(db *badger.DB) *goodsItemClient {
+func newGoodsItemClient(db *badger.DB, serializeAsJSON bool) *goodsItemClient {
+	var s itemSerializer[entity.GoodsItem]
+	if serializeAsJSON {
+		s = goodsItemJSONSerializer{}
+	} else {
+		s = goodsItemFlatbufSerializer{}
+	}
 	return &goodsItemClient{
 		db: db,
+		s:  s,
 	}
 }
 
@@ -68,17 +77,9 @@ func (c *goodsItemClient) ListIter(
 
 		for _, kv := range list.GetKv() {
 			var gooditem entity.GoodsItem
-
+			gooditem, err = c.s.Deserialize(kv.GetValue())
-			if useJSON {
+			if err != nil {
-				err = json.Unmarshal(kv.GetValue(), &gooditem)
+				return fmt.Errorf("deserializing item: %w", err)
-				if err != nil {
-					return err
-				}
-			} else {
-				gooditem, err = fbs.ParseGoodsItem(kv.GetValue())
-				if err != nil {
-					return err
-				}
 			}
 
 			bus <- gooditem
@@ -94,7 +95,6 @@ func (c *goodsItemClient) ListIter(
 		if err != nil {
 			zerolog.Ctx(ctx).Warn().Err(err).Msg("unable to orchestrate")
 		}
-		println("finished")
 	}(ctx)
 
 	return bus, nil
@@ -117,17 +117,11 @@ func (c *goodsItemClient) List(
 			current := iter.Item()
 			err = current.Value(func(val []byte) error {
 				var goodsItem entity.GoodsItem
-				if useJSON {
+				goodsItem, err = c.s.Deserialize(val)
-					err := json.Unmarshal(val, &goodsItem)
+				if err != nil {
-					if err != nil {
+					return fmt.Errorf("deserializing: %w", err)
-						return err
-					}
-				} else {
-					goodsItem, err = fbs.ParseGoodsItem(val)
-					if err != nil {
-						return err
-					}
 				}
+
 				out = append(out, goodsItem)
 
 				return nil
@@ -201,6 +195,28 @@ func (c *goodsItemClient) UpsertMany(ctx context.Context, items ...entity.GoodsI
 	return items, c.upsertByBatch(ctx, items)
 }
 
+func (c *goodsItemClient) Delete(ctx context.Context, sku string) (out entity.GoodsItem, err error) {
+	err = c.db.Update(func(txn *badger.Txn) error {
+		skuKey := c.prefixedStr(sku)
+		out, err = c.getBySKU(skuKey, txn)
+		if err != nil {
+			return err
+		}
+
+		err = txn.Delete(skuKey)
+		if err != nil {
+			return fmt.Errorf("deleting key: %w", err)
+		}
+
+		return nil
+	})
+	if err != nil {
+		return entity.GoodsItem{}, err
+	}
+
+	return out, nil
+}
+
 func (c *goodsItemClient) upsertByBatch(ctx context.Context, items []entity.GoodsItem) error {
 	batch := c.db.NewWriteBatch()
 	defer batch.Cancel()
@@ -214,11 +230,9 @@ func (c *goodsItemClient) upsertByBatch(ctx context.Context, items []entity.Good
 			}
 
 			key := c.prefixedStr(item.Articul)
-			var value []byte
+			value, err := c.s.Serialize(item)
-			if useJSON {
+			if err != nil {
-				value, _ = json.Marshal(item)
+				return fmt.Errorf("serializing item: %w", err)
-			} else {
-				value = fbs.MakeDomainGoodItemFinished(item)
 			}
 
 			idxValue := make([]byte, len(key))
@@ -257,17 +271,43 @@ func (c *goodsItemClient) getBySKU(sku []byte, txn *badger.Txn) (out entity.Good
 	}
 
 	err = item.Value(func(val []byte) error {
-		if useJSON {
+		out, err = c.s.Deserialize(val)
-			return json.Unmarshal(val, &out)
-		}
-
-		out, err = fbs.ParseGoodsItem(val)
 		return err
 	})
-
 	if err != nil {
 		return out, fmt.Errorf("reading value: %w", err)
 	}
 
 	return out, nil
 }
+
+type itemSerializer[T any] interface {
+	Serialize(T) ([]byte, error)
+	Deserialize([]byte) (T, error)
+}
+
+type goodsItemJSONSerializer struct{}
+
+func (goodsItemJSONSerializer) Serialize(in entity.GoodsItem) ([]byte, error) {
+	return json.Marshal(in)
+}
+
+func (goodsItemJSONSerializer) Deserialize(data []byte) (in entity.GoodsItem, err error) {
+	err = json.Unmarshal(data, &in)
+	return in, err
+}
+
+type goodsItemFlatbufSerializer struct{}
+
+func (goodsItemFlatbufSerializer) Serialize(in entity.GoodsItem) ([]byte, error) {
+	out := fbs.MakeDomainGoodItemFinished(in)
+	return out, nil
+}
+
+func (goodsItemFlatbufSerializer) Deserialize(data []byte) (out entity.GoodsItem, err error) {
+	out, err = fbs.ParseGoodsItem(data)
+	if err != nil {
+		return entity.GoodsItem{}, err
+	}
+	return out, nil
+}