From 08be7de1186f7f6ba7231e9ab564545bcc17d9eb Mon Sep 17 00:00:00 2001 From: Aleksandr Trushkin Date: Sun, 4 Feb 2024 13:14:56 +0300 Subject: [PATCH] actualize items list in db --- cmd/cli/main.go | 33 ++++++-- internal/entity/iter.go | 39 +++++++++- internal/entity/repository.go | 1 + internal/storage/badger/goodsitem.go | 108 ++++++++++++++++++--------- 4 files changed, 138 insertions(+), 43 deletions(-) diff --git a/cmd/cli/main.go b/cmd/cli/main.go index 71372c9..6886b0a 100644 --- a/cmd/cli/main.go +++ b/cmd/cli/main.go @@ -875,10 +875,17 @@ func parseEwayDumpAction(ctx context.Context, cmd *cli.Command) error { var i int var start int + seenItems, err := entity.IterIntoMap[string, entity.GoodsItem](repository.GoodsItem().List(ctx)).Map(func(gi entity.GoodsItem) (string, error) { + return gi.Articul, nil + }) + if err != nil { + return fmt.Errorf("making seen items map: %w", err) + } + goodsItems := make([]entity.GoodsItem, 0, batchSize) productIDs := make([]int, 0, batchSize) - knownCategories := make(map[string]struct{}) + knownCategories := make(map[string]struct{}) err = entity.IterWithErr(repository.Category().List(ctx)).Do(func(c entity.Category) error { knownCategories[c.Name] = struct{}{} return nil @@ -887,6 +894,8 @@ func parseEwayDumpAction(ctx context.Context, cmd *cli.Command) error { return fmt.Errorf("filling known categories: %w", err) } + itemsUpdated := make(map[string]struct{}, len(seenItems)) + startFrom := time.Now() for { select { @@ -918,9 +927,15 @@ func parseEwayDumpAction(ctx context.Context, cmd *cli.Command) error { goodsItems = goodsItems[:0] for _, item := range items { - pi, err := client.GetProductInfo(ctx, int64(item.Cart)) - if err != nil { - return fmt.Errorf("getting product info: %w", err) + var pi entity.GoodsItemInfo + if seenItem, ok := seenItems[item.SKU]; ok { + pi.Parameters = seenItem.Parameters + pi.PhotoURLs = seenItem.PhotoURLs + } else { + pi, err = client.GetProductInfo(ctx, int64(item.Cart)) + if err != nil { + return fmt.Errorf("getting product info: %w", err) + } } goodsItem, err := entity.MakeGoodsItem(item, remnants, pi) @@ -929,8 +944,9 @@ func parseEwayDumpAction(ctx context.Context, cmd *cli.Command) error { continue } - goodsItems = append(goodsItems, goodsItem) + itemsUpdated[goodsItem.Articul] = struct{}{} + goodsItems = append(goodsItems, goodsItem) if goodsItem.Type == "" { continue } @@ -982,6 +998,13 @@ func parseEwayDumpAction(ctx context.Context, cmd *cli.Command) error { i++ } + for k := range itemsUpdated { + delete(seenItems, k) + } + for k := range seenItems { + repository.GoodsItem().Delete(ctx, k) + } + return nil } diff --git a/internal/entity/iter.go b/internal/entity/iter.go index bc3f45d..23faaf5 100644 --- a/internal/entity/iter.go +++ b/internal/entity/iter.go @@ -1,18 +1,49 @@ package entity -func IterWithErr[T any](t []T, err error) iterWithErr[T] { - return iterWithErr[T]{ +func IterIntoMap[K comparable, V any](v []V, err error) iterIntoMap[K, V] { + bi := IterWithErr(v, err) + + return iterIntoMap[K, V]{ + baseIter: bi, + } +} + +type iterIntoMap[K comparable, V any] struct { + baseIter[V] +} + +func (i iterIntoMap[K, V]) Map(f func(V) (K, error)) (map[K]V, error) { + if i.err != nil { + return nil, i.err + } + + out := make(map[K]V, len(i.items)) + for _, item := range i.items { + var key K + key, i.err = f(item) + if i.err != nil { + return nil, i.err + } + + out[key] = item + } + + return out, nil +} + +func IterWithErr[T any](t []T, err error) baseIter[T] { + return baseIter[T]{ items: t, err: err, } } -type iterWithErr[T any] struct { +type baseIter[T any] struct { items []T err error } -func (iter iterWithErr[T]) Do(f func(T) error) error { +func (iter baseIter[T]) Do(f func(T) error) error { if iter.err != nil { return iter.err } diff --git a/internal/entity/repository.go b/internal/entity/repository.go index 1563617..5c66717 100644 --- a/internal/entity/repository.go +++ b/internal/entity/repository.go @@ -9,6 +9,7 @@ type GoodsItemRepository interface { GetByCart(context.Context, int64) (GoodsItem, error) UpsertMany(context.Context, ...GoodsItem) ([]GoodsItem, error) + Delete(context.Context, string) (GoodsItem, error) } type CategoryRepository interface { diff --git a/internal/storage/badger/goodsitem.go b/internal/storage/badger/goodsitem.go index 9650d6f..b7c7728 100644 --- a/internal/storage/badger/goodsitem.go +++ b/internal/storage/badger/goodsitem.go @@ -21,11 +21,20 @@ const useJSON = false type goodsItemClient struct { db *badger.DB + + s itemSerializer[entity.GoodsItem] } -func newGoodsItemClient(db *badger.DB) *goodsItemClient { +func newGoodsItemClient(db *badger.DB, serializeAsJSON bool) *goodsItemClient { + var s itemSerializer[entity.GoodsItem] + if serializeAsJSON { + s = goodsItemJSONSerializer{} + } else { + s = goodsItemFlatbufSerializer{} + } return &goodsItemClient{ db: db, + s: s, } } @@ -68,17 +77,9 @@ func (c *goodsItemClient) ListIter( for _, kv := range list.GetKv() { var gooditem entity.GoodsItem - - if useJSON { - err = json.Unmarshal(kv.GetValue(), &gooditem) - if err != nil { - return err - } - } else { - gooditem, err = fbs.ParseGoodsItem(kv.GetValue()) - if err != nil { - return err - } + gooditem, err = c.s.Deserialize(kv.GetValue()) + if err != nil { + return fmt.Errorf("deserializing item: %w", err) } bus <- gooditem @@ -94,7 +95,6 @@ func (c *goodsItemClient) ListIter( if err != nil { zerolog.Ctx(ctx).Warn().Err(err).Msg("unable to orchestrate") } - println("finished") }(ctx) return bus, nil @@ -117,17 +117,11 @@ func (c *goodsItemClient) List( current := iter.Item() err = current.Value(func(val []byte) error { var goodsItem entity.GoodsItem - if useJSON { - err := json.Unmarshal(val, &goodsItem) - if err != nil { - return err - } - } else { - goodsItem, err = fbs.ParseGoodsItem(val) - if err != nil { - return err - } + goodsItem, err = c.s.Deserialize(val) + if err != nil { + return fmt.Errorf("deserializing: %w", err) } + out = append(out, goodsItem) return nil @@ -201,6 +195,28 @@ func (c *goodsItemClient) UpsertMany(ctx context.Context, items ...entity.GoodsI return items, c.upsertByBatch(ctx, items) } +func (c *goodsItemClient) Delete(ctx context.Context, sku string) (out entity.GoodsItem, err error) { + err = c.db.Update(func(txn *badger.Txn) error { + skuKey := c.prefixedStr(sku) + out, err = c.getBySKU(skuKey, txn) + if err != nil { + return err + } + + err = txn.Delete(skuKey) + if err != nil { + return fmt.Errorf("deleting key: %w", err) + } + + return nil + }) + if err != nil { + return entity.GoodsItem{}, err + } + + return out, nil +} + func (c *goodsItemClient) upsertByBatch(ctx context.Context, items []entity.GoodsItem) error { batch := c.db.NewWriteBatch() defer batch.Cancel() @@ -214,11 +230,9 @@ func (c *goodsItemClient) upsertByBatch(ctx context.Context, items []entity.Good } key := c.prefixedStr(item.Articul) - var value []byte - if useJSON { - value, _ = json.Marshal(item) - } else { - value = fbs.MakeDomainGoodItemFinished(item) + value, err := c.s.Serialize(item) + if err != nil { + return fmt.Errorf("serializing item: %w", err) } idxValue := make([]byte, len(key)) @@ -257,17 +271,43 @@ func (c *goodsItemClient) getBySKU(sku []byte, txn *badger.Txn) (out entity.Good } err = item.Value(func(val []byte) error { - if useJSON { - return json.Unmarshal(val, &out) - } - - out, err = fbs.ParseGoodsItem(val) + out, err = c.s.Deserialize(val) return err }) - if err != nil { return out, fmt.Errorf("reading value: %w", err) } return out, nil } + +type itemSerializer[T any] interface { + Serialize(T) ([]byte, error) + Deserialize([]byte) (T, error) +} + +type goodsItemJSONSerializer struct{} + +func (goodsItemJSONSerializer) Serialize(in entity.GoodsItem) ([]byte, error) { + return json.Marshal(in) +} + +func (goodsItemJSONSerializer) Deserialize(data []byte) (in entity.GoodsItem, err error) { + err = json.Unmarshal(data, &in) + return in, err +} + +type goodsItemFlatbufSerializer struct{} + +func (goodsItemFlatbufSerializer) Serialize(in entity.GoodsItem) ([]byte, error) { + out := fbs.MakeDomainGoodItemFinished(in) + return out, nil +} + +func (goodsItemFlatbufSerializer) Deserialize(data []byte) (out entity.GoodsItem, err error) { + out, err = fbs.ParseGoodsItem(data) + if err != nil { + return entity.GoodsItem{}, err + } + return out, nil +}