frx/eway
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

actualize items list in db

Browse Source
This commit is contained in:
Aleksandr Trushkin
2024-02-04 13:14:56 +03:00
parent be6aec73df
commit 5c8f238fb8
4 changed files with 138 additions and 43 deletions
Download Patch File Download Diff File Expand all files Collapse all files

33
cmd/cli/main.go
View File

@ -875,10 +875,17 @@ func parseEwayDumpAction(ctx context.Context, cmd *cli.Command) error {
var i int
var start int
seenItems, err := entity.IterIntoMap[string, entity.GoodsItem](repository.GoodsItem().List(ctx)).Map(func(gi entity.GoodsItem) (string, error) {
return gi.Articul, nil
})
if err != nil {
return fmt.Errorf("making seen items map: %w", err)
}
goodsItems := make([]entity.GoodsItem, 0, batchSize)
productIDs := make([]int, 0, batchSize)
knownCategories := make(map[string]struct{})
knownCategories := make(map[string]struct{})
err = entity.IterWithErr(repository.Category().List(ctx)).Do(func(c entity.Category) error {
knownCategories[c.Name] = struct{}{}
return nil
@ -887,6 +894,8 @@ func parseEwayDumpAction(ctx context.Context, cmd *cli.Command) error {
return fmt.Errorf("filling known categories: %w", err)
}
itemsUpdated := make(map[string]struct{}, len(seenItems))
startFrom := time.Now()
for {
select {
@ -918,9 +927,15 @@ func parseEwayDumpAction(ctx context.Context, cmd *cli.Command) error {
goodsItems = goodsItems[:0]
for _, item := range items {
pi, err := client.GetProductInfo(ctx, int64(item.Cart))
if err != nil {
return fmt.Errorf("getting product info: %w", err)
var pi entity.GoodsItemInfo
if seenItem, ok := seenItems[item.SKU]; ok {
pi.Parameters = seenItem.Parameters
pi.PhotoURLs = seenItem.PhotoURLs
} else {
pi, err = client.GetProductInfo(ctx, int64(item.Cart))
if err != nil {
return fmt.Errorf("getting product info: %w", err)
}
}
goodsItem, err := entity.MakeGoodsItem(item, remnants, pi)
@ -929,8 +944,9 @@ func parseEwayDumpAction(ctx context.Context, cmd *cli.Command) error {
continue
}
goodsItems = append(goodsItems, goodsItem)
itemsUpdated[goodsItem.Articul] = struct{}{}
goodsItems = append(goodsItems, goodsItem)
if goodsItem.Type == "" {
continue
}
@ -982,6 +998,13 @@ func parseEwayDumpAction(ctx context.Context, cmd *cli.Command) error {
i++
}
for k := range itemsUpdated {
delete(seenItems, k)
}
for k := range seenItems {
repository.GoodsItem().Delete(ctx, k)
}
return nil
}

39
internal/entity/iter.go
View File

@ -1,18 +1,49 @@
package entity
func IterWithErr[T any](t []T, err error) iterWithErr[T] {
return iterWithErr[T]{
func IterIntoMap[K comparable, V any](v []V, err error) iterIntoMap[K, V] {
bi := IterWithErr(v, err)
return iterIntoMap[K, V]{
baseIter: bi,
}
}
type iterIntoMap[K comparable, V any] struct {
baseIter[V]
}
func (i iterIntoMap[K, V]) Map(f func(V) (K, error)) (map[K]V, error) {
if i.err != nil {
return nil, i.err
}
out := make(map[K]V, len(i.items))
for _, item := range i.items {
var key K
key, i.err = f(item)
if i.err != nil {
return nil, i.err
}
out[key] = item
}
return out, nil
}
func IterWithErr[T any](t []T, err error) baseIter[T] {
return baseIter[T]{
items: t,
err: err,
}
}
type iterWithErr[T any] struct {
type baseIter[T any] struct {
items []T
err error
}
func (iter iterWithErr[T]) Do(f func(T) error) error {
func (iter baseIter[T]) Do(f func(T) error) error {
if iter.err != nil {
return iter.err
}

1
internal/entity/repository.go
View File

@ -9,6 +9,7 @@ type GoodsItemRepository interface {
GetByCart(context.Context, int64) (GoodsItem, error)
UpsertMany(context.Context, ...GoodsItem) ([]GoodsItem, error)
Delete(context.Context, string) (GoodsItem, error)
}
type CategoryRepository interface {

108
internal/storage/badger/goodsitem.go
View File

@ -21,11 +21,20 @@ const useJSON = false
type goodsItemClient struct {
db *badger.DB
s itemSerializer[entity.GoodsItem]
}
func newGoodsItemClient(db *badger.DB) *goodsItemClient {
func newGoodsItemClient(db *badger.DB, serializeAsJSON bool) *goodsItemClient {
var s itemSerializer[entity.GoodsItem]
if serializeAsJSON {
s = goodsItemJSONSerializer{}
} else {
s = goodsItemFlatbufSerializer{}
}
return &goodsItemClient{
db: db,
s: s,
}
}
@ -68,17 +77,9 @@ func (c *goodsItemClient) ListIter(
for _, kv := range list.GetKv() {
var gooditem entity.GoodsItem
if useJSON {
err = json.Unmarshal(kv.GetValue(), &gooditem)
if err != nil {
return err
}
} else {
gooditem, err = fbs.ParseGoodsItem(kv.GetValue())
if err != nil {
return err
}
gooditem, err = c.s.Deserialize(kv.GetValue())
if err != nil {
return fmt.Errorf("deserializing item: %w", err)
}
bus <- gooditem
@ -94,7 +95,6 @@ func (c *goodsItemClient) ListIter(
if err != nil {
zerolog.Ctx(ctx).Warn().Err(err).Msg("unable to orchestrate")
}
println("finished")
}(ctx)
return bus, nil
@ -117,17 +117,11 @@ func (c *goodsItemClient) List(
current := iter.Item()
err = current.Value(func(val []byte) error {
var goodsItem entity.GoodsItem
if useJSON {
err := json.Unmarshal(val, &goodsItem)
if err != nil {
return err
}
} else {
goodsItem, err = fbs.ParseGoodsItem(val)
if err != nil {
return err
}
goodsItem, err = c.s.Deserialize(val)
if err != nil {
return fmt.Errorf("deserializing: %w", err)
}
out = append(out, goodsItem)
return nil
@ -201,6 +195,28 @@ func (c *goodsItemClient) UpsertMany(ctx context.Context, items ...entity.GoodsI
return items, c.upsertByBatch(ctx, items)
}
func (c *goodsItemClient) Delete(ctx context.Context, sku string) (out entity.GoodsItem, err error) {
err = c.db.Update(func(txn *badger.Txn) error {
skuKey := c.prefixedStr(sku)
out, err = c.getBySKU(skuKey, txn)
if err != nil {
return err
}
err = txn.Delete(skuKey)
if err != nil {
return fmt.Errorf("deleting key: %w", err)
}
return nil
})
if err != nil {
return entity.GoodsItem{}, err
}
return out, nil
}
func (c *goodsItemClient) upsertByBatch(ctx context.Context, items []entity.GoodsItem) error {
batch := c.db.NewWriteBatch()
defer batch.Cancel()
@ -214,11 +230,9 @@ func (c *goodsItemClient) upsertByBatch(ctx context.Context, items []entity.Good
}
key := c.prefixedStr(item.Articul)
var value []byte
if useJSON {
value, _ = json.Marshal(item)
} else {
value = fbs.MakeDomainGoodItemFinished(item)
value, err := c.s.Serialize(item)
if err != nil {
return fmt.Errorf("serializing item: %w", err)
}
idxValue := make([]byte, len(key))
@ -257,17 +271,43 @@ func (c *goodsItemClient) getBySKU(sku []byte, txn *badger.Txn) (out entity.Good
}
err = item.Value(func(val []byte) error {
if useJSON {
return json.Unmarshal(val, &out)
}
out, err = fbs.ParseGoodsItem(val)
out, err = c.s.Deserialize(val)
return err
})
if err != nil {
return out, fmt.Errorf("reading value: %w", err)
}
return out, nil
}
type itemSerializer[T any] interface {
Serialize(T) ([]byte, error)
Deserialize([]byte) (T, error)
}
type goodsItemJSONSerializer struct{}
func (goodsItemJSONSerializer) Serialize(in entity.GoodsItem) ([]byte, error) {
return json.Marshal(in)
}
func (goodsItemJSONSerializer) Deserialize(data []byte) (in entity.GoodsItem, err error) {
err = json.Unmarshal(data, &in)
return in, err
}
type goodsItemFlatbufSerializer struct{}
func (goodsItemFlatbufSerializer) Serialize(in entity.GoodsItem) ([]byte, error) {
out := fbs.MakeDomainGoodItemFinished(in)
return out, nil
}
func (goodsItemFlatbufSerializer) Deserialize(data []byte) (out entity.GoodsItem, err error) {
out, err = fbs.ParseGoodsItem(data)
if err != nil {
return entity.GoodsItem{}, err
}
return out, nil
}