frx/eway
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

able to parse xml

Browse Source
This commit is contained in:
Aleksandr Trushkin
2024-01-28 16:49:48 +03:00
parent a0b36ba83d
commit dd639995bd
11 changed files with 295 additions and 87 deletions
Download Patch File Download Diff File Expand all files Collapse all files

16
cmd/cli/components/di.go
View File

@ -20,6 +20,10 @@ import (
// Yeah, singleton is not good UNLESS you're really lazy
var diInjector *do.Injector
func GetEwayClient() (eway.Client, error) {
return do.Invoke[eway.Client](diInjector)
}
func GetRepository() (storage.Repository, error) {
adapter, err := do.Invoke[*storageRepositoryAdapter](diInjector)
if err != nil {
@ -57,7 +61,11 @@ func SetupDI(ctx context.Context, cfgpath string) error {
return nil, fmt.Errorf("getting logger: %w", err)
}
client := eway.New(eway.Config(cfg.Eway), log)
client, err := eway.New(eway.Config(cfg.Eway), log)
if err != nil {
return nil, fmt.Errorf("making new eway client: %w", err)
}
return client, nil
})
@ -103,9 +111,9 @@ func getDB() (*badger.DB, error) {
}
type settings struct {
Badger config.Badger
Log config.Log
Eway config.Eway
Badger config.Badger `toml:"badger"`
Log config.Log `toml:"log"`
Eway config.Eway `toml:"eway"`
}
func parseSettings(cfgpath string) (cfg settings, err error) {

185
cmd/cli/main.go
View File

@ -10,6 +10,7 @@ import (
"errors"
"fmt"
"io"
"math/big"
"os"
"os/signal"
"strconv"
@ -19,6 +20,7 @@ import (
"git.loyso.art/frx/eway/internal/encoding/fbs"
"git.loyso.art/frx/eway/internal/entity"
"git.loyso.art/frx/eway/internal/export"
"git.loyso.art/frx/eway/internal/interconnect/eway"
"github.com/brianvoe/gofakeit/v6"
"github.com/rodaine/table"
@ -66,10 +68,13 @@ func releaseDI(c *cli.Context) error {
return fmt.Errorf("getting logger: %w", err)
}
log.Info().Msg("shutting down env")
start := time.Now()
defer func() {
since := time.Since(start)
if err == nil {
return
}
log.Err(err).Dur("elapsed", since).Msg("shutdown finished")
}()
@ -91,14 +96,35 @@ func setupCLI(ctx context.Context) *cli.App {
app.Before = setupDI(ctx)
app.After = releaseDI
app.Commands = cli.Commands{
newParseCmd(ctx),
newImportCmd(ctx),
newExportCmd(ctx),
newViewCmd(ctx),
}
app.EnableBashCompletion = true
app.BashComplete = cli.DefaultAppComplete
return app
}
func newParseCmd(ctx context.Context) cli.Command {
return cli.Command{
Name: "parse",
Usage: "category for parsing items from various sources",
Subcommands: cli.Commands{
newParseEwayCmd(ctx),
},
}
}
func newParseEwayCmd(ctx context.Context) cli.Command {
return cli.Command{
Name: "eway",
Usage: "parse all available eway goods",
Action: decorateAction(ctx, parseEwayAction),
}
}
func newImportCmd(ctx context.Context) cli.Command {
return cli.Command{
Name: "import",
@ -111,7 +137,7 @@ func newImportCmd(ctx context.Context) cli.Command {
func newImportFromFileCmd(ctx context.Context) cli.Command {
return cli.Command{
Name: "fromfile",
Name: "file",
Usage: "imports from file into db",
Flags: []cli.Flag{
&cli.StringFlag{
@ -409,6 +435,20 @@ func importFromFileAction(ctx context.Context, c *cli.Context) error {
}
}()
failedItems, err := os.Create("failed.json")
if err != nil {
log.Warn().Err(err).Msg("unable to open file for failed results")
failedItems = os.Stdout
}
defer func() {
if failedItems == os.Stdout {
return
}
errClose := failedItems.Close()
log.Err(errClose).Msg("closing file")
}()
var (
goodsItem entity.GoodsItem
goodsItems []entity.GoodsItem
@ -439,6 +479,8 @@ func importFromFileAction(ctx context.Context, c *cli.Context) error {
err = json.Unmarshal(line, &goodsItem)
if err != nil {
log.Warn().Err(err).Str("line", string(line)).Msg("unable to unmarshal line into item")
_, _ = failedItems.Write(line)
_, _ = failedItems.Write([]byte{'\n'})
failedToInsert++
continue
}
@ -449,6 +491,12 @@ func importFromFileAction(ctx context.Context, c *cli.Context) error {
continue
}
if goodsItem.Type == "" {
log.Warn().Msg("bad item without proper type")
_ = json.NewEncoder(failedItems).Encode(goodsItem)
continue
}
_, err = r.Category().Create(ctx, goodsItem.Type)
if err != nil {
return fmt.Errorf("unable to create new category: %w", err)
@ -591,20 +639,151 @@ func exportYMLCatalogAction(ctx context.Context, c *cli.Context) error {
return enc.Encode(container)
}
func parseEwayAction(ctx context.Context, c *cli.Context) error {
client, err := components.GetEwayClient()
if err != nil {
return fmt.Errorf("getting eway client: %w", err)
}
repository, err := components.GetRepository()
if err != nil {
return fmt.Errorf("getting repository: %w", err)
}
logger, err := components.GetLogger()
if err != nil {
return fmt.Errorf("getting logger: %w", err)
}
const batchSize = 100
var i int
var start int
goodsItems := make([]entity.GoodsItem, 0, batchSize)
productIDs := make([]int, 0, batchSize)
knownCategories := make(map[string]struct{})
err = entity.IterWithErr(repository.Category().List(ctx)).Do(func(c entity.Category) error {
knownCategories[c.Name] = struct{}{}
return nil
})
if err != nil {
return fmt.Errorf("filling known categories: %w", err)
}
startFrom := time.Now()
for {
select {
case <-ctx.Done():
return ctx.Err()
default:
}
items, total, err := client.GetGoodsNew(ctx, eway.GetGoodsNewParams{
Draw: i,
Start: start,
Length: batchSize,
SearchInStocks: true,
RemmantsAtleast: 5,
})
if err != nil {
return fmt.Errorf("getting next goods batch: %w", err)
}
productIDs = productIDs[:0]
for _, item := range items {
productIDs = append(productIDs, int(item.Cart))
}
remnants, err := client.GetGoodsRemnants(ctx, productIDs)
if err != nil {
return fmt.Errorf("getting goods remnants: %w", err)
}
goodsItems = goodsItems[:0]
for _, item := range items {
goodsItem, err := entity.MakeGoodsItem(item, remnants)
if err != nil {
logger.Warn().Err(err).Any("item", item).Msg("unable to make goods item")
continue
}
goodsItems = append(goodsItems, goodsItem)
if goodsItem.Type == "" {
continue
}
if _, ok := knownCategories[goodsItem.Type]; ok {
continue
}
category, err := repository.Category().Create(ctx, goodsItem.Type)
if err != nil {
return fmt.Errorf("creating category: %w", err)
}
logger.Debug().
Str("name", category.Name).
Int64("id", category.ID).
Msg("created new category")
knownCategories[goodsItem.Type] = struct{}{}
}
_, err = repository.GoodsItem().UpsertMany(ctx, goodsItems...)
if err != nil {
return fmt.Errorf("upserting items: %w", err)
}
progressFloat := float64(start) / float64(total)
progress := big.NewFloat(progressFloat).Text('f', 3)
elapsed := time.Since(startFrom).Seconds()
var left int
if progressFloat != 0 {
left = int(((1 - progressFloat) / progressFloat) * elapsed)
}
logger.Debug().
Int("from", start).
Int("to", start+batchSize).
Int("total", total).
Str("progress", progress).
Int("seconds_left", left).
Msg("handled next batch items")
if len(items) < batchSize {
break
}
start += batchSize
i++
}
return nil
}
func goodsItemAsOffer(in entity.GoodsItem, categoryIDByName map[string]int64) (out export.Offer) {
const defaultType = "vendor.model"
const defaultCurrency = "RUR"
const defaultAvailable = true
const quantityParamName = "Количество на складе «Москва»"
const basePictureURL = "https://eway.elevel.ru"
imgurl := func(path string) string {
return basePictureURL + path
}
categoryID := categoryIDByName[in.Type]
out = export.Offer{
ID: in.Cart,
VendorCode: in.Articul,
Price: int(in.TariffPrice),
CategoryID: categoryID,
PictureURLs: []string{
in.Photo,
imgurl(in.Photo),
},
Model: in.Name,