able to parse xml
This commit is contained in:
185
cmd/cli/main.go
185
cmd/cli/main.go
@ -10,6 +10,7 @@ import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"math/big"
|
||||
"os"
|
||||
"os/signal"
|
||||
"strconv"
|
||||
@ -19,6 +20,7 @@ import (
|
||||
"git.loyso.art/frx/eway/internal/encoding/fbs"
|
||||
"git.loyso.art/frx/eway/internal/entity"
|
||||
"git.loyso.art/frx/eway/internal/export"
|
||||
"git.loyso.art/frx/eway/internal/interconnect/eway"
|
||||
|
||||
"github.com/brianvoe/gofakeit/v6"
|
||||
"github.com/rodaine/table"
|
||||
@ -66,10 +68,13 @@ func releaseDI(c *cli.Context) error {
|
||||
return fmt.Errorf("getting logger: %w", err)
|
||||
}
|
||||
|
||||
log.Info().Msg("shutting down env")
|
||||
start := time.Now()
|
||||
defer func() {
|
||||
since := time.Since(start)
|
||||
if err == nil {
|
||||
return
|
||||
}
|
||||
|
||||
log.Err(err).Dur("elapsed", since).Msg("shutdown finished")
|
||||
}()
|
||||
|
||||
@ -91,14 +96,35 @@ func setupCLI(ctx context.Context) *cli.App {
|
||||
app.Before = setupDI(ctx)
|
||||
app.After = releaseDI
|
||||
app.Commands = cli.Commands{
|
||||
newParseCmd(ctx),
|
||||
newImportCmd(ctx),
|
||||
newExportCmd(ctx),
|
||||
newViewCmd(ctx),
|
||||
}
|
||||
app.EnableBashCompletion = true
|
||||
app.BashComplete = cli.DefaultAppComplete
|
||||
|
||||
return app
|
||||
}
|
||||
|
||||
func newParseCmd(ctx context.Context) cli.Command {
|
||||
return cli.Command{
|
||||
Name: "parse",
|
||||
Usage: "category for parsing items from various sources",
|
||||
Subcommands: cli.Commands{
|
||||
newParseEwayCmd(ctx),
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func newParseEwayCmd(ctx context.Context) cli.Command {
|
||||
return cli.Command{
|
||||
Name: "eway",
|
||||
Usage: "parse all available eway goods",
|
||||
Action: decorateAction(ctx, parseEwayAction),
|
||||
}
|
||||
}
|
||||
|
||||
func newImportCmd(ctx context.Context) cli.Command {
|
||||
return cli.Command{
|
||||
Name: "import",
|
||||
@ -111,7 +137,7 @@ func newImportCmd(ctx context.Context) cli.Command {
|
||||
|
||||
func newImportFromFileCmd(ctx context.Context) cli.Command {
|
||||
return cli.Command{
|
||||
Name: "fromfile",
|
||||
Name: "file",
|
||||
Usage: "imports from file into db",
|
||||
Flags: []cli.Flag{
|
||||
&cli.StringFlag{
|
||||
@ -409,6 +435,20 @@ func importFromFileAction(ctx context.Context, c *cli.Context) error {
|
||||
}
|
||||
}()
|
||||
|
||||
failedItems, err := os.Create("failed.json")
|
||||
if err != nil {
|
||||
log.Warn().Err(err).Msg("unable to open file for failed results")
|
||||
failedItems = os.Stdout
|
||||
}
|
||||
defer func() {
|
||||
if failedItems == os.Stdout {
|
||||
return
|
||||
}
|
||||
|
||||
errClose := failedItems.Close()
|
||||
log.Err(errClose).Msg("closing file")
|
||||
}()
|
||||
|
||||
var (
|
||||
goodsItem entity.GoodsItem
|
||||
goodsItems []entity.GoodsItem
|
||||
@ -439,6 +479,8 @@ func importFromFileAction(ctx context.Context, c *cli.Context) error {
|
||||
err = json.Unmarshal(line, &goodsItem)
|
||||
if err != nil {
|
||||
log.Warn().Err(err).Str("line", string(line)).Msg("unable to unmarshal line into item")
|
||||
_, _ = failedItems.Write(line)
|
||||
_, _ = failedItems.Write([]byte{'\n'})
|
||||
failedToInsert++
|
||||
continue
|
||||
}
|
||||
@ -449,6 +491,12 @@ func importFromFileAction(ctx context.Context, c *cli.Context) error {
|
||||
continue
|
||||
}
|
||||
|
||||
if goodsItem.Type == "" {
|
||||
log.Warn().Msg("bad item without proper type")
|
||||
_ = json.NewEncoder(failedItems).Encode(goodsItem)
|
||||
continue
|
||||
}
|
||||
|
||||
_, err = r.Category().Create(ctx, goodsItem.Type)
|
||||
if err != nil {
|
||||
return fmt.Errorf("unable to create new category: %w", err)
|
||||
@ -591,20 +639,151 @@ func exportYMLCatalogAction(ctx context.Context, c *cli.Context) error {
|
||||
return enc.Encode(container)
|
||||
}
|
||||
|
||||
func parseEwayAction(ctx context.Context, c *cli.Context) error {
|
||||
client, err := components.GetEwayClient()
|
||||
if err != nil {
|
||||
return fmt.Errorf("getting eway client: %w", err)
|
||||
}
|
||||
|
||||
repository, err := components.GetRepository()
|
||||
if err != nil {
|
||||
return fmt.Errorf("getting repository: %w", err)
|
||||
}
|
||||
|
||||
logger, err := components.GetLogger()
|
||||
if err != nil {
|
||||
return fmt.Errorf("getting logger: %w", err)
|
||||
}
|
||||
|
||||
const batchSize = 100
|
||||
var i int
|
||||
var start int
|
||||
|
||||
goodsItems := make([]entity.GoodsItem, 0, batchSize)
|
||||
productIDs := make([]int, 0, batchSize)
|
||||
knownCategories := make(map[string]struct{})
|
||||
|
||||
err = entity.IterWithErr(repository.Category().List(ctx)).Do(func(c entity.Category) error {
|
||||
knownCategories[c.Name] = struct{}{}
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
return fmt.Errorf("filling known categories: %w", err)
|
||||
}
|
||||
|
||||
startFrom := time.Now()
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
default:
|
||||
}
|
||||
|
||||
items, total, err := client.GetGoodsNew(ctx, eway.GetGoodsNewParams{
|
||||
Draw: i,
|
||||
Start: start,
|
||||
Length: batchSize,
|
||||
SearchInStocks: true,
|
||||
RemmantsAtleast: 5,
|
||||
})
|
||||
if err != nil {
|
||||
return fmt.Errorf("getting next goods batch: %w", err)
|
||||
}
|
||||
|
||||
productIDs = productIDs[:0]
|
||||
for _, item := range items {
|
||||
productIDs = append(productIDs, int(item.Cart))
|
||||
}
|
||||
|
||||
remnants, err := client.GetGoodsRemnants(ctx, productIDs)
|
||||
if err != nil {
|
||||
return fmt.Errorf("getting goods remnants: %w", err)
|
||||
}
|
||||
|
||||
goodsItems = goodsItems[:0]
|
||||
for _, item := range items {
|
||||
goodsItem, err := entity.MakeGoodsItem(item, remnants)
|
||||
if err != nil {
|
||||
logger.Warn().Err(err).Any("item", item).Msg("unable to make goods item")
|
||||
continue
|
||||
}
|
||||
|
||||
goodsItems = append(goodsItems, goodsItem)
|
||||
|
||||
if goodsItem.Type == "" {
|
||||
continue
|
||||
}
|
||||
|
||||
if _, ok := knownCategories[goodsItem.Type]; ok {
|
||||
continue
|
||||
}
|
||||
|
||||
category, err := repository.Category().Create(ctx, goodsItem.Type)
|
||||
if err != nil {
|
||||
return fmt.Errorf("creating category: %w", err)
|
||||
}
|
||||
|
||||
logger.Debug().
|
||||
Str("name", category.Name).
|
||||
Int64("id", category.ID).
|
||||
Msg("created new category")
|
||||
|
||||
knownCategories[goodsItem.Type] = struct{}{}
|
||||
}
|
||||
|
||||
_, err = repository.GoodsItem().UpsertMany(ctx, goodsItems...)
|
||||
if err != nil {
|
||||
return fmt.Errorf("upserting items: %w", err)
|
||||
}
|
||||
|
||||
progressFloat := float64(start) / float64(total)
|
||||
progress := big.NewFloat(progressFloat).Text('f', 3)
|
||||
|
||||
elapsed := time.Since(startFrom).Seconds()
|
||||
var left int
|
||||
if progressFloat != 0 {
|
||||
left = int(((1 - progressFloat) / progressFloat) * elapsed)
|
||||
}
|
||||
|
||||
logger.Debug().
|
||||
Int("from", start).
|
||||
Int("to", start+batchSize).
|
||||
Int("total", total).
|
||||
Str("progress", progress).
|
||||
Int("seconds_left", left).
|
||||
Msg("handled next batch items")
|
||||
|
||||
if len(items) < batchSize {
|
||||
break
|
||||
}
|
||||
|
||||
start += batchSize
|
||||
i++
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func goodsItemAsOffer(in entity.GoodsItem, categoryIDByName map[string]int64) (out export.Offer) {
|
||||
const defaultType = "vendor.model"
|
||||
const defaultCurrency = "RUR"
|
||||
const defaultAvailable = true
|
||||
const quantityParamName = "Количество на складе «Москва»"
|
||||
const basePictureURL = "https://eway.elevel.ru"
|
||||
|
||||
imgurl := func(path string) string {
|
||||
return basePictureURL + path
|
||||
}
|
||||
|
||||
categoryID := categoryIDByName[in.Type]
|
||||
|
||||
out = export.Offer{
|
||||
ID: in.Cart,
|
||||
VendorCode: in.Articul,
|
||||
Price: int(in.TariffPrice),
|
||||
CategoryID: categoryID,
|
||||
PictureURLs: []string{
|
||||
in.Photo,
|
||||
imgurl(in.Photo),
|
||||
},
|
||||
|
||||
Model: in.Name,
|
||||
|
||||
Reference in New Issue
Block a user