add export as yml_catalog
This commit is contained in:
Aleksandr Trushkin
653
cmd/cli/main.go
Normal file
653
cmd/cli/main.go
Normal file
@ -0,0 +1,653 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"context"
|
||||
"crypto/rand"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"encoding/xml"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"os/signal"
|
||||
"strconv"
|
||||
"time"
|
||||
|
||||
"git.loyso.art/frx/eway/cmd/cli/components"
|
||||
"git.loyso.art/frx/eway/internal/encoding/fbs"
|
||||
"git.loyso.art/frx/eway/internal/entity"
|
||||
"git.loyso.art/frx/eway/internal/export"
|
||||
|
||||
"github.com/brianvoe/gofakeit/v6"
|
||||
"github.com/rodaine/table"
|
||||
"github.com/rs/zerolog"
|
||||
"github.com/urfave/cli"
|
||||
)
|
||||
|
||||
func main() {
|
||||
ctx, cancel := signal.NotifyContext(context.Background(), os.Interrupt)
|
||||
defer func() {
|
||||
cancel()
|
||||
}()
|
||||
|
||||
err := runcli(ctx)
|
||||
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "unable to handle app: %v", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
os.Exit(0)
|
||||
}
|
||||
|
||||
func runcli(ctx context.Context) (err error) {
|
||||
app := setupCLI(ctx)
|
||||
return app.Run(os.Args)
|
||||
}
|
||||
|
||||
func setupDI(ctx context.Context) cli.BeforeFunc {
|
||||
return func(c *cli.Context) error {
|
||||
cfgpath := c.String("config")
|
||||
|
||||
err := components.SetupDI(ctx, cfgpath)
|
||||
if err != nil {
|
||||
return fmt.Errorf("setting up di: %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
func releaseDI(c *cli.Context) error {
|
||||
log, err := components.GetLogger()
|
||||
if err != nil {
|
||||
return fmt.Errorf("getting logger: %w", err)
|
||||
}
|
||||
|
||||
log.Info().Msg("shutting down env")
|
||||
start := time.Now()
|
||||
defer func() {
|
||||
since := time.Since(start)
|
||||
log.Err(err).Dur("elapsed", since).Msg("shutdown finished")
|
||||
}()
|
||||
|
||||
return components.Shutdown()
|
||||
}
|
||||
|
||||
func setupCLI(ctx context.Context) *cli.App {
|
||||
app := cli.NewApp()
|
||||
app.Flags = append(
|
||||
app.Flags,
|
||||
cli.StringFlag{
|
||||
Name: "config",
|
||||
Usage: "path to config in TOML format",
|
||||
Value: "config.toml",
|
||||
TakesFile: true,
|
||||
},
|
||||
)
|
||||
|
||||
app.Before = setupDI(ctx)
|
||||
app.After = releaseDI
|
||||
app.Commands = cli.Commands{
|
||||
newImportCmd(ctx),
|
||||
newExportCmd(ctx),
|
||||
newViewCmd(ctx),
|
||||
}
|
||||
|
||||
return app
|
||||
}
|
||||
|
||||
func newImportCmd(ctx context.Context) cli.Command {
|
||||
return cli.Command{
|
||||
Name: "import",
|
||||
Usage: "category for importing data from sources",
|
||||
Subcommands: cli.Commands{
|
||||
newImportFromFileCmd(ctx),
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func newImportFromFileCmd(ctx context.Context) cli.Command {
|
||||
return cli.Command{
|
||||
Name: "fromfile",
|
||||
Usage: "imports from file into db",
|
||||
Flags: []cli.Flag{
|
||||
&cli.StringFlag{
|
||||
Name: "src",
|
||||
Value: "",
|
||||
Usage: "source of the data.",
|
||||
Required: true,
|
||||
},
|
||||
},
|
||||
Action: decorateAction(ctx, importFromFileAction),
|
||||
}
|
||||
}
|
||||
|
||||
func newExportCmd(ctx context.Context) cli.Command {
|
||||
return cli.Command{
|
||||
Name: "export",
|
||||
Usage: "category for exporting stored data",
|
||||
Subcommands: cli.Commands{
|
||||
newExportYMLCatalogCmd(ctx),
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func newExportYMLCatalogCmd(ctx context.Context) cli.Command {
|
||||
return cli.Command{
|
||||
Name: "yml_catalog",
|
||||
Usage: "export data into as yml_catalog in xml format",
|
||||
Flags: []cli.Flag{
|
||||
&cli.StringFlag{
|
||||
Name: "dst",
|
||||
Usage: "destination path",
|
||||
Required: true,
|
||||
Value: "yml_catalog.xml",
|
||||
TakesFile: true,
|
||||
},
|
||||
&cli.IntFlag{
|
||||
Name: "limit",
|
||||
Usage: "limits output offers to required value",
|
||||
},
|
||||
&cli.BoolFlag{
|
||||
Name: "pretty",
|
||||
Usage: "prettify output",
|
||||
},
|
||||
},
|
||||
Action: decorateAction(ctx, exportYMLCatalogAction),
|
||||
}
|
||||
}
|
||||
|
||||
func newTestCmd(ctx context.Context) cli.Command {
|
||||
return cli.Command{
|
||||
Name: "test",
|
||||
Usage: "various commands for testing",
|
||||
Subcommands: cli.Commands{
|
||||
newTestFBSCmd(ctx),
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func newTestFBSCmd(ctx context.Context) cli.Command {
|
||||
return cli.Command{
|
||||
Name: "fbs",
|
||||
Usage: "serialize and deserialize gooditem entity",
|
||||
Action: decorateAction(ctx, testFBSAction),
|
||||
}
|
||||
}
|
||||
|
||||
func newViewCmd(ctx context.Context) cli.Command {
|
||||
return cli.Command{
|
||||
Name: "view",
|
||||
Usage: "Set of commands to view the data inside db",
|
||||
Subcommands: []cli.Command{
|
||||
newViewCategoriesCmd(ctx),
|
||||
newViewItemsCmd(ctx),
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func newViewCategoriesCmd(ctx context.Context) cli.Command {
|
||||
return cli.Command{
|
||||
Name: "categories",
|
||||
Usage: "Set of commands to work with categories",
|
||||
Subcommands: []cli.Command{
|
||||
newViewCategoriesListCmd(ctx),
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func newViewItemsCmd(ctx context.Context) cli.Command {
|
||||
return cli.Command{
|
||||
Name: "items",
|
||||
Usage: "Set of command to work with items",
|
||||
Subcommands: cli.Commands{
|
||||
newViewItemsGetCmd(ctx),
|
||||
newViewItemsCountCmd(ctx),
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func newViewItemsCountCmd(ctx context.Context) cli.Command {
|
||||
return cli.Command{
|
||||
Name: "count",
|
||||
Usage: "iterates over collection and counts number of items",
|
||||
Action: decorateAction(ctx, viewItemsCountAction),
|
||||
}
|
||||
}
|
||||
|
||||
func newViewItemsGetCmd(ctx context.Context) cli.Command {
|
||||
return cli.Command{
|
||||
Name: "get",
|
||||
Usage: "gets goods item by its id",
|
||||
Flags: []cli.Flag{
|
||||
&cli.StringFlag{
|
||||
Name: "id",
|
||||
Usage: "id of the goods item. Either id or cart-id should be set",
|
||||
},
|
||||
&cli.Int64Flag{
|
||||
Name: "cart-id",
|
||||
Usage: "cart-id of the item. Either cart-id or id should be set",
|
||||
},
|
||||
},
|
||||
Action: decorateAction(ctx, viewItemsGetAction),
|
||||
}
|
||||
}
|
||||
|
||||
func newViewCategoriesListCmd(ctx context.Context) cli.Command {
|
||||
return cli.Command{
|
||||
Name: "list",
|
||||
Usage: "lists stored categories stored in database",
|
||||
Flags: []cli.Flag{
|
||||
&cli.IntFlag{
|
||||
Name: "limit",
|
||||
Usage: "limits output to selected items",
|
||||
Value: 20,
|
||||
},
|
||||
&cli.IntFlag{
|
||||
Name: "page",
|
||||
Usage: "in case of limit, selects page",
|
||||
Value: 0,
|
||||
},
|
||||
&cli.BoolFlag{
|
||||
Name: "with-total",
|
||||
Usage: "prints total count of categories",
|
||||
},
|
||||
},
|
||||
Action: decorateAction(ctx, viewCategoriesListAction),
|
||||
}
|
||||
}
|
||||
|
||||
func viewItemsGetAction(ctx context.Context, c *cli.Context) error {
|
||||
r, err := components.GetRepository()
|
||||
if err != nil {
|
||||
return fmt.Errorf("getting repository: %w", err)
|
||||
}
|
||||
|
||||
id := c.String("id")
|
||||
cartID := c.Int64("cart-id")
|
||||
if id == "" && cartID == 0 {
|
||||
return cli.NewExitError("oneof: id or cart-id should be set", 1)
|
||||
} else if id != "" && cartID != 0 {
|
||||
return cli.NewExitError("oneof: id or cart-id should be set", 1)
|
||||
}
|
||||
|
||||
var item entity.GoodsItem
|
||||
if id != "" {
|
||||
item, err = r.GoodsItem().Get(ctx, id)
|
||||
} else {
|
||||
item, err = r.GoodsItem().GetByCart(ctx, cartID)
|
||||
}
|
||||
if err != nil {
|
||||
return fmt.Errorf("getting item: %w", err)
|
||||
}
|
||||
|
||||
enc := json.NewEncoder(os.Stdout)
|
||||
enc.SetIndent("", " ")
|
||||
|
||||
err = enc.Encode(item)
|
||||
if err != nil {
|
||||
return fmt.Errorf("encoding item: %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func viewItemsCountAction(ctx context.Context, c *cli.Context) error {
|
||||
r, err := components.GetRepository()
|
||||
if err != nil {
|
||||
return fmt.Errorf("getting repository: %w", err)
|
||||
}
|
||||
|
||||
itemChan, err := r.GoodsItem().ListIter(ctx, 10)
|
||||
if err != nil {
|
||||
if !errors.Is(err, entity.ErrNotImplemented) {
|
||||
return fmt.Errorf("getting list iter: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
var count int
|
||||
if err == nil {
|
||||
var done bool
|
||||
for !done {
|
||||
select {
|
||||
case _, ok := <-itemChan:
|
||||
if !ok {
|
||||
done = true
|
||||
continue
|
||||
}
|
||||
count++
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
items, err := r.GoodsItem().List(ctx)
|
||||
if err != nil {
|
||||
return fmt.Errorf("getting list: %w", err)
|
||||
}
|
||||
|
||||
zerolog.Ctx(ctx).Info().Int("count", count).Int("list_count", len(items)).Msg("read all items")
|
||||
return nil
|
||||
}
|
||||
|
||||
func viewCategoriesListAction(ctx context.Context, c *cli.Context) error {
|
||||
r, err := components.GetRepository()
|
||||
if err != nil {
|
||||
return fmt.Errorf("getting repository: %w", err)
|
||||
}
|
||||
|
||||
categories, err := r.Category().List(ctx)
|
||||
if err != nil {
|
||||
return fmt.Errorf("listing categories: %w", err)
|
||||
}
|
||||
|
||||
limit := c.Int("limit")
|
||||
page := c.Int("page")
|
||||
total := len(categories)
|
||||
|
||||
if page == 0 {
|
||||
page = 1
|
||||
}
|
||||
|
||||
if limit > 0 {
|
||||
offset := (page - 1) * limit
|
||||
if offset > len(categories) {
|
||||
offset = len(categories) - 1
|
||||
}
|
||||
|
||||
limit = offset + limit
|
||||
if limit > len(categories) {
|
||||
limit = len(categories)
|
||||
}
|
||||
|
||||
categories = categories[offset:limit]
|
||||
}
|
||||
|
||||
tbl := table.New("ID", "Name")
|
||||
for _, category := range categories {
|
||||
if category.ID == 0 && category.Name == "" {
|
||||
continue
|
||||
}
|
||||
tbl.AddRow(category.ID, category.Name)
|
||||
}
|
||||
|
||||
tbl.Print()
|
||||
|
||||
if c.Bool("with-total") {
|
||||
zerolog.Ctx(ctx).Info().Int("count", total).Msg("total categories stats")
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func importFromFileAction(ctx context.Context, c *cli.Context) error {
|
||||
const maxBatch = 2000
|
||||
|
||||
r, err := components.GetRepository()
|
||||
if err != nil {
|
||||
return fmt.Errorf("getting repository: %w", err)
|
||||
}
|
||||
|
||||
filesrc := c.String("src")
|
||||
log := zerolog.Ctx(ctx)
|
||||
|
||||
log.Debug().Str("filepath", filesrc).Msg("importing data from file")
|
||||
|
||||
productsFile, err := os.Open(filesrc)
|
||||
if err != nil {
|
||||
return fmt.Errorf("opening file: %w", err)
|
||||
}
|
||||
|
||||
defer func() {
|
||||
errClose := productsFile.Close()
|
||||
if errClose != nil {
|
||||
log.Warn().Err(errClose).Msg("unable to close file")
|
||||
}
|
||||
}()
|
||||
|
||||
var (
|
||||
goodsItem entity.GoodsItem
|
||||
goodsItems []entity.GoodsItem
|
||||
failedToInsert int
|
||||
)
|
||||
|
||||
seenCategories := make(map[string]struct{})
|
||||
categories, err := r.Category().List(ctx)
|
||||
if err != nil {
|
||||
return fmt.Errorf("listing categories: %w", err)
|
||||
}
|
||||
|
||||
for _, category := range categories {
|
||||
seenCategories[category.Name] = struct{}{}
|
||||
}
|
||||
|
||||
bfile := bufio.NewReader(productsFile)
|
||||
for {
|
||||
line, _, err := bfile.ReadLine()
|
||||
if err != nil {
|
||||
if errors.Is(err, io.EOF) {
|
||||
break
|
||||
|
||||
}
|
||||
return fmt.Errorf("reading line: %w", err)
|
||||
}
|
||||
|
||||
err = json.Unmarshal(line, &goodsItem)
|
||||
if err != nil {
|
||||
log.Warn().Err(err).Str("line", string(line)).Msg("unable to unmarshal line into item")
|
||||
failedToInsert++
|
||||
continue
|
||||
}
|
||||
|
||||
goodsItems = append(goodsItems, goodsItem)
|
||||
|
||||
if _, ok := seenCategories[goodsItem.Type]; ok {
|
||||
continue
|
||||
}
|
||||
|
||||
_, err = r.Category().Create(ctx, goodsItem.Type)
|
||||
if err != nil {
|
||||
return fmt.Errorf("unable to create new category: %w", err)
|
||||
}
|
||||
log.Debug().Any("category", goodsItem.Type).Msg("inserted new category")
|
||||
seenCategories[goodsItem.Type] = struct{}{}
|
||||
}
|
||||
|
||||
log.Debug().Int("count", len(goodsItems)).Int("failed", failedToInsert).Msg("preparing to upload")
|
||||
|
||||
start := time.Now()
|
||||
batchSize := int(maxBatch)
|
||||
for i := 0; i < len(goodsItems); i += batchSize {
|
||||
to := i + batchSize
|
||||
if to > len(goodsItems) {
|
||||
to = len(goodsItems)
|
||||
}
|
||||
|
||||
_, err = r.GoodsItem().UpsertMany(ctx, goodsItems[i:to]...)
|
||||
if err != nil {
|
||||
return fmt.Errorf("upserting items: %w", err)
|
||||
}
|
||||
log.Debug().Int("count", to-i).Msg("inserted batch")
|
||||
}
|
||||
log.Debug().Dur("elapsed", time.Since(start)).Msg("upload finished")
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
type action func(ctx context.Context, c *cli.Context) error
|
||||
|
||||
func decorateAction(ctx context.Context, a action) cli.ActionFunc {
|
||||
return func(c *cli.Context) error {
|
||||
var data [3]byte
|
||||
_, _ = rand.Read(data[:])
|
||||
reqid := hex.EncodeToString(data[:])
|
||||
|
||||
log, err := components.GetLogger()
|
||||
if err != nil {
|
||||
return fmt.Errorf("getting logger: %w", err)
|
||||
}
|
||||
|
||||
log = log.With().Str("reqid", reqid).Logger()
|
||||
ctx = log.WithContext(ctx)
|
||||
|
||||
start := time.Now()
|
||||
defer func() {
|
||||
log.Info().Dur("elapsed", time.Since(start)).Msg("command completed")
|
||||
}()
|
||||
|
||||
log.Info().Msg("command execution started")
|
||||
return a(ctx, c)
|
||||
}
|
||||
}
|
||||
|
||||
func exportYMLCatalogAction(ctx context.Context, c *cli.Context) error {
|
||||
path := c.String("dst")
|
||||
limit := c.Int("limit")
|
||||
pretty := c.Bool("pretty")
|
||||
|
||||
log, err := components.GetLogger()
|
||||
if err != nil {
|
||||
return fmt.Errorf("getting logger: %w", err)
|
||||
}
|
||||
|
||||
r, err := components.GetRepository()
|
||||
if err != nil {
|
||||
return fmt.Errorf("getting repository: %w", err)
|
||||
}
|
||||
|
||||
categories, err := r.Category().List(ctx)
|
||||
if err != nil {
|
||||
return fmt.Errorf("listing categories: %w", err)
|
||||
}
|
||||
|
||||
shop := export.Shop{
|
||||
Currencies: []export.Currency{{
|
||||
ID: "RUR",
|
||||
Rate: 1,
|
||||
}},
|
||||
Categories: make([]export.Category, 0, len(categories)),
|
||||
}
|
||||
|
||||
categoryByNameIdx := make(map[string]int64, len(categories))
|
||||
for _, category := range categories {
|
||||
categoryByNameIdx[category.Name] = category.ID
|
||||
shop.Categories = append(shop.Categories, export.Category{
|
||||
ID: category.ID,
|
||||
Name: category.Name,
|
||||
})
|
||||
}
|
||||
|
||||
itemsIter, err := r.GoodsItem().ListIter(ctx, 1)
|
||||
if err != nil {
|
||||
return fmt.Errorf("getting items iterator: %w", err)
|
||||
}
|
||||
|
||||
for item := range itemsIter {
|
||||
offer := goodsItemAsOffer(item, categoryByNameIdx)
|
||||
shop.Offers = append(shop.Offers, offer)
|
||||
}
|
||||
if err = ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
f, err := os.Create(path)
|
||||
if err != nil {
|
||||
return fmt.Errorf("creating file: %w", err)
|
||||
}
|
||||
defer func() {
|
||||
errClose := f.Close()
|
||||
if err == nil {
|
||||
err = errClose
|
||||
return
|
||||
}
|
||||
|
||||
log.Err(errClose).Msg("file closed or not")
|
||||
}()
|
||||
|
||||
if limit > 0 {
|
||||
shop.Offers = shop.Offers[:limit]
|
||||
}
|
||||
|
||||
container := export.YmlContainer{
|
||||
YmlCatalog: export.YmlCatalog{
|
||||
Shop: shop,
|
||||
Date: time.Now(),
|
||||
},
|
||||
}
|
||||
|
||||
_, err = f.Write([]byte(xml.Header))
|
||||
if err != nil {
|
||||
return fmt.Errorf("writing header: %w", err)
|
||||
}
|
||||
_, err = f.Write([]byte("<!DOCTYPE yml_catalog SYSTEM \"shops.dtd\">\n"))
|
||||
enc := xml.NewEncoder(f)
|
||||
if pretty {
|
||||
enc.Indent("", " ")
|
||||
}
|
||||
return enc.Encode(container)
|
||||
}
|
||||
|
||||
func goodsItemAsOffer(in entity.GoodsItem, categoryIDByName map[string]int64) (out export.Offer) {
|
||||
const defaultType = "vendor.model"
|
||||
const defaultCurrency = "RUR"
|
||||
const defaultAvailable = true
|
||||
const quantityParamName = "Количество на складе «Москва»"
|
||||
|
||||
categoryID := categoryIDByName[in.Type]
|
||||
|
||||
out = export.Offer{
|
||||
ID: in.Cart,
|
||||
Price: int(in.TariffPrice),
|
||||
CategoryID: categoryID,
|
||||
PictureURLs: []string{
|
||||
in.Photo,
|
||||
},
|
||||
|
||||
Model: in.Name,
|
||||
Vendor: in.Producer,
|
||||
TypePrefix: in.Name,
|
||||
Description: in.Description,
|
||||
ManufacturerWarrany: true,
|
||||
Params: []export.Param{
|
||||
{
|
||||
Name: quantityParamName,
|
||||
Value: strconv.Itoa(in.Stock),
|
||||
},
|
||||
},
|
||||
|
||||
Type: defaultType,
|
||||
CurrencyID: defaultCurrency,
|
||||
Available: defaultAvailable,
|
||||
}
|
||||
|
||||
return out
|
||||
}
|
||||
|
||||
func testFBSAction(ctx context.Context, c *cli.Context) error {
|
||||
var gooditem entity.GoodsItem
|
||||
err := gofakeit.Struct(&gooditem)
|
||||
if err != nil {
|
||||
return fmt.Errorf("faking struct: %w", err)
|
||||
}
|
||||
|
||||
data := fbs.MakeDomainGoodItemFinished(gooditem)
|
||||
datahexed := hex.EncodeToString(data)
|
||||
println(datahexed)
|
||||
|
||||
got, err := fbs.ParseGoodsItem(data)
|
||||
if err != nil {
|
||||
return fmt.Errorf("parsing: %w", err)
|
||||
}
|
||||
|
||||
if got != gooditem {
|
||||
gotStr := fmt.Sprintf("%v", got)
|
||||
hasStr := fmt.Sprintf("%v", gooditem)
|
||||
println(gotStr, "\n", hasStr)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
Reference in New Issue
Block a user