eway/cmd/converter/main.go

package main

import (
	"bufio"
	"context"
	"crypto/rand"
	"encoding/hex"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"os"
	"os/signal"
	"time"

	"git.loyso.art/frx/eway/cmd/converter/components"
	"git.loyso.art/frx/eway/internal/encoding/fbs"
	"git.loyso.art/frx/eway/internal/entity"

	"github.com/rodaine/table"
	"github.com/rs/zerolog"
	"github.com/urfave/cli"
)

func main() {
	ctx, cancel := signal.NotifyContext(context.Background(), os.Interrupt)
	defer func() {
		cancel()
	}()

	err := runcli(ctx)

	if err != nil {
		fmt.Fprintf(os.Stderr, "unable to handle app: %v", err)
		os.Exit(1)
	}

	os.Exit(0)
}

func runcli(ctx context.Context) (err error) {
	app := setupCLI(ctx)
	return app.Run(os.Args)
}

func setupDI(ctx context.Context) cli.BeforeFunc {
	return func(c *cli.Context) error {
		cfgpath := c.String("config")

		err := components.SetupDI(ctx, cfgpath)
		if err != nil {
			return fmt.Errorf("setting up di: %w", err)
		}

		return nil
	}
}

func releaseDI(c *cli.Context) error {
	log, err := components.GetLogger()
	if err != nil {
		return fmt.Errorf("getting logger: %w", err)
	}

	log.Info().Msg("shutting down env")
	start := time.Now()
	defer func() {
		since := time.Since(start)
		log.Err(err).Dur("elapsed", since).Msg("shutdown finished")
	}()

	return components.Shutdown()
}

func setupCLI(ctx context.Context) *cli.App {
	app := cli.NewApp()
	app.Flags = append(
		app.Flags,
		cli.StringFlag{
			Name:      "config",
			Usage:     "path to config in TOML format",
			Value:     "config.toml",
			TakesFile: true,
		},
	)

	app.Before = setupDI(ctx)
	app.After = releaseDI
	app.Commands = cli.Commands{
		newImportCmd(ctx),
		newViewCmd(ctx),
		cli.Command{
			Name:  "test-fbs",
			Usage: "a simple check for tbs",
			Action: cli.ActionFunc(func(c *cli.Context) error {
				gooditem := entity.GoodsItem{
					Articul:     "some-sku",
					Photo:       "/photo/path.jpg",
					Name:        "some-name",
					Description: "bad-desc",
					Category:    "",
					Type:        "some-type",
					Producer:    "my-producer",
					Pack:        123,
					Step:        10,
					Price:       12.34,
					TariffPrice: 43.21,
					Cart:        1998,
					Stock:       444,
				}

				data := fbs.MakeDomainGoodItemFinished(gooditem)
				datahexed := hex.EncodeToString(data)
				println(datahexed)

				got, err := fbs.ParseGoodsItem(data)
				if err != nil {
					return fmt.Errorf("parsing: %w", err)
				}

				if got != gooditem {
					gotStr := fmt.Sprintf("%v", got)
					hasStr := fmt.Sprintf("%v", gooditem)
					println(gotStr, "\n", hasStr)
				}

				return nil
			}),
		},
	}

	return app
}

func newImportCmd(ctx context.Context) cli.Command {
	return cli.Command{
		Name:  "import",
		Usage: "category for importing data from sources",
		Subcommands: cli.Commands{
			newImportFromFileCmd(ctx),
		},
	}
}

func newViewCmd(ctx context.Context) cli.Command {
	return cli.Command{
		Name:  "view",
		Usage: "Set of commands to view the data inside db",
		Subcommands: []cli.Command{
			newViewCategoriesCmd(ctx),
			newViewItemsCmd(ctx),
		},
	}
}

func newViewCategoriesCmd(ctx context.Context) cli.Command {
	return cli.Command{
		Name:  "categories",
		Usage: "Set of commands to work with categories",
		Subcommands: []cli.Command{
			newViewCategoriesListCmd(ctx),
		},
	}
}

func newViewItemsCmd(ctx context.Context) cli.Command {
	return cli.Command{
		Name:  "items",
		Usage: "Set of command to work with items",
		Subcommands: cli.Commands{
			newViewItemsGetCmd(ctx),
			newViewItemsCountCmd(ctx),
		},
	}
}

func newViewItemsCountCmd(ctx context.Context) cli.Command {
	return cli.Command{
		Name:   "count",
		Usage:  "iterates over collection and counts number of items",
		Action: decorateAction(ctx, viewItemsCountAction),
	}
}

func newViewItemsGetCmd(ctx context.Context) cli.Command {
	return cli.Command{
		Name:  "get",
		Usage: "gets goods item by its id",
		Flags: []cli.Flag{
			&cli.StringFlag{
				Name:  "id",
				Usage: "id of the goods item. Either id or cart-id should be set",
			},
			&cli.Int64Flag{
				Name:  "cart-id",
				Usage: "cart-id of the item. Either cart-id or id should be set",
			},
		},
		Action: decorateAction(ctx, viewItemsGetAction),
	}
}

func newViewCategoriesListCmd(ctx context.Context) cli.Command {
	return cli.Command{
		Name:  "list",
		Usage: "lists stored categories stored in database",
		Flags: []cli.Flag{
			&cli.IntFlag{
				Name:  "limit",
				Usage: "limits output to selected items",
				Value: 20,
			},
			&cli.IntFlag{
				Name:  "page",
				Usage: "in case of limit, selects page",
				Value: 0,
			},
			&cli.BoolFlag{
				Name:  "with-total",
				Usage: "prints total count of categories",
			},
		},
		Action: decorateAction(ctx, viewCategoriesListAction),
	}
}

func newImportFromFileCmd(ctx context.Context) cli.Command {
	return cli.Command{
		Name:  "fromfile",
		Usage: "imports from file into db",
		Flags: []cli.Flag{
			&cli.StringFlag{
				Name:     "src",
				Value:    "",
				Usage:    "source of the data.",
				Required: true,
			},
		},
		Action: decorateAction(ctx, importFromFileAction),
	}
}

func viewItemsGetAction(ctx context.Context, c *cli.Context) error {
	r, err := components.GetRepository()
	if err != nil {
		return fmt.Errorf("getting repository: %w", err)
	}

	id := c.String("id")
	cartID := c.Int64("cart-id")
	if id == "" && cartID == 0 {
		return cli.NewExitError("oneof: id or cart-id should be set", 1)
	} else if id != "" && cartID != 0 {
		return cli.NewExitError("oneof: id or cart-id should be set", 1)
	}

	var item entity.GoodsItem
	if id != "" {
		item, err = r.GoodsItem().Get(ctx, id)
	} else {
		item, err = r.GoodsItem().GetByCart(ctx, cartID)
	}
	if err != nil {
		return fmt.Errorf("getting item: %w", err)
	}

	enc := json.NewEncoder(os.Stdout)
	enc.SetIndent("", " ")

	err = enc.Encode(item)
	if err != nil {
		return fmt.Errorf("encoding item: %w", err)
	}

	return nil
}

func viewItemsCountAction(ctx context.Context, c *cli.Context) error {
	r, err := components.GetRepository()
	if err != nil {
		return fmt.Errorf("getting repository: %w", err)
	}

	itemChan, err := r.GoodsItem().ListIter(ctx, 10)
	if err != nil {
		if !errors.Is(err, entity.ErrNotImplemented) {
			return fmt.Errorf("getting list iter: %w", err)
		}
	}

	var count int
	if err == nil {
		var done bool
		for !done {
			select {
			case _, ok := <-itemChan:
				if !ok {
					done = true
					continue
				}
				count++
			case <-ctx.Done():
				return ctx.Err()
			}
		}
	}

	items, err := r.GoodsItem().List(ctx)
	if err != nil {
		return fmt.Errorf("getting list: %w", err)
	}

	zerolog.Ctx(ctx).Info().Int("count", count).Int("list_count", len(items)).Msg("read all items")
	return nil
}

func viewCategoriesListAction(ctx context.Context, c *cli.Context) error {
	r, err := components.GetRepository()
	if err != nil {
		return fmt.Errorf("getting repository: %w", err)
	}

	categories, err := r.Category().List(ctx)
	if err != nil {
		return fmt.Errorf("listing categories: %w", err)
	}

	limit := c.Int("limit")
	page := c.Int("page")
	total := len(categories)

	if page == 0 {
		page = 1
	}

	if limit > 0 {
		offset := (page - 1) * limit
		if offset > len(categories) {
			offset = len(categories) - 1
		}

		limit = offset + limit
		if limit > len(categories) {
			limit = len(categories)
		}

		categories = categories[offset:limit]
	}

	tbl := table.New("ID", "Name")
	for _, category := range categories {
		if category.ID == 0 && category.Name == "" {
			continue
		}
		tbl.AddRow(category.ID, category.Name)
	}

	tbl.Print()

	if c.Bool("with-total") {
		zerolog.Ctx(ctx).Info().Int("count", total).Msg("total categories stats")
	}

	return nil
}

func importFromFileAction(ctx context.Context, c *cli.Context) error {
	const maxBatch = 2000

	r, err := components.GetRepository()
	if err != nil {
		return fmt.Errorf("getting repository: %w", err)
	}

	filesrc := c.String("src")
	log := zerolog.Ctx(ctx)

	log.Debug().Str("filepath", filesrc).Msg("importing data from file")

	productsFile, err := os.Open(filesrc)
	if err != nil {
		return fmt.Errorf("opening file: %w", err)
	}

	defer func() {
		errClose := productsFile.Close()
		if errClose != nil {
			log.Warn().Err(errClose).Msg("unable to close file")
		}
	}()

	var (
		goodsItem      entity.GoodsItem
		goodsItems     []entity.GoodsItem
		failedToInsert int
	)

	seenCategories := make(map[string]struct{})
	categories, err := r.Category().List(ctx)
	if err != nil {
		return fmt.Errorf("listing categories: %w", err)
	}

	for _, category := range categories {
		seenCategories[category.Name] = struct{}{}
	}

	bfile := bufio.NewReader(productsFile)
	for {
		line, _, err := bfile.ReadLine()
		if err != nil {
			if errors.Is(err, io.EOF) {
				break

			}
			return fmt.Errorf("reading line: %w", err)
		}

		err = json.Unmarshal(line, &goodsItem)
		if err != nil {
			log.Warn().Err(err).Str("line", string(line)).Msg("unable to unmarshal line into item")
			failedToInsert++
			continue
		}

		goodsItems = append(goodsItems, goodsItem)

		if _, ok := seenCategories[goodsItem.Type]; ok {
			continue
		}

		_, err = r.Category().Create(ctx, goodsItem.Type)
		if err != nil {
			return fmt.Errorf("unable to create new category: %w", err)
		}
		log.Debug().Any("category", goodsItem.Type).Msg("inserted new category")
		seenCategories[goodsItem.Type] = struct{}{}
	}

	log.Debug().Int("count", len(goodsItems)).Int("failed", failedToInsert).Msg("preparing to upload")

	start := time.Now()
	batchSize := int(maxBatch)
	for i := 0; i < len(goodsItems); i += batchSize {
		to := i + batchSize
		if to > len(goodsItems) {
			to = len(goodsItems)
		}

		_, err = r.GoodsItem().UpsertMany(ctx, goodsItems[i:to]...)
		if err != nil {
			return fmt.Errorf("upserting items: %w", err)
		}
		log.Debug().Int("count", to-i).Msg("inserted batch")
	}
	log.Debug().Dur("elapsed", time.Since(start)).Msg("upload finished")

	return nil
}

type action func(ctx context.Context, c *cli.Context) error

func decorateAction(ctx context.Context, a action) cli.ActionFunc {
	return func(c *cli.Context) error {
		var data [3]byte
		_, _ = rand.Read(data[:])
		reqid := hex.EncodeToString(data[:])

		log, err := components.GetLogger()
		if err != nil {
			return fmt.Errorf("getting logger: %w", err)
		}

		log = log.With().Str("reqid", reqid).Logger()
		ctx = log.WithContext(ctx)

		start := time.Now()
		defer func() {
			log.Info().Dur("elapsed", time.Since(start)).Msg("command completed")
		}()

		log.Info().Msg("command execution started")
		return a(ctx, c)
	}
}