Compare commits

...

32 Commits

Author SHA1 Message Date
c8243d25bf format another way 2024-02-07 17:45:00 +03:00
6a746b518f add pattern matching 2024-02-07 16:33:44 +03:00
838d25d878 add analytics cli commands 2024-02-07 12:24:02 +03:00
988b22a08f minor imporvments 2024-02-06 19:58:57 +03:00
ccc668105a use goquery instead of colly 2024-02-06 19:14:55 +03:00
5c8f52724a minor fixes 2024-02-06 17:19:36 +03:00
096c7e8157 add workers pool 2024-02-05 11:05:37 +03:00
362d4524e3 add sema and retries 2024-02-04 22:48:20 +03:00
e474c69aac organize imports 2024-02-04 22:14:36 +03:00
3c68c0b2fd allow to skip item by created 2024-02-04 22:13:39 +03:00
15d4cdb047 fix build 2024-02-04 22:00:19 +03:00
4533b90e4a fix parameter unmarshal 2024-02-04 21:57:25 +03:00
60522a7391 minor fixes 2024-02-04 21:43:38 +03:00
5d5e152429 cache and stats 2024-02-04 21:40:37 +03:00
08be7de118 actualize items list in db 2024-02-04 13:14:56 +03:00
6042ca6822 support parsing item page info 2024-02-03 20:28:33 +03:00
b0a185561d queue api 2024-02-03 18:15:26 +03:00
fae5a84182 another update runs-on 2024-02-02 19:02:57 +03:00
bbe1271620 update runs-on 2024-02-02 19:02:25 +03:00
ff4f3b1d4c try? 2024-02-02 18:57:36 +03:00
a14212702c some other try 2024-02-02 18:56:47 +03:00
cc48e97d40 update runner label 2024-02-02 18:50:31 +03:00
bd135dd3a5 update runner label 2024-02-02 18:36:25 +03:00
1767349a08 update runner 2024-02-02 18:33:15 +03:00
38e04d58c2 update runner 2024-02-02 18:32:19 +03:00
91b0dd5c4f enable gitea actions 2024-02-02 18:24:00 +03:00
3754441492 fix cli arg 2024-01-29 13:21:56 +03:00
9ef5f2bbf8 add encryption and get page 2024-01-29 00:09:30 +03:00
c814f27c54 migrate to cli v3 2024-01-28 17:08:45 +03:00
bfa105df95 able to parse xml 2024-01-28 16:49:48 +03:00
90a7797a27 add export as yml_catalog 2024-01-26 19:34:47 +03:00
6fe250896c fix saving and project improvments 2024-01-25 16:42:08 +03:00
36 changed files with 3179 additions and 694 deletions

View File

@ -0,0 +1,17 @@
name: Gitea Actions Demo
run-name: ${{ gitea.actor }} is testing out Gitea Actions 🚀
on: [push]
jobs:
Explore-Gitea-Actions:
runs-on: linux-arm64
steps:
- run: echo "The job was automatically triggered by a ${{ gitea.event_name }} event."
- run: echo "This job is now running on a ${{ runner.os }} server hosted by Gitea!"
- run: echo "The name of your branch is ${{ gitea.ref }} and your repository is ${{ gitea.repository }}."
- run: echo "The ${{ gitea.repository }} repository has been cloned to the runner."
- run: echo "The workflow is now ready to test your code on the runner."
- name: List files in the repository
run: |
ls ${{ gitea.workspace }}
- run: echo "This job's status is ${{ job.status }}."

4
.gitignore vendored
View File

@ -1,2 +1,6 @@
*.json *.json
*.zst *.zst
database
bin
*.xml
Makefile

View File

@ -14,6 +14,8 @@ table GoodItem {
tariff:float; tariff:float;
cart:long; cart:long;
stock:short; stock:short;
parameters:string;
created_at:long;
} }
table GoodItems { table GoodItems {

36
buildinfo.go Normal file
View File

@ -0,0 +1,36 @@
package eway
import (
"sync"
"time"
)
var (
version string = "v0.0.0"
commit string = "0000000"
buildTimeStr string
buildTime time.Time
parseOnce sync.Once
)
func Version() string {
return version
}
func Commit() string {
return commit
}
func BuildTime() time.Time {
parseOnce.Do(func() {
if buildTimeStr == "" {
return
}
buildTime, _ = time.Parse(buildTimeStr, time.RFC3339)
})
return buildTime
}

162
cmd/cli/components/di.go Normal file
View File

@ -0,0 +1,162 @@
package components
import (
"context"
"errors"
"fmt"
"io"
"os"
"time"
"git.loyso.art/frx/eway/internal/config"
"git.loyso.art/frx/eway/internal/interconnect/eway"
"git.loyso.art/frx/eway/internal/storage"
xbadger "git.loyso.art/frx/eway/internal/storage/badger"
"github.com/BurntSushi/toml"
"github.com/dgraph-io/badger/v4"
"github.com/rs/zerolog"
"github.com/samber/do"
)
// Yeah, singleton is not good UNLESS you're really lazy
var diInjector *do.Injector
func GetEwayClient() (eway.Client, error) {
return do.Invoke[eway.Client](diInjector)
}
func GetRepository() (storage.Repository, error) {
adapter, err := do.Invoke[*storageRepositoryAdapter](diInjector)
if err != nil {
return nil, err
}
return adapter.entity, nil
}
func GetLogger() (zerolog.Logger, error) {
return do.Invoke[zerolog.Logger](diInjector)
}
func SetupDI(ctx context.Context, cfgpath string, verbose bool, logAsJSON bool) error {
cfg, err := parseSettings(cfgpath)
if err != nil {
// if no settings provided allow cli to run without them.
if errors.Is(err, os.ErrNotExist) {
return nil
}
return err
}
diInjector = do.New()
do.Provide(diInjector, func(i *do.Injector) (zerolog.Logger, error) {
tsSet := func(wr *zerolog.ConsoleWriter) {
wr.TimeFormat = time.RFC3339
}
var writer io.Writer = zerolog.NewConsoleWriter(tsSet)
if logAsJSON {
writer = os.Stdout
}
log := zerolog.
New(writer).
With().
Timestamp().
Str("app", "converter").
Logger()
if verbose {
return log.Level(zerolog.DebugLevel), nil
}
return log.Level(zerolog.InfoLevel), nil
})
do.Provide[eway.Client](diInjector, func(i *do.Injector) (eway.Client, error) {
log, err := GetLogger()
if err != nil {
return nil, fmt.Errorf("getting logger: %w", err)
}
client, err := eway.New(ctx, eway.Config(cfg.Eway), log)
if err != nil {
return nil, fmt.Errorf("making new eway client: %w", err)
}
return client, nil
})
do.Provide[*badgerDBAdapter](diInjector, func(i *do.Injector) (*badgerDBAdapter, error) {
db, err := xbadger.Open(ctx, cfg.Badger.Dir, cfg.Badger.Debug, zerolog.Nop())
if err != nil {
return nil, fmt.Errorf("getting db: %w", err)
}
out := &badgerDBAdapter{entity: db}
return out, nil
})
do.Provide[*storageRepositoryAdapter](diInjector, func(i *do.Injector) (*storageRepositoryAdapter, error) {
db, err := getDB()
if err != nil {
return nil, err
}
client, err := xbadger.NewClient(db)
if err != nil {
return nil, fmt.Errorf("getting badger client: %w", err)
}
out := &storageRepositoryAdapter{entity: client}
return out, nil
})
return nil
}
func Shutdown() error {
if diInjector == nil {
return nil
}
return diInjector.Shutdown()
}
func getDB() (*badger.DB, error) {
adapter, err := do.Invoke[*badgerDBAdapter](diInjector)
if err != nil {
return nil, err
}
return adapter.entity, nil
}
type settings struct {
Badger config.Badger `toml:"badger"`
Log config.Log `toml:"log"`
Eway config.Eway `toml:"eway"`
}
func parseSettings(cfgpath string) (cfg settings, err error) {
_, err = toml.DecodeFile(cfgpath, &cfg)
if err != nil {
return cfg, fmt.Errorf("parsing file: %w", err)
}
return cfg, nil
}
type entityCloserAdapter[T io.Closer] struct {
entity T
}
func (a entityCloserAdapter[T]) Shutdown() error {
return a.entity.Close()
}
type storageRepositoryAdapter entityCloserAdapter[storage.Repository]
type badgerDBAdapter entityCloserAdapter[*badger.DB]

1428
cmd/cli/main.go Normal file

File diff suppressed because it is too large Load Diff

27
cmd/cli/main_encoff.go Normal file
View File

@ -0,0 +1,27 @@
//go:build !encon
// +build !encon
package main
import (
"context"
"github.com/urfave/cli/v3"
)
func newCryptoDecryptCmd() *cli.Command {
return &cli.Command{
Name: "decrypt",
Usage: "decrypt incoming text",
Flags: []cli.Flag{
&cli.StringFlag{
Name: "text",
Aliases: []string{"t"},
Required: true,
},
},
Action: func(context.Context, *cli.Command) error {
return cli.Exit("decryption is turned off", -1)
},
}
}

21
cmd/cli/main_encon.go Normal file
View File

@ -0,0 +1,21 @@
//go:build encon
// +build encon
package main
import "github.com/urfave/cli/v3"
func newCryptoDecryptCmd() *cli.Command {
return &cli.Command{
Name: "decrypt",
Usage: "decrypt incoming text",
Flags: []cli.Flag{
&cli.StringFlag{
Name: "text",
Aliases: []string{"t"},
Required: true,
},
},
Action: cryptoDeEncryptAction(false),
}
}

View File

@ -1,444 +0,0 @@
package main
import (
"bufio"
"context"
"crypto/rand"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
"io"
"os"
"os/signal"
"sync"
"sync/atomic"
"time"
"git.loyso.art/frx/eway/internal/config"
"git.loyso.art/frx/eway/internal/entity"
"git.loyso.art/frx/eway/internal/storage"
xbadger "git.loyso.art/frx/eway/internal/storage/badger"
badger "github.com/dgraph-io/badger/v4"
"github.com/dgraph-io/badger/v4/pb"
"github.com/rodaine/table"
"github.com/rs/zerolog"
"github.com/urfave/cli"
)
type appSettings struct {
Badger config.Badger
}
func main() {
ctx, cancel := signal.NotifyContext(context.Background(), os.Interrupt)
defer func() {
cancel()
}()
var corewg sync.WaitGroup
err := runcli(ctx, &corewg)
if err != nil {
fmt.Fprintf(os.Stderr, "unable to handle app: %v", err)
corewg.Wait()
os.Exit(1)
}
corewg.Wait()
os.Exit(0)
}
func runcli(ctx context.Context, wg *sync.WaitGroup) (err error) {
tsSet := func(wr *zerolog.ConsoleWriter) {
wr.TimeFormat = time.RFC3339
}
log := zerolog.New(zerolog.NewConsoleWriter(tsSet)).Level(zerolog.DebugLevel).With().Timestamp().Str("app", "converter").Logger()
defer func() {
log.Info().Err(err).Msg("app finished")
}()
ctx = log.WithContext(ctx)
log.Info().Msg("making badger")
db, err := xbadger.Open(ctx, "badger/", log)
if err != nil {
return fmt.Errorf("opening badger: %w", err)
}
go func() {
var item atomic.Uint64
err = db.Subscribe(
ctx,
func(kvlist *badger.KVList) error {
kvs := kvlist.GetKv()
for _, kv := range kvs {
count := item.Add(1)
log.Debug().Bytes("key", kv.GetKey()).Uint64("count", count).Msg("inspecting")
}
return nil
},
[]pb.Match{
{
Prefix: []byte("!!category!!"),
},
{
Prefix: []byte("!!goodsitem!!"),
},
},
)
log.Err(err).Msg("subscribing")
}()
maxBatch := db.MaxBatchCount()
log.Info().Int("max_batch", int(maxBatch)).Msg("max batch settings")
client, err := xbadger.NewClient(db)
if err != nil {
return fmt.Errorf("making new client: %w", err)
}
wg.Add(1)
defer func() {
defer wg.Done()
println("closing client")
errClose := client.Close()
if errClose != nil {
log.Warn().Err(errClose).Msg("unable to close client")
}
println("flushing db")
errSync := db.Sync()
if errSync != nil {
log.Warn().Err(errSync).Msg("unable to sync db")
}
// time.Sleep(time.Second * 5)
println("closing db")
errClose = db.Close()
if errClose != nil {
log.Warn().Err(errClose).Msg("unable to close db")
}
}()
app := setupCLI(ctx, client, maxBatch)
return app.Run(os.Args)
}
func setupCLI(ctx context.Context, r storage.Repository, maxBatch int64) *cli.App {
app := cli.NewApp()
app.Commands = cli.Commands{
newImportCmd(ctx, r, maxBatch),
newViewCmd(ctx, r),
}
return app
}
func newImportCmd(ctx context.Context, r storage.Repository, maxBatch int64) cli.Command {
return cli.Command{
Name: "import",
Usage: "category for importing data from sources",
Flags: []cli.Flag{
// &cli.StringFlag{
// Name: "config",
// Usage: "path to config",
// Value: "config.json",
// TakesFile: true,
// },
&cli.BoolFlag{
Name: "verbose",
Usage: "set logger to debug mode",
},
},
Before: func(c *cli.Context) error {
if c.Bool("verbose") {
zerolog.SetGlobalLevel(zerolog.DebugLevel)
}
return nil
},
Subcommands: cli.Commands{
newImportFromFileCmd(ctx, r, maxBatch),
},
}
}
func newViewCmd(ctx context.Context, r storage.Repository) cli.Command {
return cli.Command{
Name: "view",
Usage: "Set of commands to view the data inside db",
Subcommands: []cli.Command{
newViewCategoriesCmd(ctx, r),
newViewItemsCmd(ctx, r),
},
}
}
func newViewCategoriesCmd(ctx context.Context, r storage.Repository) cli.Command {
return cli.Command{
Name: "categories",
Usage: "Set of commands to work with categories",
Subcommands: []cli.Command{
newViewCategoriesListCmd(ctx, r),
},
}
}
func newViewItemsCmd(ctx context.Context, r storage.Repository) cli.Command {
return cli.Command{
Name: "items",
Usage: "Set of command to work with items",
Subcommands: cli.Commands{
newViewItemsCountCmd(ctx, r),
},
}
}
func newViewItemsCountCmd(ctx context.Context, r storage.Repository) cli.Command {
return cli.Command{
Name: "count",
Usage: "iterates over collection and counts number of items",
Action: viewItemsCount(ctx, r),
}
}
func newViewCategoriesListCmd(ctx context.Context, r storage.Repository) cli.Command {
return cli.Command{
Name: "list",
Usage: "lists stored categories stored in database",
Flags: []cli.Flag{
&cli.IntFlag{
Name: "limit",
Usage: "limits output to selected items",
Value: 20,
},
&cli.IntFlag{
Name: "page",
Usage: "in case of limit, selects page",
Value: 0,
},
&cli.BoolFlag{
Name: "with-total",
Usage: "prints total count of categories",
},
},
Action: viewCategoriesListAction(ctx, r),
}
}
func viewItemsCount(ctx context.Context, r storage.Repository) cli.ActionFunc {
f := func(c *cli.Context) error {
// itemChan, err := r.GoodsItem().ListIter(ctx, 10)
// if err != nil {
// return fmt.Errorf("getting list iter: %w", err)
// }
//
var count int
// for range itemChan {
// count++
// }
//
items, err := r.GoodsItem().List(ctx)
if err != nil {
return fmt.Errorf("getting list: %w", err)
}
zerolog.Ctx(ctx).Info().Int("count", count).Int("list_count", len(items)).Msg("read all items")
return nil
}
return wrapActionFunc(ctx, f)
}
func viewCategoriesListAction(ctx context.Context, r storage.Repository) cli.ActionFunc {
f := func(c *cli.Context) error {
categories, err := r.Category().List(ctx)
if err != nil {
return fmt.Errorf("listing categories: %w", err)
}
limit := c.Int("limit")
page := c.Int("page")
total := len(categories)
if page == 0 {
page = 1
}
if limit > 0 {
offset := (page - 1) * limit
if offset > len(categories) {
offset = len(categories) - 1
}
limit = offset + limit
if limit > len(categories) {
limit = len(categories)
}
categories = categories[offset:limit]
}
tbl := table.New("ID", "Name")
for _, category := range categories {
if category.ID == 0 && category.Name == "" {
continue
}
tbl.AddRow(category.ID, category.Name)
}
tbl.Print()
if c.Bool("with-total") {
zerolog.Ctx(ctx).Info().Int("count", total).Msg("total categories stats")
}
return nil
}
return wrapActionFunc(ctx, f)
}
func newImportFromFileCmd(ctx context.Context, r storage.Repository, maxBatch int64) cli.Command {
return cli.Command{
Name: "fromfile",
Usage: "imports from file into db",
Flags: []cli.Flag{
&cli.StringFlag{
Name: "src",
Value: "",
Usage: "source of the data.",
Required: true,
},
},
Action: handleConvert(ctx, r, maxBatch),
}
}
func handleConvert(ctx context.Context, r storage.Repository, maxBatch int64) cli.ActionFunc {
f := func(c *cli.Context) error {
filesrc := c.String("src")
log := zerolog.Ctx(ctx)
log.Debug().Str("filepath", filesrc).Msg("importing data from file")
productsFile, err := os.Open(filesrc)
if err != nil {
return fmt.Errorf("opening file: %w", err)
}
defer func() {
errClose := productsFile.Close()
if errClose != nil {
log.Warn().Err(errClose).Msg("unable to close file")
}
}()
var (
goodsItem entity.GoodsItem
// goodsItems []entity.GoodsItem
failedToInsert int
)
seenCategories := make(map[string]struct{})
categories, err := r.Category().List(ctx)
if err != nil {
return fmt.Errorf("listing categories: %w", err)
}
for _, category := range categories {
seenCategories[category.Name] = struct{}{}
}
bfile := bufio.NewReader(productsFile)
for {
line, _, err := bfile.ReadLine()
if err != nil {
if errors.Is(err, io.EOF) {
break
}
return fmt.Errorf("reading line: %w", err)
}
err = json.Unmarshal(line, &goodsItem)
if err != nil {
// log.Warn().Err(err).Str("line", string(line)).Msg("unable to unmarshal line into item")
failedToInsert++
continue
}
_, err = r.GoodsItem().UpsertMany(ctx, goodsItem)
if err != nil {
return fmt.Errorf("unable to upsert new item: %w", err)
}
// goodsItems = append(goodsItems, goodsItem)
if _, ok := seenCategories[goodsItem.Type]; ok {
continue
}
_, err = r.Category().Create(ctx, goodsItem.Type)
if err != nil {
return fmt.Errorf("unable to create new category: %w", err)
}
log.Debug().Any("category", goodsItem.Type).Msg("inserted new category")
seenCategories[goodsItem.Type] = struct{}{}
}
// log.Debug().Int("count", len(goodsItems)).Int("failed", failedToInsert).Msg("preparing to upload")
//
// start := time.Now()
// batchSize := int(maxBatch)
// for i := 0; i < len(goodsItems); i += batchSize {
// to := i + batchSize
// if to > len(goodsItems) {
// to = len(goodsItems)
// }
//
// _, err = r.GoodsItem().UpsertMany(ctx, goodsItems[i:to]...)
// if err != nil {
// return fmt.Errorf("upserting items: %w", err)
// }
// log.Debug().Int("count", to-i).Msg("inserted batch")
// time.Sleep(time.Second)
// }
// log.Debug().Dur("elapsed", time.Since(start)).Msg("upload finished")
//
time.Sleep(time.Second * 30)
return nil
}
time.Sleep(time.Second * 10)
return wrapActionFunc(ctx, f)
}
func wrapActionFunc(ctx context.Context, next cli.ActionFunc) cli.ActionFunc {
return func(c *cli.Context) error {
var data [3]byte
_, _ = rand.Read(data[:])
reqid := hex.EncodeToString(data[:])
log := zerolog.Ctx(ctx).With().Str("reqid", reqid).Logger()
ctx = log.WithContext(ctx)
start := time.Now()
defer func() {
log.Info().Dur("elapsed", time.Since(start)).Msg("command completed")
}()
log.Info().Msg("command execution started")
return next(c)
}
}

16
config.toml Normal file
View File

@ -0,0 +1,16 @@
[badger]
debug = false
dir = "database/"
[log]
level = "info"
format = "text"
[eway]
login = "leci@yandex.ru"
password = "2a136e113854cc5d46b868919a7d6e939156ccb55ff12e87861513f7767af98be79e62407410"
_session_id = "19b98ed56cc144f47e040e68dbcd8481"
_session_user = "1490"
owner_id = "26476"
debug = false
workers_pool = 3

31
go.mod
View File

@ -3,30 +3,33 @@ module git.loyso.art/frx/eway
go 1.21.4 go 1.21.4
require ( require (
github.com/BurntSushi/toml v1.3.2
github.com/PuerkitoBio/goquery v1.8.1
github.com/brianvoe/gofakeit/v6 v6.28.0
github.com/dgraph-io/badger/v4 v4.2.0
github.com/dgraph-io/ristretto v0.1.1
github.com/go-resty/resty/v2 v2.10.0
github.com/google/flatbuffers v23.5.26+incompatible
github.com/rodaine/table v1.1.1
github.com/rs/zerolog v1.31.0
github.com/samber/do v1.6.0
github.com/urfave/cli/v3 v3.0.0-alpha8
)
require (
github.com/andybalholm/cascadia v1.3.1 // indirect
github.com/cespare/xxhash/v2 v2.2.0 // indirect github.com/cespare/xxhash/v2 v2.2.0 // indirect
github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect
github.com/dgraph-io/badger/v4 v4.2.0 // indirect
github.com/dgraph-io/ristretto v0.1.1 // indirect
github.com/dustin/go-humanize v1.0.0 // indirect github.com/dustin/go-humanize v1.0.0 // indirect
github.com/go-resty/resty/v2 v2.10.0 // indirect
github.com/gogo/protobuf v1.3.2 // indirect github.com/gogo/protobuf v1.3.2 // indirect
github.com/golang/glog v1.0.0 // indirect github.com/golang/glog v1.0.0 // indirect
github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
github.com/golang/protobuf v1.5.2 // indirect github.com/golang/protobuf v1.5.2 // indirect
github.com/golang/snappy v0.0.3 // indirect github.com/golang/snappy v0.0.3 // indirect
github.com/google/flatbuffers v23.5.26+incompatible // indirect
github.com/karlseguin/ccache/v3 v3.0.5 // indirect
github.com/kataras/tablewriter v0.0.0-20180708051242-e063d29b7c23 // indirect
github.com/klauspost/compress v1.12.3 // indirect github.com/klauspost/compress v1.12.3 // indirect
github.com/mattn/go-colorable v0.1.13 // indirect github.com/mattn/go-colorable v0.1.13 // indirect
github.com/mattn/go-isatty v0.0.19 // indirect github.com/mattn/go-isatty v0.0.19 // indirect
github.com/mattn/go-runewidth v0.0.15 // indirect
github.com/pkg/errors v0.9.1 // indirect github.com/pkg/errors v0.9.1 // indirect
github.com/rivo/uniseg v0.2.0 // indirect github.com/xrash/smetrics v0.0.0-20231213231151-1d8dd44e695e // indirect
github.com/rodaine/table v1.1.1 // indirect
github.com/rs/zerolog v1.31.0 // indirect
github.com/russross/blackfriday/v2 v2.1.0 // indirect
github.com/urfave/cli v1.22.14 // indirect
go.opencensus.io v0.22.5 // indirect go.opencensus.io v0.22.5 // indirect
golang.org/x/net v0.17.0 // indirect golang.org/x/net v0.17.0 // indirect
golang.org/x/sys v0.13.0 // indirect golang.org/x/sys v0.13.0 // indirect

38
go.sum
View File

@ -1,19 +1,26 @@
cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
github.com/BurntSushi/toml v1.3.2 h1:o7IhLm0Msx3BaB+n3Ag7L8EVlByGnpq14C4YWiu/gL8=
github.com/BurntSushi/toml v1.3.2/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= github.com/BurntSushi/toml v1.3.2/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ=
github.com/PuerkitoBio/goquery v1.8.1 h1:uQxhNlArOIdbrH1tr0UXwdVFgDcZDrZVdcpygAcwmWM=
github.com/PuerkitoBio/goquery v1.8.1/go.mod h1:Q8ICL1kNUJ2sXGoAhPGUdYDJvgQgHzJsnnd3H7Ho5jQ=
github.com/andybalholm/cascadia v1.3.1 h1:nhxRkql1kdYCc8Snf7D5/D3spOX+dBgjA6u8x004T2c=
github.com/andybalholm/cascadia v1.3.1/go.mod h1:R4bJ1UQfqADjvDa4P6HZHLh/3OxWWEqc0Sk8XGwHqvA=
github.com/brianvoe/gofakeit/v6 v6.28.0 h1:Xib46XXuQfmlLS2EXRuJpqcw8St6qSZz75OUo0tgAW4=
github.com/brianvoe/gofakeit/v6 v6.28.0/go.mod h1:Xj58BMSnFqcn/fAQeSK+/PLtC5kSb7FJIq4JyGa8vEs=
github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44=
github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc=
github.com/cpuguy83/go-md2man/v2 v2.0.2 h1:p1EgwI/C7NhT0JmVkwCD2ZBK8j4aeHQX2pMHHBfMQ6w=
github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/dgraph-io/badger/v4 v4.2.0 h1:kJrlajbXXL9DFTNuhhu9yCx7JJa4qpYWxtE8BzuWsEs= github.com/dgraph-io/badger/v4 v4.2.0 h1:kJrlajbXXL9DFTNuhhu9yCx7JJa4qpYWxtE8BzuWsEs=
github.com/dgraph-io/badger/v4 v4.2.0/go.mod h1:qfCqhPoWDFJRx1gp5QwwyGo8xk1lbHUxvK9nK0OGAak= github.com/dgraph-io/badger/v4 v4.2.0/go.mod h1:qfCqhPoWDFJRx1gp5QwwyGo8xk1lbHUxvK9nK0OGAak=
github.com/dgraph-io/ristretto v0.1.1 h1:6CWw5tJNgpegArSHpNHJKldNeq03FQCwYvfMVWajOK8= github.com/dgraph-io/ristretto v0.1.1 h1:6CWw5tJNgpegArSHpNHJKldNeq03FQCwYvfMVWajOK8=
github.com/dgraph-io/ristretto v0.1.1/go.mod h1:S1GPSBCYCIhmVNfcth17y2zZtQT6wzkzgwUve0VDWWA= github.com/dgraph-io/ristretto v0.1.1/go.mod h1:S1GPSBCYCIhmVNfcth17y2zZtQT6wzkzgwUve0VDWWA=
github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2 h1:tdlZCpZ/P9DhczCTSixgIKmwPv6+wP5DGjqLYw5SUiA=
github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw=
github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo= github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo=
github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
@ -25,8 +32,9 @@ github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69
github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
github.com/golang/glog v1.0.0 h1:nfP3RFugxnNRyKgeWd4oI1nYvXpxrx8ck8ZrcizshdQ= github.com/golang/glog v1.0.0 h1:nfP3RFugxnNRyKgeWd4oI1nYvXpxrx8ck8ZrcizshdQ=
github.com/golang/glog v1.0.0/go.mod h1:EWib/APOK0SL3dFbYqvxE3UYd8E6s1ouQ7iEp/0LWV4= github.com/golang/glog v1.0.0/go.mod h1:EWib/APOK0SL3dFbYqvxE3UYd8E6s1ouQ7iEp/0LWV4=
github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6 h1:ZgQEtGgCBiWRM39fZuwSd1LwSqqSW0hOdXCYYDX0R3I=
github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE=
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
@ -39,11 +47,8 @@ github.com/google/flatbuffers v23.5.26+incompatible h1:M9dgRyhJemaM4Sw8+66GHBu8i
github.com/google/flatbuffers v23.5.26+incompatible/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8= github.com/google/flatbuffers v23.5.26+incompatible/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8=
github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
github.com/karlseguin/ccache/v3 v3.0.5 h1:hFX25+fxzNjsRlREYsoGNa2LoVEw5mPF8wkWq/UnevQ=
github.com/karlseguin/ccache/v3 v3.0.5/go.mod h1:qxC372+Qn+IBj8Pe3KvGjHPj0sWwEF7AeZVhsNPZ6uY=
github.com/kataras/tablewriter v0.0.0-20180708051242-e063d29b7c23 h1:M8exrBzuhWcU6aoHJlHWPe4qFjVKzkMGRal78f5jRRU=
github.com/kataras/tablewriter v0.0.0-20180708051242-e063d29b7c23/go.mod h1:kBSna6b0/RzsOcOZf515vAXwSsXYusl2U7SA0XP09yI=
github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
github.com/klauspost/compress v1.12.3 h1:G5AfA94pHPysR56qqrkO2pxEexdDzrpFJ6yt/VqWxVU= github.com/klauspost/compress v1.12.3 h1:G5AfA94pHPysR56qqrkO2pxEexdDzrpFJ6yt/VqWxVU=
@ -57,6 +62,7 @@ github.com/mattn/go-runewidth v0.0.15 h1:UNAjwbU9l54TA3KzvqLGxwWjHmMgBUVhBiTjelZ
github.com/mattn/go-runewidth v0.0.15/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= github.com/mattn/go-runewidth v0.0.15/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=
github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/rivo/uniseg v0.2.0 h1:S1pD9weZBuJdFmowNwbpi7BJ8TNftyUImj/0WQi72jY= github.com/rivo/uniseg v0.2.0 h1:S1pD9weZBuJdFmowNwbpi7BJ8TNftyUImj/0WQi72jY=
github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
@ -65,17 +71,20 @@ github.com/rodaine/table v1.1.1/go.mod h1:iqTRptjn+EVcrVBYtNMlJ2wrJZa3MpULUmcXFp
github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg=
github.com/rs/zerolog v1.31.0 h1:FcTR3NnLWW+NnTwwhFWiJSZr4ECLpqCm6QsEnyvbV4A= github.com/rs/zerolog v1.31.0 h1:FcTR3NnLWW+NnTwwhFWiJSZr4ECLpqCm6QsEnyvbV4A=
github.com/rs/zerolog v1.31.0/go.mod h1:/7mN4D5sKwJLZQ2b/znpjC3/GQWY/xaDXUM0kKWRHss= github.com/rs/zerolog v1.31.0/go.mod h1:/7mN4D5sKwJLZQ2b/znpjC3/GQWY/xaDXUM0kKWRHss=
github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= github.com/samber/do v1.6.0 h1:Jy/N++BXINDB6lAx5wBlbpHlUdl0FKpLWgGEV9YWqaU=
github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/samber/do v1.6.0/go.mod h1:DWqBvumy8dyb2vEnYZE7D7zaVEB64J45B0NjTlY/M4k=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk=
github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
github.com/urfave/cli v1.22.14 h1:ebbhrRiGK2i4naQJr+1Xj92HXZCrK7MsyTS/ob3HnAk= github.com/urfave/cli/v3 v3.0.0-alpha8 h1:H+qxFPoCkGzdF8KUMs2fEOZl5io/1QySgUiGfar8occ=
github.com/urfave/cli v1.22.14/go.mod h1:X0eDS6pD6Exaclxm99NJ3FiCDRED7vIHpx2mDOHLvkA= github.com/urfave/cli/v3 v3.0.0-alpha8/go.mod h1:0kK/RUFHyh+yIKSfWxwheGndfnrvYSmYFVeKCh03ZUc=
github.com/xrash/smetrics v0.0.0-20231213231151-1d8dd44e695e h1:+SOyEddqYF09QP7vr7CgJ1eti3pY9Fn3LHO1M1r/0sI=
github.com/xrash/smetrics v0.0.0-20231213231151-1d8dd44e695e/go.mod h1:N3UwUGtsrSj3ccvlPHLoLsHnpR27oXr4ZE984MbSER8=
github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
@ -103,8 +112,10 @@ golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLL
golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
golang.org/x/net v0.0.0-20210916014120-12bc252f5db8/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg=
golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM= golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM=
golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE=
@ -123,6 +134,7 @@ golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7w
golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
@ -141,10 +153,12 @@ golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo=
golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U= golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4=
golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
@ -171,7 +185,7 @@ google.golang.org/protobuf v1.28.1 h1:d0NfwRgPtno5B1Wa6L2DAG+KivqkdutMf1UhdNx175
google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=

View File

@ -1,5 +1,7 @@
package config package config
type Badger struct { type Badger struct {
Path string `json:"path"` Debug bool `toml:"debug"`
Dir string `toml:"dir"`
ValueDir *string `toml:"value_dir"`
} }

11
internal/config/eway.go Normal file
View File

@ -0,0 +1,11 @@
package config
type Eway struct {
Login string `toml:"login"`
Password string `toml:"password"`
SessionID string `toml:"session_id"`
SessionUser string `toml:"session_user"`
OwnerID string `toml:"owner_id"`
Debug bool `toml:"debug"`
WorkersPool int `toml:"workers_pool"`
}

55
internal/config/log.go Normal file
View File

@ -0,0 +1,55 @@
package config
import (
"strings"
"git.loyso.art/frx/eway/internal/entity"
)
type LogLevel uint8
const (
LogLevelDebug LogLevel = iota
LogLevelInfo
LogLevelWarn
)
func (l *LogLevel) UnmarshalText(data []byte) (err error) {
switch strings.ToLower(string(data)) {
case "debug":
*l = LogLevelDebug
case "info":
*l = LogLevelInfo
case "warn":
*l = LogLevelWarn
default:
return entity.SimpleError("unsupported level " + string(data))
}
return nil
}
type LogFormat uint8
const (
LogFormatText LogFormat = iota
LogFormatJSON
)
func (l *LogFormat) UnmarshalText(data []byte) (err error) {
switch strings.ToLower(string(data)) {
case "text":
*l = LogFormatText
case "info":
*l = LogFormatJSON
default:
return entity.SimpleError("unsupported format " + string(data))
}
return nil
}
type Log struct {
Level string `json:"level" toml:"level"`
Format string `json:"format" toml:"format"`
}

32
internal/crypto/cipher.go Normal file
View File

@ -0,0 +1,32 @@
package crypto
import (
"crypto/aes"
"crypto/cipher"
"crypto/rand"
"encoding/hex"
"fmt"
)
var (
someDumbKey = []byte("9530e001b619e8e98a889055f06821bb")
)
func Encrypt(plaintext string) (hexed string, err error) {
aes, err := aes.NewCipher(someDumbKey)
if err != nil {
return "", fmt.Errorf("making new cipher: %w", err)
}
gcm, err := cipher.NewGCM(aes)
if err != nil {
return "", fmt.Errorf("making new gcm: %w", err)
}
nonce := make([]byte, gcm.NonceSize())
_, err = rand.Read(nonce)
if err != nil {
return "", fmt.Errorf("generating nonce: %w", err)
}
outvalue := gcm.Seal(nonce, nonce, []byte(plaintext), nil)
return hex.EncodeToString(outvalue), nil
}

View File

@ -0,0 +1,32 @@
package crypto
import (
"crypto/aes"
"crypto/cipher"
"encoding/hex"
"fmt"
)
func Decrypt(hexedcipher string) (plaintext string, err error) {
aes, err := aes.NewCipher(someDumbKey)
if err != nil {
return "", fmt.Errorf("making new cipher: %w", err)
}
gcm, err := cipher.NewGCM(aes)
if err != nil {
return "", fmt.Errorf("making new gcm: %w", err)
}
nonceSize := gcm.NonceSize()
valueDecoded, err := hex.DecodeString(hexedcipher)
if err != nil {
return "", fmt.Errorf("decoding hexed value: %w", err)
}
nonce, cipherText := valueDecoded[:nonceSize], valueDecoded[nonceSize:]
plaintextRaw, err := gcm.Open(nil, []byte(nonce), []byte(cipherText), nil)
if err != nil {
return "", fmt.Errorf("opening: %w", err)
}
return string(plaintextRaw), nil
}

View File

@ -169,8 +169,28 @@ func (rcv *GoodItem) MutateStock(n int16) bool {
return rcv._tab.MutateInt16Slot(28, n) return rcv._tab.MutateInt16Slot(28, n)
} }
func (rcv *GoodItem) Parameters() []byte {
o := flatbuffers.UOffsetT(rcv._tab.Offset(30))
if o != 0 {
return rcv._tab.ByteVector(o + rcv._tab.Pos)
}
return nil
}
func (rcv *GoodItem) CreatedAt() int64 {
o := flatbuffers.UOffsetT(rcv._tab.Offset(32))
if o != 0 {
return rcv._tab.GetInt64(o + rcv._tab.Pos)
}
return 0
}
func (rcv *GoodItem) MutateCreatedAt(n int64) bool {
return rcv._tab.MutateInt64Slot(32, n)
}
func GoodItemStart(builder *flatbuffers.Builder) { func GoodItemStart(builder *flatbuffers.Builder) {
builder.StartObject(13) builder.StartObject(15)
} }
func GoodItemAddSku(builder *flatbuffers.Builder, sku flatbuffers.UOffsetT) { func GoodItemAddSku(builder *flatbuffers.Builder, sku flatbuffers.UOffsetT) {
builder.PrependUOffsetTSlot(0, flatbuffers.UOffsetT(sku), 0) builder.PrependUOffsetTSlot(0, flatbuffers.UOffsetT(sku), 0)
@ -211,6 +231,12 @@ func GoodItemAddCart(builder *flatbuffers.Builder, cart int64) {
func GoodItemAddStock(builder *flatbuffers.Builder, stock int16) { func GoodItemAddStock(builder *flatbuffers.Builder, stock int16) {
builder.PrependInt16Slot(12, stock, 0) builder.PrependInt16Slot(12, stock, 0)
} }
func GoodItemAddParameters(builder *flatbuffers.Builder, parameters flatbuffers.UOffsetT) {
builder.PrependUOffsetTSlot(13, flatbuffers.UOffsetT(parameters), 0)
}
func GoodItemAddCreatedAt(builder *flatbuffers.Builder, createdAt int64) {
builder.PrependInt64Slot(14, createdAt, 0)
}
func GoodItemEnd(builder *flatbuffers.Builder) flatbuffers.UOffsetT { func GoodItemEnd(builder *flatbuffers.Builder) flatbuffers.UOffsetT {
return builder.EndObject() return builder.EndObject()
} }

View File

@ -1,7 +1,12 @@
package fbs package fbs
import ( import (
"encoding/base64"
"encoding/json"
"fmt"
"strings"
"sync" "sync"
"time"
"git.loyso.art/frx/eway/internal/entity" "git.loyso.art/frx/eway/internal/entity"
@ -21,8 +26,8 @@ func getBuilder() *flatbuffers.Builder {
} }
func putBuilder(builder *flatbuffers.Builder) { func putBuilder(builder *flatbuffers.Builder) {
builder.Reset() // builder.Reset()
builderPool.Put(builder) // builderPool.Put(builder)
} }
func MakeDomainGoodItems(in ...entity.GoodsItem) []byte { func MakeDomainGoodItems(in ...entity.GoodsItem) []byte {
@ -60,14 +65,21 @@ func MakeDomainGoodItemFinished(in entity.GoodsItem) []byte {
func makeDomainGoodItem(builder *flatbuffers.Builder, in entity.GoodsItem) flatbuffers.UOffsetT { func makeDomainGoodItem(builder *flatbuffers.Builder, in entity.GoodsItem) flatbuffers.UOffsetT {
sku := builder.CreateString(in.Articul) sku := builder.CreateString(in.Articul)
photo := builder.CreateString(in.Photo) photo := builder.CreateString(strings.Join(in.PhotoURLs, ";"))
name := builder.CreateString(in.Name) name := builder.CreateString(in.Name)
desc := builder.CreateString(in.Description)
descBase64 := base64.RawStdEncoding.EncodeToString([]byte(in.Description))
desc := builder.CreateString(descBase64)
var cat flatbuffers.UOffsetT var cat flatbuffers.UOffsetT
if in.Category != "" { if in.Category != "" {
cat = builder.CreateString(in.Category) cat = builder.CreateString(in.Category)
} }
t := builder.CreateString(in.Type) t := builder.CreateString(in.Type)
parametersData, _ := json.Marshal(in.Parameters)
parameters := builder.CreateByteString(parametersData)
producer := builder.CreateString(in.Producer) producer := builder.CreateString(in.Producer)
GoodItemStart(builder) GoodItemStart(builder)
@ -86,16 +98,24 @@ func makeDomainGoodItem(builder *flatbuffers.Builder, in entity.GoodsItem) flatb
GoodItemAddTariff(builder, float32(in.TariffPrice)) GoodItemAddTariff(builder, float32(in.TariffPrice))
GoodItemAddCart(builder, int64(in.Cart)) GoodItemAddCart(builder, int64(in.Cart))
GoodItemAddStock(builder, int16(in.Stock)) GoodItemAddStock(builder, int16(in.Stock))
GoodItemAddParameters(builder, parameters)
GoodItemAddCreatedAt(builder, in.CreatedAt.Unix())
return GoodItemEnd(builder) return GoodItemEnd(builder)
} }
func ParseGoodsItem(data []byte) (item entity.GoodsItem) { func ParseGoodsItem(data []byte) (item entity.GoodsItem, err error) {
itemFBS := GetRootAsGoodItem(data, 0) itemFBS := GetRootAsGoodItem(data, 0)
item.Articul = string(itemFBS.Sku()) item.Articul = string(itemFBS.Sku())
item.Photo = string(itemFBS.Photo()) item.PhotoURLs = strings.Split(string(itemFBS.Photo()), ";")
item.Name = string(itemFBS.Name()) item.Name = string(itemFBS.Name())
item.Description = string(itemFBS.Description())
description, err := base64.RawStdEncoding.DecodeString(string(itemFBS.Description()))
if err != nil {
return item, fmt.Errorf("decoding description from base64: %w", err)
}
item.Description = string(description)
if value := itemFBS.Category(); value != nil { if value := itemFBS.Category(); value != nil {
item.Category = string(value) item.Category = string(value)
} }
@ -107,8 +127,21 @@ func ParseGoodsItem(data []byte) (item entity.GoodsItem) {
item.TariffPrice = float64(itemFBS.Tariff()) item.TariffPrice = float64(itemFBS.Tariff())
item.Cart = itemFBS.Cart() item.Cart = itemFBS.Cart()
item.Stock = int(itemFBS.Stock()) item.Stock = int(itemFBS.Stock())
item.Parameters = map[string]string{}
parameters := itemFBS.Parameters()
if len(parameters) > 0 {
err = json.Unmarshal(itemFBS.Parameters(), &item.Parameters)
if err != nil {
return item, fmt.Errorf("unmarshalling data: %w", err)
}
}
return item createdAt := itemFBS.CreatedAt()
if createdAt > 0 {
item.CreatedAt = time.Unix(createdAt, 0)
}
return item, nil
} }
func ParseCategory(data []byte) (category entity.Category) { func ParseCategory(data []byte) (category entity.Category) {

View File

@ -7,5 +7,6 @@ func (err SimpleError) Error() string {
} }
const ( const (
ErrNotFound SimpleError = "not found" ErrNotFound SimpleError = "not found"
ErrNotImplemented SimpleError = "not implemented"
) )

View File

@ -4,23 +4,26 @@ import (
"fmt" "fmt"
"strconv" "strconv"
"strings" "strings"
"time"
"unicode" "unicode"
) )
type GoodsItem struct { type GoodsItem struct {
Articul string `json:"sku"` Articul string `json:"sku"`
Photo string `json:"photo"` PhotoURLs []string `json:"photo"`
Name string `json:"name"` Name string `json:"name"`
Description string `json:"description"` Description string `json:"description"`
Category string `json:"category"` Category string `json:"category"`
Type string `json:"type"` Type string `json:"type"`
Producer string `json:"producer"` Producer string `json:"producer"`
Pack int `json:"pack"` Pack int `json:"pack"`
Step int `json:"step"` Step int `json:"step"`
Price float64 `json:"price"` Price float64 `json:"price"`
TariffPrice float64 `json:"tariff_price"` TariffPrice float64 `json:"tariff_price"`
Cart int64 `json:"cart"` Cart int64 `json:"cart"`
Stock int `json:"stock"` Stock int `json:"stock"`
Parameters map[string]string `json:"parameters"`
CreatedAt time.Time `json:"created_at"`
} }
type GoodsItemRaw struct { type GoodsItemRaw struct {
@ -42,21 +45,18 @@ type GoodsItemRaw struct {
Other string Other string
} }
type GoodsItemInfo struct {
Parameters map[string]string
PhotoURLs []string
}
type MappedGoodsRemnants map[int]GoodsRemnant type MappedGoodsRemnants map[int]GoodsRemnant
type GoodsRemnant [4]int32 type GoodsRemnant [4]int32
func ExtractProductIDs(items []GoodsItem) (out []int) {
out = make([]int, 0, len(items))
for _, item := range items {
out = append(out, int(item.Cart))
}
return out
}
func MakeGoodsItem( func MakeGoodsItem(
gi GoodsItemRaw, gi GoodsItemRaw,
remnants MappedGoodsRemnants, remnants MappedGoodsRemnants,
info GoodsItemInfo,
) (out GoodsItem, err error) { ) (out GoodsItem, err error) {
var name, desc string var name, desc string
var pack, step int var pack, step int
@ -104,9 +104,13 @@ func MakeGoodsItem(
return out, fmt.Errorf("getting step count (%s): %w", gi.Step, err) return out, fmt.Errorf("getting step count (%s): %w", gi.Step, err)
} }
photoURLs := info.PhotoURLs
if len(photoURLs) > 7 {
photoURLs = info.PhotoURLs[:7]
}
return GoodsItem{ return GoodsItem{
Articul: gi.SKU, Articul: gi.SKU,
Photo: gi.Photo,
Name: name, Name: name,
Description: desc, Description: desc,
Category: gi.Category, Category: gi.Category,
@ -118,5 +122,7 @@ func MakeGoodsItem(
TariffPrice: tariffPrice, TariffPrice: tariffPrice,
Cart: int64(gi.Cart), Cart: int64(gi.Cart),
Stock: int(remnants[int(gi.Cart)][0]), Stock: int(remnants[int(gi.Cart)][0]),
PhotoURLs: photoURLs,
Parameters: info.Parameters,
}, nil }, nil
} }

59
internal/entity/iter.go Normal file
View File

@ -0,0 +1,59 @@
package entity
func IterIntoMap[K comparable, V any](v []V, err error) iterIntoMap[K, V] {
bi := IterWithErr(v, err)
return iterIntoMap[K, V]{
baseIter: bi,
}
}
type iterIntoMap[K comparable, V any] struct {
baseIter[V]
}
func (i iterIntoMap[K, V]) Map(f func(V) (K, error)) (map[K]V, error) {
if i.err != nil {
return nil, i.err
}
out := make(map[K]V, len(i.items))
for _, item := range i.items {
var key K
key, i.err = f(item)
if i.err != nil {
return nil, i.err
}
out[key] = item
}
return out, nil
}
func IterWithErr[T any](t []T, err error) baseIter[T] {
return baseIter[T]{
items: t,
err: err,
}
}
type baseIter[T any] struct {
items []T
err error
}
func (iter baseIter[T]) Do(f func(T) error) error {
if iter.err != nil {
return iter.err
}
for _, item := range iter.items {
err := f(item)
if err != nil {
return err
}
}
return nil
}

View File

@ -9,6 +9,7 @@ type GoodsItemRepository interface {
GetByCart(context.Context, int64) (GoodsItem, error) GetByCart(context.Context, int64) (GoodsItem, error)
UpsertMany(context.Context, ...GoodsItem) ([]GoodsItem, error) UpsertMany(context.Context, ...GoodsItem) ([]GoodsItem, error)
Delete(context.Context, string) (GoodsItem, error)
} }
type CategoryRepository interface { type CategoryRepository interface {

23
internal/entity/task.go Normal file
View File

@ -0,0 +1,23 @@
package entity
import (
"context"
"time"
)
type PublishParams struct {
Body []byte
ExpiresAt time.Time
}
type MessageQueue interface {
Publish(context.Context, PublishParams) (Task, error)
Consume(context.Context) (Task, error)
}
type Task struct {
ID uint64
CreatedAt time.Time
ExpiresAt *time.Time
Body []byte
}

View File

@ -0,0 +1,61 @@
package export
import "time"
type Param struct {
Name string `xml:"name,attr"`
Value string `xml:",chardata"`
}
type Offer struct {
ID int64 `xml:"id,attr"`
Type string `xml:"type,attr"`
Available bool `xml:"available,attr"`
URL string `xml:"url"`
Price int `xml:"price"`
CurrencyID string `xml:"currencyId"`
CategoryID int64 `xml:"categoryId"`
PictureURLs []string `xml:"picture"`
Vendor string `xml:"vendor"`
Model string `xml:"model"`
VendorCode string `xml:"vendorCode"`
TypePrefix string `xml:"typePrefix"`
Description string `xml:"description"`
ManufacturerWarrany bool `xml:"manufacturer_warranty"`
Params []Param `xml:"param"`
}
type Currency struct {
ID string `xml:"id,attr"` // RUR only
Rate int64 `xml:"rate,attr"` // 1?
}
type Category struct {
ID int64 `xml:"id,attr"`
ParentID int64 `xml:"parent_id,attr,omitempty"`
Name string `xml:",chardata"`
}
type Shop struct {
Name string `xml:"name"` // r
Company string `xml:"company"` // r
URL string `xml:"url"` // r
Platform string `xml:"platform"`
Version string `xml:"version"`
Currencies []Currency `xml:"currencies"` // r RUR only
Categories []Category `xml:"categories>category"` // r
Offers []Offer `xml:"offer"` // r
}
type YmlContainer struct {
XMLName struct{} `xml:"yml_catalog"`
YmlCatalog
}
type YmlCatalog struct {
Date time.Time `xml:"date,attr"`
Shop Shop `xml:"shop"`
}

View File

@ -0,0 +1,82 @@
package export
import (
"encoding/xml"
"os"
"testing"
"time"
"github.com/brianvoe/gofakeit/v6"
)
func TestYMLSerialize(t *testing.T) {
faker := gofakeit.New(0)
categories := make([]Category, faker.Rand.Intn(4))
knownCategory := map[int64]struct{}{}
categoryIDs := make([]int64, 0, 10)
for i := range categories {
categories[i].ID = faker.Int64()
categories[i].Name = faker.HipsterWord()
categories[i].ParentID = faker.Int64()
if _, ok := knownCategory[categories[i].ID]; ok {
continue
}
knownCategory[categories[i].ID] = struct{}{}
categoryIDs = append(categoryIDs, categories[i].ID)
}
offers := make([]Offer, faker.Rand.Intn(5)+1)
for i := range offers {
offer := &offers[i]
offer.ID = faker.Int64()
offer.Type = "vendor.model"
offer.Available = true
offer.URL = faker.URL()
offer.Price = int(faker.Price(10, 1000))
offer.CurrencyID = "RUR"
offer.CategoryID = categoryIDs[faker.Rand.Intn(len(categoryIDs))]
for i := 0; i < faker.Rand.Intn(3); i++ {
offer.PictureURLs = append(offer.PictureURLs, faker.ImageURL(128, 128))
}
offer.Vendor = faker.Company()
offer.Model = faker.CarModel()
offer.VendorCode = faker.DigitN(8)
offer.TypePrefix = faker.ProductName()
offer.Description = faker.Sentence(12)
offer.ManufacturerWarrany = true
for i := 0; i < faker.Rand.Intn(8); i++ {
offer.Params = append(offer.Params, Param{
Name: faker.AdjectiveProper(),
Value: faker.Digit(),
})
}
}
var catalog YmlCatalog
catalog.Shop = Shop{
Name: faker.ProductName(),
URL: faker.URL(),
Company: faker.Company(),
Platform: "BSM/Yandex/Market",
Version: faker.AppVersion(),
Currencies: []Currency{{
ID: "RUR",
Rate: 1,
}},
Categories: categories,
Offers: offers,
}
catalog.Date = faker.Date().Truncate(time.Second)
container := YmlContainer{
YmlCatalog: catalog,
}
enc := xml.NewEncoder(os.Stdout)
enc.Indent("", " ")
_ = enc.Encode(container)
println()
t.FailNow()
}

View File

@ -11,59 +11,104 @@ import (
"reflect" "reflect"
"strconv" "strconv"
"strings" "strings"
"sync"
"time"
"git.loyso.art/frx/eway/internal/config"
"git.loyso.art/frx/eway/internal/crypto"
"git.loyso.art/frx/eway/internal/entity" "git.loyso.art/frx/eway/internal/entity"
"github.com/PuerkitoBio/goquery"
"github.com/go-resty/resty/v2" "github.com/go-resty/resty/v2"
"github.com/rs/zerolog" "github.com/rs/zerolog"
) )
type Client interface {
GetGoodsRemnants(context.Context, []int) (entity.MappedGoodsRemnants, error)
GetGoodsNew(
context.Context,
GetGoodsNewParams,
) (items []entity.GoodsItemRaw, total int, err error)
GetProductInfo(context.Context, int64) (entity.GoodsItemInfo, error)
}
type client struct { type client struct {
http *resty.Client http *resty.Client
log zerolog.Logger log zerolog.Logger
htmlParseSema chan struct{}
releaseSemaDelay time.Duration
getProductInfoBus chan getProductInfoRequest
ownerID string
workersPool int
workerswg *sync.WaitGroup
} }
func NewClientWithSession(sessionid, sessionuser string, log zerolog.Logger) client { type Config config.Eway
cookies := []*http.Cookie{
{
Name: "session_id",
Value: sessionid,
Domain: "eway.elevel.ru",
HttpOnly: true,
},
{
Name: "session_user",
Value: sessionuser,
Domain: "eway.elevel.ru",
HttpOnly: true,
},
{
Name: "contract",
Value: "6101",
Domain: "eway.elevel.ru",
HttpOnly: true,
},
}
func New(ctx context.Context, cfg Config, log zerolog.Logger) (*client, error) {
httpclient := resty.New(). httpclient := resty.New().
SetDebug(false). SetDebug(cfg.Debug).
SetCookies(cookies).
SetBaseURL("https://eway.elevel.ru/api") SetBaseURL("https://eway.elevel.ru/api")
return client{ c := client{
http: httpclient, http: httpclient,
log: log.With().Str("client", "eway").Logger(), log: log.With().Str("client", "eway").Logger(),
htmlParseSema: make(chan struct{}, 2),
getProductInfoBus: make(chan getProductInfoRequest),
releaseSemaDelay: time.Second / 2,
workerswg: &sync.WaitGroup{},
} }
}
type getGoodsNewOrder struct { for i := 0; i < cfg.WorkersPool; i++ {
Column int c.workerswg.Add(1)
Dir string go func() {
defer c.workerswg.Done()
c.productInfoWorker(ctx, c.getProductInfoBus)
}()
}
if cfg.SessionID == "" || cfg.SessionUser == "" {
if cfg.Login == "" || cfg.Password == "" {
return nil, entity.SimpleError("no auth method provided")
}
decryptedPassword, err := crypto.Decrypt(cfg.Password)
if err != nil {
return nil, fmt.Errorf("decrypting password: %w", err)
}
err = c.login(ctx, cfg.Login, decryptedPassword)
if err != nil {
return nil, err
}
log.Info().Msg("login successful")
} else if cfg.SessionID != "" && cfg.SessionUser != "" {
cookies := []*http.Cookie{
{
Name: "session_id",
Value: cfg.SessionID,
Domain: "eway.elevel.ru",
HttpOnly: true,
},
{
Name: "session_user",
Value: cfg.SessionUser,
Domain: "eway.elevel.ru",
HttpOnly: true,
},
}
c.http.SetCookies(cookies)
} else {
return nil, entity.SimpleError("bad configuration: either session_id and session_user should be set or login and password")
}
return &c, nil
} }
type GetGoodsNewParams struct { type GetGoodsNewParams struct {
Draw int Draw int
Order getGoodsNewOrder
Start int Start int
// 100 is max // 100 is max
Length int Length int
@ -79,8 +124,6 @@ type getGoodsNewResponse struct {
Replacement bool `json:"replacement"` Replacement bool `json:"replacement"`
} }
type goodRemnant [4]int
func parseGoodItem(items []any) (out entity.GoodsItemRaw) { func parseGoodItem(items []any) (out entity.GoodsItemRaw) {
valueOf := reflect.ValueOf(&out).Elem() valueOf := reflect.ValueOf(&out).Elem()
typeOf := valueOf.Type() typeOf := valueOf.Type()
@ -135,7 +178,7 @@ func mapResponseByOrder(response getGoodsNewResponse) (items []entity.GoodsItemR
return items return items
} }
func (c client) GetGoodsRemnants( func (c *client) GetGoodsRemnants(
ctx context.Context, ctx context.Context,
productIDs []int, productIDs []int,
) (out entity.MappedGoodsRemnants, err error) { ) (out entity.MappedGoodsRemnants, err error) {
@ -148,12 +191,12 @@ func (c client) GetGoodsRemnants(
productsStr = append(productsStr, strconv.Itoa(sku)) productsStr = append(productsStr, strconv.Itoa(sku))
} }
resp, err := c.http.R(). req := c.http.R().
SetFormData(map[string]string{ SetFormData(map[string]string{
"products": strings.Join(productsStr, ","), "products": strings.Join(productsStr, ","),
}). }).
SetDoNotParseResponse(true). SetDoNotParseResponse(true)
Post("/goods_remnants") resp, err := c.do(ctx, "GetGoodsRemnants", req, resty.MethodPost, "/goods_remnants")
if err != nil { if err != nil {
return nil, fmt.Errorf("getting goods new: %w", err) return nil, fmt.Errorf("getting goods new: %w", err)
} }
@ -173,8 +216,6 @@ func (c client) GetGoodsRemnants(
return nil, fmt.Errorf("reading raw body: %w", err) return nil, fmt.Errorf("reading raw body: %w", err)
} }
c.log.Debug().RawJSON("response", data).Msg("body prepared")
out = make(entity.MappedGoodsRemnants, len(productIDs)) out = make(entity.MappedGoodsRemnants, len(productIDs))
err = json.NewDecoder(bytes.NewReader(data)).Decode(&out) err = json.NewDecoder(bytes.NewReader(data)).Decode(&out)
if err != nil { if err != nil {
@ -184,27 +225,41 @@ func (c client) GetGoodsRemnants(
return out, nil return out, nil
} }
func (c client) GetGoodsNew( func (c *client) GetGoodsNew(
ctx context.Context, ctx context.Context,
params GetGoodsNewParams, params GetGoodsNewParams,
) (items []entity.GoodsItemRaw, total int, err error) { ) (items []entity.GoodsItemRaw, total int, err error) {
var response getGoodsNewResponse var response getGoodsNewResponse
resp, err := c.http.R(). formData := map[string]string{
SetFormData(map[string]string{ "draw": strconv.Itoa(params.Draw),
"draw": strconv.Itoa(params.Draw), "start": strconv.Itoa(params.Start),
"start": strconv.Itoa(params.Start), "length": strconv.Itoa(params.Length),
"length": strconv.Itoa(params.Length), "order[0][column]": "14",
"order[0][column]": "14", "order[0][dir]": "desc",
"order[0][dir]": "desc", "search[value]": "",
"search[value]": "", "search[regex]": "false",
"search[regex]": "false", }
"search_in_stocks": "on", if params.SearchInStocks {
"remnants_atleast": "5", stocksNum := strconv.Itoa(params.RemmantsAtleast)
}). formData["search_in_stocks"] = "on"
formData["remnants_atleast"] = stocksNum
}
c.log.Debug().
Int("remnants", params.RemmantsAtleast).
Bool("search_in_stocks", params.SearchInStocks).
Int("draw", params.Draw).
Int("start", params.Start).
Int("length", params.Length).
Msg("sending request")
req := c.http.R().
SetFormData(formData).
SetQueryParam("category_id", "0"). SetQueryParam("category_id", "0").
SetQueryParam("own", "26476"). // user id? SetQueryParam("own", c.ownerID). // user id?
SetDoNotParseResponse(true). SetDoNotParseResponse(true)
Post("/goods_new")
resp, err := c.do(ctx, "GetGoodsNew", req, resty.MethodPost, "/goods_new")
if err != nil { if err != nil {
return nil, -1, fmt.Errorf("getting goods new: %w", err) return nil, -1, fmt.Errorf("getting goods new: %w", err)
} }
@ -214,7 +269,6 @@ func (c client) GetGoodsNew(
c.log.Error().Err(err).Msg("unable to close body") c.log.Error().Err(err).Msg("unable to close body")
} }
}() }()
if resp.IsError() { if resp.IsError() {
return nil, -1, errors.New("request was not successful") return nil, -1, errors.New("request was not successful")
} }
@ -226,3 +280,182 @@ func (c client) GetGoodsNew(
return mapResponseByOrder(response), response.RecordsTotal, nil return mapResponseByOrder(response), response.RecordsTotal, nil
} }
func (c *client) login(ctx context.Context, user, pass string) error {
req := c.http.R().
SetDoNotParseResponse(true).
SetFormData(map[string]string{
"username": user,
"password": pass,
})
resp, err := c.do(ctx, "login", req, resty.MethodPost, "https://eway.elevel.ru/")
if err != nil {
return fmt.Errorf("sending request: %w", err)
}
if resp.IsError() {
zerolog.Ctx(ctx).Warn().Int("code", resp.StatusCode()).Msg("bad response")
return entity.SimpleError("request was not successful")
}
return nil
}
func (c *client) do(ctx context.Context, name string, req *resty.Request, method string, url string) (resp *resty.Response, err error) {
resp, err = req.
EnableTrace().
Execute(method, url)
traceInfo := resp.Request.TraceInfo()
c.log.Debug().
Str("name", name).
Str("path", url).
Str("method", method).
Float64("elapsed", traceInfo.TotalTime.Seconds()).
Float64("response_time", traceInfo.ResponseTime.Seconds()).
Int("attempt", traceInfo.RequestAttempt).
Bool("success", resp.IsSuccess()).
Msg("request processed")
if err != nil {
return nil, fmt.Errorf("executing request: %w", err)
}
return resp, err
}
func (c *client) GetProductInfo(ctx context.Context, cart int64) (pi entity.GoodsItemInfo, err error) {
if c.workersPool == 0 {
return c.getProductInfo(ctx, cart)
}
responseBus := make(chan taskResult[entity.GoodsItemInfo], 1)
c.getProductInfoBus <- getProductInfoRequest{
cartID: cart,
response: responseBus,
}
select {
case response := <-responseBus:
return response.value, response.err
case <-ctx.Done():
return pi, ctx.Err()
}
}
func (c *client) getProductInfo(ctx context.Context, cartID int64) (pi entity.GoodsItemInfo, err error) {
reqpath := "https://eway.elevel.ru/product/" + strconv.Itoa(int(cartID)) + "/"
req := c.http.R().SetDoNotParseResponse(true).AddRetryCondition(func(r *resty.Response, err error) bool {
if r.Request.Attempt > 3 {
return false
}
return strings.Contains(err.Error(), "pipe")
})
c.log.Debug().Msg("using go query")
pi.Parameters = map[string]string{}
resp, err := c.do(ctx, "getProductInfo", req, resty.MethodGet, reqpath)
if err != nil {
return pi, fmt.Errorf("getting product info: %w", err)
}
defer func() {
errClose := resp.RawBody().Close()
if errClose == nil {
return
}
if err == nil {
err = errClose
return
}
c.log.Warn().Err(errClose).Msg("unable to close body")
}()
if resp.IsError() {
return pi, errors.New("request was not successful")
}
doc, err := goquery.NewDocumentFromReader(resp.RawBody())
if err != nil {
return pi, fmt.Errorf("makind new document: %w", err)
}
cleanText := func(t string) string {
return strings.TrimSuffix(strings.TrimSpace(t), ":")
}
const parametersSelector = "body > div.page-container > div.page-content > div.content-wrapper > div.content > div.row > div.col-md-4 > div > div > div:nth-child(6)"
const parametersInnerNode = "div.display-flex"
doc.
Find(parametersSelector).
Find(parametersInnerNode).
Each(func(i int, s *goquery.Selection) {
name := cleanText(s.Find("div").Eq(0).Text())
value := cleanText(s.Find("div.text-right").Text())
pi.Parameters[name] = value
})
const galleryPanelSelector = "div.gallery_panel"
const galleryImageSelector = "div.gallery_thumbnail > img"
doc.
Find(galleryPanelSelector).
Find(galleryImageSelector).
Each(func(i int, s *goquery.Selection) {
imageURL, ok := s.Attr("src")
if !ok || len(imageURL) == 0 {
return
}
pi.PhotoURLs = append(pi.PhotoURLs, imageURL)
})
return pi, nil
}
type getProductInfoRequest struct {
cartID int64
response chan taskResult[entity.GoodsItemInfo]
}
type taskResult[T any] struct {
value T
err error
}
func (c *client) productInfoWorker(
ctx context.Context,
in <-chan getProductInfoRequest,
) {
var req getProductInfoRequest
var ok bool
for {
select {
case <-ctx.Done():
return
case req, ok = <-in:
if !ok {
return
}
}
func() {
c.htmlParseSema <- struct{}{}
defer func() {
go func() {
time.Sleep(c.releaseSemaDelay)
<-c.htmlParseSema
}()
}()
pi, err := c.getProductInfo(ctx, req.cartID)
req.response <- taskResult[entity.GoodsItemInfo]{
value: pi,
err: err,
}
}()
}
}

171
internal/matcher/radix.go Normal file
View File

@ -0,0 +1,171 @@
package matcher
import (
"regexp"
"strings"
)
const radixWildcard = '*'
type radixNode struct {
exact bool
next map[rune]*radixNode
}
type radixMatcher struct {
root *radixNode
caseInsensitive bool
saved []string
regexps []*regexp.Regexp
}
type RadixOpt func(m *radixMatcher)
func RadixCaseInsensitive() RadixOpt {
return func(m *radixMatcher) {
m.caseInsensitive = true
}
}
func NewRadix(opts ...RadixOpt) *radixMatcher {
m := &radixMatcher{
root: &radixNode{
next: make(map[rune]*radixNode),
},
}
for _, opt := range opts {
opt(m)
}
return m
}
func (r *radixMatcher) MatchByPattern(value string) (pattern string) {
originValue := value
if r.caseInsensitive {
value = strings.ToLower(value)
}
node := r.root
lastIdx := len([]rune(value)) - 1
var sb strings.Builder
for i, v := range value {
var ok bool
if _, ok := node.next[radixWildcard]; ok {
_, _ = sb.WriteRune(radixWildcard)
return sb.String()
}
node, ok = node.next[v]
if !ok {
return r.findByRegexp(originValue)
}
sb.WriteRune(v)
if i != lastIdx {
continue
}
if !node.exact {
return r.findByRegexp(value)
}
return sb.String()
}
return r.findByRegexp(originValue)
}
func (r *radixMatcher) Match(value string) bool {
originValue := value
if r.caseInsensitive {
value = strings.ToLower(value)
}
node := r.root
lastIdx := len([]rune(value)) - 1
for i, v := range value {
var ok bool
if _, ok = node.next[radixWildcard]; ok {
return true
}
node, ok = node.next[v]
if !ok {
return r.findByRegexp(originValue) != ""
}
if i == lastIdx {
return node.exact
}
}
return r.findByRegexp(originValue) != ""
}
func (r *radixMatcher) findByRegexp(value string) (regexpPattern string) {
for _, rx := range r.regexps {
if rx.MatchString(value) {
return rx.String()
}
}
return ""
}
func (r *radixMatcher) RegisterRegexp(regexpPattern string) {
pattern := regexp.MustCompile(regexpPattern)
r.regexps = append(r.regexps, pattern)
}
func (r *radixMatcher) Register(pattern string) {
if len(pattern) == 0 {
panic("unable to handle empty pattern")
}
if r.caseInsensitive {
pattern = strings.ToLower(pattern)
}
r.saved = append(r.saved, pattern)
node := r.root
lastIdx := len([]rune(pattern)) - 1
for i, v := range pattern {
nextNode, ok := node.next[v]
if !ok {
nextNode = &radixNode{
next: make(map[rune]*radixNode),
}
node.next[v] = nextNode
}
node = nextNode
if v == '*' {
return
}
if i != lastIdx {
continue
}
node.exact = true
}
}
func (r *radixMatcher) Patterns() []string {
ownIdx := len(r.saved)
out := make([]string, len(r.saved)+len(r.regexps))
copy(out, r.saved[:ownIdx])
for i := 0; i < len(r.regexps); i++ {
idx := i + ownIdx
out[idx] = r.regexps[i].String()
}
return out
}

View File

@ -0,0 +1,118 @@
package matcher_test
import (
"testing"
"git.loyso.art/frx/eway/internal/matcher"
)
func TestRadixMatcherWithPattern(t *testing.T) {
m := matcher.NewRadix()
m.Register("aloha")
m.Register("hawaii")
m.Register("te*")
var tt = []struct {
name string
in string
pattern string
}{{
name: "should match exact",
in: "aloha",
pattern: "aloha",
}, {
name: "should match exact 2",
in: "hawaii",
pattern: "hawaii",
}, {
name: "should match pattern",
in: "test",
pattern: "te*",
}, {
name: "should not match",
in: "alohe",
}, {
name: "should not match 2",
in: "whoa",
}}
for _, tc := range tt {
tc := tc
t.Run(tc.name, func(t *testing.T) {
pattern := m.MatchByPattern(tc.in)
if pattern != tc.pattern {
t.Errorf("expected %s got %s", tc.pattern, pattern)
}
})
}
}
func TestRadixMatcher(t *testing.T) {
m := matcher.NewRadix()
m.Register("aloha")
m.Register("hawaii")
m.Register("te*")
var tt = []struct {
name string
in string
match bool
}{{
name: "should match exact",
in: "aloha",
match: true,
}, {
name: "should match exact 2",
in: "hawaii",
match: true,
}, {
name: "should match pattern",
in: "test",
match: true,
}, {
name: "should not match",
in: "alohe",
}, {
name: "should not match 2",
in: "whoa",
}}
for _, tc := range tt {
tc := tc
t.Run(tc.name, func(t *testing.T) {
match := m.Match(tc.in)
if match != tc.match {
t.Errorf("expected %t got %t", tc.match, match)
}
})
}
}
func TestRadixMatcherWildcard(t *testing.T) {
m := matcher.NewRadix()
m.Register("*")
var tt = []struct {
name string
in string
match bool
}{{
name: "should match exact",
in: "aloha",
match: true,
}, {
name: "should match exact 2",
in: "hawaii",
match: true,
}}
for _, tc := range tt {
tc := tc
t.Run(tc.name, func(t *testing.T) {
match := m.Match(tc.in)
if match != tc.match {
t.Errorf("expected %t got %t", tc.match, match)
}
})
}
}

View File

@ -10,7 +10,6 @@ import (
"git.loyso.art/frx/eway/internal/entity" "git.loyso.art/frx/eway/internal/entity"
badger "github.com/dgraph-io/badger/v4" badger "github.com/dgraph-io/badger/v4"
"github.com/rs/zerolog"
) )
type categoryClient struct { type categoryClient struct {
@ -105,24 +104,14 @@ func (c categoryClient) Get(ctx context.Context, id int64) (out entity.Category,
// Create new category inside DB. It also applies new id to it. // Create new category inside DB. It also applies new id to it.
func (c categoryClient) Create(ctx context.Context, name string) (out entity.Category, err error) { func (c categoryClient) Create(ctx context.Context, name string) (out entity.Category, err error) {
seqGen, err := c.db.GetSequence(categorySequenceIDKey, 1) nextid, err := c.seqGen.Next()
if err != nil {
return out, fmt.Errorf("getting sequence for categories: %w", err)
}
defer func() {
errRelese := seqGen.Release()
if errRelese != nil {
zerolog.Ctx(ctx).Warn().Err(err).Msg("unable to release seq")
}
}()
nextid, err := seqGen.Next()
if err != nil { if err != nil {
return out, fmt.Errorf("getting next id: %w", err) return out, fmt.Errorf("getting next id: %w", err)
} }
out = entity.Category{ out = entity.Category{
ID: int64(nextid), // Because first value from sequence generator is 0
ID: int64(nextid + 1),
Name: name, Name: name,
} }

View File

@ -1,47 +1,118 @@
package badger package badger
import ( import (
"fmt"
"time"
"unsafe"
"git.loyso.art/frx/eway/internal/entity" "git.loyso.art/frx/eway/internal/entity"
badger "github.com/dgraph-io/badger/v4" badger "github.com/dgraph-io/badger/v4"
) )
var ( var (
categorySequenceIDKey = []byte("cat:") categorySequenceIDKey = []byte("!!cat_seq!!")
queueSequenceIDKey = []byte("!!que_seq!!")
) )
type client struct { type client struct {
db *badger.DB db *badger.DB
nextCategoryIDSeq *badger.Sequence
// nextCategoryIDSeq *badger.Sequence nextQueueIDSeq *badger.Sequence
} }
func NewClient(db *badger.DB) (*client, error) { func NewClient(db *badger.DB) (*client, error) {
// categorySeqGen, err := db.GetSequence(categorySequenceIDKey, 10) categorySeqGen, err := db.GetSequence(categorySequenceIDKey, 10)
// if err != nil { if err != nil {
// return nil, fmt.Errorf("getting sequence for categories: %w", err) return nil, fmt.Errorf("getting sequence for categories: %w", err)
// } }
// queueSeqGen, err := db.GetSequence(queueSequenceIDKey, 10)
if err != nil {
return nil, fmt.Errorf("getting sequence for queues: %w", err)
}
return &client{ return &client{
db: db, db: db,
// nextCategoryIDSeq: categorySeqGen, nextCategoryIDSeq: categorySeqGen,
nextQueueIDSeq: queueSeqGen,
}, nil }, nil
} }
// Close closes the underlying sequences in the client. Should be called right before // Close closes the underlying sequences in the client. Should be called right before
// underlying *badger.DB closed. // underlying *badger.DB closed.
func (c *client) Close() error { func (c *client) Close() error {
// err := c.nextCategoryIDSeq.Release() err := c.nextCategoryIDSeq.Release()
// if err != nil { if err != nil {
// return fmt.Errorf("releasing next_category_sequence: %w", err) return fmt.Errorf("releasing next_category_sequence: %w", err)
// } }
return nil return nil
} }
func (c *client) Category() entity.CategoryRepository { func (c *client) Category() entity.CategoryRepository {
return newCategoryClient(c.db, nil) return newCategoryClient(c.db, c.nextCategoryIDSeq)
} }
func (c *client) GoodsItem() entity.GoodsItemRepository { func (c *client) GoodsItem() entity.GoodsItemRepository {
return newGoodsItemClient(c.db) return newGoodsItemClient(c.db, useJSON)
}
func (c *client) QueueClient() entity.MessageQueue {
nc := c.Table("queues")
return newQueueClient(nc, c.nextQueueIDSeq)
}
func (c *client) Table(name string) namedClient {
tableBytes := unsafe.Slice(unsafe.StringData("!!"+name+"!!"), len(name)+4)
return namedClient{
table: tableBytes,
db: c.db,
}
}
type namedClient struct {
table []byte
db *badger.DB
}
type putOpt func(*badger.Entry)
func withTTL(duration time.Duration) putOpt {
return func(e *badger.Entry) {
e.WithTTL(duration)
}
}
func (c *namedClient) Put(key, value []byte, opts ...putOpt) error {
return c.db.Update(func(txn *badger.Txn) error {
tableKey := c.makeKey(key)
entry := badger.NewEntry(tableKey, value)
for _, opt := range opts {
opt(entry)
}
return txn.SetEntry(entry)
})
}
func (c *namedClient) Get(key []byte) ([]byte, error) {
var out []byte
err := c.db.View(func(txn *badger.Txn) error {
item, err := txn.Get(c.makeKey(key))
if err != nil {
return err
}
out = make([]byte, item.ValueSize())
out, err = item.ValueCopy(out)
return err
})
if err != nil {
return nil, err
}
return out, nil
}
func (c *namedClient) makeKey(key []byte) (out []byte) {
return append(c.table, key...)
} }

View File

@ -30,21 +30,22 @@ func (za zerologAdapter) fmt(event *zerolog.Event, format string, args ...any) {
event.Msgf(strings.TrimSuffix(format, "\n"), args...) event.Msgf(strings.TrimSuffix(format, "\n"), args...)
} }
func Open(ctx context.Context, path string, log zerolog.Logger) (*badger.DB, error) { func Open(ctx context.Context, path string, debug bool, log zerolog.Logger) (*badger.DB, error) {
bl := zerologAdapter{ bl := zerologAdapter{
log: log.With().Str("db", "badger").Logger(), log: log.With().Str("db", "badger").Logger(),
} }
level := badger.WARNING
if debug {
level = badger.DEBUG
}
opts := badger.DefaultOptions(path). opts := badger.DefaultOptions(path).
WithLogger(bl). WithLogger(bl).
WithLoggingLevel(badger.INFO). WithLoggingLevel(level).
WithValueLogFileSize(4 << 20). WithValueLogFileSize(4 << 20).
WithDir(path). WithDir(path).
WithValueDir(path) WithValueDir(path)
// WithMaxLevels(4).
// WithMemTableSize(8 << 20).
// WithMetricsEnabled(true).
// WithCompactL0OnClose(true).
// WithBlockCacheSize(8 << 20)
db, err := badger.Open(opts) db, err := badger.Open(opts)
if err != nil { if err != nil {

View File

@ -1,29 +1,42 @@
package badger package badger
import ( import (
"bytes"
"context" "context"
"encoding/binary" "encoding/binary"
"encoding/json"
"errors" "errors"
"fmt" "fmt"
"runtime" "time"
"unsafe" "unsafe"
"git.loyso.art/frx/eway/internal/encoding/fbs" "git.loyso.art/frx/eway/internal/encoding/fbs"
"git.loyso.art/frx/eway/internal/entity" "git.loyso.art/frx/eway/internal/entity"
badger "github.com/dgraph-io/badger/v4" badger "github.com/dgraph-io/badger/v4"
"github.com/dgraph-io/badger/v4/pb"
"github.com/dgraph-io/ristretto/z" "github.com/dgraph-io/ristretto/z"
"github.com/rs/zerolog" "github.com/rs/zerolog"
) )
const useJSON = false
type goodsItemClient struct { type goodsItemClient struct {
db *badger.DB db *badger.DB
s itemSerializer[entity.GoodsItem]
} }
func newGoodsItemClient(db *badger.DB) *goodsItemClient { func newGoodsItemClient(db *badger.DB, serializeAsJSON bool) *goodsItemClient {
var s itemSerializer[entity.GoodsItem]
if serializeAsJSON {
s = goodsItemJSONSerializer{}
} else {
s = goodsItemFlatbufSerializer{}
}
return &goodsItemClient{ return &goodsItemClient{
db: db, db: db,
s: s,
} }
} }
@ -40,7 +53,7 @@ func (c *goodsItemClient) prefixedStr(key string) []byte {
return c.prefixed(keyBytes) return c.prefixed(keyBytes)
} }
func (c *goodsItemClient) prefixedIDByCartStr(key int64) []byte { func (c *goodsItemClient) prefixedIDByCartInt64(key int64) []byte {
var keyBytes [8]byte var keyBytes [8]byte
binary.BigEndian.PutUint64(keyBytes[:], uint64(key)) binary.BigEndian.PutUint64(keyBytes[:], uint64(key))
return c.prefixedIDByCart(keyBytes[:]) return c.prefixedIDByCart(keyBytes[:])
@ -65,7 +78,13 @@ func (c *goodsItemClient) ListIter(
} }
for _, kv := range list.GetKv() { for _, kv := range list.GetKv() {
bus <- fbs.ParseGoodsItem(kv.GetValue()) var gooditem entity.GoodsItem
gooditem, err = c.s.Deserialize(kv.GetValue())
if err != nil {
return fmt.Errorf("deserializing item: %w", err)
}
bus <- gooditem
} }
return nil return nil
@ -74,7 +93,7 @@ func (c *goodsItemClient) ListIter(
go func(ctx context.Context) { go func(ctx context.Context) {
defer close(bus) defer close(bus)
err := stream.Orchestrate(context.Background()) err := stream.Orchestrate(ctx)
if err != nil { if err != nil {
zerolog.Ctx(ctx).Warn().Err(err).Msg("unable to orchestrate") zerolog.Ctx(ctx).Warn().Err(err).Msg("unable to orchestrate")
} }
@ -86,6 +105,10 @@ func (c *goodsItemClient) ListIter(
func (c *goodsItemClient) List( func (c *goodsItemClient) List(
ctx context.Context, ctx context.Context,
) (out []entity.GoodsItem, err error) { ) (out []entity.GoodsItem, err error) {
if ctx.Err() != nil {
return nil, ctx.Err()
}
err = c.db.View(func(txn *badger.Txn) error { err = c.db.View(func(txn *badger.Txn) error {
opts := badger.DefaultIteratorOptions opts := badger.DefaultIteratorOptions
opts.PrefetchValues = true opts.PrefetchValues = true
@ -94,10 +117,21 @@ func (c *goodsItemClient) List(
defer iter.Close() defer iter.Close()
prefix := c.prefix() prefix := c.prefix()
var cursor int
for iter.Seek(prefix); iter.ValidForPrefix(prefix); iter.Next() { for iter.Seek(prefix); iter.ValidForPrefix(prefix); iter.Next() {
if ctx.Err() != nil {
return ctx.Err()
}
cursor++
current := iter.Item() current := iter.Item()
err = current.Value(func(val []byte) error { err = current.Value(func(val []byte) error {
goodsItem := fbs.ParseGoodsItem(val) var goodsItem entity.GoodsItem
goodsItem, err = c.s.Deserialize(val)
if err != nil {
return fmt.Errorf("deserializing: %w", err)
}
out = append(out, goodsItem) out = append(out, goodsItem)
return nil return nil
@ -120,6 +154,10 @@ func (c *goodsItemClient) Get(
ctx context.Context, ctx context.Context,
sku string, sku string,
) (out entity.GoodsItem, err error) { ) (out entity.GoodsItem, err error) {
if ctx.Err() != nil {
return out, ctx.Err()
}
err = c.db.View(func(txn *badger.Txn) error { err = c.db.View(func(txn *badger.Txn) error {
key := unsafe.Slice(unsafe.StringData(sku), len(sku)) key := unsafe.Slice(unsafe.StringData(sku), len(sku))
out, err = c.getBySKU(key, txn) out, err = c.getBySKU(key, txn)
@ -137,21 +175,26 @@ func (c *goodsItemClient) Get(
} }
func (c *goodsItemClient) GetByCart(ctx context.Context, id int64) (out entity.GoodsItem, err error) { func (c *goodsItemClient) GetByCart(ctx context.Context, id int64) (out entity.GoodsItem, err error) {
err = c.db.View(func(txn *badger.Txn) error { if ctx.Err() != nil {
var idByte [8]byte return out, ctx.Err()
binary.BigEndian.PutUint64(idByte[:], uint64(id)) }
item, err := txn.Get(c.prefixedIDByCart(idByte[:])) err = c.db.View(func(txn *badger.Txn) error {
idxKey := c.prefixedIDByCartInt64(id)
skuByCartIDItem, err := txn.Get(idxKey)
if err != nil { if err != nil {
return fmt.Errorf("getting key: %w", err) return fmt.Errorf("getting key: %w", err)
} }
sku := make([]byte, item.ValueSize()) sku := make([]byte, skuByCartIDItem.ValueSize())
sku, err = item.ValueCopy(sku) sku, err = skuByCartIDItem.ValueCopy(sku)
if err != nil { if err != nil {
return fmt.Errorf("getting value of idx: %w", err) return fmt.Errorf("getting value of idx: %w", err)
} }
// well, yeah, that's kind of dumb to trim prefix here and
// and prefix later, but who cares.
sku = bytes.TrimPrefix(sku, c.prefix())
out, err = c.getBySKU(sku, txn) out, err = c.getBySKU(sku, txn)
return err return err
}) })
@ -167,105 +210,82 @@ func (c *goodsItemClient) GetByCart(ctx context.Context, id int64) (out entity.G
} }
func (c *goodsItemClient) UpsertMany(ctx context.Context, items ...entity.GoodsItem) ([]entity.GoodsItem, error) { func (c *goodsItemClient) UpsertMany(ctx context.Context, items ...entity.GoodsItem) ([]entity.GoodsItem, error) {
return items, c.upsertByOne(ctx, items) return items, c.upsertByBatch(ctx, items)
} }
func (c *goodsItemClient) upsertByOne(ctx context.Context, items []entity.GoodsItem) error { func (c *goodsItemClient) Delete(ctx context.Context, sku string) (out entity.GoodsItem, err error) {
return c.db.Update(func(txn *badger.Txn) error { if ctx.Err() != nil {
for _, item := range items { return out, ctx.Err()
key := c.prefixedStr(item.Articul) }
value := fbs.MakeDomainGoodItemFinished(item)
valueIdx := make([]byte, len(key))
copy(valueIdx, key)
err := txn.Set(key, value) err = c.db.Update(func(txn *badger.Txn) error {
if err != nil { skuKey := c.prefixedStr(sku)
return err out, err = c.getBySKU(skuKey, txn)
} if err != nil {
return err
}
err = txn.Set(c.prefixedIDByCartStr(item.Cart), valueIdx) err = txn.Delete(skuKey)
if err != nil { if err != nil {
return err return fmt.Errorf("deleting key: %w", err)
} }
cartID := c.prefixedIDByCartInt64(out.Cart)
err = txn.Delete(cartID)
if err != nil {
return fmt.Errorf("deleting index key: %w", err)
} }
return nil return nil
}) })
}
func (c *goodsItemClient) upsertByStream(ctx context.Context, items []entity.GoodsItem) error {
stream := c.db.NewStreamWriter()
defer stream.Cancel()
err := stream.Prepare()
if err != nil { if err != nil {
return fmt.Errorf("preparing stream: %w", err) return entity.GoodsItem{}, err
} }
buf := z.NewBuffer(len(items), "sometag") return out, nil
for _, item := range items {
key := c.prefixedStr(item.Articul)
keyIdx := c.prefixedIDByCartStr(item.Cart)
value := fbs.MakeDomainGoodItemFinished(item)
itemKV := &pb.KV{Key: key, Value: value}
itemKVIdx := &pb.KV{Key: keyIdx, Value: key}
badger.KVToBuffer(itemKV, buf)
badger.KVToBuffer(itemKVIdx, buf)
}
err = stream.Write(buf)
if err != nil {
return fmt.Errorf("writing buf: %w", err)
}
err = stream.Flush()
if err != nil {
return fmt.Errorf("flushing changes: %w", err)
}
return nil
} }
func (c *goodsItemClient) upsertByBatch(ctx context.Context, items []entity.GoodsItem) error { func (c *goodsItemClient) upsertByBatch(ctx context.Context, items []entity.GoodsItem) error {
batch := c.db.NewWriteBatch() batch := c.db.NewWriteBatch()
defer func() { defer batch.Cancel()
println("closing batch")
batch.Cancel() createdAt := time.Now()
err := func() error {
for _, item := range items {
if ctx.Err() != nil {
return ctx.Err()
}
item.CreatedAt = createdAt
value, err := c.s.Serialize(item)
if err != nil {
return fmt.Errorf("serializing item: %w", err)
}
key := c.prefixedStr(item.Articul)
idxValue := make([]byte, len(key))
copy(idxValue, key)
coreEntry := badger.NewEntry(key, value)
if err := batch.SetEntry(coreEntry); err != nil {
return fmt.Errorf("setting core entry: %w", err)
}
idxKey := c.prefixedIDByCartInt64(item.Cart)
idxEntry := badger.NewEntry(idxKey, idxValue)
if err := batch.SetEntry(idxEntry); err != nil {
return fmt.Errorf("setting index entry: %w", err)
}
}
return nil
}() }()
if err != nil && !errors.Is(err, context.Canceled) {
log := zerolog.Ctx(ctx) return err
for _, item := range items {
select {
case <-ctx.Done():
break
default:
}
key := c.prefixedStr(item.Articul)
value := fbs.MakeDomainGoodItemFinished(item)
idxValue := make([]byte, len(key))
copy(idxValue, key)
coreEntry := badger.NewEntry(key, value)
if err := batch.SetEntry(coreEntry); err != nil {
log.Warn().Err(err).Msg("unable to set item, breaking")
break
}
idxKey := c.prefixedIDByCartStr(item.Cart)
idxEntry := badger.NewEntry(idxKey, idxValue)
if err := batch.SetEntry(idxEntry); err != nil {
log.Warn().Err(err).Msg("unable to set idx, breaking")
break
}
runtime.Gosched()
} }
println("flushing") err = batch.Flush()
err := batch.Flush()
runtime.Gosched()
if err != nil { if err != nil {
println("flush err", err.Error())
return fmt.Errorf("flushing changes: %w", err) return fmt.Errorf("flushing changes: %w", err)
} }
@ -279,8 +299,8 @@ func (c *goodsItemClient) getBySKU(sku []byte, txn *badger.Txn) (out entity.Good
} }
err = item.Value(func(val []byte) error { err = item.Value(func(val []byte) error {
out = fbs.ParseGoodsItem(val) out, err = c.s.Deserialize(val)
return nil return err
}) })
if err != nil { if err != nil {
return out, fmt.Errorf("reading value: %w", err) return out, fmt.Errorf("reading value: %w", err)
@ -288,3 +308,39 @@ func (c *goodsItemClient) getBySKU(sku []byte, txn *badger.Txn) (out entity.Good
return out, nil return out, nil
} }
type itemSerializer[T any] interface {
Serialize(T) ([]byte, error)
Deserialize([]byte) (T, error)
}
type goodsItemJSONSerializer struct{}
func (goodsItemJSONSerializer) Serialize(in entity.GoodsItem) ([]byte, error) {
data, err := json.Marshal(in)
if err != nil {
return nil, fmt.Errorf("marshalling data: %w", err)
}
return data, nil
}
func (goodsItemJSONSerializer) Deserialize(data []byte) (in entity.GoodsItem, err error) {
err = json.Unmarshal(data, &in)
return in, err
}
type goodsItemFlatbufSerializer struct{}
func (goodsItemFlatbufSerializer) Serialize(in entity.GoodsItem) ([]byte, error) {
out := fbs.MakeDomainGoodItemFinished(in)
return out, nil
}
func (goodsItemFlatbufSerializer) Deserialize(data []byte) (out entity.GoodsItem, err error) {
out, err = fbs.ParseGoodsItem(data)
if err != nil {
return entity.GoodsItem{}, err
}
return out, nil
}

View File

@ -0,0 +1,130 @@
package badger
import (
"bytes"
"context"
"encoding/binary"
"errors"
"fmt"
"time"
"git.loyso.art/frx/eway/internal/entity"
"github.com/dgraph-io/badger/v4"
)
type queueClient struct {
namedClient
seqGen *badger.Sequence
}
func newQueueClient(nc namedClient, seqGen *badger.Sequence) queueClient {
return queueClient{
namedClient: nc,
seqGen: seqGen,
}
}
type taskDB struct {
createdAt int64
body []byte
}
func (t taskDB) asBinary() []byte {
buf := make([]byte, 8+len(t.body))
binary.BigEndian.PutUint64(buf[:8], uint64(t.createdAt))
copy(buf[8:], t.body)
return buf
}
func (t *taskDB) fromBinary(data []byte) error {
if len(data) < 8 {
return errors.New("bad data")
}
t.createdAt = int64(binary.BigEndian.Uint64(data[:8]))
if len(data) == 8 {
return nil
}
t.body = make([]byte, len(t.body)-8)
copy(t.body, data[8:])
return nil
}
func (c queueClient) Publish(ctx context.Context, params entity.PublishParams) (task entity.Task, err error) {
task.ID, err = c.seqGen.Next()
if err != nil {
return task, fmt.Errorf("generating id: %w", err)
}
task.CreatedAt = time.Now()
tdb := taskDB{
createdAt: task.CreatedAt.Unix(),
body: params.Body,
}
var keyData [8]byte
binary.BigEndian.PutUint64(keyData[:], task.ID)
opts := make([]putOpt, 0, 1)
if !params.ExpiresAt.IsZero() {
duration := time.Until(params.ExpiresAt)
opts = append(opts, withTTL(duration))
}
err = c.Put(keyData[:], tdb.asBinary(), opts...)
if err != nil {
return entity.Task{}, fmt.Errorf("saving data: %w", err)
}
return task, nil
}
func (c queueClient) Consume(ctx context.Context) (task entity.Task, err error) {
err = c.db.View(func(txn *badger.Txn) error {
iterOpts := badger.IteratorOptions{
PrefetchSize: 1,
PrefetchValues: true,
Reverse: false,
AllVersions: false,
Prefix: c.table,
}
iter := txn.NewIterator(iterOpts)
defer iter.Close()
iter.Seek(c.table)
if !iter.ValidForPrefix(c.table) {
return entity.ErrNotFound
}
item := iter.Item()
keyData := item.KeyCopy(nil)
valueData, err := item.ValueCopy(nil)
if err != nil {
return fmt.Errorf("getting value: %w", err)
}
var tdb taskDB
err = tdb.fromBinary(valueData)
if err != nil {
return err
}
task.ID = binary.BigEndian.Uint64(bytes.TrimPrefix(keyData, c.table))
task.CreatedAt = time.Unix(tdb.createdAt, 0)
task.Body = tdb.body
if expiresAt := item.ExpiresAt(); expiresAt > 0 {
t := time.Now().Add(time.Second * time.Duration(expiresAt))
task.ExpiresAt = &t
}
return nil
})
if err != nil {
return entity.Task{}, err
}
return task, nil
}

View File

@ -1,8 +1,14 @@
package storage package storage
import "git.loyso.art/frx/eway/internal/entity" import (
"io"
"git.loyso.art/frx/eway/internal/entity"
)
type Repository interface { type Repository interface {
io.Closer
Category() entity.CategoryRepository Category() entity.CategoryRepository
GoodsItem() entity.GoodsItemRepository GoodsItem() entity.GoodsItemRepository
} }