frx/eway
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

support parsing item page info

Browse Source
This commit is contained in:
Aleksandr Trushkin
2024-02-03 20:28:33 +03:00
parent 6044e116f8
commit be6aec73df
10 changed files with 262 additions and 61 deletions
Download Patch File Download Diff File Expand all files Collapse all files

13
internal/encoding/fbs/GoodItem.go
View File

@ -169,8 +169,16 @@ func (rcv *GoodItem) MutateStock(n int16) bool {
return rcv._tab.MutateInt16Slot(28, n)
}
func (rcv *GoodItem) Parameters() []byte {
o := flatbuffers.UOffsetT(rcv._tab.Offset(30))
if o != 0 {
return rcv._tab.ByteVector(o + rcv._tab.Pos)
}
return nil
}
func GoodItemStart(builder *flatbuffers.Builder) {
builder.StartObject(13)
builder.StartObject(14)
}
func GoodItemAddSku(builder *flatbuffers.Builder, sku flatbuffers.UOffsetT) {
builder.PrependUOffsetTSlot(0, flatbuffers.UOffsetT(sku), 0)
@ -211,6 +219,9 @@ func GoodItemAddCart(builder *flatbuffers.Builder, cart int64) {
func GoodItemAddStock(builder *flatbuffers.Builder, stock int16) {
builder.PrependInt16Slot(12, stock, 0)
}
func GoodItemAddParameters(builder *flatbuffers.Builder, parameters flatbuffers.UOffsetT) {
builder.PrependUOffsetTSlot(13, flatbuffers.UOffsetT(parameters), 0)
}
func GoodItemEnd(builder *flatbuffers.Builder) flatbuffers.UOffsetT {
return builder.EndObject()
}

0
internal/encoding/fbs/GoodsItem.go → internal/encoding/fbs/GoodItems.go
View File

16
internal/encoding/fbs/helpers.go
View File

@ -2,7 +2,9 @@ package fbs
import (
"encoding/base64"
"encoding/json"
"fmt"
"strings"
"sync"
"git.loyso.art/frx/eway/internal/entity"
@ -62,7 +64,7 @@ func MakeDomainGoodItemFinished(in entity.GoodsItem) []byte {
func makeDomainGoodItem(builder *flatbuffers.Builder, in entity.GoodsItem) flatbuffers.UOffsetT {
sku := builder.CreateString(in.Articul)
photo := builder.CreateString(in.Photo)
photo := builder.CreateString(strings.Join(in.PhotoURLs, ";"))
name := builder.CreateString(in.Name)
descBase64 := base64.RawStdEncoding.EncodeToString([]byte(in.Description))
@ -73,6 +75,10 @@ func makeDomainGoodItem(builder *flatbuffers.Builder, in entity.GoodsItem) flatb
cat = builder.CreateString(in.Category)
}
t := builder.CreateString(in.Type)
parametersData, _ := json.Marshal(in.Parameters)
parameters := builder.CreateByteString(parametersData)
producer := builder.CreateString(in.Producer)
GoodItemStart(builder)
@ -91,6 +97,7 @@ func makeDomainGoodItem(builder *flatbuffers.Builder, in entity.GoodsItem) flatb
GoodItemAddTariff(builder, float32(in.TariffPrice))
GoodItemAddCart(builder, int64(in.Cart))
GoodItemAddStock(builder, int16(in.Stock))
GoodItemAddParameters(builder, parameters)
return GoodItemEnd(builder)
}
@ -98,7 +105,7 @@ func makeDomainGoodItem(builder *flatbuffers.Builder, in entity.GoodsItem) flatb
func ParseGoodsItem(data []byte) (item entity.GoodsItem, err error) {
itemFBS := GetRootAsGoodItem(data, 0)
item.Articul = string(itemFBS.Sku())
item.Photo = string(itemFBS.Photo())
item.PhotoURLs = strings.Split(string(itemFBS.Photo()), ";")
item.Name = string(itemFBS.Name())
description, err := base64.RawStdEncoding.DecodeString(string(itemFBS.Description()))
@ -118,6 +125,11 @@ func ParseGoodsItem(data []byte) (item entity.GoodsItem, err error) {
item.TariffPrice = float64(itemFBS.Tariff())
item.Cart = itemFBS.Cart()
item.Stock = int(itemFBS.Stock())
item.Parameters = map[string]string{}
err = json.Unmarshal(itemFBS.Parameters(), &item.Parameters)
if err != nil {
return item, fmt.Errorf("unmarshalling data: %w", err)
}
return item, nil
}

41
internal/entity/gooditem.go
View File

@ -8,19 +8,20 @@ import (
)
type GoodsItem struct {
Articul string `json:"sku"`
Photo string `json:"photo"`
Name string `json:"name"`
Description string `json:"description"`
Category string `json:"category"`
Type string `json:"type"`
Producer string `json:"producer"`
Pack int `json:"pack"`
Step int `json:"step"`
Price float64 `json:"price"`
TariffPrice float64 `json:"tariff_price"`
Cart int64 `json:"cart"`
Stock int `json:"stock"`
Articul string `json:"sku"`
PhotoURLs []string `json:"photo"`
Name string `json:"name"`
Description string `json:"description"`
Category string `json:"category"`
Type string `json:"type"`
Producer string `json:"producer"`
Pack int `json:"pack"`
Step int `json:"step"`
Price float64 `json:"price"`
TariffPrice float64 `json:"tariff_price"`
Cart int64 `json:"cart"`
Stock int `json:"stock"`
Parameters map[string]string `json:"parameters"`
}
type GoodsItemRaw struct {
@ -42,12 +43,18 @@ type GoodsItemRaw struct {
Other string
}
type GoodsItemInfo struct {
Parameters map[string]string
PhotoURLs []string
}
type MappedGoodsRemnants map[int]GoodsRemnant
type GoodsRemnant [4]int32
func MakeGoodsItem(
gi GoodsItemRaw,
remnants MappedGoodsRemnants,
info GoodsItemInfo,
) (out GoodsItem, err error) {
var name, desc string
var pack, step int
@ -95,9 +102,13 @@ func MakeGoodsItem(
return out, fmt.Errorf("getting step count (%s): %w", gi.Step, err)
}
photoURLs := info.PhotoURLs
if len(photoURLs) > 7 {
photoURLs = info.PhotoURLs[:7]
}
return GoodsItem{
Articul: gi.SKU,
Photo: gi.Photo,
Name: name,
Description: desc,
Category: gi.Category,
@ -109,5 +120,7 @@ func MakeGoodsItem(
TariffPrice: tariffPrice,
Cart: int64(gi.Cart),
Stock: int(remnants[int(gi.Cart)][0]),
PhotoURLs: photoURLs,
Parameters: info.Parameters,
}, nil
}

65
internal/interconnect/eway/client.go
View File

@ -11,12 +11,14 @@ import (
"reflect"
"strconv"
"strings"
"time"
"git.loyso.art/frx/eway/internal/config"
"git.loyso.art/frx/eway/internal/crypto"
"git.loyso.art/frx/eway/internal/entity"
"github.com/go-resty/resty/v2"
"github.com/gocolly/colly"
"github.com/rs/zerolog"
)
@ -26,6 +28,7 @@ type Client interface {
context.Context,
GetGoodsNewParams,
) (items []entity.GoodsItemRaw, total int, err error)
GetProductInfo(context.Context, int64) (entity.GoodsItemInfo, error)
}
type client struct {
@ -40,7 +43,6 @@ type Config config.Eway
func New(ctx context.Context, cfg Config, log zerolog.Logger) (client, error) {
httpclient := resty.New().
SetDebug(cfg.Debug).
// SetCookies(cookies).
SetBaseURL("https://eway.elevel.ru/api")
c := client{
@ -269,3 +271,64 @@ func (c client) login(ctx context.Context, user, pass string) error {
return nil
}
type ProductInfo struct {
ImageLinks []string
Parameters map[string]string
}
type parameterSelector struct {
Name string `selector:"div"`
Value string `selector:"div.text-right"`
}
func (c client) GetProductInfo(ctx context.Context, cart int64) (pi entity.GoodsItemInfo, err error) {
collector := colly.NewCollector(
colly.AllowedDomains("eway.elevel.ru"),
colly.AllowURLRevisit(),
)
pi.Parameters = map[string]string{}
start := time.Now()
defer func() {
elapsed := time.Since(start).Seconds()
c.log.Info().Float64("elapsed", elapsed).Msg("request processed")
}()
collector.OnHTML("body > div.page-container > div.page-content > div.content-wrapper > div.content > div.row > div.col-md-4 > div > div > div:nth-child(6)", func(e *colly.HTMLElement) {
e.ForEach("div.display-flex", func(i int, h *colly.HTMLElement) {
var s parameterSelector
err = h.Unmarshal(&s)
if err != nil {
c.log.Warn().Err(err).Msg("unable to unmarshal")
return
}
if s.Name == "" || s.Value == "" {
c.log.Warn().Msg("got empty key or value, skipping")
return
}
pi.Parameters[s.Name] = s.Value
})
})
collector.OnHTML("div.gallery_panel", func(h *colly.HTMLElement) {
h.ForEach("div.gallery_thumbnail > img", func(i int, h *colly.HTMLElement) {
imageURL := h.Attr("src")
if imageURL == "" {
return
}
pi.PhotoURLs = append(pi.PhotoURLs, imageURL)
})
})
err = collector.Visit("https://eway.elevel.ru/product/" + strconv.Itoa(int(cart)) + "/")
if err != nil {
return pi, fmt.Errorf("visiting site: %w", err)
}
return pi, nil
}

17
internal/storage/badger/queue.go
View File

@ -9,6 +9,7 @@ import (
"time"
"git.loyso.art/frx/eway/internal/entity"
"github.com/dgraph-io/badger/v4"
)
@ -70,10 +71,10 @@ func (c queueClient) Publish(ctx context.Context, params entity.PublishParams) (
binary.BigEndian.PutUint64(keyData[:], task.ID)
opts := make([]putOpt, 0, 1)
if !params.ExpiresAt.IsZero() {
duration := params.ExpiresAt.Sub(time.Now())
duration := time.Until(params.ExpiresAt)
opts = append(opts, withTTL(duration))
}
err = c.namedClient.Put(keyData[:], tdb.asBinary(), opts...)
err = c.Put(keyData[:], tdb.asBinary(), opts...)
if err != nil {
return entity.Task{}, fmt.Errorf("saving data: %w", err)
}
@ -83,11 +84,13 @@ func (c queueClient) Publish(ctx context.Context, params entity.PublishParams) (
func (c queueClient) Consume(ctx context.Context) (task entity.Task, err error) {
err = c.db.View(func(txn *badger.Txn) error {
iterOpts := badger.DefaultIteratorOptions
iterOpts.PrefetchSize = 1
iterOpts.PrefetchValues = true
iterOpts.Prefix = c.table
iterOpts.Reverse = false
iterOpts := badger.IteratorOptions{
PrefetchSize: 1,
PrefetchValues: true,
Reverse: false,
AllVersions: false,
Prefix: c.table,
}
iter := txn.NewIterator(iterOpts)
defer iter.Close()