432 lines
9.7 KiB
Go
432 lines
9.7 KiB
Go
package eway
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"io"
|
|
"net/http"
|
|
"reflect"
|
|
"strconv"
|
|
"strings"
|
|
"sync"
|
|
"time"
|
|
|
|
"git.loyso.art/frx/eway/internal/config"
|
|
"git.loyso.art/frx/eway/internal/crypto"
|
|
"git.loyso.art/frx/eway/internal/entity"
|
|
|
|
"github.com/go-resty/resty/v2"
|
|
"github.com/gocolly/colly"
|
|
"github.com/rs/zerolog"
|
|
)
|
|
|
|
type Client interface {
|
|
GetGoodsRemnants(context.Context, []int) (entity.MappedGoodsRemnants, error)
|
|
GetGoodsNew(
|
|
context.Context,
|
|
GetGoodsNewParams,
|
|
) (items []entity.GoodsItemRaw, total int, err error)
|
|
GetProductInfo(context.Context, int64) (entity.GoodsItemInfo, error)
|
|
}
|
|
|
|
type client struct {
|
|
http *resty.Client
|
|
log zerolog.Logger
|
|
|
|
htmlParseSema chan struct{}
|
|
releaseSemaDelay time.Duration
|
|
getProductInfoBus chan getProductInfoRequest
|
|
ownerID string
|
|
workersPool int
|
|
workerswg *sync.WaitGroup
|
|
}
|
|
|
|
type Config config.Eway
|
|
|
|
func New(ctx context.Context, cfg Config, log zerolog.Logger) (*client, error) {
|
|
httpclient := resty.New().
|
|
SetDebug(cfg.Debug).
|
|
SetBaseURL("https://eway.elevel.ru/api")
|
|
|
|
c := client{
|
|
http: httpclient,
|
|
log: log.With().Str("client", "eway").Logger(),
|
|
htmlParseSema: make(chan struct{}, 2),
|
|
getProductInfoBus: make(chan getProductInfoRequest),
|
|
releaseSemaDelay: time.Second / 2,
|
|
workerswg: &sync.WaitGroup{},
|
|
}
|
|
|
|
for i := 0; i < cfg.WorkersPool; i++ {
|
|
c.workerswg.Add(1)
|
|
go func() {
|
|
defer c.workerswg.Done()
|
|
c.productInfoWorker(ctx, c.getProductInfoBus)
|
|
}()
|
|
}
|
|
|
|
if cfg.SessionID == "" || cfg.SessionUser == "" {
|
|
if cfg.Login == "" || cfg.Password == "" {
|
|
return nil, entity.SimpleError("no auth method provided")
|
|
}
|
|
|
|
decryptedPassword, err := crypto.Decrypt(cfg.Password)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("decrypting password: %w", err)
|
|
}
|
|
err = c.login(ctx, cfg.Login, decryptedPassword)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
log.Info().Msg("login successful")
|
|
} else if cfg.SessionID != "" && cfg.SessionUser != "" {
|
|
cookies := []*http.Cookie{
|
|
{
|
|
Name: "session_id",
|
|
Value: cfg.SessionID,
|
|
Domain: "eway.elevel.ru",
|
|
HttpOnly: true,
|
|
},
|
|
{
|
|
Name: "session_user",
|
|
Value: cfg.SessionUser,
|
|
Domain: "eway.elevel.ru",
|
|
HttpOnly: true,
|
|
},
|
|
}
|
|
|
|
c.http.SetCookies(cookies)
|
|
} else {
|
|
return nil, entity.SimpleError("bad configuration: either session_id and session_user should be set or login and password")
|
|
}
|
|
|
|
return &c, nil
|
|
}
|
|
|
|
type GetGoodsNewParams struct {
|
|
Draw int
|
|
Start int
|
|
// 100 is max
|
|
Length int
|
|
SearchInStocks bool
|
|
RemmantsAtleast int
|
|
}
|
|
|
|
type getGoodsNewResponse struct {
|
|
Draw string `json:"draw"`
|
|
RecordsFiltered int `json:"recordsFiltered"`
|
|
RecordsTotal int `json:"recordsTotal"`
|
|
Data [][]any `json:"data"`
|
|
Replacement bool `json:"replacement"`
|
|
}
|
|
|
|
type goodRemnant [4]int
|
|
|
|
func parseGoodItem(items []any) (out entity.GoodsItemRaw) {
|
|
valueOf := reflect.ValueOf(&out).Elem()
|
|
typeOf := valueOf.Type()
|
|
numField := valueOf.NumField()
|
|
|
|
for i := 0; i < numField; i++ {
|
|
field := valueOf.Field(i)
|
|
fieldType := typeOf.Field(i)
|
|
if fieldType.Type.Kind() == reflect.Slice &&
|
|
field.Type().Elem().Kind() != reflect.String {
|
|
continue
|
|
}
|
|
|
|
itemValue := reflect.ValueOf(items[i])
|
|
if items[i] == nil ||
|
|
(itemValue.CanAddr() && itemValue.IsNil()) ||
|
|
itemValue.IsZero() {
|
|
continue
|
|
}
|
|
|
|
if field.Type().Kind() != itemValue.Type().Kind() {
|
|
continue
|
|
}
|
|
|
|
// Dirty hack that accepts only strings.
|
|
if field.Type().Kind() == reflect.Slice {
|
|
values := items[i].([]any)
|
|
elemSlice := reflect.MakeSlice(typeOf.Field(i).Type, 0, field.Len())
|
|
for _, value := range values {
|
|
valueStr, ok := value.(string)
|
|
if ok {
|
|
elemSlice = reflect.Append(elemSlice, reflect.ValueOf(valueStr))
|
|
}
|
|
}
|
|
|
|
field.Set(elemSlice)
|
|
continue
|
|
}
|
|
|
|
field.Set(itemValue)
|
|
}
|
|
|
|
return out
|
|
}
|
|
|
|
func mapResponseByOrder(response getGoodsNewResponse) (items []entity.GoodsItemRaw) {
|
|
for _, columns := range response.Data {
|
|
gi := parseGoodItem(columns)
|
|
items = append(items, gi)
|
|
}
|
|
|
|
return items
|
|
}
|
|
|
|
func (c *client) GetGoodsRemnants(
|
|
ctx context.Context,
|
|
productIDs []int,
|
|
) (out entity.MappedGoodsRemnants, err error) {
|
|
if len(productIDs) == 0 {
|
|
return nil, nil
|
|
}
|
|
|
|
productsStr := make([]string, 0, len(productIDs))
|
|
for _, sku := range productIDs {
|
|
productsStr = append(productsStr, strconv.Itoa(sku))
|
|
}
|
|
|
|
resp, err := c.http.R().
|
|
SetFormData(map[string]string{
|
|
"products": strings.Join(productsStr, ","),
|
|
}).
|
|
SetDoNotParseResponse(true).
|
|
Post("/goods_remnants")
|
|
if err != nil {
|
|
return nil, fmt.Errorf("getting goods new: %w", err)
|
|
}
|
|
defer func() {
|
|
err = resp.RawBody().Close()
|
|
if err != nil {
|
|
c.log.Error().Err(err).Msg("unable to close body")
|
|
}
|
|
}()
|
|
|
|
if resp.IsError() {
|
|
return nil, errors.New("request was not successful")
|
|
}
|
|
|
|
data, err := io.ReadAll(resp.RawBody())
|
|
if err != nil {
|
|
return nil, fmt.Errorf("reading raw body: %w", err)
|
|
}
|
|
|
|
out = make(entity.MappedGoodsRemnants, len(productIDs))
|
|
err = json.NewDecoder(bytes.NewReader(data)).Decode(&out)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("decoding body: %w", err)
|
|
}
|
|
|
|
return out, nil
|
|
}
|
|
|
|
func (c *client) GetGoodsNew(
|
|
ctx context.Context,
|
|
params GetGoodsNewParams,
|
|
) (items []entity.GoodsItemRaw, total int, err error) {
|
|
var response getGoodsNewResponse
|
|
resp, err := c.http.R().
|
|
SetFormData(map[string]string{
|
|
"draw": strconv.Itoa(params.Draw),
|
|
"start": strconv.Itoa(params.Start),
|
|
"length": strconv.Itoa(params.Length),
|
|
"order[0][column]": "14",
|
|
"order[0][dir]": "desc",
|
|
"search[value]": "",
|
|
"search[regex]": "false",
|
|
"search_in_stocks": "on",
|
|
"remnants_atleast": "5",
|
|
}).
|
|
SetQueryParam("category_id", "0").
|
|
SetQueryParam("own", c.ownerID). // user id?
|
|
SetDoNotParseResponse(true).
|
|
Post("/goods_new")
|
|
if err != nil {
|
|
return nil, -1, fmt.Errorf("getting goods new: %w", err)
|
|
}
|
|
defer func() {
|
|
err = resp.RawBody().Close()
|
|
if err != nil {
|
|
c.log.Error().Err(err).Msg("unable to close body")
|
|
}
|
|
}()
|
|
|
|
if resp.IsError() {
|
|
return nil, -1, errors.New("request was not successful")
|
|
}
|
|
|
|
err = json.NewDecoder(resp.RawBody()).Decode(&response)
|
|
if err != nil {
|
|
return nil, -1, fmt.Errorf("decoding body: %w", err)
|
|
}
|
|
|
|
return mapResponseByOrder(response), response.RecordsTotal, nil
|
|
}
|
|
|
|
func (c *client) login(ctx context.Context, user, pass string) error {
|
|
resp, err := c.http.R().
|
|
SetDoNotParseResponse(true).
|
|
SetFormData(map[string]string{
|
|
"username": user,
|
|
"password": pass,
|
|
}).Post("https://eway.elevel.ru/")
|
|
if err != nil {
|
|
return fmt.Errorf("sending request: %w", err)
|
|
}
|
|
|
|
if resp.IsError() {
|
|
zerolog.Ctx(ctx).Warn().Int("code", resp.StatusCode()).Msg("bad response")
|
|
|
|
return entity.SimpleError("request was not successful")
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
type ProductInfo struct {
|
|
ImageLinks []string
|
|
Parameters map[string]string
|
|
}
|
|
|
|
type parameterSelector struct {
|
|
Name string `selector:"div"`
|
|
Value string `selector:"div.text-right"`
|
|
}
|
|
|
|
func (c *client) GetProductInfo(ctx context.Context, cart int64) (pi entity.GoodsItemInfo, err error) {
|
|
if c.workersPool == 0 {
|
|
return c.getProductInfo(ctx, cart)
|
|
}
|
|
|
|
responseBus := make(chan taskResult[entity.GoodsItemInfo], 1)
|
|
c.getProductInfoBus <- getProductInfoRequest{
|
|
cartID: cart,
|
|
response: responseBus,
|
|
}
|
|
|
|
select {
|
|
case response := <-responseBus:
|
|
return response.value, response.err
|
|
case <-ctx.Done():
|
|
return pi, ctx.Err()
|
|
}
|
|
}
|
|
|
|
func (c *client) getProductInfo(ctx context.Context, cart int64) (pi entity.GoodsItemInfo, err error) {
|
|
collector := colly.NewCollector(
|
|
colly.AllowedDomains("eway.elevel.ru"),
|
|
colly.AllowURLRevisit(),
|
|
)
|
|
|
|
pi.Parameters = map[string]string{}
|
|
|
|
start := time.Now()
|
|
defer func() {
|
|
elapsed := time.Since(start).Seconds()
|
|
c.log.Info().
|
|
Float64("elapsed", elapsed).
|
|
Int64("cart", cart).
|
|
Msg("request processed")
|
|
}()
|
|
|
|
collector.OnHTML("body > div.page-container > div.page-content > div.content-wrapper > div.content > div.row > div.col-md-4 > div > div > div:nth-child(6)", func(e *colly.HTMLElement) {
|
|
e.ForEach("div.display-flex", func(i int, h *colly.HTMLElement) {
|
|
var s parameterSelector
|
|
err = h.Unmarshal(&s)
|
|
if err != nil {
|
|
c.log.Warn().Err(err).Msg("unable to unmarshal")
|
|
return
|
|
}
|
|
|
|
if s.Name == "" || s.Value == "" {
|
|
c.log.Warn().Msg("got empty key or value, skipping")
|
|
return
|
|
}
|
|
|
|
pi.Parameters[s.Name] = s.Value
|
|
})
|
|
})
|
|
collector.OnHTML("div.gallery_panel", func(h *colly.HTMLElement) {
|
|
h.ForEach("div.gallery_thumbnail > img", func(i int, h *colly.HTMLElement) {
|
|
imageURL := h.Attr("src")
|
|
|
|
if imageURL == "" {
|
|
return
|
|
}
|
|
|
|
pi.PhotoURLs = append(pi.PhotoURLs, imageURL)
|
|
})
|
|
})
|
|
|
|
for i := 0; i < 3; i++ {
|
|
err = collector.Visit("https://eway.elevel.ru/product/" + strconv.Itoa(int(cart)) + "/")
|
|
if err != nil {
|
|
c.log.Warn().Err(err).Msg("unable to visit site, retrying...")
|
|
select {
|
|
case <-time.After(time.Second * 2):
|
|
continue
|
|
case <-ctx.Done():
|
|
return pi, ctx.Err()
|
|
}
|
|
}
|
|
|
|
break
|
|
}
|
|
if err != nil {
|
|
return pi, fmt.Errorf("visiting site: %w", err)
|
|
}
|
|
|
|
return pi, nil
|
|
}
|
|
|
|
type getProductInfoRequest struct {
|
|
cartID int64
|
|
response chan taskResult[entity.GoodsItemInfo]
|
|
}
|
|
|
|
type taskResult[T any] struct {
|
|
value T
|
|
err error
|
|
}
|
|
|
|
func (c *client) productInfoWorker(
|
|
ctx context.Context,
|
|
in <-chan getProductInfoRequest,
|
|
) {
|
|
var req getProductInfoRequest
|
|
var ok bool
|
|
for {
|
|
select {
|
|
case <-ctx.Done():
|
|
return
|
|
case req, ok = <-in:
|
|
if !ok {
|
|
return
|
|
}
|
|
}
|
|
|
|
func() {
|
|
c.htmlParseSema <- struct{}{}
|
|
defer func() {
|
|
go func() {
|
|
time.Sleep(c.releaseSemaDelay)
|
|
<-c.htmlParseSema
|
|
}()
|
|
}()
|
|
|
|
pi, err := c.getProductInfo(ctx, req.cartID)
|
|
req.response <- taskResult[entity.GoodsItemInfo]{
|
|
value: pi,
|
|
err: err,
|
|
}
|
|
}()
|
|
}
|
|
}
|