From 362d4524e33927ef4af445b56b69d8f026de64cf Mon Sep 17 00:00:00 2001 From: Aleksandr Trushkin Date: Sun, 4 Feb 2024 22:48:20 +0300 Subject: [PATCH] add sema and retries --- cmd/cli/main.go | 2 +- go.mod | 2 +- go.sum | 1 - internal/interconnect/eway/client.go | 44 ++++++++++++++++++++-------- 4 files changed, 33 insertions(+), 16 deletions(-) diff --git a/cmd/cli/main.go b/cmd/cli/main.go index 4698fa9..30c2f36 100644 --- a/cmd/cli/main.go +++ b/cmd/cli/main.go @@ -42,7 +42,7 @@ func main() { err := runcli(ctx) if err != nil { - fmt.Fprintf(os.Stderr, "unable to handle app: %v", err) + fmt.Fprintf(os.Stderr, "unable to handle app: %v\n", err) os.Exit(1) } diff --git a/go.mod b/go.mod index 850e854..b8f3249 100644 --- a/go.mod +++ b/go.mod @@ -8,6 +8,7 @@ require ( github.com/dgraph-io/badger/v4 v4.2.0 github.com/dgraph-io/ristretto v0.1.1 github.com/go-resty/resty/v2 v2.10.0 + github.com/gocolly/colly v1.2.0 github.com/google/flatbuffers v23.5.26+incompatible github.com/rodaine/table v1.1.1 github.com/rs/zerolog v1.31.0 @@ -24,7 +25,6 @@ require ( github.com/cespare/xxhash/v2 v2.2.0 // indirect github.com/dustin/go-humanize v1.0.0 // indirect github.com/gobwas/glob v0.2.3 // indirect - github.com/gocolly/colly v1.2.0 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/golang/glog v1.0.0 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect diff --git a/go.sum b/go.sum index 05d9235..304ce75 100644 --- a/go.sum +++ b/go.sum @@ -43,7 +43,6 @@ github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69 github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/glog v1.0.0 h1:nfP3RFugxnNRyKgeWd4oI1nYvXpxrx8ck8ZrcizshdQ= github.com/golang/glog v1.0.0/go.mod h1:EWib/APOK0SL3dFbYqvxE3UYd8E6s1ouQ7iEp/0LWV4= -github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6 h1:ZgQEtGgCBiWRM39fZuwSd1LwSqqSW0hOdXCYYDX0R3I= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= diff --git a/internal/interconnect/eway/client.go b/internal/interconnect/eway/client.go index 50fdbb4..2724d5d 100644 --- a/internal/interconnect/eway/client.go +++ b/internal/interconnect/eway/client.go @@ -35,33 +35,37 @@ type client struct { http *resty.Client log zerolog.Logger - ownerID string + htmlParseSema chan struct{} + releaseSemaDelay time.Duration + ownerID string } type Config config.Eway -func New(ctx context.Context, cfg Config, log zerolog.Logger) (client, error) { +func New(ctx context.Context, cfg Config, log zerolog.Logger) (*client, error) { httpclient := resty.New(). SetDebug(cfg.Debug). SetBaseURL("https://eway.elevel.ru/api") c := client{ - http: httpclient, - log: log.With().Str("client", "eway").Logger(), + http: httpclient, + log: log.With().Str("client", "eway").Logger(), + htmlParseSema: make(chan struct{}, 2), + releaseSemaDelay: time.Second / 2, } if cfg.SessionID == "" || cfg.SessionUser == "" { if cfg.Login == "" || cfg.Password == "" { - return client{}, entity.SimpleError("no auth method provided") + return nil, entity.SimpleError("no auth method provided") } decryptedPassword, err := crypto.Decrypt(cfg.Password) if err != nil { - return client{}, fmt.Errorf("decrypting password: %w", err) + return nil, fmt.Errorf("decrypting password: %w", err) } err = c.login(ctx, cfg.Login, decryptedPassword) if err != nil { - return client{}, err + return nil, err } log.Info().Msg("login successful") @@ -83,10 +87,10 @@ func New(ctx context.Context, cfg Config, log zerolog.Logger) (client, error) { c.http.SetCookies(cookies) } else { - return client{}, entity.SimpleError("bad configuration: either session_id and session_user should be set or login and password") + return nil, entity.SimpleError("bad configuration: either session_id and session_user should be set or login and password") } - return c, nil + return &c, nil } type GetGoodsNewParams struct { @@ -162,7 +166,7 @@ func mapResponseByOrder(response getGoodsNewResponse) (items []entity.GoodsItemR return items } -func (c client) GetGoodsRemnants( +func (c *client) GetGoodsRemnants( ctx context.Context, productIDs []int, ) (out entity.MappedGoodsRemnants, err error) { @@ -209,7 +213,7 @@ func (c client) GetGoodsRemnants( return out, nil } -func (c client) GetGoodsNew( +func (c *client) GetGoodsNew( ctx context.Context, params GetGoodsNewParams, ) (items []entity.GoodsItemRaw, total int, err error) { @@ -252,7 +256,7 @@ func (c client) GetGoodsNew( return mapResponseByOrder(response), response.RecordsTotal, nil } -func (c client) login(ctx context.Context, user, pass string) error { +func (c *client) login(ctx context.Context, user, pass string) error { resp, err := c.http.R(). SetDoNotParseResponse(true). SetFormData(map[string]string{ @@ -282,7 +286,19 @@ type parameterSelector struct { Value string `selector:"div.text-right"` } -func (c client) GetProductInfo(ctx context.Context, cart int64) (pi entity.GoodsItemInfo, err error) { +func (c *client) GetProductInfo(ctx context.Context, cart int64) (pi entity.GoodsItemInfo, err error) { + select { + case c.htmlParseSema <- struct{}{}: + defer func() { + go func() { + time.Sleep(c.releaseSemaDelay) + <-c.htmlParseSema + }() + }() + case <-ctx.Done(): + return pi, ctx.Err() + } + collector := colly.NewCollector( colly.AllowedDomains("eway.elevel.ru"), colly.AllowURLRevisit(), @@ -336,6 +352,8 @@ func (c client) GetProductInfo(ctx context.Context, cart int64) (pi entity.Goods return pi, ctx.Err() } } + + break } if err != nil { return pi, fmt.Errorf("visiting site: %w", err)