add sema and retries

cmd/cli/main.go

@@ -42,7 +42,7 @@ func main() {
 	err := runcli(ctx)
 
 	if err != nil {
-		fmt.Fprintf(os.Stderr, "unable to handle app: %v", err)
+		fmt.Fprintf(os.Stderr, "unable to handle app: %v\n", err)
 		os.Exit(1)
 	}
 

go.mod

@@ -8,6 +8,7 @@ require (
 	github.com/dgraph-io/badger/v4 v4.2.0
 	github.com/dgraph-io/ristretto v0.1.1
 	github.com/go-resty/resty/v2 v2.10.0
+	github.com/gocolly/colly v1.2.0
 	github.com/google/flatbuffers v23.5.26+incompatible
 	github.com/rodaine/table v1.1.1
 	github.com/rs/zerolog v1.31.0
@@ -24,7 +25,6 @@ require (
 	github.com/cespare/xxhash/v2 v2.2.0 // indirect
 	github.com/dustin/go-humanize v1.0.0 // indirect
 	github.com/gobwas/glob v0.2.3 // indirect
-	github.com/gocolly/colly v1.2.0 // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect
 	github.com/golang/glog v1.0.0 // indirect
 	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect

go.sum

@@ -43,7 +43,6 @@ github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69
 github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
 github.com/golang/glog v1.0.0 h1:nfP3RFugxnNRyKgeWd4oI1nYvXpxrx8ck8ZrcizshdQ=
 github.com/golang/glog v1.0.0/go.mod h1:EWib/APOK0SL3dFbYqvxE3UYd8E6s1ouQ7iEp/0LWV4=
-github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6 h1:ZgQEtGgCBiWRM39fZuwSd1LwSqqSW0hOdXCYYDX0R3I=
 github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
 github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE=
 github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=

internal/interconnect/eway/client.go

@@ -35,12 +35,14 @@ type client struct {
 	http *resty.Client
 	log  zerolog.Logger
 
+	htmlParseSema    chan struct{}
+	releaseSemaDelay time.Duration
 	ownerID          string
 }
 
 type Config config.Eway
 
-func New(ctx context.Context, cfg Config, log zerolog.Logger) (client, error) {
+func New(ctx context.Context, cfg Config, log zerolog.Logger) (*client, error) {
 	httpclient := resty.New().
 		SetDebug(cfg.Debug).
 		SetBaseURL("https://eway.elevel.ru/api")
@@ -48,20 +50,22 @@ func New(ctx context.Context, cfg Config, log zerolog.Logger) (client, error) {
 	c := client{
 		http:             httpclient,
 		log:              log.With().Str("client", "eway").Logger(),
+		htmlParseSema:    make(chan struct{}, 2),
+		releaseSemaDelay: time.Second / 2,
 	}
 
 	if cfg.SessionID == "" || cfg.SessionUser == "" {
 		if cfg.Login == "" || cfg.Password == "" {
-			return client{}, entity.SimpleError("no auth method provided")
+			return nil, entity.SimpleError("no auth method provided")
 		}
 
 		decryptedPassword, err := crypto.Decrypt(cfg.Password)
 		if err != nil {
-			return client{}, fmt.Errorf("decrypting password: %w", err)
+			return nil, fmt.Errorf("decrypting password: %w", err)
 		}
 		err = c.login(ctx, cfg.Login, decryptedPassword)
 		if err != nil {
-			return client{}, err
+			return nil, err
 		}
 
 		log.Info().Msg("login successful")
@@ -83,10 +87,10 @@ func New(ctx context.Context, cfg Config, log zerolog.Logger) (client, error) {
 
 		c.http.SetCookies(cookies)
 	} else {
-		return client{}, entity.SimpleError("bad configuration: either session_id and session_user should be set or login and password")
+		return nil, entity.SimpleError("bad configuration: either session_id and session_user should be set or login and password")
 	}
 
-	return c, nil
+	return &c, nil
 }
 
 type GetGoodsNewParams struct {
@@ -162,7 +166,7 @@ func mapResponseByOrder(response getGoodsNewResponse) (items []entity.GoodsItemR
 	return items
 }
 
-func (c client) GetGoodsRemnants(
+func (c *client) GetGoodsRemnants(
 	ctx context.Context,
 	productIDs []int,
 ) (out entity.MappedGoodsRemnants, err error) {
@@ -209,7 +213,7 @@ func (c client) GetGoodsRemnants(
 	return out, nil
 }
 
-func (c client) GetGoodsNew(
+func (c *client) GetGoodsNew(
 	ctx context.Context,
 	params GetGoodsNewParams,
 ) (items []entity.GoodsItemRaw, total int, err error) {
@@ -252,7 +256,7 @@ func (c client) GetGoodsNew(
 	return mapResponseByOrder(response), response.RecordsTotal, nil
 }
 
-func (c client) login(ctx context.Context, user, pass string) error {
+func (c *client) login(ctx context.Context, user, pass string) error {
 	resp, err := c.http.R().
 		SetDoNotParseResponse(true).
 		SetFormData(map[string]string{
@@ -282,7 +286,19 @@ type parameterSelector struct {
 	Value string `selector:"div.text-right"`
 }
 
-func (c client) GetProductInfo(ctx context.Context, cart int64) (pi entity.GoodsItemInfo, err error) {
+func (c *client) GetProductInfo(ctx context.Context, cart int64) (pi entity.GoodsItemInfo, err error) {
+	select {
+	case c.htmlParseSema <- struct{}{}:
+		defer func() {
+			go func() {
+				time.Sleep(c.releaseSemaDelay)
+				<-c.htmlParseSema
+			}()
+		}()
+	case <-ctx.Done():
+		return pi, ctx.Err()
+	}
+
 	collector := colly.NewCollector(
 		colly.AllowedDomains("eway.elevel.ru"),
 		colly.AllowURLRevisit(),
@@ -336,6 +352,8 @@ func (c client) GetProductInfo(ctx context.Context, cart int64) (pi entity.Goods
 				return pi, ctx.Err()
 			}
 		}
+
+		break
 	}
 	if err != nil {
 		return pi, fmt.Errorf("visiting site: %w", err)