list by pages and rate req limits

internal/common/client/sravni/client.go

@@ -8,6 +8,7 @@ import (
 	"log/slog"
 	"strconv"
 	"strings"
+	"time"
 
 	"git.loyso.art/frx/kurious/internal/common/errors"
 	"git.loyso.art/frx/kurious/pkg/slices"
@@ -16,6 +17,7 @@ import (
 	"github.com/go-resty/resty/v2"
 	"golang.org/x/net/html"
 	"golang.org/x/net/html/atom"
+	"golang.org/x/time/rate"
 )
 
 const (
@@ -38,7 +40,8 @@ type Client interface {
 
 func NewClient(ctx context.Context, log *slog.Logger, debug bool) (c *client, err error) {
 	c = &client{
-		log: log.With(slog.String("client", "sravni")),
+		log:     log.With(slog.String("client", "sravni")),
+		limiter: rate.NewLimiter(rate.Every(time.Millisecond*100), 1),
 		http: resty.New().
 			SetBaseURL(baseURL).
 			SetDebug(debug),
@@ -71,6 +74,8 @@ type client struct {
 	cachedMainPageInfo   *PageState
 	validLearningTypes   querySet
 	validCourseThematics querySet
+
+	limiter *rate.Limiter
 }
 
 func (c *client) GetMainPageState() (*PageState, error) {
@@ -146,24 +151,24 @@ type listEducationProductsRequest struct {
 	// Filters
 	LearningType               []string        `json:"learningtype"`
 	CoursesThematics           []string        `json:"coursesThematics"`
-	Organizations              []string        `json:"organizations"` // list of ids
-	DictionatyFormatFilterNew  []FilterFormat  `json:"dictionaryFormatFilterNew"`
-	DictionaryTimeFilter       []FilterTime    `json:"dictionaryTimeFilter"`
-	DictionaryGraphicFilterNew []FilterGraphic `json:"dictionaryGraphicFilterNew"`
-	DictionatyLevelFilterNew   []FilterLevel   `json:"dictionaryLevelFilterNew"`
+	Organizations              []string        `json:"organizations,omitempty"` // list of ids
+	DictionatyFormatFilterNew  []FilterFormat  `json:"dictionaryFormatFilterNew,omitempty"`
+	DictionaryTimeFilter       []FilterTime    `json:"dictionaryTimeFilter,omitempty"`
+	DictionaryGraphicFilterNew []FilterGraphic `json:"dictionaryGraphicFilterNew,omitempty"`
+	DictionatyLevelFilterNew   []FilterLevel   `json:"dictionaryLevelFilterNew,omitempty"`
 
 	// Options
-	SubMentor             []stringifiedBool `json:"sub-mentor"`             // option with mentor
-	SubTimeFree           []stringifiedBool `json:"sub-timeFree"`           // option with trial
-	SubJobGarantSub       []stringifiedBool `json:"sub-jobGarantsub"`       // option for job garantee
-	SubPriceFree          []stringifiedBool `json:"sub-priceFree"`          // only free
-	SubInstallment        []stringifiedBool `json:"sub-installment"`        // with credit
-	SubIsCourseProfession []stringifiedBool `json:"sub-isCourseProfession"` // освоить профессию с нуля
-	DevelopSkills         []stringifiedBool `json:"developSkills"`          // развить навыки
+	SubMentor             []stringifiedBool `json:"sub-mentor,omitempty"`             // option with mentor
+	SubTimeFree           []stringifiedBool `json:"sub-timeFree,omitempty"`           // option with trial
+	SubJobGarantSub       []stringifiedBool `json:"sub-jobGarantsub,omitempty"`       // option for job garantee
+	SubPriceFree          []stringifiedBool `json:"sub-priceFree,omitempty"`          // only free
+	SubInstallment        []stringifiedBool `json:"sub-installment,omitempty"`        // with credit
+	SubIsCourseProfession []stringifiedBool `json:"sub-isCourseProfession,omitempty"` // освоить профессию с нуля
+	DevelopSkills         []stringifiedBool `json:"developSkills,omitempty"`          // развить навыки
 
-	NotSubIsWebinar string `json:"not-sub-isWebinar"`
-	NotB2B          string `json:"not-b2b"`
-	AdvertisingOnly bool   `json:"advertisingOnly"`
+	NotSubIsWebinar string `json:"not-sub-isWebinar,omitempty"`
+	NotB2B          string `json:"not-b2b,omitempty"`
+	AdvertisingOnly bool   `json:"advertisingOnly,omitempty"`
 
 	// Pagination and sorting
 	Limit         int    `json:"limit"`
@@ -223,6 +228,10 @@ func (c *client) ListEducationalProducts(
 		Offset: params.Offset,
 	}
 
+	if err = c.limiter.Wait(ctx); err != nil {
+		return result, fmt.Errorf("waiting for limit: %w", err)
+	}
+
 	resp, err := c.http.R().
 		SetBody(reqParams).
 		SetResult(&result).
@@ -304,6 +313,10 @@ func (c *client) ListEducationalProductsFilterCount(
 		},
 	}
 
+	if err = c.limiter.Wait(ctx); err != nil {
+		return result, fmt.Errorf("waiting for limit: %w", err)
+	}
+
 	var respData DataContainer[ProductsFilterCount]
 	resp, err := c.http.R().
 		SetBody(reqParams).

internal/common/client/sravni/entities.go

@@ -223,6 +223,7 @@ type Course struct {
 	Discount                  CourseDiscount    `json:"discount"`
 	Link                      string            `json:"link"`
 	Learningtype              []string          `json:"learningtype"`
+	CourseThematics           []string          `json:"courseThematics"`
 	DateStart                 *time.Time        `json:"dateStart"`
 	TimeStart                 *time.Time        `json:"timeStart"`
 	TimeAllHour               *float64          `json:"timeAllHour"`

internal/common/xcontext/log.go

@@ -34,6 +34,10 @@ func LogError(ctx context.Context, log *slog.Logger, msg string, attrs ...slog.A
 	log.LogAttrs(ctx, slog.LevelError, msg, append(attrs, getLogFields(ctx)...)...)
 }
 
+func LogWithWarnError(ctx context.Context, log *slog.Logger, err error, msg string, attrs ...slog.Attr) {
+	LogWarn(ctx, log, msg, append(attrs, slog.Any("err", err))...)
+}
+
 func LogWithError(ctx context.Context, log *slog.Logger, err error, msg string, attrs ...slog.Attr) {
 	LogError(ctx, log, msg, append(attrs, slog.Any("err", err))...)
 }

internal/common/xcontext/wait.go

@@ -0,0 +1,15 @@
+package xcontext
+
+import (
+	"context"
+	"time"
+)
+
+func Wait(ctx context.Context, wait time.Duration) error {
+	select {
+	case <-ctx.Done():
+		return ctx.Err()
+	case <-time.After(wait):
+		return nil
+	}
+}