frx/kurious
1
0
Fork 0
Code Issues 2 Pull Requests Actions Packages Projects 1 Releases Wiki Activity

list by pages and rate req limits

Browse Source
This commit is contained in:
Gitea
2023-12-15 13:00:59 +03:00
parent 2f3a6e35e9
commit 06c7997491
10 changed files with 204 additions and 116 deletions
Download Patch File Download Diff File Expand all files Collapse all files

45
internal/common/client/sravni/client.go
View File

@ -8,6 +8,7 @@ import (
"log/slog"
"strconv"
"strings"
"time"
"git.loyso.art/frx/kurious/internal/common/errors"
"git.loyso.art/frx/kurious/pkg/slices"
@ -16,6 +17,7 @@ import (
"github.com/go-resty/resty/v2"
"golang.org/x/net/html"
"golang.org/x/net/html/atom"
"golang.org/x/time/rate"
)
const (
@ -38,7 +40,8 @@ type Client interface {
func NewClient(ctx context.Context, log *slog.Logger, debug bool) (c *client, err error) {
c = &client{
log: log.With(slog.String("client", "sravni")),
log: log.With(slog.String("client", "sravni")),
limiter: rate.NewLimiter(rate.Every(time.Millisecond*100), 1),
http: resty.New().
SetBaseURL(baseURL).
SetDebug(debug),
@ -71,6 +74,8 @@ type client struct {
cachedMainPageInfo *PageState
validLearningTypes querySet
validCourseThematics querySet
limiter *rate.Limiter
}
func (c *client) GetMainPageState() (*PageState, error) {
@ -146,24 +151,24 @@ type listEducationProductsRequest struct {
// Filters
LearningType []string `json:"learningtype"`
CoursesThematics []string `json:"coursesThematics"`
Organizations []string `json:"organizations"` // list of ids
DictionatyFormatFilterNew []FilterFormat `json:"dictionaryFormatFilterNew"`
DictionaryTimeFilter []FilterTime `json:"dictionaryTimeFilter"`
DictionaryGraphicFilterNew []FilterGraphic `json:"dictionaryGraphicFilterNew"`
DictionatyLevelFilterNew []FilterLevel `json:"dictionaryLevelFilterNew"`
Organizations []string `json:"organizations,omitempty"` // list of ids
DictionatyFormatFilterNew []FilterFormat `json:"dictionaryFormatFilterNew,omitempty"`
DictionaryTimeFilter []FilterTime `json:"dictionaryTimeFilter,omitempty"`
DictionaryGraphicFilterNew []FilterGraphic `json:"dictionaryGraphicFilterNew,omitempty"`
DictionatyLevelFilterNew []FilterLevel `json:"dictionaryLevelFilterNew,omitempty"`
// Options
SubMentor []stringifiedBool `json:"sub-mentor"` // option with mentor
SubTimeFree []stringifiedBool `json:"sub-timeFree"` // option with trial
SubJobGarantSub []stringifiedBool `json:"sub-jobGarantsub"` // option for job garantee
SubPriceFree []stringifiedBool `json:"sub-priceFree"` // only free
SubInstallment []stringifiedBool `json:"sub-installment"` // with credit
SubIsCourseProfession []stringifiedBool `json:"sub-isCourseProfession"` // освоить профессию с нуля
DevelopSkills []stringifiedBool `json:"developSkills"` // развить навыки
SubMentor []stringifiedBool `json:"sub-mentor,omitempty"` // option with mentor
SubTimeFree []stringifiedBool `json:"sub-timeFree,omitempty"` // option with trial
SubJobGarantSub []stringifiedBool `json:"sub-jobGarantsub,omitempty"` // option for job garantee
SubPriceFree []stringifiedBool `json:"sub-priceFree,omitempty"` // only free
SubInstallment []stringifiedBool `json:"sub-installment,omitempty"` // with credit
SubIsCourseProfession []stringifiedBool `json:"sub-isCourseProfession,omitempty"` // освоить профессию с нуля
DevelopSkills []stringifiedBool `json:"developSkills,omitempty"` // развить навыки
NotSubIsWebinar string `json:"not-sub-isWebinar"`
NotB2B string `json:"not-b2b"`
AdvertisingOnly bool `json:"advertisingOnly"`
NotSubIsWebinar string `json:"not-sub-isWebinar,omitempty"`
NotB2B string `json:"not-b2b,omitempty"`
AdvertisingOnly bool `json:"advertisingOnly,omitempty"`
// Pagination and sorting
Limit int `json:"limit"`
@ -223,6 +228,10 @@ func (c *client) ListEducationalProducts(
Offset: params.Offset,
}
if err = c.limiter.Wait(ctx); err != nil {
return result, fmt.Errorf("waiting for limit: %w", err)
}
resp, err := c.http.R().
SetBody(reqParams).
SetResult(&result).
@ -304,6 +313,10 @@ func (c *client) ListEducationalProductsFilterCount(
},
}
if err = c.limiter.Wait(ctx); err != nil {
return result, fmt.Errorf("waiting for limit: %w", err)
}
var respData DataContainer[ProductsFilterCount]
resp, err := c.http.R().
SetBody(reqParams).