frx/kurious
1
0
Fork 0
Code Issues 2 Pull Requests Actions Packages Projects 1 Releases Wiki Activity

able to list products

Browse Source
This commit is contained in:
Gitea
2023-11-23 19:13:54 +03:00
parent f382d9e73b
commit 0553ea71c3
11 changed files with 774 additions and 116 deletions
Download Patch File Download Diff File Expand all files Collapse all files

324
internal/infrastructure/interfaceadapters/courses/sravni/client.go
View File

@ -6,9 +6,11 @@ import (
"fmt"
"io"
"log/slog"
"strconv"
"strings"
"git.loyso.art/frx/kurious/internal/domain"
"git.loyso.art/frx/kurious/pkg/utilities/slices"
"github.com/go-resty/resty/v2"
"golang.org/x/net/html"
@ -19,6 +21,14 @@ const (
baseURL = "https://www.sravni.ru/kursy"
)
type Client interface {
GetMainPageState() *PageState
ListEducationalProducts(
ctx context.Context,
params ListEducationProductsParams,
) (result ListEducationProductsResponse, err error)
}
func NewClient(ctx context.Context, log *slog.Logger, debug bool) (c *client, err error) {
c = &client{
log: log.With(slog.String("client", "sravni")),
@ -32,6 +42,18 @@ func NewClient(ctx context.Context, log *slog.Logger, debug bool) (c *client, er
return nil, err
}
getQuerySet := func(fields []field) querySet {
items := slices.Map(fields, func(f field) string {
return f.Value
})
return newQuerySet(items...)
}
dicts := c.cachedMainPageInfo.Props.InitialReduxState.Dictionaries.Data
c.validLearningTypes = getQuerySet(dicts.LearningType.Fields)
c.validCourseThematics = getQuerySet(dicts.CourseThematics.Fields)
return c, nil
}
@ -39,78 +61,128 @@ type client struct {
log *slog.Logger
http *resty.Client
cachedMainPageInfo *PageState
}
type PageStateRuntimeConfig struct {
BrandingURL string `json:"brandingUrl"`
Release string `json:"release"`
Environment string `json:"environment"`
Gateway string `json:"gatewayUrl"`
APIGatewayURL string `json:"apiGatewayUrl"`
EducationURL string `json:"educationUrl"`
PhoneVerifierURL string `json:"phoneVerifierUrl"`
WebPath string `json:"webPath"`
ServiceName string `json:"serviceName"`
OrgnazationURL string `json:"organizationsUrl"`
}
type Link struct {
URL string `json:"url"`
Title string `json:"title"`
}
type ReduxStatePrefooterItem struct {
Title string `json:"title"`
Links []Link `json:"links"`
}
type ReduxMetadata struct {
Data struct {
Prefooter []ReduxStatePrefooterItem `json:"prefooter"`
} `json:"data"`
}
type InitialReduxState struct {
Metadata ReduxMetadata `json:"metadata"`
Categories struct {
Data map[string]int `json:"data"`
} `json:"categories"`
}
type PageStateProperties struct {
InitialReduxState InitialReduxState `json:"initialReduxState"`
}
type PageState struct {
Page string `json:"page"`
Query map[string]string `json:"query"`
BuildID string `json:"buildId"`
RuntimeConfig PageStateRuntimeConfig `json:"runtimeConfig"`
Props PageStateProperties `json:"props"`
}
func (p *PageState) Clone() *PageState {
copiedState := *p
copiedState.Query = make(map[string]string, len(p.Query))
for k, v := range p.Query {
copiedState.Query[k] = v
}
data := p.Props.InitialReduxState.Categories.Data
copiedData := make(map[string]int, len(data))
for k, v := range data {
copiedData[k] = v
}
copiedState.Props.InitialReduxState.Categories.Data = copiedData
return &copiedState
cachedMainPageInfo *PageState
validLearningTypes querySet
validCourseThematics querySet
}
func (c *client) GetMainPageState() *PageState {
return c.cachedMainPageInfo.Clone()
}
type ListEducationProductsParams struct {
LearningType string
CoursesThematics string
Limit int
Offset int
}
type ListEducationProductsRequest struct {
Fingerprint string `json:"fingerPrint,omitempty"`
ProductName string `json:"productName,omitempty"`
AdvertisingOnly bool `json:"advertisingOnly"`
Location string `json:"location"`
OfferTypes []string `json:"offerTypes"`
IsMix bool `json:"isMix"`
MixRepeated bool `json:"mixRepeated"`
Fields []string `json:"fields"`
SortProperty string `json:"sortProperty"`
SortDirection string `json:"sortDirection"`
LearningType []string `json:"learningtype"`
CoursesThematics []string `json:"coursesThematics"`
NotSubIsWebinar string `json:"not-sub-isWebinar"`
NotB2B string `json:"not-b2b"`
Limit int `json:"limit"`
Offset int `json:"offset"`
}
type ListEducationProductsResponse struct {
Items []Course `json:"items"`
Organizations map[string]Organization `json:"organizations"`
TotalCount int `json:"totalCount"`
TotalCountAdv int `json:"totalCountAdv"`
}
func (c *client) ListEducationalProducts(
ctx context.Context,
params ListEducationProductsParams,
) (result ListEducationProductsResponse, err error) {
const urlPath = "/v1/education/products"
const defaultLimit = 1
const defaultSortProp = "advertising.position"
const defaultSortDirection = "asc"
if err = c.checkClientInited(); err != nil {
return result, err
}
if !c.validLearningTypes.hasValue(params.LearningType) {
return result, domain.NewValidationError("learning_type", "bad value")
}
if !c.validCourseThematics.hasValue(params.CoursesThematics) {
return result, domain.NewValidationError("courses_thematics", "bad value")
}
reqParams := ListEducationProductsRequest{
LearningType: []string{
params.LearningType,
},
CoursesThematics: []string{
params.CoursesThematics,
},
Fields: defaultProductFields,
SortProperty: defaultSortProp, // mayber sort by price?
SortDirection: defaultSortDirection,
NotSubIsWebinar: strconv.FormatBool(true),
NotB2B: strconv.FormatBool(true),
IsMix: true, // not sure why, but for better parsing
MixRepeated: true, // looks like this option should force to exclude duplicates
AdvertisingOnly: false, // If true, it will show only paid items.
Location: "", // TODO: get and fill location?
Fingerprint: "", // not sure it should be set.
ProductName: "", // looks like it does not affects anything
OfferTypes: nil, // for more precise filter but not needed.
Limit: defaultLimit,
Offset: 0,
}
req := c.http.R().
SetBody(reqParams).
SetResult(&result).
EnableTrace()
resp, err := req.Post(c.makeEducationURL(urlPath))
if err != nil {
return result, fmt.Errorf("making request: %w", err)
}
if resp.IsError() {
return result, fmt.Errorf("bad status code %d: %w", resp.StatusCode(), domain.ErrUnexpectedStatus)
}
return result, nil
}
func (c *client) makeEducationURL(path string) string {
if c.cachedMainPageInfo == nil {
return ""
}
return c.cachedMainPageInfo.RuntimeConfig.EducationURL + path
}
func (c *client) checkClientInited() error {
if c.cachedMainPageInfo == nil {
return ErrClientNotInited
}
return nil
}
func (c *client) getMainPageState(ctx context.Context) (*PageState, error) {
ctxLogger := restyCtxLogger{
ctx: ctx,
@ -130,7 +202,7 @@ func (c *client) getMainPageState(ctx context.Context) (*PageState, error) {
if resp.IsError() {
c.log.ErrorContext(ctx, "unable to proceed request", slog.String("body", string(resp.Body())))
return nil, fmt.Errorf("got %d, but expected success: %w", resp.StatusCode(), domain.UnexpectedStatusError)
return nil, fmt.Errorf("got %d, but expected success: %w", resp.StatusCode(), domain.ErrUnexpectedStatus)
}
traceInfo := resp.Request.TraceInfo()
@ -195,19 +267,111 @@ func (c *client) parsePageState(ctx context.Context, body io.Reader) (*PageState
return &out, nil
}
func findNode(parent *html.Node, eq func(*html.Node) (found, deeper bool)) *html.Node {
for child := parent.FirstChild; child != nil; child = child.NextSibling {
found, deeper := eq(child)
if found {
return child
}
if deeper {
deeperChild := findNode(child, eq)
if deeperChild != nil {
return deeperChild
}
}
var educationProductFields = newQuerySet(
"id",
"name",
"organization",
"advertising",
"discount",
"link",
"learningtype",
"dateStart",
"timeStart",
"timeAllHour",
"timeAllDay",
"timeAllMonth",
"isTermApproximately",
"dictionaryFormatFilterNew",
"dictionaryLevelFilterNew",
"price",
"priceAll",
"priceInstallment",
"courseImage",
"price",
"withoutDiscountPrice",
)
var defaultProductFields = must(educationProductFields.exactSubset(
"id",
"name",
"organization",
"advertising",
"discount",
"link",
"learningtype",
"dateStart",
"timeStart",
"timeAllHour",
"timeAllDay",
"timeAllMonth",
"price",
"priceAll",
"priceInstallment",
"courseImage",
"price",
"withoutDiscountPrice",
))
func must[T any](t T, err error) T {
if err != nil {
panic(err.Error())
}
return nil
return t
}
type querySet struct {
values []string
mappedValues map[string]struct{}
}
func (qs querySet) Values() []string {
out := make([]string, len(qs.values))
copy(out, qs.values)
return out
}
func (qs querySet) hasValue(value string) bool {
_, ok := qs.mappedValues[value]
return ok
}
func (qs querySet) exactSubset(values ...string) ([]string, error) {
out := make([]string, 0, len(values))
for _, value := range values {
if !qs.hasValue(value) {
return nil, fmt.Errorf("value %s was not found in set", value)
}
out = append(out, value)
}
return out, nil
}
// func (qs querySet) subset(values ...string) []string {
// out := make([]string, 0, len(values))
// for _, value := range values {
// if qs.hasValue(value) {
// out = append(out, value)
// }
// }
//
// return out
// }
func newQuerySet(values ...string) querySet {
qs := querySet{
values: make([]string, len(values)),
mappedValues: make(map[string]struct{}, len(values)),
}
for i, v := range values {
qs.values[i] = v
qs.mappedValues[v] = struct{}{}
}
return qs
}