frx/kurious
1
0
Fork 0
Code Issues 2 Pull Requests Actions Packages Projects 1 Releases Wiki Activity

able to list products

Browse Source
This commit is contained in:
Gitea
2023-11-23 19:13:54 +03:00
parent f382d9e73b
commit 0553ea71c3
11 changed files with 774 additions and 116 deletions
Download Patch File Download Diff File Expand all files Collapse all files

324
internal/infrastructure/interfaceadapters/courses/sravni/client.go
View File

@ -6,9 +6,11 @@ import (
"fmt"
"io"
"log/slog"
"strconv"
"strings"
"git.loyso.art/frx/kurious/internal/domain"
"git.loyso.art/frx/kurious/pkg/utilities/slices"
"github.com/go-resty/resty/v2"
"golang.org/x/net/html"
@ -19,6 +21,14 @@ const (
baseURL = "https://www.sravni.ru/kursy"
)
type Client interface {
GetMainPageState() *PageState
ListEducationalProducts(
ctx context.Context,
params ListEducationProductsParams,
) (result ListEducationProductsResponse, err error)
}
func NewClient(ctx context.Context, log *slog.Logger, debug bool) (c *client, err error) {
c = &client{
log: log.With(slog.String("client", "sravni")),
@ -32,6 +42,18 @@ func NewClient(ctx context.Context, log *slog.Logger, debug bool) (c *client, er
return nil, err
}
getQuerySet := func(fields []field) querySet {
items := slices.Map(fields, func(f field) string {
return f.Value
})
return newQuerySet(items...)
}
dicts := c.cachedMainPageInfo.Props.InitialReduxState.Dictionaries.Data
c.validLearningTypes = getQuerySet(dicts.LearningType.Fields)
c.validCourseThematics = getQuerySet(dicts.CourseThematics.Fields)
return c, nil
}
@ -39,78 +61,128 @@ type client struct {
log *slog.Logger
http *resty.Client
cachedMainPageInfo *PageState
}
type PageStateRuntimeConfig struct {
BrandingURL string `json:"brandingUrl"`
Release string `json:"release"`
Environment string `json:"environment"`
Gateway string `json:"gatewayUrl"`
APIGatewayURL string `json:"apiGatewayUrl"`
EducationURL string `json:"educationUrl"`
PhoneVerifierURL string `json:"phoneVerifierUrl"`
WebPath string `json:"webPath"`
ServiceName string `json:"serviceName"`
OrgnazationURL string `json:"organizationsUrl"`
}
type Link struct {
URL string `json:"url"`
Title string `json:"title"`
}
type ReduxStatePrefooterItem struct {
Title string `json:"title"`
Links []Link `json:"links"`
}
type ReduxMetadata struct {
Data struct {
Prefooter []ReduxStatePrefooterItem `json:"prefooter"`
} `json:"data"`
}
type InitialReduxState struct {
Metadata ReduxMetadata `json:"metadata"`
Categories struct {
Data map[string]int `json:"data"`
} `json:"categories"`
}
type PageStateProperties struct {
InitialReduxState InitialReduxState `json:"initialReduxState"`
}
type PageState struct {
Page string `json:"page"`
Query map[string]string `json:"query"`
BuildID string `json:"buildId"`
RuntimeConfig PageStateRuntimeConfig `json:"runtimeConfig"`
Props PageStateProperties `json:"props"`
}
func (p *PageState) Clone() *PageState {
copiedState := *p
copiedState.Query = make(map[string]string, len(p.Query))
for k, v := range p.Query {
copiedState.Query[k] = v
}
data := p.Props.InitialReduxState.Categories.Data
copiedData := make(map[string]int, len(data))
for k, v := range data {
copiedData[k] = v
}
copiedState.Props.InitialReduxState.Categories.Data = copiedData
return &copiedState
cachedMainPageInfo *PageState
validLearningTypes querySet
validCourseThematics querySet
}
func (c *client) GetMainPageState() *PageState {
return c.cachedMainPageInfo.Clone()
}
type ListEducationProductsParams struct {
LearningType string
CoursesThematics string
Limit int
Offset int
}
type ListEducationProductsRequest struct {
Fingerprint string `json:"fingerPrint,omitempty"`
ProductName string `json:"productName,omitempty"`
AdvertisingOnly bool `json:"advertisingOnly"`
Location string `json:"location"`
OfferTypes []string `json:"offerTypes"`
IsMix bool `json:"isMix"`
MixRepeated bool `json:"mixRepeated"`
Fields []string `json:"fields"`
SortProperty string `json:"sortProperty"`
SortDirection string `json:"sortDirection"`
LearningType []string `json:"learningtype"`
CoursesThematics []string `json:"coursesThematics"`
NotSubIsWebinar string `json:"not-sub-isWebinar"`
NotB2B string `json:"not-b2b"`
Limit int `json:"limit"`
Offset int `json:"offset"`
}
type ListEducationProductsResponse struct {
Items []Course `json:"items"`
Organizations map[string]Organization `json:"organizations"`
TotalCount int `json:"totalCount"`
TotalCountAdv int `json:"totalCountAdv"`
}
func (c *client) ListEducationalProducts(
ctx context.Context,
params ListEducationProductsParams,
) (result ListEducationProductsResponse, err error) {
const urlPath = "/v1/education/products"
const defaultLimit = 1
const defaultSortProp = "advertising.position"
const defaultSortDirection = "asc"
if err = c.checkClientInited(); err != nil {
return result, err
}
if !c.validLearningTypes.hasValue(params.LearningType) {
return result, domain.NewValidationError("learning_type", "bad value")
}
if !c.validCourseThematics.hasValue(params.CoursesThematics) {
return result, domain.NewValidationError("courses_thematics", "bad value")
}
reqParams := ListEducationProductsRequest{
LearningType: []string{
params.LearningType,
},
CoursesThematics: []string{
params.CoursesThematics,
},
Fields: defaultProductFields,
SortProperty: defaultSortProp, // mayber sort by price?
SortDirection: defaultSortDirection,
NotSubIsWebinar: strconv.FormatBool(true),
NotB2B: strconv.FormatBool(true),
IsMix: true, // not sure why, but for better parsing
MixRepeated: true, // looks like this option should force to exclude duplicates
AdvertisingOnly: false, // If true, it will show only paid items.
Location: "", // TODO: get and fill location?
Fingerprint: "", // not sure it should be set.
ProductName: "", // looks like it does not affects anything
OfferTypes: nil, // for more precise filter but not needed.
Limit: defaultLimit,
Offset: 0,
}
req := c.http.R().
SetBody(reqParams).
SetResult(&result).
EnableTrace()
resp, err := req.Post(c.makeEducationURL(urlPath))
if err != nil {
return result, fmt.Errorf("making request: %w", err)
}
if resp.IsError() {
return result, fmt.Errorf("bad status code %d: %w", resp.StatusCode(), domain.ErrUnexpectedStatus)
}
return result, nil
}
func (c *client) makeEducationURL(path string) string {
if c.cachedMainPageInfo == nil {
return ""
}
return c.cachedMainPageInfo.RuntimeConfig.EducationURL + path
}
func (c *client) checkClientInited() error {
if c.cachedMainPageInfo == nil {
return ErrClientNotInited
}
return nil
}
func (c *client) getMainPageState(ctx context.Context) (*PageState, error) {
ctxLogger := restyCtxLogger{
ctx: ctx,
@ -130,7 +202,7 @@ func (c *client) getMainPageState(ctx context.Context) (*PageState, error) {
if resp.IsError() {
c.log.ErrorContext(ctx, "unable to proceed request", slog.String("body", string(resp.Body())))
return nil, fmt.Errorf("got %d, but expected success: %w", resp.StatusCode(), domain.UnexpectedStatusError)
return nil, fmt.Errorf("got %d, but expected success: %w", resp.StatusCode(), domain.ErrUnexpectedStatus)
}
traceInfo := resp.Request.TraceInfo()
@ -195,19 +267,111 @@ func (c *client) parsePageState(ctx context.Context, body io.Reader) (*PageState
return &out, nil
}
func findNode(parent *html.Node, eq func(*html.Node) (found, deeper bool)) *html.Node {
for child := parent.FirstChild; child != nil; child = child.NextSibling {
found, deeper := eq(child)
if found {
return child
}
if deeper {
deeperChild := findNode(child, eq)
if deeperChild != nil {
return deeperChild
}
}
var educationProductFields = newQuerySet(
"id",
"name",
"organization",
"advertising",
"discount",
"link",
"learningtype",
"dateStart",
"timeStart",
"timeAllHour",
"timeAllDay",
"timeAllMonth",
"isTermApproximately",
"dictionaryFormatFilterNew",
"dictionaryLevelFilterNew",
"price",
"priceAll",
"priceInstallment",
"courseImage",
"price",
"withoutDiscountPrice",
)
var defaultProductFields = must(educationProductFields.exactSubset(
"id",
"name",
"organization",
"advertising",
"discount",
"link",
"learningtype",
"dateStart",
"timeStart",
"timeAllHour",
"timeAllDay",
"timeAllMonth",
"price",
"priceAll",
"priceInstallment",
"courseImage",
"price",
"withoutDiscountPrice",
))
func must[T any](t T, err error) T {
if err != nil {
panic(err.Error())
}
return nil
return t
}
type querySet struct {
values []string
mappedValues map[string]struct{}
}
func (qs querySet) Values() []string {
out := make([]string, len(qs.values))
copy(out, qs.values)
return out
}
func (qs querySet) hasValue(value string) bool {
_, ok := qs.mappedValues[value]
return ok
}
func (qs querySet) exactSubset(values ...string) ([]string, error) {
out := make([]string, 0, len(values))
for _, value := range values {
if !qs.hasValue(value) {
return nil, fmt.Errorf("value %s was not found in set", value)
}
out = append(out, value)
}
return out, nil
}
// func (qs querySet) subset(values ...string) []string {
// out := make([]string, 0, len(values))
// for _, value := range values {
// if qs.hasValue(value) {
// out = append(out, value)
// }
// }
//
// return out
// }
func newQuerySet(values ...string) querySet {
qs := querySet{
values: make([]string, len(values)),
mappedValues: make(map[string]struct{}, len(values)),
}
for i, v := range values {
qs.values[i] = v
qs.mappedValues[v] = struct{}{}
}
return qs
}

190
internal/infrastructure/interfaceadapters/courses/sravni/entities.go Normal file
View File

@ -0,0 +1,190 @@
package sravni
import (
"time"
"git.loyso.art/frx/kurious/internal/domain"
)
const (
ErrClientNotInited domain.SimpleError = "client was not inited"
)
type PageStateRuntimeConfig struct {
BrandingURL string `json:"brandingUrl"`
Release string `json:"release"`
Environment string `json:"environment"`
Gateway string `json:"gatewayUrl"`
APIGatewayURL string `json:"apiGatewayUrl"`
EducationURL string `json:"educationUrl"`
PhoneVerifierURL string `json:"phoneVerifierUrl"`
WebPath string `json:"webPath"`
ServiceName string `json:"serviceName"`
OrgnazationURL string `json:"organizationsUrl"`
}
type Link struct {
URL string `json:"url"`
Title string `json:"title"`
}
type ReduxStatePrefooterItem struct {
Title string `json:"title"`
Links []Link `json:"links"`
}
type ReduxMetadata struct {
Data struct {
Prefooter []ReduxStatePrefooterItem `json:"prefooter"`
} `json:"data"`
}
type field struct {
Name string `json:"name"`
Value string `json:"value"`
}
type ReduxDictionaryContainer struct {
ID string `json:"_id"`
Alias string `json:"alias"`
Name string `json:"name"`
UserID string `json:"userId"`
Created time.Time `json:"created"`
Updated time.Time `json:"updated"`
Fields []field `json:"fields"`
}
type ReduxDictionaries struct {
Data struct {
CourseThematics ReduxDictionaryContainer `json:"coursesThematics"`
LearningType ReduxDictionaryContainer `json:"learningType"`
LearningTypeSelection ReduxDictionaryContainer `json:"learningTypeSelection"`
} `json:"data"`
}
type InitialReduxState struct {
Metadata ReduxMetadata `json:"metadata"`
Dictionaries ReduxDictionaries `json:"dictionaries"`
Categories struct {
Data map[string]int `json:"data"`
} `json:"categories"`
}
type PageStateProperties struct {
InitialReduxState InitialReduxState `json:"initialReduxState"`
}
type PageState struct {
Page string `json:"page"`
Query map[string]string `json:"query"`
BuildID string `json:"buildId"`
RuntimeConfig PageStateRuntimeConfig `json:"runtimeConfig"`
Props PageStateProperties `json:"props"`
}
func (p *PageState) Clone() *PageState {
copiedState := *p
copiedState.Query = make(map[string]string, len(p.Query))
for k, v := range p.Query {
copiedState.Query[k] = v
}
data := p.Props.InitialReduxState.Categories.Data
copiedData := make(map[string]int, len(data))
for k, v := range data {
copiedData[k] = v
}
copiedState.Props.InitialReduxState.Categories.Data = copiedData
return &copiedState
}
type CourseDiscount struct {
PromoCode string `json:"promoCode"`
PromoCodeType string `json:"promoCodeType"`
Percent int `json:"percent"`
EndDate time.Time `json:"endDate"`
EndTime any `json:"endTime"`
}
type CourseAdvertising struct {
Cost float64 `json:"cost"`
ButtonText string `json:"buttonText"`
ButtonMobileText string `json:"buttonMobileText"`
IsPartner bool `json:"isPartner"`
LabelText string `json:"labelText"`
Monetization struct {
Pixels struct {
Click string `json:"click"`
Display string `json:"display"`
} `json:"pixels"`
Kind string `json:"kind"`
} `json:"monetization"`
Dialog string `json:"dialog"`
SideBarBannerText string `json:"sideBarBannerText"`
OfferHighlightColor string `json:"offerHighlightColor"`
HasOffersID string `json:"hasOffersId"`
TrackingType string `json:"trackingType"`
Token []struct {
ID string `json:"_id"`
Token []string `json:"token"`
Updated time.Time `json:"updated"`
V int `json:"__v"`
} `json:"token"`
}
type Course struct {
ID string `json:"id"`
Name string `json:"name"`
Organization string `json:"organization"`
Discount CourseDiscount `json:"discount"`
Link string `json:"link"`
Learningtype []string `json:"learningtype"`
DateStart any `json:"dateStart"`
TimeStart any `json:"timeStart"`
TimeAllHour any `json:"timeAllHour"`
TimeAllDay any `json:"timeAllDay"`
TimeAllMonth int `json:"timeAllMonth"`
IsTermApproximately bool `json:"isTermApproximately"`
DictionaryFormatFilterNew []string `json:"dictionaryFormatFilterNew"`
DictionaryLevelFilterNew []string `json:"dictionaryLevelFilterNew"`
Price int `json:"price"`
PriceAll int `json:"priceAll"`
PriceInstallment int `json:"priceInstallment"`
CourseImage string `json:"courseImage"`
WithoutDiscountPrice int `json:"withoutDiscountPrice"`
Advertising CourseAdvertising `json:"advertising"`
}
type OrganizationName struct {
Short string `json:"short"`
Full string `json:"full"`
Prepositional string `json:"prepositional"`
Genitive string `json:"genitive"`
}
type RatingsInfo struct {
ComplexCalculatedRatingValue float64 `json:"complexCalculatedRatingValue"`
ParticipantsCount int `json:"participantsCount"`
Approved int `json:"approved"`
}
type Contacts struct {
Address string `json:"address"`
Phone []string `json:"phone"`
}
type Organization struct {
ID string `json:"id"`
Alias string `json:"alias"`
License string `json:"license"`
Name OrganizationName `json:"name"`
RatingsInfo RatingsInfo `json:"ratingsInfo"`
Contacts Contacts `json:"contacts"`
Logotypes struct {
Square string `json:"square"`
Web string `json:"web"`
Android string `json:"android"`
} `json:"logotypes"`
IsLabsPartner bool `json:"isLabsPartner"`
}

20
internal/infrastructure/interfaceadapters/courses/sravni/helpers.go Normal file
View File

@ -0,0 +1,20 @@
package sravni
import "golang.org/x/net/html"
func findNode(parent *html.Node, eq func(*html.Node) (found, deeper bool)) *html.Node {
for child := parent.FirstChild; child != nil; child = child.NextSibling {
found, deeper := eq(child)
if found {
return child
}
if deeper {
deeperChild := findNode(child, eq)
if deeperChild != nil {
return deeperChild
}
}
}
return nil
}