setup parser

internal/common/client/sravni/client.go

@@ -11,6 +11,7 @@ import (
 
 	"git.loyso.art/frx/kurious/internal/common/errors"
 	"git.loyso.art/frx/kurious/pkg/slices"
+	"git.loyso.art/frx/kurious/pkg/xdefault"
 
 	"github.com/go-resty/resty/v2"
 	"golang.org/x/net/html"
@@ -76,6 +77,7 @@ type ListEducationProductsParams struct {
 	LearningType     string
 	CoursesThematics string
 
+	SortBy string
 	Limit  int
 	Offset int
 }
@@ -116,6 +118,8 @@ func (c *client) ListEducationalProducts(
 	const defaultLimit = 1
 	const defaultSortProp = "advertising.position"
 	const defaultSortDirection = "asc"
+	// TODO: find out should it be settable
+	const productName = "learning-courses"
 	if err = c.checkClientInited(); err != nil {
 		return result, err
 	}
@@ -123,33 +127,28 @@ func (c *client) ListEducationalProducts(
 	if !c.validLearningTypes.hasValue(params.LearningType) {
 		return result, errors.NewValidationError("learning_type", "bad value")
 	}
-	if !c.validCourseThematics.hasValue(params.CoursesThematics) {
+	if params.CoursesThematics != "" && !c.validCourseThematics.hasValue(params.CoursesThematics) {
 		return result, errors.NewValidationError("courses_thematics", "bad value")
 	}
 
 	reqParams := ListEducationProductsRequest{
-		LearningType: []string{
-			params.LearningType,
-		},
-		CoursesThematics: []string{
-			params.CoursesThematics,
-		},
+		LearningType:     valueAsArray(params.LearningType),
+		CoursesThematics: valueAsArray(params.CoursesThematics),
+		ProductName:      productName,
+		Fields:           defaultProductFields,
+		SortProperty:     defaultSortProp, // mayber sort by price?
+		SortDirection:    defaultSortDirection,
+		NotSubIsWebinar:  strconv.FormatBool(true),
+		NotB2B:           strconv.FormatBool(true),
+		IsMix:            false,      // not sure why, but for better parsing
+		MixRepeated:      true,       // looks like this option should force to exclude duplicates
+		AdvertisingOnly:  false,      // If true, it will show only paid items.
+		Location:         "",         // TODO: get and fill location?
+		Fingerprint:      "",         // not sure it should be set.
+		OfferTypes:       []string{}, // for more precise filter but not needed.
 
-		Fields:          defaultProductFields,
-		SortProperty:    defaultSortProp, // mayber sort by price?
-		SortDirection:   defaultSortDirection,
-		NotSubIsWebinar: strconv.FormatBool(true),
-		NotB2B:          strconv.FormatBool(true),
-		IsMix:           true,  // not sure why, but for better parsing
-		MixRepeated:     true,  // looks like this option should force to exclude duplicates
-		AdvertisingOnly: false, // If true, it will show only paid items.
-		Location:        "",    // TODO: get and fill location?
-		Fingerprint:     "",    // not sure it should be set.
-		ProductName:     "",    // looks like it does not affects anything
-		OfferTypes:      nil,   // for more precise filter but not needed.
-
-		Limit:  defaultLimit,
-		Offset: 0,
+		Limit:  xdefault.WithFallback(params.Limit, defaultLimit),
+		Offset: params.Offset,
 	}
 
 	req := c.http.R().
@@ -353,17 +352,6 @@ func (qs querySet) exactSubset(values ...string) ([]string, error) {
 
 }
 
-//	func (qs querySet) subset(values ...string) []string {
-//		out := make([]string, 0, len(values))
-//		for _, value := range values {
-//			if qs.hasValue(value) {
-//				out = append(out, value)
-//			}
-//		}
-//
-//		return out
-//	}
-
 func newQuerySet(values ...string) querySet {
 	qs := querySet{
 		values:       make([]string, len(values)),
@@ -377,3 +365,11 @@ func newQuerySet(values ...string) querySet {
 
 	return qs
 }
+
+func valueAsArray(value string) []string {
+	if value == "" {
+		return nil
+	}
+
+	return []string{value}
+}

internal/common/client/sravni/entities.go

@@ -102,9 +102,9 @@ func (p *PageState) Clone() *PageState {
 type CourseDiscount struct {
 	PromoCode     string    `json:"promoCode"`
 	PromoCodeType string    `json:"promoCodeType"`
-	Percent       int       `json:"percent"`
+	Percent       any       `json:"percent"`
 	EndDate       time.Time `json:"endDate"`
-	EndTime       any       `json:"endTime"`
+	EndTime       time.Time `json:"endTime"`
 }
 
 type CourseAdvertising struct {
@@ -140,17 +140,17 @@ type Course struct {
 	Discount                  CourseDiscount    `json:"discount"`
 	Link                      string            `json:"link"`
 	Learningtype              []string          `json:"learningtype"`
-	DateStart                 any               `json:"dateStart"`
-	TimeStart                 any               `json:"timeStart"`
-	TimeAllHour               any               `json:"timeAllHour"`
-	TimeAllDay                any               `json:"timeAllDay"`
-	TimeAllMonth              int               `json:"timeAllMonth"`
+	DateStart                 *time.Time        `json:"dateStart"`
+	TimeStart                 *time.Time        `json:"timeStart"`
+	TimeAllHour               *float64          `json:"timeAllHour"`
+	TimeAllDay                *float64          `json:"timeAllDay"`
+	TimeAllMonth              *float64          `json:"timeAllMonth"`
 	IsTermApproximately       bool              `json:"isTermApproximately"`
 	DictionaryFormatFilterNew []string          `json:"dictionaryFormatFilterNew"`
 	DictionaryLevelFilterNew  []string          `json:"dictionaryLevelFilterNew"`
-	Price                     int               `json:"price"`
-	PriceAll                  int               `json:"priceAll"`
-	PriceInstallment          int               `json:"priceInstallment"`
+	Price                     float64           `json:"price"`
+	PriceAll                  float64           `json:"priceAll"`
+	PriceInstallment          float64           `json:"priceInstallment"`
 	CourseImage               string            `json:"courseImage"`
 	WithoutDiscountPrice      int               `json:"withoutDiscountPrice"`
 	Advertising               CourseAdvertising `json:"advertising"`

internal/common/generator/ids.go

@@ -0,0 +1,12 @@
+package generator
+
+import (
+	"crypto/rand"
+	"encoding/hex"
+)
+
+func RandomInt64ID() string {
+	data := [8]byte{}
+	_, _ = rand.Read(data[:])
+	return hex.EncodeToString(data[:])
+}

internal/common/xcontext/log.go

@@ -34,6 +34,10 @@ func LogError(ctx context.Context, log *slog.Logger, msg string, attrs ...slog.A
 	log.LogAttrs(ctx, slog.LevelError, msg, append(attrs, getLogFields(ctx)...)...)
 }
 
+func LogWithError(ctx context.Context, log *slog.Logger, err error, msg string, attrs ...slog.Attr) {
+	LogError(ctx, log, msg, append(attrs, slog.Any("err", err))...)
+}
+
 func getLogFields(ctx context.Context) []slog.Attr {
 	store, _ := ctx.Value(ctxLogKey{}).(ctxLogAttrStore)
 	return store.attrs

internal/common/xlog/slog.go

@@ -0,0 +1,37 @@
+package xlog
+
+import (
+	"fmt"
+	"log/slog"
+)
+
+func LevelAsSLogLevel(level Level) slog.Level {
+	switch level {
+	case LevelDebug:
+		return slog.LevelDebug
+	case LevelInfo:
+		return slog.LevelInfo
+	case LevelWarn:
+		return slog.LevelWarn
+	case LevelError:
+		return slog.LevelError
+	default:
+		panic("unsupported level " + level.String())
+	}
+}
+
+func StringerAttr(name string, value fmt.Stringer) slog.Attr {
+	return slog.Any(name, StringerValue(value))
+}
+
+func StringerValue(s fmt.Stringer) slog.LogValuer {
+	return stringerValue{s}
+}
+
+type stringerValue struct {
+	fmt.Stringer
+}
+
+func (s stringerValue) LogValue() slog.Value {
+	return slog.StringValue(s.String())
+}

internal/common/xlog/xlog.go

@@ -0,0 +1,67 @@
+package xlog
+
+import (
+	"bytes"
+	"errors"
+)
+
+type Level uint8
+
+const (
+	LevelDebug Level = iota
+	LevelInfo
+	LevelWarn
+	LevelError
+)
+
+func (l *Level) UnmarshalText(data []byte) error {
+	switch levelstr := string(bytes.ToLower(data)); levelstr {
+	case "debug":
+		*l = LevelDebug
+	case "info":
+		*l = LevelInfo
+	case "warn":
+		*l = LevelWarn
+	case "error":
+		*l = LevelError
+	default:
+		return errors.New("unsupported level " + levelstr)
+	}
+
+	return nil
+}
+
+func (l Level) String() string {
+	switch l {
+	case LevelDebug:
+		return "debug"
+	case LevelInfo:
+		return "info"
+	case LevelWarn:
+		return "warn"
+	case LevelError:
+		return "error"
+	default:
+		return ""
+	}
+}
+
+type Format uint8
+
+const (
+	FormatText Format = iota
+	FormatJSON
+)
+
+func (f *Format) UnmarshalText(data []byte) error {
+	switch formatstr := string(bytes.ToLower(data)); formatstr {
+	case "debug":
+		*f = FormatText
+	case "info":
+		*f = FormatJSON
+	default:
+		return errors.New("unsupported format " + formatstr)
+	}
+
+	return nil
+}

internal/common/xslice/filter.go

@@ -0,0 +1,36 @@
+package xslice
+
+func Filter[T any](values []T, f func(T) bool) []T {
+	out := make([]T, 0, len(values))
+	for _, value := range values {
+		if f(value) {
+			out = append(out, value)
+		}
+	}
+
+	return out
+}
+
+func FilterInplace[T any](values []T, match func(T) bool) (newlen int) {
+	next := 0
+	for i := 0; i < len(values); i++ {
+		value := values[i]
+		if !match(value) {
+			continue
+		}
+
+		if next != i {
+			values[next] = value
+		}
+
+		next++
+	}
+
+	return next
+}
+
+func Not[T any](f func(T) bool) func(T) bool {
+	return func(t T) bool {
+		return !f(t)
+	}
+}

internal/common/xslice/filter_test.go

@@ -0,0 +1,52 @@
+package xslice_test
+
+import (
+	"testing"
+
+	"git.loyso.art/frx/kurious/internal/common/xslice"
+)
+
+func TestFilterInplace(t *testing.T) {
+	tt := []struct {
+		name   string
+		in     []int
+		check  func(int) bool
+		expLen int
+	}{{
+		name:   "empty",
+		in:     nil,
+		check:  nil,
+		expLen: 0,
+	}, {
+		name: "all match",
+		in:   []int{1, 2, 3},
+		check: func(int) bool {
+			return true
+		},
+		expLen: 3,
+	}, {
+		name: "all not match",
+		in:   []int{1, 2, 3},
+		check: func(int) bool {
+			return false
+		},
+		expLen: 0,
+	}, {
+		name: "some filtered out",
+		in:   []int{1, 2, 3, 4},
+		check: func(v int) bool {
+			return v%2 == 0
+		},
+		expLen: 2,
+	}}
+
+	for _, tc := range tt {
+		tc := tc
+		t.Run(tc.name, func(t *testing.T) {
+			gotLen := xslice.FilterInplace(tc.in, tc.check)
+			if gotLen != tc.expLen {
+				t.Errorf("exp %d got %d", tc.expLen, gotLen)
+			}
+		})
+	}
+}

internal/common/xslice/foreach.go

@@ -0,0 +1,7 @@
+package xslice
+
+func ForEach[T any](items []T, f func(T)) {
+	for _, item := range items {
+		f(item)
+	}
+}

internal/common/xslice/map.go

@@ -0,0 +1,10 @@
+package xslice
+
+func Map[T, U any](in []T, f func(T) U) []U {
+	out := make([]U, len(in))
+	for i, value := range in {
+		out[i] = f(value)
+	}
+
+	return out
+}

internal/kurious/adapters/ydb_course_repository.go

@@ -79,8 +79,78 @@ type ydbCourseRepository struct {
 	log *slog.Logger
 }
 
-func (r *ydbCourseRepository) List(ctx context.Context, params domain.ListCoursesParams) ([]domain.Course, error) {
-	return nil, nil
+func (r *ydbCourseRepository) List(ctx context.Context, params domain.ListCoursesParams) (courses []domain.Course, err error) {
+	const queryName = "list"
+
+	courses = make([]domain.Course, 0, 4_000)
+	readTx := table.TxControl(
+		table.BeginTx(
+			table.WithOnlineReadOnly(),
+		),
+		table.CommitTx(),
+	)
+	err = r.db.Table().Do(
+		ctx,
+		func(ctx context.Context, s table.Session) error {
+			start := time.Now()
+			defer func() {
+				since := time.Since(start)
+				xcontext.LogInfo(
+					ctx, r.log,
+					"executed query",
+					slog.String("name", queryName),
+					slog.Duration("elapsed", since),
+				)
+			}()
+
+			_, res, err := s.Execute(
+				ctx,
+				readTx,
+				`SELECT
+          id,
+          external_id,
+          source_type,
+          source_name,
+          organization_id,
+          origin_link,
+          image_link,
+          name,
+          description,
+          full_price,
+          discount,
+          duration,
+          starts_at,
+          created_at,
+          updated_at,
+          deleted_at
+        FROM
+          courses
+        `,
+				table.NewQueryParameters(),
+				options.WithCollectStatsModeBasic(),
+			)
+			if err != nil {
+				return fmt.Errorf("executing: %w", err)
+			}
+
+			for res.NextResultSet(ctx) {
+				for res.NextRow() {
+					var cdb courseDB
+					_ = res.ScanNamed(cdb.getNamedValues()...)
+					courses = append(courses, mapCourseDB(cdb))
+				}
+			}
+			if err = res.Err(); err != nil {
+				return err
+			}
+			return nil
+		},
+		table.WithIdempotent())
+	if err != nil {
+		return nil, err
+	}
+
+	return courses, err
 }
 
 func (r *ydbCourseRepository) Get(ctx context.Context, id string) (course domain.Course, err error) {
@@ -194,6 +264,69 @@ func createCourseParamsAsStruct(params domain.CreateCourseParams) types.Value {
 	)
 }
 
+func (r *ydbCourseRepository) CreateBatch(ctx context.Context, params ...domain.CreateCourseParams) error {
+	// -- PRAGMA TablePathPrefix("courses");
+	const upsertQuery = `DECLARE $courseData AS List<Struct<
+        id: Text,
+        external_id: Optional<Text>,
+        name: Text,
+        source_type: Text,
+        source_name: Optional<Text>,
+        organization_id: Text,
+        origin_link: Text,
+        image_link: Text,
+        description: Text,
+        full_price: Double,
+        discount: Double,
+        duration: Interval,
+        starts_at: Datetime,
+        created_at: Datetime,
+        updated_at: Datetime,
+        deleted_at: Optional<Datetime>>>;
+
+    REPLACE INTO
+      courses
+    SELECT
+      id,
+      external_id,
+      name,
+      source_type,
+      source_name,
+      organization_id,
+      origin_link,
+      image_link,
+      description,
+      full_price,
+      discount,
+      duration,
+      starts_at,
+      created_at,
+      updated_at,
+      deleted_at
+    FROM AS_TABLE($courseData);`
+
+	writeTx := table.TxControl(
+		table.BeginTx(
+			table.WithSerializableReadWrite(),
+		),
+		table.CommitTx(),
+	)
+	err := r.db.Table().Do(ctx, func(ctx context.Context, s table.Session) error {
+		listValues := mapSlice(params, createCourseParamsAsStruct)
+		queryParams := table.NewQueryParameters(
+			table.ValueParam("$courseData", types.ListValue(listValues...)),
+		)
+		_, _, err := s.Execute(ctx, writeTx, upsertQuery, queryParams)
+		if err != nil {
+			return fmt.Errorf("executing query: %w", err)
+		}
+
+		return nil
+	})
+
+	return err
+}
+
 func (r *ydbCourseRepository) Create(ctx context.Context, params domain.CreateCourseParams) (domain.Course, error) {
 	// -- PRAGMA TablePathPrefix("courses");
 	const upsertQuery = `DECLARE $courseData AS List<Struct<
@@ -381,3 +514,12 @@ func mapCourseDB(cdb courseDB) domain.Course {
 		DeletedAt:      nullable.NewValuePtr(cdb.DeletedAt),
 	}
 }
+
+func mapSlice[T, U any](in []T, f func(T) U) []U {
+	out := make([]U, len(in))
+	for i, value := range in {
+		out[i] = f(value)
+	}
+
+	return out
+}

internal/kurious/app/app.go

@@ -6,8 +6,9 @@ import (
 )
 
 type Commands struct {
-	InsertCourse command.CreateCourseHandler
-	DeleteCourse command.DeleteCourseHandler
+	InsertCourses command.CreateCoursesHandler
+	InsertCourse  command.CreateCourseHandler
+	DeleteCourse  command.DeleteCourseHandler
 }
 
 type Queries struct {

internal/kurious/app/command/createcourse.go

@@ -8,6 +8,7 @@ import (
 
 	"git.loyso.art/frx/kurious/internal/common/decorator"
 	"git.loyso.art/frx/kurious/internal/common/nullable"
+	"git.loyso.art/frx/kurious/internal/common/xslice"
 	"git.loyso.art/frx/kurious/internal/kurious/domain"
 )
 
@@ -52,3 +53,36 @@ func (h createCourseHandler) Handle(ctx context.Context, cmd CreateCourse) error
 
 	return nil
 }
+
+type CreateCourses struct {
+	Courses []CreateCourse
+}
+
+type CreateCoursesHandler decorator.CommandHandler[CreateCourses]
+
+type createCoursesHandler struct {
+	repo domain.CourseRepository
+}
+
+func NewCreateCoursesHandler(
+	repo domain.CourseRepository,
+	log *slog.Logger,
+) CreateCoursesHandler {
+	h := createCoursesHandler{
+		repo: repo,
+	}
+
+	return decorator.ApplyCommandDecorators(h, log)
+}
+
+func (h createCoursesHandler) Handle(ctx context.Context, cmd CreateCourses) error {
+	params := xslice.Map(cmd.Courses, func(in CreateCourse) (out domain.CreateCourseParams) {
+		return domain.CreateCourseParams(in)
+	})
+	err := h.repo.CreateBatch(ctx, params...)
+	if err != nil {
+		return fmt.Errorf("creating course: %w", err)
+	}
+
+	return nil
+}

internal/kurious/domain/repository.go

@@ -40,6 +40,7 @@ type CourseRepository interface {
 	// Should return ErrNotFound in case course not found.
 	GetByExternalID(ctx context.Context, id string) (Course, error)
 
+	CreateBatch(context.Context, ...CreateCourseParams) error
 	// Create course, but might fail in case of
 	// unique constraint violation.
 	Create(context.Context, CreateCourseParams) (Course, error)

internal/kurious/ports/background.go

@@ -0,0 +1,129 @@
+package ports
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"log/slog"
+	"strings"
+	"time"
+
+	"git.loyso.art/frx/kurious/internal/common/client/sravni"
+	"git.loyso.art/frx/kurious/internal/common/errors"
+	"git.loyso.art/frx/kurious/internal/common/nullable"
+	"git.loyso.art/frx/kurious/internal/common/xcontext"
+	"git.loyso.art/frx/kurious/internal/common/xlog"
+	"git.loyso.art/frx/kurious/internal/kurious/ports/background"
+	"git.loyso.art/frx/kurious/internal/kurious/service"
+
+	"github.com/robfig/cron/v3"
+)
+
+type BackgroundProcess struct {
+	scheduler *cron.Cron
+	log       *slog.Logger
+
+	syncSravniHandlerEntryID nullable.Value[cron.EntryID]
+	syncSravniHandler        handler
+}
+
+func NewBackgroundProcess(ctx context.Context, log *slog.Logger) *BackgroundProcess {
+	clog := xlog.WrapSLogger(ctx, log)
+	scheduler := cron.New(
+		cron.WithSeconds(),
+		cron.WithChain(
+			cron.Recover(clog),
+		),
+	)
+
+	bp := &BackgroundProcess{
+		scheduler: scheduler,
+		log:       log,
+	}
+
+	return bp
+}
+
+func (bp *BackgroundProcess) RegisterSyncSravniHandler(
+	ctx context.Context,
+	svc service.Application,
+	client sravni.Client,
+	cronValue string,
+) (err error) {
+	const handlerName = "sync_sravni_handler"
+
+	bp.syncSravniHandler = background.NewSyncSravniHandler(svc, client, bp.log)
+
+	if cronValue == "" {
+		return nil
+	}
+	bp.syncSravniHandlerEntryID, err = bp.registerHandler(ctx, cronValue, handlerName, bp.syncSravniHandler)
+	return err
+}
+
+func (bp *BackgroundProcess) ForceExecuteSyncSravniHandler(ctx context.Context) error {
+	if bp.syncSravniHandler == nil {
+		return errors.SimpleError("sync sravni handler not mounted")
+	}
+	return bp.syncSravniHandler.Handle(ctx)
+}
+
+type NextRunStats map[string]time.Time
+
+func (s NextRunStats) String() string {
+	var sb strings.Builder
+	_ = json.NewEncoder(&sb).Encode(s)
+	return sb.String()
+}
+
+func (bp *BackgroundProcess) GetNextRunStats() NextRunStats {
+	out := make(NextRunStats)
+	if bp.syncSravniHandlerEntryID.Valid() {
+		sEntry := bp.scheduler.Entry(bp.syncSravniHandlerEntryID.Value())
+		out["sravni_handler"] = sEntry.Next
+	}
+
+	return NextRunStats(out)
+}
+
+func (bp *BackgroundProcess) Run() {
+	bp.scheduler.Run()
+}
+
+func (bp *BackgroundProcess) Shutdown(ctx context.Context) error {
+	sdctx := bp.scheduler.Stop()
+
+	select {
+	case <-ctx.Done():
+		return ctx.Err()
+	case <-sdctx.Done():
+		return nil
+	}
+}
+
+type handler interface {
+	Handle(context.Context) error
+}
+
+func (bp *BackgroundProcess) registerHandler(ctx context.Context, spec, name string, h handler) (nullable.Value[cron.EntryID], error) {
+	handlerField := slog.String("handler", name)
+	xcontext.LogInfo(ctx, bp.log, "registering handler", handlerField)
+
+	entry, err := bp.scheduler.AddJob(spec, cron.FuncJob(func() {
+		jctx := xcontext.WithLogFields(ctx, handlerField)
+		err := h.Handle(jctx)
+		if err != nil {
+			xcontext.LogWithError(jctx, bp.log, err, "unable to run iteration")
+		}
+		xcontext.LogInfo(jctx, bp.log, "iteration completed")
+	}))
+
+	var out nullable.Value[cron.EntryID]
+	if err != nil {
+		return out, fmt.Errorf("adding %s job: %w", name, err)
+	}
+
+	out.Set(entry)
+
+	return out, nil
+}

internal/kurious/ports/background/synchandler.go

@@ -0,0 +1,260 @@
+package background
+
+import (
+	"context"
+	"fmt"
+	"log/slog"
+	"time"
+
+	"golang.org/x/time/rate"
+
+	"git.loyso.art/frx/kurious/internal/common/client/sravni"
+	"git.loyso.art/frx/kurious/internal/common/generator"
+	"git.loyso.art/frx/kurious/internal/common/nullable"
+	"git.loyso.art/frx/kurious/internal/common/xcontext"
+	"git.loyso.art/frx/kurious/internal/common/xslice"
+	"git.loyso.art/frx/kurious/internal/kurious/app/command"
+	"git.loyso.art/frx/kurious/internal/kurious/app/query"
+	"git.loyso.art/frx/kurious/internal/kurious/domain"
+	"git.loyso.art/frx/kurious/internal/kurious/service"
+)
+
+func NewSyncSravniHandler(svc service.Application, client sravni.Client, log *slog.Logger) *syncSravniHandler {
+	return &syncSravniHandler{
+		svc:    svc,
+		client: client,
+		log:    log,
+	}
+}
+
+type syncSravniHandler struct {
+	svc    service.Application
+	client sravni.Client
+	log    *slog.Logger
+
+	knownExternalIDs map[string]struct{}
+}
+
+func (h *syncSravniHandler) Handle(ctx context.Context) (err error) {
+	iterationID := generator.RandomInt64ID()
+	ctx = xcontext.WithLogFields(ctx, slog.String("iteration_id", iterationID))
+	start := time.Now()
+	xcontext.LogInfo(ctx, h.log, "handling iteration")
+
+	defer func() {
+		elapsed := slog.Duration("elapsed", time.Since(start))
+		if err != nil {
+			xcontext.LogWithError(ctx, h.log, err, "unable to handle iteration", elapsed)
+			return
+		}
+		xcontext.LogInfo(ctx, h.log, "iteration finished", elapsed)
+	}()
+
+	err = h.fillCaches(ctx)
+	if err != nil {
+		return err
+	}
+
+	state, err := h.client.GetMainPageState()
+	if err != nil {
+		return fmt.Errorf("unable to get main page state: %w", err)
+	}
+
+	handleStart := time.Now()
+	defer func() {
+		elapsed := time.Since(handleStart)
+		xcontext.LogInfo(
+			ctx, h.log, "iteration finished",
+			slog.Duration("elapsed", elapsed),
+			slog.Bool("success", err == nil),
+		)
+	}()
+
+	learningTypes := state.Props.InitialReduxState.Dictionaries.Data.LearningType
+	courses := make([]sravni.Course, 0, 1024)
+	for _, learningType := range learningTypes.Fields {
+		select {
+		case <-ctx.Done():
+			return ctx.Err()
+		default:
+		}
+
+		lctx := xcontext.WithLogFields(ctx, slog.String("learning_type", learningType.Name))
+		xcontext.LogInfo(lctx, h.log, "parsing course", slog.String("name", learningType.Name))
+		start := time.Now()
+		courses = courses[:0]
+
+		courses, err = h.loadEducationalProducts(lctx, learningType.Value, courses)
+		if err != nil {
+			return fmt.Errorf("loading educational products: %w", err)
+		}
+
+		elapsed := time.Since(start)
+		xcontext.LogDebug(lctx, h.log, "parsed items", slog.Duration("elapsed", elapsed), slog.Int("amount", len(courses)))
+
+		// TODO: if the same course appears in different categories, it should be handled
+		courses = h.filterByCache(courses)
+		if len(courses) == 0 {
+			xcontext.LogInfo(lctx, h.log, "all courses were filtered out")
+			continue
+		}
+
+		xcontext.LogDebug(lctx, h.log, "filtered items", slog.Int("amount", len(courses)))
+
+		err = h.insertValues(lctx, courses)
+		elapsed = time.Since(start) - elapsed
+		elapsedField := slog.Duration("elapsed", elapsed)
+		if err != nil {
+			xcontext.LogWithError(lctx, h.log, err, "unable to insert courses", elapsedField)
+			continue
+		}
+
+		xslice.ForEach(courses, func(c sravni.Course) {
+			h.knownExternalIDs[c.ID] = struct{}{}
+		})
+
+		xcontext.LogInfo(
+			lctx, h.log, "processed items",
+			elapsedField,
+			slog.Int("count", len(courses)),
+		)
+	}
+
+	return nil
+}
+
+func (h *syncSravniHandler) loadEducationalProducts(ctx context.Context, learningType string, buf []sravni.Course) ([]sravni.Course, error) {
+	const maxDeepIteration = 10
+	const defaultLimit = 50
+
+	rateStrategy := rate.Every(time.Millisecond * 400)
+	rateLimit := rate.NewLimiter(rateStrategy, 1)
+
+	var courses []sravni.Course
+	if buf == nil || cap(buf) == 0 {
+		courses = make([]sravni.Course, 0, 256)
+	} else {
+		courses = buf
+	}
+
+	var offset int
+	params := sravni.ListEducationProductsParams{LearningType: learningType}
+	for i := 0; i < maxDeepIteration; i++ {
+		params.Limit = defaultLimit
+		params.Offset = offset
+		response, err := h.client.ListEducationalProducts(ctx, params)
+		if err != nil {
+			return nil, fmt.Errorf("listing educational products: %w", err)
+		}
+
+		offset += defaultLimit
+		courses = append(courses, response.Items...)
+		if len(response.Items) < defaultLimit {
+			break
+		}
+
+		err = rateLimit.Wait(ctx)
+		if err != nil {
+			return courses, fmt.Errorf("waiting for limit: %w", err)
+		}
+	}
+
+	return courses, nil
+}
+
+func (h *syncSravniHandler) filterByCache(courses []sravni.Course) (toInsert []sravni.Course) {
+	toCut := xslice.FilterInplace(courses, xslice.Not(h.isCached))
+	return courses[:toCut]
+}
+
+func (h *syncSravniHandler) isCached(course sravni.Course) bool {
+	_, ok := h.knownExternalIDs[course.ID]
+	return ok
+}
+
+func (h *syncSravniHandler) insertValues(ctx context.Context, courses []sravni.Course) error {
+	courseParams := xslice.Map(courses, courseAsCreateCourseParams)
+	err := h.svc.Commands.InsertCourses.Handle(ctx, command.CreateCourses{
+		Courses: courseParams,
+	})
+	if err != nil {
+		return fmt.Errorf("inserting courses: %w", err)
+	}
+
+	return nil
+}
+
+func (h *syncSravniHandler) fillCaches(ctx context.Context) error {
+	if h.knownExternalIDs != nil {
+		xcontext.LogInfo(ctx, h.log, "cache already filled")
+
+		return nil
+	}
+
+	courses, err := h.svc.Queries.ListCourses.Handle(ctx, query.ListCourse{})
+	if err != nil {
+		return fmt.Errorf("listing courses: %w", err)
+	}
+
+	h.knownExternalIDs = make(map[string]struct{}, len(courses))
+
+	xslice.ForEach(courses, func(c domain.Course) {
+		if !c.ExternalID.Valid() {
+			return
+		}
+		h.knownExternalIDs[c.ExternalID.Value()] = struct{}{}
+	})
+
+	xcontext.LogInfo(ctx, h.log, "cache filled", slog.Int("count", len(courses)))
+
+	return nil
+}
+
+func courseAsCreateCourseParams(course sravni.Course) command.CreateCourse {
+	courseid := generator.RandomInt64ID()
+	var startAt time.Time
+	if course.DateStart != nil {
+		startAt = *course.DateStart
+	}
+	if course.TimeStart != nil {
+		startAtUnix := startAt.Unix() + course.TimeStart.Unix()
+		startAt = time.Unix(startAtUnix, 0)
+	}
+
+	var courseDuration time.Duration
+	if course.TimeAllDay != nil {
+		courseDuration += time.Hour * 24 * time.Duration(*course.TimeAllDay)
+	}
+	if course.TimeAllHour != nil {
+		courseDuration += time.Hour * time.Duration(*course.TimeAllHour)
+	}
+	if course.TimeAllMonth != nil {
+		courseDuration += time.Hour * 24 * 30 * time.Duration(*course.TimeAllMonth)
+	}
+
+	var discount float64
+	switch td := course.Discount.Percent.(type) {
+	case int:
+		discount = float64(td) / 100
+	case float64:
+		discount = td / 100
+	default:
+
+	}
+
+	return command.CreateCourse{
+		ID:             courseid,
+		ExternalID:     nullable.NewValue(course.ID),
+		Name:           course.Name,
+		SourceType:     domain.SourceTypeParsed,
+		SourceName:     nullable.NewValue("sravni"),
+		OrganizationID: course.Organization,
+		OriginLink:     course.Link,
+		ImageLink:      course.CourseImage,
+		Description:    "", // should be added to parse in queue
+		FullPrice:      course.PriceAll,
+		Discount:       discount,
+		StartsAt:       startAt,
+		Duration:       courseDuration,
+	}
+}

internal/kurious/ports/cron.go

@@ -1,43 +0,0 @@
-package ports
-
-import (
-	"context"
-	"log/slog"
-
-	"git.loyso.art/frx/kurious/internal/common/xlog"
-	"git.loyso.art/frx/kurious/internal/kurious/service"
-
-	"github.com/robfig/cron/v3"
-)
-
-type BackgroundParser struct {
-	scheduler *cron.Cron
-}
-
-func NewBackgroundParser(ctx context.Context, svc service.Application, log *slog.Logger) *BackgroundParser {
-	clog := xlog.WrapSLogger(ctx, log)
-	scheduler := cron.New(cron.WithSeconds(), cron.WithChain(
-		cron.Recover(clog),
-	))
-
-	bp := &BackgroundParser{
-		scheduler: scheduler,
-	}
-
-	return bp
-}
-
-func (bp *BackgroundParser) Run() {
-	bp.scheduler.Run()
-}
-
-func (bp *BackgroundParser) Shutdown(ctx context.Context) error {
-	sdctx := bp.scheduler.Stop()
-
-	select {
-	case <-ctx.Done():
-		return ctx.Err()
-	case <-sdctx.Done():
-		return nil
-	}
-}

internal/kurious/service/service.go

@@ -36,8 +36,9 @@ func NewApplication(ctx context.Context, cfg ApplicationConfig) (Application, er
 
 	application := app.Application{
 		Commands: app.Commands{
-			InsertCourse: command.NewCreateCourseHandler(courseadapter, log),
-			DeleteCourse: command.NewDeleteCourseHandler(courseadapter, log),
+			InsertCourses: command.NewCreateCoursesHandler(courseadapter, log),
+			InsertCourse:  command.NewCreateCourseHandler(courseadapter, log),
+			DeleteCourse:  command.NewDeleteCourseHandler(courseadapter, log),
 		},
 		Queries: app.Queries{
 			GetCourse:   query.NewGetCourseHandler(courseadapter, log),