list by pages and rate req limits

internal/kurious/ports/background.go

@@ -108,10 +108,11 @@ type handler interface {
 
 func (bp *BackgroundProcess) registerHandler(ctx context.Context, spec, name string, h handler) (nullable.Value[cron.EntryID], error) {
 	handlerField := slog.String("handler", name)
-	xcontext.LogInfo(ctx, bp.log, "registering handler", handlerField)
+	jctx := xcontext.WithLogFields(ctx, handlerField)
+
+	xcontext.LogInfo(jctx, bp.log, "registering handler", handlerField)
 
 	entry, err := bp.scheduler.AddJob(spec, cron.FuncJob(func() {
-		jctx := xcontext.WithLogFields(ctx, handlerField)
 		err := h.Handle(jctx)
 		if err != nil {
 			xcontext.LogWithError(jctx, bp.log, err, "unable to run iteration")

internal/kurious/ports/background/synchandler.go

@@ -2,8 +2,10 @@ package background
 
 import (
 	"context"
+	"errors"
 	"fmt"
 	"log/slog"
+	"sync/atomic"
 	"time"
 
 	"golang.org/x/time/rate"
@@ -33,9 +35,17 @@ type syncSravniHandler struct {
 	log    *slog.Logger
 
 	knownExternalIDs map[string]struct{}
+	isRunning        uint32
 }
 
 func (h *syncSravniHandler) Handle(ctx context.Context) (err error) {
+	if !atomic.CompareAndSwapUint32(&h.isRunning, 0, 1) {
+		return nil
+	}
+	defer func() {
+		atomic.StoreUint32(&h.isRunning, 0)
+	}()
+
 	iterationID := generator.RandomInt64ID()
 	ctx = xcontext.WithLogFields(ctx, slog.String("iteration_id", iterationID))
 	start := time.Now()
@@ -72,6 +82,7 @@ func (h *syncSravniHandler) Handle(ctx context.Context) (err error) {
 
 	learningTypes := state.Props.InitialReduxState.Dictionaries.Data.LearningType
 	courses := make([]sravni.Course, 0, 1024)
+	buffer := make([]sravni.Course, 0, 512)
 	for _, learningType := range learningTypes.Fields {
 		select {
 		case <-ctx.Done():
@@ -79,14 +90,49 @@ func (h *syncSravniHandler) Handle(ctx context.Context) (err error) {
 		default:
 		}
 
-		lctx := xcontext.WithLogFields(ctx, slog.String("learning_type", learningType.Name))
-		xcontext.LogInfo(lctx, h.log, "parsing course", slog.String("name", learningType.Name))
+		lctx := xcontext.WithLogFields(ctx, slog.String("learning_type", learningType.Value))
+		xcontext.LogInfo(lctx, h.log, "parsing courses")
 		start := time.Now()
 		courses = courses[:0]
 
-		courses, err = h.loadEducationalProducts(lctx, learningType.Value, courses)
+		filterCount, err := h.client.ListEducationalProductsFilterCount(ctx, sravni.ListEducationProductsParams{
+			LearningType: learningType.Value,
+		})
 		if err != nil {
-			return fmt.Errorf("loading educational products: %w", err)
+			return fmt.Errorf("loading products filter count: %w", err)
+		}
+
+		thematics := make([]string, 0, len(filterCount.CoursesThematics))
+		for cr, count := range filterCount.CoursesThematics {
+			if count == 0 {
+				continue
+			}
+
+			thematics = append(thematics, cr)
+		}
+
+		xcontext.LogDebug(lctx, h.log, "loaded course thematics for learning type", slog.Int("count", len(thematics)))
+
+		// since count is known it might be optimized to allocate slice once per request.
+		for _, courseThematic := range thematics {
+			buffer = buffer[:0]
+			buffer, err = h.loadEducationalProducts(lctx, learningType.Value, courseThematic, buffer)
+			if err != nil {
+				if !errors.Is(err, context.Canceled) {
+					xcontext.LogWithWarnError(lctx, h.log, err, "unable to load educational products", slog.Int("count", len(thematics)))
+					continue
+				}
+
+				return fmt.Errorf("loading educational products: %w", err)
+			}
+
+			xslice.ForEach(buffer, func(c sravni.Course) {
+				c.Learningtype = []string{learningType.Value}
+				c.CourseThematics = []string{courseThematic}
+				courses = append(courses, c)
+			})
+
+			xcontext.LogInfo(lctx, h.log, "parsed subitems", slog.String("course_thematic", courseThematic), slog.Int("amount", len(buffer)))
 		}
 
 		elapsed := time.Since(start)
@@ -123,7 +169,7 @@ func (h *syncSravniHandler) Handle(ctx context.Context) (err error) {
 	return nil
 }
 
-func (h *syncSravniHandler) loadEducationalProducts(ctx context.Context, learningType string, buf []sravni.Course) ([]sravni.Course, error) {
+func (h *syncSravniHandler) loadEducationalProducts(ctx context.Context, learningType, courseThematic string, buf []sravni.Course) ([]sravni.Course, error) {
 	const maxDeepIteration = 10
 	const defaultLimit = 50
 
@@ -138,7 +184,10 @@ func (h *syncSravniHandler) loadEducationalProducts(ctx context.Context, learnin
 	}
 
 	var offset int
-	params := sravni.ListEducationProductsParams{LearningType: learningType}
+	params := sravni.ListEducationProductsParams{
+		LearningType:     learningType,
+		CoursesThematics: []string{courseThematic},
+	}
 	for i := 0; i < maxDeepIteration; i++ {
 		params.Limit = defaultLimit
 		params.Offset = offset
@@ -242,9 +291,20 @@ func courseAsCreateCourseParams(course sravni.Course) command.CreateCourse {
 
 	}
 
+	var ct string
+	if len(course.CourseThematics) > 0 {
+		ct = course.CourseThematics[0]
+	}
+	var lt string
+	if len(course.Learningtype) > 0 {
+		lt = course.Learningtype[0]
+	}
+
 	return command.CreateCourse{
 		ID:             courseid,
 		ExternalID:     nullable.NewValue(course.ID),
+		CourseThematic: ct,
+		LearningType:   lt,
 		Name:           course.Name,
 		SourceType:     domain.SourceTypeParsed,
 		SourceName:     nullable.NewValue("sravni"),