From e7c283286569bf85235c470c63d1b39699bcc135 Mon Sep 17 00:00:00 2001 From: Aleksandr Trushkin Date: Sun, 24 Mar 2024 22:59:32 +0300 Subject: [PATCH] Add command and query for organizations * Added command and query for organizations * Saving unknown organizations into database in `background` service * Added `List` method in `OrganizationRepository` --- cmd/dev/sravnicli/main.go | 1 + cmd/dev/sravnicli/products.go | 1 - internal/common/xslices/foreach.go | 8 + .../sqlite_organization_repository.go | 16 +- .../sqlite_organization_repository_test.go | 57 ++++++ .../kurious/adapters/ydb_course_repository.go | 4 + internal/kurious/app/app.go | 5 + .../kurious/app/command/createorganization.go | 53 +++++ internal/kurious/app/query/getorganization.go | 45 ++++ .../kurious/app/query/listorganizations.go | 38 ++++ .../domain/mocks/OrganizationRepository.go | 58 ++++++ internal/kurious/domain/repository.go | 16 ++ .../kurious/ports/background/synchandler.go | 192 ++++++++++++++---- internal/kurious/service/service.go | 10 +- 14 files changed, 463 insertions(+), 41 deletions(-) create mode 100644 internal/kurious/app/command/createorganization.go create mode 100644 internal/kurious/app/query/getorganization.go create mode 100644 internal/kurious/app/query/listorganizations.go diff --git a/cmd/dev/sravnicli/main.go b/cmd/dev/sravnicli/main.go index cf6b2aa..e94d8dd 100644 --- a/cmd/dev/sravnicli/main.go +++ b/cmd/dev/sravnicli/main.go @@ -61,6 +61,7 @@ func setupCLI(ctx context.Context) cli.App { case "courses": out = state.Props.InitialReduxState.Dictionaries.Data.CourseThematics } + log.InfoContext(ctx, "loaded state", slog.Any("state", out)) return 0 diff --git a/cmd/dev/sravnicli/products.go b/cmd/dev/sravnicli/products.go index ebd5db7..1f20b7b 100644 --- a/cmd/dev/sravnicli/products.go +++ b/cmd/dev/sravnicli/products.go @@ -44,7 +44,6 @@ func setupAPICommand(ctx context.Context) cli.Command { WithOption(learningSelectionOpt). WithAction(newProductsFilterCountAction(ctx)) }) - apiEducation := cli.NewCommand("education", "Education related category"). WithCommand(apiEducationListProducts). WithCommand(apiEducationFilterCount) diff --git a/internal/common/xslices/foreach.go b/internal/common/xslices/foreach.go index 05568e9..2eb8096 100644 --- a/internal/common/xslices/foreach.go +++ b/internal/common/xslices/foreach.go @@ -5,3 +5,11 @@ func ForEach[T any](items []T, f func(T)) { f(item) } } + +func AsMap[T any, U comparable](items []T, f func(T) U) map[U]struct{} { + out := make(map[U]struct{}, len(items)) + ForEach(items, func(in T) { + out[f(in)] = struct{}{} + }) + return out +} diff --git a/internal/kurious/adapters/sqlite_organization_repository.go b/internal/kurious/adapters/sqlite_organization_repository.go index d57f6b4..86c3003 100644 --- a/internal/kurious/adapters/sqlite_organization_repository.go +++ b/internal/kurious/adapters/sqlite_organization_repository.go @@ -8,6 +8,7 @@ import ( "log/slog" "time" + "git.loyso.art/frx/kurious/internal/common/xslices" "git.loyso.art/frx/kurious/internal/kurious/domain" "github.com/jmoiron/sqlx" @@ -42,7 +43,7 @@ type organizationDB struct { DeletedAt sql.NullTime `db:"deleted_at"` } -func (o *organizationDB) AsDomain() domain.Organization { +func (o organizationDB) AsDomain() domain.Organization { return domain.Organization{ ID: o.ID, ExternalID: nullStringAsDomain(o.ExternalID), @@ -82,6 +83,19 @@ type sqliteOrganizationRepository struct { log *slog.Logger } +func (r *sqliteOrganizationRepository) List(ctx context.Context) (out []domain.Organization, err error) { + const queryTemplate = `SELECT %s FROM organizations` + query := fmt.Sprintf(queryTemplate, organizationColumnsStr) + + organizations := make([]organizationDB, 0, 1<<8) + err = r.db.SelectContext(ctx, &organizations, query) + if err != nil { + return nil, fmt.Errorf("executing query: %w", err) + } + + return xslices.Map(organizations, asDomainFunc), nil +} + func (r *sqliteOrganizationRepository) Get(ctx context.Context, params domain.GetOrganizationParams) (out domain.Organization, err error) { const queryTemplate = "SELECT %s FROM organizations WHERE 1=1" query := fmt.Sprintf(queryTemplate, organizationColumnsStr) diff --git a/internal/kurious/adapters/sqlite_organization_repository_test.go b/internal/kurious/adapters/sqlite_organization_repository_test.go index 1f84fb0..98fa8c7 100644 --- a/internal/kurious/adapters/sqlite_organization_repository_test.go +++ b/internal/kurious/adapters/sqlite_organization_repository_test.go @@ -1,6 +1,8 @@ package adapters import ( + "slices" + "strconv" "testing" "git.loyso.art/frx/kurious/internal/common/nullable" @@ -21,6 +23,61 @@ func (s *sqliteOrganzationRepositorySuite) TearDownTest() { _ = s.connection.db.MustExecContext(s.ctx, "DELETE FROM organizations") } +func (s *sqliteOrganzationRepositorySuite) TestList() { + const itemscount = 3 + orgsdb := make([]domain.Organization, 0, itemscount) + + baseOrg := domain.Organization{ + Alias: "test-alias", + Name: "test-name", + Site: "test-site", + LogoLink: "test-logo", + } + for i := 0; i < itemscount; i++ { + nextitem := baseOrg + iStr := strconv.Itoa(i) + nextitem.ID = "test-id-" + iStr + nextitem.ExternalID.Set("test-ext-id-" + iStr) + + gotOrg, err := s.connection.Organization().Create(s.ctx, domain.CreateOrganizationParams{ + ID: nextitem.ID, + ExternalID: nextitem.ExternalID, + Alias: nextitem.Alias, + Name: nextitem.Name, + Site: nextitem.Site, + LogoLink: nextitem.LogoLink, + }) + s.NoError(err) + orgsdb = append(orgsdb, gotOrg) + } + + gotOrgs, err := s.connection.Organization().List(s.ctx) + s.NoError(err) + + compareF := func(lhs, rhs domain.Organization) int { + if lhs.ID < rhs.ID { + return -1 + } else if lhs.ID > rhs.ID { + return 1 + } else { + return 0 + } + } + slices.SortFunc(gotOrgs, compareF) + + for i := range gotOrgs { + s.NotEmpty(gotOrgs[i].CreatedAt) + s.NotEmpty(gotOrgs[i].UpdatedAt) + s.Empty(gotOrgs[i].DeletedAt) + + orgsdb[i].CreatedAt = gotOrgs[i].CreatedAt + orgsdb[i].UpdatedAt = gotOrgs[i].UpdatedAt + orgsdb[i].DeletedAt = gotOrgs[i].DeletedAt + } + + s.ElementsMatch(orgsdb, gotOrgs) +} + func (s *sqliteOrganzationRepositorySuite) TestGet() { var orgdb organizationDB err := s.connection.db.GetContext( diff --git a/internal/kurious/adapters/ydb_course_repository.go b/internal/kurious/adapters/ydb_course_repository.go index 86c3e88..097fc83 100644 --- a/internal/kurious/adapters/ydb_course_repository.go +++ b/internal/kurious/adapters/ydb_course_repository.go @@ -109,6 +109,10 @@ func (conn *YDBConnection) Close() error { return conn.Driver.Close(ctx) } +func (conn *YDBConnection) Organization() domain.OrganizationRepository { + return domain.NotImplementedOrganizationRepository{} +} + func (conn *YDBConnection) LearningCategory() domain.LearningCategoryRepository { return domain.NotImplementedLearningCategory{} } diff --git a/internal/kurious/app/app.go b/internal/kurious/app/app.go index 4a0941e..7ffc270 100644 --- a/internal/kurious/app/app.go +++ b/internal/kurious/app/app.go @@ -10,6 +10,8 @@ type Commands struct { InsertCourse command.CreateCourseHandler DeleteCourse command.DeleteCourseHandler UpdateCourseDescription command.UpdateCourseDescriptionHandler + + InsertOrganization command.CreateOrganizationHandler } type Queries struct { @@ -17,6 +19,9 @@ type Queries struct { ListCourses query.ListCourseHandler ListLearningTypes query.ListLearningTypesHandler ListCourseThematics query.ListCourseThematicsHandler + + ListOrganzations query.ListOrganizationsHandler + GetOrganization query.GetOrganizationHandler } type Application struct { diff --git a/internal/kurious/app/command/createorganization.go b/internal/kurious/app/command/createorganization.go new file mode 100644 index 0000000..b549558 --- /dev/null +++ b/internal/kurious/app/command/createorganization.go @@ -0,0 +1,53 @@ +package command + +import ( + "context" + "fmt" + "log/slog" + + "git.loyso.art/frx/kurious/internal/common/decorator" + "git.loyso.art/frx/kurious/internal/common/nullable" + "git.loyso.art/frx/kurious/internal/kurious/domain" +) + +type CreateOrganization struct { + ID string + ExternalID nullable.Value[string] + Alias string + Name string + Site string + Logo string +} + +type CreateOrganizationHandler decorator.CommandHandler[CreateOrganization] + +type createOrganizationHandler struct { + repo domain.OrganizationRepository +} + +func NewCreateOrganizationHandler( + repo domain.OrganizationRepository, + log *slog.Logger, +) CreateOrganizationHandler { + h := createOrganizationHandler{ + repo: repo, + } + + return decorator.ApplyCommandDecorators(h, log) +} + +func (h createOrganizationHandler) Handle(ctx context.Context, cmd CreateOrganization) error { + _, err := h.repo.Create(ctx, domain.CreateOrganizationParams{ + ID: cmd.ID, + ExternalID: cmd.ExternalID, + Alias: cmd.Alias, + Name: cmd.Name, + Site: cmd.Site, + LogoLink: cmd.Logo, + }) + if err != nil { + return fmt.Errorf("creating organization: %w", err) + } + + return nil +} diff --git a/internal/kurious/app/query/getorganization.go b/internal/kurious/app/query/getorganization.go new file mode 100644 index 0000000..e7af223 --- /dev/null +++ b/internal/kurious/app/query/getorganization.go @@ -0,0 +1,45 @@ +package query + +import ( + "context" + "fmt" + "log/slog" + + "git.loyso.art/frx/kurious/internal/common/decorator" + "git.loyso.art/frx/kurious/internal/common/nullable" + "git.loyso.art/frx/kurious/internal/kurious/domain" +) + +type GetOrganization struct { + ID nullable.Value[string] + ExternalID nullable.Value[string] +} + +type GetOrganizationHandler decorator.QueryHandler[GetOrganization, domain.Organization] + +type getOrganizationHandler struct { + repo domain.OrganizationRepository +} + +func NewGetOrganizationHandler( + repo domain.OrganizationRepository, + log *slog.Logger, +) GetOrganizationHandler { + h := getOrganizationHandler{ + repo: repo, + } + + return decorator.AddQueryDecorators(h, log) +} + +func (h getOrganizationHandler) Handle(ctx context.Context, query GetOrganization) (domain.Organization, error) { + organization, err := h.repo.Get(ctx, domain.GetOrganizationParams{ + ID: query.ID, + ExternalID: query.ExternalID, + }) + if err != nil { + return domain.Organization{}, fmt.Errorf("getting organization: %w", err) + } + + return organization, nil +} diff --git a/internal/kurious/app/query/listorganizations.go b/internal/kurious/app/query/listorganizations.go new file mode 100644 index 0000000..7f095e1 --- /dev/null +++ b/internal/kurious/app/query/listorganizations.go @@ -0,0 +1,38 @@ +package query + +import ( + "context" + "fmt" + "log/slog" + + "git.loyso.art/frx/kurious/internal/common/decorator" + "git.loyso.art/frx/kurious/internal/kurious/domain" +) + +type ListOrganizations struct{} + +type ListOrganizationsHandler decorator.QueryHandler[ListOrganizations, []domain.Organization] + +type listOrganizationsHandler struct { + repo domain.OrganizationRepository +} + +func NewListOrganizationsHandler( + repo domain.OrganizationRepository, + log *slog.Logger, +) ListOrganizationsHandler { + h := listOrganizationsHandler{ + repo: repo, + } + + return decorator.AddQueryDecorators(h, log) +} + +func (h listOrganizationsHandler) Handle(ctx context.Context, query ListOrganizations) ([]domain.Organization, error) { + organizations, err := h.repo.List(ctx) + if err != nil { + return nil, fmt.Errorf("listing organizations: %w", err) + } + + return organizations, nil +} diff --git a/internal/kurious/domain/mocks/OrganizationRepository.go b/internal/kurious/domain/mocks/OrganizationRepository.go index dcfefd5..d8a2e7e 100644 --- a/internal/kurious/domain/mocks/OrganizationRepository.go +++ b/internal/kurious/domain/mocks/OrganizationRepository.go @@ -183,6 +183,64 @@ func (_c *OrganizationRepository_Get_Call) RunAndReturn(run func(context.Context return _c } +// List provides a mock function with given fields: _a0 +func (_m *OrganizationRepository) List(_a0 context.Context) ([]domain.Organization, error) { + ret := _m.Called(_a0) + + if len(ret) == 0 { + panic("no return value specified for List") + } + + var r0 []domain.Organization + var r1 error + if rf, ok := ret.Get(0).(func(context.Context) ([]domain.Organization, error)); ok { + return rf(_a0) + } + if rf, ok := ret.Get(0).(func(context.Context) []domain.Organization); ok { + r0 = rf(_a0) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).([]domain.Organization) + } + } + + if rf, ok := ret.Get(1).(func(context.Context) error); ok { + r1 = rf(_a0) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// OrganizationRepository_List_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'List' +type OrganizationRepository_List_Call struct { + *mock.Call +} + +// List is a helper method to define mock.On call +// - _a0 context.Context +func (_e *OrganizationRepository_Expecter) List(_a0 interface{}) *OrganizationRepository_List_Call { + return &OrganizationRepository_List_Call{Call: _e.mock.On("List", _a0)} +} + +func (_c *OrganizationRepository_List_Call) Run(run func(_a0 context.Context)) *OrganizationRepository_List_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(context.Context)) + }) + return _c +} + +func (_c *OrganizationRepository_List_Call) Return(_a0 []domain.Organization, _a1 error) *OrganizationRepository_List_Call { + _c.Call.Return(_a0, _a1) + return _c +} + +func (_c *OrganizationRepository_List_Call) RunAndReturn(run func(context.Context) ([]domain.Organization, error)) *OrganizationRepository_List_Call { + _c.Call.Return(run) + return _c +} + // NewOrganizationRepository creates a new instance of OrganizationRepository. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. // The first argument is typically a *testing.T value. func NewOrganizationRepository(t interface { diff --git a/internal/kurious/domain/repository.go b/internal/kurious/domain/repository.go index 402db79..9cbde3a 100644 --- a/internal/kurious/domain/repository.go +++ b/internal/kurious/domain/repository.go @@ -93,11 +93,27 @@ type CreateOrganizationParams struct { //go:generate mockery --name OrganizationRepository type OrganizationRepository interface { + List(context.Context) ([]Organization, error) Get(context.Context, GetOrganizationParams) (Organization, error) Create(context.Context, CreateOrganizationParams) (Organization, error) Delete(ctx context.Context, id string) error } +type NotImplementedOrganizationRepository struct{} + +func (NotImplementedOrganizationRepository) List(context.Context) ([]Organization, error) { + return nil, ErrNotImplemented +} +func (NotImplementedOrganizationRepository) Get(context.Context, GetOrganizationParams) (Organization, error) { + return Organization{}, ErrNotImplemented +} +func (NotImplementedOrganizationRepository) Create(context.Context, CreateOrganizationParams) (Organization, error) { + return Organization{}, ErrNotImplemented +} +func (NotImplementedOrganizationRepository) Delete(ctx context.Context, id string) error { + return ErrNotImplemented +} + //go:generate mockery --name LearningCategoryRepository type LearningCategoryRepository interface { Upsert(context.Context, LearningCategory) error diff --git a/internal/kurious/ports/background/synchandler.go b/internal/kurious/ports/background/synchandler.go index eb603f3..ca21da0 100644 --- a/internal/kurious/ports/background/synchandler.go +++ b/internal/kurious/ports/background/synchandler.go @@ -34,8 +34,9 @@ type syncSravniHandler struct { client sravni.Client log *slog.Logger - knownExternalIDs map[string]struct{} - isRunning uint32 + knownExternalIDs map[string]struct{} + knownOrganizationsByExternalID map[string]struct{} + isRunning uint32 } func (h *syncSravniHandler) Handle(ctx context.Context) (err error) { @@ -83,6 +84,7 @@ func (h *syncSravniHandler) Handle(ctx context.Context) (err error) { learningTypes := state.Props.InitialReduxState.Dictionaries.Data.LearningType courses := make([]sravni.Course, 0, 1024) buffer := make([]sravni.Course, 0, 512) + organizations := make([]sravni.Organization, 0, 256) for _, learningType := range learningTypes.Fields { select { case <-ctx.Done(): @@ -115,61 +117,98 @@ func (h *syncSravniHandler) Handle(ctx context.Context) (err error) { // since count is known it might be optimized to allocate slice once per request. for _, courseThematic := range thematics { + var filteredCourses int + var filteredOrgs int + + var orgsByID map[string]sravni.Organization buffer = buffer[:0] - buffer, err = h.loadEducationalProducts(lctx, learningType.Value, courseThematic, buffer) + buffer, orgsByID, err = h.loadEducationalProducts(lctx, learningType.Value, courseThematic, buffer) if err != nil { - if !errors.Is(err, context.Canceled) { - xcontext.LogWithWarnError(lctx, h.log, err, "unable to load educational products", slog.Int("count", len(thematics))) - continue + if errors.Is(err, context.Canceled) { + return fmt.Errorf("loading educational products: %w", err) } - return fmt.Errorf("loading educational products: %w", err) + xcontext.LogWithWarnError(lctx, h.log, err, "unable to load educational products", slog.Int("count", len(thematics))) + + continue } xslices.ForEach(buffer, func(c sravni.Course) { + // TODO: if the same course appears in different categories, it should be handled + if !h.setCourseIfNotKnown(c) { + return + } + c.Learningtype = []string{learningType.Value} c.CourseThematics = []string{courseThematic} courses = append(courses, c) + filteredCourses++ }) - xcontext.LogInfo(lctx, h.log, "parsed subitems", slog.String("course_thematic", courseThematic), slog.Int("amount", len(buffer))) + for _, org := range orgsByID { + if !h.setOrganizationIfNotKnown(org) { + continue + } + + organizations = append(organizations, org) + filteredOrgs++ + } + + xcontext.LogInfo( + lctx, h.log, "parsed subitems", + slog.String("course_thematic", courseThematic), + slog.Int("amount", len(buffer)), + slog.Int("new_courses", filteredCourses), + slog.Int("new_organizations", filteredOrgs), + ) } elapsed := time.Since(start) xcontext.LogDebug(lctx, h.log, "parsed items", slog.Duration("elapsed", elapsed), slog.Int("amount", len(courses))) - // TODO: if the same course appears in different categories, it should be handled - courses = h.filterByCache(courses) - if len(courses) == 0 { - xcontext.LogInfo(lctx, h.log, "all courses were filtered out") - continue + xcontext.LogDebug( + lctx, h.log, "filtered items", + slog.Int("courses", len(courses)), + slog.Int("organizations", len(organizations)), + ) + + var insertCourseSuccess bool + if len(courses) > 0 { + err = h.insertCourses(lctx, courses) + if err != nil { + xcontext.LogWithError(lctx, h.log, err, "unable to insert courses") + } + + insertCourseSuccess = err == nil } - xcontext.LogDebug(lctx, h.log, "filtered items", slog.Int("amount", len(courses))) + var insertOrgsSuccess bool + if len(organizations) > 0 { + err = h.insertOrganizations(lctx, organizations) + if err != nil { + xcontext.LogWithError(lctx, h.log, err, "unable to insert courses") + } + + insertOrgsSuccess = err == nil + } - err = h.insertValues(lctx, courses) elapsed = time.Since(start) - elapsed elapsedField := slog.Duration("elapsed", elapsed) - if err != nil { - xcontext.LogWithError(lctx, h.log, err, "unable to insert courses", elapsedField) - continue - } - - xslices.ForEach(courses, func(c sravni.Course) { - h.knownExternalIDs[c.ID] = struct{}{} - }) xcontext.LogInfo( - lctx, h.log, "processed items", + lctx, h.log, "inserting finished", elapsedField, - slog.Int("count", len(courses)), + slog.Bool("courses_insert_success", insertCourseSuccess), + slog.Bool("organization_insert_success", insertOrgsSuccess), + slog.Int("courses_count", len(courses)), + slog.Int("organizations_count", len(organizations)), ) } return nil } -func (h *syncSravniHandler) loadEducationalProducts(ctx context.Context, learningType, courseThematic string, buf []sravni.Course) ([]sravni.Course, error) { +func (h *syncSravniHandler) loadEducationalProducts(ctx context.Context, learningType, courseThematic string, buf []sravni.Course) ([]sravni.Course, map[string]sravni.Organization, error) { const maxDeepIteration = 10 const defaultLimit = 50 @@ -177,6 +216,7 @@ func (h *syncSravniHandler) loadEducationalProducts(ctx context.Context, learnin rateLimit := rate.NewLimiter(rateStrategy, 1) var courses []sravni.Course + var organizationsByID = make(map[string]sravni.Organization) if buf == nil || cap(buf) == 0 { courses = make([]sravni.Course, 0, 256) } else { @@ -193,35 +233,49 @@ func (h *syncSravniHandler) loadEducationalProducts(ctx context.Context, learnin params.Offset = offset response, err := h.client.ListEducationalProducts(ctx, params) if err != nil { - return nil, fmt.Errorf("listing educational products: %w", err) + return nil, nil, fmt.Errorf("listing educational products: %w", err) } offset += defaultLimit + courses = append(courses, response.Items...) + + for oid, org := range response.Organizations { + organizationsByID[oid] = org + } + if len(response.Items) < defaultLimit { break } err = rateLimit.Wait(ctx) if err != nil { - return courses, fmt.Errorf("waiting for limit: %w", err) + return courses, organizationsByID, fmt.Errorf("waiting for limit: %w", err) } } - return courses, nil + return courses, organizationsByID, nil } -func (h *syncSravniHandler) filterByCache(courses []sravni.Course) (toInsert []sravni.Course) { - toCut := xslices.FilterInplace(courses, xslices.Not(h.isCached)) - return courses[:toCut] -} - -func (h *syncSravniHandler) isCached(course sravni.Course) bool { +func (h *syncSravniHandler) setCourseIfNotKnown(course sravni.Course) (set bool) { _, ok := h.knownExternalIDs[course.ID] - return ok + if !ok { + h.knownExternalIDs[course.ID] = struct{}{} + } + + return !ok } -func (h *syncSravniHandler) insertValues(ctx context.Context, courses []sravni.Course) error { +func (h *syncSravniHandler) setOrganizationIfNotKnown(organization sravni.Organization) bool { + _, ok := h.knownOrganizationsByExternalID[organization.ID] + if !ok { + h.knownOrganizationsByExternalID[organization.ID] = struct{}{} + } + + return !ok +} + +func (h *syncSravniHandler) insertCourses(ctx context.Context, courses []sravni.Course) error { courseParams := xslices.Map(courses, courseAsCreateCourseParams) err := h.svc.Commands.InsertCourses.Handle(ctx, command.CreateCourses{ Courses: courseParams, @@ -233,7 +287,69 @@ func (h *syncSravniHandler) insertValues(ctx context.Context, courses []sravni.C return nil } +func (h *syncSravniHandler) insertOrganizations(ctx context.Context, organizations []sravni.Organization) error { + organizationParams := xslices.Map(organizations, func(in sravni.Organization) command.CreateOrganization { + return command.CreateOrganization{ + ID: generator.RandomInt64ID(), + ExternalID: nullable.NewValue(in.ID), + Alias: in.Alias, + Name: in.Name.Short, + Site: "", + Logo: in.Logotypes.Web, + } + }) + + for _, params := range organizationParams { + err := h.svc.Commands.InsertOrganization.Handle(ctx, params) + if err != nil { + return fmt.Errorf("inserting organization: %w", err) + } + } + + return nil +} + +func (h *syncSravniHandler) fillOrganizaionCaches(ctx context.Context) error { + if h.knownOrganizationsByExternalID != nil { + xcontext.LogDebug(ctx, h.log, "organization cache already filled") + + return nil + } + + organizations, err := h.svc.Queries.ListOrganzations.Handle(ctx, query.ListOrganizations{}) + if err != nil { + return fmt.Errorf("listing organizations: %w", err) + } + + withExternalID := func(in domain.Organization) bool { + return in.ExternalID.Valid() + } + getExtID := func(in domain.Organization) string { + return in.ExternalID.Value() + } + + h.knownOrganizationsByExternalID = xslices.AsMap(xslices.Filter(organizations, withExternalID), getExtID) + + xcontext.LogInfo(ctx, h.log, "cache filled", slog.String("kind", "organizations_by_external_id"), slog.Int("count", len(organizations))) + + return nil +} + func (h *syncSravniHandler) fillCaches(ctx context.Context) error { + err := h.fillOrganizaionCaches(ctx) + if err != nil { + return err + } + + err = h.fillKnownExternalIDsCache(ctx) + if err != nil { + return err + } + + return nil +} + +func (h *syncSravniHandler) fillKnownExternalIDsCache(ctx context.Context) error { if h.knownExternalIDs != nil { xcontext.LogInfo(ctx, h.log, "cache already filled") @@ -256,7 +372,7 @@ func (h *syncSravniHandler) fillCaches(ctx context.Context) error { h.knownExternalIDs[c.ExternalID.Value()] = struct{}{} }) - xcontext.LogInfo(ctx, h.log, "cache filled", slog.Int("count", len(courses))) + xcontext.LogInfo(ctx, h.log, "cache filled", slog.String("kind", "courses_by_external_id"), slog.Int("count", len(courses))) return nil } diff --git a/internal/kurious/service/service.go b/internal/kurious/service/service.go index eb551cd..dc497f2 100644 --- a/internal/kurious/service/service.go +++ b/internal/kurious/service/service.go @@ -43,6 +43,7 @@ func NewApplication(ctx context.Context, cfg ApplicationConfig, mapper domain.Co var repoCloser io.Closer var courseadapter domain.CourseRepository + var organizationrepo domain.OrganizationRepository switch cfg.Engine { case RepositoryEngineSqlite: sqliteConnection, err := adapters.NewSqliteConnection(ctx, cfg.Sqlite, log.With(slog.String("db", "sqlite"))) @@ -51,6 +52,7 @@ func NewApplication(ctx context.Context, cfg ApplicationConfig, mapper domain.Co } courseadapter = sqliteConnection.CourseRepository() + organizationrepo = sqliteConnection.Organization() repoCloser = sqliteConnection case RepositoryEngineYDB: ydbConnection, err := adapters.NewYDBConnection(ctx, cfg.YDB, log.With(slog.String("db", "ydb"))) @@ -59,9 +61,10 @@ func NewApplication(ctx context.Context, cfg ApplicationConfig, mapper domain.Co } courseadapter = ydbConnection.CourseRepository() + organizationrepo = ydbConnection.Organization() repoCloser = ydbConnection default: - return Application{}, errors.New("unable to decide which engine to use") + return Application{}, errors.New("unable to decide which db engine to use") } err = mapper.CollectCounts(ctx, courseadapter) @@ -75,12 +78,17 @@ func NewApplication(ctx context.Context, cfg ApplicationConfig, mapper domain.Co InsertCourse: command.NewCreateCourseHandler(courseadapter, log), DeleteCourse: command.NewDeleteCourseHandler(courseadapter, log), UpdateCourseDescription: command.NewUpdateCourseDescriptionHandler(courseadapter, log), + + InsertOrganization: command.NewCreateOrganizationHandler(organizationrepo, log), }, Queries: app.Queries{ ListCourses: query.NewListCourseHandler(courseadapter, mapper, log), ListLearningTypes: query.NewListLearningTypesHandler(courseadapter, mapper, log), ListCourseThematics: query.NewListCourseThematicsHandler(courseadapter, mapper, log), GetCourse: query.NewGetCourseHandler(courseadapter, mapper, log), + + ListOrganzations: query.NewListOrganizationsHandler(organizationrepo, log), + GetOrganization: query.NewGetOrganizationHandler(organizationrepo, log), }, }