Add command and query for organizations
* Added command and query for organizations * Saving unknown organizations into database in `background` service * Added `List` method in `OrganizationRepository`
This commit is contained in:
@ -61,6 +61,7 @@ func setupCLI(ctx context.Context) cli.App {
|
|||||||
case "courses":
|
case "courses":
|
||||||
out = state.Props.InitialReduxState.Dictionaries.Data.CourseThematics
|
out = state.Props.InitialReduxState.Dictionaries.Data.CourseThematics
|
||||||
}
|
}
|
||||||
|
|
||||||
log.InfoContext(ctx, "loaded state", slog.Any("state", out))
|
log.InfoContext(ctx, "loaded state", slog.Any("state", out))
|
||||||
|
|
||||||
return 0
|
return 0
|
||||||
|
|||||||
@ -44,7 +44,6 @@ func setupAPICommand(ctx context.Context) cli.Command {
|
|||||||
WithOption(learningSelectionOpt).
|
WithOption(learningSelectionOpt).
|
||||||
WithAction(newProductsFilterCountAction(ctx))
|
WithAction(newProductsFilterCountAction(ctx))
|
||||||
})
|
})
|
||||||
|
|
||||||
apiEducation := cli.NewCommand("education", "Education related category").
|
apiEducation := cli.NewCommand("education", "Education related category").
|
||||||
WithCommand(apiEducationListProducts).
|
WithCommand(apiEducationListProducts).
|
||||||
WithCommand(apiEducationFilterCount)
|
WithCommand(apiEducationFilterCount)
|
||||||
|
|||||||
@ -5,3 +5,11 @@ func ForEach[T any](items []T, f func(T)) {
|
|||||||
f(item)
|
f(item)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func AsMap[T any, U comparable](items []T, f func(T) U) map[U]struct{} {
|
||||||
|
out := make(map[U]struct{}, len(items))
|
||||||
|
ForEach(items, func(in T) {
|
||||||
|
out[f(in)] = struct{}{}
|
||||||
|
})
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|||||||
@ -8,6 +8,7 @@ import (
|
|||||||
"log/slog"
|
"log/slog"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
|
"git.loyso.art/frx/kurious/internal/common/xslices"
|
||||||
"git.loyso.art/frx/kurious/internal/kurious/domain"
|
"git.loyso.art/frx/kurious/internal/kurious/domain"
|
||||||
|
|
||||||
"github.com/jmoiron/sqlx"
|
"github.com/jmoiron/sqlx"
|
||||||
@ -42,7 +43,7 @@ type organizationDB struct {
|
|||||||
DeletedAt sql.NullTime `db:"deleted_at"`
|
DeletedAt sql.NullTime `db:"deleted_at"`
|
||||||
}
|
}
|
||||||
|
|
||||||
func (o *organizationDB) AsDomain() domain.Organization {
|
func (o organizationDB) AsDomain() domain.Organization {
|
||||||
return domain.Organization{
|
return domain.Organization{
|
||||||
ID: o.ID,
|
ID: o.ID,
|
||||||
ExternalID: nullStringAsDomain(o.ExternalID),
|
ExternalID: nullStringAsDomain(o.ExternalID),
|
||||||
@ -82,6 +83,19 @@ type sqliteOrganizationRepository struct {
|
|||||||
log *slog.Logger
|
log *slog.Logger
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (r *sqliteOrganizationRepository) List(ctx context.Context) (out []domain.Organization, err error) {
|
||||||
|
const queryTemplate = `SELECT %s FROM organizations`
|
||||||
|
query := fmt.Sprintf(queryTemplate, organizationColumnsStr)
|
||||||
|
|
||||||
|
organizations := make([]organizationDB, 0, 1<<8)
|
||||||
|
err = r.db.SelectContext(ctx, &organizations, query)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("executing query: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
return xslices.Map(organizations, asDomainFunc), nil
|
||||||
|
}
|
||||||
|
|
||||||
func (r *sqliteOrganizationRepository) Get(ctx context.Context, params domain.GetOrganizationParams) (out domain.Organization, err error) {
|
func (r *sqliteOrganizationRepository) Get(ctx context.Context, params domain.GetOrganizationParams) (out domain.Organization, err error) {
|
||||||
const queryTemplate = "SELECT %s FROM organizations WHERE 1=1"
|
const queryTemplate = "SELECT %s FROM organizations WHERE 1=1"
|
||||||
query := fmt.Sprintf(queryTemplate, organizationColumnsStr)
|
query := fmt.Sprintf(queryTemplate, organizationColumnsStr)
|
||||||
|
|||||||
@ -1,6 +1,8 @@
|
|||||||
package adapters
|
package adapters
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"slices"
|
||||||
|
"strconv"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
"git.loyso.art/frx/kurious/internal/common/nullable"
|
"git.loyso.art/frx/kurious/internal/common/nullable"
|
||||||
@ -21,6 +23,61 @@ func (s *sqliteOrganzationRepositorySuite) TearDownTest() {
|
|||||||
_ = s.connection.db.MustExecContext(s.ctx, "DELETE FROM organizations")
|
_ = s.connection.db.MustExecContext(s.ctx, "DELETE FROM organizations")
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (s *sqliteOrganzationRepositorySuite) TestList() {
|
||||||
|
const itemscount = 3
|
||||||
|
orgsdb := make([]domain.Organization, 0, itemscount)
|
||||||
|
|
||||||
|
baseOrg := domain.Organization{
|
||||||
|
Alias: "test-alias",
|
||||||
|
Name: "test-name",
|
||||||
|
Site: "test-site",
|
||||||
|
LogoLink: "test-logo",
|
||||||
|
}
|
||||||
|
for i := 0; i < itemscount; i++ {
|
||||||
|
nextitem := baseOrg
|
||||||
|
iStr := strconv.Itoa(i)
|
||||||
|
nextitem.ID = "test-id-" + iStr
|
||||||
|
nextitem.ExternalID.Set("test-ext-id-" + iStr)
|
||||||
|
|
||||||
|
gotOrg, err := s.connection.Organization().Create(s.ctx, domain.CreateOrganizationParams{
|
||||||
|
ID: nextitem.ID,
|
||||||
|
ExternalID: nextitem.ExternalID,
|
||||||
|
Alias: nextitem.Alias,
|
||||||
|
Name: nextitem.Name,
|
||||||
|
Site: nextitem.Site,
|
||||||
|
LogoLink: nextitem.LogoLink,
|
||||||
|
})
|
||||||
|
s.NoError(err)
|
||||||
|
orgsdb = append(orgsdb, gotOrg)
|
||||||
|
}
|
||||||
|
|
||||||
|
gotOrgs, err := s.connection.Organization().List(s.ctx)
|
||||||
|
s.NoError(err)
|
||||||
|
|
||||||
|
compareF := func(lhs, rhs domain.Organization) int {
|
||||||
|
if lhs.ID < rhs.ID {
|
||||||
|
return -1
|
||||||
|
} else if lhs.ID > rhs.ID {
|
||||||
|
return 1
|
||||||
|
} else {
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
}
|
||||||
|
slices.SortFunc(gotOrgs, compareF)
|
||||||
|
|
||||||
|
for i := range gotOrgs {
|
||||||
|
s.NotEmpty(gotOrgs[i].CreatedAt)
|
||||||
|
s.NotEmpty(gotOrgs[i].UpdatedAt)
|
||||||
|
s.Empty(gotOrgs[i].DeletedAt)
|
||||||
|
|
||||||
|
orgsdb[i].CreatedAt = gotOrgs[i].CreatedAt
|
||||||
|
orgsdb[i].UpdatedAt = gotOrgs[i].UpdatedAt
|
||||||
|
orgsdb[i].DeletedAt = gotOrgs[i].DeletedAt
|
||||||
|
}
|
||||||
|
|
||||||
|
s.ElementsMatch(orgsdb, gotOrgs)
|
||||||
|
}
|
||||||
|
|
||||||
func (s *sqliteOrganzationRepositorySuite) TestGet() {
|
func (s *sqliteOrganzationRepositorySuite) TestGet() {
|
||||||
var orgdb organizationDB
|
var orgdb organizationDB
|
||||||
err := s.connection.db.GetContext(
|
err := s.connection.db.GetContext(
|
||||||
|
|||||||
@ -109,6 +109,10 @@ func (conn *YDBConnection) Close() error {
|
|||||||
return conn.Driver.Close(ctx)
|
return conn.Driver.Close(ctx)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (conn *YDBConnection) Organization() domain.OrganizationRepository {
|
||||||
|
return domain.NotImplementedOrganizationRepository{}
|
||||||
|
}
|
||||||
|
|
||||||
func (conn *YDBConnection) LearningCategory() domain.LearningCategoryRepository {
|
func (conn *YDBConnection) LearningCategory() domain.LearningCategoryRepository {
|
||||||
return domain.NotImplementedLearningCategory{}
|
return domain.NotImplementedLearningCategory{}
|
||||||
}
|
}
|
||||||
|
|||||||
@ -10,6 +10,8 @@ type Commands struct {
|
|||||||
InsertCourse command.CreateCourseHandler
|
InsertCourse command.CreateCourseHandler
|
||||||
DeleteCourse command.DeleteCourseHandler
|
DeleteCourse command.DeleteCourseHandler
|
||||||
UpdateCourseDescription command.UpdateCourseDescriptionHandler
|
UpdateCourseDescription command.UpdateCourseDescriptionHandler
|
||||||
|
|
||||||
|
InsertOrganization command.CreateOrganizationHandler
|
||||||
}
|
}
|
||||||
|
|
||||||
type Queries struct {
|
type Queries struct {
|
||||||
@ -17,6 +19,9 @@ type Queries struct {
|
|||||||
ListCourses query.ListCourseHandler
|
ListCourses query.ListCourseHandler
|
||||||
ListLearningTypes query.ListLearningTypesHandler
|
ListLearningTypes query.ListLearningTypesHandler
|
||||||
ListCourseThematics query.ListCourseThematicsHandler
|
ListCourseThematics query.ListCourseThematicsHandler
|
||||||
|
|
||||||
|
ListOrganzations query.ListOrganizationsHandler
|
||||||
|
GetOrganization query.GetOrganizationHandler
|
||||||
}
|
}
|
||||||
|
|
||||||
type Application struct {
|
type Application struct {
|
||||||
|
|||||||
53
internal/kurious/app/command/createorganization.go
Normal file
53
internal/kurious/app/command/createorganization.go
Normal file
@ -0,0 +1,53 @@
|
|||||||
|
package command
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"fmt"
|
||||||
|
"log/slog"
|
||||||
|
|
||||||
|
"git.loyso.art/frx/kurious/internal/common/decorator"
|
||||||
|
"git.loyso.art/frx/kurious/internal/common/nullable"
|
||||||
|
"git.loyso.art/frx/kurious/internal/kurious/domain"
|
||||||
|
)
|
||||||
|
|
||||||
|
type CreateOrganization struct {
|
||||||
|
ID string
|
||||||
|
ExternalID nullable.Value[string]
|
||||||
|
Alias string
|
||||||
|
Name string
|
||||||
|
Site string
|
||||||
|
Logo string
|
||||||
|
}
|
||||||
|
|
||||||
|
type CreateOrganizationHandler decorator.CommandHandler[CreateOrganization]
|
||||||
|
|
||||||
|
type createOrganizationHandler struct {
|
||||||
|
repo domain.OrganizationRepository
|
||||||
|
}
|
||||||
|
|
||||||
|
func NewCreateOrganizationHandler(
|
||||||
|
repo domain.OrganizationRepository,
|
||||||
|
log *slog.Logger,
|
||||||
|
) CreateOrganizationHandler {
|
||||||
|
h := createOrganizationHandler{
|
||||||
|
repo: repo,
|
||||||
|
}
|
||||||
|
|
||||||
|
return decorator.ApplyCommandDecorators(h, log)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (h createOrganizationHandler) Handle(ctx context.Context, cmd CreateOrganization) error {
|
||||||
|
_, err := h.repo.Create(ctx, domain.CreateOrganizationParams{
|
||||||
|
ID: cmd.ID,
|
||||||
|
ExternalID: cmd.ExternalID,
|
||||||
|
Alias: cmd.Alias,
|
||||||
|
Name: cmd.Name,
|
||||||
|
Site: cmd.Site,
|
||||||
|
LogoLink: cmd.Logo,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("creating organization: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
45
internal/kurious/app/query/getorganization.go
Normal file
45
internal/kurious/app/query/getorganization.go
Normal file
@ -0,0 +1,45 @@
|
|||||||
|
package query
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"fmt"
|
||||||
|
"log/slog"
|
||||||
|
|
||||||
|
"git.loyso.art/frx/kurious/internal/common/decorator"
|
||||||
|
"git.loyso.art/frx/kurious/internal/common/nullable"
|
||||||
|
"git.loyso.art/frx/kurious/internal/kurious/domain"
|
||||||
|
)
|
||||||
|
|
||||||
|
type GetOrganization struct {
|
||||||
|
ID nullable.Value[string]
|
||||||
|
ExternalID nullable.Value[string]
|
||||||
|
}
|
||||||
|
|
||||||
|
type GetOrganizationHandler decorator.QueryHandler[GetOrganization, domain.Organization]
|
||||||
|
|
||||||
|
type getOrganizationHandler struct {
|
||||||
|
repo domain.OrganizationRepository
|
||||||
|
}
|
||||||
|
|
||||||
|
func NewGetOrganizationHandler(
|
||||||
|
repo domain.OrganizationRepository,
|
||||||
|
log *slog.Logger,
|
||||||
|
) GetOrganizationHandler {
|
||||||
|
h := getOrganizationHandler{
|
||||||
|
repo: repo,
|
||||||
|
}
|
||||||
|
|
||||||
|
return decorator.AddQueryDecorators(h, log)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (h getOrganizationHandler) Handle(ctx context.Context, query GetOrganization) (domain.Organization, error) {
|
||||||
|
organization, err := h.repo.Get(ctx, domain.GetOrganizationParams{
|
||||||
|
ID: query.ID,
|
||||||
|
ExternalID: query.ExternalID,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
return domain.Organization{}, fmt.Errorf("getting organization: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
return organization, nil
|
||||||
|
}
|
||||||
38
internal/kurious/app/query/listorganizations.go
Normal file
38
internal/kurious/app/query/listorganizations.go
Normal file
@ -0,0 +1,38 @@
|
|||||||
|
package query
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"fmt"
|
||||||
|
"log/slog"
|
||||||
|
|
||||||
|
"git.loyso.art/frx/kurious/internal/common/decorator"
|
||||||
|
"git.loyso.art/frx/kurious/internal/kurious/domain"
|
||||||
|
)
|
||||||
|
|
||||||
|
type ListOrganizations struct{}
|
||||||
|
|
||||||
|
type ListOrganizationsHandler decorator.QueryHandler[ListOrganizations, []domain.Organization]
|
||||||
|
|
||||||
|
type listOrganizationsHandler struct {
|
||||||
|
repo domain.OrganizationRepository
|
||||||
|
}
|
||||||
|
|
||||||
|
func NewListOrganizationsHandler(
|
||||||
|
repo domain.OrganizationRepository,
|
||||||
|
log *slog.Logger,
|
||||||
|
) ListOrganizationsHandler {
|
||||||
|
h := listOrganizationsHandler{
|
||||||
|
repo: repo,
|
||||||
|
}
|
||||||
|
|
||||||
|
return decorator.AddQueryDecorators(h, log)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (h listOrganizationsHandler) Handle(ctx context.Context, query ListOrganizations) ([]domain.Organization, error) {
|
||||||
|
organizations, err := h.repo.List(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("listing organizations: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
return organizations, nil
|
||||||
|
}
|
||||||
@ -183,6 +183,64 @@ func (_c *OrganizationRepository_Get_Call) RunAndReturn(run func(context.Context
|
|||||||
return _c
|
return _c
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// List provides a mock function with given fields: _a0
|
||||||
|
func (_m *OrganizationRepository) List(_a0 context.Context) ([]domain.Organization, error) {
|
||||||
|
ret := _m.Called(_a0)
|
||||||
|
|
||||||
|
if len(ret) == 0 {
|
||||||
|
panic("no return value specified for List")
|
||||||
|
}
|
||||||
|
|
||||||
|
var r0 []domain.Organization
|
||||||
|
var r1 error
|
||||||
|
if rf, ok := ret.Get(0).(func(context.Context) ([]domain.Organization, error)); ok {
|
||||||
|
return rf(_a0)
|
||||||
|
}
|
||||||
|
if rf, ok := ret.Get(0).(func(context.Context) []domain.Organization); ok {
|
||||||
|
r0 = rf(_a0)
|
||||||
|
} else {
|
||||||
|
if ret.Get(0) != nil {
|
||||||
|
r0 = ret.Get(0).([]domain.Organization)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if rf, ok := ret.Get(1).(func(context.Context) error); ok {
|
||||||
|
r1 = rf(_a0)
|
||||||
|
} else {
|
||||||
|
r1 = ret.Error(1)
|
||||||
|
}
|
||||||
|
|
||||||
|
return r0, r1
|
||||||
|
}
|
||||||
|
|
||||||
|
// OrganizationRepository_List_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'List'
|
||||||
|
type OrganizationRepository_List_Call struct {
|
||||||
|
*mock.Call
|
||||||
|
}
|
||||||
|
|
||||||
|
// List is a helper method to define mock.On call
|
||||||
|
// - _a0 context.Context
|
||||||
|
func (_e *OrganizationRepository_Expecter) List(_a0 interface{}) *OrganizationRepository_List_Call {
|
||||||
|
return &OrganizationRepository_List_Call{Call: _e.mock.On("List", _a0)}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (_c *OrganizationRepository_List_Call) Run(run func(_a0 context.Context)) *OrganizationRepository_List_Call {
|
||||||
|
_c.Call.Run(func(args mock.Arguments) {
|
||||||
|
run(args[0].(context.Context))
|
||||||
|
})
|
||||||
|
return _c
|
||||||
|
}
|
||||||
|
|
||||||
|
func (_c *OrganizationRepository_List_Call) Return(_a0 []domain.Organization, _a1 error) *OrganizationRepository_List_Call {
|
||||||
|
_c.Call.Return(_a0, _a1)
|
||||||
|
return _c
|
||||||
|
}
|
||||||
|
|
||||||
|
func (_c *OrganizationRepository_List_Call) RunAndReturn(run func(context.Context) ([]domain.Organization, error)) *OrganizationRepository_List_Call {
|
||||||
|
_c.Call.Return(run)
|
||||||
|
return _c
|
||||||
|
}
|
||||||
|
|
||||||
// NewOrganizationRepository creates a new instance of OrganizationRepository. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations.
|
// NewOrganizationRepository creates a new instance of OrganizationRepository. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations.
|
||||||
// The first argument is typically a *testing.T value.
|
// The first argument is typically a *testing.T value.
|
||||||
func NewOrganizationRepository(t interface {
|
func NewOrganizationRepository(t interface {
|
||||||
|
|||||||
@ -93,11 +93,27 @@ type CreateOrganizationParams struct {
|
|||||||
|
|
||||||
//go:generate mockery --name OrganizationRepository
|
//go:generate mockery --name OrganizationRepository
|
||||||
type OrganizationRepository interface {
|
type OrganizationRepository interface {
|
||||||
|
List(context.Context) ([]Organization, error)
|
||||||
Get(context.Context, GetOrganizationParams) (Organization, error)
|
Get(context.Context, GetOrganizationParams) (Organization, error)
|
||||||
Create(context.Context, CreateOrganizationParams) (Organization, error)
|
Create(context.Context, CreateOrganizationParams) (Organization, error)
|
||||||
Delete(ctx context.Context, id string) error
|
Delete(ctx context.Context, id string) error
|
||||||
}
|
}
|
||||||
|
|
||||||
|
type NotImplementedOrganizationRepository struct{}
|
||||||
|
|
||||||
|
func (NotImplementedOrganizationRepository) List(context.Context) ([]Organization, error) {
|
||||||
|
return nil, ErrNotImplemented
|
||||||
|
}
|
||||||
|
func (NotImplementedOrganizationRepository) Get(context.Context, GetOrganizationParams) (Organization, error) {
|
||||||
|
return Organization{}, ErrNotImplemented
|
||||||
|
}
|
||||||
|
func (NotImplementedOrganizationRepository) Create(context.Context, CreateOrganizationParams) (Organization, error) {
|
||||||
|
return Organization{}, ErrNotImplemented
|
||||||
|
}
|
||||||
|
func (NotImplementedOrganizationRepository) Delete(ctx context.Context, id string) error {
|
||||||
|
return ErrNotImplemented
|
||||||
|
}
|
||||||
|
|
||||||
//go:generate mockery --name LearningCategoryRepository
|
//go:generate mockery --name LearningCategoryRepository
|
||||||
type LearningCategoryRepository interface {
|
type LearningCategoryRepository interface {
|
||||||
Upsert(context.Context, LearningCategory) error
|
Upsert(context.Context, LearningCategory) error
|
||||||
|
|||||||
@ -34,8 +34,9 @@ type syncSravniHandler struct {
|
|||||||
client sravni.Client
|
client sravni.Client
|
||||||
log *slog.Logger
|
log *slog.Logger
|
||||||
|
|
||||||
knownExternalIDs map[string]struct{}
|
knownExternalIDs map[string]struct{}
|
||||||
isRunning uint32
|
knownOrganizationsByExternalID map[string]struct{}
|
||||||
|
isRunning uint32
|
||||||
}
|
}
|
||||||
|
|
||||||
func (h *syncSravniHandler) Handle(ctx context.Context) (err error) {
|
func (h *syncSravniHandler) Handle(ctx context.Context) (err error) {
|
||||||
@ -83,6 +84,7 @@ func (h *syncSravniHandler) Handle(ctx context.Context) (err error) {
|
|||||||
learningTypes := state.Props.InitialReduxState.Dictionaries.Data.LearningType
|
learningTypes := state.Props.InitialReduxState.Dictionaries.Data.LearningType
|
||||||
courses := make([]sravni.Course, 0, 1024)
|
courses := make([]sravni.Course, 0, 1024)
|
||||||
buffer := make([]sravni.Course, 0, 512)
|
buffer := make([]sravni.Course, 0, 512)
|
||||||
|
organizations := make([]sravni.Organization, 0, 256)
|
||||||
for _, learningType := range learningTypes.Fields {
|
for _, learningType := range learningTypes.Fields {
|
||||||
select {
|
select {
|
||||||
case <-ctx.Done():
|
case <-ctx.Done():
|
||||||
@ -115,61 +117,98 @@ func (h *syncSravniHandler) Handle(ctx context.Context) (err error) {
|
|||||||
|
|
||||||
// since count is known it might be optimized to allocate slice once per request.
|
// since count is known it might be optimized to allocate slice once per request.
|
||||||
for _, courseThematic := range thematics {
|
for _, courseThematic := range thematics {
|
||||||
|
var filteredCourses int
|
||||||
|
var filteredOrgs int
|
||||||
|
|
||||||
|
var orgsByID map[string]sravni.Organization
|
||||||
buffer = buffer[:0]
|
buffer = buffer[:0]
|
||||||
buffer, err = h.loadEducationalProducts(lctx, learningType.Value, courseThematic, buffer)
|
buffer, orgsByID, err = h.loadEducationalProducts(lctx, learningType.Value, courseThematic, buffer)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
if !errors.Is(err, context.Canceled) {
|
if errors.Is(err, context.Canceled) {
|
||||||
xcontext.LogWithWarnError(lctx, h.log, err, "unable to load educational products", slog.Int("count", len(thematics)))
|
return fmt.Errorf("loading educational products: %w", err)
|
||||||
continue
|
|
||||||
}
|
}
|
||||||
|
|
||||||
return fmt.Errorf("loading educational products: %w", err)
|
xcontext.LogWithWarnError(lctx, h.log, err, "unable to load educational products", slog.Int("count", len(thematics)))
|
||||||
|
|
||||||
|
continue
|
||||||
}
|
}
|
||||||
|
|
||||||
xslices.ForEach(buffer, func(c sravni.Course) {
|
xslices.ForEach(buffer, func(c sravni.Course) {
|
||||||
|
// TODO: if the same course appears in different categories, it should be handled
|
||||||
|
if !h.setCourseIfNotKnown(c) {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
c.Learningtype = []string{learningType.Value}
|
c.Learningtype = []string{learningType.Value}
|
||||||
c.CourseThematics = []string{courseThematic}
|
c.CourseThematics = []string{courseThematic}
|
||||||
courses = append(courses, c)
|
courses = append(courses, c)
|
||||||
|
filteredCourses++
|
||||||
})
|
})
|
||||||
|
|
||||||
xcontext.LogInfo(lctx, h.log, "parsed subitems", slog.String("course_thematic", courseThematic), slog.Int("amount", len(buffer)))
|
for _, org := range orgsByID {
|
||||||
|
if !h.setOrganizationIfNotKnown(org) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
organizations = append(organizations, org)
|
||||||
|
filteredOrgs++
|
||||||
|
}
|
||||||
|
|
||||||
|
xcontext.LogInfo(
|
||||||
|
lctx, h.log, "parsed subitems",
|
||||||
|
slog.String("course_thematic", courseThematic),
|
||||||
|
slog.Int("amount", len(buffer)),
|
||||||
|
slog.Int("new_courses", filteredCourses),
|
||||||
|
slog.Int("new_organizations", filteredOrgs),
|
||||||
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
elapsed := time.Since(start)
|
elapsed := time.Since(start)
|
||||||
xcontext.LogDebug(lctx, h.log, "parsed items", slog.Duration("elapsed", elapsed), slog.Int("amount", len(courses)))
|
xcontext.LogDebug(lctx, h.log, "parsed items", slog.Duration("elapsed", elapsed), slog.Int("amount", len(courses)))
|
||||||
|
|
||||||
// TODO: if the same course appears in different categories, it should be handled
|
xcontext.LogDebug(
|
||||||
courses = h.filterByCache(courses)
|
lctx, h.log, "filtered items",
|
||||||
if len(courses) == 0 {
|
slog.Int("courses", len(courses)),
|
||||||
xcontext.LogInfo(lctx, h.log, "all courses were filtered out")
|
slog.Int("organizations", len(organizations)),
|
||||||
continue
|
)
|
||||||
|
|
||||||
|
var insertCourseSuccess bool
|
||||||
|
if len(courses) > 0 {
|
||||||
|
err = h.insertCourses(lctx, courses)
|
||||||
|
if err != nil {
|
||||||
|
xcontext.LogWithError(lctx, h.log, err, "unable to insert courses")
|
||||||
|
}
|
||||||
|
|
||||||
|
insertCourseSuccess = err == nil
|
||||||
}
|
}
|
||||||
|
|
||||||
xcontext.LogDebug(lctx, h.log, "filtered items", slog.Int("amount", len(courses)))
|
var insertOrgsSuccess bool
|
||||||
|
if len(organizations) > 0 {
|
||||||
|
err = h.insertOrganizations(lctx, organizations)
|
||||||
|
if err != nil {
|
||||||
|
xcontext.LogWithError(lctx, h.log, err, "unable to insert courses")
|
||||||
|
}
|
||||||
|
|
||||||
|
insertOrgsSuccess = err == nil
|
||||||
|
}
|
||||||
|
|
||||||
err = h.insertValues(lctx, courses)
|
|
||||||
elapsed = time.Since(start) - elapsed
|
elapsed = time.Since(start) - elapsed
|
||||||
elapsedField := slog.Duration("elapsed", elapsed)
|
elapsedField := slog.Duration("elapsed", elapsed)
|
||||||
if err != nil {
|
|
||||||
xcontext.LogWithError(lctx, h.log, err, "unable to insert courses", elapsedField)
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
|
|
||||||
xslices.ForEach(courses, func(c sravni.Course) {
|
|
||||||
h.knownExternalIDs[c.ID] = struct{}{}
|
|
||||||
})
|
|
||||||
|
|
||||||
xcontext.LogInfo(
|
xcontext.LogInfo(
|
||||||
lctx, h.log, "processed items",
|
lctx, h.log, "inserting finished",
|
||||||
elapsedField,
|
elapsedField,
|
||||||
slog.Int("count", len(courses)),
|
slog.Bool("courses_insert_success", insertCourseSuccess),
|
||||||
|
slog.Bool("organization_insert_success", insertOrgsSuccess),
|
||||||
|
slog.Int("courses_count", len(courses)),
|
||||||
|
slog.Int("organizations_count", len(organizations)),
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (h *syncSravniHandler) loadEducationalProducts(ctx context.Context, learningType, courseThematic string, buf []sravni.Course) ([]sravni.Course, error) {
|
func (h *syncSravniHandler) loadEducationalProducts(ctx context.Context, learningType, courseThematic string, buf []sravni.Course) ([]sravni.Course, map[string]sravni.Organization, error) {
|
||||||
const maxDeepIteration = 10
|
const maxDeepIteration = 10
|
||||||
const defaultLimit = 50
|
const defaultLimit = 50
|
||||||
|
|
||||||
@ -177,6 +216,7 @@ func (h *syncSravniHandler) loadEducationalProducts(ctx context.Context, learnin
|
|||||||
rateLimit := rate.NewLimiter(rateStrategy, 1)
|
rateLimit := rate.NewLimiter(rateStrategy, 1)
|
||||||
|
|
||||||
var courses []sravni.Course
|
var courses []sravni.Course
|
||||||
|
var organizationsByID = make(map[string]sravni.Organization)
|
||||||
if buf == nil || cap(buf) == 0 {
|
if buf == nil || cap(buf) == 0 {
|
||||||
courses = make([]sravni.Course, 0, 256)
|
courses = make([]sravni.Course, 0, 256)
|
||||||
} else {
|
} else {
|
||||||
@ -193,35 +233,49 @@ func (h *syncSravniHandler) loadEducationalProducts(ctx context.Context, learnin
|
|||||||
params.Offset = offset
|
params.Offset = offset
|
||||||
response, err := h.client.ListEducationalProducts(ctx, params)
|
response, err := h.client.ListEducationalProducts(ctx, params)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("listing educational products: %w", err)
|
return nil, nil, fmt.Errorf("listing educational products: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
offset += defaultLimit
|
offset += defaultLimit
|
||||||
|
|
||||||
courses = append(courses, response.Items...)
|
courses = append(courses, response.Items...)
|
||||||
|
|
||||||
|
for oid, org := range response.Organizations {
|
||||||
|
organizationsByID[oid] = org
|
||||||
|
}
|
||||||
|
|
||||||
if len(response.Items) < defaultLimit {
|
if len(response.Items) < defaultLimit {
|
||||||
break
|
break
|
||||||
}
|
}
|
||||||
|
|
||||||
err = rateLimit.Wait(ctx)
|
err = rateLimit.Wait(ctx)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return courses, fmt.Errorf("waiting for limit: %w", err)
|
return courses, organizationsByID, fmt.Errorf("waiting for limit: %w", err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
return courses, nil
|
return courses, organizationsByID, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (h *syncSravniHandler) filterByCache(courses []sravni.Course) (toInsert []sravni.Course) {
|
func (h *syncSravniHandler) setCourseIfNotKnown(course sravni.Course) (set bool) {
|
||||||
toCut := xslices.FilterInplace(courses, xslices.Not(h.isCached))
|
|
||||||
return courses[:toCut]
|
|
||||||
}
|
|
||||||
|
|
||||||
func (h *syncSravniHandler) isCached(course sravni.Course) bool {
|
|
||||||
_, ok := h.knownExternalIDs[course.ID]
|
_, ok := h.knownExternalIDs[course.ID]
|
||||||
return ok
|
if !ok {
|
||||||
|
h.knownExternalIDs[course.ID] = struct{}{}
|
||||||
|
}
|
||||||
|
|
||||||
|
return !ok
|
||||||
}
|
}
|
||||||
|
|
||||||
func (h *syncSravniHandler) insertValues(ctx context.Context, courses []sravni.Course) error {
|
func (h *syncSravniHandler) setOrganizationIfNotKnown(organization sravni.Organization) bool {
|
||||||
|
_, ok := h.knownOrganizationsByExternalID[organization.ID]
|
||||||
|
if !ok {
|
||||||
|
h.knownOrganizationsByExternalID[organization.ID] = struct{}{}
|
||||||
|
}
|
||||||
|
|
||||||
|
return !ok
|
||||||
|
}
|
||||||
|
|
||||||
|
func (h *syncSravniHandler) insertCourses(ctx context.Context, courses []sravni.Course) error {
|
||||||
courseParams := xslices.Map(courses, courseAsCreateCourseParams)
|
courseParams := xslices.Map(courses, courseAsCreateCourseParams)
|
||||||
err := h.svc.Commands.InsertCourses.Handle(ctx, command.CreateCourses{
|
err := h.svc.Commands.InsertCourses.Handle(ctx, command.CreateCourses{
|
||||||
Courses: courseParams,
|
Courses: courseParams,
|
||||||
@ -233,7 +287,69 @@ func (h *syncSravniHandler) insertValues(ctx context.Context, courses []sravni.C
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (h *syncSravniHandler) insertOrganizations(ctx context.Context, organizations []sravni.Organization) error {
|
||||||
|
organizationParams := xslices.Map(organizations, func(in sravni.Organization) command.CreateOrganization {
|
||||||
|
return command.CreateOrganization{
|
||||||
|
ID: generator.RandomInt64ID(),
|
||||||
|
ExternalID: nullable.NewValue(in.ID),
|
||||||
|
Alias: in.Alias,
|
||||||
|
Name: in.Name.Short,
|
||||||
|
Site: "",
|
||||||
|
Logo: in.Logotypes.Web,
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
for _, params := range organizationParams {
|
||||||
|
err := h.svc.Commands.InsertOrganization.Handle(ctx, params)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("inserting organization: %w", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (h *syncSravniHandler) fillOrganizaionCaches(ctx context.Context) error {
|
||||||
|
if h.knownOrganizationsByExternalID != nil {
|
||||||
|
xcontext.LogDebug(ctx, h.log, "organization cache already filled")
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
organizations, err := h.svc.Queries.ListOrganzations.Handle(ctx, query.ListOrganizations{})
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("listing organizations: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
withExternalID := func(in domain.Organization) bool {
|
||||||
|
return in.ExternalID.Valid()
|
||||||
|
}
|
||||||
|
getExtID := func(in domain.Organization) string {
|
||||||
|
return in.ExternalID.Value()
|
||||||
|
}
|
||||||
|
|
||||||
|
h.knownOrganizationsByExternalID = xslices.AsMap(xslices.Filter(organizations, withExternalID), getExtID)
|
||||||
|
|
||||||
|
xcontext.LogInfo(ctx, h.log, "cache filled", slog.String("kind", "organizations_by_external_id"), slog.Int("count", len(organizations)))
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
func (h *syncSravniHandler) fillCaches(ctx context.Context) error {
|
func (h *syncSravniHandler) fillCaches(ctx context.Context) error {
|
||||||
|
err := h.fillOrganizaionCaches(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
err = h.fillKnownExternalIDsCache(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (h *syncSravniHandler) fillKnownExternalIDsCache(ctx context.Context) error {
|
||||||
if h.knownExternalIDs != nil {
|
if h.knownExternalIDs != nil {
|
||||||
xcontext.LogInfo(ctx, h.log, "cache already filled")
|
xcontext.LogInfo(ctx, h.log, "cache already filled")
|
||||||
|
|
||||||
@ -256,7 +372,7 @@ func (h *syncSravniHandler) fillCaches(ctx context.Context) error {
|
|||||||
h.knownExternalIDs[c.ExternalID.Value()] = struct{}{}
|
h.knownExternalIDs[c.ExternalID.Value()] = struct{}{}
|
||||||
})
|
})
|
||||||
|
|
||||||
xcontext.LogInfo(ctx, h.log, "cache filled", slog.Int("count", len(courses)))
|
xcontext.LogInfo(ctx, h.log, "cache filled", slog.String("kind", "courses_by_external_id"), slog.Int("count", len(courses)))
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|||||||
@ -43,6 +43,7 @@ func NewApplication(ctx context.Context, cfg ApplicationConfig, mapper domain.Co
|
|||||||
|
|
||||||
var repoCloser io.Closer
|
var repoCloser io.Closer
|
||||||
var courseadapter domain.CourseRepository
|
var courseadapter domain.CourseRepository
|
||||||
|
var organizationrepo domain.OrganizationRepository
|
||||||
switch cfg.Engine {
|
switch cfg.Engine {
|
||||||
case RepositoryEngineSqlite:
|
case RepositoryEngineSqlite:
|
||||||
sqliteConnection, err := adapters.NewSqliteConnection(ctx, cfg.Sqlite, log.With(slog.String("db", "sqlite")))
|
sqliteConnection, err := adapters.NewSqliteConnection(ctx, cfg.Sqlite, log.With(slog.String("db", "sqlite")))
|
||||||
@ -51,6 +52,7 @@ func NewApplication(ctx context.Context, cfg ApplicationConfig, mapper domain.Co
|
|||||||
}
|
}
|
||||||
|
|
||||||
courseadapter = sqliteConnection.CourseRepository()
|
courseadapter = sqliteConnection.CourseRepository()
|
||||||
|
organizationrepo = sqliteConnection.Organization()
|
||||||
repoCloser = sqliteConnection
|
repoCloser = sqliteConnection
|
||||||
case RepositoryEngineYDB:
|
case RepositoryEngineYDB:
|
||||||
ydbConnection, err := adapters.NewYDBConnection(ctx, cfg.YDB, log.With(slog.String("db", "ydb")))
|
ydbConnection, err := adapters.NewYDBConnection(ctx, cfg.YDB, log.With(slog.String("db", "ydb")))
|
||||||
@ -59,9 +61,10 @@ func NewApplication(ctx context.Context, cfg ApplicationConfig, mapper domain.Co
|
|||||||
}
|
}
|
||||||
|
|
||||||
courseadapter = ydbConnection.CourseRepository()
|
courseadapter = ydbConnection.CourseRepository()
|
||||||
|
organizationrepo = ydbConnection.Organization()
|
||||||
repoCloser = ydbConnection
|
repoCloser = ydbConnection
|
||||||
default:
|
default:
|
||||||
return Application{}, errors.New("unable to decide which engine to use")
|
return Application{}, errors.New("unable to decide which db engine to use")
|
||||||
}
|
}
|
||||||
|
|
||||||
err = mapper.CollectCounts(ctx, courseadapter)
|
err = mapper.CollectCounts(ctx, courseadapter)
|
||||||
@ -75,12 +78,17 @@ func NewApplication(ctx context.Context, cfg ApplicationConfig, mapper domain.Co
|
|||||||
InsertCourse: command.NewCreateCourseHandler(courseadapter, log),
|
InsertCourse: command.NewCreateCourseHandler(courseadapter, log),
|
||||||
DeleteCourse: command.NewDeleteCourseHandler(courseadapter, log),
|
DeleteCourse: command.NewDeleteCourseHandler(courseadapter, log),
|
||||||
UpdateCourseDescription: command.NewUpdateCourseDescriptionHandler(courseadapter, log),
|
UpdateCourseDescription: command.NewUpdateCourseDescriptionHandler(courseadapter, log),
|
||||||
|
|
||||||
|
InsertOrganization: command.NewCreateOrganizationHandler(organizationrepo, log),
|
||||||
},
|
},
|
||||||
Queries: app.Queries{
|
Queries: app.Queries{
|
||||||
ListCourses: query.NewListCourseHandler(courseadapter, mapper, log),
|
ListCourses: query.NewListCourseHandler(courseadapter, mapper, log),
|
||||||
ListLearningTypes: query.NewListLearningTypesHandler(courseadapter, mapper, log),
|
ListLearningTypes: query.NewListLearningTypesHandler(courseadapter, mapper, log),
|
||||||
ListCourseThematics: query.NewListCourseThematicsHandler(courseadapter, mapper, log),
|
ListCourseThematics: query.NewListCourseThematicsHandler(courseadapter, mapper, log),
|
||||||
GetCourse: query.NewGetCourseHandler(courseadapter, mapper, log),
|
GetCourse: query.NewGetCourseHandler(courseadapter, mapper, log),
|
||||||
|
|
||||||
|
ListOrganzations: query.NewListOrganizationsHandler(organizationrepo, log),
|
||||||
|
GetOrganization: query.NewGetOrganizationHandler(organizationrepo, log),
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user