frx/kurious
1
0
Fork 0
Code Issues 1 Pull Requests Actions Packages Projects 1 Releases Wiki Activity

list by pages and rate req limits

Browse Source
This commit is contained in:
Gitea
2023-12-15 13:00:59 +03:00
parent 2f3a6e35e9
commit 06c7997491
10 changed files with 204 additions and 116 deletions
Download Patch File Download Diff File Expand all files Collapse all files

169
internal/kurious/adapters/ydb_course_repository.go
View File

@ -80,7 +80,36 @@ type ydbCourseRepository struct {
}
func (r *ydbCourseRepository) List(ctx context.Context, params domain.ListCoursesParams) (courses []domain.Course, err error) {
const limit = 1000
const queryName = "list"
const query = `
DECLARE $limit AS Int32;
DECLARE $id AS Text;
SELECT
id,
external_id,
source_type,
source_name,
course_thematic,
learning_type,
organization_id,
origin_link,
image_link,
name,
description,
full_price,
discount,
duration,
starts_at,
created_at,
updated_at,
deleted_at
FROM
courses
WHERE
id > $id
ORDER BY id
LIMIT $limit;`
courses = make([]domain.Course, 0, 4_000)
readTx := table.TxControl(
@ -103,45 +132,38 @@ func (r *ydbCourseRepository) List(ctx context.Context, params domain.ListCourse
)
}()
_, res, err := s.Execute(
ctx,
readTx,
`SELECT
id,
external_id,
source_type,
source_name,
organization_id,
origin_link,
image_link,
name,
description,
full_price,
discount,
duration,
starts_at,
created_at,
updated_at,
deleted_at
FROM
courses
`,
table.NewQueryParameters(),
options.WithCollectStatsModeBasic(),
)
if err != nil {
return fmt.Errorf("executing: %w", err)
}
var lastKnownID string
for {
queryParams := table.NewQueryParameters(
table.ValueParam("$limit", types.Int32Value(limit)),
table.ValueParam("$id", types.TextValue(lastKnownID)),
)
_, res, err := s.Execute(
ctx, readTx, query, queryParams,
options.WithCollectStatsModeBasic(),
)
if err != nil {
return fmt.Errorf("executing: %w", err)
}
if !res.NextResultSet(ctx) || !res.HasNextRow() {
break
}
for res.NextResultSet(ctx) {
for res.NextRow() {
var cdb courseDB
_ = res.ScanNamed(cdb.getNamedValues()...)
err = res.ScanNamed(cdb.getNamedValues()...)
if err != nil {
return fmt.Errorf("scanning row: %w", err)
}
courses = append(courses, mapCourseDB(cdb))
}
}
if err = res.Err(); err != nil {
return err
if err = res.Err(); err != nil {
return err
}
lastKnownID = courses[len(courses)-1].ID
}
return nil
},
@ -187,6 +209,8 @@ func (r *ydbCourseRepository) Get(ctx context.Context, id string) (course domain
external_id,
source_type,
source_name,
course_thematic,
learning_type,
organization_id,
origin_link,
image_link,
@ -249,6 +273,8 @@ func createCourseParamsAsStruct(params domain.CreateCourseParams) types.Value {
types.StructFieldValue("name", types.TextValue(params.Name)),
types.StructFieldValue("source_type", types.TextValue(st)),
types.StructFieldValue("source_name", types.NullableTextValue(params.SourceName.ValutPtr())),
types.StructFieldValue("course_thematic", types.TextValue(params.CourseThematic)),
types.StructFieldValue("learning_type", types.TextValue(params.LearningType)),
types.StructFieldValue("external_id", types.NullableTextValue(params.ExternalID.ValutPtr())),
types.StructFieldValue("organization_id", types.TextValue(params.OrganizationID)),
types.StructFieldValue("origin_link", types.TextValue(params.OriginLink)),
@ -272,6 +298,8 @@ func (r *ydbCourseRepository) CreateBatch(ctx context.Context, params ...domain.
name: Text,
source_type: Text,
source_name: Optional<Text>,
course_thematic: Text,
learning_type: Text,
organization_id: Text,
origin_link: Text,
image_link: Text,
@ -292,6 +320,8 @@ func (r *ydbCourseRepository) CreateBatch(ctx context.Context, params ...domain.
name,
source_type,
source_name,
course_thematic,
learning_type,
organization_id,
origin_link,
image_link,
@ -328,65 +358,12 @@ func (r *ydbCourseRepository) CreateBatch(ctx context.Context, params ...domain.
}
func (r *ydbCourseRepository) Create(ctx context.Context, params domain.CreateCourseParams) (domain.Course, error) {
// -- PRAGMA TablePathPrefix("courses");
const upsertQuery = `DECLARE $courseData AS List<Struct<
id: Text,
external_id: Optional<Text>,
name: Text,
source_type: Text,
source_name: Optional<Text>,
organization_id: Text,
origin_link: Text,
image_link: Text,
description: Text,
full_price: Double,
discount: Double,
duration: Interval,
starts_at: Datetime,
created_at: Datetime,
updated_at: Datetime,
deleted_at: Optional<Datetime>>>;
err := r.CreateBatch(ctx, params)
if err != nil {
return domain.Course{}, err
}
REPLACE INTO
courses
SELECT
id,
external_id,
name,
source_type,
source_name,
organization_id,
origin_link,
image_link,
description,
full_price,
discount,
duration,
starts_at,
created_at,
updated_at,
deleted_at
FROM AS_TABLE($courseData);`
writeTx := table.TxControl(
table.BeginTx(
table.WithSerializableReadWrite(),
),
table.CommitTx(),
)
err := r.db.Table().Do(ctx, func(ctx context.Context, s table.Session) error {
queryParams := table.NewQueryParameters(
table.ValueParam("$courseData", types.ListValue(createCourseParamsAsStruct(params))),
)
_, _, err := s.Execute(ctx, writeTx, upsertQuery, queryParams)
if err != nil {
return fmt.Errorf("executing query: %w", err)
}
return nil
})
return domain.Course{}, err
return domain.Course{}, nil
}
func (r *ydbCourseRepository) Delete(ctx context.Context, id string) error {
@ -403,6 +380,8 @@ func (r *ydbCourseRepository) CreateCourseTable(ctx context.Context) error {
options.WithColumn("name", types.TypeText),
options.WithColumn("source_type", types.TypeText),
options.WithColumn("source_name", types.Optional(types.TypeText)),
options.WithColumn("course_thematic", types.TypeText),
options.WithColumn("learning_type", types.TypeText),
options.WithColumn("organization_id", types.TypeText),
options.WithColumn("origin_link", types.TypeText),
options.WithColumn("image_link", types.TypeText),
@ -425,6 +404,8 @@ type courseDB struct {
Name string
SourceType string
SourceName *string
CourseThematic string
LearningType string
OrganizationID string
OriginLink string
ImageLink string
@ -445,6 +426,8 @@ func (c *courseDB) getNamedValues() []named.Value {
named.Optional("external_id", &c.ExternalID),
named.Required("source_type", &c.SourceType),
named.Optional("source_name", &c.SourceName),
named.Required("course_thematic", &c.CourseThematic),
named.Required("learning_type", &c.LearningType),
named.Required("organization_id", &c.OrganizationID),
named.Required("origin_link", &c.OriginLink),
named.Required("image_link", &c.ImageLink),
@ -501,6 +484,8 @@ func mapCourseDB(cdb courseDB) domain.Course {
Name: cdb.Name,
SourceType: st,
SourceName: nullable.NewValuePtr(cdb.SourceName),
Thematic: cdb.CourseThematic,
LearningType: cdb.LearningType,
OrganizationID: cdb.OrganizationID,
OriginLink: cdb.OriginLink,
ImageLink: cdb.ImageLink,