frx/kurious
1
0
Fork 0
Code Issues 2 Pull Requests Actions Packages Projects 1 Releases Wiki Activity

add show course and map names

Browse Source
This commit is contained in:
Aleksandr Trushkin
2024-01-08 18:31:35 +03:00
parent 48f5d80f7a
commit 2c0564f68c
22 changed files with 166 additions and 90 deletions
Download Patch File Download Diff File Expand all files Collapse all files

37
cmd/background/main.go
View File

@ -13,6 +13,7 @@ import (
"git.loyso.art/frx/kurious/internal/common/config"
"git.loyso.art/frx/kurious/internal/common/xcontext"
"git.loyso.art/frx/kurious/internal/common/xlog"
"git.loyso.art/frx/kurious/internal/kurious/adapters"
"git.loyso.art/frx/kurious/internal/kurious/ports"
"git.loyso.art/frx/kurious/internal/kurious/service"
)
@ -43,19 +44,39 @@ func app(ctx context.Context) error {
log := config.NewSLogger(cfg.Log)
app, err := service.NewApplication(ctx, service.ApplicationConfig{
LogConfig: cfg.Log,
YDB: cfg.YDB,
})
if err != nil {
return fmt.Errorf("making new application: %w", err)
}
sravniClient, err := sravni.NewClient(ctx, log, cfg.DebugHTTP)
if err != nil {
return fmt.Errorf("making sravni client: %w", err)
}
mainPageState, err := sravniClient.GetMainPageState()
if err != nil {
return fmt.Errorf("getting main page state: %w", err)
}
dictionaries := mainPageState.Props.InitialReduxState.Dictionaries.Data
dictFieldsAsMap := func(fields ...sravni.Field) map[string]string {
out := make(map[string]string, len(fields))
for _, field := range fields {
out[field.Value] = field.Name
}
return out
}
courseThematcisMapped := dictFieldsAsMap(dictionaries.CourseThematics.Fields...)
learningTypeMapped := dictFieldsAsMap(dictionaries.LearningType.Fields...)
mapper := adapters.NewMemoryMapper(courseThematcisMapped, learningTypeMapped)
app, err := service.NewApplication(ctx, service.ApplicationConfig{
LogConfig: cfg.Log,
YDB: cfg.YDB,
}, mapper)
if err != nil {
return fmt.Errorf("making new application: %w", err)
}
bgProcess := ports.NewBackgroundProcess(ctx, log)
err = bgProcess.RegisterSyncSravniHandler(ctx, app, sravniClient, cfg.SyncSravniCron)
if err != nil {

4
cmd/dev/sravnicli/products.go
View File

@ -9,7 +9,7 @@ import (
"git.loyso.art/frx/kurious/internal/common/client/sravni"
"git.loyso.art/frx/kurious/internal/common/errors"
"git.loyso.art/frx/kurious/internal/common/xslice"
"git.loyso.art/frx/kurious/internal/common/xslices"
"github.com/teris-io/cli"
)
@ -188,7 +188,7 @@ func (a *productsFilterCountAction) parse(args []string, options map[string]stri
filterNotEmpty := func(value string) bool {
return value != ""
}
a.params.courseThematic = xslice.Filter(
a.params.courseThematic = xslices.Filter(
strings.Split(options[courseThematicOptName], ","),
filterNotEmpty,
)

29
cmd/kuriweb/main.go
View File

@ -10,9 +10,11 @@ import (
"os/signal"
"time"
"git.loyso.art/frx/kurious/internal/common/client/sravni"
"git.loyso.art/frx/kurious/internal/common/config"
"git.loyso.art/frx/kurious/internal/common/generator"
"git.loyso.art/frx/kurious/internal/common/xcontext"
"git.loyso.art/frx/kurious/internal/kurious/adapters"
xhttp "git.loyso.art/frx/kurious/internal/kurious/ports/http"
"git.loyso.art/frx/kurious/internal/kurious/service"
"golang.org/x/sync/errgroup"
@ -46,10 +48,35 @@ func app(ctx context.Context) error {
log := config.NewSLogger(cfg.Log)
sravniClient, err := sravni.NewClient(ctx, log, false)
if err != nil {
return fmt.Errorf("unable to make new sravni client: %w", err)
}
mainPageState, err := sravniClient.GetMainPageState()
if err != nil {
return fmt.Errorf("getting main page state: %w", err)
}
dictionaries := mainPageState.Props.InitialReduxState.Dictionaries.Data
dictFieldsAsMap := func(fields ...sravni.Field) map[string]string {
out := make(map[string]string, len(fields))
for _, field := range fields {
out[field.Value] = field.Name
}
return out
}
courseThematcisMapped := dictFieldsAsMap(dictionaries.CourseThematics.Fields...)
learningTypeMapped := dictFieldsAsMap(dictionaries.LearningType.Fields...)
mapper := adapters.NewMemoryMapper(courseThematcisMapped, learningTypeMapped)
app, err := service.NewApplication(ctx, service.ApplicationConfig{
LogConfig: cfg.Log,
YDB: cfg.YDB,
})
}, mapper)
if err != nil {
return fmt.Errorf("making new application: %w", err)
}

6
internal/common/client/sravni/client.go
View File

@ -11,7 +11,7 @@ import (
"time"
"git.loyso.art/frx/kurious/internal/common/errors"
"git.loyso.art/frx/kurious/pkg/slices"
"git.loyso.art/frx/kurious/internal/common/xslices"
"git.loyso.art/frx/kurious/pkg/xdefault"
"github.com/go-resty/resty/v2"
@ -52,8 +52,8 @@ func NewClient(ctx context.Context, log *slog.Logger, debug bool) (c *client, er
return nil, err
}
getQuerySet := func(fields []field) querySet {
items := slices.Map(fields, func(f field) string {
getQuerySet := func(fields []Field) querySet {
items := xslices.Map(fields, func(f Field) string {
return f.Value
})

4
internal/common/client/sravni/entities.go
View File

@ -39,7 +39,7 @@ type ReduxMetadata struct {
} `json:"data"`
}
type field struct {
type Field struct {
Name string `json:"name"`
Value string `json:"value"`
}
@ -51,7 +51,7 @@ type ReduxDictionaryContainer struct {
UserID string `json:"userId"`
Created time.Time `json:"created"`
Updated time.Time `json:"updated"`
Fields []field `json:"fields"`
Fields []Field `json:"fields"`
}
type ReduxDictionaries struct {

1
internal/common/config/ydb.go
View File

@ -27,6 +27,7 @@ type YDB struct {
DSN string
Auth YCAuth
ShutdownDuration time.Duration
DebugYDB bool
}
func (ydb *YDB) UnmarshalJSON(data []byte) error {

2
internal/common/xslice/filter.go → internal/common/xslices/filter.go
View File

@ -1,4 +1,4 @@
package xslice
package xslices
func Filter[T any](values []T, f func(T) bool) []T {
out := make([]T, 0, len(values))

6
internal/common/xslice/filter_test.go → internal/common/xslices/filter_test.go
View File

@ -1,9 +1,9 @@
package xslice_test
package xslices_test
import (
"testing"
"git.loyso.art/frx/kurious/internal/common/xslice"
"git.loyso.art/frx/kurious/internal/common/xslices"
)
func TestFilterInplace(t *testing.T) {
@ -43,7 +43,7 @@ func TestFilterInplace(t *testing.T) {
for _, tc := range tt {
tc := tc
t.Run(tc.name, func(t *testing.T) {
gotLen := xslice.FilterInplace(tc.in, tc.check)
gotLen := xslices.FilterInplace(tc.in, tc.check)
if gotLen != tc.expLen {
t.Errorf("exp %d got %d", tc.expLen, gotLen)
}

2
internal/common/xslice/foreach.go → internal/common/xslices/foreach.go
View File

@ -1,4 +1,4 @@
package xslice
package xslices
func ForEach[T any](items []T, f func(T)) {
for _, item := range items {

2
internal/common/xslice/map.go → internal/common/xslices/map.go
View File

@ -1,4 +1,4 @@
package xslice
package xslices
func Map[T, U any](in []T, f func(T) U) []U {
out := make([]U, len(in))

21
internal/kurious/adapters/memory_mapper.go Normal file
View File

@ -0,0 +1,21 @@
package adapters
type inMemoryMapper struct {
courseThematicsByID map[string]string
learningTypeByID map[string]string
}
func NewMemoryMapper(courseThematics, learningType map[string]string) inMemoryMapper {
return inMemoryMapper{
courseThematicsByID: courseThematics,
learningTypeByID: learningType,
}
}
func (m inMemoryMapper) CourseThematicNameByID(id string) string {
return m.courseThematicsByID[id]
}
func (m inMemoryMapper) LearningTypeNameByID(id string) string {
return m.learningTypeByID[id]
}

51
internal/kurious/adapters/ydb_course_repository.go
View File

@ -4,6 +4,7 @@ import (
"context"
"fmt"
"log/slog"
"os"
"path"
"strings"
"text/template"
@ -17,10 +18,12 @@ import (
"git.loyso.art/frx/kurious/pkg/xdefault"
"github.com/ydb-platform/ydb-go-sdk/v3"
ydblog "github.com/ydb-platform/ydb-go-sdk/v3/log"
"github.com/ydb-platform/ydb-go-sdk/v3/table"
"github.com/ydb-platform/ydb-go-sdk/v3/table/options"
"github.com/ydb-platform/ydb-go-sdk/v3/table/result/named"
"github.com/ydb-platform/ydb-go-sdk/v3/table/types"
"github.com/ydb-platform/ydb-go-sdk/v3/trace"
yc "github.com/ydb-platform/ydb-go-yc"
)
@ -69,9 +72,11 @@ func NewYDBConnection(ctx context.Context, cfg config.YDB, log *slog.Logger) (*Y
yc.WithServiceAccountKeyFileCredentials(auth.Path),
)
}
opts = append(opts,
ydb.WithDialTimeout(time.Second*3),
)
if cfg.DebugYDB {
opts = append(opts,
ydb.WithLogger(ydblog.Default(os.Stdout, ydblog.WithMinLevel(ydblog.DEBUG)), trace.DetailsAll),
)
}
db, err := ydb.Open(
ctx,
@ -81,6 +86,14 @@ func NewYDBConnection(ctx context.Context, cfg config.YDB, log *slog.Logger) (*Y
if err != nil {
return nil, fmt.Errorf("opening connection: %w", err)
}
endpoints, err := db.Discovery().Discover(ctx)
if err != nil {
return nil, fmt.Errorf("discovering endpoints: %w", err)
}
for _, endpoint := range endpoints {
xcontext.LogInfo(ctx, log, "discovered endpoint", slog.String("value", endpoint.Address()))
}
return &YDBConnection{
Driver: db,
@ -163,11 +176,9 @@ func (r *ydbCourseRepository) List(
query, err := qtParams.render()
if err != nil {
return result, fmt.Errorf("rendering: %w", err)
return result, fmt.Errorf("rendering query params: %w", err)
}
xcontext.LogInfo(ctx, r.log, "planning to run query", slog.String("query", query), slog.String("opts", tableParamOptsToString(opts...)))
courses := make([]domain.Course, 0, 1_000)
readTx := table.TxControl(
table.BeginTx(
@ -176,11 +187,9 @@ func (r *ydbCourseRepository) List(
table.CommitTx(),
)
xcontext.LogInfo(ctx, r.log, "executing do")
err = r.db.Table().Do(
ctx,
func(ctx context.Context, s table.Session) error {
xcontext.LogInfo(ctx, r.log, "inside do")
start := time.Now()
defer func() {
since := time.Since(start)
@ -194,8 +203,6 @@ func (r *ydbCourseRepository) List(
queryParams := table.NewQueryParameters(opts...)
xcontext.LogDebug(ctx, r.log, "executing")
_, res, err := s.Execute(
ctx, readTx, query, queryParams,
options.WithCollectStatsModeBasic(),
@ -204,14 +211,10 @@ func (r *ydbCourseRepository) List(
return fmt.Errorf("executing: %w", err)
}
xcontext.LogDebug(ctx, r.log, "checking")
if !res.NextResultSet(ctx) || !res.HasNextRow() {
return nil
}
xcontext.LogDebug(ctx, r.log, "scanning")
for res.NextRow() {
var cdb courseDB
err = res.ScanNamed(cdb.getNamedValues()...)
@ -393,7 +396,6 @@ func createCourseParamsAsStruct(params domain.CreateCourseParams) types.Value {
}
func (r *ydbCourseRepository) CreateBatch(ctx context.Context, params ...domain.CreateCourseParams) error {
// -- PRAGMA TablePathPrefix("courses");
const upsertQuery = `DECLARE $courseData AS List<Struct<
id: Text,
external_id: Optional<Text>,
@ -680,8 +682,8 @@ func mapCourseDB(cdb courseDB) domain.Course {
Name: cdb.Name,
SourceType: st,
SourceName: nullable.NewValuePtr(cdb.SourceName),
Thematic: cdb.CourseThematic,
LearningType: cdb.LearningType,
ThematicID: cdb.CourseThematic,
LearningTypeID: cdb.LearningType,
OrganizationID: cdb.OrganizationID,
OriginLink: cdb.OriginLink,
ImageLink: cdb.ImageLink,
@ -738,11 +740,10 @@ WHERE {{ range .Conditions }}{{.}}{{end}}
var querySelect = template.Must(template.New("").Parse(queryTemplateSelect))
func tableParamOptsToString(in ...table.ParameterOption) string {
var sb strings.Builder
for _, opt := range in {
sb.WriteString(opt.Name() + "(" + opt.Value().Type().String() + ");")
}
return sb.String()
}
// func tableParamOptsToString(in ...table.ParameterOption) string {
// var sb strings.Builder
// for _, opt := range in {
// sb.WriteString(opt.Name() + "(" + opt.Value().Type().String() + ");")
// }
// return sb.String()
// }

4
internal/kurious/app/command/createcourse.go
View File

@ -8,7 +8,7 @@ import (
"git.loyso.art/frx/kurious/internal/common/decorator"
"git.loyso.art/frx/kurious/internal/common/nullable"
"git.loyso.art/frx/kurious/internal/common/xslice"
"git.loyso.art/frx/kurious/internal/common/xslices"
"git.loyso.art/frx/kurious/internal/kurious/domain"
)
@ -78,7 +78,7 @@ func NewCreateCoursesHandler(
}
func (h createCoursesHandler) Handle(ctx context.Context, cmd CreateCourses) error {
params := xslice.Map(cmd.Courses, func(in CreateCourse) (out domain.CreateCourseParams) {
params := xslices.Map(cmd.Courses, func(in CreateCourse) (out domain.CreateCourseParams) {
return domain.CreateCourseParams(in)
})
err := h.repo.CreateBatch(ctx, params...)

16
internal/kurious/app/query/listcourses.go
View File

@ -6,6 +6,7 @@ import (
"log/slog"
"git.loyso.art/frx/kurious/internal/common/decorator"
"git.loyso.art/frx/kurious/internal/common/xslices"
"git.loyso.art/frx/kurious/internal/kurious/domain"
)
@ -23,15 +24,18 @@ type ListCourse struct {
type ListCourseHandler decorator.QueryHandler[ListCourse, domain.ListCoursesResult]
type listCourseHandler struct {
repo domain.CourseRepository
repo domain.CourseRepository
mapper domain.CourseMapper
}
func NewListCourseHandler(
repo domain.CourseRepository,
mapper domain.CourseMapper,
log *slog.Logger,
) ListCourseHandler {
h := listCourseHandler{
repo: repo,
repo: repo,
mapper: mapper,
}
return decorator.AddQueryDecorators(h, log)
}
@ -43,7 +47,6 @@ func (h listCourseHandler) Handle(ctx context.Context, query ListCourse) (out do
out.Courses = make([]domain.Course, 0, query.Limit)
}
for {
print("listing")
result, err := h.repo.List(ctx, domain.ListCoursesParams{
CourseThematic: query.CourseThematic,
LearningType: query.LearningType,
@ -56,6 +59,13 @@ func (h listCourseHandler) Handle(ctx context.Context, query ListCourse) (out do
return out, fmt.Errorf("listing courses: %w", err)
}
result.Courses = xslices.Map(result.Courses, func(in domain.Course) domain.Course {
in.LearningType = h.mapper.LearningTypeNameByID(in.LearningTypeID)
in.Thematic = h.mapper.CourseThematicNameByID(in.ThematicID)
return in
})
out.Courses = append(out.Courses, result.Courses...)
out.NextPageToken = result.NextPageToken

10
internal/kurious/domain/course.go
View File

@ -27,9 +27,13 @@ type Course struct {
// FullPrice is a course full price without discount.
FullPrice float64
// Discount for the course.
Discount float64
Thematic string
LearningType string
Discount float64
Thematic string
ThematicID string
LearningType string
LearningTypeID string
// Duration for the course. It will be splitted in values like:
// full month / full day / full hour.

6
internal/kurious/domain/mapper.go Normal file
View File

@ -0,0 +1,6 @@
package domain
type CourseMapper interface {
CourseThematicNameByID(string) string
LearningTypeNameByID(string) string
}

12
internal/kurious/ports/background/synchandler.go
View File

@ -14,7 +14,7 @@ import (
"git.loyso.art/frx/kurious/internal/common/generator"
"git.loyso.art/frx/kurious/internal/common/nullable"
"git.loyso.art/frx/kurious/internal/common/xcontext"
"git.loyso.art/frx/kurious/internal/common/xslice"
"git.loyso.art/frx/kurious/internal/common/xslices"
"git.loyso.art/frx/kurious/internal/kurious/app/command"
"git.loyso.art/frx/kurious/internal/kurious/app/query"
"git.loyso.art/frx/kurious/internal/kurious/domain"
@ -126,7 +126,7 @@ func (h *syncSravniHandler) Handle(ctx context.Context) (err error) {
return fmt.Errorf("loading educational products: %w", err)
}
xslice.ForEach(buffer, func(c sravni.Course) {
xslices.ForEach(buffer, func(c sravni.Course) {
c.Learningtype = []string{learningType.Value}
c.CourseThematics = []string{courseThematic}
courses = append(courses, c)
@ -155,7 +155,7 @@ func (h *syncSravniHandler) Handle(ctx context.Context) (err error) {
continue
}
xslice.ForEach(courses, func(c sravni.Course) {
xslices.ForEach(courses, func(c sravni.Course) {
h.knownExternalIDs[c.ID] = struct{}{}
})
@ -212,7 +212,7 @@ func (h *syncSravniHandler) loadEducationalProducts(ctx context.Context, learnin
}
func (h *syncSravniHandler) filterByCache(courses []sravni.Course) (toInsert []sravni.Course) {
toCut := xslice.FilterInplace(courses, xslice.Not(h.isCached))
toCut := xslices.FilterInplace(courses, xslices.Not(h.isCached))
return courses[:toCut]
}
@ -222,7 +222,7 @@ func (h *syncSravniHandler) isCached(course sravni.Course) bool {
}
func (h *syncSravniHandler) insertValues(ctx context.Context, courses []sravni.Course) error {
courseParams := xslice.Map(courses, courseAsCreateCourseParams)
courseParams := xslices.Map(courses, courseAsCreateCourseParams)
err := h.svc.Commands.InsertCourses.Handle(ctx, command.CreateCourses{
Courses: courseParams,
})
@ -249,7 +249,7 @@ func (h *syncSravniHandler) fillCaches(ctx context.Context) error {
h.knownExternalIDs = make(map[string]struct{}, len(courses))
xslice.ForEach(courses, func(c domain.Course) {
xslices.ForEach(courses, func(c domain.Course) {
if !c.ExternalID.Valid() {
return
}

6
internal/kurious/ports/http/course.go
View File

@ -9,7 +9,7 @@ import (
"git.loyso.art/frx/kurious/internal/common/errors"
"git.loyso.art/frx/kurious/internal/common/xcontext"
"git.loyso.art/frx/kurious/internal/common/xslice"
"git.loyso.art/frx/kurious/internal/common/xslices"
"git.loyso.art/frx/kurious/internal/kurious/app/command"
"git.loyso.art/frx/kurious/internal/kurious/app/query"
"git.loyso.art/frx/kurious/internal/kurious/domain"
@ -90,7 +90,7 @@ type listCoursesTemplateParams struct {
func mapDomainCourseToTemplate(in ...domain.Course) listCoursesTemplateParams {
coursesBySubcategory := make(map[string][]domain.Course, len(in))
subcategoriesByCategories := make(map[string]map[string]struct{}, len(in))
xslice.ForEach(in, func(c domain.Course) {
xslices.ForEach(in, func(c domain.Course) {
coursesBySubcategory[c.Thematic] = append(coursesBySubcategory[c.Thematic], c)
if _, ok := subcategoriesByCategories[c.LearningType]; !ok {
subcategoriesByCategories[c.LearningType] = map[string]struct{}{}
@ -216,7 +216,7 @@ func (c courseServer) RenderEditDescription(w http.ResponseWriter, r *http.Reque
func (c courseServer) UpdateCourseDescription(w http.ResponseWriter, r *http.Request) {
type requestModel struct {
ID string `json:"-"`
Text string `json:"text"`
Text string `json:"description"`
}
ctx := r.Context()

4
internal/kurious/ports/http/listtemplate.go
View File

@ -9,7 +9,7 @@ import (
"path"
"git.loyso.art/frx/kurious/internal/common/xcontext"
"git.loyso.art/frx/kurious/internal/common/xslice"
"git.loyso.art/frx/kurious/internal/common/xslices"
)
const baseTemplatePath = "./internal/kurious/ports/http/templates/"
@ -22,7 +22,7 @@ func must[T any](t T, err error) T {
}
func scanFiles() []string {
entries := xslice.Map(
entries := xslices.Map(
must(os.ReadDir(baseTemplatePath)),
func(v fs.DirEntry) string {
return path.Join(baseTemplatePath, v.Name())

11
internal/kurious/ports/http/templates/get.tmpl
View File

@ -9,13 +9,16 @@
</div>
<div class="card-content">
<div class="media-content">
<p class="title is-5">{{ .Name }}</p>
<p class="title is-5" onclick="location.href='{{ .OriginLink }}'">{{ .Name }}</p>
<p class="subtitle is-8">{{ .Description }}</p>
</div>
<div class="content">
<p>{{ .FullPrice }} rub.</p>
</div>
<button class="button" onclick="location.href='{{ .OriginLink }}'">
Show Course
</button>
<button class="button" hx-get="/courses/{{ .ID }}/editdesc">
Edit description
</button>
@ -49,7 +52,7 @@
<div class="field-body">
<div class="field">
<div class="control">
<textarea class="textarea" placeholder="Description">{{ .Description }}</textarea>
<textarea class="textarea" name="description" placeholder="Description">{{ .Description }}</textarea>
</div>
</div>
</div>
@ -59,7 +62,7 @@
</div>
<div class="field is-grouped">
<p class="control">
<button class="button is-primary is-link" hx-include="closest .control">
<button class="button is-primary is-link" hx-include="[name='description']">
Submit
</button>
</p>
@ -69,8 +72,6 @@
</button>
</p>
</div>
<!-- <button class="btn">Submit</button>
<button class="btn" hx-get="/courses/{{ .ID }}/short">Cancel</button> -->
</form>
{{ end }}

5
internal/kurious/service/service.go
View File

@ -11,6 +11,7 @@ import (
"git.loyso.art/frx/kurious/internal/kurious/app"
"git.loyso.art/frx/kurious/internal/kurious/app/command"
"git.loyso.art/frx/kurious/internal/kurious/app/query"
"git.loyso.art/frx/kurious/internal/kurious/domain"
)
type ApplicationConfig struct {
@ -25,7 +26,7 @@ type Application struct {
closers []io.Closer
}
func NewApplication(ctx context.Context, cfg ApplicationConfig) (Application, error) {
func NewApplication(ctx context.Context, cfg ApplicationConfig, mapper domain.CourseMapper) (Application, error) {
log := config.NewSLogger(cfg.LogConfig)
ydbConnection, err := adapters.NewYDBConnection(ctx, cfg.YDB, log.With(slog.String("db", "ydb")))
if err != nil {
@ -43,7 +44,7 @@ func NewApplication(ctx context.Context, cfg ApplicationConfig) (Application, er
},
Queries: app.Queries{
GetCourse: query.NewGetCourseHandler(courseadapter, log),
ListCourses: query.NewListCourseHandler(courseadapter, log),
ListCourses: query.NewListCourseHandler(courseadapter, mapper, log),
},
}

17
pkg/slices/map.go
View File

@ -1,17 +0,0 @@
package slices
// Map slice from one type to another one.
func Map[S any, E any](s []S, f func(S) E) []E {
out := make([]E, len(s))
for i := range s {
out[i] = f(s[i])
}
return out
}
func ForEach[S any](s []S, f func(S)) {
for i := range s {
f(s[i])
}
}