diff --git a/Taskfile.yml b/Taskfile.yml index 53effde..6e2f54e 100644 --- a/Taskfile.yml +++ b/Taskfile.yml @@ -10,7 +10,7 @@ vars: GIT_VERSION: sh: git tag | sort -r --version-sort | head -n1 BUILD_TIME: - sh: TZ=UTC date --iso-8601=seconds + sh: TZ=UTC date -u +"%Y-%m-%dT%H:%M:%SZ" LDFLAGS: sh: echo '-X "{{.PROJECT}}.buildTime={{.BUILD_TIME}}" -X "{{.PROJECT}}.commit={{.GIT_COMMIT}}" -X "{{.PROJECT}}.version={{.GIT_VERSION}}"' @@ -24,9 +24,13 @@ tasks: test: cmds: - go test ./internal/... + build_web: + cmds: + - go build -o $GOBIN/kuriousweb -v -ldflags '{{.LDFLAGS}}' cmd/kuriweb/*.go + deps: [check, test] build_background: cmds: - - go build -o $GOBIN/sravnibackground -v -ldflags '{{.LDFLAGS}}' cmd/background/*.go + - go build -o $GOBIN/kuriousbg -v -ldflags '{{.LDFLAGS}}' cmd/background/*.go deps: [check, test] build_dev_cli: cmds: diff --git a/cmd/kuriweb/config.go b/cmd/kuriweb/config.go new file mode 100644 index 0000000..bd9f83b --- /dev/null +++ b/cmd/kuriweb/config.go @@ -0,0 +1,39 @@ +package main + +import ( + "encoding/json" + "fmt" + "os" + + "git.loyso.art/frx/kurious/internal/common/config" +) + +type Config struct { + Log config.Log `json:"log"` + YDB config.YDB `json:"ydb"` + HTTP config.HTTP `json:"http"` +} + +func readFromFile(path string, defaultConfigF func() Config) (Config, error) { + out := defaultConfigF() + payload, err := os.ReadFile(path) + if err != nil { + return out, fmt.Errorf("opening file: %w", err) + } + + err = json.Unmarshal(payload, &out) + if err != nil { + return out, fmt.Errorf("decoding as json: %w", err) + } + + return out, nil +} + +func defaultConfig() Config { + return Config{ + Log: config.Log{ + Level: config.LogLevelInfo, + Format: config.LogFormatText, + }, + } +} diff --git a/cmd/kuriweb/main.go b/cmd/kuriweb/main.go new file mode 100644 index 0000000..f6c34c3 --- /dev/null +++ b/cmd/kuriweb/main.go @@ -0,0 +1,133 @@ +package main + +import ( + "context" + "errors" + "fmt" + "log/slog" + "net/http" + "os" + "os/signal" + "time" + + "git.loyso.art/frx/kurious/internal/common/config" + "git.loyso.art/frx/kurious/internal/common/generator" + "git.loyso.art/frx/kurious/internal/common/xcontext" + xhttp "git.loyso.art/frx/kurious/internal/kurious/ports/http" + "git.loyso.art/frx/kurious/internal/kurious/service" + "golang.org/x/sync/errgroup" + + "github.com/gorilla/mux" +) + +func main() { + ctx, cancel := signal.NotifyContext(context.Background(), os.Interrupt) + defer cancel() + + err := app(ctx) + if err != nil { + println(err.Error()) + os.Exit(1) + } +} + +func app(ctx context.Context) error { + var cfgpath string + if len(os.Args) > 1 { + cfgpath = os.Args[1] + } else { + cfgpath = "config.json" + } + + cfg, err := readFromFile(cfgpath, defaultConfig) + if err != nil { + return fmt.Errorf("reading config from file: %w", err) + } + + log := config.NewSLogger(cfg.Log) + + app, err := service.NewApplication(ctx, service.ApplicationConfig{ + LogConfig: cfg.Log, + YDB: cfg.YDB, + }) + if err != nil { + return fmt.Errorf("making new application: %w", err) + } + + httpAPI := xhttp.NewServer(app, log.With(slog.String("component", "http"))) + httpServer := setupHTTP(cfg.HTTP, httpAPI, log) + + eg, egctx := errgroup.WithContext(ctx) + eg.Go(func() error { + xcontext.LogInfo( + ctx, log, "serving http", + slog.String("addr", httpServer.Addr), + ) + + err := httpServer.ListenAndServe() + if err != nil { + if !errors.Is(err, http.ErrServerClosed) { + return fmt.Errorf("listening http: %w", err) + } + } + return nil + }) + eg.Go(func() error { + <-egctx.Done() + + xcontext.LogInfo(ctx, log, "trying to shutdown http") + sdctx, sdcancel := context.WithTimeout(context.Background(), time.Second*10) + defer sdcancel() + err := httpServer.Shutdown(sdctx) + if err != nil { + return fmt.Errorf("shutting down the server: %w", err) + } + + xcontext.LogInfo(ctx, log, "server closed successfuly") + return nil + }) + + return eg.Wait() +} + +func setupHTTP(cfg config.HTTP, srv xhttp.Server, log *slog.Logger) *http.Server { + router := mux.NewRouter() + + coursesAPI := srv.Courses() + coursesRouter := router.PathPrefix("/courses").Subrouter() + coursesRouter.Use(middlewareLogger(log)) + coursesRouter.HandleFunc("/", coursesAPI.List) + + return &http.Server{ + Addr: cfg.ListenAddr, + Handler: router, + } +} + +func middlewareLogger(log *slog.Logger) mux.MiddlewareFunc { + return func(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() + requestID := r.Header.Get("x-request-id") + if requestID == "" { + requestID = generator.RandomInt64ID() + } + ctx = xcontext.WithLogFields(ctx, slog.String("request_id", requestID)) + + xcontext.LogInfo( + ctx, log, "incoming request", + slog.String("method", r.Method), + slog.String("path", r.URL.Path), + ) + + start := time.Now() + next.ServeHTTP(w, r.WithContext(ctx)) + elapsed := time.Since(start) + + xcontext.LogInfo( + ctx, log, "request processed", + slog.Duration("elapsed", elapsed), + ) + }) + } +} diff --git a/go.mod b/go.mod index f2442db..9617ea6 100644 --- a/go.mod +++ b/go.mod @@ -4,34 +4,30 @@ go 1.21 require ( github.com/go-resty/resty/v2 v2.10.0 - github.com/stretchr/testify v1.8.4 + github.com/gorilla/mux v1.8.1 + github.com/robfig/cron/v3 v3.0.0 github.com/teris-io/cli v1.0.1 + github.com/ydb-platform/ydb-go-sdk/v3 v3.54.2 + github.com/ydb-platform/ydb-go-yc v0.12.1 golang.org/x/net v0.18.0 + golang.org/x/sync v0.5.0 + golang.org/x/time v0.5.0 ) require ( - github.com/davecgh/go-spew v1.1.1 // indirect - github.com/go-chi/chi/v5 v5.0.10 // indirect github.com/golang-jwt/jwt/v4 v4.5.0 // indirect github.com/golang/protobuf v1.5.3 // indirect github.com/google/uuid v1.4.0 // indirect github.com/jonboulle/clockwork v0.4.0 // indirect - github.com/pmezard/go-difflib v1.0.0 // indirect - github.com/robfig/cron/v3 v3.0.0 // indirect - github.com/stretchr/objx v0.5.0 // indirect + github.com/stretchr/testify v1.8.4 // indirect github.com/yandex-cloud/go-genproto v0.0.0-20231120081503-a21e9fe75162 // indirect github.com/ydb-platform/ydb-go-genproto v0.0.0-20231012155159-f85a672542fd // indirect - github.com/ydb-platform/ydb-go-sdk/v3 v3.54.2 // indirect - github.com/ydb-platform/ydb-go-yc v0.12.1 // indirect github.com/ydb-platform/ydb-go-yc-metadata v0.6.1 // indirect - golang.org/x/sync v0.5.0 // indirect golang.org/x/sys v0.14.0 // indirect golang.org/x/text v0.14.0 // indirect - golang.org/x/time v0.5.0 // indirect google.golang.org/genproto v0.0.0-20231120223509-83a465c0220f // indirect google.golang.org/genproto/googleapis/api v0.0.0-20231120223509-83a465c0220f // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20231120223509-83a465c0220f // indirect google.golang.org/grpc v1.59.0 // indirect google.golang.org/protobuf v1.31.0 // indirect - gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go.sum b/go.sum index 2d99fa4..2eae5a8 100644 --- a/go.sum +++ b/go.sum @@ -572,8 +572,6 @@ github.com/envoyproxy/protoc-gen-validate v0.10.0/go.mod h1:DRjgyB0I43LtJapqN6Ni github.com/fogleman/gg v1.2.1-0.20190220221249-0403632d5b90/go.mod h1:R/bRT+9gY/C5z7JzPU0zXsXHKM4/ayA+zqcVNZzPa1k= github.com/fogleman/gg v1.3.0/go.mod h1:R/bRT+9gY/C5z7JzPU0zXsXHKM4/ayA+zqcVNZzPa1k= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= -github.com/go-chi/chi/v5 v5.0.10 h1:rLz5avzKpjqxrYwXNfmjkrYYXOyLJd37pz53UFHC6vk= -github.com/go-chi/chi/v5 v5.0.10/go.mod h1:DslCQbL2OYiznFReuXYUmQ2hGd1aDpCnlMNITLSKoi8= github.com/go-fonts/dejavu v0.1.0/go.mod h1:4Wt4I4OU2Nq9asgDCteaAaWZOV24E+0/Pwo0gppep4g= github.com/go-fonts/latin-modern v0.2.0/go.mod h1:rQVLdDMK+mK1xscDwsqM5J8U2jrRa3T0ecnM9pNujks= github.com/go-fonts/liberation v0.1.1/go.mod h1:K6qoJYypsmfVjWg8KOVDQhLc8UDgIK2HYqyqAO9z7GY= @@ -589,7 +587,6 @@ github.com/go-pdf/fpdf v0.6.0/go.mod h1:HzcnA+A23uwogo0tp9yU+l3V+KXhiESpt1PMayhO github.com/go-resty/resty/v2 v2.10.0 h1:Qla4W/+TMmv0fOeeRqzEpXPLfTUnR5HZ1+lGs+CkiCo= github.com/go-resty/resty/v2 v2.10.0/go.mod h1:iiP/OpA0CkcL3IGt1O0+/SIItFUbkkyw5BGXiVdTu+A= github.com/goccy/go-json v0.9.11/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= -github.com/golang-jwt/jwt/v4 v4.4.1 h1:pC5DB52sCeK48Wlb9oPcdhnjkz1TKt1D/P7WKJ0kUcQ= github.com/golang-jwt/jwt/v4 v4.4.1/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= github.com/golang-jwt/jwt/v4 v4.4.3/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= github.com/golang-jwt/jwt/v4 v4.5.0 h1:7cYmW1XlMY7h7ii7UhUyChSgS5wUJEnm9uZVTGqOWzg= @@ -609,6 +606,7 @@ github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4= github.com/golang/mock v1.5.0/go.mod h1:CWnOUgYIOo4TcNZ0wHX3YZCqsaM1I1Jvs6v3mP3KVu8= +github.com/golang/mock v1.6.0 h1:ErTB+efbowRARo13NNdxyJji2egdxLGQhRaY+DUumQc= github.com/golang/mock v1.6.0/go.mod h1:p6yTPP+5HYm5mzsMV8JkE6ZKdX+/wYM6Hr+LicevLPs= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= @@ -648,6 +646,7 @@ github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8/DtOE= github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= @@ -691,6 +690,8 @@ github.com/googleapis/gax-go/v2 v2.6.0/go.mod h1:1mjbznJAPHFpesgE5ucqfYEscaz5kMd github.com/googleapis/gax-go/v2 v2.7.0/go.mod h1:TEop28CZZQ2y+c0VxMUmu1lV+fQx57QpBWsYpwqHJx8= github.com/googleapis/go-type-adapters v1.0.0/go.mod h1:zHW75FOG2aur7gAO2B+MLby+cLsWGBF62rFAi7WjWO4= github.com/googleapis/google-cloud-go-testing v0.0.0-20200911160855-bcd43fbb19e8/go.mod h1:dvDLG8qkwmyD9a/MJJN3XJcT3xFxOKAvTZGvuZmac9g= +github.com/gorilla/mux v1.8.1 h1:TuBL49tXwgrFYWhqrNgrUNEY92u81SPhu7sTdzQEiWY= +github.com/gorilla/mux v1.8.1/go.mod h1:AKf9I4AEqPTmMytcMc0KkNouC66V3BtZ4qD5fmWSiMQ= github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= github.com/grpc-ecosystem/grpc-gateway/v2 v2.7.0/go.mod h1:hgWBS7lorOAVIJEQMi4ZsPv9hVvWI6+ch50m39Pf2Ks= github.com/grpc-ecosystem/grpc-gateway/v2 v2.11.3/go.mod h1:o//XUCC/F+yRGJoPO/VU0GSB0f8Nhgmxx0VIRUvaC0w= @@ -756,7 +757,6 @@ github.com/spf13/afero v1.6.0/go.mod h1:Ai8FlHk4v/PARR026UzYexafAt9roJ7LcLMAmO6Z github.com/spf13/afero v1.9.2/go.mod h1:iUV7ddyEEZPO5gA3zD4fJt6iStLlL+Lg4m2cihcDf8Y= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= -github.com/stretchr/objx v0.5.0 h1:1zr/of2m5FGMsad5YfcqgdqdWrIhu+EBEJRhR1U7z/c= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= @@ -1092,7 +1092,6 @@ golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxb golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20220922220347-f3bd1da661af/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.1.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4= golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk= golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= @@ -1427,10 +1426,8 @@ google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqw google.golang.org/protobuf v1.30.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8= google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= -gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= diff --git a/internal/kurious/adapters/ydb_course_repository.go b/internal/kurious/adapters/ydb_course_repository.go index 1881114..a7088ba 100644 --- a/internal/kurious/adapters/ydb_course_repository.go +++ b/internal/kurious/adapters/ydb_course_repository.go @@ -24,6 +24,29 @@ import ( yc "github.com/ydb-platform/ydb-go-yc" ) +var coursesFields = []string{ + "id", + "external_id", + "source_type", + "source_name", + "course_thematic", + "learning_type", + "organization_id", + "origin_link", + "image_link", + "name", + "description", + "full_price", + "discount", + "duration", + "starts_at", + "created_at", + "updated_at", + "deleted_at", +} + +var coursesFieldsStr = strings.Join(coursesFields, ",") + const ( defaultShutdownTimeout = time.Second * 10 ) @@ -87,59 +110,31 @@ func (r *ydbCourseRepository) List( ) (result domain.ListCoursesResult, err error) { const limit = 1000 const queryName = "list" - // const query = ` - // DECLARE $limit AS Int32; - // DECLARE $id AS Text; - // SELECT - // id, - // external_id, - // source_type, - // source_name, - // course_thematic, - // learning_type, - // organization_id, - // origin_link, - // image_link, - // name, - // description, - // full_price, - // discount, - // duration, - // starts_at, - // created_at, - // updated_at, - // deleted_at - // FROM - // courses - // WHERE - // id > $id - // ORDER BY id - // LIMIT $limit;` - // - const fields = `id, external_id, source_type, source_name, course_thematic, learning_type, organization_id, origin_link, image_link, name, description, full_price, discount, duration, starts_at, created_at, updated_at, deleted_at` - if params.Limit == 0 { params.Limit = limit } qtParams := queryTemplateParams{ - Fields: fields, + Fields: coursesFieldsStr, Table: "courses", Suffix: "ORDER BY id\nLIMIT $limit", - Declares: []queryTemplateDeclaration{{ - Name: "limit", - Type: "Int32", - }, { - Name: "id", - Type: "Text", - }}, + Declares: []queryTemplateDeclaration{ + { + Name: "limit", + Type: "Int32", + }, + { + Name: "id", + Type: "Text", + }, + }, Conditions: []string{ "id > $id", }, } - options := make([]table.ParameterOption, 0, 4) - appendParams := func(name string, value string) { + opts := make([]table.ParameterOption, 0, 4) + appendTextParam := func(name string, value string) { if value == "" { return } @@ -151,18 +146,17 @@ func (r *ydbCourseRepository) List( } qtParams.Declares = append(qtParams.Declares, d) qtParams.Conditions = append(qtParams.Conditions, d.Name+"="+d.Arg()) - options = append(options, table.ValueParam(d.Arg(), ydbvalue)) + opts = append(opts, table.ValueParam(d.Arg(), ydbvalue)) } - appendParams("course_thematic", params.CourseThematic) - appendParams("learning_type", params.LearningType) + appendTextParam("course_thematic", params.CourseThematic) + appendTextParam("learning_type", params.LearningType) - var sb strings.Builder - err = template.Must(template.New("").Parse(queryTemplateSelect)).Execute(&sb, qtParams) + query, err := qtParams.render() if err != nil { - return result, fmt.Errorf("executing template: %w", err) + return result, fmt.Errorf("rendering: %w", err) } - query := sb.String() + xcontext.LogInfo(ctx, r.log, "planning to run query", slog.String("query", query), slog.Any("opts", opts)) courses := make([]domain.Course, 0, 1_000) readTx := table.TxControl( @@ -171,9 +165,12 @@ func (r *ydbCourseRepository) List( ), table.CommitTx(), ) + + xcontext.LogInfo(ctx, r.log, "executing do") err = r.db.Table().Do( ctx, func(ctx context.Context, s table.Session) error { + xcontext.LogInfo(ctx, r.log, "inside do") start := time.Now() defer func() { since := time.Since(start) @@ -185,47 +182,51 @@ func (r *ydbCourseRepository) List( ) }() - var lastKnownID string - for { - queryParams := table.NewQueryParameters( - table.ValueParam("$limit", types.Int32Value(limit)), - table.ValueParam("$id", types.TextValue(lastKnownID)), - ) - _, res, err := s.Execute( - ctx, readTx, query, queryParams, - options.WithCollectStatsModeBasic(), - ) - if err != nil { - return fmt.Errorf("executing: %w", err) - } + queryParams := table.NewQueryParameters(opts...) - if !res.NextResultSet(ctx) || !res.HasNextRow() { - break - } + xcontext.LogDebug(ctx, r.log, "executing") - for res.NextRow() { - var cdb courseDB - err = res.ScanNamed(cdb.getNamedValues()...) - if err != nil { - return fmt.Errorf("scanning row: %w", err) - } - - courses = append(courses, mapCourseDB(cdb)) - } - if err = res.Err(); err != nil { - return err - } - - lastKnownID = courses[len(courses)-1].ID + _, res, err := s.Execute( + ctx, readTx, query, queryParams, + options.WithCollectStatsModeBasic(), + ) + if err != nil { + return fmt.Errorf("executing: %w", err) } + + xcontext.LogDebug(ctx, r.log, "checking") + + if !res.NextResultSet(ctx) || !res.HasNextRow() { + return nil + } + + xcontext.LogDebug(ctx, r.log, "scanning") + + for res.NextRow() { + var cdb courseDB + err = res.ScanNamed(cdb.getNamedValues()...) + if err != nil { + return fmt.Errorf("scanning row: %w", err) + } + + courses = append(courses, mapCourseDB(cdb)) + } + if err = res.Err(); err != nil { + return err + } + + result.NextPageToken = courses[len(courses)-1].ID + xcontext.LogDebug(ctx, r.log, "scanned rows", slog.Int("count", len(courses))) + return nil }, table.WithIdempotent()) if err != nil { - return nil, err + return domain.ListCoursesResult{}, err } - return courses, err + result.Courses = courses + return result, err } func (r *ydbCourseRepository) Get(ctx context.Context, id string) (course domain.Course, err error) { @@ -579,9 +580,18 @@ type queryTemplateParams struct { Suffix string } -const queryTemplateSelect = ` -{{ range .Declares }}DECLARE ${{.Name}} AS {{.Type}}\n{{end}} +func (p queryTemplateParams) render() (string, error) { + var sb strings.Builder + sb.Grow(len(queryTemplateSelect) * 3) + + err := querySelect.Execute(&sb, p) + return sb.String(), err +} + +const queryTemplateSelect = `{{ range .Declares }}DECLARE ${{.Name}} AS {{.Type}};{{end}} SELECT {{.Fields}} FROM {{.Table}} -WHERE {{ range .Conditions }}{{.}}\n{{end}} +WHERE {{ range .Conditions }}{{.}}{{end}} {{.Suffix}}` + +var querySelect = template.Must(template.New("").Parse(queryTemplateSelect)) diff --git a/internal/kurious/app/query/listcourses.go b/internal/kurious/app/query/listcourses.go index 73f77d4..0e0388b 100644 --- a/internal/kurious/app/query/listcourses.go +++ b/internal/kurious/app/query/listcourses.go @@ -16,11 +16,11 @@ type ListCourse struct { OrganizationID string Keyword string - Limit int - Offset int + Limit int + NextPageToken string } -type ListCourseHandler decorator.QueryHandler[ListCourse, []domain.Course] +type ListCourseHandler decorator.QueryHandler[ListCourse, domain.ListCoursesResult] type listCourseHandler struct { repo domain.CourseRepository @@ -36,17 +36,35 @@ func NewListCourseHandler( return decorator.AddQueryDecorators(h, log) } -func (h listCourseHandler) Handle(ctx context.Context, query ListCourse) ([]domain.Course, error) { - courses, err := h.repo.List(ctx, domain.ListCoursesParams{ - CourseThematic: query.CourseThematic, - LearningType: query.LearningType, - OrganizationID: query.OrganizationID, - Limit: query.Limit, - Offset: query.Offset, - }) - if err != nil { - return nil, fmt.Errorf("listing courses: %w", err) +func (h listCourseHandler) Handle(ctx context.Context, query ListCourse) (out domain.ListCoursesResult, err error) { + out.NextPageToken = query.NextPageToken + drainFull := query.Limit == 0 + if !drainFull { + out.Courses = make([]domain.Course, 0, query.Limit) + } + for { + print("listing") + result, err := h.repo.List(ctx, domain.ListCoursesParams{ + CourseThematic: query.CourseThematic, + LearningType: query.LearningType, + OrganizationID: query.OrganizationID, + Limit: query.Limit, + + NextPageToken: out.NextPageToken, + }) + if err != nil { + return out, fmt.Errorf("listing courses: %w", err) + } + + out.Courses = append(out.Courses, result.Courses...) + out.NextPageToken = result.NextPageToken + + if drainFull && len(result.Courses) > 0 && result.NextPageToken != "" { + continue + } + + break } - return courses, nil + return out, nil } diff --git a/internal/kurious/domain/repository.go b/internal/kurious/domain/repository.go index 4136db7..d9cde2e 100644 --- a/internal/kurious/domain/repository.go +++ b/internal/kurious/domain/repository.go @@ -14,7 +14,6 @@ type ListCoursesParams struct { NextPageToken string Limit int - Offset int } type CreateCourseParams struct { diff --git a/internal/kurious/ports/background/synchandler.go b/internal/kurious/ports/background/synchandler.go index b914b11..fd13807 100644 --- a/internal/kurious/ports/background/synchandler.go +++ b/internal/kurious/ports/background/synchandler.go @@ -240,11 +240,13 @@ func (h *syncSravniHandler) fillCaches(ctx context.Context) error { return nil } - courses, err := h.svc.Queries.ListCourses.Handle(ctx, query.ListCourse{}) + result, err := h.svc.Queries.ListCourses.Handle(ctx, query.ListCourse{}) if err != nil { return fmt.Errorf("listing courses: %w", err) } + courses := result.Courses + h.knownExternalIDs = make(map[string]struct{}, len(courses)) xslice.ForEach(courses, func(c domain.Course) { diff --git a/internal/kurious/ports/http/course.go b/internal/kurious/ports/http/course.go index ce32958..5290443 100644 --- a/internal/kurious/ports/http/course.go +++ b/internal/kurious/ports/http/course.go @@ -1,49 +1,30 @@ package http import ( - "html/template" "log/slog" "net/http" "strconv" "git.loyso.art/frx/kurious/internal/common/errors" "git.loyso.art/frx/kurious/internal/common/xslice" - "git.loyso.art/frx/kurious/internal/kurious/app" "git.loyso.art/frx/kurious/internal/kurious/app/query" "git.loyso.art/frx/kurious/internal/kurious/domain" + "git.loyso.art/frx/kurious/internal/kurious/service" ) type courseServer struct { - app app.Application + app service.Application log *slog.Logger } type pagination struct { - page int - perPage int -} - -func (p pagination) LimitOffset() (limit, offset int) { - if p.page < 0 { - p.page = 0 - } - - if p.perPage <= 0 { - p.perPage = defaultPerPage - } - - return p.perPage, p.page * p.perPage + nextPageToken string + perPage int } func parsePaginationFromQuery(r *http.Request) (out pagination, err error) { query := r.URL.Query() - - if query.Has("page") { - out.page, err = strconv.Atoi(query.Get("page")) - if err != nil { - return out, errors.NewValidationError("page", "bad page value") - } - } + out.nextPageToken = query.Get("next") if query.Has("per_page") { out.perPage, err = strconv.Atoi(query.Get("per_page")) @@ -77,42 +58,84 @@ type listCoursesParams struct { learningType string } -type templateCourse domain.Course +type baseInfo struct { + ID string + Name string + Description string +} -func mapDomainCourseToTemplate(in ...domain.Course) []templateCourse { - return xslice.Map(in, func(v domain.Course) templateCourse { - return templateCourse(v) +type categoryInfo struct { + baseInfo + + Subcategories []subcategoryInfo +} + +type subcategoryInfo struct { + baseInfo + + Courses []domain.Course +} + +type listCoursesTemplateParams struct { + Categories []categoryInfo +} + +func mapDomainCourseToTemplate(in ...domain.Course) listCoursesTemplateParams { + coursesBySubcategory := make(map[string][]domain.Course, len(in)) + subcategoriesByCategories := make(map[string]map[string]struct{}, len(in)) + xslice.ForEach(in, func(c domain.Course) { + coursesBySubcategory[c.Thematic] = append(coursesBySubcategory[c.Thematic], c) + if _, ok := subcategoriesByCategories[c.LearningType]; !ok { + subcategoriesByCategories[c.LearningType] = map[string]struct{}{} + } + subcategoriesByCategories[c.LearningType][c.Thematic] = struct{}{} }) + + var out listCoursesTemplateParams + for category, subcategoryMap := range subcategoriesByCategories { + outCategory := categoryInfo{} + outCategory.ID = category + outCategory.Name = category + outCategory.Description = "" + + for subcategory := range subcategoryMap { + outSubCategory := subcategoryInfo{ + Courses: coursesBySubcategory[subcategory], + } + outSubCategory.ID = subcategory + outSubCategory.Name = subcategory + outSubCategory.Description = "" + + outCategory.Subcategories = append(outCategory.Subcategories, outSubCategory) + } + } + + return out } func (c courseServer) List(w http.ResponseWriter, r *http.Request) { ctx := r.Context() params, err := parseListCoursesParams(r) - if err != nil { - handleError(ctx, err, w, c.log, "unable to parse list courses params") + if handleError(ctx, err, w, c.log, "unable to parse list courses params") { return } - limit, offset := params.LimitOffset() - - courses, err := c.app.Queries.ListCourses.Handle(ctx, query.ListCourse{ + result, err := c.app.Queries.ListCourses.Handle(ctx, query.ListCourse{ CourseThematic: params.courseThematic, LearningType: params.learningType, - Limit: limit, - Offset: offset, + Limit: params.perPage, + NextPageToken: params.nextPageToken, }) - if err != nil { - handleError(ctx, err, w, c.log, "unable to list courses") - } - - templateCourses := mapDomainCourseToTemplate(courses...) - - err = template.Must(template.ParseFiles("templates/list.tmpl")).Execute(w, templateCourses) - if err != nil { - handleError(ctx, err, w, c.log, "unable to execute template") + if handleError(ctx, err, w, c.log, "unable to list courses") { return } - w.WriteHeader(http.StatusOK) + courses := result.Courses + templateCourses := mapDomainCourseToTemplate(courses...) + + err = listTemplateParsed.ExecuteTemplate(w, "courses", templateCourses) + if handleError(ctx, err, w, c.log, "unable to execute template") { + return + } } diff --git a/internal/kurious/ports/http/listtemplate.go b/internal/kurious/ports/http/listtemplate.go new file mode 100644 index 0000000..40c3b25 --- /dev/null +++ b/internal/kurious/ports/http/listtemplate.go @@ -0,0 +1,99 @@ +package http + +import "html/template" + +var listTemplateParsed = template.Must( + template.New("courses"). + Parse(listTemplate), +) + +const listTemplate = `{{define "courses"}} + + + + Courses + + + +
+

Courses

+ +
+

Courses

+ {{range $category := .Categories}} +

{{$category.Name}}

+

{{$category.Description}}

+ {{range $subcategory := .Subcategories}} +

{{$subcategory.Name}}

+

{{$subcategory.Description}}

+ {{range $course := $.Courses}} +
+

{{$course.Name}}

+

{{$course.Description}}

+

Full price: {{$course.FullPrice}}

+

Discount: {{$course.Discount}}

+

Thematic: {{$course.Thematic}}

+

Learning type: {{$course.LearningType}}

+

Duration: {{$course.Duration}}

+

Starts at: {{$course.StartsAt}}

+
+ {{end}} + {{end}} + {{end}} + + +{{end}}` diff --git a/internal/kurious/ports/http/server.go b/internal/kurious/ports/http/server.go index 074bced..01e6123 100644 --- a/internal/kurious/ports/http/server.go +++ b/internal/kurious/ports/http/server.go @@ -8,30 +8,28 @@ import ( "git.loyso.art/frx/kurious/internal/common/errors" "git.loyso.art/frx/kurious/internal/common/xcontext" - "git.loyso.art/frx/kurious/internal/kurious/app" -) - -const ( - defaultPerPage = 50 + "git.loyso.art/frx/kurious/internal/kurious/service" ) type Server struct { - app app.Application + app service.Application + log *slog.Logger } -func NewServer(app app.Application) Server { - return Server{} -} - -func (s Server) Courses() courseServer { - return courseServer{ - app: s.app, +func NewServer(app service.Application, log *slog.Logger) Server { + return Server{ + app: app, + log: log, } } -func handleError(ctx context.Context, err error, w http.ResponseWriter, log *slog.Logger, msg string) { +func (s Server) Courses() courseServer { + return courseServer(s) +} + +func handleError(ctx context.Context, err error, w http.ResponseWriter, log *slog.Logger, msg string) bool { if err == nil { - return + return false } var errorString string @@ -52,4 +50,6 @@ func handleError(ctx context.Context, err error, w http.ResponseWriter, log *slo xcontext.LogWithWarnError(ctx, log, err, msg, slog.Int("status_code", code), slog.String("response", errorString)) http.Error(w, errorString, code) + + return true }