From 30e5968e0337e2f93333f10cef80f7d9ea85f657 Mon Sep 17 00:00:00 2001 From: Aleksandr Trushkin Date: Tue, 13 Aug 2024 13:15:11 +0300 Subject: [PATCH] support db migrations --- assets/db/migrations/0001_init.down.sql | 1 + .../0001_init.up.sql} | 1 + cmd/migrator/main.go | 78 +++++++++++++++++++ compose.yaml | 20 ++++- dockers/migrator/Dockerfile | 25 ++++++ Dockerfile => dockers/web/Dockerfile | 5 +- go.mod | 7 +- go.sum | 13 ++++ internal/store/mongo/store.go | 4 +- makefile | 16 +++- sqlc.yaml | 2 +- 11 files changed, 163 insertions(+), 9 deletions(-) create mode 100644 assets/db/migrations/0001_init.down.sql rename assets/db/{schema.sql => migrations/0001_init.up.sql} (99%) create mode 100644 cmd/migrator/main.go create mode 100644 dockers/migrator/Dockerfile rename Dockerfile => dockers/web/Dockerfile (89%) diff --git a/assets/db/migrations/0001_init.down.sql b/assets/db/migrations/0001_init.down.sql new file mode 100644 index 0000000..1f033b4 --- /dev/null +++ b/assets/db/migrations/0001_init.down.sql @@ -0,0 +1 @@ +DROP TABLE public.stats; diff --git a/assets/db/schema.sql b/assets/db/migrations/0001_init.up.sql similarity index 99% rename from assets/db/schema.sql rename to assets/db/migrations/0001_init.up.sql index 254134d..4521cb0 100644 --- a/assets/db/schema.sql +++ b/assets/db/migrations/0001_init.up.sql @@ -10,3 +10,4 @@ CREATE TABLE IF NOT EXISTS public.stats ( CREATE UNIQUE INDEX IF NOT EXISTS stats_by_device_id_idx ON public.stats(device_id); + diff --git a/cmd/migrator/main.go b/cmd/migrator/main.go new file mode 100644 index 0000000..0dadb3b --- /dev/null +++ b/cmd/migrator/main.go @@ -0,0 +1,78 @@ +package main + +import ( + "context" + "errors" + "fmt" + "log" + "os" + "os/signal" + + "github.com/golang-migrate/migrate/v4" + + // Use it to append postgres sql and file drivers. + _ "github.com/golang-migrate/migrate/v4/database/postgres" + _ "github.com/golang-migrate/migrate/v4/source/file" +) + +func main() { + ctx, cancel := signal.NotifyContext(context.Background(), os.Interrupt) + defer cancel() + + err := app(ctx) + if err != nil { + log.Fatalf("unable to run app: %v", err) + } +} + +func app(_ context.Context) (err error) { + const filepath = "file:///app/db/migrations" + pgdsn := os.Getenv("DEVSIM_PG_DSN") + if pgdsn == "" { + return nil + } + + filedsn := os.Getenv("DEVSIM_PG_MIGRATION") + if filedsn == "" { + filedsn = filepath + } + + log.Println("going to apply migrations from path: " + filedsn) + migrations, err := migrate.New( + filedsn, + pgdsn, + ) + if err != nil { + return fmt.Errorf("making migrator: %w", err) + } + + defer func() { + sourceErr, dbErr := migrations.Close() + err = errors.Join(err, sourceErr, dbErr) + }() + + version, dirty, err := migrations.Version() + if errors.Is(err, migrate.ErrNilVersion) { + log.Println("no migrations applied to database") + } else if err != nil { + return fmt.Errorf("getting version: %w", err) + } else { + log.Printf("current migration version: %d (dirty: %t)", version, dirty) + } + + err = migrations.Up() + if err != nil { + return fmt.Errorf("applying migrations: %w", err) + } + + version, dirty, err = migrations.Version() + if errors.Is(err, migrate.ErrNilVersion) { + log.Println("no migrations applied to database") + } else if err != nil { + return fmt.Errorf("getting version: %w", err) + } else { + log.Printf("updated to migration version: %d (dirty: %t)", version, dirty) + } + + return nil +} diff --git a/compose.yaml b/compose.yaml index d993d4f..1ef6d3f 100644 --- a/compose.yaml +++ b/compose.yaml @@ -1,6 +1,6 @@ services: web.mongo: - image: git.loyso.art/devsim:latest + image: git.loyso.art/devsim-web:latest ports: - 9124:80 depends_on: @@ -11,16 +11,26 @@ services: DEVSIM_STORE_TYPE: mongo web.pg: - image: git.loyso.art/devsim:latest + image: git.loyso.art/devsim-web:latest ports: - 9123:80 depends_on: - postgres - mongo + - postgres-migrator environment: DEVSIM_PG_DSN: "postgres://devsim:devsim@postgres:5432/devsim?sslmode=disable" DEVSIM_STORE_TYPE: pg + postgres-migrator: + image: git.loyso.art/devsim-migrator:latest + depends_on: + postgres: + condition: service_healthy + restart: true + environment: + DEVSIM_PG_DSN: "postgres://devsim:devsim@postgres:5432/devsim?sslmode=disable" + postgres: image: postgres:15-alpine environment: @@ -28,6 +38,12 @@ services: POSTGRES_PASSWORD: devsim POSTGRES_USER: devsim ports: ["5432:5432"] + healthcheck: + test: ["CMD-SHELL", "pg_isready -U devsim -d devsim"] + interval: 10s + retries: 5 + start_period: 30s + timeout: 10s mongo: image: mongo:7 diff --git a/dockers/migrator/Dockerfile b/dockers/migrator/Dockerfile new file mode 100644 index 0000000..017e01c --- /dev/null +++ b/dockers/migrator/Dockerfile @@ -0,0 +1,25 @@ +FROM golang:1.22-alpine as golang + +ARG VERSION="unknown" +ARG REVISION="unknown" +ARG BUILDTIME="" + +WORKDIR /go/src/git.loyso.art/frx/devsim +COPY . . + +RUN go mod download && \ + go mod verify && \ + CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build \ + -o /go/bin/migrator /go/src/git.loyso.art/frx/devsim/cmd/migrator/main.go + +FROM gcr.io/distroless/static-debian12@sha256:ce46866b3a5170db3b49364900fb3168dc0833dfb46c26da5c77f22abb01d8c3 + +WORKDIR /app +COPY --from=golang /go/bin/migrator /app/migrator +COPY assets/db/migrations/ /app/db/migrations/ + +ENV DEVSIM_PG_MIGRATOR="/app/migrations" + +ENTRYPOINT ["/app/migrator"] + + diff --git a/Dockerfile b/dockers/web/Dockerfile similarity index 89% rename from Dockerfile rename to dockers/web/Dockerfile index 8b26f09..b6e9c41 100644 --- a/Dockerfile +++ b/dockers/web/Dockerfile @@ -14,10 +14,11 @@ RUN go mod download && \ FROM gcr.io/distroless/static-debian12@sha256:ce46866b3a5170db3b49364900fb3168dc0833dfb46c26da5c77f22abb01d8c3 -COPY --from=golang /go/bin/app /app +WORKDIR /app +COPY --from=golang /go/bin/app /app/web ENV DEVSIM_HTTP_ADDR=":80" EXPOSE 80 -ENTRYPOINT ["/app"] +ENTRYPOINT ["/app/web"] diff --git a/go.mod b/go.mod index 572adcd..8253add 100644 --- a/go.mod +++ b/go.mod @@ -10,17 +10,22 @@ require ( require ( github.com/davecgh/go-spew v1.1.1 // indirect + github.com/golang-migrate/migrate/v4 v4.17.1 // indirect github.com/golang/snappy v0.0.4 // indirect + github.com/hashicorp/errwrap v1.1.0 // indirect + github.com/hashicorp/go-multierror v1.1.1 // indirect github.com/jackc/pgpassfile v1.0.0 // indirect github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a // indirect github.com/jackc/puddle/v2 v2.2.1 // indirect - github.com/klauspost/compress v1.13.6 // indirect + github.com/klauspost/compress v1.15.11 // indirect + github.com/lib/pq v1.10.9 // indirect github.com/montanaflynn/stats v0.7.1 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/xdg-go/pbkdf2 v1.0.0 // indirect github.com/xdg-go/scram v1.1.2 // indirect github.com/xdg-go/stringprep v1.0.4 // indirect github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d // indirect + go.uber.org/atomic v1.7.0 // indirect golang.org/x/crypto v0.26.0 // indirect golang.org/x/sync v0.8.0 // indirect golang.org/x/text v0.17.0 // indirect diff --git a/go.sum b/go.sum index 977abe3..1741612 100644 --- a/go.sum +++ b/go.sum @@ -1,10 +1,17 @@ github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/golang-migrate/migrate/v4 v4.17.1 h1:4zQ6iqL6t6AiItphxJctQb3cFqWiSpMnX7wLTPnnYO4= +github.com/golang-migrate/migrate/v4 v4.17.1/go.mod h1:m8hinFyWBn0SA4QKHuKh175Pm9wjmxj3S2Mia7dbXzM= github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= +github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I= +github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= +github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo= +github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM= github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a h1:bbPeKD0xmW/Y25WS6cokEszi5g+S0QxI/d45PkRi7Nk= @@ -15,6 +22,10 @@ github.com/jackc/puddle/v2 v2.2.1 h1:RhxXJtFG022u4ibrCSMSiu5aOq1i77R3OHKNJj77OAk github.com/jackc/puddle/v2 v2.2.1/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= github.com/klauspost/compress v1.13.6 h1:P76CopJELS0TiO2mebmnzgWaajssP/EszplttgQxcgc= github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= +github.com/klauspost/compress v1.15.11 h1:Lcadnb3RKGin4FYM/orgq0qde+nc15E5Cbqg4B9Sx9c= +github.com/klauspost/compress v1.15.11/go.mod h1:QPwzmACJjUTFsnSHH934V6woptycfrDDJnH7hvFVbGM= +github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw= +github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= github.com/montanaflynn/stats v0.7.1 h1:etflOAAHORrCC44V+aR6Ftzort912ZU+YLiSTuV8eaE= github.com/montanaflynn/stats v0.7.1/go.mod h1:etXPPgVO6n31NxCd9KQUMvCM+ve0ruNzt6R8Bnaayow= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= @@ -35,6 +46,8 @@ github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d/go.mod h1:rHwXgn7Jul github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= go.mongodb.org/mongo-driver v1.16.0 h1:tpRsfBJMROVHKpdGyc1BBEzzjDUWjItxbVSZ8Ls4BQ4= go.mongodb.org/mongo-driver v1.16.0/go.mod h1:oB6AhJQvFQL4LEHyXi6aJzQJtBiTQHiAd83l0GdFaiw= +go.uber.org/atomic v1.7.0 h1:ADUqmZGgLDDfbSL9ZmPxKTybcoEYHgpYfELNoN+7hsw= +go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.26.0 h1:RrRspgV4mU+YwB4FYnuBoKsUapNIL5cohGAmSH3azsw= diff --git a/internal/store/mongo/store.go b/internal/store/mongo/store.go index e826e03..903b65b 100644 --- a/internal/store/mongo/store.go +++ b/internal/store/mongo/store.go @@ -95,7 +95,9 @@ func (r statsRepository) Upsert(ctx context.Context, stats entities.DeviceStatis UpdatedAt: time.Now(), } - _, err := r.collection.UpdateOne(ctx, filter, document, opts) + update := bson.M{"$set": document} + + _, err := r.collection.UpdateOne(ctx, filter, update, opts) if err != nil { return fmt.Errorf("inserting: %w", err) } diff --git a/makefile b/makefile index 355c3b6..19a9e93 100644 --- a/makefile +++ b/makefile @@ -2,12 +2,24 @@ VERSION=$(shell git tag --sort=v:refname 2>/dev/null | head -n1) REVISION=$(shell git rev-parse --short HEAD) BUILDTIME=$(shell date -u +%FT%T) -build.docker: +build.docker: build.docker.web build.docker.migrator + +build.docker.web: docker build\ --build-arg VERSION=${VERSION}\ --build-arg REVISION=${REVISION}\ --build-arg BUILDTIME=${BUILDTIME}\ - -t git.loyso.art/devsim:latest\ + -t git.loyso.art/devsim-web:latest\ + -f ./dockers/web/Dockerfile\ + . + +build.docker.migrator: + docker build\ + --build-arg VERSION=${VERSION}\ + --build-arg REVISION=${REVISION}\ + --build-arg BUILDTIME=${BUILDTIME}\ + -t git.loyso.art/devsim-migrator:latest\ + -f ./dockers/migrator/Dockerfile\ . build: diff --git a/sqlc.yaml b/sqlc.yaml index fed0142..47f8989 100644 --- a/sqlc.yaml +++ b/sqlc.yaml @@ -2,7 +2,7 @@ version: "2" sql: - engine: "postgresql" queries: "assets/db/queries.sql" - schema: "assets/db/schema.sql" + schema: "assets/db/migrations" gen: go: package: "queries"