frx/devsim
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

atmost working example

Browse Source
This commit is contained in:
Aleksandr Trushkin
2024-08-11 21:44:19 +03:00
parent 0046755c7d
commit 880f67aa73
16 changed files with 566 additions and 107 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
.gitignore vendored Normal file
View File

@ -0,0 +1 @@
bin/*

20
Dockerfile
View File

@ -1,17 +1,23 @@
FROM golang:1.22-alpine as golang
WORKDIR /app
ARG VERSION="unknown"
ARG REVISION="unknown"
ARG BUILDTIME=""
WORKDIR /go/src/git.loyso.art/frx/devsim
COPY . .
RUN go mod download && go mod verify && CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -o /server .
RUN go mod download && \
go mod verify && \
CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -ldflags "-w -s -X 'git.loyso.art/frx/devsim.Version=${VERSION}' -X 'git.loyso.art/frx/devsim.Revision=${REVISION}' -X 'git.loyso.art/frx/devsim.BuildTime=${BUILDTIME}'" \
-o /go/bin/app /go/src/git.loyso.art/frx/devsim/cmd/web/main.go
FROM gcr.io/distroless/static-debian12@sha256:ce46866b3a5170db3b49364900fb3168dc0833dfb46c26da5c77f22abb01d8c3
WORKDIR /app
COPY --from=golang /server .
COPY --from=golang /go/bin/app /app
EXPOSE 9123
ENV DEVSIM_HTTP_ADDR=":9123"
ENV DEVSIM_HTTP_ADDR=":80"
EXPOSE 80
CMD ["/server"]
ENTRYPOINT ["/app"]

28
assets/db/queries.sql Normal file
View File

@ -0,0 +1,28 @@
-- name: UpsertDeviceMetrics :exec
INSERT INTO public.stats(
device_id,
inc_traffic,
out_traffic,
inc_rps,
write_rps,
read_rps,
updated_at
) VALUES (
@device_id,
@inc_traffic,
@out_traffic,
@inc_rps,
@write_rps,
@read_rps,
NOW()
) ON CONFLICT(device_id) DO UPDATE SET
device_id = EXCLUDED.device_id,
inc_traffic = EXCLUDED.inc_traffic,
out_traffic = EXCLUDED.out_traffic,
inc_rps = EXCLUDED.inc_rps,
write_rps = EXCLUDED.write_rps,
read_rps = EXCLUDED.read_rps,
updated_at = NOW();
-- name: ListDeviceStats :many
SELECT * FROM public.stats;

12
assets/db/schema.sql Normal file
View File

@ -0,0 +1,12 @@
CREATE TABLE IF NOT EXISTS public.stats (
device_id TEXT NOT NULL,
inc_traffic INT NOT NULL DEFAULT 0,
out_traffic INT NOT NULL DEFAULT 0,
inc_rps INT NOT NULL DEFAULT 0,
read_rps INT NOT NULL DEFAULT 0,
write_rps INT NOT NULL DEFAULT 0,
updated_at TIMESTAMP WITHOUT TIME ZONE DEFAULT NOW()
);
CREATE UNIQUE INDEX IF NOT EXISTS
stats_by_device_id_idx ON public.stats(device_id);

19
buildinfo.go Normal file
View File

@ -0,0 +1,19 @@
package devsim
var (
version string
revision string
buildTime string
)
func Version() string {
return version
}
func Revision() string {
return revision
}
func BuildTime() string {
return buildTime
}

145
cmd/simulator/main.go Normal file
View File

@ -0,0 +1,145 @@
package main
import (
"context"
"log"
"os"
"os/signal"
"strconv"
"sync/atomic"
"time"
"golang.org/x/sync/errgroup"
"git.loyso.art/frx/devsim/internal/entities"
"git.loyso.art/frx/devsim/internal/interconnect/collector"
)
var requestsDone = atomic.Uint64{}
func requestReporter(ctx context.Context) {
ticker := time.Tick(time.Second)
for {
select {
case <-ticker:
case <-ctx.Done():
return
}
requests := requestsDone.Swap(0)
log.Printf("rps: %d", requests)
}
}
func main() {
ctx, cancel := signal.NotifyContext(context.Background(), os.Interrupt)
defer cancel()
dstAddr := os.Getenv("DEVSIM_HTTP_ADDR")
deviceCountStr := os.Getenv("DEVSIM_DEVICE_COUNT")
delayStr := os.Getenv("DEVSIM_REQUEST_DELAY")
deviceCount, err := strconv.Atoi(deviceCountStr)
if err != nil {
log.Fatalf("parsing device count: %v", err)
}
if dstAddr == "" {
log.Fatal("no destination address provided")
}
delay, err := time.ParseDuration(delayStr)
if err != nil {
log.Fatalf("parsing delay duration: %v", err)
}
log.Printf("running application with settings: destination=%s device_count=%d delay=%s", dstAddr, deviceCount, delay)
client, err := collector.New(dstAddr)
if err != nil {
log.Fatalf("unable to create collector http client: %v", err)
}
eg, egctx := errgroup.WithContext(ctx)
eg.Go(func() error {
requestReporter(egctx)
return nil
})
for i := 0; i < deviceCount; i++ {
dh := newDeviceHandler(i+1, delay, client)
eg.Go(func() error {
dh.loop(egctx)
return nil
})
}
err = eg.Wait()
if err != nil {
log.Printf("error during execution: %v", err)
cancel()
}
}
type deviceHandler struct {
stats entities.DeviceStatistics
client collector.Client
delay time.Duration
}
func newDeviceHandler(id int, delay time.Duration, client collector.Client) *deviceHandler {
deviceID := entities.DeviceID(strconv.Itoa(id))
dh := deviceHandler{
delay: delay,
client: client,
}
dh.stats.ID = deviceID
return &dh
}
func (h *deviceHandler) loop(ctx context.Context) {
failedCount := 0
for {
start := time.Now()
h.stats.IncomingTrafficBytes++
h.stats.OutgoingTrafficBytes++
h.stats.WriteRPS = (h.stats.WriteRPS + 2) % 255
h.stats.ReadRPS = (h.stats.ReadRPS + 1) % 255
h.stats.IncomingRPS = h.stats.WriteRPS + h.stats.ReadRPS
err := h.client.Upsert(ctx, h.stats)
if err != nil {
log.Printf("%q: unable to upsert metrics: %v", h.stats.ID, err)
failedCount++
if failedCount > 10 {
log.Println("too much fails, exiting")
return
}
continue
}
requestsDone.Add(1)
failedCount = 0
elapsed := time.Since(start)
left := h.delay - elapsed
if left > 0 {
select {
case <-ctx.Done():
return
case <-time.After(left):
continue
}
} else {
if ctx.Err() != nil {
return
}
}
}
}

56
cmd/web/main.go
View File

@ -9,12 +9,14 @@ import (
"os"
"os/signal"
"golang.org/x/sync/errgroup"
"git.loyso.art/frx/devsim"
"git.loyso.art/frx/devsim/internal/api/http"
"git.loyso.art/frx/devsim/internal/store"
"git.loyso.art/frx/devsim/internal/store/mongo"
"git.loyso.art/frx/devsim/internal/store/pg"
"git.loyso.art/frx/devsim/pkg/collections"
"golang.org/x/sync/errgroup"
)
var availableStoreTypes = collections.NewSet([]string{
@ -25,20 +27,33 @@ func main() {
ctx, cancel := signal.NotifyContext(context.Background(), os.Interrupt)
defer cancel()
log := slog.New(slog.NewJSONHandler(os.Stdout, &slog.HandlerOptions{}))
log := slog.New(slog.NewJSONHandler(io.Discard, &slog.HandlerOptions{
Level: slog.LevelInfo,
}))
var settings applicationSettings
settings.fromEnv()
log.InfoContext(
ctx, "running application",
slog.Int("pid", os.Getpid()),
slog.String("version", devsim.Version()),
slog.String("revision", devsim.Revision()),
slog.String("buildtime", devsim.BuildTime()),
)
settings := loadConfigFromEnv()
err := settings.validate()
if err != nil {
log.ErrorContext(ctx, "unable to validate settings", slog.Any("err", err))
log.ErrorContext(ctx, "unable to validate settings", slog.String("err", err.Error()))
os.Exit(1)
}
log.InfoContext(
ctx, "config parsed",
slog.Any("settings", settings),
)
err = app(ctx, settings, log)
if err != nil {
log.ErrorContext(ctx, "unable to run app", slog.Any("err", err))
log.ErrorContext(ctx, "unable to run app", slog.String("err", err.Error()))
os.Exit(1)
}
}
@ -71,19 +86,20 @@ type applicationSettings struct {
mongo mongoSettings
}
func (s *applicationSettings) fromEnv() {
func loadConfigFromEnv() applicationSettings {
const webaddr = ":9123"
*s = applicationSettings{
listenAddr: valueOrDefault(os.Getenv("DEVSIM_HTTP_ADDR"), webaddr),
storeType: os.Getenv("DEVSIM_STORE_TYPE"),
}
var cfg applicationSettings
cfg.listenAddr = valueOrDefault(os.Getenv("DEVSIM_HTTP_ADDR"), webaddr)
cfg.storeType = os.Getenv("DEVSIM_STORE_TYPE")
s.pg.fromEnv()
s.mongo.fromEnv()
cfg.pg.fromEnv()
cfg.mongo.fromEnv()
return cfg
}
func (s *applicationSettings) validate() (err error) {
func (s applicationSettings) validate() (err error) {
if !availableStoreTypes.Contains(s.storeType) {
err = errors.Join(err, errors.New("store_type value is unsupported"))
}
@ -117,9 +133,9 @@ func app(ctx context.Context, settings applicationSettings, log *slog.Logger) (e
switch settings.storeType {
case "pg":
pgconn, err := pg.Dial(ctx, settings.pg.DSN)
if err != nil {
return fmt.Errorf("connecting to postgres: %w", err)
pgconn, errDial := pg.Dial(ctx, settings.pg.DSN)
if errDial != nil {
return fmt.Errorf("connecting to postgres: %w", errDial)
}
repo = pgconn.StatsRepository()
@ -128,9 +144,9 @@ func app(ctx context.Context, settings applicationSettings, log *slog.Logger) (e
closer: pgconn,
})
case "mongo":
mongoconn, err := mongo.Dial(ctx, settings.mongo.DSN)
if err != nil {
return fmt.Errorf("connecting to mongo: %w", err)
mongoconn, errDial := mongo.Dial(ctx, settings.mongo.DSN)
if errDial != nil {
return fmt.Errorf("connecting to mongo: %w", errDial)
}
repo = mongoconn.StatsRepository()

33
compose.yaml Normal file
View File

@ -0,0 +1,33 @@
services:
web.mongo:
image: git.loyso.art/devsim:latest
ports:
- 9124:80
depends_on:
- postgres
- mongo
environment:
DEVSIM_MONGO_DSN: mongodb://mongo
DEVSIM_STORE_TYPE: mongo
web.pg:
image: git.loyso.art/devsim:latest
ports:
- 9123:80
depends_on:
- postgres
- mongo
environment:
DEVSIM_PG_DSN: "postgres://devsim:devsim@postgres:5432/devsim?sslmode=disable"
DEVSIM_STORE_TYPE: pg
postgres:
image: postgres:15-alpine
environment:
POSTGRES_DB: devsim
POSTGRES_PASSWORD: devsim
POSTGRES_USER: devsim
ports: ["5432:5432"]
mongo:
image: mongo:7

5
internal/api/http/middlewares.go
View File

@ -44,7 +44,7 @@ func middlewareLogger(log *slog.Logger) middlewareFunc {
path := r.URL.Path
query := r.URL.Query().Encode()
log.InfoContext(
log.DebugContext(
r.Context(), "request processing",
slog.String("request_id", requestID),
slog.String("method", method),
@ -54,10 +54,9 @@ func middlewareLogger(log *slog.Logger) middlewareFunc {
next.ServeHTTP(w, r)
elapsed := time.Since(start)
log.InfoContext(
r.Context(), "request finished",
slog.Duration("elapsed", elapsed.Truncate(time.Millisecond)),
slog.Int64("elapsed", time.Since(start).Milliseconds()),
)
})
}

86
internal/interconnect/collector/client.go Normal file
View File

@ -0,0 +1,86 @@
package collector
import (
"bytes"
"context"
"fmt"
"io"
"net"
"net/http"
"text/template"
"time"
"git.loyso.art/frx/devsim/internal/entities"
)
type upsertRequest struct {
IncomingTraffic int `json:"incoming_traffic"`
OutgoingTraffic int `json:"outgoing_traffic"`
IncomingRPS int `json:"incoming_rps"`
ReadRPS int `json:"read_rps"`
WriteRPS int `json:"write_rps"`
}
type Client interface {
Upsert(context.Context, entities.DeviceStatistics) error
}
type client struct {
httpClient *http.Client
baseurl string
}
func New(addr string) (*client, error) {
hc := &http.Client{
Transport: &http.Transport{
Proxy: http.ProxyFromEnvironment,
DialContext: (&net.Dialer{
Timeout: time.Second * 10,
KeepAlive: time.Second * 30,
}).DialContext,
MaxIdleConns: 10,
IdleConnTimeout: time.Second * 90,
TLSHandshakeTimeout: time.Second * 5,
ExpectContinueTimeout: time.Second * 1,
ForceAttemptHTTP2: true,
},
Timeout: time.Second * 10,
}
return &client{
httpClient: hc,
baseurl: addr,
}, nil
}
var upsertRequestTemplate = template.Must(template.New("request").Parse(`{"incoming_traffic":{{.IncomingTrafficBytes}},"outgoing_traffic":{{.OutgoingTrafficBytes}},"incoming_rps":{{.IncomingRPS}},"read_rps":{{.ReadRPS}},"write_rps":{{.WriteRPS}}}`))
func (c *client) Upsert(ctx context.Context, stat entities.DeviceStatistics) error {
var buf bytes.Buffer
err := upsertRequestTemplate.Lookup("request").Execute(&buf, stat)
if err != nil {
return fmt.Errorf("executing template: %w", err)
}
path := c.baseurl + "/api/v1/stats/" + string(stat.ID)
req, err := http.NewRequestWithContext(ctx, http.MethodPost, path, &buf)
if err != nil {
return fmt.Errorf("preparing http request: %w", err)
}
resp, err := c.httpClient.Do(req)
if err != nil {
return fmt.Errorf("executing request: %w", err)
}
if resp.StatusCode == http.StatusOK {
return nil
}
data, err := io.ReadAll(io.LimitReader(resp.Body, 1<<20))
if err != nil {
return fmt.Errorf("reading body by status code %d: %w", resp.StatusCode, err)
}
return fmt.Errorf("expected status 200 got %d with body: %q", resp.StatusCode, data)
}

32
internal/store/pg/queries/db.go Normal file
View File

@ -0,0 +1,32 @@
// Code generated by sqlc. DO NOT EDIT.
// versions:
// sqlc v1.26.0
package queries
import (
"context"
"github.com/jackc/pgx/v5"
"github.com/jackc/pgx/v5/pgconn"
)
type DBTX interface {
Exec(context.Context, string, ...interface{}) (pgconn.CommandTag, error)
Query(context.Context, string, ...interface{}) (pgx.Rows, error)
QueryRow(context.Context, string, ...interface{}) pgx.Row
}
func New(db DBTX) *Queries {
return &Queries{db: db}
}
type Queries struct {
db DBTX
}
func (q *Queries) WithTx(tx pgx.Tx) *Queries {
return &Queries{
db: tx,
}
}

19
internal/store/pg/queries/models.go Normal file
View File

@ -0,0 +1,19 @@
// Code generated by sqlc. DO NOT EDIT.
// versions:
// sqlc v1.26.0
package queries
import (
"github.com/jackc/pgx/v5/pgtype"
)
type Stat struct {
DeviceID string
IncTraffic int32
OutTraffic int32
IncRps int32
ReadRps int32
WriteRps int32
UpdatedAt pgtype.Timestamp
}

90
internal/store/pg/queries/queries.sql.go Normal file
View File

@ -0,0 +1,90 @@
// Code generated by sqlc. DO NOT EDIT.
// versions:
// sqlc v1.26.0
// source: queries.sql
package queries
import (
"context"
)
const listDeviceStats = `-- name: ListDeviceStats :many
SELECT device_id, inc_traffic, out_traffic, inc_rps, read_rps, write_rps, updated_at FROM public.stats
`
func (q *Queries) ListDeviceStats(ctx context.Context) ([]Stat, error) {
rows, err := q.db.Query(ctx, listDeviceStats)
if err != nil {
return nil, err
}
defer rows.Close()
var items []Stat
for rows.Next() {
var i Stat
if err := rows.Scan(
&i.DeviceID,
&i.IncTraffic,
&i.OutTraffic,
&i.IncRps,
&i.ReadRps,
&i.WriteRps,
&i.UpdatedAt,
); err != nil {
return nil, err
}
items = append(items, i)
}
if err := rows.Err(); err != nil {
return nil, err
}
return items, nil
}
const upsertDeviceMetrics = `-- name: UpsertDeviceMetrics :exec
INSERT INTO public.stats(
device_id,
inc_traffic,
out_traffic,
inc_rps,
write_rps,
read_rps,
updated_at
) VALUES (
$1,
$2,
$3,
$4,
$5,
$6,
NOW()
) ON CONFLICT(device_id) DO UPDATE SET
device_id = EXCLUDED.device_id,
inc_traffic = EXCLUDED.inc_traffic,
out_traffic = EXCLUDED.out_traffic,
inc_rps = EXCLUDED.inc_rps,
write_rps = EXCLUDED.write_rps,
read_rps = EXCLUDED.read_rps,
updated_at = NOW()
`
type UpsertDeviceMetricsParams struct {
DeviceID string
IncTraffic int32
OutTraffic int32
IncRps int32
WriteRps int32
ReadRps int32
}
func (q *Queries) UpsertDeviceMetrics(ctx context.Context, arg UpsertDeviceMetricsParams) error {
_, err := q.db.Exec(ctx, upsertDeviceMetrics,
arg.DeviceID,
arg.IncTraffic,
arg.OutTraffic,
arg.IncRps,
arg.WriteRps,
arg.ReadRps,
)
return err
}

100
internal/store/pg/store.go
View File

@ -5,9 +5,10 @@ import (
"fmt"
"time"
"git.loyso.art/frx/devsim/internal/entities"
"github.com/jackc/pgx/v5/pgxpool"
"git.loyso.art/frx/devsim/internal/entities"
"git.loyso.art/frx/devsim/internal/store/pg/queries"
)
func Dial(ctx context.Context, addr string) (*repository, error) {
@ -66,93 +67,38 @@ func (s deviceStatsDB) asDomain() entities.DeviceStatistics {
}
func (r statsRepository) Upsert(ctx context.Context, stats entities.DeviceStatistics) error {
const query = `INSERT INTO public.stats (
device_id,
inc_traffic,
out_traffic,
inc_rps,
read_rps,
write_rps,
updated_at
) VALUES (
$1,
$2,
$3,
$4,
$5,
$6,
$7,
) ON CONFLICT(device_id) DO UPDATE SET
inc_traffic = EXCLUDED.inc_traffic,
out_traffic = EXCLUDED.out_traffic,
inc_rps = EXCLUDED.inc_rps,
read_rps = EXCLUDED.read_rps,
write_rps = EXCLUDED.write_rps,
updated_at = EXCLUDED.updated_at
RETURNING id;
`
_, err := r.db.Exec(
ctx, query,
stats.ID,
stats.IncomingTrafficBytes,
stats.OutgoingTrafficBytes,
stats.IncomingRPS,
stats.ReadRPS,
stats.WriteRPS,
)
err := queries.New(r.db).UpsertDeviceMetrics(ctx, queries.UpsertDeviceMetricsParams{
DeviceID: string(stats.ID),
IncTraffic: int32(stats.IncomingTrafficBytes),
OutTraffic: int32(stats.OutgoingTrafficBytes),
IncRps: int32(stats.IncomingRPS),
WriteRps: int32(stats.WriteRPS),
ReadRps: int32(stats.ReadRPS),
})
if err != nil {
return fmt.Errorf("executing query: %w", err)
return fmt.Errorf("upserting device metrics: %w", err)
}
return nil
}
func (r statsRepository) List(ctx context.Context) (out []entities.DeviceStatistics, err error) {
var count int
err = r.db.QueryRow(ctx, `SELECT COUNT(device_id) FROM public.stats`).Scan(&count)
stats, err := queries.New(r.db).ListDeviceStats(ctx)
if err != nil {
return nil, fmt.Errorf("getting count: %w", err)
return nil, fmt.Errorf("listing device stats: %w", err)
}
out = make([]entities.DeviceStatistics, 0, count)
out = make([]entities.DeviceStatistics, len(stats))
const query = `SELECT
device_id,
inc_traffic,
out_traffic,
inc_rps,
read_rps,
write_rps,
updated_at
FROM public.stats;
`
rows, err := r.db.Query(ctx, query)
if err != nil {
return nil, fmt.Errorf("querying: %w", err)
for i, stat := range stats {
out[i] = entities.DeviceStatistics{
IncomingTrafficBytes: int(stat.IncTraffic),
OutgoingTrafficBytes: int(stat.OutTraffic),
IncomingRPS: int(stat.IncRps),
WriteRPS: int(stat.WriteRps),
ReadRPS: int(stat.ReadRps),
UpdatedAt: stat.UpdatedAt.Time,
}
defer rows.Close()
for rows.Next() {
var stat deviceStatsDB
err = rows.Scan(
&stat.DeviceID,
&stat.IncomingTraffic,
&stat.OutgoingTraffic,
&stat.IncomingRPS,
&stat.ReadRPS,
&stat.WriteRPS,
&stat.UpdatedAt,
)
if err != nil {
return nil, fmt.Errorf("scanning row: %w", err)
}
out = append(out, stat.asDomain())
}
if err = rows.Err(); err != nil {
return nil, fmt.Errorf("checking rows err: %w", err)
}
return out, nil

17
makefile Normal file
View File

@ -0,0 +1,17 @@
VERSION=$(shell git tag --sort=v:refname 2>/dev/null | head -n1)
REVISION=$(shell git rev-parse --short HEAD)
BUILDTIME=$(shell date -u +%FT%T)
build.docker:
docker build\
--build-arg VERSION=${VERSION}\
--build-arg REVISION=${REVISION}\
--build-arg BUILDTIME=${BUILDTIME}\
-t git.loyso.art/devsim:latest\
.
build:
CGO_ENABALED=0 go build \
-ldflags "-w -s -X 'git.loyso.art/frx/devsim.version=${VERSION}' -X 'git.loyso.art/frx/devsim.revision=${REVISION}' -X 'git.loyso.art/frx/devsim.buildTime=${BUILDTIME}Z'" \
-o bin/web ./cmd/web/*.go

10
sqlc.yaml Normal file
View File

@ -0,0 +1,10 @@
version: "2"
sql:
- engine: "postgresql"
queries: "assets/db/queries.sql"
schema: "assets/db/schema.sql"
gen:
go:
package: "queries"
out: "internal/store/pg/queries"
sql_package: "pgx/v5"