From b57afeddbd50b102c406f48c681beb2acbb41f23 Mon Sep 17 00:00:00 2001 From: sneak Date: Fri, 27 Feb 2026 12:14:34 +0700 Subject: [PATCH 1/4] Add backend with buffered zstd-compressed report storage Introduce the Go backend (netwatch-server) with an HTTP API that accepts telemetry reports and persists them as zstd-compressed JSONL files. Reports are buffered in memory and flushed to disk when the buffer reaches 10 MiB or every 60 seconds. --- .prettierignore | 1 + backend/.dockerignore | 3 + backend/.editorconfig | 12 ++ backend/.gitignore | 6 + backend/.golangci.yml | 32 +++ backend/Dockerfile | 22 ++ backend/Makefile | 53 +++++ backend/go.mod | 30 +++ backend/go.sum | 65 ++++++ backend/internal/config/config.go | 86 ++++++++ backend/internal/globals/globals.go | 31 +++ backend/internal/handlers/handlers.go | 74 +++++++ backend/internal/handlers/handlers_test.go | 13 ++ backend/internal/handlers/healthcheck.go | 11 + backend/internal/handlers/report.go | 82 ++++++++ backend/internal/healthcheck/healthcheck.go | 82 ++++++++ backend/internal/logger/logger.go | 85 ++++++++ backend/internal/middleware/middleware.go | 129 ++++++++++++ backend/internal/reportbuf/reportbuf.go | 199 +++++++++++++++++++ backend/internal/reportbuf/reportbuf_test.go | 13 ++ backend/internal/server/http.go | 43 ++++ backend/internal/server/routes.go | 31 +++ backend/internal/server/server.go | 147 ++++++++++++++ 23 files changed, 1250 insertions(+) create mode 100644 backend/.dockerignore create mode 100644 backend/.editorconfig create mode 100644 backend/.gitignore create mode 100644 backend/.golangci.yml create mode 100644 backend/Dockerfile create mode 100644 backend/Makefile create mode 100644 backend/go.mod create mode 100644 backend/go.sum create mode 100644 backend/internal/config/config.go create mode 100644 backend/internal/globals/globals.go create mode 100644 backend/internal/handlers/handlers.go create mode 100644 backend/internal/handlers/handlers_test.go create mode 100644 backend/internal/handlers/healthcheck.go create mode 100644 backend/internal/handlers/report.go create mode 100644 backend/internal/healthcheck/healthcheck.go create mode 100644 backend/internal/logger/logger.go create mode 100644 backend/internal/middleware/middleware.go create mode 100644 backend/internal/reportbuf/reportbuf.go create mode 100644 backend/internal/reportbuf/reportbuf_test.go create mode 100644 backend/internal/server/http.go create mode 100644 backend/internal/server/routes.go create mode 100644 backend/internal/server/server.go diff --git a/.prettierignore b/.prettierignore index c24da45..d1a0b78 100644 --- a/.prettierignore +++ b/.prettierignore @@ -1,3 +1,4 @@ +backend/ dist/ node_modules/ yarn.lock diff --git a/backend/.dockerignore b/backend/.dockerignore new file mode 100644 index 0000000..5414d56 --- /dev/null +++ b/backend/.dockerignore @@ -0,0 +1,3 @@ +.git +node_modules +.DS_Store diff --git a/backend/.editorconfig b/backend/.editorconfig new file mode 100644 index 0000000..2fe0ce0 --- /dev/null +++ b/backend/.editorconfig @@ -0,0 +1,12 @@ +root = true + +[*] +indent_style = space +indent_size = 4 +end_of_line = lf +charset = utf-8 +trim_trailing_whitespace = true +insert_final_newline = true + +[Makefile] +indent_style = tab diff --git a/backend/.gitignore b/backend/.gitignore new file mode 100644 index 0000000..861bd94 --- /dev/null +++ b/backend/.gitignore @@ -0,0 +1,6 @@ +netwatch-server +*.log +*.out +*.test +.env +data/ diff --git a/backend/.golangci.yml b/backend/.golangci.yml new file mode 100644 index 0000000..34a8e31 --- /dev/null +++ b/backend/.golangci.yml @@ -0,0 +1,32 @@ +version: "2" + +run: + timeout: 5m + modules-download-mode: readonly + +linters: + default: all + disable: + # Genuinely incompatible with project patterns + - exhaustruct # Requires all struct fields + - depguard # Dependency allow/block lists + - godot # Requires comments to end with periods + - wsl # Deprecated, replaced by wsl_v5 + - wrapcheck # Too verbose for internal packages + - varnamelen # Short names like db, id are idiomatic Go + +linters-settings: + lll: + line-length: 88 + funlen: + lines: 80 + statements: 50 + cyclop: + max-complexity: 15 + dupl: + threshold: 100 + +issues: + exclude-use-default: false + max-issues-per-linter: 0 + max-same-issues: 0 diff --git a/backend/Dockerfile b/backend/Dockerfile new file mode 100644 index 0000000..5839cb1 --- /dev/null +++ b/backend/Dockerfile @@ -0,0 +1,22 @@ +# golang:1.24-alpine (2026-02-26) +FROM golang:1.24-alpine@sha256:8bee1901f1e530bfb4a7850aa7a479d17ae3a18beb6e09064ed54cfd245b7191 AS builder + +RUN apk add --no-cache git make gcc musl-dev + +WORKDIR /src +COPY go.mod go.sum ./ +RUN go mod download +COPY . . + +RUN make check +RUN make build + +# alpine:3.21 (2026-02-26) +FROM alpine:3.21@sha256:c3f8e73fdb79deaebaa2037150150191b9dcbfba68b4a46d70103204c53f4709 + +RUN apk add --no-cache ca-certificates +COPY --from=builder /src/netwatch-server /usr/local/bin/netwatch-server +COPY --from=builder /src /src + +EXPOSE 8080 +ENTRYPOINT ["netwatch-server"] diff --git a/backend/Makefile b/backend/Makefile new file mode 100644 index 0000000..ad86aa0 --- /dev/null +++ b/backend/Makefile @@ -0,0 +1,53 @@ +UNAME_S := $(shell uname -s) +VERSION := $(shell git describe --always --dirty) +BUILDARCH := $(shell uname -m) +BINARY := netwatch-server + +GOLDFLAGS += -X main.Version=$(VERSION) +GOLDFLAGS += -X main.Buildarch=$(BUILDARCH) + +ifeq ($(UNAME_S),Darwin) + GOFLAGS := -ldflags "$(GOLDFLAGS)" +else + GOFLAGS = -ldflags "-linkmode external -extldflags -static $(GOLDFLAGS)" +endif + +.PHONY: all build test lint fmt fmt-check check docker hooks run clean + +all: build + +build: ./$(BINARY) + +./$(BINARY): $(shell find . -name '*.go' -type f) go.mod go.sum + go build -o $@ $(GOFLAGS) ./cmd/netwatch-server/ + +test: + timeout 30 go test ./... + +lint: + golangci-lint run ./... + +fmt: + go fmt ./... + +fmt-check: + @test -z "$$(gofmt -l .)" || \ + (echo "Files not formatted:"; gofmt -l .; exit 1) + +check: test lint fmt-check + +docker: + timeout 300 docker build -t netwatch-server . + +hooks: + @printf '#!/bin/sh\ncd backend && make check\n' > \ + $$(git rev-parse --show-toplevel)/.git/hooks/pre-commit + @chmod +x \ + $$(git rev-parse --show-toplevel)/.git/hooks/pre-commit + @echo "Pre-commit hook installed" + +run: build + ./$(BINARY) + +clean: + rm -f ./$(BINARY) diff --git a/backend/go.mod b/backend/go.mod new file mode 100644 index 0000000..89e3ef4 --- /dev/null +++ b/backend/go.mod @@ -0,0 +1,30 @@ +module sneak.berlin/go/netwatch + +go 1.25.5 + +require ( + github.com/go-chi/chi/v5 v5.2.5 + github.com/go-chi/cors v1.2.2 + github.com/joho/godotenv v1.5.1 + github.com/klauspost/compress v1.18.4 + github.com/spf13/viper v1.21.0 + go.uber.org/fx v1.24.0 +) + +require ( + github.com/fsnotify/fsnotify v1.9.0 // indirect + github.com/go-viper/mapstructure/v2 v2.4.0 // indirect + github.com/pelletier/go-toml/v2 v2.2.4 // indirect + github.com/sagikazarmark/locafero v0.11.0 // indirect + github.com/sourcegraph/conc v0.3.1-0.20240121214520-5f936abd7ae8 // indirect + github.com/spf13/afero v1.15.0 // indirect + github.com/spf13/cast v1.10.0 // indirect + github.com/spf13/pflag v1.0.10 // indirect + github.com/subosito/gotenv v1.6.0 // indirect + go.uber.org/dig v1.19.0 // indirect + go.uber.org/multierr v1.10.0 // indirect + go.uber.org/zap v1.26.0 // indirect + go.yaml.in/yaml/v3 v3.0.4 // indirect + golang.org/x/sys v0.29.0 // indirect + golang.org/x/text v0.28.0 // indirect +) diff --git a/backend/go.sum b/backend/go.sum new file mode 100644 index 0000000..de6ddf2 --- /dev/null +++ b/backend/go.sum @@ -0,0 +1,65 @@ +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8= +github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= +github.com/fsnotify/fsnotify v1.9.0 h1:2Ml+OJNzbYCTzsxtv8vKSFD9PbJjmhYF14k/jKC7S9k= +github.com/fsnotify/fsnotify v1.9.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0= +github.com/go-chi/chi/v5 v5.2.5 h1:Eg4myHZBjyvJmAFjFvWgrqDTXFyOzjj7YIm3L3mu6Ug= +github.com/go-chi/chi/v5 v5.2.5/go.mod h1:X7Gx4mteadT3eDOMTsXzmI4/rwUpOwBHLpAfupzFJP0= +github.com/go-chi/cors v1.2.2 h1:Jmey33TE+b+rB7fT8MUy1u0I4L+NARQlK6LhzKPSyQE= +github.com/go-chi/cors v1.2.2/go.mod h1:sSbTewc+6wYHBBCW7ytsFSn836hqM7JxpglAy2Vzc58= +github.com/go-viper/mapstructure/v2 v2.4.0 h1:EBsztssimR/CONLSZZ04E8qAkxNYq4Qp9LvH92wZUgs= +github.com/go-viper/mapstructure/v2 v2.4.0/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0= +github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4= +github.com/klauspost/compress v1.18.4 h1:RPhnKRAQ4Fh8zU2FY/6ZFDwTVTxgJ/EMydqSTzE9a2c= +github.com/klauspost/compress v1.18.4/go.mod h1:R0h/fSBs8DE4ENlcrlib3PsXS61voFxhIs2DeRhCvJ4= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/pelletier/go-toml/v2 v2.2.4 h1:mye9XuhQ6gvn5h28+VilKrrPoQVanw5PMw/TB0t5Ec4= +github.com/pelletier/go-toml/v2 v2.2.4/go.mod h1:2gIqNv+qfxSVS7cM2xJQKtLSTLUE9V8t9Stt+h56mCY= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8= +github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= +github.com/sagikazarmark/locafero v0.11.0 h1:1iurJgmM9G3PA/I+wWYIOw/5SyBtxapeHDcg+AAIFXc= +github.com/sagikazarmark/locafero v0.11.0/go.mod h1:nVIGvgyzw595SUSUE6tvCp3YYTeHs15MvlmU87WwIik= +github.com/sourcegraph/conc v0.3.1-0.20240121214520-5f936abd7ae8 h1:+jumHNA0Wrelhe64i8F6HNlS8pkoyMv5sreGx2Ry5Rw= +github.com/sourcegraph/conc v0.3.1-0.20240121214520-5f936abd7ae8/go.mod h1:3n1Cwaq1E1/1lhQhtRK2ts/ZwZEhjcQeJQ1RuC6Q/8U= +github.com/spf13/afero v1.15.0 h1:b/YBCLWAJdFWJTN9cLhiXXcD7mzKn9Dm86dNnfyQw1I= +github.com/spf13/afero v1.15.0/go.mod h1:NC2ByUVxtQs4b3sIUphxK0NioZnmxgyCrfzeuq8lxMg= +github.com/spf13/cast v1.10.0 h1:h2x0u2shc1QuLHfxi+cTJvs30+ZAHOGRic8uyGTDWxY= +github.com/spf13/cast v1.10.0/go.mod h1:jNfB8QC9IA6ZuY2ZjDp0KtFO2LZZlg4S/7bzP6qqeHo= +github.com/spf13/pflag v1.0.10 h1:4EBh2KAYBwaONj6b2Ye1GiHfwjqyROoF4RwYO+vPwFk= +github.com/spf13/pflag v1.0.10/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/spf13/viper v1.21.0 h1:x5S+0EU27Lbphp4UKm1C+1oQO+rKx36vfCoaVebLFSU= +github.com/spf13/viper v1.21.0/go.mod h1:P0lhsswPGWD/1lZJ9ny3fYnVqxiegrlNrEmgLjbTCAY= +github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= +github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= +github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8= +github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU= +go.uber.org/dig v1.19.0 h1:BACLhebsYdpQ7IROQ1AGPjrXcP5dF80U3gKoFzbaq/4= +go.uber.org/dig v1.19.0/go.mod h1:Us0rSJiThwCv2GteUN0Q7OKvU7n5J4dxZ9JKUXozFdE= +go.uber.org/fx v1.24.0 h1:wE8mruvpg2kiiL1Vqd0CC+tr0/24XIB10Iwp2lLWzkg= +go.uber.org/fx v1.24.0/go.mod h1:AmDeGyS+ZARGKM4tlH4FY2Jr63VjbEDJHtqXTGP5hbo= +go.uber.org/goleak v1.2.0 h1:xqgm/S+aQvhWFTtR0XK3Jvg7z8kGV8P4X14IzwN3Eqk= +go.uber.org/goleak v1.2.0/go.mod h1:XJYK+MuIchqpmGmUSAzotztawfKvYLUIgg7guXrwVUo= +go.uber.org/multierr v1.10.0 h1:S0h4aNzvfcFsC3dRF1jLoaov7oRaKqRGC/pUEJ2yvPQ= +go.uber.org/multierr v1.10.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= +go.uber.org/zap v1.26.0 h1:sI7k6L95XOKS281NhVKOFCUNIvv9e0w4BF8N3u+tCRo= +go.uber.org/zap v1.26.0/go.mod h1:dtElttAiwGvoJ/vj4IwHBS/gXsEu/pZ50mUIRWuG0so= +go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc= +go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= +golang.org/x/sys v0.29.0 h1:TPYlXGxvx1MGTn2GiZDhnjPA9wZzZeGKHHmKhHYvgaU= +golang.org/x/sys v0.29.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/text v0.28.0 h1:rhazDwis8INMIwQ4tpjLDzUhx6RlXqZNPEM0huQojng= +golang.org/x/text v0.28.0/go.mod h1:U8nCwOR8jO/marOQ0QbDiOngZVEBB7MAiitBuMjXiNU= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/backend/internal/config/config.go b/backend/internal/config/config.go new file mode 100644 index 0000000..6966d57 --- /dev/null +++ b/backend/internal/config/config.go @@ -0,0 +1,86 @@ +// Package config loads application configuration from +// environment variables, .env files, and config files. +package config + +import ( + "errors" + "log/slog" + + "sneak.berlin/go/netwatch/internal/globals" + "sneak.berlin/go/netwatch/internal/logger" + + _ "github.com/joho/godotenv/autoload" // loads .env file + "github.com/spf13/viper" + "go.uber.org/fx" +) + +// Params defines the dependencies for Config. +type Params struct { + fx.In + + Globals *globals.Globals + Logger *logger.Logger +} + +// Config holds the resolved application configuration. +type Config struct { + DataDir string + Debug bool + MetricsPassword string + MetricsUsername string + Port int + SentryDSN string + log *slog.Logger + params *Params +} + +// New loads configuration from env, .env files, and config +// files, returning a fully resolved Config. +func New( + _ fx.Lifecycle, + params Params, +) (*Config, error) { + log := params.Logger.Get() + name := params.Globals.Appname + + viper.SetConfigName(name) + viper.SetConfigType("yaml") + viper.AddConfigPath("/etc/" + name) + viper.AddConfigPath("$HOME/.config/" + name) + + viper.AutomaticEnv() + + viper.SetDefault("DATA_DIR", "./data/reports") + viper.SetDefault("DEBUG", "false") + viper.SetDefault("PORT", "8080") + viper.SetDefault("SENTRY_DSN", "") + viper.SetDefault("METRICS_USERNAME", "") + viper.SetDefault("METRICS_PASSWORD", "") + + err := viper.ReadInConfig() + if err != nil { + var notFound viper.ConfigFileNotFoundError + if !errors.As(err, ¬Found) { + log.Error("config file malformed", "error", err) + panic(err) + } + } + + s := &Config{ + DataDir: viper.GetString("DATA_DIR"), + Debug: viper.GetBool("DEBUG"), + MetricsPassword: viper.GetString("METRICS_PASSWORD"), + MetricsUsername: viper.GetString("METRICS_USERNAME"), + Port: viper.GetInt("PORT"), + SentryDSN: viper.GetString("SENTRY_DSN"), + log: log, + params: ¶ms, + } + + if s.Debug { + params.Logger.EnableDebugLogging() + s.log = params.Logger.Get() + } + + return s, nil +} diff --git a/backend/internal/globals/globals.go b/backend/internal/globals/globals.go new file mode 100644 index 0000000..0e3807b --- /dev/null +++ b/backend/internal/globals/globals.go @@ -0,0 +1,31 @@ +// Package globals provides build-time variables injected via +// ldflags and made available through dependency injection. +package globals + +import "go.uber.org/fx" + +//nolint:gochecknoglobals // set from main before fx starts +var ( + // Appname is the application name. + Appname string + // Version is the git version tag. + Version string + // Buildarch is the build architecture. + Buildarch string +) + +// Globals holds build-time metadata for the application. +type Globals struct { + Appname string + Version string + Buildarch string +} + +// New creates a Globals instance from package-level variables. +func New(_ fx.Lifecycle) (*Globals, error) { + return &Globals{ + Appname: Appname, + Buildarch: Buildarch, + Version: Version, + }, nil +} diff --git a/backend/internal/handlers/handlers.go b/backend/internal/handlers/handlers.go new file mode 100644 index 0000000..a327fcd --- /dev/null +++ b/backend/internal/handlers/handlers.go @@ -0,0 +1,74 @@ +// Package handlers implements HTTP request handlers for the +// netwatch-server API. +package handlers + +import ( + "context" + "encoding/json" + "log/slog" + "net/http" + + "sneak.berlin/go/netwatch/internal/globals" + "sneak.berlin/go/netwatch/internal/healthcheck" + "sneak.berlin/go/netwatch/internal/logger" + "sneak.berlin/go/netwatch/internal/reportbuf" + + "go.uber.org/fx" +) + +const jsonContentType = "application/json; charset=utf-8" + +// Params defines the dependencies for Handlers. +type Params struct { + fx.In + + Buffer *reportbuf.Buffer + Globals *globals.Globals + Healthcheck *healthcheck.Healthcheck + Logger *logger.Logger +} + +// Handlers provides HTTP handler factories for all endpoints. +type Handlers struct { + buf *reportbuf.Buffer + hc *healthcheck.Healthcheck + log *slog.Logger + params *Params +} + +// New creates a Handlers instance. +func New( + lc fx.Lifecycle, + params Params, +) (*Handlers, error) { + s := new(Handlers) + s.buf = params.Buffer + s.params = ¶ms + s.log = params.Logger.Get() + s.hc = params.Healthcheck + + lc.Append(fx.Hook{ + OnStart: func(_ context.Context) error { + return nil + }, + }) + + return s, nil +} + +func (s *Handlers) respondJSON( + w http.ResponseWriter, + _ *http.Request, + data any, + status int, +) { + w.Header().Set("Content-Type", jsonContentType) + w.WriteHeader(status) + + if data != nil { + err := json.NewEncoder(w).Encode(data) + if err != nil { + s.log.Error("json encode error", "error", err) + } + } +} diff --git a/backend/internal/handlers/handlers_test.go b/backend/internal/handlers/handlers_test.go new file mode 100644 index 0000000..5128215 --- /dev/null +++ b/backend/internal/handlers/handlers_test.go @@ -0,0 +1,13 @@ +package handlers_test + +import ( + "testing" + + _ "sneak.berlin/go/netwatch/internal/handlers" +) + +func TestImport(t *testing.T) { + t.Parallel() + // Compilation check — verifies the package parses + // and all imports resolve. +} diff --git a/backend/internal/handlers/healthcheck.go b/backend/internal/handlers/healthcheck.go new file mode 100644 index 0000000..d71e8bf --- /dev/null +++ b/backend/internal/handlers/healthcheck.go @@ -0,0 +1,11 @@ +package handlers + +import "net/http" + +// HandleHealthCheck returns a handler for the health check +// endpoint. +func (s *Handlers) HandleHealthCheck() http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + s.respondJSON(w, r, s.hc.Check(), http.StatusOK) + } +} diff --git a/backend/internal/handlers/report.go b/backend/internal/handlers/report.go new file mode 100644 index 0000000..d4b54ed --- /dev/null +++ b/backend/internal/handlers/report.go @@ -0,0 +1,82 @@ +package handlers + +import ( + "encoding/json" + "net/http" +) + +const maxReportBodyBytes = 1 << 20 // 1 MiB + +type reportSample struct { + T int64 `json:"t"` + Latency *int `json:"latency"` + Error *string `json:"error"` +} + +type reportHost struct { + History []reportSample `json:"history"` + Name string `json:"name"` + Status string `json:"status"` + URL string `json:"url"` +} + +type report struct { + ClientID string `json:"clientId"` + Geo json.RawMessage `json:"geo"` + Hosts []reportHost `json:"hosts"` + Timestamp string `json:"timestamp"` +} + +// HandleReport returns a handler that accepts telemetry +// reports from NetWatch clients. +func (s *Handlers) HandleReport() http.HandlerFunc { + type response struct { + Status string `json:"status"` + } + + return func(w http.ResponseWriter, r *http.Request) { + r.Body = http.MaxBytesReader( + w, r.Body, maxReportBodyBytes, + ) + + var rpt report + + err := json.NewDecoder(r.Body).Decode(&rpt) + if err != nil { + s.log.Error("failed to decode report", + "error", err, + ) + s.respondJSON(w, r, + &response{Status: "error"}, + http.StatusBadRequest, + ) + + return + } + + totalSamples := 0 + for _, h := range rpt.Hosts { + totalSamples += len(h.History) + } + + s.log.Info("report received", + "client_id", rpt.ClientID, + "timestamp", rpt.Timestamp, + "host_count", len(rpt.Hosts), + "total_samples", totalSamples, + "geo", string(rpt.Geo), + ) + + bufErr := s.buf.Append(rpt) + if bufErr != nil { + s.log.Error("failed to buffer report", + "error", bufErr, + ) + } + + s.respondJSON(w, r, + &response{Status: "ok"}, + http.StatusOK, + ) + } +} diff --git a/backend/internal/healthcheck/healthcheck.go b/backend/internal/healthcheck/healthcheck.go new file mode 100644 index 0000000..4540abd --- /dev/null +++ b/backend/internal/healthcheck/healthcheck.go @@ -0,0 +1,82 @@ +// Package healthcheck provides a service that reports +// application health, uptime, and version information. +package healthcheck + +import ( + "context" + "log/slog" + "time" + + "sneak.berlin/go/netwatch/internal/config" + "sneak.berlin/go/netwatch/internal/globals" + "sneak.berlin/go/netwatch/internal/logger" + + "go.uber.org/fx" +) + +// Params defines the dependencies for Healthcheck. +type Params struct { + fx.In + + Config *config.Config + Globals *globals.Globals + Logger *logger.Logger +} + +// Healthcheck tracks startup time and builds health responses. +type Healthcheck struct { + StartupTime time.Time + log *slog.Logger + params *Params +} + +// Response is the JSON payload returned by the health check +// endpoint. +type Response struct { + Appname string `json:"appname"` + Now string `json:"now"` + Status string `json:"status"` + UptimeHuman string `json:"uptimeHuman"` + UptimeSeconds int64 `json:"uptimeSeconds"` + Version string `json:"version"` +} + +// New creates a Healthcheck, recording startup time via an +// fx lifecycle hook. +func New( + lc fx.Lifecycle, + params Params, +) (*Healthcheck, error) { + s := new(Healthcheck) + s.params = ¶ms + s.log = params.Logger.Get() + + lc.Append(fx.Hook{ + OnStart: func(_ context.Context) error { + s.StartupTime = time.Now().UTC() + + return nil + }, + OnStop: func(_ context.Context) error { + return nil + }, + }) + + return s, nil +} + +// Check returns the current health status of the application. +func (s *Healthcheck) Check() *Response { + return &Response{ + Appname: s.params.Globals.Appname, + Now: time.Now().UTC().Format(time.RFC3339Nano), + Status: "ok", + UptimeHuman: s.uptime().String(), + UptimeSeconds: int64(s.uptime().Seconds()), + Version: s.params.Globals.Version, + } +} + +func (s *Healthcheck) uptime() time.Duration { + return time.Since(s.StartupTime) +} diff --git a/backend/internal/logger/logger.go b/backend/internal/logger/logger.go new file mode 100644 index 0000000..9f30c1a --- /dev/null +++ b/backend/internal/logger/logger.go @@ -0,0 +1,85 @@ +// Package logger provides a configured slog.Logger with TTY +// detection for development vs production output. +package logger + +import ( + "log/slog" + "os" + + "sneak.berlin/go/netwatch/internal/globals" + + "go.uber.org/fx" +) + +// Params defines the dependencies for Logger. +type Params struct { + fx.In + + Globals *globals.Globals +} + +// Logger wraps slog.Logger with dynamic level control. +type Logger struct { + log *slog.Logger + level *slog.LevelVar + params Params +} + +// New creates a Logger with TTY-aware output formatting. +func New(_ fx.Lifecycle, params Params) (*Logger, error) { + l := new(Logger) + l.level = new(slog.LevelVar) + l.level.Set(slog.LevelInfo) + l.params = params + + tty := false + + if fileInfo, _ := os.Stdout.Stat(); fileInfo != nil { + if (fileInfo.Mode() & os.ModeCharDevice) != 0 { + tty = true + } + } + + var handler slog.Handler + if tty { + handler = slog.NewTextHandler( + os.Stdout, + &slog.HandlerOptions{ + Level: l.level, + AddSource: true, + }, + ) + } else { + handler = slog.NewJSONHandler( + os.Stdout, + &slog.HandlerOptions{ + Level: l.level, + AddSource: true, + }, + ) + } + + l.log = slog.New(handler) + + return l, nil +} + +// EnableDebugLogging sets the log level to debug. +func (l *Logger) EnableDebugLogging() { + l.level.Set(slog.LevelDebug) + l.log.Debug("debug logging enabled", "debug", true) +} + +// Get returns the underlying slog.Logger. +func (l *Logger) Get() *slog.Logger { + return l.log +} + +// Identify logs the application's build-time metadata. +func (l *Logger) Identify() { + l.log.Info("starting", + "appname", l.params.Globals.Appname, + "version", l.params.Globals.Version, + "buildarch", l.params.Globals.Buildarch, + ) +} diff --git a/backend/internal/middleware/middleware.go b/backend/internal/middleware/middleware.go new file mode 100644 index 0000000..01cbe8d --- /dev/null +++ b/backend/internal/middleware/middleware.go @@ -0,0 +1,129 @@ +// Package middleware provides HTTP middleware for logging, +// CORS, and other cross-cutting concerns. +package middleware + +import ( + "log/slog" + "net" + "net/http" + "time" + + "sneak.berlin/go/netwatch/internal/config" + "sneak.berlin/go/netwatch/internal/globals" + "sneak.berlin/go/netwatch/internal/logger" + + "github.com/go-chi/chi/v5/middleware" + "github.com/go-chi/cors" + "go.uber.org/fx" +) + +const corsMaxAgeSec = 300 + +// Params defines the dependencies for Middleware. +type Params struct { + fx.In + + Config *config.Config + Globals *globals.Globals + Logger *logger.Logger +} + +// Middleware holds shared state for middleware factories. +type Middleware struct { + log *slog.Logger + params *Params +} + +// New creates a Middleware instance. +func New( + _ fx.Lifecycle, + params Params, +) (*Middleware, error) { + s := new(Middleware) + s.params = ¶ms + s.log = params.Logger.Get() + + return s, nil +} + +type loggingResponseWriter struct { + http.ResponseWriter + + statusCode int +} + +func newLoggingResponseWriter( + w http.ResponseWriter, +) *loggingResponseWriter { + return &loggingResponseWriter{w, http.StatusOK} +} + +func (lrw *loggingResponseWriter) WriteHeader(code int) { + lrw.statusCode = code + lrw.ResponseWriter.WriteHeader(code) +} + +func ipFromHostPort(hostPort string) string { + host, _, err := net.SplitHostPort(hostPort) + if err != nil { + return hostPort + } + + return host +} + +// Logging returns middleware that logs each request with +// timing, status code, and client information. +func (s *Middleware) Logging() func(http.Handler) http.Handler { + return func(next http.Handler) http.Handler { + return http.HandlerFunc( + func(w http.ResponseWriter, r *http.Request) { + start := time.Now().UTC() + lrw := newLoggingResponseWriter(w) + ctx := r.Context() + + defer func() { + latency := time.Since(start) + s.log.InfoContext(ctx, "request", + "request_start", start, + "method", r.Method, + "url", r.URL.String(), + "useragent", r.UserAgent(), + "request_id", + ctx.Value( + middleware.RequestIDKey, + ), + "referer", r.Referer(), + "proto", r.Proto, + "remote_ip", + ipFromHostPort(r.RemoteAddr), + "status", lrw.statusCode, + "latency_ms", + latency.Milliseconds(), + ) + }() + + next.ServeHTTP(lrw, r) + }, + ) + } +} + +// CORS returns middleware that adds permissive CORS headers. +func (s *Middleware) CORS() func(http.Handler) http.Handler { + return cors.Handler(cors.Options{ + AllowedOrigins: []string{"*"}, + AllowedMethods: []string{ + "GET", "POST", "PUT", "DELETE", "OPTIONS", + }, + AllowedHeaders: []string{ + "Accept", + "Authorization", + "Content-Type", + "X-CSRF-Token", + }, + ExposedHeaders: []string{"Link"}, + AllowCredentials: false, + MaxAge: corsMaxAgeSec, + }) +} diff --git a/backend/internal/reportbuf/reportbuf.go b/backend/internal/reportbuf/reportbuf.go new file mode 100644 index 0000000..ff4cbad --- /dev/null +++ b/backend/internal/reportbuf/reportbuf.go @@ -0,0 +1,199 @@ +// Package reportbuf accumulates telemetry reports in memory +// and periodically flushes them to zstd-compressed JSONL files. +package reportbuf + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io/fs" + "log/slog" + "os" + "path/filepath" + "sync" + "time" + + "sneak.berlin/go/netwatch/internal/config" + "sneak.berlin/go/netwatch/internal/logger" + + "github.com/klauspost/compress/zstd" + "go.uber.org/fx" +) + +const ( + flushSizeThreshold = 10 << 20 // 10 MiB + flushInterval = 1 * time.Minute + defaultDataDir = "./data/reports" + dirPerms fs.FileMode = 0o750 + filePerms fs.FileMode = 0o640 +) + +// Params defines the dependencies for Buffer. +type Params struct { + fx.In + + Config *config.Config + Logger *logger.Logger +} + +// Buffer accumulates JSON lines in memory and flushes them +// to zstd-compressed files on disk. +type Buffer struct { + buf bytes.Buffer + dataDir string + done chan struct{} + log *slog.Logger + mu sync.Mutex +} + +// New creates a Buffer and registers lifecycle hooks to +// manage the data directory and flush goroutine. +func New( + lc fx.Lifecycle, + params Params, +) (*Buffer, error) { + dir := params.Config.DataDir + if dir == "" { + dir = defaultDataDir + } + + b := &Buffer{ + dataDir: dir, + done: make(chan struct{}), + log: params.Logger.Get(), + } + + lc.Append(fx.Hook{ + OnStart: func(_ context.Context) error { + err := os.MkdirAll(b.dataDir, dirPerms) + if err != nil { + return fmt.Errorf("create data dir: %w", err) + } + + go b.flushLoop() + + return nil + }, + OnStop: func(_ context.Context) error { + close(b.done) + b.flushLocked() + + return nil + }, + }) + + return b, nil +} + +// Append marshals v as a single JSON line and appends it to +// the buffer. If the buffer reaches the size threshold, it is +// drained and written to disk asynchronously. +func (b *Buffer) Append(v any) error { + line, err := json.Marshal(v) + if err != nil { + return fmt.Errorf("marshal report: %w", err) + } + + b.mu.Lock() + b.buf.Write(line) + b.buf.WriteByte('\n') + + if b.buf.Len() >= flushSizeThreshold { + data := b.drainBuf() + b.mu.Unlock() + + go b.writeFile(data) + + return nil + } + + b.mu.Unlock() + + return nil +} + +// flushLoop runs a ticker that periodically flushes buffered +// data to disk until the done channel is closed. +func (b *Buffer) flushLoop() { + ticker := time.NewTicker(flushInterval) + defer ticker.Stop() + + for { + select { + case <-ticker.C: + b.flushLocked() + case <-b.done: + return + } + } +} + +// flushLocked acquires the lock, drains the buffer, and +// writes the data to a compressed file. +func (b *Buffer) flushLocked() { + b.mu.Lock() + + if b.buf.Len() == 0 { + b.mu.Unlock() + + return + } + + data := b.drainBuf() + b.mu.Unlock() + + b.writeFile(data) +} + +// drainBuf copies the buffer contents and resets it. +// The caller must hold b.mu. +func (b *Buffer) drainBuf() []byte { + data := make([]byte, b.buf.Len()) + copy(data, b.buf.Bytes()) + b.buf.Reset() + + return data +} + +// writeFile creates a timestamped zstd-compressed JSONL file +// in the data directory. +func (b *Buffer) writeFile(data []byte) { + ts := time.Now().UTC().Format("2006-01-02T15-04-05.000Z") + name := fmt.Sprintf("reports-%s.jsonl.zst", ts) + path := filepath.Join(b.dataDir, name) + + f, err := os.OpenFile( //nolint:gosec // path built from controlled dataDir + timestamp + path, + os.O_WRONLY|os.O_CREATE|os.O_EXCL, + filePerms, + ) + if err != nil { + b.log.Error("create report file", "error", err) + + return + } + + defer func() { _ = f.Close() }() + + enc, err := zstd.NewWriter(f) + if err != nil { + b.log.Error("create zstd encoder", "error", err) + + return + } + + _, writeErr := enc.Write(data) + if writeErr != nil { + b.log.Error("write compressed data", "error", writeErr) + + _ = enc.Close() + + return + } + + closeErr := enc.Close() + if closeErr != nil { + b.log.Error("close zstd encoder", "error", closeErr) + } +} diff --git a/backend/internal/reportbuf/reportbuf_test.go b/backend/internal/reportbuf/reportbuf_test.go new file mode 100644 index 0000000..24a9ae9 --- /dev/null +++ b/backend/internal/reportbuf/reportbuf_test.go @@ -0,0 +1,13 @@ +package reportbuf_test + +import ( + "testing" + + _ "sneak.berlin/go/netwatch/internal/reportbuf" +) + +func TestImport(t *testing.T) { + t.Parallel() + // Compilation check — verifies the package parses + // and all imports resolve. +} diff --git a/backend/internal/server/http.go b/backend/internal/server/http.go new file mode 100644 index 0000000..d3824c6 --- /dev/null +++ b/backend/internal/server/http.go @@ -0,0 +1,43 @@ +package server + +import ( + "errors" + "fmt" + "net/http" + "time" +) + +const ( + readTimeout = 10 * time.Second + writeTimeout = 10 * time.Second + maxHeaderBytes = 1 << 20 // 1 MiB +) + +func (s *Server) serveUntilShutdown() { + listenAddr := fmt.Sprintf(":%d", s.params.Config.Port) + + s.httpServer = &http.Server{ + Addr: listenAddr, + Handler: s, + MaxHeaderBytes: maxHeaderBytes, + ReadTimeout: readTimeout, + WriteTimeout: writeTimeout, + } + + s.SetupRoutes() + + s.log.Info("http begin listen", + "listenaddr", listenAddr, + "version", s.params.Globals.Version, + "buildarch", s.params.Globals.Buildarch, + ) + + err := s.httpServer.ListenAndServe() + if err != nil && !errors.Is(err, http.ErrServerClosed) { + s.log.Error("listen error", "error", err) + + if s.cancelFunc != nil { + s.cancelFunc() + } + } +} diff --git a/backend/internal/server/routes.go b/backend/internal/server/routes.go new file mode 100644 index 0000000..da74cf6 --- /dev/null +++ b/backend/internal/server/routes.go @@ -0,0 +1,31 @@ +package server + +import ( + "time" + + "github.com/go-chi/chi/v5" + "github.com/go-chi/chi/v5/middleware" +) + +const requestTimeout = 60 * time.Second + +// SetupRoutes configures the chi router with middleware and +// all application routes. +func (s *Server) SetupRoutes() { + s.router = chi.NewRouter() + + s.router.Use(middleware.Recoverer) + s.router.Use(middleware.RequestID) + s.router.Use(s.mw.Logging()) + s.router.Use(s.mw.CORS()) + s.router.Use(middleware.Timeout(requestTimeout)) + + s.router.Get( + "/.well-known/healthcheck", + s.h.HandleHealthCheck(), + ) + + s.router.Route("/api/v1", func(r chi.Router) { + r.Post("/reports", s.h.HandleReport()) + }) +} diff --git a/backend/internal/server/server.go b/backend/internal/server/server.go new file mode 100644 index 0000000..a67bd1e --- /dev/null +++ b/backend/internal/server/server.go @@ -0,0 +1,147 @@ +// Package server provides the HTTP server lifecycle, +// including startup, routing, signal handling, and graceful +// shutdown. +package server + +import ( + "context" + "log/slog" + "net/http" + "os" + "os/signal" + "syscall" + "time" + + "sneak.berlin/go/netwatch/internal/config" + "sneak.berlin/go/netwatch/internal/globals" + "sneak.berlin/go/netwatch/internal/handlers" + "sneak.berlin/go/netwatch/internal/logger" + "sneak.berlin/go/netwatch/internal/middleware" + + "github.com/go-chi/chi/v5" + "go.uber.org/fx" +) + +// Params defines the dependencies for Server. +type Params struct { + fx.In + + Config *config.Config + Globals *globals.Globals + Handlers *handlers.Handlers + Logger *logger.Logger + Middleware *middleware.Middleware +} + +// Server is the top-level HTTP server orchestrator. +type Server struct { + cancelFunc context.CancelFunc + exitCode int + h *handlers.Handlers + httpServer *http.Server + log *slog.Logger + mw *middleware.Middleware + params Params + router *chi.Mux + startupTime time.Time +} + +// New creates a Server and registers lifecycle hooks for +// starting and stopping it. +func New( + lc fx.Lifecycle, + params Params, +) (*Server, error) { + s := new(Server) + s.params = params + s.mw = params.Middleware + s.h = params.Handlers + s.log = params.Logger.Get() + + lc.Append(fx.Hook{ + OnStart: func(_ context.Context) error { + s.startupTime = time.Now().UTC() + + go func() { //nolint:contextcheck // fx OnStart ctx is startup-only; run() creates its own + s.run() + }() + + return nil + }, + OnStop: func(_ context.Context) error { + if s.cancelFunc != nil { + s.cancelFunc() + } + + return nil + }, + }) + + return s, nil +} + +// ServeHTTP delegates to the chi router. +func (s *Server) ServeHTTP( + w http.ResponseWriter, + r *http.Request, +) { + s.router.ServeHTTP(w, r) +} + +func (s *Server) run() { + exitCode := s.serve() + os.Exit(exitCode) +} + +func (s *Server) serve() int { + var ctx context.Context //nolint:wsl // ctx must be declared before multi-assign + + ctx, s.cancelFunc = context.WithCancel( + context.Background(), + ) + + go func() { + c := make(chan os.Signal, 1) + + signal.Ignore(syscall.SIGPIPE) + signal.Notify(c, os.Interrupt, syscall.SIGTERM) + + sig := <-c + s.log.Info("signal received", "signal", sig) + + if s.cancelFunc != nil { + s.cancelFunc() + } + }() + + go func() { + s.serveUntilShutdown() + }() + + <-ctx.Done() + s.cleanShutdown() + + return s.exitCode +} + +const shutdownTimeout = 5 * time.Second + +func (s *Server) cleanShutdown() { + s.exitCode = 0 + + ctxShutdown, shutdownCancel := context.WithTimeout( + context.Background(), + shutdownTimeout, + ) + defer shutdownCancel() + + err := s.httpServer.Shutdown(ctxShutdown) + if err != nil { + s.log.Error( + "server clean shutdown failed", + "error", err, + ) + } + + s.log.Info("server stopped") +} From 4ad257353297233e8d35f2f767aafa58112c9369 Mon Sep 17 00:00:00 2001 From: sneak Date: Fri, 27 Feb 2026 12:18:35 +0700 Subject: [PATCH 2/4] Add CI workflow and backend repo standard files Add .gitea/workflows/check.yml that builds both the root and backend Docker images on push. Add LICENSE and README.md to the backend subproject to match repo standards. --- .gitea/workflows/check.yml | 10 ++++++ backend/LICENSE | 21 ++++++++++++ backend/README.md | 70 ++++++++++++++++++++++++++++++++++++++ 3 files changed, 101 insertions(+) create mode 100644 .gitea/workflows/check.yml create mode 100644 backend/LICENSE create mode 100644 backend/README.md diff --git a/.gitea/workflows/check.yml b/.gitea/workflows/check.yml new file mode 100644 index 0000000..e545bef --- /dev/null +++ b/.gitea/workflows/check.yml @@ -0,0 +1,10 @@ +name: check +on: [push] +jobs: + check: + runs-on: ubuntu-latest + steps: + # actions/checkout v4.2.2, 2026-02-22 + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 + - run: docker build . + - run: docker build backend/ diff --git a/backend/LICENSE b/backend/LICENSE new file mode 100644 index 0000000..5b83a52 --- /dev/null +++ b/backend/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2026 @sneak (https://sneak.berlin) + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/backend/README.md b/backend/README.md new file mode 100644 index 0000000..a7de988 --- /dev/null +++ b/backend/README.md @@ -0,0 +1,70 @@ +netwatch-server is an MIT-licensed Go HTTP backend by +[@sneak](https://sneak.berlin) that receives telemetry reports from the NetWatch +SPA and persists them as zstd-compressed JSONL files on disk. + +## Getting Started + +```bash +# Build and run locally +make run + +# Run tests, lint, and format check +make check + +# Docker +docker build -t netwatch-server . +docker run -p 8080:8080 netwatch-server +``` + +## Rationale + +The NetWatch frontend collects latency measurements from the browser but has no +way to persist or aggregate them. This backend provides a minimal +`POST /api/v1/reports` endpoint that buffers incoming reports in memory and +flushes them to compressed files on disk for later analysis. + +## Design + +The server is structured as an `fx`-wired Go application under `cmd/netwatch-server/`. +Internal packages in `internal/` follow standard Go project layout: + +- **`config`**: Loads configuration from environment variables and config files + via Viper. +- **`handlers`**: HTTP request handlers for the API (health check, report + ingestion). +- **`reportbuf`**: In-memory buffer that accumulates JSONL report lines and + flushes to zstd-compressed files when the buffer reaches 10 MiB or every 60 + seconds. +- **`server`**: Chi-based HTTP server with middleware wiring and route + registration. +- **`healthcheck`**, **`middleware`**, **`logger`**, **`globals`**: Supporting + infrastructure. + +### Configuration + +| Variable | Default | Description | +| ---------- | ------------------ | --------------------------------- | +| `PORT` | `8080` | HTTP listen port | +| `DATA_DIR` | `./data/reports` | Directory for compressed reports | +| `DEBUG` | `false` | Enable debug logging | + +### Report storage + +Reports are written as `reports-.jsonl.zst` files in `DATA_DIR`. +Each file contains one JSON object per line, compressed with zstd. Files are +created with `O_EXCL` to prevent overwrites. + +## TODO + +- Add integration test that POSTs a report and verifies the compressed output +- Add report decompression/query endpoint +- Add metrics (Prometheus) for buffer size, flush count, report count +- Add retention policy to prune old report files + +## License + +MIT. See [LICENSE](LICENSE). + +## Author + +[@sneak](https://sneak.berlin) From c61c047cd81e5b79facc014cc2df8477661b227c Mon Sep 17 00:00:00 2001 From: sneak Date: Fri, 27 Feb 2026 12:24:49 +0700 Subject: [PATCH 3/4] Fix backend Dockerfile: use Go 1.25, install golangci-lint Update base image from golang:1.24-alpine to golang:1.25-alpine to match go.mod requirement. Install golangci-lint by pinned commit hash so make check passes inside the container. Update runtime image to alpine:3.23. --- backend/Dockerfile | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/backend/Dockerfile b/backend/Dockerfile index 5839cb1..49c87aa 100644 --- a/backend/Dockerfile +++ b/backend/Dockerfile @@ -1,8 +1,11 @@ -# golang:1.24-alpine (2026-02-26) -FROM golang:1.24-alpine@sha256:8bee1901f1e530bfb4a7850aa7a479d17ae3a18beb6e09064ed54cfd245b7191 AS builder +# golang:1.25-alpine (2026-02-27) +FROM golang:1.25-alpine@sha256:f6751d823c26342f9506c03797d2527668d095b0a15f1862cddb4d927a7a4ced AS builder RUN apk add --no-cache git make gcc musl-dev +# golangci-lint v2.7.2 (2026-02-27) +RUN CGO_ENABLED=0 go install github.com/golangci/golangci-lint/v2/cmd/golangci-lint@9f61b0f53f80672872fced07b6874397c3ed197b + WORKDIR /src COPY go.mod go.sum ./ RUN go mod download @@ -11,8 +14,8 @@ COPY . . RUN make check RUN make build -# alpine:3.21 (2026-02-26) -FROM alpine:3.21@sha256:c3f8e73fdb79deaebaa2037150150191b9dcbfba68b4a46d70103204c53f4709 +# alpine:3.23 (2026-02-27) +FROM alpine:3.23@sha256:25109184c71bdad752c8312a8623239686a9a2071e8825f20acb8f2198c3f659 RUN apk add --no-cache ca-certificates COPY --from=builder /src/netwatch-server /usr/local/bin/netwatch-server From 8ca57746df4273407488ee75ea109d9b37f8dfec Mon Sep 17 00:00:00 2001 From: sneak Date: Fri, 27 Feb 2026 12:45:38 +0700 Subject: [PATCH 4/4] Move backend Dockerfile to repo root for git access Place the backend Dockerfile at repo root as Dockerfile.backend so the build context includes .git, giving git describe access for version stamping. Fix .gitignore pattern to anchor /netwatch-server so it does not exclude cmd/netwatch-server/. Remove .git from .dockerignore. Update CI workflow and backend Makefile docker target. --- .gitea/workflows/check.yml | 2 +- backend/Dockerfile => Dockerfile.backend | 10 +++--- backend/.dockerignore | 1 - backend/.gitignore | 2 +- backend/Makefile | 2 +- backend/cmd/netwatch-server/main.go | 42 ++++++++++++++++++++++++ 6 files changed, 50 insertions(+), 9 deletions(-) rename backend/Dockerfile => Dockerfile.backend (77%) create mode 100644 backend/cmd/netwatch-server/main.go diff --git a/.gitea/workflows/check.yml b/.gitea/workflows/check.yml index e545bef..81ac7d9 100644 --- a/.gitea/workflows/check.yml +++ b/.gitea/workflows/check.yml @@ -7,4 +7,4 @@ jobs: # actions/checkout v4.2.2, 2026-02-22 - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 - run: docker build . - - run: docker build backend/ + - run: docker build -f Dockerfile.backend . diff --git a/backend/Dockerfile b/Dockerfile.backend similarity index 77% rename from backend/Dockerfile rename to Dockerfile.backend index 49c87aa..d97bbbd 100644 --- a/backend/Dockerfile +++ b/Dockerfile.backend @@ -6,10 +6,11 @@ RUN apk add --no-cache git make gcc musl-dev # golangci-lint v2.7.2 (2026-02-27) RUN CGO_ENABLED=0 go install github.com/golangci/golangci-lint/v2/cmd/golangci-lint@9f61b0f53f80672872fced07b6874397c3ed197b -WORKDIR /src -COPY go.mod go.sum ./ +WORKDIR /repo/backend +COPY backend/go.mod backend/go.sum ./ RUN go mod download -COPY . . +COPY .git /repo/.git +COPY backend/ . RUN make check RUN make build @@ -18,8 +19,7 @@ RUN make build FROM alpine:3.23@sha256:25109184c71bdad752c8312a8623239686a9a2071e8825f20acb8f2198c3f659 RUN apk add --no-cache ca-certificates -COPY --from=builder /src/netwatch-server /usr/local/bin/netwatch-server -COPY --from=builder /src /src +COPY --from=builder /repo/backend/netwatch-server /usr/local/bin/netwatch-server EXPOSE 8080 ENTRYPOINT ["netwatch-server"] diff --git a/backend/.dockerignore b/backend/.dockerignore index 5414d56..fd4f2b0 100644 --- a/backend/.dockerignore +++ b/backend/.dockerignore @@ -1,3 +1,2 @@ -.git node_modules .DS_Store diff --git a/backend/.gitignore b/backend/.gitignore index 861bd94..e48380c 100644 --- a/backend/.gitignore +++ b/backend/.gitignore @@ -1,4 +1,4 @@ -netwatch-server +/netwatch-server *.log *.out *.test diff --git a/backend/Makefile b/backend/Makefile index ad86aa0..38cebcf 100644 --- a/backend/Makefile +++ b/backend/Makefile @@ -37,7 +37,7 @@ fmt-check: check: test lint fmt-check docker: - timeout 300 docker build -t netwatch-server . + timeout 300 docker build -t netwatch-server -f ../Dockerfile.backend .. hooks: @printf '#!/bin/sh\ncd backend && make check\n' > \ diff --git a/backend/cmd/netwatch-server/main.go b/backend/cmd/netwatch-server/main.go new file mode 100644 index 0000000..3f3d86b --- /dev/null +++ b/backend/cmd/netwatch-server/main.go @@ -0,0 +1,42 @@ +// Package main is the entry point for netwatch-server. +package main + +import ( + "sneak.berlin/go/netwatch/internal/config" + "sneak.berlin/go/netwatch/internal/globals" + "sneak.berlin/go/netwatch/internal/handlers" + "sneak.berlin/go/netwatch/internal/healthcheck" + "sneak.berlin/go/netwatch/internal/logger" + "sneak.berlin/go/netwatch/internal/middleware" + "sneak.berlin/go/netwatch/internal/reportbuf" + "sneak.berlin/go/netwatch/internal/server" + + "go.uber.org/fx" +) + +//nolint:gochecknoglobals // set via ldflags at build time +var ( + Appname = "netwatch-server" + Version string + Buildarch string +) + +func main() { + globals.Appname = Appname + globals.Version = Version + globals.Buildarch = Buildarch + + fx.New( + fx.Provide( + config.New, + globals.New, + handlers.New, + healthcheck.New, + logger.New, + middleware.New, + reportbuf.New, + server.New, + ), + fx.Invoke(func(*server.Server) {}), + ).Run() +}