Add backend with buffered zstd-compressed report storage

Introduce the Go backend (netwatch-server) with an HTTP API that
accepts telemetry reports and persists them as zstd-compressed JSONL
files. Reports are buffered in memory and flushed to disk when the
buffer reaches 10 MiB or every 60 seconds.
This commit is contained in:
2026-02-27 12:14:34 +07:00
parent 4ad98d578b
commit b57afeddbd
23 changed files with 1250 additions and 0 deletions

View File

@@ -0,0 +1,86 @@
// Package config loads application configuration from
// environment variables, .env files, and config files.
package config
import (
"errors"
"log/slog"
"sneak.berlin/go/netwatch/internal/globals"
"sneak.berlin/go/netwatch/internal/logger"
_ "github.com/joho/godotenv/autoload" // loads .env file
"github.com/spf13/viper"
"go.uber.org/fx"
)
// Params defines the dependencies for Config.
type Params struct {
fx.In
Globals *globals.Globals
Logger *logger.Logger
}
// Config holds the resolved application configuration.
type Config struct {
DataDir string
Debug bool
MetricsPassword string
MetricsUsername string
Port int
SentryDSN string
log *slog.Logger
params *Params
}
// New loads configuration from env, .env files, and config
// files, returning a fully resolved Config.
func New(
_ fx.Lifecycle,
params Params,
) (*Config, error) {
log := params.Logger.Get()
name := params.Globals.Appname
viper.SetConfigName(name)
viper.SetConfigType("yaml")
viper.AddConfigPath("/etc/" + name)
viper.AddConfigPath("$HOME/.config/" + name)
viper.AutomaticEnv()
viper.SetDefault("DATA_DIR", "./data/reports")
viper.SetDefault("DEBUG", "false")
viper.SetDefault("PORT", "8080")
viper.SetDefault("SENTRY_DSN", "")
viper.SetDefault("METRICS_USERNAME", "")
viper.SetDefault("METRICS_PASSWORD", "")
err := viper.ReadInConfig()
if err != nil {
var notFound viper.ConfigFileNotFoundError
if !errors.As(err, &notFound) {
log.Error("config file malformed", "error", err)
panic(err)
}
}
s := &Config{
DataDir: viper.GetString("DATA_DIR"),
Debug: viper.GetBool("DEBUG"),
MetricsPassword: viper.GetString("METRICS_PASSWORD"),
MetricsUsername: viper.GetString("METRICS_USERNAME"),
Port: viper.GetInt("PORT"),
SentryDSN: viper.GetString("SENTRY_DSN"),
log: log,
params: &params,
}
if s.Debug {
params.Logger.EnableDebugLogging()
s.log = params.Logger.Get()
}
return s, nil
}

View File

@@ -0,0 +1,31 @@
// Package globals provides build-time variables injected via
// ldflags and made available through dependency injection.
package globals
import "go.uber.org/fx"
//nolint:gochecknoglobals // set from main before fx starts
var (
// Appname is the application name.
Appname string
// Version is the git version tag.
Version string
// Buildarch is the build architecture.
Buildarch string
)
// Globals holds build-time metadata for the application.
type Globals struct {
Appname string
Version string
Buildarch string
}
// New creates a Globals instance from package-level variables.
func New(_ fx.Lifecycle) (*Globals, error) {
return &Globals{
Appname: Appname,
Buildarch: Buildarch,
Version: Version,
}, nil
}

View File

@@ -0,0 +1,74 @@
// Package handlers implements HTTP request handlers for the
// netwatch-server API.
package handlers
import (
"context"
"encoding/json"
"log/slog"
"net/http"
"sneak.berlin/go/netwatch/internal/globals"
"sneak.berlin/go/netwatch/internal/healthcheck"
"sneak.berlin/go/netwatch/internal/logger"
"sneak.berlin/go/netwatch/internal/reportbuf"
"go.uber.org/fx"
)
const jsonContentType = "application/json; charset=utf-8"
// Params defines the dependencies for Handlers.
type Params struct {
fx.In
Buffer *reportbuf.Buffer
Globals *globals.Globals
Healthcheck *healthcheck.Healthcheck
Logger *logger.Logger
}
// Handlers provides HTTP handler factories for all endpoints.
type Handlers struct {
buf *reportbuf.Buffer
hc *healthcheck.Healthcheck
log *slog.Logger
params *Params
}
// New creates a Handlers instance.
func New(
lc fx.Lifecycle,
params Params,
) (*Handlers, error) {
s := new(Handlers)
s.buf = params.Buffer
s.params = &params
s.log = params.Logger.Get()
s.hc = params.Healthcheck
lc.Append(fx.Hook{
OnStart: func(_ context.Context) error {
return nil
},
})
return s, nil
}
func (s *Handlers) respondJSON(
w http.ResponseWriter,
_ *http.Request,
data any,
status int,
) {
w.Header().Set("Content-Type", jsonContentType)
w.WriteHeader(status)
if data != nil {
err := json.NewEncoder(w).Encode(data)
if err != nil {
s.log.Error("json encode error", "error", err)
}
}
}

View File

@@ -0,0 +1,13 @@
package handlers_test
import (
"testing"
_ "sneak.berlin/go/netwatch/internal/handlers"
)
func TestImport(t *testing.T) {
t.Parallel()
// Compilation check — verifies the package parses
// and all imports resolve.
}

View File

@@ -0,0 +1,11 @@
package handlers
import "net/http"
// HandleHealthCheck returns a handler for the health check
// endpoint.
func (s *Handlers) HandleHealthCheck() http.HandlerFunc {
return func(w http.ResponseWriter, r *http.Request) {
s.respondJSON(w, r, s.hc.Check(), http.StatusOK)
}
}

View File

@@ -0,0 +1,82 @@
package handlers
import (
"encoding/json"
"net/http"
)
const maxReportBodyBytes = 1 << 20 // 1 MiB
type reportSample struct {
T int64 `json:"t"`
Latency *int `json:"latency"`
Error *string `json:"error"`
}
type reportHost struct {
History []reportSample `json:"history"`
Name string `json:"name"`
Status string `json:"status"`
URL string `json:"url"`
}
type report struct {
ClientID string `json:"clientId"`
Geo json.RawMessage `json:"geo"`
Hosts []reportHost `json:"hosts"`
Timestamp string `json:"timestamp"`
}
// HandleReport returns a handler that accepts telemetry
// reports from NetWatch clients.
func (s *Handlers) HandleReport() http.HandlerFunc {
type response struct {
Status string `json:"status"`
}
return func(w http.ResponseWriter, r *http.Request) {
r.Body = http.MaxBytesReader(
w, r.Body, maxReportBodyBytes,
)
var rpt report
err := json.NewDecoder(r.Body).Decode(&rpt)
if err != nil {
s.log.Error("failed to decode report",
"error", err,
)
s.respondJSON(w, r,
&response{Status: "error"},
http.StatusBadRequest,
)
return
}
totalSamples := 0
for _, h := range rpt.Hosts {
totalSamples += len(h.History)
}
s.log.Info("report received",
"client_id", rpt.ClientID,
"timestamp", rpt.Timestamp,
"host_count", len(rpt.Hosts),
"total_samples", totalSamples,
"geo", string(rpt.Geo),
)
bufErr := s.buf.Append(rpt)
if bufErr != nil {
s.log.Error("failed to buffer report",
"error", bufErr,
)
}
s.respondJSON(w, r,
&response{Status: "ok"},
http.StatusOK,
)
}
}

View File

@@ -0,0 +1,82 @@
// Package healthcheck provides a service that reports
// application health, uptime, and version information.
package healthcheck
import (
"context"
"log/slog"
"time"
"sneak.berlin/go/netwatch/internal/config"
"sneak.berlin/go/netwatch/internal/globals"
"sneak.berlin/go/netwatch/internal/logger"
"go.uber.org/fx"
)
// Params defines the dependencies for Healthcheck.
type Params struct {
fx.In
Config *config.Config
Globals *globals.Globals
Logger *logger.Logger
}
// Healthcheck tracks startup time and builds health responses.
type Healthcheck struct {
StartupTime time.Time
log *slog.Logger
params *Params
}
// Response is the JSON payload returned by the health check
// endpoint.
type Response struct {
Appname string `json:"appname"`
Now string `json:"now"`
Status string `json:"status"`
UptimeHuman string `json:"uptimeHuman"`
UptimeSeconds int64 `json:"uptimeSeconds"`
Version string `json:"version"`
}
// New creates a Healthcheck, recording startup time via an
// fx lifecycle hook.
func New(
lc fx.Lifecycle,
params Params,
) (*Healthcheck, error) {
s := new(Healthcheck)
s.params = &params
s.log = params.Logger.Get()
lc.Append(fx.Hook{
OnStart: func(_ context.Context) error {
s.StartupTime = time.Now().UTC()
return nil
},
OnStop: func(_ context.Context) error {
return nil
},
})
return s, nil
}
// Check returns the current health status of the application.
func (s *Healthcheck) Check() *Response {
return &Response{
Appname: s.params.Globals.Appname,
Now: time.Now().UTC().Format(time.RFC3339Nano),
Status: "ok",
UptimeHuman: s.uptime().String(),
UptimeSeconds: int64(s.uptime().Seconds()),
Version: s.params.Globals.Version,
}
}
func (s *Healthcheck) uptime() time.Duration {
return time.Since(s.StartupTime)
}

View File

@@ -0,0 +1,85 @@
// Package logger provides a configured slog.Logger with TTY
// detection for development vs production output.
package logger
import (
"log/slog"
"os"
"sneak.berlin/go/netwatch/internal/globals"
"go.uber.org/fx"
)
// Params defines the dependencies for Logger.
type Params struct {
fx.In
Globals *globals.Globals
}
// Logger wraps slog.Logger with dynamic level control.
type Logger struct {
log *slog.Logger
level *slog.LevelVar
params Params
}
// New creates a Logger with TTY-aware output formatting.
func New(_ fx.Lifecycle, params Params) (*Logger, error) {
l := new(Logger)
l.level = new(slog.LevelVar)
l.level.Set(slog.LevelInfo)
l.params = params
tty := false
if fileInfo, _ := os.Stdout.Stat(); fileInfo != nil {
if (fileInfo.Mode() & os.ModeCharDevice) != 0 {
tty = true
}
}
var handler slog.Handler
if tty {
handler = slog.NewTextHandler(
os.Stdout,
&slog.HandlerOptions{
Level: l.level,
AddSource: true,
},
)
} else {
handler = slog.NewJSONHandler(
os.Stdout,
&slog.HandlerOptions{
Level: l.level,
AddSource: true,
},
)
}
l.log = slog.New(handler)
return l, nil
}
// EnableDebugLogging sets the log level to debug.
func (l *Logger) EnableDebugLogging() {
l.level.Set(slog.LevelDebug)
l.log.Debug("debug logging enabled", "debug", true)
}
// Get returns the underlying slog.Logger.
func (l *Logger) Get() *slog.Logger {
return l.log
}
// Identify logs the application's build-time metadata.
func (l *Logger) Identify() {
l.log.Info("starting",
"appname", l.params.Globals.Appname,
"version", l.params.Globals.Version,
"buildarch", l.params.Globals.Buildarch,
)
}

View File

@@ -0,0 +1,129 @@
// Package middleware provides HTTP middleware for logging,
// CORS, and other cross-cutting concerns.
package middleware
import (
"log/slog"
"net"
"net/http"
"time"
"sneak.berlin/go/netwatch/internal/config"
"sneak.berlin/go/netwatch/internal/globals"
"sneak.berlin/go/netwatch/internal/logger"
"github.com/go-chi/chi/v5/middleware"
"github.com/go-chi/cors"
"go.uber.org/fx"
)
const corsMaxAgeSec = 300
// Params defines the dependencies for Middleware.
type Params struct {
fx.In
Config *config.Config
Globals *globals.Globals
Logger *logger.Logger
}
// Middleware holds shared state for middleware factories.
type Middleware struct {
log *slog.Logger
params *Params
}
// New creates a Middleware instance.
func New(
_ fx.Lifecycle,
params Params,
) (*Middleware, error) {
s := new(Middleware)
s.params = &params
s.log = params.Logger.Get()
return s, nil
}
type loggingResponseWriter struct {
http.ResponseWriter
statusCode int
}
func newLoggingResponseWriter(
w http.ResponseWriter,
) *loggingResponseWriter {
return &loggingResponseWriter{w, http.StatusOK}
}
func (lrw *loggingResponseWriter) WriteHeader(code int) {
lrw.statusCode = code
lrw.ResponseWriter.WriteHeader(code)
}
func ipFromHostPort(hostPort string) string {
host, _, err := net.SplitHostPort(hostPort)
if err != nil {
return hostPort
}
return host
}
// Logging returns middleware that logs each request with
// timing, status code, and client information.
func (s *Middleware) Logging() func(http.Handler) http.Handler {
return func(next http.Handler) http.Handler {
return http.HandlerFunc(
func(w http.ResponseWriter, r *http.Request) {
start := time.Now().UTC()
lrw := newLoggingResponseWriter(w)
ctx := r.Context()
defer func() {
latency := time.Since(start)
s.log.InfoContext(ctx, "request",
"request_start", start,
"method", r.Method,
"url", r.URL.String(),
"useragent", r.UserAgent(),
"request_id",
ctx.Value(
middleware.RequestIDKey,
),
"referer", r.Referer(),
"proto", r.Proto,
"remote_ip",
ipFromHostPort(r.RemoteAddr),
"status", lrw.statusCode,
"latency_ms",
latency.Milliseconds(),
)
}()
next.ServeHTTP(lrw, r)
},
)
}
}
// CORS returns middleware that adds permissive CORS headers.
func (s *Middleware) CORS() func(http.Handler) http.Handler {
return cors.Handler(cors.Options{
AllowedOrigins: []string{"*"},
AllowedMethods: []string{
"GET", "POST", "PUT", "DELETE", "OPTIONS",
},
AllowedHeaders: []string{
"Accept",
"Authorization",
"Content-Type",
"X-CSRF-Token",
},
ExposedHeaders: []string{"Link"},
AllowCredentials: false,
MaxAge: corsMaxAgeSec,
})
}

View File

@@ -0,0 +1,199 @@
// Package reportbuf accumulates telemetry reports in memory
// and periodically flushes them to zstd-compressed JSONL files.
package reportbuf
import (
"bytes"
"context"
"encoding/json"
"fmt"
"io/fs"
"log/slog"
"os"
"path/filepath"
"sync"
"time"
"sneak.berlin/go/netwatch/internal/config"
"sneak.berlin/go/netwatch/internal/logger"
"github.com/klauspost/compress/zstd"
"go.uber.org/fx"
)
const (
flushSizeThreshold = 10 << 20 // 10 MiB
flushInterval = 1 * time.Minute
defaultDataDir = "./data/reports"
dirPerms fs.FileMode = 0o750
filePerms fs.FileMode = 0o640
)
// Params defines the dependencies for Buffer.
type Params struct {
fx.In
Config *config.Config
Logger *logger.Logger
}
// Buffer accumulates JSON lines in memory and flushes them
// to zstd-compressed files on disk.
type Buffer struct {
buf bytes.Buffer
dataDir string
done chan struct{}
log *slog.Logger
mu sync.Mutex
}
// New creates a Buffer and registers lifecycle hooks to
// manage the data directory and flush goroutine.
func New(
lc fx.Lifecycle,
params Params,
) (*Buffer, error) {
dir := params.Config.DataDir
if dir == "" {
dir = defaultDataDir
}
b := &Buffer{
dataDir: dir,
done: make(chan struct{}),
log: params.Logger.Get(),
}
lc.Append(fx.Hook{
OnStart: func(_ context.Context) error {
err := os.MkdirAll(b.dataDir, dirPerms)
if err != nil {
return fmt.Errorf("create data dir: %w", err)
}
go b.flushLoop()
return nil
},
OnStop: func(_ context.Context) error {
close(b.done)
b.flushLocked()
return nil
},
})
return b, nil
}
// Append marshals v as a single JSON line and appends it to
// the buffer. If the buffer reaches the size threshold, it is
// drained and written to disk asynchronously.
func (b *Buffer) Append(v any) error {
line, err := json.Marshal(v)
if err != nil {
return fmt.Errorf("marshal report: %w", err)
}
b.mu.Lock()
b.buf.Write(line)
b.buf.WriteByte('\n')
if b.buf.Len() >= flushSizeThreshold {
data := b.drainBuf()
b.mu.Unlock()
go b.writeFile(data)
return nil
}
b.mu.Unlock()
return nil
}
// flushLoop runs a ticker that periodically flushes buffered
// data to disk until the done channel is closed.
func (b *Buffer) flushLoop() {
ticker := time.NewTicker(flushInterval)
defer ticker.Stop()
for {
select {
case <-ticker.C:
b.flushLocked()
case <-b.done:
return
}
}
}
// flushLocked acquires the lock, drains the buffer, and
// writes the data to a compressed file.
func (b *Buffer) flushLocked() {
b.mu.Lock()
if b.buf.Len() == 0 {
b.mu.Unlock()
return
}
data := b.drainBuf()
b.mu.Unlock()
b.writeFile(data)
}
// drainBuf copies the buffer contents and resets it.
// The caller must hold b.mu.
func (b *Buffer) drainBuf() []byte {
data := make([]byte, b.buf.Len())
copy(data, b.buf.Bytes())
b.buf.Reset()
return data
}
// writeFile creates a timestamped zstd-compressed JSONL file
// in the data directory.
func (b *Buffer) writeFile(data []byte) {
ts := time.Now().UTC().Format("2006-01-02T15-04-05.000Z")
name := fmt.Sprintf("reports-%s.jsonl.zst", ts)
path := filepath.Join(b.dataDir, name)
f, err := os.OpenFile( //nolint:gosec // path built from controlled dataDir + timestamp
path,
os.O_WRONLY|os.O_CREATE|os.O_EXCL,
filePerms,
)
if err != nil {
b.log.Error("create report file", "error", err)
return
}
defer func() { _ = f.Close() }()
enc, err := zstd.NewWriter(f)
if err != nil {
b.log.Error("create zstd encoder", "error", err)
return
}
_, writeErr := enc.Write(data)
if writeErr != nil {
b.log.Error("write compressed data", "error", writeErr)
_ = enc.Close()
return
}
closeErr := enc.Close()
if closeErr != nil {
b.log.Error("close zstd encoder", "error", closeErr)
}
}

View File

@@ -0,0 +1,13 @@
package reportbuf_test
import (
"testing"
_ "sneak.berlin/go/netwatch/internal/reportbuf"
)
func TestImport(t *testing.T) {
t.Parallel()
// Compilation check — verifies the package parses
// and all imports resolve.
}

View File

@@ -0,0 +1,43 @@
package server
import (
"errors"
"fmt"
"net/http"
"time"
)
const (
readTimeout = 10 * time.Second
writeTimeout = 10 * time.Second
maxHeaderBytes = 1 << 20 // 1 MiB
)
func (s *Server) serveUntilShutdown() {
listenAddr := fmt.Sprintf(":%d", s.params.Config.Port)
s.httpServer = &http.Server{
Addr: listenAddr,
Handler: s,
MaxHeaderBytes: maxHeaderBytes,
ReadTimeout: readTimeout,
WriteTimeout: writeTimeout,
}
s.SetupRoutes()
s.log.Info("http begin listen",
"listenaddr", listenAddr,
"version", s.params.Globals.Version,
"buildarch", s.params.Globals.Buildarch,
)
err := s.httpServer.ListenAndServe()
if err != nil && !errors.Is(err, http.ErrServerClosed) {
s.log.Error("listen error", "error", err)
if s.cancelFunc != nil {
s.cancelFunc()
}
}
}

View File

@@ -0,0 +1,31 @@
package server
import (
"time"
"github.com/go-chi/chi/v5"
"github.com/go-chi/chi/v5/middleware"
)
const requestTimeout = 60 * time.Second
// SetupRoutes configures the chi router with middleware and
// all application routes.
func (s *Server) SetupRoutes() {
s.router = chi.NewRouter()
s.router.Use(middleware.Recoverer)
s.router.Use(middleware.RequestID)
s.router.Use(s.mw.Logging())
s.router.Use(s.mw.CORS())
s.router.Use(middleware.Timeout(requestTimeout))
s.router.Get(
"/.well-known/healthcheck",
s.h.HandleHealthCheck(),
)
s.router.Route("/api/v1", func(r chi.Router) {
r.Post("/reports", s.h.HandleReport())
})
}

View File

@@ -0,0 +1,147 @@
// Package server provides the HTTP server lifecycle,
// including startup, routing, signal handling, and graceful
// shutdown.
package server
import (
"context"
"log/slog"
"net/http"
"os"
"os/signal"
"syscall"
"time"
"sneak.berlin/go/netwatch/internal/config"
"sneak.berlin/go/netwatch/internal/globals"
"sneak.berlin/go/netwatch/internal/handlers"
"sneak.berlin/go/netwatch/internal/logger"
"sneak.berlin/go/netwatch/internal/middleware"
"github.com/go-chi/chi/v5"
"go.uber.org/fx"
)
// Params defines the dependencies for Server.
type Params struct {
fx.In
Config *config.Config
Globals *globals.Globals
Handlers *handlers.Handlers
Logger *logger.Logger
Middleware *middleware.Middleware
}
// Server is the top-level HTTP server orchestrator.
type Server struct {
cancelFunc context.CancelFunc
exitCode int
h *handlers.Handlers
httpServer *http.Server
log *slog.Logger
mw *middleware.Middleware
params Params
router *chi.Mux
startupTime time.Time
}
// New creates a Server and registers lifecycle hooks for
// starting and stopping it.
func New(
lc fx.Lifecycle,
params Params,
) (*Server, error) {
s := new(Server)
s.params = params
s.mw = params.Middleware
s.h = params.Handlers
s.log = params.Logger.Get()
lc.Append(fx.Hook{
OnStart: func(_ context.Context) error {
s.startupTime = time.Now().UTC()
go func() { //nolint:contextcheck // fx OnStart ctx is startup-only; run() creates its own
s.run()
}()
return nil
},
OnStop: func(_ context.Context) error {
if s.cancelFunc != nil {
s.cancelFunc()
}
return nil
},
})
return s, nil
}
// ServeHTTP delegates to the chi router.
func (s *Server) ServeHTTP(
w http.ResponseWriter,
r *http.Request,
) {
s.router.ServeHTTP(w, r)
}
func (s *Server) run() {
exitCode := s.serve()
os.Exit(exitCode)
}
func (s *Server) serve() int {
var ctx context.Context //nolint:wsl // ctx must be declared before multi-assign
ctx, s.cancelFunc = context.WithCancel(
context.Background(),
)
go func() {
c := make(chan os.Signal, 1)
signal.Ignore(syscall.SIGPIPE)
signal.Notify(c, os.Interrupt, syscall.SIGTERM)
sig := <-c
s.log.Info("signal received", "signal", sig)
if s.cancelFunc != nil {
s.cancelFunc()
}
}()
go func() {
s.serveUntilShutdown()
}()
<-ctx.Done()
s.cleanShutdown()
return s.exitCode
}
const shutdownTimeout = 5 * time.Second
func (s *Server) cleanShutdown() {
s.exitCode = 0
ctxShutdown, shutdownCancel := context.WithTimeout(
context.Background(),
shutdownTimeout,
)
defer shutdownCancel()
err := s.httpServer.Shutdown(ctxShutdown)
if err != nil {
s.log.Error(
"server clean shutdown failed",
"error", err,
)
}
s.log.Info("server stopped")
}