Add backend with buffered zstd-compressed report storage
Introduce the Go backend (netwatch-server) with an HTTP API that accepts telemetry reports and persists them as zstd-compressed JSONL files. Reports are buffered in memory and flushed to disk when the buffer reaches 10 MiB or every 60 seconds.
This commit is contained in:
86
backend/internal/config/config.go
Normal file
86
backend/internal/config/config.go
Normal file
@@ -0,0 +1,86 @@
|
||||
// Package config loads application configuration from
|
||||
// environment variables, .env files, and config files.
|
||||
package config
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"log/slog"
|
||||
|
||||
"sneak.berlin/go/netwatch/internal/globals"
|
||||
"sneak.berlin/go/netwatch/internal/logger"
|
||||
|
||||
_ "github.com/joho/godotenv/autoload" // loads .env file
|
||||
"github.com/spf13/viper"
|
||||
"go.uber.org/fx"
|
||||
)
|
||||
|
||||
// Params defines the dependencies for Config.
|
||||
type Params struct {
|
||||
fx.In
|
||||
|
||||
Globals *globals.Globals
|
||||
Logger *logger.Logger
|
||||
}
|
||||
|
||||
// Config holds the resolved application configuration.
|
||||
type Config struct {
|
||||
DataDir string
|
||||
Debug bool
|
||||
MetricsPassword string
|
||||
MetricsUsername string
|
||||
Port int
|
||||
SentryDSN string
|
||||
log *slog.Logger
|
||||
params *Params
|
||||
}
|
||||
|
||||
// New loads configuration from env, .env files, and config
|
||||
// files, returning a fully resolved Config.
|
||||
func New(
|
||||
_ fx.Lifecycle,
|
||||
params Params,
|
||||
) (*Config, error) {
|
||||
log := params.Logger.Get()
|
||||
name := params.Globals.Appname
|
||||
|
||||
viper.SetConfigName(name)
|
||||
viper.SetConfigType("yaml")
|
||||
viper.AddConfigPath("/etc/" + name)
|
||||
viper.AddConfigPath("$HOME/.config/" + name)
|
||||
|
||||
viper.AutomaticEnv()
|
||||
|
||||
viper.SetDefault("DATA_DIR", "./data/reports")
|
||||
viper.SetDefault("DEBUG", "false")
|
||||
viper.SetDefault("PORT", "8080")
|
||||
viper.SetDefault("SENTRY_DSN", "")
|
||||
viper.SetDefault("METRICS_USERNAME", "")
|
||||
viper.SetDefault("METRICS_PASSWORD", "")
|
||||
|
||||
err := viper.ReadInConfig()
|
||||
if err != nil {
|
||||
var notFound viper.ConfigFileNotFoundError
|
||||
if !errors.As(err, ¬Found) {
|
||||
log.Error("config file malformed", "error", err)
|
||||
panic(err)
|
||||
}
|
||||
}
|
||||
|
||||
s := &Config{
|
||||
DataDir: viper.GetString("DATA_DIR"),
|
||||
Debug: viper.GetBool("DEBUG"),
|
||||
MetricsPassword: viper.GetString("METRICS_PASSWORD"),
|
||||
MetricsUsername: viper.GetString("METRICS_USERNAME"),
|
||||
Port: viper.GetInt("PORT"),
|
||||
SentryDSN: viper.GetString("SENTRY_DSN"),
|
||||
log: log,
|
||||
params: ¶ms,
|
||||
}
|
||||
|
||||
if s.Debug {
|
||||
params.Logger.EnableDebugLogging()
|
||||
s.log = params.Logger.Get()
|
||||
}
|
||||
|
||||
return s, nil
|
||||
}
|
||||
31
backend/internal/globals/globals.go
Normal file
31
backend/internal/globals/globals.go
Normal file
@@ -0,0 +1,31 @@
|
||||
// Package globals provides build-time variables injected via
|
||||
// ldflags and made available through dependency injection.
|
||||
package globals
|
||||
|
||||
import "go.uber.org/fx"
|
||||
|
||||
//nolint:gochecknoglobals // set from main before fx starts
|
||||
var (
|
||||
// Appname is the application name.
|
||||
Appname string
|
||||
// Version is the git version tag.
|
||||
Version string
|
||||
// Buildarch is the build architecture.
|
||||
Buildarch string
|
||||
)
|
||||
|
||||
// Globals holds build-time metadata for the application.
|
||||
type Globals struct {
|
||||
Appname string
|
||||
Version string
|
||||
Buildarch string
|
||||
}
|
||||
|
||||
// New creates a Globals instance from package-level variables.
|
||||
func New(_ fx.Lifecycle) (*Globals, error) {
|
||||
return &Globals{
|
||||
Appname: Appname,
|
||||
Buildarch: Buildarch,
|
||||
Version: Version,
|
||||
}, nil
|
||||
}
|
||||
74
backend/internal/handlers/handlers.go
Normal file
74
backend/internal/handlers/handlers.go
Normal file
@@ -0,0 +1,74 @@
|
||||
// Package handlers implements HTTP request handlers for the
|
||||
// netwatch-server API.
|
||||
package handlers
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
|
||||
"sneak.berlin/go/netwatch/internal/globals"
|
||||
"sneak.berlin/go/netwatch/internal/healthcheck"
|
||||
"sneak.berlin/go/netwatch/internal/logger"
|
||||
"sneak.berlin/go/netwatch/internal/reportbuf"
|
||||
|
||||
"go.uber.org/fx"
|
||||
)
|
||||
|
||||
const jsonContentType = "application/json; charset=utf-8"
|
||||
|
||||
// Params defines the dependencies for Handlers.
|
||||
type Params struct {
|
||||
fx.In
|
||||
|
||||
Buffer *reportbuf.Buffer
|
||||
Globals *globals.Globals
|
||||
Healthcheck *healthcheck.Healthcheck
|
||||
Logger *logger.Logger
|
||||
}
|
||||
|
||||
// Handlers provides HTTP handler factories for all endpoints.
|
||||
type Handlers struct {
|
||||
buf *reportbuf.Buffer
|
||||
hc *healthcheck.Healthcheck
|
||||
log *slog.Logger
|
||||
params *Params
|
||||
}
|
||||
|
||||
// New creates a Handlers instance.
|
||||
func New(
|
||||
lc fx.Lifecycle,
|
||||
params Params,
|
||||
) (*Handlers, error) {
|
||||
s := new(Handlers)
|
||||
s.buf = params.Buffer
|
||||
s.params = ¶ms
|
||||
s.log = params.Logger.Get()
|
||||
s.hc = params.Healthcheck
|
||||
|
||||
lc.Append(fx.Hook{
|
||||
OnStart: func(_ context.Context) error {
|
||||
return nil
|
||||
},
|
||||
})
|
||||
|
||||
return s, nil
|
||||
}
|
||||
|
||||
func (s *Handlers) respondJSON(
|
||||
w http.ResponseWriter,
|
||||
_ *http.Request,
|
||||
data any,
|
||||
status int,
|
||||
) {
|
||||
w.Header().Set("Content-Type", jsonContentType)
|
||||
w.WriteHeader(status)
|
||||
|
||||
if data != nil {
|
||||
err := json.NewEncoder(w).Encode(data)
|
||||
if err != nil {
|
||||
s.log.Error("json encode error", "error", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
13
backend/internal/handlers/handlers_test.go
Normal file
13
backend/internal/handlers/handlers_test.go
Normal file
@@ -0,0 +1,13 @@
|
||||
package handlers_test
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
_ "sneak.berlin/go/netwatch/internal/handlers"
|
||||
)
|
||||
|
||||
func TestImport(t *testing.T) {
|
||||
t.Parallel()
|
||||
// Compilation check — verifies the package parses
|
||||
// and all imports resolve.
|
||||
}
|
||||
11
backend/internal/handlers/healthcheck.go
Normal file
11
backend/internal/handlers/healthcheck.go
Normal file
@@ -0,0 +1,11 @@
|
||||
package handlers
|
||||
|
||||
import "net/http"
|
||||
|
||||
// HandleHealthCheck returns a handler for the health check
|
||||
// endpoint.
|
||||
func (s *Handlers) HandleHealthCheck() http.HandlerFunc {
|
||||
return func(w http.ResponseWriter, r *http.Request) {
|
||||
s.respondJSON(w, r, s.hc.Check(), http.StatusOK)
|
||||
}
|
||||
}
|
||||
82
backend/internal/handlers/report.go
Normal file
82
backend/internal/handlers/report.go
Normal file
@@ -0,0 +1,82 @@
|
||||
package handlers
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
)
|
||||
|
||||
const maxReportBodyBytes = 1 << 20 // 1 MiB
|
||||
|
||||
type reportSample struct {
|
||||
T int64 `json:"t"`
|
||||
Latency *int `json:"latency"`
|
||||
Error *string `json:"error"`
|
||||
}
|
||||
|
||||
type reportHost struct {
|
||||
History []reportSample `json:"history"`
|
||||
Name string `json:"name"`
|
||||
Status string `json:"status"`
|
||||
URL string `json:"url"`
|
||||
}
|
||||
|
||||
type report struct {
|
||||
ClientID string `json:"clientId"`
|
||||
Geo json.RawMessage `json:"geo"`
|
||||
Hosts []reportHost `json:"hosts"`
|
||||
Timestamp string `json:"timestamp"`
|
||||
}
|
||||
|
||||
// HandleReport returns a handler that accepts telemetry
|
||||
// reports from NetWatch clients.
|
||||
func (s *Handlers) HandleReport() http.HandlerFunc {
|
||||
type response struct {
|
||||
Status string `json:"status"`
|
||||
}
|
||||
|
||||
return func(w http.ResponseWriter, r *http.Request) {
|
||||
r.Body = http.MaxBytesReader(
|
||||
w, r.Body, maxReportBodyBytes,
|
||||
)
|
||||
|
||||
var rpt report
|
||||
|
||||
err := json.NewDecoder(r.Body).Decode(&rpt)
|
||||
if err != nil {
|
||||
s.log.Error("failed to decode report",
|
||||
"error", err,
|
||||
)
|
||||
s.respondJSON(w, r,
|
||||
&response{Status: "error"},
|
||||
http.StatusBadRequest,
|
||||
)
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
totalSamples := 0
|
||||
for _, h := range rpt.Hosts {
|
||||
totalSamples += len(h.History)
|
||||
}
|
||||
|
||||
s.log.Info("report received",
|
||||
"client_id", rpt.ClientID,
|
||||
"timestamp", rpt.Timestamp,
|
||||
"host_count", len(rpt.Hosts),
|
||||
"total_samples", totalSamples,
|
||||
"geo", string(rpt.Geo),
|
||||
)
|
||||
|
||||
bufErr := s.buf.Append(rpt)
|
||||
if bufErr != nil {
|
||||
s.log.Error("failed to buffer report",
|
||||
"error", bufErr,
|
||||
)
|
||||
}
|
||||
|
||||
s.respondJSON(w, r,
|
||||
&response{Status: "ok"},
|
||||
http.StatusOK,
|
||||
)
|
||||
}
|
||||
}
|
||||
82
backend/internal/healthcheck/healthcheck.go
Normal file
82
backend/internal/healthcheck/healthcheck.go
Normal file
@@ -0,0 +1,82 @@
|
||||
// Package healthcheck provides a service that reports
|
||||
// application health, uptime, and version information.
|
||||
package healthcheck
|
||||
|
||||
import (
|
||||
"context"
|
||||
"log/slog"
|
||||
"time"
|
||||
|
||||
"sneak.berlin/go/netwatch/internal/config"
|
||||
"sneak.berlin/go/netwatch/internal/globals"
|
||||
"sneak.berlin/go/netwatch/internal/logger"
|
||||
|
||||
"go.uber.org/fx"
|
||||
)
|
||||
|
||||
// Params defines the dependencies for Healthcheck.
|
||||
type Params struct {
|
||||
fx.In
|
||||
|
||||
Config *config.Config
|
||||
Globals *globals.Globals
|
||||
Logger *logger.Logger
|
||||
}
|
||||
|
||||
// Healthcheck tracks startup time and builds health responses.
|
||||
type Healthcheck struct {
|
||||
StartupTime time.Time
|
||||
log *slog.Logger
|
||||
params *Params
|
||||
}
|
||||
|
||||
// Response is the JSON payload returned by the health check
|
||||
// endpoint.
|
||||
type Response struct {
|
||||
Appname string `json:"appname"`
|
||||
Now string `json:"now"`
|
||||
Status string `json:"status"`
|
||||
UptimeHuman string `json:"uptimeHuman"`
|
||||
UptimeSeconds int64 `json:"uptimeSeconds"`
|
||||
Version string `json:"version"`
|
||||
}
|
||||
|
||||
// New creates a Healthcheck, recording startup time via an
|
||||
// fx lifecycle hook.
|
||||
func New(
|
||||
lc fx.Lifecycle,
|
||||
params Params,
|
||||
) (*Healthcheck, error) {
|
||||
s := new(Healthcheck)
|
||||
s.params = ¶ms
|
||||
s.log = params.Logger.Get()
|
||||
|
||||
lc.Append(fx.Hook{
|
||||
OnStart: func(_ context.Context) error {
|
||||
s.StartupTime = time.Now().UTC()
|
||||
|
||||
return nil
|
||||
},
|
||||
OnStop: func(_ context.Context) error {
|
||||
return nil
|
||||
},
|
||||
})
|
||||
|
||||
return s, nil
|
||||
}
|
||||
|
||||
// Check returns the current health status of the application.
|
||||
func (s *Healthcheck) Check() *Response {
|
||||
return &Response{
|
||||
Appname: s.params.Globals.Appname,
|
||||
Now: time.Now().UTC().Format(time.RFC3339Nano),
|
||||
Status: "ok",
|
||||
UptimeHuman: s.uptime().String(),
|
||||
UptimeSeconds: int64(s.uptime().Seconds()),
|
||||
Version: s.params.Globals.Version,
|
||||
}
|
||||
}
|
||||
|
||||
func (s *Healthcheck) uptime() time.Duration {
|
||||
return time.Since(s.StartupTime)
|
||||
}
|
||||
85
backend/internal/logger/logger.go
Normal file
85
backend/internal/logger/logger.go
Normal file
@@ -0,0 +1,85 @@
|
||||
// Package logger provides a configured slog.Logger with TTY
|
||||
// detection for development vs production output.
|
||||
package logger
|
||||
|
||||
import (
|
||||
"log/slog"
|
||||
"os"
|
||||
|
||||
"sneak.berlin/go/netwatch/internal/globals"
|
||||
|
||||
"go.uber.org/fx"
|
||||
)
|
||||
|
||||
// Params defines the dependencies for Logger.
|
||||
type Params struct {
|
||||
fx.In
|
||||
|
||||
Globals *globals.Globals
|
||||
}
|
||||
|
||||
// Logger wraps slog.Logger with dynamic level control.
|
||||
type Logger struct {
|
||||
log *slog.Logger
|
||||
level *slog.LevelVar
|
||||
params Params
|
||||
}
|
||||
|
||||
// New creates a Logger with TTY-aware output formatting.
|
||||
func New(_ fx.Lifecycle, params Params) (*Logger, error) {
|
||||
l := new(Logger)
|
||||
l.level = new(slog.LevelVar)
|
||||
l.level.Set(slog.LevelInfo)
|
||||
l.params = params
|
||||
|
||||
tty := false
|
||||
|
||||
if fileInfo, _ := os.Stdout.Stat(); fileInfo != nil {
|
||||
if (fileInfo.Mode() & os.ModeCharDevice) != 0 {
|
||||
tty = true
|
||||
}
|
||||
}
|
||||
|
||||
var handler slog.Handler
|
||||
if tty {
|
||||
handler = slog.NewTextHandler(
|
||||
os.Stdout,
|
||||
&slog.HandlerOptions{
|
||||
Level: l.level,
|
||||
AddSource: true,
|
||||
},
|
||||
)
|
||||
} else {
|
||||
handler = slog.NewJSONHandler(
|
||||
os.Stdout,
|
||||
&slog.HandlerOptions{
|
||||
Level: l.level,
|
||||
AddSource: true,
|
||||
},
|
||||
)
|
||||
}
|
||||
|
||||
l.log = slog.New(handler)
|
||||
|
||||
return l, nil
|
||||
}
|
||||
|
||||
// EnableDebugLogging sets the log level to debug.
|
||||
func (l *Logger) EnableDebugLogging() {
|
||||
l.level.Set(slog.LevelDebug)
|
||||
l.log.Debug("debug logging enabled", "debug", true)
|
||||
}
|
||||
|
||||
// Get returns the underlying slog.Logger.
|
||||
func (l *Logger) Get() *slog.Logger {
|
||||
return l.log
|
||||
}
|
||||
|
||||
// Identify logs the application's build-time metadata.
|
||||
func (l *Logger) Identify() {
|
||||
l.log.Info("starting",
|
||||
"appname", l.params.Globals.Appname,
|
||||
"version", l.params.Globals.Version,
|
||||
"buildarch", l.params.Globals.Buildarch,
|
||||
)
|
||||
}
|
||||
129
backend/internal/middleware/middleware.go
Normal file
129
backend/internal/middleware/middleware.go
Normal file
@@ -0,0 +1,129 @@
|
||||
// Package middleware provides HTTP middleware for logging,
|
||||
// CORS, and other cross-cutting concerns.
|
||||
package middleware
|
||||
|
||||
import (
|
||||
"log/slog"
|
||||
"net"
|
||||
"net/http"
|
||||
"time"
|
||||
|
||||
"sneak.berlin/go/netwatch/internal/config"
|
||||
"sneak.berlin/go/netwatch/internal/globals"
|
||||
"sneak.berlin/go/netwatch/internal/logger"
|
||||
|
||||
"github.com/go-chi/chi/v5/middleware"
|
||||
"github.com/go-chi/cors"
|
||||
"go.uber.org/fx"
|
||||
)
|
||||
|
||||
const corsMaxAgeSec = 300
|
||||
|
||||
// Params defines the dependencies for Middleware.
|
||||
type Params struct {
|
||||
fx.In
|
||||
|
||||
Config *config.Config
|
||||
Globals *globals.Globals
|
||||
Logger *logger.Logger
|
||||
}
|
||||
|
||||
// Middleware holds shared state for middleware factories.
|
||||
type Middleware struct {
|
||||
log *slog.Logger
|
||||
params *Params
|
||||
}
|
||||
|
||||
// New creates a Middleware instance.
|
||||
func New(
|
||||
_ fx.Lifecycle,
|
||||
params Params,
|
||||
) (*Middleware, error) {
|
||||
s := new(Middleware)
|
||||
s.params = ¶ms
|
||||
s.log = params.Logger.Get()
|
||||
|
||||
return s, nil
|
||||
}
|
||||
|
||||
type loggingResponseWriter struct {
|
||||
http.ResponseWriter
|
||||
|
||||
statusCode int
|
||||
}
|
||||
|
||||
func newLoggingResponseWriter(
|
||||
w http.ResponseWriter,
|
||||
) *loggingResponseWriter {
|
||||
return &loggingResponseWriter{w, http.StatusOK}
|
||||
}
|
||||
|
||||
func (lrw *loggingResponseWriter) WriteHeader(code int) {
|
||||
lrw.statusCode = code
|
||||
lrw.ResponseWriter.WriteHeader(code)
|
||||
}
|
||||
|
||||
func ipFromHostPort(hostPort string) string {
|
||||
host, _, err := net.SplitHostPort(hostPort)
|
||||
if err != nil {
|
||||
return hostPort
|
||||
}
|
||||
|
||||
return host
|
||||
}
|
||||
|
||||
// Logging returns middleware that logs each request with
|
||||
// timing, status code, and client information.
|
||||
func (s *Middleware) Logging() func(http.Handler) http.Handler {
|
||||
return func(next http.Handler) http.Handler {
|
||||
return http.HandlerFunc(
|
||||
func(w http.ResponseWriter, r *http.Request) {
|
||||
start := time.Now().UTC()
|
||||
lrw := newLoggingResponseWriter(w)
|
||||
ctx := r.Context()
|
||||
|
||||
defer func() {
|
||||
latency := time.Since(start)
|
||||
s.log.InfoContext(ctx, "request",
|
||||
"request_start", start,
|
||||
"method", r.Method,
|
||||
"url", r.URL.String(),
|
||||
"useragent", r.UserAgent(),
|
||||
"request_id",
|
||||
ctx.Value(
|
||||
middleware.RequestIDKey,
|
||||
),
|
||||
"referer", r.Referer(),
|
||||
"proto", r.Proto,
|
||||
"remote_ip",
|
||||
ipFromHostPort(r.RemoteAddr),
|
||||
"status", lrw.statusCode,
|
||||
"latency_ms",
|
||||
latency.Milliseconds(),
|
||||
)
|
||||
}()
|
||||
|
||||
next.ServeHTTP(lrw, r)
|
||||
},
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
// CORS returns middleware that adds permissive CORS headers.
|
||||
func (s *Middleware) CORS() func(http.Handler) http.Handler {
|
||||
return cors.Handler(cors.Options{
|
||||
AllowedOrigins: []string{"*"},
|
||||
AllowedMethods: []string{
|
||||
"GET", "POST", "PUT", "DELETE", "OPTIONS",
|
||||
},
|
||||
AllowedHeaders: []string{
|
||||
"Accept",
|
||||
"Authorization",
|
||||
"Content-Type",
|
||||
"X-CSRF-Token",
|
||||
},
|
||||
ExposedHeaders: []string{"Link"},
|
||||
AllowCredentials: false,
|
||||
MaxAge: corsMaxAgeSec,
|
||||
})
|
||||
}
|
||||
199
backend/internal/reportbuf/reportbuf.go
Normal file
199
backend/internal/reportbuf/reportbuf.go
Normal file
@@ -0,0 +1,199 @@
|
||||
// Package reportbuf accumulates telemetry reports in memory
|
||||
// and periodically flushes them to zstd-compressed JSONL files.
|
||||
package reportbuf
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io/fs"
|
||||
"log/slog"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"sneak.berlin/go/netwatch/internal/config"
|
||||
"sneak.berlin/go/netwatch/internal/logger"
|
||||
|
||||
"github.com/klauspost/compress/zstd"
|
||||
"go.uber.org/fx"
|
||||
)
|
||||
|
||||
const (
|
||||
flushSizeThreshold = 10 << 20 // 10 MiB
|
||||
flushInterval = 1 * time.Minute
|
||||
defaultDataDir = "./data/reports"
|
||||
dirPerms fs.FileMode = 0o750
|
||||
filePerms fs.FileMode = 0o640
|
||||
)
|
||||
|
||||
// Params defines the dependencies for Buffer.
|
||||
type Params struct {
|
||||
fx.In
|
||||
|
||||
Config *config.Config
|
||||
Logger *logger.Logger
|
||||
}
|
||||
|
||||
// Buffer accumulates JSON lines in memory and flushes them
|
||||
// to zstd-compressed files on disk.
|
||||
type Buffer struct {
|
||||
buf bytes.Buffer
|
||||
dataDir string
|
||||
done chan struct{}
|
||||
log *slog.Logger
|
||||
mu sync.Mutex
|
||||
}
|
||||
|
||||
// New creates a Buffer and registers lifecycle hooks to
|
||||
// manage the data directory and flush goroutine.
|
||||
func New(
|
||||
lc fx.Lifecycle,
|
||||
params Params,
|
||||
) (*Buffer, error) {
|
||||
dir := params.Config.DataDir
|
||||
if dir == "" {
|
||||
dir = defaultDataDir
|
||||
}
|
||||
|
||||
b := &Buffer{
|
||||
dataDir: dir,
|
||||
done: make(chan struct{}),
|
||||
log: params.Logger.Get(),
|
||||
}
|
||||
|
||||
lc.Append(fx.Hook{
|
||||
OnStart: func(_ context.Context) error {
|
||||
err := os.MkdirAll(b.dataDir, dirPerms)
|
||||
if err != nil {
|
||||
return fmt.Errorf("create data dir: %w", err)
|
||||
}
|
||||
|
||||
go b.flushLoop()
|
||||
|
||||
return nil
|
||||
},
|
||||
OnStop: func(_ context.Context) error {
|
||||
close(b.done)
|
||||
b.flushLocked()
|
||||
|
||||
return nil
|
||||
},
|
||||
})
|
||||
|
||||
return b, nil
|
||||
}
|
||||
|
||||
// Append marshals v as a single JSON line and appends it to
|
||||
// the buffer. If the buffer reaches the size threshold, it is
|
||||
// drained and written to disk asynchronously.
|
||||
func (b *Buffer) Append(v any) error {
|
||||
line, err := json.Marshal(v)
|
||||
if err != nil {
|
||||
return fmt.Errorf("marshal report: %w", err)
|
||||
}
|
||||
|
||||
b.mu.Lock()
|
||||
b.buf.Write(line)
|
||||
b.buf.WriteByte('\n')
|
||||
|
||||
if b.buf.Len() >= flushSizeThreshold {
|
||||
data := b.drainBuf()
|
||||
b.mu.Unlock()
|
||||
|
||||
go b.writeFile(data)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
b.mu.Unlock()
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// flushLoop runs a ticker that periodically flushes buffered
|
||||
// data to disk until the done channel is closed.
|
||||
func (b *Buffer) flushLoop() {
|
||||
ticker := time.NewTicker(flushInterval)
|
||||
defer ticker.Stop()
|
||||
|
||||
for {
|
||||
select {
|
||||
case <-ticker.C:
|
||||
b.flushLocked()
|
||||
case <-b.done:
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// flushLocked acquires the lock, drains the buffer, and
|
||||
// writes the data to a compressed file.
|
||||
func (b *Buffer) flushLocked() {
|
||||
b.mu.Lock()
|
||||
|
||||
if b.buf.Len() == 0 {
|
||||
b.mu.Unlock()
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
data := b.drainBuf()
|
||||
b.mu.Unlock()
|
||||
|
||||
b.writeFile(data)
|
||||
}
|
||||
|
||||
// drainBuf copies the buffer contents and resets it.
|
||||
// The caller must hold b.mu.
|
||||
func (b *Buffer) drainBuf() []byte {
|
||||
data := make([]byte, b.buf.Len())
|
||||
copy(data, b.buf.Bytes())
|
||||
b.buf.Reset()
|
||||
|
||||
return data
|
||||
}
|
||||
|
||||
// writeFile creates a timestamped zstd-compressed JSONL file
|
||||
// in the data directory.
|
||||
func (b *Buffer) writeFile(data []byte) {
|
||||
ts := time.Now().UTC().Format("2006-01-02T15-04-05.000Z")
|
||||
name := fmt.Sprintf("reports-%s.jsonl.zst", ts)
|
||||
path := filepath.Join(b.dataDir, name)
|
||||
|
||||
f, err := os.OpenFile( //nolint:gosec // path built from controlled dataDir + timestamp
|
||||
path,
|
||||
os.O_WRONLY|os.O_CREATE|os.O_EXCL,
|
||||
filePerms,
|
||||
)
|
||||
if err != nil {
|
||||
b.log.Error("create report file", "error", err)
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
defer func() { _ = f.Close() }()
|
||||
|
||||
enc, err := zstd.NewWriter(f)
|
||||
if err != nil {
|
||||
b.log.Error("create zstd encoder", "error", err)
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
_, writeErr := enc.Write(data)
|
||||
if writeErr != nil {
|
||||
b.log.Error("write compressed data", "error", writeErr)
|
||||
|
||||
_ = enc.Close()
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
closeErr := enc.Close()
|
||||
if closeErr != nil {
|
||||
b.log.Error("close zstd encoder", "error", closeErr)
|
||||
}
|
||||
}
|
||||
13
backend/internal/reportbuf/reportbuf_test.go
Normal file
13
backend/internal/reportbuf/reportbuf_test.go
Normal file
@@ -0,0 +1,13 @@
|
||||
package reportbuf_test
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
_ "sneak.berlin/go/netwatch/internal/reportbuf"
|
||||
)
|
||||
|
||||
func TestImport(t *testing.T) {
|
||||
t.Parallel()
|
||||
// Compilation check — verifies the package parses
|
||||
// and all imports resolve.
|
||||
}
|
||||
43
backend/internal/server/http.go
Normal file
43
backend/internal/server/http.go
Normal file
@@ -0,0 +1,43 @@
|
||||
package server
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"time"
|
||||
)
|
||||
|
||||
const (
|
||||
readTimeout = 10 * time.Second
|
||||
writeTimeout = 10 * time.Second
|
||||
maxHeaderBytes = 1 << 20 // 1 MiB
|
||||
)
|
||||
|
||||
func (s *Server) serveUntilShutdown() {
|
||||
listenAddr := fmt.Sprintf(":%d", s.params.Config.Port)
|
||||
|
||||
s.httpServer = &http.Server{
|
||||
Addr: listenAddr,
|
||||
Handler: s,
|
||||
MaxHeaderBytes: maxHeaderBytes,
|
||||
ReadTimeout: readTimeout,
|
||||
WriteTimeout: writeTimeout,
|
||||
}
|
||||
|
||||
s.SetupRoutes()
|
||||
|
||||
s.log.Info("http begin listen",
|
||||
"listenaddr", listenAddr,
|
||||
"version", s.params.Globals.Version,
|
||||
"buildarch", s.params.Globals.Buildarch,
|
||||
)
|
||||
|
||||
err := s.httpServer.ListenAndServe()
|
||||
if err != nil && !errors.Is(err, http.ErrServerClosed) {
|
||||
s.log.Error("listen error", "error", err)
|
||||
|
||||
if s.cancelFunc != nil {
|
||||
s.cancelFunc()
|
||||
}
|
||||
}
|
||||
}
|
||||
31
backend/internal/server/routes.go
Normal file
31
backend/internal/server/routes.go
Normal file
@@ -0,0 +1,31 @@
|
||||
package server
|
||||
|
||||
import (
|
||||
"time"
|
||||
|
||||
"github.com/go-chi/chi/v5"
|
||||
"github.com/go-chi/chi/v5/middleware"
|
||||
)
|
||||
|
||||
const requestTimeout = 60 * time.Second
|
||||
|
||||
// SetupRoutes configures the chi router with middleware and
|
||||
// all application routes.
|
||||
func (s *Server) SetupRoutes() {
|
||||
s.router = chi.NewRouter()
|
||||
|
||||
s.router.Use(middleware.Recoverer)
|
||||
s.router.Use(middleware.RequestID)
|
||||
s.router.Use(s.mw.Logging())
|
||||
s.router.Use(s.mw.CORS())
|
||||
s.router.Use(middleware.Timeout(requestTimeout))
|
||||
|
||||
s.router.Get(
|
||||
"/.well-known/healthcheck",
|
||||
s.h.HandleHealthCheck(),
|
||||
)
|
||||
|
||||
s.router.Route("/api/v1", func(r chi.Router) {
|
||||
r.Post("/reports", s.h.HandleReport())
|
||||
})
|
||||
}
|
||||
147
backend/internal/server/server.go
Normal file
147
backend/internal/server/server.go
Normal file
@@ -0,0 +1,147 @@
|
||||
// Package server provides the HTTP server lifecycle,
|
||||
// including startup, routing, signal handling, and graceful
|
||||
// shutdown.
|
||||
package server
|
||||
|
||||
import (
|
||||
"context"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
"os"
|
||||
"os/signal"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
"sneak.berlin/go/netwatch/internal/config"
|
||||
"sneak.berlin/go/netwatch/internal/globals"
|
||||
"sneak.berlin/go/netwatch/internal/handlers"
|
||||
"sneak.berlin/go/netwatch/internal/logger"
|
||||
"sneak.berlin/go/netwatch/internal/middleware"
|
||||
|
||||
"github.com/go-chi/chi/v5"
|
||||
"go.uber.org/fx"
|
||||
)
|
||||
|
||||
// Params defines the dependencies for Server.
|
||||
type Params struct {
|
||||
fx.In
|
||||
|
||||
Config *config.Config
|
||||
Globals *globals.Globals
|
||||
Handlers *handlers.Handlers
|
||||
Logger *logger.Logger
|
||||
Middleware *middleware.Middleware
|
||||
}
|
||||
|
||||
// Server is the top-level HTTP server orchestrator.
|
||||
type Server struct {
|
||||
cancelFunc context.CancelFunc
|
||||
exitCode int
|
||||
h *handlers.Handlers
|
||||
httpServer *http.Server
|
||||
log *slog.Logger
|
||||
mw *middleware.Middleware
|
||||
params Params
|
||||
router *chi.Mux
|
||||
startupTime time.Time
|
||||
}
|
||||
|
||||
// New creates a Server and registers lifecycle hooks for
|
||||
// starting and stopping it.
|
||||
func New(
|
||||
lc fx.Lifecycle,
|
||||
params Params,
|
||||
) (*Server, error) {
|
||||
s := new(Server)
|
||||
s.params = params
|
||||
s.mw = params.Middleware
|
||||
s.h = params.Handlers
|
||||
s.log = params.Logger.Get()
|
||||
|
||||
lc.Append(fx.Hook{
|
||||
OnStart: func(_ context.Context) error {
|
||||
s.startupTime = time.Now().UTC()
|
||||
|
||||
go func() { //nolint:contextcheck // fx OnStart ctx is startup-only; run() creates its own
|
||||
s.run()
|
||||
}()
|
||||
|
||||
return nil
|
||||
},
|
||||
OnStop: func(_ context.Context) error {
|
||||
if s.cancelFunc != nil {
|
||||
s.cancelFunc()
|
||||
}
|
||||
|
||||
return nil
|
||||
},
|
||||
})
|
||||
|
||||
return s, nil
|
||||
}
|
||||
|
||||
// ServeHTTP delegates to the chi router.
|
||||
func (s *Server) ServeHTTP(
|
||||
w http.ResponseWriter,
|
||||
r *http.Request,
|
||||
) {
|
||||
s.router.ServeHTTP(w, r)
|
||||
}
|
||||
|
||||
func (s *Server) run() {
|
||||
exitCode := s.serve()
|
||||
os.Exit(exitCode)
|
||||
}
|
||||
|
||||
func (s *Server) serve() int {
|
||||
var ctx context.Context //nolint:wsl // ctx must be declared before multi-assign
|
||||
|
||||
ctx, s.cancelFunc = context.WithCancel(
|
||||
context.Background(),
|
||||
)
|
||||
|
||||
go func() {
|
||||
c := make(chan os.Signal, 1)
|
||||
|
||||
signal.Ignore(syscall.SIGPIPE)
|
||||
signal.Notify(c, os.Interrupt, syscall.SIGTERM)
|
||||
|
||||
sig := <-c
|
||||
s.log.Info("signal received", "signal", sig)
|
||||
|
||||
if s.cancelFunc != nil {
|
||||
s.cancelFunc()
|
||||
}
|
||||
}()
|
||||
|
||||
go func() {
|
||||
s.serveUntilShutdown()
|
||||
}()
|
||||
|
||||
<-ctx.Done()
|
||||
s.cleanShutdown()
|
||||
|
||||
return s.exitCode
|
||||
}
|
||||
|
||||
const shutdownTimeout = 5 * time.Second
|
||||
|
||||
func (s *Server) cleanShutdown() {
|
||||
s.exitCode = 0
|
||||
|
||||
ctxShutdown, shutdownCancel := context.WithTimeout(
|
||||
context.Background(),
|
||||
shutdownTimeout,
|
||||
)
|
||||
defer shutdownCancel()
|
||||
|
||||
err := s.httpServer.Shutdown(ctxShutdown)
|
||||
if err != nil {
|
||||
s.log.Error(
|
||||
"server clean shutdown failed",
|
||||
"error", err,
|
||||
)
|
||||
}
|
||||
|
||||
s.log.Info("server stopped")
|
||||
}
|
||||
Reference in New Issue
Block a user