Compare commits

..
Author SHA1 Message Date
sneak 3669abe761 config: each waf setting's code in one place (closes #449)
check / check (push) In progress
internal/config/waf.go is reordered by setting, in the order of the
settings sections of SPEC and docs/settings.md. The struct WAF and
WAFFromEnvironment stay first. Each setting's reader, parser, errors
and check sit together. A helper several settings share sits before
the first of them.

Moves only. The one error block and the one block of constants are
split into a block beside each user, so gofmt re-aligns the = of some
of those lines.

Model: opus-5-5
2026-11-20 00:17:29 +00:00
17 changed files with 1694 additions and 1614 deletions

No files matched your search

+1621 -1444
View File
File diff suppressed because it is too large. Load diff
+1 -9
View File
@@ -1,14 +1,6 @@
package rproxy package rproxy
import ( import "time"
"net/http"
"time"
)
// HTTPServer is Go's server s holds, for tests that read its settings.
func (s *Server) HTTPServer() *http.Server {
return s.httpServer
}
// StopBreakerClock has the circuit breaker of s read at as the time, so that how // StopBreakerClock has the circuit breaker of s read at as the time, so that how
// long requests take cannot matter to it. It is called before s serves. // long requests take cannot matter to it. It is called before s serves.
+5 -5
View File
@@ -56,7 +56,7 @@ func TestServerHasTheFixedLimits(t *testing.T) {
InstanceName: "test", InstanceName: "test",
Level: slog.LevelInfo, Level: slog.LevelInfo,
}), }),
}).HTTPServer() })
if server.Addr != ":8080" || server.MaxHeaderBytes != 28<<10 || if server.Addr != ":8080" || server.MaxHeaderBytes != 28<<10 ||
server.IdleTimeout != 2*time.Minute || server.ReadHeaderTimeout != time.Minute { server.IdleTimeout != 2*time.Minute || server.ReadHeaderTimeout != time.Minute {
@@ -709,7 +709,7 @@ func TestABodyLeftUnreadIsReadOnlyUntilTheRequestTimeLimit(t *testing.T) {
before := time.Now() before := time.Now()
server.ServeHTTP(w, req) server.Handler.ServeHTTP(w, req)
if w.Code != http.StatusNoContent { if w.Code != http.StatusNoContent {
t.Errorf("status %d, want the app's 204", w.Code) t.Errorf("status %d, want the app's 204", w.Code)
@@ -762,7 +762,7 @@ func TestABodyLeftUnreadIsReadNoMoreOnceTheTimeToSendTheAnswerRunsOut(
before := time.Now() before := time.Now()
server.ServeHTTP(w, req) server.Handler.ServeHTTP(w, req)
if w.Code != http.StatusOK { if w.Code != http.StatusOK {
t.Errorf("status %d, want the app's 200", w.Code) t.Errorf("status %d, want the app's 200", w.Code)
@@ -815,7 +815,7 @@ func TestAnAnswerRProxyGivesItselfHasTheResponseTimeToBeSent(t *testing.T) {
before := time.Now() before := time.Now()
server.ServeHTTP(w, req) server.Handler.ServeHTTP(w, req)
if w.Code != tc.status { if w.Code != tc.status {
t.Errorf("status %d, want %d", w.Code, tc.status) t.Errorf("status %d, want %d", w.Code, tc.status)
@@ -856,7 +856,7 @@ func TestAnAnswerRProxyGivesItselfHasTenSecondsToBeSentAtLeast(
before := time.Now() before := time.Now()
server.ServeHTTP(w, req) server.Handler.ServeHTTP(w, req)
if w.writeDeadline.Before(before.Add(least)) { if w.writeDeadline.Before(before.Add(least)) {
t.Errorf( t.Errorf(
+1 -1
View File
@@ -583,7 +583,7 @@ func TestAQuestionNamesNoZone(t *testing.T) {
req.RemoteAddr = tc.remoteAddr req.RemoteAddr = tc.remoteAddr
req.Header.Set(forwardedForHeader, tc.forwardedFor) req.Header.Set(forwardedForHeader, tc.forwardedFor)
server.ServeHTTP(httptest.NewRecorder(), req) server.Handler.ServeHTTP(httptest.NewRecorder(), req)
asked := standIn.question(t) asked := standIn.question(t)
if asked.header.Get(wafprotocol.ClientHeader) != tc.client || if asked.header.Get(wafprotocol.ClientHeader) != tc.client ||
+24 -39
View File
@@ -16,11 +16,11 @@ import (
"io" "io"
"log" "log"
"log/slog" "log/slog"
"net"
"net/http" "net/http"
"net/url" "net/url"
"slices" "slices"
"strings" "strings"
"sync"
"time" "time"
"sneak.berlin/go/smallwebwaf/internal/config" "sneak.berlin/go/smallwebwaf/internal/config"
@@ -64,8 +64,9 @@ type Params struct {
// Server is the server `sww rproxy` runs. // Server is the server `sww rproxy` runs.
type Server struct { type Server struct {
httpServer *http.Server *http.Server
h *handler
h *handler
} }
// New returns the server `sww rproxy` runs. Go's server itself refuses // New returns the server `sww rproxy` runs. Go's server itself refuses
@@ -94,47 +95,28 @@ func New(params Params) *Server {
breakerClock: time.Now, breakerClock: time.Now,
} }
s := &Server{h: h} return &Server{
s.httpServer = &http.Server{ Server: &http.Server{
Addr: params.Config.ListenAddr, Addr: params.Config.ListenAddr,
Handler: s, Handler: h,
ErrorLog: errorLog, ErrorLog: errorLog,
ReadHeaderTimeout: params.Config.MaxRequestTime, ReadHeaderTimeout: params.Config.MaxRequestTime,
IdleTimeout: idleTimeout, IdleTimeout: idleTimeout,
MaxHeaderBytes: requestHeaderMaxBytes - goServerExtraHeaderBytes, MaxHeaderBytes: requestHeaderMaxBytes - goServerExtraHeaderBytes,
},
h: h,
} }
return s
}
// Serve serves requests on listener until Shutdown or Close, as Go's
// http.Server.Serve does.
func (s *Server) Serve(listener net.Listener) error {
return s.httpServer.Serve(listener)
}
// Shutdown stops serving once the requests in progress have finished, or
// when ctx is done, as Go's http.Server.Shutdown does.
func (s *Server) Shutdown(ctx context.Context) error {
return s.httpServer.Shutdown(ctx)
}
// Close stops serving at once and closes every connection, as Go's
// http.Server.Close does.
func (s *Server) Close() error {
return s.httpServer.Close()
}
// ServeHTTP handles one request from a visitor, as handler.ServeHTTP says.
func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) {
s.h.ServeHTTP(w, r)
} }
// WaitForReports returns once the reports being sent have gone or failed, // WaitForReports returns once the reports being sent have gone or failed,
// which SWW_RPROXY_WAF_BACKEND_TIMEOUT_MS bounds. Any report after this call // which SWW_RPROXY_WAF_BACKEND_TIMEOUT_MS bounds. Any report after this call
// is dropped: that of a WebSocket that closes while rproxy stops, for one. // is dropped: that of a WebSocket that closes while rproxy stops, for one.
func (s *Server) WaitForReports() { func (s *Server) WaitForReports() {
s.h.reports.stopAndWait() s.h.reportsMu.Lock()
s.h.reportsStopped = true
s.h.reportsMu.Unlock()
s.h.reports.Wait()
} }
// handler is the proxy. It holds what every request shares; what belongs to // handler is the proxy. It holds what every request shares; what belongs to
@@ -154,8 +136,11 @@ type handler struct {
// the time. // the time.
breaker breaker breaker breaker
breakerClock func() time.Time breakerClock func() time.Time
// reports are the goroutines sending reports to waf. // reports waits for the reports being sent. reportsMu guards starting one
reports reportGoroutines // against reportsStopped, which is set once nothing is to wait for another.
reports sync.WaitGroup
reportsMu sync.Mutex
reportsStopped bool
} }
// newTransport returns what carries requests to the app. It never goes through // newTransport returns what carries requests to the app. It never goes through
+1 -1
View File
@@ -710,7 +710,7 @@ func TestAnAnswerWithoutAContentTypeGetsNone(t *testing.T) {
) )
w := httptest.NewRecorder() w := httptest.NewRecorder()
server.ServeHTTP(w, req) server.Handler.ServeHTTP(w, req)
// Go's server guesses a type when the first part of the body goes before the // Go's server guesses a type when the first part of the body goes before the
// headers are flushed, which is a matter of timing here. So this looks for what // headers are flushed, which is a matter of timing here. So this looks for what
+8 -33
View File
@@ -8,7 +8,6 @@ import (
"maps" "maps"
"net/http" "net/http"
"strings" "strings"
"sync"
"sneak.berlin/go/smallwebwaf/internal/rules" "sneak.berlin/go/smallwebwaf/internal/rules"
"sneak.berlin/go/smallwebwaf/internal/wafprotocol" "sneak.berlin/go/smallwebwaf/internal/wafprotocol"
@@ -211,7 +210,14 @@ func (h *handler) sendReport(
ctx context.Context, ctx context.Context,
report *wafprotocol.Report, report *wafprotocol.Report,
) { ) {
h.reports.startUnlessStopped(func() { h.reportsMu.Lock()
defer h.reportsMu.Unlock()
if h.reportsStopped {
return
}
h.reports.Go(func() {
err := h.postReport(context.WithoutCancel(ctx), report) err := h.postReport(context.WithoutCancel(ctx), report)
if err != nil { if err != nil {
h.processLog.Warn("a report is dropped", "error", err.Error()) h.processLog.Warn("a report is dropped", "error", err.Error())
@@ -219,37 +225,6 @@ func (h *handler) sendReport(
}) })
} }
// reportGoroutines are the goroutines sending reports to waf. None starts once
// stopAndWait has been called.
type reportGoroutines struct {
mu sync.Mutex
stopped bool
running sync.WaitGroup
}
// startUnlessStopped runs send on a goroutine of its own, or drops it once
// stopAndWait has been called.
func (g *reportGoroutines) startUnlessStopped(send func()) {
g.mu.Lock()
defer g.mu.Unlock()
if g.stopped {
return
}
g.running.Go(send)
}
// stopAndWait stops any more goroutines from starting, and returns once those
// running have ended.
func (g *reportGoroutines) stopAndWait() {
g.mu.Lock()
g.stopped = true
g.mu.Unlock()
g.running.Wait()
}
// postReport posts report to waf, and waits for its answer for up to // postReport posts report to waf, and waits for its answer for up to
// SWW_RPROXY_WAF_BACKEND_TIMEOUT_MS. // SWW_RPROXY_WAF_BACKEND_TIMEOUT_MS.
func (h *handler) postReport( func (h *handler) postReport(
+1 -9
View File
@@ -95,20 +95,12 @@ func background(loop func(context.Context)) fx.Hook {
) )
} }
// daemonServer is the server a daemon runs, waf.Server or rproxy.Server: the
// methods of Go's http.Server that serve starts and stops it with.
type daemonServer interface {
Serve(listener net.Listener) error
Shutdown(ctx context.Context) error
Close() error
}
// serve serves requests on listener from the start. At the stop it gives the // serve serves requests on listener from the start. At the stop it gives the
// requests in progress shutdownTimeout to finish, then closes their // requests in progress shutdownTimeout to finish, then closes their
// connections. When serving fails, it logs why and sends the error to // connections. When serving fails, it logs why and sends the error to
// failed. // failed.
func serve( func serve(
lc fx.Lifecycle, server daemonServer, listener net.Listener, lc fx.Lifecycle, server *http.Server, listener net.Listener,
processLog *slog.Logger, failed servingFailed, processLog *slog.Logger, failed servingFailed,
) { ) {
served := make(chan error, 1) served := make(chan error, 1)
+3 -7
View File
@@ -4,6 +4,7 @@ import (
"context" "context"
"log/slog" "log/slog"
"net" "net"
"net/http"
"os" "os"
"go.uber.org/fx" "go.uber.org/fx"
@@ -43,13 +44,8 @@ func RunRProxy(ctx context.Context, params Params) int {
app := fx.New( app := fx.New(
fx.Supply(cfg, processLog, requestLog{params.Stdout}, failed), fx.Supply(cfg, processLog, requestLog{params.Stdout}, failed),
fx.Provide( fx.Provide(
// run waits for rproxy.Server's reports, and serve takes it as a newRProxyServer,
// daemonServer. func(server *rproxy.Server) *http.Server { return server.Server },
fx.Annotate(
newRProxyServer,
fx.As(fx.Self()),
fx.As(new(daemonServer)),
),
newRProxyListener, newRProxyListener,
), ),
fx.Invoke(serve), fx.Invoke(serve),
+3 -2
View File
@@ -6,6 +6,7 @@ import (
"io" "io"
"log/slog" "log/slog"
"net" "net"
"net/http"
"time" "time"
"go.uber.org/fx" "go.uber.org/fx"
@@ -96,8 +97,8 @@ func RunWAF(ctx context.Context, params Params) int {
newAlertQueue, newRuleFiles, newCoreRuleSet, newLookupFile, newAlertQueue, newRuleFiles, newCoreRuleSet, newLookupFile,
newLimiter, newLedger, newAnomalyCounters, newLimiter, newLedger, newAnomalyCounters,
newLists, newDNSBL, newAbuseIPDB, newRouteWatch, newLists, newDNSBL, newAbuseIPDB, newRouteWatch,
newMetrics, newGeoJS, newMetrics, newGeoJS, newWAFServer,
fx.Annotate(newWAFServer, fx.As(new(daemonServer))), func(server *waf.Server) *http.Server { return server.Server },
newWAFListener, newStateFiles, newWAFListener, newStateFiles,
), ),
// fx stops hooks in the reverse of the order they were added in. The // fx stops hooks in the reverse of the order they were added in. The
+1 -1
View File
@@ -717,7 +717,7 @@ func askDirectly(
recorder := httptest.NewRecorder() recorder := httptest.NewRecorder()
began := time.Now() began := time.Now()
server.ServeHTTP(recorder, req) server.Handler.ServeHTTP(recorder, req)
return recorder.Code, time.Since(began) return recorder.Code, time.Since(began)
} }
+1 -1
View File
@@ -592,7 +592,7 @@ func serveFromDE(
req.Header.Set(wafprotocol.ClientHeader, fromDE) req.Header.Set(wafprotocol.ClientHeader, fromDE)
req.Header.Set(requestIDHeader, "the-id") req.Header.Set(requestIDHeader, "the-id")
server.ServeHTTP(httptest.NewRecorder(), req) server.Handler.ServeHTTP(httptest.NewRecorder(), req)
} }
// wantPercent checks a limit percentage that a log line or a ban's notes // wantPercent checks a limit percentage that a log line or a ban's notes
-7
View File
@@ -1,11 +1,4 @@
package waf package waf
import "net/http"
// BesideLookupTimeLimit is besideLookupTimeLimit. // BesideLookupTimeLimit is besideLookupTimeLimit.
const BesideLookupTimeLimit = besideLookupTimeLimit const BesideLookupTimeLimit = besideLookupTimeLimit
// HTTPServer is Go's server s holds, for tests that read its settings.
func (s *Server) HTTPServer() *http.Server {
return s.httpServer
}
+1 -2
View File
@@ -109,8 +109,7 @@ type CoreRuleSetInspector interface {
// Server is the server waf runs. // Server is the server waf runs.
type Server struct { type Server struct {
httpServer *http.Server *http.Server
h *handler
} }
// newHandler returns the handler with what every request shares, the parts // newHandler returns the handler with what every request shares, the parts
+3 -6
View File
@@ -344,8 +344,8 @@ func startProxyWithAlerts(
return startRProxy(t, server.Addr, appURL, env), out, server, alertQueue return startRProxy(t, server.Addr, appURL, env), out, server, alertQueue
} }
// serve serves server on a port of its own until the test ends, and sets // serve serves server on a port of its own until the test ends, and sets its
// server.Addr to where it listens. // Addr to where it listens.
func serve(t *testing.T, server *wafAndParts) { func serve(t *testing.T, server *wafAndParts) {
t.Helper() t.Helper()
@@ -423,13 +423,10 @@ func startRProxy(
} }
// wafAndParts is waf's server with the Params it was built from, so that a // wafAndParts is waf's server with the Params it was built from, so that a
// test can read the parts the server uses. Addr is where it listens once serve // test can read the parts the server uses.
// has it serving.
type wafAndParts struct { type wafAndParts struct {
*waf.Server *waf.Server
waf.Params waf.Params
Addr string
} }
// banAbuseIPDBBlacklist does on w what a fetch of AbuseIPDB's blacklist with // banAbuseIPDBBlacklist does on w what a fetch of AbuseIPDB's blacklist with
+12 -37
View File
@@ -4,7 +4,6 @@ import (
"context" "context"
"errors" "errors"
"log/slog" "log/slog"
"net"
"net/http" "net/http"
"strings" "strings"
"time" "time"
@@ -31,43 +30,19 @@ const (
// is far above rproxy's, and a question refused for its size would have rproxy // is far above rproxy's, and a question refused for its size would have rproxy
// let the request through. // let the request through.
func New(params Params) *Server { func New(params Params) *Server {
s := &Server{h: newHandler(params)} return &Server{
s.httpServer = &http.Server{ Server: &http.Server{
Addr: params.Config.ListenAddr, Addr: params.Config.ListenAddr,
Handler: s, Handler: newHandler(params),
ErrorLog: slog.NewLogLogger( ErrorLog: slog.NewLogLogger(
params.ProcessLog.Handler(), params.ProcessLog.Handler(),
slog.LevelWarn, slog.LevelWarn,
), ),
ReadTimeout: wafTimeout, ReadTimeout: wafTimeout,
WriteTimeout: wafTimeout, WriteTimeout: wafTimeout,
IdleTimeout: wafIdleTimeout, IdleTimeout: wafIdleTimeout,
},
} }
return s
}
// Serve serves requests on listener until Shutdown or Close, as Go's
// http.Server.Serve does.
func (s *Server) Serve(listener net.Listener) error {
return s.httpServer.Serve(listener)
}
// Shutdown stops serving once the requests in progress have finished, or
// when ctx is done, as Go's http.Server.Shutdown does.
func (s *Server) Shutdown(ctx context.Context) error {
return s.httpServer.Shutdown(ctx)
}
// Close stops serving at once and closes every connection, as Go's
// http.Server.Close does.
func (s *Server) Close() error {
return s.httpServer.Close()
}
// ServeHTTP handles one request to waf, as handler.ServeHTTP says.
func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) {
s.h.ServeHTTP(w, r)
} }
// ServeHTTP handles one request to waf: a request carrying // ServeHTTP handles one request to waf: a request carrying
+8 -10
View File
@@ -703,9 +703,7 @@ func TestQuestionWhoseBodyStopsArrivingGetsNoAnswer(t *testing.T) {
t, t,
"", "",
map[string]string{metricsToken: token}, map[string]string{metricsToken: token},
func(server *wafAndParts) { func(server *wafAndParts) { server.ReadTimeout = readTimeout },
server.HTTPServer().ReadTimeout = readTimeout
},
) )
conn := dial(t, s.addr) conn := dial(t, s.addr)
@@ -1836,16 +1834,16 @@ func TestLogLevelHoldsBackNoRequestLine(t *testing.T) {
func TestServerHasTheTimeLimitsSPECGives(t *testing.T) { func TestServerHasTheTimeLimitsSPECGives(t *testing.T) {
t.Parallel() t.Parallel()
server := startWAF(t, "", nil).server.HTTPServer() s := startWAF(t, "", nil)
if server.ReadTimeout != time.Minute || if s.server.ReadTimeout != time.Minute ||
server.WriteTimeout != time.Minute || s.server.WriteTimeout != time.Minute ||
server.IdleTimeout != 2*time.Minute { s.server.IdleTimeout != 2*time.Minute {
t.Errorf( t.Errorf(
"the read, write and idle limits are %v, %v and %v, want 1m, 1m and 2m", "the read, write and idle limits are %v, %v and %v, want 1m, 1m and 2m",
server.ReadTimeout, s.server.ReadTimeout,
server.WriteTimeout, s.server.WriteTimeout,
server.IdleTimeout, s.server.IdleTimeout,
) )
} }
} }