Compare commits

..
Author SHA1 Message Date
clawbot ef88f309fc waf: the server's types and constructor in waf.go, active_ban in its own file (closes #435)
check / check (push) Waiting to run
Moves only. waf.go now holds the package comment, Params, Server, New,
newHandler, handler and ServeHTTP. The code that answers questions and
takes reports moves to questionsandreports.go, and activeBan to
activeban.go. refusedWhateverTheAnswer moves to lookup.go, beside its
only caller. proxy.go is gone. The only changed lines are imports and
the file comment of bans.go, split between it and activeban.go.

Model: opus-5-5
2026-11-20 01:09:32 +00:00
clawbot 1b9ca94774 Hold Go's server privately in waf.Server and rproxy.Server (closes #437)
check / check (push) Waiting to run
Neither Server embeds *http.Server any more. Each holds it as a private
field, is its handler, and has Serve, Shutdown, Close and ServeHTTP.
sww's serve hook takes the new daemonServer interface of those first
three, and fx provides each Server as it.

Model: opus-5-5
2026-11-20 01:58:25 +01:00
clawbot 9afbd1b683 config: each waf setting's code in one place (closes #449)
check / check (push) Waiting to run
internal/config/waf.go is reordered by setting, in the order of the
settings sections of SPEC and docs/settings.md. The struct WAF and
WAFFromEnvironment stay first. Each setting's reader, parser, errors
and check sit together. A helper several settings share sits before
the first of them.

Moves only. The one error block and the one block of constants are
split into a block beside each user, so gofmt re-aligns the = of some
of those lines.

Model: opus-5-5
2026-11-20 01:44:08 +01:00
clawbot cc77435f2a rproxy: one type for the goroutines sending reports (closes #433)
check / check (push) Waiting to run
handler held a WaitGroup, a mutex and a stopped flag for the reports,
and the rule that no report starts once waiting has begun was split
between sendReport and Server.WaitForReports. reportGoroutines now
holds all three, and its two methods are the only code that touches
the lock and the flag.

Model: opus-5-5
2026-11-20 01:21:06 +01:00
20 changed files with 794 additions and 688 deletions

No files matched your search

+9 -1
View File
@@ -1,6 +1,14 @@
package rproxy
import "time"
import (
"net/http"
"time"
)
// HTTPServer is Go's server s holds, for tests that read its settings.
func (s *Server) HTTPServer() *http.Server {
return s.httpServer
}
// StopBreakerClock has the circuit breaker of s read at as the time, so that how
// long requests take cannot matter to it. It is called before s serves.
+5 -5
View File
@@ -56,7 +56,7 @@ func TestServerHasTheFixedLimits(t *testing.T) {
InstanceName: "test",
Level: slog.LevelInfo,
}),
})
}).HTTPServer()
if server.Addr != ":8080" || server.MaxHeaderBytes != 28<<10 ||
server.IdleTimeout != 2*time.Minute || server.ReadHeaderTimeout != time.Minute {
@@ -709,7 +709,7 @@ func TestABodyLeftUnreadIsReadOnlyUntilTheRequestTimeLimit(t *testing.T) {
before := time.Now()
server.Handler.ServeHTTP(w, req)
server.ServeHTTP(w, req)
if w.Code != http.StatusNoContent {
t.Errorf("status %d, want the app's 204", w.Code)
@@ -762,7 +762,7 @@ func TestABodyLeftUnreadIsReadNoMoreOnceTheTimeToSendTheAnswerRunsOut(
before := time.Now()
server.Handler.ServeHTTP(w, req)
server.ServeHTTP(w, req)
if w.Code != http.StatusOK {
t.Errorf("status %d, want the app's 200", w.Code)
@@ -815,7 +815,7 @@ func TestAnAnswerRProxyGivesItselfHasTheResponseTimeToBeSent(t *testing.T) {
before := time.Now()
server.Handler.ServeHTTP(w, req)
server.ServeHTTP(w, req)
if w.Code != tc.status {
t.Errorf("status %d, want %d", w.Code, tc.status)
@@ -856,7 +856,7 @@ func TestAnAnswerRProxyGivesItselfHasTenSecondsToBeSentAtLeast(
before := time.Now()
server.Handler.ServeHTTP(w, req)
server.ServeHTTP(w, req)
if w.writeDeadline.Before(before.Add(least)) {
t.Errorf(
+1 -1
View File
@@ -583,7 +583,7 @@ func TestAQuestionNamesNoZone(t *testing.T) {
req.RemoteAddr = tc.remoteAddr
req.Header.Set(forwardedForHeader, tc.forwardedFor)
server.Handler.ServeHTTP(httptest.NewRecorder(), req)
server.ServeHTTP(httptest.NewRecorder(), req)
asked := standIn.question(t)
if asked.header.Get(wafprotocol.ClientHeader) != tc.client ||
+39 -24
View File
@@ -16,11 +16,11 @@ import (
"io"
"log"
"log/slog"
"net"
"net/http"
"net/url"
"slices"
"strings"
"sync"
"time"
"sneak.berlin/go/smallwebwaf/internal/config"
@@ -64,9 +64,8 @@ type Params struct {
// Server is the server `sww rproxy` runs.
type Server struct {
*http.Server
h *handler
httpServer *http.Server
h *handler
}
// New returns the server `sww rproxy` runs. Go's server itself refuses
@@ -95,28 +94,47 @@ func New(params Params) *Server {
breakerClock: time.Now,
}
return &Server{
Server: &http.Server{
Addr: params.Config.ListenAddr,
Handler: h,
ErrorLog: errorLog,
ReadHeaderTimeout: params.Config.MaxRequestTime,
IdleTimeout: idleTimeout,
MaxHeaderBytes: requestHeaderMaxBytes - goServerExtraHeaderBytes,
},
h: h,
s := &Server{h: h}
s.httpServer = &http.Server{
Addr: params.Config.ListenAddr,
Handler: s,
ErrorLog: errorLog,
ReadHeaderTimeout: params.Config.MaxRequestTime,
IdleTimeout: idleTimeout,
MaxHeaderBytes: requestHeaderMaxBytes - goServerExtraHeaderBytes,
}
return s
}
// Serve serves requests on listener until Shutdown or Close, as Go's
// http.Server.Serve does.
func (s *Server) Serve(listener net.Listener) error {
return s.httpServer.Serve(listener)
}
// Shutdown stops serving once the requests in progress have finished, or
// when ctx is done, as Go's http.Server.Shutdown does.
func (s *Server) Shutdown(ctx context.Context) error {
return s.httpServer.Shutdown(ctx)
}
// Close stops serving at once and closes every connection, as Go's
// http.Server.Close does.
func (s *Server) Close() error {
return s.httpServer.Close()
}
// ServeHTTP handles one request from a visitor, as handler.ServeHTTP says.
func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) {
s.h.ServeHTTP(w, r)
}
// WaitForReports returns once the reports being sent have gone or failed,
// which SWW_RPROXY_WAF_BACKEND_TIMEOUT_MS bounds. Any report after this call
// is dropped: that of a WebSocket that closes while rproxy stops, for one.
func (s *Server) WaitForReports() {
s.h.reportsMu.Lock()
s.h.reportsStopped = true
s.h.reportsMu.Unlock()
s.h.reports.Wait()
s.h.reports.stopAndWait()
}
// handler is the proxy. It holds what every request shares; what belongs to
@@ -136,11 +154,8 @@ type handler struct {
// the time.
breaker breaker
breakerClock func() time.Time
// reports waits for the reports being sent. reportsMu guards starting one
// against reportsStopped, which is set once nothing is to wait for another.
reports sync.WaitGroup
reportsMu sync.Mutex
reportsStopped bool
// reports are the goroutines sending reports to waf.
reports reportGoroutines
}
// newTransport returns what carries requests to the app. It never goes through
+1 -1
View File
@@ -710,7 +710,7 @@ func TestAnAnswerWithoutAContentTypeGetsNone(t *testing.T) {
)
w := httptest.NewRecorder()
server.Handler.ServeHTTP(w, req)
server.ServeHTTP(w, req)
// Go's server guesses a type when the first part of the body goes before the
// headers are flushed, which is a matter of timing here. So this looks for what
+33 -8
View File
@@ -8,6 +8,7 @@ import (
"maps"
"net/http"
"strings"
"sync"
"sneak.berlin/go/smallwebwaf/internal/rules"
"sneak.berlin/go/smallwebwaf/internal/wafprotocol"
@@ -210,14 +211,7 @@ func (h *handler) sendReport(
ctx context.Context,
report *wafprotocol.Report,
) {
h.reportsMu.Lock()
defer h.reportsMu.Unlock()
if h.reportsStopped {
return
}
h.reports.Go(func() {
h.reports.startUnlessStopped(func() {
err := h.postReport(context.WithoutCancel(ctx), report)
if err != nil {
h.processLog.Warn("a report is dropped", "error", err.Error())
@@ -225,6 +219,37 @@ func (h *handler) sendReport(
})
}
// reportGoroutines are the goroutines sending reports to waf. None starts once
// stopAndWait has been called.
type reportGoroutines struct {
mu sync.Mutex
stopped bool
running sync.WaitGroup
}
// startUnlessStopped runs send on a goroutine of its own, or drops it once
// stopAndWait has been called.
func (g *reportGoroutines) startUnlessStopped(send func()) {
g.mu.Lock()
defer g.mu.Unlock()
if g.stopped {
return
}
g.running.Go(send)
}
// stopAndWait stops any more goroutines from starting, and returns once those
// running have ended.
func (g *reportGoroutines) stopAndWait() {
g.mu.Lock()
g.stopped = true
g.mu.Unlock()
g.running.Wait()
}
// postReport posts report to waf, and waits for its answer for up to
// SWW_RPROXY_WAF_BACKEND_TIMEOUT_MS.
func (h *handler) postReport(
+9 -1
View File
@@ -95,12 +95,20 @@ func background(loop func(context.Context)) fx.Hook {
)
}
// daemonServer is the server a daemon runs, waf.Server or rproxy.Server: the
// methods of Go's http.Server that serve starts and stops it with.
type daemonServer interface {
Serve(listener net.Listener) error
Shutdown(ctx context.Context) error
Close() error
}
// serve serves requests on listener from the start. At the stop it gives the
// requests in progress shutdownTimeout to finish, then closes their
// connections. When serving fails, it logs why and sends the error to
// failed.
func serve(
lc fx.Lifecycle, server *http.Server, listener net.Listener,
lc fx.Lifecycle, server daemonServer, listener net.Listener,
processLog *slog.Logger, failed servingFailed,
) {
served := make(chan error, 1)
+7 -3
View File
@@ -4,7 +4,6 @@ import (
"context"
"log/slog"
"net"
"net/http"
"os"
"go.uber.org/fx"
@@ -44,8 +43,13 @@ func RunRProxy(ctx context.Context, params Params) int {
app := fx.New(
fx.Supply(cfg, processLog, requestLog{params.Stdout}, failed),
fx.Provide(
newRProxyServer,
func(server *rproxy.Server) *http.Server { return server.Server },
// run waits for rproxy.Server's reports, and serve takes it as a
// daemonServer.
fx.Annotate(
newRProxyServer,
fx.As(fx.Self()),
fx.As(new(daemonServer)),
),
newRProxyListener,
),
fx.Invoke(serve),
+2 -3
View File
@@ -6,7 +6,6 @@ import (
"io"
"log/slog"
"net"
"net/http"
"time"
"go.uber.org/fx"
@@ -97,8 +96,8 @@ func RunWAF(ctx context.Context, params Params) int {
newAlertQueue, newRuleFiles, newCoreRuleSet, newLookupFile,
newLimiter, newLedger, newAnomalyCounters,
newLists, newDNSBL, newAbuseIPDB, newRouteWatch,
newMetrics, newGeoJS, newWAFServer,
func(server *waf.Server) *http.Server { return server.Server },
newMetrics, newGeoJS,
fx.Annotate(newWAFServer, fx.As(new(daemonServer))),
newWAFListener, newStateFiles,
),
// fx stops hooks in the reverse of the order they were added in. The
+56
View File
@@ -0,0 +1,56 @@
package waf
// The heuristic active_ban. It reads SWW_WAF_MODE; the bans are the ban
// ledger's.
import (
"context"
"time"
)
// activeBan refuses a client a ban covers: a ban on a netblock the client is
// in, in force at now. It notes for the log line when that ban ends. A request
// that makes the ban permanent, or in observe mode would have, raises the alert
// for it. Its figures are the ban's netblock and when it ends.
type activeBan struct {
banMaker banMaker
}
func (activeBan) name() string { return "active_ban" }
func (a activeBan) check(
_ context.Context,
rq *request,
now time.Time,
) *verdict {
ledger := a.banMaker.ledger
check := ledger.Check
if a.banMaker.observe {
check = ledger.Find // the ban refuses nothing, and stays as it is
}
ban, banned, madePermanent := check(rq.client, now)
var expires string
if banned {
expires = banExpires(ban)
rq.line.BanExpires = expires
}
if madePermanent {
ban.Expires = time.Time{} // the ban made permanent, which Find leaves as it is
a.banMaker.alertBan(rq, ban)
}
if !banned {
return nil
}
return &verdict{
action: banAction(ban.Permanent()),
figures: map[string]any{
figureNetblock: ban.Netblock.String(), "ban_expires": expires,
},
}
}
+1 -1
View File
@@ -717,7 +717,7 @@ func askDirectly(
recorder := httptest.NewRecorder()
began := time.Now()
server.Handler.ServeHTTP(recorder, req)
server.ServeHTTP(recorder, req)
return recorder.Code, time.Since(began)
}
+2 -50
View File
@@ -1,10 +1,9 @@
package waf
// The heuristic active_ban, and banMaker, which makes the bans of several
// heuristics. active_ban reads SWW_WAF_MODE; the bans are the ban ledger's.
// banMaker, which makes the bans of several heuristics. The bans are the ban
// ledger's.
import (
"context"
"net/netip"
"time"
@@ -22,53 +21,6 @@ func (rq *request) banResponse(action requestlog.Action) *refusal {
return &refusal{status: rq.h.config.BanResponse, action: action}
}
// activeBan refuses a client a ban covers: a ban on a netblock the client is
// in, in force at now. It notes for the log line when that ban ends. A request
// that makes the ban permanent, or in observe mode would have, raises the alert
// for it. Its figures are the ban's netblock and when it ends.
type activeBan struct {
banMaker banMaker
}
func (activeBan) name() string { return "active_ban" }
func (a activeBan) check(
_ context.Context,
rq *request,
now time.Time,
) *verdict {
ledger := a.banMaker.ledger
check := ledger.Check
if a.banMaker.observe {
check = ledger.Find // the ban refuses nothing, and stays as it is
}
ban, banned, madePermanent := check(rq.client, now)
var expires string
if banned {
expires = banExpires(ban)
rq.line.BanExpires = expires
}
if madePermanent {
ban.Expires = time.Time{} // the ban made permanent, which Find leaves as it is
a.banMaker.alertBan(rq, ban)
}
if !banned {
return nil
}
return &verdict{
action: banAction(ban.Permanent()),
figures: map[string]any{
figureNetblock: ban.Netblock.String(), "ban_expires": expires,
},
}
}
// banAction is the action of a request refused under a ban or for making one:
// permanent says the ban never runs out.
func banAction(permanent bool) requestlog.Action {
+1 -1
View File
@@ -592,7 +592,7 @@ func serveFromDE(
req.Header.Set(wafprotocol.ClientHeader, fromDE)
req.Header.Set(requestIDHeader, "the-id")
server.Handler.ServeHTTP(httptest.NewRecorder(), req)
server.ServeHTTP(httptest.NewRecorder(), req)
}
// wantPercent checks a limit percentage that a log line or a ban's notes
+7
View File
@@ -1,4 +1,11 @@
package waf
import "net/http"
// BesideLookupTimeLimit is besideLookupTimeLimit.
const BesideLookupTimeLimit = besideLookupTimeLimit
// HTTPServer is Go's server s holds, for tests that read its settings.
func (s *Server) HTTPServer() *http.Server {
return s.httpServer
}
+20
View File
@@ -80,6 +80,26 @@ func (rq *request) startLookUp(
return result
}
// refusedWhateverTheAnswer reports whether the request is one that does not wait
// for GeoJS, since the checks will refuse it at now whatever GeoJS answers about
// its client: a heuristic of afterLookup that implements refusesWithoutLookup
// says it will refuse it. In observe mode, which refuses nothing, none is.
// rproxy lets a request through when waf answers too late.
func (rq *request) refusedWhateverTheAnswer(now time.Time) bool {
if rq.h.config.Observe {
return false
}
for _, each := range rq.h.afterLookup {
knows, ok := each.(refusesWithoutLookup)
if ok && knows.refusesWhateverTheLookupAnswers(rq, now) {
return true
}
}
return false
}
// noteLookup notes result for the log line and as the request's asn= and
// country= tags.
func (rq *request) noteLookup(result lookupResult) {
-321
View File
@@ -1,321 +0,0 @@
// Package waf is the server the waf command runs. It answers
// rproxy's questions, takes rproxy's reports and answers its own endpoints, and
// writes a log line for each. It also bans the clients on AbuseIPDB's
// blacklist each time the reputation package fetches it.
package waf
import (
"cmp"
"io"
"log/slog"
"net/http"
"net/netip"
"time"
lru "github.com/hashicorp/golang-lru/v2"
"sneak.berlin/go/smallwebwaf/internal/alerts"
"sneak.berlin/go/smallwebwaf/internal/anomaly"
"sneak.berlin/go/smallwebwaf/internal/bans"
"sneak.berlin/go/smallwebwaf/internal/config"
"sneak.berlin/go/smallwebwaf/internal/coreruleset"
"sneak.berlin/go/smallwebwaf/internal/geojs"
"sneak.berlin/go/smallwebwaf/internal/lookup"
"sneak.berlin/go/smallwebwaf/internal/metrics"
"sneak.berlin/go/smallwebwaf/internal/ratelimit"
"sneak.berlin/go/smallwebwaf/internal/reputation"
"sneak.berlin/go/smallwebwaf/internal/routewatch"
"sneak.berlin/go/smallwebwaf/internal/rules"
)
// MetricsPath is where the metrics are, for a request that carries
// SWW_WAF_METRICS_TOKEN.
const MetricsPath = "/_sww/metrics"
// BansPath is where an admin lists and adds bans, and, followed by / and
// a client's address, lifts them, with SWW_WAF_ADMIN_TOKEN.
const BansPath = "/_sww/bans"
// ClientsPath is where an admin asks what waf knows of a client, by
// the client's address after it, with SWW_WAF_ADMIN_TOKEN.
const ClientsPath = "/_sww/clients/"
// Params are what New needs.
type Params struct {
Config *config.WAF
// RequestLog receives two JSON lines per question, one at its decision and
// one when its report comes, and one per request to waf's own
// endpoints.
RequestLog io.Writer
// ProcessLog receives the process's own messages.
ProcessLog *slog.Logger
// Metrics count what waf does. Whoever builds them adds the metrics
// read from the other parts.
Metrics *metrics.Metrics
// Limiter is the table of clients the rate limits count, with each
// client's history.
Limiter *ratelimit.Limiter
// Ledger holds the bans.
Ledger *bans.Ledger
// Anomalies are the counters of the anomaly thresholds.
Anomalies *anomaly.Counters
// Lists are the lists fetched from URLs, DNSBL the DNSBL zones'
// verdicts, and AbuseIPDB AbuseIPDB's scores and the checks spent.
Lists *reputation.Lists
DNSBL *reputation.DNSBL
AbuseIPDB *reputation.AbuseIPDB
// RouteWatch asks routewatch about clients, and is nil while
// SWW_WAF_ROUTEWATCH_URL is unset.
RouteWatch *routewatch.Client
// GeoJS looks clients' AS numbers and countries up while
// SWW_WAF_LOOKUP_SOURCE is geojs. It is given AddLookup as its Answered
// and GeoJSFailed as its Failed.
GeoJS *geojs.GeoJS
// LookupFile is the lookup database they are looked up in while
// SWW_WAF_LOOKUP_SOURCE is file, and nil otherwise.
LookupFile *lookup.File
// CoreRuleSet is what inspects requests while
// SWW_WAF_CORE_RULE_SET_MODE is not off, and nil while it is.
CoreRuleSet CoreRuleSetInspector
// Now tells the time by which requests are counted for the rate
// limits, bans are made and run out, and GeoJS's answers are kept,
// normally time.Now in UTC, the time the state files give.
Now func() time.Time
// Rules are the rule files' rules, which each request is checked
// against.
Rules *rules.Files
// Alerts receive the alert for each ban waf makes or makes
// permanent, for each count over an anomaly threshold, for each request
// whose client a blocklist, a DNSBL zone or AbuseIPDB lists, for each
// request the Core Rule Set scores at or over
// SWW_WAF_CORE_RULE_SET_SCORE_THRESHOLD, and for GeoJS failing, a fetch of a list
// failing, a query to a DNSBL zone or a check with AbuseIPDB failing,
// or the day's AbuseIPDB checks used up.
Alerts *alerts.Queue
// BesideLookupTimeLimit is how long the heuristics of besideLookup have
// for a request. Zero means besideLookupTimeLimit. Tests that run the
// Core Rule Set lengthen it, since the race detector and a busy host can
// slow an inspection past besideLookupTimeLimit.
BesideLookupTimeLimit time.Duration
}
// CoreRuleSetInspector inspects a request with the Core Rule Set, as
// coreruleset.CoreRuleSet's Inspect says. Tests give waf a stand-in for it.
type CoreRuleSetInspector interface {
Inspect(
r *http.Request, client netip.Addr, body io.Reader,
) (coreruleset.Result, []byte, error)
}
// Server is the server waf runs.
type Server struct {
*http.Server
}
// newHandler returns the handler with what every request shares, the parts
// in params among it. It builds the heuristics, each with the settings and
// parts it reads, and the lists they run in.
//
//nolint:funlen // a few lines for each heuristic, a list that grows with them
func newHandler(params Params) *handler {
awaitingReport, err := lru.New[string, *decision](maxAwaitingReport)
if err != nil {
panic(err) // New fails only for a size below one
}
cfg := params.Config
h := &handler{
config: cfg,
requestLog: params.RequestLog,
processLog: params.ProcessLog,
now: params.Now,
metrics: params.Metrics,
awaitingReport: awaitingReport,
limiter: params.Limiter,
ledger: params.Ledger,
geojs: params.GeoJS,
anomalies: params.Anomalies,
lookupFile: params.LookupFile,
// The country list, the headers, the tag settings that name an asn=
// or country= tag and the biased thresholds act on the answer before
// the request goes on.
waitForGeoJS: len(cfg.ExclusivelyAllowedCountries) > 0 ||
cfg.AddLookupHeaders ||
cfg.TagsNeedLookup() ||
biasedThresholdsSet(cfg),
besideLookupLimit: cmp.Or(
params.BesideLookupTimeLimit,
besideLookupTimeLimit,
),
}
sharedBanMaker := banMaker{
ledger: params.Ledger,
limiter: params.Limiter,
metrics: params.Metrics,
alerts: params.Alerts,
observe: cfg.Observe,
banResponse: cfg.BanResponse,
banScopeV4Prefix: cfg.BanScopeV4Prefix,
ipv6GroupPrefix: cfg.IPv6GroupPrefix,
}
hitNoter := reputationHitNoter{
metrics: params.Metrics,
alerts: params.Alerts,
ipv6GroupPrefix: cfg.IPv6GroupPrefix,
}
h.trapPaths = trapPaths{paths: cfg.TrapPaths, banMaker: sharedBanMaker}
h.ruleFiles = ruleFiles{
rules: params.Rules,
metrics: params.Metrics,
banMaker: sharedBanMaker,
}
h.coreRuleSet = coreRuleSet{
inspector: params.CoreRuleSet,
metrics: params.Metrics,
alerts: params.Alerts,
mode: cfg.CoreRuleSetMode,
exemptPaths: cfg.CoreRuleSetExemptPaths,
scoreThreshold: cfg.CoreRuleSetScoreThreshold,
observe: cfg.Observe,
ipv6GroupPrefix: cfg.IPv6GroupPrefix,
}
h.beforeLookup = []heuristic{
headerLimits{},
tagLists{lists: params.Lists},
routeWatchTags{
routeWatch: params.RouteWatch,
ipv6GroupPrefix: cfg.IPv6GroupPrefix,
},
denyNets{netblocks: cfg.DenyNets},
activeBan{banMaker: sharedBanMaker},
}
h.afterLookup = []heuristic{
countries{allowed: cfg.ExclusivelyAllowedCountries},
blocklists{
lists: params.Lists,
action: cfg.BlocklistAction,
hitNoter: hitNoter,
},
dnsbl{
dnsbl: params.DNSBL,
action: cfg.DNSBLAbuseIPDBAction,
hitNoter: hitNoter,
},
abuseIPDB{
abuseIPDB: params.AbuseIPDB,
limiter: params.Limiter,
keySet: cfg.AbuseIPDBKey != "",
minScore: cfg.AbuseIPDBMinScore,
action: cfg.DNSBLAbuseIPDBAction,
ipv6GroupPrefix: cfg.IPv6GroupPrefix,
hitNoter: hitNoter,
},
tagDeny{tags: cfg.TagDeny},
rateLimits{
limiter: params.Limiter,
exemptNets: cfg.RateLimitExemptNets,
exemptPaths: cfg.RateLimitExemptPaths,
banResponse: cfg.BanResponse,
ipv6GroupPrefix: cfg.IPv6GroupPrefix,
biasedThresholds: biasedThresholds{
lists: params.Lists,
tagLimitPercent: cfg.TagLimitPercent,
asnBytesLimitPercent: cfg.ASNBytesLimitPercent,
countryBytesLimitPercent: cfg.CountryBytesLimitPercent,
unknownCountryLimitPercent: cfg.UnknownCountryLimitPercent,
blocklistAction: cfg.BlocklistAction,
blocklistLimitPercent: cfg.BlocklistLimitPercent,
dnsblAbuseIPDBAction: cfg.DNSBLAbuseIPDBAction,
dnsblAbuseIPDBLimitPercent: cfg.DNSBLAbuseIPDBLimitPercent,
},
banMaker: sharedBanMaker,
},
}
h.besideLookup = []heuristic{h.trapPaths, h.ruleFiles, h.coreRuleSet}
h.afterBesideLookup = []heuristic{tagDeny{tags: cfg.TagDeny}}
h.afterAnswer = []heuristic{
refusalLimit{
limiter: params.Limiter,
perMinute: cfg.RefusalsLimitPerMinute,
banResponse: cfg.BanResponse,
ipv6GroupPrefix: cfg.IPv6GroupPrefix,
banMaker: sharedBanMaker,
},
}
h.onReport = []heuristic{
byteLimits{
limiter: params.Limiter,
bytesCounted: cfg.BytesCounted,
ipv6GroupPrefix: cfg.IPv6GroupPrefix,
banMaker: sharedBanMaker,
},
}
h.ownEndpoints = h.routes()
return h
}
// handler is waf's handler. It holds what every request shares; what
// belongs to one request is in a request.
type handler struct {
config *config.WAF
requestLog io.Writer
processLog *slog.Logger
now func() time.Time
metrics *metrics.Metrics
limiter *ratelimit.Limiter
ledger *bans.Ledger
geojs *geojs.GeoJS
anomalies *anomaly.Counters
lookupFile *lookup.File
// waitForGeoJS is true when a setting needs each request's answer from
// GeoJS before the request goes on.
waitForGeoJS bool
// besideLookupLimit is how long the heuristics of besideLookup have for a
// request, as Params.BesideLookupTimeLimit says.
besideLookupLimit time.Duration
// awaitingReport holds the decision on each answered question by its id
// until its report comes, and nil for a question still being handled.
awaitingReport *lru.Cache[string, *decision]
// ownEndpoints answers every request that is not a question, as routes
// lists the endpoints.
ownEndpoints http.Handler
// The lists of heuristics, each in the order its heuristics run. A client
// in SWW_WAF_ALLOW_NETS skips them all.
// beforeLookup are the heuristics that run before the client's AS number
// and country are looked up, so that a client they refuse is not looked
// up. header_limits comes first: it only measures and counts, and a
// request it refuses causes no other work.
beforeLookup []heuristic
// afterLookup are the heuristics that run once the lookup has been made.
afterLookup []heuristic
// besideLookup are the heuristics that only read the request. Their work
// runs beside the lookup and afterLookup, in findBesideLookup, which
// changes nothing of the request. Their check then applies what that work
// found, and runs only once waf takes it.
besideLookup []heuristic
// afterBesideLookup are the heuristics that run once afterLookup and
// besideLookup have both let the request through. tag_deny runs a second
// time, on the tag rule_files and core_rule_set can add.
afterBesideLookup []heuristic
// afterAnswer are the heuristics that run on a request once waf has
// answered it, before its decision line is written. They ban the client,
// and do not change what the request got: their verdict's action is not
// used.
afterAnswer []heuristic
// onReport are the heuristics that run on an allowed request when
// rproxy's report of it comes, before its report line is written. Like
// afterAnswer, they only ban.
onReport []heuristic
// The heuristics of besideLookup, whose work findBesideLookup does.
trapPaths trapPaths
ruleFiles ruleFiles
coreRuleSet coreRuleSet
}
+6 -3
View File
@@ -344,8 +344,8 @@ func startProxyWithAlerts(
return startRProxy(t, server.Addr, appURL, env), out, server, alertQueue
}
// serve serves server on a port of its own until the test ends, and sets its
// Addr to where it listens.
// serve serves server on a port of its own until the test ends, and sets
// server.Addr to where it listens.
func serve(t *testing.T, server *wafAndParts) {
t.Helper()
@@ -423,10 +423,13 @@ func startRProxy(
}
// wafAndParts is waf's server with the Params it was built from, so that a
// test can read the parts the server uses.
// test can read the parts the server uses. Addr is where it listens once serve
// has it serving.
type wafAndParts struct {
*waf.Server
waf.Params
Addr string
}
// banAbuseIPDBBlacklist does on w what a fetch of AbuseIPDB's blacklist with
+235
View File
@@ -0,0 +1,235 @@
package waf
import (
"context"
"errors"
"net/http"
"strings"
"time"
"sneak.berlin/go/smallwebwaf/internal/requestlog"
"sneak.berlin/go/smallwebwaf/internal/wafprotocol"
)
// answerQuestion runs the checks on the visitor's request a question
// carries, and answers rproxy what to do with it. A denied request's counting
// is finished here, and an allowed one's by its report.
func (h *handler) answerQuestion(w http.ResponseWriter, r *http.Request) {
question, ok := h.readQuestion(w, r)
if !ok {
return
}
rq := h.startRequest(
w,
question.Visitor,
question.Client,
question.Peer,
true,
)
// These run even when a check panics. finish runs first, so that what is
// held has the finished log line. Go's server sends the answer only once
// this function has returned, so the report cannot come before it is held.
defer func() {
rq.finish()
rq.holdForReport()
}()
rq.site = r.Host
rq.line.WAFLine = requestlog.WAFLineDecision
refused := rq.check(r.Context())
rq.checked = time.Now()
underOwnPrefix := strings.HasPrefix(
r.URL.Path,
wafprotocol.OwnEndpointPrefix,
)
if refused == nil && !underOwnPrefix {
rq.allow()
// In observe mode, what enforce mode would have refused counts here.
rq.runBanningHeuristics(r.Context(), rq.h.afterAnswer)
return
}
if refused != nil {
rq.deny(*refused)
} else {
// The app never sees this prefix. waf's own endpoints are at its
// own address, so this answers as a path with no endpoint does.
rq.line.Action = requestlog.ActionOwnEndpoint
wafprotocol.SetAnswerHeaders(
rq.out.Header(),
wafprotocol.Answer{Disposition: wafprotocol.DispositionDeny},
)
http.Error(
rq.out,
http.StatusText(http.StatusNotFound),
http.StatusNotFound,
)
}
rq.banAndCount(r.Context(), rq.h.afterAnswer)
}
// readQuestion reads a question, and reports whether waf takes it. It answers
// a question it does not take with 400 and no disposition. rproxy passes that
// 400 to the visitor, and SWW_RPROXY_WAF_BACKEND_NO_ANSWER_DISPOSITION does
// not apply. The id of a question waf takes is held from here on, without a
// decision yet, so that another question with that id is refused while this
// one is being handled.
//
// A question whose body stops arriving gets no answer, and is not looked at.
func (h *handler) readQuestion(
w http.ResponseWriter, r *http.Request,
) (wafprotocol.Question, bool) {
question, err := wafprotocol.ReadQuestion(w, r)
if errors.Is(err, wafprotocol.ErrBodyStoppedArriving) {
h.processLog.Warn(
"a question's body stopped arriving",
"error",
err.Error(),
)
// This panic has Go's server close the connection without an answer,
// and log nothing.
panic(http.ErrAbortHandler)
}
if err != nil {
h.refuseQuestion(w, err.Error())
return question, false
}
held, _ := h.awaitingReport.ContainsOrAdd(question.RequestID, nil)
if held {
h.refuseQuestion(
w,
wafprotocol.RequestIDHeader+" is the id of a request waf holds",
)
return question, false
}
return question, true
}
// refuseQuestion answers a question waf does not take, for reason.
func (h *handler) refuseQuestion(w http.ResponseWriter, reason string) {
h.processLog.Warn("a question is refused", "reason", reason)
http.Error(w, reason, http.StatusBadRequest)
}
// allow answers a question with wafprotocol.DispositionAllow and, while
// SWW_WAF_ADD_LOOKUP_HEADERS is true, the client's AS number and country.
func (rq *request) allow() {
rq.allowed = true
answer := wafprotocol.Answer{Disposition: wafprotocol.DispositionAllow}
if rq.h.config.AddLookupHeaders {
answer.ASN = rq.lookupAnswer.ASN
answer.Country = rq.lookupAnswer.Country
}
wafprotocol.SetAnswerHeaders(rq.out.Header(), answer)
rq.out.WriteHeader(http.StatusOK)
}
// holdForReport holds a copy of what the report's steps need, by the request's
// id. A report only ever reaches that copy, never the request being handled.
func (rq *request) holdForReport() {
held := rq.decision
rq.h.awaitingReport.Add(rq.line.RequestID, &held)
}
// deny answers a question with what the visitor is to get for r.
func (rq *request) deny(r refusal) {
rq.refused = &r
if r.status == 0 {
// SWW_WAF_BAN_RESPONSE is close. Written past rq.out, since the log line's
// status stays 0: the visitor is sent nothing.
wafprotocol.SetAnswerHeaders(
rq.out.Header(),
wafprotocol.Answer{Disposition: wafprotocol.DispositionClose},
)
rq.out.ResponseWriter.WriteHeader(http.StatusOK)
return
}
wafprotocol.SetAnswerHeaders(
rq.out.Header(),
wafprotocol.Answer{Disposition: wafprotocol.DispositionDeny},
)
http.Error(rq.out, http.StatusText(r.status), r.status)
}
// handleReport answers POST wafprotocol.ReportPath: it finishes the request
// the report names, if it holds that request's decision, and ignores the
// report otherwise.
func (h *handler) handleReport() http.HandlerFunc {
return func(w http.ResponseWriter, r *http.Request) {
report, err := wafprotocol.ReadReport(w, r)
if err != nil {
h.processLog.Warn("a report is refused", "error", err.Error())
http.Error(w, "the body is not a report", http.StatusBadRequest)
return
}
held, found := h.awaitingReport.Peek(report.RequestID)
// Remove reports true to one report only, of several that name the
// id. A question still being handled is held as nil.
if found && held != nil &&
h.awaitingReport.Remove(report.RequestID) {
h.finishWith(r.Context(), held, report)
}
w.WriteHeader(http.StatusNoContent)
}
}
// finishWith finishes a request with what its report says, and writes the
// request's last log line, which holds its decision's fields and the report's.
// For an allowed request it first runs the heuristics of onReport, and adds the
// request to its client's history, the anomaly counters and the metrics, with
// the status and bytes the report gives. A denied request's counting was
// finished at its decision. ctx is the report's own context.
func (h *handler) finishWith(
ctx context.Context,
held *decision,
report wafprotocol.Report,
) {
rq := &request{
decision: *held,
h: h,
out: &responseWriter{
status: report.Status,
bytes: report.ResponseBytes,
},
requestBytes: report.RequestBytes,
}
rq.line.WAFLine = requestlog.WAFLineReport
rq.line.Status = report.Status
rq.line.RequestBytes = report.RequestBytes
rq.line.ResponseBytes = report.ResponseBytes
rq.line.DurationTotalMS = report.DurationTotalMS
if rq.allowed {
rq.line.DurationUpstreamTotalMS = &report.DurationUpstreamTotalMS
rq.banAndCount(ctx, rq.h.onReport)
rq.h.metrics.RequestReported(
&rq.line,
time.Duration(report.DurationTotalMS*float64(time.Millisecond)),
time.Duration(
report.DurationUpstreamTotalMS*float64(time.Millisecond),
),
)
}
rq.writeLine()
}
+350 -257
View File
@@ -1,17 +1,48 @@
// Package waf is the server the waf command runs. It answers
// rproxy's questions, takes rproxy's reports and answers its own endpoints, and
// writes a log line for each. It also bans the clients on AbuseIPDB's
// blacklist each time the reputation package fetches it.
package waf
import (
"cmp"
"context"
"errors"
"io"
"log/slog"
"net"
"net/http"
"strings"
"net/netip"
"time"
"sneak.berlin/go/smallwebwaf/internal/requestlog"
lru "github.com/hashicorp/golang-lru/v2"
"sneak.berlin/go/smallwebwaf/internal/alerts"
"sneak.berlin/go/smallwebwaf/internal/anomaly"
"sneak.berlin/go/smallwebwaf/internal/bans"
"sneak.berlin/go/smallwebwaf/internal/config"
"sneak.berlin/go/smallwebwaf/internal/coreruleset"
"sneak.berlin/go/smallwebwaf/internal/geojs"
"sneak.berlin/go/smallwebwaf/internal/lookup"
"sneak.berlin/go/smallwebwaf/internal/metrics"
"sneak.berlin/go/smallwebwaf/internal/ratelimit"
"sneak.berlin/go/smallwebwaf/internal/reputation"
"sneak.berlin/go/smallwebwaf/internal/routewatch"
"sneak.berlin/go/smallwebwaf/internal/rules"
"sneak.berlin/go/smallwebwaf/internal/wafprotocol"
)
// MetricsPath is where the metrics are, for a request that carries
// SWW_WAF_METRICS_TOKEN.
const MetricsPath = "/_sww/metrics"
// BansPath is where an admin lists and adds bans, and, followed by / and
// a client's address, lifts them, with SWW_WAF_ADMIN_TOKEN.
const BansPath = "/_sww/bans"
// ClientsPath is where an admin asks what waf knows of a client, by
// the client's address after it, with SWW_WAF_ADMIN_TOKEN.
const ClientsPath = "/_sww/clients/"
const (
// maxAwaitingReport is the most answered requests held until their
// reports come. Past it, the request answered longest ago is dropped.
@@ -23,6 +54,79 @@ const (
wafIdleTimeout = 120 * time.Second
)
// Params are what New needs.
type Params struct {
Config *config.WAF
// RequestLog receives two JSON lines per question, one at its decision and
// one when its report comes, and one per request to waf's own
// endpoints.
RequestLog io.Writer
// ProcessLog receives the process's own messages.
ProcessLog *slog.Logger
// Metrics count what waf does. Whoever builds them adds the metrics
// read from the other parts.
Metrics *metrics.Metrics
// Limiter is the table of clients the rate limits count, with each
// client's history.
Limiter *ratelimit.Limiter
// Ledger holds the bans.
Ledger *bans.Ledger
// Anomalies are the counters of the anomaly thresholds.
Anomalies *anomaly.Counters
// Lists are the lists fetched from URLs, DNSBL the DNSBL zones'
// verdicts, and AbuseIPDB AbuseIPDB's scores and the checks spent.
Lists *reputation.Lists
DNSBL *reputation.DNSBL
AbuseIPDB *reputation.AbuseIPDB
// RouteWatch asks routewatch about clients, and is nil while
// SWW_WAF_ROUTEWATCH_URL is unset.
RouteWatch *routewatch.Client
// GeoJS looks clients' AS numbers and countries up while
// SWW_WAF_LOOKUP_SOURCE is geojs. It is given AddLookup as its Answered
// and GeoJSFailed as its Failed.
GeoJS *geojs.GeoJS
// LookupFile is the lookup database they are looked up in while
// SWW_WAF_LOOKUP_SOURCE is file, and nil otherwise.
LookupFile *lookup.File
// CoreRuleSet is what inspects requests while
// SWW_WAF_CORE_RULE_SET_MODE is not off, and nil while it is.
CoreRuleSet CoreRuleSetInspector
// Now tells the time by which requests are counted for the rate
// limits, bans are made and run out, and GeoJS's answers are kept,
// normally time.Now in UTC, the time the state files give.
Now func() time.Time
// Rules are the rule files' rules, which each request is checked
// against.
Rules *rules.Files
// Alerts receive the alert for each ban waf makes or makes
// permanent, for each count over an anomaly threshold, for each request
// whose client a blocklist, a DNSBL zone or AbuseIPDB lists, for each
// request the Core Rule Set scores at or over
// SWW_WAF_CORE_RULE_SET_SCORE_THRESHOLD, and for GeoJS failing, a fetch of a list
// failing, a query to a DNSBL zone or a check with AbuseIPDB failing,
// or the day's AbuseIPDB checks used up.
Alerts *alerts.Queue
// BesideLookupTimeLimit is how long the heuristics of besideLookup have
// for a request. Zero means besideLookupTimeLimit. Tests that run the
// Core Rule Set lengthen it, since the race detector and a busy host can
// slow an inspection past besideLookupTimeLimit.
BesideLookupTimeLimit time.Duration
}
// CoreRuleSetInspector inspects a request with the Core Rule Set, as
// coreruleset.CoreRuleSet's Inspect says. Tests give waf a stand-in for it.
type CoreRuleSetInspector interface {
Inspect(
r *http.Request, client netip.Addr, body io.Reader,
) (coreruleset.Result, []byte, error)
}
// Server is the server waf runs.
type Server struct {
httpServer *http.Server
h *handler
}
// New returns the server `sww waf` runs. It answers
// rproxy's questions, takes rproxy's reports and answers its own endpoints. It
// passes nothing to an app. rproxy applies the size and time limits to the
@@ -30,19 +134,251 @@ const (
// is far above rproxy's, and a question refused for its size would have rproxy
// let the request through.
func New(params Params) *Server {
return &Server{
Server: &http.Server{
Addr: params.Config.ListenAddr,
Handler: newHandler(params),
ErrorLog: slog.NewLogLogger(
params.ProcessLog.Handler(),
slog.LevelWarn,
),
ReadTimeout: wafTimeout,
WriteTimeout: wafTimeout,
IdleTimeout: wafIdleTimeout,
s := &Server{h: newHandler(params)}
s.httpServer = &http.Server{
Addr: params.Config.ListenAddr,
Handler: s,
ErrorLog: slog.NewLogLogger(
params.ProcessLog.Handler(),
slog.LevelWarn,
),
ReadTimeout: wafTimeout,
WriteTimeout: wafTimeout,
IdleTimeout: wafIdleTimeout,
}
return s
}
// Serve serves requests on listener until Shutdown or Close, as Go's
// http.Server.Serve does.
func (s *Server) Serve(listener net.Listener) error {
return s.httpServer.Serve(listener)
}
// Shutdown stops serving once the requests in progress have finished, or
// when ctx is done, as Go's http.Server.Shutdown does.
func (s *Server) Shutdown(ctx context.Context) error {
return s.httpServer.Shutdown(ctx)
}
// Close stops serving at once and closes every connection, as Go's
// http.Server.Close does.
func (s *Server) Close() error {
return s.httpServer.Close()
}
// ServeHTTP handles one request to waf, as handler.ServeHTTP says.
func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) {
s.h.ServeHTTP(w, r)
}
// newHandler returns the handler with what every request shares, the parts
// in params among it. It builds the heuristics, each with the settings and
// parts it reads, and the lists they run in.
//
//nolint:funlen // a few lines for each heuristic, a list that grows with them
func newHandler(params Params) *handler {
awaitingReport, err := lru.New[string, *decision](maxAwaitingReport)
if err != nil {
panic(err) // New fails only for a size below one
}
cfg := params.Config
h := &handler{
config: cfg,
requestLog: params.RequestLog,
processLog: params.ProcessLog,
now: params.Now,
metrics: params.Metrics,
awaitingReport: awaitingReport,
limiter: params.Limiter,
ledger: params.Ledger,
geojs: params.GeoJS,
anomalies: params.Anomalies,
lookupFile: params.LookupFile,
// The country list, the headers, the tag settings that name an asn=
// or country= tag and the biased thresholds act on the answer before
// the request goes on.
waitForGeoJS: len(cfg.ExclusivelyAllowedCountries) > 0 ||
cfg.AddLookupHeaders ||
cfg.TagsNeedLookup() ||
biasedThresholdsSet(cfg),
besideLookupLimit: cmp.Or(
params.BesideLookupTimeLimit,
besideLookupTimeLimit,
),
}
sharedBanMaker := banMaker{
ledger: params.Ledger,
limiter: params.Limiter,
metrics: params.Metrics,
alerts: params.Alerts,
observe: cfg.Observe,
banResponse: cfg.BanResponse,
banScopeV4Prefix: cfg.BanScopeV4Prefix,
ipv6GroupPrefix: cfg.IPv6GroupPrefix,
}
hitNoter := reputationHitNoter{
metrics: params.Metrics,
alerts: params.Alerts,
ipv6GroupPrefix: cfg.IPv6GroupPrefix,
}
h.trapPaths = trapPaths{paths: cfg.TrapPaths, banMaker: sharedBanMaker}
h.ruleFiles = ruleFiles{
rules: params.Rules,
metrics: params.Metrics,
banMaker: sharedBanMaker,
}
h.coreRuleSet = coreRuleSet{
inspector: params.CoreRuleSet,
metrics: params.Metrics,
alerts: params.Alerts,
mode: cfg.CoreRuleSetMode,
exemptPaths: cfg.CoreRuleSetExemptPaths,
scoreThreshold: cfg.CoreRuleSetScoreThreshold,
observe: cfg.Observe,
ipv6GroupPrefix: cfg.IPv6GroupPrefix,
}
h.beforeLookup = []heuristic{
headerLimits{},
tagLists{lists: params.Lists},
routeWatchTags{
routeWatch: params.RouteWatch,
ipv6GroupPrefix: cfg.IPv6GroupPrefix,
},
denyNets{netblocks: cfg.DenyNets},
activeBan{banMaker: sharedBanMaker},
}
h.afterLookup = []heuristic{
countries{allowed: cfg.ExclusivelyAllowedCountries},
blocklists{
lists: params.Lists,
action: cfg.BlocklistAction,
hitNoter: hitNoter,
},
dnsbl{
dnsbl: params.DNSBL,
action: cfg.DNSBLAbuseIPDBAction,
hitNoter: hitNoter,
},
abuseIPDB{
abuseIPDB: params.AbuseIPDB,
limiter: params.Limiter,
keySet: cfg.AbuseIPDBKey != "",
minScore: cfg.AbuseIPDBMinScore,
action: cfg.DNSBLAbuseIPDBAction,
ipv6GroupPrefix: cfg.IPv6GroupPrefix,
hitNoter: hitNoter,
},
tagDeny{tags: cfg.TagDeny},
rateLimits{
limiter: params.Limiter,
exemptNets: cfg.RateLimitExemptNets,
exemptPaths: cfg.RateLimitExemptPaths,
banResponse: cfg.BanResponse,
ipv6GroupPrefix: cfg.IPv6GroupPrefix,
biasedThresholds: biasedThresholds{
lists: params.Lists,
tagLimitPercent: cfg.TagLimitPercent,
asnBytesLimitPercent: cfg.ASNBytesLimitPercent,
countryBytesLimitPercent: cfg.CountryBytesLimitPercent,
unknownCountryLimitPercent: cfg.UnknownCountryLimitPercent,
blocklistAction: cfg.BlocklistAction,
blocklistLimitPercent: cfg.BlocklistLimitPercent,
dnsblAbuseIPDBAction: cfg.DNSBLAbuseIPDBAction,
dnsblAbuseIPDBLimitPercent: cfg.DNSBLAbuseIPDBLimitPercent,
},
banMaker: sharedBanMaker,
},
}
h.besideLookup = []heuristic{h.trapPaths, h.ruleFiles, h.coreRuleSet}
h.afterBesideLookup = []heuristic{tagDeny{tags: cfg.TagDeny}}
h.afterAnswer = []heuristic{
refusalLimit{
limiter: params.Limiter,
perMinute: cfg.RefusalsLimitPerMinute,
banResponse: cfg.BanResponse,
ipv6GroupPrefix: cfg.IPv6GroupPrefix,
banMaker: sharedBanMaker,
},
}
h.onReport = []heuristic{
byteLimits{
limiter: params.Limiter,
bytesCounted: cfg.BytesCounted,
ipv6GroupPrefix: cfg.IPv6GroupPrefix,
banMaker: sharedBanMaker,
},
}
h.ownEndpoints = h.routes()
return h
}
// handler is waf's handler. It holds what every request shares; what
// belongs to one request is in a request.
type handler struct {
config *config.WAF
requestLog io.Writer
processLog *slog.Logger
now func() time.Time
metrics *metrics.Metrics
limiter *ratelimit.Limiter
ledger *bans.Ledger
geojs *geojs.GeoJS
anomalies *anomaly.Counters
lookupFile *lookup.File
// waitForGeoJS is true when a setting needs each request's answer from
// GeoJS before the request goes on.
waitForGeoJS bool
// besideLookupLimit is how long the heuristics of besideLookup have for a
// request, as Params.BesideLookupTimeLimit says.
besideLookupLimit time.Duration
// awaitingReport holds the decision on each answered question by its id
// until its report comes, and nil for a question still being handled.
awaitingReport *lru.Cache[string, *decision]
// ownEndpoints answers every request that is not a question, as routes
// lists the endpoints.
ownEndpoints http.Handler
// The lists of heuristics, each in the order its heuristics run. A client
// in SWW_WAF_ALLOW_NETS skips them all.
// beforeLookup are the heuristics that run before the client's AS number
// and country are looked up, so that a client they refuse is not looked
// up. header_limits comes first: it only measures and counts, and a
// request it refuses causes no other work.
beforeLookup []heuristic
// afterLookup are the heuristics that run once the lookup has been made.
afterLookup []heuristic
// besideLookup are the heuristics that only read the request. Their work
// runs beside the lookup and afterLookup, in findBesideLookup, which
// changes nothing of the request. Their check then applies what that work
// found, and runs only once waf takes it.
besideLookup []heuristic
// afterBesideLookup are the heuristics that run once afterLookup and
// besideLookup have both let the request through. tag_deny runs a second
// time, on the tag rule_files and core_rule_set can add.
afterBesideLookup []heuristic
// afterAnswer are the heuristics that run on a request once waf has
// answered it, before its decision line is written. They ban the client,
// and do not change what the request got: their verdict's action is not
// used.
afterAnswer []heuristic
// onReport are the heuristics that run on an allowed request when
// rproxy's report of it comes, before its report line is written. Like
// afterAnswer, they only ban.
onReport []heuristic
// The heuristics of besideLookup, whose work findBesideLookup does.
trapPaths trapPaths
ruleFiles ruleFiles
coreRuleSet coreRuleSet
}
// ServeHTTP handles one request to waf: a request carrying
@@ -57,246 +393,3 @@ func (h *handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
h.ownEndpoints.ServeHTTP(w, r)
}
// refusedWhateverTheAnswer reports whether the request is one that does not wait
// for GeoJS, since the checks will refuse it at now whatever GeoJS answers about
// its client: a heuristic of afterLookup that implements refusesWithoutLookup
// says it will refuse it. In observe mode, which refuses nothing, none is.
// rproxy lets a request through when waf answers too late.
func (rq *request) refusedWhateverTheAnswer(now time.Time) bool {
if rq.h.config.Observe {
return false
}
for _, each := range rq.h.afterLookup {
knows, ok := each.(refusesWithoutLookup)
if ok && knows.refusesWhateverTheLookupAnswers(rq, now) {
return true
}
}
return false
}
// answerQuestion runs the checks on the visitor's request a question
// carries, and answers rproxy what to do with it. A denied request's counting
// is finished here, and an allowed one's by its report.
func (h *handler) answerQuestion(w http.ResponseWriter, r *http.Request) {
question, ok := h.readQuestion(w, r)
if !ok {
return
}
rq := h.startRequest(
w,
question.Visitor,
question.Client,
question.Peer,
true,
)
// These run even when a check panics. finish runs first, so that what is
// held has the finished log line. Go's server sends the answer only once
// this function has returned, so the report cannot come before it is held.
defer func() {
rq.finish()
rq.holdForReport()
}()
rq.site = r.Host
rq.line.WAFLine = requestlog.WAFLineDecision
refused := rq.check(r.Context())
rq.checked = time.Now()
underOwnPrefix := strings.HasPrefix(
r.URL.Path,
wafprotocol.OwnEndpointPrefix,
)
if refused == nil && !underOwnPrefix {
rq.allow()
// In observe mode, what enforce mode would have refused counts here.
rq.runBanningHeuristics(r.Context(), rq.h.afterAnswer)
return
}
if refused != nil {
rq.deny(*refused)
} else {
// The app never sees this prefix. waf's own endpoints are at its
// own address, so this answers as a path with no endpoint does.
rq.line.Action = requestlog.ActionOwnEndpoint
wafprotocol.SetAnswerHeaders(
rq.out.Header(),
wafprotocol.Answer{Disposition: wafprotocol.DispositionDeny},
)
http.Error(
rq.out,
http.StatusText(http.StatusNotFound),
http.StatusNotFound,
)
}
rq.banAndCount(r.Context(), rq.h.afterAnswer)
}
// readQuestion reads a question, and reports whether waf takes it. It answers
// a question it does not take with 400 and no disposition. rproxy passes that
// 400 to the visitor, and SWW_RPROXY_WAF_BACKEND_NO_ANSWER_DISPOSITION does
// not apply. The id of a question waf takes is held from here on, without a
// decision yet, so that another question with that id is refused while this
// one is being handled.
//
// A question whose body stops arriving gets no answer, and is not looked at.
func (h *handler) readQuestion(
w http.ResponseWriter, r *http.Request,
) (wafprotocol.Question, bool) {
question, err := wafprotocol.ReadQuestion(w, r)
if errors.Is(err, wafprotocol.ErrBodyStoppedArriving) {
h.processLog.Warn(
"a question's body stopped arriving",
"error",
err.Error(),
)
// This panic has Go's server close the connection without an answer,
// and log nothing.
panic(http.ErrAbortHandler)
}
if err != nil {
h.refuseQuestion(w, err.Error())
return question, false
}
held, _ := h.awaitingReport.ContainsOrAdd(question.RequestID, nil)
if held {
h.refuseQuestion(
w,
wafprotocol.RequestIDHeader+" is the id of a request waf holds",
)
return question, false
}
return question, true
}
// refuseQuestion answers a question waf does not take, for reason.
func (h *handler) refuseQuestion(w http.ResponseWriter, reason string) {
h.processLog.Warn("a question is refused", "reason", reason)
http.Error(w, reason, http.StatusBadRequest)
}
// allow answers a question with wafprotocol.DispositionAllow and, while
// SWW_WAF_ADD_LOOKUP_HEADERS is true, the client's AS number and country.
func (rq *request) allow() {
rq.allowed = true
answer := wafprotocol.Answer{Disposition: wafprotocol.DispositionAllow}
if rq.h.config.AddLookupHeaders {
answer.ASN = rq.lookupAnswer.ASN
answer.Country = rq.lookupAnswer.Country
}
wafprotocol.SetAnswerHeaders(rq.out.Header(), answer)
rq.out.WriteHeader(http.StatusOK)
}
// holdForReport holds a copy of what the report's steps need, by the request's
// id. A report only ever reaches that copy, never the request being handled.
func (rq *request) holdForReport() {
held := rq.decision
rq.h.awaitingReport.Add(rq.line.RequestID, &held)
}
// deny answers a question with what the visitor is to get for r.
func (rq *request) deny(r refusal) {
rq.refused = &r
if r.status == 0 {
// SWW_WAF_BAN_RESPONSE is close. Written past rq.out, since the log line's
// status stays 0: the visitor is sent nothing.
wafprotocol.SetAnswerHeaders(
rq.out.Header(),
wafprotocol.Answer{Disposition: wafprotocol.DispositionClose},
)
rq.out.ResponseWriter.WriteHeader(http.StatusOK)
return
}
wafprotocol.SetAnswerHeaders(
rq.out.Header(),
wafprotocol.Answer{Disposition: wafprotocol.DispositionDeny},
)
http.Error(rq.out, http.StatusText(r.status), r.status)
}
// handleReport answers POST wafprotocol.ReportPath: it finishes the request
// the report names, if it holds that request's decision, and ignores the
// report otherwise.
func (h *handler) handleReport() http.HandlerFunc {
return func(w http.ResponseWriter, r *http.Request) {
report, err := wafprotocol.ReadReport(w, r)
if err != nil {
h.processLog.Warn("a report is refused", "error", err.Error())
http.Error(w, "the body is not a report", http.StatusBadRequest)
return
}
held, found := h.awaitingReport.Peek(report.RequestID)
// Remove reports true to one report only, of several that name the
// id. A question still being handled is held as nil.
if found && held != nil &&
h.awaitingReport.Remove(report.RequestID) {
h.finishWith(r.Context(), held, report)
}
w.WriteHeader(http.StatusNoContent)
}
}
// finishWith finishes a request with what its report says, and writes the
// request's last log line, which holds its decision's fields and the report's.
// For an allowed request it first runs the heuristics of onReport, and adds the
// request to its client's history, the anomaly counters and the metrics, with
// the status and bytes the report gives. A denied request's counting was
// finished at its decision. ctx is the report's own context.
func (h *handler) finishWith(
ctx context.Context,
held *decision,
report wafprotocol.Report,
) {
rq := &request{
decision: *held,
h: h,
out: &responseWriter{
status: report.Status,
bytes: report.ResponseBytes,
},
requestBytes: report.RequestBytes,
}
rq.line.WAFLine = requestlog.WAFLineReport
rq.line.Status = report.Status
rq.line.RequestBytes = report.RequestBytes
rq.line.ResponseBytes = report.ResponseBytes
rq.line.DurationTotalMS = report.DurationTotalMS
if rq.allowed {
rq.line.DurationUpstreamTotalMS = &report.DurationUpstreamTotalMS
rq.banAndCount(ctx, rq.h.onReport)
rq.h.metrics.RequestReported(
&rq.line,
time.Duration(report.DurationTotalMS*float64(time.Millisecond)),
time.Duration(
report.DurationUpstreamTotalMS*float64(time.Millisecond),
),
)
}
rq.writeLine()
}
+10 -8
View File
@@ -703,7 +703,9 @@ func TestQuestionWhoseBodyStopsArrivingGetsNoAnswer(t *testing.T) {
t,
"",
map[string]string{metricsToken: token},
func(server *wafAndParts) { server.ReadTimeout = readTimeout },
func(server *wafAndParts) {
server.HTTPServer().ReadTimeout = readTimeout
},
)
conn := dial(t, s.addr)
@@ -1834,16 +1836,16 @@ func TestLogLevelHoldsBackNoRequestLine(t *testing.T) {
func TestServerHasTheTimeLimitsSPECGives(t *testing.T) {
t.Parallel()
s := startWAF(t, "", nil)
server := startWAF(t, "", nil).server.HTTPServer()
if s.server.ReadTimeout != time.Minute ||
s.server.WriteTimeout != time.Minute ||
s.server.IdleTimeout != 2*time.Minute {
if server.ReadTimeout != time.Minute ||
server.WriteTimeout != time.Minute ||
server.IdleTimeout != 2*time.Minute {
t.Errorf(
"the read, write and idle limits are %v, %v and %v, want 1m, 1m and 2m",
s.server.ReadTimeout,
s.server.WriteTimeout,
s.server.IdleTimeout,
server.ReadTimeout,
server.WriteTimeout,
server.IdleTimeout,
)
}
}