check / check (push) Failing after 1m56s
While the lookup waits for GeoJS, the trap paths, the rule files and the Core Rule Set run at the same time as the checks after the lookup. The first side to refuse answers the request; what the other side finds leaves no trace. Those three checks have 500 ms together: past it, the request is refused with 503 as checks_timed_out, with no ban and no offence. Tests hold GeoJS and the Core Rule Set with stand-ins under synctest. The test helper lifts the 500 ms limit for tests that run the real Core Rule Set, which the race detector slows past it. Model: opus-5-5
301 lines
11 KiB
Go
301 lines
11 KiB
Go
// Package waf is the server the waf command runs. It answers
|
|
// rproxy's questions, takes rproxy's reports and answers its own endpoints, and
|
|
// writes a log line for each.
|
|
package waf
|
|
|
|
import (
|
|
"fmt"
|
|
"io"
|
|
"log/slog"
|
|
"net/http"
|
|
"net/netip"
|
|
"time"
|
|
|
|
lru "github.com/hashicorp/golang-lru/v2"
|
|
|
|
"sneak.berlin/go/smallwebwaf/internal/alerts"
|
|
"sneak.berlin/go/smallwebwaf/internal/anomaly"
|
|
"sneak.berlin/go/smallwebwaf/internal/bans"
|
|
"sneak.berlin/go/smallwebwaf/internal/config"
|
|
"sneak.berlin/go/smallwebwaf/internal/coreruleset"
|
|
"sneak.berlin/go/smallwebwaf/internal/geojs"
|
|
"sneak.berlin/go/smallwebwaf/internal/lookup"
|
|
"sneak.berlin/go/smallwebwaf/internal/metrics"
|
|
"sneak.berlin/go/smallwebwaf/internal/ratelimit"
|
|
"sneak.berlin/go/smallwebwaf/internal/reputation"
|
|
"sneak.berlin/go/smallwebwaf/internal/routewatch"
|
|
"sneak.berlin/go/smallwebwaf/internal/rules"
|
|
)
|
|
|
|
// ownEndpointPrefix starts the path of each of waf's own endpoints. The app
|
|
// never sees a path under it.
|
|
const ownEndpointPrefix = "/_sww/"
|
|
|
|
// HealthPath is the health endpoint of waf and of rproxy, which the
|
|
// container's health check asks.
|
|
const HealthPath = "/_sww/healthz"
|
|
|
|
// MetricsPath is where the metrics are, for a request that carries
|
|
// SWW_WAF_METRICS_TOKEN.
|
|
const MetricsPath = "/_sww/metrics"
|
|
|
|
// BansPath is where an admin lists and adds bans, and, followed by / and
|
|
// a client's address, lifts them, with SWW_WAF_ADMIN_TOKEN.
|
|
const BansPath = "/_sww/bans"
|
|
|
|
// ClientsPath is where an admin asks what waf knows of a client, by
|
|
// the client's address after it, with SWW_WAF_ADMIN_TOKEN.
|
|
const ClientsPath = "/_sww/clients/"
|
|
|
|
// Params are what New needs.
|
|
type Params struct {
|
|
Config *config.WAF
|
|
// RequestLog receives two JSON lines per question, one at its decision and
|
|
// one when its report comes, and one per request to waf's own
|
|
// endpoints.
|
|
RequestLog io.Writer
|
|
// ProcessLog receives the process's own messages.
|
|
ProcessLog *slog.Logger
|
|
// GeoJSURL is where clients' AS numbers and countries are looked up
|
|
// while SWW_WAF_LOOKUP_SOURCE is geojs, normally geojs.URL.
|
|
GeoJSURL string
|
|
// AbuseIPDBURL is where clients are checked with AbuseIPDB while
|
|
// SWW_WAF_ABUSEIPDB_KEY is set, normally reputation.AbuseIPDBURL.
|
|
AbuseIPDBURL string
|
|
// LookupFile is the lookup database they are looked up in while
|
|
// SWW_WAF_LOOKUP_SOURCE is file, and nil otherwise.
|
|
LookupFile *lookup.File
|
|
// CoreRuleSet is what NewCoreRuleSet returns: what inspects requests
|
|
// while SWW_WAF_CORE_RULE_SET_MODE is not off, and nil while it is.
|
|
CoreRuleSet CoreRuleSetInspector
|
|
// Now tells the time by which requests are counted for the rate
|
|
// limits, bans are made and run out, and GeoJS's answers are kept,
|
|
// normally time.Now in UTC, the time the state files give.
|
|
Now func() time.Time
|
|
// Rules are the rule files' rules, which each request is checked
|
|
// against.
|
|
Rules *rules.Files
|
|
// Alerts receive the alert for each ban waf makes or makes
|
|
// permanent, for each count over an anomaly threshold, for each request
|
|
// whose client a blocklist, a DNSBL zone or AbuseIPDB lists, for each
|
|
// request the Core Rule Set scores at or over
|
|
// SWW_WAF_CORE_RULE_SET_SCORE_THRESHOLD, and for GeoJS failing, a fetch of a list
|
|
// failing, a query to a DNSBL zone or a check with AbuseIPDB failing,
|
|
// or the day's AbuseIPDB checks used up.
|
|
Alerts *alerts.Queue
|
|
}
|
|
|
|
// CoreRuleSetInspector inspects a request with the Core Rule Set, as
|
|
// coreruleset.CoreRuleSet's Inspect says. Tests give waf a stand-in for it.
|
|
type CoreRuleSetInspector interface {
|
|
Inspect(
|
|
r *http.Request, client netip.Addr, body io.Reader,
|
|
) (coreruleset.Result, []byte, error)
|
|
}
|
|
|
|
// Server is the server waf runs, with the parts whose state the state
|
|
// files keep, the lookup database, nil unless SWW_WAF_LOOKUP_SOURCE is file, the
|
|
// lists fetched from URLs, which its Run fetches, the DNSBL zones' verdicts,
|
|
// AbuseIPDB's scores and checks spent, routewatch's answers, and the metrics.
|
|
// RouteWatch is nil while SWW_WAF_ROUTEWATCH_URL is unset.
|
|
type Server struct {
|
|
*http.Server
|
|
|
|
Ledger *bans.Ledger
|
|
Limiter *ratelimit.Limiter
|
|
GeoJS *geojs.GeoJS
|
|
Anomalies *anomaly.Counters
|
|
LookupFile *lookup.File
|
|
Lists *reputation.Lists
|
|
DNSBL *reputation.DNSBL
|
|
AbuseIPDB *reputation.AbuseIPDB
|
|
RouteWatch *routewatch.Client
|
|
Metrics *metrics.Metrics
|
|
}
|
|
|
|
// newHandler returns the handler with what every request shares, as the
|
|
// settings in params make it.
|
|
func newHandler(params Params) *handler {
|
|
m := metrics.New(
|
|
params.Config.MetricsBusiestCountriesAndASNs,
|
|
params.Config.InstanceName,
|
|
)
|
|
lists, dnsbl, abuseIPDB := newReputation(params, m)
|
|
|
|
awaitingReport, err := lru.New[string, *decision](maxAwaitingReport)
|
|
if err != nil {
|
|
panic(err) // New fails only for a size below one
|
|
}
|
|
|
|
h := &handler{
|
|
config: params.Config,
|
|
requestLog: params.RequestLog,
|
|
processLog: params.ProcessLog,
|
|
now: params.Now,
|
|
metrics: m,
|
|
awaitingReport: awaitingReport,
|
|
limiter: ratelimit.New(ratelimit.Limits{
|
|
PerMinute: params.Config.RateLimitPerMinute,
|
|
PerHour: params.Config.RateLimitPerHour,
|
|
PerDay: params.Config.RateLimitPerDay,
|
|
BytesPerMinute: params.Config.BytesLimitPerMinute,
|
|
BytesPerHour: params.Config.BytesLimitPerHour,
|
|
BytesPerDay: params.Config.BytesLimitPerDay,
|
|
}, params.Config.MaxTrackedClients),
|
|
ledger: bans.New(bans.Params{
|
|
LimitBanDuration: params.Config.LimitBanDuration,
|
|
LimitBanRepeatWindow: params.Config.LimitBanRepeatWindow,
|
|
LimitBanPermanentAbove: params.Config.LimitBanPermanentAbove,
|
|
AttackBanDuration: params.Config.AttackBanDuration,
|
|
MaxBans: params.Config.MaxBans,
|
|
}),
|
|
anomalies: anomaly.New(anomaly.Params{
|
|
Client: params.Config.AnomalyClient,
|
|
Net: params.Config.AnomalyNet,
|
|
ASN: params.Config.AnomalyASN,
|
|
Total: params.Config.AnomalyTotal,
|
|
WatchedNet: params.Config.AnomalyWatchedNet,
|
|
NetV4Prefix: params.Config.AnomalyNetV4Prefix,
|
|
NetV6Prefix: params.Config.AnomalyNetV6Prefix,
|
|
NamedNetblocks: params.Config.AnomalyWatchedNets,
|
|
Alerts: params.Alerts,
|
|
}),
|
|
lookupFile: params.LookupFile,
|
|
lists: lists,
|
|
dnsbl: dnsbl,
|
|
abuseIPDB: abuseIPDB,
|
|
rules: params.Rules,
|
|
coreRuleSet: params.CoreRuleSet,
|
|
alerts: params.Alerts,
|
|
|
|
besideLookupLimit: besideLookupTimeLimit,
|
|
}
|
|
if params.Config.RouteWatchURL != "" {
|
|
h.routeWatch = routewatch.New(routewatch.Params{
|
|
URL: params.Config.RouteWatchURL, Now: params.Now, ProcessLog: params.ProcessLog,
|
|
})
|
|
}
|
|
|
|
// The country list, the headers, the tag settings that name an asn= or
|
|
// country= tag and the biased thresholds act on the answer before the
|
|
// request goes on.
|
|
h.waitForGeoJS = len(params.Config.ExclusivelyAllowedCountries) > 0 ||
|
|
params.Config.AddLookupHeaders || params.Config.TagsNeedLookup() ||
|
|
biasedThresholdsSet(params.Config)
|
|
h.geojs = geojs.New(geojs.Params{
|
|
URL: params.GeoJSURL,
|
|
Answered: h.addLookup,
|
|
Now: params.Now,
|
|
Asked: m.GeoJSRequests.Inc,
|
|
Failed: h.geojsFailed,
|
|
WentWithoutAnswer: m.RequestsWithoutGeoJSAnswer.Inc,
|
|
})
|
|
m.AddBansAndClients(h.ledger, h.limiter, params.Now)
|
|
m.AddRules(params.Rules)
|
|
|
|
return h
|
|
}
|
|
|
|
// newReputation returns the lists fetched from URLs, the DNSBL zones'
|
|
// verdicts and AbuseIPDB's scores, as the settings in params name them,
|
|
// with none fetched, asked for or checked yet, and adds their metrics to
|
|
// m, AbuseIPDB's while SWW_WAF_ABUSEIPDB_KEY is set.
|
|
func newReputation(
|
|
params Params, m *metrics.Metrics,
|
|
) (*reputation.Lists, *reputation.DNSBL, *reputation.AbuseIPDB) {
|
|
cfg := params.Config
|
|
lists := reputation.New(reputation.Params{
|
|
BlocklistURLs: cfg.BlocklistURLs,
|
|
Refresh: cfg.BlocklistRefresh,
|
|
ASNLimitPercentURL: cfg.ASNLimitPercentURL,
|
|
TagLists: cfg.TagLists,
|
|
Now: params.Now,
|
|
ProcessLog: params.ProcessLog,
|
|
Alerts: params.Alerts,
|
|
})
|
|
dnsbl := reputation.NewDNSBL(reputation.DNSBLParams{
|
|
Zones: cfg.DNSBLZones, Resolver: cfg.DNSBLResolver,
|
|
CacheTTL: cfg.IPReputationCacheTTL,
|
|
Timeout: cfg.DNSBLAbuseIPDBTimeout, Now: params.Now, ProcessLog: params.ProcessLog,
|
|
Alerts: params.Alerts,
|
|
})
|
|
abuseIPDB := reputation.NewAbuseIPDB(reputation.AbuseIPDBParams{
|
|
URL: params.AbuseIPDBURL, Key: cfg.AbuseIPDBKey, MinScore: cfg.AbuseIPDBMinScore,
|
|
DailyBudget: cfg.AbuseIPDBDailyBudget, CacheTTL: cfg.IPReputationCacheTTL,
|
|
Timeout: cfg.DNSBLAbuseIPDBTimeout, Now: params.Now, ProcessLog: params.ProcessLog,
|
|
Alerts: params.Alerts,
|
|
})
|
|
|
|
m.AddReputation(lists, dnsbl)
|
|
|
|
if cfg.AbuseIPDBKey != "" {
|
|
m.AddAbuseIPDB(abuseIPDB)
|
|
}
|
|
|
|
return lists, dnsbl, abuseIPDB
|
|
}
|
|
|
|
// NewCoreRuleSet returns the Core Rule Set at SWW_WAF_CORE_RULE_SET_PARANOIA_LEVEL,
|
|
// without the rules SWW_WAF_CORE_RULE_SET_DISABLED_RULES switches off, reading bodies
|
|
// up to SWW_WAF_CORE_RULE_SET_BODY_INSPECTION_SIZE, with the exclusions in
|
|
// SWW_WAF_CORE_RULE_SET_EXCLUSIONS_FILE and the fixes of the apps in SWW_WAF_APPS,
|
|
// or nil while SWW_WAF_CORE_RULE_SET_MODE is off. An exclusions file that
|
|
// coreruleset.New refuses is an error in every mode. While the mode is off, a
|
|
// set file is loaded with the Core Rule Set only to check it.
|
|
func NewCoreRuleSet(cfg *config.WAF) (*coreruleset.CoreRuleSet, error) {
|
|
if cfg.CoreRuleSetMode == config.CoreRuleSetModeOff &&
|
|
cfg.CoreRuleSetExclusionsFile == "" {
|
|
return nil, nil //nolint:nilnil // off is no Core Rule Set, and no error
|
|
}
|
|
|
|
coreRuleSet, err := coreruleset.New(coreruleset.Params{
|
|
ParanoiaLevel: cfg.CoreRuleSetParanoiaLevel,
|
|
DisabledRules: cfg.CoreRuleSetDisabledRules,
|
|
BodyInspectionSize: cfg.CoreRuleSetBodyInspectionSize,
|
|
ExclusionsFile: cfg.CoreRuleSetExclusionsFile,
|
|
Apps: cfg.Apps,
|
|
})
|
|
if err != nil {
|
|
return nil, fmt.Errorf(
|
|
"SWW_WAF_CORE_RULE_SET_EXCLUSIONS_FILE: %w",
|
|
err,
|
|
)
|
|
}
|
|
|
|
if cfg.CoreRuleSetMode == config.CoreRuleSetModeOff {
|
|
return nil, nil //nolint:nilnil // off is no Core Rule Set, and no error
|
|
}
|
|
|
|
return coreRuleSet, nil
|
|
}
|
|
|
|
// handler is waf's handler. It holds what every request shares; what
|
|
// belongs to one request is in a request.
|
|
type handler struct {
|
|
config *config.WAF
|
|
requestLog io.Writer
|
|
processLog *slog.Logger
|
|
now func() time.Time
|
|
metrics *metrics.Metrics
|
|
limiter *ratelimit.Limiter
|
|
ledger *bans.Ledger
|
|
geojs *geojs.GeoJS
|
|
anomalies *anomaly.Counters
|
|
lookupFile *lookup.File
|
|
lists *reputation.Lists
|
|
dnsbl *reputation.DNSBL
|
|
abuseIPDB *reputation.AbuseIPDB
|
|
routeWatch *routewatch.Client
|
|
rules *rules.Files
|
|
coreRuleSet CoreRuleSetInspector
|
|
alerts *alerts.Queue
|
|
// waitForGeoJS is true when a setting needs each request's answer from
|
|
// GeoJS before the request goes on.
|
|
waitForGeoJS bool
|
|
// besideLookupLimit is besideLookupTimeLimit. Only tests change it.
|
|
besideLookupLimit time.Duration
|
|
// awaitingReport holds the decision on each answered question by its id
|
|
// until its report comes, and nil for a question still being handled.
|
|
awaitingReport *lru.Cache[string, *decision]
|
|
}
|