check / check (push) Waiting to run
While SWW_WAF_ABUSEIPDB_KEY is set, waf fetches AbuseIPDB's blacklist at the start and every SWW_WAF_ABUSEIPDB_BLACKLIST_REFRESH_INTERVAL_MINS (60), cut at SWW_WAF_ABUSEIPDB_THRESHOLD (90, from 25 to 100). Each listed client's netblock gets a ban of SWW_WAF_ABUSEIPDB_BAN_DAYS (30) with the new cause abuseipdb. The blacklist itself is not kept. These bans do not count toward SWW_WAF_MAX_BANS, so a 10,000-address blacklist cannot push out the bans for limits and attacks. An ended abuseipdb ban is forgotten at each fetch and each read of bans.json. A netblock with an active ban is left alone, and so is one whose abuseipdb ban an admin lifted, until that ban would have ended. Observe mode makes no ban. Model: opus-5-5
241 lines
8.8 KiB
Go
241 lines
8.8 KiB
Go
// Package waf is the server the waf command runs. It answers
|
|
// rproxy's questions, takes rproxy's reports and answers its own endpoints, and
|
|
// writes a log line for each. It also bans the clients on AbuseIPDB's
|
|
// blacklist each time the reputation package fetches it.
|
|
package waf
|
|
|
|
import (
|
|
"cmp"
|
|
"fmt"
|
|
"io"
|
|
"log/slog"
|
|
"net/http"
|
|
"net/netip"
|
|
"time"
|
|
|
|
lru "github.com/hashicorp/golang-lru/v2"
|
|
|
|
"sneak.berlin/go/smallwebwaf/internal/alerts"
|
|
"sneak.berlin/go/smallwebwaf/internal/anomaly"
|
|
"sneak.berlin/go/smallwebwaf/internal/bans"
|
|
"sneak.berlin/go/smallwebwaf/internal/config"
|
|
"sneak.berlin/go/smallwebwaf/internal/coreruleset"
|
|
"sneak.berlin/go/smallwebwaf/internal/geojs"
|
|
"sneak.berlin/go/smallwebwaf/internal/lookup"
|
|
"sneak.berlin/go/smallwebwaf/internal/metrics"
|
|
"sneak.berlin/go/smallwebwaf/internal/ratelimit"
|
|
"sneak.berlin/go/smallwebwaf/internal/reputation"
|
|
"sneak.berlin/go/smallwebwaf/internal/routewatch"
|
|
"sneak.berlin/go/smallwebwaf/internal/rules"
|
|
)
|
|
|
|
// ownEndpointPrefix starts the path of each of waf's own endpoints. The app
|
|
// never sees a path under it.
|
|
const ownEndpointPrefix = "/_sww/"
|
|
|
|
// HealthPath is the health endpoint of waf and of rproxy, which the
|
|
// container's health check asks.
|
|
const HealthPath = "/_sww/healthz"
|
|
|
|
// MetricsPath is where the metrics are, for a request that carries
|
|
// SWW_WAF_METRICS_TOKEN.
|
|
const MetricsPath = "/_sww/metrics"
|
|
|
|
// BansPath is where an admin lists and adds bans, and, followed by / and
|
|
// a client's address, lifts them, with SWW_WAF_ADMIN_TOKEN.
|
|
const BansPath = "/_sww/bans"
|
|
|
|
// ClientsPath is where an admin asks what waf knows of a client, by
|
|
// the client's address after it, with SWW_WAF_ADMIN_TOKEN.
|
|
const ClientsPath = "/_sww/clients/"
|
|
|
|
// Params are what New needs.
|
|
type Params struct {
|
|
Config *config.WAF
|
|
// RequestLog receives two JSON lines per question, one at its decision and
|
|
// one when its report comes, and one per request to waf's own
|
|
// endpoints.
|
|
RequestLog io.Writer
|
|
// ProcessLog receives the process's own messages.
|
|
ProcessLog *slog.Logger
|
|
// Metrics count what waf does. Whoever builds them adds the metrics
|
|
// read from the other parts.
|
|
Metrics *metrics.Metrics
|
|
// Limiter is the table of clients the rate limits count, with each
|
|
// client's history.
|
|
Limiter *ratelimit.Limiter
|
|
// Ledger holds the bans.
|
|
Ledger *bans.Ledger
|
|
// Anomalies are the counters of the anomaly thresholds.
|
|
Anomalies *anomaly.Counters
|
|
// Lists are the lists fetched from URLs, DNSBL the DNSBL zones'
|
|
// verdicts, and AbuseIPDB AbuseIPDB's scores and the checks spent.
|
|
Lists *reputation.Lists
|
|
DNSBL *reputation.DNSBL
|
|
AbuseIPDB *reputation.AbuseIPDB
|
|
// RouteWatch asks routewatch about clients, and is nil while
|
|
// SWW_WAF_ROUTEWATCH_URL is unset.
|
|
RouteWatch *routewatch.Client
|
|
// GeoJS looks clients' AS numbers and countries up while
|
|
// SWW_WAF_LOOKUP_SOURCE is geojs. It is given AddLookup as its Answered
|
|
// and GeoJSFailed as its Failed.
|
|
GeoJS *geojs.GeoJS
|
|
// LookupFile is the lookup database they are looked up in while
|
|
// SWW_WAF_LOOKUP_SOURCE is file, and nil otherwise.
|
|
LookupFile *lookup.File
|
|
// CoreRuleSet is what NewCoreRuleSet returns: what inspects requests
|
|
// while SWW_WAF_CORE_RULE_SET_MODE is not off, and nil while it is.
|
|
CoreRuleSet CoreRuleSetInspector
|
|
// Now tells the time by which requests are counted for the rate
|
|
// limits, bans are made and run out, and GeoJS's answers are kept,
|
|
// normally time.Now in UTC, the time the state files give.
|
|
Now func() time.Time
|
|
// Rules are the rule files' rules, which each request is checked
|
|
// against.
|
|
Rules *rules.Files
|
|
// Alerts receive the alert for each ban waf makes or makes
|
|
// permanent, for each count over an anomaly threshold, for each request
|
|
// whose client a blocklist, a DNSBL zone or AbuseIPDB lists, for each
|
|
// request the Core Rule Set scores at or over
|
|
// SWW_WAF_CORE_RULE_SET_SCORE_THRESHOLD, and for GeoJS failing, a fetch of a list
|
|
// failing, a query to a DNSBL zone or a check with AbuseIPDB failing,
|
|
// or the day's AbuseIPDB checks used up.
|
|
Alerts *alerts.Queue
|
|
// BesideLookupTimeLimit is how long the heuristics of besideLookup have
|
|
// for a request. Zero means besideLookupTimeLimit. Tests that run the
|
|
// Core Rule Set lengthen it, since the race detector and a busy host can
|
|
// slow an inspection past besideLookupTimeLimit.
|
|
BesideLookupTimeLimit time.Duration
|
|
}
|
|
|
|
// CoreRuleSetInspector inspects a request with the Core Rule Set, as
|
|
// coreruleset.CoreRuleSet's Inspect says. Tests give waf a stand-in for it.
|
|
type CoreRuleSetInspector interface {
|
|
Inspect(
|
|
r *http.Request, client netip.Addr, body io.Reader,
|
|
) (coreruleset.Result, []byte, error)
|
|
}
|
|
|
|
// Server is the server waf runs, with the parts of its Params. The tests
|
|
// read them.
|
|
type Server struct {
|
|
*http.Server
|
|
|
|
Ledger *bans.Ledger
|
|
Limiter *ratelimit.Limiter
|
|
GeoJS *geojs.GeoJS
|
|
Anomalies *anomaly.Counters
|
|
LookupFile *lookup.File
|
|
Lists *reputation.Lists
|
|
DNSBL *reputation.DNSBL
|
|
AbuseIPDB *reputation.AbuseIPDB
|
|
RouteWatch *routewatch.Client
|
|
Metrics *metrics.Metrics
|
|
}
|
|
|
|
// newHandler returns the handler with what every request shares, the parts
|
|
// in params among it.
|
|
func newHandler(params Params) *handler {
|
|
awaitingReport, err := lru.New[string, *decision](maxAwaitingReport)
|
|
if err != nil {
|
|
panic(err) // New fails only for a size below one
|
|
}
|
|
|
|
return &handler{
|
|
config: params.Config,
|
|
requestLog: params.RequestLog,
|
|
processLog: params.ProcessLog,
|
|
now: params.Now,
|
|
metrics: params.Metrics,
|
|
awaitingReport: awaitingReport,
|
|
limiter: params.Limiter,
|
|
ledger: params.Ledger,
|
|
geojs: params.GeoJS,
|
|
anomalies: params.Anomalies,
|
|
lookupFile: params.LookupFile,
|
|
lists: params.Lists,
|
|
dnsbl: params.DNSBL,
|
|
abuseIPDB: params.AbuseIPDB,
|
|
routeWatch: params.RouteWatch,
|
|
rules: params.Rules,
|
|
coreRuleSet: params.CoreRuleSet,
|
|
alerts: params.Alerts,
|
|
// The country list, the headers, the tag settings that name an asn=
|
|
// or country= tag and the biased thresholds act on the answer before
|
|
// the request goes on.
|
|
waitForGeoJS: len(params.Config.ExclusivelyAllowedCountries) > 0 ||
|
|
params.Config.AddLookupHeaders ||
|
|
params.Config.TagsNeedLookup() ||
|
|
biasedThresholdsSet(params.Config),
|
|
besideLookupLimit: cmp.Or(
|
|
params.BesideLookupTimeLimit,
|
|
besideLookupTimeLimit,
|
|
),
|
|
}
|
|
}
|
|
|
|
// NewCoreRuleSet returns the Core Rule Set at SWW_WAF_CORE_RULE_SET_PARANOIA_LEVEL,
|
|
// without the rules SWW_WAF_CORE_RULE_SET_DISABLED_RULES switches off, reading bodies
|
|
// up to SWW_WAF_CORE_RULE_SET_BODY_INSPECTION_SIZE, with the exclusions in
|
|
// SWW_WAF_CORE_RULE_SET_EXCLUSIONS_FILE and the fixes of the apps in SWW_WAF_APPS,
|
|
// or nil while SWW_WAF_CORE_RULE_SET_MODE is off. An exclusions file that
|
|
// coreruleset.New refuses is an error in every mode. While the mode is off, a
|
|
// set file is loaded with the Core Rule Set only to check it.
|
|
func NewCoreRuleSet(cfg *config.WAF) (*coreruleset.CoreRuleSet, error) {
|
|
if cfg.CoreRuleSetMode == config.CoreRuleSetModeOff &&
|
|
cfg.CoreRuleSetExclusionsFile == "" {
|
|
return nil, nil //nolint:nilnil // off is no Core Rule Set, and no error
|
|
}
|
|
|
|
coreRuleSet, err := coreruleset.New(coreruleset.Params{
|
|
ParanoiaLevel: cfg.CoreRuleSetParanoiaLevel,
|
|
DisabledRules: cfg.CoreRuleSetDisabledRules,
|
|
BodyInspectionSize: cfg.CoreRuleSetBodyInspectionSize,
|
|
ExclusionsFile: cfg.CoreRuleSetExclusionsFile,
|
|
Apps: cfg.Apps,
|
|
})
|
|
if err != nil {
|
|
return nil, fmt.Errorf(
|
|
"SWW_WAF_CORE_RULE_SET_EXCLUSIONS_FILE: %w",
|
|
err,
|
|
)
|
|
}
|
|
|
|
if cfg.CoreRuleSetMode == config.CoreRuleSetModeOff {
|
|
return nil, nil //nolint:nilnil // off is no Core Rule Set, and no error
|
|
}
|
|
|
|
return coreRuleSet, nil
|
|
}
|
|
|
|
// handler is waf's handler. It holds what every request shares; what
|
|
// belongs to one request is in a request.
|
|
type handler struct {
|
|
config *config.WAF
|
|
requestLog io.Writer
|
|
processLog *slog.Logger
|
|
now func() time.Time
|
|
metrics *metrics.Metrics
|
|
limiter *ratelimit.Limiter
|
|
ledger *bans.Ledger
|
|
geojs *geojs.GeoJS
|
|
anomalies *anomaly.Counters
|
|
lookupFile *lookup.File
|
|
lists *reputation.Lists
|
|
dnsbl *reputation.DNSBL
|
|
abuseIPDB *reputation.AbuseIPDB
|
|
routeWatch *routewatch.Client
|
|
rules *rules.Files
|
|
coreRuleSet CoreRuleSetInspector
|
|
alerts *alerts.Queue
|
|
// waitForGeoJS is true when a setting needs each request's answer from
|
|
// GeoJS before the request goes on.
|
|
waitForGeoJS bool
|
|
// besideLookupLimit is how long the heuristics of besideLookup have for a
|
|
// request, as Params.BesideLookupTimeLimit says.
|
|
besideLookupLimit time.Duration
|
|
// awaitingReport holds the decision on each answered question by its id
|
|
// until its report comes, and nil for a question still being handled.
|
|
awaitingReport *lru.Cache[string, *decision]
|
|
}
|