check / check (push) Waiting to run
refusedWhateverTheAnswer asks each heuristic of afterLookup that implements the new refusesWithoutLookup interface, instead of naming blocklists and rate_limits. The comment on adding a heuristic names the places outside its file that may change with it. Model: opus-5-5
329 lines
12 KiB
Go
329 lines
12 KiB
Go
// Package waf is the server the waf command runs. It answers
|
|
// rproxy's questions, takes rproxy's reports and answers its own endpoints, and
|
|
// writes a log line for each. It also bans the clients on AbuseIPDB's
|
|
// blacklist each time the reputation package fetches it.
|
|
package waf
|
|
|
|
import (
|
|
"cmp"
|
|
"io"
|
|
"log/slog"
|
|
"net/http"
|
|
"net/netip"
|
|
"time"
|
|
|
|
lru "github.com/hashicorp/golang-lru/v2"
|
|
|
|
"sneak.berlin/go/smallwebwaf/internal/alerts"
|
|
"sneak.berlin/go/smallwebwaf/internal/anomaly"
|
|
"sneak.berlin/go/smallwebwaf/internal/bans"
|
|
"sneak.berlin/go/smallwebwaf/internal/config"
|
|
"sneak.berlin/go/smallwebwaf/internal/coreruleset"
|
|
"sneak.berlin/go/smallwebwaf/internal/geojs"
|
|
"sneak.berlin/go/smallwebwaf/internal/lookup"
|
|
"sneak.berlin/go/smallwebwaf/internal/metrics"
|
|
"sneak.berlin/go/smallwebwaf/internal/ratelimit"
|
|
"sneak.berlin/go/smallwebwaf/internal/reputation"
|
|
"sneak.berlin/go/smallwebwaf/internal/routewatch"
|
|
"sneak.berlin/go/smallwebwaf/internal/rules"
|
|
)
|
|
|
|
// MetricsPath is where the metrics are, for a request that carries
|
|
// SWW_WAF_METRICS_TOKEN.
|
|
const MetricsPath = "/_sww/metrics"
|
|
|
|
// BansPath is where an admin lists and adds bans, and, followed by / and
|
|
// a client's address, lifts them, with SWW_WAF_ADMIN_TOKEN.
|
|
const BansPath = "/_sww/bans"
|
|
|
|
// ClientsPath is where an admin asks what waf knows of a client, by
|
|
// the client's address after it, with SWW_WAF_ADMIN_TOKEN.
|
|
const ClientsPath = "/_sww/clients/"
|
|
|
|
// Params are what New needs.
|
|
type Params struct {
|
|
Config *config.WAF
|
|
// RequestLog receives two JSON lines per question, one at its decision and
|
|
// one when its report comes, and one per request to waf's own
|
|
// endpoints.
|
|
RequestLog io.Writer
|
|
// ProcessLog receives the process's own messages.
|
|
ProcessLog *slog.Logger
|
|
// Metrics count what waf does. Whoever builds them adds the metrics
|
|
// read from the other parts.
|
|
Metrics *metrics.Metrics
|
|
// Limiter is the table of clients the rate limits count, with each
|
|
// client's history.
|
|
Limiter *ratelimit.Limiter
|
|
// Ledger holds the bans.
|
|
Ledger *bans.Ledger
|
|
// Anomalies are the counters of the anomaly thresholds.
|
|
Anomalies *anomaly.Counters
|
|
// Lists are the lists fetched from URLs, DNSBL the DNSBL zones'
|
|
// verdicts, and AbuseIPDB AbuseIPDB's scores and the checks spent.
|
|
Lists *reputation.Lists
|
|
DNSBL *reputation.DNSBL
|
|
AbuseIPDB *reputation.AbuseIPDB
|
|
// RouteWatch asks routewatch about clients, and is nil while
|
|
// SWW_WAF_ROUTEWATCH_URL is unset.
|
|
RouteWatch *routewatch.Client
|
|
// GeoJS looks clients' AS numbers and countries up while
|
|
// SWW_WAF_LOOKUP_SOURCE is geojs. It is given AddLookup as its Answered
|
|
// and GeoJSFailed as its Failed.
|
|
GeoJS *geojs.GeoJS
|
|
// LookupFile is the lookup database they are looked up in while
|
|
// SWW_WAF_LOOKUP_SOURCE is file, and nil otherwise.
|
|
LookupFile *lookup.File
|
|
// CoreRuleSet is what inspects requests while
|
|
// SWW_WAF_CORE_RULE_SET_MODE is not off, and nil while it is.
|
|
CoreRuleSet CoreRuleSetInspector
|
|
// Now tells the time by which requests are counted for the rate
|
|
// limits, bans are made and run out, and GeoJS's answers are kept,
|
|
// normally time.Now in UTC, the time the state files give.
|
|
Now func() time.Time
|
|
// Rules are the rule files' rules, which each request is checked
|
|
// against.
|
|
Rules *rules.Files
|
|
// Alerts receive the alert for each ban waf makes or makes
|
|
// permanent, for each count over an anomaly threshold, for each request
|
|
// whose client a blocklist, a DNSBL zone or AbuseIPDB lists, for each
|
|
// request the Core Rule Set scores at or over
|
|
// SWW_WAF_CORE_RULE_SET_SCORE_THRESHOLD, and for GeoJS failing, a fetch of a list
|
|
// failing, a query to a DNSBL zone or a check with AbuseIPDB failing,
|
|
// or the day's AbuseIPDB checks used up.
|
|
Alerts *alerts.Queue
|
|
// BesideLookupTimeLimit is how long the heuristics of besideLookup have
|
|
// for a request. Zero means besideLookupTimeLimit. Tests that run the
|
|
// Core Rule Set lengthen it, since the race detector and a busy host can
|
|
// slow an inspection past besideLookupTimeLimit.
|
|
BesideLookupTimeLimit time.Duration
|
|
}
|
|
|
|
// CoreRuleSetInspector inspects a request with the Core Rule Set, as
|
|
// coreruleset.CoreRuleSet's Inspect says. Tests give waf a stand-in for it.
|
|
type CoreRuleSetInspector interface {
|
|
Inspect(
|
|
r *http.Request, client netip.Addr, body io.Reader,
|
|
) (coreruleset.Result, []byte, error)
|
|
}
|
|
|
|
// Server is the server waf runs, with the parts of its Params. The tests
|
|
// read them.
|
|
type Server struct {
|
|
*http.Server
|
|
|
|
Ledger *bans.Ledger
|
|
Limiter *ratelimit.Limiter
|
|
GeoJS *geojs.GeoJS
|
|
Anomalies *anomaly.Counters
|
|
LookupFile *lookup.File
|
|
Lists *reputation.Lists
|
|
DNSBL *reputation.DNSBL
|
|
AbuseIPDB *reputation.AbuseIPDB
|
|
RouteWatch *routewatch.Client
|
|
Metrics *metrics.Metrics
|
|
}
|
|
|
|
// newHandler returns the handler with what every request shares, the parts
|
|
// in params among it. It builds the heuristics, each with the settings and
|
|
// parts it reads, and the lists they run in.
|
|
//
|
|
//nolint:funlen // a few lines for each heuristic, a list that grows with them
|
|
func newHandler(params Params) *handler {
|
|
awaitingReport, err := lru.New[string, *decision](maxAwaitingReport)
|
|
if err != nil {
|
|
panic(err) // New fails only for a size below one
|
|
}
|
|
|
|
cfg := params.Config
|
|
h := &handler{
|
|
config: cfg,
|
|
requestLog: params.RequestLog,
|
|
processLog: params.ProcessLog,
|
|
now: params.Now,
|
|
metrics: params.Metrics,
|
|
awaitingReport: awaitingReport,
|
|
limiter: params.Limiter,
|
|
ledger: params.Ledger,
|
|
geojs: params.GeoJS,
|
|
anomalies: params.Anomalies,
|
|
lookupFile: params.LookupFile,
|
|
// The country list, the headers, the tag settings that name an asn=
|
|
// or country= tag and the biased thresholds act on the answer before
|
|
// the request goes on.
|
|
waitForGeoJS: len(cfg.ExclusivelyAllowedCountries) > 0 ||
|
|
cfg.AddLookupHeaders ||
|
|
cfg.TagsNeedLookup() ||
|
|
biasedThresholdsSet(cfg),
|
|
besideLookupLimit: cmp.Or(
|
|
params.BesideLookupTimeLimit,
|
|
besideLookupTimeLimit,
|
|
),
|
|
}
|
|
|
|
sharedBanMaker := banMaker{
|
|
ledger: params.Ledger,
|
|
limiter: params.Limiter,
|
|
metrics: params.Metrics,
|
|
alerts: params.Alerts,
|
|
observe: cfg.Observe,
|
|
banResponse: cfg.BanResponse,
|
|
banScopeV4Prefix: cfg.BanScopeV4Prefix,
|
|
ipv6GroupPrefix: cfg.IPv6GroupPrefix,
|
|
}
|
|
hitNoter := reputationHitNoter{
|
|
metrics: params.Metrics,
|
|
alerts: params.Alerts,
|
|
ipv6GroupPrefix: cfg.IPv6GroupPrefix,
|
|
}
|
|
|
|
h.trapPaths = trapPaths{paths: cfg.TrapPaths, banMaker: sharedBanMaker}
|
|
h.ruleFiles = ruleFiles{
|
|
rules: params.Rules,
|
|
metrics: params.Metrics,
|
|
banMaker: sharedBanMaker,
|
|
}
|
|
h.coreRuleSet = coreRuleSet{
|
|
inspector: params.CoreRuleSet,
|
|
metrics: params.Metrics,
|
|
alerts: params.Alerts,
|
|
mode: cfg.CoreRuleSetMode,
|
|
exemptPaths: cfg.CoreRuleSetExemptPaths,
|
|
scoreThreshold: cfg.CoreRuleSetScoreThreshold,
|
|
observe: cfg.Observe,
|
|
ipv6GroupPrefix: cfg.IPv6GroupPrefix,
|
|
}
|
|
|
|
h.beforeLookup = []heuristic{
|
|
headerLimits{},
|
|
tagLists{lists: params.Lists},
|
|
routeWatchTags{
|
|
routeWatch: params.RouteWatch,
|
|
ipv6GroupPrefix: cfg.IPv6GroupPrefix,
|
|
},
|
|
denyNets{netblocks: cfg.DenyNets},
|
|
activeBan{banMaker: sharedBanMaker},
|
|
}
|
|
h.afterLookup = []heuristic{
|
|
countries{allowed: cfg.ExclusivelyAllowedCountries},
|
|
blocklists{
|
|
lists: params.Lists,
|
|
action: cfg.BlocklistAction,
|
|
hitNoter: hitNoter,
|
|
},
|
|
dnsbl{
|
|
dnsbl: params.DNSBL,
|
|
action: cfg.DNSBLAbuseIPDBAction,
|
|
hitNoter: hitNoter,
|
|
},
|
|
abuseIPDB{
|
|
abuseIPDB: params.AbuseIPDB,
|
|
limiter: params.Limiter,
|
|
keySet: cfg.AbuseIPDBKey != "",
|
|
minScore: cfg.AbuseIPDBMinScore,
|
|
action: cfg.DNSBLAbuseIPDBAction,
|
|
ipv6GroupPrefix: cfg.IPv6GroupPrefix,
|
|
hitNoter: hitNoter,
|
|
},
|
|
tagDeny{tags: cfg.TagDeny},
|
|
rateLimits{
|
|
limiter: params.Limiter,
|
|
exemptNets: cfg.RateLimitExemptNets,
|
|
exemptPaths: cfg.RateLimitExemptPaths,
|
|
banResponse: cfg.BanResponse,
|
|
ipv6GroupPrefix: cfg.IPv6GroupPrefix,
|
|
biasedThresholds: biasedThresholds{
|
|
lists: params.Lists,
|
|
tagLimitPercent: cfg.TagLimitPercent,
|
|
asnBytesLimitPercent: cfg.ASNBytesLimitPercent,
|
|
countryBytesLimitPercent: cfg.CountryBytesLimitPercent,
|
|
unknownCountryLimitPercent: cfg.UnknownCountryLimitPercent,
|
|
blocklistAction: cfg.BlocklistAction,
|
|
blocklistLimitPercent: cfg.BlocklistLimitPercent,
|
|
dnsblAbuseIPDBAction: cfg.DNSBLAbuseIPDBAction,
|
|
dnsblAbuseIPDBLimitPercent: cfg.DNSBLAbuseIPDBLimitPercent,
|
|
},
|
|
banMaker: sharedBanMaker,
|
|
},
|
|
}
|
|
h.besideLookup = []heuristic{h.trapPaths, h.ruleFiles, h.coreRuleSet}
|
|
h.afterBesideLookup = []heuristic{tagDeny{tags: cfg.TagDeny}}
|
|
h.afterAnswer = []heuristic{
|
|
refusalLimit{
|
|
limiter: params.Limiter,
|
|
perMinute: cfg.RefusalsLimitPerMinute,
|
|
banResponse: cfg.BanResponse,
|
|
ipv6GroupPrefix: cfg.IPv6GroupPrefix,
|
|
banMaker: sharedBanMaker,
|
|
},
|
|
}
|
|
h.onReport = []heuristic{
|
|
byteLimits{
|
|
limiter: params.Limiter,
|
|
bytesCounted: cfg.BytesCounted,
|
|
ipv6GroupPrefix: cfg.IPv6GroupPrefix,
|
|
banMaker: sharedBanMaker,
|
|
},
|
|
}
|
|
|
|
return h
|
|
}
|
|
|
|
// handler is waf's handler. It holds what every request shares; what
|
|
// belongs to one request is in a request.
|
|
type handler struct {
|
|
config *config.WAF
|
|
requestLog io.Writer
|
|
processLog *slog.Logger
|
|
now func() time.Time
|
|
metrics *metrics.Metrics
|
|
limiter *ratelimit.Limiter
|
|
ledger *bans.Ledger
|
|
geojs *geojs.GeoJS
|
|
anomalies *anomaly.Counters
|
|
lookupFile *lookup.File
|
|
// waitForGeoJS is true when a setting needs each request's answer from
|
|
// GeoJS before the request goes on.
|
|
waitForGeoJS bool
|
|
// besideLookupLimit is how long the heuristics of besideLookup have for a
|
|
// request, as Params.BesideLookupTimeLimit says.
|
|
besideLookupLimit time.Duration
|
|
// awaitingReport holds the decision on each answered question by its id
|
|
// until its report comes, and nil for a question still being handled.
|
|
awaitingReport *lru.Cache[string, *decision]
|
|
|
|
// The lists of heuristics, each in the order its heuristics run. A client
|
|
// in SWW_WAF_ALLOW_NETS skips them all.
|
|
|
|
// beforeLookup are the heuristics that run before the client's AS number
|
|
// and country are looked up, so that a client they refuse is not looked
|
|
// up. header_limits comes first: it only measures and counts, and a
|
|
// request it refuses causes no other work.
|
|
beforeLookup []heuristic
|
|
// afterLookup are the heuristics that run once the lookup has been made.
|
|
afterLookup []heuristic
|
|
// besideLookup are the heuristics that only read the request. Their work
|
|
// runs beside the lookup and afterLookup, in findBesideLookup, which
|
|
// changes nothing of the request. Their check then applies what that work
|
|
// found, and runs only once waf takes it.
|
|
besideLookup []heuristic
|
|
// afterBesideLookup are the heuristics that run once afterLookup and
|
|
// besideLookup have both let the request through. tag_deny runs a second
|
|
// time, on the tag rule_files and core_rule_set can add.
|
|
afterBesideLookup []heuristic
|
|
// afterAnswer are the heuristics that run on a request once waf has
|
|
// answered it, before its decision line is written. They ban the client,
|
|
// and do not change what the request got: their verdict's action is not
|
|
// used.
|
|
afterAnswer []heuristic
|
|
// onReport are the heuristics that run on an allowed request when
|
|
// rproxy's report of it comes, before its report line is written. Like
|
|
// afterAnswer, they only ban.
|
|
onReport []heuristic
|
|
|
|
// The heuristics of besideLookup, whose work findBesideLookup does.
|
|
trapPaths trapPaths
|
|
ruleFiles ruleFiles
|
|
coreRuleSet coreRuleSet
|
|
}
|