// Package waf is the server the waf command runs. It answers // rproxy's questions, takes rproxy's reports and answers its own endpoints, and // writes a log line for each. It also bans the clients on AbuseIPDB's // blacklist each time the reputation package fetches it. package waf import ( "cmp" "io" "log/slog" "net/http" "net/netip" "time" lru "github.com/hashicorp/golang-lru/v2" "sneak.berlin/go/smallwebwaf/internal/alerts" "sneak.berlin/go/smallwebwaf/internal/anomaly" "sneak.berlin/go/smallwebwaf/internal/bans" "sneak.berlin/go/smallwebwaf/internal/config" "sneak.berlin/go/smallwebwaf/internal/coreruleset" "sneak.berlin/go/smallwebwaf/internal/geojs" "sneak.berlin/go/smallwebwaf/internal/lookup" "sneak.berlin/go/smallwebwaf/internal/metrics" "sneak.berlin/go/smallwebwaf/internal/ratelimit" "sneak.berlin/go/smallwebwaf/internal/reputation" "sneak.berlin/go/smallwebwaf/internal/routewatch" "sneak.berlin/go/smallwebwaf/internal/rules" ) // ownEndpointPrefix starts the path of each of waf's own endpoints. The app // never sees a path under it. const ownEndpointPrefix = "/_sww/" // HealthPath is the health endpoint of waf and of rproxy, which the // container's health check asks. const HealthPath = "/_sww/healthz" // MetricsPath is where the metrics are, for a request that carries // SWW_WAF_METRICS_TOKEN. const MetricsPath = "/_sww/metrics" // BansPath is where an admin lists and adds bans, and, followed by / and // a client's address, lifts them, with SWW_WAF_ADMIN_TOKEN. const BansPath = "/_sww/bans" // ClientsPath is where an admin asks what waf knows of a client, by // the client's address after it, with SWW_WAF_ADMIN_TOKEN. const ClientsPath = "/_sww/clients/" // Params are what New needs. type Params struct { Config *config.WAF // RequestLog receives two JSON lines per question, one at its decision and // one when its report comes, and one per request to waf's own // endpoints. RequestLog io.Writer // ProcessLog receives the process's own messages. ProcessLog *slog.Logger // Metrics count what waf does. Whoever builds them adds the metrics // read from the other parts. Metrics *metrics.Metrics // Limiter is the table of clients the rate limits count, with each // client's history. Limiter *ratelimit.Limiter // Ledger holds the bans. Ledger *bans.Ledger // Anomalies are the counters of the anomaly thresholds. Anomalies *anomaly.Counters // Lists are the lists fetched from URLs, DNSBL the DNSBL zones' // verdicts, and AbuseIPDB AbuseIPDB's scores and the checks spent. Lists *reputation.Lists DNSBL *reputation.DNSBL AbuseIPDB *reputation.AbuseIPDB // RouteWatch asks routewatch about clients, and is nil while // SWW_WAF_ROUTEWATCH_URL is unset. RouteWatch *routewatch.Client // GeoJS looks clients' AS numbers and countries up while // SWW_WAF_LOOKUP_SOURCE is geojs. It is given AddLookup as its Answered // and GeoJSFailed as its Failed. GeoJS *geojs.GeoJS // LookupFile is the lookup database they are looked up in while // SWW_WAF_LOOKUP_SOURCE is file, and nil otherwise. LookupFile *lookup.File // CoreRuleSet is what inspects requests while // SWW_WAF_CORE_RULE_SET_MODE is not off, and nil while it is. CoreRuleSet CoreRuleSetInspector // Now tells the time by which requests are counted for the rate // limits, bans are made and run out, and GeoJS's answers are kept, // normally time.Now in UTC, the time the state files give. Now func() time.Time // Rules are the rule files' rules, which each request is checked // against. Rules *rules.Files // Alerts receive the alert for each ban waf makes or makes // permanent, for each count over an anomaly threshold, for each request // whose client a blocklist, a DNSBL zone or AbuseIPDB lists, for each // request the Core Rule Set scores at or over // SWW_WAF_CORE_RULE_SET_SCORE_THRESHOLD, and for GeoJS failing, a fetch of a list // failing, a query to a DNSBL zone or a check with AbuseIPDB failing, // or the day's AbuseIPDB checks used up. Alerts *alerts.Queue // BesideLookupTimeLimit is how long the heuristics of besideLookup have // for a request. Zero means besideLookupTimeLimit. Tests that run the // Core Rule Set lengthen it, since the race detector and a busy host can // slow an inspection past besideLookupTimeLimit. BesideLookupTimeLimit time.Duration } // CoreRuleSetInspector inspects a request with the Core Rule Set, as // coreruleset.CoreRuleSet's Inspect says. Tests give waf a stand-in for it. type CoreRuleSetInspector interface { Inspect( r *http.Request, client netip.Addr, body io.Reader, ) (coreruleset.Result, []byte, error) } // Server is the server waf runs. type Server struct { *http.Server } // newHandler returns the handler with what every request shares, the parts // in params among it. It builds the heuristics, each with the settings and // parts it reads, and the lists they run in. // //nolint:funlen // a few lines for each heuristic, a list that grows with them func newHandler(params Params) *handler { awaitingReport, err := lru.New[string, *decision](maxAwaitingReport) if err != nil { panic(err) // New fails only for a size below one } cfg := params.Config h := &handler{ config: cfg, requestLog: params.RequestLog, processLog: params.ProcessLog, now: params.Now, metrics: params.Metrics, awaitingReport: awaitingReport, limiter: params.Limiter, ledger: params.Ledger, geojs: params.GeoJS, anomalies: params.Anomalies, lookupFile: params.LookupFile, // The country list, the headers, the tag settings that name an asn= // or country= tag and the biased thresholds act on the answer before // the request goes on. waitForGeoJS: len(cfg.ExclusivelyAllowedCountries) > 0 || cfg.AddLookupHeaders || cfg.TagsNeedLookup() || biasedThresholdsSet(cfg), besideLookupLimit: cmp.Or( params.BesideLookupTimeLimit, besideLookupTimeLimit, ), } sharedBanMaker := banMaker{ ledger: params.Ledger, limiter: params.Limiter, metrics: params.Metrics, alerts: params.Alerts, observe: cfg.Observe, banResponse: cfg.BanResponse, banScopeV4Prefix: cfg.BanScopeV4Prefix, ipv6GroupPrefix: cfg.IPv6GroupPrefix, } hitNoter := reputationHitNoter{ metrics: params.Metrics, alerts: params.Alerts, ipv6GroupPrefix: cfg.IPv6GroupPrefix, } h.blocklists = blocklists{ lists: params.Lists, action: cfg.BlocklistAction, hitNoter: hitNoter, } h.rateLimits = rateLimits{ limiter: params.Limiter, exemptNets: cfg.RateLimitExemptNets, exemptPaths: cfg.RateLimitExemptPaths, banResponse: cfg.BanResponse, ipv6GroupPrefix: cfg.IPv6GroupPrefix, biasedThresholds: biasedThresholds{ lists: params.Lists, tagLimitPercent: cfg.TagLimitPercent, asnBytesLimitPercent: cfg.ASNBytesLimitPercent, countryBytesLimitPercent: cfg.CountryBytesLimitPercent, unknownCountryLimitPercent: cfg.UnknownCountryLimitPercent, blocklistAction: cfg.BlocklistAction, blocklistLimitPercent: cfg.BlocklistLimitPercent, dnsblAbuseIPDBAction: cfg.DNSBLAbuseIPDBAction, dnsblAbuseIPDBLimitPercent: cfg.DNSBLAbuseIPDBLimitPercent, }, banMaker: sharedBanMaker, } h.trapPaths = trapPaths{paths: cfg.TrapPaths, banMaker: sharedBanMaker} h.ruleFiles = ruleFiles{ rules: params.Rules, metrics: params.Metrics, banMaker: sharedBanMaker, } h.coreRuleSet = coreRuleSet{ inspector: params.CoreRuleSet, metrics: params.Metrics, alerts: params.Alerts, mode: cfg.CoreRuleSetMode, exemptPaths: cfg.CoreRuleSetExemptPaths, scoreThreshold: cfg.CoreRuleSetScoreThreshold, observe: cfg.Observe, ipv6GroupPrefix: cfg.IPv6GroupPrefix, } h.beforeLookup = []heuristic{ headerLimits{}, tagLists{lists: params.Lists}, routeWatchTags{ routeWatch: params.RouteWatch, ipv6GroupPrefix: cfg.IPv6GroupPrefix, }, denyNets{netblocks: cfg.DenyNets}, activeBan{banMaker: sharedBanMaker}, } h.afterLookup = []heuristic{ countries{allowed: cfg.ExclusivelyAllowedCountries}, h.blocklists, dnsbl{ dnsbl: params.DNSBL, action: cfg.DNSBLAbuseIPDBAction, hitNoter: hitNoter, }, abuseIPDB{ abuseIPDB: params.AbuseIPDB, limiter: params.Limiter, keySet: cfg.AbuseIPDBKey != "", minScore: cfg.AbuseIPDBMinScore, action: cfg.DNSBLAbuseIPDBAction, ipv6GroupPrefix: cfg.IPv6GroupPrefix, hitNoter: hitNoter, }, tagDeny{tags: cfg.TagDeny}, h.rateLimits, } h.besideLookup = []heuristic{h.trapPaths, h.ruleFiles, h.coreRuleSet} h.afterBesideLookup = []heuristic{tagDeny{tags: cfg.TagDeny}} h.afterAnswer = []heuristic{ refusalLimit{ limiter: params.Limiter, perMinute: cfg.RefusalsLimitPerMinute, banResponse: cfg.BanResponse, ipv6GroupPrefix: cfg.IPv6GroupPrefix, banMaker: sharedBanMaker, }, } h.onReport = []heuristic{ byteLimits{ limiter: params.Limiter, bytesCounted: cfg.BytesCounted, ipv6GroupPrefix: cfg.IPv6GroupPrefix, banMaker: sharedBanMaker, }, } return h } // handler is waf's handler. It holds what every request shares; what // belongs to one request is in a request. type handler struct { config *config.WAF requestLog io.Writer processLog *slog.Logger now func() time.Time metrics *metrics.Metrics limiter *ratelimit.Limiter ledger *bans.Ledger geojs *geojs.GeoJS anomalies *anomaly.Counters lookupFile *lookup.File // waitForGeoJS is true when a setting needs each request's answer from // GeoJS before the request goes on. waitForGeoJS bool // besideLookupLimit is how long the heuristics of besideLookup have for a // request, as Params.BesideLookupTimeLimit says. besideLookupLimit time.Duration // awaitingReport holds the decision on each answered question by its id // until its report comes, and nil for a question still being handled. awaitingReport *lru.Cache[string, *decision] // The lists of heuristics, each in the order its heuristics run. A client // in SWW_WAF_ALLOW_NETS skips them all. // beforeLookup are the heuristics that run before the client's AS number // and country are looked up, so that a client they refuse is not looked // up. header_limits comes first: it only measures and counts, and a // request it refuses causes no other work. beforeLookup []heuristic // afterLookup are the heuristics that run once the lookup has been made. afterLookup []heuristic // besideLookup are the heuristics that only read the request. Their work // runs beside the lookup and afterLookup, in findBesideLookup, which // changes nothing of the request. Their check then applies what that work // found, and runs only once waf takes it. besideLookup []heuristic // afterBesideLookup are the heuristics that run once afterLookup and // besideLookup have both let the request through. tag_deny runs a second // time, on the tag rule_files and core_rule_set can add. afterBesideLookup []heuristic // afterAnswer are the heuristics that run on a request once waf has // answered it, before its decision line is written. They ban the client, // and do not change what the request got: their verdict's action is not // used. afterAnswer []heuristic // onReport are the heuristics that run on an allowed request when // rproxy's report of it comes, before its report line is written. Like // afterAnswer, they only ban. onReport []heuristic // The heuristics of the lists that other steps of a request use too: // refusedWhateverTheAnswer asks blocklists and rateLimits, and // findBesideLookup does the work of trapPaths, ruleFiles and coreRuleSet. blocklists blocklists rateLimits rateLimits trapPaths trapPaths ruleFiles ruleFiles coreRuleSet coreRuleSet }