check / check (push) Failing after 4m2s
go.uber.org/fx builds the parts of waf and rproxy in internal/sww, one provider per part. Each background loop is a lifecycle hook added where its part is built. waf.Params takes every part built, and waf.New builds none. GeoJS calls back into waf, so waf exports AddLookup and GeoJSFailed, and clientGroup and banNetblock take the settings. newServer, loadStateFiles, the inBackground list and serveRProxy are gone. serve is the one hook that serves both daemons, and reports a failure to run on a channel of its own; fx's signal handling is not used. Once fx has stopped, run takes each daemon's last step: waf writes the state files, and rproxy waits for its reports. Model: opus-5-5
230 lines
8.3 KiB
Go
230 lines
8.3 KiB
Go
// Package waf is the server the waf command runs. It answers
|
|
// rproxy's questions, takes rproxy's reports and answers its own endpoints, and
|
|
// writes a log line for each.
|
|
package waf
|
|
|
|
import (
|
|
"fmt"
|
|
"io"
|
|
"log/slog"
|
|
"net/http"
|
|
"net/netip"
|
|
"time"
|
|
|
|
lru "github.com/hashicorp/golang-lru/v2"
|
|
|
|
"sneak.berlin/go/smallwebwaf/internal/alerts"
|
|
"sneak.berlin/go/smallwebwaf/internal/anomaly"
|
|
"sneak.berlin/go/smallwebwaf/internal/bans"
|
|
"sneak.berlin/go/smallwebwaf/internal/config"
|
|
"sneak.berlin/go/smallwebwaf/internal/coreruleset"
|
|
"sneak.berlin/go/smallwebwaf/internal/geojs"
|
|
"sneak.berlin/go/smallwebwaf/internal/lookup"
|
|
"sneak.berlin/go/smallwebwaf/internal/metrics"
|
|
"sneak.berlin/go/smallwebwaf/internal/ratelimit"
|
|
"sneak.berlin/go/smallwebwaf/internal/reputation"
|
|
"sneak.berlin/go/smallwebwaf/internal/routewatch"
|
|
"sneak.berlin/go/smallwebwaf/internal/rules"
|
|
)
|
|
|
|
// ownEndpointPrefix starts the path of each of waf's own endpoints. The app
|
|
// never sees a path under it.
|
|
const ownEndpointPrefix = "/_sww/"
|
|
|
|
// HealthPath is the health endpoint of waf and of rproxy, which the
|
|
// container's health check asks.
|
|
const HealthPath = "/_sww/healthz"
|
|
|
|
// MetricsPath is where the metrics are, for a request that carries
|
|
// SWW_WAF_METRICS_TOKEN.
|
|
const MetricsPath = "/_sww/metrics"
|
|
|
|
// BansPath is where an admin lists and adds bans, and, followed by / and
|
|
// a client's address, lifts them, with SWW_WAF_ADMIN_TOKEN.
|
|
const BansPath = "/_sww/bans"
|
|
|
|
// ClientsPath is where an admin asks what waf knows of a client, by
|
|
// the client's address after it, with SWW_WAF_ADMIN_TOKEN.
|
|
const ClientsPath = "/_sww/clients/"
|
|
|
|
// Params are what New needs.
|
|
type Params struct {
|
|
Config *config.WAF
|
|
// RequestLog receives two JSON lines per question, one at its decision and
|
|
// one when its report comes, and one per request to waf's own
|
|
// endpoints.
|
|
RequestLog io.Writer
|
|
// ProcessLog receives the process's own messages.
|
|
ProcessLog *slog.Logger
|
|
// Metrics count what waf does. Whoever builds them adds the metrics
|
|
// read from the other parts.
|
|
Metrics *metrics.Metrics
|
|
// Limiter is the table of clients the rate limits count, with each
|
|
// client's history.
|
|
Limiter *ratelimit.Limiter
|
|
// Ledger holds the bans.
|
|
Ledger *bans.Ledger
|
|
// Anomalies are the counters of the anomaly thresholds.
|
|
Anomalies *anomaly.Counters
|
|
// Lists are the lists fetched from URLs, DNSBL the DNSBL zones'
|
|
// verdicts, and AbuseIPDB AbuseIPDB's scores and the checks spent.
|
|
Lists *reputation.Lists
|
|
DNSBL *reputation.DNSBL
|
|
AbuseIPDB *reputation.AbuseIPDB
|
|
// RouteWatch asks routewatch about clients, and is nil while
|
|
// SWW_WAF_ROUTEWATCH_URL is unset.
|
|
RouteWatch *routewatch.Client
|
|
// GeoJS looks clients' AS numbers and countries up while
|
|
// SWW_WAF_LOOKUP_SOURCE is geojs. It is given AddLookup as its Answered
|
|
// and GeoJSFailed as its Failed.
|
|
GeoJS *geojs.GeoJS
|
|
// LookupFile is the lookup database they are looked up in while
|
|
// SWW_WAF_LOOKUP_SOURCE is file, and nil otherwise.
|
|
LookupFile *lookup.File
|
|
// CoreRuleSet is what NewCoreRuleSet returns: what inspects requests
|
|
// while SWW_WAF_CORE_RULE_SET_MODE is not off, and nil while it is.
|
|
CoreRuleSet CoreRuleSetInspector
|
|
// Now tells the time by which requests are counted for the rate
|
|
// limits, bans are made and run out, and GeoJS's answers are kept,
|
|
// normally time.Now in UTC, the time the state files give.
|
|
Now func() time.Time
|
|
// Rules are the rule files' rules, which each request is checked
|
|
// against.
|
|
Rules *rules.Files
|
|
// Alerts receive the alert for each ban waf makes or makes
|
|
// permanent, for each count over an anomaly threshold, for each request
|
|
// whose client a blocklist, a DNSBL zone or AbuseIPDB lists, for each
|
|
// request the Core Rule Set scores at or over
|
|
// SWW_WAF_CORE_RULE_SET_SCORE_THRESHOLD, and for GeoJS failing, a fetch of a list
|
|
// failing, a query to a DNSBL zone or a check with AbuseIPDB failing,
|
|
// or the day's AbuseIPDB checks used up.
|
|
Alerts *alerts.Queue
|
|
}
|
|
|
|
// CoreRuleSetInspector inspects a request with the Core Rule Set, as
|
|
// coreruleset.CoreRuleSet's Inspect says. Tests give waf a stand-in for it.
|
|
type CoreRuleSetInspector interface {
|
|
Inspect(
|
|
r *http.Request, client netip.Addr, body io.Reader,
|
|
) (coreruleset.Result, []byte, error)
|
|
}
|
|
|
|
// Server is the server waf runs, with the parts of its Params. The tests
|
|
// read them.
|
|
type Server struct {
|
|
*http.Server
|
|
|
|
Ledger *bans.Ledger
|
|
Limiter *ratelimit.Limiter
|
|
GeoJS *geojs.GeoJS
|
|
Anomalies *anomaly.Counters
|
|
LookupFile *lookup.File
|
|
Lists *reputation.Lists
|
|
DNSBL *reputation.DNSBL
|
|
AbuseIPDB *reputation.AbuseIPDB
|
|
RouteWatch *routewatch.Client
|
|
Metrics *metrics.Metrics
|
|
}
|
|
|
|
// newHandler returns the handler with what every request shares, the parts
|
|
// in params among it.
|
|
func newHandler(params Params) *handler {
|
|
awaitingReport, err := lru.New[string, *decision](maxAwaitingReport)
|
|
if err != nil {
|
|
panic(err) // New fails only for a size below one
|
|
}
|
|
|
|
return &handler{
|
|
config: params.Config,
|
|
requestLog: params.RequestLog,
|
|
processLog: params.ProcessLog,
|
|
now: params.Now,
|
|
metrics: params.Metrics,
|
|
awaitingReport: awaitingReport,
|
|
limiter: params.Limiter,
|
|
ledger: params.Ledger,
|
|
geojs: params.GeoJS,
|
|
anomalies: params.Anomalies,
|
|
lookupFile: params.LookupFile,
|
|
lists: params.Lists,
|
|
dnsbl: params.DNSBL,
|
|
abuseIPDB: params.AbuseIPDB,
|
|
routeWatch: params.RouteWatch,
|
|
rules: params.Rules,
|
|
coreRuleSet: params.CoreRuleSet,
|
|
alerts: params.Alerts,
|
|
// The country list, the headers, the tag settings that name an asn=
|
|
// or country= tag and the biased thresholds act on the answer before
|
|
// the request goes on.
|
|
waitForGeoJS: len(params.Config.ExclusivelyAllowedCountries) > 0 ||
|
|
params.Config.AddLookupHeaders ||
|
|
params.Config.TagsNeedLookup() ||
|
|
biasedThresholdsSet(params.Config),
|
|
besideLookupLimit: besideLookupTimeLimit,
|
|
}
|
|
}
|
|
|
|
// NewCoreRuleSet returns the Core Rule Set at SWW_WAF_CORE_RULE_SET_PARANOIA_LEVEL,
|
|
// without the rules SWW_WAF_CORE_RULE_SET_DISABLED_RULES switches off, reading bodies
|
|
// up to SWW_WAF_CORE_RULE_SET_BODY_INSPECTION_SIZE, with the exclusions in
|
|
// SWW_WAF_CORE_RULE_SET_EXCLUSIONS_FILE and the fixes of the apps in SWW_WAF_APPS,
|
|
// or nil while SWW_WAF_CORE_RULE_SET_MODE is off. An exclusions file that
|
|
// coreruleset.New refuses is an error in every mode. While the mode is off, a
|
|
// set file is loaded with the Core Rule Set only to check it.
|
|
func NewCoreRuleSet(cfg *config.WAF) (*coreruleset.CoreRuleSet, error) {
|
|
if cfg.CoreRuleSetMode == config.CoreRuleSetModeOff &&
|
|
cfg.CoreRuleSetExclusionsFile == "" {
|
|
return nil, nil //nolint:nilnil // off is no Core Rule Set, and no error
|
|
}
|
|
|
|
coreRuleSet, err := coreruleset.New(coreruleset.Params{
|
|
ParanoiaLevel: cfg.CoreRuleSetParanoiaLevel,
|
|
DisabledRules: cfg.CoreRuleSetDisabledRules,
|
|
BodyInspectionSize: cfg.CoreRuleSetBodyInspectionSize,
|
|
ExclusionsFile: cfg.CoreRuleSetExclusionsFile,
|
|
Apps: cfg.Apps,
|
|
})
|
|
if err != nil {
|
|
return nil, fmt.Errorf(
|
|
"SWW_WAF_CORE_RULE_SET_EXCLUSIONS_FILE: %w",
|
|
err,
|
|
)
|
|
}
|
|
|
|
if cfg.CoreRuleSetMode == config.CoreRuleSetModeOff {
|
|
return nil, nil //nolint:nilnil // off is no Core Rule Set, and no error
|
|
}
|
|
|
|
return coreRuleSet, nil
|
|
}
|
|
|
|
// handler is waf's handler. It holds what every request shares; what
|
|
// belongs to one request is in a request.
|
|
type handler struct {
|
|
config *config.WAF
|
|
requestLog io.Writer
|
|
processLog *slog.Logger
|
|
now func() time.Time
|
|
metrics *metrics.Metrics
|
|
limiter *ratelimit.Limiter
|
|
ledger *bans.Ledger
|
|
geojs *geojs.GeoJS
|
|
anomalies *anomaly.Counters
|
|
lookupFile *lookup.File
|
|
lists *reputation.Lists
|
|
dnsbl *reputation.DNSBL
|
|
abuseIPDB *reputation.AbuseIPDB
|
|
routeWatch *routewatch.Client
|
|
rules *rules.Files
|
|
coreRuleSet CoreRuleSetInspector
|
|
alerts *alerts.Queue
|
|
// waitForGeoJS is true when a setting needs each request's answer from
|
|
// GeoJS before the request goes on.
|
|
waitForGeoJS bool
|
|
// besideLookupLimit is besideLookupTimeLimit. Only tests change it.
|
|
besideLookupLimit time.Duration
|
|
// awaitingReport holds the decision on each answered question by its id
|
|
// until its report comes, and nil for a question still being handled.
|
|
awaitingReport *lru.Cache[string, *decision]
|
|
}
|