smallwebwaf/internal/discriminator/proxy.go

// Package discriminator is the server the discriminator command runs. It answers
// rproxy's questions, takes rproxy's reports and answers its own endpoints, and
// writes a log line for each.
package discriminator

import (
	"fmt"
	"io"
	"log/slog"
	"net/http"
	"time"

	lru "github.com/hashicorp/golang-lru/v2"

	"sneak.berlin/go/smallwebwaf/internal/alerts"
	"sneak.berlin/go/smallwebwaf/internal/anomaly"
	"sneak.berlin/go/smallwebwaf/internal/bans"
	"sneak.berlin/go/smallwebwaf/internal/config"
	"sneak.berlin/go/smallwebwaf/internal/lookup"
	"sneak.berlin/go/smallwebwaf/internal/metrics"
	"sneak.berlin/go/smallwebwaf/internal/ratelimit"
	"sneak.berlin/go/smallwebwaf/internal/reputation"
	"sneak.berlin/go/smallwebwaf/internal/rules"
	"sneak.berlin/go/smallwebwaf/internal/waf"
)

// adminPrefix starts the path of each of discriminator's own endpoints. The app never
// sees a path under it.
const adminPrefix = "/_sww/"

// HealthPath is the health endpoint of discriminator and of rproxy, which the
// container's health check asks.
const HealthPath = "/_sww/healthz"

// MetricsPath is where the metrics are, for a request that carries
// SWW_WAF_METRICS_TOKEN.
const MetricsPath = "/_sww/metrics"

// BansPath is where an admin lists and adds bans, and, followed by / and
// a client's address, lifts them, with SWW_WAF_ADMIN_TOKEN.
const BansPath = "/_sww/bans"

// ClientsPath is where an admin asks what discriminator knows of a client, by
// the client's address after it, with SWW_WAF_ADMIN_TOKEN.
const ClientsPath = "/_sww/clients/"

// Params are what New needs.
type Params struct {
	Config *config.Config
	// RequestLog receives two JSON lines per question, one at its decision and
	// one when its report comes, and one per request to discriminator's own
	// endpoints.
	RequestLog io.Writer
	// ProcessLog receives the process's own messages.
	ProcessLog *slog.Logger
	// GeoJSURL is where clients' AS numbers and countries are looked up
	// while SWW_WAF_LOOKUP_SOURCE is geojs, normally lookup.URL.
	GeoJSURL string
	// AbuseIPDBURL is where clients are checked with AbuseIPDB while
	// SWW_WAF_ABUSEIPDB_KEY is set, normally reputation.AbuseIPDBURL.
	AbuseIPDBURL string
	// LookupFile is the lookup database they are looked up in while
	// SWW_WAF_LOOKUP_SOURCE is file, and nil otherwise.
	LookupFile *lookup.File
	// CoreRuleSet is what NewCoreRuleSet returns: what inspects requests
	// while SWW_WAF_CORE_RULE_SET_MODE is not off, and nil while it is.
	CoreRuleSet *waf.CoreRuleSet
	// Now tells the time by which requests are counted for the rate
	// limits, bans are made and run out, and GeoJS's answers are kept,
	// normally time.Now in UTC, the time the state files give.
	Now func() time.Time
	// Rules are the rule files' rules, which each request is checked
	// against.
	Rules *rules.Files
	// Alerts receive the alert for each ban discriminator makes or makes
	// permanent, for each count over an anomaly threshold, for each request
	// whose client a blocklist, the CrowdSec decision list, a DNSBL zone or
	// AbuseIPDB lists, for each request the Core Rule Set scores at or over
	// SWW_WAF_CORE_RULE_SET_ANOMALY_THRESHOLD, and for GeoJS failing, a fetch of a list
	// failing, a query to a DNSBL zone or a check with AbuseIPDB failing,
	// or the day's AbuseIPDB checks used up.
	Alerts *alerts.Queue
}

// Server is the server discriminator runs, with the parts whose state the state
// files keep, the lookup database, nil unless SWW_WAF_LOOKUP_SOURCE is file, the
// lists fetched from URLs, which its Run fetches, the DNSBL zones' verdicts,
// AbuseIPDB's scores and checks spent, and the metrics.
type Server struct {
	*http.Server

	Ledger     *bans.Ledger
	Limiter    *ratelimit.Limiter
	GeoJS      *lookup.GeoJS
	Anomalies  *anomaly.Counters
	LookupFile *lookup.File
	Lists      *reputation.Lists
	DNSBL      *reputation.DNSBL
	AbuseIPDB  *reputation.AbuseIPDB
	Metrics    *metrics.Metrics
}

// newHandler returns the handler with what every request shares, as the
// settings in params make it.
func newHandler(params Params) *handler {
	m := metrics.New(params.Config.MetricsTopN, params.Config.InstanceName)
	lists, dnsbl, abuseIPDB := newReputation(params, m)

	awaitingReport, err := lru.New[string, *decision](maxAwaitingReport)
	if err != nil {
		panic(err) // New fails only for a size below one
	}

	h := &handler{
		config:         params.Config,
		requestLog:     params.RequestLog,
		processLog:     params.ProcessLog,
		now:            params.Now,
		metrics:        m,
		awaitingReport: awaitingReport,
		limiter: ratelimit.New(ratelimit.Limits{
			PerMinute:      params.Config.RateLimitPerMinute,
			PerHour:        params.Config.RateLimitPerHour,
			PerDay:         params.Config.RateLimitPerDay,
			BytesPerMinute: params.Config.BytesLimitPerMinute,
			BytesPerHour:   params.Config.BytesLimitPerHour,
			BytesPerDay:    params.Config.BytesLimitPerDay,
		}, params.Config.MaxTrackedClients),
		ledger: bans.New(bans.Rules{
			LimitBanDuration:     params.Config.LimitBanDuration,
			LimitBanRepeatWindow: params.Config.LimitBanRepeatWindow,
			MaxBanDuration:       params.Config.MaxBanDuration,
			AttackBanDuration:    params.Config.AttackBanDuration,
			MaxBans:              params.Config.MaxBans,
		}),
		anomalies: anomaly.New(anomaly.Params{
			Client:         params.Config.AnomalyClient,
			Net:            params.Config.AnomalyNet,
			ASN:            params.Config.AnomalyASN,
			Total:          params.Config.AnomalyTotal,
			Watch:          params.Config.AnomalyWatch,
			NetV4Prefix:    params.Config.AnomalyNetV4Prefix,
			NetV6Prefix:    params.Config.AnomalyNetV6Prefix,
			NamedNetblocks: params.Config.WatchNets,
			Alerts:         params.Alerts,
		}),
		lookupFile:  params.LookupFile,
		lists:       lists,
		dnsbl:       dnsbl,
		abuseIPDB:   abuseIPDB,
		rules:       params.Rules,
		coreRuleSet: params.CoreRuleSet,
		alerts:      params.Alerts,
	}
	h.geojs = lookup.New(lookup.Params{
		URL:     params.GeoJSURL,
		Timeout: params.Config.LookupTimeout,
		// The country lists, the headers and the biased thresholds act on
		// the answer before the request goes on.
		Wait: len(params.Config.DeniedCountries) > 0 ||
			len(params.Config.ExclusivelyAllowedCountries) > 0 ||
			params.Config.AddLookupHeaders || biasedThresholdsSet(params.Config),
		Answered:   h.addLookup,
		Now:        params.Now,
		ProcessLog: params.ProcessLog,
		Metrics:    m,
		Alerts:     params.Alerts,
	})
	m.AddBansAndClients(h.ledger, h.limiter, params.Now)
	m.AddRules(params.Rules)

	return h
}

// newReputation returns the lists fetched from URLs, the DNSBL zones'
// verdicts and AbuseIPDB's scores, as the settings in params name them,
// with none fetched, asked for or checked yet, and adds their metrics to
// m, AbuseIPDB's while SWW_WAF_ABUSEIPDB_KEY is set.
func newReputation(
	params Params, m *metrics.Metrics,
) (*reputation.Lists, *reputation.DNSBL, *reputation.AbuseIPDB) {
	cfg := params.Config
	lists := reputation.New(reputation.Params{
		BlocklistURLs: cfg.BlocklistURLs, Refresh: cfg.BlocklistRefresh,
		ASNLimitPercentURL:   cfg.ASNLimitPercentURL,
		CrowdSecDecisionsURL: cfg.CrowdSecDecisionsURL, CrowdSecKey: cfg.CrowdSecKey,
		Now: params.Now, ProcessLog: params.ProcessLog, Alerts: params.Alerts,
	})
	dnsbl := reputation.NewDNSBL(reputation.DNSBLParams{
		Zones: cfg.DNSBLZones, Resolver: cfg.DNSBLResolver, CacheTTL: cfg.ReputationCacheTTL,
		Timeout: cfg.ReputationTimeout, Now: params.Now, ProcessLog: params.ProcessLog,
		Alerts: params.Alerts,
	})
	abuseIPDB := reputation.NewAbuseIPDB(reputation.AbuseIPDBParams{
		URL: params.AbuseIPDBURL, Key: cfg.AbuseIPDBKey, MinScore: cfg.AbuseIPDBMinScore,
		DailyBudget: cfg.AbuseIPDBDailyBudget, CacheTTL: cfg.ReputationCacheTTL,
		Timeout: cfg.ReputationTimeout, Now: params.Now, ProcessLog: params.ProcessLog,
		Alerts: params.Alerts,
	})

	m.AddReputation(lists, dnsbl)

	if cfg.AbuseIPDBKey != "" {
		m.AddAbuseIPDB(abuseIPDB)
	}

	return lists, dnsbl, abuseIPDB
}

// NewCoreRuleSet returns the Core Rule Set at SWW_WAF_CORE_RULE_SET_PARANOIA_LEVEL,
// without the rules SWW_WAF_CORE_RULE_SET_DISABLED_RULES switches off, reading bodies
// up to SWW_WAF_CORE_RULE_SET_BODY_LIMIT, with the exclusions in
// SWW_WAF_CORE_RULE_SET_EXCLUSIONS_FILE, or nil while SWW_WAF_CORE_RULE_SET_MODE is
// off. It reads at most QuestionBodyMaxBytes less one byte: a question's body of
// QuestionBodyMaxBytes may be cut short, and the Core Rule Set does not read a
// JSON or XML body over its limit. An exclusions file that waf.New refuses is
// an error in every mode. While the mode is off, a set file is loaded with the
// Core Rule Set only to check it.
func NewCoreRuleSet(cfg *config.Config) (*waf.CoreRuleSet, error) {
	if cfg.WAFMode == config.WAFModeOff && cfg.CoreRuleSetExclusionsFile == "" {
		return nil, nil //nolint:nilnil // off is no Core Rule Set, and no error
	}

	coreRuleSet, err := waf.New(waf.Params{
		ParanoiaLevel: cfg.WAFParanoiaLevel, DisabledRules: cfg.WAFDisabledRules,
		BodyLimit:      min(cfg.WAFBodyLimit, QuestionBodyMaxBytes-1),
		ExclusionsFile: cfg.CoreRuleSetExclusionsFile,
	})
	if err != nil {
		return nil, fmt.Errorf("SWW_WAF_CORE_RULE_SET_EXCLUSIONS_FILE: %w", err)
	}

	if cfg.WAFMode == config.WAFModeOff {
		return nil, nil //nolint:nilnil // off is no Core Rule Set, and no error
	}

	return coreRuleSet, nil
}

// handler is discriminator's handler. It holds what every request shares; what
// belongs to one request is in a request.
type handler struct {
	config      *config.Config
	requestLog  io.Writer
	processLog  *slog.Logger
	now         func() time.Time
	metrics     *metrics.Metrics
	limiter     *ratelimit.Limiter
	ledger      *bans.Ledger
	geojs       *lookup.GeoJS
	anomalies   *anomaly.Counters
	lookupFile  *lookup.File
	lists       *reputation.Lists
	dnsbl       *reputation.DNSBL
	abuseIPDB   *reputation.AbuseIPDB
	rules       *rules.Files
	coreRuleSet *waf.CoreRuleSet
	alerts      *alerts.Queue
	// awaitingReport holds the decision on each answered question by its id
	// until its report comes, and nil for a question still being handled.
	awaitingReport *lru.Cache[string, *decision]
}