Anomaly thresholds: alerts for unusual traffic, nothing refused (closes #101)

SWWAF_ANOMALY_CLIENT_*, _NET_*, _ASN_*, _TOTAL_* and SWWAF_WATCH_* with
SWWAF_WATCH_NETS: requests and bytes per minute and per hour, each off by
default; with all off, nothing is counted. Otherwise every request but the
health check is counted, allow-listed and exempt ones included; a count
over its threshold raises an anomaly alert, with a cooldown per scope. At
most 20,000 counters, kept in alerts.json. A per-AS-number threshold with
lookups off, or a malformed SWWAF_WATCH_NETS, stops the start. A cooldown
that has run out is dropped as the hour ends, whatever it held back; the
hour's summary gives its repeats.

Judgement call: refused requests are counted too.
Judgement call: per-client counters are kept in alerts.json, which SPEC.md does not list.
Judgement call: a request counts for an AS number only if the lookup answered before it ended.

Model: opus-5-5

internal/config/config.go

@@ -24,6 +24,7 @@ import (
 	"unicode/utf8"
 
 	"sneak.berlin/go/smallwebwaf/internal/alerts"
+	"sneak.berlin/go/smallwebwaf/internal/anomaly"
 	"sneak.berlin/go/smallwebwaf/internal/remotelog"
 )
 
@@ -220,6 +221,23 @@ type Config struct {
 	AlertEvents          []string
 	AlertCooldown        time.Duration
 	AlertMaxPerHour      int
+	// The anomaly thresholds, which only raise alerts: the most requests
+	// and bytes a minute and an hour per client (SWWAF_ANOMALY_CLIENT_*),
+	// per netblock around a client (SWWAF_ANOMALY_NET_*), per AS number
+	// (SWWAF_ANOMALY_ASN_*), for the whole service (SWWAF_ANOMALY_TOTAL_*)
+	// and per named netblock (SWWAF_WATCH_*), each 0 while it is off.
+	// AnomalyNetV4Prefix and AnomalyNetV6Prefix are the lengths of the
+	// netblock around a client (SWWAF_ANOMALY_NET_V4_PREFIX and
+	// SWWAF_ANOMALY_NET_V6_PREFIX), and WatchNets the named netblocks
+	// (SWWAF_WATCH_NETS).
+	AnomalyClient      anomaly.Thresholds
+	AnomalyNet         anomaly.Thresholds
+	AnomalyASN         anomaly.Thresholds
+	AnomalyTotal       anomaly.Thresholds
+	AnomalyWatch       anomaly.Thresholds
+	AnomalyNetV4Prefix int
+	AnomalyNetV6Prefix int
+	WatchNets          []anomaly.NamedNetblock
 
 	// settings are the values read, as given or by default, and the
 	// files they were read from, for the log line at start.
@@ -240,6 +258,7 @@ const (
 	mebibyte = 1 << 20
 	gibibyte = 1 << 30
 	ipv4Bits = 32
+	ipv6Bits = 128
 	// minTokenLength is the fewest characters a token may have.
 	minTokenLength = 32
 	// masked is what the log shows for a token that is set, and in place of
@@ -287,6 +306,10 @@ var (
 	errNotBanResponse = errors.New("is not 403, 429 or close")
 	errNotV4Prefix    = errors.New(
 		"is not the length of an IPv4 netblock, from 0 to 32, such as 24")
+	errNotV6Prefix = errors.New(
+		"is not the length of an IPv6 netblock, from 0 to 128, such as 48")
+	errNotNamedNetblock = errors.New(
+		"is not a name, = and a netblock, such as office=203.0.113.0/24")
 	errNotAbsolutePath = errors.New(
 		"is not an absolute path, such as /var/lib/smallwebwaf")
 	errShortToken    = errors.New("is shorter than 32 characters")
@@ -398,8 +421,16 @@ func FromEnvironment(lookupEnv func(string) (string, bool)) (*Config, error) {
 		AlertNtfyToken:       env.secret("SWWAF_ALERT_NTFY_TOKEN"),
 		AlertEvents: env.alertEvents("SWWAF_ALERT_EVENTS",
 			strings.Join(alerts.Events(), ",")),
-		AlertCooldown:   env.duration("SWWAF_ALERT_COOLDOWN", "15m"),
-		AlertMaxPerHour: env.numberOrOff("SWWAF_ALERT_MAX_PER_HOUR", "60"),
+		AlertCooldown:      env.duration("SWWAF_ALERT_COOLDOWN", "15m"),
+		AlertMaxPerHour:    env.numberOrOff("SWWAF_ALERT_MAX_PER_HOUR", "60"),
+		AnomalyClient:      env.thresholds("SWWAF_ANOMALY_CLIENT_"),
+		AnomalyNet:         env.thresholds("SWWAF_ANOMALY_NET_"),
+		AnomalyASN:         env.thresholds("SWWAF_ANOMALY_ASN_"),
+		AnomalyTotal:       env.thresholds("SWWAF_ANOMALY_TOTAL_"),
+		AnomalyWatch:       env.thresholds("SWWAF_WATCH_"),
+		AnomalyNetV4Prefix: env.v4Prefix("SWWAF_ANOMALY_NET_V4_PREFIX", "24"),
+		AnomalyNetV6Prefix: env.v6Prefix("SWWAF_ANOMALY_NET_V6_PREFIX", "48"),
+		WatchNets:          env.namedNetblocks("SWWAF_WATCH_NETS"),
 	}
 
 	cfg.LogRemoteAppName = env.appName("SWWAF_LOG_REMOTE_APP_NAME",
@@ -666,9 +697,9 @@ func (e *environment) checkLookupDBPath(cfg *Config) {
 
 // checkCountriesAndLookups refuses a country on both country lists, and,
 // while SWWAF_LOOKUP_SOURCE is off, each setting that needs clients looked
-// up: the country lists, SWWAF_ADD_LOOKUP_HEADERS, and the biased
-// thresholds, of which SWWAF_UNKNOWN_LIMIT_PERCENT needs them only below
-// 100, where it lowers a limit.
+// up: the country lists, SWWAF_ADD_LOOKUP_HEADERS, the biased thresholds,
+// of which SWWAF_UNKNOWN_LIMIT_PERCENT needs them only below 100, where it
+// lowers a limit, and the anomaly thresholds per AS number.
 func (e *environment) checkCountriesAndLookups(cfg *Config) {
 	for _, country := range cfg.ExclusivelyAllowedCountries {
 		if slices.Contains(cfg.DeniedCountries, country) {
@@ -693,6 +724,10 @@ func (e *environment) checkCountriesAndLookups(cfg *Config) {
 		{"SWWAF_ASN_BYTES_PERCENT", len(cfg.ASNBytesPercent) > 0},
 		{"SWWAF_COUNTRY_BYTES_PERCENT", len(cfg.CountryBytesPercent) > 0},
 		{"SWWAF_UNKNOWN_LIMIT_PERCENT", cfg.UnknownLimitPercent < 100},
+		{"SWWAF_ANOMALY_ASN_REQUESTS_PER_MINUTE", cfg.AnomalyASN.RequestsPerMinute > 0},
+		{"SWWAF_ANOMALY_ASN_REQUESTS_PER_HOUR", cfg.AnomalyASN.RequestsPerHour > 0},
+		{"SWWAF_ANOMALY_ASN_BYTES_PER_MINUTE", cfg.AnomalyASN.BytesPerMinute > 0},
+		{"SWWAF_ANOMALY_ASN_BYTES_PER_HOUR", cfg.AnomalyASN.BytesPerHour > 0},
 	} {
 		if setting.set {
 			e.check(setting.name, fmt.Errorf("is set while SWWAF_LOOKUP_SOURCE is off; %w",
@@ -744,6 +779,35 @@ func (e *environment) v4Prefix(name, defaultValue string) int {
 	return length
 }
 
+// v6Prefix reads a setting that is the length of an IPv6 netblock.
+func (e *environment) v6Prefix(name, defaultValue string) int {
+	length, err := parseV6Prefix(e.value(name, defaultValue))
+	e.check(name, err)
+
+	return length
+}
+
+// thresholds reads the four anomaly thresholds whose settings' names
+// start with prefix: requests and bytes per minute and per hour. Each is
+// off by default.
+func (e *environment) thresholds(prefix string) anomaly.Thresholds {
+	return anomaly.Thresholds{
+		RequestsPerMinute: e.count(prefix+"REQUESTS_PER_MINUTE", off),
+		RequestsPerHour:   e.count(prefix+"REQUESTS_PER_HOUR", off),
+		BytesPerMinute:    e.size(prefix+"BYTES_PER_MINUTE", off),
+		BytesPerHour:      e.size(prefix+"BYTES_PER_HOUR", off),
+	}
+}
+
+// namedNetblocks reads a setting that is a list of named netblocks. It is
+// empty by default.
+func (e *environment) namedNetblocks(name string) []anomaly.NamedNetblock {
+	named, err := parseNamedNetblocks(e.value(name, ""))
+	e.check(name, err)
+
+	return named
+}
+
 // absolutePath reads a setting that is an absolute path.
 func (e *environment) absolutePath(name, defaultValue string) string {
 	path := e.value(name, defaultValue)
@@ -1085,6 +1149,16 @@ func parseV4Prefix(value string) (int, error) {
 	return n, nil
 }
 
+// parseV6Prefix reads the length of an IPv6 netblock, from 0 to 128.
+func parseV6Prefix(value string) (int, error) {
+	n, err := strconv.Atoi(value)
+	if err != nil || n < 0 || n > ipv6Bits {
+		return 0, fmt.Errorf("%q %w", value, errNotV6Prefix)
+	}
+
+	return n, nil
+}
+
 // parseList splits a comma-separated list and trims the spaces around
 // each item. An empty value is an empty list.
 func parseList(value string) ([]string, error) {
@@ -1144,6 +1218,42 @@ func parseNetblock(value string) (netip.Prefix, error) {
 	return netip.PrefixFrom(addr, addr.BitLen()), nil
 }
 
+// parseNamedNetblocks reads a comma-separated list of named netblocks,
+// each a name, = and a netblock, such as office=203.0.113.0/24. An empty
+// value is an empty list. A name listed twice is an error.
+func parseNamedNetblocks(value string) ([]anomaly.NamedNetblock, error) {
+	items, err := parseList(value)
+	if err != nil {
+		return nil, err
+	}
+
+	named := make([]anomaly.NamedNetblock, 0, len(items))
+
+	for _, item := range items {
+		name, netblockText, found := strings.Cut(item, "=")
+		name = strings.TrimSpace(name)
+
+		if !found || name == "" {
+			return nil, fmt.Errorf("%q %w", item, errNotNamedNetblock)
+		}
+
+		netblock, err := parseNetblock(strings.TrimSpace(netblockText))
+		if err != nil {
+			return nil, err
+		}
+
+		if slices.ContainsFunc(named, func(n anomaly.NamedNetblock) bool {
+			return n.Name == name
+		}) {
+			return nil, fmt.Errorf("%q %w", name, errListedTwice)
+		}
+
+		named = append(named, anomaly.NamedNetblock{Name: name, Netblock: netblock})
+	}
+
+	return named, nil
+}
+
 // parsePathPrefixes reads a comma-separated list of path prefixes, each
 // starting with /.
 func parsePathPrefixes(value string) ([]string, error) {

internal/config/config_test.go

@@ -12,10 +12,12 @@ import (
 	"path/filepath"
 	"reflect"
 	"slices"
+	"strconv"
 	"strings"
 	"testing"
 	"time"
 
+	"sneak.berlin/go/smallwebwaf/internal/anomaly"
 	"sneak.berlin/go/smallwebwaf/internal/config"
 )
 
@@ -85,8 +87,59 @@ const (
 	alertEvents             = "SWWAF_ALERT_EVENTS"
 	alertCooldown           = "SWWAF_ALERT_COOLDOWN"
 	alertMaxPerHour         = "SWWAF_ALERT_MAX_PER_HOUR"
+	anomalyNetV4Prefix      = "SWWAF_ANOMALY_NET_V4_PREFIX"
+	anomalyNetV6Prefix      = "SWWAF_ANOMALY_NET_V6_PREFIX"
+	watchNets               = "SWWAF_WATCH_NETS"
 )
 
+// The anomaly thresholds: each of the prefixes below, which name a scope,
+// followed by each of the four ends.
+const (
+	anomalyClient = "SWWAF_ANOMALY_CLIENT_"
+	anomalyNet    = "SWWAF_ANOMALY_NET_"
+	anomalyASN    = "SWWAF_ANOMALY_ASN_"
+	anomalyTotal  = "SWWAF_ANOMALY_TOTAL_"
+	watch         = "SWWAF_WATCH_"
+
+	requestsPerMinute = "REQUESTS_PER_MINUTE"
+	requestsPerHour   = "REQUESTS_PER_HOUR"
+	bytesPerMinute    = "BYTES_PER_MINUTE"
+	bytesPerHour      = "BYTES_PER_HOUR"
+)
+
+// anomalyScopes returns the prefixes of the anomaly thresholds, one for
+// each scope.
+func anomalyScopes() []string {
+	return []string{anomalyClient, anomalyNet, anomalyASN, anomalyTotal, watch}
+}
+
+// anomalyThresholds returns the names of the twenty anomaly thresholds.
+func anomalyThresholds() []string {
+	ends := []string{requestsPerMinute, requestsPerHour, bytesPerMinute, bytesPerHour}
+	names := make([]string, 0, len(anomalyScopes())*len(ends))
+
+	for _, scope := range anomalyScopes() {
+		for _, end := range ends {
+			names = append(names, scope+end)
+		}
+	}
+
+	return names
+}
+
+// loggedAnomalyDefaults returns the anomaly settings as the settings
+// logged at start give them by default.
+func loggedAnomalyDefaults() map[string]string {
+	logged := map[string]string{
+		anomalyNetV4Prefix: "24", anomalyNetV6Prefix: "48", watchNets: "",
+	}
+	for _, name := range anomalyThresholds() {
+		logged[name] = off
+	}
+
+	return logged
+}
+
 // defaultAlertEvents is the default of SWWAF_ALERT_EVENTS, and
 // defaultAlertCooldown that of SWWAF_ALERT_COOLDOWN.
 const (
@@ -897,14 +950,18 @@ func TestSettingNeedingLookupsStopsTheStartWhileTheyAreOff(t *testing.T) {
 	t.Parallel()
 
 	for name, value := range map[string]string{
-		deniedCountries:     "kp",
-		allowedCountries:    "de",
-		addLookupHeaders:    enabled,
-		asnLimitPercent:     "AS64496:50",
-		countryLimitPercent: "cn:25",
-		asnBytesPercent:     "AS64496:50",
-		countryBytesPercent: "cn:25",
-		unknownLimitPercent: "99",
+		deniedCountries:                "kp",
+		allowedCountries:               "de",
+		addLookupHeaders:               enabled,
+		asnLimitPercent:                "AS64496:50",
+		countryLimitPercent:            "cn:25",
+		asnBytesPercent:                "AS64496:50",
+		countryBytesPercent:            "cn:25",
+		unknownLimitPercent:            "99",
+		anomalyASN + requestsPerMinute: "1000",
+		anomalyASN + requestsPerHour:   "10000",
+		anomalyASN + bytesPerMinute:    "1G",
+		anomalyASN + bytesPerHour:      "10G",
 	} {
 		t.Run(name, func(t *testing.T) {
 			t.Parallel()
@@ -920,12 +977,153 @@ func TestSettingNeedingLookupsStopsTheStartWhileTheyAreOff(t *testing.T) {
 	}
 
 	// Set empty, the lists need nothing looked up, and nor does
-	// SWWAF_UNKNOWN_LIMIT_PERCENT at 100, which lowers no limit.
-	fromEnvironment(t, environment{
+	// SWWAF_UNKNOWN_LIMIT_PERCENT at 100, which lowers no limit, an anomaly
+	// threshold per AS number that is off, or any other anomaly threshold.
+	env := environment{
 		lookupSource: off, deniedCountries: "", allowedCountries: "",
 		asnLimitPercent: "", countryLimitPercent: "", asnBytesPercent: "",
 		countryBytesPercent: "", unknownLimitPercent: "100",
-	})
+	}
+	for _, name := range anomalyThresholds() {
+		env[name] = "1000"
+		if strings.HasPrefix(name, anomalyASN) {
+			env[name] = off
+		}
+	}
+
+	fromEnvironment(t, env)
+}
+
+func TestAnomalySettingsDefaults(t *testing.T) {
+	t.Parallel()
+
+	cfg := fromEnvironment(t, environment{})
+	wantAllOff(t, cfg)
+
+	if cfg.AnomalyNetV4Prefix != 24 || cfg.AnomalyNetV6Prefix != 48 ||
+		len(cfg.WatchNets) != 0 {
+		t.Errorf("%s, %s and %s gave %d, %d and %v, want 24, 48 and none",
+			anomalyNetV4Prefix, anomalyNetV6Prefix, watchNets, cfg.AnomalyNetV4Prefix,
+			cfg.AnomalyNetV6Prefix, cfg.WatchNets)
+	}
+}
+
+func TestAnomalySettingsAsSet(t *testing.T) {
+	t.Parallel()
+
+	// Each threshold of a scope its own value; bytes are sizes.
+	env := environment{
+		anomalyNetV4Prefix: "16",
+		anomalyNetV6Prefix: "56",
+		// Spaces around a name or a netblock, and a bare address.
+		watchNets: "office = 203.0.113.0/24, scraper-x=198.51.100.7,v6=2001:db8::/32",
+	}
+	want := map[string]anomaly.Thresholds{}
+
+	for i, scope := range anomalyScopes() {
+		n := int64(i + 1)
+		env[scope+requestsPerMinute] = strconv.FormatInt(n, 10)
+		env[scope+requestsPerHour] = strconv.FormatInt(10*n, 10)
+		env[scope+bytesPerMinute] = strconv.FormatInt(n, 10) + "K"
+		env[scope+bytesPerHour] = strconv.FormatInt(n, 10) + "G"
+		want[scope] = anomaly.Thresholds{
+			RequestsPerMinute: n, RequestsPerHour: 10 * n,
+			BytesPerMinute: n << 10, BytesPerHour: n << 30,
+		}
+	}
+
+	cfg := fromEnvironment(t, env)
+
+	if got := thresholdsByScope(cfg); !maps.Equal(got, want) {
+		t.Errorf("thresholds by scope\n%+v\nwant\n%+v", got, want)
+	}
+
+	wantNamed := []anomaly.NamedNetblock{
+		{Name: "office", Netblock: netip.MustParsePrefix("203.0.113.0/24")},
+		{Name: "scraper-x", Netblock: netip.MustParsePrefix("198.51.100.7/32")},
+		{Name: "v6", Netblock: netip.MustParsePrefix("2001:db8::/32")},
+	}
+	if cfg.AnomalyNetV4Prefix != 16 || cfg.AnomalyNetV6Prefix != 56 ||
+		!slices.Equal(cfg.WatchNets, wantNamed) {
+		t.Errorf("%s, %s and %s gave %d, %d and %v, want 16, 56 and %v",
+			anomalyNetV4Prefix, anomalyNetV6Prefix, watchNets, cfg.AnomalyNetV4Prefix,
+			cfg.AnomalyNetV6Prefix, cfg.WatchNets, wantNamed)
+	}
+
+	// off switches each threshold off.
+	for _, name := range anomalyThresholds() {
+		env[name] = off
+	}
+
+	wantAllOff(t, fromEnvironment(t, env))
+}
+
+// thresholdsByScope returns cfg's anomaly thresholds, each by the prefix
+// of its scope's settings.
+func thresholdsByScope(cfg *config.Config) map[string]anomaly.Thresholds {
+	return map[string]anomaly.Thresholds{
+		anomalyClient: cfg.AnomalyClient, anomalyNet: cfg.AnomalyNet,
+		anomalyASN: cfg.AnomalyASN, anomalyTotal: cfg.AnomalyTotal,
+		watch: cfg.AnomalyWatch,
+	}
+}
+
+// wantAllOff checks that every anomaly threshold of cfg is off.
+func wantAllOff(t *testing.T, cfg *config.Config) {
+	t.Helper()
+
+	for scope, thresholds := range thresholdsByScope(cfg) {
+		if thresholds != (anomaly.Thresholds{}) {
+			t.Errorf("%s* gave %+v, want every one off", scope, thresholds)
+		}
+	}
+}
+
+func TestInvalidAnomalySettingStopsTheStartSayingWhatIsWrong(t *testing.T) {
+	t.Parallel()
+
+	const (
+		notCount        = " is not a whole number of requests such as 1000, or off"
+		notSize         = " is not a size such as 512K, 100M or 5G, or off"
+		notPositive     = " must be more than zero, or off"
+		notNamed        = " is not a name, = and a netblock, such as office=203.0.113.0/24"
+		notNetblock     = " is not a netblock such as 10.0.0.0/8, or an address"
+		notV4Prefix     = " is not the length of an IPv4 netblock, from 0 to 32, such as 24"
+		notV6Prefix     = " is not the length of an IPv6 netblock, from 0 to 128, such as 48"
+		officeNetblock  = "office=203.0.113.0/24"
+		scraperNetblock = "scraper=198.51.100.0/24"
+	)
+
+	for _, tc := range []struct{ name, value, want string }{
+		{anomalyClient + requestsPerMinute, "1K", `"1K"` + notCount},
+		{anomalyNet + requestsPerHour, "0", `"0"` + notPositive},
+		{anomalyTotal + bytesPerMinute, "1T", `"1T"` + notSize},
+		{watch + bytesPerHour, "-1G", `"-1G"` + notPositive},
+		{anomalyNetV4Prefix, "33", `"33"` + notV4Prefix},
+		{anomalyNetV4Prefix, off, `"off"` + notV4Prefix},
+		{anomalyNetV6Prefix, "129", `"129"` + notV6Prefix},
+		{anomalyNetV6Prefix, "/48", `"/48"` + notV6Prefix},
+		{watchNets, "office", `"office"` + notNamed},
+		{watchNets, "=203.0.113.0/24", `"=203.0.113.0/24"` + notNamed},
+		{watchNets, "office=203.0.113.300/24", `"203.0.113.300/24"` + notNetblock},
+		{watchNets, officeNetblock + ",", `"` + officeNetblock + `," has an empty item ` +
+			`in its list`},
+		{
+			watchNets, officeNetblock + "," + scraperNetblock + ",office=192.0.2.0/24",
+			`"office" is listed twice`,
+		},
+	} {
+		t.Run(tc.name+"="+tc.value, func(t *testing.T) {
+			t.Parallel()
+
+			_, err := config.FromEnvironment(environment{tc.name: tc.value}.lookupEnv)
+
+			want := tc.name + ": " + tc.want
+			if err == nil || err.Error() != want {
+				t.Errorf("error %v, want %s", err, want)
+			}
+		})
+	}
 }
 
 func TestBiasedThresholdsAsSet(t *testing.T) {
@@ -1461,6 +1659,8 @@ func TestLogsEachSettingWithItsValue(t *testing.T) {
 		alertCooldown:           defaultAlertCooldown,
 		alertMaxPerHour:         "60",
 	}
+	maps.Copy(want, loggedAnomalyDefaults())
+
 	if got := loggedSettings(t, cfg); !maps.Equal(got, want) {
 		t.Errorf("logged settings\n%v\nwant\n%v", got, want)
 	}