Anomaly thresholds: alerts for unusual traffic, nothing refused (closes #101)

SWWAF_ANOMALY_CLIENT_*, _NET_*, _ASN_*, _TOTAL_* and SWWAF_WATCH_* with
SWWAF_WATCH_NETS: requests and bytes per minute and per hour, each off by
default; with all off, nothing is counted. Otherwise every request but the
health check is counted, allow-listed and exempt ones included; a count
over its threshold raises an anomaly alert, with a cooldown per scope. At
most 20,000 counters, kept in alerts.json. A per-AS-number threshold with
lookups off, or a malformed SWWAF_WATCH_NETS, stops the start. A cooldown
that has run out is dropped as the hour ends, whatever it held back; the
hour's summary gives its repeats.

Judgement call: refused requests are counted too.
Judgement call: per-client counters are kept in alerts.json, which SPEC.md does not list.
Judgement call: a request counts for an AS number only if the lookup answered before it ended.

Model: opus-5-5

internal/alerts/alerts_test.go

@@ -381,7 +381,7 @@ func TestAlertsPastTheHourlyLimitAreRolledIntoOneSummary(t *testing.T) {
 	})
 }
 
-func TestRepeatsBeforeAnAlertPastTheHourlyLimitAreGivenByTheNextSent(t *testing.T) {
+func TestRepeatsBeforeAnAlertPastTheHourlyLimitAreGivenByTheSummary(t *testing.T) {
 	t.Parallel()
 
 	synctest.Test(t, func(t *testing.T) {
@@ -401,8 +401,8 @@ func TestRepeatsBeforeAnAlertPastTheHourlyLimitAreGivenByTheNextSent(t *testing.
 		time.Sleep(cooldown)
 		raise()
 
-		// The next hour's first alert gives the two repeats, and the summary
-		// the alert past the limit.
+		// The summary gives the alert past the limit and the two repeats,
+		// and the next hour's first alert none.
 		time.Sleep(time.Hour - cooldown)
 		synctest.Wait()
 		raise()
@@ -413,11 +413,14 @@ func TestRepeatsBeforeAnAlertPastTheHourlyLimitAreGivenByTheNextSent(t *testing.
 		got := webhook.received()
 		if len(got) == 3 {
 			detail, _ := got[1].alert["detail"].(map[string]any)
-			repeats := got[2].alert["suppressed_repeats"]
+			summaryRepeats := got[1].alert["suppressed_repeats"]
+			lastRepeats := got[2].alert["suppressed_repeats"]
 
-			if detail["count"] != float64(1) || repeats != float64(2) {
-				t.Errorf("the summary counts %v alerts, and the last alert gives %v "+
-					"repeats, want 1 and 2", detail["count"], repeats)
+			if detail["count"] != float64(1) || summaryRepeats != float64(2) ||
+				lastRepeats != float64(0) {
+				t.Errorf("the summary counts %v alerts and %v repeats, and the last "+
+					"alert gives %v repeats, want 1, 2 and 0", detail["count"],
+					summaryRepeats, lastRepeats)
 			}
 		}
 
@@ -425,6 +428,70 @@ func TestRepeatsBeforeAnAlertPastTheHourlyLimitAreGivenByTheNextSent(t *testing.
 	})
 }
 
+func TestCooldownsThatHaveRunOutAreDroppedAndTheirRepeatsSummedUp(t *testing.T) {
+	t.Parallel()
+
+	for name, maxPerHour := range map[string]int{"limit off": 0, "limit set": 60} {
+		t.Run(name, func(t *testing.T) {
+			t.Parallel()
+
+			synctest.Test(t, func(t *testing.T) {
+				params := newParams()
+				params.MaxPerHour = maxPerHour
+				webhook, q := start(t, params)
+
+				// For four hours, a netblock of its own each minute is over an
+				// anomaly threshold twice: an alert, and a repeat the cooldown
+				// holds back.
+				netblocks := 0
+
+				for range 4 {
+					for range 60 {
+						anomaly := alerts.Alert{
+							Event: alerts.EventAnomaly, Netblock: netblock(netblocks),
+							Detail: map[string]any{"scope": "net"},
+						}
+						q.Raise(anomaly)
+						q.Raise(anomaly)
+
+						netblocks++
+
+						time.Sleep(time.Minute)
+					}
+
+					// As the hour ends, only the cooldowns started less than the
+					// cooldown before are kept, in memory and for alerts.json.
+					synctest.Wait()
+
+					kept := len(q.Snapshot().Cooldowns)
+					if kept > int(cooldown/time.Minute) {
+						t.Errorf("after %d netblocks, %d cooldowns are kept, want at most %d",
+							netblocks, kept, int(cooldown/time.Minute))
+					}
+				}
+
+				// An hour on, every cooldown has been dropped, and the summaries
+				// have given every repeat.
+				time.Sleep(time.Hour)
+				synctest.Wait()
+
+				repeats := 0.0
+
+				for _, request := range webhook.received() {
+					count, _ := request.alert["suppressed_repeats"].(float64)
+					repeats += count
+				}
+
+				kept := len(q.Snapshot().Cooldowns)
+				if kept != 0 || repeats != float64(netblocks) {
+					t.Errorf("%d cooldowns are kept and the webhook was given %v repeats, "+
+						"want 0 and %d", kept, repeats, netblocks)
+				}
+			})
+		})
+	}
+}
+
 func TestFailedRequestIsSentAgainWithBackoff(t *testing.T) {
 	t.Parallel()
 
@@ -635,7 +702,8 @@ func TestStateLoadedIntoANewQueueCarriesOn(t *testing.T) {
 		after.Load(roundTrip(t, before.Snapshot()))
 
 		// The new queue sends the alert waiting, holds back the repeat as
-		// the cooldown still runs, and sends the summary of the hour.
+		// the cooldown still runs, and sends the summary of the hour, which
+		// gives both repeats, as the cooldown has run out.
 		after.Raise(alerts.Alert{Event: alerts.EventBan, Netblock: netblock(1)})
 		synctest.Wait()
 		wantEvents(t, webhook, alerts.EventBan)
@@ -644,18 +712,12 @@ func TestStateLoadedIntoANewQueueCarriesOn(t *testing.T) {
 		synctest.Wait()
 		wantEvents(t, webhook, alerts.EventBan, alerts.EventSummary)
 
-		detail, _ := webhook.received()[1].alert["detail"].(map[string]any)
-		if detail["count"] != float64(1) {
-			t.Errorf("the summary counts %v alerts, want 1", detail["count"])
-		}
+		summary := webhook.received()[1].alert
+		detail, _ := summary["detail"].(map[string]any)
 
-		// The cooldown has run out, and the next one gives both repeats.
-		after.Raise(alerts.Alert{Event: alerts.EventBan, Netblock: netblock(1)})
-		synctest.Wait()
-
-		got := webhook.received()
-		if repeats := got[len(got)-1].alert["suppressed_repeats"]; repeats != float64(2) {
-			t.Errorf("the last alert gives %v repeats, want 2", repeats)
+		if detail["count"] != float64(1) || summary["suppressed_repeats"] != float64(2) {
+			t.Errorf("the summary counts %v alerts and %v repeats, want 1 and 2",
+				detail["count"], summary["suppressed_repeats"])
 		}
 	})
 }
@@ -701,8 +763,8 @@ func TestSlackAndNtfyAreSentTheSummaryAndTheRepeatsHeldBack(t *testing.T) {
 		ban := alerts.Alert{Event: alerts.EventBan, Netblock: netblock(1), Reason: "a ban"}
 
 		// The hour's one alert, a repeat of it the cooldown holds back, and
-		// an alert past the limit; once the hour has ended, its summary, and
-		// the next alert, which gives the repeat.
+		// an alert past the limit; once the hour has ended, its summary,
+		// which gives the repeat, and the next alert, which gives none.
 		q.Raise(ban)
 		q.Raise(ban)
 		q.Raise(alerts.Alert{Event: alerts.EventFileError, Reason: "a file error"})
@@ -720,14 +782,14 @@ func TestSlackAndNtfyAreSentTheSummaryAndTheRepeatsHeldBack(t *testing.T) {
 		}
 
 		const summary = "1 alerts held back in the hour from 2000-01-01T00:00:00Z, " +
-			"past the 1 an hour SWWAF_ALERT_MAX_PER_HOUR allows"
+			"past the 1 an hour SWWAF_ALERT_MAX_PER_HOUR allows; 1 repeats held back " +
+			"by SWWAF_ALERT_COOLDOWN that no later alert gives\nsuppressed repeats: 1"
 
 		wantSlackMessage(t, slack[1], "*"+instance+": summary*\n"+summary)
 		wantNtfyMessage(t, ntfy[1], instance+": summary", "default bar_chart", summary)
-		wantSlackMessage(t, slack[2],
-			"*"+instance+": ban*\na ban\nnetblock: 203.0.113.1/32\nsuppressed repeats: 1")
+		wantSlackMessage(t, slack[2], "*"+instance+": ban*\na ban\nnetblock: 203.0.113.1/32")
 		wantNtfyMessage(t, ntfy[2], instance+": ban", "default no_entry",
-			"a ban\nnetblock: 203.0.113.1/32\nsuppressed repeats: 1")
+			"a ban\nnetblock: 203.0.113.1/32")
 	})
 }