Anomaly thresholds: alerts for unusual traffic, nothing refused (closes #101)

SWWAF_ANOMALY_CLIENT_*, _NET_*, _ASN_*, _TOTAL_* and SWWAF_WATCH_* with
SWWAF_WATCH_NETS: requests and bytes per minute and per hour, each off by
default; with all off, nothing is counted. Otherwise every request but the
health check is counted, allow-listed and exempt ones included; a count
over its threshold raises an anomaly alert, with a cooldown per scope. At
most 20,000 counters, kept in alerts.json. A per-AS-number threshold with
lookups off, or a malformed SWWAF_WATCH_NETS, stops the start. A cooldown
that has run out is dropped as the hour ends, whatever it held back; the
hour's summary gives its repeats.

Judgement call: refused requests are counted too.
Judgement call: per-client counters are kept in alerts.json, which SPEC.md does not list.
Judgement call: a request counts for an AS number only if the lookup answered before it ended.

Model: opus-5-5

internal/alerts/alerts.go

@@ -1,5 +1,6 @@
-// Package alerts sends alerts on bans, on a source that fails and on a
-// file with an error to each destination set: to the webhook
+// Package alerts sends alerts on bans, on traffic over an anomaly
+// threshold, on a source that fails and on a file with an error to each
+// destination set: to the webhook
 // SWWAF_ALERT_WEBHOOK_URL names, each as one JSON object, as the "Alert
 // webhook schema" section of SPEC.md describes, to the Slack incoming
 // webhook SWWAF_ALERT_SLACK_WEBHOOK_URL names, as a message, and to the
@@ -40,11 +41,12 @@ const (
 	// EventPermanentBan is a permanent ban smallwebwaf made, or a ban it
 	// made permanent.
 	EventPermanentBan = "permanent_ban"
-	// EventWAFBlock, EventAnomaly and EventReputationHit come with the
-	// Core Rule Set, the anomaly thresholds and the reputation sources;
-	// nothing raises them yet.
+	// EventAnomaly is a count of requests or bytes over an anomaly
+	// threshold.
+	EventAnomaly = "anomaly"
+	// EventWAFBlock and EventReputationHit come with the Core Rule Set and
+	// the reputation sources; nothing raises them yet.
 	EventWAFBlock      = "waf_block"
-	EventAnomaly       = "anomaly"
 	EventReputationHit = "reputation_hit"
 	// EventSourceFailure is GeoJS failing or refusing smallwebwaf.
 	EventSourceFailure = "source_failure"
@@ -153,18 +155,22 @@ type Alert struct {
 	ASName   string       `json:"as_name"`
 	Country  string       `json:"country"`
 	// Reason is a short sentence, and Detail what is particular to the
-	// event: for a file_error, its "file", and for a source_failure, its
-	// "source", which the cooldown tells repeats by.
+	// event: for a file_error, its "file", for a source_failure, its
+	// "source", and for an anomaly, its "scope", with the "asn" or the
+	// "name" of some scopes, which the cooldown tells repeats by.
 	Reason string         `json:"reason"`
 	Detail map[string]any `json:"detail"`
 	// SuppressedRepeats is how many repeats of the alert the cooldown
-	// held back since the last one let through.
+	// held back since the last one let through. For a summary, it is how
+	// many the cooldowns dropped as the hour ended had held back that no
+	// alert let through gave.
 	SuppressedRepeats int `json:"suppressed_repeats"`
 }
 
-// Cooldown is, for an event on a netblock, or about a file or a source,
-// when the last alert let through was raised, and how many repeats the
-// cooldown has held back since, as alerts.json holds it.
+// Cooldown is, for an event on a netblock, about a file or a source, or
+// for an anomaly in a scope, when the last alert let through was raised,
+// and how many repeats the cooldown has held back since, as alerts.json
+// holds it.
 //
 //nolint:tagliatelle // the state files use snake_case, as the request log does
 type Cooldown struct {
@@ -172,6 +178,9 @@ type Cooldown struct {
 	Netblock          netip.Prefix `json:"netblock"`
 	File              string       `json:"file,omitempty"`
 	Source            string       `json:"source,omitempty"`
+	Scope             string       `json:"scope,omitempty"`
+	ASN               string       `json:"asn,omitempty"`
+	Name              string       `json:"name,omitempty"`
 	Sent              time.Time    `json:"sent"`
 	SuppressedRepeats int          `json:"suppressed_repeats"`
 }
@@ -214,7 +223,7 @@ type Queue struct {
 
 	mu sync.Mutex
 	// cooldowns are the alerts last let through, by event and netblock,
-	// file or source.
+	// file, source or scope.
 	cooldowns map[cooldownKey]*Cooldown
 	hour      Hour
 
@@ -248,21 +257,28 @@ type destination struct {
 }
 
 // cooldownKey is what makes an alert a repeat of another: the same event
-// on the same netblock, and about the same file or source, as its detail
-// names them. Each is empty for an alert without one.
+// on the same netblock, and about the same file or source, or in the same
+// scope with the same AS number or name, as its detail names them. Each
+// is empty for an alert without one.
 type cooldownKey struct {
 	event    string
 	netblock netip.Prefix
 	file     string
 	source   string
+	scope    string
+	asn      string
+	name     string
 }
 
 // cooldownKeyOf returns what makes another alert a repeat of alert.
 func cooldownKeyOf(alert *Alert) cooldownKey {
 	file, _ := alert.Detail["file"].(string)
 	source, _ := alert.Detail["source"].(string)
+	scope, _ := alert.Detail["scope"].(string)
+	asn, _ := alert.Detail["asn"].(string)
+	name, _ := alert.Detail["name"].(string)
 
-	return cooldownKey{alert.Event, alert.Netblock, file, source}
+	return cooldownKey{alert.Event, alert.Netblock, file, source, scope, asn, name}
 }
 
 // New returns a Queue with no alert yet.
@@ -299,7 +315,8 @@ func New(params Params) *Queue {
 // alerts let through in the hour under way, by the clock, an alert is
 // held back for that hour's summary instead, which is sent once the hour
 // has ended; it starts no cooldown, and the repeats held back before it
-// are given by the next alert let through. Raise never waits: an alert
+// are given by that summary, as their cooldown, which has run out, is
+// dropped when the hour ends. Raise never waits: an alert
 // let through joins the queue of each destination, from which Run sends
 // it, and with queueSize alerts waiting for a destination, the oldest is
 // dropped.
@@ -425,7 +442,8 @@ func (q *Queue) Suppressed() int64 {
 }
 
 // Snapshot returns the queue's state, as alerts.json holds it, with the
-// cooldowns sorted by netblock, then by event, file and source.
+// cooldowns sorted by netblock, then by event, file, source, scope, AS
+// number and name.
 func (q *Queue) Snapshot() State {
 	q.mu.Lock()
 	defer q.mu.Unlock()
@@ -443,7 +461,9 @@ func (q *Queue) Snapshot() State {
 
 	slices.SortFunc(state.Cooldowns, func(a, b Cooldown) int {
 		return cmp.Or(a.Netblock.Compare(b.Netblock), cmp.Compare(a.Event, b.Event),
-			cmp.Compare(a.File, b.File), cmp.Compare(a.Source, b.Source))
+			cmp.Compare(a.File, b.File), cmp.Compare(a.Source, b.Source),
+			cmp.Compare(a.Scope, b.Scope), cmp.Compare(a.ASN, b.ASN),
+			cmp.Compare(a.Name, b.Name))
 	})
 
 	for _, d := range q.destinations {
@@ -466,7 +486,10 @@ func (q *Queue) Load(state State) {
 
 	for _, cooldown := range state.Cooldowns {
 		cooldown.Netblock = cooldown.Netblock.Masked()
-		key := cooldownKey{cooldown.Event, cooldown.Netblock, cooldown.File, cooldown.Source}
+		key := cooldownKey{
+			cooldown.Event, cooldown.Netblock, cooldown.File, cooldown.Source,
+			cooldown.Scope, cooldown.ASN, cooldown.Name,
+		}
 		q.cooldowns[key] = &cooldown
 	}
 
@@ -538,46 +561,64 @@ func (q *Queue) startCooldown(alert *Alert, now time.Time) {
 
 	q.cooldowns[key] = &Cooldown{
 		Event: alert.Event, Netblock: alert.Netblock, File: key.file, Source: key.source,
-		Sent: now,
+		Scope: key.scope, ASN: key.asn, Name: key.name, Sent: now,
 	}
 }
 
-// endHour ends the hour under way, if now is past it: it queues that
-// hour's summary when alerts were held back in it past MaxPerHour, and
-// forgets the cooldowns that have run out with no repeat held back, which
-// no alert needs any more.
+// endHour ends the hour under way, if now is past it. It drops the
+// cooldowns that have run out, whatever repeats they held back, so that
+// they do not pile up, and queues that hour's summary when alerts were
+// held back in it past MaxPerHour, or when a cooldown dropped had held
+// back repeats, which no alert let through has given: the summary gives
+// them.
 func (q *Queue) endHour(now time.Time) {
 	start := now.Truncate(time.Hour)
 	if !start.After(q.hour.Start) {
 		return
 	}
 
+	repeats := 0
+
+	for key, cooldown := range q.cooldowns {
+		if now.Sub(cooldown.Sent) >= q.params.Cooldown {
+			repeats += cooldown.SuppressedRepeats
+
+			delete(q.cooldowns, key)
+		}
+	}
+
 	heldBack := 0
 	for _, count := range q.hour.HeldBack {
 		heldBack += count
 	}
 
+	var reasons []string
+
 	if heldBack > 0 {
+		reasons = append(reasons, fmt.Sprintf("%d alerts held back in the hour from %s, "+
+			"past the %d an hour SWWAF_ALERT_MAX_PER_HOUR allows", heldBack,
+			q.hour.Start.Format(time.RFC3339), q.params.MaxPerHour))
+	}
+
+	if repeats > 0 {
+		reasons = append(reasons, fmt.Sprintf("%d repeats held back by "+
+			"SWWAF_ALERT_COOLDOWN that no later alert gives", repeats))
+	}
+
+	if len(reasons) > 0 {
 		q.queue(&Alert{
 			Instance: q.params.Instance,
 			Time:     now,
 			Event:    EventSummary,
-			Reason: fmt.Sprintf("%d alerts held back in the hour from %s, past the %d "+
-				"an hour SWWAF_ALERT_MAX_PER_HOUR allows", heldBack,
-				q.hour.Start.Format(time.RFC3339), q.params.MaxPerHour),
+			Reason:   strings.Join(reasons, "; "),
 			Detail: map[string]any{
 				"hour": q.hour.Start, "count": heldBack, "events": q.hour.HeldBack,
 			},
+			SuppressedRepeats: repeats,
 		})
 	}
 
 	q.hour = Hour{Start: start, HeldBack: map[string]int{}}
-
-	for key, cooldown := range q.cooldowns {
-		if now.Sub(cooldown.Sent) >= q.params.Cooldown && cooldown.SuppressedRepeats == 0 {
-			delete(q.cooldowns, key)
-		}
-	}
 }
 
 // queue adds alert to the alerts waiting for each destination.

internal/alerts/alerts_test.go

@@ -381,7 +381,7 @@ func TestAlertsPastTheHourlyLimitAreRolledIntoOneSummary(t *testing.T) {
 	})
 }
 
-func TestRepeatsBeforeAnAlertPastTheHourlyLimitAreGivenByTheNextSent(t *testing.T) {
+func TestRepeatsBeforeAnAlertPastTheHourlyLimitAreGivenByTheSummary(t *testing.T) {
 	t.Parallel()
 
 	synctest.Test(t, func(t *testing.T) {
@@ -401,8 +401,8 @@ func TestRepeatsBeforeAnAlertPastTheHourlyLimitAreGivenByTheNextSent(t *testing.
 		time.Sleep(cooldown)
 		raise()
 
-		// The next hour's first alert gives the two repeats, and the summary
-		// the alert past the limit.
+		// The summary gives the alert past the limit and the two repeats,
+		// and the next hour's first alert none.
 		time.Sleep(time.Hour - cooldown)
 		synctest.Wait()
 		raise()
@@ -413,11 +413,14 @@ func TestRepeatsBeforeAnAlertPastTheHourlyLimitAreGivenByTheNextSent(t *testing.
 		got := webhook.received()
 		if len(got) == 3 {
 			detail, _ := got[1].alert["detail"].(map[string]any)
-			repeats := got[2].alert["suppressed_repeats"]
+			summaryRepeats := got[1].alert["suppressed_repeats"]
+			lastRepeats := got[2].alert["suppressed_repeats"]
 
-			if detail["count"] != float64(1) || repeats != float64(2) {
-				t.Errorf("the summary counts %v alerts, and the last alert gives %v "+
-					"repeats, want 1 and 2", detail["count"], repeats)
+			if detail["count"] != float64(1) || summaryRepeats != float64(2) ||
+				lastRepeats != float64(0) {
+				t.Errorf("the summary counts %v alerts and %v repeats, and the last "+
+					"alert gives %v repeats, want 1, 2 and 0", detail["count"],
+					summaryRepeats, lastRepeats)
 			}
 		}
 
@@ -425,6 +428,70 @@ func TestRepeatsBeforeAnAlertPastTheHourlyLimitAreGivenByTheNextSent(t *testing.
 	})
 }
 
+func TestCooldownsThatHaveRunOutAreDroppedAndTheirRepeatsSummedUp(t *testing.T) {
+	t.Parallel()
+
+	for name, maxPerHour := range map[string]int{"limit off": 0, "limit set": 60} {
+		t.Run(name, func(t *testing.T) {
+			t.Parallel()
+
+			synctest.Test(t, func(t *testing.T) {
+				params := newParams()
+				params.MaxPerHour = maxPerHour
+				webhook, q := start(t, params)
+
+				// For four hours, a netblock of its own each minute is over an
+				// anomaly threshold twice: an alert, and a repeat the cooldown
+				// holds back.
+				netblocks := 0
+
+				for range 4 {
+					for range 60 {
+						anomaly := alerts.Alert{
+							Event: alerts.EventAnomaly, Netblock: netblock(netblocks),
+							Detail: map[string]any{"scope": "net"},
+						}
+						q.Raise(anomaly)
+						q.Raise(anomaly)
+
+						netblocks++
+
+						time.Sleep(time.Minute)
+					}
+
+					// As the hour ends, only the cooldowns started less than the
+					// cooldown before are kept, in memory and for alerts.json.
+					synctest.Wait()
+
+					kept := len(q.Snapshot().Cooldowns)
+					if kept > int(cooldown/time.Minute) {
+						t.Errorf("after %d netblocks, %d cooldowns are kept, want at most %d",
+							netblocks, kept, int(cooldown/time.Minute))
+					}
+				}
+
+				// An hour on, every cooldown has been dropped, and the summaries
+				// have given every repeat.
+				time.Sleep(time.Hour)
+				synctest.Wait()
+
+				repeats := 0.0
+
+				for _, request := range webhook.received() {
+					count, _ := request.alert["suppressed_repeats"].(float64)
+					repeats += count
+				}
+
+				kept := len(q.Snapshot().Cooldowns)
+				if kept != 0 || repeats != float64(netblocks) {
+					t.Errorf("%d cooldowns are kept and the webhook was given %v repeats, "+
+						"want 0 and %d", kept, repeats, netblocks)
+				}
+			})
+		})
+	}
+}
+
 func TestFailedRequestIsSentAgainWithBackoff(t *testing.T) {
 	t.Parallel()
 
@@ -635,7 +702,8 @@ func TestStateLoadedIntoANewQueueCarriesOn(t *testing.T) {
 		after.Load(roundTrip(t, before.Snapshot()))
 
 		// The new queue sends the alert waiting, holds back the repeat as
-		// the cooldown still runs, and sends the summary of the hour.
+		// the cooldown still runs, and sends the summary of the hour, which
+		// gives both repeats, as the cooldown has run out.
 		after.Raise(alerts.Alert{Event: alerts.EventBan, Netblock: netblock(1)})
 		synctest.Wait()
 		wantEvents(t, webhook, alerts.EventBan)
@@ -644,18 +712,12 @@ func TestStateLoadedIntoANewQueueCarriesOn(t *testing.T) {
 		synctest.Wait()
 		wantEvents(t, webhook, alerts.EventBan, alerts.EventSummary)
 
-		detail, _ := webhook.received()[1].alert["detail"].(map[string]any)
-		if detail["count"] != float64(1) {
-			t.Errorf("the summary counts %v alerts, want 1", detail["count"])
-		}
+		summary := webhook.received()[1].alert
+		detail, _ := summary["detail"].(map[string]any)
 
-		// The cooldown has run out, and the next one gives both repeats.
-		after.Raise(alerts.Alert{Event: alerts.EventBan, Netblock: netblock(1)})
-		synctest.Wait()
-
-		got := webhook.received()
-		if repeats := got[len(got)-1].alert["suppressed_repeats"]; repeats != float64(2) {
-			t.Errorf("the last alert gives %v repeats, want 2", repeats)
+		if detail["count"] != float64(1) || summary["suppressed_repeats"] != float64(2) {
+			t.Errorf("the summary counts %v alerts and %v repeats, want 1 and 2",
+				detail["count"], summary["suppressed_repeats"])
 		}
 	})
 }
@@ -701,8 +763,8 @@ func TestSlackAndNtfyAreSentTheSummaryAndTheRepeatsHeldBack(t *testing.T) {
 		ban := alerts.Alert{Event: alerts.EventBan, Netblock: netblock(1), Reason: "a ban"}
 
 		// The hour's one alert, a repeat of it the cooldown holds back, and
-		// an alert past the limit; once the hour has ended, its summary, and
-		// the next alert, which gives the repeat.
+		// an alert past the limit; once the hour has ended, its summary,
+		// which gives the repeat, and the next alert, which gives none.
 		q.Raise(ban)
 		q.Raise(ban)
 		q.Raise(alerts.Alert{Event: alerts.EventFileError, Reason: "a file error"})
@@ -720,14 +782,14 @@ func TestSlackAndNtfyAreSentTheSummaryAndTheRepeatsHeldBack(t *testing.T) {
 		}
 
 		const summary = "1 alerts held back in the hour from 2000-01-01T00:00:00Z, " +
-			"past the 1 an hour SWWAF_ALERT_MAX_PER_HOUR allows"
+			"past the 1 an hour SWWAF_ALERT_MAX_PER_HOUR allows; 1 repeats held back " +
+			"by SWWAF_ALERT_COOLDOWN that no later alert gives\nsuppressed repeats: 1"
 
 		wantSlackMessage(t, slack[1], "*"+instance+": summary*\n"+summary)
 		wantNtfyMessage(t, ntfy[1], instance+": summary", "default bar_chart", summary)
-		wantSlackMessage(t, slack[2],
-			"*"+instance+": ban*\na ban\nnetblock: 203.0.113.1/32\nsuppressed repeats: 1")
+		wantSlackMessage(t, slack[2], "*"+instance+": ban*\na ban\nnetblock: 203.0.113.1/32")
 		wantNtfyMessage(t, ntfy[2], instance+": ban", "default no_entry",
-			"a ban\nnetblock: 203.0.113.1/32\nsuppressed repeats: 1")
+			"a ban\nnetblock: 203.0.113.1/32")
 	})
 }