Anomaly thresholds: alerts for unusual traffic, nothing refused (closes #101)

SWWAF_ANOMALY_CLIENT_*, _NET_*, _ASN_*, _TOTAL_* and SWWAF_WATCH_* with
SWWAF_WATCH_NETS: requests and bytes per minute and per hour, each off by
default; with all off, nothing is counted. Otherwise every request but the
health check is counted, allow-listed and exempt ones included; a count
over its threshold raises an anomaly alert, with a cooldown per scope. At
most 20,000 counters, kept in alerts.json. A per-AS-number threshold with
lookups off, or a malformed SWWAF_WATCH_NETS, stops the start. A cooldown
that has run out is dropped as the hour ends, whatever it held back; the
hour's summary gives its repeats.

Judgement call: refused requests are counted too.
Judgement call: per-client counters are kept in alerts.json, which SPEC.md does not list.
Judgement call: a request counts for an AS number only if the lookup answered before it ended.

Model: opus-5-5

internal/alerts/alerts.go

@@ -1,5 +1,6 @@
-// Package alerts sends alerts on bans, on a source that fails and on a
-// file with an error to each destination set: to the webhook
+// Package alerts sends alerts on bans, on traffic over an anomaly
+// threshold, on a source that fails and on a file with an error to each
+// destination set: to the webhook
 // SWWAF_ALERT_WEBHOOK_URL names, each as one JSON object, as the "Alert
 // webhook schema" section of SPEC.md describes, to the Slack incoming
 // webhook SWWAF_ALERT_SLACK_WEBHOOK_URL names, as a message, and to the
@@ -40,11 +41,12 @@ const (
 	// EventPermanentBan is a permanent ban smallwebwaf made, or a ban it
 	// made permanent.
 	EventPermanentBan = "permanent_ban"
-	// EventWAFBlock, EventAnomaly and EventReputationHit come with the
-	// Core Rule Set, the anomaly thresholds and the reputation sources;
-	// nothing raises them yet.
+	// EventAnomaly is a count of requests or bytes over an anomaly
+	// threshold.
+	EventAnomaly = "anomaly"
+	// EventWAFBlock and EventReputationHit come with the Core Rule Set and
+	// the reputation sources; nothing raises them yet.
 	EventWAFBlock      = "waf_block"
-	EventAnomaly       = "anomaly"
 	EventReputationHit = "reputation_hit"
 	// EventSourceFailure is GeoJS failing or refusing smallwebwaf.
 	EventSourceFailure = "source_failure"
@@ -52,8 +54,11 @@ const (
 	// runs that does not parse, a replacement of the lookup database that
 	// cannot be read, or a state file that cannot be written.
 	EventFileError = "file_error"
-	// EventSummary is the summary of the alerts an hour held back past
-	// SWWAF_ALERT_MAX_PER_HOUR. SWWAF_ALERT_EVENTS does not name it.
+	// EventSummary is the summary sent as an hour ends: of the alerts held
+	// back in it past SWWAF_ALERT_MAX_PER_HOUR, and of the repeats held
+	// back by the cooldowns dropped as it ends, which no alert let through
+	// has given. It is sent with SWWAF_ALERT_MAX_PER_HOUR off too, for
+	// those repeats. SWWAF_ALERT_EVENTS does not name it.
 	EventSummary = "summary"
 )
 
@@ -153,18 +158,22 @@ type Alert struct {
 	ASName   string       `json:"as_name"`
 	Country  string       `json:"country"`
 	// Reason is a short sentence, and Detail what is particular to the
-	// event: for a file_error, its "file", and for a source_failure, its
-	// "source", which the cooldown tells repeats by.
+	// event: for a file_error, its "file", for a source_failure, its
+	// "source", and for an anomaly, its "scope", with the "asn" or the
+	// "name" of some scopes, which the cooldown tells repeats by.
 	Reason string         `json:"reason"`
 	Detail map[string]any `json:"detail"`
 	// SuppressedRepeats is how many repeats of the alert the cooldown
-	// held back since the last one let through.
+	// held back since the last one let through. For a summary, it is how
+	// many the cooldowns dropped as the hour ended had held back that no
+	// alert let through gave.
 	SuppressedRepeats int `json:"suppressed_repeats"`
 }
 
-// Cooldown is, for an event on a netblock, or about a file or a source,
-// when the last alert let through was raised, and how many repeats the
-// cooldown has held back since, as alerts.json holds it.
+// Cooldown is, for an event on a netblock, about a file or a source, or
+// for an anomaly in a scope, when the last alert let through was raised,
+// and how many repeats the cooldown has held back since, as alerts.json
+// holds it.
 //
 //nolint:tagliatelle // the state files use snake_case, as the request log does
 type Cooldown struct {
@@ -172,6 +181,9 @@ type Cooldown struct {
 	Netblock          netip.Prefix `json:"netblock"`
 	File              string       `json:"file,omitempty"`
 	Source            string       `json:"source,omitempty"`
+	Scope             string       `json:"scope,omitempty"`
+	ASN               string       `json:"asn,omitempty"`
+	Name              string       `json:"name,omitempty"`
 	Sent              time.Time    `json:"sent"`
 	SuppressedRepeats int          `json:"suppressed_repeats"`
 }
@@ -214,7 +226,7 @@ type Queue struct {
 
 	mu sync.Mutex
 	// cooldowns are the alerts last let through, by event and netblock,
-	// file or source.
+	// file, source or scope.
 	cooldowns map[cooldownKey]*Cooldown
 	hour      Hour
 
@@ -248,21 +260,28 @@ type destination struct {
 }
 
 // cooldownKey is what makes an alert a repeat of another: the same event
-// on the same netblock, and about the same file or source, as its detail
-// names them. Each is empty for an alert without one.
+// on the same netblock, and about the same file or source, or in the same
+// scope with the same AS number or name, as its detail names them. Each
+// is empty for an alert without one.
 type cooldownKey struct {
 	event    string
 	netblock netip.Prefix
 	file     string
 	source   string
+	scope    string
+	asn      string
+	name     string
 }
 
 // cooldownKeyOf returns what makes another alert a repeat of alert.
 func cooldownKeyOf(alert *Alert) cooldownKey {
 	file, _ := alert.Detail["file"].(string)
 	source, _ := alert.Detail["source"].(string)
+	scope, _ := alert.Detail["scope"].(string)
+	asn, _ := alert.Detail["asn"].(string)
+	name, _ := alert.Detail["name"].(string)
 
-	return cooldownKey{alert.Event, alert.Netblock, file, source}
+	return cooldownKey{alert.Event, alert.Netblock, file, source, scope, asn, name}
 }
 
 // New returns a Queue with no alert yet.
@@ -295,14 +314,14 @@ func New(params Params) *Queue {
 // unless no destination is set or SWWAF_ALERT_EVENTS leaves its event
 // out. It gives alert the instance and the time. An alert that repeats
 // the last one let through less than Cooldown before is held back and
-// counted, and the next one let through gives that count. Past MaxPerHour
-// alerts let through in the hour under way, by the clock, an alert is
-// held back for that hour's summary instead, which is sent once the hour
-// has ended; it starts no cooldown, and the repeats held back before it
-// are given by the next alert let through. Raise never waits: an alert
-// let through joins the queue of each destination, from which Run sends
-// it, and with queueSize alerts waiting for a destination, the oldest is
-// dropped.
+// counted. The next one let through gives that count, unless an hour of
+// the clock ends first after the cooldown has run out: the cooldown is
+// then dropped, and that hour's summary gives the count. Past MaxPerHour
+// alerts let through in the hour under way, an alert is held back for
+// that hour's summary instead, which is sent once the hour has ended; it
+// starts no cooldown. Raise never waits: an alert let through joins the
+// queue of each destination, from which Run sends it, and with queueSize
+// alerts waiting for a destination, the oldest is dropped.
 func (q *Queue) Raise(alert Alert) {
 	if len(q.destinations) == 0 || !slices.Contains(q.params.Events, alert.Event) {
 		return
@@ -425,7 +444,8 @@ func (q *Queue) Suppressed() int64 {
 }
 
 // Snapshot returns the queue's state, as alerts.json holds it, with the
-// cooldowns sorted by netblock, then by event, file and source.
+// cooldowns sorted by netblock, then by event, file, source, scope, AS
+// number and name.
 func (q *Queue) Snapshot() State {
 	q.mu.Lock()
 	defer q.mu.Unlock()
@@ -443,7 +463,9 @@ func (q *Queue) Snapshot() State {
 
 	slices.SortFunc(state.Cooldowns, func(a, b Cooldown) int {
 		return cmp.Or(a.Netblock.Compare(b.Netblock), cmp.Compare(a.Event, b.Event),
-			cmp.Compare(a.File, b.File), cmp.Compare(a.Source, b.Source))
+			cmp.Compare(a.File, b.File), cmp.Compare(a.Source, b.Source),
+			cmp.Compare(a.Scope, b.Scope), cmp.Compare(a.ASN, b.ASN),
+			cmp.Compare(a.Name, b.Name))
 	})
 
 	for _, d := range q.destinations {
@@ -466,7 +488,10 @@ func (q *Queue) Load(state State) {
 
 	for _, cooldown := range state.Cooldowns {
 		cooldown.Netblock = cooldown.Netblock.Masked()
-		key := cooldownKey{cooldown.Event, cooldown.Netblock, cooldown.File, cooldown.Source}
+		key := cooldownKey{
+			cooldown.Event, cooldown.Netblock, cooldown.File, cooldown.Source,
+			cooldown.Scope, cooldown.ASN, cooldown.Name,
+		}
 		q.cooldowns[key] = &cooldown
 	}
 
@@ -538,46 +563,64 @@ func (q *Queue) startCooldown(alert *Alert, now time.Time) {
 
 	q.cooldowns[key] = &Cooldown{
 		Event: alert.Event, Netblock: alert.Netblock, File: key.file, Source: key.source,
-		Sent: now,
+		Scope: key.scope, ASN: key.asn, Name: key.name, Sent: now,
 	}
 }
 
-// endHour ends the hour under way, if now is past it: it queues that
-// hour's summary when alerts were held back in it past MaxPerHour, and
-// forgets the cooldowns that have run out with no repeat held back, which
-// no alert needs any more.
+// endHour ends the hour under way, if now is past it. It drops the
+// cooldowns that have run out, whatever repeats they held back, so that
+// they do not pile up, and queues that hour's summary when alerts were
+// held back in it past MaxPerHour, or when a cooldown dropped had held
+// back repeats, which no alert let through has given: the summary gives
+// them.
 func (q *Queue) endHour(now time.Time) {
 	start := now.Truncate(time.Hour)
 	if !start.After(q.hour.Start) {
 		return
 	}
 
+	repeats := 0
+
+	for key, cooldown := range q.cooldowns {
+		if now.Sub(cooldown.Sent) >= q.params.Cooldown {
+			repeats += cooldown.SuppressedRepeats
+
+			delete(q.cooldowns, key)
+		}
+	}
+
 	heldBack := 0
 	for _, count := range q.hour.HeldBack {
 		heldBack += count
 	}
 
+	var reasons []string
+
 	if heldBack > 0 {
+		reasons = append(reasons, fmt.Sprintf("%d alerts held back in the hour from %s, "+
+			"past the %d an hour SWWAF_ALERT_MAX_PER_HOUR allows", heldBack,
+			q.hour.Start.Format(time.RFC3339), q.params.MaxPerHour))
+	}
+
+	if repeats > 0 {
+		reasons = append(reasons, fmt.Sprintf("%d repeats held back by "+
+			"SWWAF_ALERT_COOLDOWN that no later alert gives", repeats))
+	}
+
+	if len(reasons) > 0 {
 		q.queue(&Alert{
 			Instance: q.params.Instance,
 			Time:     now,
 			Event:    EventSummary,
-			Reason: fmt.Sprintf("%d alerts held back in the hour from %s, past the %d "+
-				"an hour SWWAF_ALERT_MAX_PER_HOUR allows", heldBack,
-				q.hour.Start.Format(time.RFC3339), q.params.MaxPerHour),
+			Reason:   strings.Join(reasons, "; "),
 			Detail: map[string]any{
 				"hour": q.hour.Start, "count": heldBack, "events": q.hour.HeldBack,
 			},
+			SuppressedRepeats: repeats,
 		})
 	}
 
 	q.hour = Hour{Start: start, HeldBack: map[string]int{}}
-
-	for key, cooldown := range q.cooldowns {
-		if now.Sub(cooldown.Sent) >= q.params.Cooldown && cooldown.SuppressedRepeats == 0 {
-			delete(q.cooldowns, key)
-		}
-	}
 }
 
 // queue adds alert to the alerts waiting for each destination.