Alerts to a JSON webhook, with a cooldown and an hourly summary (closes #26)

SWWAF_ALERT_WEBHOOK_URL gets one JSON POST per alert, in SPEC.md's
schema, with SWWAF_ALERT_WEBHOOK_HEADERS: ban and permanent_ban, with
the ban's notes, in observe mode too, marked mode observe;
source_failure for GeoJS; file_error for a rule or state file with an
error. SWWAF_ALERT_EVENTS chooses; SWWAF_ALERT_COOLDOWN holds back
repeats by netblock, file or source; past SWWAF_ALERT_MAX_PER_HOUR the
hour ends in one summary. A bounded queue, retried with backoff, holds
up no request; a 4xx other than 408 and 429 gives the alert up.
alerts.json keeps the queue, the cooldowns and the hour. Nothing shows
the URL's path or query.

Judgement call: the summary's event is summary, which SPEC.md omits.
Judgement call: an admin's ban raises no alert.

Model: opus-5-5

internal/state/state_test.go

@@ -10,8 +10,10 @@ import (
 	"net/http"
 	"net/http/httptest"
 	"net/netip"
+	"net/url"
 	"os"
 	"path/filepath"
+	"reflect"
 	"slices"
 	"strconv"
 	"strings"
@@ -19,6 +21,7 @@ import (
 	"testing/synctest"
 	"time"
 
+	"sneak.berlin/go/smallwebwaf/internal/alerts"
 	"sneak.berlin/go/smallwebwaf/internal/bans"
 	"sneak.berlin/go/smallwebwaf/internal/lookup"
 	"sneak.berlin/go/smallwebwaf/internal/metrics"
@@ -31,6 +34,7 @@ const (
 	bansJSON    = "bans.json"
 	clientsJSON = "clients.json"
 	lookupsJSON = "lookups.json"
+	alertsJSON  = "alerts.json"
 	// What the process log says once Watch watches the directory, and as
 	// it takes in an edit.
 	watching = "watching the state files for edits"
@@ -85,6 +89,67 @@ const liftedBansJSON = `{"version": 1, "bans": [{"netblock": "203.0.113.9/32", `
 	`"start": "2026-10-06T00:00:00Z", "expires": "2026-10-06T01:00:00Z", ` +
 	`"cause": "limit", "lifted": "2026-10-06T00:10:00Z"}]}`
 
+// filledAlertsJSON is alerts.json holding the alerts of fill.
+const filledAlertsJSON = `{
+    "version": 1,
+    "cooldowns": [
+        {
+            "event": "file_error",
+            "netblock": "",
+            "file": "/var/lib/smallwebwaf/bans.json",
+            "sent": "2026-10-06T00:00:00Z",
+            "suppressed_repeats": 0
+        },
+        {
+            "event": "ban",
+            "netblock": "203.0.113.9/32",
+            "sent": "2026-10-06T00:00:00Z",
+            "suppressed_repeats": 1
+        }
+    ],
+    "hour": {
+        "start": "2026-10-06T00:00:00Z",
+        "sent": 2,
+        "held_back": {
+            "source_failure": 1
+        }
+    },
+    "waiting": [
+        {
+            "instance": "fsn1app1/gitea",
+            "time": "2026-10-06T00:00:00Z",
+            "event": "ban",
+            "client": "203.0.113.9",
+            "netblock": "203.0.113.9/32",
+            "asn": "",
+            "as_name": "",
+            "country": "DE",
+            "reason": "requests per minute over the limit of 1",
+            "detail": {
+                "cause": "limit"
+            },
+            "suppressed_repeats": 0
+        },
+        {
+            "instance": "fsn1app1/gitea",
+            "time": "2026-10-06T00:00:00Z",
+            "event": "file_error",
+            "client": "",
+            "netblock": "",
+            "asn": "",
+            "as_name": "",
+            "country": "",
+            "reason": "writing the state files failed",
+            "detail": {
+                "error": "no space left on device",
+                "file": "/var/lib/smallwebwaf/bans.json"
+            },
+            "suppressed_repeats": 0
+        }
+    ]
+}
+`
+
 func TestFilesWrittenAndReadBack(t *testing.T) {
 	t.Parallel()
 
@@ -111,13 +176,38 @@ func TestFilesWrittenAndReadBack(t *testing.T) {
 	wantEqual(t, clientsJSON, after.Limiter.Snapshot(), before.Limiter.Snapshot())
 	wantEqual(t, lookupsJSON, after.GeoJS.Snapshot(), before.GeoJS.Snapshot())
 
+	if got, want := after.Alerts.Snapshot(), before.Alerts.Snapshot(); !reflect.DeepEqual(
+		got, want) {
+		t.Errorf("%s read back\n%+v\nwant\n%+v", alertsJSON, got, want)
+	}
+
 	// Each one-per-line file lists its entries by client, and nothing
-	// but the three files is left in the directory.
+	// but the four files is left in the directory.
 	wantEntries(t, filepath.Join(dir, clientsJSON), "clients",
 		"192.0.2.1/32", "203.0.113.9/32", "2001:db8::/64")
 	wantEntries(t, filepath.Join(dir, lookupsJSON), "lookups",
 		"192.0.2.1/32", "203.0.113.9/32")
-	wantFiles(t, dir, bansJSON, clientsJSON, lookupsJSON)
+	wantFiles(t, dir, alertsJSON, bansJSON, clientsJSON, lookupsJSON)
+}
+
+func TestAlertsJSONIsIndentedWithTheCooldownsTheHourAndTheAlertsWaiting(t *testing.T) {
+	t.Parallel()
+
+	dir := t.TempDir()
+	params := newParams(dir)
+	fill(params)
+
+	files := load(t, params)
+
+	err := files.WriteAll()
+	if err != nil {
+		t.Fatalf("write: %v", err)
+	}
+
+	got := readFile(t, filepath.Join(dir, alertsJSON))
+	if got != filledAlertsJSON {
+		t.Errorf("alerts.json\n%s\nwant\n%s", got, filledAlertsJSON)
+	}
 }
 
 func TestBansJSONIsIndentedWithNullForAPermanentBan(t *testing.T) {
@@ -146,8 +236,10 @@ func TestMissingFilesAreEmptyState(t *testing.T) {
 	params := newParams(t.TempDir())
 	load(t, params)
 
+	held := params.Alerts.Snapshot()
 	if len(params.Ledger.Snapshot()) != 0 || len(params.Limiter.Snapshot()) != 0 ||
-		len(params.GeoJS.Snapshot()) != 0 {
+		len(params.GeoJS.Snapshot()) != 0 || len(held.Cooldowns) != 0 ||
+		len(held.Waiting) != 0 || held.Hour.Sent != 0 {
 		t.Error("state from no files")
 	}
 }
@@ -189,6 +281,11 @@ func TestFileThatDoesNotParseStopsTheStart(t *testing.T) {
 			`{"version": 1, "bans": [{"netblock": "203.0.113.300/32"}]}`,
 			`: netip.ParsePrefix("203.0.113.300/32")`,
 		},
+		{
+			"an unknown field of an alert waiting", alertsJSON,
+			`{"version": 1, "waiting": [{"event": "ban", "evnet": "ban"}]}`,
+			`: json: unknown field "evnet"`,
+		},
 	} {
 		t.Run(tc.name, func(t *testing.T) {
 			t.Parallel()
@@ -278,6 +375,43 @@ func TestEntryWithoutAFieldItNeedsStopsTheStart(t *testing.T) {
 	}
 }
 
+func TestAlertsJSONEntryWithoutAFieldItNeedsStopsTheStart(t *testing.T) {
+	t.Parallel()
+
+	for _, tc := range []struct {
+		name, content string
+		// want is what the error says after the file's path.
+		want string
+	}{
+		{
+			"a cooldown without its event",
+			`{"version": 1, "cooldowns": [{"sent": "2026-10-06T00:00:00Z"}]}`,
+			`: cooldowns entry 1 has no "event"`,
+		},
+		{
+			"a cooldown without when it was sent",
+			`{"version": 1, "cooldowns": [{"event": "ban"}]}`,
+			`: cooldowns entry 1 has no "sent"`,
+		},
+		{
+			"an alert waiting without its event",
+			`{"version": 1, "waiting": [{"time": "2026-10-06T00:00:00Z"}]}`,
+			`: waiting entry 1 has no "event"`,
+		},
+		{
+			"an alert waiting without its time",
+			`{"version": 1, "waiting": [{"event": "ban"}]}`,
+			`: waiting entry 1 has no "time"`,
+		},
+	} {
+		t.Run(tc.name, func(t *testing.T) {
+			t.Parallel()
+
+			wantRefused(t, alertsJSON, tc.content, tc.want)
+		})
+	}
+}
+
 func TestBanWithAnotherCauseStopsTheStart(t *testing.T) {
 	t.Parallel()
 
@@ -294,7 +428,7 @@ func TestBanWithAnotherCauseStopsTheStart(t *testing.T) {
 func TestUnknownVersionStopsTheStart(t *testing.T) {
 	t.Parallel()
 
-	for _, file := range []string{bansJSON, clientsJSON, lookupsJSON} {
+	for _, file := range []string{bansJSON, clientsJSON, lookupsJSON, alertsJSON} {
 		for _, content := range []string{`{"version": 2}`, `{}`} {
 			t.Run(file+" "+content, func(t *testing.T) {
 				t.Parallel()
@@ -344,12 +478,12 @@ func TestBansWrittenOnceWriteDelayAfterABan(t *testing.T) {
 
 		// A second ban, made while the first waits to be written, puts the
 		// write off no further, and is written with it.
-		first := params.Ledger.BanForLimit(netip.MustParsePrefix("203.0.113.9/32"),
+		first, _ := params.Ledger.BanForLimit(netip.MustParsePrefix("203.0.113.9/32"),
 			midnight(), bans.Notes{})
 
 		time.Sleep(5 * time.Second)
 
-		second := params.Ledger.BanForLimit(netip.MustParsePrefix("203.0.113.10/32"),
+		second, _ := params.Ledger.BanForLimit(netip.MustParsePrefix("203.0.113.10/32"),
 			midnight(), bans.Notes{})
 
 		time.Sleep(5*time.Second - time.Nanosecond)
@@ -396,8 +530,8 @@ func TestEveryFileWrittenEveryCounterInterval(t *testing.T) {
 
 			time.Sleep(time.Nanosecond)
 			synctest.Wait()
-			wantFiles(t, dir, bansJSON, clientsJSON, lookupsJSON)
-			removeFiles(t, dir, bansJSON, clientsJSON, lookupsJSON)
+			wantFiles(t, dir, alertsJSON, bansJSON, clientsJSON, lookupsJSON)
+			removeFiles(t, dir, alertsJSON, bansJSON, clientsJSON, lookupsJSON)
 		}
 	})
 }
@@ -440,6 +574,55 @@ func TestEditJustBeforeAScheduledWriteSurvivesIt(t *testing.T) {
 	})
 }
 
+func TestWriteThatFailsWhileRunningRaisesAFileErrorAlertOncePerCooldown(t *testing.T) {
+	t.Parallel()
+
+	synctest.Test(t, func(t *testing.T) {
+		dir := t.TempDir()
+		params := newParams(dir)
+		params.CounterInterval = time.Minute
+		run(t, load(t, params).Run)
+
+		// A directory in the way of bans.json's temporary file fails each of
+		// its writes. It holds a file, so that the write cannot remove it.
+		err := os.Mkdir(filepath.Join(dir, bansJSON+".tmp"), 0o700)
+		if err == nil {
+			err = os.WriteFile(filepath.Join(dir, bansJSON+".tmp", "kept"), nil, 0o600)
+		}
+
+		if err != nil {
+			t.Fatalf("put a directory in the way: %v", err)
+		}
+
+		time.Sleep(time.Minute)
+		synctest.Wait()
+
+		waiting := params.Alerts.Snapshot().Waiting
+		if len(waiting) != 1 {
+			t.Fatalf("%d alerts wait, want 1", len(waiting))
+		}
+
+		message, _ := waiting[0].Detail["error"].(string)
+
+		if waiting[0].Event != alerts.EventFileError ||
+			waiting[0].Reason != "writing the state files failed" ||
+			waiting[0].Detail["file"] != filepath.Join(dir, bansJSON) ||
+			!strings.Contains(message, bansJSON+".tmp") {
+			t.Fatalf("alerts waiting %+v, want a file_error alert for bans.json, naming "+
+				"its temporary file", waiting)
+		}
+
+		// The next write fails too, within the cooldown, which holds it back.
+		time.Sleep(time.Minute)
+		synctest.Wait()
+
+		if len(params.Alerts.Snapshot().Waiting) != 1 || params.Alerts.Suppressed() != 1 {
+			t.Errorf("%d alerts wait and %d are held back, want 1 and 1",
+				len(params.Alerts.Snapshot().Waiting), params.Alerts.Suppressed())
+		}
+	})
+}
+
 func TestFailedWriteLeavesTheFileAsItWas(t *testing.T) {
 	t.Parallel()
 
@@ -565,7 +748,7 @@ func TestFileThatCannotBeReadIsNotWrittenOver(t *testing.T) {
 		t.Errorf("bans.json is now %v (%v), want the socket", info, err)
 	}
 
-	wantFiles(t, dir, bansJSON, clientsJSON, lookupsJSON)
+	wantFiles(t, dir, alertsJSON, bansJSON, clientsJSON, lookupsJSON)
 	wantWriteFailed(t, params, bansJSON)
 }
 
@@ -637,6 +820,25 @@ func TestEditOfEachFileTakenIn(t *testing.T) {
 	wantTakenIn(t, lines, dir, lookupsJSON)
 	wantEqual(t, lookupsJSON, params.GeoJS.Snapshot(),
 		[]lookup.Answer{{Client: client, Country: "FR", Answered: midnight()}})
+
+	// A netblock with bits past its length is read as the netblock it is
+	// in.
+	edit(t, dir, alertsJSON, `{"version": 1, "cooldowns": [{"event": "ban", `+
+		`"netblock": "198.51.100.9/24", "sent": "2026-10-06T00:00:00Z"}], `+
+		`"waiting": [{"event": "file_error", "time": "2026-10-06T00:00:00Z"}]}`)
+	wantTakenIn(t, lines, dir, alertsJSON)
+
+	want := alerts.State{
+		Cooldowns: []alerts.Cooldown{{
+			Event: alerts.EventBan, Netblock: netip.MustParsePrefix("198.51.100.0/24"),
+			Sent: midnight(),
+		}},
+		Hour:    alerts.Hour{HeldBack: map[string]int{}},
+		Waiting: []alerts.Alert{{Event: alerts.EventFileError, Time: midnight()}},
+	}
+	if got := params.Alerts.Snapshot(); !reflect.DeepEqual(got, want) {
+		t.Errorf("%s taken in as\n%+v\nwant\n%+v", alertsJSON, got, want)
+	}
 }
 
 func TestOwnWritesAreNotTakenIn(t *testing.T) {
@@ -708,7 +910,7 @@ func TestBanAddedAndLiftedThroughBansJSON(t *testing.T) {
 		`"start": "2026-10-06T00:00:00Z", "expires": null}]}`)
 	wantTakenIn(t, lines, dir, bansJSON)
 
-	_, banned := params.Ledger.Check(client, midnight())
+	_, banned, _ := params.Ledger.Check(client, midnight())
 	if !banned {
 		t.Error("the ban added to bans.json does not refuse")
 	}
@@ -717,7 +919,7 @@ func TestBanAddedAndLiftedThroughBansJSON(t *testing.T) {
 	edit(t, dir, bansJSON, `{"version": 1, "bans": []}`)
 	wantTakenIn(t, lines, dir, bansJSON)
 
-	_, banned = params.Ledger.Check(client, midnight())
+	_, banned, _ = params.Ledger.Check(client, midnight())
 	if banned {
 		t.Error("the ban removed from bans.json still refuses")
 	}
@@ -807,7 +1009,7 @@ func TestBanLiftedByAnEditWhileRunning(t *testing.T) {
 	netblock := netip.MustParsePrefix(liftedClient + "/32")
 	params.Ledger.BanForLimit(netblock, midnight(), bans.Notes{})
 
-	_, banned := params.Ledger.Find(netblock.Addr(), afterLifting())
+	_, banned, _ := params.Ledger.Find(netblock.Addr(), afterLifting())
 	if !banned {
 		t.Fatal("the ban does not refuse before it is lifted")
 	}
@@ -844,7 +1046,7 @@ func TestBrokenEditSetAsideAtTheNextWrite(t *testing.T) {
 	edit(t, dir, bansJSON, broken)
 	edit(t, dir, clientsJSON, `{"version": 1, "clients": []}`)
 	wantTakenIn(t, lines, dir, clientsJSON)
-	wantFiles(t, dir, bansJSON, clientsJSON, lookupsJSON)
+	wantFiles(t, dir, alertsJSON, bansJSON, clientsJSON, lookupsJSON)
 
 	// The next write sets it aside, logged with where the error is, and
 	// writes bans.json again from what smallwebwaf still holds.
@@ -861,7 +1063,14 @@ func TestBrokenEditSetAsideAtTheNextWrite(t *testing.T) {
 		t.Errorf("set aside with %v", line)
 	}
 
-	wantFiles(t, dir, bansJSON, bansJSON+".bad", clientsJSON, lookupsJSON)
+	// It is raised as a file_error alert, with the same file and error.
+	waiting := params.Alerts.Snapshot().Waiting
+	if len(waiting) != 1 || waiting[0].Event != alerts.EventFileError ||
+		waiting[0].Detail["file"] != path+".bad" || waiting[0].Detail["error"] != message {
+		t.Errorf("alerts waiting %+v, want a file_error alert for %s", waiting, path+".bad")
+	}
+
+	wantFiles(t, dir, alertsJSON, bansJSON, bansJSON+".bad", clientsJSON, lookupsJSON)
 
 	if got := readFile(t, path+".bad"); got != broken {
 		t.Errorf("bans.json.bad holds\n%s\nwant the edit", got)
@@ -990,7 +1199,8 @@ func midnight() time.Time {
 }
 
 // newParams returns Params for the state files in dir, with parts that
-// hold nothing yet. GeoJS is never asked.
+// hold nothing yet. GeoJS is never asked, and the alerts, at most two an
+// hour, are never sent.
 func newParams(dir string) state.Params {
 	discard := slog.New(slog.DiscardHandler)
 	m := metrics.New(1)
@@ -1010,6 +1220,14 @@ func newParams(dir string) state.Params {
 		GeoJS: lookup.New(lookup.Params{
 			Now: midnight, ProcessLog: discard, Metrics: m,
 		}),
+		Alerts: alerts.New(alerts.Params{
+			WebhookURL: &url.URL{Scheme: "https", Host: "alerts.example"},
+			Events:     alerts.Events(),
+			Cooldown:   15 * time.Minute,
+			MaxPerHour: 2,
+			Instance:   "fsn1app1/gitea",
+			Now:        midnight,
+		}),
 		Now:        midnight,
 		ProcessLog: discard,
 		Metrics:    m,
@@ -1017,8 +1235,9 @@ func newParams(dir string) state.Params {
 }
 
 // fill puts a permanent ban an admin made, a ban for a broken limit and
-// one for a clear sign of attack, clients with counts and histories, and
-// GeoJS answers into the parts of params.
+// one for a clear sign of attack, clients with counts and histories,
+// GeoJS answers, and alerts, as filledAlertsJSON holds them, into the
+// parts of params.
 func fill(params state.Params) {
 	now := midnight()
 	client := netip.MustParsePrefix("203.0.113.9/32")
@@ -1043,6 +1262,25 @@ func fill(params state.Params) {
 			Answered: now.Add(-time.Hour), Used: now.Add(-time.Minute),
 		},
 	})
+
+	// An alert waiting, a repeat of it the cooldown holds back, another
+	// alert waiting, and one past the two an hour, for the hour's summary.
+	ban := alerts.Alert{
+		Event: alerts.EventBan, Client: client.Addr(), Netblock: client, Country: "DE",
+		Reason: "requests per minute over the limit of 1",
+		Detail: map[string]any{"cause": "limit"},
+	}
+	params.Alerts.Raise(ban)
+	params.Alerts.Raise(ban)
+	params.Alerts.Raise(alerts.Alert{
+		Event: alerts.EventFileError, Reason: "writing the state files failed",
+		Detail: map[string]any{
+			"file": "/var/lib/smallwebwaf/bans.json", "error": "no space left on device",
+		},
+	})
+	params.Alerts.Raise(alerts.Alert{
+		Event: alerts.EventSourceFailure, Reason: "asking GeoJS failed",
+	})
 }
 
 // permanentBan is the ban permanentBansJSON holds.
@@ -1098,13 +1336,13 @@ func wantLiftedBanKept(
 
 	netblock := netip.MustParsePrefix(liftedClient + "/32")
 
-	_, banned := ledger.Check(netblock.Addr(), afterLifting())
+	_, banned, _ := ledger.Check(netblock.Addr(), afterLifting())
 	if banned {
 		t.Error("the lifted ban refuses")
 	}
 
 	// Were the lifted ban counted, the next would last three hours.
-	ban := ledger.BanForLimit(netblock, afterLifting(), bans.Notes{})
+	ban, _ := ledger.BanForLimit(netblock, afterLifting(), bans.Notes{})
 	if ban.Expires.Sub(ban.Start) != time.Hour {
 		t.Errorf("the next ban lasts %s, want 1h", ban.Expires.Sub(ban.Start))
 	}