Alerts to Slack and ntfy, each destination with its own queue (closes #90)

Each alert is posted as a message to the Slack incoming webhook
SWWAF_ALERT_SLACK_WEBHOOK_URL names, and published to the ntfy topic
SWWAF_ALERT_NTFY_URL names, with SWWAF_ALERT_NTFY_TOKEN as a bearer
token and a priority and tag by event. The cooldown and the hourly
limit stay shared; past them, each destination has its own bounded
queue and backoff, and its own sent, failed and dropped counts.
alerts.json keeps the alerts waiting by destination; one whose waiting
is still a list stops the start, saying what to change. A control
character in the ntfy token, or in the instance name ntfy is sent,
stops the start.

Judgement call: messages also give the detail's file, source, error and mode.
Judgement call: alerts_suppressed_total is the same for every destination.

Model: opus-5-5

internal/state/state.go

@@ -2,11 +2,12 @@
 // SWWAF_STATE_DIR, as the "Persistent state" section of SPEC.md describes:
 // bans.json holds the bans, clients.json each client's counters and
 // history, lookups.json GeoJS's answers, and alerts.json the cooldowns,
-// the hour under way and the alerts waiting. Load reads them at start,
-// Watch takes in an admin's edit of one while smallwebwaf runs, and Run
-// and WriteAll write them. The disk is read and written outside the
-// parts' locks, which are held only to take a snapshot or to put in what
-// a file holds, so that no request waits on the disk.
+// the hour under way and the alerts waiting for each destination. Load
+// reads them at start, Watch takes in an admin's edit of one while
+// smallwebwaf runs, and Run and WriteAll write them. The disk is read and
+// written outside the parts' locks, which are held only to take a
+// snapshot or to put in what a file holds, so that no request waits on
+// the disk.
 package state
 
 import (
@@ -18,9 +19,11 @@ import (
 	"fmt"
 	"io/fs"
 	"log/slog"
+	"maps"
 	"net/netip"
 	"os"
 	"path/filepath"
+	"slices"
 	"sync"
 	"time"
 
@@ -50,8 +53,12 @@ const (
 var (
 	errVersion = errors.New("unknown version")
 	// errMissing is for an entry without a field it needs.
-	errMissing = errors.New("has no")
-	errCause   = errors.New("is not limit, attack or admin")
+	errMissing     = errors.New("has no")
+	errCause       = errors.New("is not limit, attack or admin")
+	errDestination = errors.New("is not webhook, slack or ntfy")
+	errWaitingList = errors.New(`waiting is a list, but now lists the alerts by ` +
+		`destination: put the list under "webhook", as "waiting": {"webhook": [...]}, ` +
+		`or remove the file`)
 )
 
 // Params are what Load needs.
@@ -129,10 +136,10 @@ type lookupsFile struct {
 
 // alertsFile is alerts.json, indented for an admin to read and edit.
 type alertsFile struct {
-	Version   int               `json:"version"`
-	Cooldowns []alerts.Cooldown `json:"cooldowns"`
-	Hour      alerts.Hour       `json:"hour"`
-	Waiting   []alerts.Alert    `json:"waiting"`
+	Version   int                       `json:"version"`
+	Cooldowns []alerts.Cooldown         `json:"cooldowns"`
+	Hour      alerts.Hour               `json:"hour"`
+	Waiting   map[string][]alerts.Alert `json:"waiting"`
 }
 
 // stateFile is the struct of a state file. Once the file is decoded, its
@@ -392,6 +399,17 @@ func (f *Files) takeIn(name string, data []byte, edit bool) (int, error) {
 		f.params.GeoJS.Load(file.Lookups)
 		entries = len(file.Lookups)
 	case alertsJSON:
+		// waiting was a list, of the alerts waiting for the webhook, before
+		// alerts went to Slack and ntfy too.
+		var written struct {
+			Waiting json.RawMessage `json:"waiting"`
+		}
+
+		if json.Unmarshal(data, &written) == nil &&
+			bytes.HasPrefix(written.Waiting, []byte("[")) {
+			return 0, fmt.Errorf("%s: %w", path, errWaitingList)
+		}
+
 		var file alertsFile
 
 		err := parse(path, data, &file)
@@ -402,7 +420,10 @@ func (f *Files) takeIn(name string, data []byte, edit bool) (int, error) {
 		f.params.Alerts.Load(alerts.State{
 			Cooldowns: file.Cooldowns, Hour: file.Hour, Waiting: file.Waiting,
 		})
-		entries = len(file.Waiting)
+
+		for _, waiting := range file.Waiting {
+			entries += len(waiting)
+		}
 	}
 
 	f.sums[name] = sha256.Sum256(data)
@@ -645,8 +666,9 @@ func (f *lookupsFile) check(data []byte) error {
 }
 
 // check refuses a cooldown without its event or when its alert was sent,
-// which would hold back no repeat, and an alert waiting without its event
-// or its time.
+// which would hold back no repeat, alerts waiting for a destination with
+// another name than webhook, slack or ntfy, most likely misspelt, and an
+// alert waiting without its event or its time.
 func (f *alertsFile) check([]byte) error {
 	for i, cooldown := range f.Cooldowns {
 		switch {
@@ -657,12 +679,18 @@ func (f *alertsFile) check([]byte) error {
 		}
 	}
 
-	for i, alert := range f.Waiting {
-		switch {
-		case alert.Event == "":
-			return fmt.Errorf("waiting %w", missing(i, "event"))
-		case alert.Time.IsZero():
-			return fmt.Errorf("waiting %w", missing(i, "time"))
+	for _, destination := range slices.Sorted(maps.Keys(f.Waiting)) {
+		if !slices.Contains(alerts.Destinations(), destination) {
+			return fmt.Errorf("waiting %q %w", destination, errDestination)
+		}
+
+		for i, alert := range f.Waiting[destination] {
+			switch {
+			case alert.Event == "":
+				return fmt.Errorf("waiting %s %w", destination, missing(i, "event"))
+			case alert.Time.IsZero():
+				return fmt.Errorf("waiting %s %w", destination, missing(i, "time"))
+			}
 		}
 	}
 

internal/state/state_test.go

@@ -114,39 +114,41 @@ const filledAlertsJSON = `{
             "source_failure": 1
         }
     },
-    "waiting": [
-        {
-            "instance": "fsn1app1/gitea",
-            "time": "2026-10-06T00:00:00Z",
-            "event": "ban",
-            "client": "203.0.113.9",
-            "netblock": "203.0.113.9/32",
-            "asn": "",
-            "as_name": "",
-            "country": "DE",
-            "reason": "requests per minute over the limit of 1",
-            "detail": {
-                "cause": "limit"
+    "waiting": {
+        "webhook": [
+            {
+                "instance": "fsn1app1/gitea",
+                "time": "2026-10-06T00:00:00Z",
+                "event": "ban",
+                "client": "203.0.113.9",
+                "netblock": "203.0.113.9/32",
+                "asn": "",
+                "as_name": "",
+                "country": "DE",
+                "reason": "requests per minute over the limit of 1",
+                "detail": {
+                    "cause": "limit"
+                },
+                "suppressed_repeats": 0
             },
-            "suppressed_repeats": 0
-        },
-        {
-            "instance": "fsn1app1/gitea",
-            "time": "2026-10-06T00:00:00Z",
-            "event": "file_error",
-            "client": "",
-            "netblock": "",
-            "asn": "",
-            "as_name": "",
-            "country": "",
-            "reason": "writing the state files failed",
-            "detail": {
-                "error": "no space left on device",
-                "file": "/var/lib/smallwebwaf/bans.json"
-            },
-            "suppressed_repeats": 0
-        }
-    ]
+            {
+                "instance": "fsn1app1/gitea",
+                "time": "2026-10-06T00:00:00Z",
+                "event": "file_error",
+                "client": "",
+                "netblock": "",
+                "asn": "",
+                "as_name": "",
+                "country": "",
+                "reason": "writing the state files failed",
+                "detail": {
+                    "error": "no space left on device",
+                    "file": "/var/lib/smallwebwaf/bans.json"
+                },
+                "suppressed_repeats": 0
+            }
+        ]
+    }
 }
 `
 
@@ -234,7 +236,7 @@ func TestSourceFailureCooldownKeptInAlertsJSONAcrossARestart(t *testing.T) {
 	load(t, after)
 	after.Alerts.Raise(failure)
 
-	waiting := after.Alerts.Snapshot().Waiting
+	waiting := after.Alerts.Snapshot().Waiting[alerts.DestinationWebhook]
 	if len(waiting) != 1 || after.Alerts.Suppressed() != 1 {
 		t.Errorf("%d alerts wait and %d are held back, want the one read back and 1",
 			len(waiting), after.Alerts.Suppressed())
@@ -270,7 +272,7 @@ func TestMissingFilesAreEmptyState(t *testing.T) {
 	held := params.Alerts.Snapshot()
 	if len(params.Ledger.Snapshot()) != 0 || len(params.Limiter.Snapshot()) != 0 ||
 		len(params.GeoJS.Snapshot()) != 0 || len(held.Cooldowns) != 0 ||
-		len(held.Waiting) != 0 || held.Hour.Sent != 0 {
+		len(held.Waiting[alerts.DestinationWebhook]) != 0 || held.Hour.Sent != 0 {
 		t.Error("state from no files")
 	}
 }
@@ -314,9 +316,14 @@ func TestFileThatDoesNotParseStopsTheStart(t *testing.T) {
 		},
 		{
 			"an unknown field of an alert waiting", alertsJSON,
-			`{"version": 1, "waiting": [{"event": "ban", "evnet": "ban"}]}`,
+			`{"version": 1, "waiting": {"webhook": [{"event": "ban", "evnet": "ban"}]}}`,
 			`: json: unknown field "evnet"`,
 		},
+		{
+			"alerts waiting for an unknown destination", alertsJSON,
+			`{"version": 1, "waiting": {"webhook": [], "slak": []}}`,
+			`: waiting "slak" is not webhook, slack or ntfy`,
+		},
 	} {
 		t.Run(tc.name, func(t *testing.T) {
 			t.Parallel()
@@ -426,13 +433,13 @@ func TestAlertsJSONEntryWithoutAFieldItNeedsStopsTheStart(t *testing.T) {
 		},
 		{
 			"an alert waiting without its event",
-			`{"version": 1, "waiting": [{"time": "2026-10-06T00:00:00Z"}]}`,
-			`: waiting entry 1 has no "event"`,
+			`{"version": 1, "waiting": {"ntfy": [{"time": "2026-10-06T00:00:00Z"}]}}`,
+			`: waiting ntfy entry 1 has no "event"`,
 		},
 		{
 			"an alert waiting without its time",
-			`{"version": 1, "waiting": [{"event": "ban"}]}`,
-			`: waiting entry 1 has no "time"`,
+			`{"version": 1, "waiting": {"slack": [{"event": "ban"}]}}`,
+			`: waiting slack entry 1 has no "time"`,
 		},
 	} {
 		t.Run(tc.name, func(t *testing.T) {
@@ -443,6 +450,23 @@ func TestAlertsJSONEntryWithoutAFieldItNeedsStopsTheStart(t *testing.T) {
 	}
 }
 
+func TestAlertsJSONWithWaitingAsAListStopsTheStartSayingWhatToChange(t *testing.T) {
+	t.Parallel()
+
+	// alerts.json as it was written before alerts went to Slack and ntfy too,
+	// with no alert waiting, or one.
+	for _, waiting := range []string{
+		`[]`,
+		`[{"event": "ban", "time": "2026-10-06T00:00:00Z"}]`,
+	} {
+		wantRefused(t, alertsJSON, `{"version": 1, "cooldowns": [], `+
+			`"hour": {"start": "2026-10-06T00:00:00Z", "sent": 0, "held_back": {}}, `+
+			`"waiting": `+waiting+`}`,
+			`: waiting is a list, but now lists the alerts by destination: put the `+
+				`list under "webhook", as "waiting": {"webhook": [...]}, or remove the file`)
+	}
+}
+
 func TestBanWithAnotherCauseStopsTheStart(t *testing.T) {
 	t.Parallel()
 
@@ -629,7 +653,7 @@ func TestWriteThatFailsWhileRunningRaisesAFileErrorAlertOncePerCooldown(t *testi
 		time.Sleep(time.Minute)
 		synctest.Wait()
 
-		waiting := params.Alerts.Snapshot().Waiting
+		waiting := params.Alerts.Snapshot().Waiting[alerts.DestinationWebhook]
 		if len(waiting) != 1 {
 			t.Fatalf("%d alerts wait, want 1", len(waiting))
 		}
@@ -648,9 +672,10 @@ func TestWriteThatFailsWhileRunningRaisesAFileErrorAlertOncePerCooldown(t *testi
 		time.Sleep(time.Minute)
 		synctest.Wait()
 
-		if len(params.Alerts.Snapshot().Waiting) != 1 || params.Alerts.Suppressed() != 1 {
+		waiting = params.Alerts.Snapshot().Waiting[alerts.DestinationWebhook]
+		if len(waiting) != 1 || params.Alerts.Suppressed() != 1 {
 			t.Errorf("%d alerts wait and %d are held back, want 1 and 1",
-				len(params.Alerts.Snapshot().Waiting), params.Alerts.Suppressed())
+				len(waiting), params.Alerts.Suppressed())
 		}
 	})
 }
@@ -857,7 +882,7 @@ func TestEditOfEachFileTakenIn(t *testing.T) {
 	// in.
 	edit(t, dir, alertsJSON, `{"version": 1, "cooldowns": [{"event": "ban", `+
 		`"netblock": "198.51.100.9/24", "sent": "2026-10-06T00:00:00Z"}], `+
-		`"waiting": [{"event": "file_error", "time": "2026-10-06T00:00:00Z"}]}`)
+		`"waiting": {"webhook": [{"event": "file_error", "time": "2026-10-06T00:00:00Z"}]}}`)
 	wantTakenIn(t, lines, dir, alertsJSON)
 
 	want := alerts.State{
@@ -865,8 +890,10 @@ func TestEditOfEachFileTakenIn(t *testing.T) {
 			Event: alerts.EventBan, Netblock: netip.MustParsePrefix("198.51.100.0/24"),
 			Sent: midnight(),
 		}},
-		Hour:    alerts.Hour{HeldBack: map[string]int{}},
-		Waiting: []alerts.Alert{{Event: alerts.EventFileError, Time: midnight()}},
+		Hour: alerts.Hour{HeldBack: map[string]int{}},
+		Waiting: map[string][]alerts.Alert{
+			alerts.DestinationWebhook: {{Event: alerts.EventFileError, Time: midnight()}},
+		},
 	}
 	if got := params.Alerts.Snapshot(); !reflect.DeepEqual(got, want) {
 		t.Errorf("%s taken in as\n%+v\nwant\n%+v", alertsJSON, got, want)
@@ -1096,7 +1123,7 @@ func TestBrokenEditSetAsideAtTheNextWrite(t *testing.T) {
 	}
 
 	// It is raised as a file_error alert, with the same file and error.
-	waiting := params.Alerts.Snapshot().Waiting
+	waiting := params.Alerts.Snapshot().Waiting[alerts.DestinationWebhook]
 	if len(waiting) != 1 || waiting[0].Event != alerts.EventFileError ||
 		waiting[0].Detail["file"] != path+".bad" || waiting[0].Detail["error"] != message {
 		t.Errorf("alerts waiting %+v, want a file_error alert for %s", waiting, path+".bad")