Alerts to Slack and ntfy, each destination with its own queue (closes #90)

Each alert is posted as a message to the Slack incoming webhook SWWAF_ALERT_SLACK_WEBHOOK_URL names, and published to the ntfy topic SWWAF_ALERT_NTFY_URL names, with SWWAF_ALERT_NTFY_TOKEN as a bearer token and a priority and tag by event. The cooldown and the hourly limit stay shared; past them, each destination has its own bounded queue and backoff, and its own sent, failed and dropped counts. alerts.json keeps the alerts waiting by destination. Judgement call: messages also give the detail's file, source, error and mode. Judgement call: alerts_suppressed_total is the same for every destination. Judgement call: an alerts.json with waiting as a list stops the start. Model: opus-5-5
2026-10-07 03:07:29 +00:00
parent 432097ee3f
commit 177bf8a29d
14 changed files with 1467 additions and 428 deletions
@@ -114,39 +114,41 @@ const filledAlertsJSON = `{
            "source_failure": 1
        }
    },
-    "waiting": [
-        {
-            "instance": "fsn1app1/gitea",
-            "time": "2026-10-06T00:00:00Z",
-            "event": "ban",
-            "client": "203.0.113.9",
-            "netblock": "203.0.113.9/32",
-            "asn": "",
-            "as_name": "",
-            "country": "DE",
-            "reason": "requests per minute over the limit of 1",
-            "detail": {
-                "cause": "limit"
+    "waiting": {
+        "webhook": [
+            {
+                "instance": "fsn1app1/gitea",
+                "time": "2026-10-06T00:00:00Z",
+                "event": "ban",
+                "client": "203.0.113.9",
+                "netblock": "203.0.113.9/32",
+                "asn": "",
+                "as_name": "",
+                "country": "DE",
+                "reason": "requests per minute over the limit of 1",
+                "detail": {
+                    "cause": "limit"
+                },
+                "suppressed_repeats": 0
            },
-            "suppressed_repeats": 0
-        },
-        {
-            "instance": "fsn1app1/gitea",
-            "time": "2026-10-06T00:00:00Z",
-            "event": "file_error",
-            "client": "",
-            "netblock": "",
-            "asn": "",
-            "as_name": "",
-            "country": "",
-            "reason": "writing the state files failed",
-            "detail": {
-                "error": "no space left on device",
-                "file": "/var/lib/smallwebwaf/bans.json"
-            },
-            "suppressed_repeats": 0
-        }
-    ]
+            {
+                "instance": "fsn1app1/gitea",
+                "time": "2026-10-06T00:00:00Z",
+                "event": "file_error",
+                "client": "",
+                "netblock": "",
+                "asn": "",
+                "as_name": "",
+                "country": "",
+                "reason": "writing the state files failed",
+                "detail": {
+                    "error": "no space left on device",
+                    "file": "/var/lib/smallwebwaf/bans.json"
+                },
+                "suppressed_repeats": 0
+            }
+        ]
+    }
 }
 `

@@ -234,7 +236,7 @@ func TestSourceFailureCooldownKeptInAlertsJSONAcrossARestart(t *testing.T) {
 	load(t, after)
 	after.Alerts.Raise(failure)

-	waiting := after.Alerts.Snapshot().Waiting
+	waiting := after.Alerts.Snapshot().Waiting[alerts.DestinationWebhook]
 	if len(waiting) != 1 || after.Alerts.Suppressed() != 1 {
 		t.Errorf("%d alerts wait and %d are held back, want the one read back and 1",
 			len(waiting), after.Alerts.Suppressed())
@@ -270,7 +272,7 @@ func TestMissingFilesAreEmptyState(t *testing.T) {
 	held := params.Alerts.Snapshot()
 	if len(params.Ledger.Snapshot()) != 0 || len(params.Limiter.Snapshot()) != 0 ||
 		len(params.GeoJS.Snapshot()) != 0 || len(held.Cooldowns) != 0 ||
-		len(held.Waiting) != 0 || held.Hour.Sent != 0 {
+		len(held.Waiting[alerts.DestinationWebhook]) != 0 || held.Hour.Sent != 0 {
 		t.Error("state from no files")
 	}
 }
@@ -314,9 +316,14 @@ func TestFileThatDoesNotParseStopsTheStart(t *testing.T) {
 		},
 		{
 			"an unknown field of an alert waiting", alertsJSON,
-			`{"version": 1, "waiting": [{"event": "ban", "evnet": "ban"}]}`,
+			`{"version": 1, "waiting": {"webhook": [{"event": "ban", "evnet": "ban"}]}}`,
 			`: json: unknown field "evnet"`,
 		},
+		{
+			"alerts waiting for an unknown destination", alertsJSON,
+			`{"version": 1, "waiting": {"webhook": [], "slak": []}}`,
+			`: waiting "slak" is not webhook, slack or ntfy`,
+		},
 	} {
 		t.Run(tc.name, func(t *testing.T) {
 			t.Parallel()
@@ -426,13 +433,13 @@ func TestAlertsJSONEntryWithoutAFieldItNeedsStopsTheStart(t *testing.T) {
 		},
 		{
 			"an alert waiting without its event",
-			`{"version": 1, "waiting": [{"time": "2026-10-06T00:00:00Z"}]}`,
-			`: waiting entry 1 has no "event"`,
+			`{"version": 1, "waiting": {"ntfy": [{"time": "2026-10-06T00:00:00Z"}]}}`,
+			`: waiting ntfy entry 1 has no "event"`,
 		},
 		{
 			"an alert waiting without its time",
-			`{"version": 1, "waiting": [{"event": "ban"}]}`,
-			`: waiting entry 1 has no "time"`,
+			`{"version": 1, "waiting": {"slack": [{"event": "ban"}]}}`,
+			`: waiting slack entry 1 has no "time"`,
 		},
 	} {
 		t.Run(tc.name, func(t *testing.T) {
@@ -629,7 +636,7 @@ func TestWriteThatFailsWhileRunningRaisesAFileErrorAlertOncePerCooldown(t *testi
 		time.Sleep(time.Minute)
 		synctest.Wait()

-		waiting := params.Alerts.Snapshot().Waiting
+		waiting := params.Alerts.Snapshot().Waiting[alerts.DestinationWebhook]
 		if len(waiting) != 1 {
 			t.Fatalf("%d alerts wait, want 1", len(waiting))
 		}
@@ -648,9 +655,10 @@ func TestWriteThatFailsWhileRunningRaisesAFileErrorAlertOncePerCooldown(t *testi
 		time.Sleep(time.Minute)
 		synctest.Wait()

-		if len(params.Alerts.Snapshot().Waiting) != 1 || params.Alerts.Suppressed() != 1 {
+		waiting = params.Alerts.Snapshot().Waiting[alerts.DestinationWebhook]
+		if len(waiting) != 1 || params.Alerts.Suppressed() != 1 {
 			t.Errorf("%d alerts wait and %d are held back, want 1 and 1",
-				len(params.Alerts.Snapshot().Waiting), params.Alerts.Suppressed())
+				len(waiting), params.Alerts.Suppressed())
 		}
 	})
 }
@@ -857,7 +865,7 @@ func TestEditOfEachFileTakenIn(t *testing.T) {
 	// in.
 	edit(t, dir, alertsJSON, `{"version": 1, "cooldowns": [{"event": "ban", `+
 		`"netblock": "198.51.100.9/24", "sent": "2026-10-06T00:00:00Z"}], `+
-		`"waiting": [{"event": "file_error", "time": "2026-10-06T00:00:00Z"}]}`)
+		`"waiting": {"webhook": [{"event": "file_error", "time": "2026-10-06T00:00:00Z"}]}}`)
 	wantTakenIn(t, lines, dir, alertsJSON)

 	want := alerts.State{
@@ -865,8 +873,10 @@ func TestEditOfEachFileTakenIn(t *testing.T) {
 			Event: alerts.EventBan, Netblock: netip.MustParsePrefix("198.51.100.0/24"),
 			Sent: midnight(),
 		}},
-		Hour:    alerts.Hour{HeldBack: map[string]int{}},
-		Waiting: []alerts.Alert{{Event: alerts.EventFileError, Time: midnight()}},
+		Hour: alerts.Hour{HeldBack: map[string]int{}},
+		Waiting: map[string][]alerts.Alert{
+			alerts.DestinationWebhook: {{Event: alerts.EventFileError, Time: midnight()}},
+		},
 	}
 	if got := params.Alerts.Snapshot(); !reflect.DeepEqual(got, want) {
 		t.Errorf("%s taken in as\n%+v\nwant\n%+v", alertsJSON, got, want)
@@ -1096,7 +1106,7 @@ func TestBrokenEditSetAsideAtTheNextWrite(t *testing.T) {
 	}

 	// It is raised as a file_error alert, with the same file and error.
-	waiting := params.Alerts.Snapshot().Waiting
+	waiting := params.Alerts.Snapshot().Waiting[alerts.DestinationWebhook]
 	if len(waiting) != 1 || waiting[0].Event != alerts.EventFileError ||
 		waiting[0].Detail["file"] != path+".bad" || waiting[0].Detail["error"] != message {
 		t.Errorf("alerts waiting %+v, want a file_error alert for %s", waiting, path+".bad")