Alerts to Slack and ntfy, each destination with its own queue (closes #90)

Each alert is posted as a message to the Slack incoming webhook
SWWAF_ALERT_SLACK_WEBHOOK_URL names, and published to the ntfy topic
SWWAF_ALERT_NTFY_URL names, with SWWAF_ALERT_NTFY_TOKEN as a bearer
token and a priority and tag by event. The cooldown and the hourly
limit stay shared; past them, each destination has its own bounded
queue and backoff, and its own sent, failed and dropped counts.
alerts.json keeps the alerts waiting by destination; one whose waiting
is still a list stops the start, saying what to change. A control
character in the ntfy token, or in the instance name ntfy is sent,
stops the start.

Judgement call: messages also give the detail's file, source, error and mode.
Judgement call: alerts_suppressed_total is the same for every destination.

Model: opus-5-5

internal/state/state.go

@@ -2,11 +2,12 @@
 // SWWAF_STATE_DIR, as the "Persistent state" section of SPEC.md describes:
 // bans.json holds the bans, clients.json each client's counters and
 // history, lookups.json GeoJS's answers, and alerts.json the cooldowns,
-// the hour under way and the alerts waiting. Load reads them at start,
-// Watch takes in an admin's edit of one while smallwebwaf runs, and Run
-// and WriteAll write them. The disk is read and written outside the
-// parts' locks, which are held only to take a snapshot or to put in what
-// a file holds, so that no request waits on the disk.
+// the hour under way and the alerts waiting for each destination. Load
+// reads them at start, Watch takes in an admin's edit of one while
+// smallwebwaf runs, and Run and WriteAll write them. The disk is read and
+// written outside the parts' locks, which are held only to take a
+// snapshot or to put in what a file holds, so that no request waits on
+// the disk.
 package state
 
 import (
@@ -18,9 +19,11 @@ import (
 	"fmt"
 	"io/fs"
 	"log/slog"
+	"maps"
 	"net/netip"
 	"os"
 	"path/filepath"
+	"slices"
 	"sync"
 	"time"
 
@@ -50,8 +53,12 @@ const (
 var (
 	errVersion = errors.New("unknown version")
 	// errMissing is for an entry without a field it needs.
-	errMissing = errors.New("has no")
-	errCause   = errors.New("is not limit, attack or admin")
+	errMissing     = errors.New("has no")
+	errCause       = errors.New("is not limit, attack or admin")
+	errDestination = errors.New("is not webhook, slack or ntfy")
+	errWaitingList = errors.New(`waiting is a list, but now lists the alerts by ` +
+		`destination: put the list under "webhook", as "waiting": {"webhook": [...]}, ` +
+		`or remove the file`)
 )
 
 // Params are what Load needs.
@@ -129,10 +136,10 @@ type lookupsFile struct {
 
 // alertsFile is alerts.json, indented for an admin to read and edit.
 type alertsFile struct {
-	Version   int               `json:"version"`
-	Cooldowns []alerts.Cooldown `json:"cooldowns"`
-	Hour      alerts.Hour       `json:"hour"`
-	Waiting   []alerts.Alert    `json:"waiting"`
+	Version   int                       `json:"version"`
+	Cooldowns []alerts.Cooldown         `json:"cooldowns"`
+	Hour      alerts.Hour               `json:"hour"`
+	Waiting   map[string][]alerts.Alert `json:"waiting"`
 }
 
 // stateFile is the struct of a state file. Once the file is decoded, its
@@ -392,6 +399,17 @@ func (f *Files) takeIn(name string, data []byte, edit bool) (int, error) {
 		f.params.GeoJS.Load(file.Lookups)
 		entries = len(file.Lookups)
 	case alertsJSON:
+		// waiting was a list, of the alerts waiting for the webhook, before
+		// alerts went to Slack and ntfy too.
+		var written struct {
+			Waiting json.RawMessage `json:"waiting"`
+		}
+
+		if json.Unmarshal(data, &written) == nil &&
+			bytes.HasPrefix(written.Waiting, []byte("[")) {
+			return 0, fmt.Errorf("%s: %w", path, errWaitingList)
+		}
+
 		var file alertsFile
 
 		err := parse(path, data, &file)
@@ -402,7 +420,10 @@ func (f *Files) takeIn(name string, data []byte, edit bool) (int, error) {
 		f.params.Alerts.Load(alerts.State{
 			Cooldowns: file.Cooldowns, Hour: file.Hour, Waiting: file.Waiting,
 		})
-		entries = len(file.Waiting)
+
+		for _, waiting := range file.Waiting {
+			entries += len(waiting)
+		}
 	}
 
 	f.sums[name] = sha256.Sum256(data)
@@ -645,8 +666,9 @@ func (f *lookupsFile) check(data []byte) error {
 }
 
 // check refuses a cooldown without its event or when its alert was sent,
-// which would hold back no repeat, and an alert waiting without its event
-// or its time.
+// which would hold back no repeat, alerts waiting for a destination with
+// another name than webhook, slack or ntfy, most likely misspelt, and an
+// alert waiting without its event or its time.
 func (f *alertsFile) check([]byte) error {
 	for i, cooldown := range f.Cooldowns {
 		switch {
@@ -657,12 +679,18 @@ func (f *alertsFile) check([]byte) error {
 		}
 	}
 
-	for i, alert := range f.Waiting {
-		switch {
-		case alert.Event == "":
-			return fmt.Errorf("waiting %w", missing(i, "event"))
-		case alert.Time.IsZero():
-			return fmt.Errorf("waiting %w", missing(i, "time"))
+	for _, destination := range slices.Sorted(maps.Keys(f.Waiting)) {
+		if !slices.Contains(alerts.Destinations(), destination) {
+			return fmt.Errorf("waiting %q %w", destination, errDestination)
+		}
+
+		for i, alert := range f.Waiting[destination] {
+			switch {
+			case alert.Event == "":
+				return fmt.Errorf("waiting %s %w", destination, missing(i, "event"))
+			case alert.Time.IsZero():
+				return fmt.Errorf("waiting %s %w", destination, missing(i, "time"))
+			}
 		}
 	}