Alerts to a JSON webhook, with a cooldown and an hourly summary (closes #26)

SWWAF_ALERT_WEBHOOK_URL gets one JSON POST per alert, in SPEC.md's
schema, with SWWAF_ALERT_WEBHOOK_HEADERS: ban and permanent_ban, with
the ban's notes, in observe mode too, marked mode observe;
source_failure for GeoJS; file_error for a rule or state file with an
error. SWWAF_ALERT_EVENTS chooses; SWWAF_ALERT_COOLDOWN holds back
repeats by netblock, file or source; past SWWAF_ALERT_MAX_PER_HOUR the
hour ends in one summary. A bounded queue, retried with backoff, holds
up no request; a 4xx other than 408 and 429 gives the alert up.
alerts.json keeps the queue, the cooldowns and the hour. Nothing shows
the URL's path or query.

Judgement call: the summary's event is summary, which SPEC.md omits.
Judgement call: an admin's ban raises no alert.

Model: opus-5-5

internal/state/state.go

@@ -1,7 +1,8 @@
 // Package state keeps smallwebwaf's state in JSON files in
 // SWWAF_STATE_DIR, as the "Persistent state" section of SPEC.md describes:
 // bans.json holds the bans, clients.json each client's counters and
-// history, and lookups.json GeoJS's answers. Load reads them at start,
+// history, lookups.json GeoJS's answers, and alerts.json the cooldowns,
+// the hour under way and the alerts waiting. Load reads them at start,
 // Watch takes in an admin's edit of one while smallwebwaf runs, and Run
 // and WriteAll write them. The disk is read and written outside the
 // parts' locks, which are held only to take a snapshot or to put in what
@@ -24,6 +25,7 @@ import (
 	"time"
 
 	"github.com/fsnotify/fsnotify"
+	"sneak.berlin/go/smallwebwaf/internal/alerts"
 	"sneak.berlin/go/smallwebwaf/internal/bans"
 	"sneak.berlin/go/smallwebwaf/internal/lookup"
 	"sneak.berlin/go/smallwebwaf/internal/metrics"
@@ -42,6 +44,7 @@ const (
 	bansJSON    = "bans.json"
 	clientsJSON = "clients.json"
 	lookupsJSON = "lookups.json"
+	alertsJSON  = "alerts.json"
 )
 
 var (
@@ -60,10 +63,13 @@ type Params struct {
 	// is (SWWAF_STATE_COUNTER_INTERVAL).
 	WriteDelay      time.Duration
 	CounterInterval time.Duration
-	// Ledger, Limiter and GeoJS hold the state.
+	// Ledger, Limiter, GeoJS and Alerts hold the state. Alerts also
+	// receive a file_error alert for an edit set aside, and for a write
+	// that fails while smallwebwaf runs.
 	Ledger  *bans.Ledger
 	Limiter *ratelimit.Limiter
 	GeoJS   *lookup.GeoJS
+	Alerts  *alerts.Queue
 	// Now tells the time by which the counters' buckets run out, normally
 	// time.Now in UTC.
 	Now func() time.Time
@@ -121,6 +127,14 @@ type lookupsFile struct {
 	Lookups []lookup.Answer `json:"lookups"`
 }
 
+// alertsFile is alerts.json, indented for an admin to read and edit.
+type alertsFile struct {
+	Version   int               `json:"version"`
+	Cooldowns []alerts.Cooldown `json:"cooldowns"`
+	Hour      alerts.Hour       `json:"hour"`
+	Waiting   []alerts.Alert    `json:"waiting"`
+}
+
 // stateFile is the struct of a state file. Once the file is decoded, its
 // check refuses the first entry without a field it needs, which would
 // otherwise be read as something the entry does not say. data is the
@@ -147,23 +161,25 @@ func Load(params Params) (*Files, error) {
 	bansRead, bansErr := f.read(bansJSON)
 	clientsRead, clientsErr := f.read(clientsJSON)
 	lookupsRead, lookupsErr := f.read(lookupsJSON)
+	alertsRead, alertsErr := f.read(alertsJSON)
 
-	err = errors.Join(bansErr, clientsErr, lookupsErr)
+	err = errors.Join(bansErr, clientsErr, lookupsErr, alertsErr)
 	if err != nil {
 		return nil, err
 	}
 
 	params.ProcessLog.Info("read the state files", "directory", params.Dir,
-		"bans", bansRead, "clients", clientsRead, "lookups", lookupsRead)
+		"bans", bansRead, "clients", clientsRead, "lookups", lookupsRead,
+		"alerts_waiting", alertsRead)
 
 	return f, nil
 }
 
 // Run writes bans.json WriteDelay after a ban is made, with every ban
 // made in between, and every file every CounterInterval, until ctx is
-// done. A write that fails is logged, and the file is written again at
-// its next write. Each write takes in an admin's edit of its file first,
-// as writeFile describes.
+// done. A write that fails is logged, raised as a file_error alert, and
+// the file is written again at its next write. Each write takes in an
+// admin's edit of its file first, as writeFile describes.
 func (f *Files) Run(ctx context.Context) {
 	interval := time.NewTicker(f.params.CounterInterval)
 	defer interval.Stop()
@@ -181,9 +197,11 @@ func (f *Files) Run(ctx context.Context) {
 		case <-bansDue:
 			bansDue = nil
 
-			f.logFailure(f.writeFile(bansJSON))
+			f.logFailure(bansJSON, f.writeFile(bansJSON))
 		case <-interval.C:
-			f.logFailure(f.WriteAll())
+			for _, name := range []string{bansJSON, clientsJSON, lookupsJSON, alertsJSON} {
+				f.logFailure(name, f.writeFile(name))
+			}
 		}
 	}
 }
@@ -192,7 +210,7 @@ func (f *Files) Run(ctx context.Context) {
 // fails does not keep the others from being written.
 func (f *Files) WriteAll() error {
 	return errors.Join(f.writeFile(bansJSON), f.writeFile(clientsJSON),
-		f.writeFile(lookupsJSON))
+		f.writeFile(lookupsJSON), f.writeFile(alertsJSON))
 }
 
 // Watch watches Dir until ctx is done, and takes in an admin's edit of a
@@ -227,7 +245,7 @@ func (f *Files) Watch(ctx context.Context) {
 			return
 		case event := <-watcher.Events:
 			switch name := filepath.Base(event.Name); name {
-			case bansJSON, clientsJSON, lookupsJSON:
+			case bansJSON, clientsJSON, lookupsJSON, alertsJSON:
 				f.fileChanged(name)
 			}
 		case err = <-watcher.Errors:
@@ -237,11 +255,22 @@ func (f *Files) Watch(ctx context.Context) {
 	}
 }
 
-// logFailure logs a write that failed.
-func (f *Files) logFailure(err error) {
+// logFailure logs a write of the state file name that failed, and raises
+// a file_error alert for it.
+func (f *Files) logFailure(name string, err error) {
 	if err != nil {
-		f.params.ProcessLog.Error("writing the state files failed",
-			"error", err.Error())
+		const failed = "writing the state files failed"
+
+		// Raised before it is logged, so that the alert is there once the
+		// log line is.
+		f.params.Alerts.Raise(alerts.Alert{
+			Event:  alerts.EventFileError,
+			Reason: failed,
+			Detail: map[string]any{
+				"file": filepath.Join(f.params.Dir, name), "error": err.Error(),
+			},
+		})
+		f.params.ProcessLog.Error(failed, "error", err.Error())
 	}
 }
 
@@ -362,6 +391,18 @@ func (f *Files) takeIn(name string, data []byte, edit bool) (int, error) {
 
 		f.params.GeoJS.Load(file.Lookups)
 		entries = len(file.Lookups)
+	case alertsJSON:
+		var file alertsFile
+
+		err := parse(path, data, &file)
+		if err != nil {
+			return 0, err
+		}
+
+		f.params.Alerts.Load(alerts.State{
+			Cooldowns: file.Cooldowns, Hour: file.Hour, Waiting: file.Waiting,
+		})
+		entries = len(file.Waiting)
 	}
 
 	f.sums[name] = sha256.Sum256(data)
@@ -413,8 +454,9 @@ func (f *Files) writeFile(name string) error {
 
 // setAside renames the state file name, an edit that does not parse with
 // parseErr, to name.bad, for the admin to mend, and logs it with where in
-// the file the error is. If the rename fails, the edit is left as it is,
-// and the error returned is parseErr joined with the rename's.
+// the file the error is, and raises a file_error alert for it. If the
+// rename fails, the edit is left as it is, and the error returned is
+// parseErr joined with the rename's.
 func (f *Files) setAside(name string, parseErr error) error {
 	path := filepath.Join(f.params.Dir, name)
 
@@ -423,8 +465,16 @@ func (f *Files) setAside(name string, parseErr error) error {
 		return errors.Join(parseErr, err)
 	}
 
-	f.params.ProcessLog.Error("set aside an edit of a state file that does not parse",
-		"file", path+".bad", "error", parseErr.Error())
+	const setAside = "set aside an edit of a state file that does not parse"
+
+	// Raised before it is logged, so that the alert is there once the log
+	// line is.
+	f.params.Alerts.Raise(alerts.Alert{
+		Event:  alerts.EventFileError,
+		Reason: setAside,
+		Detail: map[string]any{"file": path + ".bad", "error": parseErr.Error()},
+	})
+	f.params.ProcessLog.Error(setAside, "file", path+".bad", "error", parseErr.Error())
 	f.params.Metrics.StateFileEditSetAside(name)
 
 	return nil
@@ -445,8 +495,21 @@ func (f *Files) encode(name string) ([]byte, error) {
 		return append(data, '\n'), nil
 	case clientsJSON:
 		return encodeOnePerLine("clients", f.params.Limiter.Snapshot())
-	default: // lookups.json
+	case lookupsJSON:
 		return encodeOnePerLine("lookups", f.params.GeoJS.Snapshot())
+	default: // alerts.json
+		held := f.params.Alerts.Snapshot()
+		file := alertsFile{
+			Version: version, Cooldowns: held.Cooldowns, Hour: held.Hour,
+			Waiting: held.Waiting,
+		}
+
+		data, err := json.MarshalIndent(file, "", "    ")
+		if err != nil {
+			return nil, err
+		}
+
+		return append(data, '\n'), nil
 	}
 }
 
@@ -581,6 +644,31 @@ func (f *lookupsFile) check(data []byte) error {
 	return nil
 }
 
+// check refuses a cooldown without its event or when its alert was sent,
+// which would hold back no repeat, and an alert waiting without its event
+// or its time.
+func (f *alertsFile) check([]byte) error {
+	for i, cooldown := range f.Cooldowns {
+		switch {
+		case cooldown.Event == "":
+			return fmt.Errorf("cooldowns %w", missing(i, "event"))
+		case cooldown.Sent.IsZero():
+			return fmt.Errorf("cooldowns %w", missing(i, "sent"))
+		}
+	}
+
+	for i, alert := range f.Waiting {
+		switch {
+		case alert.Event == "":
+			return fmt.Errorf("waiting %w", missing(i, "event"))
+		case alert.Time.IsZero():
+			return fmt.Errorf("waiting %w", missing(i, "time"))
+		}
+	}
+
+	return nil
+}
+
 // countsWithoutStart reports whether b holds requests but no start, which
 // places them in time.
 func countsWithoutStart(b ratelimit.Buckets) bool {