Alerts to a JSON webhook, with a cooldown and an hourly summary (closes #26)

SWWAF_ALERT_WEBHOOK_URL gets one JSON POST per alert, in SPEC.md's
schema, with SWWAF_ALERT_WEBHOOK_HEADERS: ban and permanent_ban, with
the ban's notes, in observe mode too, marked mode observe;
source_failure for GeoJS; file_error for a rule or state file with an
error. SWWAF_ALERT_EVENTS chooses; SWWAF_ALERT_COOLDOWN holds back
repeats by netblock, file or source; past SWWAF_ALERT_MAX_PER_HOUR the
hour ends in one summary. A bounded queue, retried with backoff, holds
up no request; a 4xx other than 408 and 429 gives the alert up.
alerts.json keeps the queue, the cooldowns and the hour. Nothing shows
the URL's path or query.

Judgement call: the summary's event is summary, which SPEC.md omits.
Judgement call: an admin's ban raises no alert.

Model: opus-5-5

internal/metrics/metrics.go

@@ -11,6 +11,7 @@ import (
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/client_golang/prometheus/collectors"
 	"github.com/prometheus/client_golang/prometheus/promhttp"
+	"sneak.berlin/go/smallwebwaf/internal/alerts"
 	"sneak.berlin/go/smallwebwaf/internal/bans"
 	"sneak.berlin/go/smallwebwaf/internal/ratelimit"
 	"sneak.berlin/go/smallwebwaf/internal/remotelog"
@@ -225,6 +226,47 @@ func (m *Metrics) AddRemoteLog(remote *remotelog.Sender) {
 	)
 }
 
+// AddAlerts adds the metrics of the alerts sent to
+// SWWAF_ALERT_WEBHOOK_URL, read from queue as the metrics are asked for,
+// with the destination webhook: the alerts sent, the requests to the
+// webhook that failed, and the alerts held back and dropped.
+func (m *Metrics) AddAlerts(queue *alerts.Queue) {
+	webhook := prometheus.Labels{"destination": "webhook"}
+
+	m.registry.MustRegister(
+		prometheus.NewCounterFunc(prometheus.CounterOpts{
+			Name:        "smallwebwaf_alerts_sent_total",
+			Help:        "Alerts the destination took.",
+			ConstLabels: webhook,
+		}, func() float64 {
+			return float64(queue.Sent())
+		}),
+		prometheus.NewCounterFunc(prometheus.CounterOpts{
+			Name:        "smallwebwaf_alerts_failed_total",
+			Help:        "Requests to the destination that failed.",
+			ConstLabels: webhook,
+		}, func() float64 {
+			return float64(queue.Failed())
+		}),
+		prometheus.NewCounterFunc(prometheus.CounterOpts{
+			Name: "smallwebwaf_alerts_suppressed_total",
+			Help: "Alerts held back: repeats within SWWAF_ALERT_COOLDOWN, and " +
+				"alerts past SWWAF_ALERT_MAX_PER_HOUR, for the hour's summary.",
+			ConstLabels: webhook,
+		}, func() float64 {
+			return float64(queue.Suppressed())
+		}),
+		prometheus.NewCounterFunc(prometheus.CounterOpts{
+			Name: "smallwebwaf_alerts_dropped_total",
+			Help: "Alerts dropped, the oldest first, from a full queue, and alerts " +
+				"given up as the destination refused them.",
+			ConstLabels: webhook,
+		}, func() float64 {
+			return float64(queue.Dropped())
+		}),
+	)
+}
+
 // ServeHTTP answers with the metrics in the Prometheus text format.
 func (m *Metrics) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 	m.handler.ServeHTTP(w, r)