Alerts to Slack and ntfy, each destination with its own queue (closes #90)

Each alert is posted as a message to the Slack incoming webhook
SWWAF_ALERT_SLACK_WEBHOOK_URL names, and published to the ntfy topic
SWWAF_ALERT_NTFY_URL names, with SWWAF_ALERT_NTFY_TOKEN as a bearer
token and a priority and tag by event. The cooldown and the hourly
limit stay shared; past them, each destination has its own bounded
queue and backoff, and its own sent, failed and dropped counts.
alerts.json keeps the alerts waiting by destination.

Judgement call: messages also give the detail's file, source, error and mode.
Judgement call: alerts_suppressed_total is the same for every destination.
Judgement call: an alerts.json with waiting as a list stops the start.

Model: opus-5-5

internal/smallwebwaf/smallwebwaf_test.go

@@ -38,7 +38,8 @@ const (
 	stateCounterInterval = "SWWAF_STATE_COUNTER_INTERVAL"
 	rateLimitPerDay      = "SWWAF_RATE_LIMIT_PER_DAY"
 	rulesDir             = "SWWAF_RULES_DIR"
-	adminToken           = "SWWAF_ADMIN_TOKEN" //nolint:gosec // the setting's name
+	adminToken           = "SWWAF_ADMIN_TOKEN"   //nolint:gosec // the setting's name
+	metricsToken         = "SWWAF_METRICS_TOKEN" //nolint:gosec // the setting's name
 	// adminSecret is the SWWAF_ADMIN_TOKEN the tests set.
 	adminSecret = "fedcba9876543210fedcba9876543210"
 	// greeting is what the tests' app answers.
@@ -136,7 +137,7 @@ func TestShortTokenStopsTheStartUnshown(t *testing.T) {
 
 	const token = "a-token-of-31-characters-at-all" //nolint:gosec // too short to use
 
-	for _, name := range []string{adminToken, "SWWAF_METRICS_TOKEN"} {
+	for _, name := range []string{adminToken, metricsToken} {
 		t.Run(name, func(t *testing.T) {
 			t.Parallel()
 
@@ -517,7 +518,7 @@ func TestStalledRemoteLogEndpointHoldsUpNoRequest(t *testing.T) {
 		rulesDir:                  t.TempDir(),
 		"SWWAF_LOG_REMOTE_URL":    "syslog+tls://" + endpoint.Addr().String(),
 		"SWWAF_LOG_REMOTE_BUFFER": "1",
-		"SWWAF_METRICS_TOKEN":     token,
+		metricsToken:              token,
 	}
 
 	out := runUntilStopped(t, env, func(url string) {
@@ -591,7 +592,7 @@ func TestBanIsAlertedAndAnAlertNotSentIsKeptAcrossARestart(t *testing.T) {
 	// alerts.json keeps it as smallwebwaf stops, and once started again,
 	// smallwebwaf sends it.
 	var file struct {
-		Waiting []struct {
+		Waiting map[string][]struct {
 			Event string `json:"event"`
 		} `json:"waiting"`
 	}
@@ -603,8 +604,10 @@ func TestBanIsAlertedAndAnAlertNotSentIsKeptAcrossARestart(t *testing.T) {
 		err = json.Unmarshal(data, &file)
 	}
 
-	if err != nil || len(file.Waiting) != 1 || file.Waiting[0].Event != "permanent_ban" {
-		t.Fatalf("alerts.json holds %s (%v), want the permanent_ban alert waiting", data, err)
+	waiting := file.Waiting["webhook"]
+	if err != nil || len(waiting) != 1 || waiting[0].Event != "permanent_ban" {
+		t.Fatalf("alerts.json holds %s (%v), want the permanent_ban alert waiting for "+
+			"the webhook", data, err)
 	}
 
 	// It counts the alert sent in the metrics, read here from a client the
@@ -614,7 +617,7 @@ func TestBanIsAlertedAndAnAlertNotSentIsKeptAcrossARestart(t *testing.T) {
 	webhook.failing.Store(false)
 
 	env["SWWAF_ALLOW_NETS"] = localhost
-	env["SWWAF_METRICS_TOKEN"] = token
+	env[metricsToken] = token
 
 	runUntilStopped(t, env, func(url string) {
 		webhook.waitFor(t, "permanent_ban", true)
@@ -639,6 +642,79 @@ func TestBanIsAlertedAndAnAlertNotSentIsKeptAcrossARestart(t *testing.T) {
 	})
 }
 
+func TestBanIsAlertedToSlackAndNtfy(t *testing.T) {
+	t.Parallel()
+
+	const (
+		ntfyToken = "tk_0123456789abcdefghijklmnopq"
+		token     = "abcdef0123456789abcdef0123456789"
+		client    = "203.0.113.9"
+	)
+
+	slack, ntfy := startDestination(t), startDestination(t)
+	env := map[string]string{
+		listenAddr:      localhost + ":0",
+		upstreamURL:     startApp(t),
+		stateDir:        t.TempDir(),
+		rulesDir:        t.TempDir(),
+		trustedProxies:  localhost + "/32",
+		rateLimitPerDay: "1",
+		// The metrics are read from 127.0.0.1, which no limit counts.
+		"SWWAF_ALLOW_NETS":              localhost + "/32",
+		metricsToken:                    token,
+		"SWWAF_INSTANCE_NAME":           "fsn1app1/gitea",
+		"SWWAF_ALERT_SLACK_WEBHOOK_URL": slack.url,
+		"SWWAF_ALERT_NTFY_URL":          ntfy.url,
+		"SWWAF_ALERT_NTFY_TOKEN":        ntfyToken,
+	}
+
+	runUntilStopped(t, env, func(url string) {
+		// The client's second request breaks the day limit, and bans it;
+		// Slack and ntfy are each sent the alert.
+		wantStatus(t, url, client, http.StatusOK)
+		wantStatus(t, url, client, http.StatusForbidden)
+
+		var message struct {
+			Text string `json:"text"`
+		}
+
+		slackPost := slack.firstPost(t)
+		err := json.Unmarshal([]byte(slackPost.body), &message)
+
+		if err != nil || !strings.HasPrefix(message.Text, "*fsn1app1/gitea: ban*\n") ||
+			!strings.Contains(message.Text, "\nclient: "+client+"\n") {
+			t.Errorf("Slack was sent %s", slackPost.body)
+		}
+
+		ntfyPost := ntfy.firstPost(t)
+		if ntfyPost.header.Get("Title") != "fsn1app1/gitea: ban" ||
+			ntfyPost.header.Get("Authorization") != "Bearer "+ntfyToken ||
+			!strings.Contains(ntfyPost.body, "\nclient: "+client+"\n") {
+			t.Errorf("ntfy was sent %s, with the headers %v", ntfyPost.body,
+				ntfyPost.header)
+		}
+
+		// The metrics count it for each, and give no series for the
+		// webhook, which is not set. As long as that takes, so that a slow
+		// test process cannot fail the test.
+		sent := []string{
+			"\nsmallwebwaf_alerts_sent_total{destination=\"slack\"} 1\n",
+			"\nsmallwebwaf_alerts_sent_total{destination=\"ntfy\"} 1\n",
+		}
+
+		metrics := metricsText(t, url+"_smallwebwaf/metrics", token)
+		for !strings.Contains(metrics, sent[0]) || !strings.Contains(metrics, sent[1]) {
+			time.Sleep(pollInterval)
+
+			metrics = metricsText(t, url+"_smallwebwaf/metrics", token)
+		}
+
+		if strings.Contains(metrics, `destination="webhook"`) {
+			t.Errorf("the metrics give the webhook:\n%s", metrics)
+		}
+	})
+}
+
 func TestStateFileThatDoesNotParseStopsTheStart(t *testing.T) {
 	t.Parallel()
 
@@ -1009,6 +1085,66 @@ func saveUntilAnswered(t *testing.T, path, content, url, from string, status int
 	}
 }
 
+// destination is a stand-in for SWWAF_ALERT_SLACK_WEBHOOK_URL or
+// SWWAF_ALERT_NTFY_URL. It notes each request it is sent, and answers
+// 200.
+type destination struct {
+	url string
+
+	mu    sync.Mutex
+	posts []destinationPost
+}
+
+// destinationPost is a request a destination was sent: its headers and
+// its body.
+type destinationPost struct {
+	header http.Header
+	body   string
+}
+
+// startDestination starts a destination that takes every alert.
+func startDestination(t *testing.T) *destination {
+	t.Helper()
+
+	d := &destination{}
+	server := httptest.NewServer(http.HandlerFunc(
+		func(_ http.ResponseWriter, r *http.Request) {
+			body, _ := io.ReadAll(r.Body)
+
+			d.mu.Lock()
+			d.posts = append(d.posts, destinationPost{
+				header: r.Header.Clone(), body: string(body),
+			})
+			d.mu.Unlock()
+		}))
+	t.Cleanup(server.Close)
+
+	d.url = server.URL + "/alerts"
+
+	return d
+}
+
+// firstPost waits until the destination has been sent a request, and
+// returns the first. It waits as long as that takes, so that a slow test
+// process cannot fail the test.
+func (d *destination) firstPost(t *testing.T) destinationPost {
+	t.Helper()
+
+	for {
+		d.mu.Lock()
+
+		if len(d.posts) > 0 {
+			post := d.posts[0]
+			d.mu.Unlock()
+
+			return post
+		}
+
+		d.mu.Unlock()
+		time.Sleep(pollInterval)
+	}
+}
+
 // webhook is a stand-in for SWWAF_ALERT_WEBHOOK_URL. It notes each alert
 // it is sent, and answers 204, or 503 while failing.
 type webhook struct {