Alerts to Slack and ntfy, each destination with its own queue (closes #90)

Each alert is posted as a message to the Slack incoming webhook SWWAF_ALERT_SLACK_WEBHOOK_URL names, and published to the ntfy topic SWWAF_ALERT_NTFY_URL names, with SWWAF_ALERT_NTFY_TOKEN as a bearer token and a priority and tag by event. The cooldown and the hourly limit stay shared; past them, each destination has its own bounded queue and backoff, and its own sent, failed and dropped counts. alerts.json keeps the alerts waiting by destination; one whose waiting is still a list stops the start, saying what to change. A control character in the ntfy token, or in the instance name ntfy is sent, stops the start. Judgement call: messages also give the detail's file, source, error and mode. Judgement call: alerts_suppressed_total is the same for every destination. Model: opus-5-5
2026-10-07 05:54:34 +02:00
parent 432097ee3f
commit 70a8ea1b92
14 changed files with 1586 additions and 428 deletions
@@ -26,13 +26,18 @@ import (
 // clock of the test's own, which starts at 2000-01-01T00:00:00Z, the start
 // of an hour: a wait lasts exactly as long as it should, however slowly
 // the test process runs, and synctest.Wait returns once the queue has
-// done all it can before time passes. The stand-in for the webhook
-// answers without the network, since a request waiting on the network
-// would keep that clock from moving on.
+// done all it can before time passes. The stand-ins for the webhook,
+// Slack and ntfy answer without the network, since a request waiting on
+// the network would keep that clock from moving on.

 const (
-	// webhookURL is where the alerts are posted.
+	// webhookURL is where the alerts are posted, slackURL the Slack
+	// incoming webhook, and ntfyURL the ntfy topic. ntfyToken is the ntfy
+	// token of the tests that set one.
 	webhookURL = "https://alerts.example/smallwebwaf?team=ops"
+	slackURL   = "https://hooks.slack.example/services/T0123/B4567/abcdef"
+	ntfyURL    = "https://ntfy.example/smallwebwaf-alerts"
+	ntfyToken  = "tk_0123456789abcdefghijklmnopq"
 	// instance is the instance name every alert gives.
 	instance = "fsn1app1/gitea"
 	// started is when each test starts, as an alert gives it, and
@@ -135,7 +140,7 @@ func TestWouldSendOnlyForTheChosenEvents(t *testing.T) {
 	}
 }

-func TestNothingIsQueuedWithoutAWebhook(t *testing.T) {
+func TestRaiseDoesNothingWithoutADestination(t *testing.T) {
 	t.Parallel()

 	params := newParams()
@@ -144,12 +149,18 @@ func TestNothingIsQueuedWithoutAWebhook(t *testing.T) {

 	q.Raise(alerts.Alert{Event: alerts.EventBan, Netblock: netblock(1)})

-	if waiting := q.Snapshot().Waiting; len(waiting) != 0 {
-		t.Errorf("%d alerts wait, want none", len(waiting))
+	// No alert waits, no cooldown has started, and the hour counts none.
+	want := alerts.State{
+		Cooldowns: []alerts.Cooldown{},
+		Hour:      alerts.Hour{HeldBack: map[string]int{}},
+		Waiting:   map[string][]alerts.Alert{},
+	}
+	if got := q.Snapshot(); !reflect.DeepEqual(got, want) {
+		t.Errorf("state %+v, want %+v", got, want)
 	}
 }

-func TestWouldSendNothingWithoutAWebhook(t *testing.T) {
+func TestWouldSendNothingWithoutADestination(t *testing.T) {
 	t.Parallel()

 	params := newParams()
@@ -161,6 +172,27 @@ func TestWouldSendNothingWithoutAWebhook(t *testing.T) {
 	}
 }

+func TestWouldSendWithOnlySlackOrOnlyNtfySet(t *testing.T) {
+	t.Parallel()
+
+	onlySlack := newParams()
+	onlySlack.WebhookURL = nil
+	onlySlack.SlackURL = parseURL(slackURL)
+
+	onlyNtfy := newParams()
+	onlyNtfy.WebhookURL = nil
+	onlyNtfy.NtfyURL = parseURL(ntfyURL)
+
+	for setting, params := range map[string]alerts.Params{
+		"SWWAF_ALERT_SLACK_WEBHOOK_URL": onlySlack,
+		"SWWAF_ALERT_NTFY_URL":          onlyNtfy,
+	} {
+		if !alerts.New(params).WouldSend(alerts.EventBan, netblock(1)) {
+			t.Errorf("a ban alert would not be sent with only %s set", setting)
+		}
+	}
+}
+
 func TestRepeatWithinTheCooldownIsHeldBackAndCountedInTheNext(t *testing.T) {
 	t.Parallel()

@@ -265,7 +297,7 @@ func TestFileErrorAndSourceFailureRepeatOnlyForTheSameFileOrSource(t *testing.T)
 		after.Raise(fileError("/rules.d/50-a.rules"))
 		after.Raise(fileError("/rules.d/50-c.rules"))

-		waiting := after.Snapshot().Waiting
+		waiting := after.Snapshot().Waiting[alerts.DestinationWebhook]
 		if len(waiting) != 1 || waiting[0].Detail["file"] != "/rules.d/50-c.rules" {
 			t.Errorf("after loading, alerts wait %+v, want the one for 50-c.rules", waiting)
 		}
@@ -444,7 +476,8 @@ func TestFailedRequestIsSentAgainWithBackoff(t *testing.T) {

 		wantCounts(t, q, 1, int64(len(want)), 0, 0)

-		if waiting := q.Snapshot().Waiting; len(waiting) != 0 {
+		waiting := q.Snapshot().Waiting[alerts.DestinationWebhook]
+		if len(waiting) != 0 {
 			t.Errorf("%d alerts still wait, want none", len(waiting))
 		}
 	})
@@ -552,7 +585,7 @@ func TestFullQueueDropsTheOldestAndRaiseNeverWaits(t *testing.T) {

 		wantCounts(t, q, 0, 0, 0, 1)

-		waiting := q.Snapshot().Waiting
+		waiting := q.Snapshot().Waiting[alerts.DestinationWebhook]
 		if len(waiting) != alerts.QueueSize || waiting[0].Netblock != netblock(1) {
 			t.Fatalf("%d alerts wait, the first for %s, want %d, the first for %s",
 				len(waiting), waiting[0].Netblock, alerts.QueueSize, netblock(1))
@@ -627,7 +660,264 @@ func TestStateLoadedIntoANewQueueCarriesOn(t *testing.T) {
 	})
 }

-// How the stand-in for the webhook answers: with a status, or, hanging,
+func TestSlackAndNtfyAreSentAMessageForEachEvent(t *testing.T) {
+	t.Parallel()
+
+	synctest.Test(t, func(t *testing.T) {
+		params := withSlackAndNtfy(newParams())
+		params.WebhookURL = nil
+		standIns, q := startAll(t, params)
+
+		for _, each := range anAlertForEachEvent() {
+			q.Raise(each.alert)
+		}
+
+		synctest.Wait()
+
+		want := anAlertForEachEvent()
+		slack := standIns[alerts.DestinationSlack].received()
+		ntfy := standIns[alerts.DestinationNtfy].received()
+
+		if len(slack) != len(want) || len(ntfy) != len(want) {
+			t.Fatalf("Slack was sent %d messages and ntfy %d, want %d each",
+				len(slack), len(ntfy), len(want))
+		}
+
+		for i, each := range want {
+			wantSlackMessage(t, slack[i], "*"+each.title+"*\n"+each.text)
+			wantNtfyMessage(t, ntfy[i], each.title, each.priorityAndTag, each.text)
+		}
+	})
+}
+
+func TestSlackAndNtfyAreSentTheSummaryAndTheRepeatsHeldBack(t *testing.T) {
+	t.Parallel()
+
+	synctest.Test(t, func(t *testing.T) {
+		params := withSlackAndNtfy(newParams())
+		params.WebhookURL = nil
+		params.MaxPerHour = 1
+		standIns, q := startAll(t, params)
+		ban := alerts.Alert{Event: alerts.EventBan, Netblock: netblock(1), Reason: "a ban"}
+
+		// The hour's one alert, a repeat of it the cooldown holds back, and
+		// an alert past the limit; once the hour has ended, its summary, and
+		// the next alert, which gives the repeat.
+		q.Raise(ban)
+		q.Raise(ban)
+		q.Raise(alerts.Alert{Event: alerts.EventFileError, Reason: "a file error"})
+		time.Sleep(time.Hour)
+		synctest.Wait()
+		q.Raise(ban)
+		synctest.Wait()
+
+		slack := standIns[alerts.DestinationSlack].received()
+		ntfy := standIns[alerts.DestinationNtfy].received()
+
+		if len(slack) != 3 || len(ntfy) != 3 {
+			t.Fatalf("Slack was sent %d messages and ntfy %d, want 3 each",
+				len(slack), len(ntfy))
+		}
+
+		const summary = "1 alerts held back in the hour from 2000-01-01T00:00:00Z, " +
+			"past the 1 an hour SWWAF_ALERT_MAX_PER_HOUR allows"
+
+		wantSlackMessage(t, slack[1], "*"+instance+": summary*\n"+summary)
+		wantNtfyMessage(t, ntfy[1], instance+": summary", "default bar_chart", summary)
+		wantSlackMessage(t, slack[2],
+			"*"+instance+": ban*\na ban\nnetblock: 203.0.113.1/32\nsuppressed repeats: 1")
+		wantNtfyMessage(t, ntfy[2], instance+": ban", "default no_entry",
+			"a ban\nnetblock: 203.0.113.1/32\nsuppressed repeats: 1")
+	})
+}
+
+func TestSlackMessageEscapesAmpersandsAndAngleBrackets(t *testing.T) {
+	t.Parallel()
+
+	synctest.Test(t, func(t *testing.T) {
+		params := withSlackAndNtfy(newParams())
+		params.Instance = "app<1>"
+		standIns, q := startAll(t, params)
+
+		q.Raise(alerts.Alert{
+			Event: alerts.EventFileError, Reason: "<!channel> & <https://x.example|y>",
+		})
+		synctest.Wait()
+
+		got := standIns[alerts.DestinationSlack].received()
+		if len(got) != 1 {
+			t.Fatalf("Slack was sent %d messages, want 1", len(got))
+		}
+
+		wantSlackMessage(t, got[0], "*app&lt;1&gt;: file_error*\n"+
+			"&lt;!channel&gt; &amp; &lt;https://x.example|y&gt;")
+		wantBodies(t, standIns[alerts.DestinationNtfy], "<!channel> & <https://x.example|y>")
+	})
+}
+
+func TestNtfyTokenIsSentToNtfyAlone(t *testing.T) {
+	t.Parallel()
+
+	for _, tc := range []struct {
+		name, token string
+		want        []string
+	}{
+		{"with a token", ntfyToken, []string{"Bearer " + ntfyToken}},
+		{"without one", "", nil},
+	} {
+		t.Run(tc.name, func(t *testing.T) {
+			t.Parallel()
+
+			synctest.Test(t, func(t *testing.T) {
+				params := withSlackAndNtfy(newParams())
+				params.NtfyToken = tc.token
+				standIns, q := startAll(t, params)
+
+				q.Raise(alerts.Alert{Event: alerts.EventBan, Netblock: netblock(1)})
+				synctest.Wait()
+
+				for destination, want := range map[string][]string{
+					alerts.DestinationWebhook: nil,
+					alerts.DestinationSlack:   nil,
+					alerts.DestinationNtfy:    tc.want,
+				} {
+					got := standIns[destination].received()
+					if len(got) != 1 {
+						t.Fatalf("%s was sent %d requests, want 1", destination, len(got))
+					}
+
+					authorization := got[0].header.Values("Authorization")
+					if !slices.Equal(authorization, want) {
+						t.Errorf("%s was sent Authorization %q, want %q", destination,
+							authorization, want)
+					}
+				}
+			})
+		})
+	}
+}
+
+func TestDestinationThatDoesNotAnswerHoldsUpNeitherOther(t *testing.T) {
+	t.Parallel()
+
+	synctest.Test(t, func(t *testing.T) {
+		params := withSlackAndNtfy(newParams())
+		log := &lockedBuffer{}
+		params.ProcessLog = slog.New(slog.NewJSONHandler(log, nil))
+		standIns, q := startAll(t, params)
+		standIns[alerts.DestinationSlack].set(hanging)
+
+		for n := range 3 {
+			q.Raise(alerts.Alert{Event: alerts.EventBan, Netblock: netblock(n)})
+		}
+
+		synctest.Wait()
+
+		// With no time passed, the webhook and ntfy have taken every alert,
+		// while Slack has not answered the first.
+		for destination, want := range map[string]alerts.Counts{
+			alerts.DestinationWebhook: {Sent: 3},
+			alerts.DestinationSlack:   {},
+			alerts.DestinationNtfy:    {Sent: 3},
+		} {
+			wantDestinationCounts(t, q, destination, want)
+		}
+
+		if got := standIns[alerts.DestinationSlack].received(); len(got) != 1 {
+			t.Errorf("Slack was sent %d requests, want 1", len(got))
+		}
+
+		// Slack's request is abandoned after 10 seconds, and logged without
+		// its URL; Slack, which answers again, is sent every alert a second
+		// later.
+		standIns[alerts.DestinationSlack].set(answering)
+		time.Sleep(11 * time.Second)
+		synctest.Wait()
+
+		wantDestinationCounts(t, q, alerts.DestinationSlack,
+			alerts.Counts{Sent: 3, Failed: 1})
+
+		logged := log.String()
+		if !strings.Contains(logged,
+			`"msg":"sending an alert to SWWAF_ALERT_SLACK_WEBHOOK_URL failed"`) ||
+			strings.Contains(logged, "hooks.slack.example") ||
+			strings.Contains(logged, "T0123") {
+			t.Errorf("process log %q names no failure, or names the URL", logged)
+		}
+	})
+}
+
+func TestDropsAreCountedForEachDestination(t *testing.T) {
+	t.Parallel()
+
+	synctest.Test(t, func(t *testing.T) {
+		params := withSlackAndNtfy(newParams())
+		params.MaxPerHour = 0
+		standIns, q := startAll(t, params)
+		standIns[alerts.DestinationSlack].set(hanging)
+		standIns[alerts.DestinationNtfy].set(refusing)
+
+		// Each alert is raised once each destination has done all it can
+		// with those before it.
+		for n := range alerts.QueueSize + 1 {
+			q.Raise(alerts.Alert{Event: alerts.EventBan, Netblock: netblock(n)})
+			synctest.Wait()
+		}
+
+		// Slack, which does not answer the first alert, drops it from its
+		// full queue for the last; ntfy refuses each, which is given up; and
+		// the webhook takes every one.
+		const all = alerts.QueueSize + 1
+
+		for destination, want := range map[string]alerts.Counts{
+			alerts.DestinationWebhook: {Sent: all},
+			alerts.DestinationSlack:   {Dropped: 1},
+			alerts.DestinationNtfy:    {Failed: all, Dropped: all},
+		} {
+			wantDestinationCounts(t, q, destination, want)
+		}
+	})
+}
+
+func TestEachDestinationIsSentOnlyTheAlertsWaitingForIt(t *testing.T) {
+	t.Parallel()
+
+	synctest.Test(t, func(t *testing.T) {
+		params := withSlackAndNtfy(newParams())
+		params.WebhookURL = nil
+		standIns, q := startAll(t, params)
+		waiting := func(reason string) alerts.Alert {
+			return alerts.Alert{
+				Instance: instance, Time: midnight(), Event: alerts.EventFileError,
+				Reason: reason,
+			}
+		}
+
+		// As read from alerts.json. The alert waiting for the webhook, which
+		// is not set, is dropped.
+		q.Load(roundTrip(t, alerts.State{Waiting: map[string][]alerts.Alert{
+			alerts.DestinationWebhook: {waiting("first")},
+			alerts.DestinationSlack:   {waiting("second"), waiting("third")},
+			alerts.DestinationNtfy:    {waiting("fourth")},
+		}}))
+		synctest.Wait()
+
+		wantBodies(t, standIns[alerts.DestinationSlack],
+			`{"text":"*fsn1app1/gitea: file_error*\nsecond"}`,
+			`{"text":"*fsn1app1/gitea: file_error*\nthird"}`)
+		wantBodies(t, standIns[alerts.DestinationNtfy], "fourth")
+
+		// alerts.json then lists Slack and ntfy alone, with no alert waiting.
+		want := map[string][]alerts.Alert{
+			alerts.DestinationSlack: {}, alerts.DestinationNtfy: {},
+		}
+		if got := q.Snapshot().Waiting; !reflect.DeepEqual(got, want) {
+			t.Errorf("alerts waiting %v, want %v", got, want)
+		}
+	})
+}
+
+// How a stand-in for a destination answers: with a status, or, hanging,
 // not at all, until the request is abandoned.
 const (
 	answering = http.StatusNoContent
@@ -636,7 +926,7 @@ const (
 	hanging   = 0
 )

-// standIn is a stand-in for the webhook. It notes each request it is
+// standIn is a stand-in for a destination. It notes each request it is
 // sent.
 type standIn struct {
 	mu       sync.Mutex
@@ -644,14 +934,16 @@ type standIn struct {
 	requests []post
 }

-// post is a request the webhook was sent: when, its method, URL and
-// headers, the alert it carried, and whether the webhook answered it with
-// a 2xx status.
+// post is a request a destination was sent: when, its method, URL and
+// headers, its body, and that body read as a JSON object, which for the
+// webhook is the alert it carried, and whether the destination answered
+// it with a 2xx status.
 type post struct {
 	at       time.Time
 	method   string
 	url      string
 	header   http.Header
+	body     string
 	alert    map[string]any
 	answered bool
 }
@@ -685,7 +977,7 @@ func (s *standIn) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 	answers := s.answers
 	s.requests = append(s.requests, post{
 		at: time.Now(), method: r.Method, url: r.URL.String(), header: r.Header.Clone(),
-		alert: alert, answered: answers == answering,
+		body: string(body), alert: alert, answered: answers == answering,
 	})
 	s.mu.Unlock()

@@ -739,13 +1031,8 @@ func (b *lockedBuffer) String() string {
 // every event, the default cooldown and hourly limit, and the bubble's
 // clock in UTC.
 func newParams() alerts.Params {
-	webhook, err := url.Parse(webhookURL)
-	if err != nil {
-		panic(err)
-	}
-
 	return alerts.Params{
-		WebhookURL: webhook,
+		WebhookURL: parseURL(webhookURL),
 		Events:     alerts.Events(),
 		Cooldown:   cooldown,
 		MaxPerHour: 60,
@@ -755,14 +1042,48 @@ func newParams() alerts.Params {
 	}
 }

+// withSlackAndNtfy returns params with Slack at slackURL and ntfy at
+// ntfyURL set as well.
+func withSlackAndNtfy(params alerts.Params) alerts.Params {
+	params.SlackURL = parseURL(slackURL)
+	params.NtfyURL = parseURL(ntfyURL)
+
+	return params
+}
+
+// parseURL returns rawURL, parsed.
+func parseURL(rawURL string) *url.URL {
+	parsed, err := url.Parse(rawURL)
+	if err != nil {
+		panic(err)
+	}
+
+	return parsed
+}
+
 // start returns a stand-in for the webhook that answers, and a Queue that
 // sends to it, run until the test ends.
 func start(t *testing.T, params alerts.Params) (*standIn, *alerts.Queue) {
 	t.Helper()

-	webhook := &standIn{answers: answering}
+	standIns, q := startAll(t, params)
+
+	return standIns[alerts.DestinationWebhook], q
+}
+
+// startAll returns, by destination, a stand-in that answers for each
+// destination params sets, and a Queue that sends to them, run until the
+// test ends.
+func startAll(t *testing.T, params alerts.Params) (map[string]*standIn, *alerts.Queue) {
+	t.Helper()
+
 	q := alerts.New(params)
-	q.SetTransport(webhook)
+	standIns := map[string]*standIn{}
+
+	for _, destination := range q.DestinationsSet() {
+		standIns[destination] = &standIn{answers: answering}
+		q.SetTransport(destination, standIns[destination])
+	}

 	ctx, stop := context.WithCancel(t.Context())
 	stopped := make(chan struct{})
@@ -777,7 +1098,7 @@ func start(t *testing.T, params alerts.Params) (*standIn, *alerts.Queue) {
 		<-stopped
 	})

-	return webhook, q
+	return standIns, q
 }

 // midnight is when each test starts.
@@ -836,17 +1157,158 @@ func wantEvents(t *testing.T, webhook *standIn, want ...string) {
 	}
 }

-// wantCounts checks the alerts q counts as sent, the requests it counts as
-// failed, and the alerts it counts as held back and as dropped.
+// wantCounts checks the alerts q counts as sent to the webhook, the
+// requests to it that it counts as failed, the alerts it counts as held
+// back, and those it counts as dropped for the webhook.
 func wantCounts(
 	t *testing.T, q *alerts.Queue, sent, failed, suppressed, dropped int64,
 ) {
 	t.Helper()

-	if q.Sent() != sent || q.Failed() != failed || q.Suppressed() != suppressed ||
-		q.Dropped() != dropped {
-		t.Errorf("counts sent %d, failed %d, suppressed %d and dropped %d, "+
-			"want %d, %d, %d and %d", q.Sent(), q.Failed(), q.Suppressed(), q.Dropped(),
-			sent, failed, suppressed, dropped)
+	wantDestinationCounts(t, q, alerts.DestinationWebhook,
+		alerts.Counts{Sent: sent, Failed: failed, Dropped: dropped})
+
+	if q.Suppressed() != suppressed {
+		t.Errorf("%d alerts held back, want %d", q.Suppressed(), suppressed)
+	}
+}
+
+// wantDestinationCounts checks q's counts for destination.
+func wantDestinationCounts(
+	t *testing.T, q *alerts.Queue, destination string, want alerts.Counts,
+) {
+	t.Helper()
+
+	if got := q.Counts(destination); got != want {
+		t.Errorf("%s counts %+v, want %+v", destination, got, want)
+	}
+}
+
+// eventMessage is an alert, and the message Slack and ntfy are sent for
+// it: its title, the priority and the tag ntfy is sent, with a space
+// between them, and its text.
+type eventMessage struct {
+	alert                       alerts.Alert
+	title, priorityAndTag, text string
+}
+
+// anAlertForEachEvent returns an alert for each event SWWAF_ALERT_EVENTS
+// names, the first in observe mode, each with its message.
+func anAlertForEachEvent() []eventMessage {
+	return []eventMessage{
+		{
+			alerts.Alert{
+				Event: alerts.EventBan, Client: netip.MustParseAddr("203.0.113.9"),
+				Netblock: netip.MustParsePrefix("203.0.113.0/24"), Country: "DE",
+				Reason: "requests per hour over the limit of 10000",
+				Detail: map[string]any{"mode": "observe"},
+			},
+			"fsn1app1/gitea: ban", "default no_entry",
+			"requests per hour over the limit of 10000\nclient: 203.0.113.9\n" +
+				"netblock: 203.0.113.0/24\ncountry: DE\nmode: observe",
+		},
+		{
+			alerts.Alert{
+				Event: alerts.EventPermanentBan, Client: netip.MustParseAddr("198.51.100.7"),
+				Netblock: netip.MustParsePrefix("198.51.100.7/32"), Country: "FR",
+				Reason: "matched the rule env-file",
+			},
+			"fsn1app1/gitea: permanent_ban", "high no_entry",
+			"matched the rule env-file\nclient: 198.51.100.7\n" +
+				"netblock: 198.51.100.7/32\ncountry: FR",
+		},
+		{
+			alerts.Alert{
+				Event: alerts.EventWAFBlock, Client: netip.MustParseAddr("192.0.2.1"),
+				Netblock: netip.MustParsePrefix("192.0.2.1/32"),
+				Reason:   "refused by the Core Rule Set",
+			},
+			"fsn1app1/gitea: waf_block", "default shield",
+			"refused by the Core Rule Set\nclient: 192.0.2.1\nnetblock: 192.0.2.1/32",
+		},
+		{
+			alerts.Alert{
+				Event: alerts.EventAnomaly, Netblock: netip.MustParsePrefix("192.0.2.0/24"),
+				Reason: "requests per minute over the threshold of 5000",
+			},
+			"fsn1app1/gitea: anomaly", "high chart_with_upwards_trend",
+			"requests per minute over the threshold of 5000\nnetblock: 192.0.2.0/24",
+		},
+		{
+			alerts.Alert{
+				Event: alerts.EventReputationHit, Client: netip.MustParseAddr("192.0.2.2"),
+				Netblock: netip.MustParsePrefix("192.0.2.2/32"),
+				Reason:   "listed by a DNS blocklist",
+			},
+			"fsn1app1/gitea: reputation_hit", "low label",
+			"listed by a DNS blocklist\nclient: 192.0.2.2\nnetblock: 192.0.2.2/32",
+		},
+		{
+			alerts.Alert{
+				Event: alerts.EventSourceFailure, Reason: "asking GeoJS failed",
+				Detail: map[string]any{"source": "geojs"},
+			},
+			"fsn1app1/gitea: source_failure", "high warning",
+			"asking GeoJS failed\nsource: geojs",
+		},
+		{
+			alerts.Alert{
+				Event:  alerts.EventFileError,
+				Reason: "a rule file has an error, and the rules stay as they were",
+				Detail: map[string]any{
+					"file": "/etc/smallwebwaf/rules.d/50-app.rules", "error": "line 2: no action",
+				},
+			},
+			"fsn1app1/gitea: file_error", "high warning",
+			"a rule file has an error, and the rules stay as they were\n" +
+				"file: /etc/smallwebwaf/rules.d/50-app.rules\nerror: line 2: no action",
+		},
+	}
+}
+
+// wantSlackMessage checks that request, one Slack was sent, posted the
+// message text, as JSON.
+func wantSlackMessage(t *testing.T, request post, text string) {
+	t.Helper()
+
+	if request.method != http.MethodPost || request.url != slackURL ||
+		request.header.Get("Content-Type") != "application/json" ||
+		!reflect.DeepEqual(request.alert, map[string]any{"text": text}) {
+		t.Errorf("Slack was sent %s %s, Content-Type %q, %s, want POST %s, "+
+			"application/json, the text %q", request.method, request.url,
+			request.header.Get("Content-Type"), request.body, slackURL, text)
+	}
+}
+
+// wantNtfyMessage checks that request, one ntfy was sent, posted the
+// message text with the title, and with the priority and the tag
+// priorityAndTag gives, with a space between them.
+func wantNtfyMessage(t *testing.T, request post, title, priorityAndTag, text string) {
+	t.Helper()
+
+	got := []string{
+		request.method, request.url, request.header.Get("Title"),
+		request.header.Get("Priority") + " " + request.header.Get("Tags"), request.body,
+	}
+
+	want := []string{http.MethodPost, ntfyURL, title, priorityAndTag, text}
+	if !slices.Equal(got, want) {
+		t.Errorf("ntfy was sent the method, URL, title, priority and tag, and text "+
+			"%q, want %q", got, want)
+	}
+}
+
+// wantBodies checks the bodies of the requests the stand-in was sent, in
+// order.
+func wantBodies(t *testing.T, s *standIn, want ...string) {
+	t.Helper()
+
+	got := make([]string, 0, len(want))
+	for _, request := range s.received() {
+		got = append(got, request.body)
+	}
+
+	if !slices.Equal(got, want) {
+		t.Errorf("bodies %q, want %q", got, want)
 	}
 }