Alerts to Slack and ntfy, each destination with its own queue (closes #90)

Each alert is posted as a message to the Slack incoming webhook
SWWAF_ALERT_SLACK_WEBHOOK_URL names, and published to the ntfy topic
SWWAF_ALERT_NTFY_URL names, with SWWAF_ALERT_NTFY_TOKEN as a bearer
token and a priority and tag by event. The cooldown and the hourly
limit stay shared; past them, each destination has its own bounded
queue and backoff, and its own sent, failed and dropped counts.
alerts.json keeps the alerts waiting by destination.

Judgement call: messages also give the detail's file, source, error and mode.
Judgement call: alerts_suppressed_total is the same for every destination.
Judgement call: an alerts.json with waiting as a list stops the start.

Model: opus-5-5

internal/config/config_test.go

@@ -66,6 +66,9 @@ const (
 	logRemoteAppName        = "SWWAF_LOG_REMOTE_APP_NAME"
 	alertWebhookURL         = "SWWAF_ALERT_WEBHOOK_URL"
 	alertWebhookHeaders     = "SWWAF_ALERT_WEBHOOK_HEADERS"
+	alertSlackWebhookURL    = "SWWAF_ALERT_SLACK_WEBHOOK_URL"
+	alertNtfyURL            = "SWWAF_ALERT_NTFY_URL"
+	alertNtfyToken          = "SWWAF_ALERT_NTFY_TOKEN" //nolint:gosec // the setting's name
 	alertEvents             = "SWWAF_ALERT_EVENTS"
 	alertCooldown           = "SWWAF_ALERT_COOLDOWN"
 	alertMaxPerHour         = "SWWAF_ALERT_MAX_PER_HOUR"
@@ -498,44 +501,65 @@ func TestAlertSettingsDefaults(t *testing.T) {
 	cfg := fromEnvironment(t, environment{})
 
 	if cfg.AlertWebhookURL != nil || len(cfg.AlertWebhookHeaders) != 0 ||
+		cfg.AlertSlackWebhookURL != nil || cfg.AlertNtfyURL != nil ||
+		cfg.AlertNtfyToken != "" ||
 		strings.Join(cfg.AlertEvents, ",") != defaultAlertEvents ||
 		cfg.AlertCooldown != 15*time.Minute || cfg.AlertMaxPerHour != 60 {
-		t.Errorf("alert settings %v, %v, %v, %s and %d, want no URL, no headers, "+
-			"%s, 15m and 60", cfg.AlertWebhookURL, cfg.AlertWebhookHeaders,
-			cfg.AlertEvents, cfg.AlertCooldown, cfg.AlertMaxPerHour, defaultAlertEvents)
+		t.Errorf("alert settings %v, %v, %v, %v, %q, %v, %s and %d, want no URLs, "+
+			"no headers, no token, %s, 15m and 60", cfg.AlertWebhookURL,
+			cfg.AlertWebhookHeaders, cfg.AlertSlackWebhookURL, cfg.AlertNtfyURL,
+			cfg.AlertNtfyToken, cfg.AlertEvents, cfg.AlertCooldown, cfg.AlertMaxPerHour,
+			defaultAlertEvents)
 	}
 }
 
 func TestAlertSettingsAsSet(t *testing.T) {
 	t.Parallel()
 
-	const webhook = "https://alerts.example:8443/hooks/waf?team=ops"
+	const (
+		webhook = "https://alerts.example:8443/hooks/waf?team=ops"
+		slack   = "https://hooks.slack.example/services/T0123/B4567/abcdef"
+		ntfy    = "https://ntfy.example/smallwebwaf-alerts"
+		token   = "tk_0123456789abcdefghijklmnopq"
+	)
 
 	cfg := fromEnvironment(t, environment{
-		alertWebhookURL:     webhook,
-		alertWebhookHeaders: "Authorization: Bearer abc:def , x-team:ops",
-		alertEvents:         "ban, file_error",
-		alertCooldown:       "1h",
-		alertMaxPerHour:     "10",
+		alertWebhookURL:      webhook,
+		alertWebhookHeaders:  "Authorization: Bearer abc:def , x-team:ops",
+		alertSlackWebhookURL: slack,
+		alertNtfyURL:         ntfy,
+		alertNtfyToken:       token,
+		alertEvents:          "ban, file_error",
+		alertCooldown:        "1h",
+		alertMaxPerHour:      "10",
 	})
 
 	headers := http.Header{"Authorization": {"Bearer abc:def"}, "X-Team": {"ops"}}
 	if cfg.AlertWebhookURL.String() != webhook ||
 		!reflect.DeepEqual(cfg.AlertWebhookHeaders, headers) ||
+		cfg.AlertSlackWebhookURL.String() != slack || cfg.AlertNtfyURL.String() != ntfy ||
+		cfg.AlertNtfyToken != token ||
 		!slices.Equal(cfg.AlertEvents, []string{"ban", "file_error"}) ||
 		cfg.AlertCooldown != time.Hour || cfg.AlertMaxPerHour != 10 {
-		t.Errorf("alert settings %v, %v, %v, %s and %d", cfg.AlertWebhookURL,
-			cfg.AlertWebhookHeaders, cfg.AlertEvents, cfg.AlertCooldown,
-			cfg.AlertMaxPerHour)
+		t.Errorf("alert settings %v, %v, %v, %v, %q, %v, %s and %d", cfg.AlertWebhookURL,
+			cfg.AlertWebhookHeaders, cfg.AlertSlackWebhookURL, cfg.AlertNtfyURL,
+			cfg.AlertNtfyToken, cfg.AlertEvents, cfg.AlertCooldown, cfg.AlertMaxPerHour)
 	}
+}
 
-	cfg = fromEnvironment(t, environment{
-		alertWebhookURL: "", alertEvents: "", alertCooldown: off, alertMaxPerHour: off,
+func TestAlertSettingsSetEmptyOrOff(t *testing.T) {
+	t.Parallel()
+
+	cfg := fromEnvironment(t, environment{
+		alertWebhookURL: "", alertSlackWebhookURL: "", alertNtfyURL: "",
+		alertNtfyToken: "", alertEvents: "", alertCooldown: off, alertMaxPerHour: off,
 	})
-	if cfg.AlertWebhookURL != nil || len(cfg.AlertEvents) != 0 ||
-		cfg.AlertCooldown != 0 || cfg.AlertMaxPerHour != 0 {
-		t.Errorf("set empty or off, alert settings %v, %v, %s and %d",
-			cfg.AlertWebhookURL, cfg.AlertEvents, cfg.AlertCooldown, cfg.AlertMaxPerHour)
+	if cfg.AlertWebhookURL != nil || cfg.AlertSlackWebhookURL != nil ||
+		cfg.AlertNtfyURL != nil || cfg.AlertNtfyToken != "" ||
+		len(cfg.AlertEvents) != 0 || cfg.AlertCooldown != 0 || cfg.AlertMaxPerHour != 0 {
+		t.Errorf("set empty or off, alert settings %v, %v, %v, %q, %v, %s and %d",
+			cfg.AlertWebhookURL, cfg.AlertSlackWebhookURL, cfg.AlertNtfyURL,
+			cfg.AlertNtfyToken, cfg.AlertEvents, cfg.AlertCooldown, cfg.AlertMaxPerHour)
 	}
 }
 
@@ -550,6 +574,10 @@ func TestInvalidAlertSettingStopsTheStart(t *testing.T) {
 		{alertWebhookURL, "https://alerts.example/#top"},
 		{alertWebhookURL, "https://alerts.example:0/"},
 		{alertWebhookURL, "https://alerts.example:65536/"},
+		{alertSlackWebhookURL, "hooks.slack.example/services/T0123"},
+		{alertSlackWebhookURL, "https://user:password@hooks.slack.example/"},
+		{alertNtfyURL, "ntfy://ntfy.example/smallwebwaf-alerts"},
+		{alertNtfyURL, "https://ntfy.example/smallwebwaf-alerts#top"},
 		{alertWebhookHeaders, "Authorization"},
 		{alertWebhookHeaders, "X Team:ops"},
 		{alertWebhookHeaders, ":ops"},
@@ -636,6 +664,39 @@ func TestWebhookURLIsLoggedWithoutItsPathOrQueryAndNeverShown(t *testing.T) {
 	}
 }
 
+func TestSlackAndNtfySettingsAreLoggedWithoutTheirSecrets(t *testing.T) {
+	t.Parallel()
+
+	const token = "tk_0123456789abcdefghijklmnopq"
+
+	cfg := fromEnvironment(t, environment{
+		alertSlackWebhookURL: "https://hooks.slack.example/services/T0123/B4567/abcdef",
+		alertNtfyURL:         "https://ntfy.example/smallwebwaf-alerts",
+		alertNtfyToken:       token,
+	})
+
+	var out bytes.Buffer
+
+	slog.New(slog.NewJSONHandler(&out, nil)).Info("starting", "settings", cfg)
+
+	logged := out.String()
+	for _, want := range []string{
+		`"` + alertSlackWebhookURL + `":"https://hooks.slack.example/********"`,
+		`"` + alertNtfyURL + `":"https://ntfy.example/********"`,
+		`"` + alertNtfyToken + `":"********"`,
+	} {
+		if !strings.Contains(logged, want) {
+			t.Errorf("no %s in the settings logged: %s", want, logged)
+		}
+	}
+
+	for _, secret := range []string{"T0123", "smallwebwaf-alerts", token} {
+		if strings.Contains(logged, secret) {
+			t.Errorf("%s in the settings logged: %s", secret, logged)
+		}
+	}
+}
+
 func TestCodeOnBothCountryListsStopsTheStart(t *testing.T) {
 	t.Parallel()
 
@@ -1087,6 +1148,9 @@ func TestLogsEachSettingWithItsValue(t *testing.T) {
 		logRemoteAppName:        hostname,
 		alertWebhookURL:         "",
 		alertWebhookHeaders:     "",
+		alertSlackWebhookURL:    "",
+		alertNtfyURL:            "",
+		alertNtfyToken:          "",
 		alertEvents:             defaultAlertEvents,
 		alertCooldown:           defaultAlertCooldown,
 		alertMaxPerHour:         "60",