Alerts to Slack and ntfy, each destination with its own queue (closes #90)

Each alert is posted as a message to the Slack incoming webhook
SWWAF_ALERT_SLACK_WEBHOOK_URL names, and published to the ntfy topic
SWWAF_ALERT_NTFY_URL names, with SWWAF_ALERT_NTFY_TOKEN as a bearer
token and a priority and tag by event. The cooldown and the hourly
limit stay shared; past them, each destination has its own bounded
queue and backoff, and its own sent, failed and dropped counts.
alerts.json keeps the alerts waiting by destination.

Judgement call: messages also give the detail's file, source, error and mode.
Judgement call: alerts_suppressed_total is the same for every destination.
Judgement call: an alerts.json with waiting as a list stops the start.

Model: opus-5-5

internal/config/config.go

@@ -160,18 +160,26 @@ type Config struct {
 	LogRemoteFacility int
 	LogRemoteAppName  string
 	// AlertWebhookURL is where each alert is posted as JSON
-	// (SWWAF_ALERT_WEBHOOK_URL), nil while it is unset and no alert is
-	// sent. AlertWebhookHeaders are sent with each
-	// (SWWAF_ALERT_WEBHOOK_HEADERS). AlertEvents are the events alerts are
-	// sent for (SWWAF_ALERT_EVENTS). A repeat of an alert within
-	// AlertCooldown is held back (SWWAF_ALERT_COOLDOWN), and so is an alert
-	// past AlertMaxPerHour in an hour, for the hour's summary
-	// (SWWAF_ALERT_MAX_PER_HOUR); 0 is off for both.
-	AlertWebhookURL     *url.URL
-	AlertWebhookHeaders http.Header
-	AlertEvents         []string
-	AlertCooldown       time.Duration
-	AlertMaxPerHour     int
+	// (SWWAF_ALERT_WEBHOOK_URL), nil while it is unset. AlertWebhookHeaders
+	// are sent with each (SWWAF_ALERT_WEBHOOK_HEADERS).
+	// AlertSlackWebhookURL is the Slack incoming webhook each alert is
+	// posted to as a message (SWWAF_ALERT_SLACK_WEBHOOK_URL), and
+	// AlertNtfyURL the ntfy topic each is published to
+	// (SWWAF_ALERT_NTFY_URL), each nil while it is unset; AlertNtfyToken,
+	// unless empty, is sent to ntfy with each (SWWAF_ALERT_NTFY_TOKEN).
+	// With none of the three URLs set, no alert is sent. AlertEvents are
+	// the events alerts are sent for (SWWAF_ALERT_EVENTS). A repeat of an
+	// alert within AlertCooldown is held back (SWWAF_ALERT_COOLDOWN), and
+	// so is an alert past AlertMaxPerHour in an hour, for the hour's
+	// summary (SWWAF_ALERT_MAX_PER_HOUR); 0 is off for both.
+	AlertWebhookURL      *url.URL
+	AlertWebhookHeaders  http.Header
+	AlertSlackWebhookURL *url.URL
+	AlertNtfyURL         *url.URL
+	AlertNtfyToken       string
+	AlertEvents          []string
+	AlertCooldown        time.Duration
+	AlertMaxPerHour      int
 
 	// settings are the values read, as given or by default, and the
 	// files they were read from, for the log line at start.
@@ -303,17 +311,20 @@ func FromEnvironment(lookupEnv func(string) (string, bool)) (*Config, error) {
 		StateCounterInterval: env.durationNotOff("SWWAF_STATE_COUNTER_INTERVAL", "15m"),
 		LogRequestHeaders: env.headerNames("SWWAF_LOG_REQUEST_HEADERS",
 			"accept,accept-language,accept-encoding,content-type,origin,range"),
-		AdminToken:          env.token("SWWAF_ADMIN_TOKEN"),
-		MetricsToken:        env.token("SWWAF_METRICS_TOKEN"),
-		MetricsTopN:         env.numberNotOff("SWWAF_METRICS_TOP_N", "50"),
-		RulesDir:            env.value("SWWAF_RULES_DIR", "/etc/smallwebwaf/rules.d"),
-		RulesEnabled:        env.boolean("SWWAF_RULES_ENABLED", "true"),
-		LogRemoteURL:        env.logRemoteURL("SWWAF_LOG_REMOTE_URL"),
-		LogRemoteTLSCAs:     env.certificates("SWWAF_LOG_REMOTE_TLS_CA_FILE"),
-		LogRemoteBuffer:     env.numberNotOff("SWWAF_LOG_REMOTE_BUFFER", "10000"),
-		LogRemoteFacility:   env.facility("SWWAF_LOG_REMOTE_FACILITY", "local0"),
-		AlertWebhookURL:     env.webhookURL("SWWAF_ALERT_WEBHOOK_URL"),
-		AlertWebhookHeaders: env.webhookHeaders("SWWAF_ALERT_WEBHOOK_HEADERS"),
+		AdminToken:           env.token("SWWAF_ADMIN_TOKEN"),
+		MetricsToken:         env.token("SWWAF_METRICS_TOKEN"),
+		MetricsTopN:          env.numberNotOff("SWWAF_METRICS_TOP_N", "50"),
+		RulesDir:             env.value("SWWAF_RULES_DIR", "/etc/smallwebwaf/rules.d"),
+		RulesEnabled:         env.boolean("SWWAF_RULES_ENABLED", "true"),
+		LogRemoteURL:         env.logRemoteURL("SWWAF_LOG_REMOTE_URL"),
+		LogRemoteTLSCAs:      env.certificates("SWWAF_LOG_REMOTE_TLS_CA_FILE"),
+		LogRemoteBuffer:      env.numberNotOff("SWWAF_LOG_REMOTE_BUFFER", "10000"),
+		LogRemoteFacility:    env.facility("SWWAF_LOG_REMOTE_FACILITY", "local0"),
+		AlertWebhookURL:      env.webhookURL("SWWAF_ALERT_WEBHOOK_URL"),
+		AlertWebhookHeaders:  env.webhookHeaders("SWWAF_ALERT_WEBHOOK_HEADERS"),
+		AlertSlackWebhookURL: env.webhookURL("SWWAF_ALERT_SLACK_WEBHOOK_URL"),
+		AlertNtfyURL:         env.webhookURL("SWWAF_ALERT_NTFY_URL"),
+		AlertNtfyToken:       env.secret("SWWAF_ALERT_NTFY_TOKEN"),
 		AlertEvents: env.alertEvents("SWWAF_ALERT_EVENTS",
 			strings.Join(alerts.Events(), ",")),
 		AlertCooldown:   env.duration("SWWAF_ALERT_COOLDOWN", "15m"),
@@ -667,10 +678,12 @@ func (e *environment) appName(name, instanceName string, sending bool) string {
 	return value
 }
 
-// webhookURL reads the setting that is where each alert is posted. Unset
-// or empty, it is nil, and no alert is sent. The log shows ******** in
-// place of its path and query, and an error shows none of it, since many
-// webhooks carry their secret there.
+// webhookURL reads a setting that is a URL each alert is posted to:
+// SWWAF_ALERT_WEBHOOK_URL, SWWAF_ALERT_SLACK_WEBHOOK_URL or
+// SWWAF_ALERT_NTFY_URL. Unset or empty, it is nil, and no alert is posted
+// there. The log shows ******** in place of its path and query, and an
+// error shows none of it, since a webhook or an ntfy topic can carry its
+// secret there.
 func (e *environment) webhookURL(name string) *url.URL {
 	value, _ := e.lookup(name)
 	webhook, logged, err := parseWebhookURL(value)
@@ -692,6 +705,22 @@ func (e *environment) webhookHeaders(name string) http.Header {
 	return headers
 }
 
+// secret reads a setting that is a secret another service gave, such as
+// an ntfy token, "" while it is unset. The log shows ******** in place of
+// a value that is not empty.
+func (e *environment) secret(name string) string {
+	value, _ := e.lookup(name)
+
+	logged := ""
+	if value != "" {
+		logged = masked
+	}
+
+	e.settings = append(e.settings, slog.String(name, logged))
+
+	return value
+}
+
 // alertEvents reads the setting that is the events alerts are sent for.
 func (e *environment) alertEvents(name, defaultValue string) []string {
 	events, err := parseAlertEvents(e.value(name, defaultValue))

internal/config/config_test.go

@@ -66,6 +66,9 @@ const (
 	logRemoteAppName        = "SWWAF_LOG_REMOTE_APP_NAME"
 	alertWebhookURL         = "SWWAF_ALERT_WEBHOOK_URL"
 	alertWebhookHeaders     = "SWWAF_ALERT_WEBHOOK_HEADERS"
+	alertSlackWebhookURL    = "SWWAF_ALERT_SLACK_WEBHOOK_URL"
+	alertNtfyURL            = "SWWAF_ALERT_NTFY_URL"
+	alertNtfyToken          = "SWWAF_ALERT_NTFY_TOKEN" //nolint:gosec // the setting's name
 	alertEvents             = "SWWAF_ALERT_EVENTS"
 	alertCooldown           = "SWWAF_ALERT_COOLDOWN"
 	alertMaxPerHour         = "SWWAF_ALERT_MAX_PER_HOUR"
@@ -498,44 +501,65 @@ func TestAlertSettingsDefaults(t *testing.T) {
 	cfg := fromEnvironment(t, environment{})
 
 	if cfg.AlertWebhookURL != nil || len(cfg.AlertWebhookHeaders) != 0 ||
+		cfg.AlertSlackWebhookURL != nil || cfg.AlertNtfyURL != nil ||
+		cfg.AlertNtfyToken != "" ||
 		strings.Join(cfg.AlertEvents, ",") != defaultAlertEvents ||
 		cfg.AlertCooldown != 15*time.Minute || cfg.AlertMaxPerHour != 60 {
-		t.Errorf("alert settings %v, %v, %v, %s and %d, want no URL, no headers, "+
-			"%s, 15m and 60", cfg.AlertWebhookURL, cfg.AlertWebhookHeaders,
-			cfg.AlertEvents, cfg.AlertCooldown, cfg.AlertMaxPerHour, defaultAlertEvents)
+		t.Errorf("alert settings %v, %v, %v, %v, %q, %v, %s and %d, want no URLs, "+
+			"no headers, no token, %s, 15m and 60", cfg.AlertWebhookURL,
+			cfg.AlertWebhookHeaders, cfg.AlertSlackWebhookURL, cfg.AlertNtfyURL,
+			cfg.AlertNtfyToken, cfg.AlertEvents, cfg.AlertCooldown, cfg.AlertMaxPerHour,
+			defaultAlertEvents)
 	}
 }
 
 func TestAlertSettingsAsSet(t *testing.T) {
 	t.Parallel()
 
-	const webhook = "https://alerts.example:8443/hooks/waf?team=ops"
+	const (
+		webhook = "https://alerts.example:8443/hooks/waf?team=ops"
+		slack   = "https://hooks.slack.example/services/T0123/B4567/abcdef"
+		ntfy    = "https://ntfy.example/smallwebwaf-alerts"
+		token   = "tk_0123456789abcdefghijklmnopq"
+	)
 
 	cfg := fromEnvironment(t, environment{
-		alertWebhookURL:     webhook,
-		alertWebhookHeaders: "Authorization: Bearer abc:def , x-team:ops",
-		alertEvents:         "ban, file_error",
-		alertCooldown:       "1h",
-		alertMaxPerHour:     "10",
+		alertWebhookURL:      webhook,
+		alertWebhookHeaders:  "Authorization: Bearer abc:def , x-team:ops",
+		alertSlackWebhookURL: slack,
+		alertNtfyURL:         ntfy,
+		alertNtfyToken:       token,
+		alertEvents:          "ban, file_error",
+		alertCooldown:        "1h",
+		alertMaxPerHour:      "10",
 	})
 
 	headers := http.Header{"Authorization": {"Bearer abc:def"}, "X-Team": {"ops"}}
 	if cfg.AlertWebhookURL.String() != webhook ||
 		!reflect.DeepEqual(cfg.AlertWebhookHeaders, headers) ||
+		cfg.AlertSlackWebhookURL.String() != slack || cfg.AlertNtfyURL.String() != ntfy ||
+		cfg.AlertNtfyToken != token ||
 		!slices.Equal(cfg.AlertEvents, []string{"ban", "file_error"}) ||
 		cfg.AlertCooldown != time.Hour || cfg.AlertMaxPerHour != 10 {
-		t.Errorf("alert settings %v, %v, %v, %s and %d", cfg.AlertWebhookURL,
-			cfg.AlertWebhookHeaders, cfg.AlertEvents, cfg.AlertCooldown,
-			cfg.AlertMaxPerHour)
+		t.Errorf("alert settings %v, %v, %v, %v, %q, %v, %s and %d", cfg.AlertWebhookURL,
+			cfg.AlertWebhookHeaders, cfg.AlertSlackWebhookURL, cfg.AlertNtfyURL,
+			cfg.AlertNtfyToken, cfg.AlertEvents, cfg.AlertCooldown, cfg.AlertMaxPerHour)
 	}
+}
 
-	cfg = fromEnvironment(t, environment{
-		alertWebhookURL: "", alertEvents: "", alertCooldown: off, alertMaxPerHour: off,
+func TestAlertSettingsSetEmptyOrOff(t *testing.T) {
+	t.Parallel()
+
+	cfg := fromEnvironment(t, environment{
+		alertWebhookURL: "", alertSlackWebhookURL: "", alertNtfyURL: "",
+		alertNtfyToken: "", alertEvents: "", alertCooldown: off, alertMaxPerHour: off,
 	})
-	if cfg.AlertWebhookURL != nil || len(cfg.AlertEvents) != 0 ||
-		cfg.AlertCooldown != 0 || cfg.AlertMaxPerHour != 0 {
-		t.Errorf("set empty or off, alert settings %v, %v, %s and %d",
-			cfg.AlertWebhookURL, cfg.AlertEvents, cfg.AlertCooldown, cfg.AlertMaxPerHour)
+	if cfg.AlertWebhookURL != nil || cfg.AlertSlackWebhookURL != nil ||
+		cfg.AlertNtfyURL != nil || cfg.AlertNtfyToken != "" ||
+		len(cfg.AlertEvents) != 0 || cfg.AlertCooldown != 0 || cfg.AlertMaxPerHour != 0 {
+		t.Errorf("set empty or off, alert settings %v, %v, %v, %q, %v, %s and %d",
+			cfg.AlertWebhookURL, cfg.AlertSlackWebhookURL, cfg.AlertNtfyURL,
+			cfg.AlertNtfyToken, cfg.AlertEvents, cfg.AlertCooldown, cfg.AlertMaxPerHour)
 	}
 }
 
@@ -550,6 +574,10 @@ func TestInvalidAlertSettingStopsTheStart(t *testing.T) {
 		{alertWebhookURL, "https://alerts.example/#top"},
 		{alertWebhookURL, "https://alerts.example:0/"},
 		{alertWebhookURL, "https://alerts.example:65536/"},
+		{alertSlackWebhookURL, "hooks.slack.example/services/T0123"},
+		{alertSlackWebhookURL, "https://user:password@hooks.slack.example/"},
+		{alertNtfyURL, "ntfy://ntfy.example/smallwebwaf-alerts"},
+		{alertNtfyURL, "https://ntfy.example/smallwebwaf-alerts#top"},
 		{alertWebhookHeaders, "Authorization"},
 		{alertWebhookHeaders, "X Team:ops"},
 		{alertWebhookHeaders, ":ops"},
@@ -636,6 +664,39 @@ func TestWebhookURLIsLoggedWithoutItsPathOrQueryAndNeverShown(t *testing.T) {
 	}
 }
 
+func TestSlackAndNtfySettingsAreLoggedWithoutTheirSecrets(t *testing.T) {
+	t.Parallel()
+
+	const token = "tk_0123456789abcdefghijklmnopq"
+
+	cfg := fromEnvironment(t, environment{
+		alertSlackWebhookURL: "https://hooks.slack.example/services/T0123/B4567/abcdef",
+		alertNtfyURL:         "https://ntfy.example/smallwebwaf-alerts",
+		alertNtfyToken:       token,
+	})
+
+	var out bytes.Buffer
+
+	slog.New(slog.NewJSONHandler(&out, nil)).Info("starting", "settings", cfg)
+
+	logged := out.String()
+	for _, want := range []string{
+		`"` + alertSlackWebhookURL + `":"https://hooks.slack.example/********"`,
+		`"` + alertNtfyURL + `":"https://ntfy.example/********"`,
+		`"` + alertNtfyToken + `":"********"`,
+	} {
+		if !strings.Contains(logged, want) {
+			t.Errorf("no %s in the settings logged: %s", want, logged)
+		}
+	}
+
+	for _, secret := range []string{"T0123", "smallwebwaf-alerts", token} {
+		if strings.Contains(logged, secret) {
+			t.Errorf("%s in the settings logged: %s", secret, logged)
+		}
+	}
+}
+
 func TestCodeOnBothCountryListsStopsTheStart(t *testing.T) {
 	t.Parallel()
 
@@ -1087,6 +1148,9 @@ func TestLogsEachSettingWithItsValue(t *testing.T) {
 		logRemoteAppName:        hostname,
 		alertWebhookURL:         "",
 		alertWebhookHeaders:     "",
+		alertSlackWebhookURL:    "",
+		alertNtfyURL:            "",
+		alertNtfyToken:          "",
 		alertEvents:             defaultAlertEvents,
 		alertCooldown:           defaultAlertCooldown,
 		alertMaxPerHour:         "60",