Alerts to Slack and ntfy, each destination with its own queue (closes #90)

Each alert is posted as a message to the Slack incoming webhook
SWWAF_ALERT_SLACK_WEBHOOK_URL names, and published to the ntfy topic
SWWAF_ALERT_NTFY_URL names, with SWWAF_ALERT_NTFY_TOKEN as a bearer
token and a priority and tag by event. The cooldown and the hourly
limit stay shared; past them, each destination has its own bounded
queue and backoff, and its own sent, failed and dropped counts.
alerts.json keeps the alerts waiting by destination.

Judgement call: messages also give the detail's file, source, error and mode.
Judgement call: alerts_suppressed_total is the same for every destination.
Judgement call: an alerts.json with waiting as a list stops the start.

Model: opus-5-5

internal/config/config.go

@@ -160,18 +160,26 @@ type Config struct {
 	LogRemoteFacility int
 	LogRemoteAppName  string
 	// AlertWebhookURL is where each alert is posted as JSON
-	// (SWWAF_ALERT_WEBHOOK_URL), nil while it is unset and no alert is
-	// sent. AlertWebhookHeaders are sent with each
-	// (SWWAF_ALERT_WEBHOOK_HEADERS). AlertEvents are the events alerts are
-	// sent for (SWWAF_ALERT_EVENTS). A repeat of an alert within
-	// AlertCooldown is held back (SWWAF_ALERT_COOLDOWN), and so is an alert
-	// past AlertMaxPerHour in an hour, for the hour's summary
-	// (SWWAF_ALERT_MAX_PER_HOUR); 0 is off for both.
-	AlertWebhookURL     *url.URL
-	AlertWebhookHeaders http.Header
-	AlertEvents         []string
-	AlertCooldown       time.Duration
-	AlertMaxPerHour     int
+	// (SWWAF_ALERT_WEBHOOK_URL), nil while it is unset. AlertWebhookHeaders
+	// are sent with each (SWWAF_ALERT_WEBHOOK_HEADERS).
+	// AlertSlackWebhookURL is the Slack incoming webhook each alert is
+	// posted to as a message (SWWAF_ALERT_SLACK_WEBHOOK_URL), and
+	// AlertNtfyURL the ntfy topic each is published to
+	// (SWWAF_ALERT_NTFY_URL), each nil while it is unset; AlertNtfyToken,
+	// unless empty, is sent to ntfy with each (SWWAF_ALERT_NTFY_TOKEN).
+	// With none of the three URLs set, no alert is sent. AlertEvents are
+	// the events alerts are sent for (SWWAF_ALERT_EVENTS). A repeat of an
+	// alert within AlertCooldown is held back (SWWAF_ALERT_COOLDOWN), and
+	// so is an alert past AlertMaxPerHour in an hour, for the hour's
+	// summary (SWWAF_ALERT_MAX_PER_HOUR); 0 is off for both.
+	AlertWebhookURL      *url.URL
+	AlertWebhookHeaders  http.Header
+	AlertSlackWebhookURL *url.URL
+	AlertNtfyURL         *url.URL
+	AlertNtfyToken       string
+	AlertEvents          []string
+	AlertCooldown        time.Duration
+	AlertMaxPerHour      int
 
 	// settings are the values read, as given or by default, and the
 	// files they were read from, for the log line at start.
@@ -303,17 +311,20 @@ func FromEnvironment(lookupEnv func(string) (string, bool)) (*Config, error) {
 		StateCounterInterval: env.durationNotOff("SWWAF_STATE_COUNTER_INTERVAL", "15m"),
 		LogRequestHeaders: env.headerNames("SWWAF_LOG_REQUEST_HEADERS",
 			"accept,accept-language,accept-encoding,content-type,origin,range"),
-		AdminToken:          env.token("SWWAF_ADMIN_TOKEN"),
-		MetricsToken:        env.token("SWWAF_METRICS_TOKEN"),
-		MetricsTopN:         env.numberNotOff("SWWAF_METRICS_TOP_N", "50"),
-		RulesDir:            env.value("SWWAF_RULES_DIR", "/etc/smallwebwaf/rules.d"),
-		RulesEnabled:        env.boolean("SWWAF_RULES_ENABLED", "true"),
-		LogRemoteURL:        env.logRemoteURL("SWWAF_LOG_REMOTE_URL"),
-		LogRemoteTLSCAs:     env.certificates("SWWAF_LOG_REMOTE_TLS_CA_FILE"),
-		LogRemoteBuffer:     env.numberNotOff("SWWAF_LOG_REMOTE_BUFFER", "10000"),
-		LogRemoteFacility:   env.facility("SWWAF_LOG_REMOTE_FACILITY", "local0"),
-		AlertWebhookURL:     env.webhookURL("SWWAF_ALERT_WEBHOOK_URL"),
-		AlertWebhookHeaders: env.webhookHeaders("SWWAF_ALERT_WEBHOOK_HEADERS"),
+		AdminToken:           env.token("SWWAF_ADMIN_TOKEN"),
+		MetricsToken:         env.token("SWWAF_METRICS_TOKEN"),
+		MetricsTopN:          env.numberNotOff("SWWAF_METRICS_TOP_N", "50"),
+		RulesDir:             env.value("SWWAF_RULES_DIR", "/etc/smallwebwaf/rules.d"),
+		RulesEnabled:         env.boolean("SWWAF_RULES_ENABLED", "true"),
+		LogRemoteURL:         env.logRemoteURL("SWWAF_LOG_REMOTE_URL"),
+		LogRemoteTLSCAs:      env.certificates("SWWAF_LOG_REMOTE_TLS_CA_FILE"),
+		LogRemoteBuffer:      env.numberNotOff("SWWAF_LOG_REMOTE_BUFFER", "10000"),
+		LogRemoteFacility:    env.facility("SWWAF_LOG_REMOTE_FACILITY", "local0"),
+		AlertWebhookURL:      env.webhookURL("SWWAF_ALERT_WEBHOOK_URL"),
+		AlertWebhookHeaders:  env.webhookHeaders("SWWAF_ALERT_WEBHOOK_HEADERS"),
+		AlertSlackWebhookURL: env.webhookURL("SWWAF_ALERT_SLACK_WEBHOOK_URL"),
+		AlertNtfyURL:         env.webhookURL("SWWAF_ALERT_NTFY_URL"),
+		AlertNtfyToken:       env.secret("SWWAF_ALERT_NTFY_TOKEN"),
 		AlertEvents: env.alertEvents("SWWAF_ALERT_EVENTS",
 			strings.Join(alerts.Events(), ",")),
 		AlertCooldown:   env.duration("SWWAF_ALERT_COOLDOWN", "15m"),
@@ -667,10 +678,12 @@ func (e *environment) appName(name, instanceName string, sending bool) string {
 	return value
 }
 
-// webhookURL reads the setting that is where each alert is posted. Unset
-// or empty, it is nil, and no alert is sent. The log shows ******** in
-// place of its path and query, and an error shows none of it, since many
-// webhooks carry their secret there.
+// webhookURL reads a setting that is a URL each alert is posted to:
+// SWWAF_ALERT_WEBHOOK_URL, SWWAF_ALERT_SLACK_WEBHOOK_URL or
+// SWWAF_ALERT_NTFY_URL. Unset or empty, it is nil, and no alert is posted
+// there. The log shows ******** in place of its path and query, and an
+// error shows none of it, since a webhook or an ntfy topic can carry its
+// secret there.
 func (e *environment) webhookURL(name string) *url.URL {
 	value, _ := e.lookup(name)
 	webhook, logged, err := parseWebhookURL(value)
@@ -692,6 +705,22 @@ func (e *environment) webhookHeaders(name string) http.Header {
 	return headers
 }
 
+// secret reads a setting that is a secret another service gave, such as
+// an ntfy token, "" while it is unset. The log shows ******** in place of
+// a value that is not empty.
+func (e *environment) secret(name string) string {
+	value, _ := e.lookup(name)
+
+	logged := ""
+	if value != "" {
+		logged = masked
+	}
+
+	e.settings = append(e.settings, slog.String(name, logged))
+
+	return value
+}
+
 // alertEvents reads the setting that is the events alerts are sent for.
 func (e *environment) alertEvents(name, defaultValue string) []string {
 	events, err := parseAlertEvents(e.value(name, defaultValue))