Alerts to Slack and ntfy, each destination with its own queue (closes #90)

Each alert is posted as a message to the Slack incoming webhook
SWWAF_ALERT_SLACK_WEBHOOK_URL names, and published to the ntfy topic
SWWAF_ALERT_NTFY_URL names, with SWWAF_ALERT_NTFY_TOKEN as a bearer
token and a priority and tag by event. The cooldown and the hourly
limit stay shared; past them, each destination has its own bounded
queue and backoff, and its own sent, failed and dropped counts.
alerts.json keeps the alerts waiting by destination; one whose waiting
is still a list stops the start, saying what to change. A control
character in the ntfy token, or in the instance name ntfy is sent,
stops the start.

Judgement call: messages also give the detail's file, source, error and mode.
Judgement call: alerts_suppressed_total is the same for every destination.

Model: opus-5-5

internal/config/config.go

@@ -20,6 +20,7 @@ import (
 	"strconv"
 	"strings"
 	"time"
+	"unicode"
 	"unicode/utf8"
 
 	"sneak.berlin/go/smallwebwaf/internal/alerts"
@@ -160,18 +161,26 @@ type Config struct {
 	LogRemoteFacility int
 	LogRemoteAppName  string
 	// AlertWebhookURL is where each alert is posted as JSON
-	// (SWWAF_ALERT_WEBHOOK_URL), nil while it is unset and no alert is
-	// sent. AlertWebhookHeaders are sent with each
-	// (SWWAF_ALERT_WEBHOOK_HEADERS). AlertEvents are the events alerts are
-	// sent for (SWWAF_ALERT_EVENTS). A repeat of an alert within
-	// AlertCooldown is held back (SWWAF_ALERT_COOLDOWN), and so is an alert
-	// past AlertMaxPerHour in an hour, for the hour's summary
-	// (SWWAF_ALERT_MAX_PER_HOUR); 0 is off for both.
-	AlertWebhookURL     *url.URL
-	AlertWebhookHeaders http.Header
-	AlertEvents         []string
-	AlertCooldown       time.Duration
-	AlertMaxPerHour     int
+	// (SWWAF_ALERT_WEBHOOK_URL), nil while it is unset. AlertWebhookHeaders
+	// are sent with each (SWWAF_ALERT_WEBHOOK_HEADERS).
+	// AlertSlackWebhookURL is the Slack incoming webhook each alert is
+	// posted to as a message (SWWAF_ALERT_SLACK_WEBHOOK_URL), and
+	// AlertNtfyURL the ntfy topic each is published to
+	// (SWWAF_ALERT_NTFY_URL), each nil while it is unset; AlertNtfyToken,
+	// unless empty, is sent to ntfy with each (SWWAF_ALERT_NTFY_TOKEN).
+	// With none of the three URLs set, no alert is sent. AlertEvents are
+	// the events alerts are sent for (SWWAF_ALERT_EVENTS). A repeat of an
+	// alert within AlertCooldown is held back (SWWAF_ALERT_COOLDOWN), and
+	// so is an alert past AlertMaxPerHour in an hour, for the hour's
+	// summary (SWWAF_ALERT_MAX_PER_HOUR); 0 is off for both.
+	AlertWebhookURL      *url.URL
+	AlertWebhookHeaders  http.Header
+	AlertSlackWebhookURL *url.URL
+	AlertNtfyURL         *url.URL
+	AlertNtfyToken       string
+	AlertEvents          []string
+	AlertCooldown        time.Duration
+	AlertMaxPerHour      int
 
 	// settings are the values read, as given or by default, and the
 	// files they were read from, for the log line at start.
@@ -252,6 +261,8 @@ var (
 	errNotWebhookHeader = errors.New(
 		"is not a header name followed by : and the header's value, " +
 			"such as Authorization:Bearer <token>")
+	errControlCharacter = errors.New(
+		"holds a control character, such as the carriage return of a Windows line end")
 	errNotAlertEvent = errors.New(
 		"is not ban, permanent_ban, waf_block, anomaly, reputation_hit, " +
 			"source_failure or file_error")
@@ -303,17 +314,20 @@ func FromEnvironment(lookupEnv func(string) (string, bool)) (*Config, error) {
 		StateCounterInterval: env.durationNotOff("SWWAF_STATE_COUNTER_INTERVAL", "15m"),
 		LogRequestHeaders: env.headerNames("SWWAF_LOG_REQUEST_HEADERS",
 			"accept,accept-language,accept-encoding,content-type,origin,range"),
-		AdminToken:          env.token("SWWAF_ADMIN_TOKEN"),
-		MetricsToken:        env.token("SWWAF_METRICS_TOKEN"),
-		MetricsTopN:         env.numberNotOff("SWWAF_METRICS_TOP_N", "50"),
-		RulesDir:            env.value("SWWAF_RULES_DIR", "/etc/smallwebwaf/rules.d"),
-		RulesEnabled:        env.boolean("SWWAF_RULES_ENABLED", "true"),
-		LogRemoteURL:        env.logRemoteURL("SWWAF_LOG_REMOTE_URL"),
-		LogRemoteTLSCAs:     env.certificates("SWWAF_LOG_REMOTE_TLS_CA_FILE"),
-		LogRemoteBuffer:     env.numberNotOff("SWWAF_LOG_REMOTE_BUFFER", "10000"),
-		LogRemoteFacility:   env.facility("SWWAF_LOG_REMOTE_FACILITY", "local0"),
-		AlertWebhookURL:     env.webhookURL("SWWAF_ALERT_WEBHOOK_URL"),
-		AlertWebhookHeaders: env.webhookHeaders("SWWAF_ALERT_WEBHOOK_HEADERS"),
+		AdminToken:           env.token("SWWAF_ADMIN_TOKEN"),
+		MetricsToken:         env.token("SWWAF_METRICS_TOKEN"),
+		MetricsTopN:          env.numberNotOff("SWWAF_METRICS_TOP_N", "50"),
+		RulesDir:             env.value("SWWAF_RULES_DIR", "/etc/smallwebwaf/rules.d"),
+		RulesEnabled:         env.boolean("SWWAF_RULES_ENABLED", "true"),
+		LogRemoteURL:         env.logRemoteURL("SWWAF_LOG_REMOTE_URL"),
+		LogRemoteTLSCAs:      env.certificates("SWWAF_LOG_REMOTE_TLS_CA_FILE"),
+		LogRemoteBuffer:      env.numberNotOff("SWWAF_LOG_REMOTE_BUFFER", "10000"),
+		LogRemoteFacility:    env.facility("SWWAF_LOG_REMOTE_FACILITY", "local0"),
+		AlertWebhookURL:      env.webhookURL("SWWAF_ALERT_WEBHOOK_URL"),
+		AlertWebhookHeaders:  env.webhookHeaders("SWWAF_ALERT_WEBHOOK_HEADERS"),
+		AlertSlackWebhookURL: env.webhookURL("SWWAF_ALERT_SLACK_WEBHOOK_URL"),
+		AlertNtfyURL:         env.webhookURL("SWWAF_ALERT_NTFY_URL"),
+		AlertNtfyToken:       env.secret("SWWAF_ALERT_NTFY_TOKEN"),
 		AlertEvents: env.alertEvents("SWWAF_ALERT_EVENTS",
 			strings.Join(alerts.Events(), ",")),
 		AlertCooldown:   env.duration("SWWAF_ALERT_COOLDOWN", "15m"),
@@ -323,6 +337,8 @@ func FromEnvironment(lookupEnv func(string) (string, bool)) (*Config, error) {
 	cfg.LogRemoteAppName = env.appName("SWWAF_LOG_REMOTE_APP_NAME",
 		cfg.InstanceName, cfg.LogRemoteURL != nil)
 
+	env.checkInstanceNameForNtfy(cfg.InstanceName, cfg.AlertNtfyURL != nil)
+
 	for _, country := range cfg.ExclusivelyAllowedCountries {
 		if slices.Contains(cfg.DeniedCountries, country) {
 			env.check("SWWAF_EXCLUSIVELY_ALLOWED_COUNTRIES",
@@ -667,10 +683,23 @@ func (e *environment) appName(name, instanceName string, sending bool) string {
 	return value
 }
 
-// webhookURL reads the setting that is where each alert is posted. Unset
-// or empty, it is nil, and no alert is sent. The log shows ******** in
-// place of its path and query, and an error shows none of it, since many
-// webhooks carry their secret there.
+// checkInstanceNameForNtfy refuses an instance name that holds a control
+// character while ntfySet, SWWAF_ALERT_NTFY_URL being set: ntfy is sent
+// the instance name in a header, which cannot hold one.
+func (e *environment) checkInstanceNameForNtfy(instanceName string, ntfySet bool) {
+	if ntfySet && strings.ContainsFunc(instanceName, unicode.IsControl) {
+		e.check("SWWAF_INSTANCE_NAME", fmt.Errorf(
+			"%q %w, and is sent to ntfy in a header while SWWAF_ALERT_NTFY_URL is set",
+			instanceName, errControlCharacter))
+	}
+}
+
+// webhookURL reads a setting that is a URL each alert is posted to:
+// SWWAF_ALERT_WEBHOOK_URL, SWWAF_ALERT_SLACK_WEBHOOK_URL or
+// SWWAF_ALERT_NTFY_URL. Unset or empty, it is nil, and no alert is posted
+// there. The log shows ******** in place of its path and query, and an
+// error shows none of it, since a webhook or an ntfy topic can carry its
+// secret there.
 func (e *environment) webhookURL(name string) *url.URL {
 	value, _ := e.lookup(name)
 	webhook, logged, err := parseWebhookURL(value)
@@ -692,6 +721,28 @@ func (e *environment) webhookHeaders(name string) http.Header {
 	return headers
 }
 
+// secret reads a setting that is a secret another service gave, such as
+// an ntfy token, "" while it is unset. It is sent in a header, which
+// cannot hold a control character, so one in it is an error. The log
+// shows ******** in place of a value that is not empty, and an error
+// shows none of it.
+func (e *environment) secret(name string) string {
+	value, _ := e.lookup(name)
+
+	logged := ""
+	if value != "" {
+		logged = masked
+	}
+
+	e.settings = append(e.settings, slog.String(name, logged))
+
+	if strings.ContainsFunc(value, unicode.IsControl) {
+		e.check(name, errControlCharacter)
+	}
+
+	return value
+}
+
 // alertEvents reads the setting that is the events alerts are sent for.
 func (e *environment) alertEvents(name, defaultValue string) []string {
 	events, err := parseAlertEvents(e.value(name, defaultValue))