Alerts to a JSON webhook, with a cooldown and an hourly summary (closes #26)

SWWAF_ALERT_WEBHOOK_URL gets one JSON POST per alert, in SPEC.md's schema, with SWWAF_ALERT_WEBHOOK_HEADERS: ban and permanent_ban, with the ban's notes, in observe mode too, marked mode observe and worked out only when the alert would be sent; source_failure for GeoJS; file_error for a rule or state file with an error. SWWAF_ALERT_EVENTS chooses; SWWAF_ALERT_COOLDOWN holds back repeats by netblock, file or source; past SWWAF_ALERT_MAX_PER_HOUR the hour ends in one summary. A bounded queue, retried with backoff, holds up no request; a 4xx other than 408 and 429 gives the alert up. alerts.json keeps the queue, the cooldowns and the hour. Nothing shows the URL's path or query. Judgement call: the summary's event is summary, which SPEC.md omits. Judgement call: an admin's ban raises no alert. Model: opus-5-5
2026-10-07 04:21:24 +02:00
parent 5d6f6ffaf9
commit 432097ee3f
25 changed files with 3474 additions and 255 deletions
@@ -22,6 +22,7 @@ import (
 	"time"
 	"unicode/utf8"

+	"sneak.berlin/go/smallwebwaf/internal/alerts"
 	"sneak.berlin/go/smallwebwaf/internal/remotelog"
 )

@@ -158,6 +159,19 @@ type Config struct {
 	LogRemoteBuffer   int
 	LogRemoteFacility int
 	LogRemoteAppName  string
+	// AlertWebhookURL is where each alert is posted as JSON
+	// (SWWAF_ALERT_WEBHOOK_URL), nil while it is unset and no alert is
+	// sent. AlertWebhookHeaders are sent with each
+	// (SWWAF_ALERT_WEBHOOK_HEADERS). AlertEvents are the events alerts are
+	// sent for (SWWAF_ALERT_EVENTS). A repeat of an alert within
+	// AlertCooldown is held back (SWWAF_ALERT_COOLDOWN), and so is an alert
+	// past AlertMaxPerHour in an hour, for the hour's summary
+	// (SWWAF_ALERT_MAX_PER_HOUR); 0 is off for both.
+	AlertWebhookURL     *url.URL
+	AlertWebhookHeaders http.Header
+	AlertEvents         []string
+	AlertCooldown       time.Duration
+	AlertMaxPerHour     int

 	// settings are the values read, as given or by default, and the
 	// files they were read from, for the log line at start.
@@ -176,7 +190,8 @@ const (
 	ipv4Bits = 32
 	// minTokenLength is the fewest characters a token may have.
 	minTokenLength = 32
-	// masked is what the log shows for a token that is set.
+	// masked is what the log shows for a token that is set, and in place of
+	// a secret in another setting.
 	masked = "********"
 	// defaultListenAddr and defaultUpstreamURL are the defaults of
 	// SWWAF_LISTEN_ADDR and SWWAF_UPSTREAM_URL.
@@ -230,7 +245,17 @@ var (
 	errNotFacility   = errors.New("is not a syslog facility such as local0 or daemon")
 	errNotAppName    = errors.New(
 		"is not 1 to 48 printable ASCII characters without a space, such as gitea")
-	errSetTwice = errors.New("set only one of them")
+	errSetTwice      = errors.New("set only one of them")
+	errNotWebhookURL = errors.New(
+		"is not an http or https URL without a user or a fragment, " +
+			"such as https://alerts.example/smallwebwaf")
+	errNotWebhookHeader = errors.New(
+		"is not a header name followed by : and the header's value, " +
+			"such as Authorization:Bearer <token>")
+	errNotAlertEvent = errors.New(
+		"is not ban, permanent_ban, waf_block, anomaly, reputation_hit, " +
+			"source_failure or file_error")
+	errNotNumberOrOff = errors.New("is not a whole number above zero, such as 60, or off")
 )

 // FromEnvironment reads the settings with lookupEnv, normally
@@ -278,15 +303,21 @@ func FromEnvironment(lookupEnv func(string) (string, bool)) (*Config, error) {
 		StateCounterInterval: env.durationNotOff("SWWAF_STATE_COUNTER_INTERVAL", "15m"),
 		LogRequestHeaders: env.headerNames("SWWAF_LOG_REQUEST_HEADERS",
 			"accept,accept-language,accept-encoding,content-type,origin,range"),
-		AdminToken:        env.token("SWWAF_ADMIN_TOKEN"),
-		MetricsToken:      env.token("SWWAF_METRICS_TOKEN"),
-		MetricsTopN:       env.numberNotOff("SWWAF_METRICS_TOP_N", "50"),
-		RulesDir:          env.value("SWWAF_RULES_DIR", "/etc/smallwebwaf/rules.d"),
-		RulesEnabled:      env.boolean("SWWAF_RULES_ENABLED", "true"),
-		LogRemoteURL:      env.logRemoteURL("SWWAF_LOG_REMOTE_URL"),
-		LogRemoteTLSCAs:   env.certificates("SWWAF_LOG_REMOTE_TLS_CA_FILE"),
-		LogRemoteBuffer:   env.numberNotOff("SWWAF_LOG_REMOTE_BUFFER", "10000"),
-		LogRemoteFacility: env.facility("SWWAF_LOG_REMOTE_FACILITY", "local0"),
+		AdminToken:          env.token("SWWAF_ADMIN_TOKEN"),
+		MetricsToken:        env.token("SWWAF_METRICS_TOKEN"),
+		MetricsTopN:         env.numberNotOff("SWWAF_METRICS_TOP_N", "50"),
+		RulesDir:            env.value("SWWAF_RULES_DIR", "/etc/smallwebwaf/rules.d"),
+		RulesEnabled:        env.boolean("SWWAF_RULES_ENABLED", "true"),
+		LogRemoteURL:        env.logRemoteURL("SWWAF_LOG_REMOTE_URL"),
+		LogRemoteTLSCAs:     env.certificates("SWWAF_LOG_REMOTE_TLS_CA_FILE"),
+		LogRemoteBuffer:     env.numberNotOff("SWWAF_LOG_REMOTE_BUFFER", "10000"),
+		LogRemoteFacility:   env.facility("SWWAF_LOG_REMOTE_FACILITY", "local0"),
+		AlertWebhookURL:     env.webhookURL("SWWAF_ALERT_WEBHOOK_URL"),
+		AlertWebhookHeaders: env.webhookHeaders("SWWAF_ALERT_WEBHOOK_HEADERS"),
+		AlertEvents: env.alertEvents("SWWAF_ALERT_EVENTS",
+			strings.Join(alerts.Events(), ",")),
+		AlertCooldown:   env.duration("SWWAF_ALERT_COOLDOWN", "15m"),
+		AlertMaxPerHour: env.numberOrOff("SWWAF_ALERT_MAX_PER_HOUR", "60"),
 	}

 	cfg.LogRemoteAppName = env.appName("SWWAF_LOG_REMOTE_APP_NAME",
@@ -636,6 +667,48 @@ func (e *environment) appName(name, instanceName string, sending bool) string {
 	return value
 }

+// webhookURL reads the setting that is where each alert is posted. Unset
+// or empty, it is nil, and no alert is sent. The log shows ******** in
+// place of its path and query, and an error shows none of it, since many
+// webhooks carry their secret there.
+func (e *environment) webhookURL(name string) *url.URL {
+	value, _ := e.lookup(name)
+	webhook, logged, err := parseWebhookURL(value)
+	e.settings = append(e.settings, slog.String(name, logged))
+	e.check(name, err)
+
+	return webhook
+}
+
+// webhookHeaders reads the setting that is the headers sent with each
+// alert. The log shows each header's value as ********, since a header
+// such as Authorization carries a secret.
+func (e *environment) webhookHeaders(name string) http.Header {
+	value, _ := e.lookup(name)
+	headers, logged, err := parseWebhookHeaders(value)
+	e.settings = append(e.settings, slog.String(name, logged))
+	e.check(name, err)
+
+	return headers
+}
+
+// alertEvents reads the setting that is the events alerts are sent for.
+func (e *environment) alertEvents(name, defaultValue string) []string {
+	events, err := parseAlertEvents(e.value(name, defaultValue))
+	e.check(name, err)
+
+	return events
+}
+
+// numberOrOff reads a setting that is a whole number above zero, or off,
+// which is 0.
+func (e *environment) numberOrOff(name, defaultValue string) int {
+	number, err := parseNumberOrOff(e.value(name, defaultValue))
+	e.check(name, err)
+
+	return number
+}
+
 // parseDuration reads a duration in Go's syntax, such as 90s or 15m, a
 // whole number of days such as 7d, or off.
 func parseDuration(value string) (time.Duration, error) {
@@ -1051,6 +1124,96 @@ func parseFacility(value string) (int, error) {
 	return number, nil
 }

+// parseWebhookURL reads where each alert is posted: http or https, a
+// host, and an optional port from 1 to 65535, path and query, without a
+// user or a fragment. It returns the URL, and how the log shows it: its
+// scheme and host, and ******** in place of its path and query, if it has
+// either. An error shows no part of the value. An empty value is no URL.
+func parseWebhookURL(value string) (*url.URL, string, error) {
+	if value == "" {
+		return nil, "", nil
+	}
+
+	webhook, err := url.Parse(value)
+	if err != nil {
+		return nil, "", errNotWebhookURL
+	}
+
+	port, err := strconv.ParseUint(webhook.Port(), 10, 16)
+
+	valid := (webhook.Scheme == "http" || webhook.Scheme == "https") &&
+		webhook.Hostname() != "" && (webhook.Port() == "" || (err == nil && port != 0)) &&
+		webhook.User == nil && webhook.Opaque == "" && webhook.Fragment == ""
+	if !valid {
+		return nil, "", errNotWebhookURL
+	}
+
+	logged := webhook.Scheme + "://" + webhook.Host
+	if webhook.Path != "" || webhook.RawQuery != "" {
+		logged += "/" + masked
+	}
+
+	return webhook, logged, nil
+}
+
+// parseWebhookHeaders reads a comma-separated list of headers, each its
+// name, :, and its value, and returns them, and how the log shows them,
+// with each value as ********. An error names the item by its place in
+// the list, so that it shows no value. An empty value is an empty list.
+func parseWebhookHeaders(value string) (http.Header, string, error) {
+	headers := http.Header{}
+	if strings.TrimSpace(value) == "" {
+		return headers, "", nil
+	}
+
+	logged := []string{}
+
+	for i, item := range strings.Split(value, ",") {
+		name, headerValue, found := strings.Cut(item, ":")
+		name = strings.TrimSpace(name)
+
+		if !found || !IsHeaderName(name) || strings.ContainsAny(headerValue, "\r\n\x00") {
+			return nil, "", fmt.Errorf("item %d %w", i+1, errNotWebhookHeader)
+		}
+
+		headers.Add(name, strings.TrimSpace(headerValue))
+		logged = append(logged, name+":"+masked)
+	}
+
+	return headers, strings.Join(logged, ","), nil
+}
+
+// parseAlertEvents reads a comma-separated list of the events alerts can
+// be sent for.
+func parseAlertEvents(value string) ([]string, error) {
+	events, err := parseList(value)
+	if err != nil {
+		return nil, err
+	}
+
+	for _, event := range events {
+		if !slices.Contains(alerts.Events(), event) {
+			return nil, fmt.Errorf("%q %w", event, errNotAlertEvent)
+		}
+	}
+
+	return events, nil
+}
+
+// parseNumberOrOff reads a whole number above zero, or off, which is 0.
+func parseNumberOrOff(value string) (int, error) {
+	if value == off {
+		return 0, nil
+	}
+
+	n, err := strconv.Atoi(value)
+	if err != nil || n <= 0 {
+		return 0, fmt.Errorf("%q %w", value, errNotNumberOrOff)
+	}
+
+	return n, nil
+}
+
 // appNameMaxLength is the most characters RFC 5424 allows in an
 // APP-NAME.
 const appNameMaxLength = 48
@@ -6,9 +6,11 @@ import (
 	"encoding/json"
 	"log/slog"
 	"maps"
+	"net/http"
 	"net/netip"
 	"os"
 	"path/filepath"
+	"reflect"
 	"slices"
 	"strings"
 	"testing"
@@ -62,6 +64,19 @@ const (
 	logRemoteBuffer         = "SWWAF_LOG_REMOTE_BUFFER"
 	logRemoteFacility       = "SWWAF_LOG_REMOTE_FACILITY"
 	logRemoteAppName        = "SWWAF_LOG_REMOTE_APP_NAME"
+	alertWebhookURL         = "SWWAF_ALERT_WEBHOOK_URL"
+	alertWebhookHeaders     = "SWWAF_ALERT_WEBHOOK_HEADERS"
+	alertEvents             = "SWWAF_ALERT_EVENTS"
+	alertCooldown           = "SWWAF_ALERT_COOLDOWN"
+	alertMaxPerHour         = "SWWAF_ALERT_MAX_PER_HOUR"
+)
+
+// defaultAlertEvents is the default of SWWAF_ALERT_EVENTS, and
+// defaultAlertCooldown that of SWWAF_ALERT_COOLDOWN.
+const (
+	defaultAlertEvents = "ban,permanent_ban,waf_block,anomaly,reputation_hit," +
+		"source_failure,file_error"
+	defaultAlertCooldown = "15m"
 )

 // defaultLogRequestHeaders is the default of SWWAF_LOG_REQUEST_HEADERS.
@@ -477,6 +492,150 @@ func TestAppNameSetStopsTheStartWhileSending(t *testing.T) {
 	}
 }

+func TestAlertSettingsDefaults(t *testing.T) {
+	t.Parallel()
+
+	cfg := fromEnvironment(t, environment{})
+
+	if cfg.AlertWebhookURL != nil || len(cfg.AlertWebhookHeaders) != 0 ||
+		strings.Join(cfg.AlertEvents, ",") != defaultAlertEvents ||
+		cfg.AlertCooldown != 15*time.Minute || cfg.AlertMaxPerHour != 60 {
+		t.Errorf("alert settings %v, %v, %v, %s and %d, want no URL, no headers, "+
+			"%s, 15m and 60", cfg.AlertWebhookURL, cfg.AlertWebhookHeaders,
+			cfg.AlertEvents, cfg.AlertCooldown, cfg.AlertMaxPerHour, defaultAlertEvents)
+	}
+}
+
+func TestAlertSettingsAsSet(t *testing.T) {
+	t.Parallel()
+
+	const webhook = "https://alerts.example:8443/hooks/waf?team=ops"
+
+	cfg := fromEnvironment(t, environment{
+		alertWebhookURL:     webhook,
+		alertWebhookHeaders: "Authorization: Bearer abc:def , x-team:ops",
+		alertEvents:         "ban, file_error",
+		alertCooldown:       "1h",
+		alertMaxPerHour:     "10",
+	})
+
+	headers := http.Header{"Authorization": {"Bearer abc:def"}, "X-Team": {"ops"}}
+	if cfg.AlertWebhookURL.String() != webhook ||
+		!reflect.DeepEqual(cfg.AlertWebhookHeaders, headers) ||
+		!slices.Equal(cfg.AlertEvents, []string{"ban", "file_error"}) ||
+		cfg.AlertCooldown != time.Hour || cfg.AlertMaxPerHour != 10 {
+		t.Errorf("alert settings %v, %v, %v, %s and %d", cfg.AlertWebhookURL,
+			cfg.AlertWebhookHeaders, cfg.AlertEvents, cfg.AlertCooldown,
+			cfg.AlertMaxPerHour)
+	}
+
+	cfg = fromEnvironment(t, environment{
+		alertWebhookURL: "", alertEvents: "", alertCooldown: off, alertMaxPerHour: off,
+	})
+	if cfg.AlertWebhookURL != nil || len(cfg.AlertEvents) != 0 ||
+		cfg.AlertCooldown != 0 || cfg.AlertMaxPerHour != 0 {
+		t.Errorf("set empty or off, alert settings %v, %v, %s and %d",
+			cfg.AlertWebhookURL, cfg.AlertEvents, cfg.AlertCooldown, cfg.AlertMaxPerHour)
+	}
+}
+
+func TestInvalidAlertSettingStopsTheStart(t *testing.T) {
+	t.Parallel()
+
+	wantStartStopped(t, []struct{ name, value string }{
+		{alertWebhookURL, "alerts.example/smallwebwaf"},
+		{alertWebhookURL, "ftp://alerts.example/"},
+		{alertWebhookURL, "https:///smallwebwaf"},
+		{alertWebhookURL, "https://user:password@alerts.example/"},
+		{alertWebhookURL, "https://alerts.example/#top"},
+		{alertWebhookURL, "https://alerts.example:0/"},
+		{alertWebhookURL, "https://alerts.example:65536/"},
+		{alertWebhookHeaders, "Authorization"},
+		{alertWebhookHeaders, "X Team:ops"},
+		{alertWebhookHeaders, ":ops"},
+		{alertWebhookHeaders, "X-Team:ops,"},
+		{alertWebhookHeaders, "X-Team:o\r\nps"},
+		{alertEvents, "bans"},
+		{alertEvents, "summary"},
+		{alertEvents, "ban,,file_error"},
+		{alertCooldown, "0"},
+		{alertCooldown, "soon"},
+		{alertMaxPerHour, "0"},
+		{alertMaxPerHour, "-1"},
+		{alertMaxPerHour, "1.5"},
+	})
+}
+
+func TestWebhookHeadersAreLoggedMaskedAndNeverShown(t *testing.T) {
+	t.Parallel()
+
+	const secret = "Bearer 0123456789abcdef"
+
+	cfg := fromEnvironment(t, environment{
+		alertWebhookHeaders: "Authorization:" + secret + ",X-Team:ops",
+	})
+
+	var out bytes.Buffer
+
+	slog.New(slog.NewJSONHandler(&out, nil)).Info("starting", "settings", cfg)
+
+	logged := out.String()
+	if strings.Contains(logged, secret) || strings.Contains(logged, "ops") ||
+		!strings.Contains(logged,
+			`"`+alertWebhookHeaders+`":"Authorization:********,X-Team:********"`) {
+		t.Errorf("the headers are not logged masked: %s", logged)
+	}
+
+	// An item that is not a header is named by its place, not shown.
+	_, err := config.FromEnvironment(environment{
+		alertWebhookHeaders: "X-Team:ops," + secret,
+	}.lookupEnv)
+
+	want := alertWebhookHeaders + ": item 2 is not a header name followed by : " +
+		"and the header's value, such as Authorization:Bearer <token>"
+	if err == nil || err.Error() != want {
+		t.Errorf("error %v, want %s", err, want)
+	}
+}
+
+func TestWebhookURLIsLoggedWithoutItsPathOrQueryAndNeverShown(t *testing.T) {
+	t.Parallel()
+
+	const secret = "T0123/B4567/abcdef"
+
+	for value, want := range map[string]string{
+		"https://hooks.example/services/" + secret:   "https://hooks.example/********",
+		"https://hooks.example:8443?token=" + secret: "https://hooks.example:8443/********",
+		"http://[2001:db8::1]:8080":                  "http://[2001:db8::1]:8080",
+	} {
+		cfg := fromEnvironment(t, environment{alertWebhookURL: value})
+
+		var out bytes.Buffer
+
+		slog.New(slog.NewJSONHandler(&out, nil)).Info("starting", "settings", cfg)
+
+		logged := out.String()
+		if strings.Contains(logged, secret) ||
+			!strings.Contains(logged, `"`+alertWebhookURL+`":"`+want+`"`) {
+			t.Errorf("%s is not logged as %s: %s", value, want, logged)
+		}
+	}
+
+	// A value that is not such a URL is not shown either.
+	for _, value := range []string{
+		"ftp://hooks.example/services/" + secret,
+		"https://hooks.example/services/%zz" + secret,
+	} {
+		_, err := config.FromEnvironment(environment{alertWebhookURL: value}.lookupEnv)
+
+		want := alertWebhookURL + ": is not an http or https URL without a user or " +
+			"a fragment, such as https://alerts.example/smallwebwaf"
+		if err == nil || err.Error() != want {
+			t.Errorf("error %v, want %s", err, want)
+		}
+	}
+}
+
 func TestCodeOnBothCountryListsStopsTheStart(t *testing.T) {
 	t.Parallel()

@@ -926,6 +1085,11 @@ func TestLogsEachSettingWithItsValue(t *testing.T) {
 		logRemoteBuffer:         "10000",
 		logRemoteFacility:       "local0",
 		logRemoteAppName:        hostname,
+		alertWebhookURL:         "",
+		alertWebhookHeaders:     "",
+		alertEvents:             defaultAlertEvents,
+		alertCooldown:           defaultAlertCooldown,
+		alertMaxPerHour:         "60",
 	}
 	if !maps.Equal(line.Settings, want) {
 		t.Errorf("logged settings\n%v\nwant\n%v", line.Settings, want)