Alerts to a JSON webhook, with a cooldown and an hourly summary (closes #26)

SWWAF_ALERT_WEBHOOK_URL gets one JSON POST per alert, in SPEC.md's
schema, with SWWAF_ALERT_WEBHOOK_HEADERS: ban and permanent_ban, with
the ban's notes; source_failure for GeoJS; file_error for a rule or
state file edit that does not parse and a failed state write.
SWWAF_ALERT_EVENTS chooses, SWWAF_ALERT_COOLDOWN holds back repeats,
and past SWWAF_ALERT_MAX_PER_HOUR the hour ends in one summary. A
bounded queue, retried with backoff, holds up no request; alerts.json
keeps it, the cooldowns and the hour. The ledger now reports whether it
made a ban, or made one permanent.

Judgement call: the summary's event is summary, which SPEC.md omits.
Judgement call: admin bans and observe mode raise no alert.

Model: opus-5-5

internal/proxy/alerts_test.go

@@ -0,0 +1,174 @@
+package proxy_test
+
+import (
+	"maps"
+	"net/http"
+	"net/netip"
+	"reflect"
+	"testing"
+	"time"
+
+	"sneak.berlin/go/smallwebwaf/internal/alerts"
+	"sneak.berlin/go/smallwebwaf/internal/bans"
+	"sneak.berlin/go/smallwebwaf/internal/proxy"
+	"sneak.berlin/go/smallwebwaf/internal/requestlog"
+)
+
+const (
+	alertWebhookURL = "SWWAF_ALERT_WEBHOOK_URL"
+	// alertInstance is the instance every alert of these tests gives.
+	alertInstance = "fsn1app1/gitea"
+)
+
+func TestBanForABrokenLimitRaisesABanAlertWithItsNotes(t *testing.T) {
+	t.Parallel()
+
+	s, clk, server, queue := startWithAlerts(t, map[string]string{
+		rateLimitPerMinute: "1",
+		banScopeV4Prefix:   "24",
+	})
+	start := clk.Now()
+
+	s.get(client, http.StatusOK, requestlog.ActionForward)
+	s.get(client, http.StatusForbidden, requestlog.ActionRateLimited)
+
+	netblock := netip.MustParsePrefix("203.0.113.0/24")
+	ban := server.Ledger.Bans(netblock)[0]
+
+	// A request refused under the ban raises no other alert.
+	clk.advance(time.Minute)
+	s.get(client, http.StatusForbidden, requestlog.ActionBanned)
+
+	wantAlerts(t, queue, alerts.Alert{
+		Instance: alertInstance,
+		Time:     start,
+		Event:    alerts.EventBan,
+		Client:   netip.MustParseAddr(client),
+		Netblock: netblock,
+		Reason:   "requests per minute over the limit of 1",
+		Detail: map[string]any{
+			"cause":       bans.CauseLimit,
+			"ban_expires": requestlog.FormatTime(start.Add(time.Hour)),
+			"notes":       ban.Notes,
+		},
+	})
+
+	if ban.Notes.Limit != 1 || ban.Notes.Request.Path != "/" {
+		t.Errorf("the alert's notes are %+v, want those of the broken limit", ban.Notes)
+	}
+}
+
+func TestAttackBanRaisesABanAlertThenAPermanentBanAlert(t *testing.T) {
+	t.Parallel()
+
+	s, clk, server, queue := startWithAlerts(t, map[string]string{
+		rulesDir: writeRules(t, testRules),
+	})
+	start := clk.Now()
+	netblock := netip.MustParsePrefix(client + "/32")
+	other := netip.MustParsePrefix(otherClient + "/32")
+
+	// The probe bans the client for seven days, and its next request makes
+	// the ban permanent. The request after that changes nothing.
+	s.request(client, "/.env", http.StatusForbidden, requestlog.ActionBanned)
+
+	attackBan := server.Ledger.Bans(netblock)[0]
+
+	clk.advance(time.Minute)
+	s.get(client, http.StatusForbidden, requestlog.ActionBanned)
+
+	permanentBan := server.Ledger.Bans(netblock)[0]
+
+	s.get(client, http.StatusForbidden, requestlog.ActionBanned)
+
+	// Another client's probe after its first ban has run out without a
+	// request makes a permanent ban at once.
+	s.request(otherClient, "/.env", http.StatusForbidden, requestlog.ActionBanned)
+	clk.advance(7 * 24 * time.Hour)
+	s.request(otherClient, "/.env", http.StatusForbidden, requestlog.ActionBanned)
+
+	otherBans := server.Ledger.Bans(other)
+
+	wantAlerts(t, queue,
+		attackAlert(alerts.EventBan, start, client, attackBan,
+			requestlog.FormatTime(start.Add(7*24*time.Hour))),
+		attackAlert(alerts.EventPermanentBan, start.Add(time.Minute), client,
+			permanentBan, "permanent"),
+		attackAlert(alerts.EventBan, start.Add(time.Minute), otherClient, otherBans[0],
+			requestlog.FormatTime(start.Add(time.Minute+7*24*time.Hour))),
+		attackAlert(alerts.EventPermanentBan, start.Add(time.Minute+7*24*time.Hour),
+			otherClient, otherBans[1], "permanent"),
+	)
+}
+
+func TestObserveModeRaisesNoBanAlert(t *testing.T) {
+	t.Parallel()
+
+	s, _, _, queue := startWithAlerts(t, map[string]string{
+		mode:               "observe",
+		rateLimitPerMinute: "1",
+		rulesDir:           writeRules(t, testRules),
+	})
+
+	s.get(client, http.StatusOK, requestlog.ActionForward)
+	s.get(client, http.StatusOK, requestlog.ActionForward)
+	s.request(otherClient, "/.env", http.StatusOK, requestlog.ActionForward)
+
+	wantAlerts(t, queue)
+}
+
+// startWithAlerts is startWithClock with alerts to a webhook, which is
+// never sent them, and returns the queue they wait in as well.
+func startWithAlerts(
+	t *testing.T, env map[string]string,
+) (*sender, *clock, *proxy.Server, *alerts.Queue) {
+	t.Helper()
+
+	app := startApp(t, func(http.ResponseWriter, *http.Request) {})
+	clk := &clock{now: time.Date(2026, 10, 6, 0, 0, 0, 0, time.UTC)}
+	settings := map[string]string{
+		trustedProxies:  trustLocalhost,
+		alertWebhookURL: "https://alerts.example/smallwebwaf",
+		instanceName:    alertInstance,
+	}
+	maps.Copy(settings, env)
+
+	addr, out, server, queue := startProxyWithAlerts(t, app.URL, "", clk.Now, settings)
+
+	return &sender{t: t, addr: addr, out: out}, clk, server, queue
+}
+
+// attackAlert returns the alert for event, raised by a request from client
+// at the time raised, for ban, a ban for the probe rule of testRules,
+// which ends at expires, as the log line gives it.
+func attackAlert(
+	event string, raised time.Time, client string, ban bans.Ban, expires string,
+) alerts.Alert {
+	return alerts.Alert{
+		Instance: alertInstance,
+		Time:     raised,
+		Event:    event,
+		Client:   netip.MustParseAddr(client),
+		Netblock: ban.Netblock,
+		Reason:   "matched the rule probe",
+		Detail: map[string]any{
+			"cause": bans.CauseAttack, "ban_expires": expires, "notes": ban.Notes,
+		},
+	}
+}
+
+// wantAlerts checks the alerts waiting in queue, in order.
+func wantAlerts(t *testing.T, queue *alerts.Queue, want ...alerts.Alert) {
+	t.Helper()
+
+	got := queue.Snapshot().Waiting
+	if len(got) != len(want) {
+		t.Fatalf("%d alerts wait, want %d: %+v", len(got), len(want), got)
+	}
+
+	for i := range want {
+		if !reflect.DeepEqual(got[i], want[i]) {
+			t.Errorf("alert %d is\n%+v\nwant\n%+v", i, got[i], want[i])
+		}
+	}
+}

internal/proxy/bans.go

@@ -4,6 +4,7 @@ import (
 	"net/netip"
 	"time"
 
+	"sneak.berlin/go/smallwebwaf/internal/alerts"
 	"sneak.berlin/go/smallwebwaf/internal/bans"
 	"sneak.berlin/go/smallwebwaf/internal/requestlog"
 	"sneak.berlin/go/smallwebwaf/internal/rules"
@@ -16,14 +17,25 @@ func (rq *request) banResponse(action string) *refusal {
 }
 
 // banned reports whether a ban on a netblock the client is in covers the
-// request at now, and notes for the log line when that ban ends.
+// request at now, and notes for the log line when that ban ends. A
+// request that makes the ban permanent raises the alert for it.
 func (rq *request) banned(now time.Time) bool {
-	check := rq.h.ledger.Check
+	var (
+		ban           bans.Ban
+		banned        bool
+		madePermanent bool
+	)
+
 	if rq.h.config.Observe {
-		check = rq.h.ledger.Find // in observe mode the ban refuses nothing
+		ban, banned = rq.h.ledger.Find(rq.client, now) // the ban refuses nothing
+	} else {
+		ban, banned, madePermanent = rq.h.ledger.Check(rq.client, now)
+	}
+
+	if madePermanent {
+		rq.alertBan(ban)
 	}
 
-	ban, banned := check(rq.client, now)
 	if banned {
 		rq.line.BanExpires = banExpires(ban)
 	}
@@ -54,7 +66,7 @@ func (rq *request) limitBroken(now time.Time) bool {
 	}
 
 	netblock := rq.h.netblock(rq.client)
-	ban := rq.h.ledger.BanForLimit(netblock, now, bans.Notes{
+	ban, made := rq.h.ledger.BanForLimit(netblock, now, bans.Notes{
 		Country:  rq.line.Country,
 		Limit:    hit.Limit,
 		Window:   hit.Window,
@@ -65,6 +77,10 @@ func (rq *request) limitBroken(now time.Time) bool {
 	rq.h.limiter.Reset(group)
 	rq.line.BanExpires = banExpires(ban)
 
+	if made {
+		rq.alertBan(ban)
+	}
+
 	return true
 }
 
@@ -72,7 +88,7 @@ func (rq *request) limitBroken(now time.Time) bool {
 // attack, the match of rule, a ban rule.
 func (rq *request) banForAttack(now time.Time, rule rules.Rule) {
 	netblock := rq.h.netblock(rq.client)
-	ban := rq.h.ledger.BanForAttack(netblock, now, bans.Notes{
+	ban, made := rq.h.ledger.BanForAttack(netblock, now, bans.Notes{
 		Country:  rq.line.Country,
 		RuleID:   rule.ID,
 		Target:   rule.Target,
@@ -80,6 +96,31 @@ func (rq *request) banForAttack(now time.Time, rule rules.Rule) {
 		Requests: rq.netblockRequests(netblock),
 	})
 	rq.line.BanExpires = banExpires(ban)
+
+	if made {
+		rq.alertBan(ban)
+	}
+}
+
+// alertBan raises the alert for ban, which the request made, or made
+// permanent: permanent_ban for a permanent ban, ban for another. Its
+// detail gives the ban's cause, when it ends, and its notes.
+func (rq *request) alertBan(ban bans.Ban) {
+	event := alerts.EventBan
+	if ban.Permanent() {
+		event = alerts.EventPermanentBan
+	}
+
+	rq.h.alerts.Raise(alerts.Alert{
+		Event:    event,
+		Client:   rq.client,
+		Netblock: ban.Netblock,
+		Country:  ban.Notes.Country,
+		Reason:   ban.Reason,
+		Detail: map[string]any{
+			"cause": ban.Cause, "ban_expires": banExpires(ban), "notes": ban.Notes,
+		},
+	})
 }
 
 // noted is the request, refused at now with SWWAF_BAN_RESPONSE, as the

internal/proxy/proxy.go

@@ -11,6 +11,7 @@ import (
 	"strings"
 	"time"
 
+	"sneak.berlin/go/smallwebwaf/internal/alerts"
 	"sneak.berlin/go/smallwebwaf/internal/bans"
 	"sneak.berlin/go/smallwebwaf/internal/config"
 	"sneak.berlin/go/smallwebwaf/internal/lookup"
@@ -63,6 +64,9 @@ type Params struct {
 	// Rules are the rule files' rules, which each request is checked
 	// against.
 	Rules *rules.Files
+	// Alerts receive the alert for each ban the proxy makes or makes
+	// permanent, and for GeoJS failing.
+	Alerts *alerts.Queue
 }
 
 // Server is the server smallwebwaf runs, with the parts of the proxy
@@ -110,8 +114,10 @@ func New(params Params) *Server {
 			Now:        params.Now,
 			ProcessLog: params.ProcessLog,
 			Metrics:    m,
+			Alerts:     params.Alerts,
 		}),
-		rules: params.Rules,
+		rules:  params.Rules,
+		alerts: params.Alerts,
 	}
 	m.AddBansAndClients(h.ledger, h.limiter, params.Now)
 	m.AddRules(params.Rules)
@@ -150,6 +156,7 @@ type handler struct {
 	ledger     *bans.Ledger
 	geojs      *lookup.GeoJS
 	rules      *rules.Files
+	alerts     *alerts.Queue
 }
 
 // newTransport returns what carries requests to the app. It never goes

internal/proxy/proxy_test.go

@@ -14,6 +14,7 @@ import (
 	"testing"
 	"time"
 
+	"sneak.berlin/go/smallwebwaf/internal/alerts"
 	"sneak.berlin/go/smallwebwaf/internal/config"
 	"sneak.berlin/go/smallwebwaf/internal/proxy"
 	"sneak.berlin/go/smallwebwaf/internal/requestlog"
@@ -223,6 +224,20 @@ func startProxyWithClock(
 ) (string, *output, *proxy.Server) {
 	t.Helper()
 
+	addr, out, server, _ := startProxyWithAlerts(t, appURL, geojsURL, now, env)
+
+	return addr, out, server
+}
+
+// startProxyWithAlerts is startProxyWithClock, and returns the queue of
+// the alerts the proxy raises as well, as the settings in env make it. No
+// alert is sent from it: they wait in it, for the test to look at.
+func startProxyWithAlerts(
+	t *testing.T, appURL, geojsURL string, now func() time.Time,
+	env map[string]string,
+) (string, *output, *proxy.Server, *alerts.Queue) {
+	t.Helper()
+
 	settings := map[string]string{"SWWAF_UPSTREAM_URL": appURL, rulesDir: t.TempDir()}
 	maps.Copy(settings, env)
 
@@ -245,6 +260,16 @@ func startProxyWithClock(
 		t.Fatalf("rule files: %v", err)
 	}
 
+	alertQueue := alerts.New(alerts.Params{
+		WebhookURL: cfg.AlertWebhookURL,
+		Events:     cfg.AlertEvents,
+		Cooldown:   cfg.AlertCooldown,
+		MaxPerHour: cfg.AlertMaxPerHour,
+		Instance:   cfg.InstanceName,
+		Now:        now,
+		ProcessLog: processLog,
+	})
+
 	server := proxy.New(proxy.Params{
 		Config:     cfg,
 		RequestLog: out,
@@ -252,6 +277,7 @@ func startProxyWithClock(
 		GeoJSURL:   geojsURL,
 		Now:        now,
 		Rules:      ruleFiles,
+		Alerts:     alertQueue,
 	})
 
 	listener, err := (&net.ListenConfig{}).Listen(t.Context(), "tcp", localhost+":0")
@@ -267,7 +293,7 @@ func startProxyWithClock(
 		_ = server.Close()
 	})
 
-	return listener.Addr().String(), out, server
+	return listener.Addr().String(), out, server, alertQueue
 }
 
 // newClient returns an HTTP client that sends requests as they are made,