Alerts to a JSON webhook, with a cooldown and an hourly summary (closes #26)

SWWAF_ALERT_WEBHOOK_URL gets one JSON POST per alert, in SPEC.md's
schema, with SWWAF_ALERT_WEBHOOK_HEADERS: ban and permanent_ban, with
the ban's notes, in observe mode too, marked mode observe and worked
out only when the alert would be sent; source_failure for GeoJS;
file_error for a rule or state file with an error. SWWAF_ALERT_EVENTS
chooses; SWWAF_ALERT_COOLDOWN holds back repeats by netblock, file or
source; past SWWAF_ALERT_MAX_PER_HOUR the hour ends in one summary. A
bounded queue, retried with backoff, holds up no request; a 4xx other
than 408 and 429 gives the alert up. alerts.json keeps the queue, the
cooldowns and the hour. Nothing shows the URL's path or query.

Judgement call: the summary's event is summary, which SPEC.md omits.
Judgement call: an admin's ban raises no alert.

Model: opus-5-5

internal/proxy/alerts_test.go

@@ -0,0 +1,263 @@
+package proxy_test
+
+import (
+	"maps"
+	"net/http"
+	"net/netip"
+	"reflect"
+	"testing"
+	"time"
+
+	"sneak.berlin/go/smallwebwaf/internal/alerts"
+	"sneak.berlin/go/smallwebwaf/internal/bans"
+	"sneak.berlin/go/smallwebwaf/internal/proxy"
+	"sneak.berlin/go/smallwebwaf/internal/requestlog"
+)
+
+const (
+	alertWebhookURL = "SWWAF_ALERT_WEBHOOK_URL"
+	alertMaxPerHour = "SWWAF_ALERT_MAX_PER_HOUR"
+	// alertInstance is the instance every alert of these tests gives.
+	alertInstance = "fsn1app1/gitea"
+)
+
+func TestBanForABrokenLimitRaisesABanAlertWithItsNotes(t *testing.T) {
+	t.Parallel()
+
+	s, clk, server, queue := startWithAlerts(t, map[string]string{
+		rateLimitPerMinute: "1",
+		banScopeV4Prefix:   "24",
+	})
+	start := clk.Now()
+
+	s.get(client, http.StatusOK, requestlog.ActionForward)
+	s.get(client, http.StatusForbidden, requestlog.ActionRateLimited)
+
+	netblock := netip.MustParsePrefix("203.0.113.0/24")
+	ban := server.Ledger.Bans(netblock)[0]
+
+	// A request refused under the ban raises no other alert.
+	clk.advance(time.Minute)
+	s.get(client, http.StatusForbidden, requestlog.ActionBanned)
+
+	wantAlerts(t, queue, banAlert(alerts.EventBan, start, client, bans.Ban{
+		Netblock: netblock, Cause: bans.CauseLimit,
+		Reason: "requests per minute over the limit of 1", Notes: ban.Notes,
+	}, requestlog.FormatTime(start.Add(time.Hour))))
+
+	if ban.Notes.Limit != 1 || ban.Notes.Request.Path != "/" {
+		t.Errorf("the alert's notes are %+v, want those of the broken limit", ban.Notes)
+	}
+}
+
+func TestAttackBanRaisesABanAlertThenAPermanentBanAlert(t *testing.T) {
+	t.Parallel()
+
+	s, clk, server, queue := startWithAlerts(t, map[string]string{
+		rulesDir: writeRules(t, testRules),
+	})
+	start := clk.Now()
+	netblock := netip.MustParsePrefix(client + "/32")
+	other := netip.MustParsePrefix(otherClient + "/32")
+
+	// The probe bans the client for seven days, and its next request makes
+	// the ban permanent. The request after that changes nothing.
+	s.request(client, "/.env", http.StatusForbidden, requestlog.ActionBanned)
+
+	attackBan := server.Ledger.Bans(netblock)[0]
+
+	clk.advance(time.Minute)
+	s.get(client, http.StatusForbidden, requestlog.ActionBanned)
+
+	permanentBan := server.Ledger.Bans(netblock)[0]
+
+	s.get(client, http.StatusForbidden, requestlog.ActionBanned)
+
+	// Another client's probe after its first ban has run out without a
+	// request makes a permanent ban at once.
+	s.request(otherClient, "/.env", http.StatusForbidden, requestlog.ActionBanned)
+	clk.advance(7 * 24 * time.Hour)
+	s.request(otherClient, "/.env", http.StatusForbidden, requestlog.ActionBanned)
+
+	otherBans := server.Ledger.Bans(other)
+
+	wantAlerts(t, queue,
+		attackAlert(alerts.EventBan, start, client, attackBan,
+			requestlog.FormatTime(start.Add(7*24*time.Hour))),
+		attackAlert(alerts.EventPermanentBan, start.Add(time.Minute), client,
+			permanentBan, "permanent"),
+		attackAlert(alerts.EventBan, start.Add(time.Minute), otherClient, otherBans[0],
+			requestlog.FormatTime(start.Add(time.Minute+7*24*time.Hour))),
+		attackAlert(alerts.EventPermanentBan, start.Add(time.Minute+7*24*time.Hour),
+			otherClient, otherBans[1], "permanent"),
+	)
+}
+
+func TestObserveModeRaisesTheBanAlertsItWouldHave(t *testing.T) {
+	t.Parallel()
+
+	s, clk, server, queue := startWithAlerts(t, map[string]string{
+		mode:               observe,
+		rateLimitPerMinute: "2",
+		rulesDir:           writeRules(t, testRules),
+	})
+	start := clk.Now()
+
+	// A ban for a clear sign of attack, which a request under it would make
+	// permanent.
+	group := netip.MustParsePrefix(ipv6Group)
+	attackBan, _ := server.Ledger.BanForAttack(group, start, bans.Notes{RuleID: "probe"})
+
+	// The third request breaks the limit, and so does the fourth, within the
+	// cooldown, which raises nothing. The probe is a clear sign of attack.
+	for range 4 {
+		s.get(client, http.StatusOK, requestlog.ActionForward)
+	}
+
+	s.request(otherClient, "/.env", http.StatusOK, requestlog.ActionForward)
+	line := s.get(ipv6Client, http.StatusOK, requestlog.ActionForward)
+
+	// No ban is made, and none made permanent.
+	if held := server.Ledger.Snapshot(); len(held) != 1 || held[0] != attackBan ||
+		line.BanExpires != requestlog.FormatTime(attackBan.Expires) {
+		t.Errorf("the ledger holds %+v, and the log line gives %s, want the ban "+
+			"for the attack alone, as it was", held, line.BanExpires)
+	}
+
+	waiting := queue.Snapshot().Waiting
+	if len(waiting) != 3 || queue.Suppressed() != 0 {
+		t.Fatalf("%d alerts wait and %d are held back, want 3 and 0: %+v",
+			len(waiting), queue.Suppressed(), waiting)
+	}
+
+	limitNotes, _ := waiting[0].Detail["notes"].(bans.Notes)
+	attackNotes, _ := waiting[1].Detail["notes"].(bans.Notes)
+
+	if limitNotes.Limit != 2 || limitNotes.Request.Path != "/" ||
+		attackNotes.Request.Path != "/.env" {
+		t.Errorf("the notes are %+v and %+v, want those of the broken limit and "+
+			"of the probe", limitNotes, attackNotes)
+	}
+
+	// Each alert is the one enforce mode would have raised, with mode
+	// observe in its detail.
+	want := []alerts.Alert{
+		banAlert(alerts.EventBan, start, client, bans.Ban{
+			Netblock: netip.MustParsePrefix(client + "/32"), Cause: bans.CauseLimit,
+			Reason: "requests per minute over the limit of 2", Notes: limitNotes,
+		}, requestlog.FormatTime(start.Add(time.Hour))),
+		attackAlert(alerts.EventBan, start, otherClient, bans.Ban{
+			Netblock: netip.MustParsePrefix(otherClient + "/32"), Notes: attackNotes,
+		}, requestlog.FormatTime(start.Add(7*24*time.Hour))),
+		attackAlert(alerts.EventPermanentBan, start, ipv6Client, attackBan, permanent),
+	}
+	for _, alert := range want {
+		alert.Detail["mode"] = observe
+	}
+
+	wantAlerts(t, queue, want...)
+}
+
+func TestObserveModeWorksOutABanOnlyWhenItsAlertWouldBeSent(t *testing.T) {
+	t.Parallel()
+
+	s, _, _, queue := startWithAlerts(t, map[string]string{
+		mode:               observe,
+		rateLimitPerMinute: "2",
+		rulesDir:           writeRules(t, testRules),
+		alertMaxPerHour:    "2",
+	})
+
+	// The client's third request breaks the limit, and raises the first
+	// alert of the hour. Its fourth is within the cooldown.
+	for range 4 {
+		s.get(client, http.StatusOK, requestlog.ActionForward)
+	}
+
+	// The other client's first probe raises the second. Its second probe is
+	// within the cooldown.
+	for range 2 {
+		s.request(otherClient, "/.env", http.StatusOK, requestlog.ActionForward)
+	}
+
+	// The IPv6 client's third request breaks the limit past the two alerts
+	// an hour.
+	for range 3 {
+		s.get(ipv6Client, http.StatusOK, requestlog.ActionForward)
+	}
+
+	// Had the ban been worked out for any of the requests within the
+	// cooldown or past the two an hour, its alert would have been raised,
+	// held back and counted.
+	if waiting := queue.Snapshot().Waiting; len(waiting) != 2 || queue.Suppressed() != 0 {
+		t.Errorf("%d alerts wait and %d are held back, want 2 and 0: %+v",
+			len(waiting), queue.Suppressed(), waiting)
+	}
+}
+
+// startWithAlerts is startWithClock with alerts to a webhook, which is
+// never sent them, and returns the queue they wait in as well.
+func startWithAlerts(
+	t *testing.T, env map[string]string,
+) (*sender, *clock, *proxy.Server, *alerts.Queue) {
+	t.Helper()
+
+	app := startApp(t, func(http.ResponseWriter, *http.Request) {})
+	clk := &clock{now: time.Date(2026, 10, 6, 0, 0, 0, 0, time.UTC)}
+	settings := map[string]string{
+		trustedProxies:  trustLocalhost,
+		alertWebhookURL: "https://alerts.example/smallwebwaf",
+		instanceName:    alertInstance,
+	}
+	maps.Copy(settings, env)
+
+	addr, out, server, queue := startProxyWithAlerts(t, app.URL, "", clk.Now, settings)
+
+	return &sender{t: t, addr: addr, out: out}, clk, server, queue
+}
+
+// banAlert returns the alert for event, raised by a request from client at
+// the time raised, for ban, with its netblock, cause, reason and notes,
+// which ends at expires, as the log line gives it.
+func banAlert(
+	event string, raised time.Time, client string, ban bans.Ban, expires string,
+) alerts.Alert {
+	return alerts.Alert{
+		Instance: alertInstance,
+		Time:     raised,
+		Event:    event,
+		Client:   netip.MustParseAddr(client),
+		Netblock: ban.Netblock,
+		Reason:   ban.Reason,
+		Detail: map[string]any{
+			"cause": ban.Cause, "ban_expires": expires, "notes": ban.Notes,
+		},
+	}
+}
+
+// attackAlert is banAlert for a ban for the probe rule of testRules, with
+// the netblock and the notes of ban.
+func attackAlert(
+	event string, raised time.Time, client string, ban bans.Ban, expires string,
+) alerts.Alert {
+	return banAlert(event, raised, client, bans.Ban{
+		Netblock: ban.Netblock, Cause: bans.CauseAttack, Reason: "matched the rule probe",
+		Notes: ban.Notes,
+	}, expires)
+}
+
+// wantAlerts checks the alerts waiting in queue, in order.
+func wantAlerts(t *testing.T, queue *alerts.Queue, want ...alerts.Alert) {
+	t.Helper()
+
+	got := queue.Snapshot().Waiting
+	if len(got) != len(want) {
+		t.Fatalf("%d alerts wait, want %d: %+v", len(got), len(want), got)
+	}
+
+	for i := range want {
+		if !reflect.DeepEqual(got[i], want[i]) {
+			t.Errorf("alert %d is\n%+v\nwant\n%+v", i, got[i], want[i])
+		}
+	}
+}

internal/proxy/bans.go

@@ -4,6 +4,7 @@ import (
 	"net/netip"
 	"time"
 
+	"sneak.berlin/go/smallwebwaf/internal/alerts"
 	"sneak.berlin/go/smallwebwaf/internal/bans"
 	"sneak.berlin/go/smallwebwaf/internal/requestlog"
 	"sneak.berlin/go/smallwebwaf/internal/rules"
@@ -16,18 +17,25 @@ func (rq *request) banResponse(action string) *refusal {
 }
 
 // banned reports whether a ban on a netblock the client is in covers the
-// request at now, and notes for the log line when that ban ends.
+// request at now, and notes for the log line when that ban ends. A
+// request that makes the ban permanent, or in observe mode would have,
+// raises the alert for it.
 func (rq *request) banned(now time.Time) bool {
 	check := rq.h.ledger.Check
 	if rq.h.config.Observe {
-		check = rq.h.ledger.Find // in observe mode the ban refuses nothing
+		check = rq.h.ledger.Find // the ban refuses nothing, and stays as it is
 	}
 
-	ban, banned := check(rq.client, now)
+	ban, banned, madePermanent := check(rq.client, now)
 	if banned {
 		rq.line.BanExpires = banExpires(ban)
 	}
 
+	if madePermanent {
+		ban.Expires = time.Time{} // the ban made permanent, which Find leaves as it is
+		rq.alertBan(ban)
+	}
+
 	return banned
 }
 
@@ -35,7 +43,8 @@ func (rq *request) banned(now time.Time) bool {
 // client's counts for the log line, and reports whether the request takes
 // the client over a limit. In enforce mode such a request bans the
 // client's netblock, and sets the client's counters back to zero; in
-// observe mode it does neither.
+// observe mode it does neither, and raises the alert for the ban it would
+// have made, if that alert would be sent.
 func (rq *request) limitBroken(now time.Time) bool {
 	group := clientGroup(rq.client)
 
@@ -49,41 +58,122 @@ func (rq *request) limitBroken(now time.Time) bool {
 	rq.line.LimitHit = hit.Window
 	rq.line.Offence = requestlog.OffenceLimit
 
-	if rq.h.config.Observe {
+	netblock := rq.h.netblock(rq.client)
+	if rq.h.config.Observe && !rq.wouldAlertBan(netblock, now, bans.CauseLimit) {
 		return true
 	}
 
-	netblock := rq.h.netblock(rq.client)
-	ban := rq.h.ledger.BanForLimit(netblock, now, bans.Notes{
+	notes := bans.Notes{
 		Country:  rq.line.Country,
 		Limit:    hit.Limit,
 		Window:   hit.Window,
 		Count:    hit.Requests,
 		Request:  rq.noted(now),
 		Requests: rq.netblockRequests(netblock),
-	})
+	}
+
+	if rq.h.config.Observe {
+		ban, wouldBan := rq.h.ledger.WouldBanForLimit(netblock, now, notes)
+		if wouldBan {
+			rq.alertBan(ban)
+		}
+
+		return true
+	}
+
+	ban, made := rq.h.ledger.BanForLimit(netblock, now, notes)
 	rq.h.limiter.Reset(group)
 	rq.line.BanExpires = banExpires(ban)
 
+	if made {
+		rq.alertBan(ban)
+	}
+
 	return true
 }
 
 // banForAttack bans the client's netblock at now for a clear sign of
-// attack, the match of rule, a ban rule.
+// attack, the match of rule, a ban rule. In observe mode it makes no ban,
+// and raises the alert for the ban it would have made, if that alert
+// would be sent.
 func (rq *request) banForAttack(now time.Time, rule rules.Rule) {
 	netblock := rq.h.netblock(rq.client)
-	ban := rq.h.ledger.BanForAttack(netblock, now, bans.Notes{
+	if rq.h.config.Observe && !rq.wouldAlertBan(netblock, now, bans.CauseAttack) {
+		return
+	}
+
+	notes := bans.Notes{
 		Country:  rq.line.Country,
 		RuleID:   rule.ID,
 		Target:   rule.Target,
 		Request:  rq.noted(now),
 		Requests: rq.netblockRequests(netblock),
-	})
+	}
+
+	if rq.h.config.Observe {
+		ban, wouldBan := rq.h.ledger.WouldBanForAttack(netblock, now, notes)
+		if wouldBan {
+			rq.alertBan(ban)
+		}
+
+		return
+	}
+
+	ban, made := rq.h.ledger.BanForAttack(netblock, now, notes)
 	rq.line.BanExpires = banExpires(ban)
+
+	if made {
+		rq.alertBan(ban)
+	}
 }
 
-// noted is the request, refused at now with SWWAF_BAN_RESPONSE, as the
-// notes of the ban it makes keep it.
+// wouldAlertBan reports whether the alert for a ban on netblock for cause
+// made at now would be sent. In observe mode the ban the request would
+// have made is worked out only then, at most once per
+// SWWAF_ALERT_COOLDOWN and never with no webhook set: its notes count the
+// netblock's requests, which can mean going through every client.
+func (rq *request) wouldAlertBan(
+	netblock netip.Prefix, now time.Time, cause string,
+) bool {
+	event := alerts.EventBan
+	if rq.h.ledger.WouldBePermanent(netblock, now, cause) {
+		event = alerts.EventPermanentBan
+	}
+
+	return rq.h.alerts.WouldSend(event, netblock)
+}
+
+// alertBan raises the alert for ban, which the request made, or made
+// permanent: permanent_ban for a permanent ban, ban for another. Its
+// detail gives the ban's cause, when it ends, and its notes, and in
+// observe mode, where ban is the ban that would have been made, or made
+// permanent, mode, observe.
+func (rq *request) alertBan(ban bans.Ban) {
+	event := alerts.EventBan
+	if ban.Permanent() {
+		event = alerts.EventPermanentBan
+	}
+
+	detail := map[string]any{
+		"cause": ban.Cause, "ban_expires": banExpires(ban), "notes": ban.Notes,
+	}
+	if rq.h.config.Observe {
+		detail["mode"] = "observe"
+	}
+
+	rq.h.alerts.Raise(alerts.Alert{
+		Event:    event,
+		Client:   rq.client,
+		Netblock: ban.Netblock,
+		Country:  ban.Notes.Country,
+		Reason:   ban.Reason,
+		Detail:   detail,
+	})
+}
+
+// noted is the request, refused at now with SWWAF_BAN_RESPONSE, or in
+// observe mode as it would have been, as the notes of the ban it makes
+// keep it.
 func (rq *request) noted(now time.Time) bans.Request {
 	return bans.Request{
 		Time:      now,

internal/proxy/proxy.go

@@ -11,6 +11,7 @@ import (
 	"strings"
 	"time"
 
+	"sneak.berlin/go/smallwebwaf/internal/alerts"
 	"sneak.berlin/go/smallwebwaf/internal/bans"
 	"sneak.berlin/go/smallwebwaf/internal/config"
 	"sneak.berlin/go/smallwebwaf/internal/lookup"
@@ -63,6 +64,9 @@ type Params struct {
 	// Rules are the rule files' rules, which each request is checked
 	// against.
 	Rules *rules.Files
+	// Alerts receive the alert for each ban the proxy makes or makes
+	// permanent, and for GeoJS failing.
+	Alerts *alerts.Queue
 }
 
 // Server is the server smallwebwaf runs, with the parts of the proxy
@@ -110,8 +114,10 @@ func New(params Params) *Server {
 			Now:        params.Now,
 			ProcessLog: params.ProcessLog,
 			Metrics:    m,
+			Alerts:     params.Alerts,
 		}),
-		rules: params.Rules,
+		rules:  params.Rules,
+		alerts: params.Alerts,
 	}
 	m.AddBansAndClients(h.ledger, h.limiter, params.Now)
 	m.AddRules(params.Rules)
@@ -150,6 +156,7 @@ type handler struct {
 	ledger     *bans.Ledger
 	geojs      *lookup.GeoJS
 	rules      *rules.Files
+	alerts     *alerts.Queue
 }
 
 // newTransport returns what carries requests to the app. It never goes

internal/proxy/proxy_test.go

@@ -14,6 +14,7 @@ import (
 	"testing"
 	"time"
 
+	"sneak.berlin/go/smallwebwaf/internal/alerts"
 	"sneak.berlin/go/smallwebwaf/internal/config"
 	"sneak.berlin/go/smallwebwaf/internal/proxy"
 	"sneak.berlin/go/smallwebwaf/internal/requestlog"
@@ -223,6 +224,20 @@ func startProxyWithClock(
 ) (string, *output, *proxy.Server) {
 	t.Helper()
 
+	addr, out, server, _ := startProxyWithAlerts(t, appURL, geojsURL, now, env)
+
+	return addr, out, server
+}
+
+// startProxyWithAlerts is startProxyWithClock, and returns the queue of
+// the alerts the proxy raises as well, as the settings in env make it. No
+// alert is sent from it: they wait in it, for the test to look at.
+func startProxyWithAlerts(
+	t *testing.T, appURL, geojsURL string, now func() time.Time,
+	env map[string]string,
+) (string, *output, *proxy.Server, *alerts.Queue) {
+	t.Helper()
+
 	settings := map[string]string{"SWWAF_UPSTREAM_URL": appURL, rulesDir: t.TempDir()}
 	maps.Copy(settings, env)
 
@@ -245,6 +260,16 @@ func startProxyWithClock(
 		t.Fatalf("rule files: %v", err)
 	}
 
+	alertQueue := alerts.New(alerts.Params{
+		WebhookURL: cfg.AlertWebhookURL,
+		Events:     cfg.AlertEvents,
+		Cooldown:   cfg.AlertCooldown,
+		MaxPerHour: cfg.AlertMaxPerHour,
+		Instance:   cfg.InstanceName,
+		Now:        now,
+		ProcessLog: processLog,
+	})
+
 	server := proxy.New(proxy.Params{
 		Config:     cfg,
 		RequestLog: out,
@@ -252,6 +277,7 @@ func startProxyWithClock(
 		GeoJSURL:   geojsURL,
 		Now:        now,
 		Rules:      ruleFiles,
+		Alerts:     alertQueue,
 	})
 
 	listener, err := (&net.ListenConfig{}).Listen(t.Context(), "tcp", localhost+":0")
@@ -267,7 +293,7 @@ func startProxyWithClock(
 		_ = server.Close()
 	})
 
-	return listener.Addr().String(), out, server
+	return listener.Addr().String(), out, server, alertQueue
 }
 
 // newClient returns an HTTP client that sends requests as they are made,

internal/proxy/rules.go

@@ -10,8 +10,9 @@ import (
 // checkRules checks the request against the rules of the rule files at
 // now, notes the ids of those it matches in the log line, and returns the
 // action of the rule that refuses it, ActionRuleBlocked for a block rule
-// and ActionBanned for a ban rule, or "" when none does. In enforce mode
-// a ban rule bans the client's netblock for a clear sign of attack.
+// and ActionBanned for a ban rule, or "" when none does. A ban rule bans
+// the client's netblock for a clear sign of attack, or in observe mode
+// raises the alert for the ban it would have made.
 func (rq *request) checkRules(now time.Time) string {
 	matched := rq.h.rules.Match(rq.in)
 
@@ -29,9 +30,7 @@ func (rq *request) checkRules(now time.Time) string {
 	case rules.ActionBlock:
 		return requestlog.ActionRuleBlocked
 	case rules.ActionBan:
-		if !rq.h.config.Observe {
-			rq.banForAttack(now, last)
-		}
+		rq.banForAttack(now, last)
 
 		return requestlog.ActionBanned
 	default: