Instance name on process log lines and every metric (closes #91)

Process log lines carry instance, as request lines do; the instance name
is read before the other settings, so the line saying a setting is
invalid carries it too. Every metric, Go's and the process's included,
carries the label instance, set once on the registry. README.md says so,
and that Prometheus keeps it as exported_instance unless the scrape sets
honor_labels. An instance name that is not valid UTF-8 stops the start,
as the metrics library panics on such a label.

Tests that read metrics expect the label; one helper replaces the alert
tests' loops that wait for them.

Judgement call: the label is named instance, as in the log lines and
alerts, although Prometheus gives each target a label of that name.

Model: opus-5-5

Dockerfile

@@ -167,6 +167,10 @@ RUN groupadd --system --gid 65532 smallwebwaf \
 # smallwebwaf user at each start.
 RUN mkdir /var/lib/smallwebwaf
 
+# The default rule file, in SWWAF_RULES_DIR by default, where an app's
+# Dockerfile can copy rule files of its own beside it.
+COPY share/rules.d/00-default.rules /etc/smallwebwaf/rules.d/00-default.rules
+
 # runsvinit starts runit's runsvdir on /etc/service, where Ubuntu's sv
 # looks too.
 COPY --chmod=755 share/smallwebwaf.run /etc/service/smallwebwaf/run

SPEC.md

@@ -32,7 +32,7 @@ from a directory of hand-editable text files.
 - Defence against traffic floods that saturate the host's network link. That
   needs help upstream of the host.
 - A web UI or a configuration file. Settings are environment variables. Apart
-  from settings given as files (the `_FILE` form of any setting, such as
+  from settings given as files (the `_FILE` form of a setting, such as
   `SWWAF_ADMIN_TOKEN_FILE`, and `SWWAF_LOG_REMOTE_TLS_CA_FILE`), its own state
   files and the lookup database, the only files read are the rule files, which
   hold one regex per line and nothing more elaborate.
@@ -298,7 +298,8 @@ it.
 - A list set to an empty value is an empty list, and replaces the default.
 - Every setting may instead be given as a file holding the value, named by the
   setting's name with `_FILE` added, such as `SWWAF_ADMIN_TOKEN_FILE`, for
-  secrets and long lists.
+  secrets and long lists. `SWWAF_LOG_REMOTE_TLS_CA_FILE`, whose value names a
+  file already, has no `_FILE` form.
 - Settings, including those given as files, are read once at start; changing one
   means restarting the container. The files `smallwebwaf` watches while it runs
   are its state files, its rule files and the lookup database.

go.mod

@@ -3,6 +3,7 @@ module sneak.berlin/go/smallwebwaf
 go 1.26.0
 
 require (
+	github.com/fsnotify/fsnotify v1.10.1
 	github.com/hashicorp/golang-lru/v2 v2.0.7
 	github.com/prometheus/client_golang v1.24.1
 )

go.sum

@@ -4,6 +4,8 @@ github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UF
 github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/fsnotify/fsnotify v1.10.1 h1:b0/UzAf9yR5rhf3RPm9gf3ehBPpf0oZKIjtpKrx59Ho=
+github.com/fsnotify/fsnotify v1.10.1/go.mod h1:TLheqan6HD6GBK6PrDWyDPBaEV8LspOxvPSjC+bVfgo=
 github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
 github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU=
 github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k=

internal/alerts/alerts.go

@@ -0,0 +1,882 @@
+// Package alerts sends alerts on bans, on a source that fails and on a
+// file with an error to each destination set: to the webhook
+// SWWAF_ALERT_WEBHOOK_URL names, each as one JSON object, as the "Alert
+// webhook schema" section of SPEC.md describes, to the Slack incoming
+// webhook SWWAF_ALERT_SLACK_WEBHOOK_URL names, as a message, and to the
+// ntfy topic SWWAF_ALERT_NTFY_URL names. A repeat within
+// SWWAF_ALERT_COOLDOWN is held back, and so is an alert past
+// SWWAF_ALERT_MAX_PER_HOUR, for the hour's summary. The others wait in a
+// bounded queue of each destination's own, so that a destination that is
+// slow or unreachable holds up neither the others nor any request. The
+// state is written to alerts.json and read from it by the state package.
+// Nothing logged names a destination's URL, whose path or query can carry
+// a secret.
+package alerts
+
+import (
+	"bytes"
+	"cmp"
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"log/slog"
+	"maps"
+	"net/http"
+	"net/netip"
+	"net/url"
+	"slices"
+	"strings"
+	"sync"
+	"sync/atomic"
+	"time"
+)
+
+// The events an alert is for, as SWWAF_ALERT_EVENTS names them.
+const (
+	// EventBan is a ban smallwebwaf made.
+	EventBan = "ban"
+	// EventPermanentBan is a permanent ban smallwebwaf made, or a ban it
+	// made permanent.
+	EventPermanentBan = "permanent_ban"
+	// EventWAFBlock, EventAnomaly and EventReputationHit come with the
+	// Core Rule Set, the anomaly thresholds and the reputation sources;
+	// nothing raises them yet.
+	EventWAFBlock      = "waf_block"
+	EventAnomaly       = "anomaly"
+	EventReputationHit = "reputation_hit"
+	// EventSourceFailure is GeoJS failing or refusing smallwebwaf.
+	EventSourceFailure = "source_failure"
+	// EventFileError is a rule file or state file edited while smallwebwaf
+	// runs that does not parse, or a state file that cannot be written.
+	EventFileError = "file_error"
+	// EventSummary is the summary of the alerts an hour held back past
+	// SWWAF_ALERT_MAX_PER_HOUR. SWWAF_ALERT_EVENTS does not name it.
+	EventSummary = "summary"
+)
+
+// Events returns every event SWWAF_ALERT_EVENTS can name, which is its
+// default.
+func Events() []string {
+	return []string{
+		EventBan, EventPermanentBan, EventWAFBlock, EventAnomaly,
+		EventReputationHit, EventSourceFailure, EventFileError,
+	}
+}
+
+// The destinations alerts are sent to, as the metrics and alerts.json
+// name them.
+const (
+	// DestinationWebhook is the webhook SWWAF_ALERT_WEBHOOK_URL names.
+	DestinationWebhook = "webhook"
+	// DestinationSlack is the Slack incoming webhook
+	// SWWAF_ALERT_SLACK_WEBHOOK_URL names.
+	DestinationSlack = "slack"
+	// DestinationNtfy is the ntfy topic SWWAF_ALERT_NTFY_URL names.
+	DestinationNtfy = "ntfy"
+)
+
+// Destinations returns every destination alerts can be sent to.
+func Destinations() []string {
+	return []string{DestinationWebhook, DestinationSlack, DestinationNtfy}
+}
+
+const (
+	// queueSize is the most alerts that wait to be sent to a destination.
+	// Past it, the oldest is dropped.
+	queueSize = 1000
+	// sendTimeout bounds one request to a destination.
+	sendTimeout = 10 * time.Second
+	// After a request to a destination fails, the alert is sent again a
+	// second later, and retryDelayFactor times as long after each further
+	// failure in a row, up to a minute.
+	firstRetryDelay  = time.Second
+	retryDelayFactor = 2
+	maxRetryDelay    = time.Minute
+	// maxAnswerBytes is the most of a destination's answer that is read.
+	maxAnswerBytes = 64 << 10
+)
+
+var (
+	errStatus = errors.New("the destination answered")
+	// errRefused is a 4xx answer other than 408 and 429: the destination
+	// refuses the alert itself, and would refuse it again.
+	errRefused = errors.New("the destination refused the alert, answering")
+)
+
+// Params are what New needs. With none of WebhookURL, SlackURL and
+// NtfyURL set, no alert is sent.
+type Params struct {
+	// WebhookURL is where each alert is posted as JSON
+	// (SWWAF_ALERT_WEBHOOK_URL), nil while it is unset. WebhookHeaders
+	// are sent with each (SWWAF_ALERT_WEBHOOK_HEADERS).
+	WebhookURL     *url.URL
+	WebhookHeaders http.Header
+	// SlackURL is the Slack incoming webhook each alert is posted to as a
+	// message (SWWAF_ALERT_SLACK_WEBHOOK_URL), nil while it is unset.
+	SlackURL *url.URL
+	// NtfyURL is the ntfy topic each alert is published to
+	// (SWWAF_ALERT_NTFY_URL), nil while it is unset. NtfyToken, unless
+	// empty, is sent with each as a bearer token (SWWAF_ALERT_NTFY_TOKEN).
+	NtfyURL   *url.URL
+	NtfyToken string
+	// Events are the events alerts are sent for (SWWAF_ALERT_EVENTS).
+	Events []string
+	// Cooldown is how long a repeat of an alert is held back
+	// (SWWAF_ALERT_COOLDOWN), 0 for no time. MaxPerHour is the most alerts
+	// sent in an hour (SWWAF_ALERT_MAX_PER_HOUR), 0 for no limit.
+	Cooldown   time.Duration
+	MaxPerHour int
+	// Instance is SWWAF_INSTANCE_NAME, which every alert gives.
+	Instance string
+	// Now tells the time of an alert, normally time.Now in UTC.
+	Now func() time.Time
+	// ProcessLog receives the requests to a destination that fail.
+	ProcessLog *slog.Logger
+}
+
+// Alert is one alert, as the webhook is sent it and alerts.json holds it,
+// with the fields of the "Alert webhook schema" section of SPEC.md. ASN
+// and ASName are empty until AS numbers are looked up.
+//
+//nolint:tagliatelle // SPEC.md's alert webhook schema names its fields in snake_case
+type Alert struct {
+	Instance string       `json:"instance"`
+	Time     time.Time    `json:"time"`
+	Event    string       `json:"event"`
+	Client   netip.Addr   `json:"client"`
+	Netblock netip.Prefix `json:"netblock"`
+	ASN      string       `json:"asn"`
+	ASName   string       `json:"as_name"`
+	Country  string       `json:"country"`
+	// Reason is a short sentence, and Detail what is particular to the
+	// event: for a file_error, its "file", and for a source_failure, its
+	// "source", which the cooldown tells repeats by.
+	Reason string         `json:"reason"`
+	Detail map[string]any `json:"detail"`
+	// SuppressedRepeats is how many repeats of the alert the cooldown
+	// held back since the last one let through.
+	SuppressedRepeats int `json:"suppressed_repeats"`
+}
+
+// Cooldown is, for an event on a netblock, or about a file or a source,
+// when the last alert let through was raised, and how many repeats the
+// cooldown has held back since, as alerts.json holds it.
+//
+//nolint:tagliatelle // the state files use snake_case, as the request log does
+type Cooldown struct {
+	Event             string       `json:"event"`
+	Netblock          netip.Prefix `json:"netblock"`
+	File              string       `json:"file,omitempty"`
+	Source            string       `json:"source,omitempty"`
+	Sent              time.Time    `json:"sent"`
+	SuppressedRepeats int          `json:"suppressed_repeats"`
+}
+
+// Hour is the hour under way, by the clock, as alerts.json holds it: when
+// it started, how many alerts were let through in it, and how many were
+// held back in it past MaxPerHour, by event, for its summary.
+//
+//nolint:tagliatelle // the state files use snake_case, as the request log does
+type Hour struct {
+	Start    time.Time      `json:"start"`
+	Sent     int            `json:"sent"`
+	HeldBack map[string]int `json:"held_back"`
+}
+
+// State is what alerts.json holds: the cooldowns, the hour under way, and
+// for each destination set, the alerts waiting to be sent to it, oldest
+// first.
+type State struct {
+	Cooldowns []Cooldown         `json:"cooldowns"`
+	Hour      Hour               `json:"hour"`
+	Waiting   map[string][]Alert `json:"waiting"`
+}
+
+// Counts are, for a destination, how many alerts it took, how many
+// requests to it failed, and how many alerts were dropped from its full
+// queue or given up as it refused them.
+type Counts struct {
+	Sent, Failed, Dropped int64
+}
+
+// Queue takes the alerts raised, holds back those it must, and sends the
+// others to each destination set, from a queue of the destination's own.
+// It is safe for concurrent use.
+type Queue struct {
+	params Params
+	// destinations are the destinations set, in the order of
+	// Destinations.
+	destinations []*destination
+
+	mu sync.Mutex
+	// cooldowns are the alerts last let through, by event and netblock,
+	// file or source.
+	cooldowns map[cooldownKey]*Cooldown
+	hour      Hour
+
+	suppressed atomic.Int64
+}
+
+// destination is a destination set, with the alerts waiting to be sent
+// to it. Its mu is taken after the Queue's, never before.
+type destination struct {
+	// name is how the metrics and alerts.json name the destination, and
+	// setting the setting that is its URL, which the log names in place
+	// of the URL.
+	name    string
+	setting string
+	url     *url.URL
+	// message returns the body an alert is posted with, and the headers
+	// sent with it.
+	message func(alert *Alert) ([]byte, http.Header, error)
+	// httpClient follows no redirect: a redirect is a failure.
+	httpClient *http.Client
+	processLog *slog.Logger
+	// queued receives a value when an alert joins the queue, unless one
+	// waits already, so that run looks at the queue again.
+	queued chan struct{}
+
+	mu sync.Mutex
+	// waiting are the alerts waiting to be sent, oldest first.
+	waiting []*Alert
+
+	sent, failed, dropped atomic.Int64
+}
+
+// cooldownKey is what makes an alert a repeat of another: the same event
+// on the same netblock, and about the same file or source, as its detail
+// names them. Each is empty for an alert without one.
+type cooldownKey struct {
+	event    string
+	netblock netip.Prefix
+	file     string
+	source   string
+}
+
+// cooldownKeyOf returns what makes another alert a repeat of alert.
+func cooldownKeyOf(alert *Alert) cooldownKey {
+	file, _ := alert.Detail["file"].(string)
+	source, _ := alert.Detail["source"].(string)
+
+	return cooldownKey{alert.Event, alert.Netblock, file, source}
+}
+
+// New returns a Queue with no alert yet.
+func New(params Params) *Queue {
+	q := &Queue{
+		params:    params,
+		cooldowns: map[cooldownKey]*Cooldown{},
+		hour:      Hour{HeldBack: map[string]int{}},
+	}
+
+	if params.WebhookURL != nil {
+		q.addDestination(DestinationWebhook, "SWWAF_ALERT_WEBHOOK_URL",
+			params.WebhookURL, q.webhookMessage)
+	}
+
+	if params.SlackURL != nil {
+		q.addDestination(DestinationSlack, "SWWAF_ALERT_SLACK_WEBHOOK_URL",
+			params.SlackURL, slackMessage)
+	}
+
+	if params.NtfyURL != nil {
+		q.addDestination(DestinationNtfy, "SWWAF_ALERT_NTFY_URL",
+			params.NtfyURL, q.ntfyMessage)
+	}
+
+	return q
+}
+
+// Raise sends alert, which names its event and what is particular to it,
+// unless no destination is set or SWWAF_ALERT_EVENTS leaves its event
+// out. It gives alert the instance and the time. An alert that repeats
+// the last one let through less than Cooldown before is held back and
+// counted, and the next one let through gives that count. Past MaxPerHour
+// alerts let through in the hour under way, by the clock, an alert is
+// held back for that hour's summary instead, which is sent once the hour
+// has ended; it starts no cooldown, and the repeats held back before it
+// are given by the next alert let through. Raise never waits: an alert
+// let through joins the queue of each destination, from which Run sends
+// it, and with queueSize alerts waiting for a destination, the oldest is
+// dropped.
+func (q *Queue) Raise(alert Alert) {
+	if len(q.destinations) == 0 || !slices.Contains(q.params.Events, alert.Event) {
+		return
+	}
+
+	q.mu.Lock()
+	defer q.mu.Unlock()
+
+	now := q.params.Now()
+	alert.Instance = q.params.Instance
+	alert.Time = now
+
+	if q.repeat(&alert, now) {
+		q.suppressed.Add(1)
+
+		return
+	}
+
+	q.endHour(now)
+
+	if q.params.MaxPerHour > 0 && q.hour.Sent >= q.params.MaxPerHour {
+		q.hour.HeldBack[alert.Event]++
+		q.suppressed.Add(1)
+
+		return
+	}
+
+	q.startCooldown(&alert, now)
+	q.hour.Sent++
+	q.queue(&alert)
+}
+
+// WouldSend reports whether Raise would let an alert for event on
+// netblock through now: a destination is set, SWWAF_ALERT_EVENTS chooses
+// event, no alert for event on netblock was let through less than
+// Cooldown before, and fewer than MaxPerHour alerts have been let through
+// in the hour under way. Unlike Raise, it counts nothing.
+func (q *Queue) WouldSend(event string, netblock netip.Prefix) bool {
+	if len(q.destinations) == 0 || !slices.Contains(q.params.Events, event) {
+		return false
+	}
+
+	q.mu.Lock()
+	defer q.mu.Unlock()
+
+	now := q.params.Now()
+
+	last, found := q.cooldowns[cooldownKey{event: event, netblock: netblock}]
+	if q.params.Cooldown > 0 && found && now.Sub(last.Sent) < q.params.Cooldown {
+		return false
+	}
+
+	q.endHour(now)
+
+	return q.params.MaxPerHour == 0 || q.hour.Sent < q.params.MaxPerHour
+}
+
+// Run sends the alerts waiting to each destination, from its own queue,
+// as destination.run does, until ctx is done. It also ends each hour as
+// Raise does, so that the hour's summary is sent as it ends. With no
+// destination set, it returns at once.
+func (q *Queue) Run(ctx context.Context) {
+	if len(q.destinations) == 0 {
+		return
+	}
+
+	var sending sync.WaitGroup
+
+	for _, d := range q.destinations {
+		sending.Go(func() { d.run(ctx) })
+	}
+
+	for {
+		q.mu.Lock()
+		untilHourEnds := q.hour.Start.Add(time.Hour).Sub(q.params.Now())
+		q.mu.Unlock()
+
+		select {
+		case <-ctx.Done():
+			sending.Wait()
+
+			return
+		case <-time.After(untilHourEnds):
+			q.mu.Lock()
+			q.endHour(q.params.Now())
+			q.mu.Unlock()
+		}
+	}
+}
+
+// Counts returns the counts of the destination name, all 0 for one not
+// set.
+func (q *Queue) Counts(name string) Counts {
+	for _, d := range q.destinations {
+		if d.name == name {
+			return Counts{
+				Sent: d.sent.Load(), Failed: d.failed.Load(), Dropped: d.dropped.Load(),
+			}
+		}
+	}
+
+	return Counts{}
+}
+
+// DestinationsSet returns the destinations set, in the order of
+// Destinations.
+func (q *Queue) DestinationsSet() []string {
+	names := make([]string, 0, len(q.destinations))
+	for _, d := range q.destinations {
+		names = append(names, d.name)
+	}
+
+	return names
+}
+
+// Suppressed is how many alerts were held back: by the cooldown, and past
+// MaxPerHour. No destination is sent such an alert.
+func (q *Queue) Suppressed() int64 {
+	return q.suppressed.Load()
+}
+
+// Snapshot returns the queue's state, as alerts.json holds it, with the
+// cooldowns sorted by netblock, then by event, file and source.
+func (q *Queue) Snapshot() State {
+	q.mu.Lock()
+	defer q.mu.Unlock()
+
+	state := State{
+		Cooldowns: make([]Cooldown, 0, len(q.cooldowns)),
+		Hour:      q.hour,
+		Waiting:   map[string][]Alert{},
+	}
+	state.Hour.HeldBack = maps.Clone(q.hour.HeldBack)
+
+	for _, cooldown := range q.cooldowns {
+		state.Cooldowns = append(state.Cooldowns, *cooldown)
+	}
+
+	slices.SortFunc(state.Cooldowns, func(a, b Cooldown) int {
+		return cmp.Or(a.Netblock.Compare(b.Netblock), cmp.Compare(a.Event, b.Event),
+			cmp.Compare(a.File, b.File), cmp.Compare(a.Source, b.Source))
+	})
+
+	for _, d := range q.destinations {
+		state.Waiting[d.name] = d.snapshot()
+	}
+
+	return state
+}
+
+// Load puts state, read from alerts.json, in place of the queue's state.
+// Each cooldown's netblock is masked to its length, so that
+// 203.0.113.9/24 is 203.0.113.0/24. The alerts waiting for a destination
+// that is not set are dropped, and so are the oldest past queueSize
+// alerts waiting for one that is.
+func (q *Queue) Load(state State) {
+	q.mu.Lock()
+	defer q.mu.Unlock()
+
+	q.cooldowns = map[cooldownKey]*Cooldown{}
+
+	for _, cooldown := range state.Cooldowns {
+		cooldown.Netblock = cooldown.Netblock.Masked()
+		key := cooldownKey{cooldown.Event, cooldown.Netblock, cooldown.File, cooldown.Source}
+		q.cooldowns[key] = &cooldown
+	}
+
+	q.hour = state.Hour
+	q.hour.HeldBack = maps.Clone(state.Hour.HeldBack)
+
+	if q.hour.HeldBack == nil {
+		q.hour.HeldBack = map[string]int{}
+	}
+
+	for _, d := range q.destinations {
+		d.load(state.Waiting[d.name])
+	}
+}
+
+// addDestination adds a destination: its name, the setting that gives
+// its URL, that URL, target, and message, which makes the messages sent
+// to it.
+func (q *Queue) addDestination(
+	name, setting string, target *url.URL,
+	message func(alert *Alert) ([]byte, http.Header, error),
+) {
+	q.destinations = append(q.destinations, &destination{
+		name:    name,
+		setting: setting,
+		url:     target,
+		message: message,
+		httpClient: &http.Client{
+			CheckRedirect: func(*http.Request, []*http.Request) error {
+				return http.ErrUseLastResponse
+			},
+		},
+		processLog: q.params.ProcessLog,
+		queued:     make(chan struct{}, 1),
+	})
+}
+
+// repeat reports whether alert, raised at now, repeats the last one let
+// through less than Cooldown before, and counts it if it does.
+func (q *Queue) repeat(alert *Alert, now time.Time) bool {
+	if q.params.Cooldown == 0 {
+		return false
+	}
+
+	last, found := q.cooldowns[cooldownKeyOf(alert)]
+	if !found || now.Sub(last.Sent) >= q.params.Cooldown {
+		return false
+	}
+
+	last.SuppressedRepeats++
+
+	return true
+}
+
+// startCooldown gives alert, let through at now, the count of the repeats
+// held back since the last one let through, and notes alert as the last
+// one let through.
+func (q *Queue) startCooldown(alert *Alert, now time.Time) {
+	if q.params.Cooldown == 0 {
+		return
+	}
+
+	key := cooldownKeyOf(alert)
+
+	last, found := q.cooldowns[key]
+	if found {
+		alert.SuppressedRepeats = last.SuppressedRepeats
+	}
+
+	q.cooldowns[key] = &Cooldown{
+		Event: alert.Event, Netblock: alert.Netblock, File: key.file, Source: key.source,
+		Sent: now,
+	}
+}
+
+// endHour ends the hour under way, if now is past it: it queues that
+// hour's summary when alerts were held back in it past MaxPerHour, and
+// forgets the cooldowns that have run out with no repeat held back, which
+// no alert needs any more.
+func (q *Queue) endHour(now time.Time) {
+	start := now.Truncate(time.Hour)
+	if !start.After(q.hour.Start) {
+		return
+	}
+
+	heldBack := 0
+	for _, count := range q.hour.HeldBack {
+		heldBack += count
+	}
+
+	if heldBack > 0 {
+		q.queue(&Alert{
+			Instance: q.params.Instance,
+			Time:     now,
+			Event:    EventSummary,
+			Reason: fmt.Sprintf("%d alerts held back in the hour from %s, past the %d "+
+				"an hour SWWAF_ALERT_MAX_PER_HOUR allows", heldBack,
+				q.hour.Start.Format(time.RFC3339), q.params.MaxPerHour),
+			Detail: map[string]any{
+				"hour": q.hour.Start, "count": heldBack, "events": q.hour.HeldBack,
+			},
+		})
+	}
+
+	q.hour = Hour{Start: start, HeldBack: map[string]int{}}
+
+	for key, cooldown := range q.cooldowns {
+		if now.Sub(cooldown.Sent) >= q.params.Cooldown && cooldown.SuppressedRepeats == 0 {
+			delete(q.cooldowns, key)
+		}
+	}
+}
+
+// queue adds alert to the alerts waiting for each destination.
+func (q *Queue) queue(alert *Alert) {
+	for _, d := range q.destinations {
+		d.add(alert)
+	}
+}
+
+// run sends the alerts waiting, oldest first, until ctx is done. An alert
+// stays in the queue until the destination answers it with a 2xx status,
+// or refuses it with a 4xx status other than 408 and 429: a refused alert
+// is logged, counted as dropped, and given up, so that the next is sent.
+// Any other request that fails is logged, and the alert sent again
+// firstRetryDelay later, retryDelayFactor times as long after each
+// further failure in a row, up to maxRetryDelay.
+func (d *destination) run(ctx context.Context) {
+	var (
+		retryDelay time.Duration
+		retryAt    time.Time
+	)
+
+	for {
+		alert := d.oldest()
+
+		var due <-chan time.Time // nil while no alert waits
+		if alert != nil {
+			due = time.After(time.Until(retryAt))
+		}
+
+		select {
+		case <-ctx.Done():
+			return
+		case <-d.queued:
+		case <-due:
+			err := d.send(ctx, alert)
+
+			switch {
+			case err == nil:
+				d.remove(alert)
+				d.sent.Add(1)
+
+				retryDelay = 0
+				retryAt = time.Time{}
+			case errors.Is(err, errRefused):
+				d.remove(alert)
+				d.failed.Add(1)
+				d.dropped.Add(1)
+
+				retryDelay = 0
+				retryAt = time.Time{}
+
+				d.processLog.Warn("gave up an alert "+d.setting+" refused",
+					"event", alert.Event, "error", err.Error())
+			case ctx.Err() == nil: // not cut off as smallwebwaf stops
+				d.failed.Add(1)
+
+				retryDelay = min(max(retryDelayFactor*retryDelay, firstRetryDelay),
+					maxRetryDelay)
+				retryAt = time.Now().Add(retryDelay)
+
+				d.processLog.Warn("sending an alert to "+d.setting+" failed",
+					"error", err.Error(), "sending_again_in", retryDelay.String())
+			}
+		}
+	}
+}
+
+// add adds alert to the alerts waiting, first dropping the oldest while
+// queueSize wait, and has run look at the queue again.
+func (d *destination) add(alert *Alert) {
+	d.mu.Lock()
+	defer d.mu.Unlock()
+
+	if len(d.waiting) == queueSize {
+		d.waiting = slices.Delete(d.waiting, 0, 1)
+		d.dropped.Add(1)
+	}
+
+	d.waiting = append(d.waiting, alert)
+
+	select {
+	case d.queued <- struct{}{}:
+	default: // a value waits already
+	}
+}
+
+// load puts waiting, read from alerts.json, in place of the alerts
+// waiting, as add adds them.
+func (d *destination) load(waiting []Alert) {
+	d.mu.Lock()
+	d.waiting = nil
+	d.mu.Unlock()
+
+	for _, alert := range waiting {
+		d.add(&alert)
+	}
+}
+
+// snapshot returns the alerts waiting, oldest first.
+func (d *destination) snapshot() []Alert {
+	d.mu.Lock()
+	defer d.mu.Unlock()
+
+	waiting := make([]Alert, 0, len(d.waiting))
+	for _, alert := range d.waiting {
+		waiting = append(waiting, *alert)
+	}
+
+	return waiting
+}
+
+// oldest returns the oldest alert waiting, nil when none waits.
+func (d *destination) oldest() *Alert {
+	d.mu.Lock()
+	defer d.mu.Unlock()
+
+	if len(d.waiting) == 0 {
+		return nil
+	}
+
+	return d.waiting[0]
+}
+
+// remove takes alert, which run has sent or given up, out of the queue,
+// unless it has been dropped from it, or load has replaced the queue,
+// since run took it. Only the oldest alert is ever dropped, so alert is
+// the oldest if it is there at all.
+func (d *destination) remove(alert *Alert) {
+	d.mu.Lock()
+	defer d.mu.Unlock()
+
+	if len(d.waiting) > 0 && d.waiting[0] == alert {
+		d.waiting = slices.Delete(d.waiting, 0, 1)
+	}
+}
+
+// send posts alert to the destination, as message makes it, and returns
+// an error unless the destination answers with a 2xx status: one that
+// wraps errRefused for a 4xx status other than 408 and 429. No error
+// names the destination's URL, whose path or query can carry a secret.
+func (d *destination) send(ctx context.Context, alert *Alert) error {
+	body, header, err := d.message(alert)
+	if err != nil {
+		return err
+	}
+
+	ctx, cancel := context.WithTimeout(ctx, sendTimeout)
+	defer cancel()
+
+	req, err := http.NewRequestWithContext(ctx, http.MethodPost, d.url.String(),
+		bytes.NewReader(body))
+	if err != nil {
+		return fmt.Errorf("make the request: %w", err)
+	}
+
+	maps.Copy(req.Header, header)
+
+	res, err := d.httpClient.Do(req)
+	if err != nil {
+		// The client's error names the URL: only what went wrong is kept.
+		if urlErr, ok := errors.AsType[*url.Error](err); ok {
+			return urlErr.Err
+		}
+
+		return err
+	}
+
+	defer func() {
+		_ = res.Body.Close()
+	}()
+
+	// Read, so that the connection can be used again.
+	_, _ = io.Copy(io.Discard, io.LimitReader(res.Body, maxAnswerBytes))
+
+	switch status := res.StatusCode; {
+	case status >= http.StatusOK && status < http.StatusMultipleChoices:
+		return nil
+	case status >= http.StatusBadRequest && status < http.StatusInternalServerError &&
+		status != http.StatusRequestTimeout && status != http.StatusTooManyRequests:
+		return fmt.Errorf("%w %s", errRefused, res.Status)
+	default:
+		return fmt.Errorf("%w %s", errStatus, res.Status)
+	}
+}
+
+// webhookMessage returns alert as JSON, for the webhook, and the headers
+// sent with it: WebhookHeaders, and its Content-Type.
+func (q *Queue) webhookMessage(alert *Alert) ([]byte, http.Header, error) {
+	body, err := json.Marshal(alert)
+	if err != nil {
+		return nil, nil, fmt.Errorf("encode the alert: %w", err)
+	}
+
+	header := http.Header{}
+	maps.Copy(header, q.params.WebhookHeaders)
+	header.Set("Content-Type", "application/json")
+
+	return body, header, nil
+}
+
+// slackMessage returns alert as a message for a Slack incoming webhook,
+// in JSON: its title in bold, then its text, with &, < and > escaped, as
+// Slack asks, so that nothing in them is read as a link or a mention.
+func slackMessage(alert *Alert) ([]byte, http.Header, error) {
+	escape := strings.NewReplacer("&", "&amp;", "<", "&lt;", ">", "&gt;").Replace
+
+	body, err := json.Marshal(map[string]string{
+		"text": "*" + escape(title(alert)) + "*\n" + escape(text(alert)),
+	})
+	if err != nil {
+		return nil, nil, fmt.Errorf("encode the message: %w", err)
+	}
+
+	return body, http.Header{"Content-Type": {"application/json"}}, nil
+}
+
+// ntfyMessage returns alert's text, as the message published to ntfy,
+// and the headers sent with it: its title, the priority and the tag of
+// its event, and NtfyToken, unless it is empty, as a bearer token.
+func (q *Queue) ntfyMessage(alert *Alert) ([]byte, http.Header, error) {
+	header := http.Header{
+		"Title":    {title(alert)},
+		"Priority": {ntfyPriority(alert.Event)},
+		"Tags":     {ntfyTag(alert.Event)},
+	}
+
+	if q.params.NtfyToken != "" {
+		header.Set("Authorization", "Bearer "+q.params.NtfyToken)
+	}
+
+	return []byte(text(alert)), header, nil
+}
+
+// ntfyPriority returns the priority an alert for event is published to
+// ntfy with: high for an event the admin needs to look at.
+func ntfyPriority(event string) string {
+	switch event {
+	case EventPermanentBan, EventAnomaly, EventSourceFailure, EventFileError:
+		return "high"
+	case EventReputationHit:
+		return "low"
+	default: // ban, waf_block and summary
+		return "default"
+	}
+}
+
+// ntfyTag returns the tag an alert for event is published to ntfy with,
+// which ntfy shows as an emoji.
+func ntfyTag(event string) string {
+	switch event {
+	case EventBan, EventPermanentBan:
+		return "no_entry"
+	case EventWAFBlock:
+		return "shield"
+	case EventAnomaly:
+		return "chart_with_upwards_trend"
+	case EventReputationHit:
+		return "label"
+	case EventSourceFailure, EventFileError:
+		return "warning"
+	default: // summary
+		return "bar_chart"
+	}
+}
+
+// title returns the title of alert in Slack and ntfy: the instance and
+// the event.
+func title(alert *Alert) string {
+	return alert.Instance + ": " + alert.Event
+}
+
+// text returns the text of alert in Slack and ntfy: its reason, then a
+// line for each of its client, netblock and country, the file, source,
+// error and mode its detail gives, and its suppressed repeats, that it
+// has.
+func text(alert *Alert) string {
+	lines := []string{alert.Reason}
+
+	if alert.Client.IsValid() {
+		lines = append(lines, "client: "+alert.Client.String())
+	}
+
+	if alert.Netblock.IsValid() {
+		lines = append(lines, "netblock: "+alert.Netblock.String())
+	}
+
+	if alert.Country != "" {
+		lines = append(lines, "country: "+alert.Country)
+	}
+
+	for _, name := range []string{"file", "source", "error", "mode"} {
+		value, _ := alert.Detail[name].(string)
+		if value != "" {
+			lines = append(lines, name+": "+value)
+		}
+	}
+
+	if alert.SuppressedRepeats > 0 {
+		lines = append(lines, fmt.Sprintf("suppressed repeats: %d", alert.SuppressedRepeats))
+	}
+
+	return strings.Join(lines, "\n")
+}

internal/alerts/export_test.go

@@ -0,0 +1,16 @@
+package alerts
+
+import "net/http"
+
+// QueueSize is the most alerts that wait to be sent to a destination.
+const QueueSize = queueSize
+
+// SetTransport has q's requests to the destination name go through
+// transport instead of the network.
+func (q *Queue) SetTransport(name string, transport http.RoundTripper) {
+	for _, d := range q.destinations {
+		if d.name == name {
+			d.httpClient.Transport = transport
+		}
+	}
+}

internal/bans/admin_test.go

@@ -0,0 +1,286 @@
+package bans_test
+
+import (
+	"net/netip"
+	"testing"
+	"time"
+
+	"sneak.berlin/go/smallwebwaf/internal/bans"
+)
+
+func TestBanWithoutACauseIsAnAdmins(t *testing.T) {
+	t.Parallel()
+
+	netblock := netip.MustParsePrefix("203.0.113.0/24")
+	ledger := bans.New(defaultRules())
+	ledger.Load([]bans.Ban{{Netblock: netblock, Start: midnight()}})
+
+	if got := ledger.Bans(netblock)[0].Cause; got != bans.CauseAdmin {
+		t.Errorf("the ban's cause is %q, want admin", got)
+	}
+}
+
+func TestAdminsBansAreNeverDroppedAndDoNotCountTowardMaxBans(t *testing.T) {
+	t.Parallel()
+
+	rules := defaultRules()
+	rules.MaxBans = 1
+	ledger := bans.New(rules)
+	adminsOnly := netip.MustParsePrefix("198.51.100.0/24")
+	both := netip.MustParsePrefix("203.0.113.1/32")
+	second := netip.MustParsePrefix("203.0.113.2/32")
+	third := netip.MustParsePrefix("203.0.113.3/32")
+
+	// Seen longest ago, a netblock with two of an admin's bans alone, and
+	// then one with an admin's ban before a ban smallwebwaf made: the one
+	// ban counted toward MaxBans.
+	ledger.Load([]bans.Ban{
+		{Netblock: adminsOnly, Start: midnight().Add(-3 * time.Hour), Cause: bans.CauseAdmin},
+		{Netblock: adminsOnly, Start: midnight().Add(-2 * time.Hour), Cause: bans.CauseAdmin},
+		{Netblock: both, Start: midnight().Add(-time.Hour), Cause: bans.CauseAdmin},
+		{
+			Netblock: both,
+			Start:    midnight(),
+			Expires:  midnight().Add(time.Hour),
+			Cause:    bans.CauseLimit,
+		},
+	})
+	wantBans(t, ledger, map[netip.Prefix]int{adminsOnly: 2, both: 2})
+
+	// A new ban drops the ban smallwebwaf made, and only that one.
+	ledger.BanForLimit(second, midnight(), bans.Notes{})
+	wantBans(t, ledger, map[netip.Prefix]int{adminsOnly: 2, both: 1, second: 1})
+
+	if ledger.Bans(both)[0].Cause != bans.CauseAdmin {
+		t.Errorf("%s kept %+v, want the admin's ban", both, ledger.Bans(both))
+	}
+
+	// And the next drops that one.
+	ledger.BanForLimit(third, midnight(), bans.Notes{})
+	wantBans(t, ledger, map[netip.Prefix]int{adminsOnly: 2, both: 1, second: 0, third: 1})
+}
+
+func TestReasonOfTheBansSmallwebwafMakes(t *testing.T) {
+	t.Parallel()
+
+	ledger := bans.New(defaultRules())
+
+	limit, _ := ledger.BanForLimit(netip.MustParsePrefix("203.0.113.1/32"), midnight(),
+		bans.Notes{Limit: 1000, Window: "minute"})
+	attack, _ := ledger.BanForAttack(netip.MustParsePrefix("203.0.113.2/32"), midnight(),
+		bans.Notes{RuleID: "git-dir", Target: "path"})
+
+	for _, tc := range []struct{ got, want string }{
+		{limit.Reason, "requests per minute over the limit of 1000"},
+		{attack.Reason, "matched the rule git-dir"},
+	} {
+		if tc.got != tc.want {
+			t.Errorf("the reason is %q, want %q", tc.got, tc.want)
+		}
+	}
+}
+
+func TestLiftedBanForALimitRefusesNothingAndMakesNoBanLonger(t *testing.T) {
+	t.Parallel()
+
+	// An hour's ban lifted ten minutes after it started.
+	netblock := netip.MustParsePrefix("203.0.113.9/32")
+	lifted := bans.Ban{
+		Netblock: netblock,
+		Start:    midnight(),
+		Expires:  midnight().Add(time.Hour),
+		Cause:    bans.CauseLimit,
+		Lifted:   midnight().Add(10 * time.Minute),
+	}
+
+	ledger := bans.New(defaultRules())
+	ledger.Load([]bans.Ban{lifted})
+
+	// While it would still last, it refuses nothing, and a limit broken
+	// bans for an hour, as a first broken limit does; the lifted ban is
+	// kept, and counted among the earlier bans.
+	now := midnight().Add(30 * time.Minute)
+
+	_, banned, _ := ledger.Check(netblock.Addr(), now)
+	if banned {
+		t.Error("the lifted ban refuses")
+	}
+
+	ban, _ := ledger.BanForLimit(netblock, now, bans.Notes{})
+	if ban.Expires.Sub(ban.Start) != time.Hour ||
+		ban.Notes.EarlierBans != (bans.EarlierBans{Limit: 1}) {
+		t.Errorf("the next ban lasts %s with earlier bans %+v, want 1h and 1 for a limit",
+			ban.Expires.Sub(ban.Start), ban.Notes.EarlierBans)
+	}
+
+	held := ledger.Bans(netblock)
+	if len(held) != 2 || held[0] != lifted {
+		t.Errorf("the ledger holds %+v, want the lifted ban and the new one", held)
+	}
+}
+
+func TestLiftedBanForAnAttackRefusesNothingAndMakesNoBanLonger(t *testing.T) {
+	t.Parallel()
+
+	// A permanent ban for a clear sign of attack, lifted.
+	netblock := netip.MustParsePrefix("203.0.113.9/32")
+	ledger := bans.New(defaultRules())
+	ledger.Load([]bans.Ban{{
+		Netblock: netblock,
+		Start:    midnight(),
+		Cause:    bans.CauseAttack,
+		Lifted:   midnight().Add(time.Hour),
+	}})
+
+	now := midnight().Add(2 * time.Hour)
+
+	_, banned, _ := ledger.Find(netblock.Addr(), now)
+	if banned {
+		t.Error("the lifted ban refuses")
+	}
+
+	active, permanent := ledger.Count(now)
+	if active != 0 || permanent != 0 {
+		t.Errorf("%d bans are active and %d permanent, want none", active, permanent)
+	}
+
+	// The next clear sign of attack bans for seven days, as a first does.
+	ban, _ := ledger.BanForAttack(netblock, now, bans.Notes{})
+	if ban.Expires.Sub(ban.Start) != 7*day {
+		t.Errorf("the next ban for an attack ends at %s, want seven days on", ban.Expires)
+	}
+}
+
+func TestLoadEditCountsTheBansAnAdminMade(t *testing.T) {
+	t.Parallel()
+
+	ledger := bans.New(defaultRules())
+	made, _ := ledger.BanForLimit(netip.MustParsePrefix("203.0.113.1/32"), midnight(),
+		bans.Notes{})
+	atStart := bans.Ban{
+		Netblock: netip.MustParsePrefix("203.0.113.2/32"),
+		Start:    midnight(),
+	}
+
+	// The bans read at the start were made before it.
+	ledger.Load([]bans.Ban{made, atStart})
+
+	if got := ledger.Made(bans.CauseAdmin); got != 0 {
+		t.Fatalf("%d bans made by an admin after the start's, want none", got)
+	}
+
+	// The admin keeps the ban smallwebwaf made, keeps the one read at the
+	// start, and adds one without a cause: that one alone is made.
+	kept := made
+	kept.Cause = bans.CauseAdmin
+	added := bans.Ban{
+		Netblock: netip.MustParsePrefix("203.0.113.3/32"),
+		Start:    midnight(),
+	}
+	ledger.LoadEdit([]bans.Ban{kept, atStart, added})
+
+	if ledger.Made(bans.CauseAdmin) != 1 || ledger.Made(bans.CauseLimit) != 1 {
+		t.Errorf("%d bans made by an admin and %d for a limit, want 1 of each",
+			ledger.Made(bans.CauseAdmin), ledger.Made(bans.CauseLimit))
+	}
+}
+
+func TestAdminsBanIsMadeWhileAnotherLasts(t *testing.T) {
+	t.Parallel()
+
+	netblock := netip.MustParsePrefix("203.0.113.0/24")
+	ledger := bans.New(defaultRules())
+
+	// An hour's ban for a broken limit.
+	ledger.BanForLimit(netblock, midnight(), bans.Notes{})
+	wantChanged(t, ledger, true)
+
+	// A minute later an admin bans the netblock for good, named by an
+	// address in it: that ban is made, and counts the other among the
+	// earlier bans.
+	now := midnight().Add(time.Minute)
+	want := bans.Ban{
+		Netblock: netblock,
+		Start:    now,
+		Cause:    bans.CauseAdmin,
+		Reason:   "probes for logins",
+		Notes:    bans.Notes{EarlierBans: bans.EarlierBans{Limit: 1}},
+	}
+
+	got := ledger.BanForAdmin(netip.MustParsePrefix("203.0.113.9/24"), now, time.Time{},
+		"probes for logins")
+	if got != want {
+		t.Errorf("the admin's ban is\n%+v\nwant\n%+v", got, want)
+	}
+
+	wantChanged(t, ledger, true)
+
+	if made := ledger.Made(bans.CauseAdmin); made != 1 {
+		t.Errorf("%d bans made by an admin, want 1", made)
+	}
+
+	// It refuses once the ban for the limit has ended.
+	ban, banned, _ := ledger.Find(netblock.Addr(), midnight().Add(2*time.Hour))
+	if !banned || ban != want {
+		t.Errorf("after the limit's ban the netblock is under %+v (%t), want %+v",
+			ban, banned, want)
+	}
+}
+
+func TestLiftLiftsEveryActiveBanCoveringTheClient(t *testing.T) {
+	t.Parallel()
+
+	client := netip.MustParseAddr("203.0.113.9")
+	own := netip.MustParsePrefix("203.0.113.9/32")
+	wide := netip.MustParsePrefix("203.0.113.0/24")
+	other := netip.MustParsePrefix("203.0.113.10/32")
+
+	ledger := bans.New(defaultRules())
+	ledger.Load([]bans.Ban{
+		// Ended an hour ago.
+		{
+			Netblock: own, Start: midnight().Add(-2 * time.Hour),
+			Expires: midnight().Add(-time.Hour), Cause: bans.CauseLimit,
+		},
+		// Active, on the client's address and on its /24.
+		{
+			Netblock: own, Start: midnight(), Expires: midnight().Add(time.Hour),
+			Cause: bans.CauseLimit,
+		},
+		{Netblock: wide, Start: midnight(), Cause: bans.CauseAdmin},
+		// Another client's.
+		{Netblock: other, Start: midnight(), Cause: bans.CauseAdmin},
+	})
+
+	now := midnight().Add(time.Minute)
+
+	lifted := ledger.Lift(client, now)
+	if len(lifted) != 2 || lifted[0].Lifted != now || lifted[1].Lifted != now {
+		t.Errorf("lifted %+v, want the two active bans covering the client", lifted)
+	}
+
+	wantChanged(t, ledger, true)
+
+	if _, banned, _ := ledger.Check(client, now); banned {
+		t.Error("the client is still banned")
+	}
+
+	if _, banned, _ := ledger.Check(other.Addr(), now); !banned {
+		t.Error("the other client's ban was lifted")
+	}
+
+	// The lifted bans are kept, and the one that had ended is not lifted.
+	covering := ledger.Covering(client)
+	if len(covering) != 3 || covering[0].Netblock != wide ||
+		!covering[1].Lifted.IsZero() || covering[2].Lifted != now {
+		t.Errorf("the bans covering the client are %+v, want the /24's and both "+
+			"of its own, the earlier not lifted", covering)
+	}
+
+	// With none active, nothing is lifted or changed.
+	if lifted = ledger.Lift(client, now); len(lifted) != 0 {
+		t.Errorf("lifted %+v again", lifted)
+	}
+
+	wantChanged(t, ledger, false)
+}

internal/bans/bans.go

@@ -1,10 +1,13 @@
 // Package bans is the ban ledger: the bans smallwebwaf makes on the
-// netblocks of clients that break a rate limit, with their notes, as the
-// "Bans" section of SPEC.md describes. The bans are kept in memory, and
-// written to bans.json and read from it by the state package.
+// netblocks of clients that break a rate limit or show a clear sign of
+// attack, and those an admin makes, with their notes, as the "Bans"
+// section of SPEC.md describes. The bans are kept in memory, and written
+// to bans.json and read from it by the state package.
 package bans
 
 import (
+	"fmt"
+	"math"
 	"net/netip"
 	"slices"
 	"strings"
@@ -14,6 +17,17 @@ import (
 	"github.com/hashicorp/golang-lru/v2/simplelru"
 )
 
+// The causes of bans.
+const (
+	// CauseLimit is a ban smallwebwaf made for a broken limit.
+	CauseLimit = "limit"
+	// CauseAttack is a ban smallwebwaf made for a clear sign of attack.
+	CauseAttack = "attack"
+	// CauseAdmin is a ban an admin made, or one smallwebwaf made that an
+	// admin keeps. It is never dropped.
+	CauseAdmin = "admin"
+)
+
 // repeatFactor is how many times as long as the netblock's last ban a ban
 // for a limit broken again within the repeat window lasts.
 const repeatFactor = 3
@@ -21,32 +35,44 @@ const repeatFactor = 3
 // maxTextBytes is how much of each text in a ban's notes is kept.
 const maxTextBytes = 256
 
-// Rules are how long a ban for a broken limit lasts, and how many bans
-// are held.
+// Rules are how long a ban lasts, and how many bans are held.
 type Rules struct {
-	// LimitBanDuration is how long a first ban lasts.
+	// LimitBanDuration is how long a first ban for a broken limit lasts.
 	LimitBanDuration time.Duration
-	// LimitBanRepeatWindow is how soon after the netblock's last ban
-	// ended a broken limit counts as a repeat, which bans for
-	// repeatFactor times as long as that ban.
+	// LimitBanRepeatWindow is how soon after the end of the netblock's
+	// ban that ended last, other than one for a clear sign of attack, a
+	// broken limit counts as a repeat, which bans for repeatFactor times as
+	// long as that ban.
 	LimitBanRepeatWindow time.Duration
-	// MaxBanDuration is the longest ban; a ban that would be longer is
-	// permanent instead.
+	// MaxBanDuration is the longest ban for a broken limit; one that would
+	// be longer is permanent instead.
 	MaxBanDuration time.Duration
-	// MaxBans is the most bans held, at least one. Past it, the earliest
-	// ban of the netblock that has gone longest without a request is
-	// dropped.
+	// AttackBanDuration is how long a first ban for a clear sign of attack
+	// lasts.
+	AttackBanDuration time.Duration
+	// MaxBans is the most bans held whose cause is not CauseAdmin, at
+	// least one. Past it, the earliest such ban of the netblock that has
+	// gone longest without a request is dropped. Bans whose cause is
+	// CauseAdmin are held besides, and never dropped.
 	MaxBans int
 }
 
-// Ban is a ban on a netblock for a broken limit, the only kind of ban
-// smallwebwaf makes so far.
+// Ban is a ban on a netblock.
 type Ban struct {
 	Netblock netip.Prefix
 	Start    time.Time
 	// Expires is when the ban ends, zero for a permanent ban.
 	Expires time.Time
-	Notes   Notes
+	// Cause is CauseLimit, CauseAttack or CauseAdmin.
+	Cause string
+	// Reason is a short text: for a ban smallwebwaf made, the limit broken
+	// or the rule that matched; for an admin's, what the admin wrote.
+	Reason string
+	// Lifted is when an admin lifted the ban, zero while no admin has. A
+	// lifted ban refuses nothing, and does not make the netblock's next
+	// ban longer.
+	Lifted time.Time
+	Notes  Notes
 }
 
 // Permanent reports whether the ban never runs out.
@@ -54,9 +80,10 @@ func (b Ban) Permanent() bool {
 	return b.Expires.IsZero()
 }
 
-// ActiveAt reports whether the ban refuses requests at now.
+// ActiveAt reports whether the ban refuses requests at now: it has not
+// been lifted, and has not run out.
 func (b Ban) ActiveAt(now time.Time) bool {
-	return b.Permanent() || now.Before(b.Expires)
+	return b.Lifted.IsZero() && (b.Permanent() || now.Before(b.Expires))
 }
 
 // Notes are what an admin needs to decide whether to lift a ban. The
@@ -66,23 +93,36 @@ func (b Ban) ActiveAt(now time.Time) bool {
 type Notes struct {
 	// Country is the client's country, when it was looked up.
 	Country string `json:"country"`
-	// Limit, Window and Count are the limit that was broken, its window,
-	// "minute", "hour" or "day", and the count reached: the client's
-	// requests in the window, the one that broke the limit included.
-	// These are the requests that counted toward the ban, and the window
-	// is the time over which they came.
-	Limit  int64   `json:"limit"`
-	Window string  `json:"window"`
-	Count  float64 `json:"count"`
-	// Request is the request that broke the limit.
+	// Limit, Window and Count are, for a ban for a broken limit, the limit
+	// that was broken, its window, "minute", "hour" or "day", and the
+	// count reached: the client's requests in the window, the one that
+	// broke the limit included. These are the requests that counted
+	// toward the ban, and the window is the time over which they came.
+	Limit  int64   `json:"limit,omitempty"`
+	Window string  `json:"window,omitempty"`
+	Count  float64 `json:"count,omitempty"`
+	// RuleID and Target are, for a ban for a clear sign of attack, the id
+	// of the rule file rule that matched, and its target.
+	RuleID string `json:"rule_id,omitempty"`
+	Target string `json:"target,omitempty"`
+	// Request is the request that broke the limit, or that was the clear
+	// sign of attack.
 	Request Request `json:"request"`
 	// Requests is how many requests the netblock has sent since it was
 	// first seen, and Refused how many of them the ban has refused so
 	// far. Both go up with each request the ban refuses.
 	Requests int64 `json:"requests"`
 	Refused  int64 `json:"refused"`
-	// EarlierBans is how many bans the netblock had before this one.
-	EarlierBans int `json:"earlier_bans"`
+	// EarlierBans is how many bans the netblock had before this one, by
+	// cause.
+	EarlierBans EarlierBans `json:"earlier_bans"`
+}
+
+// EarlierBans counts a netblock's bans before a ban, by cause.
+type EarlierBans struct {
+	Limit  int `json:"limit"`
+	Attack int `json:"attack"`
+	Admin  int `json:"admin"`
 }
 
 // Request is a request in a ban's notes. Each text is cut to 256 bytes.
@@ -107,13 +147,16 @@ type Ledger struct {
 	changed chan struct{}
 
 	mu sync.Mutex
-	// netblocks holds each banned netblock's bans, oldest first. Check
-	// makes each netblock it finds the most recently seen.
+	// netblocks holds each banned netblock's bans, oldest first. Check and
+	// Find make each netblock they find the most recently seen.
 	netblocks *simplelru.LRU[netip.Prefix, *[]Ban]
-	// held is how many bans netblocks holds, at most rules.MaxBans.
+	// held is how many bans netblocks holds whose cause is not CauseAdmin,
+	// at most rules.MaxBans.
 	held int
-	// made is how many bans BanForLimit has made since the start.
-	made int
+	// made is how many bans have been made since the start, by cause: by
+	// the ledger, and by an admin, through BanForAdmin or in an edit of
+	// bans.json.
+	made map[string]int
 	// v4Lengths and v6Lengths are the lengths of the IPv4 and IPv6
 	// netblocks that have been banned. Check looks for a ban at each of
 	// them, so that a ban read from bans.json refuses every client in its
@@ -124,9 +167,10 @@ type Ledger struct {
 
 // New returns a Ledger with no ban yet.
 func New(rules Rules) *Ledger {
-	// Every netblock held has a ban, so there are never more netblocks
-	// than rules.MaxBans, and the LRU never drops one itself.
-	netblocks, err := simplelru.NewLRU[netip.Prefix, *[]Ban](rules.MaxBans, nil)
+	// The ledger drops bans itself, and never those whose cause is
+	// CauseAdmin, however many there are, so the LRU has no limit of its
+	// own: it keeps the netblocks in the order they were last seen.
+	netblocks, err := simplelru.NewLRU[netip.Prefix, *[]Ban](math.MaxInt, nil)
 	if err != nil {
 		panic(err) // NewLRU fails only for a size below one
 	}
@@ -135,89 +179,224 @@ func New(rules Rules) *Ledger {
 		rules:     rules,
 		changed:   make(chan struct{}, 1),
 		netblocks: netblocks,
+		made:      map[string]int{},
 	}
 }
 
-// Changed receives a value after a ban is made, so that bans.json can be
-// written. Several bans made before it is read leave one value.
+// Changed receives a value after a ban is made, lifted or made permanent,
+// so that bans.json can be written. Several changes before it is read
+// leave one value.
 func (l *Ledger) Changed() <-chan struct{} {
 	return l.changed
 }
 
-// Check is called for each request from client, at now. It reports
-// whether a ban on a netblock client is in is active, and returns that
-// ban, with the request counted among those it refused.
-func (l *Ledger) Check(client netip.Addr, now time.Time) (Ban, bool) {
+// Check is called for a request from client, at now. It reports whether
+// a ban on a netblock client is in is active, and returns that ban, with
+// the request counted among those it refused. A ban for a clear sign of
+// attack is made permanent by the request: the netblock is malicious.
+// The last result reports whether the request made the ban permanent.
+func (l *Ledger) Check(client netip.Addr, now time.Time) (Ban, bool, bool) {
 	l.mu.Lock()
 	defer l.mu.Unlock()
 
-	lengths := l.v6Lengths
-	if client.Is4() {
-		lengths = l.v4Lengths
+	ban := l.active(client, now)
+	if ban == nil {
+		return Ban{}, false, false
 	}
 
-	for _, length := range lengths {
-		bans, found := l.netblocks.Get(netip.PrefixFrom(client, length).Masked())
-		if !found {
-			continue
-		}
+	ban.Notes.Requests++
+	ban.Notes.Refused++
 
-		// A ban is made only once the one before has ended, so only the
-		// last can be active.
-		last := &(*bans)[len(*bans)-1]
-		if last.ActiveAt(now) {
-			last.Notes.Requests++
-			last.Notes.Refused++
+	madePermanent := ban.Cause == CauseAttack && !ban.Permanent()
+	if madePermanent {
+		ban.Expires = time.Time{}
 
-			return *last, true
+		l.markChanged()
+	}
+
+	return *ban, true, madePermanent
+}
+
+// Find is Check without counting the request among those the ban
+// refused, and without making the ban permanent: in observe mode a ban
+// refuses nothing. The last result reports whether Check would have made
+// the ban permanent.
+func (l *Ledger) Find(client netip.Addr, now time.Time) (Ban, bool, bool) {
+	l.mu.Lock()
+	defer l.mu.Unlock()
+
+	ban := l.active(client, now)
+	if ban == nil {
+		return Ban{}, false, false
+	}
+
+	return *ban, true, ban.Cause == CauseAttack && !ban.Permanent()
+}
+
+// activeBan returns the ban in bans, a netblock's bans oldest first, that
+// is active at now, or nil when none is. If several are, it returns the
+// one that started last. Every ban is looked at, since a ban an admin adds
+// to bans.json can start before the netblock's others and outlast them.
+func activeBan(bans []Ban, now time.Time) *Ban {
+	for i := len(bans) - 1; i >= 0; i-- {
+		if bans[i].ActiveAt(now) {
+			return &bans[i]
 		}
 	}
 
-	return Ban{}, false
+	return nil
 }
 
 // BanForLimit bans netblock at now for a broken limit, with notes, and
-// returns the ban. A first ban lasts LimitBanDuration. A ban made within
-// LimitBanRepeatWindow after the netblock's last ban ended lasts
+// returns the ban, and true. A first ban lasts LimitBanDuration. A ban
+// made within LimitBanRepeatWindow after the netblock's ban that ended
+// last, other than one for a clear sign of attack or a lifted one, lasts
 // repeatFactor times as long as that one. A ban that would be longer
 // than MaxBanDuration is permanent instead. If a ban on netblock is still
 // active, as when two of its requests break a limit at once, that ban is
-// returned and no other is made. The ledger fills in the notes' Refused
-// and EarlierBans itself.
-func (l *Ledger) BanForLimit(netblock netip.Prefix, now time.Time, notes Notes) Ban {
+// returned with false, and no other is made. The ledger fills in the
+// notes' Refused and EarlierBans itself, and gives the ban the reason
+// "requests per <Window> over the limit of <Limit>", from the notes.
+func (l *Ledger) BanForLimit(
+	netblock netip.Prefix, now time.Time, notes Notes,
+) (Ban, bool) {
+	return l.ban(netblock, now, CauseLimit, limitReason(notes), notes, true)
+}
+
+// WouldBanForLimit returns what BanForLimit would, without making the ban:
+// what observe mode would have done.
+func (l *Ledger) WouldBanForLimit(
+	netblock netip.Prefix, now time.Time, notes Notes,
+) (Ban, bool) {
+	return l.ban(netblock, now, CauseLimit, limitReason(notes), notes, false)
+}
+
+// BanForAttack bans netblock at now for a clear sign of attack, with
+// notes, and returns the ban, and whether it made it, as BanForLimit
+// does. A first ban lasts AttackBanDuration; once the netblock has had
+// one that was not lifted, the next is permanent. Its reason is "matched
+// the rule <RuleID>".
+func (l *Ledger) BanForAttack(
+	netblock netip.Prefix, now time.Time, notes Notes,
+) (Ban, bool) {
+	return l.ban(netblock, now, CauseAttack, attackReason(notes), notes, true)
+}
+
+// WouldBanForAttack returns what BanForAttack would, without making the
+// ban: what observe mode would have done.
+func (l *Ledger) WouldBanForAttack(
+	netblock netip.Prefix, now time.Time, notes Notes,
+) (Ban, bool) {
+	return l.ban(netblock, now, CauseAttack, attackReason(notes), notes, false)
+}
+
+// WouldBePermanent reports whether a ban on netblock for cause, CauseLimit
+// or CauseAttack, made at now would be permanent, as BanForLimit or
+// BanForAttack would make it. It works out nothing else of the ban.
+func (l *Ledger) WouldBePermanent(
+	netblock netip.Prefix, now time.Time, cause string,
+) bool {
 	l.mu.Lock()
 	defer l.mu.Unlock()
 
-	var last *Ban
-
-	bans, found := l.netblocks.Get(netblock)
-	if found {
-		last = &(*bans)[len(*bans)-1]
-		if last.ActiveAt(now) {
-			return *last
-		}
-
-		notes.EarlierBans = last.Notes.EarlierBans + 1
+	var held []Ban
+	if bans, found := l.netblocks.Peek(netblock); found {
+		held = *bans
 	}
 
-	notes.Request = notes.Request.cut()
+	if cause == CauseAttack {
+		return l.attackExpiry(held, now).IsZero()
+	}
+
+	return l.limitExpiry(held, now).IsZero()
+}
+
+// limitReason is the reason of a ban for a broken limit, with notes.
+func limitReason(notes Notes) string {
+	return fmt.Sprintf("requests per %s over the limit of %d", notes.Window, notes.Limit)
+}
+
+// attackReason is the reason of a ban for a clear sign of attack, with
+// notes.
+func attackReason(notes Notes) string {
+	return "matched the rule " + notes.RuleID
+}
+
+// BanForAdmin bans netblock at now for an admin, with reason, until
+// expires, or for good when expires is zero, and returns the ban, whose
+// cause is CauseAdmin. Unlike BanForLimit and BanForAttack, it makes the
+// ban even while another on netblock is active, since the admin asked
+// for this one. The ledger fills in the notes' EarlierBans, and counts
+// the ban among those made.
+func (l *Ledger) BanForAdmin(
+	netblock netip.Prefix, now, expires time.Time, reason string,
+) Ban {
+	l.mu.Lock()
+	defer l.mu.Unlock()
+
 	ban := Ban{
-		Netblock: netblock,
-		Start:    now,
-		Expires:  l.expiry(last, now),
-		Notes:    notes,
+		Netblock: netblock.Masked(), Start: now, Expires: expires, Cause: CauseAdmin,
+		Reason: reason,
 	}
-	l.add(ban)
-	l.made++
 
-	select {
-	case l.changed <- struct{}{}:
-	default: // a value is waiting already
+	held, found := l.netblocks.Get(ban.Netblock)
+	if found {
+		ban.Notes.EarlierBans = earlierBans(*held)
 	}
 
+	l.add(ban)
+	l.made[CauseAdmin]++
+	l.markChanged()
+
 	return ban
 }
 
+// Lift lifts, at now, every ban active then on a netblock client is in,
+// as an admin does, and returns those bans. A lifted ban is kept, refuses
+// nothing, and does not make the netblock's next ban longer.
+func (l *Ledger) Lift(client netip.Addr, now time.Time) []Ban {
+	l.mu.Lock()
+	defer l.mu.Unlock()
+
+	var lifted []Ban
+
+	for _, bans := range l.covering(client) {
+		for i := range *bans {
+			ban := &(*bans)[i]
+			if ban.ActiveAt(now) {
+				ban.Lifted = now
+				lifted = append(lifted, *ban)
+			}
+		}
+	}
+
+	if len(lifted) > 0 {
+		l.markChanged()
+	}
+
+	return lifted
+}
+
+// Covering returns every ban held on a netblock client is in, active or
+// not, sorted by netblock, and each netblock's bans oldest first. It is
+// not a request from client, and leaves when the netblocks were last seen
+// unchanged.
+func (l *Ledger) Covering(client netip.Addr) []Ban {
+	l.mu.Lock()
+	defer l.mu.Unlock()
+
+	var held []Ban
+	for _, bans := range l.covering(client) {
+		held = append(held, *bans...)
+	}
+
+	slices.SortStableFunc(held, func(a, b Ban) int {
+		return a.Netblock.Compare(b.Netblock)
+	})
+
+	return held
+}
+
 // Bans returns the bans held on netblock, oldest first. It is not a
 // request from netblock, and leaves when it was last seen unchanged.
 func (l *Ledger) Bans(netblock netip.Prefix) []Ban {
@@ -232,17 +411,19 @@ func (l *Ledger) Bans(netblock netip.Prefix) []Ban {
 	return slices.Clone(*bans)
 }
 
-// Made returns how many bans the ledger has made since the start; bans
-// read from bans.json are not among them.
-func (l *Ledger) Made() int {
+// Made returns how many bans for cause have been made since the start:
+// for CauseLimit and CauseAttack, by the ledger; for CauseAdmin, by an
+// admin, with BanForAdmin or in an edit of bans.json, as LoadEdit counts
+// them. The bans read from bans.json at the start are not among them.
+func (l *Ledger) Made(cause string) int {
 	l.mu.Lock()
 	defer l.mu.Unlock()
 
-	return l.made
+	return l.made[cause]
 }
 
 // Count returns how many of the bans held are active at now, and how many
-// are permanent.
+// of those are permanent. A lifted ban is neither.
 func (l *Ledger) Count(now time.Time) (int, int) {
 	l.mu.Lock()
 	defer l.mu.Unlock()
@@ -251,10 +432,12 @@ func (l *Ledger) Count(now time.Time) (int, int) {
 
 	for _, bans := range l.netblocks.Values() {
 		for _, ban := range *bans {
-			if ban.ActiveAt(now) {
-				active++
+			if !ban.ActiveAt(now) {
+				continue
 			}
 
+			active++
+
 			if ban.Permanent() {
 				permanent++
 			}
@@ -282,32 +465,199 @@ func (l *Ledger) Snapshot() []Ban {
 	return held
 }
 
-// Load puts bans read from bans.json into a ledger that holds none yet,
-// in the order they started, so that a netblock whose last ban started
-// latest counts as the most recently seen. Each netblock is masked to its
-// length, so that 203.0.113.9/24 is 203.0.113.0/24, and each text in the
-// notes is cut to 256 bytes. Past MaxBans the earliest bans are dropped,
-// as when they are made.
+// Load puts bans read from bans.json at the start into the ledger, in
+// place of the bans it holds, in the order they started, so that a
+// netblock whose last ban started latest counts as the most recently
+// seen. A ban without a cause is an admin's, and gets CauseAdmin. Each
+// netblock is masked to its length, so that 203.0.113.9/24 is
+// 203.0.113.0/24, and each text in the notes is cut to 256 bytes. Past
+// MaxBans the earliest bans whose cause is not CauseAdmin are dropped, as
+// when they are made.
 func (l *Ledger) Load(bans []Ban) {
 	l.mu.Lock()
 	defer l.mu.Unlock()
 
+	l.load(bans)
+}
+
+// LoadEdit is Load for an admin's edit of bans.json, taken in while
+// smallwebwaf runs. Each ban in it whose cause is CauseAdmin, and which
+// the ledger did not hold, with the same netblock and start, is one the
+// admin made, and is counted among the bans made.
+func (l *Ledger) LoadEdit(bans []Ban) {
+	l.mu.Lock()
+	defer l.mu.Unlock()
+
+	l.made[CauseAdmin] += l.load(bans)
+}
+
+// load does what Load describes, and returns how many of bans are bans
+// whose cause is CauseAdmin that the ledger did not hold before.
+func (l *Ledger) load(bans []Ban) int {
 	bans = slices.Clone(bans)
+	added := 0
+
+	for i := range bans {
+		ban := &bans[i]
+		ban.Netblock = ban.Netblock.Masked()
+		ban.Notes.Request = ban.Notes.Request.cut()
+
+		if ban.Cause == "" {
+			ban.Cause = CauseAdmin
+		}
+
+		if ban.Cause == CauseAdmin && !l.holds(ban.Netblock, ban.Start) {
+			added++
+		}
+	}
+
 	slices.SortStableFunc(bans, func(a, b Ban) int {
 		return a.Start.Compare(b.Start)
 	})
 
+	l.netblocks.Purge()
+	l.held = 0
+	l.v4Lengths, l.v6Lengths = nil, nil
+
 	for _, ban := range bans {
-		ban.Netblock = ban.Netblock.Masked()
-		ban.Notes.Request = ban.Notes.Request.cut()
 		l.add(ban)
 	}
+
+	return added
+}
+
+// holds reports whether the ledger holds a ban on netblock that started
+// at start.
+func (l *Ledger) holds(netblock netip.Prefix, start time.Time) bool {
+	bans, found := l.netblocks.Peek(netblock)
+
+	return found && slices.ContainsFunc(*bans, func(ban Ban) bool {
+		return ban.Start.Equal(start)
+	})
+}
+
+// ban bans netblock at now for cause, with reason and notes, as
+// BanForLimit and BanForAttack describe, and returns the ban, and whether
+// it made it. Unless keep is true, the ban is not made, only returned: it
+// is the ban that would have been made.
+func (l *Ledger) ban(
+	netblock netip.Prefix, now time.Time, cause, reason string, notes Notes, keep bool,
+) (Ban, bool) {
+	l.mu.Lock()
+	defer l.mu.Unlock()
+
+	// held are the netblock's bans, none of them active.
+	var held []Ban
+
+	bans, found := l.netblocks.Get(netblock)
+	if found {
+		active := activeBan(*bans, now)
+		if active != nil {
+			return *active, false
+		}
+
+		held = *bans
+		notes.EarlierBans = earlierBans(held)
+	}
+
+	notes.Request = notes.Request.cut()
+	ban := Ban{Netblock: netblock, Start: now, Cause: cause, Reason: reason, Notes: notes}
+
+	if cause == CauseAttack {
+		ban.Expires = l.attackExpiry(held, now)
+	} else {
+		ban.Expires = l.limitExpiry(held, now)
+	}
+
+	if !keep {
+		return ban, true
+	}
+
+	l.add(ban)
+	l.made[cause]++
+	l.markChanged()
+
+	return ban, true
+}
+
+// earlierBans returns how many bans a netblock with the bans held, oldest
+// first, has had, by cause: the first ban held counts the bans the
+// netblock had before that one, since dropped to make room, and each ban
+// held adds one.
+func earlierBans(held []Ban) EarlierBans {
+	earlier := held[0].Notes.EarlierBans
+
+	for _, ban := range held {
+		switch ban.Cause {
+		case CauseLimit:
+			earlier.Limit++
+		case CauseAttack:
+			earlier.Attack++
+		case CauseAdmin:
+			earlier.Admin++
+		}
+	}
+
+	return earlier
+}
+
+// markChanged has Changed receive a value, unless one is waiting already.
+func (l *Ledger) markChanged() {
+	select {
+	case l.changed <- struct{}{}:
+	default: // a value is waiting already
+	}
+}
+
+// active returns the ban active at now on a netblock client is in, or
+// nil.
+func (l *Ledger) active(client netip.Addr, now time.Time) *Ban {
+	lengths := l.v6Lengths
+	if client.Is4() {
+		lengths = l.v4Lengths
+	}
+
+	for _, length := range lengths {
+		bans, found := l.netblocks.Get(netip.PrefixFrom(client, length).Masked())
+		if !found {
+			continue
+		}
+
+		ban := activeBan(*bans, now)
+		if ban != nil {
+			return ban
+		}
+	}
+
+	return nil
+}
+
+// covering returns the bans of each netblock held that client is in,
+// leaving when the netblocks were last seen unchanged.
+func (l *Ledger) covering(client netip.Addr) []*[]Ban {
+	lengths := l.v6Lengths
+	if client.Is4() {
+		lengths = l.v4Lengths
+	}
+
+	var found []*[]Ban
+
+	for _, length := range lengths {
+		bans, ok := l.netblocks.Peek(netip.PrefixFrom(client, length).Masked())
+		if ok {
+			found = append(found, bans)
+		}
+	}
+
+	return found
 }
 
 // add adds ban to its netblock's bans, after the last, and makes its
-// netblock the most recently seen. With MaxBans held, it drops one first.
+// netblock the most recently seen. With MaxBans held, it drops one first,
+// unless ban's cause is CauseAdmin, which does not count toward MaxBans.
 func (l *Ledger) add(ban Ban) {
-	if l.held == l.rules.MaxBans {
+	counted := ban.Cause != CauseAdmin
+	if counted && l.held == l.rules.MaxBans {
 		l.dropOne()
 	}
 
@@ -320,7 +670,10 @@ func (l *Ledger) add(ban Ban) {
 	}
 
 	*bans = append(*bans, ban)
-	l.held++
+
+	if counted {
+		l.held++
+	}
 
 	lengths := &l.v6Lengths
 	if ban.Netblock.Addr().Is4() {
@@ -332,12 +685,24 @@ func (l *Ledger) add(ban Ban) {
 	}
 }
 
-// expiry returns when a ban for a broken limit made at now ends, or zero
-// when it is permanent. last is the netblock's last ban, which has ended,
-// or nil when it has none.
-func (l *Ledger) expiry(last *Ban, now time.Time) time.Time {
+// limitExpiry returns when a ban for a broken limit made at now ends, or
+// zero when it is permanent. held are the netblock's bans, none of them
+// active, of which the one that ended last, other than a ban for a clear
+// sign of attack or a lifted one, can make the new ban longer. A ban an
+// admin adds to bans.json can start after another and end before it, so
+// that one is looked for among them all.
+func (l *Ledger) limitExpiry(held []Ban, now time.Time) time.Time {
 	length := l.rules.LimitBanDuration
 
+	var last *Ban
+
+	for i, ban := range held {
+		if ban.Cause != CauseAttack && ban.Lifted.IsZero() &&
+			(last == nil || ban.Expires.After(last.Expires)) {
+			last = &held[i]
+		}
+	}
+
 	if last != nil && now.Sub(last.Expires) <= l.rules.LimitBanRepeatWindow {
 		lastLength := last.Expires.Sub(last.Start)
 		// This is repeatFactor * lastLength > MaxBanDuration, written so
@@ -356,17 +721,53 @@ func (l *Ledger) expiry(last *Ban, now time.Time) time.Time {
 	return now.Add(length)
 }
 
-// dropOne drops the earliest ban of the netblock that has gone longest
-// without a request, and the netblock with it if that was its only ban.
-func (l *Ledger) dropOne() {
-	netblock, bans, _ := l.netblocks.GetOldest()
-	if len(*bans) == 1 {
-		l.netblocks.Remove(netblock)
-	} else {
-		*bans = slices.Delete(*bans, 0, 1)
+// attackExpiry returns when a ban for a clear sign of attack made at now
+// ends. held are the netblock's bans, none of them active: if one of them
+// is for a clear sign of attack too, and was not lifted, the new ban is
+// permanent, and its end zero; otherwise it ends AttackBanDuration later.
+func (l *Ledger) attackExpiry(held []Ban, now time.Time) time.Time {
+	for _, ban := range held {
+		if ban.Cause == CauseAttack && ban.Lifted.IsZero() {
+			return time.Time{}
+		}
 	}
 
-	l.held--
+	return now.Add(l.rules.AttackBanDuration)
+}
+
+// dropOne drops the earliest ban whose cause is not CauseAdmin of the
+// netblock that has gone longest without a request, of those that hold
+// such a ban, and the netblock with it if that was its only ban. It is
+// called with at least one such ban held. It looks at each netblock once
+// at most, and drops nothing when none holds such a ban.
+func (l *Ledger) dropOne() {
+	for range l.netblocks.Len() {
+		netblock, bans, _ := l.netblocks.GetOldest()
+
+		i := slices.IndexFunc(*bans, func(ban Ban) bool {
+			return ban.Cause != CauseAdmin
+		})
+		if i < 0 {
+			// Its bans are all an admin's, and never dropped. Get makes
+			// it the most recently seen, so that the next netblock is
+			// looked at; when it was seen matters only for dropping a
+			// ban, and a ban added to it makes it the most recently seen
+			// anyway.
+			l.netblocks.Get(netblock)
+
+			continue
+		}
+
+		if len(*bans) == 1 {
+			l.netblocks.Remove(netblock)
+		} else {
+			*bans = slices.Delete(*bans, i, i+1)
+		}
+
+		l.held--
+
+		return
+	}
 }
 
 // cut returns r with each text cut to maxTextBytes and copied, so that

internal/bans/bans_test.go

@@ -21,11 +21,12 @@ func TestRepeatsTripleUntilPermanent(t *testing.T) {
 	// Each ban is followed by another as soon as it ends: 1, 3, 9, 27 and
 	// 81 hours.
 	for i, hours := range []int{1, 3, 9, 27, 81} {
-		ban := ledger.BanForLimit(netblock, now, bans.Notes{})
+		ban, _ := ledger.BanForLimit(netblock, now, bans.Notes{})
 
 		length := time.Duration(hours) * time.Hour
-		if !ban.Expires.Equal(now.Add(length)) || ban.Notes.EarlierBans != i {
-			t.Fatalf("ban %d lasts %s with %d earlier bans, want %d hours and %d",
+		if !ban.Expires.Equal(now.Add(length)) ||
+			ban.Notes.EarlierBans != (bans.EarlierBans{Limit: i}) {
+			t.Fatalf("ban %d lasts %s with earlier bans %+v, want %d hours and %d for a limit",
 				i+1, ban.Expires.Sub(now), ban.Notes.EarlierBans, hours, i)
 		}
 
@@ -34,12 +35,12 @@ func TestRepeatsTripleUntilPermanent(t *testing.T) {
 
 	// The sixth would last 243 hours, more than seven days: it is
 	// permanent, and never ends.
-	ban := ledger.BanForLimit(netblock, now, bans.Notes{})
+	ban, _ := ledger.BanForLimit(netblock, now, bans.Notes{})
 	if !ban.Permanent() {
 		t.Fatalf("sixth ban ends at %s, want a permanent one", ban.Expires)
 	}
 
-	_, banned := ledger.Check(netblock.Addr(), now.Add(100*365*day))
+	_, banned, _ := ledger.Check(netblock.Addr(), now.Add(100*365*day))
 	if !banned {
 		t.Error("a permanent ban ended")
 	}
@@ -63,11 +64,12 @@ func TestRepeatWindowRunsOut(t *testing.T) {
 			ledger := bans.New(defaultRules())
 			netblock := netip.MustParsePrefix("203.0.113.9/32")
 
-			first := ledger.BanForLimit(netblock, midnight(), bans.Notes{})
-			second := ledger.BanForLimit(netblock, first.Expires.Add(tc.gap), bans.Notes{})
+			first, _ := ledger.BanForLimit(netblock, midnight(), bans.Notes{})
+			second, _ := ledger.BanForLimit(netblock, first.Expires.Add(tc.gap), bans.Notes{})
 
-			if second.Expires.Sub(second.Start) != tc.want || second.Notes.EarlierBans != 1 {
-				t.Errorf("second ban lasts %s with %d earlier bans, want %s and 1",
+			if second.Expires.Sub(second.Start) != tc.want ||
+				second.Notes.EarlierBans != (bans.EarlierBans{Limit: 1}) {
+				t.Errorf("second ban lasts %s with earlier bans %+v, want %s and 1 for a limit",
 					second.Expires.Sub(second.Start), second.Notes.EarlierBans, tc.want)
 			}
 		})
@@ -81,7 +83,7 @@ func TestFirstBanLongerThanTheMaximumIsPermanent(t *testing.T) {
 	rules.LimitBanDuration = rules.MaxBanDuration + time.Hour
 	ledger := bans.New(rules)
 
-	ban := ledger.BanForLimit(netip.MustParsePrefix("203.0.113.9/32"), midnight(),
+	ban, _ := ledger.BanForLimit(netip.MustParsePrefix("203.0.113.9/32"), midnight(),
 		bans.Notes{})
 	if !ban.Permanent() {
 		t.Errorf("first ban ends at %s, want a permanent one", ban.Expires)
@@ -101,7 +103,7 @@ func TestLongestBanSetFarOffDoesNotOverflow(t *testing.T) {
 	now := midnight()
 
 	for i := range 14 {
-		ban := ledger.BanForLimit(netblock, now, bans.Notes{})
+		ban, _ := ledger.BanForLimit(netblock, now, bans.Notes{})
 		if !ban.Expires.After(ban.Start) {
 			t.Fatalf("ban %d starts at %s and ends at %s", i+1, ban.Start, ban.Expires)
 		}
@@ -109,7 +111,7 @@ func TestLongestBanSetFarOffDoesNotOverflow(t *testing.T) {
 		now = ban.Expires
 	}
 
-	ban := ledger.BanForLimit(netblock, now, bans.Notes{})
+	ban, _ := ledger.BanForLimit(netblock, now, bans.Notes{})
 	if !ban.Permanent() {
 		t.Errorf("15th ban ends at %s, want a permanent one", ban.Expires)
 	}
@@ -121,12 +123,22 @@ func TestBrokenLimitDuringABanMakesNoOther(t *testing.T) {
 	ledger := bans.New(defaultRules())
 	netblock := netip.MustParsePrefix("203.0.113.9/32")
 
-	first := ledger.BanForLimit(netblock, midnight(), bans.Notes{})
-	again := ledger.BanForLimit(netblock, midnight().Add(time.Minute), bans.Notes{})
+	first, made := ledger.BanForLimit(netblock, midnight(), bans.Notes{})
+	if !made {
+		t.Error("the first ban was not made")
+	}
 
-	if again != first || len(ledger.Bans(netblock)) != 1 {
-		t.Errorf("a limit broken during a ban gave %+v and %d bans, want %+v and 1",
-			again, len(ledger.Bans(netblock)), first)
+	again, made := ledger.BanForLimit(netblock, midnight().Add(time.Minute), bans.Notes{})
+
+	if made || again != first || len(ledger.Bans(netblock)) != 1 {
+		t.Errorf("a limit broken during a ban gave %+v, made %t, and %d bans, "+
+			"want %+v, not made, and 1", again, made, len(ledger.Bans(netblock)), first)
+	}
+
+	again, made = ledger.BanForAttack(netblock, midnight().Add(time.Minute), bans.Notes{})
+	if made || again != first {
+		t.Errorf("an attack during a ban gave %+v, made %t, want %+v, not made",
+			again, made, first)
 	}
 }
 
@@ -135,21 +147,21 @@ func TestCheckRefusesWhileTheBanLastsAndCountsTheRefusals(t *testing.T) {
 
 	ledger := bans.New(defaultRules())
 	netblock := netip.MustParsePrefix("203.0.113.9/32")
-	ban := ledger.BanForLimit(netblock, midnight(), bans.Notes{Requests: 5})
+	ban, _ := ledger.BanForLimit(netblock, midnight(), bans.Notes{Requests: 5})
 
 	for range 3 {
-		got, banned := ledger.Check(netblock.Addr(), ban.Expires.Add(-time.Nanosecond))
+		got, banned, _ := ledger.Check(netblock.Addr(), ban.Expires.Add(-time.Nanosecond))
 		if !banned || got.Start != ban.Start {
 			t.Fatalf("check during the ban gives %+v and %t", got, banned)
 		}
 	}
 
-	_, banned := ledger.Check(netip.MustParseAddr("203.0.113.10"), midnight())
+	_, banned, _ := ledger.Check(netip.MustParseAddr("203.0.113.10"), midnight())
 	if banned {
 		t.Error("another netblock is banned")
 	}
 
-	_, banned = ledger.Check(netblock.Addr(), ban.Expires)
+	_, banned, _ = ledger.Check(netblock.Addr(), ban.Expires)
 	if banned {
 		t.Error("the ban did not end")
 	}
@@ -162,6 +174,28 @@ func TestCheckRefusesWhileTheBanLastsAndCountsTheRefusals(t *testing.T) {
 	}
 }
 
+func TestFindCountsNothing(t *testing.T) {
+	t.Parallel()
+
+	ledger := bans.New(defaultRules())
+	netblock := netip.MustParsePrefix("203.0.113.9/32")
+	ban, _ := ledger.BanForLimit(netblock, midnight(), bans.Notes{Requests: 5})
+
+	got, banned, _ := ledger.Find(netblock.Addr(), ban.Expires.Add(-time.Nanosecond))
+	if !banned || got != ban {
+		t.Errorf("find during the ban gives %+v and %t, want %+v", got, banned, ban)
+	}
+
+	_, banned, _ = ledger.Find(netblock.Addr(), ban.Expires)
+	if banned {
+		t.Error("the ban did not end")
+	}
+
+	if notes := ledger.Bans(netblock)[0].Notes; notes != ban.Notes {
+		t.Errorf("the notes are %+v, want them unchanged, %+v", notes, ban.Notes)
+	}
+}
+
 func TestMaxBansDropsTheEarliestBanOfTheNetblockSeenLongestAgo(t *testing.T) {
 	t.Parallel()
 
@@ -174,7 +208,7 @@ func TestMaxBansDropsTheEarliestBanOfTheNetblockSeenLongestAgo(t *testing.T) {
 	d := netip.MustParsePrefix("2001:db8::/64")
 	now := midnight()
 
-	first := ledger.BanForLimit(a, now, bans.Notes{})
+	first, _ := ledger.BanForLimit(a, now, bans.Notes{})
 	ledger.BanForLimit(b, now, bans.Notes{})
 	ledger.BanForLimit(c, now, bans.Notes{})
 
@@ -209,13 +243,192 @@ func TestFullLedgerDropsTheEarlierBanOfTheNetblockBannedAgain(t *testing.T) {
 	ledger := bans.New(rules)
 	netblock := netip.MustParsePrefix("203.0.113.9/32")
 
-	first := ledger.BanForLimit(netblock, midnight(), bans.Notes{})
-	second := ledger.BanForLimit(netblock, first.Expires, bans.Notes{})
+	first, _ := ledger.BanForLimit(netblock, midnight(), bans.Notes{})
+	second, _ := ledger.BanForLimit(netblock, first.Expires, bans.Notes{})
 
 	held := ledger.Bans(netblock)
-	if len(held) != 1 || held[0] != second || held[0].Notes.EarlierBans != 1 {
-		t.Errorf("the ledger holds %+v, want only the second ban, with 1 earlier ban",
-			held)
+	if len(held) != 1 || held[0] != second ||
+		held[0].Notes.EarlierBans != (bans.EarlierBans{Limit: 1}) {
+		t.Errorf("the ledger holds %+v, want only the second ban, "+
+			"with 1 earlier ban for a limit", held)
+	}
+}
+
+func TestRequestDuringAnAttackBanMakesItPermanent(t *testing.T) {
+	t.Parallel()
+
+	ledger := bans.New(defaultRules())
+	netblock := netip.MustParsePrefix("203.0.113.9/32")
+	notes := bans.Notes{RuleID: "env-file", Target: "path"}
+
+	ban, _ := ledger.BanForAttack(netblock, midnight(), notes)
+	if !ban.Expires.Equal(midnight().Add(7*day)) || ban.Cause != bans.CauseAttack ||
+		ban.Notes.RuleID != "env-file" || ledger.Made(bans.CauseAttack) != 1 ||
+		ledger.Made(bans.CauseLimit) != 0 {
+		t.Fatalf("the ban is %+v, with %d made for an attack and %d for a limit, "+
+			"want one for an attack, of seven days", ban,
+			ledger.Made(bans.CauseAttack), ledger.Made(bans.CauseLimit))
+	}
+
+	wantChanged(t, ledger, true)
+
+	// In observe mode the ban refuses nothing, and stays as it is, while
+	// Find tells that the request would have made it permanent.
+	got, _, wouldMakePermanent := ledger.Find(netblock.Addr(), midnight().Add(time.Hour))
+	if got.Permanent() || ledger.Bans(netblock)[0].Permanent() || !wouldMakePermanent {
+		t.Fatalf("a request found under the ban left it %+v, would have made it "+
+			"permanent %t, want it as it was, and true", got, wouldMakePermanent)
+	}
+
+	wantChanged(t, ledger, false)
+
+	// A request it refuses makes it permanent, says so, and makes
+	// bans.json due.
+	got, _, madePermanent := ledger.Check(netblock.Addr(), midnight().Add(time.Hour))
+	if !madePermanent || !got.Permanent() || !ledger.Bans(netblock)[0].Permanent() {
+		t.Fatalf("after a request during the ban, it is %+v, made permanent %t, "+
+			"want it made permanent", got, madePermanent)
+	}
+
+	wantChanged(t, ledger, true)
+
+	// The next request finds it permanent already.
+	_, banned, madePermanent := ledger.Check(netblock.Addr(), midnight().Add(100*365*day))
+	if !banned || madePermanent {
+		t.Errorf("a later request is banned %t, and made the ban permanent %t, "+
+			"want banned by the permanent ban", banned, madePermanent)
+	}
+}
+
+func TestAttackAfterAnAttackBanHasEndedBansPermanently(t *testing.T) {
+	t.Parallel()
+
+	ledger := bans.New(defaultRules())
+	netblock := netip.MustParsePrefix("203.0.113.9/32")
+
+	// A ban for a broken limit before does not count.
+	first, _ := ledger.BanForLimit(netblock, midnight(), bans.Notes{})
+	second, _ := ledger.BanForAttack(netblock, first.Expires, bans.Notes{})
+
+	if second.Expires.Sub(second.Start) != 7*day {
+		t.Fatalf("the first ban for an attack lasts %s, want 7 days",
+			second.Expires.Sub(second.Start))
+	}
+
+	// Once that has run out without a request, the netblock is served, and
+	// its next clear sign of attack bans it for good.
+	_, banned, _ := ledger.Check(netblock.Addr(), second.Expires)
+	if banned {
+		t.Fatal("the ban did not end")
+	}
+
+	// Its notes show the earlier ban for an attack that makes it permanent,
+	// beside the one for a limit.
+	third, _ := ledger.BanForAttack(netblock, second.Expires.Add(30*day), bans.Notes{})
+	if !third.Permanent() ||
+		third.Notes.EarlierBans != (bans.EarlierBans{Limit: 1, Attack: 1}) {
+		t.Errorf("the next ban for an attack is %+v, want a permanent one, "+
+			"with 1 earlier ban for a limit and 1 for an attack", third)
+	}
+}
+
+func TestWouldBanGivesTheBanWithoutMakingIt(t *testing.T) {
+	t.Parallel()
+
+	ledger := bans.New(defaultRules())
+	netblock := netip.MustParsePrefix("203.0.113.9/32")
+	first, _ := ledger.BanForLimit(netblock, midnight(), bans.Notes{})
+	wantChanged(t, ledger, true)
+
+	// While the first ban lasts, none would be made.
+	during, would := ledger.WouldBanForAttack(netblock, midnight(), bans.Notes{})
+	if would || during != first {
+		t.Errorf("during the first ban, would ban %t with %+v, want false with %+v",
+			would, during, first)
+	}
+
+	// As it ends, a clear sign of attack would ban for seven days, and a
+	// limit broken again for three hours, but neither is made.
+	limitNotes := bans.Notes{Limit: 1, Window: "minute"}
+	attack, wouldAttack := ledger.WouldBanForAttack(netblock, first.Expires,
+		bans.Notes{RuleID: "git-dir"})
+	limit, wouldLimit := ledger.WouldBanForLimit(netblock, first.Expires, limitNotes)
+
+	if !wouldAttack || !attack.Expires.Equal(first.Expires.Add(7*day)) ||
+		attack.Reason != "matched the rule git-dir" || !wouldLimit ||
+		!limit.Expires.Equal(first.Expires.Add(3*time.Hour)) ||
+		limit.Reason != "requests per minute over the limit of 1" {
+		t.Errorf("would ban with %+v and %+v, want seven days for the attack and "+
+			"three hours for the limit", attack, limit)
+	}
+
+	if len(ledger.Bans(netblock)) != 1 || ledger.Made(bans.CauseLimit) != 1 ||
+		ledger.Made(bans.CauseAttack) != 0 {
+		t.Errorf("the ledger holds %+v, want the first ban alone", ledger.Bans(netblock))
+	}
+
+	wantChanged(t, ledger, false)
+
+	// The ban made is the one that would have been.
+	made, _ := ledger.BanForLimit(netblock, first.Expires, limitNotes)
+	if made != limit {
+		t.Errorf("the ban made is %+v, want %+v", made, limit)
+	}
+}
+
+func TestWouldBePermanentAnswersAsTheBanWouldBeMade(t *testing.T) {
+	t.Parallel()
+
+	ledger := bans.New(defaultRules())
+	netblock := netip.MustParsePrefix("203.0.113.9/32")
+	now := midnight()
+
+	// Five bans for a limit in a row, of 1, 3, 9, 27 and 81 hours, are not
+	// permanent. The sixth, of 243 hours, would be, while a first ban for
+	// an attack would not.
+	for i := range 5 {
+		if ledger.WouldBePermanent(netblock, now, bans.CauseLimit) {
+			t.Fatalf("ban %d for a limit would be permanent", i+1)
+		}
+
+		ban, _ := ledger.BanForLimit(netblock, now, bans.Notes{})
+		now = ban.Expires
+	}
+
+	if !ledger.WouldBePermanent(netblock, now, bans.CauseLimit) {
+		t.Error("the sixth ban for a limit would not be permanent")
+	}
+
+	if ledger.WouldBePermanent(netblock, now, bans.CauseAttack) {
+		t.Error("a first ban for an attack would be permanent")
+	}
+
+	// Once a first ban for an attack has ended, the next would be permanent.
+	attack, _ := ledger.BanForAttack(netblock, now, bans.Notes{})
+	if !ledger.WouldBePermanent(netblock, attack.Expires, bans.CauseAttack) {
+		t.Error("a second ban for an attack would not be permanent")
+	}
+}
+
+func TestAttackBanDoesNotLengthenTheNextBanForALimit(t *testing.T) {
+	t.Parallel()
+
+	ledger := bans.New(defaultRules())
+	netblock := netip.MustParsePrefix("203.0.113.9/32")
+
+	// Three times the seven days would be permanent; a limit broken as the
+	// ban for an attack ends bans for an hour, as a first broken limit does.
+	attack, _ := ledger.BanForAttack(netblock, midnight(), bans.Notes{})
+	limit, _ := ledger.BanForLimit(netblock, attack.Expires, bans.Notes{})
+
+	if limit.Expires.Sub(limit.Start) != time.Hour || limit.Cause != bans.CauseLimit {
+		t.Errorf("the ban for a limit is %+v, want one of an hour", limit)
+	}
+
+	// And a request during the ban for a limit leaves it as it is.
+	got, _, madePermanent := ledger.Check(netblock.Addr(), limit.Start)
+	if got.Permanent() || madePermanent {
+		t.Error("a request during a ban for a limit made it permanent")
 	}
 }
 
@@ -229,7 +442,7 @@ func TestRequestTextsAreCutTo256Bytes(t *testing.T) {
 		Time: midnight(), Method: long, Host: long, Path: long, Status: 403, UserAgent: long,
 	}
 
-	ban := ledger.BanForLimit(netblock, midnight(), bans.Notes{Request: request})
+	ban, _ := ledger.BanForLimit(netblock, midnight(), bans.Notes{Request: request})
 
 	cut := long[:256]
 	want := bans.Request{
@@ -247,6 +460,7 @@ func defaultRules() bans.Rules {
 		LimitBanDuration:     time.Hour,
 		LimitBanRepeatWindow: day,
 		MaxBanDuration:       7 * day,
+		AttackBanDuration:    7 * day,
 		MaxBans:              5000,
 	}
 }

internal/bans/snapshot_test.go

@@ -42,7 +42,7 @@ func TestSnapshotListsEveryBanByNetblock(t *testing.T) {
 	high := netip.MustParsePrefix("203.0.113.10/32")
 	low := netip.MustParsePrefix("203.0.113.9/32")
 
-	first := ledger.BanForLimit(v6, midnight(), bans.Notes{})
+	first, _ := ledger.BanForLimit(v6, midnight(), bans.Notes{})
 	ledger.BanForLimit(high, midnight(), bans.Notes{})
 	ledger.BanForLimit(low, midnight(), bans.Notes{})
 	ledger.BanForLimit(v6, first.Expires, bans.Notes{})
@@ -68,7 +68,7 @@ func TestLoadedBansCarryOn(t *testing.T) {
 
 	before := bans.New(defaultRules())
 	netblock := netip.MustParsePrefix("203.0.113.9/32")
-	ban := before.BanForLimit(netblock, midnight(), bans.Notes{Limit: 1})
+	ban, _ := before.BanForLimit(netblock, midnight(), bans.Notes{Limit: 1})
 
 	// Loaded into a new ledger, as across a restart, the ban still refuses
 	// while it lasts, and once it has ended a broken limit bans for three
@@ -76,14 +76,15 @@ func TestLoadedBansCarryOn(t *testing.T) {
 	after := bans.New(defaultRules())
 	after.Load(before.Snapshot())
 
-	_, banned := after.Check(netblock.Addr(), ban.Expires.Add(-time.Second))
+	_, banned, _ := after.Check(netblock.Addr(), ban.Expires.Add(-time.Second))
 	if !banned {
 		t.Error("the loaded ban does not refuse")
 	}
 
-	again := after.BanForLimit(netblock, ban.Expires, bans.Notes{})
-	if again.Expires.Sub(again.Start) != 3*time.Hour || again.Notes.EarlierBans != 1 {
-		t.Errorf("the next ban lasts %s with %d earlier bans, want 3h and 1",
+	again, _ := after.BanForLimit(netblock, ban.Expires, bans.Notes{})
+	if again.Expires.Sub(again.Start) != 3*time.Hour ||
+		again.Notes.EarlierBans != (bans.EarlierBans{Limit: 1}) {
+		t.Errorf("the next ban lasts %s with earlier bans %+v, want 3h and 1 for a limit",
 			again.Expires.Sub(again.Start), again.Notes.EarlierBans)
 	}
 }
@@ -110,7 +111,7 @@ func TestLoadedBanRefusesEveryClientInItsNetblock(t *testing.T) {
 		"198.51.100.7":    true,
 		"198.51.100.8":    false,
 	} {
-		_, banned := ledger.Check(netip.MustParseAddr(client), midnight())
+		_, banned, _ := ledger.Check(netip.MustParseAddr(client), midnight())
 		if banned != want {
 			t.Errorf("%s is refused: %t, want %t", client, banned, want)
 		}
@@ -130,14 +131,91 @@ func TestLoadedBanRefusesEveryClientInItsNetblock(t *testing.T) {
 	}
 }
 
+func TestPermanentBanStartedBeforeAnEndedOneRefuses(t *testing.T) {
+	t.Parallel()
+
+	// As when an admin adds a permanent ban to bans.json with a start
+	// before that of the netblock's ban that has ended.
+	netblock := netip.MustParsePrefix("203.0.113.0/24")
+	permanent := bans.Ban{Netblock: netblock, Start: midnight().Add(-time.Hour)}
+	ended := bans.Ban{
+		Netblock: netblock,
+		Start:    midnight(),
+		Expires:  midnight().Add(time.Hour),
+	}
+
+	ledger := bans.New(defaultRules())
+	ledger.Load([]bans.Ban{permanent, ended})
+
+	now := midnight().Add(2 * time.Hour)
+	client := netip.MustParseAddr("203.0.113.9")
+
+	ban, banned, _ := ledger.Find(client, now)
+	if !banned || !ban.Permanent() {
+		t.Errorf("find gives %+v and %t, want the permanent ban", ban, banned)
+	}
+
+	ban, banned, _ = ledger.Check(client, now)
+	if !banned || !ban.Permanent() {
+		t.Errorf("the client is refused: %t, under %+v, want under the permanent ban",
+			banned, ban)
+	}
+
+	// A limit broken now makes no shorter ban over the permanent one.
+	ban, _ = ledger.BanForLimit(netblock, now, bans.Notes{})
+	if !ban.Permanent() || len(ledger.Bans(netblock)) != 2 {
+		t.Errorf("a broken limit returned %+v and left the netblock %d bans, "+
+			"want the permanent ban and 2", ban, len(ledger.Bans(netblock)))
+	}
+}
+
+func TestNextBanWorkedOutFromTheBanThatEndedLast(t *testing.T) {
+	t.Parallel()
+
+	// A 9-hour ban smallwebwaf made, the third in a row, and an admin's
+	// 1-hour ban added to bans.json over it, with no cause and no notes.
+	netblock := netip.MustParsePrefix("203.0.113.9/32")
+	nineHours := bans.Ban{
+		Netblock: netblock,
+		Start:    midnight(),
+		Expires:  midnight().Add(9 * time.Hour),
+		Cause:    bans.CauseLimit,
+		Notes:    bans.Notes{EarlierBans: bans.EarlierBans{Limit: 2}},
+	}
+	admins := bans.Ban{
+		Netblock: netblock,
+		Start:    midnight().Add(time.Hour),
+		Expires:  midnight().Add(2 * time.Hour),
+	}
+
+	ledger := bans.New(defaultRules())
+	ledger.Load([]bans.Ban{nineHours, admins})
+
+	// Once both have ended, a limit broken within the repeat window bans
+	// for three times the 9 hours, and the notes count the two bans
+	// before the 9-hour one and it, for a limit, and the admin's.
+	ban, _ := ledger.BanForLimit(netblock, nineHours.Expires.Add(time.Hour), bans.Notes{})
+	if ban.Expires.Sub(ban.Start) != 27*time.Hour ||
+		ban.Notes.EarlierBans != (bans.EarlierBans{Limit: 3, Admin: 1}) {
+		t.Errorf("the next ban lasts %s with earlier bans %+v, "+
+			"want 27h, 3 for a limit and 1 an admin's",
+			ban.Expires.Sub(ban.Start), ban.Notes.EarlierBans)
+	}
+}
+
 func TestLoadKeepsAtMostMaxBansDroppingTheEarliest(t *testing.T) {
 	t.Parallel()
 
 	// bans.json lists the bans by netblock, not in the order they began.
-	later := bans.Ban{Netblock: netip.MustParsePrefix("203.0.113.1/32"), Start: midnight()}
+	later := bans.Ban{
+		Netblock: netip.MustParsePrefix("203.0.113.1/32"),
+		Start:    midnight(),
+		Cause:    bans.CauseLimit,
+	}
 	earlier := bans.Ban{
 		Netblock: netip.MustParsePrefix("203.0.113.2/32"),
 		Start:    midnight().Add(-time.Hour),
+		Cause:    bans.CauseLimit,
 	}
 
 	rules := defaultRules()
@@ -151,6 +229,49 @@ func TestLoadKeepsAtMostMaxBansDroppingTheEarliest(t *testing.T) {
 	}
 }
 
+func TestLoadReplacesTheBansHeld(t *testing.T) {
+	t.Parallel()
+
+	// Room for three bans, so that the second load, were it added to the
+	// two bans held, would drop none of them to make room.
+	rules := defaultRules()
+	rules.MaxBans = 3
+	ledger := bans.New(rules)
+	kept := bans.Ban{
+		Netblock: netip.MustParsePrefix("2001:db8::/64"),
+		Start:    midnight(),
+		Cause:    bans.CauseLimit,
+	}
+	ledger.Load([]bans.Ban{
+		{
+			Netblock: netip.MustParsePrefix("203.0.113.0/24"),
+			Start:    midnight(),
+			Cause:    bans.CauseLimit,
+		},
+		kept,
+	})
+
+	// Loaded again without the first ban, as when an admin's edit of
+	// bans.json is taken in, that ban is lifted.
+	ledger.Load([]bans.Ban{kept})
+
+	_, banned, _ := ledger.Check(netip.MustParseAddr("203.0.113.9"), midnight())
+	if banned {
+		t.Error("a ban left out of the second load still refuses")
+	}
+
+	// The ledger holds one ban, so it makes two more without dropping any.
+	first, _ := ledger.BanForLimit(netip.MustParsePrefix("198.51.100.7/32"), midnight(),
+		bans.Notes{})
+	second, _ := ledger.BanForLimit(netip.MustParsePrefix("198.51.100.8/32"), midnight(),
+		bans.Notes{})
+
+	want := []bans.Ban{first, second, kept}
+	if got := ledger.Snapshot(); !slices.Equal(got, want) {
+		t.Errorf("the ledger holds %+v, want %+v", got, want)
+	}
+}
+
 func TestLoadCutsTheTextsTo256Bytes(t *testing.T) {
 	t.Parallel()
 

internal/config/config.go

@@ -1,9 +1,11 @@
 // Package config reads smallwebwaf's settings. Every setting is an
-// environment variable whose name starts with SWWAF_, every setting has a
-// default, and this package is the one place they are read.
+// environment variable whose name starts with SWWAF_, or a file such a
+// variable names, every setting has a default, and this package is the
+// one place they are read.
 package config
 
 import (
+	"crypto/x509"
 	"errors"
 	"fmt"
 	"log/slog"
@@ -12,12 +14,17 @@ import (
 	"net/http"
 	"net/netip"
 	"net/url"
+	"os"
 	"path/filepath"
 	"slices"
 	"strconv"
 	"strings"
 	"time"
+	"unicode"
 	"unicode/utf8"
+
+	"sneak.berlin/go/smallwebwaf/internal/alerts"
+	"sneak.berlin/go/smallwebwaf/internal/remotelog"
 )
 
 // Config is smallwebwaf's settings. A timeout, size or rate limit of zero
@@ -27,6 +34,16 @@ type Config struct {
 	ListenAddr string
 	// UpstreamURL is the app (SWWAF_UPSTREAM_URL).
 	UpstreamURL *url.URL
+	// InstanceName is the name every log line and alert gives as instance,
+	// and every metric carries as its label instance (SWWAF_INSTANCE_NAME),
+	// by default the host's name, which docker sets to the first 12
+	// characters of the container's id.
+	InstanceName string
+	// Observe is true in observe mode, when SWWAF_MODE is observe rather
+	// than enforce: a request that SWWAF_DENY_NETS, a ban, the country
+	// lists, a rate limit or a rule would refuse is passed to the app
+	// instead, and no ban is made.
+	Observe bool
 	// TrustedProxies are the netblocks whose X-Forwarded-For is
 	// believed (SWWAF_TRUSTED_PROXIES).
 	TrustedProxies []netip.Prefix
@@ -84,7 +101,8 @@ type Config struct {
 	// BanResponse is the status a refused client is answered with, 403
 	// or 429, or 0 to close the connection without an answer
 	// (SWWAF_BAN_RESPONSE). It answers a banned client, a request that
-	// breaks a rate limit, SWWAF_DENY_NETS and the country lists.
+	// breaks a rate limit or matches a ban rule, SWWAF_DENY_NETS and the
+	// country lists.
 	BanResponse int
 	// LimitBanDuration is the ban for a first broken rate limit
 	// (SWWAF_LIMIT_BAN_DURATION). A limit broken again within
@@ -95,6 +113,9 @@ type Config struct {
 	LimitBanDuration     time.Duration
 	LimitBanRepeatWindow time.Duration
 	MaxBanDuration       time.Duration
+	// AttackBanDuration is the ban for a first clear sign of attack
+	// (SWWAF_ATTACK_BAN_DURATION). It cannot be off.
+	AttackBanDuration time.Duration
 	// MaxBans is the most bans held (SWWAF_MAX_BANS).
 	MaxBans int
 	// BanScopeV4Prefix is the length of the netblock around an IPv4
@@ -108,15 +129,62 @@ type Config struct {
 	StateDir             string
 	StateWriteDelay      time.Duration
 	StateCounterInterval time.Duration
+	// LogRequestHeaders are the request headers whose values the request
+	// log gives, in lower case (SWWAF_LOG_REQUEST_HEADERS).
+	LogRequestHeaders []string
+	// AdminToken is the bearer token an admin sends for the ban endpoints
+	// and /_smallwebwaf/clients/<ip> (SWWAF_ADMIN_TOKEN), "" while it is
+	// unset and they are off.
+	AdminToken string
 	// MetricsToken is the bearer token a scraper sends for the metrics
 	// (SWWAF_METRICS_TOKEN), "" while it is unset and the metrics are off.
 	// MetricsTopN is how many countries get series of their own in the
 	// metrics (SWWAF_METRICS_TOP_N).
 	MetricsToken string
 	MetricsTopN  int
+	// RulesDir is the directory of the rule files (SWWAF_RULES_DIR), read
+	// unless RulesEnabled is false (SWWAF_RULES_ENABLED).
+	RulesDir     string
+	RulesEnabled bool
+	// LogRemoteURL is where every line on stdout is also sent
+	// (SWWAF_LOG_REMOTE_URL), nil while it is unset and nothing is sent.
+	// LogRemoteTLSCAs are the certificates a syslog+tls endpoint's
+	// certificate must chain to (SWWAF_LOG_REMOTE_TLS_CA_FILE), nil while
+	// it is unset and the host's own are used. LogRemoteBuffer is the most
+	// lines held while they wait to be sent (SWWAF_LOG_REMOTE_BUFFER).
+	// LogRemoteFacility is the number of the syslog facility
+	// (SWWAF_LOG_REMOTE_FACILITY), and LogRemoteAppName the APP-NAME
+	// (SWWAF_LOG_REMOTE_APP_NAME, by default InstanceName), of the records
+	// the lines are sent in.
+	LogRemoteURL      *url.URL
+	LogRemoteTLSCAs   *x509.CertPool
+	LogRemoteBuffer   int
+	LogRemoteFacility int
+	LogRemoteAppName  string
+	// AlertWebhookURL is where each alert is posted as JSON
+	// (SWWAF_ALERT_WEBHOOK_URL), nil while it is unset. AlertWebhookHeaders
+	// are sent with each (SWWAF_ALERT_WEBHOOK_HEADERS).
+	// AlertSlackWebhookURL is the Slack incoming webhook each alert is
+	// posted to as a message (SWWAF_ALERT_SLACK_WEBHOOK_URL), and
+	// AlertNtfyURL the ntfy topic each is published to
+	// (SWWAF_ALERT_NTFY_URL), each nil while it is unset; AlertNtfyToken,
+	// unless empty, is sent to ntfy with each (SWWAF_ALERT_NTFY_TOKEN).
+	// With none of the three URLs set, no alert is sent. AlertEvents are
+	// the events alerts are sent for (SWWAF_ALERT_EVENTS). A repeat of an
+	// alert within AlertCooldown is held back (SWWAF_ALERT_COOLDOWN), and
+	// so is an alert past AlertMaxPerHour in an hour, for the hour's
+	// summary (SWWAF_ALERT_MAX_PER_HOUR); 0 is off for both.
+	AlertWebhookURL      *url.URL
+	AlertWebhookHeaders  http.Header
+	AlertSlackWebhookURL *url.URL
+	AlertNtfyURL         *url.URL
+	AlertNtfyToken       string
+	AlertEvents          []string
+	AlertCooldown        time.Duration
+	AlertMaxPerHour      int
 
-	// settings are the values read, as given or by default, for the
-	// log line at start.
+	// settings are the values read, as given or by default, and the
+	// files they were read from, for the log line at start.
 	settings []slog.Attr
 }
 
@@ -132,8 +200,13 @@ const (
 	ipv4Bits = 32
 	// minTokenLength is the fewest characters a token may have.
 	minTokenLength = 32
-	// masked is what the log shows for a token that is set.
+	// masked is what the log shows for a token that is set, and in place of
+	// a secret in another setting.
 	masked = "********"
+	// defaultListenAddr and defaultUpstreamURL are the defaults of
+	// SWWAF_LISTEN_ADDR and SWWAF_UPSTREAM_URL.
+	defaultListenAddr  = ":8080"
+	defaultUpstreamURL = "http://127.0.0.1:8081"
 )
 
 var (
@@ -154,6 +227,11 @@ var (
 			"such as http://127.0.0.1:8081")
 	errNotCountry = errors.New(
 		"is not a two-letter country code such as de or kp")
+	errNotHeaderName = errors.New(
+		"is not a header name such as accept-language")
+	errHeaderTakenOut = errors.New(
+		"is taken out of every request by Go's HTTP server, so it can never " +
+			"be logged")
 	errOnBothLists          = errors.New("is in SWWAF_DENIED_COUNTRIES too")
 	errNotOver4K            = errors.New("is not a size of more than 4K, such as 32K")
 	errNotDurationAboveZero = errors.New(
@@ -166,18 +244,45 @@ var (
 	errNotAbsolutePath = errors.New(
 		"is not an absolute path, such as /var/lib/smallwebwaf")
 	errShortToken    = errors.New("is shorter than 32 characters")
+	errNotMode       = errors.New("is not enforce or observe")
 	errNotPathPrefix = errors.New(
 		"is not a path prefix starting with /, such as /assets/")
+	errNotBoolean      = errors.New("is not true or false")
+	errNotLogRemoteURL = errors.New(
+		"is not syslog+udp, syslog+tcp or syslog+tls with a host and a port, " +
+			"and nothing more, such as syslog+tls://logs.example:6514")
+	errNoCertificate = errors.New("holds no PEM certificate")
+	errNotFacility   = errors.New("is not a syslog facility such as local0 or daemon")
+	errNotAppName    = errors.New(
+		"is not 1 to 48 printable ASCII characters without a space, such as gitea")
+	errSetTwice      = errors.New("set only one of them")
+	errNotWebhookURL = errors.New(
+		"is not an http or https URL without a user or a fragment, " +
+			"such as https://alerts.example/smallwebwaf")
+	errNotWebhookHeader = errors.New(
+		"is not a header name followed by : and the header's value, " +
+			"such as Authorization:Bearer <token>")
+	errControlCharacter = errors.New(
+		"holds a control character, such as the carriage return of a Windows line end")
+	errNotAlertEvent = errors.New(
+		"is not ban, permanent_ban, waf_block, anomaly, reputation_hit, " +
+			"source_failure or file_error")
+	errNotNumberOrOff = errors.New("is not a whole number above zero, such as 60, or off")
+	errNotUTF8        = errors.New("is not valid UTF-8")
 )
 
 // FromEnvironment reads the settings with lookupEnv, normally
-// os.LookupEnv. A setting that is not set takes its default. A setting
-// that is set but invalid is an error that names it.
+// os.LookupEnv. A setting may instead be given as a file: the variable
+// named by the setting's name with _FILE added names the file, which is
+// read now (see lookup). A setting that is not set takes its default. A
+// setting that is set but invalid is an error that names it.
 func FromEnvironment(lookupEnv func(string) (string, bool)) (*Config, error) {
 	env := &environment{lookupEnv: lookupEnv}
 	cfg := &Config{
-		ListenAddr:           env.address("SWWAF_LISTEN_ADDR", ":8080"),
-		UpstreamURL:          env.appURL("SWWAF_UPSTREAM_URL", "http://127.0.0.1:8081"),
+		ListenAddr:           env.address("SWWAF_LISTEN_ADDR", defaultListenAddr),
+		UpstreamURL:          env.appURL("SWWAF_UPSTREAM_URL", defaultUpstreamURL),
+		InstanceName:         env.instanceName(),
+		Observe:              env.observe("SWWAF_MODE", "enforce"),
 		TrustedProxies:       env.netblocks("SWWAF_TRUSTED_PROXIES", privateRanges),
 		ClientRequestTimeout: env.duration("SWWAF_CLIENT_REQUEST_TIMEOUT", "60s"),
 		ClientRequestHeaderMaxBytes: env.headerSize(
@@ -202,15 +307,39 @@ func FromEnvironment(lookupEnv func(string) (string, bool)) (*Config, error) {
 		LimitBanDuration:     env.durationNotOff("SWWAF_LIMIT_BAN_DURATION", "1h"),
 		LimitBanRepeatWindow: env.durationNotOff("SWWAF_LIMIT_BAN_REPEAT_WINDOW", "24h"),
 		MaxBanDuration:       env.durationNotOff("SWWAF_MAX_BAN_DURATION", "7d"),
+		AttackBanDuration:    env.durationNotOff("SWWAF_ATTACK_BAN_DURATION", "7d"),
 		MaxBans:              env.numberNotOff("SWWAF_MAX_BANS", "5000"),
 		BanScopeV4Prefix:     env.v4Prefix("SWWAF_BAN_SCOPE_V4_PREFIX", "32"),
 		StateDir:             env.absolutePath("SWWAF_STATE_DIR", "/var/lib/smallwebwaf"),
 		StateWriteDelay:      env.durationNotOff("SWWAF_STATE_WRITE_DELAY", "10s"),
 		StateCounterInterval: env.durationNotOff("SWWAF_STATE_COUNTER_INTERVAL", "15m"),
+		LogRequestHeaders: env.headerNames("SWWAF_LOG_REQUEST_HEADERS",
+			"accept,accept-language,accept-encoding,content-type,origin,range"),
+		AdminToken:           env.token("SWWAF_ADMIN_TOKEN"),
 		MetricsToken:         env.token("SWWAF_METRICS_TOKEN"),
 		MetricsTopN:          env.numberNotOff("SWWAF_METRICS_TOP_N", "50"),
+		RulesDir:             env.value("SWWAF_RULES_DIR", "/etc/smallwebwaf/rules.d"),
+		RulesEnabled:         env.boolean("SWWAF_RULES_ENABLED", "true"),
+		LogRemoteURL:         env.logRemoteURL("SWWAF_LOG_REMOTE_URL"),
+		LogRemoteTLSCAs:      env.certificates("SWWAF_LOG_REMOTE_TLS_CA_FILE"),
+		LogRemoteBuffer:      env.numberNotOff("SWWAF_LOG_REMOTE_BUFFER", "10000"),
+		LogRemoteFacility:    env.facility("SWWAF_LOG_REMOTE_FACILITY", "local0"),
+		AlertWebhookURL:      env.webhookURL("SWWAF_ALERT_WEBHOOK_URL"),
+		AlertWebhookHeaders:  env.webhookHeaders("SWWAF_ALERT_WEBHOOK_HEADERS"),
+		AlertSlackWebhookURL: env.webhookURL("SWWAF_ALERT_SLACK_WEBHOOK_URL"),
+		AlertNtfyURL:         env.webhookURL("SWWAF_ALERT_NTFY_URL"),
+		AlertNtfyToken:       env.secret("SWWAF_ALERT_NTFY_TOKEN"),
+		AlertEvents: env.alertEvents("SWWAF_ALERT_EVENTS",
+			strings.Join(alerts.Events(), ",")),
+		AlertCooldown:   env.duration("SWWAF_ALERT_COOLDOWN", "15m"),
+		AlertMaxPerHour: env.numberOrOff("SWWAF_ALERT_MAX_PER_HOUR", "60"),
 	}
 
+	cfg.LogRemoteAppName = env.appName("SWWAF_LOG_REMOTE_APP_NAME",
+		cfg.InstanceName, cfg.LogRemoteURL != nil)
+
+	env.checkInstanceNameForNtfy(cfg.InstanceName, cfg.AlertNtfyURL != nil)
+
 	for _, country := range cfg.ExclusivelyAllowedCountries {
 		if slices.Contains(cfg.DeniedCountries, country) {
 			env.check("SWWAF_EXCLUSIVELY_ALLOWED_COUNTRIES",
@@ -227,6 +356,34 @@ func FromEnvironment(lookupEnv func(string) (string, bool)) (*Config, error) {
 	return cfg, nil
 }
 
+// ListenAddrAndUpstreamURL reads only SWWAF_LISTEN_ADDR and
+// SWWAF_UPSTREAM_URL, either of which may be given as a file, as
+// FromEnvironment does. The health check needs no other setting, so it
+// reads no other, nor a file that another names.
+func ListenAddrAndUpstreamURL(
+	lookupEnv func(string) (string, bool),
+) (string, *url.URL, error) {
+	env := &environment{lookupEnv: lookupEnv}
+	listenAddr := env.address("SWWAF_LISTEN_ADDR", defaultListenAddr)
+	upstreamURL := env.appURL("SWWAF_UPSTREAM_URL", defaultUpstreamURL)
+
+	if env.err != nil {
+		return "", nil, env.err
+	}
+
+	return listenAddr, upstreamURL, nil
+}
+
+// InstanceName reads only SWWAF_INSTANCE_NAME, which may be given as a
+// file, as FromEnvironment does, so that the line saying a setting is
+// invalid carries it too. A file that cannot be read gives the default
+// here, and FromEnvironment then stops the start over it.
+func InstanceName(lookupEnv func(string) (string, bool)) string {
+	env := &environment{lookupEnv: lookupEnv}
+
+	return env.instanceName()
+}
+
 // privateRanges are the private address ranges, the default trusted
 // proxies.
 const privateRanges = "10.0.0.0/8,172.16.0.0/12,192.168.0.0/16"
@@ -248,8 +405,8 @@ type environment struct {
 // value returns a setting's value, or its default when it is not set,
 // and notes it for the log.
 func (e *environment) value(name, defaultValue string) string {
-	value, ok := e.lookupEnv(name)
-	if !ok {
+	value, set := e.lookup(name)
+	if !set {
 		value = defaultValue
 	}
 
@@ -258,6 +415,37 @@ func (e *environment) value(name, defaultValue string) string {
 	return value
 }
 
+// lookup returns a setting's value and whether it is set: the value of the
+// variable name, or the contents of the file that the variable name_FILE
+// names, less one newline at their end. It notes that file's path for the
+// log. Both variables set, or a file that cannot be read, is an error.
+func (e *environment) lookup(name string) (string, bool) {
+	value, set := e.lookupEnv(name)
+	fileName := name + "_FILE"
+
+	path, inFile := e.lookupEnv(fileName)
+	if !inFile {
+		return value, set
+	}
+
+	if set {
+		e.check(name, fmt.Errorf("is set, and so is %s; %w", fileName, errSetTwice))
+
+		return value, set
+	}
+
+	e.settings = append(e.settings, slog.String(fileName, path))
+
+	contents, err := os.ReadFile(path) //nolint:gosec // a file the admin names
+	if err != nil {
+		e.check(fileName, fmt.Errorf("cannot be read: %w", err))
+
+		return "", false
+	}
+
+	return strings.TrimSuffix(string(contents), "\n"), true
+}
+
 // check keeps the first error, naming the setting it is about.
 func (e *environment) check(name string, err error) {
 	if err != nil && e.err == nil {
@@ -281,6 +469,27 @@ func (e *environment) appURL(name, defaultValue string) *url.URL {
 	return upstream
 }
 
+// observe reads the setting that is the mode, enforce or observe, and
+// reports whether it is observe.
+func (e *environment) observe(name, defaultValue string) bool {
+	mode := e.value(name, defaultValue)
+	if mode != "enforce" && mode != "observe" {
+		e.check(name, fmt.Errorf("%q %w", mode, errNotMode))
+	}
+
+	return mode == "observe"
+}
+
+// boolean reads a setting that is true or false.
+func (e *environment) boolean(name, defaultValue string) bool {
+	value := e.value(name, defaultValue)
+	if value != "true" && value != "false" {
+		e.check(name, fmt.Errorf("%q %w", value, errNotBoolean))
+	}
+
+	return value == "true"
+}
+
 // netblocks reads a setting that is a list of netblocks.
 func (e *environment) netblocks(name, defaultValue string) []netip.Prefix {
 	netblocks, err := parseNetblocks(e.value(name, defaultValue))
@@ -338,10 +547,19 @@ func (e *environment) countries(name, defaultValue string) []string {
 	return countries
 }
 
+// headerNames reads a setting that is a list of header names, and
+// returns them in lower case.
+func (e *environment) headerNames(name, defaultValue string) []string {
+	headers, err := parseHeaderNames(e.value(name, defaultValue))
+	e.check(name, err)
+
+	return headers
+}
+
 // durationNotOff reads a setting that is a duration and, unlike a
 // timeout, cannot be off.
 func (e *environment) durationNotOff(name, defaultValue string) time.Duration {
-	duration, err := parseDurationNotOff(e.value(name, defaultValue))
+	duration, err := ParseDurationNotOff(e.value(name, defaultValue))
 	e.check(name, err)
 
 	return duration
@@ -386,7 +604,7 @@ func (e *environment) absolutePath(name, defaultValue string) string {
 // switches off what it guards; set, it must be at least minTokenLength
 // characters. Neither the log nor an error shows its value.
 func (e *environment) token(name string) string {
-	value, set := e.lookupEnv(name)
+	value, set := e.lookup(name)
 	if !set {
 		e.settings = append(e.settings, slog.String(name, ""))
 
@@ -402,6 +620,170 @@ func (e *environment) token(name string) string {
 	return value
 }
 
+// logRemoteURL reads the setting that is where every log line is also
+// sent. Unset or empty, it is nil, and nothing is sent.
+func (e *environment) logRemoteURL(name string) *url.URL {
+	value := e.value(name, "")
+	if value == "" {
+		return nil
+	}
+
+	remote, err := parseLogRemoteURL(value)
+	e.check(name, err)
+
+	return remote
+}
+
+// certificates reads a setting that is the path of a file of PEM
+// certificates. Unset or empty, it is nil. Its value names a file
+// already, so, unlike the other settings, it has no _FILE form.
+func (e *environment) certificates(name string) *x509.CertPool {
+	path, _ := e.lookupEnv(name)
+	e.settings = append(e.settings, slog.String(name, path))
+
+	if path == "" {
+		return nil
+	}
+
+	pem, err := os.ReadFile(path) //nolint:gosec // a file the admin names
+	if err != nil {
+		e.check(name, fmt.Errorf("cannot be read: %w", err))
+
+		return nil
+	}
+
+	pool := x509.NewCertPool()
+	if !pool.AppendCertsFromPEM(pem) {
+		e.check(name, fmt.Errorf("%q %w", path, errNoCertificate))
+
+		return nil
+	}
+
+	return pool
+}
+
+// facility reads a setting that is a syslog facility, and returns its
+// number.
+func (e *environment) facility(name, defaultValue string) int {
+	number, err := parseFacility(e.value(name, defaultValue))
+	e.check(name, err)
+
+	return number
+}
+
+// instanceName reads SWWAF_INSTANCE_NAME, by default the host's name. It
+// must be valid UTF-8: the metrics library panics on a label that is not.
+func (e *environment) instanceName() string {
+	hostname, _ := os.Hostname() // "" when the host has no name to give
+
+	value := e.value("SWWAF_INSTANCE_NAME", hostname)
+	if !utf8.ValidString(value) {
+		e.check("SWWAF_INSTANCE_NAME", fmt.Errorf("%q %w", value, errNotUTF8))
+	}
+
+	return value
+}
+
+// appName reads the setting that is the APP-NAME of the records the log
+// lines are sent in, by default the instance name. Its value is checked
+// when it is set, and, while lines are sent, when it is the instance name.
+func (e *environment) appName(name, instanceName string, sending bool) string {
+	value, set := e.lookup(name)
+	if !set {
+		value = instanceName
+	}
+
+	e.settings = append(e.settings, slog.String(name, value))
+
+	switch {
+	case isAppName(value):
+	case set:
+		e.check(name, fmt.Errorf("%q %w", value, errNotAppName))
+	case sending:
+		e.check(name, fmt.Errorf("is unset, and SWWAF_INSTANCE_NAME %q, its default, %w",
+			value, errNotAppName))
+	}
+
+	return value
+}
+
+// checkInstanceNameForNtfy refuses an instance name that holds a control
+// character while ntfySet, SWWAF_ALERT_NTFY_URL being set: ntfy is sent
+// the instance name in a header, which cannot hold one.
+func (e *environment) checkInstanceNameForNtfy(instanceName string, ntfySet bool) {
+	if ntfySet && strings.ContainsFunc(instanceName, unicode.IsControl) {
+		e.check("SWWAF_INSTANCE_NAME", fmt.Errorf(
+			"%q %w, and is sent to ntfy in a header while SWWAF_ALERT_NTFY_URL is set",
+			instanceName, errControlCharacter))
+	}
+}
+
+// webhookURL reads a setting that is a URL each alert is posted to:
+// SWWAF_ALERT_WEBHOOK_URL, SWWAF_ALERT_SLACK_WEBHOOK_URL or
+// SWWAF_ALERT_NTFY_URL. Unset or empty, it is nil, and no alert is posted
+// there. The log shows ******** in place of its path and query, and an
+// error shows none of it, since a webhook or an ntfy topic can carry its
+// secret there.
+func (e *environment) webhookURL(name string) *url.URL {
+	value, _ := e.lookup(name)
+	webhook, logged, err := parseWebhookURL(value)
+	e.settings = append(e.settings, slog.String(name, logged))
+	e.check(name, err)
+
+	return webhook
+}
+
+// webhookHeaders reads the setting that is the headers sent with each
+// alert. The log shows each header's value as ********, since a header
+// such as Authorization carries a secret.
+func (e *environment) webhookHeaders(name string) http.Header {
+	value, _ := e.lookup(name)
+	headers, logged, err := parseWebhookHeaders(value)
+	e.settings = append(e.settings, slog.String(name, logged))
+	e.check(name, err)
+
+	return headers
+}
+
+// secret reads a setting that is a secret another service gave, such as
+// an ntfy token, "" while it is unset. It is sent in a header, which
+// cannot hold a control character, so one in it is an error. The log
+// shows ******** in place of a value that is not empty, and an error
+// shows none of it.
+func (e *environment) secret(name string) string {
+	value, _ := e.lookup(name)
+
+	logged := ""
+	if value != "" {
+		logged = masked
+	}
+
+	e.settings = append(e.settings, slog.String(name, logged))
+
+	if strings.ContainsFunc(value, unicode.IsControl) {
+		e.check(name, errControlCharacter)
+	}
+
+	return value
+}
+
+// alertEvents reads the setting that is the events alerts are sent for.
+func (e *environment) alertEvents(name, defaultValue string) []string {
+	events, err := parseAlertEvents(e.value(name, defaultValue))
+	e.check(name, err)
+
+	return events
+}
+
+// numberOrOff reads a setting that is a whole number above zero, or off,
+// which is 0.
+func (e *environment) numberOrOff(name, defaultValue string) int {
+	number, err := parseNumberOrOff(e.value(name, defaultValue))
+	e.check(name, err)
+
+	return number
+}
+
 // parseDuration reads a duration in Go's syntax, such as 90s or 15m, a
 // whole number of days such as 7d, or off.
 func parseDuration(value string) (time.Duration, error) {
@@ -503,9 +885,9 @@ func parseCount(value string) (int64, error) {
 	return n, nil
 }
 
-// parseDurationNotOff reads a duration above zero, as parseDuration does,
-// but not off.
-func parseDurationNotOff(value string) (time.Duration, error) {
+// ParseDurationNotOff reads a duration above zero, as parseDuration does,
+// but not off. The ban endpoint reads the duration of a ban with it too.
+func ParseDurationNotOff(value string) (time.Duration, error) {
 	duration, err := parseDuration(value)
 	if err != nil || duration == 0 {
 		return 0, fmt.Errorf("%q %w", value, errNotDurationAboveZero)
@@ -681,6 +1063,58 @@ func parseCountries(value string) ([]string, error) {
 	return countries, nil
 }
 
+// headerNameChars are the characters RFC 9110 allows in a header name:
+// letters, digits and these marks.
+const headerNameChars = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz" +
+	"0123456789!#$%&'*+-.^_`|~"
+
+// IsHeaderName reports whether name can be a header name: one or more of
+// the characters RFC 9110 allows in one.
+func IsHeaderName(name string) bool {
+	if name == "" {
+		return false
+	}
+
+	for _, char := range name {
+		if !strings.ContainsRune(headerNameChars, char) {
+			return false
+		}
+	}
+
+	return true
+}
+
+// parseHeaderNames reads a comma-separated list of header names in either
+// case, and returns them in lower case. Host and Transfer-Encoding are
+// refused: Go's HTTP server takes them out of the request's headers.
+func parseHeaderNames(value string) ([]string, error) {
+	items, err := parseList(value)
+	if err != nil {
+		return nil, err
+	}
+
+	headers := make([]string, 0, len(items))
+
+	for _, item := range items {
+		if !IsHeaderName(item) {
+			return nil, fmt.Errorf("%q %w", item, errNotHeaderName)
+		}
+
+		header := strings.ToLower(item)
+		switch header {
+		case "host":
+			return nil, fmt.Errorf("%q %w; the request's host is the field host",
+				item, errHeaderTakenOut)
+		case "transfer-encoding":
+			return nil, fmt.Errorf("%q %w", item, errHeaderTakenOut)
+		}
+
+		headers = append(headers, header)
+	}
+
+	return headers, nil
+}
+
 // parseListenAddr checks an address to listen on: an optional host and a
 // port number.
 func parseListenAddr(value string) (string, error) {
@@ -723,3 +1157,154 @@ func parseUpstreamURL(value string) (*url.URL, error) {
 
 	return upstream, nil
 }
+
+// parseLogRemoteURL reads where every log line is also sent:
+// syslog+udp, syslog+tcp or syslog+tls, a host and a port from 1 to
+// 65535, and nothing else.
+func parseLogRemoteURL(value string) (*url.URL, error) {
+	remote, err := url.Parse(value)
+	if err != nil {
+		return nil, fmt.Errorf("%q %w", value, errNotLogRemoteURL)
+	}
+
+	schemes := []string{remotelog.SchemeUDP, remotelog.SchemeTCP, remotelog.SchemeTLS}
+	port, err := strconv.ParseUint(remote.Port(), 10, 16)
+
+	onlySchemeHostAndPort := slices.Contains(schemes, remote.Scheme) &&
+		remote.Hostname() != "" && err == nil && port != 0 &&
+		remote.User == nil && remote.Opaque == "" &&
+		(remote.Path == "" || remote.Path == "/") &&
+		remote.RawQuery == "" && remote.Fragment == ""
+	if !onlySchemeHostAndPort {
+		return nil, fmt.Errorf("%q %w", value, errNotLogRemoteURL)
+	}
+
+	return remote, nil
+}
+
+// parseFacility reads the name of a syslog facility, and returns its
+// number, as RFC 5424 numbers them.
+func parseFacility(value string) (int, error) {
+	//nolint:mnd // the facilities' numbers in RFC 5424
+	number, known := map[string]int{
+		"kern": 0, "user": 1, "mail": 2, "daemon": 3, "auth": 4, "syslog": 5,
+		"lpr": 6, "news": 7, "uucp": 8, "cron": 9, "authpriv": 10, "ftp": 11,
+		"local0": 16, "local1": 17, "local2": 18, "local3": 19,
+		"local4": 20, "local5": 21, "local6": 22, "local7": 23,
+	}[value]
+	if !known {
+		return 0, fmt.Errorf("%q %w", value, errNotFacility)
+	}
+
+	return number, nil
+}
+
+// parseWebhookURL reads where each alert is posted: http or https, a
+// host, and an optional port from 1 to 65535, path and query, without a
+// user or a fragment. It returns the URL, and how the log shows it: its
+// scheme and host, and ******** in place of its path and query, if it has
+// either. An error shows no part of the value. An empty value is no URL.
+func parseWebhookURL(value string) (*url.URL, string, error) {
+	if value == "" {
+		return nil, "", nil
+	}
+
+	webhook, err := url.Parse(value)
+	if err != nil {
+		return nil, "", errNotWebhookURL
+	}
+
+	port, err := strconv.ParseUint(webhook.Port(), 10, 16)
+
+	valid := (webhook.Scheme == "http" || webhook.Scheme == "https") &&
+		webhook.Hostname() != "" && (webhook.Port() == "" || (err == nil && port != 0)) &&
+		webhook.User == nil && webhook.Opaque == "" && webhook.Fragment == ""
+	if !valid {
+		return nil, "", errNotWebhookURL
+	}
+
+	logged := webhook.Scheme + "://" + webhook.Host
+	if webhook.Path != "" || webhook.RawQuery != "" {
+		logged += "/" + masked
+	}
+
+	return webhook, logged, nil
+}
+
+// parseWebhookHeaders reads a comma-separated list of headers, each its
+// name, :, and its value, and returns them, and how the log shows them,
+// with each value as ********. An error names the item by its place in
+// the list, so that it shows no value. An empty value is an empty list.
+func parseWebhookHeaders(value string) (http.Header, string, error) {
+	headers := http.Header{}
+	if strings.TrimSpace(value) == "" {
+		return headers, "", nil
+	}
+
+	logged := []string{}
+
+	for i, item := range strings.Split(value, ",") {
+		name, headerValue, found := strings.Cut(item, ":")
+		name = strings.TrimSpace(name)
+
+		if !found || !IsHeaderName(name) || strings.ContainsAny(headerValue, "\r\n\x00") {
+			return nil, "", fmt.Errorf("item %d %w", i+1, errNotWebhookHeader)
+		}
+
+		headers.Add(name, strings.TrimSpace(headerValue))
+		logged = append(logged, name+":"+masked)
+	}
+
+	return headers, strings.Join(logged, ","), nil
+}
+
+// parseAlertEvents reads a comma-separated list of the events alerts can
+// be sent for.
+func parseAlertEvents(value string) ([]string, error) {
+	events, err := parseList(value)
+	if err != nil {
+		return nil, err
+	}
+
+	for _, event := range events {
+		if !slices.Contains(alerts.Events(), event) {
+			return nil, fmt.Errorf("%q %w", event, errNotAlertEvent)
+		}
+	}
+
+	return events, nil
+}
+
+// parseNumberOrOff reads a whole number above zero, or off, which is 0.
+func parseNumberOrOff(value string) (int, error) {
+	if value == off {
+		return 0, nil
+	}
+
+	n, err := strconv.Atoi(value)
+	if err != nil || n <= 0 {
+		return 0, fmt.Errorf("%q %w", value, errNotNumberOrOff)
+	}
+
+	return n, nil
+}
+
+// appNameMaxLength is the most characters RFC 5424 allows in an
+// APP-NAME.
+const appNameMaxLength = 48
+
+// isAppName reports whether value can be an APP-NAME: 1 to
+// appNameMaxLength printable ASCII characters, none of them a space.
+func isAppName(value string) bool {
+	if value == "" || len(value) > appNameMaxLength {
+		return false
+	}
+
+	for _, char := range []byte(value) {
+		if char < '!' || char > '~' {
+			return false
+		}
+	}
+
+	return true
+}

internal/lookup/lookup.go

@@ -19,6 +19,7 @@ import (
 	"time"
 
 	"github.com/hashicorp/golang-lru/v2/simplelru"
+	"sneak.berlin/go/smallwebwaf/internal/alerts"
 	"sneak.berlin/go/smallwebwaf/internal/metrics"
 )
 
@@ -68,6 +69,8 @@ type Params struct {
 	// Metrics count the requests to GeoJS, those that failed, and the
 	// clients that go without an answer.
 	Metrics *metrics.Metrics
+	// Alerts receive a source_failure alert each time GeoJS fails.
+	Alerts *alerts.Queue
 }
 
 // GeoJS looks up clients' countries through GeoJS. At most one request
@@ -78,6 +81,7 @@ type GeoJS struct {
 	now        func() time.Time
 	processLog *slog.Logger
 	metrics    *metrics.Metrics
+	alerts     *alerts.Queue
 	// httpClient follows no redirect, so that visitors' addresses go to
 	// GeoJS alone: a redirect is a failure.
 	httpClient *http.Client
@@ -127,6 +131,7 @@ func New(params Params) *GeoJS {
 		now:        params.Now,
 		processLog: params.ProcessLog,
 		metrics:    params.Metrics,
+		alerts:     params.Alerts,
 		httpClient: &http.Client{
 			CheckRedirect: func(*http.Request, []*http.Request) error {
 				return http.ErrUseLastResponse
@@ -197,19 +202,21 @@ func (g *GeoJS) Snapshot() []Answer {
 	return answers
 }
 
-// Load keeps answers read from lookups.json, in a GeoJS that keeps none
-// yet, in the order they were last used, so that the one used longest
+// Load keeps answers read from lookups.json, in place of the answers it
+// keeps, in the order they were last used, so that the one used longest
 // ago is dropped first. Answers GeoJS gave keepFor ago or more are
 // dropped.
 func (g *GeoJS) Load(answers []Answer) {
-	g.mu.Lock()
-	defer g.mu.Unlock()
-
 	answers = slices.Clone(answers)
 	slices.SortStableFunc(answers, func(a, b Answer) int {
 		return a.Used.Compare(b.Used)
 	})
 
+	g.mu.Lock()
+	defer g.mu.Unlock()
+
+	g.answers.Purge()
+
 	now := g.now()
 
 	for _, answer := range answers {
@@ -384,6 +391,14 @@ func (g *GeoJS) keep(
 
 		g.processLog.Warn("asking GeoJS failed",
 			"error", err.Error(), "asking_again_in", g.retryDelay.String())
+		g.alerts.Raise(alerts.Alert{
+			Event:  alerts.EventSourceFailure,
+			Reason: "asking GeoJS failed",
+			Detail: map[string]any{
+				"source": "geojs", "error": err.Error(),
+				"asking_again_in": g.retryDelay.String(),
+			},
+		})
 
 		return false
 	}

internal/lookup/lookup_test.go

@@ -6,6 +6,8 @@ import (
 	"net/http"
 	"net/http/httptest"
 	"net/netip"
+	"net/url"
+	"reflect"
 	"slices"
 	"strings"
 	"sync"
@@ -14,6 +16,7 @@ import (
 	"time"
 
 	"github.com/prometheus/client_golang/prometheus/testutil"
+	"sneak.berlin/go/smallwebwaf/internal/alerts"
 	"sneak.berlin/go/smallwebwaf/internal/lookup"
 	"sneak.berlin/go/smallwebwaf/internal/metrics"
 )
@@ -196,7 +199,8 @@ func TestFailureIsLoggedWithoutTheAddressesAskedAbout(t *testing.T) {
 			URL:        lookup.URL,
 			Now:        time.Now,
 			ProcessLog: slog.New(slog.NewTextHandler(&log, nil)),
-			Metrics:    metrics.New(1),
+			Metrics:    metrics.New(1, "app"),
+			Alerts:     alerts.New(alerts.Params{}),
 		})
 		g.SetTransport(geojs)
 
@@ -211,6 +215,45 @@ func TestFailureIsLoggedWithoutTheAddressesAskedAbout(t *testing.T) {
 	})
 }
 
+func TestFailureRaisesASourceFailureAlertOncePerCooldown(t *testing.T) {
+	t.Parallel()
+
+	synctest.Test(t, func(t *testing.T) {
+		geojs, clock, g, queue := startWithAlerts()
+		clients := newClients()
+
+		geojs.set(failing)
+
+		wantCountry(t, g, clients(), "")
+
+		want := alerts.Alert{
+			Time:   clock.Now(),
+			Event:  alerts.EventSourceFailure,
+			Reason: "asking GeoJS failed",
+			Detail: map[string]any{
+				"source":          "geojs",
+				"error":           "GeoJS answered 503 Service Unavailable",
+				"asking_again_in": "1s",
+			},
+		}
+
+		// The next failure, a second later, is a repeat within the
+		// cooldown.
+		clock.advance(time.Second)
+		wantCountry(t, g, clients(), "")
+		wantRequests(t, geojs, 2)
+
+		waiting := queue.Snapshot().Waiting[alerts.DestinationWebhook]
+		if len(waiting) != 1 || !reflect.DeepEqual(waiting[0], want) {
+			t.Errorf("alerts waiting %+v, want only %+v", waiting, want)
+		}
+
+		if queue.Suppressed() != 1 {
+			t.Errorf("%d alerts held back, want the repeat", queue.Suppressed())
+		}
+	})
+}
+
 func TestWaitingClientsAreAskedAboutInOneRequest(t *testing.T) {
 	t.Parallel()
 
@@ -355,12 +398,13 @@ func TestClientsWithoutAnAnswerAreCounted(t *testing.T) {
 	t.Parallel()
 
 	synctest.Test(t, func(t *testing.T) {
-		m := metrics.New(1)
+		m := metrics.New(1, "app")
 		g := lookup.New(lookup.Params{
 			URL:        lookup.URL,
 			Now:        time.Now,
 			ProcessLog: slog.New(slog.DiscardHandler),
 			Metrics:    m,
+			Alerts:     alerts.New(alerts.Params{}),
 		})
 		g.SetTransport(&standIn{answers: failing})
 
@@ -524,17 +568,33 @@ func (c *testClock) advance(d time.Duration) {
 // start returns a stand-in for GeoJS that answers, a clock, and a GeoJS
 // asking the stand-in by that clock.
 func start() (*standIn, *testClock, *lookup.GeoJS) {
+	geojs, clock, g, _ := startWithAlerts()
+
+	return geojs, clock, g
+}
+
+// startWithAlerts is start, and returns the queue of the alerts GeoJS
+// raises as well, for a webhook that is never sent them, with the default
+// cooldown, by the same clock.
+func startWithAlerts() (*standIn, *testClock, *lookup.GeoJS, *alerts.Queue) {
 	geojs := &standIn{}
 	clock := &testClock{now: time.Date(2026, 10, 4, 0, 0, 0, 0, time.UTC)}
+	queue := alerts.New(alerts.Params{
+		WebhookURL: &url.URL{Scheme: "https", Host: "alerts.example"},
+		Events:     alerts.Events(),
+		Cooldown:   15 * time.Minute,
+		Now:        clock.Now,
+	})
 	g := lookup.New(lookup.Params{
 		URL:        lookup.URL,
 		Now:        clock.Now,
 		ProcessLog: slog.New(slog.DiscardHandler),
-		Metrics:    metrics.New(1),
+		Metrics:    metrics.New(1, "app"),
+		Alerts:     queue,
 	})
 	g.SetTransport(geojs)
 
-	return geojs, clock, g
+	return geojs, clock, g, queue
 }
 
 // newClients returns what returns a new IPv4 client each time it is

internal/metrics/metrics.go

@@ -11,14 +11,18 @@ import (
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/client_golang/prometheus/collectors"
 	"github.com/prometheus/client_golang/prometheus/promhttp"
+	"sneak.berlin/go/smallwebwaf/internal/alerts"
 	"sneak.berlin/go/smallwebwaf/internal/bans"
 	"sneak.berlin/go/smallwebwaf/internal/ratelimit"
+	"sneak.berlin/go/smallwebwaf/internal/remotelog"
 	"sneak.berlin/go/smallwebwaf/internal/requestlog"
+	"sneak.berlin/go/smallwebwaf/internal/rules"
 )
 
 // Metrics are smallwebwaf's metrics. They are safe for concurrent use.
 type Metrics struct {
-	registry *prometheus.Registry
+	// registry gives every metric registered with it the label instance.
+	registry prometheus.Registerer
 	handler  http.Handler
 
 	inFlight             prometheus.Gauge
@@ -30,7 +34,9 @@ type Metrics struct {
 	rateLimitHits        *prometheus.CounterVec
 	sizeAndTimeLimitHits *prometheus.CounterVec
 	offences             *prometheus.CounterVec
-	countries            *countries
+	// ruleMatches are made by AddRules.
+	ruleMatches *prometheus.CounterVec
+	countries   *countries
 
 	// GeoJSRequests are the requests to GeoJS, and GeoJSFailures those
 	// that failed. GeoJSUnanswered are the requests whose client counted
@@ -44,17 +50,23 @@ type Metrics struct {
 	stateFileWriteFailures *prometheus.CounterVec
 	stateFileLastWrite     *prometheus.GaugeVec
 	stateFileSize          *prometheus.GaugeVec
+	stateFileEditsTakenIn  *prometheus.CounterVec
+	stateFileEditsSetAside *prometheus.CounterVec
 }
 
 // New returns the metrics, with the Go runtime's and the process's own.
 // topN is how many countries get series of their own
-// (SWWAF_METRICS_TOP_N).
-func New(topN int) *Metrics {
+// (SWWAF_METRICS_TOP_N). Every metric carries instanceName
+// (SWWAF_INSTANCE_NAME) as its label instance.
+func New(topN int, instanceName string) *Metrics {
 	byStatus := []string{"status_class", "action"}
 	byFile := []string{"file"}
+	registry := prometheus.NewRegistry()
 
 	m := &Metrics{
-		registry: prometheus.NewRegistry(),
+		registry: prometheus.WrapRegistererWith(
+			prometheus.Labels{"instance": instanceName}, registry),
+		handler: promhttp.HandlerFor(registry, promhttp.HandlerOpts{}),
 		inFlight: prometheus.NewGauge(prometheus.GaugeOpts{
 			Name: "smallwebwaf_requests_in_flight",
 			Help: "Requests under way.",
@@ -105,10 +117,13 @@ func New(topN int) *Metrics {
 			"When each state file was last written, in seconds since 1970.", byFile),
 		stateFileSize: gaugeVec("smallwebwaf_state_file_size_bytes",
 			"The size of each state file, as it was last written.", byFile),
+		stateFileEditsTakenIn: counterVec("smallwebwaf_state_file_edits_taken_in_total",
+			"Edits of each state file taken in while running.", byFile),
+		stateFileEditsSetAside: counterVec("smallwebwaf_state_file_edits_set_aside_total",
+			"Edits of each state file renamed to <name>.bad because they did not parse.",
+			byFile),
 	}
 
-	m.handler = promhttp.HandlerFor(m.registry, promhttp.HandlerOpts{})
-
 	m.registry.MustRegister(
 		collectors.NewGoCollector(),
 		collectors.NewProcessCollector(collectors.ProcessCollectorOpts{}),
@@ -120,6 +135,7 @@ func New(topN int) *Metrics {
 		m.GeoJSRequests, m.GeoJSFailures, m.GeoJSUnanswered,
 		m.stateFileWrites, m.stateFileWriteFailures,
 		m.stateFileLastWrite, m.stateFileSize,
+		m.stateFileEditsTakenIn, m.stateFileEditsSetAside,
 	)
 
 	return m
@@ -127,19 +143,22 @@ func New(topN int) *Metrics {
 
 // AddBansAndClients adds the metrics read from the ledger and the table
 // of clients as the metrics are asked for: the bans made since the start,
-// the bans active and permanent at now, and the clients in the table.
+// by cause, the bans active and permanent at now, and the clients in the
+// table.
 func (m *Metrics) AddBansAndClients(
 	ledger *bans.Ledger, limiter *ratelimit.Limiter, now func() time.Time,
 ) {
-	m.registry.MustRegister(
-		// Every ban smallwebwaf makes so far is for a broken limit.
-		prometheus.NewCounterFunc(prometheus.CounterOpts{
+	for _, cause := range []string{bans.CauseLimit, bans.CauseAttack, bans.CauseAdmin} {
+		m.registry.MustRegister(prometheus.NewCounterFunc(prometheus.CounterOpts{
 			Name:        "smallwebwaf_bans_made_total",
 			Help:        "Bans made, by cause.",
-			ConstLabels: prometheus.Labels{"cause": "limit"},
+			ConstLabels: prometheus.Labels{"cause": cause},
 		}, func() float64 {
-			return float64(ledger.Made())
-		}),
+			return float64(ledger.Made(cause))
+		}))
+	}
+
+	m.registry.MustRegister(
 		prometheus.NewGaugeFunc(prometheus.GaugeOpts{
 			Name: "smallwebwaf_active_bans",
 			Help: "Bans active now, the permanent ones included.",
@@ -150,7 +169,7 @@ func (m *Metrics) AddBansAndClients(
 		}),
 		prometheus.NewGaugeFunc(prometheus.GaugeOpts{
 			Name: "smallwebwaf_permanent_bans",
-			Help: "Permanent bans.",
+			Help: "Permanent bans not lifted.",
 		}, func() float64 {
 			_, permanent := ledger.Count(now())
 
@@ -165,6 +184,95 @@ func (m *Metrics) AddBansAndClients(
 	)
 }
 
+// AddRules adds the metrics of the rule files: the requests that matched
+// each rule, which RuleMatched counts, and the rules loaded from
+// ruleFiles, read as the metrics are asked for. It is called once, before
+// RuleMatched.
+func (m *Metrics) AddRules(ruleFiles *rules.Files) {
+	m.ruleMatches = counterVec("smallwebwaf_rule_matches_total",
+		"Requests that matched a rule of the rule files, by its id and action.",
+		[]string{"rule_id", "action"})
+
+	m.registry.MustRegister(m.ruleMatches,
+		prometheus.NewGaugeFunc(prometheus.GaugeOpts{
+			Name: "smallwebwaf_rules_loaded",
+			Help: "Rules loaded from the rule files.",
+		}, func() float64 {
+			return float64(ruleFiles.Len())
+		}))
+}
+
+// AddRemoteLog adds the metrics of sending the log lines to
+// SWWAF_LOG_REMOTE_URL, read from remote as the metrics are asked for: the
+// lines sent, those dropped, and those waiting in the buffer.
+func (m *Metrics) AddRemoteLog(remote *remotelog.Sender) {
+	m.registry.MustRegister(
+		prometheus.NewCounterFunc(prometheus.CounterOpts{
+			Name: "smallwebwaf_remote_log_lines_sent_total",
+			Help: "Log lines sent to SWWAF_LOG_REMOTE_URL.",
+		}, func() float64 {
+			return float64(remote.Sent())
+		}),
+		prometheus.NewCounterFunc(prometheus.CounterOpts{
+			Name: "smallwebwaf_remote_log_lines_dropped_total",
+			Help: "Log lines dropped: the oldest in a full buffer, and those " +
+				"whose sending failed.",
+		}, func() float64 {
+			return float64(remote.Dropped())
+		}),
+		prometheus.NewGaugeFunc(prometheus.GaugeOpts{
+			Name: "smallwebwaf_remote_log_buffer_depth",
+			Help: "Log lines in the buffer, waiting to be sent.",
+		}, func() float64 {
+			return float64(remote.Depth())
+		}),
+	)
+}
+
+// AddAlerts adds the metrics of the alerts sent to each destination set,
+// read from queue as the metrics are asked for, by destination: the
+// alerts sent, the requests to the destination that failed, the alerts
+// held back, which are the same for every destination, and those
+// dropped. With no destination set, it adds none.
+func (m *Metrics) AddAlerts(queue *alerts.Queue) {
+	for _, name := range queue.DestinationsSet() {
+		destination := prometheus.Labels{"destination": name}
+
+		m.registry.MustRegister(
+			prometheus.NewCounterFunc(prometheus.CounterOpts{
+				Name:        "smallwebwaf_alerts_sent_total",
+				Help:        "Alerts the destination took.",
+				ConstLabels: destination,
+			}, func() float64 {
+				return float64(queue.Counts(name).Sent)
+			}),
+			prometheus.NewCounterFunc(prometheus.CounterOpts{
+				Name:        "smallwebwaf_alerts_failed_total",
+				Help:        "Requests to the destination that failed.",
+				ConstLabels: destination,
+			}, func() float64 {
+				return float64(queue.Counts(name).Failed)
+			}),
+			prometheus.NewCounterFunc(prometheus.CounterOpts{
+				Name: "smallwebwaf_alerts_suppressed_total",
+				Help: "Alerts held back: repeats within SWWAF_ALERT_COOLDOWN, and " +
+					"alerts past SWWAF_ALERT_MAX_PER_HOUR, for the hour's summary.",
+				ConstLabels: destination,
+			}, func() float64 {
+				return float64(queue.Suppressed())
+			}),
+			prometheus.NewCounterFunc(prometheus.CounterOpts{
+				Name: "smallwebwaf_alerts_dropped_total",
+				Help: "Alerts dropped, the oldest first, from a full queue, and " +
+					"alerts given up as the destination refused them.",
+				ConstLabels: destination,
+			}, func() float64 {
+				return float64(queue.Counts(name).Dropped)
+			}),
+		)
+	}
+}
+
 // ServeHTTP answers with the metrics in the Prometheus text format.
 func (m *Metrics) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 	m.handler.ServeHTTP(w, r)
@@ -211,6 +319,12 @@ func (m *Metrics) RequestEnded(
 	}
 }
 
+// RuleMatched counts a request that matched the rule id, whose action is
+// action.
+func (m *Metrics) RuleMatched(id, action string) {
+	m.ruleMatches.WithLabelValues(id, action).Inc()
+}
+
 // StateFileWritten counts a write of the state file name, of size bytes,
 // that ended with err.
 func (m *Metrics) StateFileWritten(name string, size int, err error) {
@@ -230,6 +344,18 @@ func (m *Metrics) StateFileWritten(name string, size int, err error) {
 	m.stateFileSize.WithLabelValues(name).Set(float64(size))
 }
 
+// StateFileEditTakenIn counts an admin's edit of the state file name
+// taken in while smallwebwaf runs.
+func (m *Metrics) StateFileEditTakenIn(name string) {
+	m.stateFileEditsTakenIn.WithLabelValues(name).Inc()
+}
+
+// StateFileEditSetAside counts an admin's edit of the state file name
+// renamed to name.bad because it did not parse.
+func (m *Metrics) StateFileEditSetAside(name string) {
+	m.stateFileEditsSetAside.WithLabelValues(name).Inc()
+}
+
 // statusClass returns the class of status, such as 2xx, or none when no
 // status was sent.
 func statusClass(status int) string {

internal/proxy/admin.go

@@ -1,26 +1,60 @@
 package proxy
 
 import (
+	"bytes"
 	"crypto/subtle"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
 	"net/http"
+	"net/netip"
+	"os"
 	"strings"
+	"time"
 
+	"sneak.berlin/go/smallwebwaf/internal/bans"
+	"sneak.berlin/go/smallwebwaf/internal/config"
+	"sneak.berlin/go/smallwebwaf/internal/ratelimit"
 	"sneak.berlin/go/smallwebwaf/internal/requestlog"
+	"sneak.berlin/go/smallwebwaf/internal/state"
+)
+
+// banBodyMaxBytes is the most of the body of a request to add a ban that
+// is read; its three fields need far less.
+const banBodyMaxBytes = 4 << 10
+
+// permanent is how the log line and the ban endpoint name a ban that
+// never ends.
+const permanent = "permanent"
+
+var (
+	errNotBanToAdd = errors.New(
+		"the body is not a JSON object of netblock, duration and reason")
+	errNotNetblock = errors.New(
+		"is not an address or a netblock, such as 203.0.113.9 or 203.0.113.0/24")
+	errMappedNetblock = errors.New(
+		"is IPv4-mapped: give the IPv4 netblock, such as 203.0.113.0/24")
+	errZone        = errors.New("has a zone, which a netblock cannot have")
+	errNotDuration = errors.New(
+		"is not a duration above zero, such as 1h or 7d, or permanent")
+	errNotAddress = errors.New("is not an address, such as 203.0.113.9")
 )
 
 // answerAdmin answers a request for smallwebwaf itself, under
-// /_smallwebwaf/, once it has passed the checks: GET MetricsPath with
-// SWWAF_METRICS_TOKEN gets the metrics, and without it is refused with
-// 401. Any other request gets 404, as the metrics do while
-// SWWAF_METRICS_TOKEN is unset.
+// /_smallwebwaf/, once it has passed the checks. Each endpoint needs a
+// token, sent as Authorization: Bearer <token>: the metrics
+// SWWAF_METRICS_TOKEN, the others SWWAF_ADMIN_TOKEN. A request without
+// it is refused with 401. An endpoint whose token is unset answers 404,
+// as any other request under /_smallwebwaf/ does.
 func (rq *request) answerAdmin() {
 	rq.line.Action = requestlog.ActionAdmin
 	rq.startClientResponseTimeout()
 
-	token := rq.h.config.MetricsToken
+	token, answer := rq.endpoint()
 
 	switch {
-	case token == "" || rq.in.Method != http.MethodGet || rq.in.URL.Path != MetricsPath:
+	case token == "":
 		http.Error(rq.out, http.StatusText(http.StatusNotFound), http.StatusNotFound)
 	case !hasToken(rq.in, token):
 		rq.out.Header().Set("WWW-Authenticate", "Bearer")
@@ -29,7 +63,29 @@ func (rq *request) answerAdmin() {
 			action: requestlog.ActionAdmin,
 		})
 	default:
-		rq.h.metrics.ServeHTTP(rq.out, rq.in)
+		answer()
+	}
+}
+
+// endpoint returns the token the request's endpoint needs, and what
+// answers the request there; "" when there is no such endpoint.
+func (rq *request) endpoint() (string, func()) {
+	cfg := rq.h.config
+	method, path := rq.in.Method, rq.in.URL.Path
+
+	switch {
+	case method == http.MethodGet && path == MetricsPath:
+		return cfg.MetricsToken, func() { rq.h.metrics.ServeHTTP(rq.out, rq.in) }
+	case method == http.MethodGet && path == BansPath:
+		return cfg.AdminToken, rq.listBans
+	case method == http.MethodPost && path == BansPath:
+		return cfg.AdminToken, rq.addBan
+	case method == http.MethodDelete && strings.HasPrefix(path, BansPath+"/"):
+		return cfg.AdminToken, rq.liftBans
+	case method == http.MethodGet && strings.HasPrefix(path, ClientsPath):
+		return cfg.AdminToken, rq.showClient
+	default:
+		return "", nil
 	}
 }
 
@@ -41,3 +97,229 @@ func hasToken(r *http.Request, token string) bool {
 	return strings.EqualFold(scheme, "Bearer") &&
 		subtle.ConstantTimeCompare([]byte(sent), []byte(token)) == 1
 }
+
+// listBans answers GET BansPath with every ban held.
+func (rq *request) listBans() {
+	rq.answerBans(rq.h.ledger.Snapshot())
+}
+
+// banToAdd is the body of POST BansPath.
+type banToAdd struct {
+	// Netblock is a netblock, or a client's address, which stands for the
+	// netblock a ban on that client covers.
+	Netblock string `json:"netblock"`
+	// Duration is how long the ban lasts, as a setting gives a duration,
+	// or permanent.
+	Duration string `json:"duration"`
+	Reason   string `json:"reason"`
+}
+
+// addBan answers POST BansPath: it bans the netblock the body names, as
+// an admin, from now for the duration the body gives, with its reason,
+// and answers with that ban.
+func (rq *request) addBan() {
+	// The body must arrive within SWWAF_CLIENT_REQUEST_TIMEOUT, as any
+	// other request's must.
+	rq.stopReadingBody(rq.clientRequestDeadline())
+
+	toAdd, err := rq.readBanToAdd()
+	if refused := rq.refused.Load(); refused != nil {
+		rq.answer(*refused) // the body is over SWWAF_REQUEST_MAX_BYTES
+
+		return
+	}
+
+	if errors.Is(err, os.ErrDeadlineExceeded) {
+		rq.answer(refusal{
+			status: http.StatusRequestTimeout,
+			action: requestlog.ActionTimedOut,
+			limit:  "SWWAF_CLIENT_REQUEST_TIMEOUT",
+		})
+
+		return
+	}
+
+	var (
+		netblock netip.Prefix
+		expires  time.Time
+		now      = rq.h.now()
+	)
+
+	if err == nil {
+		netblock, err = rq.h.banNetblock(toAdd.Netblock)
+	}
+
+	if err == nil {
+		expires, err = expiry(toAdd.Duration, now)
+	}
+
+	if err != nil {
+		http.Error(rq.out, err.Error(), http.StatusBadRequest)
+
+		return
+	}
+
+	ban := rq.h.ledger.BanForAdmin(netblock, now, expires, toAdd.Reason)
+	rq.answerBans([]bans.Ban{ban})
+}
+
+// readBanToAdd reads the body of POST BansPath: a JSON object with
+// nothing but whitespace after it, in at most banBodyMaxBytes.
+func (rq *request) readBanToAdd() (banToAdd, error) {
+	var body io.ReadCloser = http.NoBody
+	if rq.body != nil {
+		body = rq.body
+	}
+
+	data, err := io.ReadAll(http.MaxBytesReader(nil, body, banBodyMaxBytes))
+	if err != nil {
+		return banToAdd{}, fmt.Errorf("%w: %w", errNotBanToAdd, err)
+	}
+
+	var toAdd banToAdd
+
+	decoder := json.NewDecoder(bytes.NewReader(data))
+	decoder.DisallowUnknownFields()
+
+	err = decoder.Decode(&toAdd)
+	if err != nil {
+		return banToAdd{}, fmt.Errorf("%w: %w", errNotBanToAdd, err)
+	}
+
+	// Token returns io.EOF only when nothing but whitespace is left.
+	_, err = decoder.Token()
+	if !errors.Is(err, io.EOF) {
+		return banToAdd{}, fmt.Errorf("%w: more follows the object", errNotBanToAdd)
+	}
+
+	return toAdd, nil
+}
+
+// banNetblock reads value, a netblock such as 203.0.113.0/24, or a
+// client's address, which stands for the netblock a ban on that client
+// covers. An IPv4-mapped netblock, such as ::ffff:203.0.113.0/120, is
+// refused, since a client's address is looked up as IPv4 and a ban on it
+// would refuse nothing, and so is a value with a zone.
+func (h *handler) banNetblock(value string) (netip.Prefix, error) {
+	netblock, err := netip.ParsePrefix(value)
+	if err == nil {
+		if netblock.Addr().Is4In6() {
+			return netip.Prefix{}, fmt.Errorf("netblock %q %w", value, errMappedNetblock)
+		}
+
+		return netblock, nil
+	}
+
+	// ParsePrefix refuses a zone, but ParseAddr reads the /48 of
+	// 2001:db8::1%x/48 as part of the zone.
+	addr, err := netip.ParseAddr(value)
+	if err != nil {
+		return netip.Prefix{}, fmt.Errorf("netblock %q %w", value, errNotNetblock)
+	}
+
+	if addr.Zone() != "" {
+		return netip.Prefix{}, fmt.Errorf("netblock %q %w", value, errZone)
+	}
+
+	return h.netblock(addr), nil
+}
+
+// expiry returns when a ban made at now for duration ends: duration
+// later, for a duration as a setting gives one, or zero for permanent.
+func expiry(duration string, now time.Time) (time.Time, error) {
+	if duration == permanent {
+		return time.Time{}, nil
+	}
+
+	length, err := config.ParseDurationNotOff(duration)
+	if err != nil {
+		return time.Time{}, fmt.Errorf("duration %q %w", duration, errNotDuration)
+	}
+
+	return now.Add(length), nil
+}
+
+// liftBans answers DELETE BansPath/<client>: it lifts every ban active on
+// a netblock the client's address is in, and answers with those bans, or
+// with 404 when none is active.
+func (rq *request) liftBans() {
+	client, err := pathAddress(rq.in.URL.Path, BansPath+"/")
+	if err != nil {
+		http.Error(rq.out, err.Error(), http.StatusBadRequest)
+
+		return
+	}
+
+	lifted := rq.h.ledger.Lift(client, rq.h.now())
+	if len(lifted) == 0 {
+		http.Error(rq.out, "no ban is active on "+client.String(), http.StatusNotFound)
+
+		return
+	}
+
+	rq.answerBans(lifted)
+}
+
+// clientAnswer is the answer to GET ClientsPath<ip>: the client the
+// address is, as clients.json holds it, or null when the table of
+// clients does not hold it, and the bans on each netblock the address is
+// in, as bans.json lists them.
+type clientAnswer struct {
+	Client *ratelimit.Client `json:"client"`
+	Bans   []state.BanEntry  `json:"bans"`
+}
+
+// showClient answers GET ClientsPath<ip> with what smallwebwaf knows of
+// the client: its counters, its history, which holds its country as last
+// looked up and its offences, and its bans with their notes.
+func (rq *request) showClient() {
+	addr, err := pathAddress(rq.in.URL.Path, ClientsPath)
+	if err != nil {
+		http.Error(rq.out, err.Error(), http.StatusBadRequest)
+
+		return
+	}
+
+	answer := clientAnswer{Bans: state.BanEntries(rq.h.ledger.Covering(addr))}
+
+	client, seen := rq.h.limiter.Client(clientGroup(addr))
+	if seen {
+		answer.Client = &client
+	}
+
+	rq.answerJSON(answer)
+}
+
+// pathAddress reads the client's address that follows prefix in path.
+func pathAddress(path, prefix string) (netip.Addr, error) {
+	value := strings.TrimPrefix(path, prefix)
+
+	addr, err := netip.ParseAddr(value)
+	if err != nil {
+		return netip.Addr{}, fmt.Errorf("%q %w", value, errNotAddress)
+	}
+
+	return addr.Unmap(), nil
+}
+
+// answerBans answers with held under bans, as bans.json lists them.
+func (rq *request) answerBans(held []bans.Ban) {
+	rq.answerJSON(struct {
+		Bans []state.BanEntry `json:"bans"`
+	}{state.BanEntries(held)})
+}
+
+// answerJSON answers with value as indented JSON.
+func (rq *request) answerJSON(value any) {
+	body, err := json.MarshalIndent(value, "", "    ")
+	if err != nil {
+		rq.h.processLog.Error("encoding an answer failed", "error", err.Error())
+		http.Error(rq.out, http.StatusText(http.StatusInternalServerError),
+			http.StatusInternalServerError)
+
+		return
+	}
+
+	rq.out.Header().Set("Content-Type", "application/json")
+	_, _ = rq.out.Write(append(body, '\n'))
+}

internal/proxy/admin_test.go

@@ -0,0 +1,535 @@
+package proxy_test
+
+import (
+	"encoding/json"
+	"net/http"
+	"net/netip"
+	"slices"
+	"strconv"
+	"strings"
+	"testing"
+	"time"
+
+	"sneak.berlin/go/smallwebwaf/internal/bans"
+	"sneak.berlin/go/smallwebwaf/internal/proxy"
+	"sneak.berlin/go/smallwebwaf/internal/ratelimit"
+	"sneak.berlin/go/smallwebwaf/internal/requestlog"
+	"sneak.berlin/go/smallwebwaf/internal/state"
+)
+
+const (
+	adminToken = "SWWAF_ADMIN_TOKEN" //nolint:gosec // the setting's name
+	// adminSecret is the SWWAF_ADMIN_TOKEN the tests set, and adminBearer
+	// how a request carries it.
+	adminSecret = "fedcba9876543210fedcba9876543210"
+	adminBearer = "Bearer " + adminSecret
+	// adminClient is the client the tests' admin sends its requests from.
+	adminClient = "192.0.2.10"
+	// banOtherClient is the body of a request to ban otherClient for an
+	// hour.
+	banOtherClient = `{"netblock": "` + otherClient + `", "duration": "1h", ` +
+		`"reason": "probes for logins"}`
+)
+
+func TestAdminEndpointsAreOffWhileTheTokenIsUnset(t *testing.T) {
+	t.Parallel()
+
+	// The metrics token is set, and opens none of them.
+	s, clk, server := startWithClock(t, "", map[string]string{metricsToken: token})
+	server.Ledger.BanForLimit(netip.MustParsePrefix(otherClient+"/32"), clk.Now(),
+		bans.Notes{})
+	before := server.Ledger.Snapshot()
+
+	// An empty token does not match the unset one either.
+	for _, authorization := range []string{adminBearer, bearer, "Bearer ", ""} {
+		for _, e := range adminEndpoints() {
+			s.adminRequest(adminClient, authorization, e.method, e.path, e.body,
+				http.StatusNotFound, requestlog.ActionAdmin)
+		}
+	}
+
+	if after := server.Ledger.Snapshot(); !slices.Equal(after, before) {
+		t.Errorf("the bans are now\n%+v\nwant them unchanged\n%+v", after, before)
+	}
+}
+
+func TestAdminEndpointsNeedTheAdminToken(t *testing.T) {
+	t.Parallel()
+
+	s, _, server := startWithClock(t, "", map[string]string{
+		adminToken:   adminSecret,
+		metricsToken: token,
+	})
+
+	// Listing the bans, banning otherClient, lifting that ban, and asking
+	// about otherClient, in that order. Without the admin token, with the
+	// metrics token, or with one that differs, each is refused, and
+	// changes nothing; with the admin token, it is answered.
+	for _, e := range adminEndpoints() {
+		before := server.Ledger.Snapshot()
+
+		for _, authorization := range []string{
+			"", bearer, "Bearer " + strings.ToUpper(adminSecret), "Basic " + adminSecret,
+		} {
+			got := s.adminRequest(adminClient, authorization, e.method, e.path, e.body,
+				http.StatusUnauthorized, requestlog.ActionAdmin)
+			if got.header.Get("WWW-Authenticate") != "Bearer" {
+				t.Errorf("%s %s with %q was answered without WWW-Authenticate: Bearer",
+					e.method, e.path, authorization)
+			}
+		}
+
+		if after := server.Ledger.Snapshot(); !slices.Equal(after, before) {
+			t.Errorf("%s %s without the token changed the bans to\n%+v\nfrom\n%+v",
+				e.method, e.path, after, before)
+		}
+
+		got := s.admin(e.method, e.path, e.body, http.StatusOK)
+		if got.header.Get("Content-Type") != "application/json" {
+			t.Errorf("%s %s answered %q", e.method, e.path, got.header.Get("Content-Type"))
+		}
+	}
+
+	// Any other request under /_smallwebwaf/ is not found.
+	for _, e := range []adminEndpoint{
+		{http.MethodPut, proxy.BansPath, banOtherClient},
+		{http.MethodDelete, proxy.BansPath, ""},
+		{http.MethodGet, proxy.BansPath + "/" + otherClient, ""},
+		{http.MethodPost, proxy.ClientsPath + otherClient, ""},
+		{http.MethodGet, strings.TrimSuffix(proxy.ClientsPath, "/"), ""},
+	} {
+		s.admin(e.method, e.path, e.body, http.StatusNotFound)
+	}
+}
+
+func TestBanAddedListedAndLiftedThroughTheEndpoints(t *testing.T) {
+	t.Parallel()
+
+	s, clk, _ := startWithClock(t, "", map[string]string{
+		adminToken:       adminSecret,
+		banScopeV4Prefix: "24",
+	})
+
+	// A ban on otherClient bans the /24 a ban on that client covers, so it
+	// refuses client too, for an hour.
+	start := clk.Now()
+	expires := start.Add(time.Hour)
+	want := state.BanEntry{
+		Netblock: netip.MustParsePrefix("203.0.113.0/24"),
+		Start:    start,
+		Expires:  &expires,
+		Cause:    bans.CauseAdmin,
+		Reason:   "probes for logins",
+	}
+
+	wantBans(t, s.admin(http.MethodPost, proxy.BansPath, banOtherClient, http.StatusOK),
+		want)
+
+	line := s.get(client, http.StatusForbidden, requestlog.ActionBanned)
+	if line.BanExpires != requestlog.FormatTime(expires) {
+		t.Errorf("the ban ends at %s, want %s", line.BanExpires, expires)
+	}
+
+	// Its notes count the request it refused.
+	want.Notes.Requests, want.Notes.Refused = 1, 1
+
+	wantBans(t, s.admin(http.MethodGet, proxy.BansPath, "", http.StatusOK), want)
+
+	// Ten minutes on, lifting the bans on client lifts that one, which is
+	// kept, marked lifted.
+	clk.advance(10 * time.Minute)
+
+	lifted := clk.Now()
+	want.Lifted = &lifted
+
+	wantBans(t, s.admin(http.MethodDelete, proxy.BansPath+"/"+client, "",
+		http.StatusOK), want)
+	s.get(client, http.StatusOK, requestlog.ActionForward)
+	wantBans(t, s.admin(http.MethodGet, proxy.BansPath, "", http.StatusOK), want)
+
+	// No ban on it is active any more.
+	s.admin(http.MethodDelete, proxy.BansPath+"/"+client, "", http.StatusNotFound)
+}
+
+func TestBanToAddGivesItsNetblockAndDuration(t *testing.T) {
+	t.Parallel()
+
+	s, clk, _ := startWithClock(t, "", map[string]string{
+		adminToken:       adminSecret,
+		banScopeV4Prefix: "24",
+	})
+	start := clk.Now()
+
+	for _, tc := range []struct {
+		netblock, duration string
+		want               string
+		length             time.Duration // 0 for a permanent ban
+	}{
+		// An address stands for the netblock a ban on that client covers.
+		{client, "7d", "203.0.113.0/24", 7 * 24 * time.Hour},
+		{"::ffff:198.51.100.7", "90m", "198.51.100.0/24", 90 * time.Minute},
+		{"2001:db8:5::1", "permanent", "2001:db8:5::/64", 0},
+		// A netblock stands for itself, its bits past its length cleared.
+		{"198.51.100.7/16", "1h", "198.51.0.0/16", time.Hour},
+		{"2001:db8:6::/48", "1h", "2001:db8:6::/48", time.Hour},
+	} {
+		// Whitespace may follow the object.
+		body := `{"netblock": "` + tc.netblock + `", "duration": "` + tc.duration + `"}` +
+			"\r\n"
+		want := state.BanEntry{
+			Netblock: netip.MustParsePrefix(tc.want), Start: start, Cause: bans.CauseAdmin,
+		}
+
+		if tc.length != 0 {
+			expires := start.Add(tc.length)
+			want.Expires = &expires
+		}
+
+		wantBans(t, s.admin(http.MethodPost, proxy.BansPath, body, http.StatusOK), want)
+	}
+}
+
+func TestBanToAddThatCannotBeReadIsRefused(t *testing.T) {
+	t.Parallel()
+
+	s, _, server := startWithClock(t, "", map[string]string{adminToken: adminSecret})
+
+	for _, tc := range []struct{ body, want string }{
+		{"", "the body is not a JSON object of netblock, duration and reason: EOF"},
+		{"netblock=203.0.113.9", "the body is not a JSON object"},
+		{
+			`{"netblock": "203.0.113.9", "duration": "1h", "until": "2027"}`,
+			`unknown field "until"`,
+		},
+		{
+			`{"netblock": "203.0.113", "duration": "1h"}`,
+			`netblock "203.0.113" is not an address or a netblock`,
+		},
+		// A client's address is looked up as IPv4, so a ban on an
+		// IPv4-mapped netblock would refuse nothing.
+		{
+			`{"netblock": "::ffff:203.0.113.0/120", "duration": "1h"}`,
+			`netblock "::ffff:203.0.113.0/120" is IPv4-mapped`,
+		},
+		// Read as an address, its zone would be "x/48", and its ban on the
+		// /64 around it.
+		{
+			`{"netblock": "2001:db8::1%x/48", "duration": "1h"}`,
+			`netblock "2001:db8::1%x/48" has a zone`,
+		},
+		{
+			`{"netblock": "fe80::1%eth0", "duration": "1h"}`,
+			`netblock "fe80::1%eth0" has a zone`,
+		},
+		// Anything but whitespace after the object.
+		{
+			`{"netblock": "203.0.113.9", "duration": "1h"}` +
+				`{"netblock": "198.51.100.0/24", "duration": "1h"}`,
+			"more follows the object",
+		},
+		{`{"netblock": "203.0.113.9", "duration": "1h"} x`, "more follows the object"},
+		{`{"duration": "1h"}`, `netblock "" is not an address or a netblock`},
+		{`{"netblock": "203.0.113.9"}`, `duration "" is not a duration above zero`},
+		{
+			`{"netblock": "203.0.113.9", "duration": "off"}`,
+			`duration "off" is not a duration above zero`,
+		},
+		{
+			`{"netblock": "203.0.113.9", "duration": "0s"}`,
+			`duration "0s" is not a duration above zero`,
+		},
+		{
+			`{"netblock": "203.0.113.9", "duration": "forever"}`,
+			`duration "forever" is not a duration above zero, such as 1h or 7d, ` +
+				`or permanent`,
+		},
+		// Over the 4 KiB read of a body, even when the object comes first.
+		{
+			`{"netblock": "203.0.113.9", "duration": "1h", "reason": "` +
+				strings.Repeat("x", 4<<10) + `"}`,
+			"request body too large",
+		},
+		{
+			`{"netblock": "203.0.113.9", "duration": "1h"}` + strings.Repeat(" ", 4<<10),
+			"request body too large",
+		},
+	} {
+		got := s.admin(http.MethodPost, proxy.BansPath, tc.body, http.StatusBadRequest)
+		if !strings.Contains(string(got.body), tc.want) {
+			t.Errorf("%.80s was answered %q, want it to say %q", tc.body, got.body, tc.want)
+		}
+	}
+
+	if held := server.Ledger.Snapshot(); len(held) != 0 {
+		t.Errorf("the ledger holds %+v, want no ban", held)
+	}
+}
+
+func TestBanToAddOverTheRequestSizeLimitIsRefused(t *testing.T) {
+	t.Parallel()
+
+	s, _, server := startWithClock(t, "", map[string]string{
+		adminToken:      adminSecret,
+		requestMaxBytes: "16",
+	})
+
+	// Sent in a chunk, its length is not announced, so that it is found
+	// over SWWAF_REQUEST_MAX_BYTES only as it is read.
+	chunk := `{"netblock": "203.0.113.9", "duration": "1h"}`
+	s.adminRequest(adminClient, adminBearer+"\r\nTransfer-Encoding: chunked",
+		http.MethodPost, proxy.BansPath,
+		strconv.FormatInt(int64(len(chunk)), 16)+"\r\n"+chunk+"\r\n0\r\n\r\n",
+		http.StatusRequestEntityTooLarge, requestlog.ActionTooLarge)
+
+	if held := server.Ledger.Snapshot(); len(held) != 0 {
+		t.Errorf("the ledger holds %+v, want no ban", held)
+	}
+}
+
+func TestBanToAddSlowerThanTheClientRequestTimeoutIsRefused(t *testing.T) {
+	t.Parallel()
+
+	s, _, server := startWithClock(t, "", map[string]string{
+		adminToken:           adminSecret,
+		metricsToken:         token,
+		clientRequestTimeout: shortTimeoutSetting,
+	})
+
+	// The chunk announces 256 bytes and the rest of it never comes, so only
+	// the timeout ends the wait. A hold-up of the test process can only
+	// make the answer later, so the time is checked only for not being
+	// shorter than the timeout.
+	start := time.Now()
+
+	s.adminRequest(adminClient, adminBearer+"\r\nTransfer-Encoding: chunked",
+		http.MethodPost, proxy.BansPath, "100\r\n"+`{"netblock": "203.0.113.9", `,
+		http.StatusRequestTimeout, requestlog.ActionTimedOut)
+
+	if took := time.Since(start); took < shortTimeout {
+		t.Errorf("answered after %s, before the timeout of %s ran out", took, shortTimeout)
+	}
+
+	wantLimitHits(t, s.addr, clientRequestTimeout, 1)
+
+	if held := server.Ledger.Snapshot(); len(held) != 0 {
+		t.Errorf("the ledger holds %+v, want no ban", held)
+	}
+}
+
+func TestClientEndpointShowsTheClientAndItsBans(t *testing.T) {
+	t.Parallel()
+
+	s, clk, _ := startWithClock(t, "", map[string]string{
+		adminToken:          adminSecret,
+		rateLimitPerMinute:  "2",
+		rateLimitExemptNets: adminClient,
+	})
+	start := clk.Now()
+
+	// Two of otherClient's requests are let through; the third breaks the
+	// limit of two a minute, and bans it.
+	s.get(otherClient, http.StatusOK, requestlog.ActionForward)
+	s.get(otherClient, http.StatusOK, requestlog.ActionForward)
+	s.get(otherClient, http.StatusForbidden, requestlog.ActionRateLimited)
+
+	// Asked about by its address in IPv6 form too.
+	for _, addr := range []string{otherClient, "::ffff:" + otherClient} {
+		var got struct {
+			Client *ratelimit.Client `json:"client"`
+			Bans   []state.BanEntry  `json:"bans"`
+		}
+
+		decode(t, s.admin(http.MethodGet, proxy.ClientsPath+addr, "", http.StatusOK), &got)
+
+		if got.Client == nil {
+			t.Fatalf("%s: no client", addr)
+		}
+
+		history := got.Client.History
+		if got.Client.Client != netip.MustParsePrefix(otherClient+"/32") ||
+			history.Requests != 3 || history.Forwarded != 2 || history.Refused != 1 ||
+			history.Offences.Limit != 1 || !history.FirstSeen.Equal(start) {
+			t.Errorf("%s: client %+v", addr, got.Client)
+		}
+
+		if len(got.Bans) != 1 || got.Bans[0].Cause != bans.CauseLimit ||
+			got.Bans[0].Reason != "requests per minute over the limit of 2" ||
+			got.Bans[0].Notes.Count != 3 {
+			t.Errorf("%s: bans %+v, want the one for the broken limit", addr, got.Bans)
+		}
+	}
+
+	// Of an address no request came from and no ban covers, nothing is
+	// known.
+	got := s.admin(http.MethodGet, proxy.ClientsPath+"198.51.100.99", "", http.StatusOK)
+	if string(got.body) != "{\n    \"client\": null,\n    \"bans\": []\n}\n" {
+		t.Errorf("an unknown client is answered\n%s", got.body)
+	}
+
+	s.admin(http.MethodGet, proxy.ClientsPath+"203.0.113", "", http.StatusBadRequest)
+	s.admin(http.MethodDelete, proxy.BansPath+"/203.0.113.0/24", "",
+		http.StatusBadRequest)
+}
+
+func TestBannedClientIsRefusedAtTheEndpointsEvenWithTheToken(t *testing.T) {
+	t.Parallel()
+
+	s, _, _ := startWithClock(t, "", map[string]string{adminToken: adminSecret})
+
+	s.admin(http.MethodPost, proxy.BansPath, banOtherClient, http.StatusOK)
+
+	// otherClient cannot lift its own ban either.
+	for _, e := range adminEndpoints() {
+		s.adminRequest(otherClient, adminBearer, e.method, e.path, e.body,
+			http.StatusForbidden, requestlog.ActionBanned)
+	}
+}
+
+func TestAdminRequestsCountTowardTheLimits(t *testing.T) {
+	t.Parallel()
+
+	s, _, _ := startWithClock(t, "", map[string]string{
+		adminToken:         adminSecret,
+		rateLimitPerMinute: "2",
+	})
+
+	// A request refused for a missing token and one answered count toward
+	// the limit of two a minute, so the next breaks it.
+	s.adminRequest(client, "", http.MethodGet, proxy.BansPath, "",
+		http.StatusUnauthorized, requestlog.ActionAdmin)
+	s.adminRequest(client, adminBearer, http.MethodGet, proxy.BansPath, "",
+		http.StatusOK, requestlog.ActionAdmin)
+	s.adminRequest(client, adminBearer, http.MethodGet, proxy.BansPath, "",
+		http.StatusForbidden, requestlog.ActionRateLimited)
+}
+
+func TestClientInAllowNetsSkipsTheChecksButNeedsTheToken(t *testing.T) {
+	t.Parallel()
+
+	const allowed = "192.0.2.60" // in SWWAF_ALLOW_NETS
+
+	s, clk, server := startWithClock(t, "", map[string]string{
+		adminToken:         adminSecret,
+		allowNets:          allowed,
+		rateLimitPerMinute: "1",
+	})
+
+	// A ban on it refuses nothing, and its requests are not counted.
+	server.Ledger.BanForAdmin(netip.MustParsePrefix(allowed+"/32"), clk.Now(),
+		time.Time{}, "")
+
+	for range 2 {
+		s.adminRequest(allowed, "", http.MethodGet, proxy.BansPath, "",
+			http.StatusUnauthorized, requestlog.ActionAdmin)
+		s.adminRequest(allowed, adminBearer, http.MethodGet, proxy.BansPath, "",
+			http.StatusOK, requestlog.ActionAdmin)
+	}
+}
+
+func TestAdminEndpointsNeedTheTokenInObserveMode(t *testing.T) {
+	t.Parallel()
+
+	s, _, server := startWithClock(t, "", map[string]string{
+		adminToken: adminSecret,
+		mode:       observe,
+	})
+
+	for _, e := range adminEndpoints() {
+		s.adminRequest(adminClient, "", e.method, e.path, e.body,
+			http.StatusUnauthorized, requestlog.ActionAdmin)
+	}
+
+	if held := server.Ledger.Snapshot(); len(held) != 0 {
+		t.Errorf("the ledger holds %+v, want no ban", held)
+	}
+}
+
+// adminEndpoint is a request to an endpoint SWWAF_ADMIN_TOKEN opens.
+type adminEndpoint struct {
+	method, path, body string
+}
+
+// adminEndpoints returns a request to each endpoint SWWAF_ADMIN_TOKEN
+// opens: listing the bans, banning otherClient for an hour, lifting the
+// bans on otherClient, and asking about otherClient.
+func adminEndpoints() []adminEndpoint {
+	return []adminEndpoint{
+		{http.MethodGet, proxy.BansPath, ""},
+		{http.MethodPost, proxy.BansPath, banOtherClient},
+		{http.MethodDelete, proxy.BansPath + "/" + otherClient, ""},
+		{http.MethodGet, proxy.ClientsPath + otherClient, ""},
+	}
+}
+
+// admin sends a request with method for path, with body, from
+// adminClient, with the admin token, and checks that it is answered with
+// status, its log line's action admin. It returns the answer.
+func (s *sender) admin(method, path, body string, status int) answer {
+	s.t.Helper()
+
+	return s.adminRequest(adminClient, adminBearer, method, path, body, status,
+		requestlog.ActionAdmin)
+}
+
+// adminRequest sends a request with method for path, with body, from the
+// client at from, with authorization as its Authorization header unless
+// it is "", and checks its answer's status and its log line's action, as
+// request does. authorization may end in more header lines. A body that
+// is not "" has its length announced, unless authorization names
+// Transfer-Encoding. It returns the answer.
+func (s *sender) adminRequest(
+	from, authorization, method, path, body string, status int, action string,
+) answer {
+	s.t.Helper()
+
+	var header []string
+
+	if authorization != "" {
+		header = append(header, "Authorization: "+authorization)
+	}
+
+	if body != "" && !strings.Contains(authorization, "Transfer-Encoding") {
+		header = append(header, "Content-Length: "+strconv.Itoa(len(body)))
+	}
+
+	_, got := s.requestWithBody(method, from, path, strings.Join(header, "\r\n"),
+		body, status, action)
+
+	return got
+}
+
+// wantBans checks that a ban endpoint answered with want, and no other
+// ban.
+func wantBans(t *testing.T, got answer, want ...state.BanEntry) {
+	t.Helper()
+
+	var decoded struct {
+		Bans []state.BanEntry `json:"bans"`
+	}
+
+	decode(t, got, &decoded)
+
+	gotJSON, err := json.Marshal(decoded.Bans)
+	if err != nil {
+		t.Fatalf("encode %+v: %v", decoded.Bans, err)
+	}
+
+	wantJSON, err := json.Marshal(want)
+	if err != nil {
+		t.Fatalf("encode %+v: %v", want, err)
+	}
+
+	if string(gotJSON) != string(wantJSON) {
+		t.Errorf("bans\n%s\nwant\n%s", gotJSON, wantJSON)
+	}
+}
+
+// decode reads the JSON answer of an endpoint into value.
+func decode(t *testing.T, got answer, value any) {
+	t.Helper()
+
+	err := json.Unmarshal(got.body, value)
+	if err != nil {
+		t.Fatalf("decode %s: %v", got.body, err)
+	}
+}

internal/proxy/alerts_test.go

@@ -0,0 +1,264 @@
+package proxy_test
+
+import (
+	"maps"
+	"net/http"
+	"net/netip"
+	"reflect"
+	"testing"
+	"time"
+
+	"sneak.berlin/go/smallwebwaf/internal/alerts"
+	"sneak.berlin/go/smallwebwaf/internal/bans"
+	"sneak.berlin/go/smallwebwaf/internal/proxy"
+	"sneak.berlin/go/smallwebwaf/internal/requestlog"
+)
+
+const (
+	alertWebhookURL = "SWWAF_ALERT_WEBHOOK_URL"
+	alertMaxPerHour = "SWWAF_ALERT_MAX_PER_HOUR"
+	// alertInstance is the instance every alert of these tests gives.
+	alertInstance = "fsn1app1/gitea"
+)
+
+func TestBanForABrokenLimitRaisesABanAlertWithItsNotes(t *testing.T) {
+	t.Parallel()
+
+	s, clk, server, queue := startWithAlerts(t, map[string]string{
+		rateLimitPerMinute: "1",
+		banScopeV4Prefix:   "24",
+	})
+	start := clk.Now()
+
+	s.get(client, http.StatusOK, requestlog.ActionForward)
+	s.get(client, http.StatusForbidden, requestlog.ActionRateLimited)
+
+	netblock := netip.MustParsePrefix("203.0.113.0/24")
+	ban := server.Ledger.Bans(netblock)[0]
+
+	// A request refused under the ban raises no other alert.
+	clk.advance(time.Minute)
+	s.get(client, http.StatusForbidden, requestlog.ActionBanned)
+
+	wantAlerts(t, queue, banAlert(alerts.EventBan, start, client, bans.Ban{
+		Netblock: netblock, Cause: bans.CauseLimit,
+		Reason: "requests per minute over the limit of 1", Notes: ban.Notes,
+	}, requestlog.FormatTime(start.Add(time.Hour))))
+
+	if ban.Notes.Limit != 1 || ban.Notes.Request.Path != "/" {
+		t.Errorf("the alert's notes are %+v, want those of the broken limit", ban.Notes)
+	}
+}
+
+func TestAttackBanRaisesABanAlertThenAPermanentBanAlert(t *testing.T) {
+	t.Parallel()
+
+	s, clk, server, queue := startWithAlerts(t, map[string]string{
+		rulesDir: writeRules(t, testRules),
+	})
+	start := clk.Now()
+	netblock := netip.MustParsePrefix(client + "/32")
+	other := netip.MustParsePrefix(otherClient + "/32")
+
+	// The probe bans the client for seven days, and its next request makes
+	// the ban permanent. The request after that changes nothing.
+	s.request(client, "/.env", http.StatusForbidden, requestlog.ActionBanned)
+
+	attackBan := server.Ledger.Bans(netblock)[0]
+
+	clk.advance(time.Minute)
+	s.get(client, http.StatusForbidden, requestlog.ActionBanned)
+
+	permanentBan := server.Ledger.Bans(netblock)[0]
+
+	s.get(client, http.StatusForbidden, requestlog.ActionBanned)
+
+	// Another client's probe after its first ban has run out without a
+	// request makes a permanent ban at once.
+	s.request(otherClient, "/.env", http.StatusForbidden, requestlog.ActionBanned)
+	clk.advance(7 * 24 * time.Hour)
+	s.request(otherClient, "/.env", http.StatusForbidden, requestlog.ActionBanned)
+
+	otherBans := server.Ledger.Bans(other)
+
+	wantAlerts(t, queue,
+		attackAlert(alerts.EventBan, start, client, attackBan,
+			requestlog.FormatTime(start.Add(7*24*time.Hour))),
+		attackAlert(alerts.EventPermanentBan, start.Add(time.Minute), client,
+			permanentBan, "permanent"),
+		attackAlert(alerts.EventBan, start.Add(time.Minute), otherClient, otherBans[0],
+			requestlog.FormatTime(start.Add(time.Minute+7*24*time.Hour))),
+		attackAlert(alerts.EventPermanentBan, start.Add(time.Minute+7*24*time.Hour),
+			otherClient, otherBans[1], "permanent"),
+	)
+}
+
+func TestObserveModeRaisesTheBanAlertsItWouldHave(t *testing.T) {
+	t.Parallel()
+
+	s, clk, server, queue := startWithAlerts(t, map[string]string{
+		mode:               observe,
+		rateLimitPerMinute: "2",
+		rulesDir:           writeRules(t, testRules),
+	})
+	start := clk.Now()
+
+	// A ban for a clear sign of attack, which a request under it would make
+	// permanent.
+	group := netip.MustParsePrefix(ipv6Group)
+	attackBan, _ := server.Ledger.BanForAttack(group, start, bans.Notes{RuleID: "probe"})
+
+	// The third request breaks the limit, and so does the fourth, within the
+	// cooldown, which raises nothing. The probe is a clear sign of attack.
+	for range 4 {
+		s.get(client, http.StatusOK, requestlog.ActionForward)
+	}
+
+	s.request(otherClient, "/.env", http.StatusOK, requestlog.ActionForward)
+	line := s.get(ipv6Client, http.StatusOK, requestlog.ActionForward)
+
+	// No ban is made, and none made permanent.
+	if held := server.Ledger.Snapshot(); len(held) != 1 || held[0] != attackBan ||
+		line.BanExpires != requestlog.FormatTime(attackBan.Expires) {
+		t.Errorf("the ledger holds %+v, and the log line gives %s, want the ban "+
+			"for the attack alone, as it was", held, line.BanExpires)
+	}
+
+	waiting := queue.Snapshot().Waiting[alerts.DestinationWebhook]
+	if len(waiting) != 3 || queue.Suppressed() != 0 {
+		t.Fatalf("%d alerts wait and %d are held back, want 3 and 0: %+v",
+			len(waiting), queue.Suppressed(), waiting)
+	}
+
+	limitNotes, _ := waiting[0].Detail["notes"].(bans.Notes)
+	attackNotes, _ := waiting[1].Detail["notes"].(bans.Notes)
+
+	if limitNotes.Limit != 2 || limitNotes.Request.Path != "/" ||
+		attackNotes.Request.Path != "/.env" {
+		t.Errorf("the notes are %+v and %+v, want those of the broken limit and "+
+			"of the probe", limitNotes, attackNotes)
+	}
+
+	// Each alert is the one enforce mode would have raised, with mode
+	// observe in its detail.
+	want := []alerts.Alert{
+		banAlert(alerts.EventBan, start, client, bans.Ban{
+			Netblock: netip.MustParsePrefix(client + "/32"), Cause: bans.CauseLimit,
+			Reason: "requests per minute over the limit of 2", Notes: limitNotes,
+		}, requestlog.FormatTime(start.Add(time.Hour))),
+		attackAlert(alerts.EventBan, start, otherClient, bans.Ban{
+			Netblock: netip.MustParsePrefix(otherClient + "/32"), Notes: attackNotes,
+		}, requestlog.FormatTime(start.Add(7*24*time.Hour))),
+		attackAlert(alerts.EventPermanentBan, start, ipv6Client, attackBan, permanent),
+	}
+	for _, alert := range want {
+		alert.Detail["mode"] = observe
+	}
+
+	wantAlerts(t, queue, want...)
+}
+
+func TestObserveModeWorksOutABanOnlyWhenItsAlertWouldBeSent(t *testing.T) {
+	t.Parallel()
+
+	s, _, _, queue := startWithAlerts(t, map[string]string{
+		mode:               observe,
+		rateLimitPerMinute: "2",
+		rulesDir:           writeRules(t, testRules),
+		alertMaxPerHour:    "2",
+	})
+
+	// The client's third request breaks the limit, and raises the first
+	// alert of the hour. Its fourth is within the cooldown.
+	for range 4 {
+		s.get(client, http.StatusOK, requestlog.ActionForward)
+	}
+
+	// The other client's first probe raises the second. Its second probe is
+	// within the cooldown.
+	for range 2 {
+		s.request(otherClient, "/.env", http.StatusOK, requestlog.ActionForward)
+	}
+
+	// The IPv6 client's third request breaks the limit past the two alerts
+	// an hour.
+	for range 3 {
+		s.get(ipv6Client, http.StatusOK, requestlog.ActionForward)
+	}
+
+	// Had the ban been worked out for any of the requests within the
+	// cooldown or past the two an hour, its alert would have been raised,
+	// held back and counted.
+	waiting := queue.Snapshot().Waiting[alerts.DestinationWebhook]
+	if len(waiting) != 2 || queue.Suppressed() != 0 {
+		t.Errorf("%d alerts wait and %d are held back, want 2 and 0: %+v",
+			len(waiting), queue.Suppressed(), waiting)
+	}
+}
+
+// startWithAlerts is startWithClock with alerts to a webhook, which is
+// never sent them, and returns the queue they wait in as well.
+func startWithAlerts(
+	t *testing.T, env map[string]string,
+) (*sender, *clock, *proxy.Server, *alerts.Queue) {
+	t.Helper()
+
+	app := startApp(t, func(http.ResponseWriter, *http.Request) {})
+	clk := &clock{now: time.Date(2026, 10, 6, 0, 0, 0, 0, time.UTC)}
+	settings := map[string]string{
+		trustedProxies:  trustLocalhost,
+		alertWebhookURL: "https://alerts.example/smallwebwaf",
+		instanceName:    alertInstance,
+	}
+	maps.Copy(settings, env)
+
+	addr, out, server, queue := startProxyWithAlerts(t, app.URL, "", clk.Now, settings)
+
+	return &sender{t: t, addr: addr, out: out}, clk, server, queue
+}
+
+// banAlert returns the alert for event, raised by a request from client at
+// the time raised, for ban, with its netblock, cause, reason and notes,
+// which ends at expires, as the log line gives it.
+func banAlert(
+	event string, raised time.Time, client string, ban bans.Ban, expires string,
+) alerts.Alert {
+	return alerts.Alert{
+		Instance: alertInstance,
+		Time:     raised,
+		Event:    event,
+		Client:   netip.MustParseAddr(client),
+		Netblock: ban.Netblock,
+		Reason:   ban.Reason,
+		Detail: map[string]any{
+			"cause": ban.Cause, "ban_expires": expires, "notes": ban.Notes,
+		},
+	}
+}
+
+// attackAlert is banAlert for a ban for the probe rule of testRules, with
+// the netblock and the notes of ban.
+func attackAlert(
+	event string, raised time.Time, client string, ban bans.Ban, expires string,
+) alerts.Alert {
+	return banAlert(event, raised, client, bans.Ban{
+		Netblock: ban.Netblock, Cause: bans.CauseAttack, Reason: "matched the rule probe",
+		Notes: ban.Notes,
+	}, expires)
+}
+
+// wantAlerts checks the alerts waiting in queue, in order.
+func wantAlerts(t *testing.T, queue *alerts.Queue, want ...alerts.Alert) {
+	t.Helper()
+
+	got := queue.Snapshot().Waiting[alerts.DestinationWebhook]
+	if len(got) != len(want) {
+		t.Fatalf("%d alerts wait, want %d: %+v", len(got), len(want), got)
+	}
+
+	for i := range want {
+		if !reflect.DeepEqual(got[i], want[i]) {
+			t.Errorf("alert %d is\n%+v\nwant\n%+v", i, got[i], want[i])
+		}
+	}
+}

internal/proxy/bans.go

@@ -4,8 +4,10 @@ import (
 	"net/netip"
 	"time"
 
+	"sneak.berlin/go/smallwebwaf/internal/alerts"
 	"sneak.berlin/go/smallwebwaf/internal/bans"
 	"sneak.berlin/go/smallwebwaf/internal/requestlog"
+	"sneak.berlin/go/smallwebwaf/internal/rules"
 )
 
 // banResponse is a refusal answered with SWWAF_BAN_RESPONSE, and logged
@@ -14,61 +16,189 @@ func (rq *request) banResponse(action string) *refusal {
 	return &refusal{status: rq.h.config.BanResponse, action: action}
 }
 
-// banned reports whether a ban on a netblock the client is in refuses
-// the request at now, and notes for the log line when that ban ends.
+// banned reports whether a ban on a netblock the client is in covers the
+// request at now, and notes for the log line when that ban ends. A
+// request that makes the ban permanent, or in observe mode would have,
+// raises the alert for it.
 func (rq *request) banned(now time.Time) bool {
-	ban, banned := rq.h.ledger.Check(rq.client, now)
+	check := rq.h.ledger.Check
+	if rq.h.config.Observe {
+		check = rq.h.ledger.Find // the ban refuses nothing, and stays as it is
+	}
+
+	ban, banned, madePermanent := check(rq.client, now)
 	if banned {
 		rq.line.BanExpires = banExpires(ban)
 	}
 
+	if madePermanent {
+		ban.Expires = time.Time{} // the ban made permanent, which Find leaves as it is
+		rq.alertBan(ban)
+	}
+
 	return banned
 }
 
-// limitBroken counts the request for the rate limits at now, and reports
-// whether it takes the client over one. Such a request bans the client's
-// netblock, and sets the client's counters back to zero.
+// limitBroken counts the request for the rate limits at now, notes the
+// client's counts for the log line, and reports whether the request takes
+// the client over a limit. In enforce mode such a request bans the
+// client's netblock, and sets the client's counters back to zero; in
+// observe mode it does neither, and raises the alert for the ban it would
+// have made, if that alert would be sent.
 func (rq *request) limitBroken(now time.Time) bool {
 	group := clientGroup(rq.client)
 
-	hit, over := rq.h.limiter.Count(group, now)
+	counts, hit, over := rq.h.limiter.Count(group, now)
+	rq.line.Counts = counts
+
 	if !over {
 		return false
 	}
 
-	netblock := rq.netblock()
-	ban := rq.h.ledger.BanForLimit(netblock, now, bans.Notes{
-		Country: rq.line.Country,
-		Limit:   hit.Limit,
-		Window:  hit.Window,
-		Count:   hit.Requests,
-		Request: bans.Request{
-			Time:      now,
-			Method:    rq.in.Method,
-			Host:      rq.in.Host,
-			Path:      rq.in.URL.RequestURI(),
-			Status:    rq.h.config.BanResponse,
-			UserAgent: rq.in.UserAgent(),
-		},
-		// The histories count this request only once it has ended.
-		Requests: rq.h.limiter.Requests(netblock) + 1,
-	})
-	rq.h.limiter.Reset(group)
-
 	rq.line.LimitHit = hit.Window
 	rq.line.Offence = requestlog.OffenceLimit
+
+	netblock := rq.h.netblock(rq.client)
+	if rq.h.config.Observe && !rq.wouldAlertBan(netblock, now, bans.CauseLimit) {
+		return true
+	}
+
+	notes := bans.Notes{
+		Country:  rq.line.Country,
+		Limit:    hit.Limit,
+		Window:   hit.Window,
+		Count:    hit.Requests,
+		Request:  rq.noted(now),
+		Requests: rq.netblockRequests(netblock),
+	}
+
+	if rq.h.config.Observe {
+		ban, wouldBan := rq.h.ledger.WouldBanForLimit(netblock, now, notes)
+		if wouldBan {
+			rq.alertBan(ban)
+		}
+
+		return true
+	}
+
+	ban, made := rq.h.ledger.BanForLimit(netblock, now, notes)
+	rq.h.limiter.Reset(group)
 	rq.line.BanExpires = banExpires(ban)
 
+	if made {
+		rq.alertBan(ban)
+	}
+
 	return true
 }
 
-// netblock is the netblock a ban on the client covers: its IPv4 address,
+// banForAttack bans the client's netblock at now for a clear sign of
+// attack, the match of rule, a ban rule. In observe mode it makes no ban,
+// and raises the alert for the ban it would have made, if that alert
+// would be sent.
+func (rq *request) banForAttack(now time.Time, rule rules.Rule) {
+	netblock := rq.h.netblock(rq.client)
+	if rq.h.config.Observe && !rq.wouldAlertBan(netblock, now, bans.CauseAttack) {
+		return
+	}
+
+	notes := bans.Notes{
+		Country:  rq.line.Country,
+		RuleID:   rule.ID,
+		Target:   rule.Target,
+		Request:  rq.noted(now),
+		Requests: rq.netblockRequests(netblock),
+	}
+
+	if rq.h.config.Observe {
+		ban, wouldBan := rq.h.ledger.WouldBanForAttack(netblock, now, notes)
+		if wouldBan {
+			rq.alertBan(ban)
+		}
+
+		return
+	}
+
+	ban, made := rq.h.ledger.BanForAttack(netblock, now, notes)
+	rq.line.BanExpires = banExpires(ban)
+
+	if made {
+		rq.alertBan(ban)
+	}
+}
+
+// wouldAlertBan reports whether the alert for a ban on netblock for cause
+// made at now would be sent. In observe mode the ban the request would
+// have made is worked out only then, at most once per
+// SWWAF_ALERT_COOLDOWN and never with no webhook set: its notes count the
+// netblock's requests, which can mean going through every client.
+func (rq *request) wouldAlertBan(
+	netblock netip.Prefix, now time.Time, cause string,
+) bool {
+	event := alerts.EventBan
+	if rq.h.ledger.WouldBePermanent(netblock, now, cause) {
+		event = alerts.EventPermanentBan
+	}
+
+	return rq.h.alerts.WouldSend(event, netblock)
+}
+
+// alertBan raises the alert for ban, which the request made, or made
+// permanent: permanent_ban for a permanent ban, ban for another. Its
+// detail gives the ban's cause, when it ends, and its notes, and in
+// observe mode, where ban is the ban that would have been made, or made
+// permanent, mode, observe.
+func (rq *request) alertBan(ban bans.Ban) {
+	event := alerts.EventBan
+	if ban.Permanent() {
+		event = alerts.EventPermanentBan
+	}
+
+	detail := map[string]any{
+		"cause": ban.Cause, "ban_expires": banExpires(ban), "notes": ban.Notes,
+	}
+	if rq.h.config.Observe {
+		detail["mode"] = "observe"
+	}
+
+	rq.h.alerts.Raise(alerts.Alert{
+		Event:    event,
+		Client:   rq.client,
+		Netblock: ban.Netblock,
+		Country:  ban.Notes.Country,
+		Reason:   ban.Reason,
+		Detail:   detail,
+	})
+}
+
+// noted is the request, refused at now with SWWAF_BAN_RESPONSE, or in
+// observe mode as it would have been, as the notes of the ban it makes
+// keep it.
+func (rq *request) noted(now time.Time) bans.Request {
+	return bans.Request{
+		Time:      now,
+		Method:    rq.in.Method,
+		Host:      rq.in.Host,
+		Path:      rq.in.URL.RequestURI(),
+		Status:    rq.h.config.BanResponse,
+		UserAgent: rq.in.UserAgent(),
+	}
+}
+
+// netblockRequests is how many requests netblock has sent since it was
+// first seen, this one included: the histories count it only once it has
+// ended.
+func (rq *request) netblockRequests(netblock netip.Prefix) int64 {
+	return rq.h.limiter.Requests(netblock) + 1
+}
+
+// netblock is the netblock a ban on client covers: its IPv4 address,
 // widened to SWWAF_BAN_SCOPE_V4_PREFIX, or the IPv6 group clientGroup
 // counts it in.
-func (rq *request) netblock() netip.Prefix {
-	addr := rq.client.Unmap()
+func (h *handler) netblock(client netip.Addr) netip.Prefix {
+	addr := client.Unmap()
 	if addr.Is4() {
-		return netip.PrefixFrom(addr, rq.h.config.BanScopeV4Prefix).Masked()
+		return netip.PrefixFrom(addr, h.config.BanScopeV4Prefix).Masked()
 	}
 
 	return clientGroup(addr)
@@ -78,7 +208,7 @@ func (rq *request) netblock() netip.Prefix {
 // permanent.
 func banExpires(ban bans.Ban) string {
 	if ban.Permanent() {
-		return "permanent"
+		return permanent
 	}
 
 	return requestlog.FormatTime(ban.Expires)

internal/proxy/bans_test.go

@@ -278,6 +278,8 @@ func TestBanNotes(t *testing.T) {
 		Netblock: netblock,
 		Start:    start,
 		Expires:  start.Add(time.Hour),
+		Cause:    bans.CauseLimit,
+		Reason:   "requests per minute over the limit of 1",
 		Notes: bans.Notes{
 			Country: "DE",
 			Limit:   1,
@@ -295,7 +297,7 @@ func TestBanNotes(t *testing.T) {
 			// refused under the ban.
 			Requests:    4,
 			Refused:     2,
-			EarlierBans: 0,
+			EarlierBans: bans.EarlierBans{},
 		},
 	}
 
@@ -312,8 +314,8 @@ func TestBanNotes(t *testing.T) {
 	s.get(fromDE, http.StatusForbidden, requestlog.ActionRateLimited)
 
 	got = ledger.Bans(netblock)
-	if len(got) != 2 || got[1].Notes.EarlierBans != 1 {
-		t.Errorf("bans %+v, want two, the second with one earlier ban", got)
+	if len(got) != 2 || got[1].Notes.EarlierBans != (bans.EarlierBans{Limit: 1}) {
+		t.Errorf("bans %+v, want two, the second with one earlier ban for a limit", got)
 	}
 }
 
@@ -415,14 +417,28 @@ func (s *sender) requestWithHeader(
 ) (logLine, string) {
 	s.t.Helper()
 
+	line, got := s.requestWithBody(http.MethodGet, from, path, header, "", status, action)
+
+	return line, string(got.body)
+}
+
+// requestWithBody is requestWithHeader for a request with method, whose
+// body is sent as it is after the headers, header holding its
+// Content-Length or Transfer-Encoding. header may hold several lines,
+// separated by "\r\n". It returns the whole answer.
+func (s *sender) requestWithBody(
+	method, from, path, header, body string, status int, action string,
+) (logLine, answer) {
+	s.t.Helper()
+
 	if header != "" {
 		header += "\r\n"
 	}
 
 	conn := dial(s.t, s.addr)
-	send(s.t, conn, "GET "+path+" HTTP/1.1\r\nHost: "+appHost+
+	send(s.t, conn, method+" "+path+" HTTP/1.1\r\nHost: "+appHost+
 		"\r\nUser-Agent: "+userAgent+"\r\n"+forwardedFor+": "+from+"\r\n"+
-		header+"\r\n")
+		header+"\r\n"+body)
 
 	err := conn.SetReadDeadline(time.Now().Add(waitLimit))
 	if err != nil {
@@ -451,5 +467,5 @@ func (s *sender) requestWithHeader(
 	s.sent++
 	wantLine(s.t, line, status, action)
 
-	return line, string(got.body)
+	return line, got
 }

internal/proxy/client.go

@@ -1,6 +1,7 @@
 package proxy
 
 import (
+	"crypto/rand"
 	"net/http"
 	"net/netip"
 	"slices"
@@ -48,6 +49,33 @@ func clientAddress(
 	return client
 }
 
+// requestIDHeader carries the request's id, from traefik and to the app.
+const requestIDHeader = "X-Request-ID"
+
+// requestID is the request's id: the one a trusted proxy sent, or a new
+// random one. A peer outside the trusted proxies did not come through
+// traefik, so the id it sends is its own claim, and is replaced.
+func requestID(r *http.Request, peerTrusted bool) string {
+	id := r.Header.Get(requestIDHeader)
+	if !peerTrusted || id == "" {
+		id = rand.Text()
+	}
+
+	return id
+}
+
+// scheme is how the client reached traefik, as a trusted proxy says in
+// X-Forwarded-Proto, or otherwise http, the only scheme smallwebwaf
+// serves.
+func scheme(r *http.Request, peerTrusted bool) string {
+	proto := r.Header.Get("X-Forwarded-Proto")
+	if !peerTrusted || proto == "" {
+		return "http"
+	}
+
+	return proto
+}
+
 // ipv6GroupPrefix is the length of the IPv6 netblock that is one client.
 const ipv6GroupPrefix = 64
 

internal/proxy/client_test.go

@@ -14,10 +14,14 @@ const (
 	appHost = "app.example"
 	// client is the client's address, as a proxy names it.
 	client = "203.0.113.9"
-	// forwardedFor is the header that lists the client and its proxies.
-	forwardedFor = "X-Forwarded-For"
-	// secure is the scheme a client reached traefik with.
+	// forwardedFor is the header that lists the client and its proxies,
+	// and forwardedProto the one that gives the scheme the client used.
+	forwardedFor   = "X-Forwarded-For"
+	forwardedProto = "X-Forwarded-Proto"
+	// secure is the scheme a client reached traefik with, and plain the
+	// one smallwebwaf serves.
 	secure = "https"
+	plain  = "http"
 )
 
 // appHeaders is what the app tells about the headers it received.
@@ -65,13 +69,13 @@ func TestClientAddressAndForwardedHeaders(t *testing.T) {
 func clientAddressCases() []clientAddressCase {
 	trusted := map[string]string{trustedProxies: trustLocalhost}
 	forged := http.Header{
-		forwardedFor:        {client},
-		"X-Forwarded-Host":  {"forged.example"},
-		"X-Forwarded-Proto": {secure},
-		"X-Real-Ip":         {client},
+		forwardedFor:       {client},
+		"X-Forwarded-Host": {"forged.example"},
+		forwardedProto:     {secure},
+		"X-Real-Ip":        {client},
 	}
 	replaced := appHeaders{
-		ForwardedFor: localhost, ForwardedHost: appHost, ForwardedProto: "http",
+		ForwardedFor: localhost, ForwardedHost: appHost, ForwardedProto: plain,
 	}
 
 	return []clientAddressCase{{
@@ -87,10 +91,10 @@ func clientAddressCases() []clientAddressCase {
 			"outside the trusted proxies from the right",
 		env: trusted,
 		header: http.Header{
-			forwardedFor:        {"198.51.100.7, " + client + ", 10.0.0.2"},
-			"X-Forwarded-Host":  {appHost},
-			"X-Forwarded-Proto": {secure},
-			"X-Real-Ip":         {client},
+			forwardedFor:       {"198.51.100.7, " + client + ", 10.0.0.2"},
+			"X-Forwarded-Host": {appHost},
+			forwardedProto:     {secure},
+			"X-Real-Ip":        {client},
 		},
 		wantClient: client,
 		wantApp: appHeaders{
@@ -138,7 +142,7 @@ func requestWithHeaders(
 			Host:           r.Host,
 			ForwardedFor:   r.Header.Get(forwardedFor),
 			ForwardedHost:  r.Header.Get("X-Forwarded-Host"),
-			ForwardedProto: r.Header.Get("X-Forwarded-Proto"),
+			ForwardedProto: r.Header.Get(forwardedProto),
 			RealIP:         r.Header.Get("X-Real-IP"),
 		})
 	})

internal/proxy/health_test.go

@@ -21,14 +21,18 @@ func TestHealthEndpointIsAnsweredBeforeAnyCheck(t *testing.T) {
 	// the last one would have it refused.
 	addr, out := startProxy(t, app.URL, map[string]string{rateLimitPerMinute: "1"})
 
-	const healthChecks = 3
+	const (
+		healthChecks = 3
+		contentType  = "text/plain; charset=utf-8"
+	)
 
 	for range healthChecks {
 		got := get(t, addr, proxy.HealthPath)
 		wantStatus(t, got, http.StatusOK)
 
-		if string(got.body) != "ok\n" {
-			t.Errorf("health endpoint answered %q, want ok", got.body)
+		if string(got.body) != "ok\n" || got.header.Get("Content-Type") != contentType {
+			t.Errorf("health endpoint answered %q with Content-Type %q, want ok "+
+				"with %q", got.body, got.header.Get("Content-Type"), contentType)
 		}
 	}
 
@@ -37,6 +41,11 @@ func TestHealthEndpointIsAnsweredBeforeAnyCheck(t *testing.T) {
 	lines := out.requestLines(t, healthChecks+1)
 	for _, line := range lines[:healthChecks] {
 		wantLine(t, line, http.StatusOK, requestlog.ActionAdmin)
+
+		if line.ResponseContentType != contentType {
+			t.Errorf("health check's log line has response_content_type %q, "+
+				"want %q", line.ResponseContentType, contentType)
+		}
 	}
 
 	wantLine(t, lines[healthChecks], http.StatusOK, requestlog.ActionForward)

internal/proxy/metrics_test.go

@@ -12,6 +12,7 @@ import (
 	"testing"
 	"time"
 
+	"sneak.berlin/go/smallwebwaf/internal/bans"
 	"sneak.berlin/go/smallwebwaf/internal/lookup"
 	"sneak.berlin/go/smallwebwaf/internal/proxy"
 	"sneak.berlin/go/smallwebwaf/internal/requestlog"
@@ -147,8 +148,8 @@ func TestMetricsCountTheTraffic(t *testing.T) {
 	wantStatus(t, get(t, addr, "/_smallwebwaf/nothing"), http.StatusNotFound)
 	out.requestLines(t, 2)
 
-	forward := `{action="forward",status_class="2xx"}`
-	notFound := `{action="admin",status_class="4xx"}`
+	forward := `{action="forward",instance="app",status_class="2xx"}`
+	notFound := `{action="admin",instance="app",status_class="4xx"}`
 
 	// The request for the metrics is itself under way.
 	metrics := scrape(t, addr)
@@ -158,11 +159,13 @@ func TestMetricsCountTheTraffic(t *testing.T) {
 	wantMetric(t, metrics, "smallwebwaf_response_bytes_total"+forward, 5)
 	wantMetric(t, metrics, "smallwebwaf_response_bytes_total"+notFound,
 		float64(len("Not Found\n")))
-	wantMetric(t, metrics, "smallwebwaf_request_duration_seconds_count", 2)
-	wantMetric(t, metrics, "smallwebwaf_upstream_duration_seconds_count", 1)
-	wantMetric(t, metrics, "smallwebwaf_requests_in_flight", 1)
-	metric(t, metrics, "go_goroutines")
-	metric(t, metrics, "process_start_time_seconds")
+	wantMetric(t, metrics,
+		`smallwebwaf_request_duration_seconds_count{instance="app"}`, 2)
+	wantMetric(t, metrics,
+		`smallwebwaf_upstream_duration_seconds_count{instance="app"}`, 1)
+	wantMetric(t, metrics, `smallwebwaf_requests_in_flight{instance="app"}`, 1)
+	metric(t, metrics, `go_goroutines{instance="app"}`)
+	metric(t, metrics, `process_start_time_seconds{instance="app"}`)
 
 	// A request the app holds is under way until it ends.
 	httpClient := newClient(t)
@@ -179,7 +182,7 @@ func TestMetricsCountTheTraffic(t *testing.T) {
 	}()
 
 	<-arrived
-	wantMetric(t, scrape(t, addr), "smallwebwaf_requests_in_flight", 2)
+	wantMetric(t, scrape(t, addr), `smallwebwaf_requests_in_flight{instance="app"}`, 2)
 	releaseApp()
 
 	err := <-ended
@@ -188,7 +191,7 @@ func TestMetricsCountTheTraffic(t *testing.T) {
 	}
 
 	out.requestLines(t, 5)
-	wantMetric(t, scrape(t, addr), "smallwebwaf_requests_in_flight", 1)
+	wantMetric(t, scrape(t, addr), `smallwebwaf_requests_in_flight{instance="app"}`, 1)
 }
 
 func TestMetricsCountLimitsAndBans(t *testing.T) {
@@ -218,15 +221,16 @@ func TestMetricsCountLimitsAndBans(t *testing.T) {
 
 	metrics := s.scrape(scraper)
 	wantMetric(t, metrics,
-		`smallwebwaf_requests_total{action="denied",status_class="none"}`, 1)
-	wantMetric(t, metrics, `smallwebwaf_rate_limit_hits_total{window="minute"}`, 1)
-	wantMetric(t, metrics, `smallwebwaf_offences_total{kind="limit"}`, 1)
-	wantMetric(t, metrics, `smallwebwaf_bans_made_total{cause="limit"}`, 1)
-	wantMetric(t, metrics, "smallwebwaf_active_bans", 1)
-	wantMetric(t, metrics, "smallwebwaf_permanent_bans", 0)
+		`smallwebwaf_requests_total{action="denied",instance="app",status_class="none"}`, 1)
+	wantMetric(t, metrics,
+		`smallwebwaf_rate_limit_hits_total{instance="app",window="minute"}`, 1)
+	wantMetric(t, metrics, `smallwebwaf_offences_total{instance="app",kind="limit"}`, 1)
+	wantMetric(t, metrics, `smallwebwaf_bans_made_total{cause="limit",instance="app"}`, 1)
+	wantMetric(t, metrics, `smallwebwaf_active_bans{instance="app"}`, 1)
+	wantMetric(t, metrics, `smallwebwaf_permanent_bans{instance="app"}`, 0)
 
 	clk.advance(time.Hour)
-	wantMetric(t, s.scrape(scraper), "smallwebwaf_active_bans", 0)
+	wantMetric(t, s.scrape(scraper), `smallwebwaf_active_bans{instance="app"}`, 0)
 
 	// A limit broken again right after would ban for three hours, longer
 	// than SWWAF_MAX_BAN_DURATION, so the ban is permanent.
@@ -234,13 +238,42 @@ func TestMetricsCountLimitsAndBans(t *testing.T) {
 	s.get(client, 0, requestlog.ActionRateLimited)
 
 	metrics = s.scrape(scraper)
-	wantMetric(t, metrics, `smallwebwaf_rate_limit_hits_total{window="minute"}`, 2)
-	wantMetric(t, metrics, `smallwebwaf_offences_total{kind="limit"}`, 2)
-	wantMetric(t, metrics, `smallwebwaf_bans_made_total{cause="limit"}`, 2)
-	wantMetric(t, metrics, "smallwebwaf_active_bans", 1)
-	wantMetric(t, metrics, "smallwebwaf_permanent_bans", 1)
+	wantMetric(t, metrics,
+		`smallwebwaf_rate_limit_hits_total{instance="app",window="minute"}`, 2)
+	wantMetric(t, metrics, `smallwebwaf_offences_total{instance="app",kind="limit"}`, 2)
+	wantMetric(t, metrics, `smallwebwaf_bans_made_total{cause="limit",instance="app"}`, 2)
+	wantMetric(t, metrics, `smallwebwaf_active_bans{instance="app"}`, 1)
+	wantMetric(t, metrics, `smallwebwaf_permanent_bans{instance="app"}`, 1)
 	// denied, client, and the scraper as of its earlier requests.
-	wantMetric(t, metrics, "smallwebwaf_tracked_clients", 3)
+	wantMetric(t, metrics, `smallwebwaf_tracked_clients{instance="app"}`, 3)
+}
+
+func TestMetricsCountTheBansAnAdminMakes(t *testing.T) {
+	t.Parallel()
+
+	const scraper = "192.0.2.200" // in SWWAF_RATE_LIMIT_EXEMPT_NETS
+
+	s, clk, server := startWithClock(t, "", map[string]string{
+		metricsToken:        token,
+		adminToken:          adminSecret,
+		rateLimitExemptNets: scraper,
+	})
+
+	const admins = `smallwebwaf_bans_made_total{cause="admin",instance="app"}`
+
+	wantMetric(t, s.scrape(scraper), admins, 0)
+
+	// As an admin's edit of bans.json that adds a ban is taken in.
+	server.Ledger.LoadEdit([]bans.Ban{{
+		Netblock: netip.MustParsePrefix(client + "/32"),
+		Start:    clk.Now(),
+	}})
+
+	wantMetric(t, s.scrape(scraper), admins, 1)
+
+	// And a ban made through the endpoint.
+	s.admin(http.MethodPost, proxy.BansPath, banOtherClient, http.StatusOK)
+	wantMetric(t, s.scrape(scraper), admins, 2)
 }
 
 func TestMetricsByCountryKeepTheBusiestAndCountTheRestAsOther(t *testing.T) {
@@ -290,28 +323,42 @@ func TestMetricsByCountryKeepTheBusiestAndCountTheRestAsOther(t *testing.T) {
 	metrics := scrape(t, addr)
 	lines++
 
-	wantMetric(t, metrics, `smallwebwaf_country_requests_total{country="KP"}`, 3)
-	wantMetric(t, metrics, `smallwebwaf_country_requests_total{country="DE"}`, 2)
-	wantMetric(t, metrics, `smallwebwaf_country_requests_total{country="other"}`, 1)
-	wantMetric(t, metrics, `smallwebwaf_country_list_refusals_total{country="KP"}`, 3)
-	wantMetric(t, metrics, `smallwebwaf_country_request_bytes_total{country="KP"}`, 0)
-	wantMetric(t, metrics, `smallwebwaf_country_request_bytes_total{country="DE"}`, 6)
-	wantMetric(t, metrics, `smallwebwaf_country_response_bytes_total{country="KP"}`,
+	wantMetric(t, metrics,
+		`smallwebwaf_country_requests_total{country="KP",instance="app"}`, 3)
+	wantMetric(t, metrics,
+		`smallwebwaf_country_requests_total{country="DE",instance="app"}`, 2)
+	wantMetric(t, metrics,
+		`smallwebwaf_country_requests_total{country="other",instance="app"}`, 1)
+	wantMetric(t, metrics,
+		`smallwebwaf_country_list_refusals_total{country="KP",instance="app"}`, 3)
+	wantMetric(t, metrics,
+		`smallwebwaf_country_request_bytes_total{country="KP",instance="app"}`, 0)
+	wantMetric(t, metrics,
+		`smallwebwaf_country_request_bytes_total{country="DE",instance="app"}`, 6)
+	wantMetric(t, metrics,
+		`smallwebwaf_country_response_bytes_total{country="KP",instance="app"}`,
 		float64(3*len("Forbidden\n")))
-	wantMetric(t, metrics, `smallwebwaf_country_response_bytes_total{country="other"}`,
+	wantMetric(t, metrics,
+		`smallwebwaf_country_response_bytes_total{country="other",instance="app"}`,
 		float64(len("hello")))
-	wantNoSeries(t, metrics, `smallwebwaf_country_requests_total{country="FR"}`)
+	wantNoSeries(t, metrics,
+		`smallwebwaf_country_requests_total{country="FR",instance="app"}`)
 
 	// Once FR is busier than DE, it takes DE's place: its series counts
 	// from then on, and DE's is gone.
 	send(fromFR, 3, http.StatusOK)
 
 	metrics = scrape(t, addr)
-	wantMetric(t, metrics, `smallwebwaf_country_requests_total{country="KP"}`, 3)
-	wantMetric(t, metrics, `smallwebwaf_country_requests_total{country="FR"}`, 2)
-	wantMetric(t, metrics, `smallwebwaf_country_requests_total{country="other"}`, 2)
-	wantNoSeries(t, metrics, `smallwebwaf_country_requests_total{country="DE"}`)
-	wantNoSeries(t, metrics, `smallwebwaf_country_request_bytes_total{country="DE"}`)
+	wantMetric(t, metrics,
+		`smallwebwaf_country_requests_total{country="KP",instance="app"}`, 3)
+	wantMetric(t, metrics,
+		`smallwebwaf_country_requests_total{country="FR",instance="app"}`, 2)
+	wantMetric(t, metrics,
+		`smallwebwaf_country_requests_total{country="other",instance="app"}`, 2)
+	wantNoSeries(t, metrics,
+		`smallwebwaf_country_requests_total{country="DE",instance="app"}`)
+	wantNoSeries(t, metrics,
+		`smallwebwaf_country_request_bytes_total{country="DE",instance="app"}`)
 }
 
 func TestMetricsCountGeoJSRequestsAndFailures(t *testing.T) {
@@ -341,16 +388,16 @@ func TestMetricsCountGeoJSRequestsAndFailures(t *testing.T) {
 	deadline := time.Now().Add(waitLimit)
 	metrics := scrape(t, addr)
 
-	for metric(t, metrics, "smallwebwaf_geojs_failures_total") == 0 &&
+	for metric(t, metrics, `smallwebwaf_geojs_failures_total{instance="app"}`) == 0 &&
 		time.Now().Before(deadline) {
 		time.Sleep(pollInterval)
 
 		metrics = scrape(t, addr)
 	}
 
-	wantMetric(t, metrics, "smallwebwaf_geojs_requests_total", 1)
-	wantMetric(t, metrics, "smallwebwaf_geojs_failures_total", 1)
-	wantMetric(t, metrics, "smallwebwaf_geojs_unanswered_total", 1)
+	wantMetric(t, metrics, `smallwebwaf_geojs_requests_total{instance="app"}`, 1)
+	wantMetric(t, metrics, `smallwebwaf_geojs_failures_total{instance="app"}`, 1)
+	wantMetric(t, metrics, `smallwebwaf_geojs_unanswered_total{instance="app"}`, 1)
 }
 
 // keptAnswer returns GeoJS's answer that the client at addr is in
@@ -393,8 +440,9 @@ func (s *sender) scrape(from string) string {
 
 // metric returns the value of series in metrics, which are in the
 // Prometheus text format. series is a name and its labels in the order of
-// their names, such as smallwebwaf_offences_total{kind="limit"}. It fails
-// the test if there is no such series.
+// their names, such as
+// smallwebwaf_offences_total{instance="app",kind="limit"}. It fails the
+// test if there is no such series.
 func metric(t *testing.T, metrics, series string) float64 {
 	t.Helper()
 
@@ -442,7 +490,8 @@ func wantNoSeries(t *testing.T, metrics, series string) {
 func wantLimitHits(t *testing.T, addr, limit string, hits int) {
 	t.Helper()
 
-	series := `smallwebwaf_size_and_time_limit_hits_total{limit="` + limit + `"}`
+	series := `smallwebwaf_size_and_time_limit_hits_total{instance="app",limit="` +
+		limit + `"}`
 	metrics := scrape(t, addr)
 
 	if hits == 0 {

internal/proxy/observe_test.go

@@ -0,0 +1,203 @@
+package proxy_test
+
+import (
+	"bytes"
+	"io"
+	"net/http"
+	"net/netip"
+	"sync/atomic"
+	"testing"
+	"time"
+
+	"sneak.berlin/go/smallwebwaf/internal/bans"
+	"sneak.berlin/go/smallwebwaf/internal/proxy"
+	"sneak.berlin/go/smallwebwaf/internal/requestlog"
+)
+
+// observe is the value of SWWAF_MODE for observe mode.
+const observe = "observe"
+
+func TestObserveModeForwardsWhatEnforceModeRefuses(t *testing.T) {
+	t.Parallel()
+
+	const (
+		denied = "192.0.2.50" // in SWWAF_DENY_NETS
+		banned = otherClient  // under a ban read from bans.json
+	)
+
+	for _, tc := range []struct {
+		setting string // "" leaves SWWAF_MODE at its default
+		observe bool
+	}{
+		{"", false},
+		{"enforce", false},
+		{observe, true},
+	} {
+		t.Run(mode+"="+tc.setting, func(t *testing.T) {
+			t.Parallel()
+
+			geojsURL, _ := startGeoJS(t)
+			env := map[string]string{
+				rateLimitPerMinute: "1",
+				denyNets:           denied,
+				deniedCountries:    "kp",
+			}
+
+			if tc.setting != "" {
+				env[mode] = tc.setting
+			}
+
+			s, clk, server := startWithClock(t, geojsURL, env)
+			server.Ledger.Load([]bans.Ban{{
+				Netblock: netip.MustParsePrefix(banned + "/32"),
+				Start:    clk.Now(),
+				Expires:  clk.Now().Add(time.Hour),
+			}})
+
+			// fromDE's first request is within the limit of one a minute,
+			// and its second breaks it.
+			s.get(fromDE, http.StatusOK, requestlog.ActionForward)
+
+			for _, sent := range []struct{ from, refusal string }{
+				{denied, requestlog.ActionDenied},
+				{banned, requestlog.ActionBanned},
+				{fromKP, requestlog.ActionCountryDenied},
+				{fromDE, requestlog.ActionRateLimited},
+			} {
+				if !tc.observe {
+					line := s.get(sent.from, http.StatusForbidden, sent.refusal)
+					wantWouldAction(t, line, "")
+
+					continue
+				}
+
+				// Passed to the app, which answered it.
+				line := s.get(sent.from, http.StatusOK, requestlog.ActionForward)
+				wantWouldAction(t, line, sent.refusal)
+
+				if line.UpstreamStatus != http.StatusOK {
+					t.Errorf("log line has upstream_status %d, want 200",
+						line.UpstreamStatus)
+				}
+			}
+		})
+	}
+}
+
+func TestObserveModeMakesNoBanAndKeepsTheBansItHas(t *testing.T) {
+	t.Parallel()
+
+	s, clk, server := startWithClock(t, "", map[string]string{
+		mode:               observe,
+		rateLimitPerMinute: "1",
+	})
+	kept := bans.Ban{
+		Netblock: netip.MustParsePrefix(otherClient + "/32"),
+		Start:    clk.Now(),
+		Expires:  clk.Now().Add(time.Hour),
+		Cause:    bans.CauseAdmin,
+	}
+	server.Ledger.Load([]bans.Ban{kept})
+
+	// No ban sets client's counters back to zero, so each request after
+	// the first breaks the limit of one a minute.
+	s.get(client, http.StatusOK, requestlog.ActionForward)
+
+	for range 2 {
+		line := s.get(client, http.StatusOK, requestlog.ActionForward)
+		wantWouldAction(t, line, requestlog.ActionRateLimited)
+
+		if line.LimitHit != minute || line.Offence != requestlog.OffenceLimit ||
+			line.BanExpires != "" {
+			t.Errorf("log line has limit_hit %q, offence %q and ban_expires %q, "+
+				"want minute, limit and none", line.LimitHit, line.Offence,
+				line.BanExpires)
+		}
+	}
+
+	// The ban read from bans.json refuses nothing, and so counts no
+	// refusal in its notes, but is kept.
+	line := s.get(otherClient, http.StatusOK, requestlog.ActionForward)
+	wantWouldAction(t, line, requestlog.ActionBanned)
+
+	if line.BanExpires != requestlog.FormatTime(kept.Expires) {
+		t.Errorf("log line has ban_expires %q, want %s", line.BanExpires,
+			requestlog.FormatTime(kept.Expires))
+	}
+
+	got := server.Ledger.Snapshot()
+	if len(got) != 1 || got[0] != kept {
+		t.Errorf("bans\n%+v\nwant only\n%+v", got, kept)
+	}
+}
+
+func TestObserveModeKeepsTheSizeLimitsAndTheToken(t *testing.T) {
+	t.Parallel()
+
+	const denied = "192.0.2.50" // in SWWAF_DENY_NETS
+
+	var calls atomic.Int32
+
+	app := startApp(t, func(w http.ResponseWriter, _ *http.Request) {
+		calls.Add(1)
+		answerWithSize(w, 2*sizeLimit, true)
+	})
+	addr, out := startProxy(t, app.URL, map[string]string{
+		mode:             observe,
+		trustedProxies:   trustLocalhost,
+		denyNets:         denied,
+		requestMaxBytes:  sizeLimitSetting,
+		responseMaxBytes: sizeLimitSetting,
+		metricsToken:     token,
+	})
+
+	// SWWAF_DENY_NETS would refuse each request; instead a size limit or
+	// the missing token does.
+	for i, tc := range []struct {
+		method, path string
+		body         io.Reader
+		status       int
+		action       string
+	}{
+		{
+			http.MethodPost, "/upload", bytes.NewReader(make([]byte, 2*sizeLimit)),
+			http.StatusRequestEntityTooLarge, requestlog.ActionTooLarge,
+		},
+		{
+			http.MethodGet, "/download", http.NoBody,
+			http.StatusBadGateway, requestlog.ActionTooLarge,
+		},
+		{
+			http.MethodGet, proxy.MetricsPath, http.NoBody,
+			http.StatusUnauthorized, requestlog.ActionAdmin,
+		},
+	} {
+		req := newRequest(t, tc.method, addr, tc.path, tc.body)
+		req.Header.Set(forwardedFor, denied)
+		wantStatus(t, do(t, req), tc.status)
+
+		line := out.requestLines(t, i+1)[i]
+		wantLine(t, line, tc.status, tc.action)
+		wantWouldAction(t, line, requestlog.ActionDenied)
+	}
+
+	// The upload was refused before it reached the app.
+	if calls.Load() != 1 {
+		t.Errorf("the app was called %d times, want once", calls.Load())
+	}
+}
+
+// wantWouldAction checks the request log line's would_action, and that a
+// line that should have none has no such field.
+func wantWouldAction(t *testing.T, line logLine, want string) {
+	t.Helper()
+
+	got, present := line.fields["would_action"]
+
+	switch {
+	case want == "" && present:
+		t.Errorf("log line has would_action %v, want none", got)
+	case want != "" && got != want:
+		t.Errorf("log line has would_action %v, want %s", got, want)
+	}
+}

internal/proxy/passthrough_test.go

@@ -6,6 +6,7 @@ import (
 	"errors"
 	"io"
 	"net/http"
+	"reflect"
 	"slices"
 	"strings"
 	"sync/atomic"
@@ -14,6 +15,7 @@ import (
 
 	"sneak.berlin/go/smallwebwaf/internal/config"
 	"sneak.berlin/go/smallwebwaf/internal/proxy"
+	"sneak.berlin/go/smallwebwaf/internal/ratelimit"
 	"sneak.berlin/go/smallwebwaf/internal/requestlog"
 )
 
@@ -115,27 +117,32 @@ func wantAnswer(t *testing.T, got answer, body []byte) {
 	}
 }
 
-// wantRequestFields checks the log line's fields about the request.
+// wantRequestFields checks the log line's fields about the request. Its
+// time, its id and its timings are checked only for being there.
 func wantRequestFields(t *testing.T, line logLine, host string, sent, received int) {
 	t.Helper()
 
-	want := requestlog.Line{
-		Type: "request", Time: line.Time, ClientIP: localhost, PeerIP: localhost,
-		Method: http.MethodPatch, Host: host, Path: rawPath, Query: rawQuery,
-		Protocol: "HTTP/1.1", Status: http.StatusTeapot,
-		UpstreamStatus: http.StatusTeapot, RequestBytes: int64(sent),
+	want := withTimings(line, requestlog.Line{
+		Type: requestType, Time: line.Time, Instance: "app",
+		ClientIP: localhost, Method: http.MethodPatch, Scheme: plain, Host: host,
+		Path: rawPath, Query: rawQuery, Protocol: protocol,
+		Status: http.StatusTeapot, RequestBytes: int64(sent),
 		ResponseBytes: int64(received), UserAgent: "test-agent",
-		Action: requestlog.ActionForward, DurationTotal: line.DurationTotal,
-		DurationUpstreamTotal: line.DurationUpstreamTotal,
-	}
-	if line.Line != want {
+		RequestID: line.RequestID, PeerIP: localhost, ClientGroup: localhost + "/32",
+		ContentLength: int64(sent), ResponseContentType: "text/plain; charset=utf-8",
+		UpstreamStatus: http.StatusTeapot, Action: requestlog.ActionForward,
+		Counts: ratelimit.Counts{Minute: 1, Hour: 1, Day: 1},
+	})
+	if !reflect.DeepEqual(line.Line, want) {
 		t.Errorf("log line\n%+v\nwant\n%+v", line.Line, want)
 	}
 
 	_, err := time.Parse(time.RFC3339, line.Time)
-	if err != nil || line.DurationTotal <= 0 || line.DurationUpstreamTotal <= 0 {
-		t.Errorf("log line has time %q and durations %v and %v",
-			line.Time, line.DurationTotal, line.DurationUpstreamTotal)
+	if err != nil || line.RequestID == "" || line.DurationTotal <= 0 ||
+		line.DurationUpstreamTotal == nil || *line.DurationUpstreamTotal <= 0 {
+		t.Errorf("log line has time %q, request_id %q and durations %v and %v",
+			line.Time, line.RequestID, line.DurationTotal,
+			line.fields["duration_upstream_total"])
 	}
 }
 
@@ -308,7 +315,7 @@ func TestServerHasTheDefaultLimits(t *testing.T) {
 	server := proxy.New(proxy.Params{
 		Config:     cfg,
 		RequestLog: io.Discard,
-		ProcessLog: requestlog.NewProcessLogger(io.Discard),
+		ProcessLog: requestlog.NewProcessLogger(io.Discard, cfg.InstanceName),
 	})
 
 	if server.Addr != ":8080" || server.MaxHeaderBytes != 28<<10 ||
@@ -371,8 +378,13 @@ func TestAnswers502WhenTheAppCannotBeReached(t *testing.T) {
 	addr, out := startProxy(t, "http://"+localhost+":1", nil)
 
 	wantStatus(t, get(t, addr, "/"), http.StatusBadGateway)
-	wantLine(t, out.requestLine(t), http.StatusBadGateway,
-		requestlog.ActionUpstreamError)
+
+	line := out.requestLine(t)
+	wantLine(t, line, http.StatusBadGateway, requestlog.ActionUpstreamError)
+
+	// There never was a connection to the app, nor an answer from it.
+	wantTimings(t, line, "duration_total", "duration_checks",
+		"duration_upstream_total")
 
 	logged := slices.ContainsFunc(out.lines(t), func(line map[string]any) bool {
 		return line["type"] == "process" && line["msg"] == "request to the app failed"

internal/proxy/proxy.go

@@ -11,12 +11,14 @@ import (
 	"strings"
 	"time"
 
+	"sneak.berlin/go/smallwebwaf/internal/alerts"
 	"sneak.berlin/go/smallwebwaf/internal/bans"
 	"sneak.berlin/go/smallwebwaf/internal/config"
 	"sneak.berlin/go/smallwebwaf/internal/lookup"
 	"sneak.berlin/go/smallwebwaf/internal/metrics"
 	"sneak.berlin/go/smallwebwaf/internal/ratelimit"
 	"sneak.berlin/go/smallwebwaf/internal/requestlog"
+	"sneak.berlin/go/smallwebwaf/internal/rules"
 )
 
 // How smallwebwaf keeps connections to the app open between requests.
@@ -37,6 +39,14 @@ const HealthPath = "/_smallwebwaf/healthz"
 // SWWAF_METRICS_TOKEN.
 const MetricsPath = "/_smallwebwaf/metrics"
 
+// BansPath is where an admin lists and adds bans, and, followed by / and
+// a client's address, lifts them, with SWWAF_ADMIN_TOKEN.
+const BansPath = "/_smallwebwaf/bans"
+
+// ClientsPath is where an admin asks what smallwebwaf knows of a client,
+// by the client's address after it, with SWWAF_ADMIN_TOKEN.
+const ClientsPath = "/_smallwebwaf/clients/"
+
 // Params are what New needs.
 type Params struct {
 	Config *config.Config
@@ -51,6 +61,12 @@ type Params struct {
 	// limits, bans are made and run out, and GeoJS's answers are kept,
 	// normally time.Now in UTC, the time the state files give.
 	Now func() time.Time
+	// Rules are the rule files' rules, which each request is checked
+	// against.
+	Rules *rules.Files
+	// Alerts receive the alert for each ban the proxy makes or makes
+	// permanent, and for GeoJS failing.
+	Alerts *alerts.Queue
 }
 
 // Server is the server smallwebwaf runs, with the parts of the proxy
@@ -72,7 +88,7 @@ type Server struct {
 // applies the timeouts and size limits from then on.
 func New(params Params) *Server {
 	errorLog := slog.NewLogLogger(params.ProcessLog.Handler(), slog.LevelWarn)
-	m := metrics.New(params.Config.MetricsTopN)
+	m := metrics.New(params.Config.MetricsTopN, params.Config.InstanceName)
 	h := &handler{
 		config:     params.Config,
 		requestLog: params.RequestLog,
@@ -90,6 +106,7 @@ func New(params Params) *Server {
 			LimitBanDuration:     params.Config.LimitBanDuration,
 			LimitBanRepeatWindow: params.Config.LimitBanRepeatWindow,
 			MaxBanDuration:       params.Config.MaxBanDuration,
+			AttackBanDuration:    params.Config.AttackBanDuration,
 			MaxBans:              params.Config.MaxBans,
 		}),
 		geojs: lookup.New(lookup.Params{
@@ -97,9 +114,13 @@ func New(params Params) *Server {
 			Now:        params.Now,
 			ProcessLog: params.ProcessLog,
 			Metrics:    m,
+			Alerts:     params.Alerts,
 		}),
+		rules:  params.Rules,
+		alerts: params.Alerts,
 	}
 	m.AddBansAndClients(h.ledger, h.limiter, params.Now)
+	m.AddRules(params.Rules)
 
 	return &Server{
 		Server: &http.Server{
@@ -134,6 +155,8 @@ type handler struct {
 	limiter    *ratelimit.Limiter
 	ledger     *bans.Ledger
 	geojs      *lookup.GeoJS
+	rules      *rules.Files
+	alerts     *alerts.Queue
 }
 
 // newTransport returns what carries requests to the app. It never goes
@@ -160,6 +183,9 @@ func (h *handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 	// a health checker is never refused. It does not ask the app.
 	if r.Method == http.MethodGet && r.URL.Path == HealthPath {
 		rq.line.Action = requestlog.ActionAdmin
+		// Set here rather than left to Go's server, which would set it only
+		// after the log line has taken the response's headers.
+		rq.out.Header().Set("Content-Type", "text/plain; charset=utf-8")
 		_, _ = io.WriteString(rq.out, "ok\n")
 
 		return
@@ -169,6 +195,8 @@ func (h *handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 	defer rq.addToHistory()
 
 	refused := rq.check(r.Context())
+	rq.checked = time.Now()
+
 	if refused != nil {
 		rq.answer(*refused)
 

internal/proxy/proxy_test.go

@@ -14,9 +14,11 @@ import (
 	"testing"
 	"time"
 
+	"sneak.berlin/go/smallwebwaf/internal/alerts"
 	"sneak.berlin/go/smallwebwaf/internal/config"
 	"sneak.berlin/go/smallwebwaf/internal/proxy"
 	"sneak.berlin/go/smallwebwaf/internal/requestlog"
+	"sneak.berlin/go/smallwebwaf/internal/rules"
 )
 
 const (
@@ -35,6 +37,10 @@ const (
 	// localhost is where every test server listens, and so the address
 	// smallwebwaf sees each test's requests come from.
 	localhost = "127.0.0.1"
+	// requestType is the type that marks a request log line.
+	requestType = "request"
+	// protocol is the protocol of every test's requests.
+	protocol = "HTTP/1.1"
 )
 
 // shortTimeoutSetting is shortTimeout as a setting's value.
@@ -50,6 +56,7 @@ const (
 	clientResponseTimeout   = "SWWAF_CLIENT_RESPONSE_TIMEOUT"
 	upstreamRequestTimeout  = "SWWAF_UPSTREAM_REQUEST_TIMEOUT"
 	upstreamResponseTimeout = "SWWAF_UPSTREAM_RESPONSE_TIMEOUT"
+	mode                    = "SWWAF_MODE"
 	requestMaxBytes         = "SWWAF_REQUEST_MAX_BYTES"
 	responseMaxBytes        = "SWWAF_RESPONSE_MAX_BYTES"
 	trustedProxies          = "SWWAF_TRUSTED_PROXIES"
@@ -67,6 +74,10 @@ const (
 	maxBanDuration          = "SWWAF_MAX_BAN_DURATION"
 	maxBans                 = "SWWAF_MAX_BANS"
 	banScopeV4Prefix        = "SWWAF_BAN_SCOPE_V4_PREFIX"
+	instanceName            = "SWWAF_INSTANCE_NAME"
+	logRequestHeaders       = "SWWAF_LOG_REQUEST_HEADERS"
+	attackBanDuration       = "SWWAF_ATTACK_BAN_DURATION"
+	rulesDir                = "SWWAF_RULES_DIR"
 )
 
 // output collects what smallwebwaf writes on stdout.
@@ -83,6 +94,14 @@ func (o *output) Write(p []byte) (int, error) {
 	return o.buf.Write(p)
 }
 
+// text returns everything written so far.
+func (o *output) text() string {
+	o.mu.Lock()
+	defer o.mu.Unlock()
+
+	return o.buf.String()
+}
+
 // lines returns every line written so far, decoded.
 func (o *output) lines(t *testing.T) []map[string]any {
 	t.Helper()
@@ -122,7 +141,7 @@ func (o *output) requestLines(t *testing.T, count int) []logLine {
 		var found []logLine
 
 		for _, fields := range o.lines(t) {
-			if fields["type"] == "request" {
+			if fields["type"] == requestType {
 				found = append(found, decodeLine(t, fields))
 			}
 		}
@@ -197,14 +216,33 @@ func startProxyWithGeoJS(
 }
 
 // startProxyWithClock is startProxyWithGeoJS with requests counted and
-// bans made by the time now tells, and returns the server as well.
+// bans made by the time now tells, and returns the server as well. Unless
+// env sets SWWAF_RULES_DIR, it is an empty directory, of no rules, and
+// unless it sets SWWAF_INSTANCE_NAME, that is app, the label instance of
+// every metric.
 func startProxyWithClock(
 	t *testing.T, appURL, geojsURL string, now func() time.Time,
 	env map[string]string,
 ) (string, *output, *proxy.Server) {
 	t.Helper()
 
-	settings := map[string]string{"SWWAF_UPSTREAM_URL": appURL}
+	addr, out, server, _ := startProxyWithAlerts(t, appURL, geojsURL, now, env)
+
+	return addr, out, server
+}
+
+// startProxyWithAlerts is startProxyWithClock, and returns the queue of
+// the alerts the proxy raises as well, as the settings in env make it. No
+// alert is sent from it: they wait in it, for the test to look at.
+func startProxyWithAlerts(
+	t *testing.T, appURL, geojsURL string, now func() time.Time,
+	env map[string]string,
+) (string, *output, *proxy.Server, *alerts.Queue) {
+	t.Helper()
+
+	settings := map[string]string{
+		"SWWAF_UPSTREAM_URL": appURL, rulesDir: t.TempDir(), instanceName: "app",
+	}
 	maps.Copy(settings, env)
 
 	cfg, err := config.FromEnvironment(func(name string) (string, bool) {
@@ -217,12 +255,33 @@ func startProxyWithClock(
 	}
 
 	out := &output{}
+	processLog := requestlog.NewProcessLogger(out, cfg.InstanceName)
+
+	ruleFiles, err := rules.Load(rules.Params{
+		Dir: cfg.RulesDir, Enabled: cfg.RulesEnabled, ProcessLog: processLog,
+	})
+	if err != nil {
+		t.Fatalf("rule files: %v", err)
+	}
+
+	alertQueue := alerts.New(alerts.Params{
+		WebhookURL: cfg.AlertWebhookURL,
+		Events:     cfg.AlertEvents,
+		Cooldown:   cfg.AlertCooldown,
+		MaxPerHour: cfg.AlertMaxPerHour,
+		Instance:   cfg.InstanceName,
+		Now:        now,
+		ProcessLog: processLog,
+	})
+
 	server := proxy.New(proxy.Params{
 		Config:     cfg,
 		RequestLog: out,
-		ProcessLog: requestlog.NewProcessLogger(out),
+		ProcessLog: processLog,
 		GeoJSURL:   geojsURL,
 		Now:        now,
+		Rules:      ruleFiles,
+		Alerts:     alertQueue,
 	})
 
 	listener, err := (&net.ListenConfig{}).Listen(t.Context(), "tcp", localhost+":0")
@@ -238,7 +297,7 @@ func startProxyWithClock(
 		_ = server.Close()
 	})
 
-	return listener.Addr().String(), out, server
+	return listener.Addr().String(), out, server, alertQueue
 }
 
 // newClient returns an HTTP client that sends requests as they are made,

internal/proxy/ratelimits_test.go

@@ -5,6 +5,8 @@ import (
 	"sync/atomic"
 	"testing"
 
+	"sneak.berlin/go/smallwebwaf/internal/lookup"
+	"sneak.berlin/go/smallwebwaf/internal/ratelimit"
 	"sneak.berlin/go/smallwebwaf/internal/requestlog"
 )
 
@@ -75,10 +77,17 @@ func TestRateLimitExemptPathsAreNeitherCountedNorRefused(t *testing.T) {
 
 	const denied = "192.0.2.50" // in SWWAF_DENY_NETS
 
-	s, _, _ := startWithClock(t, "", map[string]string{
+	s, _, server := startWithClock(t, "", map[string]string{
 		rateLimitPerMinute:   "1",
 		rateLimitExemptPaths: "/assets/,/favicon.ico",
 		denyNets:             denied,
+		deniedCountries:      "kp",
+	})
+
+	// The answers are kept before the requests, so that none waits for
+	// GeoJS.
+	server.GeoJS.Load([]lookup.Answer{
+		keptAnswer(client, "DE"), keptAnswer(fromKP, "KP"),
 	})
 
 	// With a limit of one request a minute, the requests for paths under a
@@ -89,15 +98,62 @@ func TestRateLimitExemptPathsAreNeitherCountedNorRefused(t *testing.T) {
 	s.get(client, http.StatusOK, requestlog.ActionForward)
 
 	line := s.request(client, "/assets/app.js", http.StatusOK, requestlog.ActionForward)
-	if line.LimitHit != "" {
-		t.Errorf("log line has limit_hit %q, want none", line.LimitHit)
+	if line.LimitHit != "" || line.Counts != (ratelimit.Counts{}) {
+		t.Errorf("log line has limit_hit %q and counts %+v, want neither",
+			line.LimitHit, line.Counts)
 	}
 
 	// A path outside every prefix is counted: /assets is not under
 	// /assets/, and breaks the limit.
 	s.request(client, "/assets", http.StatusForbidden, requestlog.ActionRateLimited)
 
-	// A ban and SWWAF_DENY_NETS still refuse a path under a prefix.
+	// A ban, SWWAF_DENY_NETS and the country lists still refuse a path
+	// under a prefix.
 	s.request(client, "/assets/app.js", http.StatusForbidden, requestlog.ActionBanned)
 	s.request(denied, "/assets/app.js", http.StatusForbidden, requestlog.ActionDenied)
+	s.request(fromKP, "/assets/app.js",
+		http.StatusForbidden, requestlog.ActionCountryDenied)
+}
+
+func TestRateLimitCountsPathsThatAreNotExempt(t *testing.T) {
+	t.Parallel()
+
+	for _, sent := range []string{
+		// A prefix matches only at the start of the path.
+		"/static/assets/app.js",
+		// A prefix matches the path as sent: a router that matches the
+		// path as received does not take /%61ssets/x for a path under
+		// /assets/.
+		"/%61ssets/x",
+		// .. once percent-decoded: an app may act on these as /login, the
+		// last as a path under /sneak/app/ or as /assets/x.
+		"/assets/../login",
+		"/assets/%2e%2e/login",
+		"/assets/..%2Flogin",
+		"/assets/..;/login",
+		"/sneak/app/src/branch/main/..%2F..%2F..%2F..%2F..%2F..%2Fassets/x",
+		// Not under /assets/ as sent: Go's router takes /assets%2Fx for one
+		// path segment, not a path under /assets/.
+		"/assets%2Fx",
+		"/assets%2fx",
+		// Under /assets/ as sent, but holding an encoded slash, in either
+		// case, or a backslash: never exempt, whatever the prefix.
+		"/assets/x%2Fy",
+		"/assets/x%2fy",
+		`/assets/x\y`,
+	} {
+		t.Run(sent, func(t *testing.T) {
+			t.Parallel()
+
+			s, _, _ := startWithClock(t, "", map[string]string{
+				rateLimitPerMinute:   "1",
+				rateLimitExemptPaths: "/assets/",
+			})
+
+			// Counted, the second request breaks the limit of one request
+			// a minute.
+			s.request(client, sent, http.StatusOK, requestlog.ActionForward)
+			s.request(client, sent, http.StatusForbidden, requestlog.ActionRateLimited)
+		})
+	}
 }

internal/proxy/request.go

@@ -7,6 +7,7 @@ import (
 	"net/http/httptrace"
 	"net/http/httputil"
 	"net/netip"
+	"net/url"
 	"os"
 	"slices"
 	"strings"
@@ -48,7 +49,9 @@ type request struct {
 	peer        netip.Addr
 	peerTrusted bool
 	start       time.Time
-	// upstreamStart is when the request was handed to the app.
+	// checked is when the checks were done, and upstreamStart when the
+	// request was handed to the app.
+	checked       time.Time
 	upstreamStart time.Time
 	// cancel ends the request to the app.
 	cancel context.CancelFunc
@@ -58,26 +61,34 @@ type request struct {
 	complete bool
 
 	// mu guards what follows. The timeouts run on goroutines of their
-	// own, and the transport starts and stops them from its own; once
-	// timersStopped is set, none of them acts any more.
+	// own, and the transport starts and stops them, and notes the times
+	// below, from its own; once timersStopped is set, none of the timeouts
+	// acts any more.
 	mu                    sync.Mutex
 	timersStopped         bool
 	clientRequestTimer    *time.Timer
 	upstreamRequestTimer  *time.Timer
 	upstreamResponseTimer *time.Timer
-	// requestSent is when the app had been sent the whole request.
-	requestSent time.Time
+	// connected is when there was a connection to the app, requestSent
+	// when the app had been sent the whole request, and answerStarted
+	// when the first byte of its answer arrived.
+	connected     time.Time
+	requestSent   time.Time
+	answerStarted time.Time
 }
 
 // newRequest starts handling r: it notes the time, counts the request as
-// under way, and works out the client.
+// under way, works out the client, and starts the log line with what is
+// known of the request.
 func (h *handler) newRequest(w http.ResponseWriter, r *http.Request) *request {
 	h.metrics.RequestStarted()
 
 	start := time.Now()
 	peer := peerAddress(r)
 	trusted := h.config.TrustedProxies
-	client := clientAddress(peer, r.Header.Values("X-Forwarded-For"), trusted)
+	peerTrusted := isInside(peer, trusted)
+	forwardedFor := r.Header.Values("X-Forwarded-For")
+	client := clientAddress(peer, forwardedFor, trusted)
 
 	rq := &request{
 		h:           h,
@@ -86,22 +97,37 @@ func (h *handler) newRequest(w http.ResponseWriter, r *http.Request) *request {
 		out:         &responseWriter{ResponseWriter: w},
 		client:      client,
 		peer:        peer,
-		peerTrusted: isInside(peer, trusted),
+		peerTrusted: peerTrusted,
 		start:       start,
 		line: requestlog.Line{
-			Time:      requestlog.FormatTime(start),
-			ClientIP:  client.String(),
-			PeerIP:    peer.String(),
-			Method:    r.Method,
-			Host:      r.Host,
-			Path:      r.URL.EscapedPath(),
-			Query:     r.URL.RawQuery,
-			Protocol:  r.Proto,
-			Referer:   r.Referer(),
-			UserAgent: r.UserAgent(),
-			Action:    requestlog.ActionForward,
+			Time:             requestlog.FormatTime(start),
+			Instance:         h.config.InstanceName,
+			ClientIP:         client.String(),
+			Method:           r.Method,
+			Scheme:           scheme(r, peerTrusted),
+			Host:             r.Host,
+			Path:             r.URL.EscapedPath(),
+			Query:            r.URL.RawQuery,
+			Protocol:         r.Proto,
+			Referer:          r.Referer(),
+			UserAgent:        r.UserAgent(),
+			RequestID:        requestID(r, peerTrusted),
+			PeerIP:           peer.String(),
+			ForwardedFor:     strings.Join(forwardedFor, ", "),
+			ClientGroup:      clientGroup(client).String(),
+			ContentType:      r.Header.Get("Content-Type"),
+			RequestHeaders:   requestHeaders(r, h.config.LogRequestHeaders),
+			HasAuthorization: len(r.Header.Values("Authorization")) > 0,
+			HasCookie:        len(r.Header.Values("Cookie")) > 0,
+			Action:           requestlog.ActionForward,
 		},
 	}
+
+	// A length of -1 is a body whose length was not announced.
+	if r.ContentLength > 0 {
+		rq.line.ContentLength = r.ContentLength
+	}
+
 	if r.Body != http.NoBody {
 		rq.body = &requestBody{body: limitBody(r.Body, h.config.RequestMaxBytes), rq: rq}
 	}
@@ -109,43 +135,50 @@ func (h *handler) newRequest(w http.ResponseWriter, r *http.Request) *request {
 	return rq
 }
 
+// requestHeaders returns the headers of r that names lists, by name in
+// lower case, each with its values joined by ", ". Authorization, Cookie
+// and Set-Cookie are never among them, whatever names says.
+func requestHeaders(r *http.Request, names []string) map[string]string {
+	headers := map[string]string{}
+
+	for _, name := range names {
+		switch name {
+		case "authorization", "cookie", "set-cookie":
+			continue
+		}
+
+		values := r.Header.Values(name)
+		if len(values) > 0 {
+			headers[name] = strings.Join(values, ", ")
+		}
+	}
+
+	return headers
+}
+
 // check is the one place where a request can be refused once its client
 // is known, before its body is read or anything reaches the app. It
-// returns nil to let the request through. A client in SWWAF_ALLOW_NETS
-// skips every check but the size limit. For any other client,
-// SWWAF_DENY_NETS comes first, then a ban on its netblock, so that a
-// client either refuses is not looked up, and then the country lists; a
-// request any of them refuses is not counted for the rate limits. Then
-// come the rate limits, unless the client is in
-// SWWAF_RATE_LIMIT_EXEMPT_NETS or the request's path, as the client sent
-// it and the log line shows it, starts with one of
-// SWWAF_RATE_LIMIT_EXEMPT_PATHS, so that every other request is counted,
-// one refused for its size too. Every refusal but the size limit's is
-// answered with SWWAF_BAN_RESPONSE. ctx is the request's own context.
+// returns nil to let the request through. The checks of checkClient come
+// first, answered with SWWAF_BAN_RESPONSE, or 403 for a block rule, and
+// then the size limit, so that a request the rate limits count is counted
+// even when it is refused for its size. In observe mode a request
+// checkClient refuses goes on to the size limit like any other. ctx is
+// the request's own context.
 func (rq *request) check(ctx context.Context) *refusal {
-	cfg := rq.h.config
-	allowed := isInside(rq.client, cfg.AllowNets)
-	exempt := isInside(rq.client, cfg.RateLimitExemptNets) ||
-		startsWithAny(rq.in.URL.EscapedPath(), cfg.RateLimitExemptPaths)
-	now := rq.h.now()
+	action := rq.checkClient(ctx)
 
-	if !allowed && isInside(rq.client, cfg.DenyNets) {
-		return rq.banResponse(requestlog.ActionDenied)
+	switch {
+	case action == "":
+	case rq.h.config.Observe:
+		// The log line names what enforce mode would have done.
+		rq.line.WouldAction = action
+	case action == requestlog.ActionRuleBlocked:
+		return &refusal{status: http.StatusForbidden, action: action}
+	default:
+		return rq.banResponse(action)
 	}
 
-	if !allowed && rq.banned(now) {
-		return rq.banResponse(requestlog.ActionBanned)
-	}
-
-	if !allowed && rq.countryDenied(ctx) {
-		return rq.banResponse(requestlog.ActionCountryDenied)
-	}
-
-	if !allowed && !exempt && rq.limitBroken(now) {
-		return rq.banResponse(requestlog.ActionRateLimited)
-	}
-
-	maxBytes := cfg.RequestMaxBytes
+	maxBytes := rq.h.config.RequestMaxBytes
 	if maxBytes > 0 && rq.in.ContentLength > maxBytes {
 		return &refusal{
 			status: http.StatusRequestEntityTooLarge,
@@ -157,10 +190,66 @@ func (rq *request) check(ctx context.Context) *refusal {
 	return nil
 }
 
-// startsWithAny reports whether path starts with one of prefixes.
-func startsWithAny(path string, prefixes []string) bool {
+// checkClient runs the checks on the request's client, and returns the
+// action of the first that refuses the request, or "" when none does. A
+// client in SWWAF_ALLOW_NETS skips them. For any other client,
+// SWWAF_DENY_NETS comes first, then a ban on its netblock, so that a
+// client either refuses is not looked up, and then the country lists; a
+// request any of them refuses is not counted for the rate limits. Then
+// come the rate limits, unless the client is in
+// SWWAF_RATE_LIMIT_EXEMPT_NETS or the request's path is exempt under
+// SWWAF_RATE_LIMIT_EXEMPT_PATHS, so that every other request is counted,
+// and last the rule files. ctx is the request's own context.
+func (rq *request) checkClient(ctx context.Context) string {
+	cfg := rq.h.config
+	if isInside(rq.client, cfg.AllowNets) {
+		return ""
+	}
+
+	now := rq.h.now()
+
+	if isInside(rq.client, cfg.DenyNets) {
+		return requestlog.ActionDenied
+	}
+
+	if rq.banned(now) {
+		return requestlog.ActionBanned
+	}
+
+	if rq.countryDenied(ctx) {
+		return requestlog.ActionCountryDenied
+	}
+
+	exempt := isInside(rq.client, cfg.RateLimitExemptNets) ||
+		pathExempt(rq.in.URL, cfg.RateLimitExemptPaths)
+	if !exempt && rq.limitBroken(now) {
+		return requestlog.ActionRateLimited
+	}
+
+	return rq.checkRules(now)
+}
+
+// pathExempt reports whether the rate limits leave out a request for u
+// because of SWWAF_RATE_LIMIT_EXEMPT_PATHS: whether its path as sent, the
+// path the app receives, not percent-decoded, starts with one of
+// prefixes, so that /%61ssets/x is not under /assets/ for an app whose
+// router matches the path as received. A request whose decoded path
+// contains .. anywhere or a backslash, or whose path as sent holds an
+// encoded slash (%2F or %2f), never is, since an app may act on it as a
+// path outside every prefix: /assets/..%2Flogin as /login, or /assets%2Fx
+// as one path segment, as Go's router does.
+func pathExempt(u *url.URL, prefixes []string) bool {
+	decoded := u.Path
+	// EscapedPath is the path as the app receives it, not decoded.
+	sent := u.EscapedPath()
+
+	if strings.Contains(decoded, "..") || strings.Contains(decoded, `\`) ||
+		strings.Contains(strings.ToLower(sent), "%2f") {
+		return false
+	}
+
 	return slices.ContainsFunc(prefixes, func(prefix string) bool {
-		return strings.HasPrefix(path, prefix)
+		return strings.HasPrefix(sent, prefix)
 	})
 }
 
@@ -172,7 +261,9 @@ func (rq *request) forward(ctx context.Context) {
 
 	rq.cancel = cancel
 	ctx = httptrace.WithClientTrace(ctx, &httptrace.ClientTrace{
-		WroteRequest: rq.wroteRequest,
+		GotConn:              rq.gotConn,
+		WroteRequest:         rq.wroteRequest,
+		GotFirstResponseByte: rq.gotFirstResponseByte,
 	})
 
 	out := rq.in.WithContext(ctx)
@@ -195,7 +286,8 @@ func (rq *request) forward(ctx context.Context) {
 }
 
 // rewrite makes the request the app receives: the client's request,
-// unchanged, sent to SWWAF_UPSTREAM_URL, with the forwarded headers set.
+// unchanged, sent to SWWAF_UPSTREAM_URL, with the forwarded headers and
+// the request's id set.
 func (rq *request) rewrite(pr *httputil.ProxyRequest) {
 	upstream := rq.h.config.UpstreamURL
 	pr.Out.URL.Scheme = upstream.Scheme
@@ -204,6 +296,7 @@ func (rq *request) rewrite(pr *httputil.ProxyRequest) {
 	// the query as the client sent it.
 	pr.Out.URL.RawQuery = pr.In.URL.RawQuery
 	setForwardedHeaders(pr.In, pr.Out, rq.peer, rq.peerTrusted)
+	pr.Out.Header.Set(requestIDHeader, rq.line.RequestID)
 }
 
 // modifyResponse looks at the app's answer before ReverseProxy passes it
@@ -217,6 +310,7 @@ func (rq *request) modifyResponse(res *http.Response) error {
 		// connection it takes over, not through rq.out.
 		rq.stopTimers()
 		rq.out.status = res.StatusCode
+		rq.line.Websocket = true
 
 		return nil
 	}
@@ -293,10 +387,14 @@ func (rq *request) answer(r refusal) {
 }
 
 // refuse records r, unless an earlier refusal was, and ends the request
-// to the app.
+// to the app, if one was made: smallwebwaf reads the body of a request
+// it answers itself too.
 func (rq *request) refuse(r refusal) {
 	rq.refused.CompareAndSwap(nil, &r)
-	rq.cancel()
+
+	if rq.cancel != nil {
+		rq.cancel()
+	}
 }
 
 // finish ends the request's timeouts, counts it in the metrics and writes
@@ -312,6 +410,10 @@ func (rq *request) finish() {
 	line := &rq.line
 	line.Status = rq.out.status
 	line.ResponseBytes = rq.out.bytes
+	header := rq.out.Header()
+	line.ResponseContentType = header.Get("Content-Type")
+	line.CacheControl = header.Get("Cache-Control")
+	line.Location = header.Get("Location")
 
 	if rq.body != nil {
 		line.RequestBytes = rq.body.bytes.Load()
@@ -336,12 +438,18 @@ func (rq *request) finish() {
 	now := time.Now()
 	duration := now.Sub(rq.start)
 	line.DurationTotal = requestlog.Milliseconds(duration)
+	line.DurationChecks = timing(rq.start, rq.checked)
 
 	var upstreamDuration time.Duration
 
 	if !rq.upstreamStart.IsZero() {
 		upstreamDuration = now.Sub(rq.upstreamStart)
-		line.DurationUpstreamTotal = requestlog.Milliseconds(upstreamDuration)
+		line.DurationUpstreamTotal = new(requestlog.Milliseconds(upstreamDuration))
+
+		rq.mu.Lock()
+		line.DurationUpstreamConnect = timing(rq.upstreamStart, rq.connected)
+		line.DurationUpstreamFirstByte = timing(rq.upstreamStart, rq.answerStarted)
+		rq.mu.Unlock()
 	}
 
 	// Counted before the log line is written, so that the metrics count
@@ -354,6 +462,17 @@ func (rq *request) finish() {
 	}
 }
 
+// timing is the time from start to end in milliseconds, for one of the
+// log line's timings, or nil when end is zero: what it times never
+// happened.
+func timing(start, end time.Time) *float64 {
+	if end.IsZero() {
+		return nil
+	}
+
+	return new(requestlog.Milliseconds(end.Sub(start)))
+}
+
 // addToHistory adds the request, which has ended, to its client's
 // history.
 func (rq *request) addToHistory() {
@@ -463,6 +582,24 @@ func (rq *request) bodyReceived() {
 	stopTimer(rq.clientRequestTimer)
 }
 
+// gotConn is called once there is a connection to the app, a new one or
+// one kept open from an earlier request.
+func (rq *request) gotConn(httptrace.GotConnInfo) {
+	rq.mu.Lock()
+	defer rq.mu.Unlock()
+
+	rq.connected = time.Now()
+}
+
+// gotFirstResponseByte is called once the first byte of the app's answer
+// has arrived.
+func (rq *request) gotFirstResponseByte() {
+	rq.mu.Lock()
+	defer rq.mu.Unlock()
+
+	rq.answerStarted = time.Now()
+}
+
 // wroteRequest is called once the app has been sent the whole request:
 // the request timeouts end and SWWAF_UPSTREAM_RESPONSE_TIMEOUT starts.
 func (rq *request) wroteRequest(info httptrace.WroteRequestInfo) {

internal/proxy/requestlog_test.go

@@ -0,0 +1,368 @@
+package proxy_test
+
+import (
+	"io"
+	"maps"
+	"math"
+	"net/http"
+	"reflect"
+	"slices"
+	"strings"
+	"testing"
+	"time"
+
+	"sneak.berlin/go/smallwebwaf/internal/proxy"
+	"sneak.berlin/go/smallwebwaf/internal/ratelimit"
+	"sneak.berlin/go/smallwebwaf/internal/requestlog"
+)
+
+const (
+	// requestIDHeader carries the request's id.
+	requestIDHeader = "X-Request-ID"
+	// instance is the SWWAF_INSTANCE_NAME a test sets.
+	instance = "fsn1app1/gitea"
+	// ipv6Client is a client on IPv6, and ipv6Group the netblock the rate
+	// limits count it as.
+	ipv6Client = "2001:db8::7"
+	ipv6Group  = "2001:db8::/64"
+)
+
+func TestLogLineHasEachFieldWhereItApplies(t *testing.T) {
+	t.Parallel()
+
+	received := make(chan string, 2) // the request ids the app received
+	app := startApp(t, func(w http.ResponseWriter, r *http.Request) {
+		received <- r.Header.Get(requestIDHeader)
+
+		_, _ = io.Copy(io.Discard, r.Body)
+
+		if r.URL.Path != "/full" {
+			w.WriteHeader(http.StatusNoContent)
+
+			return
+		}
+
+		w.Header().Set("Content-Type", "text/html")
+		w.Header().Set("Cache-Control", "no-store")
+		w.Header().Set("Location", "/elsewhere")
+		w.WriteHeader(http.StatusFound)
+		_, _ = io.WriteString(w, "moved")
+	})
+	addr, out := startProxy(t, app.URL, map[string]string{
+		trustedProxies:      trustLocalhost,
+		rateLimitExemptNets: localhost,
+		instanceName:        instance,
+		logRequestHeaders:   "Accept,x-custom,Authorization,cookie,SET-COOKIE",
+	})
+
+	// This request comes from ipv6Client through a trusted proxy, with a
+	// body and each header the log line looks at, and is answered with a
+	// redirect.
+	conn := dial(t, addr)
+	send(t, conn, "POST /full HTTP/1.1\r\nHost: "+appHost+"\r\n"+
+		forwardedFor+": 198.51.100.7, "+ipv6Client+"\r\n"+
+		forwardedProto+": "+secure+"\r\n"+requestIDHeader+": from-traefik\r\n"+
+		"Content-Type: application/x-www-form-urlencoded\r\nContent-Length: 3\r\n"+
+		"Accept: text/html\r\nX-Custom: one\r\nX-Custom: two\r\n"+
+		"Authorization: Bearer secret-token\r\nCookie: session=secret-cookie\r\n"+
+		"Set-Cookie: secret-set-cookie\r\n\r\na=b")
+	wantStatus(t, readResponse(t, conn), http.StatusFound)
+
+	// A request's log line can come after its answer: each is waited for
+	// before the next request, so that the lines are in order.
+	full := out.requestLines(t, 1)[0]
+
+	// This one comes from 127.0.0.1, which the rate limits do not count,
+	// with a body of 4 bytes whose length it does not announce, so that its
+	// request_bytes is not its content_length, and no header the log line
+	// looks at, and is answered with 204 and no header.
+	conn = dial(t, addr)
+	send(t, conn, "POST /bare HTTP/1.1\r\nHost: "+appHost+"\r\n"+
+		"Transfer-Encoding: chunked\r\n\r\n4\r\nbody\r\n0\r\n\r\n")
+	wantStatus(t, readResponse(t, conn), http.StatusNoContent)
+
+	bare := out.requestLines(t, 2)[1]
+
+	wantFullLine(t, full)
+	wantBareLine(t, bare)
+
+	for _, line := range []logLine{full, bare} {
+		got := <-received
+		if got != line.RequestID {
+			t.Errorf("the app received request id %q, the log line has %q",
+				got, line.RequestID)
+		}
+	}
+
+	if strings.Contains(out.text(), "secret") {
+		t.Errorf("a value of Authorization, Cookie or Set-Cookie is logged:\n%s",
+			out.text())
+	}
+}
+
+// wantFullLine checks the log line of the request with every header the
+// line looks at. Its timings are checked by TestTimingsAreInOrder.
+func wantFullLine(t *testing.T, line logLine) {
+	t.Helper()
+
+	headers := map[string]string{"accept": "text/html", "x-custom": "one, two"}
+
+	want := withTimings(line, requestlog.Line{
+		Type: requestType, Time: line.Time, Instance: instance,
+		ClientIP: ipv6Client, Method: http.MethodPost, Scheme: secure,
+		Host: appHost, Path: "/full", Protocol: protocol,
+		Status: http.StatusFound, RequestBytes: 3, ResponseBytes: 5,
+		RequestID: "from-traefik", PeerIP: localhost,
+		ForwardedFor: "198.51.100.7, " + ipv6Client, ClientGroup: ipv6Group,
+		ContentType: "application/x-www-form-urlencoded", ContentLength: 3,
+		RequestHeaders: headers, HasAuthorization: true, HasCookie: true,
+		ResponseContentType: "text/html", UpstreamStatus: http.StatusFound,
+		CacheControl: "no-store", Location: "/elsewhere",
+		Action: requestlog.ActionForward,
+		Counts: ratelimit.Counts{Minute: 1, Hour: 1, Day: 1},
+	})
+	if !reflect.DeepEqual(line.Line, want) {
+		t.Errorf("log line\n%+v\nwant\n%+v", line.Line, want)
+	}
+}
+
+// wantBareLine checks the log line of the request with none of them, and
+// that the fields that do not apply to it are left out.
+func wantBareLine(t *testing.T, line logLine) {
+	t.Helper()
+
+	want := withTimings(line, requestlog.Line{
+		Type: requestType, Time: line.Time, Instance: instance,
+		ClientIP: localhost, Method: http.MethodPost, Scheme: plain,
+		Host: appHost, Path: "/bare", Protocol: protocol,
+		Status: http.StatusNoContent, RequestBytes: 4, RequestID: line.RequestID,
+		PeerIP: localhost, ClientGroup: localhost + "/32",
+		UpstreamStatus: http.StatusNoContent, Action: requestlog.ActionForward,
+	})
+	if !reflect.DeepEqual(line.Line, want) || line.RequestID == "" {
+		t.Errorf("log line\n%+v\nwant\n%+v, with a request id", line.Line, want)
+	}
+
+	for _, name := range []string{
+		"forwarded_for", "content_type", "content_length", "request_headers",
+		"has_authorization", "has_cookie", "websocket", "response_content_type",
+		"cache_control", "location", "counts",
+	} {
+		_, present := line.fields[name]
+		if present {
+			t.Errorf("log line has %s, which does not apply", name)
+		}
+	}
+}
+
+// withTimings returns want with the timings of line.
+func withTimings(line logLine, want requestlog.Line) requestlog.Line {
+	want.DurationTotal = line.DurationTotal
+	want.DurationChecks = line.DurationChecks
+	want.DurationUpstreamConnect = line.DurationUpstreamConnect
+	want.DurationUpstreamFirstByte = line.DurationUpstreamFirstByte
+	want.DurationUpstreamTotal = line.DurationUpstreamTotal
+
+	return want
+}
+
+func TestHasAuthorizationAndHasCookieEachComeFromTheirOwnHeader(t *testing.T) {
+	t.Parallel()
+
+	const hasAuthorization, hasCookie = "has_authorization", "has_cookie"
+
+	for _, tc := range []struct{ header, field, other string }{
+		{"Authorization", hasAuthorization, hasCookie},
+		{"Cookie", hasCookie, hasAuthorization},
+	} {
+		t.Run("only "+tc.header, func(t *testing.T) {
+			t.Parallel()
+
+			app := startApp(t, func(http.ResponseWriter, *http.Request) {})
+			addr, out := startProxy(t, app.URL, nil)
+
+			req := newRequest(t, http.MethodGet, addr, "/", http.NoBody)
+			req.Header.Set(tc.header, "secret")
+			wantStatus(t, do(t, req), http.StatusOK)
+
+			line := out.requestLine(t)
+
+			_, otherPresent := line.fields[tc.other]
+			if line.fields[tc.field] != true || otherPresent {
+				t.Errorf("log line has %s %v and %s %v, want true and none",
+					tc.field, line.fields[tc.field], tc.other, line.fields[tc.other])
+			}
+		})
+	}
+}
+
+func TestRequestIDAndSchemeComeOnlyFromATrustedProxy(t *testing.T) {
+	t.Parallel()
+
+	const sentID = "from-traefik"
+
+	sent := http.Header{requestIDHeader: {sentID}, forwardedProto: {secure}}
+	trusted := map[string]string{trustedProxies: trustLocalhost}
+
+	for _, tc := range []struct {
+		name   string
+		env    map[string]string
+		header http.Header
+		// wantID is the request id logged, "" for a new one.
+		wantID, wantScheme string
+	}{
+		{"a trusted proxy's are kept", trusted, sent, sentID, secure},
+		{"without them, the id is new and the scheme http", trusted, nil, "", plain},
+		{"another peer's are replaced", nil, sent, "", plain},
+	} {
+		t.Run(tc.name, func(t *testing.T) {
+			t.Parallel()
+
+			received := make(chan string, 2)
+			app := startApp(t, func(_ http.ResponseWriter, r *http.Request) {
+				received <- r.Header.Get(requestIDHeader)
+			})
+			addr, out := startProxy(t, app.URL, tc.env)
+
+			// Two requests, so that two new ids can be told apart.
+			ids := make([]string, 0, 2)
+
+			for i := range 2 {
+				req := newRequest(t, http.MethodGet, addr, "/", http.NoBody)
+				maps.Copy(req.Header, tc.header)
+				wantStatus(t, do(t, req), http.StatusOK)
+
+				line := out.requestLines(t, i+1)[i]
+				ids = append(ids, line.RequestID)
+
+				got := <-received
+				if line.RequestID != got || line.Scheme != tc.wantScheme {
+					t.Errorf("log line has request_id %q and scheme %q, and the "+
+						"app received id %q; want the same id and scheme %q",
+						line.RequestID, line.Scheme, got, tc.wantScheme)
+				}
+			}
+
+			switch {
+			case tc.wantID != "" && (ids[0] != tc.wantID || ids[1] != tc.wantID):
+				t.Errorf("request ids %q, want %q", ids, tc.wantID)
+			case tc.wantID == "" && (slices.Contains(ids, sentID) ||
+				slices.Contains(ids, "") || ids[0] == ids[1]):
+				t.Errorf("request ids %q, want two new ones", ids)
+			}
+		})
+	}
+}
+
+func TestTimingsAreInOrder(t *testing.T) {
+	t.Parallel()
+
+	const denied = "192.0.2.50" // in SWWAF_DENY_NETS
+
+	app := startApp(t, func(w http.ResponseWriter, _ *http.Request) {
+		// The pauses set the times apart; a hold-up of the test only
+		// lengthens them.
+		time.Sleep(time.Millisecond)
+		w.WriteHeader(http.StatusOK)
+		_ = http.NewResponseController(w).Flush()
+
+		time.Sleep(time.Millisecond)
+
+		_, _ = io.WriteString(w, "done")
+	})
+	addr, out := startProxy(t, app.URL, map[string]string{
+		trustedProxies: trustLocalhost,
+		denyNets:       denied,
+	})
+
+	// Each log line is waited for before the next request, so that the
+	// lines are in order.
+	wantStatus(t, get(t, addr, "/"), http.StatusOK)
+	forwarded := out.requestLines(t, 1)[0]
+
+	req := newRequest(t, http.MethodGet, addr, "/", http.NoBody)
+	req.Header.Set(forwardedFor, denied)
+	wantStatus(t, do(t, req), http.StatusForbidden)
+	refused := out.requestLines(t, 2)[1]
+
+	wantStatus(t, get(t, addr, proxy.HealthPath), http.StatusOK)
+	health := out.requestLines(t, 3)[2]
+
+	// A request passed to the app has every timing; one refused, none of
+	// the app's; the health check, which runs no check, only the total.
+	wantTimings(t, forwarded, "duration_total", "duration_checks",
+		"duration_upstream_connect", "duration_upstream_first_byte",
+		"duration_upstream_total")
+	wantTimings(t, refused, "duration_total", "duration_checks")
+	wantTimings(t, health, "duration_total")
+
+	if t.Failed() {
+		return
+	}
+
+	// In whole microseconds, as they are logged, so that the sum below is
+	// exact.
+	total := microseconds(forwarded.DurationTotal)
+	checks := microseconds(*forwarded.DurationChecks)
+	connect := microseconds(*forwarded.DurationUpstreamConnect)
+	firstByte := microseconds(*forwarded.DurationUpstreamFirstByte)
+	upstream := microseconds(*forwarded.DurationUpstreamTotal)
+
+	// The checks end before the request is handed to the app, and the
+	// connection comes before the answer, which the app ends after a
+	// pause.
+	if checks+upstream > total || connect >= firstByte || firstByte >= upstream {
+		t.Errorf("timings in microseconds: total %d, checks %d, connect %d, "+
+			"first byte %d, upstream total %d", total, checks, connect, firstByte,
+			upstream)
+	}
+
+	if *refused.DurationChecks > refused.DurationTotal {
+		t.Errorf("refused request's checks took %v of %v milliseconds",
+			*refused.DurationChecks, refused.DurationTotal)
+	}
+}
+
+// wantTimings checks that the timings named are the only ones line has.
+func wantTimings(t *testing.T, line logLine, want ...string) {
+	t.Helper()
+
+	var got []string
+
+	for name := range line.fields {
+		if strings.HasPrefix(name, "duration_") {
+			got = append(got, name)
+		}
+	}
+
+	slices.Sort(got)
+	slices.Sort(want)
+
+	if !slices.Equal(got, want) {
+		t.Errorf("log line of %s has timings %v, want %v", line.Path, got, want)
+	}
+}
+
+// microseconds is a timing in whole microseconds.
+func microseconds(milliseconds float64) int64 {
+	return int64(math.Round(milliseconds * 1000))
+}
+
+func TestLogsAnUpgradedConnection(t *testing.T) {
+	t.Parallel()
+
+	app := startApp(t, echoAfterUpgrade)
+	addr, out := startProxy(t, app.URL, nil)
+
+	conn := dial(t, addr)
+	send(t, conn, "GET /socket HTTP/1.1\r\nHost: app\r\n"+
+		"Connection: Upgrade\r\nUpgrade: websocket\r\n\r\n")
+	wantStatus(t, readResponse(t, conn), http.StatusSwitchingProtocols)
+
+	_ = conn.Close()
+
+	line := out.requestLine(t)
+	if line.fields["websocket"] != true {
+		t.Errorf("log line has websocket %v, want true", line.fields["websocket"])
+	}
+}

internal/proxy/rulefiles_test.go

@@ -0,0 +1,233 @@
+package proxy_test
+
+import (
+	"net/http"
+	"net/netip"
+	"os"
+	"path/filepath"
+	"slices"
+	"testing"
+	"time"
+
+	"sneak.berlin/go/smallwebwaf/internal/bans"
+	"sneak.berlin/go/smallwebwaf/internal/requestlog"
+)
+
+// testRules are the rules most tests here load: a block rule for
+// /blocked and a ban rule for /.env.
+const testRules = `
+blocked  path  block  ^/blocked$
+probe    path  ban    ^/\.env$
+`
+
+func TestEachRuleAction(t *testing.T) {
+	t.Parallel()
+
+	s, clk, server := startWithClock(t, "", map[string]string{
+		rulesDir:    writeRules(t, "noted path log ^/\n"+testRules),
+		banResponse: "429",
+	})
+	start := clk.Now()
+
+	// A log rule notes its match, and lets the request through.
+	line := s.get(client, http.StatusOK, requestlog.ActionForward)
+	wantRuleIDs(t, line, "noted")
+
+	// A block rule refuses with 403, whatever SWWAF_BAN_RESPONSE is, and
+	// bans no one.
+	line = s.request(client, "/blocked", http.StatusForbidden,
+		requestlog.ActionRuleBlocked)
+	wantRuleIDs(t, line, "noted", "blocked")
+	s.get(client, http.StatusOK, requestlog.ActionForward)
+
+	// A ban rule refuses with SWWAF_BAN_RESPONSE, and bans the client for
+	// seven days, the default.
+	line = s.request(client, "/.env", http.StatusTooManyRequests, requestlog.ActionBanned)
+	wantRuleIDs(t, line, "noted", "probe")
+
+	if line.BanExpires != requestlog.FormatTime(start.Add(7*24*time.Hour)) {
+		t.Errorf("log line has ban_expires %q, want seven days on", line.BanExpires)
+	}
+
+	netblock := netip.MustParsePrefix(client + "/32")
+	want := bans.Ban{
+		Netblock: netblock,
+		Start:    start,
+		Expires:  start.Add(7 * 24 * time.Hour),
+		Cause:    bans.CauseAttack,
+		Reason:   "matched the rule probe",
+		Notes: bans.Notes{
+			RuleID: "probe",
+			Target: "path",
+			Request: bans.Request{
+				Time:      start,
+				Method:    http.MethodGet,
+				Host:      appHost,
+				Path:      "/.env",
+				Status:    http.StatusTooManyRequests,
+				UserAgent: userAgent,
+			},
+			// The four requests up to and including the probe.
+			Requests: 4,
+		},
+	}
+
+	got := server.Ledger.Bans(netblock)
+	if len(got) != 1 || got[0] != want {
+		t.Fatalf("bans\n%+v\nwant\n%+v", got, want)
+	}
+
+	// The next request is refused under the ban, without being checked
+	// against the rules, and makes the ban permanent.
+	clk.advance(time.Hour)
+
+	line = s.get(client, http.StatusTooManyRequests, requestlog.ActionBanned)
+	wantRuleIDs(t, line)
+
+	if line.BanExpires != permanent {
+		t.Errorf("log line has ban_expires %q, want permanent", line.BanExpires)
+	}
+
+	clk.advance(365 * 24 * time.Hour)
+	s.get(client, http.StatusTooManyRequests, requestlog.ActionBanned)
+}
+
+func TestNextClearSignOfAttackAfterABanBansPermanently(t *testing.T) {
+	t.Parallel()
+
+	s, clk, _ := startWithClock(t, "", map[string]string{
+		rulesDir:          writeRules(t, testRules),
+		attackBanDuration: "1h",
+	})
+
+	// The first probe bans for SWWAF_ATTACK_BAN_DURATION.
+	line := s.request(client, "/.env", http.StatusForbidden, requestlog.ActionBanned)
+	if line.BanExpires != requestlog.FormatTime(clk.Now().Add(time.Hour)) {
+		t.Errorf("log line has ban_expires %q, want an hour on", line.BanExpires)
+	}
+
+	// Once that ban has run out without a request, the client is served,
+	// and its next probe bans it for good.
+	clk.advance(time.Hour)
+	s.get(client, http.StatusOK, requestlog.ActionForward)
+
+	line = s.request(client, "/.env", http.StatusForbidden, requestlog.ActionBanned)
+	if line.BanExpires != permanent {
+		t.Errorf("log line has ban_expires %q, want permanent", line.BanExpires)
+	}
+}
+
+func TestRulesComeAfterTheOtherChecks(t *testing.T) {
+	t.Parallel()
+
+	const (
+		allowed = "192.0.2.60" // in SWWAF_ALLOW_NETS
+		exempt  = "192.0.2.50" // in SWWAF_RATE_LIMIT_EXEMPT_NETS
+	)
+
+	s, _, server := startWithClock(t, "", map[string]string{
+		rulesDir:            writeRules(t, testRules),
+		allowNets:           allowed,
+		rateLimitExemptNets: exempt,
+		rateLimitPerMinute:  "1",
+	})
+
+	// A client in SWWAF_ALLOW_NETS is not checked.
+	line := s.request(allowed, "/.env", http.StatusOK, requestlog.ActionForward)
+	wantRuleIDs(t, line)
+
+	// A probe over the rate limit breaks the limit before any rule sees
+	// it.
+	s.get(client, http.StatusOK, requestlog.ActionForward)
+	line = s.request(client, "/.env", http.StatusForbidden, requestlog.ActionRateLimited)
+	wantRuleIDs(t, line)
+
+	limitBan := server.Ledger.Bans(netip.MustParsePrefix(client + "/32"))
+	if len(limitBan) != 1 || limitBan[0].Cause != bans.CauseLimit {
+		t.Errorf("bans %+v, want one for a broken limit", limitBan)
+	}
+
+	// A client the rate limits do not apply to is still checked.
+	s.get(exempt, http.StatusOK, requestlog.ActionForward)
+	s.request(exempt, "/.env", http.StatusForbidden, requestlog.ActionBanned)
+}
+
+func TestObserveModeLogsWhatTheRulesWouldDo(t *testing.T) {
+	t.Parallel()
+
+	s, _, server := startWithClock(t, "", map[string]string{
+		rulesDir: writeRules(t, testRules),
+		mode:     observe,
+	})
+
+	line := s.request(client, "/blocked", http.StatusOK, requestlog.ActionForward)
+	wantWouldAction(t, line, requestlog.ActionRuleBlocked)
+	wantRuleIDs(t, line, "blocked")
+
+	line = s.request(client, "/.env", http.StatusOK, requestlog.ActionForward)
+	wantWouldAction(t, line, requestlog.ActionBanned)
+	wantRuleIDs(t, line, "probe")
+
+	if line.BanExpires != "" {
+		t.Errorf("log line has ban_expires %q, want none", line.BanExpires)
+	}
+
+	// No ban was made.
+	line = s.get(client, http.StatusOK, requestlog.ActionForward)
+	wantWouldAction(t, line, "")
+
+	if got := server.Ledger.Snapshot(); len(got) != 0 {
+		t.Errorf("bans %+v, want none", got)
+	}
+}
+
+func TestMetricsCountRuleMatchesAndBansForAnAttack(t *testing.T) {
+	t.Parallel()
+
+	const scraper = "192.0.2.200"
+
+	s, _, _ := startWithClock(t, "", map[string]string{
+		rulesDir:     writeRules(t, testRules),
+		metricsToken: token,
+	})
+
+	s.request(client, "/blocked", http.StatusForbidden, requestlog.ActionRuleBlocked)
+	s.request(client, "/.env", http.StatusForbidden, requestlog.ActionBanned)
+	s.get(client, http.StatusForbidden, requestlog.ActionBanned)
+
+	metrics := s.scrape(scraper)
+	wantMetric(t, metrics,
+		`smallwebwaf_rule_matches_total{action="block",instance="app",rule_id="blocked"}`, 1)
+	wantMetric(t, metrics,
+		`smallwebwaf_rule_matches_total{action="ban",instance="app",rule_id="probe"}`, 1)
+	wantMetric(t, metrics, `smallwebwaf_rules_loaded{instance="app"}`, 2)
+	wantMetric(t, metrics, `smallwebwaf_requests_total{action="rule_blocked",`+
+		`instance="app",status_class="4xx"}`, 1)
+	wantMetric(t, metrics, `smallwebwaf_bans_made_total{cause="attack",instance="app"}`, 1)
+	wantMetric(t, metrics, `smallwebwaf_bans_made_total{cause="limit",instance="app"}`, 0)
+	wantMetric(t, metrics, `smallwebwaf_permanent_bans{instance="app"}`, 1)
+}
+
+// writeRules writes content as a rule file into a new directory, and
+// returns the directory.
+func writeRules(t *testing.T, content string) string {
+	t.Helper()
+
+	dir := t.TempDir()
+
+	err := os.WriteFile(filepath.Join(dir, "test.rules"), []byte(content), 0o600)
+	if err != nil {
+		t.Fatalf("write the rule file: %v", err)
+	}
+
+	return dir
+}
+
+// wantRuleIDs checks the request log line's rule_ids.
+func wantRuleIDs(t *testing.T, line logLine, want ...string) {
+	t.Helper()
+
+	if !slices.Equal(line.RuleIDs, want) {
+		t.Errorf("log line has rule_ids %v, want %v", line.RuleIDs, want)
+	}
+}

internal/proxy/rules.go

@@ -0,0 +1,39 @@
+package proxy
+
+import (
+	"time"
+
+	"sneak.berlin/go/smallwebwaf/internal/requestlog"
+	"sneak.berlin/go/smallwebwaf/internal/rules"
+)
+
+// checkRules checks the request against the rules of the rule files at
+// now, notes the ids of those it matches in the log line, and returns the
+// action of the rule that refuses it, ActionRuleBlocked for a block rule
+// and ActionBanned for a ban rule, or "" when none does. A ban rule bans
+// the client's netblock for a clear sign of attack, or in observe mode
+// raises the alert for the ban it would have made.
+func (rq *request) checkRules(now time.Time) string {
+	matched := rq.h.rules.Match(rq.in)
+
+	for _, rule := range matched {
+		rq.line.RuleIDs = append(rq.line.RuleIDs, rule.ID)
+		rq.h.metrics.RuleMatched(rule.ID, rule.Action)
+	}
+
+	if len(matched) == 0 {
+		return ""
+	}
+
+	// Only the last rule matched can refuse the request.
+	switch last := matched[len(matched)-1]; last.Action {
+	case rules.ActionBlock:
+		return requestlog.ActionRuleBlocked
+	case rules.ActionBan:
+		rq.banForAttack(now, last)
+
+		return requestlog.ActionBanned
+	default:
+		return ""
+	}
+}

internal/ratelimit/ratelimit.go

@@ -149,26 +149,39 @@ type Hit struct {
 	Requests float64
 }
 
+// Counts are a client's requests in the minute, the hour and the day that
+// end at a request, that request included.
+type Counts struct {
+	Minute float64 `json:"minute"`
+	Hour   float64 `json:"hour"`
+	Day    float64 `json:"day"`
+}
+
 // Count counts a request from client at now, in every window, whether or
-// not it is refused. It reports whether the request takes the client over
-// a limit, and the window whose limit it goes over, the shortest if it is
-// over several.
-func (l *Limiter) Count(client netip.Prefix, now time.Time) (Hit, bool) {
+// not it is refused, and returns the client's requests in each window. It
+// reports whether the request takes the client over a limit, and the
+// window whose limit it goes over, the shortest if it is over several.
+func (l *Limiter) Count(client netip.Prefix, now time.Time) (Counts, Hit, bool) {
 	l.mu.Lock()
 	defer l.mu.Unlock()
 
-	var hit Hit
+	var (
+		requests [3]float64
+		hit      Hit
+	)
 
 	for i, b := range l.get(client).buckets() {
 		w := l.windows[i]
 
-		requests := b.add(now, w.length)
-		if hit.Window == "" && w.limit > 0 && requests > float64(w.limit) {
-			hit = Hit{Window: w.name, Limit: w.limit, Requests: requests}
+		requests[i] = b.add(now, w.length)
+		if hit.Window == "" && w.limit > 0 && requests[i] > float64(w.limit) {
+			hit = Hit{Window: w.name, Limit: w.limit, Requests: requests[i]}
 		}
 	}
 
-	return hit, hit.Window != ""
+	counts := Counts{Minute: requests[0], Hour: requests[1], Day: requests[2]}
+
+	return counts, hit, hit.Window != ""
 }
 
 // Reset sets client's counts in every window back to zero. Its history
@@ -242,6 +255,20 @@ func (l *Limiter) Requests(netblock netip.Prefix) int64 {
 	return requests
 }
 
+// Client returns client as the table holds it, and whether it does. It is
+// not a request from client, and leaves when it was last seen unchanged.
+func (l *Limiter) Client(client netip.Prefix) (Client, bool) {
+	l.mu.Lock()
+	defer l.mu.Unlock()
+
+	c, seen := l.clients.Peek(client)
+	if !seen {
+		return Client{}, false
+	}
+
+	return *c, true
+}
+
 // Len returns how many clients are in the table.
 func (l *Limiter) Len() int {
 	l.mu.Lock()
@@ -269,18 +296,21 @@ func (l *Limiter) Snapshot() []Client {
 	return clients
 }
 
-// Load puts clients read from clients.json into a table that holds none
-// yet, in the order they were last seen, so that the least recently seen
-// is dropped first. Buckets whose time has passed at now are emptied.
+// Load puts clients read from clients.json into the table, in place of
+// the clients it holds, in the order they were last seen, so that the
+// least recently seen is dropped first. Buckets whose time has passed at
+// now are emptied.
 func (l *Limiter) Load(clients []Client, now time.Time) {
-	l.mu.Lock()
-	defer l.mu.Unlock()
-
 	clients = slices.Clone(clients)
 	slices.SortStableFunc(clients, func(a, b Client) int {
 		return a.History.LastSeen.Compare(b.History.LastSeen)
 	})
 
+	l.mu.Lock()
+	defer l.mu.Unlock()
+
+	l.clients.Purge()
+
 	for _, c := range clients {
 		for i, b := range c.buckets() {
 			// The window that ends at now covers neither bucket once it

internal/ratelimit/ratelimit_test.go

@@ -62,14 +62,14 @@ func TestHitGivesTheLimitAndTheRequestsCounted(t *testing.T) {
 	start := midnight()
 
 	for range limit {
-		_, over := limiter.Count(client, start)
+		_, _, over := limiter.Count(client, start)
 		if over {
 			t.Fatal("a request within the limit is over it")
 		}
 	}
 
 	// Over both limits; the minute's is named, with the four requests.
-	hit, over := limiter.Count(client, start)
+	_, hit, over := limiter.Count(client, start)
 
 	want := ratelimit.Hit{Window: minute, Limit: limit, Requests: limit + 1}
 	if !over || hit != want {
@@ -78,6 +78,29 @@ func TestHitGivesTheLimitAndTheRequestsCounted(t *testing.T) {
 	}
 }
 
+func TestCountGivesTheRequestsInEachWindow(t *testing.T) {
+	t.Parallel()
+
+	limiter := ratelimit.New(ratelimit.Limits{})
+	client := netip.MustParsePrefix("203.0.113.9/32")
+	start := midnight()
+
+	for range 3 {
+		limiter.Count(client, start)
+	}
+
+	// A quarter into the next hour, the minute has only this request. The
+	// hour still covers three quarters of the bucket before, with its three
+	// requests, which count 2.25, and this one: 3.25. The day covers all
+	// four.
+	counts, _, _ := limiter.Count(client, start.Add(time.Hour+time.Hour/4))
+
+	want := ratelimit.Counts{Minute: 1, Hour: 3.25, Day: 4}
+	if counts != want {
+		t.Errorf("counts %+v, want %+v", counts, want)
+	}
+}
+
 func TestResetSetsTheCountsBackToZero(t *testing.T) {
 	t.Parallel()
 
@@ -238,7 +261,7 @@ func wantCount(
 ) {
 	t.Helper()
 
-	hit, _ := limiter.Count(client, now)
+	_, hit, _ := limiter.Count(client, now)
 	if hit.Window != want {
 		t.Errorf("request from %s at %s is over %q, want %q",
 			client, now.Format(time.RFC3339), hit.Window, want)

internal/remotelog/remotelog.go

@@ -0,0 +1,310 @@
+// Package remotelog sends the lines smallwebwaf writes on stdout to the
+// remote log endpoint, SWWAF_LOG_REMOTE_URL, as the "Request log" section
+// of SPEC.md describes: each line as the message of an RFC 5424 syslog
+// record, over UDP, TCP or TLS. Lines wait in a bounded buffer, so a slow
+// or unreachable endpoint never holds up a request or stdout.
+package remotelog
+
+import (
+	"bytes"
+	"context"
+	"crypto/tls"
+	"crypto/x509"
+	"errors"
+	"fmt"
+	"log/slog"
+	"net"
+	"net/url"
+	"os"
+	"strconv"
+	"sync/atomic"
+	"syscall"
+	"time"
+
+	"sneak.berlin/go/smallwebwaf/internal/requestlog"
+)
+
+// The forms of SWWAF_LOG_REMOTE_URL, by its scheme.
+const (
+	SchemeUDP = "syslog+udp"
+	SchemeTCP = "syslog+tcp"
+	SchemeTLS = "syslog+tls"
+)
+
+// A record's priority is the number of its facility times the number of
+// severities there are, plus the number of its severity. Every record's
+// severity is informational.
+const (
+	severities    = 8
+	informational = 6
+)
+
+const (
+	// dialTimeout bounds connecting to the endpoint, the TLS handshake
+	// included.
+	dialTimeout = 10 * time.Second
+	// After a failed attempt to connect, or a connection on which a record
+	// fails, the next attempt to connect is made a second later, and
+	// retryDelayFactor times as long after each further failure in a row,
+	// up to a minute. A connection that fails after it has stayed up for
+	// resetRetryDelayAfter ends the row.
+	firstRetryDelay      = time.Second
+	retryDelayFactor     = 2
+	maxRetryDelay        = time.Minute
+	resetRetryDelayAfter = time.Minute
+)
+
+// Params are what New needs.
+type Params struct {
+	// URL is the endpoint (SWWAF_LOG_REMOTE_URL): SchemeUDP, SchemeTCP or
+	// SchemeTLS, a host and a port.
+	URL *url.URL
+	// RootCAs are the certificates a SchemeTLS endpoint's certificate
+	// must chain to (SWWAF_LOG_REMOTE_TLS_CA_FILE), nil for the host's.
+	RootCAs *x509.CertPool
+	// Buffer is the most lines held while they wait to be sent
+	// (SWWAF_LOG_REMOTE_BUFFER).
+	Buffer int
+	// Facility is the number of the records' syslog facility
+	// (SWWAF_LOG_REMOTE_FACILITY), and AppName their APP-NAME
+	// (SWWAF_LOG_REMOTE_APP_NAME).
+	Facility int
+	AppName  string
+}
+
+// Sender sends lines to the endpoint. Write puts them in its buffer, and
+// Run sends them from there.
+type Sender struct {
+	url       *url.URL
+	tlsConfig *tls.Config
+	// beforeTime and afterTime are the parts of every record's header
+	// before and after its time, as RFC 5424 lays the header out.
+	beforeTime string
+	afterTime  string
+	// records is the buffer: each line's record, framed to be sent.
+	records chan []byte
+	sent    atomic.Int64
+	dropped atomic.Int64
+}
+
+// New returns a Sender for the endpoint params.URL.
+func New(params Params) *Sender {
+	hostname, err := os.Hostname()
+	if err != nil || hostname == "" {
+		hostname = "-" // RFC 5424's value for a field that has none
+	}
+
+	priority := params.Facility*severities + informational
+
+	return &Sender{
+		url: params.URL,
+		tlsConfig: &tls.Config{
+			RootCAs:    params.RootCAs,
+			MinVersion: tls.VersionTLS12,
+		},
+		// The 1 is the version of the format. The process id, the message
+		// id and the structured data have no value.
+		beforeTime: "<" + strconv.Itoa(priority) + ">1 ",
+		afterTime:  " " + hostname + " " + params.AppName + " - - - ",
+		records:    make(chan []byte, params.Buffer),
+	}
+}
+
+// Write puts each line in p in the buffer, as the message of a record of
+// its own, and never waits: when the buffer is full, the oldest record in
+// it is dropped to make room. It is safe for concurrent use.
+func (s *Sender) Write(p []byte) (int, error) {
+	at := requestlog.FormatTime(time.Now())
+
+	for line := range bytes.Lines(p) {
+		line = bytes.TrimSuffix(line, []byte("\n"))
+		if len(line) > 0 {
+			s.put(s.record(at, line))
+		}
+	}
+
+	return len(p), nil
+}
+
+// Sent is how many records have been sent.
+func (s *Sender) Sent() int64 {
+	return s.sent.Load()
+}
+
+// Dropped is how many records were dropped: the oldest in a full buffer,
+// and those whose sending failed.
+func (s *Sender) Dropped() int64 {
+	return s.dropped.Load()
+}
+
+// Depth is how many records are in the buffer.
+func (s *Sender) Depth() int {
+	return len(s.records)
+}
+
+// Run connects to the endpoint and sends each record as it comes into the
+// buffer, until ctx is done. Then it sends the records still in the buffer,
+// on the connection open at that time or, if there is none, on a new one,
+// until none is left or one fails, and returns. How long it may take over
+// that is for the caller to bound.
+//
+// A connection on which a record fails is closed and the record dropped.
+// That failure, like a failed attempt to connect, is logged to processLog
+// and followed by the next attempt after firstRetryDelay, retryDelayFactor
+// times as long after each further failure in a row up to maxRetryDelay,
+// and firstRetryDelay again after a connection that stayed up for
+// resetRetryDelayAfter. Meanwhile the records wait in the buffer.
+func (s *Sender) Run(ctx context.Context, processLog *slog.Logger) {
+	conn := s.send(ctx, processLog)
+	if conn == nil && len(s.records) > 0 {
+		conn, _ = s.dial(context.WithoutCancel(ctx))
+	}
+
+	if conn == nil {
+		return
+	}
+
+	defer func() {
+		_ = conn.Close()
+	}()
+
+	for {
+		select {
+		case record := <-s.records:
+			if s.write(conn, record) != nil {
+				return
+			}
+		default:
+			return
+		}
+	}
+}
+
+// record returns line as an RFC 5424 record made at the time at, framed
+// for the endpoint: on its own over UDP, since each datagram holds one,
+// and over TCP and TLS after its length in bytes and a space, the
+// octet-counted framing of RFC 6587 and RFC 5425.
+func (s *Sender) record(at string, line []byte) []byte {
+	record := make([]byte, 0, len(s.beforeTime)+len(at)+len(s.afterTime)+len(line))
+	record = append(record, s.beforeTime...)
+	record = append(record, at...)
+	record = append(record, s.afterTime...)
+	record = append(record, line...)
+
+	if s.url.Scheme == SchemeUDP {
+		return record
+	}
+
+	return append([]byte(strconv.Itoa(len(record))+" "), record...)
+}
+
+// put adds record to the buffer, first dropping the oldest record in it
+// while it is full.
+func (s *Sender) put(record []byte) {
+	for {
+		select {
+		case s.records <- record:
+			return
+		default:
+		}
+
+		select {
+		case <-s.records:
+			s.dropped.Add(1)
+		default:
+		}
+	}
+}
+
+// send connects to the endpoint and sends each record as it comes into
+// the buffer, until ctx is done, and returns the connection then open, or
+// nil.
+func (s *Sender) send(ctx context.Context, processLog *slog.Logger) net.Conn {
+	delay := firstRetryDelay
+
+	for {
+		conn, err := s.dial(ctx)
+		if ctx.Err() != nil {
+			return conn
+		}
+
+		if err == nil {
+			connected := time.Now()
+
+			err = s.sendOn(ctx, conn)
+			if err == nil {
+				return conn
+			}
+
+			_ = conn.Close()
+
+			if time.Since(connected) >= resetRetryDelayAfter {
+				delay = firstRetryDelay
+			}
+		}
+
+		processLog.Warn("sending to SWWAF_LOG_REMOTE_URL failed",
+			"error", err.Error(), "connecting_again_in", delay.String())
+
+		select {
+		case <-time.After(delay):
+		case <-ctx.Done():
+			return nil
+		}
+
+		delay = min(retryDelayFactor*delay, maxRetryDelay)
+	}
+}
+
+// sendOn sends each record on conn as it comes into the buffer, until one
+// fails, whose error it returns, or ctx is done.
+func (s *Sender) sendOn(ctx context.Context, conn net.Conn) error {
+	for {
+		select {
+		case record := <-s.records:
+			err := s.write(conn, record)
+			if err != nil {
+				return err
+			}
+		case <-ctx.Done():
+			return nil
+		}
+	}
+}
+
+// write sends record on conn, and counts it as sent or, if that fails,
+// as dropped. A record too long for one UDP datagram is dropped without
+// an error, since the connection has not failed: a long request must not
+// hold up the lines after it.
+func (s *Sender) write(conn net.Conn, record []byte) error {
+	_, err := conn.Write(record)
+	if err != nil {
+		s.dropped.Add(1)
+
+		if errors.Is(err, syscall.EMSGSIZE) {
+			return nil
+		}
+
+		return fmt.Errorf("send a record: %w", err)
+	}
+
+	s.sent.Add(1)
+
+	return nil
+}
+
+// dial connects to the endpoint.
+func (s *Sender) dial(ctx context.Context) (net.Conn, error) {
+	dialer := &net.Dialer{Timeout: dialTimeout}
+
+	switch s.url.Scheme {
+	case SchemeUDP:
+		return dialer.DialContext(ctx, "udp", s.url.Host)
+	case SchemeTLS:
+		tlsDialer := &tls.Dialer{NetDialer: dialer, Config: s.tlsConfig}
+
+		return tlsDialer.DialContext(ctx, "tcp", s.url.Host)
+	default:
+		return dialer.DialContext(ctx, "tcp", s.url.Host)
+	}
+}

internal/remotelog/remotelog_test.go

@@ -0,0 +1,640 @@
+package remotelog_test
+
+import (
+	"bufio"
+	"bytes"
+	"context"
+	"crypto/ecdsa"
+	"crypto/elliptic"
+	"crypto/rand"
+	"crypto/tls"
+	"crypto/x509"
+	"crypto/x509/pkix"
+	"encoding/json"
+	"fmt"
+	"io"
+	"log/slog"
+	"math/big"
+	"net"
+	"net/url"
+	"os"
+	"slices"
+	"strconv"
+	"strings"
+	"sync"
+	"sync/atomic"
+	"testing"
+	"testing/synctest"
+	"time"
+
+	"sneak.berlin/go/smallwebwaf/internal/remotelog"
+)
+
+// The tests run in a synctest bubble, where the time package runs on a
+// clock of the test's own, which starts at 2000-01-01T00:00:00Z: a wait
+// lasts exactly as long as it should, however slowly the test process
+// runs, and synctest.Wait returns once the sender has done all it can
+// before time passes. The endpoint is a listener on the loopback address.
+// A test reads from it only once the records are on their way, and checks
+// the sender's counts first, since a goroutine of the bubble that waits on
+// the network keeps that clock from moving on. For the same reason the
+// endpoint that refuses connections, a tlsEndpoint, runs outside the
+// bubble: a sender connecting over TLS waits on the endpoint's answer.
+
+const (
+	// started is the time a record made as a test starts gives.
+	started = "2000-01-01T00:00:00.000Z"
+	appName = "fsn1app1/gitea"
+	// local0 is the number of the default facility, and local0Info the
+	// priority of its records.
+	local0     = 16
+	local0Info = "<134>"
+	// loopback is where the endpoints listen.
+	loopback = "127.0.0.1:0"
+)
+
+func TestRecordsOverUDPGoOnePerDatagram(t *testing.T) {
+	t.Parallel()
+
+	synctest.Test(t, func(t *testing.T) {
+		endpoint, err := (&net.ListenConfig{}).ListenPacket(t.Context(), "udp", loopback)
+		if err != nil {
+			t.Fatalf("listen: %v", err)
+		}
+
+		t.Cleanup(func() { _ = endpoint.Close() })
+
+		sender, _, _ := run(t, params(remotelog.SchemeUDP, endpoint.LocalAddr()))
+
+		_, _ = sender.Write([]byte(`{"type":"request"}` + "\n" + `{"type":"process"}` + "\n"))
+
+		synctest.Wait()
+		wantCounts(t, sender, 2, 0, 0)
+
+		for _, line := range []string{`{"type":"request"}`, `{"type":"process"}`} {
+			datagram := make([]byte, 1024)
+
+			n, _, err := endpoint.ReadFrom(datagram)
+			if err != nil {
+				t.Fatalf("read: %v", err)
+			}
+
+			want := record(t, local0Info, appName, line)
+			if string(datagram[:n]) != want {
+				t.Errorf("datagram %q, want %q", datagram[:n], want)
+			}
+		}
+	})
+}
+
+func TestRecordsOverTCPAreOctetCountedWithTheirFacility(t *testing.T) {
+	t.Parallel()
+
+	synctest.Test(t, func(t *testing.T) {
+		endpoint := listen(t)
+		endpointParams := params(remotelog.SchemeTCP, endpoint.Addr())
+		endpointParams.Facility = 19 // local3
+		endpointParams.AppName = "gitea"
+		sender, _, _ := run(t, endpointParams)
+
+		_, _ = sender.Write([]byte("first\nsecond\n"))
+
+		synctest.Wait()
+		wantCounts(t, sender, 2, 0, 0)
+
+		frames := bufio.NewReader(accept(t, endpoint))
+		wantFrame(t, frames, record(t, "<158>", "gitea", "first"))
+		wantFrame(t, frames, record(t, "<158>", "gitea", "second"))
+	})
+}
+
+func TestStalledEndpointHoldsUpNoWriteAndOldestRecordsAreDropped(t *testing.T) {
+	t.Parallel()
+
+	synctest.Test(t, func(t *testing.T) {
+		certificate, roots := testCertificate(t)
+		endpoint := listen(t)
+		endpointParams := params(remotelog.SchemeTLS, endpoint.Addr())
+		endpointParams.RootCAs = roots
+		endpointParams.Buffer = 3
+		sender, _, _ := run(t, endpointParams)
+
+		// The sender connects, and its TLS handshake waits for an answer
+		// the endpoint does not give yet.
+		conn := accept(t, endpoint)
+
+		var stdout bytes.Buffer
+
+		out := io.MultiWriter(&stdout, sender)
+		for i := range 5 {
+			_, _ = fmt.Fprintf(out, "line %d\n", i+1)
+		}
+
+		if stdout.String() != "line 1\nline 2\nline 3\nline 4\nline 5\n" {
+			t.Errorf("stdout has %q", stdout.String())
+		}
+
+		wantCounts(t, sender, 0, 2, 3)
+
+		// Once the endpoint answers, the three newest records are sent.
+		server := tls.Server(conn, &tls.Config{
+			Certificates: []tls.Certificate{certificate},
+			MinVersion:   tls.VersionTLS12,
+		})
+
+		err := server.HandshakeContext(t.Context())
+		if err != nil {
+			t.Fatalf("handshake: %v", err)
+		}
+
+		synctest.Wait()
+		wantCounts(t, sender, 3, 2, 0)
+
+		frames := bufio.NewReader(server)
+		for _, line := range []string{"line 3", "line 4", "line 5"} {
+			wantFrame(t, frames, record(t, local0Info, appName, line))
+		}
+	})
+}
+
+func TestReconnectsWithBackoffAfterTheEndpointGoesAway(t *testing.T) {
+	t.Parallel()
+
+	certificate, roots := testCertificate(t)
+	endpoint := startTLSEndpoint(t, certificate)
+
+	synctest.Test(t, func(t *testing.T) {
+		endpointParams := params(remotelog.SchemeTLS, endpoint.addr)
+		endpointParams.RootCAs = roots
+		sender, logged, _ := run(t, endpointParams)
+
+		_, _ = sender.Write([]byte("one\n"))
+
+		synctest.Wait()
+		wantCounts(t, sender, 1, 0, 0)
+
+		conn := endpoint.next(t)
+		wantFrame(t, bufio.NewReader(conn), record(t, local0Info, appName, "one"))
+
+		// The endpoint goes away: it closes the connection, and refuses the
+		// next ones. The sender notices when a record fails, and tries to
+		// connect again a second later, then two seconds after that.
+		endpoint.refusing.Store(true)
+
+		_ = conn.Close()
+
+		writeUntilDropped(t, sender, 1)
+		sent := sender.Sent()
+
+		_, _ = sender.Write([]byte("two\n"))
+
+		time.Sleep(time.Second)
+		synctest.Wait()
+
+		endpoint.refusing.Store(false)
+
+		time.Sleep(2*time.Second - time.Nanosecond)
+		synctest.Wait()
+		wantCounts(t, sender, sent, 1, 1)
+
+		// The endpoint is back, and the record waiting is sent.
+		time.Sleep(time.Nanosecond)
+		synctest.Wait()
+		wantCounts(t, sender, sent+1, 1, 0)
+
+		conn = endpoint.next(t)
+		wantFrame(t, bufio.NewReader(conn), record(t, local0Info, appName, "two"))
+		wantRetries(t, logged, "1s", "2s")
+	})
+}
+
+func TestAConnectionClosedAtOnceIsMadeAgainAfterAGrowingDelay(t *testing.T) {
+	t.Parallel()
+
+	synctest.Test(t, func(t *testing.T) {
+		endpoint := listen(t)
+		sender, logged, _ := run(t, params(remotelog.SchemeTCP, endpoint.Addr()))
+
+		// The endpoint closes each connection as soon as it takes it. The
+		// sender notices when a record fails, and connects again a second
+		// later, then two seconds after that, then four.
+		delays := []time.Duration{time.Second, 2 * time.Second, 4 * time.Second}
+		for i, delay := range delays {
+			_ = accept(t, endpoint).Close()
+
+			writeUntilDropped(t, sender, int64(i+1))
+			wantConnectedAgainAfter(t, sender, delay)
+		}
+
+		wantRetries(t, logged, "1s", "2s", "4s")
+	})
+}
+
+func TestTheDelayStartsAgainAfterAConnectionThatStayedUpAMinute(t *testing.T) {
+	t.Parallel()
+
+	synctest.Test(t, func(t *testing.T) {
+		endpoint := listen(t)
+		sender, logged, _ := run(t, params(remotelog.SchemeTCP, endpoint.Addr()))
+
+		_ = accept(t, endpoint).Close()
+
+		writeUntilDropped(t, sender, 1)
+		wantConnectedAgainAfter(t, sender, time.Second)
+
+		// A connection that fails just short of a minute after it was made
+		// leaves the delay growing.
+		conn := accept(t, endpoint)
+
+		time.Sleep(time.Minute - time.Nanosecond)
+
+		_ = conn.Close()
+
+		writeUntilDropped(t, sender, 2)
+		wantConnectedAgainAfter(t, sender, 2*time.Second)
+
+		// One that fails a minute after it was made starts it again from a
+		// second.
+		conn = accept(t, endpoint)
+
+		time.Sleep(time.Minute)
+
+		_ = conn.Close()
+
+		writeUntilDropped(t, sender, 3)
+		wantConnectedAgainAfter(t, sender, time.Second)
+		wantRetries(t, logged, "1s", "2s", "1s")
+	})
+}
+
+func TestALineTooLongForADatagramIsDroppedAlone(t *testing.T) {
+	t.Parallel()
+
+	synctest.Test(t, func(t *testing.T) {
+		endpoint, err := (&net.ListenConfig{}).ListenPacket(t.Context(), "udp", loopback)
+		if err != nil {
+			t.Fatalf("listen: %v", err)
+		}
+
+		t.Cleanup(func() { _ = endpoint.Close() })
+
+		sender, logged, _ := run(t, params(remotelog.SchemeUDP, endpoint.LocalAddr()))
+
+		// With its header, the first line's record is longer than the 65507
+		// bytes a UDP datagram over IPv4 holds. It is dropped, nothing is
+		// logged, and the next line is sent at once.
+		_, _ = sender.Write([]byte(strings.Repeat("x", 65507) + "\nnext\n"))
+
+		synctest.Wait()
+		wantCounts(t, sender, 1, 1, 0)
+		wantRetries(t, logged)
+
+		datagram := make([]byte, 1024)
+
+		n, _, err := endpoint.ReadFrom(datagram)
+		if err != nil {
+			t.Fatalf("read: %v", err)
+		}
+
+		want := record(t, local0Info, appName, "next")
+		if string(datagram[:n]) != want {
+			t.Errorf("datagram %q, want %q", datagram[:n], want)
+		}
+	})
+}
+
+func TestRecordsWaitingAtTheStopAreSent(t *testing.T) {
+	t.Parallel()
+
+	certificate, roots := testCertificate(t)
+	endpoint := startTLSEndpoint(t, certificate)
+
+	synctest.Test(t, func(t *testing.T) {
+		// The endpoint refuses the sender's first connection: it fails to
+		// connect, and waits a second to try again.
+		endpoint.refusing.Store(true)
+
+		endpointParams := params(remotelog.SchemeTLS, endpoint.addr)
+		endpointParams.RootCAs = roots
+		sender, logged, stop := run(t, endpointParams)
+
+		synctest.Wait()
+		wantRetries(t, logged, "1s")
+
+		_, _ = sender.Write([]byte("one\ntwo\n"))
+
+		endpoint.refusing.Store(false)
+
+		// Stopped before that second is over, it connects to send them.
+		stop()
+		wantCounts(t, sender, 2, 0, 0)
+
+		frames := bufio.NewReader(endpoint.next(t))
+		wantFrame(t, frames, record(t, local0Info, appName, "one"))
+		wantFrame(t, frames, record(t, local0Info, appName, "two"))
+	})
+}
+
+// output collects what the sender logs.
+type output struct {
+	mu  sync.Mutex
+	buf bytes.Buffer
+}
+
+// Write adds lines the sender logs.
+func (o *output) Write(p []byte) (int, error) {
+	o.mu.Lock()
+	defer o.mu.Unlock()
+
+	return o.buf.Write(p)
+}
+
+// text returns everything logged so far.
+func (o *output) text() string {
+	o.mu.Lock()
+	defer o.mu.Unlock()
+
+	return o.buf.String()
+}
+
+// params returns the settings of a Sender for the endpoint at addr, in
+// the form scheme names: room for ten lines, the default facility, and
+// appName.
+func params(scheme string, addr net.Addr) remotelog.Params {
+	return remotelog.Params{
+		URL:      &url.URL{Scheme: scheme, Host: addr.String()},
+		Buffer:   10,
+		Facility: local0,
+		AppName:  appName,
+	}
+}
+
+// run runs a Sender with settings until the test ends or the function
+// it returns is called, which waits for Run to return. It returns the
+// Sender, and what it logs.
+func run(t *testing.T, settings remotelog.Params) (*remotelog.Sender, *output, func()) {
+	t.Helper()
+
+	sender := remotelog.New(settings)
+	logged := &output{}
+	ctx, cancel := context.WithCancel(t.Context())
+	ran := make(chan struct{})
+
+	go func() {
+		sender.Run(ctx, slog.New(slog.NewJSONHandler(logged, nil)))
+		close(ran)
+	}()
+
+	stop := func() {
+		cancel()
+		<-ran
+	}
+	t.Cleanup(stop)
+
+	return sender, logged, stop
+}
+
+// listen returns a TCP listener on the loopback address, closed when the
+// test ends.
+func listen(t *testing.T) net.Listener {
+	t.Helper()
+
+	listener, err := (&net.ListenConfig{}).Listen(t.Context(), "tcp", loopback)
+	if err != nil {
+		t.Fatalf("listen: %v", err)
+	}
+
+	t.Cleanup(func() { _ = listener.Close() })
+
+	return listener
+}
+
+// accept returns the next connection to listener, closed when the test
+// ends.
+func accept(t *testing.T, listener net.Listener) net.Conn {
+	t.Helper()
+
+	conn, err := listener.Accept()
+	if err != nil {
+		t.Fatalf("accept: %v", err)
+	}
+
+	t.Cleanup(func() { _ = conn.Close() })
+
+	return conn
+}
+
+// tlsEndpoint is a syslog+tls endpoint on the loopback address, which a
+// test starts outside its bubble. It keeps its listener until the test
+// ends, and either takes each connection or refuses it.
+type tlsEndpoint struct {
+	addr net.Addr
+	// refusing is set while the endpoint closes each connection before the
+	// TLS handshake, which fails the sender's attempt to connect.
+	refusing atomic.Bool
+	// conns are the connections it has taken, after the handshake.
+	conns chan net.Conn
+}
+
+// startTLSEndpoint starts a tlsEndpoint with certificate, which takes
+// connections until it is told to refuse them.
+func startTLSEndpoint(t *testing.T, certificate tls.Certificate) *tlsEndpoint {
+	t.Helper()
+
+	listener := listen(t)
+	endpoint := &tlsEndpoint{addr: listener.Addr(), conns: make(chan net.Conn, 10)}
+	config := &tls.Config{
+		Certificates: []tls.Certificate{certificate},
+		MinVersion:   tls.VersionTLS12,
+	}
+
+	go func() {
+		for {
+			conn, err := listener.Accept()
+			if err != nil {
+				return
+			}
+
+			server := tls.Server(conn, config)
+			if endpoint.refusing.Load() || server.HandshakeContext(t.Context()) != nil {
+				_ = conn.Close()
+
+				continue
+			}
+
+			endpoint.conns <- server
+		}
+	}()
+
+	return endpoint
+}
+
+// next returns the next connection the endpoint has taken, closed when
+// the test ends.
+func (e *tlsEndpoint) next(t *testing.T) net.Conn {
+	t.Helper()
+
+	conn := <-e.conns
+
+	t.Cleanup(func() { _ = conn.Close() })
+
+	return conn
+}
+
+// record returns the record of line made as the test started, with the
+// priority and the app name given.
+func record(t *testing.T, priority, app, line string) string {
+	t.Helper()
+
+	hostname, err := os.Hostname()
+	if err != nil || hostname == "" {
+		hostname = "-"
+	}
+
+	return priority + "1 " + started + " " + hostname + " " + app + " - - - " + line
+}
+
+// wantFrame reads the next octet-counted frame from frames, and checks
+// that it holds want.
+func wantFrame(t *testing.T, frames *bufio.Reader, want string) {
+	t.Helper()
+
+	count, err := frames.ReadString(' ')
+	if err != nil {
+		t.Fatalf("read a frame's length: %v", err)
+	}
+
+	length, err := strconv.Atoi(strings.TrimSuffix(count, " "))
+	if err != nil {
+		t.Fatalf("frame starts %q, not with its length", count)
+	}
+
+	got := make([]byte, length)
+
+	_, err = io.ReadFull(frames, got)
+	if err != nil {
+		t.Fatalf("read a frame: %v", err)
+	}
+
+	if string(got) != want {
+		t.Errorf("frame %q, want %q", got, want)
+	}
+}
+
+// wantCounts checks the records sender has sent, dropped and holds in
+// its buffer.
+func wantCounts(
+	t *testing.T, sender *remotelog.Sender, sent, dropped int64, depth int,
+) {
+	t.Helper()
+
+	if sender.Sent() != sent || sender.Dropped() != dropped || sender.Depth() != depth {
+		t.Fatalf("sent %d, dropped %d, %d in the buffer; want %d, %d and %d",
+			sender.Sent(), sender.Dropped(), sender.Depth(), sent, dropped, depth)
+	}
+}
+
+// writeUntilDropped writes a line at a time until the count of records
+// sender has dropped reaches dropped. The records it sends on a
+// connection the endpoint has closed are lost before one fails; how many
+// depends on when the endpoint's host answers that the connection is
+// gone.
+func writeUntilDropped(t *testing.T, sender *remotelog.Sender, dropped int64) {
+	t.Helper()
+
+	for sender.Dropped() < dropped {
+		_, _ = sender.Write([]byte("lost\n"))
+
+		synctest.Wait()
+	}
+}
+
+// wantConnectedAgainAfter writes a line while the sender waits to connect
+// again, and checks that it connects, and takes the line from the buffer,
+// only once delay is over.
+func wantConnectedAgainAfter(
+	t *testing.T, sender *remotelog.Sender, delay time.Duration,
+) {
+	t.Helper()
+
+	_, _ = sender.Write([]byte("waiting\n"))
+
+	time.Sleep(delay - time.Nanosecond)
+	synctest.Wait()
+
+	if sender.Depth() != 1 {
+		t.Fatalf("connected again before %v", delay)
+	}
+
+	time.Sleep(time.Nanosecond)
+	synctest.Wait()
+
+	if sender.Depth() != 0 {
+		t.Fatalf("not connected again after %v", delay)
+	}
+}
+
+// wantRetries checks that the sender logged a failure, of an attempt to
+// connect or of a connection, for each of delays, the time until the next
+// attempt, in order, and logged nothing else.
+func wantRetries(t *testing.T, logged *output, delays ...string) {
+	t.Helper()
+
+	var got []string
+
+	for line := range strings.Lines(logged.text()) {
+		var fields map[string]any
+
+		err := json.Unmarshal([]byte(line), &fields)
+		if err != nil || fields["msg"] != "sending to SWWAF_LOG_REMOTE_URL failed" {
+			t.Fatalf("logged %q", line)
+		}
+
+		delay, _ := fields["connecting_again_in"].(string)
+		got = append(got, delay)
+	}
+
+	if !slices.Equal(got, delays) {
+		t.Errorf("logged failures to connect again in %v, want %v", got, delays)
+	}
+}
+
+// testCertificate returns a certificate for 127.0.0.1 that is its own
+// CA, and a pool that holds it. It is valid on the bubble's clock, which
+// starts at 2000-01-01T00:00:00Z.
+func testCertificate(t *testing.T) (tls.Certificate, *x509.CertPool) {
+	t.Helper()
+
+	key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
+	if err != nil {
+		t.Fatalf("generate a key: %v", err)
+	}
+
+	template := &x509.Certificate{
+		SerialNumber:          big.NewInt(1),
+		Subject:               pkix.Name{CommonName: "smallwebwaf test CA"},
+		NotBefore:             time.Date(1999, 12, 31, 0, 0, 0, 0, time.UTC),
+		NotAfter:              time.Date(2000, 1, 2, 0, 0, 0, 0, time.UTC),
+		IsCA:                  true,
+		BasicConstraintsValid: true,
+		KeyUsage:              x509.KeyUsageCertSign | x509.KeyUsageDigitalSignature,
+		ExtKeyUsage:           []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth},
+		IPAddresses:           []net.IP{net.IPv4(127, 0, 0, 1)},
+	}
+
+	der, err := x509.CreateCertificate(rand.Reader, template, template,
+		&key.PublicKey, key)
+	if err != nil {
+		t.Fatalf("create a certificate: %v", err)
+	}
+
+	certificate, err := x509.ParseCertificate(der)
+	if err != nil {
+		t.Fatalf("parse the certificate: %v", err)
+	}
+
+	roots := x509.NewCertPool()
+	roots.AddCert(certificate)
+
+	return tls.Certificate{Certificate: [][]byte{der}, PrivateKey: key}, roots
+}

internal/requestlog/requestlog.go

@@ -9,6 +9,8 @@ import (
 	"io"
 	"log/slog"
 	"time"
+
+	"sneak.berlin/go/smallwebwaf/internal/ratelimit"
 )
 
 // The action a request line names: what smallwebwaf did with the
@@ -26,8 +28,12 @@ const (
 	// ActionRateLimited is a request refused because it took its client
 	// over a rate limit, which bans the client.
 	ActionRateLimited = "rate_limited"
-	// ActionBanned is a request refused because a ban covers its client.
+	// ActionBanned is a request refused because a ban covers its client,
+	// or because it matched a ban rule, which bans the client.
 	ActionBanned = "banned"
+	// ActionRuleBlocked is a request refused because it matched a block
+	// rule.
+	ActionRuleBlocked = "rule_blocked"
 	// ActionDenied is a request refused because its client is in
 	// SWWAF_DENY_NETS.
 	ActionDenied = "denied"
@@ -45,28 +51,74 @@ const OffenceLimit = "limit"
 // timeLayout is RFC 3339 with milliseconds.
 const timeLayout = "2006-01-02T15:04:05.000Z07:00"
 
-// Line is one request's line in the request log. The field names are
-// those of the "Request log" section of SPEC.md.
+// Line is one request's line in the request log. The field names, and
+// their order, are those of the "Request log" section of SPEC.md. A field
+// that may not apply to a request is left out of its line when it does
+// not.
 //
 //nolint:tagliatelle // SPEC.md's request log names its fields in snake_case
 type Line struct {
-	Type           string `json:"type"`
-	Time           string `json:"time"`
-	ClientIP       string `json:"client_ip"`
-	PeerIP         string `json:"peer_ip"`
-	Country        string `json:"country"`
-	Method         string `json:"method"`
-	Host           string `json:"host"`
-	Path           string `json:"path"`
-	Query          string `json:"query"`
-	Protocol       string `json:"protocol"`
-	Status         int    `json:"status"`
-	UpstreamStatus int    `json:"upstream_status,omitempty"`
-	RequestBytes   int64  `json:"request_bytes"`
-	ResponseBytes  int64  `json:"response_bytes"`
-	Referer        string `json:"referer"`
-	UserAgent      string `json:"user_agent"`
-	Action         string `json:"action"`
+	Type string `json:"type"`
+
+	// The standard web log fields. Scheme is how the client reached
+	// smallwebwaf, or the trusted proxy in front of it.
+	Time          string `json:"time"`
+	Instance      string `json:"instance"`
+	ClientIP      string `json:"client_ip"`
+	Method        string `json:"method"`
+	Scheme        string `json:"scheme"`
+	Host          string `json:"host"`
+	Path          string `json:"path"`
+	Query         string `json:"query"`
+	Protocol      string `json:"protocol"`
+	Status        int    `json:"status"`
+	RequestBytes  int64  `json:"request_bytes"`
+	ResponseBytes int64  `json:"response_bytes"`
+	Referer       string `json:"referer"`
+	UserAgent     string `json:"user_agent"`
+
+	// Request detail. RequestID is the X-Request-ID a trusted proxy sent,
+	// or a new one, and is sent on to the app. ForwardedFor is the
+	// X-Forwarded-For header as received. ClientGroup is the netblock the
+	// client is counted as.
+	RequestID    string `json:"request_id"`
+	PeerIP       string `json:"peer_ip"`
+	ForwardedFor string `json:"forwarded_for,omitempty"`
+	ClientGroup  string `json:"client_group"`
+	Country      string `json:"country"`
+	ContentType  string `json:"content_type,omitempty"`
+	// ContentLength is the length of its body the request announced.
+	ContentLength int64 `json:"content_length,omitempty"`
+	// RequestHeaders are the headers SWWAF_LOG_REQUEST_HEADERS names that
+	// the request carried, by name in lower case.
+	RequestHeaders   map[string]string `json:"request_headers,omitempty"`
+	HasAuthorization bool              `json:"has_authorization,omitempty"`
+	HasCookie        bool              `json:"has_cookie,omitempty"`
+	// Websocket is true when the connection was upgraded, as for a
+	// WebSocket.
+	Websocket bool `json:"websocket,omitempty"`
+
+	// Response detail, from the headers of the answer: the app's, as
+	// passed on, or those of smallwebwaf's own. Aborted is true when the
+	// client went away early.
+	ResponseContentType string `json:"response_content_type,omitempty"`
+	UpstreamStatus      int    `json:"upstream_status,omitempty"`
+	CacheControl        string `json:"cache_control,omitempty"`
+	Location            string `json:"location,omitempty"`
+	Aborted             bool   `json:"aborted,omitempty"`
+
+	// The decision.
+	Action string `json:"action"`
+	// WouldAction is, in observe mode, the action enforce mode would have
+	// taken with a request it would have refused: ActionDenied,
+	// ActionBanned, ActionCountryDenied, ActionRateLimited or
+	// ActionRuleBlocked.
+	WouldAction string `json:"would_action,omitempty"`
+	// Counts are the client's requests as the rate limits counted them
+	// with this one, for a request they counted.
+	Counts ratelimit.Counts `json:"counts,omitzero"`
+	// RuleIDs are the ids of the rule file rules the request matched.
+	RuleIDs []string `json:"rule_ids,omitempty"`
 	// LimitHit is the window whose rate limit the request went over:
 	// minute, hour or day.
 	LimitHit string `json:"limit_hit,omitempty"`
@@ -75,11 +127,18 @@ type Line struct {
 	// BanExpires is when the ban the request made, or was refused under,
 	// ends: a time, or "permanent".
 	BanExpires string `json:"ban_expires,omitempty"`
-	// Aborted is true when the client went away early.
-	Aborted bool `json:"aborted,omitempty"`
-	// DurationTotal and DurationUpstreamTotal are in milliseconds.
-	DurationTotal         float64 `json:"duration_total"`
-	DurationUpstreamTotal float64 `json:"duration_upstream_total,omitempty"`
+
+	// The timings, in milliseconds. DurationChecks is the time until the
+	// checks were done. DurationUpstreamConnect, DurationUpstreamFirstByte
+	// and DurationUpstreamTotal run from when the request was handed to the
+	// app: until there was a connection to it, until the first byte of its
+	// answer arrived, and until the end. Each but DurationTotal is nil for
+	// a request that did not get that far.
+	DurationTotal             float64  `json:"duration_total"`
+	DurationChecks            *float64 `json:"duration_checks,omitempty"`
+	DurationUpstreamConnect   *float64 `json:"duration_upstream_connect,omitempty"`
+	DurationUpstreamFirstByte *float64 `json:"duration_upstream_first_byte,omitempty"`
+	DurationUpstreamTotal     *float64 `json:"duration_upstream_total,omitempty"`
 }
 
 // Write writes line to w as one JSON line marked "type":"request".
@@ -112,8 +171,8 @@ func Milliseconds(d time.Duration) float64 {
 
 // NewProcessLogger returns the logger for the process's own messages:
 // JSON lines on w, marked "type":"process", with the time in the same form
-// as a request line's.
-func NewProcessLogger(w io.Writer) *slog.Logger {
+// as a request line's, and instanceName, SWWAF_INSTANCE_NAME, as instance.
+func NewProcessLogger(w io.Writer, instanceName string) *slog.Logger {
 	handler := slog.NewJSONHandler(w, &slog.HandlerOptions{
 		ReplaceAttr: func(groups []string, attr slog.Attr) slog.Attr {
 			if attr.Key == slog.TimeKey && len(groups) == 0 {
@@ -124,5 +183,5 @@ func NewProcessLogger(w io.Writer) *slog.Logger {
 		},
 	})
 
-	return slog.New(handler).With("type", "process")
+	return slog.New(handler).With("type", "process", "instance", instanceName)
 }

internal/requestlog/requestlog_test.go

@@ -50,7 +50,11 @@ func TestWriteWritesOneJSONLineMarkedRequest(t *testing.T) {
 	}
 
 	unset := []string{
-		"upstream_status", "limit_hit", "offence", "ban_expires", "aborted",
+		"forwarded_for", "content_type", "content_length", "request_headers",
+		"has_authorization", "has_cookie", "websocket", "response_content_type",
+		"upstream_status", "cache_control", "location", "aborted", "counts",
+		"limit_hit", "offence", "ban_expires", "duration_checks",
+		"duration_upstream_connect", "duration_upstream_first_byte",
 		"duration_upstream_total",
 	}
 	for _, name := range unset {
@@ -61,12 +65,12 @@ func TestWriteWritesOneJSONLineMarkedRequest(t *testing.T) {
 	}
 }
 
-func TestProcessLinesAreMarkedProcess(t *testing.T) {
+func TestProcessLinesAreMarkedProcessAndGiveTheInstance(t *testing.T) {
 	t.Parallel()
 
 	var out bytes.Buffer
 
-	requestlog.NewProcessLogger(&out).Info("starting", "version", "v1")
+	requestlog.NewProcessLogger(&out, "fsn1app1/gitea").Info("starting", "version", "v1")
 
 	var fields map[string]any
 
@@ -75,8 +79,9 @@ func TestProcessLinesAreMarkedProcess(t *testing.T) {
 		t.Fatalf("decode %q: %v", out.String(), err)
 	}
 
-	if fields["type"] != "process" || fields["msg"] != "starting" ||
-		fields["level"] != "INFO" || fields["version"] != "v1" {
+	if fields["type"] != "process" || fields["instance"] != "fsn1app1/gitea" ||
+		fields["msg"] != "starting" || fields["level"] != "INFO" ||
+		fields["version"] != "v1" {
 		t.Errorf("process line %v", fields)
 	}
 

internal/rules/rules.go

@@ -0,0 +1,483 @@
+// Package rules reads the rule files: the plain text files in
+// SWWAF_RULES_DIR, one rule to a line, that each request is checked
+// against, as the "Rule files" section of SPEC.md describes. They are read
+// at start, and again once the directory has had no change for a short
+// time after one is edited, added or removed.
+package rules
+
+import (
+	"context"
+	"encoding/hex"
+	"errors"
+	"fmt"
+	"log/slog"
+	"net/http"
+	"os"
+	"path/filepath"
+	"regexp"
+	"slices"
+	"strings"
+	"sync/atomic"
+	"time"
+
+	"github.com/fsnotify/fsnotify"
+
+	"sneak.berlin/go/smallwebwaf/internal/alerts"
+	"sneak.berlin/go/smallwebwaf/internal/config"
+)
+
+// The actions a rule takes when it matches.
+const (
+	// ActionLog notes the match in the request log, and does nothing else.
+	ActionLog = "log"
+	// ActionBlock refuses the request with 403.
+	ActionBlock = "block"
+	// ActionBan refuses the request and bans the client's netblock: the
+	// request is a clear sign of attack.
+	ActionBan = "ban"
+)
+
+// extension ends the name of every rule file.
+const extension = ".rules"
+
+// quietTime is how long SWWAF_RULES_DIR must go without a change before
+// the rule files are read again, so that a file still being written, such
+// as one saved in place, appended to or copied in with scp, is read only
+// once whole.
+const quietTime = 2 * time.Second
+
+// headerTarget starts the target that is one request header,
+// header:<Name>.
+const headerTarget = "header:"
+
+// escapeLength is the length of a percent escape, such as %2e.
+const escapeLength = 3
+
+var (
+	// ruleLine is a rule: four fields separated by spaces or tabs, of
+	// which the fourth, the regex, runs to the end of the line.
+	ruleLine = regexp.MustCompile(`^([^ \t]+)[ \t]+([^ \t]+)[ \t]+([^ \t]+)[ \t]+(.+)$`)
+	// idChars are the characters of a rule's id.
+	idChars = regexp.MustCompile(`^[A-Za-z0-9_-]+$`)
+)
+
+var (
+	errNotRule = errors.New(
+		"is not a rule: an id, a target, an action and a regex, " +
+			"separated by spaces or tabs")
+	errNotID     = errors.New("is not an id of letters, digits, - and _")
+	errNotTarget = errors.New(
+		"is not path, query, uri, method, host, user_agent, referer or header:<Name>")
+	errNotHeaderName = errors.New(
+		"has a character after header: that no header name can have")
+	errHeaderTakenOut = errors.New(
+		"names a header that Go's HTTP server takes out of every request, " +
+			"so a rule never sees it")
+	errNotAction = errors.New("is not log, block or ban")
+	errNotRegex  = errors.New("does not compile")
+	errUsedTwice = errors.New("is already the id of the rule at")
+)
+
+// Rule is one rule of a rule file.
+type Rule struct {
+	// ID names the rule in the request log, the metrics and ban notes.
+	ID string
+	// Target is what the regex is matched against, such as path or
+	// header:Accept.
+	Target string
+	// Action is ActionLog, ActionBlock or ActionBan.
+	Action string
+
+	regex *regexp.Regexp
+}
+
+// Params are what Load needs.
+type Params struct {
+	// Dir is the directory of the rule files (SWWAF_RULES_DIR).
+	Dir string
+	// Enabled is SWWAF_RULES_ENABLED: while it is false, no file is read
+	// and no rule loaded.
+	Enabled bool
+	// ProcessLog receives how many rules were read, and the error in a
+	// rule file edited while smallwebwaf runs.
+	ProcessLog *slog.Logger
+	// Alerts receive a file_error alert for that error.
+	Alerts *alerts.Queue
+}
+
+// Files are the rule files of a running smallwebwaf, and the rules read
+// from them. They are safe for concurrent use.
+type Files struct {
+	params Params
+	// rules are the rules loaded, in the order of their files' names, and
+	// then of their lines.
+	rules atomic.Pointer[[]Rule]
+}
+
+// Load reads the rules of every *.rules file in Dir, in the order of the
+// files' names, unless Enabled is false. A Dir that cannot be read is an
+// error, and so is a line that is not a rule, a header name with a
+// character no header name can have, a rule for the Host or the
+// Transfer-Encoding header, which Go's HTTP server takes out of every
+// request, a regex that does not compile and an id used twice, each named
+// with its file and line.
+func Load(params Params) (*Files, error) {
+	f := &Files{params: params}
+	f.rules.Store(&[]Rule{})
+
+	if !params.Enabled {
+		return f, nil
+	}
+
+	rules, _, err := read(params.Dir)
+	if err != nil {
+		return nil, err
+	}
+
+	f.rules.Store(&rules)
+	f.logRead(len(rules))
+
+	return f, nil
+}
+
+// Match checks r against the rules, in order, and returns those it
+// matches, up to the first whose action refuses it, block or ban, which
+// is then the last one returned.
+func (f *Files) Match(r *http.Request) []Rule {
+	var matched []Rule
+
+	for _, rule := range *f.rules.Load() {
+		if !rule.matches(r) {
+			continue
+		}
+
+		matched = append(matched, rule)
+		if rule.Action != ActionLog {
+			break
+		}
+	}
+
+	return matched
+}
+
+// Len returns how many rules are loaded.
+func (f *Files) Len() int {
+	return len(*f.rules.Load())
+}
+
+// Watch watches Dir until ctx is done, and reads the rule files again
+// once Dir has had no change for quietTime, after one is edited, added or
+// removed, and after Watch starts watching. If they then hold an error,
+// the rules stay as they were, the error is logged with its file and
+// line, and the files are read again after the next change. If Dir cannot
+// be watched, that is logged, and the rules stay as they were loaded.
+// While Enabled is false, Watch returns at once.
+func (f *Files) Watch(ctx context.Context) {
+	if !f.params.Enabled {
+		return
+	}
+
+	watcher, err := fsnotify.NewWatcher()
+	if err == nil {
+		defer func() {
+			_ = watcher.Close()
+		}()
+
+		err = watcher.Add(f.params.Dir)
+	}
+
+	if err != nil {
+		f.params.ProcessLog.Error("cannot watch the rule files for edits",
+			"error", err.Error())
+
+		return
+	}
+
+	f.params.ProcessLog.Info("watching the rule files for edits",
+		"directory", f.params.Dir)
+
+	f.readAfterChanges(ctx, watcher.Events, watcher.Errors)
+}
+
+// readAfterChanges reads the rule files again once quietTime has passed
+// without a change from events, until ctx is done, and logs the errors
+// from errs. The wait starts at once, as if for a change, so that an edit
+// saved after Load read the files, and before Dir was watched, is taken
+// in too.
+func (f *Files) readAfterChanges(
+	ctx context.Context, events <-chan fsnotify.Event, errs <-chan error,
+) {
+	quiet := time.NewTimer(quietTime)
+	defer quiet.Stop()
+
+	for {
+		select {
+		case <-ctx.Done():
+			return
+		case <-events:
+			quiet.Reset(quietTime)
+		case <-quiet.C:
+			f.readAgain()
+		case err := <-errs:
+			f.params.ProcessLog.Warn("watching the rule files failed",
+				"error", err.Error())
+		}
+	}
+}
+
+// readAgain reads the rule files again, in place of the rules loaded, or
+// logs the error that keeps the rules as they were, and raises a
+// file_error alert for it, for the file it is in.
+func (f *Files) readAgain() {
+	rules, path, err := read(f.params.Dir)
+	if err != nil {
+		const kept = "a rule file has an error, and the rules stay as they were"
+
+		// Raised before it is logged, so that the alert is there once the
+		// log line is.
+		f.params.Alerts.Raise(alerts.Alert{
+			Event:  alerts.EventFileError,
+			Reason: kept,
+			Detail: map[string]any{"file": path, "error": err.Error()},
+		})
+		f.params.ProcessLog.Error(kept, "error", err.Error())
+
+		return
+	}
+
+	f.rules.Store(&rules)
+	f.logRead(len(rules))
+}
+
+// logRead logs that the rule files were read, and how many rules they
+// hold, which can be none.
+func (f *Files) logRead(count int) {
+	f.params.ProcessLog.Info("read the rule files",
+		"directory", f.params.Dir, "rules", count)
+}
+
+// read returns the rules of every rule file in dir, in the order of the
+// files' names, and then of their lines, or an error, with the path of the
+// rule file it is in, or dir. A file whose name starts with a dot, such as
+// an editor's lock file .#50-app.rules, is not a rule file, as a shell's
+// *.rules would not match it.
+func read(dir string) ([]Rule, string, error) {
+	entries, err := os.ReadDir(dir)
+	if err != nil {
+		return nil, dir, fmt.Errorf("SWWAF_RULES_DIR cannot be read: %w", err)
+	}
+
+	var rules []Rule
+
+	// places are where each id is, as "<file>, line <n>".
+	places := map[string]string{}
+
+	for _, entry := range entries {
+		name := entry.Name()
+		if entry.IsDir() || strings.HasPrefix(name, ".") || filepath.Ext(name) != extension {
+			continue
+		}
+
+		path := filepath.Join(dir, name)
+
+		rules, err = readFile(path, rules, places)
+		if err != nil {
+			return nil, path, err
+		}
+	}
+
+	return rules, "", nil
+}
+
+// readFile appends the rules of the rule file at path to rules. places
+// are where each id read so far is, and gain those of the file.
+func readFile(path string, rules []Rule, places map[string]string) ([]Rule, error) {
+	data, err := os.ReadFile(path) //nolint:gosec // a rule file, in SWWAF_RULES_DIR
+	if err != nil {
+		return nil, err
+	}
+
+	number := 0
+
+	for line := range strings.Lines(string(data)) {
+		number++
+		place := fmt.Sprintf("%s, line %d", path, number)
+
+		text := strings.TrimSuffix(strings.TrimSuffix(line, "\n"), "\r")
+
+		rule, isRule, err := parse(text)
+		if err != nil {
+			return nil, fmt.Errorf("%s: %w", place, err)
+		}
+
+		if !isRule {
+			continue
+		}
+
+		first, used := places[rule.ID]
+		if used {
+			return nil, fmt.Errorf("%s: the id %q %w %s", place, rule.ID, errUsedTwice, first)
+		}
+
+		places[rule.ID] = place
+		rules = append(rules, rule)
+	}
+
+	return rules, nil
+}
+
+// parse reads a line of a rule file. It returns false for a blank line
+// and for a comment, a line that starts with #. Spaces and tabs at the
+// end of the line are not part of its regex, so a line with only those
+// after its action has no regex, and is not a rule.
+func parse(line string) (Rule, bool, error) {
+	line = strings.Trim(line, " \t")
+	if line == "" || strings.HasPrefix(line, "#") {
+		return Rule{}, false, nil
+	}
+
+	fields := ruleLine.FindStringSubmatch(line)
+	if fields == nil {
+		return Rule{}, false, errNotRule
+	}
+
+	rule := Rule{ID: fields[1], Target: fields[2], Action: fields[3]}
+	headerName, isHeader := strings.CutPrefix(rule.Target, headerTarget)
+
+	switch {
+	case !idChars.MatchString(rule.ID):
+		return Rule{}, false, fmt.Errorf("the id %q %w", rule.ID, errNotID)
+	case !isTarget(rule.Target):
+		return Rule{}, false, fmt.Errorf("the target %q %w", rule.Target, errNotTarget)
+	case isHeader && !config.IsHeaderName(headerName):
+		return Rule{}, false, fmt.Errorf("the target %q %w", rule.Target, errNotHeaderName)
+	case strings.EqualFold(rule.Target, headerTarget+"Host"):
+		return Rule{}, false, fmt.Errorf(
+			"the target %q %w; the request's host is the target host",
+			rule.Target, errHeaderTakenOut)
+	case strings.EqualFold(rule.Target, headerTarget+"Transfer-Encoding"):
+		return Rule{}, false, fmt.Errorf("the target %q %w", rule.Target, errHeaderTakenOut)
+	case !slices.Contains([]string{ActionLog, ActionBlock, ActionBan}, rule.Action):
+		return Rule{}, false, fmt.Errorf("the action %q %w", rule.Action, errNotAction)
+	}
+
+	regex, err := regexp.Compile(fields[4])
+	if err != nil {
+		return Rule{}, false, fmt.Errorf("the regex %w: %w", errNotRegex, err)
+	}
+
+	rule.regex = regex
+
+	return rule, true, nil
+}
+
+// isTarget reports whether target is one a rule may have.
+func isTarget(target string) bool {
+	switch target {
+	case "path", "query", "uri", "method", "host", "user_agent", "referer":
+		return true
+	}
+
+	name, isHeader := strings.CutPrefix(target, headerTarget)
+
+	return isHeader && name != ""
+}
+
+// matches reports whether the rule's regex matches its target in r. For
+// uri it is matched against the path and query as received, and against
+// them once percent-decoded, so that an encoded probe cannot slip past.
+func (rule Rule) matches(r *http.Request) bool {
+	if rule.Target == "uri" {
+		uri := pathAndQuery(r)
+
+		return rule.regex.MatchString(uri) || rule.regex.MatchString(decodeOnce(uri))
+	}
+
+	return rule.regex.MatchString(value(rule.Target, r))
+}
+
+// value returns what a rule with target, other than uri, is matched
+// against in r: the path and the query as the client sent them, before
+// any decoding or re-encoding, split at the first ?, and a header's values
+// joined by ", ", as HTTP joins those of a header sent more than once.
+func value(target string, r *http.Request) string {
+	switch target {
+	case "path":
+		path, _, _ := strings.Cut(pathAndQuery(r), "?")
+
+		return path
+	case "query":
+		_, query, _ := strings.Cut(pathAndQuery(r), "?")
+
+		return query
+	case "method":
+		return r.Method
+	case "host":
+		return r.Host
+	case "user_agent":
+		return header(r, "User-Agent")
+	case "referer":
+		return header(r, "Referer")
+	default:
+		return header(r, strings.TrimPrefix(target, headerTarget))
+	}
+}
+
+// pathAndQuery returns the target of r's request line, r.RequestURI, as
+// the client sent it, less any scheme and host: a target with a scheme
+// gives what follows the scheme and its :, and the host when // follows.
+// So http://host/path, as a client sends it to a proxy, gives /path, and
+// so does http:/path, which Go reads as a target with a scheme and no
+// host. r.URL is not used: when the path holds a character it escapes,
+// such as \ or a non-ASCII byte, it decodes the whole path and escapes it
+// again, so that \ becomes %5C and %2e a dot.
+func pathAndQuery(r *http.Request) string {
+	if !r.URL.IsAbs() {
+		return r.RequestURI
+	}
+
+	_, afterScheme, _ := strings.Cut(r.RequestURI, ":")
+
+	hostAndRest, hasHost := strings.CutPrefix(afterScheme, "//")
+	if !hasHost {
+		return afterScheme
+	}
+
+	start := strings.IndexAny(hostAndRest, "/?")
+	if start < 0 {
+		return ""
+	}
+
+	return hostAndRest[start:]
+}
+
+// header returns the values of r's header name joined by ", ", or "" if
+// r has no such header.
+func header(r *http.Request, name string) string {
+	return strings.Join(r.Header.Values(name), ", ")
+}
+
+// decodeOnce returns s with each percent escape, such as %2e, replaced by
+// the byte it stands for. A % that is not followed by two hex digits is
+// left as it is, so that a malformed escape cannot keep the rest of s
+// from being decoded.
+func decodeOnce(s string) string {
+	var decoded strings.Builder
+
+	for i := 0; i < len(s); i++ {
+		if s[i] == '%' && i+escapeLength <= len(s) {
+			b, err := hex.DecodeString(s[i+1 : i+escapeLength])
+			if err == nil {
+				decoded.Write(b)
+
+				i += escapeLength - 1
+
+				continue
+			}
+		}
+
+		decoded.WriteByte(s[i])
+	}
+
+	return decoded.String()
+}

internal/rules/rules_test.go

@@ -0,0 +1,687 @@
+package rules_test
+
+import (
+	"context"
+	"encoding/json"
+	"log/slog"
+	"maps"
+	"net/http"
+	"net/http/httptest"
+	"net/url"
+	"os"
+	"path/filepath"
+	"slices"
+	"strconv"
+	"testing"
+	"time"
+
+	"sneak.berlin/go/smallwebwaf/internal/alerts"
+	"sneak.berlin/go/smallwebwaf/internal/rules"
+)
+
+const (
+	// What the process log says once Watch watches the directory, after
+	// each reading of the rule files, and for one that has an error.
+	watching = "watching the rule files for edits"
+	read     = "read the rule files"
+	hasError = "a rule file has an error, and the rules stay as they were"
+	// maxLogLines is how many lines of the process log wait for a test to
+	// read them.
+	maxLogLines = 64
+	// browser is the user agent of an ordinary visitor.
+	browser = "Mozilla/5.0 (X11; Linux x86_64; rv:140.0) Gecko/20100101 Firefox/140.0"
+	// testFile is the rule file of a test that needs only one, and
+	// firstFile the first of a test's rule files.
+	testFile  = "test.rules"
+	firstFile = "00-a.rules"
+	// userAgent is the header that carries the user agent.
+	userAgent = "User-Agent"
+)
+
+func TestEachTargetMatchesWhatItNames(t *testing.T) {
+	t.Parallel()
+
+	for _, tc := range []struct {
+		name   string
+		rule   string // its target, action and regex
+		uri    string // the request's path and query
+		header http.Header
+		want   bool
+	}{
+		{"path as received", `path log ^/%2eenv$`, "/%2eenv", nil, true},
+		{"path not decoded", `path log ^/\.env$`, "/%2eenv", nil, false},
+		{"path without the query", `path log ^/a$`, "/a?b=c", nil, true},
+		{"query as received", `query log ^b=%2e$`, "/a?b=%2e", nil, true},
+		{"uri as received", `uri log ^/a\?b=%2e$`, "/a?b=%2e", nil, true},
+		{"uri decoded", `uri log (\.\./){2}`, "/a?f=%2e%2e%2f%2e%2e%2f", nil, true},
+		{
+			"uri decoded past malformed escapes", `uri log (\.\./){2}&h=%$`,
+			"/a?g=%zz&f=%2e%2e%2f%2e%2e%2f&h=%", nil, true,
+		},
+		{"uri decoded only once", `uri log ^/a\.b$`, "/a%252eb", nil, false},
+		{"method", `method log ^PUT$`, "/", nil, true},
+		{"host", `host log ^app\.example$`, "/", nil, true},
+		{
+			"user_agent", `user_agent log ^sqlmap/`, "/",
+			http.Header{userAgent: {"sqlmap/1.8"}}, true,
+		},
+		{
+			"user_agent sent twice", `user_agent log ^curl/8, sqlmap/`, "/",
+			http.Header{userAgent: {"curl/8", "sqlmap/1.8"}}, true,
+		},
+		{"user_agent missing", `user_agent log ^$`, "/", nil, true},
+		{
+			"referer", `referer log ^https://spam\.example/`, "/",
+			http.Header{"Referer": {"https://spam.example/buy"}}, true,
+		},
+		{
+			"a header sent twice", `header:x-api-version log ^2, 3$`, "/",
+			http.Header{"X-Api-Version": {"2", "3"}}, true,
+		},
+		{"a header missing", `header:X-Api-Version log ^$`, "/", nil, true},
+	} {
+		t.Run(tc.name, func(t *testing.T) {
+			t.Parallel()
+
+			files := load(t, ruleFiles{testFile: "a-rule " + tc.rule + "\n"})
+
+			// Every request is a PUT, which the method rule looks for.
+			r := httptest.NewRequestWithContext(t.Context(), http.MethodPut,
+				"http://app.example"+tc.uri, nil)
+			maps.Copy(r.Header, tc.header)
+
+			got := len(files.Match(r)) == 1
+			if got != tc.want {
+				t.Errorf("%s matches %s: %t, want %t", tc.rule, tc.uri, got, tc.want)
+			}
+		})
+	}
+}
+
+func TestPathMatchedAsTheClientSentIt(t *testing.T) {
+	t.Parallel()
+
+	// Each path holds a character Go's URL type would escape again, \ or
+	// a non-ASCII byte, and each rule is written for the path as sent.
+	for _, tc := range []struct {
+		rule string // its target, action and regex
+		sent string // the path and query the client sent
+	}{
+		{`path log ^/\.\.\\\.\.\\windows\\win\.ini$`, `/..\..\windows\win.ini`},
+		{`path log ^/%2e%2e\\%2e%2e\\windows\\win\.ini$`, `/%2e%2e\%2e%2e\windows\win.ini`},
+		{`path log ^/café$`, "/café?x=1"},
+		{`uri log ^/%2e%2e\\%2e%2e\\boot\.ini\?x=1$`, `/%2e%2e\%2e%2e\boot.ini?x=1`},
+	} {
+		files := load(t, ruleFiles{testFile: "as-sent " + tc.rule + "\n"})
+
+		// The target in origin form, as traefik sends it, in absolute form,
+		// as a client sends it to a proxy, and with a scheme but no host,
+		// which Go reads as absolute form with no host, sending the app
+		// the path.
+		for _, target := range []string{
+			tc.sent, "http://app.example" + tc.sent, "http:" + tc.sent, "foo:" + tc.sent,
+		} {
+			r := httptest.NewRequestWithContext(t.Context(), http.MethodGet, target, nil)
+			wantMatched(t, files, r, "as-sent")
+		}
+	}
+}
+
+func TestMatchingStopsAtTheFirstRuleThatRefuses(t *testing.T) {
+	t.Parallel()
+
+	files := load(t, ruleFiles{testFile: `
+every-path     path  log    ^/
+no-path        path  log    ^$
+first-refusal  path  block  ^/probe
+later-ban      path  ban    ^/probe
+after          path  log    ^/
+`})
+
+	// Every log rule that matches is noted, and the block rule ends the
+	// matching.
+	wantMatched(t, files, get(t, "/probe"), "every-path", "first-refusal")
+	wantMatched(t, files, get(t, "/page"), "every-path", "after")
+
+	// A ban rule ends it too.
+	files = load(t, ruleFiles{testFile: "ban path ban ^/\nlater path block ^/\n"})
+	wantMatched(t, files, get(t, "/"), "ban")
+}
+
+func TestSpacesAndTabsEndingALineAreNotPartOfItsRegex(t *testing.T) {
+	t.Parallel()
+
+	files := load(t, ruleFiles{testFile: "env-file path block ^/\\.env$ \t \n"})
+
+	wantMatched(t, files, get(t, "/.env"), "env-file")
+}
+
+func TestFilesReadInNameOrderThenLineOrder(t *testing.T) {
+	t.Parallel()
+
+	files := load(t, ruleFiles{
+		"50-b.rules": "b1 path log ^/\n\n# a comment\n  # an indented one\nb2 path log ^/\n",
+		firstFile:    "a1 path log ^/\r\n",
+		// None is a rule file.
+		"notes.txt":       "notes, not rules\n",
+		"10-c.rules.bak":  "an old copy\n",
+		"20-d.rules/keep": "a file in a directory\n",
+	})
+
+	wantMatched(t, files, get(t, "/"), "a1", "b1", "b2")
+
+	if files.Len() != 3 {
+		t.Errorf("%d rules loaded, want 3", files.Len())
+	}
+}
+
+func TestFileWhoseNameStartsWithADotIsNotARuleFile(t *testing.T) {
+	t.Parallel()
+
+	dir := writeFiles(t, ruleFiles{firstFile: "probe path block ^/probe\n"})
+
+	// The lock file Emacs makes beside a file while it is edited: a link to
+	// nothing, which cannot be read.
+	err := os.Symlink("user@host.1234:1700000000", filepath.Join(dir, ".#"+firstFile))
+	if err != nil {
+		t.Fatalf("symlink: %v", err)
+	}
+
+	params, _ := newParams(dir)
+
+	files, err := rules.Load(params)
+	if err != nil {
+		t.Fatalf("load: %v", err)
+	}
+
+	wantMatched(t, files, get(t, "/probe"), "probe")
+}
+
+func TestFaultStopsTheStartNamingTheFileAndLine(t *testing.T) {
+	t.Parallel()
+
+	for _, tc := range []struct {
+		name    string
+		content string
+		line    int
+		want    string
+	}{
+		{
+			"too few fields", "env-file path ban\n", 1,
+			"is not a rule: an id, a target, an action and a regex, " +
+				"separated by spaces or tabs",
+		},
+		{
+			// Else its regex would be a space, found in nearly every user agent.
+			"a regex of only spaces and tabs", "scanner user_agent ban\t \n", 1,
+			"is not a rule: an id, a target, an action and a regex, " +
+				"separated by spaces or tabs",
+		},
+		{
+			"an id of other characters", "# ids\n\nenv.file path ban ^/\n", 3,
+			`the id "env.file" is not an id of letters, digits, - and _`,
+		},
+		{
+			"an unknown target", "env-file paths ban ^/\n", 1,
+			`the target "paths" is not path, query, uri, method, host, ` +
+				"user_agent, referer or header:<Name>",
+		},
+		{
+			"a header without a name", "env-file header: ban ^/\n", 1,
+			`the target "header:" is not path, query, uri, method, host, ` +
+				"user_agent, referer or header:<Name>",
+		},
+		{
+			"a header name written with its colon", "sqlmap header:User-Agent: ban sqlmap\n", 1,
+			`the target "header:User-Agent:" has a character after header: ` +
+				"that no header name can have",
+		},
+		{
+			"a header name with a semicolon", "accept header:Accept;q log ^$\n", 1,
+			`the target "header:Accept;q" has a character after header: ` +
+				"that no header name can have",
+		},
+		{
+			"a header name with brackets", "x-header header:X(y) log ^$\n", 1,
+			`the target "header:X(y)" has a character after header: ` +
+				"that no header name can have",
+		},
+		{
+			"the Host header", "host-header header:host block ^$\n", 1,
+			`the target "header:host" names a header that Go's HTTP server ` +
+				"takes out of every request, so a rule never sees it; " +
+				"the request's host is the target host",
+		},
+		{
+			"the Transfer-Encoding header",
+			"# bodies sent in chunks\nchunked header:Transfer-Encoding block ^chunked$\n", 2,
+			`the target "header:Transfer-Encoding" names a header that Go's ` +
+				"HTTP server takes out of every request, so a rule never sees it",
+		},
+		{
+			"an unknown action", "env-file path deny ^/\n", 1,
+			`the action "deny" is not log, block or ban`,
+		},
+		{
+			"a regex that does not compile", "env-file path ban ^/(\n", 1,
+			"the regex does not compile: error parsing regexp: " +
+				"missing closing ): `^/(`",
+		},
+	} {
+		t.Run(tc.name, func(t *testing.T) {
+			t.Parallel()
+
+			dir := writeFiles(t, ruleFiles{"00-default.rules": tc.content})
+			path := filepath.Join(dir, "00-default.rules")
+
+			wantRefused(t, dir, path+", line "+strconv.Itoa(tc.line)+": "+tc.want)
+		})
+	}
+}
+
+func TestIDUsedTwiceStopsTheStartNamingBothPlaces(t *testing.T) {
+	t.Parallel()
+
+	dir := writeFiles(t, ruleFiles{
+		"00-a.rules": "probe path log ^/a\n",
+		"50-b.rules": "other path log ^/b\nprobe path ban ^/c\n",
+	})
+
+	wantRefused(t, dir, filepath.Join(dir, "50-b.rules")+`, line 2: the id "probe" `+
+		"is already the id of the rule at "+filepath.Join(dir, "00-a.rules")+", line 1")
+}
+
+func TestDirectoryThatDoesNotExistStopsTheStart(t *testing.T) {
+	t.Parallel()
+
+	dir := filepath.Join(t.TempDir(), "rules.d")
+
+	wantRefused(t, dir, "SWWAF_RULES_DIR cannot be read: open "+dir+
+		": no such file or directory")
+}
+
+func TestEmptyDirectoryLoadsNoRulesAndSaysSo(t *testing.T) {
+	t.Parallel()
+
+	params, lines := newParams(writeFiles(t, ruleFiles{"00-default.rules": "# none\n"}))
+
+	files, err := rules.Load(params)
+	if err != nil {
+		t.Fatalf("load: %v", err)
+	}
+
+	line := lines.waitFor(t, read)
+	if files.Len() != 0 || line["rules"] != 0.0 {
+		t.Errorf("%d rules loaded, and the log says %v, want none", files.Len(), line)
+	}
+}
+
+func TestRuleFilesOffReadNothing(t *testing.T) {
+	t.Parallel()
+
+	// SWWAF_RULES_DIR does not exist, which would stop the start.
+	params, _ := newParams(filepath.Join(t.TempDir(), "rules.d"))
+	params.Enabled = false
+
+	files, err := rules.Load(params)
+	if err != nil {
+		t.Fatalf("load: %v", err)
+	}
+
+	if files.Len() != 0 || files.Match(get(t, "/")) != nil {
+		t.Errorf("%d rules loaded with the rule files off", files.Len())
+	}
+
+	// It would watch until the test ends.
+	files.Watch(t.Context())
+}
+
+func TestEditsTakenInWhileRunning(t *testing.T) {
+	t.Parallel()
+
+	dir := writeFiles(t, ruleFiles{firstFile: "first path block ^/first\n"})
+	files, lines, _ := watch(t, dir)
+
+	// matches reports whether path matches a rule.
+	matches := func(path string) bool { return len(files.Match(get(t, path))) == 1 }
+
+	// A file added.
+	save(t, dir, "50-b.rules", "second path block ^/second\n")
+	lines.waitUntil(t, func() bool { return matches("/second") })
+	wantMatched(t, files, get(t, "/first"), "first")
+
+	// A file edited.
+	save(t, dir, firstFile, "first path block ^/edited\n")
+	lines.waitUntil(t, func() bool { return !matches("/first") })
+	wantMatched(t, files, get(t, "/edited"), "first")
+
+	// A file removed.
+	err := os.Remove(filepath.Join(dir, "50-b.rules"))
+	if err != nil {
+		t.Fatalf("remove: %v", err)
+	}
+
+	lines.waitUntil(t, func() bool { return !matches("/second") })
+	wantMatched(t, files, get(t, "/edited"), "first")
+}
+
+func TestBrokenEditKeepsTheRulesAsTheyWere(t *testing.T) {
+	t.Parallel()
+
+	dir := writeFiles(t, ruleFiles{firstFile: "first path block ^/first\n"})
+	files, lines, queue := watch(t, dir)
+
+	// The edit's second line has an unknown action, so the rules stay as
+	// they were, the first line's earlier version included.
+	save(t, dir, firstFile, "first path block ^/edited\nsecond path bann ^/second\n")
+
+	line := lines.waitFor(t, hasError)
+	want := filepath.Join(dir, firstFile) +
+		`, line 2: the action "bann" is not log, block or ban`
+
+	if line["error"] != want || line["level"] != "ERROR" {
+		t.Errorf("logged %v, want an error %q", line, want)
+	}
+
+	// The error is raised as a file_error alert too, for the file.
+	wantFileError := func() {
+		t.Helper()
+
+		waiting := queue.Snapshot().Waiting[alerts.DestinationWebhook]
+		if len(waiting) != 1 || waiting[0].Event != alerts.EventFileError ||
+			waiting[0].Reason != hasError || waiting[0].Detail["error"] != want ||
+			waiting[0].Detail["file"] != filepath.Join(dir, firstFile) {
+			t.Errorf("alerts waiting %+v, want one file_error alert for %q", waiting, want)
+		}
+	}
+	wantFileError()
+
+	wantMatched(t, files, get(t, "/first"), "first")
+	wantMatched(t, files, get(t, "/second"))
+
+	// Once mended, the file is read again, and raises no alert.
+	save(t, dir, firstFile, "first path block ^/edited\nsecond path ban ^/second\n")
+	lines.waitUntil(t, func() bool { return len(files.Match(get(t, "/second"))) == 1 })
+	wantMatched(t, files, get(t, "/edited"), "first")
+	wantFileError()
+}
+
+func TestDefaultFileBansProbesAtTheSiteRootAlone(t *testing.T) {
+	t.Parallel()
+
+	params, _ := newParams(filepath.Join("..", "..", "share", "rules.d"))
+
+	files, err := rules.Load(params)
+	if err != nil {
+		t.Fatalf("load the default file: %v", err)
+	}
+
+	// Probes sent by a browser, by the rule that refuses them.
+	for rule, targets := range map[string][]string{
+		"env-file":    {"/.env", "/.env.production", "/.ENV"},
+		"vcs-dir":     {"/.git/config", "/.git", "/.svn/entries"},
+		"secrets-dir": {"/.aws/credentials", "/.ssh/id_rsa"},
+		"secret-file": {"/.htpasswd", "/.DS_Store", "/.git-credentials"},
+		"editor-dir":  {"/.vscode/sftp.json"},
+		"backup-file": {
+			"/wp-config.php.bak", "/index.php~", "/dump.sql", "/backup.sql.gz",
+		},
+		"log-file":     {"/debug.log"},
+		"compose-file": {"/docker-compose.yml", "/compose.yaml"},
+		"php-shell":    {"/shell.php"},
+		"path-traversal": {
+			"/static/../../etc/passwd", "/f?f=%2e%2e%2f%2e%2e%2fetc%2fpasswd",
+		},
+	} {
+		for _, target := range targets {
+			wantRefusedBy(t, files, target, browser, rule)
+		}
+	}
+
+	// Scanners, by their user agents.
+	for _, scanner := range []string{
+		"sqlmap/1.8.4#stable (https://sqlmap.org)",
+		"Mozilla/5.0 (compatible; Nuclei - Open-source project)",
+	} {
+		wantRefusedBy(t, files, "/", scanner, "scanner-agent")
+	}
+
+	// Ordinary requests to a code forge for files of those names deeper
+	// in its paths, and for other files at its root.
+	for _, target := range []string{
+		"/owner/repo/src/branch/main/.env.example",
+		"/owner/repo/src/branch/main/.env",
+		"/owner/repo/src/branch/main/.github/workflows/ci.yml",
+		"/owner/repo/src/branch/main/.vscode/settings.json",
+		"/owner/repo/src/branch/main/.htaccess",
+		"/owner/repo/src/branch/main/docker-compose.yml",
+		"/owner/repo/src/branch/main/db/schema.sql",
+		"/owner/repo/raw/branch/main/debug.log",
+		"/owner/repo.git/info/refs?service=git-upload-pack",
+		"/owner/repo/src/branch/main/docs/../README.md",
+		"/user/login?redirect_to=%2fowner%2frepo",
+		"/index.php",
+		"/.well-known/security.txt",
+	} {
+		r := get(t, target)
+		r.Header.Set(userAgent, browser)
+
+		matched := files.Match(r)
+		if len(matched) != 0 {
+			t.Errorf("%s matched %v, want no rule", target, ids(matched))
+		}
+	}
+
+	// A request without a user agent is only noted.
+	wantMatched(t, files, get(t, "/"), "empty-agent")
+}
+
+// ruleFiles are files to write into a directory of rule files, by name.
+type ruleFiles map[string]string
+
+// writeFiles writes files into a new directory, and returns it.
+func writeFiles(t *testing.T, files ruleFiles) string {
+	t.Helper()
+
+	dir := t.TempDir()
+
+	for name, content := range files {
+		path := filepath.Join(dir, name)
+
+		err := os.MkdirAll(filepath.Dir(path), 0o700)
+		if err != nil {
+			t.Fatalf("mkdir: %v", err)
+		}
+
+		err = os.WriteFile(path, []byte(content), 0o600)
+		if err != nil {
+			t.Fatalf("write %s: %v", name, err)
+		}
+	}
+
+	return dir
+}
+
+// save writes content to the rule file name in dir as an editor that
+// saves by renaming does, so that the file is never seen half written.
+func save(t *testing.T, dir, name, content string) {
+	t.Helper()
+
+	path := filepath.Join(dir, name)
+
+	err := os.WriteFile(path+".tmp", []byte(content), 0o600)
+	if err != nil {
+		t.Fatalf("write %s: %v", name, err)
+	}
+
+	err = os.Rename(path+".tmp", path)
+	if err != nil {
+		t.Fatalf("rename: %v", err)
+	}
+}
+
+// newParams returns Params for the rule files in dir, switched on, with
+// the process log in the processLog returned, and the alerts waiting in a
+// queue for a webhook that is never sent them.
+func newParams(dir string) (rules.Params, processLog) {
+	lines := make(processLog, maxLogLines)
+
+	return rules.Params{
+		Dir:        dir,
+		Enabled:    true,
+		ProcessLog: slog.New(slog.NewJSONHandler(lines, nil)),
+		Alerts: alerts.New(alerts.Params{
+			WebhookURL: &url.URL{Scheme: "https", Host: "alerts.example"},
+			Events:     alerts.Events(),
+			Cooldown:   15 * time.Minute,
+			Now:        time.Now,
+		}),
+	}, lines
+}
+
+// load writes files into a new directory and loads the rules in it.
+func load(t *testing.T, files ruleFiles) *rules.Files {
+	t.Helper()
+
+	params, _ := newParams(writeFiles(t, files))
+	params.ProcessLog = slog.New(slog.DiscardHandler)
+
+	loaded, err := rules.Load(params)
+	if err != nil {
+		t.Fatalf("load: %v", err)
+	}
+
+	return loaded
+}
+
+// watch loads the rules in dir, runs their Watch until the test ends, and
+// waits until it watches the directory. It returns the alerts' queue as
+// well.
+func watch(t *testing.T, dir string) (*rules.Files, processLog, *alerts.Queue) {
+	t.Helper()
+
+	params, lines := newParams(dir)
+
+	files, err := rules.Load(params)
+	if err != nil {
+		t.Fatalf("load: %v", err)
+	}
+
+	ctx, stop := context.WithCancel(t.Context())
+	stopped := make(chan struct{})
+
+	go func() {
+		files.Watch(ctx)
+		close(stopped)
+	}()
+
+	t.Cleanup(func() {
+		stop()
+		<-stopped
+	})
+
+	lines.waitFor(t, watching)
+
+	return files, lines, params.Alerts
+}
+
+// wantRefused checks that loading the rule files in dir fails with the
+// error want.
+func wantRefused(t *testing.T, dir, want string) {
+	t.Helper()
+
+	params, _ := newParams(dir)
+
+	_, err := rules.Load(params)
+	if err == nil || err.Error() != want {
+		t.Errorf("error %v, want %s", err, want)
+	}
+}
+
+// get returns a GET request for target, a path and an optional query, as
+// smallwebwaf's server reads it, without a user agent.
+func get(t *testing.T, target string) *http.Request {
+	t.Helper()
+
+	return httptest.NewRequestWithContext(t.Context(), http.MethodGet,
+		"http://app.example"+target, nil)
+}
+
+// wantRefusedBy checks that a GET request for target with the user agent
+// sent matches rule alone, and that rule refuses it.
+func wantRefusedBy(t *testing.T, files *rules.Files, target, sent, rule string) {
+	t.Helper()
+
+	r := get(t, target)
+	r.Header.Set(userAgent, sent)
+
+	matched := files.Match(r)
+	if len(matched) != 1 || matched[0].ID != rule || matched[0].Action == rules.ActionLog {
+		t.Errorf("%s from %q matched %v, want %s alone, refusing it", target,
+			sent, ids(matched), rule)
+	}
+}
+
+// wantMatched checks the ids of the rules r matches, in order.
+func wantMatched(t *testing.T, files *rules.Files, r *http.Request, want ...string) {
+	t.Helper()
+
+	got := ids(files.Match(r))
+	if !slices.Equal(got, want) {
+		t.Errorf("%s matched %v, want %v", r.URL, got, want)
+	}
+}
+
+// ids returns the ids of matched.
+func ids(matched []rules.Rule) []string {
+	got := make([]string, 0, len(matched))
+	for _, rule := range matched {
+		got = append(got, rule.ID)
+	}
+
+	return got
+}
+
+// processLog receives the lines of a process log, each a JSON object, for
+// a test to wait for.
+type processLog chan string
+
+// Write receives a line of the process log.
+func (l processLog) Write(line []byte) (int, error) {
+	l <- string(line)
+
+	return len(line), nil
+}
+
+// waitFor returns the next line of the process log whose message is msg,
+// passing over the lines before it. It waits as long as that takes, so
+// that a slow test process cannot fail the test.
+func (l processLog) waitFor(t *testing.T, msg string) map[string]any {
+	t.Helper()
+
+	for line := range l {
+		var fields map[string]any
+
+		err := json.Unmarshal([]byte(line), &fields)
+		if err != nil {
+			t.Fatalf("process log line %q is not JSON: %v", line, err)
+		}
+
+		if fields["msg"] == msg {
+			return fields
+		}
+	}
+
+	return nil
+}
+
+// waitUntil waits for the rule files to be read until done reports true,
+// as it does once they have been read after the test's last change. They
+// can be read before then too, as they are once Watch starts watching.
+func (l processLog) waitUntil(t *testing.T, done func() bool) {
+	t.Helper()
+
+	for !done() {
+		l.waitFor(t, read)
+	}
+}

internal/rules/watch_internal_test.go

@@ -0,0 +1,165 @@
+package rules
+
+import (
+	"context"
+	"log/slog"
+	"net/http"
+	"net/http/httptest"
+	"os"
+	"path/filepath"
+	"slices"
+	"testing"
+	"testing/synctest"
+	"time"
+
+	"github.com/fsnotify/fsnotify"
+
+	"sneak.berlin/go/smallwebwaf/internal/alerts"
+)
+
+// The tests below run readAfterChanges in a synctest bubble, where time is
+// a clock of the test's own: time.Sleep moves it on at once, and
+// synctest.Wait returns once readAfterChanges waits again, so that every
+// reading due by then is done. The test sends the changes itself, as the
+// watch of a directory cannot run in a bubble.
+
+func TestFileWrittenInTwoPartsTakenInOnlyWhole(t *testing.T) {
+	t.Parallel()
+
+	synctest.Test(t, func(t *testing.T) {
+		dir := t.TempDir()
+		path := filepath.Join(dir, "50-app.rules")
+		writeFile(t, path, "first path block ^/first\n")
+		files := load(t, dir)
+		changes := run(t, files)
+
+		file, err := os.Create(path) //nolint:gosec // a file the test wrote
+		if err != nil {
+			t.Fatalf("create: %v", err)
+		}
+
+		defer func() {
+			_ = file.Close()
+		}()
+
+		// The first part ends in the middle of a ban rule's regex, which,
+		// read then, would ban every request.
+		write(t, file, "first path block ^/first\nprobe path ban ^/")
+
+		changes <- fsnotify.Event{Name: path, Op: fsnotify.Write}
+
+		time.Sleep(quietTime - time.Nanosecond)
+		synctest.Wait()
+		wantMatched(t, files, "/anything")
+
+		// The second part starts the wait again.
+		write(t, file, `\.env$`+"\n")
+
+		changes <- fsnotify.Event{Name: path, Op: fsnotify.Write}
+
+		time.Sleep(quietTime - time.Nanosecond)
+		synctest.Wait()
+		wantMatched(t, files, "/.env")
+
+		time.Sleep(time.Nanosecond)
+		synctest.Wait()
+		wantMatched(t, files, "/.env", "probe")
+		wantMatched(t, files, "/anything")
+	})
+}
+
+func TestEditSavedBeforeTheWatchStartsTakenIn(t *testing.T) {
+	t.Parallel()
+
+	synctest.Test(t, func(t *testing.T) {
+		dir := t.TempDir()
+		path := filepath.Join(dir, "50-app.rules")
+		writeFile(t, path, "first path block ^/first\n")
+		files := load(t, dir)
+
+		// Saved after Load read the files, and before the directory was
+		// watched, so that no change is seen for it.
+		writeFile(t, path, "first path block ^/edited\n")
+		run(t, files)
+		time.Sleep(quietTime)
+		synctest.Wait()
+		wantMatched(t, files, "/edited", "first")
+	})
+}
+
+// load loads the rules in dir.
+func load(t *testing.T, dir string) *Files {
+	t.Helper()
+
+	files, err := Load(Params{
+		Dir: dir, Enabled: true, ProcessLog: slog.New(slog.DiscardHandler),
+		Alerts: alerts.New(alerts.Params{}),
+	})
+	if err != nil {
+		t.Fatalf("load: %v", err)
+	}
+
+	return files
+}
+
+// run runs files' readAfterChanges until the test ends, and returns the
+// channel that sends it changes.
+func run(t *testing.T, files *Files) chan<- fsnotify.Event {
+	t.Helper()
+
+	changes := make(chan fsnotify.Event)
+	ctx, stop := context.WithCancel(t.Context())
+	stopped := make(chan struct{})
+
+	go func() {
+		files.readAfterChanges(ctx, changes, nil)
+		close(stopped)
+	}()
+
+	t.Cleanup(func() {
+		stop()
+		<-stopped
+	})
+
+	return changes
+}
+
+// writeFile writes content to the file at path.
+func writeFile(t *testing.T, path, content string) {
+	t.Helper()
+
+	err := os.WriteFile(path, []byte(content), 0o600)
+	if err != nil {
+		t.Fatalf("write %s: %v", path, err)
+	}
+}
+
+// write writes text to the end of file.
+func write(t *testing.T, file *os.File, text string) {
+	t.Helper()
+
+	_, err := file.WriteString(text)
+	if err != nil {
+		t.Fatalf("write: %v", err)
+	}
+}
+
+// wantMatched checks the ids of the rules that a GET request for path
+// matches, in order.
+func wantMatched(t *testing.T, files *Files, path string, want ...string) {
+	t.Helper()
+
+	r := httptest.NewRequestWithContext(t.Context(), http.MethodGet,
+		"http://app.example"+path, nil)
+
+	matched := files.Match(r)
+
+	got := make([]string, 0, len(matched))
+	for _, rule := range matched {
+		got = append(got, rule.ID)
+	}
+
+	if !slices.Equal(got, want) {
+		t.Errorf("%s matched %v, want %v", path, got, want)
+	}
+}

internal/smallwebwaf/healthcheck.go

@@ -23,6 +23,8 @@ var errHealthEndpoint = errors.New("smallwebwaf's health endpoint answered")
 // smallwebwaf answers its health endpoint on 127.0.0.1, at the port in
 // SWWAF_LISTEN_ADDR, and the app accepts connections at the address in
 // SWWAF_UPSTREAM_URL. Otherwise it writes why to stderr and returns 1.
+// It reads no other setting, nor a file that another names, so neither
+// can fail it.
 // args are the arguments after `healthcheck`; it takes none, and given
 // one it names it on stderr and returns 1 without checking anything.
 func HealthCheck(
@@ -50,13 +52,13 @@ func healthCheck(ctx context.Context, lookupEnv func(string) (string, bool)) err
 	ctx, cancel := context.WithTimeout(ctx, healthCheckTimeout)
 	defer cancel()
 
-	cfg, err := config.FromEnvironment(lookupEnv)
+	listenAddr, upstreamURL, err := config.ListenAddrAndUpstreamURL(lookupEnv)
 	if err != nil {
 		return fmt.Errorf("invalid setting: %w", err)
 	}
 
 	// The settings have checked that the address has a port.
-	_, port, _ := net.SplitHostPort(cfg.ListenAddr)
+	_, port, _ := net.SplitHostPort(listenAddr)
 	health := "http://" + net.JoinHostPort("127.0.0.1", port) + proxy.HealthPath
 
 	req, err := http.NewRequestWithContext(ctx, http.MethodGet, health, http.NoBody)
@@ -75,7 +77,7 @@ func healthCheck(ctx context.Context, lookupEnv func(string) (string, bool)) err
 		return fmt.Errorf("%w %s", errHealthEndpoint, res.Status)
 	}
 
-	conn, err := (&net.Dialer{}).DialContext(ctx, "tcp", appAddress(cfg.UpstreamURL))
+	conn, err := (&net.Dialer{}).DialContext(ctx, "tcp", appAddress(upstreamURL))
 	if err != nil {
 		return fmt.Errorf("connect to the app: %w", err)
 	}

internal/smallwebwaf/healthcheck_test.go

@@ -6,6 +6,8 @@ import (
 	"net"
 	"net/http"
 	"net/http/httptest"
+	"os"
+	"path/filepath"
 	"strings"
 	"testing"
 	"time"
@@ -28,6 +30,7 @@ func TestHealthCheck(t *testing.T) {
 		listenAddr:  localhost + ":0",
 		upstreamURL: app.URL,
 		stateDir:    t.TempDir(),
+		rulesDir:    t.TempDir(),
 	}
 
 	go func() {
@@ -42,6 +45,21 @@ func TestHealthCheck(t *testing.T) {
 
 	wantHealthCheck(t, env, 0, "")
 
+	// The health check reads those two settings alone, here given as
+	// files: a removed or invalid token file, or an invalid value of
+	// another setting, does not fail it.
+	for _, other := range []struct{ name, value string }{
+		{"SWWAF_METRICS_TOKEN_FILE", filepath.Join(t.TempDir(), "removed")},
+		{"SWWAF_METRICS_TOKEN_FILE", writeFile(t, "too short\n")},
+		{"SWWAF_MODE", "neither"},
+	} {
+		wantHealthCheck(t, map[string]string{
+			listenAddr + "_FILE":  writeFile(t, ":"+port+"\n"),
+			upstreamURL + "_FILE": writeFile(t, app.URL+"\n"),
+			other.name:            other.value,
+		}, 0, "")
+	}
+
 	app.Close()
 	wantHealthCheck(t, env, 1, "unhealthy: connect to the app: ")
 
@@ -97,3 +115,18 @@ func wantHealthCheck(t *testing.T, env map[string]string, status int, message st
 			got, wrote, status, message)
 	}
 }
+
+// writeFile writes contents to a file in a directory of its own, removed
+// when the test ends, and returns the file's path.
+func writeFile(t *testing.T, contents string) string {
+	t.Helper()
+
+	path := filepath.Join(t.TempDir(), "setting")
+
+	err := os.WriteFile(path, []byte(contents), 0o600)
+	if err != nil {
+		t.Fatalf("write %s: %v", path, err)
+	}
+
+	return path
+}

internal/smallwebwaf/smallwebwaf.go

@@ -1,6 +1,6 @@
-// Package smallwebwaf runs the smallwebwaf process: it reads the settings
-// and the state files, serves requests until it is told to stop, and then
-// stops in an orderly way, writing the state files.
+// Package smallwebwaf runs the smallwebwaf process: it reads the settings,
+// the rule files and the state files, serves requests until it is told to
+// stop, and then stops in an orderly way, writing the state files.
 package smallwebwaf
 
 import (
@@ -15,10 +15,13 @@ import (
 	"syscall"
 	"time"
 
+	"sneak.berlin/go/smallwebwaf/internal/alerts"
 	"sneak.berlin/go/smallwebwaf/internal/config"
 	"sneak.berlin/go/smallwebwaf/internal/lookup"
 	"sneak.berlin/go/smallwebwaf/internal/proxy"
+	"sneak.berlin/go/smallwebwaf/internal/remotelog"
 	"sneak.berlin/go/smallwebwaf/internal/requestlog"
+	"sneak.berlin/go/smallwebwaf/internal/rules"
 	"sneak.berlin/go/smallwebwaf/internal/state"
 )
 
@@ -27,6 +30,11 @@ import (
 // runit and docker wait a little longer before they kill the process.
 const shutdownTimeout = 5 * time.Second
 
+// remoteLogStopTimeout is how long, as smallwebwaf stops, the log lines
+// still waiting are sent to SWWAF_LOG_REMOTE_URL before they are given
+// up. stdout has carried them.
+const remoteLogStopTimeout = 2 * time.Second
+
 // Params are what Run needs from the process.
 type Params struct {
 	// Version is the version of the binary, set when it is built.
@@ -56,11 +64,12 @@ func Main(version string) int {
 	})
 }
 
-// Run reads the settings and the state files, then serves requests until
-// ctx is done. It returns the process's exit status, 1 when smallwebwaf
-// cannot start.
+// Run reads the settings, the rule files and the state files, then serves
+// requests until ctx is done. It returns the process's exit status, 1
+// when smallwebwaf cannot start.
 func Run(ctx context.Context, params Params) int {
-	processLog := requestlog.NewProcessLogger(params.Stdout)
+	processLog := requestlog.NewProcessLogger(params.Stdout,
+		config.InstanceName(params.LookupEnv))
 
 	cfg, err := config.FromEnvironment(params.LookupEnv)
 	if err != nil {
@@ -69,28 +78,54 @@ func Run(ctx context.Context, params Params) int {
 		return 1
 	}
 
-	// The state files give times in UTC.
+	// While SWWAF_LOG_REMOTE_URL is set, every line on stdout from here on
+	// is sent there too.
+	stdout := params.Stdout
+
+	var remote *remotelog.Sender
+
+	if cfg.LogRemoteURL != nil {
+		remote = newRemoteLogSender(cfg)
+		stdout = io.MultiWriter(params.Stdout, remote)
+		processLog = requestlog.NewProcessLogger(stdout, cfg.InstanceName)
+
+		stopSending := startSending(ctx, remote, processLog)
+		defer stopSending()
+	}
+
+	// The state files and the alerts give times in UTC.
 	now := func() time.Time { return time.Now().UTC() }
 
+	alertQueue := newAlertQueue(cfg, now, processLog)
+
+	ruleFiles, err := rules.Load(rules.Params{
+		Dir:        cfg.RulesDir,
+		Enabled:    cfg.RulesEnabled,
+		ProcessLog: processLog,
+		Alerts:     alertQueue,
+	})
+	if err != nil {
+		processLog.Error("cannot use the rule files", "error", err.Error())
+
+		return 1
+	}
+
 	server := proxy.New(proxy.Params{
 		Config:     cfg,
-		RequestLog: params.Stdout,
+		RequestLog: stdout,
 		ProcessLog: processLog,
 		GeoJSURL:   lookup.URL,
 		Now:        now,
+		Rules:      ruleFiles,
+		Alerts:     alertQueue,
 	})
+	if remote != nil {
+		server.Metrics.AddRemoteLog(remote)
+	}
 
-	files, err := state.Load(state.Params{
-		Dir:             cfg.StateDir,
-		WriteDelay:      cfg.StateWriteDelay,
-		CounterInterval: cfg.StateCounterInterval,
-		Ledger:          server.Ledger,
-		Limiter:         server.Limiter,
-		GeoJS:           server.GeoJS,
-		Now:             now,
-		ProcessLog:      processLog,
-		Metrics:         server.Metrics,
-	})
+	server.Metrics.AddAlerts(alertQueue)
+
+	files, err := loadStateFiles(cfg, server, alertQueue, now, processLog)
 	if err != nil {
 		processLog.Error("cannot use the state files", "error", err.Error())
 
@@ -110,15 +145,95 @@ func Run(ctx context.Context, params Params) int {
 		"address", listener.Addr().String(),
 		"settings", cfg)
 
-	return serve(ctx, server.Server, listener, files, processLog)
+	return serve(ctx, server.Server, listener, files, ruleFiles, alertQueue, processLog)
 }
 
-// serve serves requests on listener, and writes the state files as they
-// are due, until ctx is done. Then it gives the requests in progress
-// shutdownTimeout to finish, and writes every state file.
+// loadStateFiles reads the state files into the parts of server and into
+// alertQueue, as state.Load does.
+func loadStateFiles(
+	cfg *config.Config, server *proxy.Server, alertQueue *alerts.Queue,
+	now func() time.Time, processLog *slog.Logger,
+) (*state.Files, error) {
+	return state.Load(state.Params{
+		Dir:             cfg.StateDir,
+		WriteDelay:      cfg.StateWriteDelay,
+		CounterInterval: cfg.StateCounterInterval,
+		Ledger:          server.Ledger,
+		Limiter:         server.Limiter,
+		GeoJS:           server.GeoJS,
+		Alerts:          alertQueue,
+		Now:             now,
+		ProcessLog:      processLog,
+		Metrics:         server.Metrics,
+	})
+}
+
+// newAlertQueue returns the queue of the alerts to the webhook, Slack and
+// ntfy, with the settings for them.
+func newAlertQueue(
+	cfg *config.Config, now func() time.Time, processLog *slog.Logger,
+) *alerts.Queue {
+	return alerts.New(alerts.Params{
+		WebhookURL:     cfg.AlertWebhookURL,
+		WebhookHeaders: cfg.AlertWebhookHeaders,
+		SlackURL:       cfg.AlertSlackWebhookURL,
+		NtfyURL:        cfg.AlertNtfyURL,
+		NtfyToken:      cfg.AlertNtfyToken,
+		Events:         cfg.AlertEvents,
+		Cooldown:       cfg.AlertCooldown,
+		MaxPerHour:     cfg.AlertMaxPerHour,
+		Instance:       cfg.InstanceName,
+		Now:            now,
+		ProcessLog:     processLog,
+	})
+}
+
+// newRemoteLogSender returns a sender of the log lines to
+// SWWAF_LOG_REMOTE_URL, with the settings for it.
+func newRemoteLogSender(cfg *config.Config) *remotelog.Sender {
+	return remotelog.New(remotelog.Params{
+		URL:      cfg.LogRemoteURL,
+		RootCAs:  cfg.LogRemoteTLSCAs,
+		Buffer:   cfg.LogRemoteBuffer,
+		Facility: cfg.LogRemoteFacility,
+		AppName:  cfg.LogRemoteAppName,
+	})
+}
+
+// startSending runs remote until the function it returns is called, which
+// then waits at most remoteLogStopTimeout for the lines still waiting to
+// be sent. Sending goes on after ctx is done, so that the lines written
+// while smallwebwaf stops are sent too.
+func startSending(
+	ctx context.Context, remote *remotelog.Sender, processLog *slog.Logger,
+) func() {
+	sending, stop := context.WithCancel(context.WithoutCancel(ctx))
+	sent := make(chan struct{})
+
+	go func() {
+		remote.Run(sending, processLog)
+		close(sent)
+	}()
+
+	return func() {
+		stop()
+
+		select {
+		case <-sent:
+		case <-time.After(remoteLogStopTimeout):
+		}
+	}
+}
+
+// serve serves requests on listener, writes the state files as they are
+// due, takes in an admin's edits of them, reads the rule files again as
+// they change, and sends the alerts, until ctx is done. Then it gives the
+// requests in progress shutdownTimeout to finish, and writes every state
+// file, alerts.json with the alerts still waiting.
 func serve(
 	ctx context.Context, server *http.Server, listener net.Listener,
-	files *state.Files, processLog *slog.Logger,
+	files *state.Files, ruleFiles *rules.Files, alertQueue *alerts.Queue,
+	processLog *slog.Logger,
 ) int {
 	served := make(chan error, 1)
 
@@ -129,12 +244,10 @@ func serve(
 	writing, stopWriting := context.WithCancel(ctx)
 	defer stopWriting()
 
-	written := make(chan struct{})
-
-	go func() {
-		files.Run(writing)
-		close(written)
-	}()
+	written := inBackground(func() { files.Run(writing) })
+	watched := inBackground(func() { files.Watch(writing) })
+	rulesWatched := inBackground(func() { ruleFiles.Watch(writing) })
+	alertsSent := inBackground(func() { alertQueue.Run(writing) })
 
 	select {
 	case err := <-served:
@@ -165,13 +278,18 @@ func serve(
 		return 1
 	}
 
-	// Run's last write has ended, so nothing else writes the files. Every
-	// request has ended too, but for two kinds that Go's server does not
-	// wait for: one cut off because Shutdown timed out, and one whose
-	// connection switched protocols, such as a WebSocket. Such a request
-	// adds to its client's history only as it ends, which can be after
-	// this write, and then that request is missing from clients.json.
+	// Run and Watch have ended, so nothing else reads or writes the
+	// files, and no alert is being sent, so that alerts.json keeps every
+	// alert not yet sent. Every request has ended too, but for two kinds
+	// that Go's server does not wait for: one cut off because Shutdown
+	// timed out, and one whose connection switched protocols, such as a
+	// WebSocket. Such a request adds to its client's history only as it
+	// ends, which can be after this write, and then that request is
+	// missing from clients.json.
 	<-written
+	<-watched
+	<-rulesWatched
+	<-alertsSent
 
 	err = files.WriteAll()
 	if err != nil {
@@ -184,3 +302,16 @@ func serve(
 
 	return 0
 }
+
+// inBackground runs task on a goroutine of its own, and returns a channel
+// that is closed once task has returned.
+func inBackground(task func()) <-chan struct{} {
+	done := make(chan struct{})
+
+	go func() {
+		task()
+		close(done)
+	}()
+
+	return done
+}

internal/smallwebwaf/smallwebwaf_internal_test.go

@@ -0,0 +1,82 @@
+package smallwebwaf
+
+import (
+	"log/slog"
+	"net/url"
+	"testing"
+	"testing/synctest"
+	"time"
+
+	"sneak.berlin/go/smallwebwaf/internal/remotelog"
+)
+
+// The stop's tests run in a synctest bubble, where the time package runs
+// on a clock of the test's own, so that how long the stop takes can be
+// told exactly. The sender is held up by its process log, not by the
+// network: a goroutine of the bubble that waits on the network keeps that
+// clock from moving on.
+
+func TestStopWaitsForTheSenderToFinish(t *testing.T) {
+	t.Parallel()
+
+	synctest.Test(t, func(t *testing.T) {
+		took := stopHeldSender(t, time.Second)
+		if took != time.Second {
+			t.Errorf("the stop took %s, want the second the sender took", took)
+		}
+	})
+}
+
+func TestStopWaitsForTheSenderAtMostTwoSeconds(t *testing.T) {
+	t.Parallel()
+
+	synctest.Test(t, func(t *testing.T) {
+		took := stopHeldSender(t, time.Minute)
+		if took != 2*time.Second {
+			t.Errorf("the stop took %s, want 2s", took)
+		}
+	})
+}
+
+// heldLog holds each line written to it until it is closed.
+type heldLog chan struct{}
+
+// Write waits until the log is closed.
+func (l heldLog) Write(p []byte) (int, error) {
+	<-l
+
+	return len(p), nil
+}
+
+// stopHeldSender starts sending to an endpoint the sender cannot connect
+// to, holds the sender as it logs that failure until release has passed,
+// stops the sending, and returns how long the stop took. It returns once
+// the sender has ended, as a bubble must.
+func stopHeldSender(t *testing.T, release time.Duration) time.Duration {
+	t.Helper()
+
+	log := make(heldLog)
+	sender := remotelog.New(remotelog.Params{
+		// No port is 65536, so each attempt to connect fails at once,
+		// before it reaches the network.
+		URL:    &url.URL{Scheme: remotelog.SchemeTCP, Host: "127.0.0.1:65536"},
+		Buffer: 1,
+	})
+
+	stopSending := startSending(t.Context(), sender,
+		slog.New(slog.NewJSONHandler(log, nil)))
+
+	synctest.Wait()
+	time.AfterFunc(release, func() { close(log) })
+
+	stopped := time.Now()
+
+	stopSending()
+
+	took := time.Since(stopped)
+
+	time.Sleep(release)
+	synctest.Wait()
+
+	return took
+}

internal/state/state.go

@@ -1,24 +1,34 @@
 // Package state keeps smallwebwaf's state in JSON files in
 // SWWAF_STATE_DIR, as the "Persistent state" section of SPEC.md describes:
 // bans.json holds the bans, clients.json each client's counters and
-// history, and lookups.json GeoJS's answers. Load reads them at start, and
-// Run and WriteAll write them, each from a snapshot its part takes under
-// its own lock, so that no request waits on the disk.
+// history, lookups.json GeoJS's answers, and alerts.json the cooldowns,
+// the hour under way and the alerts waiting for each destination. Load
+// reads them at start, Watch takes in an admin's edit of one while
+// smallwebwaf runs, and Run and WriteAll write them. The disk is read and
+// written outside the parts' locks, which are held only to take a
+// snapshot or to put in what a file holds, so that no request waits on
+// the disk.
 package state
 
 import (
 	"bytes"
 	"context"
+	"crypto/sha256"
 	"encoding/json"
 	"errors"
 	"fmt"
 	"io/fs"
 	"log/slog"
+	"maps"
 	"net/netip"
 	"os"
 	"path/filepath"
+	"slices"
+	"sync"
 	"time"
 
+	"github.com/fsnotify/fsnotify"
+	"sneak.berlin/go/smallwebwaf/internal/alerts"
 	"sneak.berlin/go/smallwebwaf/internal/bans"
 	"sneak.berlin/go/smallwebwaf/internal/lookup"
 	"sneak.berlin/go/smallwebwaf/internal/metrics"
@@ -37,12 +47,18 @@ const (
 	bansJSON    = "bans.json"
 	clientsJSON = "clients.json"
 	lookupsJSON = "lookups.json"
+	alertsJSON  = "alerts.json"
 )
 
 var (
 	errVersion = errors.New("unknown version")
 	// errMissing is for an entry without a field it needs.
-	errMissing = errors.New("has no")
+	errMissing     = errors.New("has no")
+	errCause       = errors.New("is not limit, attack or admin")
+	errDestination = errors.New("is not webhook, slack or ntfy")
+	errWaitingList = errors.New(`waiting is a list, but now lists the alerts by ` +
+		`destination: put the list under "webhook", as "waiting": {"webhook": [...]}, ` +
+		`or remove the file`)
 )
 
 // Params are what Load needs.
@@ -54,36 +70,55 @@ type Params struct {
 	// is (SWWAF_STATE_COUNTER_INTERVAL).
 	WriteDelay      time.Duration
 	CounterInterval time.Duration
-	// Ledger, Limiter and GeoJS hold the state.
+	// Ledger, Limiter, GeoJS and Alerts hold the state. Alerts also
+	// receive a file_error alert for an edit set aside, and for a write
+	// that fails while smallwebwaf runs.
 	Ledger  *bans.Ledger
 	Limiter *ratelimit.Limiter
 	GeoJS   *lookup.GeoJS
+	Alerts  *alerts.Queue
 	// Now tells the time by which the counters' buckets run out, normally
 	// time.Now in UTC.
 	Now func() time.Time
-	// ProcessLog receives what was read, and the writes that fail.
+	// ProcessLog receives what was read and taken in, the edits set aside,
+	// and the writes that fail.
 	ProcessLog *slog.Logger
-	// Metrics count each file's writes.
+	// Metrics count each file's writes, and the edits taken in and set
+	// aside.
 	Metrics *metrics.Metrics
 }
 
 // Files are the state files of a running smallwebwaf.
 type Files struct {
 	params Params
+
+	// mu is held while a file is read for an edit, and while it is
+	// written, so that Watch and the writes take turns. No request takes
+	// it.
+	mu sync.Mutex
+	// sums are the SHA-256 sums of what each file held, by name, when
+	// smallwebwaf last read or wrote it. A file that holds anything else
+	// has been edited since.
+	sums map[string][sha256.Size]byte
 }
 
 // bansFile is bans.json, indented for an admin to read and edit.
 type bansFile struct {
 	Version int        `json:"version"`
-	Bans    []banEntry `json:"bans"`
+	Bans    []BanEntry `json:"bans"`
 }
 
-// banEntry is a ban as bans.json holds it: a permanent ban's expires is
-// null.
-type banEntry struct {
+// BanEntry is a ban as bans.json holds it: a permanent ban's expires is
+// null, a ban an admin added may have no cause, which makes it an
+// admin's, and lifted is left out until an admin lifts the ban. The ban
+// endpoints answer with bans in this form too.
+type BanEntry struct {
 	Netblock netip.Prefix `json:"netblock"`
 	Start    time.Time    `json:"start"`
 	Expires  *time.Time   `json:"expires"`
+	Cause    string       `json:"cause"`
+	Reason   string       `json:"reason,omitempty"`
+	Lifted   *time.Time   `json:"lifted,omitempty"`
 	Notes    bans.Notes   `json:"notes"`
 }
 
@@ -99,6 +134,14 @@ type lookupsFile struct {
 	Lookups []lookup.Answer `json:"lookups"`
 }
 
+// alertsFile is alerts.json, indented for an admin to read and edit.
+type alertsFile struct {
+	Version   int                       `json:"version"`
+	Cooldowns []alerts.Cooldown         `json:"cooldowns"`
+	Hour      alerts.Hour               `json:"hour"`
+	Waiting   map[string][]alerts.Alert `json:"waiting"`
+}
+
 // stateFile is the struct of a state file. Once the file is decoded, its
 // check refuses the first entry without a field it needs, which would
 // otherwise be read as something the entry does not say. data is the
@@ -120,41 +163,30 @@ func Load(params Params) (*Files, error) {
 		return nil, fmt.Errorf("SWWAF_STATE_DIR cannot be written: %w", err)
 	}
 
-	var (
-		bansIn    bansFile
-		clientsIn clientsFile
-		lookupsIn lookupsFile
-	)
+	f := &Files{params: params, sums: map[string][sha256.Size]byte{}}
 
-	err = errors.Join(
-		read(params.Dir, bansJSON, &bansIn),
-		read(params.Dir, clientsJSON, &clientsIn),
-		read(params.Dir, lookupsJSON, &lookupsIn),
-	)
+	bansRead, bansErr := f.read(bansJSON)
+	clientsRead, clientsErr := f.read(clientsJSON)
+	lookupsRead, lookupsErr := f.read(lookupsJSON)
+	alertsRead, alertsErr := f.read(alertsJSON)
+
+	err = errors.Join(bansErr, clientsErr, lookupsErr, alertsErr)
 	if err != nil {
 		return nil, err
 	}
 
-	held := make([]bans.Ban, 0, len(bansIn.Bans))
-	for _, entry := range bansIn.Bans {
-		held = append(held, entry.ban())
-	}
-
-	params.Ledger.Load(held)
-	params.Limiter.Load(clientsIn.Clients, params.Now())
-	params.GeoJS.Load(lookupsIn.Lookups)
-
 	params.ProcessLog.Info("read the state files", "directory", params.Dir,
-		"bans", len(bansIn.Bans), "clients", len(clientsIn.Clients),
-		"lookups", len(lookupsIn.Lookups))
+		"bans", bansRead, "clients", clientsRead, "lookups", lookupsRead,
+		"alerts_waiting", alertsRead)
 
-	return &Files{params: params}, nil
+	return f, nil
 }
 
 // Run writes bans.json WriteDelay after a ban is made, with every ban
 // made in between, and every file every CounterInterval, until ctx is
-// done. A write that fails is logged, and the file is written again at
-// its next write.
+// done. A write that fails is logged, raised as a file_error alert, and
+// the file is written again at its next write. Each write takes in an
+// admin's edit of its file first, as writeFile describes.
 func (f *Files) Run(ctx context.Context) {
 	interval := time.NewTicker(f.params.CounterInterval)
 	defer interval.Stop()
@@ -172,9 +204,11 @@ func (f *Files) Run(ctx context.Context) {
 		case <-bansDue:
 			bansDue = nil
 
-			f.logFailure(f.writeBans())
+			f.logFailure(bansJSON, f.writeFile(bansJSON))
 		case <-interval.C:
-			f.logFailure(f.WriteAll())
+			for _, name := range []string{bansJSON, clientsJSON, lookupsJSON, alertsJSON} {
+				f.logFailure(name, f.writeFile(name))
+			}
 		}
 	}
 }
@@ -182,80 +216,366 @@ func (f *Files) Run(ctx context.Context) {
 // WriteAll writes every state file, as smallwebwaf stops. A file that
 // fails does not keep the others from being written.
 func (f *Files) WriteAll() error {
-	return errors.Join(f.writeBans(), f.writeClients(), f.writeLookups())
+	return errors.Join(f.writeFile(bansJSON), f.writeFile(clientsJSON),
+		f.writeFile(lookupsJSON), f.writeFile(alertsJSON))
 }
 
-// logFailure logs a write that failed.
-func (f *Files) logFailure(err error) {
+// Watch watches Dir until ctx is done, and takes in an admin's edit of a
+// state file as soon as it is saved: what the file holds replaces what
+// smallwebwaf held for it. An edit that does not parse is left for the
+// file's next write, which sets it aside, since a file can be read while
+// an editor is still writing it. If Dir cannot be watched, that is
+// logged, and an edit is taken in only before its file is written.
+func (f *Files) Watch(ctx context.Context) {
+	watcher, err := fsnotify.NewWatcher()
+	if err == nil {
+		defer func() {
+			_ = watcher.Close()
+		}()
+
+		err = watcher.Add(f.params.Dir)
+	}
+
 	if err != nil {
-		f.params.ProcessLog.Error("writing the state files failed",
+		f.params.ProcessLog.Error("cannot watch the state files for edits",
 			"error", err.Error())
+
+		return
+	}
+
+	f.params.ProcessLog.Info("watching the state files for edits",
+		"directory", f.params.Dir)
+
+	for {
+		select {
+		case <-ctx.Done():
+			return
+		case event := <-watcher.Events:
+			switch name := filepath.Base(event.Name); name {
+			case bansJSON, clientsJSON, lookupsJSON, alertsJSON:
+				f.fileChanged(name)
+			}
+		case err = <-watcher.Errors:
+			f.params.ProcessLog.Warn("watching the state files failed",
+				"error", err.Error())
+		}
 	}
 }
 
-// writeBans writes bans.json.
-func (f *Files) writeBans() error {
-	held := f.params.Ledger.Snapshot()
-
-	file := bansFile{Version: version, Bans: make([]banEntry, 0, len(held))}
-	for _, ban := range held {
-		file.Bans = append(file.Bans, newBanEntry(ban))
-	}
-
-	data, err := json.MarshalIndent(file, "", "    ")
+// logFailure logs a write of the state file name that failed, and raises
+// a file_error alert for it.
+func (f *Files) logFailure(name string, err error) {
 	if err != nil {
-		return fmt.Errorf("encode %s: %w", bansJSON, err)
-	}
+		const failed = "writing the state files failed"
 
-	return f.writeCounted(bansJSON, append(data, '\n'))
+		// Raised before it is logged, so that the alert is there once the
+		// log line is.
+		f.params.Alerts.Raise(alerts.Alert{
+			Event:  alerts.EventFileError,
+			Reason: failed,
+			Detail: map[string]any{
+				"file": filepath.Join(f.params.Dir, name), "error": err.Error(),
+			},
+		})
+		f.params.ProcessLog.Error(failed, "error", err.Error())
+	}
 }
 
-// writeClients writes clients.json.
-func (f *Files) writeClients() error {
-	data, err := encodeOnePerLine("clients", f.params.Limiter.Snapshot())
+// fileChanged takes in what the state file name holds, as Watch sees it
+// change, if that is an edit made since smallwebwaf last read or wrote
+// the file. A file that cannot be read or does not parse is left for its
+// next write.
+func (f *Files) fileChanged(name string) {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+
+	data, changed, err := f.readChanged(name)
+	if err != nil || !changed {
+		return
+	}
+
+	_ = f.takeInEdit(name, data)
+}
+
+// takeInEdit takes in data, an edit of the state file name, as takeIn
+// does, and counts and logs it. Every edit taken in while smallwebwaf
+// runs, by Watch or by a write, is taken in here. An edit that does not
+// parse is neither counted nor logged, and takeIn's error returned.
+func (f *Files) takeInEdit(name string, data []byte) error {
+	_, err := f.takeIn(name, data, true)
 	if err != nil {
-		return fmt.Errorf("encode %s: %w", clientsJSON, err)
+		return err
 	}
 
-	return f.writeCounted(clientsJSON, data)
+	// Counted before it is logged, so that the count is there once the
+	// log line is.
+	f.params.Metrics.StateFileEditTakenIn(name)
+	f.params.ProcessLog.Info("took in an edit of a state file",
+		"file", filepath.Join(f.params.Dir, name))
+
+	return nil
 }
 
-// writeLookups writes lookups.json.
-func (f *Files) writeLookups() error {
-	data, err := encodeOnePerLine("lookups", f.params.GeoJS.Snapshot())
+// read takes in the state file name at start, and returns how many
+// entries it holds. A missing file holds none.
+func (f *Files) read(name string) (int, error) {
+	data, changed, err := f.readChanged(name)
+	if err != nil || !changed {
+		return 0, err
+	}
+
+	return f.takeIn(name, data, false)
+}
+
+// readChanged returns what the state file name holds, and whether that
+// has changed since smallwebwaf last read or wrote the file, as it has
+// for a file smallwebwaf never read or wrote. A missing file has not
+// changed: it is written again at its next write.
+func (f *Files) readChanged(name string) ([]byte, bool, error) {
+	path := filepath.Join(f.params.Dir, name)
+
+	data, err := os.ReadFile(path) //nolint:gosec // a state file, in SWWAF_STATE_DIR
+	if errors.Is(err, fs.ErrNotExist) {
+		return nil, false, nil
+	}
+
 	if err != nil {
-		return fmt.Errorf("encode %s: %w", lookupsJSON, err)
+		return nil, false, err
 	}
 
-	return f.writeCounted(lookupsJSON, data)
+	return data, sha256.Sum256(data) != f.sums[name], nil
 }
 
-// writeCounted writes data to the state file name, as write does, and
-// counts the write in the metrics.
-func (f *Files) writeCounted(name string, data []byte) error {
-	err := write(f.params.Dir, name, data)
+// takeIn parses data, what the state file name holds, puts it into the
+// part that keeps that state, in place of what the part held, and returns
+// how many entries the file holds. edit is whether data is an admin's
+// edit taken in while smallwebwaf runs, rather than the file read at the
+// start. An error names the file and, where the JSON decoder tells it,
+// the line and column, or else the entry.
+func (f *Files) takeIn(name string, data []byte, edit bool) (int, error) {
+	path := filepath.Join(f.params.Dir, name)
+
+	var entries int
+
+	switch name {
+	case bansJSON:
+		var file bansFile
+
+		err := parse(path, data, &file)
+		if err != nil {
+			return 0, err
+		}
+
+		held := make([]bans.Ban, 0, len(file.Bans))
+		for _, entry := range file.Bans {
+			held = append(held, entry.ban())
+		}
+
+		if edit {
+			f.params.Ledger.LoadEdit(held)
+		} else {
+			f.params.Ledger.Load(held)
+		}
+
+		entries = len(held)
+	case clientsJSON:
+		var file clientsFile
+
+		err := parse(path, data, &file)
+		if err != nil {
+			return 0, err
+		}
+
+		f.params.Limiter.Load(file.Clients, f.params.Now())
+		entries = len(file.Clients)
+	case lookupsJSON:
+		var file lookupsFile
+
+		err := parse(path, data, &file)
+		if err != nil {
+			return 0, err
+		}
+
+		f.params.GeoJS.Load(file.Lookups)
+		entries = len(file.Lookups)
+	case alertsJSON:
+		// waiting was a list, of the alerts waiting for the webhook, before
+		// alerts went to Slack and ntfy too.
+		var written struct {
+			Waiting json.RawMessage `json:"waiting"`
+		}
+
+		if json.Unmarshal(data, &written) == nil &&
+			bytes.HasPrefix(written.Waiting, []byte("[")) {
+			return 0, fmt.Errorf("%s: %w", path, errWaitingList)
+		}
+
+		var file alertsFile
+
+		err := parse(path, data, &file)
+		if err != nil {
+			return 0, err
+		}
+
+		f.params.Alerts.Load(alerts.State{
+			Cooldowns: file.Cooldowns, Hour: file.Hour, Waiting: file.Waiting,
+		})
+
+		for _, waiting := range file.Waiting {
+			entries += len(waiting)
+		}
+	}
+
+	f.sums[name] = sha256.Sum256(data)
+
+	return entries, nil
+}
+
+// writeFile writes the state file name from what smallwebwaf holds. An
+// edit made since smallwebwaf last read or wrote the file is taken in
+// first, so that it is not overwritten, or set aside if it does not
+// parse. A file that cannot be read, or an edit that cannot be set
+// aside, is left as it is, and the write given up. Every write is counted
+// in the metrics, and one that fails or is given up as a failure.
+func (f *Files) writeFile(name string) error {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+
+	data, changed, err := f.readChanged(name)
+	if err == nil && changed {
+		err = f.takeInEdit(name, data)
+		if err != nil {
+			err = f.setAside(name, err)
+		}
+	}
+
+	if err == nil {
+		data, err = f.encode(name)
+		if err != nil {
+			err = fmt.Errorf("encode %s: %w", name, err)
+		}
+	}
+
+	if err == nil {
+		err = write(f.params.Dir, name, data)
+	}
+
+	if err == nil {
+		// The file holds data from here on, even if the directory sync
+		// fails, so that its next read does not take it for an admin's
+		// edit.
+		f.sums[name] = sha256.Sum256(data)
+		err = syncDirectory(f.params.Dir)
+	}
+
 	f.params.Metrics.StateFileWritten(name, len(data), err)
 
 	return err
 }
 
+// setAside renames the state file name, an edit that does not parse with
+// parseErr, to name.bad, for the admin to mend, and logs it with where in
+// the file the error is, and raises a file_error alert for it. If the
+// rename fails, the edit is left as it is, and the error returned is
+// parseErr joined with the rename's.
+func (f *Files) setAside(name string, parseErr error) error {
+	path := filepath.Join(f.params.Dir, name)
+
+	err := os.Rename(path, path+".bad")
+	if err != nil {
+		return errors.Join(parseErr, err)
+	}
+
+	const setAside = "set aside an edit of a state file that does not parse"
+
+	// Raised before it is logged, so that the alert is there once the log
+	// line is.
+	f.params.Alerts.Raise(alerts.Alert{
+		Event:  alerts.EventFileError,
+		Reason: setAside,
+		Detail: map[string]any{"file": path + ".bad", "error": parseErr.Error()},
+	})
+	f.params.ProcessLog.Error(setAside, "file", path+".bad", "error", parseErr.Error())
+	f.params.Metrics.StateFileEditSetAside(name)
+
+	return nil
+}
+
+// encode returns the state file name as smallwebwaf writes it, from a
+// snapshot of the part that keeps that state.
+func (f *Files) encode(name string) ([]byte, error) {
+	switch name {
+	case bansJSON:
+		file := bansFile{Version: version, Bans: BanEntries(f.params.Ledger.Snapshot())}
+
+		data, err := json.MarshalIndent(file, "", "    ")
+		if err != nil {
+			return nil, err
+		}
+
+		return append(data, '\n'), nil
+	case clientsJSON:
+		return encodeOnePerLine("clients", f.params.Limiter.Snapshot())
+	case lookupsJSON:
+		return encodeOnePerLine("lookups", f.params.GeoJS.Snapshot())
+	default: // alerts.json
+		held := f.params.Alerts.Snapshot()
+		file := alertsFile{
+			Version: version, Cooldowns: held.Cooldowns, Hour: held.Hour,
+			Waiting: held.Waiting,
+		}
+
+		data, err := json.MarshalIndent(file, "", "    ")
+		if err != nil {
+			return nil, err
+		}
+
+		return append(data, '\n'), nil
+	}
+}
+
+// BanEntries returns held as bans.json lists them, an empty list for
+// none.
+func BanEntries(held []bans.Ban) []BanEntry {
+	entries := make([]BanEntry, 0, len(held))
+	for _, ban := range held {
+		entries = append(entries, newBanEntry(ban))
+	}
+
+	return entries
+}
+
 // newBanEntry returns ban as bans.json holds it.
-func newBanEntry(ban bans.Ban) banEntry {
-	entry := banEntry{Netblock: ban.Netblock, Start: ban.Start, Notes: ban.Notes}
+func newBanEntry(ban bans.Ban) BanEntry {
+	entry := BanEntry{
+		Netblock: ban.Netblock, Start: ban.Start, Cause: ban.Cause, Reason: ban.Reason,
+		Notes: ban.Notes,
+	}
 	if !ban.Permanent() {
 		entry.Expires = &ban.Expires
 	}
 
+	if !ban.Lifted.IsZero() {
+		entry.Lifted = &ban.Lifted
+	}
+
 	return entry
 }
 
 // ban returns the ban an entry of bans.json holds.
-func (e banEntry) ban() bans.Ban {
-	ban := bans.Ban{Netblock: e.Netblock, Start: e.Start, Notes: e.Notes}
+func (e BanEntry) ban() bans.Ban {
+	ban := bans.Ban{
+		Netblock: e.Netblock, Start: e.Start, Cause: e.Cause, Reason: e.Reason,
+		Notes: e.Notes,
+	}
 	if e.Expires != nil {
 		ban.Expires = *e.Expires
 	}
 
+	if e.Lifted != nil {
+		ban.Lifted = *e.Lifted
+	}
+
 	return ban
 }
 
@@ -263,7 +583,8 @@ func (e banEntry) ban() bans.Ban {
 // client, a start, from which the length of the netblock's next ban is
 // worked out, or an expires, which would make it permanent. A permanent
 // ban's expires is null, which Bans cannot tell from a missing one, so
-// each expires is read again as written.
+// each expires is read again as written. A cause other than limit,
+// attack or admin, most likely misspelt, is refused too.
 func (f *bansFile) check(data []byte) error {
 	var written struct {
 		Bans []struct {
@@ -284,6 +605,9 @@ func (f *bansFile) check(data []byte) error {
 			return missing(i, "start")
 		case written.Bans[i].Expires == nil:
 			return missing(i, "expires")
+		case entry.Cause != "" && entry.Cause != bans.CauseLimit &&
+			entry.Cause != bans.CauseAttack && entry.Cause != bans.CauseAdmin:
+			return fmt.Errorf("entry %d's cause %q %w", i+1, entry.Cause, errCause)
 		}
 	}
 
@@ -341,6 +665,38 @@ func (f *lookupsFile) check(data []byte) error {
 	return nil
 }
 
+// check refuses a cooldown without its event or when its alert was sent,
+// which would hold back no repeat, alerts waiting for a destination with
+// another name than webhook, slack or ntfy, most likely misspelt, and an
+// alert waiting without its event or its time.
+func (f *alertsFile) check([]byte) error {
+	for i, cooldown := range f.Cooldowns {
+		switch {
+		case cooldown.Event == "":
+			return fmt.Errorf("cooldowns %w", missing(i, "event"))
+		case cooldown.Sent.IsZero():
+			return fmt.Errorf("cooldowns %w", missing(i, "sent"))
+		}
+	}
+
+	for _, destination := range slices.Sorted(maps.Keys(f.Waiting)) {
+		if !slices.Contains(alerts.Destinations(), destination) {
+			return fmt.Errorf("waiting %q %w", destination, errDestination)
+		}
+
+		for i, alert := range f.Waiting[destination] {
+			switch {
+			case alert.Event == "":
+				return fmt.Errorf("waiting %s %w", destination, missing(i, "event"))
+			case alert.Time.IsZero():
+				return fmt.Errorf("waiting %s %w", destination, missing(i, "time"))
+			}
+		}
+	}
+
+	return nil
+}
+
 // countsWithoutStart reports whether b holds requests but no start, which
 // places them in time.
 func countsWithoutStart(b ratelimit.Buckets) bool {
@@ -389,28 +745,16 @@ func checkWritable(dir string) error {
 	return errors.Join(file.Close(), os.Remove(file.Name()))
 }
 
-// read reads the state file name in dir into file, a pointer to that
-// file's struct, and checks its entries. A missing file leaves file as it
-// is.
-func read(dir, name string, file stateFile) error {
-	path := filepath.Join(dir, name)
-
-	data, err := os.ReadFile(path) //nolint:gosec // a state file, in SWWAF_STATE_DIR
-	if errors.Is(err, fs.ErrNotExist) {
-		return nil
-	}
-
-	if err != nil {
-		return err
-	}
-
+// parse reads data, what the state file at path holds, into file, a
+// pointer to that file's struct, and checks its entries.
+func parse(path string, data []byte, file stateFile) error {
 	// The version is read first, so that a file of another version is
 	// refused for that, and not for an entry this version cannot read.
 	var header struct {
 		Version int `json:"version"`
 	}
 
-	err = json.Unmarshal(data, &header)
+	err := json.Unmarshal(data, &header)
 	if err == nil && header.Version != version {
 		err = fmt.Errorf("%w %d, where this smallwebwaf reads version %d",
 			errVersion, header.Version, version)
@@ -463,7 +807,7 @@ func position(data []byte, err error) string {
 // write writes data to the file name in dir so that a crash at any
 // moment leaves either the old file or the new one, whole: data goes to a
 // temporary file in the same directory, which is synced and renamed over
-// name, and then the directory is synced, so that the rename lasts.
+// name. syncDirectory must follow, so that the rename lasts.
 func write(dir, name string, data []byte) error {
 	path := filepath.Join(dir, name)
 	temporary := path + ".tmp"
@@ -475,10 +819,13 @@ func write(dir, name string, data []byte) error {
 
 	if err != nil {
 		_ = os.Remove(temporary)
-
-		return err
 	}
 
+	return err
+}
+
+// syncDirectory syncs dir to the disk, so that a rename in it lasts.
+func syncDirectory(dir string) error {
 	directory, err := os.Open(dir) //nolint:gosec // SWWAF_STATE_DIR itself
 	if err != nil {
 		return err

internal/state/write_internal_test.go

@@ -0,0 +1,35 @@
+package state
+
+import (
+	"os"
+	"path/filepath"
+	"testing"
+)
+
+// The test is on write itself: a state file is read before it is
+// written, and a directory in its place fails that read first.
+func TestFailedRenameLeavesNoTemporaryFile(t *testing.T) {
+	t.Parallel()
+
+	dir := t.TempDir()
+
+	// A directory named bans.json cannot be renamed over.
+	err := os.Mkdir(filepath.Join(dir, bansJSON), 0o700)
+	if err != nil {
+		t.Fatalf("mkdir: %v", err)
+	}
+
+	err = write(dir, bansJSON, []byte("{}\n"))
+	if err == nil {
+		t.Error("writing over a directory did not fail")
+	}
+
+	entries, err := os.ReadDir(dir)
+	if err != nil {
+		t.Fatalf("read %s: %v", dir, err)
+	}
+
+	if len(entries) != 1 || entries[0].Name() != bansJSON {
+		t.Errorf("%s holds %v, want only bans.json", dir, entries)
+	}
+}

script/example-app

@@ -3,9 +3,10 @@
 # deploy/example-app, then run the app's container with a volume for the
 # state files and check that the health check passes, that a request is
 # served through smallwebwaf, that a second one in a minute bans the
-# client, that `sv stop` stops smallwebwaf in order, that `docker stop`
-# stops the container without having to kill it, and that a new
-# container on the same volume still refuses the banned client. The
+# client, that a probe for /.env bans another client, which its next
+# request bans for good, that `sv stop` stops smallwebwaf in order, that
+# `docker stop` stops the container without having to kill it, and that
+# a new container on the same volume still refuses the banned client. The
 # containers, the volume and both images are removed however the script
 # ends. Building the app needs network access, for nixpkgs' binary cache.
 # script/check does not run this.
@@ -52,9 +53,13 @@ healthy() {
     [ "$status" = healthy ]
 }
 
-# logged <text>: the container's output holds text.
+# logged <text>...: a line of the container's output holds every text,
+# in any order.
 logged() {
-    docker logs "$CONTAINER" 2>&1 | grep -qF "$1"
+    lines="$(docker logs "$CONTAINER" 2>&1)"
+    for text in "$@"; do
+        lines="$(printf '%s\n' "$lines" | grep -F "$text")" || return 1
+    done
 }
 
 # start_container: run the app's container, with the state files on the
@@ -77,6 +82,15 @@ refused() {
     [ "$code" = 403 ]
 }
 
+# refused_from <client> <path>: a request for path from client, as
+# X-Forwarded-For names it, gets 403. smallwebwaf believes the header
+# from docker's gateway, a private address.
+refused_from() {
+    code="$(curl --silent --output /dev/null --write-out '%{http_code}' \
+        --max-time 10 --header "X-Forwarded-For: $1" "http://$address$2")" || true
+    [ "$code" = 403 ]
+}
+
 main() {
     cd "$ROOT"
     trap cleanup EXIT
@@ -100,6 +114,14 @@ main() {
     wait_for "smallwebwaf logged no ban" logged '"action":"rate_limited"'
     echo "example-app: a second request in a minute bans the client"
 
+    refused_from 203.0.113.9 /.env || fail "a probe for /.env was not refused"
+    wait_for "smallwebwaf logged no ban for the probe" \
+        logged '"action":"banned"' '"rule_ids":["env-file"]'
+    refused_from 203.0.113.9 / || fail "the client of the probe was let through"
+    wait_for "the client's next request did not make its ban permanent" \
+        logged '"ban_expires":"permanent"'
+    echo "example-app: a probe for /.env bans the client, its next request for good"
+
     docker exec "$CONTAINER" sv stop smallwebwaf >/dev/null ||
         fail "sv stop smallwebwaf failed"
     wait_for "smallwebwaf did not stop in order" logged '"msg":"stopped"'

script/run

@@ -1,7 +1,9 @@
 #!/bin/sh
 # script/run: build bin/smallwebwaf with script/build and run it, with
 # the settings in the environment. Unless SWWAF_STATE_DIR is set, the
-# state files go in bin/state, beside the binary.
+# state files go in bin/state, beside the binary, and unless
+# SWWAF_RULES_DIR is set, the rule files are those of share/rules.d,
+# which the image ships.
 set -eu
 
 SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd -P)"
@@ -14,6 +16,10 @@ main() {
         export SWWAF_STATE_DIR
         mkdir -p "$SWWAF_STATE_DIR"
     fi
+    if [ -z "${SWWAF_RULES_DIR+set}" ]; then
+        SWWAF_RULES_DIR="$ROOT/share/rules.d"
+        export SWWAF_RULES_DIR
+    fi
     exec "$ROOT/bin/smallwebwaf"
 }
 

share/rules.d/00-default.rules

@@ -0,0 +1,15 @@
+# 00-default.rules: probes no real visitor sends, anchored at the site root
+
+# id             target      action  regex
+env-file         path        ban     (?i)^/\.env(\.[a-z]+)?$
+vcs-dir          path        ban     (?i)^/\.(git|svn|hg|bzr)(/|$)
+secrets-dir      path        ban     (?i)^/\.(aws|ssh|docker|kube)/
+secret-file      path        ban     (?i)^/\.(htpasswd|htaccess|npmrc|netrc|pgpass|git-credentials|bash_history|DS_Store)$
+editor-dir       path        ban     (?i)^/\.(vscode|idea)/
+backup-file      path        ban     (?i)^/[^/]+\.(php(\.[a-z0-9]+|~)|sql(\.[a-z0-9]+)?)$
+log-file         path        ban     (?i)^/(debug|error|access)\.log$
+compose-file     path        ban     (?i)^/(docker-)?compose\.ya?ml$
+php-shell        path        ban     (?i)^/(shell|c99|r57|wso|alfa)\.php$
+scanner-agent    user_agent  ban     (?i)\b(sqlmap|nikto|nuclei|masscan|zgrab|wpscan)\b
+path-traversal   uri         block   (\.\./){2,}
+empty-agent      user_agent  log     ^$