smallwebwaf/internal/sww/sww_test.go

package sww_test

import (
	"bytes"
	"context"
	"crypto/rand"
	"encoding/json"
	"io"
	"net"
	"net/http"
	"net/http/httptest"
	"os"
	"path/filepath"
	"slices"
	"strconv"
	"strings"
	"sync"
	"sync/atomic"
	"testing"
	"time"

	"github.com/spf13/viper"

	"sneak.berlin/go/smallwebwaf/internal/config"
	"sneak.berlin/go/smallwebwaf/internal/discriminator"
	"sneak.berlin/go/smallwebwaf/internal/lookup/lookuptest"
	"sneak.berlin/go/smallwebwaf/internal/sww"
)

const (
	// waitLimit bounds how long a test waits for what should happen.
	waitLimit = 10 * time.Second
	// pollInterval is how often a test looks for a line.
	pollInterval = 10 * time.Millisecond
	// testVersion is the version the tests give sww.
	testVersion = "test"
	// localhost is where the tests listen.
	localhost            = "127.0.0.1"
	listenAddr           = "SWW_WAF_LISTEN_ADDR"
	stateDir             = "SWW_WAF_STATE_DIR"
	stateWriteDelay      = "SWW_WAF_STATE_WRITE_DELAY"
	stateCounterInterval = "SWW_WAF_STATE_COUNTER_INTERVAL"
	rateLimitPerDay      = "SWW_WAF_RATE_LIMIT_PER_DAY"
	rateLimitExemptNets  = "SWW_WAF_RATE_LIMIT_EXEMPT_NETS"
	rulesDir             = "SWW_WAF_RULES_DIR"
	lookupSource         = "SWW_WAF_LOOKUP_SOURCE"
	lookupDBPath         = "SWW_WAF_LOOKUP_DB_PATH"
	instanceName         = "SWW_WAF_INSTANCE_NAME"
	adminToken           = "SWW_WAF_ADMIN_TOKEN"   //nolint:gosec // the setting's name
	metricsToken         = "SWW_WAF_METRICS_TOKEN" //nolint:gosec // the setting's name
	// adminSecret is the SWW_WAF_ADMIN_TOKEN the tests set.
	adminSecret = "fedcba9876543210fedcba9876543210"
	// instance is the SWW_WAF_INSTANCE_NAME the tests set where they look at
	// it.
	instance = "fsn1app1/blog"
	// greeting is what the tests' app answers.
	greeting = "hello from the app"
	// placed is the client the tests' lookup databases place.
	placed = "203.0.113.9"
)

// output collects what sww writes on stdout.
type output struct {
	mu  sync.Mutex
	buf bytes.Buffer
}

// Write adds lines sww writes.
func (o *output) Write(p []byte) (int, error) {
	o.mu.Lock()
	defer o.mu.Unlock()

	return o.buf.Write(p)
}

// line returns the first line whose field key is value, waiting for it.
func (o *output) line(t *testing.T, key, value string) map[string]any {
	t.Helper()

	deadline := time.Now().Add(waitLimit)
	for time.Now().Before(deadline) {
		fields := o.findLine(t, func(fields map[string]any) bool {
			return fields[key] == value
		})
		if fields != nil {
			return fields
		}

		time.Sleep(pollInterval)
	}

	t.Fatalf("no line with %s %q in the output:\n%s", key, value, o.text())

	return nil
}

// findLine returns the first line written so far for which matches is true, or
// nil.
func (o *output) findLine(
	t *testing.T, matches func(map[string]any) bool,
) map[string]any {
	t.Helper()

	for line := range strings.Lines(o.text()) {
		var fields map[string]any

		err := json.Unmarshal([]byte(line), &fields)
		if err != nil {
			t.Fatalf("output line %q is not JSON: %v", line, err)
		}

		if matches(fields) {
			return fields
		}
	}

	return nil
}

// text returns everything written so far.
func (o *output) text() string {
	o.mu.Lock()
	defer o.mu.Unlock()

	return o.buf.String()
}

// viperFrom returns a viper Set with each of settings, as the viper
// config.NewViper returns one bound to a process's environment.
func viperFrom(settings map[string]string) *viper.Viper {
	v := viper.New()
	for name, value := range settings {
		v.Set(name, value)
	}

	return v
}

// run runs `sww discriminator` with the settings in env until ctx is done,
// and returns its exit status. SWW_WAF_LOOKUP_SOURCE is off unless env sets it,
// so that no test sends GeoJS its clients' addresses.
func run(ctx context.Context, env map[string]string, out *output) int {
	v := viperFrom(env)
	if _, set := env["SWW_WAF_LOOKUP_SOURCE"]; !set {
		v.Set("SWW_WAF_LOOKUP_SOURCE", "off")
	}

	return sww.RunDiscriminator(ctx, sww.Params{
		Version: testVersion,
		Viper:   v,
		Stdout:  out,
	})
}

func TestInvalidSettingStopsTheStart(t *testing.T) {
	t.Parallel()

	out := &output{}

	status := run(t.Context(), map[string]string{
		"SWW_WAF_BYTES_LIMIT_PER_MINUTE": "lots",
		instanceName:                     instance,
	}, out)
	if status != 1 {
		t.Errorf("exit status %d, want 1", status)
	}

	line := out.line(t, "msg", "invalid setting")
	message, _ := line["error"].(string)

	if line["type"] != "process" || line["instance"] != instance ||
		line["level"] != "ERROR" ||
		!strings.HasPrefix(message, "SWW_WAF_BYTES_LIMIT_PER_MINUTE: ") {
		t.Errorf("start refused with %v", line)
	}
}

func TestShortTokenStopsTheStartUnshown(t *testing.T) {
	t.Parallel()

	const token = "a-token-of-31-characters-at-all" //nolint:gosec // too short to use

	for _, name := range []string{adminToken, metricsToken} {
		t.Run(name, func(t *testing.T) {
			t.Parallel()

			out := &output{}

			status := run(t.Context(), map[string]string{name: token}, out)
			if status != 1 {
				t.Errorf("exit status %d, want 1", status)
			}

			line := out.line(t, "msg", "invalid setting")
			if line["error"] != name+": is shorter than 32 characters" {
				t.Errorf("start refused with %v", line)
			}

			if strings.Contains(out.text(), token) {
				t.Errorf("the output shows the token:\n%s", out.text())
			}
		})
	}
}

func TestAddressInUseStopsTheStart(t *testing.T) {
	t.Parallel()

	taken, err := (&net.ListenConfig{}).Listen(t.Context(), "tcp", localhost+":0")
	if err != nil {
		t.Fatalf("listen: %v", err)
	}

	defer func() {
		_ = taken.Close()
	}()

	out := &output{}

	status := run(t.Context(), map[string]string{
		listenAddr: taken.Addr().String(),
		stateDir:   t.TempDir(),
		rulesDir:   t.TempDir(),
	}, out)
	if status != 1 {
		t.Errorf("exit status %d, want 1", status)
	}

	out.line(t, "msg", "cannot listen on SWW_WAF_LISTEN_ADDR")
}

func TestServesUntilToldToStop(t *testing.T) {
	t.Parallel()

	dir := t.TempDir()

	ctx, stop := context.WithCancel(t.Context())
	out := &output{}
	exited := make(chan int, 1)

	go func() {
		exited <- run(ctx, map[string]string{
			listenAddr: localhost + ":0",
			stateDir:   dir,
			rulesDir:   filepath.Join("..", "..", "share", "rules.d"),
		}, out)
	}()

	// The default rule file is read.
	if rules := out.line(t, "msg", "read the rule files")["rules"]; rules != 12.0 {
		t.Errorf("read %v rules from the default rule file, want 12", rules)
	}

	starting := out.line(t, "msg", "starting")
	wantStartingLine(t, starting, dir)

	addr, _ := starting["address"].(string)
	wantAllowed(t, "http://"+addr+"/")
	out.line(t, "type", "request")

	stop()

	select {
	case status := <-exited:
		if status != 0 {
			t.Errorf("exit status %d, want 0", status)
		}
	case <-time.After(waitLimit):
		t.Fatal("still running after being told to stop")
	}

	out.line(t, "msg", "stopped")
}

func TestLogLevelHoldsBackTheLessSevereProcessLines(t *testing.T) {
	t.Parallel()

	// A list that cannot be fetched has a warning written once sww
	// serves, after its starting line.
	lists := httptest.NewServer(http.HandlerFunc(
		func(w http.ResponseWriter, _ *http.Request) {
			w.WriteHeader(http.StatusServiceUnavailable)
		}))
	t.Cleanup(lists.Close)

	ctx, stop := context.WithCancel(t.Context())
	out := &output{}
	exited := make(chan int, 1)

	go func() {
		exited <- run(ctx, map[string]string{
			listenAddr:               localhost + ":0",
			stateDir:                 t.TempDir(),
			rulesDir:                 t.TempDir(),
			"SWW_WAF_BLOCKLIST_URLS": lists.URL + "/tor.txt",
			"SWW_WAF_LOG_LEVEL":      "warn",
		}, out)
	}()

	out.line(t, "msg", "fetching a list failed")
	stop()

	select {
	case status := <-exited:
		if status != 0 {
			t.Fatalf("exit status %d, want 0; output:\n%s", status, out.text())
		}
	case <-time.After(waitLimit):
		t.Fatal("still running after being told to stop")
	}

	// Not one of the info lines from the start to the stop.
	for line := range strings.Lines(out.text()) {
		var fields map[string]any

		err := json.Unmarshal([]byte(line), &fields)
		if err != nil || fields["level"] == "INFO" {
			t.Errorf("line %q (%v), want none at info", line, err)
		}
	}
}

func TestEveryLogLineAndMetricCarriesTheInstanceName(t *testing.T) {
	t.Parallel()

	const token = "0123456789abcdef0123456789abcdef"

	env := map[string]string{
		listenAddr:   localhost + ":0",
		stateDir:     t.TempDir(),
		rulesDir:     t.TempDir(),
		metricsToken: token,
		instanceName: instance,
	}

	var metrics string

	out := runUntilStopped(t, env, func(url string) {
		metrics = metricsText(t, url+"_sww/metrics", token)
	})

	// The process's lines from its start to its stop, and the request's.
	for line := range strings.Lines(out.text()) {
		var fields map[string]any

		err := json.Unmarshal([]byte(line), &fields)
		if err != nil || fields["instance"] != instance {
			t.Errorf("line %q (%v), want instance %s", line, err, instance)
		}
	}

	// Each series, Go's and the process's included; the other lines are
	// the comments.
	for line := range strings.Lines(metrics) {
		if !strings.HasPrefix(line, "#") &&
			!strings.Contains(line, `instance="fsn1app1/blog"`) {
			t.Errorf("series %q, without instance=\"fsn1app1/blog\"", line)
		}
	}
}

func TestStateKeptAcrossRestarts(t *testing.T) {
	t.Parallel()

	env := map[string]string{
		listenAddr:      localhost + ":0",
		stateDir:        t.TempDir(),
		rulesDir:        t.TempDir(),
		rateLimitPerDay: "2",
		// Neither comes due in the test: the files are written as
		// sww stops.
		stateWriteDelay:      "1h",
		stateCounterInterval: "1h",
	}

	// The two requests a day allows, and a stop.
	runUntilStopped(t, env, func(url string) {
		wantAllowed(t, url)
		wantAllowed(t, url)
	})

	// After a restart the client has no fresh allowance: its third
	// request breaks the day limit, and bans it.
	out := runUntilStopped(t, env, func(url string) {
		wantRefused(t, url)
	})
	out.line(t, "action", "rate_limited")

	// After another, the ban still refuses it.
	out = runUntilStopped(t, env, func(url string) {
		wantRefused(t, url)
	})
	out.line(t, "action", "banned")
}

func TestBanRefusesItsNetblockAfterARestartWithAnotherScope(t *testing.T) {
	t.Parallel()

	const scope = "SWW_WAF_BAN_SCOPE_V4_PREFIX"

	env := map[string]string{
		listenAddr:      localhost + ":0",
		stateDir:        t.TempDir(),
		rulesDir:        t.TempDir(),
		rateLimitPerDay: "1",
		scope:           "24",
	}

	// 203.0.113.9's second request breaks the day limit, and bans
	// 203.0.113.0/24.
	runUntilStopped(t, env, func(url string) {
		wantStatus(t, url, "203.0.113.9", http.StatusOK)
		wantStatus(t, url, "203.0.113.9", http.StatusForbidden)
	})

	// With each address a netblock of its own after a restart, that ban
	// still refuses all of 203.0.113.0/24. 198.51.100.7 is banned alone.
	env[scope] = "32"
	runUntilStopped(t, env, func(url string) {
		wantStatus(t, url, "203.0.113.200", http.StatusForbidden)
		wantStatus(t, url, "203.0.114.1", http.StatusOK)
		wantStatus(t, url, "198.51.100.7", http.StatusOK)
		wantStatus(t, url, "198.51.100.7", http.StatusForbidden)
	})

	// With /24 netblocks again, that ban still refuses 198.51.100.7, and
	// no other address.
	env[scope] = "24"
	runUntilStopped(t, env, func(url string) {
		wantStatus(t, url, "198.51.100.7", http.StatusForbidden)
		wantStatus(t, url, "198.51.100.8", http.StatusOK)
	})
}

func TestBanAddedAndLiftedByEditingBansJSON(t *testing.T) {
	t.Parallel()

	const (
		// bans.json as an admin writes it with a ban, permanent, on
		// 203.0.113.0/24, and with none.
		oneBan = `{"version": 1, "bans": [{"netblock": "203.0.113.0/24", ` +
			`"start": "2026-10-06T00:00:00Z", "expires": null}]}`
		noBan = `{"version": 1, "bans": []}`
	)

	dir := t.TempDir()
	env := map[string]string{
		listenAddr: localhost + ":0",
		stateDir:   dir,
		rulesDir:   t.TempDir(),
		// No write comes due in the test, so only the watch on the
		// directory can take the edits in.
		stateWriteDelay:      "1h",
		stateCounterInterval: "1h",
	}

	runUntilStopped(t, env, func(url string) {
		path := filepath.Join(dir, "bans.json")

		saveUntilAnswered(t, path, oneBan, url, "203.0.113.9", http.StatusForbidden)
		wantStatus(t, url, "198.51.100.7", http.StatusOK)
		saveUntilAnswered(t, path, noBan, url, "203.0.113.9", http.StatusOK)
	})
}

func TestBanAddedAndLiftedThroughTheEndpointsKeptInBansJSON(t *testing.T) {
	t.Parallel()

	dir := t.TempDir()
	env := map[string]string{
		listenAddr: localhost + ":0",
		stateDir:   dir,
		rulesDir:   t.TempDir(),
		adminToken: adminSecret,
		// Neither comes due in the test: the files are written as
		// sww stops.
		stateWriteDelay:      "1h",
		stateCounterInterval: "1h",
	}

	runUntilStopped(t, env, func(url string) {
		askAsAdmin(t, http.MethodPost, url+"_sww/bans",
			`{"netblock": "203.0.113.0/24", "duration": "permanent", `+
				`"reason": "probes for logins"}`)
		wantStatus(t, url, "203.0.113.9", http.StatusForbidden)
	})

	ban := onlyBan(t, dir)
	if ban["netblock"] != "203.0.113.0/24" || ban["cause"] != "admin" ||
		ban["reason"] != "probes for logins" || ban["expires"] != nil ||
		ban["lifted"] != nil {
		t.Errorf("bans.json holds %v, want the admin's permanent ban", ban)
	}

	// After a restart the ban still refuses; once lifted, it refuses no
	// more, and bans.json keeps it, marked lifted.
	runUntilStopped(t, env, func(url string) {
		wantStatus(t, url, "203.0.113.9", http.StatusForbidden)
		askAsAdmin(t, http.MethodDelete, url+"_sww/bans/203.0.113.9", "")
		wantStatus(t, url, "203.0.113.9", http.StatusOK)
	})

	ban = onlyBan(t, dir)
	if ban["netblock"] != "203.0.113.0/24" || ban["lifted"] == nil {
		t.Errorf("bans.json holds %v, want the admin's ban, lifted", ban)
	}
}

func TestRuleFileAddedWhileRunningTakesEffect(t *testing.T) {
	t.Parallel()

	dir := t.TempDir()
	env := map[string]string{
		listenAddr: localhost + ":0",
		stateDir:   t.TempDir(),
		rulesDir:   dir,
		// The requests sent until the rule takes effect must not break a
		// rate limit, whose ban would refuse them too.
		"SWW_WAF_RATE_LIMIT_PER_MINUTE": "off",
	}

	out := runUntilStopped(t, env, func(url string) {
		wantAllowed(t, url)

		// Written once: each change would start the rule files' wait
		// again. A file written before sww watches the directory is
		// read once it does.
		err := os.WriteFile(filepath.Join(dir, "50-app.rules"),
			[]byte("everything path block ^/\n"), 0o600)
		if err != nil {
			t.Fatalf("write the rule file: %v", err)
		}

		// As long as that takes, so that a slow test process cannot fail
		// the test.
		for statusFrom(t, url, "203.0.113.9") != http.StatusForbidden {
			time.Sleep(pollInterval)
		}
	})
	out.line(t, "action", "rule_blocked")
}

func TestRuleFileThatDoesNotParseStopsTheStart(t *testing.T) {
	t.Parallel()

	dir := t.TempDir()
	path := filepath.Join(dir, "00-default.rules")

	err := os.WriteFile(path, []byte("# probes\nenv-file path bann ^/\\.env$\n"), 0o600)
	if err != nil {
		t.Fatalf("write the rule file: %v", err)
	}

	wantRulesRefused(t, dir, path+`, line 2: the action "bann" is not log, block or ban`)
}

func TestRulesDirThatDoesNotExistStopsTheStart(t *testing.T) {
	t.Parallel()

	dir := filepath.Join(t.TempDir(), "rules.d")

	wantRulesRefused(t, dir, "SWW_WAF_RULES_DIR cannot be read: open "+dir+
		": no such file or directory")
}

func TestCoreRuleSetExclusionsFileThatCannotBeReadStopsTheStart(t *testing.T) {
	t.Parallel()

	path := filepath.Join(t.TempDir(), "crs-exclusions.conf")
	out := &output{}

	status := run(t.Context(), map[string]string{
		listenAddr: localhost + ":0", stateDir: t.TempDir(), rulesDir: t.TempDir(),
		"SWW_WAF_CORE_RULE_SET_EXCLUSIONS_FILE": path,
	}, out)
	if status != 1 {
		t.Fatalf("exit status %d, want 1", status)
	}

	line := out.line(t, "msg", "cannot use the Core Rule Set")
	want := "SWW_WAF_CORE_RULE_SET_EXCLUSIONS_FILE: read the exclusions file: open " +
		path + ": no such file or directory"

	if line["error"] != want {
		t.Errorf("start refused with %v, want the error %q", line, want)
	}
}

func TestCoreRuleSetExclusionsFileThatCannotBeUsedStopsTheStartWhileTheCoreRuleSetIsOff(
	t *testing.T,
) {
	t.Parallel()

	broken := filepath.Join(t.TempDir(), "broken.conf")

	err := os.WriteFile(broken, []byte("SecNoSuchDirective On\n"), 0o600)
	if err != nil {
		t.Fatal(err)
	}

	for _, path := range []string{filepath.Join(t.TempDir(), "missing.conf"), broken} {
		out := &output{}

		status := run(t.Context(), map[string]string{
			listenAddr: localhost + ":0", stateDir: t.TempDir(), rulesDir: t.TempDir(),
			"SWW_WAF_CORE_RULE_SET_MODE":            config.WAFModeOff,
			"SWW_WAF_CORE_RULE_SET_EXCLUSIONS_FILE": path,
		}, out)
		if status != 1 {
			t.Fatalf("%s: exit status %d, want 1", path, status)
		}

		out.line(t, "msg", "cannot use the Core Rule Set")
	}
}

func TestLookupDatabaseReplacedWhileRunningTakesEffect(t *testing.T) {
	t.Parallel()

	const token = "0123456789abcdef0123456789abcdef"

	path := filepath.Join(t.TempDir(), "ipinfo_lite.mmdb")
	writeLookupDatabase(t, path, "DE")

	env := map[string]string{
		listenAddr:                 localhost + ":0",
		stateDir:                   t.TempDir(),
		rulesDir:                   t.TempDir(),
		metricsToken:               token,
		instanceName:               instance,
		lookupSource:               "file",
		lookupDBPath:               path,
		"SWW_WAF_DENIED_COUNTRIES": "kp",
		// The requests sent until a replacement takes effect, and those
		// for the metrics, must not break a rate limit, whose ban would
		// refuse them too.
		rateLimitExemptNets: placed + "," + localhost,
	}
	began := time.Now()
	// Each replacement is written beside the file and renamed over it, as
	// a refresh is.
	replacement := path + ".new"

	var lastRead float64

	runUntilStopped(t, env, func(url string) {
		wantStatus(t, url, placed, http.StatusOK)

		// sww reads the file again once it watches its directory,
		// which may be after the first replacement. Once that has taken
		// effect, only the watch can show the next. Each takes as long as
		// it takes, so that a slow test process cannot fail the test.
		writeLookupDatabase(t, replacement, "KP")
		rename(t, replacement, path)

		for statusFrom(t, url, placed) != http.StatusForbidden {
			time.Sleep(pollInterval)
		}

		writeLookupDatabase(t, replacement, "DE")
		rename(t, replacement, path)

		for statusFrom(t, url, placed) != http.StatusOK {
			time.Sleep(pollInterval)
		}

		// One that cannot be read leaves the file read before in use.
		err := os.WriteFile(replacement, []byte("not a lookup database\n"), 0o600)
		if err != nil {
			t.Fatalf("write %s: %v", replacement, err)
		}

		rename(t, replacement, path)

		metrics := metricsWith(t, url+"_sww/metrics", token,
			`sww_lookup_database_read_failures_total{instance="fsn1app1/blog"} 1`)
		lastRead = seriesValue(t, metrics,
			`sww_lookup_database_last_read_timestamp_seconds{instance="fsn1app1/blog"}`)

		wantStatus(t, url, placed, http.StatusOK)
	})

	if lastRead < float64(began.Unix()) || lastRead > float64(time.Now().Unix()) {
		t.Errorf("the lookup database was last read at %v, want a time in the test", lastRead)
	}
}

func TestBlocklistTriesAndCopiesKeptInReputationJSONAcrossRestarts(t *testing.T) {
	t.Parallel()

	const (
		token    = "0123456789abcdef0123456789abcdef"
		torPath  = "/tor.txt"
		dropPath = "/drop.txt"
		// copyright is the DROP list's date and copyright line.
		copyright = "; Spamhaus DROP List 2026/10/07 - (c) 2026 The Spamhaus Project SLL"
	)

	failing, torFetches := new(atomic.Bool), new(atomic.Int32)
	lists := map[string]string{
		torPath:  "198.51.100.0/24\n",
		dropPath: copyright + "\n" + placed + " ; SBL1\n",
	}
	server := httptest.NewServer(http.HandlerFunc(
		func(w http.ResponseWriter, r *http.Request) {
			if r.URL.Path == torPath {
				torFetches.Add(1)
			}

			if failing.Load() {
				w.WriteHeader(http.StatusServiceUnavailable)

				return
			}

			_, _ = io.WriteString(w, lists[r.URL.Path])
		}))
	t.Cleanup(server.Close)

	torURL, dropURL := server.URL+torPath, server.URL+dropPath
	dir := t.TempDir()
	env := map[string]string{
		listenAddr:               localhost + ":0",
		stateDir:                 dir,
		rulesDir:                 t.TempDir(),
		metricsToken:             token,
		instanceName:             instance,
		"SWW_WAF_BLOCKLIST_URLS": torURL,
		// The requests sent until the list takes effect, and those for the
		// metrics, must not break a rate limit, whose ban would refuse them
		// too.
		rateLimitExemptNets: placed + "," + localhost,
	}
	failures := `sww_reputation_failures_total{instance="fsn1app1/blog",` +
		`source="` + torURL + `"} `

	// tor.txt cannot be fetched at first, which is counted.
	failing.Store(true)
	runUntilStopped(t, env, func(url string) {
		metricsWith(t, url+"_sww/metrics", token, failures+"1")
	})

	// Restarted with drop.txt named after it and the server answering, tor.txt
	// waits SWW_WAF_BLOCKLIST_REFRESH after its failed try, kept in reputation.json,
	// while drop.txt, never tried, is fetched at once. Lists are fetched in the
	// order named, so once drop.txt refuses the client, tor.txt has had its turn.
	failing.Store(false)

	env["SWW_WAF_BLOCKLIST_URLS"] = torURL + "," + dropURL
	out := runUntilStopped(t, env, func(url string) {
		for statusFrom(t, url, placed) != http.StatusForbidden {
			time.Sleep(pollInterval)
		}
	})
	wantDeniedByList(t, out.line(t, "action", "denied"), dropURL)

	if fetches := torFetches.Load(); fetches != 1 {
		t.Errorf("tor.txt fetched %d times, want once, before the restart", fetches)
	}

	// After another restart, with the server failing, the copy of drop.txt
	// kept in reputation.json, its copyright line included, refuses the
	// client from the first request.
	failing.Store(true)

	out = runUntilStopped(t, env, func(url string) {
		wantStatus(t, url, placed, http.StatusForbidden)
	})
	wantDeniedByList(t, out.line(t, "type", "request"), dropURL)

	path := filepath.Join(dir, "reputation.json")

	kept, err := os.ReadFile(path) //nolint:gosec // a file in the test's directory
	if err != nil || !strings.Contains(string(kept), `"`+copyright+`"`) {
		t.Errorf("reputation.json holds\n%s\nwant the copy with %q (%v)", kept, copyright,
			err)
	}
}

func TestCrowdSecDecisionListKeptInReputationJSONAcrossARestart(t *testing.T) {
	t.Parallel()

	const bouncerKey = "crowdsec-key-0123456789abcdef"

	// A stand-in for the engine's local API, which bans 203.0.113.0/24 for
	// four hours, and answers only a request with its key, while it is up.
	down := new(atomic.Bool)
	engine := httptest.NewServer(http.HandlerFunc(
		func(w http.ResponseWriter, r *http.Request) {
			switch {
			case r.URL.Path != "/v1/decisions" || r.Header.Get("X-Api-Key") != bouncerKey:
				w.WriteHeader(http.StatusForbidden)
			case down.Load():
				w.WriteHeader(http.StatusServiceUnavailable)
			default:
				_, _ = io.WriteString(w, `[{"duration": "4h0m0s", "origin": "crowdsec", `+
					`"scenario": "crowdsecurity/http-probing", "scope": "Range", `+
					`"type": "ban", "value": "203.0.113.0/24"}]`)
			}
		}))
	t.Cleanup(engine.Close)

	dir := t.TempDir()
	env := map[string]string{
		listenAddr:                  localhost + ":0",
		stateDir:                    dir,
		rulesDir:                    t.TempDir(),
		"SWW_WAF_CROWDSEC_LAPI_URL": engine.URL,
		"SWW_WAF_CROWDSEC_LAPI_KEY": bouncerKey,
	}

	// Once the list is fetched, the client's request bans it.
	first := runUntilStopped(t, env, func(url string) {
		for statusFrom(t, url, placed) != http.StatusForbidden {
			time.Sleep(pollInterval)
		}
	})

	ban := onlyBan(t, dir)
	if ban["netblock"] != placed+"/32" || ban["cause"] != "crowdsec" ||
		ban["reason"] != "CrowdSec's decision for crowdsecurity/http-probing" {
		t.Errorf("bans.json holds %v, want the ban for crowdsec on %s", ban, placed)
	}

	// Restarted with the engine down, the copy kept in reputation.json bans
	// another client in the netblock from its first request.
	down.Store(true)

	second := runUntilStopped(t, env, func(url string) {
		wantStatus(t, url, "203.0.113.10", http.StatusForbidden)
	})

	// The key is in neither run's output, nor in a state file.
	files := []string{"bans.json", "reputation.json", "clients.json"}
	shown := make([]string, 0, len(files)+2)
	shown = append(shown, first.text(), second.text())

	for _, name := range files {
		data, err := os.ReadFile(filepath.Join(dir, name)) //nolint:gosec // the test's
		if err != nil {
			t.Fatalf("read %s: %v", name, err)
		}

		shown = append(shown, string(data))
	}

	if all := strings.Join(shown, "\n"); strings.Contains(all, bouncerKey) {
		t.Errorf("the key is shown in the output or the state files:\n%s", all)
	}
}

// wantDeniedByList checks that the request log line is of a request the
// blocklist at listURL refused.
func wantDeniedByList(t *testing.T, line map[string]any, listURL string) {
	t.Helper()

	reputation, _ := line["reputation"].([]any)
	if line["action"] != "denied" || len(reputation) != 1 || reputation[0] != listURL {
		t.Errorf("request log line %v, want one denied for %s", line, listURL)
	}
}

func TestLookupDatabaseThatCannotBeReadStopsTheStart(t *testing.T) {
	t.Parallel()

	path := filepath.Join(t.TempDir(), "ipinfo_lite.mmdb")

	// If it starts instead, it is stopped after waitLimit.
	ctx, stop := context.WithTimeout(t.Context(), waitLimit)
	defer stop()

	out := &output{}

	status := run(ctx, map[string]string{
		listenAddr: localhost + ":0", stateDir: t.TempDir(), rulesDir: t.TempDir(),
		lookupSource: "file", lookupDBPath: path,
	}, out)
	if status != 1 {
		t.Fatalf("exit status %d, want 1", status)
	}

	want := "SWW_WAF_LOOKUP_DB_PATH cannot be read: open " + path +
		": no such file or directory"

	line := out.line(t, "msg", "cannot use the lookup database")
	if line["error"] != want {
		t.Errorf("start refused with %v, want the error %q", line, want)
	}
}

func TestEveryLineIsAlsoSentToTheRemoteLogEndpoint(t *testing.T) {
	t.Parallel()

	endpoint, err := (&net.ListenConfig{}).Listen(t.Context(), "tcp", localhost+":0")
	if err != nil {
		t.Fatalf("listen: %v", err)
	}

	defer func() {
		_ = endpoint.Close()
	}()

	env := map[string]string{
		listenAddr:               localhost + ":0",
		stateDir:                 t.TempDir(),
		rulesDir:                 t.TempDir(),
		"SWW_WAF_LOG_REMOTE_URL": "syslog+tcp://" + endpoint.Addr().String(),
	}

	out := runUntilStopped(t, env, func(url string) {
		wantAllowed(t, url)
	})
	out.line(t, "type", "request")

	// sww connected as it started, and closes the connection once
	// it has sent the lines written as it stopped.
	conn, err := endpoint.Accept()
	if err != nil {
		t.Fatalf("accept: %v", err)
	}

	received, err := io.ReadAll(conn)
	_ = conn.Close()

	if err != nil {
		t.Fatalf("read: %v", err)
	}

	// Lines written at once by several goroutines may reach stdout and
	// the endpoint in different orders.
	sent := messages(t, string(received))
	written := slices.Collect(strings.Lines(out.text()))

	slices.Sort(sent)
	slices.Sort(written)

	if !slices.Equal(sent, written) {
		t.Errorf("sent\n%v\nwrote\n%v", sent, written)
	}
}

func TestStalledRemoteLogEndpointHoldsUpNoRequest(t *testing.T) {
	t.Parallel()

	const token = "0123456789abcdef0123456789abcdef"

	// The endpoint takes connections and never answers, so the TLS
	// handshake of each waits on it, and no line is ever sent.
	endpoint, err := (&net.ListenConfig{}).Listen(t.Context(), "tcp", localhost+":0")
	if err != nil {
		t.Fatalf("listen: %v", err)
	}

	defer func() {
		_ = endpoint.Close()
	}()

	env := map[string]string{
		listenAddr:                  localhost + ":0",
		stateDir:                    t.TempDir(),
		rulesDir:                    t.TempDir(),
		"SWW_WAF_LOG_REMOTE_URL":    "syslog+tls://" + endpoint.Addr().String(),
		"SWW_WAF_LOG_REMOTE_BUFFER": "1",
		metricsToken:                token,
		instanceName:                instance,
	}

	out := runUntilStopped(t, env, func(url string) {
		wantAllowed(t, url)

		// More than one line has been written, and the buffer holds the
		// last.
		metrics := metricsText(t, url+"_sww/metrics", token)
		for _, series := range []string{
			`sww_remote_log_lines_sent_total{instance="fsn1app1/blog"} 0`,
			`sww_remote_log_buffer_depth{instance="fsn1app1/blog"} 1`,
		} {
			if !strings.Contains(metrics, "\n"+series+"\n") {
				t.Errorf("no %q in the metrics:\n%s", series, metrics)
			}
		}

		const dropped = "\nsww_remote_log_lines_dropped_total" +
			`{instance="fsn1app1/blog"} `
		if strings.Contains(metrics, dropped+"0\n") || !strings.Contains(metrics, dropped) {
			t.Errorf("no line dropped in the metrics:\n%s", metrics)
		}

		// Closed, the endpoint refuses the connection made to send the
		// lines still waiting at the stop, which then does not wait.
		_ = endpoint.Close()
	})

	out.line(t, "type", "request")

	// While the lines are sent, the process's lines give the instance name
	// too.
	line := out.line(t, "msg", "starting")
	if line["instance"] != instance {
		t.Errorf("start logged with instance %v, want %s", line["instance"], instance)
	}
}

func TestBanIsAlertedAndAnAlertNotSentIsKeptAcrossARestart(t *testing.T) {
	t.Parallel()

	webhook := startWebhook(t)
	rules := t.TempDir()

	err := os.WriteFile(filepath.Join(rules, "50-app.rules"),
		[]byte(`probe path ban ^/\.env$`+"\n"), 0o600)
	if err != nil {
		t.Fatalf("write the rule file: %v", err)
	}

	dir := t.TempDir()
	env := map[string]string{
		listenAddr:                      localhost + ":0",
		stateDir:                        dir,
		rulesDir:                        rules,
		"SWW_WAF_ALERT_WEBHOOK_URL":     webhook.url,
		"SWW_WAF_ALERT_WEBHOOK_HEADERS": "Authorization:Bearer " + adminSecret,
		instanceName:                    instance,
	}

	runUntilStopped(t, env, func(url string) {
		// The probe bans the client, and the webhook is sent the alert.
		wantRefused(t, url+".env")

		post := webhook.waitFor(t, "ban", true)
		if post.alert["client"] != localhost || post.alert["netblock"] != localhost+"/32" ||
			post.authorization != "Bearer "+adminSecret {
			t.Errorf("the webhook was sent %v, with Authorization %q", post.alert,
				post.authorization)
		}

		// The webhook fails, so the alert for the ban made permanent by the
		// client's next request waits.
		webhook.failing.Store(true)
		wantRefused(t, url)
		webhook.waitFor(t, "permanent_ban", false)
	})

	// alerts.json keeps it as sww stops, and once started again,
	// sww sends it.
	var file struct {
		Waiting map[string][]struct {
			Event string `json:"event"`
		} `json:"waiting"`
	}

	path := filepath.Join(dir, "alerts.json")

	data, err := os.ReadFile(path) //nolint:gosec // a file in the test's directory
	if err == nil {
		err = json.Unmarshal(data, &file)
	}

	waiting := file.Waiting["webhook"]
	if err != nil || len(waiting) != 1 || waiting[0].Event != "permanent_ban" {
		t.Fatalf("alerts.json holds %s (%v), want the permanent_ban alert waiting for "+
			"the webhook", data, err)
	}

	// It counts the alert sent in the metrics, read here from a client the
	// ban does not cover.
	const token = "0123456789abcdef0123456789abcdef"

	webhook.failing.Store(false)

	env["SWW_WAF_ALLOW_NETS"] = localhost
	env[metricsToken] = token

	runUntilStopped(t, env, func(url string) {
		webhook.waitFor(t, "permanent_ban", true)

		const ofWebhook = `{destination="webhook",instance="fsn1app1/blog"}`

		metrics := metricsWith(t, url+"_sww/metrics", token,
			"\nsww_alerts_sent_total"+ofWebhook+" 1\n")

		for _, series := range []string{"failed", "suppressed", "dropped"} {
			zero := "\nsww_alerts_" + series + "_total" + ofWebhook + " 0\n"
			if !strings.Contains(metrics, zero) {
				t.Errorf("no %q in the metrics:\n%s", zero, metrics)
			}
		}
	})
}

func TestBanIsAlertedToSlackAndNtfy(t *testing.T) {
	t.Parallel()

	const (
		ntfyToken = "tk_0123456789abcdefghijklmnopq"
		token     = "abcdef0123456789abcdef0123456789"
		client    = "203.0.113.9"
	)

	slack, ntfy := startDestination(t), startDestination(t)
	env := map[string]string{
		listenAddr:      localhost + ":0",
		stateDir:        t.TempDir(),
		rulesDir:        t.TempDir(),
		rateLimitPerDay: "1",
		// The metrics are read from 127.0.0.1, which no limit counts.
		"SWW_WAF_ALLOW_NETS":              localhost + "/32",
		metricsToken:                      token,
		instanceName:                      instance,
		"SWW_WAF_ALERT_SLACK_WEBHOOK_URL": slack.url,
		"SWW_WAF_ALERT_NTFY_URL":          ntfy.url,
		"SWW_WAF_ALERT_NTFY_TOKEN":        ntfyToken,
	}

	runUntilStopped(t, env, func(url string) {
		// The client's second request breaks the day limit, and bans it;
		// Slack and ntfy are each sent the alert.
		wantStatus(t, url, client, http.StatusOK)
		wantStatus(t, url, client, http.StatusForbidden)

		var message struct {
			Text string `json:"text"`
		}

		slackPost := slack.firstPost(t)
		err := json.Unmarshal([]byte(slackPost.body), &message)

		if err != nil || !strings.HasPrefix(message.Text, "*fsn1app1/blog: ban*\n") ||
			!strings.Contains(message.Text, "\nclient: "+client+"\n") {
			t.Errorf("Slack was sent %s", slackPost.body)
		}

		ntfyPost := ntfy.firstPost(t)
		if ntfyPost.header.Get("Title") != "fsn1app1/blog: ban" ||
			ntfyPost.header.Get("Authorization") != "Bearer "+ntfyToken ||
			!strings.Contains(ntfyPost.body, "\nclient: "+client+"\n") {
			t.Errorf("ntfy was sent %s, with the headers %v", ntfyPost.body,
				ntfyPost.header)
		}

		// The metrics count it for each, and give no series for the
		// webhook, which is not set. As long as that takes, so that a slow
		// test process cannot fail the test.
		sent := []string{
			"\nsww_alerts_sent_total{destination=\"slack\"," +
				"instance=\"fsn1app1/blog\"} 1\n",
			"\nsww_alerts_sent_total{destination=\"ntfy\"," +
				"instance=\"fsn1app1/blog\"} 1\n",
		}

		metrics := metricsWith(t, url+"_sww/metrics", token, sent...)
		if strings.Contains(metrics, `destination="webhook"`) {
			t.Errorf("the metrics give the webhook:\n%s", metrics)
		}
	})
}

func TestStateFileThatDoesNotParseStopsTheStart(t *testing.T) {
	t.Parallel()

	dir := t.TempDir()

	err := os.WriteFile(filepath.Join(dir, "bans.json"), []byte("{\n"), 0o600)
	if err != nil {
		t.Fatalf("write bans.json: %v", err)
	}

	// The file ends at the newline that is the second byte of its first
	// line.
	wantStartRefused(t, dir, filepath.Join(dir, "bans.json")+", line 1, column 2: ")
}

func TestUnwritableStateDirStopsTheStart(t *testing.T) {
	t.Parallel()

	wantStartRefused(t, filepath.Join(t.TempDir(), "missing"),
		"SWW_WAF_STATE_DIR cannot be written: ")
}

// wantStartRefused runs sww with its state files in dir, and
// checks that it stops at start, with an error that starts with want. If
// it starts instead, it is stopped after waitLimit.
func wantStartRefused(t *testing.T, dir, want string) {
	t.Helper()

	ctx, stop := context.WithTimeout(t.Context(), waitLimit)
	defer stop()

	out := &output{}

	status := run(ctx, map[string]string{
		listenAddr: localhost + ":0", stateDir: dir, rulesDir: t.TempDir(),
	}, out)
	if status != 1 {
		t.Fatalf("exit status %d, want 1", status)
	}

	line := out.line(t, "msg", "cannot use the state files")
	message, _ := line["error"].(string)

	if !strings.HasPrefix(message, want) {
		t.Errorf("start refused with %q, want an error starting %q", message, want)
	}
}

// wantRulesRefused runs sww with its rule files in dir, and
// checks that it stops at start, with the error want. If it starts
// instead, it is stopped after waitLimit.
func wantRulesRefused(t *testing.T, dir, want string) {
	t.Helper()

	ctx, stop := context.WithTimeout(t.Context(), waitLimit)
	defer stop()

	out := &output{}

	status := run(ctx, map[string]string{
		listenAddr: localhost + ":0", stateDir: t.TempDir(), rulesDir: dir,
	}, out)
	if status != 1 {
		t.Fatalf("exit status %d, want 1", status)
	}

	line := out.line(t, "msg", "cannot use the rule files")
	if line["error"] != want || line["level"] != "ERROR" {
		t.Errorf("start refused with %v, want the error %q", line, want)
	}
}

// startApp starts an app that answers every request with greeting, and
// returns its URL.
func startApp(t *testing.T) string {
	t.Helper()

	app := httptest.NewServer(http.HandlerFunc(
		func(w http.ResponseWriter, _ *http.Request) {
			_, _ = io.WriteString(w, greeting)
		}))
	t.Cleanup(app.Close)

	return app.URL
}

// runUntilStopped runs `sww discriminator` with the settings in env, has use
// send it requests at url, then stops it as SIGTERM does, checks that it
// stopped in order, and returns its output.
func runUntilStopped(
	t *testing.T, env map[string]string, use func(url string),
) *output {
	t.Helper()

	ctx, stop := context.WithCancel(t.Context())
	out := &output{}
	exited := make(chan int, 1)

	go func() {
		exited <- run(ctx, env, out)
	}()

	addr, _ := out.line(t, "msg", "starting")["address"].(string)
	use("http://" + addr + "/")
	stop()

	select {
	case status := <-exited:
		if status != 0 {
			t.Fatalf("exit status %d, want 0; output:\n%s", status, out.text())
		}
	case <-time.After(waitLimit):
		t.Fatal("still running after being told to stop")
	}

	return out
}

// wantStartingLine checks that the line at start gives the version and
// every setting's value.
func wantStartingLine(t *testing.T, line map[string]any, dir string) {
	t.Helper()

	settings, _ := line["settings"].(map[string]any)
	want := map[string]any{
		listenAddr:                              localhost + ":0",
		stateDir:                                dir,
		"SWW_WAF_MODE":                          "enforce",
		stateWriteDelay:                         "10s",
		stateCounterInterval:                    "15m",
		"SWW_WAF_ALLOW_NETS":                    "",
		rateLimitExemptNets:                     "",
		"SWW_WAF_DENY_NETS":                     "",
		"SWW_WAF_RATE_LIMIT_PER_MINUTE":         "120",
		"SWW_WAF_RATE_LIMIT_PER_HOUR":           "1500",
		rateLimitPerDay:                         "6000",
		"SWW_WAF_RATE_LIMIT_EXEMPT_PATHS":       "",
		"SWW_WAF_DENIED_COUNTRIES":              "",
		"SWW_WAF_EXCLUSIVELY_ALLOWED_COUNTRIES": "",
		"SWW_WAF_BAN_RESPONSE":                  "403",
		"SWW_WAF_LIMIT_BAN_DURATION":            "1h",
		"SWW_WAF_LIMIT_BAN_REPEAT_WINDOW":       "24h",
		"SWW_WAF_MAX_BAN_DURATION":              "7d",
		"SWW_WAF_ATTACK_BAN_DURATION":           "7d",
		"SWW_WAF_MAX_BANS":                      "5000",
		"SWW_WAF_BAN_SCOPE_V4_PREFIX":           "32",
		"SWW_WAF_RULES_ENABLED":                 "true",
	}

	for name, value := range want {
		if settings[name] != value {
			t.Errorf("starting line gives %s=%v, want %v", name, settings[name], value)
		}
	}

	if line["version"] != testVersion || line["type"] != "process" {
		t.Errorf("starting line %v", line)
	}
}

// wantAllowed checks that discriminator at url allows a request from localhost.
func wantAllowed(t *testing.T, url string) {
	t.Helper()

	wantStatus(t, url, localhost, http.StatusOK)
}

// messages returns the message of each record in received, octet-counted
// frames of RFC 5424 records with the default facility and app name, each
// with the newline that ends a line on stdout.
func messages(t *testing.T, received string) []string {
	t.Helper()

	hostname, _ := os.Hostname()
	header := " " + hostname + " " + hostname + " - - - "

	var found []string

	for received != "" {
		count, rest, _ := strings.Cut(received, " ")

		length, err := strconv.Atoi(count)
		if err != nil || length > len(rest) {
			t.Fatalf("no frame at %q", received)
		}

		record := rest[:length]
		received = rest[length:]

		_, message, ok := strings.Cut(record, header)
		if !ok || !strings.HasPrefix(record, "<134>1 ") {
			t.Fatalf("record %q, want priority <134> and header %q", record, header)
		}

		found = append(found, message+"\n")
	}

	return found
}

// metricsText asks for the metrics at url with token, and returns them.
func metricsText(t *testing.T, url, token string) string {
	t.Helper()

	req, err := http.NewRequestWithContext(t.Context(), http.MethodGet, url,
		http.NoBody)
	if err != nil {
		t.Fatalf("new request: %v", err)
	}

	req.Header.Set("Authorization", "Bearer "+token)

	transport := &http.Transport{}
	defer transport.CloseIdleConnections()

	res, err := (&http.Client{Transport: transport}).Do(req)
	if err != nil {
		t.Fatalf("request: %v", err)
	}

	body, err := io.ReadAll(res.Body)
	_ = res.Body.Close()

	if err != nil || res.StatusCode != http.StatusOK {
		t.Fatalf("metrics answered %d (%v)", res.StatusCode, err)
	}

	return string(body)
}

// metricsWith asks for the metrics at url with token until they hold each
// of series, as long as that takes, so that a slow test process cannot
// fail the test, and returns them.
func metricsWith(t *testing.T, url, token string, series ...string) string {
	t.Helper()

	for {
		metrics := metricsText(t, url, token)

		missing := slices.ContainsFunc(series, func(one string) bool {
			return !strings.Contains(metrics, one)
		})
		if !missing {
			return metrics
		}

		time.Sleep(pollInterval)
	}
}

// seriesValue returns the value of series, such as
// name{instance="app"}, in metrics.
func seriesValue(t *testing.T, metrics, series string) float64 {
	t.Helper()

	for line := range strings.Lines(metrics) {
		value, found := strings.CutPrefix(strings.TrimSpace(line), series+" ")
		if !found {
			continue
		}

		number, err := strconv.ParseFloat(value, 64)
		if err != nil {
			t.Fatalf("%s has the value %q: %v", series, value, err)
		}

		return number
	}

	t.Fatalf("no %s in the metrics:\n%s", series, metrics)

	return 0
}

// rename renames the file at from to, replacing any file there.
func rename(t *testing.T, from, to string) {
	t.Helper()

	err := os.Rename(from, to)
	if err != nil {
		t.Fatalf("rename %s to %s: %v", from, to, err)
	}
}

// writeLookupDatabase writes a lookup database at path that places the
// client placed in country, and no other address.
func writeLookupDatabase(t *testing.T, path, country string) {
	t.Helper()

	lookuptest.Write(t, path, map[string]lookuptest.Network{
		placed + "/32": {ASN: "AS64496", ASName: "Example Net", Country: country},
	})
}

// askAsAdmin sends a request with method to url, with body and
// adminSecret, and checks that it is answered 200.
func askAsAdmin(t *testing.T, method, url, body string) {
	t.Helper()

	req, err := http.NewRequestWithContext(t.Context(), method, url,
		strings.NewReader(body))
	if err != nil {
		t.Fatalf("new request: %v", err)
	}

	req.Header.Set("Authorization", "Bearer "+adminSecret)

	transport := &http.Transport{}
	defer transport.CloseIdleConnections()

	res, err := (&http.Client{Transport: transport}).Do(req)
	if err != nil {
		t.Fatalf("request: %v", err)
	}

	_ = res.Body.Close()

	if res.StatusCode != http.StatusOK {
		t.Fatalf("%s %s answered %d", method, url, res.StatusCode)
	}
}

// onlyBan returns the one ban bans.json in dir holds.
func onlyBan(t *testing.T, dir string) map[string]any {
	t.Helper()

	path := filepath.Join(dir, "bans.json")

	data, err := os.ReadFile(path) //nolint:gosec // a file in the test's directory
	if err != nil {
		t.Fatalf("read bans.json: %v", err)
	}

	var file struct {
		Bans []map[string]any `json:"bans"`
	}

	err = json.Unmarshal(data, &file)
	if err != nil || len(file.Bans) != 1 {
		t.Fatalf("bans.json holds\n%s\nwant one ban (%v)", data, err)
	}

	return file.Bans[0]
}

// wantRefused checks that discriminator at url refuses a request from localhost with
// 403, the default SWW_WAF_BAN_RESPONSE.
func wantRefused(t *testing.T, url string) {
	t.Helper()

	wantStatus(t, url, localhost, http.StatusForbidden)
}

// wantStatus checks that a request to url from the client at from, as statusFrom
// sends it, is answered with status.
func wantStatus(t *testing.T, url, from string, status int) {
	t.Helper()

	got := statusFrom(t, url, from)
	if got != status {
		t.Errorf("request from %s: status %d, want %d", from, got, status)
	}
}

// saveUntilAnswered writes content to the state file at path, as an
// admin saves an edit of it, until a request to url from the client at
// from is answered with status. The file is written again before each
// request, since sww may not watch its directory yet when it is
// first written. It waits as long as that takes, so that a slow test
// process cannot fail the test.
func saveUntilAnswered(t *testing.T, path, content, url, from string, status int) {
	t.Helper()

	for {
		err := os.WriteFile(path, []byte(content), 0o600)
		if err != nil {
			t.Fatalf("write %s: %v", path, err)
		}

		if statusFrom(t, url, from) == status {
			return
		}

		time.Sleep(pollInterval)
	}
}

// destination is a stand-in for SWW_WAF_ALERT_SLACK_WEBHOOK_URL or
// SWW_WAF_ALERT_NTFY_URL. It notes each request it is sent, and answers
// 200.
type destination struct {
	url string

	mu    sync.Mutex
	posts []destinationPost
}

// destinationPost is a request a destination was sent: its headers and
// its body.
type destinationPost struct {
	header http.Header
	body   string
}

// startDestination starts a destination that takes every alert.
func startDestination(t *testing.T) *destination {
	t.Helper()

	d := &destination{}
	server := httptest.NewServer(http.HandlerFunc(
		func(_ http.ResponseWriter, r *http.Request) {
			body, _ := io.ReadAll(r.Body)

			d.mu.Lock()
			d.posts = append(d.posts, destinationPost{
				header: r.Header.Clone(), body: string(body),
			})
			d.mu.Unlock()
		}))
	t.Cleanup(server.Close)

	d.url = server.URL + "/alerts"

	return d
}

// firstPost waits until the destination has been sent a request, and
// returns the first. It waits as long as that takes, so that a slow test
// process cannot fail the test.
func (d *destination) firstPost(t *testing.T) destinationPost {
	t.Helper()

	for {
		d.mu.Lock()

		if len(d.posts) > 0 {
			post := d.posts[0]
			d.mu.Unlock()

			return post
		}

		d.mu.Unlock()
		time.Sleep(pollInterval)
	}
}

// webhook is a stand-in for SWW_WAF_ALERT_WEBHOOK_URL. It notes each alert
// it is sent, and answers 204, or 503 while failing.
type webhook struct {
	url     string
	failing atomic.Bool

	mu    sync.Mutex
	posts []webhookPost
}

// webhookPost is an alert the webhook was sent, with the Authorization
// header sent with it, and whether the webhook took it.
type webhookPost struct {
	alert         map[string]any
	authorization string
	answered      bool
}

// startWebhook starts a webhook that takes every alert.
func startWebhook(t *testing.T) *webhook {
	t.Helper()

	w := &webhook{}
	server := httptest.NewServer(http.HandlerFunc(
		func(rw http.ResponseWriter, r *http.Request) {
			var alert map[string]any

			_ = json.NewDecoder(r.Body).Decode(&alert)
			failing := w.failing.Load()

			w.mu.Lock()
			w.posts = append(w.posts, webhookPost{
				alert: alert, authorization: r.Header.Get("Authorization"),
				answered: !failing,
			})
			w.mu.Unlock()

			if failing {
				rw.WriteHeader(http.StatusServiceUnavailable)

				return
			}

			rw.WriteHeader(http.StatusNoContent)
		}))
	t.Cleanup(server.Close)

	w.url = server.URL + "/alerts"

	return w
}

// waitFor waits until the webhook has been sent an alert for event that
// it took, or, unless answered, failed, and returns it. It waits as long
// as that takes, so that a slow test process cannot fail the test.
func (w *webhook) waitFor(t *testing.T, event string, answered bool) webhookPost {
	t.Helper()

	for {
		w.mu.Lock()

		for _, post := range w.posts {
			if post.alert["event"] == event && post.answered == answered {
				w.mu.Unlock()

				return post
			}
		}

		w.mu.Unlock()
		time.Sleep(pollInterval)
	}
}

// statusFrom returns the status a request to url from the client at from is
// answered with. The request serves rproxy and discriminator alike. rproxy reads the
// client from X-Forwarded-For, and replaces the other two headers. To discriminator,
// they make it a question about that client.
func statusFrom(t *testing.T, url, from string) int {
	t.Helper()

	req, err := http.NewRequestWithContext(t.Context(), http.MethodGet, url,
		http.NoBody)
	if err != nil {
		t.Fatalf("new request: %v", err)
	}

	req.Header.Set("X-Forwarded-For", from)
	req.Header.Set(discriminator.ClientHeader, from)
	req.Header.Set("X-Request-ID", rand.Text())

	transport := &http.Transport{}
	defer transport.CloseIdleConnections()

	res, err := (&http.Client{Transport: transport}).Do(req)
	if err != nil {
		t.Fatalf("request: %v", err)
	}

	_ = res.Body.Close()

	return res.StatusCode
}