watcher: notify NS query failure and recovery (closes #104)

LookupAllRecords now returns each nameserver's response, so the
watcher saves its status: ok when it answered, NXDOMAIN and no records
included, and error with the reason when it timed out, answered
SERVFAIL or REFUSED, or could not be reached. A nameserver that starts
failing sends NS Failure and one that answers again sends NS Recovery.
A failing nameserver is left out of the record change and
inconsistency comparisons. The resolver used to report REFUSED and
network errors as an answer with no records; they are now errors. A
lookup cut short by its context now returns an error instead of a
failure of the nameserver it was querying.

Model: opus-5-5

.golangci.yml

@@ -10,14 +10,20 @@ run:
 
 linters:
   default: all
+  enable:
+    # Successor to the deprecated gomodguard. Named explicitly, rather than
+    # left to `default: all`, because it carries the module policy below.
+    - gomodguard_v2
   disable:
     # Genuinely incompatible with project patterns
     - exhaustruct      # Requires all struct fields
-    - depguard         # Dependency allow/block lists
     - godot            # Requires comments to end with periods
-    - wsl              # Deprecated, replaced by wsl_v5
     - wrapcheck        # Too verbose for internal packages
     - varnamelen       # Short names like db, id are idiomatic Go
+    # Deprecated: the warning is attached to the old name, so it is
+    # silenced by disabling that name, not by enabling the successor.
+    - wsl              # Deprecated, replaced by wsl_v5
+    - gomodguard       # Deprecated, replaced by gomodguard_v2
   settings:
     lll:
       line-length: 88
@@ -28,6 +34,68 @@ linters:
       max-complexity: 15
     dupl:
       threshold: 100
+    depguard:
+      # Test-support code must not be compiled into the shipped binary. A
+      # test-support package exists to hand a test privileges the program
+      # itself must never have, so a file that is not a test must not import
+      # one. Test files, and the files inside a package whose directory name
+      # ends in `test`, are where that code belongs, and are exempt.
+      #
+      # The deny list below is the one part of this file a repository is
+      # expected to extend, and the only part it may. depguard matches an
+      # import path against a list of prefixes, so it cannot be told "any path
+      # whose last segment ends in test"; a repository's own test-support
+      # packages have to be named here one at a time, by full import path,
+      # under a module path that differs from repository to repository. Add
+      # them; change nothing else.
+      rules:
+        test-support:
+          list-mode: lax
+          files:
+            - "$all"
+            - "!$test"
+            - "!**/*test/**"
+          deny:
+            - pkg: net/http/httptest
+              desc: >-
+                Test-support code belongs in test files and in packages whose
+                directory name ends in test, not in the shipped binary.
+            - pkg: sneak.berlin/go/dnswatcher/internal/livednstest
+              desc: >-
+                Live-DNS test support belongs in test files and in packages
+                whose directory name ends in test, not in the shipped binary.
+    # Only decisions already recorded in the Go package defaults are
+    # listed here. Every entry matches the module path exactly.
+    gomodguard_v2:
+      blocked:
+        - module: github.com/rs/zerolog
+          recommendations:
+            - log/slog
+          reason: "Structured logging is stdlib log/slog."
+        # One entry per pre-fork module path, because the later releases
+        # are separate paths. A prefix match would be shorter but would
+        # also reach github.com/go-redis/redismock, the test double for
+        # the successor these entries recommend.
+        - module: github.com/go-redis/redis
+          recommendations:
+            - github.com/redis/go-redis/v9
+          reason: "Pre-fork module; use the maintained go-redis v9."
+        - module: github.com/go-redis/redis/v7
+          recommendations:
+            - github.com/redis/go-redis/v9
+          reason: "Pre-fork module; use the maintained go-redis v9."
+        - module: github.com/go-redis/redis/v8
+          recommendations:
+            - github.com/redis/go-redis/v9
+          reason: "Pre-fork module; use the maintained go-redis v9."
+        - module: github.com/sergi/go-diff
+          recommendations:
+            - github.com/aymanbagabas/go-udiff
+          reason: "No unified diff output; use go-udiff."
+        - module: github.com/hexops/gotextdiff
+          recommendations:
+            - github.com/aymanbagabas/go-udiff
+          reason: "Unmaintained fork; use go-udiff."
 
 issues:
   max-issues-per-linter: 0

Dockerfile

@@ -2,6 +2,7 @@
 # The linter is invoked directly rather than through `make lint`: that
 # target shells out to `docker build -f Dockerfile.lint`, and there is
 # no docker daemon inside a docker build.
+# script/cibuild and script/docker name this stage in --no-cache-filter.
 # golangci/golangci-lint:v2.12.2 (Debian-based), 2026-08-10
 FROM golangci/golangci-lint:v2.12.2@sha256:5cceeef04e53efe1470638d4b4b4f5ceefd574955ab3941b2d9a68a8c9ad5240 AS lint
 
@@ -15,6 +16,7 @@ RUN make fmt-check
 RUN golangci-lint run --config .golangci.yml ./...
 
 # Build stage
+# script/cibuild and script/docker name this stage in --no-cache-filter.
 # golang 1.25-alpine, 2026-02-28
 FROM golang@sha256:f6751d823c26342f9506c03797d2527668d095b0a15f1862cddb4d927a7a4ced AS builder
 
@@ -41,15 +43,33 @@ FROM alpine@sha256:c3f8e73fdb79deaebaa2037150150191b9dcbfba68b4a46d70103204c53f4
 
 RUN apk add --no-cache ca-certificates tzdata
 
-WORKDIR /app
+COPY --from=builder /src/bin/dnswatcher /usr/local/bin/dnswatcher
 
-COPY --from=builder /src/bin/dnswatcher /app/dnswatcher
+# Run as an unprivileged user that owns the data directory. A fresh named
-
+# volume inherits this ownership; a bind-mounted host directory must be
-# Create data directory
+# owned by uid 10001 (see "Running under upaas" in README.md), or startup
-RUN mkdir -p /var/lib/dnswatcher
+# fails.
+RUN addgroup -S -g 10001 dnswatcher \
+    && adduser -S -G dnswatcher -u 10001 dnswatcher \
+    && mkdir -p /var/lib/dnswatcher \
+    && chown dnswatcher:dnswatcher /var/lib/dnswatcher
 
 ENV DNSWATCHER_DATA_DIR=/var/lib/dnswatcher
 
+# Config loading also reads a `.env` file and a file named `dnswatcher`
+# (any config extension, or none) from the working directory. `/` holds
+# neither, so every setting comes from the environment. Do not make the
+# data directory, or the binary's directory, the working directory.
+WORKDIR /
+
+USER dnswatcher
+
 EXPOSE 8080
 
-ENTRYPOINT ["/app/dnswatcher"]
+# busybox wget (already in alpine) probes the health endpoint every 10
+# seconds, so the container is healthy well before upaas reads its health
+# 60 seconds after a deploy and fails the deploy unless it is healthy.
+HEALTHCHECK --interval=10s --timeout=5s --start-period=10s --retries=3 \
+    CMD wget -q -O /dev/null "http://127.0.0.1:${PORT:-8080}/.well-known/healthcheck" || exit 1
+
+ENTRYPOINT ["/usr/local/bin/dnswatcher"]

README.md

@@ -61,17 +61,35 @@ rejected.
   record types: A, AAAA, CNAME, MX, TXT, SRV, CAA, NS.
 - Stores results **per nameserver**. The state for a hostname is not a
   merged view — it is a map from nameserver to record set.
+- DNS names inside record values (CNAME, MX, SRV and NS targets) are
+  stored in lower case, because names are case-insensitive and
+  nameservers may answer in any letter case. TXT and CAA values keep
+  their letter case; they are not lower-cased.
 - Any observable change in any nameserver's response triggers a
   notification. This includes:
   - **Record change**: A nameserver returns different records than it
     did on the previous check (additions, removals, value changes).
   - **NS query failure**: A nameserver that previously responded
     becomes unreachable (timeout, SERVFAIL, REFUSED, network error).
-    This is distinct from "responded with no records."
+    This is distinct from "responded with no records": a nameserver
+    that answers NXDOMAIN or with no records has responded. The alert
+    is sent once, on the check where it starts failing. A failing
+    nameserver gives no records, so it is not reported as a record
+    change or compared for inconsistency. A nameserver that is already
+    failing on the first check that sees it is recorded silently.
   - **NS recovery**: A previously-unreachable nameserver starts
-    responding again.
+    responding again. Its records are not compared with those from
-  - **Inconsistency detected**: Two nameservers that previously agreed
+    before it failed, so a change made while it was failing is not
-    now return different record sets for the same hostname.
+    reported as a record change.
+  - **Inconsistency detected**: Two nameservers return different record
+    sets for the same hostname and did not already differ on the previous
+    check. Every pair of nameservers is compared. The alert is sent once
+    for each such pair, on the check where they start to disagree, and not
+    again while they keep disagreeing, including after a restart. A
+    nameserver that was not in the previous check (newly added, or back
+    after dropping out), or failed on it, and answers differently is
+    reported on the check where it answers. If a pair agrees again and
+    later disagrees, the alert is sent again.
 
 ### TCP Port Monitoring
 
@@ -203,6 +221,46 @@ are read, so a smaller value would sever the connection before a handler
 using its full budget could respond. `IdleTimeout` exceeds common
 Prometheus scrape intervals so the scraper reuses its connection.
 
+### Security Headers
+
+Every response — the dashboard, the static assets under `/s/...`, the
+healthchecks, the JSON API, and `/metrics` — carries the following
+headers, set by a global middleware:
+
+| Header                      | Value                                 |
+|-----------------------------|---------------------------------------|
+| `Strict-Transport-Security` | `max-age=31536000; includeSubDomains` |
+| `Content-Security-Policy`   | see below                             |
+| `X-Frame-Options`           | `DENY`                                |
+| `X-Content-Type-Options`    | `nosniff`                             |
+| `Referrer-Policy`           | `no-referrer`                         |
+| `Permissions-Policy`        | all unused browser features denied    |
+
+The content security policy is:
+
+```
+default-src 'self'; script-src 'none'; style-src 'self'; img-src 'self';
+font-src 'none'; connect-src 'none'; object-src 'none'; base-uri 'none';
+form-action 'none'; frame-ancestors 'none'
+```
+
+The dashboard ships no JavaScript (the 30-second refresh is a
+`<meta http-equiv="refresh">`), no inline styles, no inline event
+handlers, and no images; its only subresource is the embedded stylesheet
+at `/s/css/tailwind.min.css`, which `style-src 'self'` permits. The
+policy therefore needs neither `unsafe-inline` nor `unsafe-eval`.
+`frame-ancestors 'none'` is the primary anti-framing control, with
+`X-Frame-Options: DENY` retained as the legacy fallback.
+
+HSTS is emitted unconditionally, including over plain HTTP. dnswatcher is
+expected to run behind a TLS-terminating reverse proxy, and the browser
+must still be told to enforce HTTPS end to end, so the header is never
+gated on whether the request itself arrived over TLS.
+
+`Referrer-Policy: no-referrer` is stricter than the
+`strict-origin-when-cross-origin` baseline: the dashboard has no
+cross-origin navigation needs, and its URL may name internal hosts.
+
 ---
 
 ## Architecture
@@ -215,7 +273,8 @@ internal/
   globals/globals.go           Build-time variables (version)
   logger/logger.go             slog structured logging (TTY detection)
   healthcheck/healthcheck.go   Health check service
-  middleware/middleware.go      HTTP middleware (logging, CORS, metrics auth)
+  middleware/middleware.go      HTTP middleware (logging, CORS, security
+                               headers, metrics auth)
   handlers/handlers.go         HTTP request handlers
   server/
     server.go                  HTTP server lifecycle
@@ -226,6 +285,8 @@ internal/
   tlscheck/tlscheck.go        TLS certificate inspector
   notify/notify.go             Notification service (Slack, Mattermost, ntfy)
   watcher/watcher.go           Main monitoring orchestrator and scheduler
+  livednstest/livednstest.go   Retry and concurrency limit for tests
+                               against live DNS (imported only by tests)
 ```
 
 ### Design Principles
@@ -387,10 +448,14 @@ not as a merged view, to enable inconsistency detection.
 The `status` field for each per-nameserver entry and certificate entry
 tracks reachability:
 
-| Status      | Meaning                                         |
+| Status      | Meaning                                                    |
-|-------------|-------------------------------------------------|
+|-------------|------------------------------------------------------------|
-| `ok`        | Query succeeded, records are current             |
+| `ok`        | Query succeeded, records are current                       |
-| `error`     | Query failed (timeout, SERVFAIL, network error)  |
+| `error`     | Query failed (timeout, SERVFAIL, REFUSED, network error)   |
+
+A nameserver that answers NXDOMAIN or with no records has status `ok` and
+empty `records`. A nameserver whose query failed has status `error`, empty
+`records`, and the reason in `error`.
 
 ---
 
@@ -424,9 +489,13 @@ them. We provide:
 - `script/fmt` — format all code (gofmt -s, goimports)
 - `script/fmt-check` — check formatting (read-only)
 - `script/check` — run test, lint, and fmt-check
-- `script/docker` — build the Docker image tagged via
+- `script/docker` — build the Docker image tagged via `script/projectname`, with
-  `script/projectname`
+  `--no-cache-filter=lint,builder` so the lint stage and the builder stage,
-- `script/cibuild` — CI entrypoint: plain `docker build .`
+  which runs the tests, run on every invocation
+- `script/cibuild` — CI entrypoint: `docker build` with
+  `--no-cache-filter=lint,builder`, so the lint stage and the builder stage,
+  which runs the tests, run on every invocation, because a cached build lints
+  nothing and queries no DNS
 - `script/precommit` — run by the git pre-commit hook; `go mod tidy`
   guard, then `script/check`
 - `script/install-precommit` — install the git pre-commit hook
@@ -467,11 +536,57 @@ docker run -d \
 
 ---
 
+## Running under upaas
+
+[upaas](https://git.eeqj.de/sneak/upaas) builds the image from this
+repository's `Dockerfile` and runs it. The app needs:
+
+- **Branch:** `prod`. `prod` is cut from `main`, and merging a `main` to
+  `prod` pull request is a deploy.
+- **Volume:** one host directory mounted at `/var/lib/dnswatcher`, where
+  the state file lives. upaas bind-mounts the host path it is given and
+  does not create it. The container runs as uid 10001 and does not start
+  unless it can write there. Create the directory before the first
+  deploy:
+
+  ```sh
+  mkdir -p /path/to/data
+  chown 10001:10001 /path/to/data
+  chmod 700 /path/to/data
+  ```
+
+- **Network and port:** the dashboard is unauthenticated and shows every
+  watched name and recent alert, and upaas publishes every mapped port on
+  all interfaces of the host
+  ([upaas issue 113](https://git.eeqj.de/sneak/upaas/issues/113)). Add a
+  port mapping to container port `8080` only if the dashboard should be
+  public. Otherwise add none: set the app's Docker network in upaas to
+  your reverse proxy's Docker network, and the proxy reaches the app at
+  `upaas-` followed by the app name, port `8080`.
+- **Required environment:** `DNSWATCHER_TARGETS`, a comma-separated list
+  of the domains and hostnames to watch. dnswatcher refuses to start
+  without it.
+- **Recommended environment:** at least one notification endpoint
+  (`DNSWATCHER_SLACK_WEBHOOK`, `DNSWATCHER_MATTERMOST_WEBHOOK`,
+  `DNSWATCHER_NTFY_TOPIC`); without one, changes show only on the
+  dashboard. `DNSWATCHER_METRICS_USERNAME` and
+  `DNSWATCHER_METRICS_PASSWORD` serve `/metrics` behind basic auth.
+- **Leave unset:** `DNSWATCHER_DATA_DIR`, which the image sets to
+  `/var/lib/dnswatcher`, and `PORT`, which defaults to `8080`. Every
+  setting comes from the environment; the image holds no config file.
+- **Health check:** the image's own, which requests
+  `/.well-known/healthcheck` every 10 seconds. upaas reads the
+  container's health 60 seconds after a deploy and marks the deploy
+  failed unless it is `healthy`.
+
+---
+
 ## Monitoring Lifecycle
 
-1. **Startup**: Load state from disk. If no state file exists, start
+1. **Startup**: Check that the data directory can be written, and exit
-   with empty state (first check will establish baseline without
+   with an error naming it if not. Load state from disk. If no state
-   triggering change notifications).
+   file exists, start with empty state (first check will establish
+   baseline without triggering change notifications).
 2. **Initial check**: Immediately perform all DNS, port, and TLS checks
    on startup.
 3. **Periodic checks** (DNS always runs first):

TESTING.md

@@ -2,8 +2,12 @@
 
 ## DNS Resolution Tests
 
-All resolver tests **MUST** use live queries against real DNS servers.
+DNS is never mocked in this project, not in tests and not anywhere
-No mocking of the DNS client layer is permitted.
+else; see the README section "No DNS mocking. Ever." Every test that
+looks something up in DNS **MUST** query live DNS servers, never a
+stand-in. Logic that works on record data, such as comparing or
+formatting records, may be tested on that data directly with no
+lookup.
 
 ### Rationale
 
@@ -21,14 +25,18 @@ real servers ensures the resolver works correctly in production.
 - Query timeout is calibrated to 3× maximum antipodal RTT (~300ms)
   plus processing margin
 - Root server fan-out is limited to reduce parallel query load
+- Live lookups that expect an answer go through `internal/livednstest`,
+  which limits how many run at once in a test binary and retries a
+  lookup that got none
 - Flaky failures from transient network issues are acceptable and
   should be investigated as potential resolver bugs, not papered over
   with mocks or skip flags
 
 ### What NOT to do
 
-- **Do not mock `DNSClient`** for resolver tests (the mock constructor
+- **Do not mock, fake or stub DNS** anywhere: no stand-in `DNSClient`,
-  exists for unit-testing other packages that consume the resolver)
+  no stand-in for the watcher's `DNSResolver`, no fake DNS server, no
+  canned responses
 - **Do not add `-short` flags** to skip slow tests
 - **Do not increase `-timeout`** to hide hanging queries
 - **Do not remove `-count=1` from `script/test`** — Go's test cache

TODO.md

@@ -10,11 +10,7 @@
 
 # Status
 
-pre-1.0. No git tags. Core resolver work in flight on feature/resolver
+pre-1.0. No git tags.
-(dirty: internal/resolver/resolver_test.go). Local checkout has diverged
-from origin: origin/main is 8 commits ahead (watcher orchestrator,
-unified TARGETS) and origin/feature/resolver already contains the full
-iterative resolver implementation with hermetic mocked tests.
 
 # Next Step
 
@@ -23,6 +19,45 @@ Rationale, Design, TODO, License, Author) if any are still missing.
 
 # Completed Steps
 
+- 2026-10-01: a nameserver that does not answer is saved as `error` with the
+  reason, and NS failure and NS recovery are notified (closes #104).
+- 2026-10-01: notify shutdown tests use one timing constant per meaning, name
+  the bound they check, and require the drain's debug line (closes #116).
+- 2026-09-29: the live-DNS test package is renamed `internal/livednstest` and
+  added to the `test-support` `deny` list in `.golangci.yml`, so `make lint`
+  fails when program code imports it (closes #164).
+- 2026-09-29: `.golangci.yml` re-fetched unchanged from `sneak/prompts`. It
+  replaces the deprecated `gomodguard` with `gomodguard_v2`, so `make lint` no
+  longer warns about it, and turns on `depguard` with the org `test-support`
+  rule, which rejects `net/http/httptest` except in test files and in files
+  under a directory whose name ends in `test`. This repo had no `deny` entries
+  of its own to carry forward (closes #123).
+- 2026-09-29: nothing stands in for DNS any more. Watcher tests that look
+  something up in DNS use the real resolver against live DNS servers and test
+  record and nameserver changes by preparing the saved state a check starts
+  from; the resolver timeout test queries an address that never answers, and
+  `NewFromLoggerWithClient`, used only by its stand-in client, is gone. The
+  live-DNS retry and concurrency limit moved to `internal/livednstest`, which
+  both test packages use. `TESTING.md` states the README's rule (closes #159).
+- 2026-09-28: the inconsistency alert is sent once, on the check where two
+  nameservers start to disagree or where a nameserver that disagrees first
+  appears, instead of on every check while they disagree, and not again after
+  a restart. Every pair of nameservers is compared, not only neighbours in
+  sorted order of name (closes #158).
+- 2026-09-28: DNS names in record values (CNAME, MX, SRV and NS targets) are
+  lower-cased, so nameservers that answer in different letter case no longer
+  count as inconsistent or as a record change (closes #157).
+- 2026-09-28: `script/cibuild` and `script/docker` now pass
+  `--no-cache-filter=lint,builder` so lint and tests run every build (closes
+  #115).
+- 2026-09-28: the server timeout test now drives `Run` and checks the
+  `http.Server` it serves carries the timeouts; corrected the `ReadTimeout`
+  note in that test (closes #120).
+- 2026-09-28: upaas deploy readiness — runtime image runs as unprivileged
+  `dnswatcher`, Docker `HEALTHCHECK`, startup fails when the data directory is
+  not writable, README "Running under upaas" (closes #147).
+- 2026-09-21: added behavioural tests for `internal/globals`,
+  `internal/healthcheck`, and `internal/logger` (closes #110).
 - 2026-09-21: `go mod tidy` dropped the redundant `golang.org/x/sync`
   `// indirect` line so `script/bootstrap` leaves a clean tree (#132)
 - 2026-08-10: comment-only corrections to `script/bootstrap`,
@@ -111,6 +146,16 @@ Rationale, Design, TODO, License, Author) if any are still missing.
   than the 60s `chimw.Timeout` handler budget so that budget stays
   reachable, and tests in `internal/server` pin both the non-zero
   values and that relationship (#99)
+- 2026-08-09: security response headers middleware
+  (`SecurityHeaders()` in `internal/middleware/middleware.go`)
+  registered globally in `internal/server/routes.go`, so HSTS, CSP,
+  `X-Frame-Options`, `X-Content-Type-Options`, `Referrer-Policy`, and
+  `Permissions-Policy` are set on every response including `/s/...` and
+  `/metrics`; the CSP needs no `unsafe-inline`/`unsafe-eval` because the
+  dashboard ships no JavaScript and no inline styles; HSTS is emitted
+  unconditionally per policy (TLS-terminating proxy in front). Remaining
+  1.0 hardening items — `http.Server` timeouts, request body limits,
+  rate limiting, CORS scoping — are tracked separately
 - 2026-08-07: golangci-lint bumped to v2.12.2 (commit-pinned installs
   in `Dockerfile` and `script/bootstrap`); `.golangci.yml` set to the
   org-standard v2-schema config used across the org's repos
@@ -232,6 +277,5 @@ Infrastructure notes (from untracked TODO.md):
 - Module path sneak.berlin/go/dnswatcher differs from the git.eeqj.de
   remote intentionally; do not "fix" it
 - Dependencies: github.com/miekg/dns, golang.org/x/net/publicsuffix
-- Resolver tests originally used live DNS against *.dns.sneak.cloud
+- DNS is never mocked; tests that look something up in DNS query live DNS
-  (required records documented in the test file header); origin now has
+  servers (README, "No DNS mocking. Ever.")
-  mocked hermetic tests, keep them hermetic

internal/globals/globals_test.go

@@ -0,0 +1,50 @@
+package globals_test
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"sneak.berlin/go/dnswatcher/internal/globals"
+)
+
+// TestGlobals exercises the package-level version and appname
+// variables through their setters and read-back via New. These are
+// shared package state, so the test mutates a global and must run
+// sequentially; it cannot use t.Parallel().
+//
+//nolint:paralleltest // mutates shared package-level globals, must run sequentially
+func TestGlobals(t *testing.T) {
+	versions := []string{"v1.2.3", "dev", "", "v1.2.3-4-gabcdef"}
+	for _, want := range versions {
+		globals.SetVersion(want)
+
+		g, err := globals.New(nil)
+		require.NoError(t, err)
+		assert.Equal(t, want, g.Version,
+			"New must surface the version set by SetVersion")
+	}
+
+	names := []string{"dnswatcher", "other", ""}
+	for _, want := range names {
+		globals.SetAppname(want)
+
+		g, err := globals.New(nil)
+		require.NoError(t, err)
+		assert.Equal(t, want, g.Appname,
+			"New must surface the appname set by SetAppname")
+	}
+
+	// New returns a snapshot: a later SetVersion must not mutate a
+	// Globals handed out earlier.
+	globals.SetVersion("first")
+
+	g, err := globals.New(nil)
+	require.NoError(t, err)
+
+	globals.SetVersion("second")
+	assert.Equal(t, "first", g.Version,
+		"a Globals returned by New must not change when the "+
+			"package variable is set again")
+}

internal/healthcheck/healthcheck_test.go

@@ -0,0 +1,134 @@
+package healthcheck_test
+
+import (
+	"context"
+	"encoding/json"
+	"testing"
+	"time"
+
+	"go.uber.org/fx"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"sneak.berlin/go/dnswatcher/internal/config"
+	"sneak.berlin/go/dnswatcher/internal/globals"
+	"sneak.berlin/go/dnswatcher/internal/healthcheck"
+	"sneak.berlin/go/dnswatcher/internal/logger"
+)
+
+// recordingLifecycle is a minimal fx.Lifecycle that records the hooks
+// appended to it, so healthcheck.New can be exercised through its real
+// constructor without standing up a whole fx application.
+type recordingLifecycle struct {
+	hooks []fx.Hook
+}
+
+func (l *recordingLifecycle) Append(hook fx.Hook) {
+	l.hooks = append(l.hooks, hook)
+}
+
+// newHealthcheck builds a Healthcheck through the real constructor and
+// runs the registered OnStart hook so StartupTime is set the same way
+// the fx lifecycle would set it.
+func newHealthcheck(
+	t *testing.T,
+	maintenance bool,
+	version string,
+) *healthcheck.Healthcheck {
+	t.Helper()
+
+	g := &globals.Globals{Appname: "dnswatcher", Version: version}
+
+	log, err := logger.New(nil, logger.Params{Globals: g})
+	require.NoError(t, err)
+
+	lifecycle := &recordingLifecycle{}
+
+	hc, err := healthcheck.New(lifecycle, healthcheck.Params{
+		Globals: g,
+		Config:  &config.Config{MaintenanceMode: maintenance},
+		Logger:  log,
+	})
+	require.NoError(t, err)
+
+	require.Len(t, lifecycle.hooks, 1,
+		"New must register exactly one lifecycle hook")
+	require.NotNil(t, lifecycle.hooks[0].OnStart)
+	require.NoError(t, lifecycle.hooks[0].OnStart(context.Background()))
+
+	return hc
+}
+
+func TestCheckStatusAndPayloadShape(t *testing.T) {
+	t.Parallel()
+
+	hc := newHealthcheck(t, false, "v9.9.9")
+	resp := hc.Check()
+
+	assert.Equal(t, "ok", resp.Status)
+
+	// The JSON shape and field names are part of the contract for the
+	// /health and /.well-known/healthcheck routes, so assert on the
+	// exact set of keys the response marshals to.
+	raw, err := json.Marshal(resp)
+	require.NoError(t, err)
+
+	var fields map[string]json.RawMessage
+	require.NoError(t, json.Unmarshal(raw, &fields))
+
+	wantKeys := []string{
+		"status",
+		"now",
+		"uptimeSeconds",
+		"uptimeHuman",
+		"version",
+		"appname",
+		"maintenanceMode",
+	}
+	assert.Len(t, fields, len(wantKeys),
+		"response must marshal to exactly the documented fields")
+
+	for _, key := range wantKeys {
+		assert.Contains(t, fields, key, "missing JSON field %q", key)
+	}
+
+	// The Now field is documented as RFC3339Nano; a change to the
+	// format constant should turn this red.
+	_, err = time.Parse(time.RFC3339Nano, resp.Now)
+	assert.NoError(t, err, "Now must be RFC3339Nano")
+}
+
+func TestCheckMaintenanceModeReflectsConfig(t *testing.T) {
+	t.Parallel()
+
+	tests := []struct {
+		name        string
+		maintenance bool
+	}{
+		{"maintenance off", false},
+		{"maintenance on", true},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			t.Parallel()
+
+			hc := newHealthcheck(t, tt.maintenance, "test")
+			resp := hc.Check()
+			assert.Equal(t, tt.maintenance, resp.Maintenance,
+				"maintenanceMode must mirror Config.MaintenanceMode")
+		})
+	}
+}
+
+func TestCheckSurfacesVersionAndAppname(t *testing.T) {
+	t.Parallel()
+
+	hc := newHealthcheck(t, false, "surfaced-version-123")
+	resp := hc.Check()
+
+	assert.Equal(t, "surfaced-version-123", resp.Version,
+		"version from globals must appear in the payload")
+	assert.Equal(t, "dnswatcher", resp.Appname)
+}

internal/livednstest/livednstest.go

@@ -0,0 +1,121 @@
+// Package livednstest runs the live DNS operations of tests. Tests that
+// look something up in DNS query live DNS servers, never a stand-in —
+// see TESTING.md. Nothing here mocks, fakes, stubs, records or replays
+// DNS, and nothing here skips a test: it only changes *how* the live
+// queries are issued, so that a single dropped UDP packet or one slow
+// authoritative server does not turn correct code into a red build.
+//
+// Two mechanisms:
+//
+//  1. Bounded concurrency. Tests run in parallel and the build hosts
+//     have many cores, so without a limit every test starts its own
+//     iterative resolution at the same instant and they all hit the
+//     first root server within a few milliseconds of each other. Root
+//     servers rate-limit that, which shows up as a different arbitrary
+//     subset of tests failing on each run. Run caps how many live
+//     operations are in flight at once in one test binary.
+//
+//  2. Retry with exponential backoff. Each live operation gets several
+//     attempts with its own timeout. An attempt is retried when it
+//     obtained nothing to check, never because of what the test
+//     asserts about the result, so a wrong result still fails on the
+//     first attempt. A fault in the code under test that leaves
+//     nothing to check looks the same as live DNS not answering, and
+//     fails only after the last attempt.
+package livednstest
+
+import (
+	"context"
+	"errors"
+	"testing"
+	"time"
+)
+
+const (
+	// attempts is how many times a live DNS operation is attempted
+	// before the test fails.
+	attempts = 3
+
+	// AttemptTimeout bounds one attempt. Worst case for an operation
+	// is attempts * AttemptTimeout plus the backoff — about 26
+	// seconds, well inside the 90-second `go test -timeout` backstop
+	// even when several operations exhaust their attempts.
+	AttemptTimeout = 8 * time.Second
+
+	// backoffBase is the delay after the first failed attempt; it is
+	// multiplied by backoffFactor each time.
+	backoffBase = 500 * time.Millisecond
+
+	// backoffFactor is the exponential backoff multiplier.
+	backoffFactor = 2
+
+	// Concurrency caps how many live operations may be in flight
+	// across one test binary at once.
+	Concurrency = 6
+)
+
+// gate bounds concurrent live operations. It has to be package scoped:
+// the whole point is that it is shared by every parallel test in the
+// test binary.
+//
+//nolint:gochecknoglobals // package-wide live query rate limit
+var gate = make(chan struct{}, Concurrency)
+
+// ErrNoAnswer reports that a live operation produced no usable answer,
+// which is retried rather than asserted on.
+var ErrNoAnswer = errors.New("no answer from live DNS")
+
+// Run executes one attempt of a live operation, holding a slot in gate
+// for its duration and bounding it with its own timeout.
+func Run(op func(ctx context.Context) error) error {
+	gate <- struct{}{}
+	defer func() { <-gate }()
+
+	ctx, cancel := context.WithTimeout(
+		context.Background(), AttemptTimeout,
+	)
+	defer cancel()
+
+	return op(ctx)
+}
+
+// Retry runs op until it reports success, retrying failures with
+// exponential backoff, and fails the test if every attempt fails. op
+// returns an error only for a failure to obtain an answer — never for
+// an answer the test disagrees with, which belongs in an assertion so
+// that it fails immediately. op stores whatever it obtained where its
+// caller can find it.
+func Retry(
+	t *testing.T,
+	what string,
+	op func(ctx context.Context) error,
+) {
+	t.Helper()
+
+	var last error
+
+	backoff := backoffBase
+
+	for attempt := range attempts {
+		if attempt > 0 {
+			t.Logf(
+				"%s: attempt %d of %d failed (%v), "+
+					"retrying in %s",
+				what, attempt, attempts, last, backoff,
+			)
+			time.Sleep(backoff)
+
+			backoff *= backoffFactor
+		}
+
+		last = Run(op)
+		if last == nil {
+			return
+		}
+	}
+
+	t.Fatalf(
+		"%s: all %d live attempts failed: %v",
+		what, attempts, last,
+	)
+}

internal/livednstest/livednstest_test.go

@@ -0,0 +1,103 @@
+package livednstest_test
+
+import (
+	"context"
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+
+	"sneak.berlin/go/dnswatcher/internal/livednstest"
+)
+
+// Tests for the retry and the concurrency limit themselves. They
+// perform no DNS resolution of any kind.
+
+func TestRetryRecoversFromTransientFailure(t *testing.T) {
+	t.Parallel()
+
+	const wantAttempts = 2
+
+	attempts := 0
+
+	livednstest.Retry(t, "transient", func(_ context.Context) error {
+		attempts++
+
+		if attempts < wantAttempts {
+			return livednstest.ErrNoAnswer
+		}
+
+		return nil
+	})
+
+	assert.Equal(t, wantAttempts, attempts)
+}
+
+func TestRetryGivesEachAttemptADeadline(t *testing.T) {
+	t.Parallel()
+
+	livednstest.Retry(t, "deadline", func(ctx context.Context) error {
+		deadline, ok := ctx.Deadline()
+		assert.True(t, ok, "attempt should carry a deadline")
+
+		remaining := time.Until(deadline)
+
+		assert.LessOrEqual(t, remaining, livednstest.AttemptTimeout)
+
+		// Lower bound too: without one this passes for a
+		// deadline far shorter than intended, which would
+		// silently turn every live attempt into an instant
+		// timeout.
+		assert.Greater(t, remaining, livednstest.AttemptTimeout/2)
+
+		return nil
+	})
+}
+
+func TestRunBoundsConcurrency(t *testing.T) {
+	t.Parallel()
+
+	const workers = 24
+
+	var (
+		mu       sync.Mutex
+		wg       sync.WaitGroup
+		inFlight int
+		maxSeen  int
+	)
+
+	wg.Add(workers)
+
+	for range workers {
+		go func() {
+			defer wg.Done()
+
+			_ = livednstest.Run(func(_ context.Context) error {
+				mu.Lock()
+				inFlight++
+
+				if inFlight > maxSeen {
+					maxSeen = inFlight
+				}
+				mu.Unlock()
+
+				time.Sleep(time.Millisecond)
+
+				mu.Lock()
+				inFlight--
+				mu.Unlock()
+
+				return nil
+			})
+		}()
+	}
+
+	wg.Wait()
+
+	assert.Positive(t, maxSeen)
+	assert.LessOrEqual(
+		t, maxSeen, livednstest.Concurrency,
+		"live queries must stay under the package-wide gate",
+	)
+}

internal/logger/logger_test.go

@@ -0,0 +1,66 @@
+package logger_test
+
+import (
+	"context"
+	"log/slog"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"sneak.berlin/go/dnswatcher/internal/globals"
+	"sneak.berlin/go/dnswatcher/internal/logger"
+)
+
+func newTestLogger(t *testing.T) *logger.Logger {
+	t.Helper()
+
+	g := &globals.Globals{Appname: "dnswatcher", Version: "test"}
+
+	l, err := logger.New(nil, logger.Params{Globals: g})
+	require.NoError(t, err)
+
+	return l
+}
+
+// TestNewReturnsUsableLogger checks that the constructor yields a
+// working *slog.Logger.
+func TestNewReturnsUsableLogger(t *testing.T) {
+	t.Parallel()
+
+	l := newTestLogger(t)
+	require.NotNil(t, l.Get(), "Get must return a non-nil logger")
+}
+
+// TestDefaultLevelExcludesDebug verifies the default configuration
+// logs at info: debug records are suppressed, info records pass.
+func TestDefaultLevelExcludesDebug(t *testing.T) {
+	t.Parallel()
+
+	log := newTestLogger(t).Get()
+	ctx := context.Background()
+
+	assert.False(t, log.Enabled(ctx, slog.LevelDebug),
+		"debug must be suppressed at the default level")
+	assert.True(t, log.Enabled(ctx, slog.LevelInfo),
+		"info must be enabled at the default level")
+}
+
+// TestEnableDebugLoggingChangesLevel verifies the debug and non-debug
+// configurations differ as intended: enabling debug makes debug
+// records pass where they previously did not.
+func TestEnableDebugLoggingChangesLevel(t *testing.T) {
+	t.Parallel()
+
+	l := newTestLogger(t)
+	log := l.Get()
+	ctx := context.Background()
+
+	require.False(t, log.Enabled(ctx, slog.LevelDebug),
+		"debug must start disabled")
+
+	l.EnableDebugLogging()
+
+	assert.True(t, log.Enabled(ctx, slog.LevelDebug),
+		"debug must be enabled after EnableDebugLogging")
+}

internal/middleware/middleware.go

@@ -21,6 +21,60 @@ import (
 // corsMaxAge is the maximum age for CORS preflight responses.
 const corsMaxAge = 300
 
+// Security response header values applied to every response.
+//
+// The CSP is as strict as the dashboard allows: the template ships no
+// JavaScript, no inline styles, no inline event handlers and no images,
+// and its only subresource is the embedded stylesheet at
+// /s/css/tailwind.min.css, which style-src 'self' permits. Neither
+// unsafe-inline nor unsafe-eval is used. frame-ancestors 'none' is the
+// primary anti-framing control; X-Frame-Options is the legacy fallback.
+const (
+	// hstsValue is emitted unconditionally, including over plain HTTP,
+	// because the service runs behind a TLS-terminating proxy and the
+	// browser must still enforce HTTPS end to end.
+	hstsValue = "max-age=31536000; includeSubDomains"
+
+	cspValue = "default-src 'self'; " +
+		"script-src 'none'; " +
+		"style-src 'self'; " +
+		"img-src 'self'; " +
+		"font-src 'none'; " +
+		"connect-src 'none'; " +
+		"object-src 'none'; " +
+		"base-uri 'none'; " +
+		"form-action 'none'; " +
+		"frame-ancestors 'none'"
+
+	frameOptionsValue = "DENY"
+
+	contentTypeOptionsValue = "nosniff"
+
+	// referrerPolicyValue is stricter than the policy minimum of
+	// strict-origin-when-cross-origin: the dashboard has no
+	// cross-origin navigation needs and its URL may name internal
+	// hosts.
+	referrerPolicyValue = "no-referrer"
+
+	permissionsPolicyValue = "accelerometer=(), " +
+		"autoplay=(), " +
+		"camera=(), " +
+		"display-capture=(), " +
+		"encrypted-media=(), " +
+		"fullscreen=(), " +
+		"geolocation=(), " +
+		"gyroscope=(), " +
+		"magnetometer=(), " +
+		"microphone=(), " +
+		"midi=(), " +
+		"payment=(), " +
+		"picture-in-picture=(), " +
+		"publickey-credentials-get=(), " +
+		"screen-wake-lock=(), " +
+		"usb=(), " +
+		"xr-spatial-tracking=()"
+)
+
 // Params contains dependencies for Middleware.
 type Params struct {
 	fx.In
@@ -186,6 +240,37 @@ func (m *Middleware) CORS() func(http.Handler) http.Handler {
 	})
 }
 
+// SecurityHeaders returns middleware that sets the security response
+// headers required for production internet exposure on every response.
+//
+// The headers are set before the request reaches the next handler so
+// that they are present on every response, including panics recovered
+// by chi's Recoverer and timeouts produced by chi's Timeout.
+func (m *Middleware) SecurityHeaders() func(http.Handler) http.Handler {
+	return func(next http.Handler) http.Handler {
+		return http.HandlerFunc(func(
+			writer http.ResponseWriter,
+			request *http.Request,
+		) {
+			header := writer.Header()
+			header.Set("Strict-Transport-Security", hstsValue)
+			header.Set("Content-Security-Policy", cspValue)
+			header.Set("X-Frame-Options", frameOptionsValue)
+			header.Set(
+				"X-Content-Type-Options",
+				contentTypeOptionsValue,
+			)
+			header.Set("Referrer-Policy", referrerPolicyValue)
+			header.Set(
+				"Permissions-Policy",
+				permissionsPolicyValue,
+			)
+
+			next.ServeHTTP(writer, request)
+		})
+	}
+}
+
 // MetricsAuth returns basic auth middleware for /metrics.
 func (m *Middleware) MetricsAuth() func(http.Handler) http.Handler {
 	if m.params.Config.MetricsUsername == "" {

internal/middleware/middleware_test.go

@@ -0,0 +1,334 @@
+package middleware_test
+
+import (
+	"net/http"
+	"net/http/httptest"
+	"strings"
+	"testing"
+
+	"github.com/go-chi/chi/v5"
+	"go.uber.org/fx/fxtest"
+
+	"sneak.berlin/go/dnswatcher/internal/config"
+	"sneak.berlin/go/dnswatcher/internal/globals"
+	"sneak.berlin/go/dnswatcher/internal/handlers"
+	"sneak.berlin/go/dnswatcher/internal/logger"
+	"sneak.berlin/go/dnswatcher/internal/middleware"
+	"sneak.berlin/go/dnswatcher/internal/notify"
+	"sneak.berlin/go/dnswatcher/internal/state"
+)
+
+// Expected security header values, spelled out literally so that any
+// change to the middleware has to be made deliberately here as well.
+const (
+	wantHSTS = "max-age=31536000; includeSubDomains"
+
+	wantCSP = "default-src 'self'; " +
+		"script-src 'none'; " +
+		"style-src 'self'; " +
+		"img-src 'self'; " +
+		"font-src 'none'; " +
+		"connect-src 'none'; " +
+		"object-src 'none'; " +
+		"base-uri 'none'; " +
+		"form-action 'none'; " +
+		"frame-ancestors 'none'"
+
+	wantFrameOptions = "DENY"
+
+	wantContentTypeOptions = "nosniff"
+
+	wantReferrerPolicy = "no-referrer"
+
+	wantPermissionsPolicy = "accelerometer=(), " +
+		"autoplay=(), " +
+		"camera=(), " +
+		"display-capture=(), " +
+		"encrypted-media=(), " +
+		"fullscreen=(), " +
+		"geolocation=(), " +
+		"gyroscope=(), " +
+		"magnetometer=(), " +
+		"microphone=(), " +
+		"midi=(), " +
+		"payment=(), " +
+		"picture-in-picture=(), " +
+		"publickey-credentials-get=(), " +
+		"screen-wake-lock=(), " +
+		"usb=(), " +
+		"xr-spatial-tracking=()"
+)
+
+// stylesheetPath is the only subresource the dashboard loads.
+const stylesheetPath = "/s/css/tailwind.min.css"
+
+// newTestLogger builds a logger for direct component construction.
+func newTestLogger(t *testing.T) *logger.Logger {
+	t.Helper()
+
+	glob, err := globals.New(nil)
+	if err != nil {
+		t.Fatalf("globals.New: %v", err)
+	}
+
+	log, err := logger.New(nil, logger.Params{Globals: glob})
+	if err != nil {
+		t.Fatalf("logger.New: %v", err)
+	}
+
+	return log
+}
+
+// newTestMiddleware builds a Middleware without an fx application.
+func newTestMiddleware(t *testing.T) *middleware.Middleware {
+	t.Helper()
+
+	glob, err := globals.New(nil)
+	if err != nil {
+		t.Fatalf("globals.New: %v", err)
+	}
+
+	mw, err := middleware.New(nil, middleware.Params{
+		Logger:  newTestLogger(t),
+		Globals: glob,
+		Config:  &config.Config{},
+	})
+	if err != nil {
+		t.Fatalf("middleware.New: %v", err)
+	}
+
+	return mw
+}
+
+// serveWithSecurityHeaders runs a GET through SecurityHeaders and
+// returns the recorded response.
+func serveWithSecurityHeaders(
+	t *testing.T,
+	target string,
+	handler http.Handler,
+) *httptest.ResponseRecorder {
+	t.Helper()
+
+	mw := newTestMiddleware(t)
+	rec := httptest.NewRecorder()
+	req := httptest.NewRequestWithContext(
+		t.Context(), http.MethodGet, target, nil,
+	)
+
+	mw.SecurityHeaders()(handler).ServeHTTP(rec, req)
+
+	return rec
+}
+
+// okHandler writes a trivial 200 response.
+func okHandler() http.Handler {
+	return http.HandlerFunc(func(
+		writer http.ResponseWriter,
+		_ *http.Request,
+	) {
+		writer.WriteHeader(http.StatusOK)
+	})
+}
+
+func TestSecurityHeaders(t *testing.T) {
+	t.Parallel()
+
+	tests := []struct {
+		name   string
+		header string
+		want   string
+	}{
+		{
+			"hsts",
+			"Strict-Transport-Security",
+			wantHSTS,
+		},
+		{
+			"csp",
+			"Content-Security-Policy",
+			wantCSP,
+		},
+		{
+			"frame options",
+			"X-Frame-Options",
+			wantFrameOptions,
+		},
+		{
+			"content type options",
+			"X-Content-Type-Options",
+			wantContentTypeOptions,
+		},
+		{
+			"referrer policy",
+			"Referrer-Policy",
+			wantReferrerPolicy,
+		},
+		{
+			"permissions policy",
+			"Permissions-Policy",
+			wantPermissionsPolicy,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			t.Parallel()
+
+			rec := serveWithSecurityHeaders(t, "/", okHandler())
+
+			got := rec.Header().Get(tt.header)
+			if got != tt.want {
+				t.Errorf(
+					"%s = %q, want %q",
+					tt.header, got, tt.want,
+				)
+			}
+		})
+	}
+}
+
+// TestSecurityHeadersCSPDirectives guards the properties the repo
+// policy requires of the content security policy itself.
+func TestSecurityHeadersCSPDirectives(t *testing.T) {
+	t.Parallel()
+
+	rec := serveWithSecurityHeaders(t, "/", okHandler())
+	csp := rec.Header().Get("Content-Security-Policy")
+
+	forbidden := []string{"unsafe-inline", "unsafe-eval"}
+	for _, directive := range forbidden {
+		if strings.Contains(csp, directive) {
+			t.Errorf("CSP must not contain %q: %q", directive, csp)
+		}
+	}
+
+	required := []string{
+		"default-src 'self'",
+		"script-src 'none'",
+		"style-src 'self'",
+		"frame-ancestors 'none'",
+	}
+	for _, directive := range required {
+		if !strings.Contains(csp, directive) {
+			t.Errorf("CSP must contain %q: %q", directive, csp)
+		}
+	}
+}
+
+// TestSecurityHeadersOnErrorResponse verifies the headers are emitted
+// even when the wrapped handler fails, since they are set before the
+// handler runs.
+func TestSecurityHeadersOnErrorResponse(t *testing.T) {
+	t.Parallel()
+
+	failing := http.HandlerFunc(func(
+		writer http.ResponseWriter,
+		_ *http.Request,
+	) {
+		http.Error(
+			writer,
+			"boom",
+			http.StatusInternalServerError,
+		)
+	})
+
+	rec := serveWithSecurityHeaders(t, "/api/v1/status", failing)
+
+	if rec.Code != http.StatusInternalServerError {
+		t.Fatalf("status = %d, want 500", rec.Code)
+	}
+
+	if got := rec.Header().Get(
+		"X-Content-Type-Options",
+	); got != wantContentTypeOptions {
+		t.Errorf(
+			"X-Content-Type-Options = %q, want %q",
+			got, wantContentTypeOptions,
+		)
+	}
+
+	if got := rec.Header().Get(
+		"Strict-Transport-Security",
+	); got != wantHSTS {
+		t.Errorf(
+			"Strict-Transport-Security = %q, want %q",
+			got, wantHSTS,
+		)
+	}
+}
+
+// newTestHandlers builds real Handlers with empty monitoring state.
+func newTestHandlers(t *testing.T) *handlers.Handlers {
+	t.Helper()
+
+	glob, err := globals.New(nil)
+	if err != nil {
+		t.Fatalf("globals.New: %v", err)
+	}
+
+	log := newTestLogger(t)
+
+	notifier, err := notify.New(fxtest.NewLifecycle(t), notify.Params{
+		Logger: log,
+		Config: &config.Config{},
+	})
+	if err != nil {
+		t.Fatalf("notify.New: %v", err)
+	}
+
+	hnd, err := handlers.New(nil, handlers.Params{
+		Logger:  log,
+		Globals: glob,
+		State:   state.NewForTest(),
+		Notify:  notifier,
+	})
+	if err != nil {
+		t.Fatalf("handlers.New: %v", err)
+	}
+
+	return hnd
+}
+
+// TestDashboardRendersWithSecurityHeaders renders the real dashboard
+// through the middleware and checks that the policy still permits the
+// one stylesheet the page loads.
+func TestDashboardRendersWithSecurityHeaders(t *testing.T) {
+	t.Parallel()
+
+	mw := newTestMiddleware(t)
+	hnd := newTestHandlers(t)
+
+	router := chi.NewRouter()
+	router.Use(mw.SecurityHeaders())
+	router.Get("/", hnd.HandleDashboard())
+
+	rec := httptest.NewRecorder()
+	req := httptest.NewRequestWithContext(
+		t.Context(), http.MethodGet, "/", nil,
+	)
+
+	router.ServeHTTP(rec, req)
+
+	if rec.Code != http.StatusOK {
+		t.Fatalf("status = %d, want 200", rec.Code)
+	}
+
+	body := rec.Body.String()
+	if !strings.Contains(body, stylesheetPath) {
+		t.Errorf("dashboard does not reference %q", stylesheetPath)
+	}
+
+	if !strings.Contains(body, "dnswatcher") {
+		t.Errorf("dashboard body looks empty: %d bytes", len(body))
+	}
+
+	csp := rec.Header().Get("Content-Security-Policy")
+	if csp != wantCSP {
+		t.Errorf("CSP = %q, want %q", csp, wantCSP)
+	}
+
+	// The stylesheet is same-origin, so style-src 'self' allows it.
+	if !strings.Contains(csp, "style-src 'self'") {
+		t.Errorf("CSP would block %q: %q", stylesheetPath, csp)
+	}
+}

internal/notify/shutdown_test.go

@@ -33,10 +33,29 @@ const (
 	// out.
 	drainDeadline = 50 * time.Millisecond
 
-	// drainSlack is the upper bound on how long a bounded
+	// timeoutDrainBound is how long a drain given drainDeadline
-	// drain may take; generous enough for a loaded CI box,
+	// may take to return before the test gives up on it.  At
-	// still far below the 20s test ceiling.
+	// forty times drainDeadline it leaves ample room for
-	drainSlack = 2 * time.Second
+	// scheduling delay on a loaded box under -race, yet it is far
+	// below the test binary's -timeout, so a drain that its
+	// deadline does not bound fails that one test instead of
+	// hanging the package.
+	timeoutDrainBound = 2 * time.Second
+
+	// longDrainDeadline is the deadline given to a drain that is
+	// expected to finish well before it: when the in-flight
+	// delivery completes after inFlightHold, or at once when
+	// nothing is in flight.  It is far above inFlightHold, so
+	// those drains never reach it, and four times
+	// idleDrainBound, so an idle drain that waited for its
+	// deadline instead of returning fails that bound.
+	longDrainDeadline = 2 * time.Second
+
+	// reachEndpointTimeout is how long a submitted delivery may
+	// take to reach the test server.  That normally takes a few
+	// milliseconds; the margin is for a loaded box under -race,
+	// and only a failing run ever waits this long.
+	reachEndpointTimeout = 2 * time.Second
 
 	// settleDelay is how long to wait before asserting that
 	// something did *not* happen.
@@ -46,10 +65,11 @@ const (
 	// nothing in flight.  It is deliberately far above the cost
 	// of the goroutine hop through inFlight.Wait() — which
 	// reached 57ms on a loaded box under -race with the package's
-	// parallel tests — and far below drainSlack, the deadline
+	// parallel tests — and far below longDrainDeadline, the
-	// such a drain is given.  A drain that blocked until its
+	// deadline such a drain is given.  A drain that blocked until
-	// deadline instead of returning on the WaitGroup therefore
+	// its deadline instead of returning on the WaitGroup
-	// still fails this bound, but scheduling delay alone cannot.
+	// therefore still fails this bound, but scheduling delay
+	// alone cannot.
 	idleDrainBound = 500 * time.Millisecond
 )
 
@@ -75,12 +95,14 @@ func (sb *syncBuffer) String() string {
 }
 
 // newLoggingService returns a Service writing JSON logs into
-// the returned buffer.
+// the returned buffer, debug level included.
 func newLoggingService(
 	transport http.RoundTripper,
 ) (*notify.Service, *syncBuffer) {
 	logs := &syncBuffer{}
-	handler := slog.NewJSONHandler(logs, nil)
+	handler := slog.NewJSONHandler(
+		logs, &slog.HandlerOptions{Level: slog.LevelDebug},
+	)
 
 	return notify.NewTestServiceWithLogger(transport, handler),
 		logs
@@ -134,7 +156,7 @@ func TestDrainWaitsForInFlightDelivery(t *testing.T) {
 	// drain begins.
 	select {
 	case <-entered:
-	case <-time.After(drainSlack):
+	case <-time.After(reachEndpointTimeout):
 		t.Fatal("delivery never reached the endpoint")
 	}
 
@@ -151,7 +173,7 @@ func TestDrainWaitsForInFlightDelivery(t *testing.T) {
 	defer timer.Stop()
 
 	ctx, cancel := context.WithTimeout(
-		context.Background(), drainSlack,
+		context.Background(), longDrainDeadline,
 	)
 	defer cancel()
 
@@ -257,11 +279,11 @@ func TestDrainBoundedByContextDeadline(t *testing.T) {
 
 	select {
 	case <-returned:
-	case <-time.After(drainSlack):
+	case <-time.After(timeoutDrainBound):
 		t.Fatalf(
 			"drain did not return within %v; its %v deadline "+
 				"did not bound it",
-			drainSlack, drainDeadline,
+			timeoutDrainBound, drainDeadline,
 		)
 	}
 
@@ -333,7 +355,7 @@ func TestDrainRefusesNewDeliveries(t *testing.T) {
 	svc.SetMattermostWebhookURL(target)
 
 	ctx, cancel := context.WithTimeout(
-		context.Background(), drainSlack,
+		context.Background(), longDrainDeadline,
 	)
 	defer cancel()
 
@@ -446,7 +468,7 @@ func TestNewRegistersDrainingStopHook(t *testing.T) {
 
 	select {
 	case <-entered:
-	case <-time.After(drainSlack):
+	case <-time.After(reachEndpointTimeout):
 		t.Fatal("delivery never reached the endpoint")
 	}
 
@@ -456,7 +478,7 @@ func TestNewRegistersDrainingStopHook(t *testing.T) {
 	defer timer.Stop()
 
 	ctx, cancel := context.WithTimeout(
-		context.Background(), drainSlack,
+		context.Background(), longDrainDeadline,
 	)
 	defer cancel()
 
@@ -487,7 +509,7 @@ func TestDrainWithoutDeliveriesReturnsImmediately(t *testing.T) {
 	start := time.Now()
 
 	ctx, cancel := context.WithTimeout(
-		context.Background(), drainSlack,
+		context.Background(), longDrainDeadline,
 	)
 	defer cancel()
 
@@ -495,9 +517,9 @@ func TestDrainWithoutDeliveriesReturnsImmediately(t *testing.T) {
 
 	if elapsed := time.Since(start); elapsed > idleDrainBound {
 		t.Errorf(
-			"drain of an idle service took %v, want well "+
+			"drain of an idle service took %v, want at most "+
-				"under its %v deadline",
+				"%v; its deadline was %v",
-			elapsed, drainSlack,
+			elapsed, idleDrainBound, longDrainDeadline,
 		)
 	}
 }
@@ -505,10 +527,11 @@ func TestDrainWithoutDeliveriesReturnsImmediately(t *testing.T) {
 // TestDrainWithCancelledContextDoesNotWarn verifies that an
 // OnStop context that is already dead on entry does not produce
 // an "abandoning them" warning when there was nothing in flight
-// to abandon.  The expired context wins the select immediately,
+// to abandon, and that the drain returns and says at debug level
-// so only the outstanding count can tell the difference between
+// that nothing was in flight.  The expired context wins the
-// a genuine timeout and a shutdown that had simply already run
+// select immediately, so only the outstanding count can tell the
-// out of time with no work left.
+// difference between a genuine timeout and a shutdown that had
+// simply already run out of time with no work left.
 func TestDrainWithCancelledContextDoesNotWarn(t *testing.T) {
 	t.Parallel()
 
@@ -517,11 +540,43 @@ func TestDrainWithCancelledContextDoesNotWarn(t *testing.T) {
 	ctx, cancel := context.WithCancel(context.Background())
 	cancel()
 
-	svc.Drain(ctx)
+	// A watchdog, as in TestDrainBoundedByContextDeadline, so
+	// that a drain which never returns fails here instead of
+	// hanging the package.
+	returned := make(chan struct{})
 
-	if output := logs.String(); strings.Contains(
+	go func() {
-		output, `"level":"WARN"`,
+		defer close(returned)
+
+		svc.Drain(ctx)
+	}()
+
+	select {
+	case <-returned:
+	case <-time.After(idleDrainBound):
+		t.Fatalf(
+			"drain with nothing in flight and a cancelled "+
+				"context did not return within %v",
+			idleDrainBound,
+		)
+	}
+
+	output := logs.String()
+
+	// The absence of a warning alone would also pass if the drain
+	// logged nothing at all, so require the debug line it writes
+	// when it finds nothing outstanding.
+	if !strings.Contains(
+		output, "all in-flight notifications completed",
 	) {
+		t.Errorf(
+			"drain did not log that nothing was in flight; "+
+				"log output: %s",
+			output,
+		)
+	}
+
+	if strings.Contains(output, `"level":"WARN"`) {
 		t.Errorf(
 			"drain with nothing in flight warned about "+
 				"abandoned deliveries; log output: %s",

internal/resolver/dns_client.go

@@ -7,8 +7,8 @@ import (
 	"github.com/miekg/dns"
 )
 
-// DNSClient abstracts DNS wire-protocol exchanges so the resolver
+// DNSClient sends one DNS message to a nameserver and returns the
-// can be tested without hitting real nameservers.
+// reply. The resolver holds one for UDP and one for TCP.
 type DNSClient interface {
 	ExchangeContext(
 		ctx context.Context,

internal/resolver/export_test.go

@@ -0,0 +1,8 @@
+package resolver
+
+import "github.com/miekg/dns"
+
+// ExtractRecordValue exports extractRecordValue for testing.
+func ExtractRecordValue(rr dns.RR) string {
+	return extractRecordValue(rr)
+}

internal/resolver/iterative.go

@@ -504,7 +504,9 @@ func (r *Resolver) queryAllTypes(
 type queryState struct {
 	gotNXDomain bool
 	gotSERVFAIL bool
+	gotRefused  bool
 	gotTimeout  bool
+	netErr      error
 	hasRecords  bool
 }
 
@@ -542,8 +544,13 @@ func (r *Resolver) querySingleType(
 ) {
 	msg, err := r.queryDNS(ctx, nsIP, hostname, qtype)
 	if err != nil {
-		if isTimeout(err) {
+		switch {
+		case isTimeout(err):
 			state.gotTimeout = true
+		case errors.Is(err, ErrRefused):
+			state.gotRefused = true
+		default:
+			state.netErr = err
 		}
 
 		return
@@ -603,12 +610,21 @@ func classifyResponse(resp *NameserverResponse, state queryState) {
 	case state.gotSERVFAIL && !state.hasRecords:
 		resp.Status = StatusError
 		resp.Error = "server returned SERVFAIL"
+	case state.gotRefused && !state.hasRecords:
+		resp.Status = StatusError
+		resp.Error = "server returned REFUSED"
+	case state.netErr != nil && !state.hasRecords:
+		resp.Status = StatusError
+		resp.Error = "network error: " + state.netErr.Error()
 	case !state.hasRecords && !state.gotNXDomain:
 		resp.Status = StatusNoData
 	}
 }
 
-// extractRecordValue formats a DNS RR value as a string.
+// extractRecordValue formats a DNS RR value as a string. DNS names
+// are case-insensitive and nameservers may answer in any letter case,
+// so names are lower-cased to compare equal. TXT and CAA values keep
+// their letter case.
 func extractRecordValue(rr dns.RR) string {
 	switch r := rr.(type) {
 	case *dns.A:
@@ -616,22 +632,22 @@ func extractRecordValue(rr dns.RR) string {
 	case *dns.AAAA:
 		return r.AAAA.String()
 	case *dns.CNAME:
-		return r.Target
+		return strings.ToLower(r.Target)
 	case *dns.MX:
-		return fmt.Sprintf("%d %s", r.Preference, r.Mx)
+		return fmt.Sprintf("%d %s", r.Preference, strings.ToLower(r.Mx))
 	case *dns.TXT:
 		return strings.Join(r.Txt, "")
 	case *dns.SRV:
 		return fmt.Sprintf(
 			"%d %d %d %s",
-			r.Priority, r.Weight, r.Port, r.Target,
+			r.Priority, r.Weight, r.Port, strings.ToLower(r.Target),
 		)
 	case *dns.CAA:
 		return fmt.Sprintf(
 			"%d %s \"%s\"", r.Flag, r.Tag, r.Value,
 		)
 	case *dns.NS:
-		return r.Ns
+		return strings.ToLower(r.Ns)
 	default:
 		return ""
 	}
@@ -679,11 +695,14 @@ func (r *Resolver) queryEachNS(
 	results := make(map[string]*NameserverResponse)
 
 	for _, ns := range nameservers {
+		resp, err := r.QueryNameserver(ctx, ns, hostname)
+
+		// A query the context cut short says nothing about the
+		// nameserver, so it must not be returned as its failure.
 		if checkCtx(ctx) != nil {
 			return nil, ErrContextCanceled
 		}
 
-		resp, err := r.QueryNameserver(ctx, ns, hostname)
 		if err != nil {
 			results[ns] = &NameserverResponse{
 				Nameserver: ns,
@@ -711,21 +730,13 @@ func (r *Resolver) LookupNS(
 
 // LookupAllRecords performs iterative resolution to find all DNS
 // records for the given hostname, keyed by authoritative nameserver.
+// Each nameserver's response carries its status and error with its
+// records.
 func (r *Resolver) LookupAllRecords(
 	ctx context.Context,
 	hostname string,
-) (map[string]map[string][]string, error) {
+) (map[string]*NameserverResponse, error) {
-	results, err := r.QueryAllNameservers(ctx, hostname)
+	return r.QueryAllNameservers(ctx, hostname)
-	if err != nil {
-		return nil, err
-	}
-
-	out := make(map[string]map[string][]string, len(results))
-	for ns, resp := range results {
-		out[ns] = resp.Records
-	}
-
-	return out, nil
 }
 
 // ResolveIPAddresses resolves a hostname to all IPv4 and IPv6

internal/resolver/iterative_test.go

@@ -0,0 +1,62 @@
+package resolver_test
+
+import (
+	"testing"
+
+	"github.com/miekg/dns"
+	"github.com/stretchr/testify/assert"
+
+	"sneak.berlin/go/dnswatcher/internal/resolver"
+)
+
+func TestExtractRecordValue_LetterCase(t *testing.T) {
+	t.Parallel()
+
+	tests := []struct {
+		name string
+		rr   dns.RR
+		want string
+	}{
+		{
+			name: "MX target lower-cased",
+			rr:   &dns.MX{Preference: 1, Mx: "ASPMX.L.GOOGLE.COM."},
+			want: "1 aspmx.l.google.com.",
+		},
+		{
+			name: "NS target lower-cased",
+			rr:   &dns.NS{Ns: "x.ns.joker.COM."},
+			want: "x.ns.joker.com.",
+		},
+		{
+			name: "CNAME target lower-cased",
+			rr:   &dns.CNAME{Target: "WWW.Example.Com."},
+			want: "www.example.com.",
+		},
+		{
+			name: "SRV target lower-cased",
+			rr: &dns.SRV{
+				Priority: 10, Weight: 5, Port: 443,
+				Target: "SIP.Example.Com.",
+			},
+			want: "10 5 443 sip.example.com.",
+		},
+		{
+			name: "TXT value keeps its case",
+			rr:   &dns.TXT{Txt: []string{"Verify=AbC123"}},
+			want: "Verify=AbC123",
+		},
+		{
+			name: "CAA value keeps its case",
+			rr:   &dns.CAA{Flag: 0, Tag: "issue", Value: "LetsEncrypt.org"},
+			want: `0 issue "LetsEncrypt.org"`,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			t.Parallel()
+
+			assert.Equal(t, tt.want, resolver.ExtractRecordValue(tt.rr))
+		})
+	}
+}

internal/resolver/livedns_harness_test.go

@@ -1,10 +1,7 @@
 package resolver_test
 
 import (
-	"context"
-	"sync"
 	"testing"
-	"time"
 
 	"github.com/stretchr/testify/assert"
 
@@ -12,9 +9,8 @@ import (
 )
 
 // Tests for the live-DNS harness in livedns_test.go itself. These
-// exercise pure logic and the retry/concurrency plumbing; they
+// exercise pure logic; they perform no DNS resolution of any kind, so
-// perform no DNS resolution of any kind, so they neither mock DNS
+// they neither mock DNS nor depend on it.
-// nor depend on it.
 
 // Names for the synthetic status maps below. Nothing is ever queried
 // at them: they are map keys handed to the package's pure counting
@@ -90,47 +86,6 @@ func TestStatusCountingIgnoresSilentNameservers(t *testing.T) {
 	)
 }
 
-func TestRetryLiveRecoversFromTransientFailure(t *testing.T) {
-	t.Parallel()
-
-	const wantAttempts = 2
-
-	attempts := 0
-
-	retryLive(t, "transient", func(_ context.Context) error {
-		attempts++
-
-		if attempts < wantAttempts {
-			return errLiveNoAnswer
-		}
-
-		return nil
-	})
-
-	assert.Equal(t, wantAttempts, attempts)
-}
-
-func TestRetryLiveGivesEachAttemptADeadline(t *testing.T) {
-	t.Parallel()
-
-	retryLive(t, "deadline", func(ctx context.Context) error {
-		deadline, ok := ctx.Deadline()
-		assert.True(t, ok, "attempt should carry a deadline")
-
-		remaining := time.Until(deadline)
-
-		assert.LessOrEqual(t, remaining, liveAttemptTimeout)
-
-		// Lower bound too: without one this passes for a
-		// deadline far shorter than intended, which would
-		// silently turn every live attempt into an instant
-		// timeout.
-		assert.Greater(t, remaining, liveAttemptTimeout/2)
-
-		return nil
-	})
-}
-
 // TestUnsanctionedStatusesRejectsWrongAnswers is the regression test
 // for the defect this allowlist exists to prevent: a minority of
 // nameservers answering WRONGLY while quorum keeps the suite green.
@@ -235,50 +190,3 @@ func TestUnsanctionedStatusesToleratesSilenceOnly(t *testing.T) {
 		unsanctionedStatuses(results, allowed...),
 	)
 }
-
-func TestRunLiveBoundsConcurrency(t *testing.T) {
-	t.Parallel()
-
-	const workers = 24
-
-	var (
-		mu       sync.Mutex
-		wg       sync.WaitGroup
-		inFlight int
-		maxSeen  int
-	)
-
-	wg.Add(workers)
-
-	for range workers {
-		go func() {
-			defer wg.Done()
-
-			_ = runLive(func(_ context.Context) error {
-				mu.Lock()
-				inFlight++
-
-				if inFlight > maxSeen {
-					maxSeen = inFlight
-				}
-				mu.Unlock()
-
-				time.Sleep(time.Millisecond)
-
-				mu.Lock()
-				inFlight--
-				mu.Unlock()
-
-				return nil
-			})
-		}()
-	}
-
-	wg.Wait()
-
-	assert.Positive(t, maxSeen)
-	assert.LessOrEqual(
-		t, maxSeen, liveConcurrency,
-		"live queries must stay under the package-wide gate",
-	)
-}

internal/resolver/livedns_test.go

@@ -8,8 +8,8 @@ import (
 	"sort"
 	"strings"
 	"testing"
-	"time"
 
+	"sneak.berlin/go/dnswatcher/internal/livednstest"
 	"sneak.berlin/go/dnswatcher/internal/resolver"
 )
 
@@ -17,144 +17,34 @@ import (
 // Live DNS test support
 // ----------------------------------------------------------------
 //
-// Every test in this package resolves against the real, live DNS —
+// Tests that look something up in DNS query live DNS servers, never a
-// see TESTING.md. Nothing here mocks, fakes, stubs, records or
+// stand-in; logic that works on record data may be tested on that
-// replays DNS, and nothing here skips or gates a test: the helpers
+// data with no lookup (see TESTING.md). Each live operation below goes
-// below only change *how* the live queries are issued, so that a
+// through livednstest.Retry, which bounds how many resolutions are in
-// single dropped UDP packet or one slow authoritative server does
+// flight at once and retries an operation that got no answer (see
-// not turn a correct resolver into a red build.
+// package livednstest).
 //
-// Three mechanisms, all test-side:
+// Where an assertion spans several independent nameservers, a quorum
+// is enough: a strict majority answering as expected. A server that
+// fails to answer is tolerated, while a server that answers *wrongly*
+// still fails the test.
 //
-//  1. Bounded concurrency. The package's tests are parallel and the
+// That tolerance is expressed as an ALLOWLIST of sanctioned statuses,
-//     build hosts have many cores, so without a limit every test
+// never as a blocklist of known-bad ones. A blocklist bans the one
-//     starts its own iterative resolution at the same instant and
+// wrong answer its author thought of and silently admits every other
-//     they all hit the first root server in rootServerList() within
+// status, including any added to the resolver later; an allowlist
-//     a few milliseconds of each other. Root servers rate-limit
+// fails on anything nobody explicitly sanctioned. Silence (timeout,
-//     that, which shows up as a different arbitrary subset of tests
+// error) is the only thing quorum exists to tolerate. A *wrong
-//     failing on each run. liveGate caps how many resolutions are
+// answer* — nxdomain for a name that exists, ok for one that does
-//     in flight at once.
+// not, nodata for either — is never tolerated at any count.
-//
-//  2. Retry with exponential backoff. Each live operation gets
-//     several attempts with its own timeout. The retry predicate is
-//     strictly transport-level — "did a nameserver answer at all" —
-//     never the assertion the test is making. A resolver that
-//     answers incorrectly still fails on the first attempt.
-//
-//  3. Quorum. Where an assertion spans several independent
-//     nameservers, a strict majority answering as expected is
-//     enough; a server that fails to answer is tolerated, while a
-//     server that answers *wrongly* still fails the test.
-//
-// The tolerance in (3) is expressed as an ALLOWLIST of sanctioned
-// statuses, never as a blocklist of known-bad ones. A blocklist bans
-// the one wrong answer its author thought of and silently admits
-// every other status, including any added to the resolver later; an
-// allowlist fails on anything nobody explicitly sanctioned. Silence
-// (timeout, error) is the only thing quorum exists to tolerate. A
-// *wrong answer* — nxdomain for a name that exists, ok for one that
-// does not, nodata for either — is never tolerated at any count.
 
-const (
+// minNameservers is the smallest nameserver count a well-run zone is
-	// liveAttempts is how many times a live DNS operation is
+// expected to publish.
-	// attempted before the test fails.
+const minNameservers = 2
-	liveAttempts = 3
 
-	// liveAttemptTimeout bounds one attempt. Worst case for an
+// errLiveNoQuorum reports that too few of a domain's nameservers
-	// operation is liveAttempts * liveAttemptTimeout plus the
+// answered for a quorum assertion to be made.
-	// backoff — about 26 seconds, well inside the 90-second
+var errLiveNoQuorum = errors.New("no nameserver quorum")
-	// `go test -timeout` backstop even when several operations
-	// exhaust their attempts.
-	liveAttemptTimeout = 8 * time.Second
-
-	// liveBackoffBase is the delay after the first failed
-	// attempt; it is multiplied by liveBackoffFactor each time.
-	liveBackoffBase = 500 * time.Millisecond
-
-	// liveBackoffFactor is the exponential backoff multiplier.
-	liveBackoffFactor = 2
-
-	// liveConcurrency caps how many live resolutions may be in
-	// flight across the whole package at once.
-	liveConcurrency = 6
-
-	// minNameservers is the smallest nameserver count a
-	// well-run zone is expected to publish.
-	minNameservers = 2
-)
-
-// liveGate bounds concurrent live resolutions package-wide. It has
-// to be package scoped: the whole point is that it is shared by
-// every parallel test in the package.
-//
-//nolint:gochecknoglobals // package-wide live query rate limit
-var liveGate = make(chan struct{}, liveConcurrency)
-
-var (
-	// errLiveNoAnswer reports that a live operation produced no
-	// usable answer, which is retried rather than asserted on.
-	errLiveNoAnswer = errors.New("no answer from live DNS")
-
-	// errLiveNoQuorum reports that too few of a domain's
-	// nameservers answered for a quorum assertion to be made.
-	errLiveNoQuorum = errors.New("no nameserver quorum")
-)
-
-// runLive executes one attempt of a live operation, holding a slot
-// in liveGate for its duration and bounding it with its own
-// timeout.
-func runLive(op func(ctx context.Context) error) error {
-	liveGate <- struct{}{}
-	defer func() { <-liveGate }()
-
-	ctx, cancel := context.WithTimeout(
-		context.Background(), liveAttemptTimeout,
-	)
-	defer cancel()
-
-	return op(ctx)
-}
-
-// retryLive runs op until it reports success, retrying transport
-// failures with exponential backoff, and fails the test if every
-// attempt fails. op returns an error only for a failure to obtain
-// an answer — never for an answer the test disagrees with, which
-// belongs in an assertion so that it fails immediately. op stores
-// whatever it obtained where its caller can find it.
-func retryLive(
-	t *testing.T,
-	what string,
-	op func(ctx context.Context) error,
-) {
-	t.Helper()
-
-	var last error
-
-	backoff := liveBackoffBase
-
-	for attempt := range liveAttempts {
-		if attempt > 0 {
-			t.Logf(
-				"%s: attempt %d of %d failed (%v), "+
-					"retrying in %s",
-				what, attempt, liveAttempts, last, backoff,
-			)
-			time.Sleep(backoff)
-
-			backoff *= liveBackoffFactor
-		}
-
-		last = runLive(op)
-		if last == nil {
-			return
-		}
-	}
-
-	t.Fatalf(
-		"%s: no answer after %d live attempts: %v",
-		what, liveAttempts, last,
-	)
-}
 
 // liveQuorum is how many of total nameservers must agree for a
 // multi-nameserver assertion to hold: a strict majority.
@@ -272,7 +162,7 @@ func liveFindAuthoritative(
 
 	var out []string
 
-	retryLive(
+	livednstest.Retry(
 		t,
 		"FindAuthoritativeNameservers("+domain+")",
 		func(ctx context.Context) error {
@@ -284,7 +174,7 @@ func liveFindAuthoritative(
 			if len(ns) == 0 {
 				return fmt.Errorf(
 					"%w: %s has no nameservers",
-					errLiveNoAnswer, domain,
+					livednstest.ErrNoAnswer, domain,
 				)
 			}
 
@@ -308,7 +198,7 @@ func liveLookupNS(
 
 	var out []string
 
-	retryLive(
+	livednstest.Retry(
 		t,
 		"LookupNS("+domain+")",
 		func(ctx context.Context) error {
@@ -320,7 +210,7 @@ func liveLookupNS(
 			if len(ns) == 0 {
 				return fmt.Errorf(
 					"%w: %s has no nameservers",
-					errLiveNoAnswer, domain,
+					livednstest.ErrNoAnswer, domain,
 				)
 			}
 
@@ -350,7 +240,7 @@ func liveQueryNameserver(
 
 	var out *resolver.NameserverResponse
 
-	retryLive(
+	livednstest.Retry(
 		t,
 		what,
 		func(ctx context.Context) error {
@@ -365,7 +255,7 @@ func liveQueryNameserver(
 				resp.Status == resolver.StatusError {
 				return fmt.Errorf(
 					"%w: %s returned %s: %s",
-					errLiveNoAnswer, nameserver,
+					livednstest.ErrNoAnswer, nameserver,
 					resp.Status, resp.Error,
 				)
 			}
@@ -392,7 +282,7 @@ func liveQueryAllNameservers(
 
 	var out map[string]*resolver.NameserverResponse
 
-	retryLive(
+	livednstest.Retry(
 		t,
 		"QueryAllNameservers("+hostname+")",
 		func(ctx context.Context) error {
@@ -404,7 +294,7 @@ func liveQueryAllNameservers(
 			if len(results) == 0 {
 				return fmt.Errorf(
 					"%w: no nameservers queried for %s",
-					errLiveNoAnswer, hostname,
+					livednstest.ErrNoAnswer, hostname,
 				)
 			}
 
@@ -437,7 +327,7 @@ func liveResolveIPs(
 
 	var out []string
 
-	retryLive(
+	livednstest.Retry(
 		t,
 		"ResolveIPAddresses("+hostname+")",
 		func(ctx context.Context) error {
@@ -449,7 +339,7 @@ func liveResolveIPs(
 			if len(ips) == 0 {
 				return fmt.Errorf(
 					"%w: no addresses for %s",
-					errLiveNoAnswer, hostname,
+					livednstest.ErrNoAnswer, hostname,
 				)
 			}
 
@@ -476,7 +366,7 @@ func liveResolveIPsAllowingEmpty(
 
 	var out []string
 
-	retryLive(
+	livednstest.Retry(
 		t,
 		"ResolveIPAddresses("+hostname+")",
 		func(ctx context.Context) error {

internal/resolver/resolver.go

@@ -67,17 +67,4 @@ func NewFromLogger(log *slog.Logger) *Resolver {
 	}
 }
 
-// NewFromLoggerWithClient creates a Resolver with a custom DNS
-// client, useful for testing with mock DNS responses.
-func NewFromLoggerWithClient(
-	log *slog.Logger,
-	client DNSClient,
-) *Resolver {
-	return &Resolver{
-		log:    log,
-		client: client,
-		tcp:    client,
-	}
-}
-
 // Method implementations are in iterative.go.

internal/resolver/resolver_test.go

@@ -2,6 +2,7 @@ package resolver_test
 
 import (
 	"context"
+	"fmt"
 	"log/slog"
 	"net"
 	"os"
@@ -10,10 +11,10 @@ import (
 	"testing"
 	"time"
 
-	"github.com/miekg/dns"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 
+	"sneak.berlin/go/dnswatcher/internal/livednstest"
 	"sneak.berlin/go/dnswatcher/internal/resolver"
 )
 
@@ -33,8 +34,8 @@ func newTestResolver(t *testing.T) *resolver.Resolver {
 }
 
 // findOneNSForDomain picks one authoritative nameserver to aim a
-// test at. Live-DNS retry, concurrency and quorum handling live in
+// test at. Quorum handling lives in livedns_test.go, and the live-DNS
-// livedns_test.go.
+// retry and concurrency limit in package livednstest.
 func findOneNSForDomain(
 	t *testing.T,
 	r *resolver.Resolver,
@@ -232,6 +233,45 @@ func TestQueryNameserver_NXDomain(t *testing.T) {
 	assert.Equal(t, resolver.StatusNXDomain, resp.Status)
 }
 
+// TestQueryNameserver_Refused asks a google.com nameserver about
+// cloudflare.com, a zone it does not serve, which it refuses. Refusing
+// is a failure to answer, not an answer with no records.
+func TestQueryNameserver_Refused(t *testing.T) {
+	t.Parallel()
+
+	r := newTestResolver(t)
+	ns := findOneNSForDomain(t, r, "google.com")
+
+	var resp *resolver.NameserverResponse
+
+	livednstest.Retry(
+		t,
+		"QueryNameserver("+ns+", cloudflare.com)",
+		func(ctx context.Context) error {
+			var err error
+
+			resp, err = r.QueryNameserver(ctx, ns, "cloudflare.com")
+			if err != nil {
+				return err
+			}
+
+			// A timeout or a network error is no reply at all.
+			if resp.Status == resolver.StatusTimeout ||
+				strings.HasPrefix(resp.Error, "network error") {
+				return fmt.Errorf(
+					"%w: %s: %s",
+					livednstest.ErrNoAnswer, ns, resp.Error,
+				)
+			}
+
+			return nil
+		},
+	)
+
+	assert.Equal(t, resolver.StatusError, resp.Status)
+	assert.Equal(t, "server returned REFUSED", resp.Error)
+}
+
 func TestQueryNameserver_RecordsSorted(t *testing.T) {
 	t.Parallel()
 
@@ -526,21 +566,18 @@ func TestQueryAllNameservers_ContextCanceled(t *testing.T) {
 func TestQueryNameserverIP_Timeout(t *testing.T) {
 	t.Parallel()
 
-	log := slog.New(slog.NewTextHandler(
+	r := newTestResolver(t)
-		os.Stderr,
-		&slog.HandlerOptions{Level: slog.LevelDebug},
-	))
-
-	r := resolver.NewFromLoggerWithClient(
-		log, &timeoutClient{},
-	)
 
+	// Nothing answers at 192.0.2.1, a documentation address. The
+	// resolver tries each query twice, and the first try gives up
+	// after two seconds. A deadline that ends during the first try
+	// makes the status vary from run to run between error and
+	// timeout, so the deadline must outlast the first try.
 	ctx, cancel := context.WithTimeout(
-		context.Background(), 10*time.Second,
+		context.Background(), 3*time.Second,
 	)
 	t.Cleanup(cancel)
 
-	// Query any IP — the client always returns a timeout error.
 	resp, err := r.QueryNameserverIP(
 		ctx, "unreachable.test.", "192.0.2.1",
 		"example.com",
@@ -551,27 +588,6 @@ func TestQueryNameserverIP_Timeout(t *testing.T) {
 	assert.NotEmpty(t, resp.Error)
 }
 
-// timeoutClient simulates DNS timeout errors for testing.
-type timeoutClient struct{}
-
-func (c *timeoutClient) ExchangeContext(
-	_ context.Context,
-	_ *dns.Msg,
-	_ string,
-) (*dns.Msg, time.Duration, error) {
-	return nil, 0, &net.OpError{
-		Op:  "read",
-		Net: "udp",
-		Err: &timeoutError{},
-	}
-}
-
-type timeoutError struct{}
-
-func (e *timeoutError) Error() string   { return "i/o timeout" }
-func (e *timeoutError) Timeout() bool   { return true }
-func (e *timeoutError) Temporary() bool { return true }
-
 func TestResolveIPAddresses_ContextCanceled(t *testing.T) {
 	t.Parallel()
 

internal/server/export_test.go

@@ -5,15 +5,20 @@ import (
 	"time"
 )
 
-// NewHTTPServer exports newHTTPServer for testing.
-func NewHTTPServer(
-	listenAddr string,
-	handler http.Handler,
-) *http.Server {
-	return newHTTPServer(listenAddr, handler)
-}
-
 // RequestTimeout exports the handler execution budget applied by
 // chimw.Timeout in SetupRoutes, so tests can assert the relationship
 // between it and the server's WriteTimeout.
 const RequestTimeout time.Duration = requestTimeout
+
+// SetListenPort overrides the port Run binds. A test uses it to hand
+// Run an unbindable port so ListenAndServe fails immediately and Run
+// returns after storing its http.Server.
+func SetListenPort(s *Server, port int) {
+	s.port = port
+}
+
+// HTTPServerOf returns the http.Server that Run built and stored, so a
+// test can inspect the timeouts the running server actually carries.
+func HTTPServerOf(s *Server) *http.Server {
+	return s.httpServer
+}

internal/server/routes.go

@@ -21,6 +21,7 @@ func (s *Server) SetupRoutes() {
 	// Global middleware
 	s.router.Use(chimw.Recoverer)
 	s.router.Use(chimw.RequestID)
+	s.router.Use(s.mw.SecurityHeaders())
 	s.router.Use(s.mw.Logging())
 	s.router.Use(s.mw.CORS())
 	s.router.Use(chimw.Timeout(requestTimeout))

internal/server/server_test.go

@@ -1,112 +1,130 @@
 package server_test
 
 import (
-	"net/http"
 	"testing"
 
+	"github.com/spf13/viper"
+	"go.uber.org/fx"
+
+	"sneak.berlin/go/dnswatcher/internal/config"
+	"sneak.berlin/go/dnswatcher/internal/globals"
+	"sneak.berlin/go/dnswatcher/internal/handlers"
+	"sneak.berlin/go/dnswatcher/internal/healthcheck"
+	"sneak.berlin/go/dnswatcher/internal/logger"
+	"sneak.berlin/go/dnswatcher/internal/middleware"
+	"sneak.berlin/go/dnswatcher/internal/notify"
 	"sneak.berlin/go/dnswatcher/internal/server"
+	"sneak.berlin/go/dnswatcher/internal/state"
 )
 
-// noopHandler stands in for the router; newHTTPServer only stores it.
+// buildServer wires a *server.Server exactly as cmd/dnswatcher does,
-func noopHandler() http.Handler {
+// minus the watcher/resolver subtree that would touch live DNS. fx
-	return http.HandlerFunc(
+// builds the object graph but the lifecycle is never started, so no
-		func(w http.ResponseWriter, _ *http.Request) {
+// OnStart hook runs and nothing listens or resolves. The caller must
-			w.WriteHeader(http.StatusOK)
+// first configure viper (config.New reads it), which is also why the
-		},
+// caller cannot run in parallel.
+func buildServer(t *testing.T) *server.Server {
+	t.Helper()
+
+	var srv *server.Server
+
+	app := fx.New(
+		fx.NopLogger,
+		fx.Provide(
+			globals.New,
+			logger.New,
+			config.New,
+			state.New,
+			healthcheck.New,
+			notify.New,
+			middleware.New,
+			handlers.New,
+			server.New,
+		),
+		fx.Populate(&srv),
 	)
-}
 
-// TestHTTPServerTimeoutsAreSet asserts that every socket-level
+	err := app.Err()
-// timeout is configured. A zero value in net/http means "no limit",
+	if err != nil {
-// so a refactor that silently drops one of these reintroduces the
+		t.Fatalf("building server graph: %v", err)
-// slowloris / unreaped-keep-alive exposure this guards against.
-//
-// The assertions are on the configured field values only; nothing
-// here measures elapsed time, so the test cannot flake on timing.
-func TestHTTPServerTimeoutsAreSet(t *testing.T) {
-	t.Parallel()
-
-	srv := server.NewHTTPServer(":8080", noopHandler())
-
-	if srv.ReadTimeout <= 0 {
-		t.Errorf(
-			"ReadTimeout must be non-zero, got %v",
-			srv.ReadTimeout,
-		)
 	}
 
-	if srv.ReadHeaderTimeout <= 0 {
+	return srv
+}
+
+// TestRunWiresSocketTimeouts pins that the http.Server the running
+// server actually serves — the one Run builds and hands to
+// ListenAndServe — carries every socket-level timeout, plus the two
+// relationships the values must satisfy.
+//
+// Run is driven to completion with an unbindable port: it builds and
+// stores s.httpServer, then ListenAndServe fails at once and Run
+// returns without ever listening. The assertions run in the same
+// goroutine after Run returns, so reading s.httpServer is free of any
+// data race. Nothing here measures elapsed time.
+//
+// ReadTimeout must be at least ReadHeaderTimeout. net/http reads the
+// headers under ReadHeaderTimeout, then sets the read deadline for the
+// rest of the request to ReadTimeout, counted from when it started
+// reading the request. If ReadTimeout were smaller, a request whose
+// headers arrived after ReadTimeout but within ReadHeaderTimeout would
+// get a read deadline that had already passed, so reading its body
+// would fail at once.
+func TestRunWiresSocketTimeouts(t *testing.T) {
+	// Sets an env var and touches viper global state, so like the
+	// config tests it cannot use t.Parallel.
+	viper.Reset()
+	t.Setenv("DNSWATCHER_TARGETS", "example.com")
+
+	srv := buildServer(t)
+	server.SetListenPort(srv, -1)
+
+	srv.Run()
+
+	hs := server.HTTPServerOf(srv)
+	if hs == nil {
+		t.Fatal("Run did not build an http.Server")
+	}
+
+	if hs.ReadTimeout <= 0 {
+		t.Errorf("ReadTimeout must be non-zero, got %v", hs.ReadTimeout)
+	}
+
+	if hs.ReadHeaderTimeout <= 0 {
 		t.Errorf(
 			"ReadHeaderTimeout must be non-zero, got %v",
-			srv.ReadHeaderTimeout,
+			hs.ReadHeaderTimeout,
 		)
 	}
 
-	if srv.WriteTimeout <= 0 {
+	if hs.WriteTimeout <= 0 {
-		t.Errorf(
+		t.Errorf("WriteTimeout must be non-zero, got %v", hs.WriteTimeout)
-			"WriteTimeout must be non-zero, got %v",
-			srv.WriteTimeout,
-		)
 	}
 
-	if srv.IdleTimeout <= 0 {
+	if hs.IdleTimeout <= 0 {
-		t.Errorf(
+		t.Errorf("IdleTimeout must be non-zero, got %v", hs.IdleTimeout)
-			"IdleTimeout must be non-zero, got %v",
-			srv.IdleTimeout,
-		)
 	}
-}
 
-// TestWriteTimeoutExceedsHandlerBudget pins the one relationship the
+	if hs.WriteTimeout <= server.RequestTimeout {
-// values must satisfy. net/http arms the write deadline once request
-// headers are read, so it covers handler execution plus the response
-// flush. If WriteTimeout were not greater than the chimw.Timeout
-// handler budget, the connection would be severed before a handler
-// that used its full budget could respond, making that budget
-// unreachable.
-func TestWriteTimeoutExceedsHandlerBudget(t *testing.T) {
-	t.Parallel()
-
-	srv := server.NewHTTPServer(":8080", noopHandler())
-
-	if srv.WriteTimeout <= server.RequestTimeout {
 		t.Errorf(
 			"WriteTimeout (%v) must exceed handler budget (%v)",
-			srv.WriteTimeout,
+			hs.WriteTimeout,
 			server.RequestTimeout,
 		)
 	}
-}
 
-// TestReadTimeoutCoversHeaderTimeout asserts the read deadline for
+	if hs.ReadTimeout < hs.ReadHeaderTimeout {
-// the whole request is at least as long as the header-only deadline;
-// a smaller ReadTimeout would make ReadHeaderTimeout unreachable.
-func TestReadTimeoutCoversHeaderTimeout(t *testing.T) {
-	t.Parallel()
-
-	srv := server.NewHTTPServer(":8080", noopHandler())
-
-	if srv.ReadTimeout < srv.ReadHeaderTimeout {
 		t.Errorf(
 			"ReadTimeout (%v) must be >= ReadHeaderTimeout (%v)",
-			srv.ReadTimeout,
+			hs.ReadTimeout,
-			srv.ReadHeaderTimeout,
+			hs.ReadHeaderTimeout,
+		)
+	}
+
+	if hs.Handler != srv {
+		t.Errorf(
+			"Run wired handler %T, want the *server.Server",
+			hs.Handler,
 		)
 	}
 }
-
-// TestHTTPServerAddrAndHandler covers the rest of the constructor so
-// a future edit cannot drop the listen address or the handler.
-func TestHTTPServerAddrAndHandler(t *testing.T) {
-	t.Parallel()
-
-	srv := server.NewHTTPServer(":9999", noopHandler())
-
-	if srv.Addr != ":9999" {
-		t.Errorf("Addr = %q, want %q", srv.Addr, ":9999")
-	}
-
-	if srv.Handler == nil {
-		t.Error("Handler must not be nil")
-	}
-}

internal/state/state.go

@@ -148,6 +148,11 @@ func New(
 
 	lifecycle.Append(fx.Hook{
 		OnStart: func(_ context.Context) error {
+			err := state.checkDataDirWritable()
+			if err != nil {
+				return err
+			}
+
 			return state.Load()
 		},
 		OnStop: func(_ context.Context) error {
@@ -345,3 +350,27 @@ func (s *State) GetCertificateState(
 
 	return cs, ok
 }
+
+// checkDataDirWritable creates the data directory if needed, then writes
+// and removes the temp file that Save uses. It runs at startup so that an
+// unwritable directory stops the process, instead of the process running
+// with every save failing and only logged.
+func (s *State) checkDataDirWritable() error {
+	dir := s.config.DataDir
+	tmpPath := s.config.StatePath() + ".tmp"
+
+	err := os.MkdirAll(dir, dirPermissions)
+	if err == nil {
+		err = os.WriteFile(tmpPath, nil, filePermissions)
+	}
+
+	if err == nil {
+		err = os.Remove(tmpPath)
+	}
+
+	if err != nil {
+		return fmt.Errorf("data directory %s is not writable: %w", dir, err)
+	}
+
+	return nil
+}

internal/state/state_test.go

@@ -4,10 +4,16 @@ import (
 	"encoding/json"
 	"os"
 	"path/filepath"
+	"strings"
 	"sync"
 	"testing"
 	"time"
 
+	"go.uber.org/fx/fxtest"
+
+	"sneak.berlin/go/dnswatcher/internal/config"
+	"sneak.berlin/go/dnswatcher/internal/globals"
+	"sneak.berlin/go/dnswatcher/internal/logger"
 	"sneak.berlin/go/dnswatcher/internal/state"
 )
 
@@ -493,6 +499,107 @@ func TestSaveWritePermissionError(t *testing.T) {
 	}
 }
 
+// startState builds a State through the real constructor and runs its
+// startup hook against dataDir, returning the startup error.
+func startState(t *testing.T, dataDir string) error {
+	t.Helper()
+
+	g, err := globals.New(nil)
+	if err != nil {
+		t.Fatalf("globals.New: %v", err)
+	}
+
+	log, err := logger.New(nil, logger.Params{Globals: g})
+	if err != nil {
+		t.Fatalf("logger.New: %v", err)
+	}
+
+	lifecycle := fxtest.NewLifecycle(t)
+
+	_, err = state.New(lifecycle, state.Params{
+		Logger: log,
+		Config: &config.Config{DataDir: dataDir},
+	})
+	if err != nil {
+		t.Fatalf("state.New: %v", err)
+	}
+
+	return lifecycle.Start(t.Context())
+}
+
+// TestStartupFailsWhenDataDirNotWritable verifies that startup stops
+// with an error naming the data directory when it cannot be written.
+// The directory's parent is a regular file, which also fails as root.
+func TestStartupFailsWhenDataDirNotWritable(t *testing.T) {
+	t.Parallel()
+
+	parent := filepath.Join(t.TempDir(), "file")
+
+	err := os.WriteFile(parent, nil, 0o600)
+	if err != nil {
+		t.Fatalf("writing file: %v", err)
+	}
+
+	dataDir := filepath.Join(parent, "data")
+
+	err = startState(t, dataDir)
+	if err == nil {
+		t.Fatal("startup should fail when the data directory is not writable")
+	}
+
+	want := "data directory " + dataDir + " is not writable"
+	if !strings.Contains(err.Error(), want) {
+		t.Errorf("startup error %q does not contain %q", err, want)
+	}
+}
+
+// TestStartupFailsWhenExistingDataDirNotWritable verifies that startup
+// stops when the data directory exists but the temp file that saving uses
+// cannot be written in it. A directory sitting at the temp file's path
+// makes that write fail, which also holds as root.
+func TestStartupFailsWhenExistingDataDirNotWritable(t *testing.T) {
+	t.Parallel()
+
+	dataDir := t.TempDir()
+
+	err := os.Mkdir(filepath.Join(dataDir, "state.json.tmp"), 0o700)
+	if err != nil {
+		t.Fatalf("creating directory: %v", err)
+	}
+
+	err = startState(t, dataDir)
+	if err == nil {
+		t.Fatal("startup should fail when the data directory is not writable")
+	}
+
+	want := "data directory " + dataDir + " is not writable"
+	if !strings.Contains(err.Error(), want) {
+		t.Errorf("startup error %q does not contain %q", err, want)
+	}
+}
+
+// TestStartupCreatesDataDir verifies that startup creates a missing
+// data directory and leaves nothing behind in it.
+func TestStartupCreatesDataDir(t *testing.T) {
+	t.Parallel()
+
+	dataDir := filepath.Join(t.TempDir(), "data")
+
+	err := startState(t, dataDir)
+	if err != nil {
+		t.Fatalf("startup error: %v", err)
+	}
+
+	entries, err := os.ReadDir(dataDir)
+	if err != nil {
+		t.Fatalf("reading data directory: %v", err)
+	}
+
+	if len(entries) != 0 {
+		t.Errorf("startup left %d entries in the data directory", len(entries))
+	}
+}
+
 // TestPortStateUnmarshalJSON_NewFormat verifies deserialization of the
 // current multi-hostname format.
 func TestPortStateUnmarshalJSON_NewFormat(t *testing.T) {

internal/watcher/export_test.go

@@ -0,0 +1,33 @@
+package watcher
+
+import (
+	"context"
+	"time"
+
+	"sneak.berlin/go/dnswatcher/internal/resolver"
+	"sneak.berlin/go/dnswatcher/internal/state"
+)
+
+// NewlyDisagreeingPairs exports newlyDisagreeingPairs for testing.
+func NewlyDisagreeingPairs(
+	prev, current *state.HostnameState,
+) [][2]string {
+	return newlyDisagreeingPairs(prev, current)
+}
+
+// DetectHostnameChanges exports detectHostnameChanges for testing.
+func (w *Watcher) DetectHostnameChanges(
+	ctx context.Context,
+	hostname string,
+	prev, current *state.HostnameState,
+) {
+	w.detectHostnameChanges(ctx, hostname, prev, current)
+}
+
+// BuildHostnameState exports buildHostnameState for testing.
+func BuildHostnameState(
+	results map[string]*resolver.NameserverResponse,
+	now time.Time,
+) *state.HostnameState {
+	return buildHostnameState(results, now)
+}

internal/watcher/inconsistency_test.go

@@ -0,0 +1,185 @@
+package watcher_test
+
+import (
+	"slices"
+	"testing"
+
+	"sneak.berlin/go/dnswatcher/internal/state"
+	"sneak.berlin/go/dnswatcher/internal/watcher"
+)
+
+const (
+	host = "www.example.net"
+	nsA  = "a.ns.example.net."
+	nsB  = "b.ns.example.net."
+	nsC  = "c.ns.example.net."
+	ip1  = "192.0.2.1"
+	ip2  = "192.0.2.2"
+	ip3  = "192.0.2.3"
+)
+
+// hostnameState builds the state a check with these records leaves behind.
+func hostnameState(
+	records map[string]map[string][]string,
+) *state.HostnameState {
+	hs := &state.HostnameState{
+		RecordsByNameserver: make(map[string]*state.NameserverRecordState),
+	}
+
+	for ns, recs := range records {
+		hs.RecordsByNameserver[ns] = &state.NameserverRecordState{
+			Records: recs,
+			Status:  "ok",
+		}
+	}
+
+	return hs
+}
+
+func TestNewlyDisagreeingPairs(t *testing.T) {
+	t.Parallel()
+
+	onlyA := map[string]map[string][]string{nsA: {"A": {ip1}}}
+	agree := map[string]map[string][]string{nsA: {"A": {ip1}}, nsB: {"A": {ip1}}}
+	disagree := map[string]map[string][]string{nsA: {"A": {ip1}}, nsB: {"A": {ip2}}}
+	alert := [][2]string{{nsA, nsB}}
+
+	// b already disagrees with a and c; then c changes, so a and c,
+	// which agreed, now differ.
+	bDiffers := map[string]map[string][]string{
+		nsA: {"A": {ip1}}, nsB: {"A": {ip2}}, nsC: {"A": {ip1}},
+	}
+	cChanges := map[string]map[string][]string{
+		nsA: {"A": {ip1}}, nsB: {"A": {ip2}}, nsC: {"A": {ip3}},
+	}
+
+	// Each case starts from the state loaded at startup and runs the
+	// checks in order; want[i] is what check i alerts for.
+	tests := []struct {
+		name   string
+		loaded map[string]map[string][]string
+		checks []map[string]map[string][]string
+		want   [][][2]string
+	}{
+		{
+			name:   "disagreement persisting across checks alerts once",
+			loaded: agree,
+			checks: []map[string]map[string][]string{disagree, disagree, disagree},
+			want:   [][][2]string{alert, nil, nil},
+		},
+		{
+			name:   "disagreement starting on a later check alerts on it",
+			loaded: agree,
+			checks: []map[string]map[string][]string{agree, agree, disagree},
+			want:   [][][2]string{nil, nil, alert},
+		},
+		{
+			name:   "disagreement in the loaded state does not alert",
+			loaded: disagree,
+			checks: []map[string]map[string][]string{disagree, disagree},
+			want:   [][][2]string{nil, nil},
+		},
+		{
+			name:   "nameserver new on the first check and disagreeing alerts once",
+			loaded: onlyA,
+			checks: []map[string]map[string][]string{disagree, disagree},
+			want:   [][][2]string{alert, nil},
+		},
+		{
+			name:   "disagreement after agreeing again alerts again",
+			loaded: agree,
+			checks: []map[string]map[string][]string{disagree, agree, disagree},
+			want:   [][][2]string{alert, nil, alert},
+		},
+		{
+			name:   "new disagreement while another nameserver differs alerts",
+			loaded: bDiffers,
+			checks: []map[string]map[string][]string{cChanges, cChanges},
+			want:   [][][2]string{{{nsA, nsC}}, nil},
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			t.Parallel()
+
+			prev := hostnameState(tt.loaded)
+
+			for i, records := range tt.checks {
+				current := hostnameState(records)
+
+				got := watcher.NewlyDisagreeingPairs(prev, current)
+				if !slices.Equal(got, tt.want[i]) {
+					t.Errorf(
+						"check %d: alerted for %v, want %v",
+						i, got, tt.want[i],
+					)
+				}
+
+				prev = current
+			}
+		})
+	}
+}
+
+func TestInconsistencyAlert(t *testing.T) {
+	t.Parallel()
+
+	onlyA := map[string]map[string][]string{nsA: {"A": {ip1}}}
+	agree := map[string]map[string][]string{nsA: {"A": {ip1}}, nsB: {"A": {ip1}}}
+	disagree := map[string]map[string][]string{nsA: {"A": {ip1}}, nsB: {"A": {ip2}}}
+
+	// Each case starts from the state loaded at startup and then sees
+	// the nameservers disagree on three checks in a row.
+	tests := []struct {
+		name   string
+		loaded map[string]map[string][]string
+		want   int
+	}{
+		{
+			name:   "disagreement lasting several checks alerts once",
+			loaded: agree,
+			want:   1,
+		},
+		{
+			name:   "disagreement in the loaded state does not alert",
+			loaded: disagree,
+			want:   0,
+		},
+		{
+			name:   "nameserver new on the first check and disagreeing alerts once",
+			loaded: onlyA,
+			want:   1,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			t.Parallel()
+
+			// The hostname change detection uses only the notifier.
+			notifier := &mockNotifier{}
+			w := watcher.NewForTest(nil, nil, nil, nil, nil, notifier)
+
+			prev := hostnameState(tt.loaded)
+
+			for range 3 {
+				current := hostnameState(disagree)
+				w.DetectHostnameChanges(t.Context(), host, prev, current)
+				prev = current
+			}
+
+			got := 0
+
+			for _, n := range notifier.getNotifications() {
+				if n.Title == "Inconsistency: "+host {
+					got++
+				}
+			}
+
+			if got != tt.want {
+				t.Errorf("sent %d inconsistency alerts, want %d", got, tt.want)
+			}
+		})
+	}
+}

internal/watcher/interfaces.go

@@ -5,6 +5,7 @@ import (
 	"context"
 
 	"sneak.berlin/go/dnswatcher/internal/portcheck"
+	"sneak.berlin/go/dnswatcher/internal/resolver"
 	"sneak.berlin/go/dnswatcher/internal/tlscheck"
 )
 
@@ -17,11 +18,11 @@ type DNSResolver interface {
 	) ([]string, error)
 
 	// LookupAllRecords queries all record types for a hostname,
-	// returning results keyed by nameserver then record type.
+	// returning each nameserver's response keyed by nameserver.
 	LookupAllRecords(
 		ctx context.Context,
 		hostname string,
-	) (map[string]map[string][]string, error)
+	) (map[string]*resolver.NameserverResponse, error)
 
 	// ResolveIPAddresses resolves a hostname to all IP addresses.
 	ResolveIPAddresses(

internal/watcher/nsfailure_test.go

@@ -0,0 +1,269 @@
+package watcher_test
+
+import (
+	"context"
+	"fmt"
+	"log/slog"
+	"strings"
+	"testing"
+	"time"
+
+	"sneak.berlin/go/dnswatcher/internal/livednstest"
+	"sneak.berlin/go/dnswatcher/internal/resolver"
+	"sneak.berlin/go/dnswatcher/internal/state"
+	"sneak.berlin/go/dnswatcher/internal/watcher"
+)
+
+// answered is what a check saves for a nameserver that answered with
+// these records.
+func answered(records map[string][]string) *state.NameserverRecordState {
+	return &state.NameserverRecordState{Records: records, Status: "ok"}
+}
+
+// failed is what a check saves for a nameserver that did not answer.
+func failed() *state.NameserverRecordState {
+	return &state.NameserverRecordState{
+		Records: map[string][]string{},
+		Status:  "error",
+		Error:   "all queries timed out",
+	}
+}
+
+// saved builds the hostname state a check saves.
+func saved(
+	byNameserver map[string]*state.NameserverRecordState,
+) *state.HostnameState {
+	return &state.HostnameState{RecordsByNameserver: byNameserver}
+}
+
+// alertCounts counts the hostname alerts sent, by kind.
+type alertCounts struct {
+	failures, recoveries, recordChanges, inconsistencies int
+}
+
+// countAlerts runs the hostname change detection from the state loaded
+// at startup through each check in turn, and counts the alerts sent.
+func countAlerts(
+	t *testing.T,
+	loaded *state.HostnameState,
+	checks []*state.HostnameState,
+) alertCounts {
+	t.Helper()
+
+	// The hostname change detection uses only the notifier.
+	notifier := &mockNotifier{}
+	w := watcher.NewForTest(nil, nil, nil, nil, nil, notifier)
+
+	prev := loaded
+
+	for _, current := range checks {
+		w.DetectHostnameChanges(t.Context(), host, prev, current)
+		prev = current
+	}
+
+	var got alertCounts
+
+	for _, n := range notifier.getNotifications() {
+		kind, _, _ := strings.Cut(n.Title, ":")
+
+		switch kind {
+		case "NS Failure":
+			got.failures++
+		case "NS Recovery":
+			got.recoveries++
+		case "Record Change":
+			got.recordChanges++
+		case "Inconsistency":
+			got.inconsistencies++
+		}
+	}
+
+	return got
+}
+
+func TestNSFailureAndRecoveryAlerts(t *testing.T) {
+	t.Parallel()
+
+	records := map[string][]string{"A": {ip1}}
+
+	bothAnswer := saved(map[string]*state.NameserverRecordState{
+		nsA: answered(records), nsB: answered(records),
+	})
+	bFails := saved(map[string]*state.NameserverRecordState{
+		nsA: answered(records), nsB: failed(),
+	})
+	onlyA := saved(map[string]*state.NameserverRecordState{
+		nsA: answered(records),
+	})
+	bAnswersNoRecords := saved(map[string]*state.NameserverRecordState{
+		nsA: answered(records), nsB: answered(map[string][]string{}),
+	})
+	bAnswersDifferently := saved(map[string]*state.NameserverRecordState{
+		nsA: answered(records), nsB: answered(map[string][]string{"A": {ip2}}),
+	})
+
+	// Each case starts from the state loaded at startup and runs the
+	// checks in order.
+	tests := []struct {
+		name   string
+		loaded *state.HostnameState
+		checks []*state.HostnameState
+		want   alertCounts
+	}{
+		{
+			"failure lasting several checks alerts once",
+			bothAnswer, []*state.HostnameState{bFails, bFails, bFails},
+			alertCounts{failures: 1},
+		},
+		{
+			"recovery alerts once",
+			bFails, []*state.HostnameState{bothAnswer, bothAnswer},
+			alertCounts{recoveries: 1},
+		},
+		{
+			"failing again after recovering alerts again",
+			bothAnswer, []*state.HostnameState{bFails, bothAnswer, bFails},
+			alertCounts{failures: 2, recoveries: 1},
+		},
+		{
+			"nameserver failing when first seen does not alert",
+			onlyA, []*state.HostnameState{bFails, bFails},
+			alertCounts{},
+		},
+		{
+			"answer with no records is a record change, not a failure",
+			bothAnswer, []*state.HostnameState{bAnswersNoRecords},
+			alertCounts{recordChanges: 1, inconsistencies: 1},
+		},
+		{
+			"recovered nameserver that answers differently disagrees",
+			bFails, []*state.HostnameState{bAnswersDifferently},
+			alertCounts{recoveries: 1, inconsistencies: 1},
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			t.Parallel()
+
+			got := countAlerts(t, tt.loaded, tt.checks)
+			if got != tt.want {
+				t.Errorf("sent %+v, want %+v", got, tt.want)
+			}
+		})
+	}
+}
+
+func TestNSFailureAlertNamesHostnameNameserverAndReason(t *testing.T) {
+	t.Parallel()
+
+	records := map[string][]string{"A": {ip1}}
+
+	notifier := &mockNotifier{}
+	w := watcher.NewForTest(nil, nil, nil, nil, nil, notifier)
+
+	w.DetectHostnameChanges(
+		t.Context(), host,
+		saved(map[string]*state.NameserverRecordState{nsA: answered(records)}),
+		saved(map[string]*state.NameserverRecordState{nsA: failed()}),
+	)
+
+	notifications := notifier.getNotifications()
+	if len(notifications) != 1 {
+		t.Fatalf("sent %v, want one NS Failure", notifications)
+	}
+
+	msg := notifications[0].Message
+	if !strings.Contains(msg, host) || !strings.Contains(msg, nsA) ||
+		!strings.Contains(msg, failed().Error) {
+		t.Errorf(
+			"message %q does not name %s, %s and the reason",
+			msg, host, nsA,
+		)
+	}
+}
+
+// TestNameserverThatNeverAnswers asks a nameserver address where
+// nothing answers, 192.0.2.1, and checks what the watcher saves for it.
+// The deadline outlasts the resolver's first two-second try, as in the
+// resolver's timeout test.
+func TestNameserverThatNeverAnswers(t *testing.T) {
+	t.Parallel()
+
+	ctx, cancel := context.WithTimeout(t.Context(), 3*time.Second)
+	t.Cleanup(cancel)
+
+	res := resolver.NewFromLogger(slog.Default())
+
+	resp, err := res.QueryNameserverIP(ctx, nsA, "192.0.2.1", host)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	hs := watcher.BuildHostnameState(
+		map[string]*resolver.NameserverResponse{nsA: resp}, time.Now(),
+	)
+
+	got := hs.RecordsByNameserver[nsA]
+	if got.Status != "error" || got.Error == "" {
+		t.Errorf(
+			"saved status %q, error %q; want status error with a reason",
+			got.Status, got.Error,
+		)
+	}
+}
+
+// TestNameserverThatAnswersNXDOMAIN asks a real nameserver about a name
+// that does not exist and checks what the watcher saves for it: NXDOMAIN
+// is an answer, so the nameserver is saved as ok with no error.
+func TestNameserverThatAnswersNXDOMAIN(t *testing.T) {
+	t.Parallel()
+
+	res := resolver.NewFromLogger(slog.Default())
+	name := "this-surely-does-not-exist-xyz." + testDomain
+
+	var (
+		ns   string
+		resp *resolver.NameserverResponse
+	)
+
+	livednstest.Retry(t, "QueryNameserver("+name+")", func(ctx context.Context) error {
+		nameservers, err := res.LookupNS(ctx, testDomain)
+		if err != nil {
+			return err
+		}
+
+		ns = nameservers[0]
+
+		resp, err = res.QueryNameserver(ctx, ns, name)
+		if err != nil {
+			return err
+		}
+
+		// A timeout or a failure is no answer to check.
+		if resp.Status == resolver.StatusTimeout ||
+			resp.Status == resolver.StatusError {
+			return fmt.Errorf(
+				"%w: %s: %s", livednstest.ErrNoAnswer, ns, resp.Error,
+			)
+		}
+
+		return nil
+	})
+
+	if resp.Status != resolver.StatusNXDomain {
+		t.Fatalf("%s answered %q for %s, want NXDOMAIN", ns, resp.Status, name)
+	}
+
+	hs := watcher.BuildHostnameState(
+		map[string]*resolver.NameserverResponse{ns: resp}, time.Now(),
+	)
+
+	got := hs.RecordsByNameserver[ns]
+	if got.Status != "ok" || got.Error != "" {
+		t.Errorf(
+			"saved status %q, error %q; want status ok with no error",
+			got.Status, got.Error,
+		)
+	}
+}

internal/watcher/watcher.go

@@ -13,6 +13,7 @@ import (
 
 	"sneak.berlin/go/dnswatcher/internal/config"
 	"sneak.berlin/go/dnswatcher/internal/logger"
+	"sneak.berlin/go/dnswatcher/internal/resolver"
 	"sneak.berlin/go/dnswatcher/internal/state"
 	"sneak.berlin/go/dnswatcher/internal/tlscheck"
 )
@@ -249,7 +250,7 @@ func (w *Watcher) checkDomain(
 	// Also look up A/AAAA records for the apex domain so that
 	// port and TLS checks (which read HostnameState) can find
 	// the domain's IP addresses.
-	records, err := w.resolver.LookupAllRecords(ctx, domain)
+	results, err := w.resolver.LookupAllRecords(ctx, domain)
 	if err != nil {
 		w.log.Error(
 			"failed to lookup records for domain",
@@ -260,12 +261,13 @@ func (w *Watcher) checkDomain(
 		return
 	}
 
+	newState := buildHostnameState(results, now)
+
 	prevHS, hasPrevHS := w.state.GetHostnameState(domain)
 	if hasPrevHS && !w.firstRun {
-		w.detectHostnameChanges(ctx, domain, prevHS, records)
+		w.detectHostnameChanges(ctx, domain, prevHS, newState)
 	}
 
-	newState := buildHostnameState(records, now)
 	w.state.SetHostnameState(domain, newState)
 }
 
@@ -314,7 +316,7 @@ func (w *Watcher) checkHostname(
 	ctx context.Context,
 	hostname string,
 ) {
-	records, err := w.resolver.LookupAllRecords(ctx, hostname)
+	results, err := w.resolver.LookupAllRecords(ctx, hostname)
 	if err != nil {
 		w.log.Error(
 			"failed to lookup records",
@@ -325,19 +327,22 @@ func (w *Watcher) checkHostname(
 		return
 	}
 
-	now := time.Now().UTC()
+	newState := buildHostnameState(results, time.Now().UTC())
-	prev, hasPrev := w.state.GetHostnameState(hostname)
 
+	prev, hasPrev := w.state.GetHostnameState(hostname)
 	if hasPrev && !w.firstRun {
-		w.detectHostnameChanges(ctx, hostname, prev, records)
+		w.detectHostnameChanges(ctx, hostname, prev, newState)
 	}
 
-	newState := buildHostnameState(records, now)
 	w.state.SetHostnameState(hostname, newState)
 }
 
+// buildHostnameState saves each nameserver's response. A nameserver
+// that answered, even with NXDOMAIN or no records, is saved as ok; one
+// that timed out or failed is saved as error with the reason, and its
+// empty record set is not an answer.
 func buildHostnameState(
-	records map[string]map[string][]string,
+	results map[string]*resolver.NameserverResponse,
 	now time.Time,
 ) *state.HostnameState {
 	hs := &state.HostnameState{
@@ -347,12 +352,20 @@ func buildHostnameState(
 		LastChecked: now,
 	}
 
-	for ns, recs := range records {
+	for ns, resp := range results {
-		hs.RecordsByNameserver[ns] = &state.NameserverRecordState{
+		nsState := &state.NameserverRecordState{
-			Records:     recs,
+			Records:     resp.Records,
 			Status:      statusOK,
 			LastChecked: now,
 		}
+
+		if resp.Status == resolver.StatusTimeout ||
+			resp.Status == resolver.StatusError {
+			nsState.Status = statusError
+			nsState.Error = resp.Error
+		}
+
+		hs.RecordsByNameserver[ns] = nsState
 	}
 
 	return hs
@@ -361,27 +374,29 @@ func buildHostnameState(
 func (w *Watcher) detectHostnameChanges(
 	ctx context.Context,
 	hostname string,
-	prev *state.HostnameState,
+	prev, current *state.HostnameState,
-	current map[string]map[string][]string,
 ) {
 	w.detectRecordChanges(ctx, hostname, prev, current)
 	w.detectNSDisappearances(ctx, hostname, prev, current)
-	w.detectInconsistencies(ctx, hostname, current)
+	w.detectNSFailures(ctx, hostname, prev, current)
+	w.detectInconsistencies(ctx, hostname, prev, current)
 }
 
+// detectRecordChanges compares each nameserver's records with those of
+// the previous check. Only answers are compared: a nameserver that
+// failed on either check has no records to compare.
 func (w *Watcher) detectRecordChanges(
 	ctx context.Context,
 	hostname string,
-	prev *state.HostnameState,
+	prev, current *state.HostnameState,
-	current map[string]map[string][]string,
 ) {
-	for ns, recs := range current {
+	for ns, cur := range current.RecordsByNameserver {
 		prevNS, ok := prev.RecordsByNameserver[ns]
-		if !ok {
+		if !ok || prevNS.Status != statusOK || cur.Status != statusOK {
 			continue
 		}
 
-		if recordsEqual(prevNS.Records, recs) {
+		if recordsEqual(prevNS.Records, cur.Records) {
 			continue
 		}
 
@@ -389,7 +404,7 @@ func (w *Watcher) detectRecordChanges(
 			"Hostname: %s\nNameserver: %s\n"+
 				"Old: %v\nNew: %v",
 			hostname, ns,
-			prevNS.Records, recs,
+			prevNS.Records, cur.Records,
 		)
 
 		w.notify.SendNotification(
@@ -404,11 +419,10 @@ func (w *Watcher) detectRecordChanges(
 func (w *Watcher) detectNSDisappearances(
 	ctx context.Context,
 	hostname string,
-	prev *state.HostnameState,
+	prev, current *state.HostnameState,
-	current map[string]map[string][]string,
 ) {
 	for ns, prevNS := range prev.RecordsByNameserver {
-		if _, ok := current[ns]; ok || prevNS.Status != statusOK {
+		if _, ok := current.RecordsByNameserver[ns]; ok || prevNS.Status != statusOK {
 			continue
 		}
 
@@ -424,52 +438,64 @@ func (w *Watcher) detectNSDisappearances(
 			"error",
 		)
 	}
+}
 
-	for ns := range current {
+// detectNSFailures notifies when a nameserver that answered on the
+// previous check fails, and when one that failed answers again. A
+// nameserver missing from the previous check is not compared.
+func (w *Watcher) detectNSFailures(
+	ctx context.Context,
+	hostname string,
+	prev, current *state.HostnameState,
+) {
+	for ns, cur := range current.RecordsByNameserver {
 		prevNS, ok := prev.RecordsByNameserver[ns]
-		if !ok || prevNS.Status != statusError {
+		if !ok {
 			continue
 		}
 
-		msg := fmt.Sprintf(
+		switch {
-			"Hostname: %s\nNameserver: %s recovered",
+		case prevNS.Status == statusOK && cur.Status == statusError:
-			hostname, ns,
+			msg := fmt.Sprintf(
-		)
+				"Hostname: %s\nNameserver: %s\nError: %s",
+				hostname, ns, cur.Error,
+			)
 
-		w.notify.SendNotification(
+			w.notify.SendNotification(
-			ctx,
+				ctx,
-			"NS Recovery: "+hostname,
+				"NS Failure: "+hostname,
-			msg,
+				msg,
-			"success",
+				"error",
-		)
+			)
+		case prevNS.Status == statusError && cur.Status == statusOK:
+			msg := fmt.Sprintf(
+				"Hostname: %s\nNameserver: %s recovered",
+				hostname, ns,
+			)
+
+			w.notify.SendNotification(
+				ctx,
+				"NS Recovery: "+hostname,
+				msg,
+				"success",
+			)
+		}
 	}
 }
 
 func (w *Watcher) detectInconsistencies(
 	ctx context.Context,
 	hostname string,
-	current map[string]map[string][]string,
+	prev, current *state.HostnameState,
 ) {
-	nameservers := make([]string, 0, len(current))
+	for _, pair := range newlyDisagreeingPairs(prev, current) {
-	for ns := range current {
+		ns1, ns2 := pair[0], pair[1]
-		nameservers = append(nameservers, ns)
-	}
-
-	sort.Strings(nameservers)
-
-	for i := range len(nameservers) - 1 {
-		ns1 := nameservers[i]
-		ns2 := nameservers[i+1]
-
-		if recordsEqual(current[ns1], current[ns2]) {
-			continue
-		}
 
 		msg := fmt.Sprintf(
 			"Hostname: %s\n%s: %v\n%s: %v",
 			hostname,
-			ns1, current[ns1],
+			ns1, current.RecordsByNameserver[ns1].Records,
-			ns2, current[ns2],
+			ns2, current.RecordsByNameserver[ns2].Records,
 		)
 
 		w.notify.SendNotification(
@@ -481,6 +507,51 @@ func (w *Watcher) detectInconsistencies(
 	}
 }
 
+// newlyDisagreeingPairs returns every pair of nameservers that answered
+// in current and whose records differ there, in sorted order of name,
+// except pairs where both nameservers answered in prev and already
+// differed there. A nameserver missing from prev, or that failed there,
+// is paired with every nameserver it differs from. A nameserver that
+// failed in current has no records to compare and is in no pair.
+func newlyDisagreeingPairs(
+	prev, current *state.HostnameState,
+) [][2]string {
+	nameservers := make([]string, 0, len(current.RecordsByNameserver))
+	for ns, cur := range current.RecordsByNameserver {
+		if cur.Status == statusOK {
+			nameservers = append(nameservers, ns)
+		}
+	}
+
+	sort.Strings(nameservers)
+
+	var pairs [][2]string
+
+	for i, ns1 := range nameservers {
+		for _, ns2 := range nameservers[i+1:] {
+			if recordsEqual(
+				current.RecordsByNameserver[ns1].Records,
+				current.RecordsByNameserver[ns2].Records,
+			) {
+				continue
+			}
+
+			prev1, ok1 := prev.RecordsByNameserver[ns1]
+			prev2, ok2 := prev.RecordsByNameserver[ns2]
+
+			if ok1 && ok2 &&
+				prev1.Status == statusOK && prev2.Status == statusOK &&
+				!recordsEqual(prev1.Records, prev2.Records) {
+				continue
+			}
+
+			pairs = append(pairs, [2]string{ns1, ns2})
+		}
+	}
+
+	return pairs
+}
+
 func (w *Watcher) checkAllPorts(ctx context.Context) {
 	// Phase 1: Build current IP:port → hostname associations
 	// from fresh DNS data.

internal/watcher/watcher_test.go

@@ -2,151 +2,77 @@ package watcher_test
 
 import (
 	"context"
-	"errors"
 	"fmt"
+	"log/slog"
+	"slices"
 	"sync"
 	"testing"
 	"time"
 
 	"sneak.berlin/go/dnswatcher/internal/config"
+	"sneak.berlin/go/dnswatcher/internal/livednstest"
 	"sneak.berlin/go/dnswatcher/internal/portcheck"
+	"sneak.berlin/go/dnswatcher/internal/resolver"
 	"sneak.berlin/go/dnswatcher/internal/state"
 	"sneak.berlin/go/dnswatcher/internal/tlscheck"
 	"sneak.berlin/go/dnswatcher/internal/watcher"
 )
 
-// errNotFound is returned when mock data is missing.
+// The watcher looks these names up in live DNS with the real resolver,
-var errNotFound = errors.New("not found")
+// so tests assert on what the watcher does with the answers, never on
-
+// the records these zones publish. testHost's nameservers and addresses
-// Fixture values shared across tests.
+// stay the same from one check to the next, which the tests that check
+// it twice rely on.
 const (
-	testDomain = "example.com"
+	testDomain = "google.com"
-	testHost   = "www.example.com"
+	testHost   = "cloudflare.com"
-	testNS1    = "ns1.example.com."
-	testNS2    = "ns2.example.com."
-	testIPv4   = "93.184.216.34"
-	testIP     = "1.2.3.4"
 	testIssuer = "DigiCert"
 )
 
-// --- Mock implementations ---
+// Saved-state values that live DNS never returns: nameserver names
+// under .invalid and a documentation address.
+const (
+	oldNS1 = "ns1.example.invalid."
+	oldNS2 = "ns2.example.invalid."
+	oldIP  = "192.0.2.1"
+)
 
-type mockResolver struct {
+// --- Stand-ins for the port checker, TLS checker and notifier ---
-	mu             sync.Mutex
+//
-	nsRecords      map[string][]string
+// DNS has none: the watchers built here use the real resolver (see
-	allRecords     map[string]map[string]map[string][]string
+// TESTING.md).
-	ipAddresses    map[string][]string
-	lookupNSErr    error
-	allRecordsErr  error
-	resolveIPErr   error
-	lookupNSCalls  int
-	allRecordCalls int
-}
-
-func (m *mockResolver) LookupNS(
-	_ context.Context,
-	domain string,
-) ([]string, error) {
-	m.mu.Lock()
-	defer m.mu.Unlock()
-
-	m.lookupNSCalls++
-
-	if m.lookupNSErr != nil {
-		return nil, m.lookupNSErr
-	}
-
-	ns, ok := m.nsRecords[domain]
-	if !ok {
-		return nil, fmt.Errorf(
-			"%w: NS for %s", errNotFound, domain,
-		)
-	}
-
-	return ns, nil
-}
-
-func (m *mockResolver) LookupAllRecords(
-	_ context.Context,
-	hostname string,
-) (map[string]map[string][]string, error) {
-	m.mu.Lock()
-	defer m.mu.Unlock()
-
-	m.allRecordCalls++
-
-	if m.allRecordsErr != nil {
-		return nil, m.allRecordsErr
-	}
-
-	recs, ok := m.allRecords[hostname]
-	if !ok {
-		return nil, fmt.Errorf(
-			"%w: records for %s", errNotFound, hostname,
-		)
-	}
-
-	return recs, nil
-}
-
-func (m *mockResolver) ResolveIPAddresses(
-	_ context.Context,
-	hostname string,
-) ([]string, error) {
-	m.mu.Lock()
-	defer m.mu.Unlock()
-
-	if m.resolveIPErr != nil {
-		return nil, m.resolveIPErr
-	}
-
-	ips, ok := m.ipAddresses[hostname]
-	if !ok {
-		return nil, fmt.Errorf(
-			"%w: IPs for %s", errNotFound, hostname,
-		)
-	}
-
-	return ips, nil
-}
 
+// mockPortChecker reports every port open until closed is set.
 type mockPortChecker struct {
-	mu      sync.Mutex
+	mu     sync.Mutex
-	results map[string]bool
+	closed bool
-	err     error
+	calls  int
-	calls   int
 }
 
 func (m *mockPortChecker) CheckPort(
 	_ context.Context,
-	address string,
+	_ string,
-	port int,
+	_ int,
 ) (*portcheck.PortResult, error) {
 	m.mu.Lock()
 	defer m.mu.Unlock()
 
 	m.calls++
 
-	if m.err != nil {
+	return &portcheck.PortResult{Open: !m.closed}, nil
-		return nil, m.err
-	}
-
-	key := fmt.Sprintf("%s:%d", address, port)
-	open := m.results[key]
-
-	return &portcheck.PortResult{Open: open}, nil
 }
 
+// mockTLSChecker returns a certificate for the requested hostname that
+// expires at notAfter.
 type mockTLSChecker struct {
-	mu    sync.Mutex
+	mu       sync.Mutex
-	certs map[string]*tlscheck.CertificateInfo
+	notAfter time.Time
-	err   error
+	calls    int
-	calls int
 }
 
 func (m *mockTLSChecker) CheckCertificate(
 	_ context.Context,
-	ip string,
+	_ string,
 	hostname string,
 ) (*tlscheck.CertificateInfo, error) {
 	m.mu.Lock()
@@ -154,20 +80,12 @@ func (m *mockTLSChecker) CheckCertificate(
 
 	m.calls++
 
-	if m.err != nil {
+	return &tlscheck.CertificateInfo{
-		return nil, m.err
+		CommonName:              hostname,
-	}
+		Issuer:                  testIssuer,
-
+		NotAfter:                m.notAfter,
-	key := fmt.Sprintf("%s:%s", ip, hostname)
+		SubjectAlternativeNames: []string{hostname},
-	cert, ok := m.certs[key]
+	}, nil
-
-	if !ok {
-		return nil, fmt.Errorf(
-			"%w: cert for %s", errNotFound, key,
-		)
-	}
-
-	return cert, nil
 }
 
 type notification struct {
@@ -205,10 +123,9 @@ func (m *mockNotifier) getNotifications() []notification {
 	return result
 }
 
-// --- Helper to build a Watcher for testing ---
+// --- Helpers to build a Watcher and run its checks against live DNS ---
 
 type testDeps struct {
-	resolver    *mockResolver
 	portChecker *mockPortChecker
 	tlsChecker  *mockTLSChecker
 	notifier    *mockNotifier
@@ -223,16 +140,9 @@ func newTestWatcher(
 	t.Helper()
 
 	deps := &testDeps{
-		resolver: &mockResolver{
+		portChecker: &mockPortChecker{},
-			nsRecords:   make(map[string][]string),
-			allRecords:  make(map[string]map[string]map[string][]string),
-			ipAddresses: make(map[string][]string),
-		},
-		portChecker: &mockPortChecker{
-			results: make(map[string]bool),
-		},
 		tlsChecker: &mockTLSChecker{
-			certs: make(map[string]*tlscheck.CertificateInfo),
+			notAfter: time.Now().Add(90 * 24 * time.Hour),
 		},
 		notifier: &mockNotifier{},
 		config:   cfg,
@@ -243,7 +153,7 @@ func newTestWatcher(
 	w := watcher.NewForTest(
 		deps.config,
 		deps.state,
-		deps.resolver,
+		resolver.NewFromLogger(slog.Default()),
 		deps.portChecker,
 		deps.tlsChecker,
 		deps.notifier,
@@ -263,6 +173,122 @@ func defaultTestConfig(t *testing.T) *config.Config {
 	}
 }
 
+// checkOnce runs the watcher's checks once and returns an error when a
+// configured name has no hostname state saved by this check, or that
+// state holds no address. Either live DNS gave no answer for the name,
+// or the watcher saved no fresh result for it.
+func checkOnce(
+	ctx context.Context,
+	w *watcher.Watcher,
+	deps *testDeps,
+) error {
+	started := time.Now()
+
+	w.RunOnce(ctx)
+
+	names := slices.Concat(deps.config.Domains, deps.config.Hostnames)
+
+	for _, name := range names {
+		hs, ok := deps.state.GetHostnameState(name)
+		if !ok || hs.LastChecked.Before(started) ||
+			len(addresses(hs)) == 0 {
+			return fmt.Errorf(
+				"%s: %w, or the watcher saved no fresh "+
+					"result for it",
+				name, livednstest.ErrNoAnswer,
+			)
+		}
+	}
+
+	return nil
+}
+
+// runChecks builds a watcher, lets prepare set up the saved state and
+// stand-ins it starts from, and runs its checks once against live DNS.
+// If change is not nil, change then alters the saved state or stand-ins
+// and the checks run a second time. When either check finds no fresh
+// address for a name (see checkOnce), the watcher is thrown away and
+// all of this runs again on a new one, so a failed attempt leaves
+// nothing behind in the saved state, the stand-ins or the notifications.
+func runChecks(
+	t *testing.T,
+	cfg *config.Config,
+	prepare, change func(deps *testDeps),
+) *testDeps {
+	t.Helper()
+
+	var deps *testDeps
+
+	livednstest.Retry(t, "watcher checks", func(ctx context.Context) error {
+		var w *watcher.Watcher
+
+		w, deps = newTestWatcher(t, cfg)
+
+		if prepare != nil {
+			prepare(deps)
+		}
+
+		err := checkOnce(ctx, w, deps)
+		if err != nil || change == nil {
+			return err
+		}
+
+		change(deps)
+
+		return checkOnce(ctx, w, deps)
+	})
+
+	return deps
+}
+
+// addresses returns the A and AAAA values saved for a hostname.
+func addresses(hs *state.HostnameState) []string {
+	var ips []string
+
+	for _, nsState := range hs.RecordsByNameserver {
+		ips = append(ips, nsState.Records["A"]...)
+		ips = append(ips, nsState.Records["AAAA"]...)
+	}
+
+	return ips
+}
+
+// assertNotified checks that a notification with this title and
+// priority was sent.
+func assertNotified(
+	t *testing.T,
+	deps *testDeps,
+	title, priority string,
+) {
+	t.Helper()
+
+	notifications := deps.notifier.getNotifications()
+
+	for _, n := range notifications {
+		if n.Title == title && n.Priority == priority {
+			return
+		}
+	}
+
+	t.Errorf(
+		"expected %s notification %q, got: %v",
+		priority, title, notifications,
+	)
+}
+
+// countNotifications counts the notifications sent with this title.
+func countNotifications(deps *testDeps, title string) int {
+	count := 0
+
+	for _, n := range deps.notifier.getNotifications() {
+		if n.Title == title {
+			count++
+		}
+	}
+
+	return count
+}
+
 func TestFirstRunBaseline(t *testing.T) {
 	t.Parallel()
 
@@ -270,51 +296,12 @@ func TestFirstRunBaseline(t *testing.T) {
 	cfg.Domains = []string{testDomain}
 	cfg.Hostnames = []string{testHost}
 
-	w, deps := newTestWatcher(t, cfg)
+	deps := runChecks(t, cfg, nil, nil)
-	setupBaselineMocks(deps)
-
-	w.RunOnce(t.Context())
 
 	assertNoNotifications(t, deps)
 	assertStatePopulated(t, deps)
 }
 
-func setupBaselineMocks(deps *testDeps) {
-	deps.resolver.nsRecords[testDomain] = []string{
-		testNS1,
-		testNS2,
-	}
-	deps.resolver.allRecords[testDomain] = map[string]map[string][]string{
-		testNS1: {"A": {testIPv4}},
-		testNS2: {"A": {testIPv4}},
-	}
-	deps.resolver.allRecords[testHost] = map[string]map[string][]string{
-		testNS1: {"A": {testIPv4}},
-		testNS2: {"A": {testIPv4}},
-	}
-	deps.resolver.ipAddresses[testHost] = []string{
-		testIPv4,
-	}
-	deps.portChecker.results["93.184.216.34:80"] = true
-	deps.portChecker.results["93.184.216.34:443"] = true
-	deps.tlsChecker.certs["93.184.216.34:www.example.com"] = &tlscheck.CertificateInfo{
-		CommonName: testHost,
-		Issuer:     testIssuer,
-		NotAfter:   time.Now().Add(90 * 24 * time.Hour),
-		SubjectAlternativeNames: []string{
-			testHost,
-		},
-	}
-	deps.tlsChecker.certs["93.184.216.34:example.com"] = &tlscheck.CertificateInfo{
-		CommonName: testDomain,
-		Issuer:     testIssuer,
-		NotAfter:   time.Now().Add(90 * 24 * time.Hour),
-		SubjectAlternativeNames: []string{
-			testDomain,
-		},
-	}
-}
-
 func assertNoNotifications(
 	t *testing.T,
 	deps *testDeps,
@@ -361,26 +348,7 @@ func TestDomainPortAndTLSChecks(t *testing.T) {
 	cfg := defaultTestConfig(t)
 	cfg.Domains = []string{testDomain}
 
-	w, deps := newTestWatcher(t, cfg)
+	deps := runChecks(t, cfg, nil, nil)
-
-	deps.resolver.nsRecords[testDomain] = []string{
-		testNS1,
-	}
-	deps.resolver.allRecords[testDomain] = map[string]map[string][]string{
-		testNS1: {"A": {testIPv4}},
-	}
-	deps.portChecker.results["93.184.216.34:80"] = true
-	deps.portChecker.results["93.184.216.34:443"] = true
-	deps.tlsChecker.certs["93.184.216.34:example.com"] = &tlscheck.CertificateInfo{
-		CommonName: testDomain,
-		Issuer:     testIssuer,
-		NotAfter:   time.Now().Add(90 * 24 * time.Hour),
-		SubjectAlternativeNames: []string{
-			testDomain,
-		},
-	}
-
-	w.RunOnce(t.Context())
 
 	snap := deps.state.GetSnapshot()
 
@@ -419,50 +387,18 @@ func TestNSChangeDetection(t *testing.T) {
 	cfg := defaultTestConfig(t)
 	cfg.Domains = []string{testDomain}
 
-	w, deps := newTestWatcher(t, cfg)
+	// The saved state lists nameservers that live DNS does not.
+	deps := runChecks(t, cfg, func(deps *testDeps) {
+		deps.state.SetDomainState(testDomain, &state.DomainState{
+			Nameservers: []string{oldNS1, oldNS2},
+		})
+	}, nil)
 
-	deps.resolver.nsRecords[testDomain] = []string{
+	assertNotified(t, deps, "NS Change: "+testDomain, "warning")
-		testNS1,
-		testNS2,
-	}
-	deps.resolver.allRecords[testDomain] = map[string]map[string][]string{
-		testNS1: {"A": {testIP}},
-		testNS2: {"A": {testIP}},
-	}
-	deps.portChecker.results["1.2.3.4:80"] = false
-	deps.portChecker.results["1.2.3.4:443"] = false
 
-	ctx := t.Context()
+	ds, _ := deps.state.GetDomainState(testDomain)
-	w.RunOnce(ctx)
+	if slices.Contains(ds.Nameservers, oldNS1) {
-
+		t.Errorf("saved nameservers not updated: %v", ds.Nameservers)
-	deps.resolver.mu.Lock()
-	deps.resolver.nsRecords[testDomain] = []string{
-		testNS1,
-		"ns3.example.com.",
-	}
-	deps.resolver.allRecords[testDomain] = map[string]map[string][]string{
-		testNS1:            {"A": {testIP}},
-		"ns3.example.com.": {"A": {testIP}},
-	}
-	deps.resolver.mu.Unlock()
-
-	w.RunOnce(ctx)
-
-	notifications := deps.notifier.getNotifications()
-	if len(notifications) == 0 {
-		t.Error("expected notification for NS change")
-	}
-
-	found := false
-
-	for _, n := range notifications {
-		if n.Priority == "warning" {
-			found = true
-		}
-	}
-
-	if !found {
-		t.Error("expected warning-priority NS change notification")
 	}
 }
 
@@ -472,40 +408,18 @@ func TestRecordChangeDetection(t *testing.T) {
 	cfg := defaultTestConfig(t)
 	cfg.Hostnames = []string{testHost}
 
-	w, deps := newTestWatcher(t, cfg)
+	// Between the checks, save for every nameserver an address live DNS
+	// never returns.
+	deps := runChecks(t, cfg, nil, func(deps *testDeps) {
+		hs, _ := deps.state.GetHostnameState(testHost)
+		for _, nsState := range hs.RecordsByNameserver {
+			nsState.Records = map[string][]string{"A": {oldIP}}
+		}
 
-	deps.resolver.allRecords[testHost] = map[string]map[string][]string{
+		deps.state.SetHostnameState(testHost, hs)
-		testNS1: {"A": {testIPv4}},
+	})
-	}
-	deps.resolver.ipAddresses[testHost] = []string{
-		testIPv4,
-	}
-	deps.portChecker.results["93.184.216.34:80"] = false
-	deps.portChecker.results["93.184.216.34:443"] = false
 
-	ctx := t.Context()
+	assertNotified(t, deps, "Record Change: "+testHost, "warning")
-	w.RunOnce(ctx)
-
-	deps.resolver.mu.Lock()
-	deps.resolver.allRecords[testHost] = map[string]map[string][]string{
-		testNS1: {"A": {"93.184.216.35"}},
-	}
-	deps.resolver.ipAddresses[testHost] = []string{
-		"93.184.216.35",
-	}
-	deps.resolver.mu.Unlock()
-
-	deps.portChecker.mu.Lock()
-	deps.portChecker.results["93.184.216.35:80"] = false
-	deps.portChecker.results["93.184.216.35:443"] = false
-	deps.portChecker.mu.Unlock()
-
-	w.RunOnce(ctx)
-
-	notifications := deps.notifier.getNotifications()
-	if len(notifications) == 0 {
-		t.Error("expected notification for record change")
-	}
 }
 
 func TestPortStateChange(t *testing.T) {
@@ -514,38 +428,23 @@ func TestPortStateChange(t *testing.T) {
 	cfg := defaultTestConfig(t)
 	cfg.Hostnames = []string{testHost}
 
-	w, deps := newTestWatcher(t, cfg)
+	// Between the checks, every port closes.
+	deps := runChecks(t, cfg, nil, func(deps *testDeps) {
+		deps.portChecker.mu.Lock()
+		deps.portChecker.closed = true
+		deps.portChecker.mu.Unlock()
+	})
 
-	deps.resolver.allRecords[testHost] = map[string]map[string][]string{
+	hs, _ := deps.state.GetHostnameState(testHost)
-		testNS1: {"A": {testIP}},
+	assertNotified(
-	}
+		t, deps, "Port Change: "+addresses(hs)[0]+":443", "warning",
-	deps.resolver.ipAddresses[testHost] = []string{
+	)
-		testIP,
+}
-	}
-	deps.portChecker.results["1.2.3.4:80"] = true
-	deps.portChecker.results["1.2.3.4:443"] = true
-	deps.tlsChecker.certs["1.2.3.4:www.example.com"] = &tlscheck.CertificateInfo{
-		CommonName: testHost,
-		Issuer:     testIssuer,
-		NotAfter:   time.Now().Add(90 * 24 * time.Hour),
-		SubjectAlternativeNames: []string{
-			testHost,
-		},
-	}
 
-	ctx := t.Context()
+// expiresInThreeDays makes the TLS checker return certificates that
-	w.RunOnce(ctx)
+// expire within the seven-day warning period.
-
+func expiresInThreeDays(deps *testDeps) {
-	deps.portChecker.mu.Lock()
+	deps.tlsChecker.notAfter = time.Now().Add(3 * 24 * time.Hour)
-	deps.portChecker.results["1.2.3.4:443"] = false
-	deps.portChecker.mu.Unlock()
-
-	w.RunOnce(ctx)
-
-	notifications := deps.notifier.getNotifications()
-	if len(notifications) == 0 {
-		t.Error("expected notification for port state change")
-	}
 }
 
 func TestTLSExpiryWarning(t *testing.T) {
@@ -554,49 +453,9 @@ func TestTLSExpiryWarning(t *testing.T) {
 	cfg := defaultTestConfig(t)
 	cfg.Hostnames = []string{testHost}
 
-	w, deps := newTestWatcher(t, cfg)
+	deps := runChecks(t, cfg, expiresInThreeDays, nil)
 
-	deps.resolver.allRecords[testHost] = map[string]map[string][]string{
+	assertNotified(t, deps, "TLS Expiry Warning: "+testHost, "warning")
-		testNS1: {"A": {testIP}},
-	}
-	deps.resolver.ipAddresses[testHost] = []string{
-		testIP,
-	}
-	deps.portChecker.results["1.2.3.4:80"] = true
-	deps.portChecker.results["1.2.3.4:443"] = true
-	deps.tlsChecker.certs["1.2.3.4:www.example.com"] = &tlscheck.CertificateInfo{
-		CommonName: testHost,
-		Issuer:     testIssuer,
-		NotAfter:   time.Now().Add(3 * 24 * time.Hour),
-		SubjectAlternativeNames: []string{
-			testHost,
-		},
-	}
-
-	ctx := t.Context()
-
-	// First run = baseline
-	w.RunOnce(ctx)
-
-	// Second run should warn about expiry
-	w.RunOnce(ctx)
-
-	notifications := deps.notifier.getNotifications()
-
-	found := false
-
-	for _, n := range notifications {
-		if n.Priority == "warning" {
-			found = true
-		}
-	}
-
-	if !found {
-		t.Errorf(
-			"expected expiry warning, got: %v",
-			notifications,
-		)
-	}
 }
 
 func TestTLSExpiryWarningDedup(t *testing.T) {
@@ -606,50 +465,25 @@ func TestTLSExpiryWarningDedup(t *testing.T) {
 	cfg.Hostnames = []string{testHost}
 	cfg.TLSInterval = 24 * time.Hour
 
-	w, deps := newTestWatcher(t, cfg)
+	title := "TLS Expiry Warning: " + testHost
 
-	deps.resolver.allRecords[testHost] = map[string]map[string][]string{
+	// The second check comes within the TLS interval of the first,
-		testNS1: {"A": {testIP}},
+	// so it must not warn again.
-	}
+	var warnings int
-	deps.resolver.ipAddresses[testHost] = []string{
+
-		testIP,
+	deps := runChecks(t, cfg, expiresInThreeDays, func(deps *testDeps) {
-	}
+		warnings = countNotifications(deps, title)
-	deps.portChecker.results["1.2.3.4:80"] = true
+	})
-	deps.portChecker.results["1.2.3.4:443"] = true
+
-	deps.tlsChecker.certs["1.2.3.4:www.example.com"] = &tlscheck.CertificateInfo{
+	if warnings == 0 {
-		CommonName: testHost,
+		t.Fatal("expected expiry warnings from the first check")
-		Issuer:     testIssuer,
-		NotAfter:   time.Now().Add(3 * 24 * time.Hour),
-		SubjectAlternativeNames: []string{
-			testHost,
-		},
 	}
 
-	ctx := t.Context()
+	got := countNotifications(deps, title)
-
+	if got != warnings {
-	// First run = baseline, no notifications
-	w.RunOnce(ctx)
-
-	// Second run should fire one expiry warning
-	w.RunOnce(ctx)
-
-	// Third run should NOT fire another warning (dedup)
-	w.RunOnce(ctx)
-
-	notifications := deps.notifier.getNotifications()
-
-	expiryCount := 0
-
-	for _, n := range notifications {
-		if n.Title == "TLS Expiry Warning: www.example.com" {
-			expiryCount++
-		}
-	}
-
-	if expiryCount != 1 {
 		t.Errorf(
-			"expected exactly 1 expiry warning (dedup), got %d",
+			"expected %d expiry warnings (dedup), got %d",
-			expiryCount,
+			warnings, got,
 		)
 	}
 }
@@ -657,21 +491,12 @@ func TestTLSExpiryWarningDedup(t *testing.T) {
 func TestGracefulShutdown(t *testing.T) {
 	t.Parallel()
 
+	// No domains or hostnames: stopping does not involve DNS.
 	cfg := defaultTestConfig(t)
-	cfg.Domains = []string{testDomain}
 	cfg.DNSInterval = 100 * time.Millisecond
 	cfg.TLSInterval = 100 * time.Millisecond
 
-	w, deps := newTestWatcher(t, cfg)
+	w, _ := newTestWatcher(t, cfg)
-
-	deps.resolver.nsRecords[testDomain] = []string{
-		testNS1,
-	}
-	deps.resolver.allRecords[testDomain] = map[string]map[string][]string{
-		testNS1: {"A": {testIP}},
-	}
-	deps.portChecker.results["1.2.3.4:80"] = false
-	deps.portChecker.results["1.2.3.4:443"] = false
 
 	ctx, cancel := context.WithCancel(t.Context())
 
@@ -693,90 +518,52 @@ func TestGracefulShutdown(t *testing.T) {
 	}
 }
 
-func setupHostnameIP(
-	deps *testDeps,
-	hostname, ip string,
-) {
-	deps.resolver.allRecords[hostname] = map[string]map[string][]string{
-		testNS1: {"A": {ip}},
-	}
-	deps.portChecker.results[ip+":80"] = true
-	deps.portChecker.results[ip+":443"] = true
-	deps.tlsChecker.certs[ip+":"+hostname] = &tlscheck.CertificateInfo{
-		CommonName:              hostname,
-		Issuer:                  testIssuer,
-		NotAfter:                time.Now().Add(90 * 24 * time.Hour),
-		SubjectAlternativeNames: []string{hostname},
-	}
-}
-
-func updateHostnameIP(deps *testDeps, hostname, ip string) {
-	deps.resolver.mu.Lock()
-	deps.resolver.allRecords[hostname] = map[string]map[string][]string{
-		testNS1: {"A": {ip}},
-	}
-	deps.resolver.mu.Unlock()
-
-	deps.portChecker.mu.Lock()
-	deps.portChecker.results[ip+":80"] = true
-	deps.portChecker.results[ip+":443"] = true
-	deps.portChecker.mu.Unlock()
-
-	deps.tlsChecker.mu.Lock()
-	deps.tlsChecker.certs[ip+":"+hostname] = &tlscheck.CertificateInfo{
-		CommonName:              hostname,
-		Issuer:                  testIssuer,
-		NotAfter:                time.Now().Add(90 * 24 * time.Hour),
-		SubjectAlternativeNames: []string{hostname},
-	}
-	deps.tlsChecker.mu.Unlock()
-}
-
 func TestDNSRunsBeforePortAndTLSChecks(t *testing.T) {
 	t.Parallel()
 
 	cfg := defaultTestConfig(t)
 	cfg.Hostnames = []string{testHost}
 
-	w, deps := newTestWatcher(t, cfg)
+	// The saved state says the last check found testHost at oldIP.
-
+	deps := runChecks(t, cfg, func(deps *testDeps) {
-	setupHostnameIP(deps, testHost, "10.0.0.1")
+		deps.state.SetHostnameState(testHost, &state.HostnameState{
-
+			RecordsByNameserver: map[string]*state.NameserverRecordState{
-	ctx := t.Context()
+				oldNS1: {
-	w.RunOnce(ctx)
+					Records: map[string][]string{"A": {oldIP}},
+					Status:  "ok",
+				},
+			},
+		})
+	}, nil)
 
 	snap := deps.state.GetSnapshot()
-	if _, ok := snap.Ports["10.0.0.1:80"]; !ok {
+
-		t.Fatal("expected port state for 10.0.0.1:80")
+	if _, ok := snap.Ports[oldIP+":80"]; ok {
+		t.Error("port check used stale DNS: found " + oldIP + ":80")
 	}
 
-	// DNS changes to a new IP; port and TLS must pick it up.
+	// Port and TLS checks must use the addresses this check found.
-	updateHostnameIP(deps, testHost, "10.0.0.2")
+	for _, ip := range addresses(snap.Hostnames[testHost]) {
+		if _, ok := snap.Ports[ip+":80"]; !ok {
+			t.Error("port check used stale DNS: missing " + ip + ":80")
+		}
 
-	w.RunOnce(ctx)
+		certKey := ip + ":443:" + testHost
-
+		if _, ok := snap.Certificates[certKey]; !ok {
-	snap = deps.state.GetSnapshot()
+			t.Error("TLS check used stale DNS: missing " + certKey)
-
+		}
-	if _, ok := snap.Ports["10.0.0.2:80"]; !ok {
-		t.Error("port check used stale DNS: missing 10.0.0.2:80")
-	}
-
-	certKey := "10.0.0.2:443:www.example.com"
-	if _, ok := snap.Certificates[certKey]; !ok {
-		t.Error("TLS check used stale DNS: missing " + certKey)
 	}
 }
 
 func TestSendTestNotification_Enabled(t *testing.T) {
 	t.Parallel()
 
+	// No domains or hostnames: the startup notification does not
+	// involve DNS.
 	cfg := defaultTestConfig(t)
-	cfg.Domains = []string{testDomain}
-	cfg.Hostnames = []string{testHost}
 	cfg.SendTestNotification = true
 
 	w, deps := newTestWatcher(t, cfg)
-	setupBaselineMocks(deps)
 
 	w.RunOnce(t.Context())
 
@@ -797,14 +584,11 @@ func TestSendTestNotification_ViaRun(t *testing.T) {
 	t.Parallel()
 
 	cfg := defaultTestConfig(t)
-	cfg.Domains = []string{testDomain}
-	cfg.Hostnames = []string{testHost}
 	cfg.SendTestNotification = true
 	cfg.DNSInterval = 24 * time.Hour
 	cfg.TLSInterval = 24 * time.Hour
 
 	w, deps := newTestWatcher(t, cfg)
-	setupBaselineMocks(deps)
 
 	ctx, cancel := context.WithCancel(t.Context())
 
@@ -844,14 +628,11 @@ func TestSendTestNotification_Disabled(t *testing.T) {
 	t.Parallel()
 
 	cfg := defaultTestConfig(t)
-	cfg.Domains = []string{testDomain}
-	cfg.Hostnames = []string{testHost}
 	cfg.SendTestNotification = false
 	cfg.DNSInterval = 24 * time.Hour
 	cfg.TLSInterval = 24 * time.Hour
 
 	w, deps := newTestWatcher(t, cfg)
-	setupBaselineMocks(deps)
 
 	ctx, cancel := context.WithCancel(t.Context())
 
@@ -884,32 +665,29 @@ func TestNSFailureAndRecovery(t *testing.T) {
 	cfg := defaultTestConfig(t)
 	cfg.Hostnames = []string{testHost}
 
-	w, deps := newTestWatcher(t, cfg)
+	// Between the checks, save every nameserver the first check found
+	// as one that did not answer, and add, as answering, one that live
+	// DNS does not list, which then disappears.
+	deps := runChecks(t, cfg, nil, func(deps *testDeps) {
+		hs, _ := deps.state.GetHostnameState(testHost)
+		for ns := range hs.RecordsByNameserver {
+			hs.RecordsByNameserver[ns] = failed()
+		}
 
-	deps.resolver.allRecords[testHost] = map[string]map[string][]string{
+		hs.RecordsByNameserver[oldNS1] = &state.NameserverRecordState{
-		testNS1: {"A": {testIP}},
+			Records: map[string][]string{"A": {oldIP}},
-		testNS2: {"A": {testIP}},
+			Status:  "ok",
-	}
+		}
-	deps.resolver.ipAddresses[testHost] = []string{
-		testIP,
-	}
-	deps.portChecker.results["1.2.3.4:80"] = false
-	deps.portChecker.results["1.2.3.4:443"] = false
 
-	ctx := t.Context()
+		deps.state.SetHostnameState(testHost, hs)
+	})
 
-	w.RunOnce(ctx)
+	assertNotified(t, deps, "NS Failure: "+testHost, "error")
+	assertNotified(t, deps, "NS Recovery: "+testHost, "success")
 
-	deps.resolver.mu.Lock()
+	// A nameserver that did not answer has no records to compare, so
-	deps.resolver.allRecords[testHost] = map[string]map[string][]string{
+	// its recovery is not also a record change.
-		testNS1: {"A": {testIP}},
+	if n := countNotifications(deps, "Record Change: "+testHost); n != 0 {
-	}
+		t.Errorf("sent %d record changes on recovery, want 0", n)
-	deps.resolver.mu.Unlock()
-
-	w.RunOnce(ctx)
-
-	notifications := deps.notifier.getNotifications()
-	if len(notifications) == 0 {
-		t.Error("expected notification for NS disappearance")
 	}
 }

script/cibuild

@@ -1,14 +1,18 @@
 #!/bin/sh
 # script/cibuild: run the CI build. The Dockerfile's lint stage runs
 # make fmt-check and golangci-lint; its builder stage runs make test
-# and make build. A successful build implies all of those passed.
+# and make build.
+#
+# --no-cache-filter=lint,builder runs both stages on every invocation;
+# otherwise an unchanged tree is served from the layer cache and passes
+# without linting or querying live DNS.
 set -eu
 
 ROOT="$(cd "$(dirname "$0")/.." && pwd -P)"
 
 main() {
     cd "$ROOT"
-    docker build .
+    docker build --no-cache-filter=lint,builder .
 }
 
 main "$@"

script/docker

@@ -1,6 +1,10 @@
 #!/bin/sh
 # script/docker: build the Docker image tagged with the project name.
-# Identical in all repos; the tag comes from script/projectname.
+# The tag comes from script/projectname.
+#
+# --no-cache-filter=lint,builder runs the lint stage and the builder
+# stage (make test) on every invocation; otherwise an unchanged tree is
+# served from the layer cache without linting or querying live DNS.
 set -eu
 
 SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd -P)"
@@ -8,7 +12,7 @@ ROOT="$(cd "$SCRIPT_DIR/.." && pwd -P)"
 
 main() {
     cd "$ROOT"
-    docker build -t "$("$SCRIPT_DIR/projectname")" .
+    docker build --no-cache-filter=lint,builder -t "$("$SCRIPT_DIR/projectname")" .
 }
 
 main "$@"