metrics: rate limit /metrics per client address before Basic Auth (closes #101)

/metrics is behind a password, and REPO_POLICIES.md requires rate
limiting on password logins. Each client address may now send it 30
requests a minute, counted by httprate before Basic Auth, so failed
logins use up the allowance and a request over it gets 429 without
the password being checked. The address is the one the existing
trusted-proxy logic in internal/middleware works out, with IPv6
addresses grouped by /64. A Prometheus server scraping every 15
seconds sends 4 requests a minute.

Model: opus-5-5

.golangci.yml

@@ -10,14 +10,20 @@ run:
 
 linters:
   default: all
+  enable:
+    # Successor to the deprecated gomodguard. Named explicitly, rather than
+    # left to `default: all`, because it carries the module policy below.
+    - gomodguard_v2
   disable:
     # Genuinely incompatible with project patterns
     - exhaustruct      # Requires all struct fields
-    - depguard         # Dependency allow/block lists
     - godot            # Requires comments to end with periods
-    - wsl              # Deprecated, replaced by wsl_v5
     - wrapcheck        # Too verbose for internal packages
     - varnamelen       # Short names like db, id are idiomatic Go
+    # Deprecated: the warning is attached to the old name, so it is
+    # silenced by disabling that name, not by enabling the successor.
+    - wsl              # Deprecated, replaced by wsl_v5
+    - gomodguard       # Deprecated, replaced by gomodguard_v2
   settings:
     lll:
       line-length: 88
@@ -28,6 +34,68 @@ linters:
       max-complexity: 15
     dupl:
       threshold: 100
+    depguard:
+      # Test-support code must not be compiled into the shipped binary. A
+      # test-support package exists to hand a test privileges the program
+      # itself must never have, so a file that is not a test must not import
+      # one. Test files, and the files inside a package whose directory name
+      # ends in `test`, are where that code belongs, and are exempt.
+      #
+      # The deny list below is the one part of this file a repository is
+      # expected to extend, and the only part it may. depguard matches an
+      # import path against a list of prefixes, so it cannot be told "any path
+      # whose last segment ends in test"; a repository's own test-support
+      # packages have to be named here one at a time, by full import path,
+      # under a module path that differs from repository to repository. Add
+      # them; change nothing else.
+      rules:
+        test-support:
+          list-mode: lax
+          files:
+            - "$all"
+            - "!$test"
+            - "!**/*test/**"
+          deny:
+            - pkg: net/http/httptest
+              desc: >-
+                Test-support code belongs in test files and in packages whose
+                directory name ends in test, not in the shipped binary.
+            - pkg: sneak.berlin/go/dnswatcher/internal/livednstest
+              desc: >-
+                Live-DNS test support belongs in test files and in packages
+                whose directory name ends in test, not in the shipped binary.
+    # Only decisions already recorded in the Go package defaults are
+    # listed here. Every entry matches the module path exactly.
+    gomodguard_v2:
+      blocked:
+        - module: github.com/rs/zerolog
+          recommendations:
+            - log/slog
+          reason: "Structured logging is stdlib log/slog."
+        # One entry per pre-fork module path, because the later releases
+        # are separate paths. A prefix match would be shorter but would
+        # also reach github.com/go-redis/redismock, the test double for
+        # the successor these entries recommend.
+        - module: github.com/go-redis/redis
+          recommendations:
+            - github.com/redis/go-redis/v9
+          reason: "Pre-fork module; use the maintained go-redis v9."
+        - module: github.com/go-redis/redis/v7
+          recommendations:
+            - github.com/redis/go-redis/v9
+          reason: "Pre-fork module; use the maintained go-redis v9."
+        - module: github.com/go-redis/redis/v8
+          recommendations:
+            - github.com/redis/go-redis/v9
+          reason: "Pre-fork module; use the maintained go-redis v9."
+        - module: github.com/sergi/go-diff
+          recommendations:
+            - github.com/aymanbagabas/go-udiff
+          reason: "No unified diff output; use go-udiff."
+        - module: github.com/hexops/gotextdiff
+          recommendations:
+            - github.com/aymanbagabas/go-udiff
+          reason: "Unmaintained fork; use go-udiff."
 
 issues:
   max-issues-per-linter: 0

Dockerfile

@@ -2,6 +2,7 @@
 # The linter is invoked directly rather than through `make lint`: that
 # target shells out to `docker build -f Dockerfile.lint`, and there is
 # no docker daemon inside a docker build.
+# script/cibuild and script/docker name this stage in --no-cache-filter.
 # golangci/golangci-lint:v2.12.2 (Debian-based), 2026-08-10
 FROM golangci/golangci-lint:v2.12.2@sha256:5cceeef04e53efe1470638d4b4b4f5ceefd574955ab3941b2d9a68a8c9ad5240 AS lint
 
@@ -15,6 +16,7 @@ RUN make fmt-check
 RUN golangci-lint run --config .golangci.yml ./...
 
 # Build stage
+# script/cibuild and script/docker name this stage in --no-cache-filter.
 # golang 1.25-alpine, 2026-02-28
 FROM golang@sha256:f6751d823c26342f9506c03797d2527668d095b0a15f1862cddb4d927a7a4ced AS builder
 
@@ -39,17 +41,33 @@ RUN make build
 # alpine 3.21, 2026-02-28
 FROM alpine@sha256:c3f8e73fdb79deaebaa2037150150191b9dcbfba68b4a46d70103204c53f4709
 
-RUN apk add --no-cache ca-certificates tzdata
+RUN apk add --no-cache ca-certificates tzdata su-exec
 
-WORKDIR /app
+COPY --from=builder /src/bin/dnswatcher /usr/local/bin/dnswatcher
+COPY deploy/docker-entrypoint.sh /usr/local/bin/docker-entrypoint.sh
 
-COPY --from=builder /src/bin/dnswatcher /app/dnswatcher
+# dnswatcher runs as this unprivileged user. The entrypoint creates the
-
+# data directory and gives it to this user on every start.
-# Create data directory
+RUN addgroup -S -g 10001 dnswatcher \
-RUN mkdir -p /var/lib/dnswatcher
+    && adduser -S -G dnswatcher -u 10001 dnswatcher
 
 ENV DNSWATCHER_DATA_DIR=/var/lib/dnswatcher
 
+# Config loading also reads a `.env` file and a file named `dnswatcher`
+# (any config extension, or none) from the working directory. `/` holds
+# neither, so every setting comes from the environment. Do not make the
+# data directory, or the binary's directory, the working directory.
+WORKDIR /
+
+# No USER: the entrypoint must start as root to set up the data
+# directory; it then runs dnswatcher as the dnswatcher user.
+
 EXPOSE 8080
 
-ENTRYPOINT ["/app/dnswatcher"]
+# busybox wget (already in alpine) probes the health endpoint every 10
+# seconds, so the container is healthy well before upaas reads its health
+# 60 seconds after a deploy and fails the deploy unless it is healthy.
+HEALTHCHECK --interval=10s --timeout=5s --start-period=10s --retries=3 \
+    CMD wget -q -O /dev/null "http://127.0.0.1:${PORT:-8080}/.well-known/healthcheck" || exit 1
+
+ENTRYPOINT ["/usr/local/bin/docker-entrypoint.sh"]

README.md

@@ -61,6 +61,10 @@ rejected.
   record types: A, AAAA, CNAME, MX, TXT, SRV, CAA, NS.
 - Stores results **per nameserver**. The state for a hostname is not a
   merged view — it is a map from nameserver to record set.
+- DNS names inside record values (CNAME, MX, SRV and NS targets) are
+  stored in lower case, because names are case-insensitive and
+  nameservers may answer in any letter case. TXT and CAA values keep
+  their letter case; they are not lower-cased.
 - Any observable change in any nameserver's response triggers a
   notification. This includes:
   - **Record change**: A nameserver returns different records than it
@@ -70,8 +74,15 @@ rejected.
     This is distinct from "responded with no records."
   - **NS recovery**: A previously-unreachable nameserver starts
     responding again.
-  - **Inconsistency detected**: Two nameservers that previously agreed
+  - **Inconsistency detected**: Two nameservers return different record
-    now return different record sets for the same hostname.
+    sets for the same hostname and did not already differ on the previous
+    check. Every pair of nameservers is compared. The alert is sent once
+    for each such pair, on the check where they start to disagree, and not
+    again while they keep disagreeing, including after a restart. A
+    nameserver that was not in the previous check (newly added, or back
+    after dropping out) and answers differently is reported on the check
+    where it appears. If a pair agrees again and later disagrees, the
+    alert is sent again.
 
 ### TCP Port Monitoring
 
@@ -203,6 +214,46 @@ are read, so a smaller value would sever the connection before a handler
 using its full budget could respond. `IdleTimeout` exceeds common
 Prometheus scrape intervals so the scraper reuses its connection.
 
+### Security Headers
+
+Every response — the dashboard, the static assets under `/s/...`, the
+healthchecks, the JSON API, and `/metrics` — carries the following
+headers, set by a global middleware:
+
+| Header                      | Value                                 |
+|-----------------------------|---------------------------------------|
+| `Strict-Transport-Security` | `max-age=31536000; includeSubDomains` |
+| `Content-Security-Policy`   | see below                             |
+| `X-Frame-Options`           | `DENY`                                |
+| `X-Content-Type-Options`    | `nosniff`                             |
+| `Referrer-Policy`           | `no-referrer`                         |
+| `Permissions-Policy`        | all unused browser features denied    |
+
+The content security policy is:
+
+```
+default-src 'self'; script-src 'none'; style-src 'self'; img-src 'self';
+font-src 'none'; connect-src 'none'; object-src 'none'; base-uri 'none';
+form-action 'none'; frame-ancestors 'none'
+```
+
+The dashboard ships no JavaScript (the 30-second refresh is a
+`<meta http-equiv="refresh">`), no inline styles, no inline event
+handlers, and no images; its only subresource is the embedded stylesheet
+at `/s/css/tailwind.min.css`, which `style-src 'self'` permits. The
+policy therefore needs neither `unsafe-inline` nor `unsafe-eval`.
+`frame-ancestors 'none'` is the primary anti-framing control, with
+`X-Frame-Options: DENY` retained as the legacy fallback.
+
+HSTS is emitted unconditionally, including over plain HTTP. dnswatcher is
+expected to run behind a TLS-terminating reverse proxy, and the browser
+must still be told to enforce HTTPS end to end, so the header is never
+gated on whether the request itself arrived over TLS.
+
+`Referrer-Policy: no-referrer` is stricter than the
+`strict-origin-when-cross-origin` baseline: the dashboard has no
+cross-origin navigation needs, and its URL may name internal hosts.
+
 ---
 
 ## Architecture
@@ -215,7 +266,8 @@ internal/
   globals/globals.go           Build-time variables (version)
   logger/logger.go             slog structured logging (TTY detection)
   healthcheck/healthcheck.go   Health check service
-  middleware/middleware.go      HTTP middleware (logging, CORS, metrics auth)
+  middleware/middleware.go      HTTP middleware (logging, CORS, security
+                               headers, metrics auth and rate limit)
   handlers/handlers.go         HTTP request handlers
   server/
     server.go                  HTTP server lifecycle
@@ -226,6 +278,8 @@ internal/
   tlscheck/tlscheck.go        TLS certificate inspector
   notify/notify.go             Notification service (Slack, Mattermost, ntfy)
   watcher/watcher.go           Main monitoring orchestrator and scheduler
+  livednstest/livednstest.go   Retry and concurrency limit for tests
+                               against live DNS (imported only by tests)
 ```
 
 ### Design Principles
@@ -281,6 +335,15 @@ is a misconfiguration, so dnswatcher fails fast with a clear error message
 rather than running silently. Set `DNSWATCHER_TARGETS` to a comma-separated
 list of DNS names before starting.
 
+**`/metrics` is rate limited.** Each client address may send it 30 requests a
+minute, failed logins included; beyond that it answers `429 Too Many Requests`
+without checking the password. A Prometheus server scraping every 15 seconds
+sends 4 a minute. IPv6 addresses in one /64 count as one client. When the
+request comes from a private or loopback address, such as a reverse proxy's,
+the client address is taken from the `X-Real-IP` or `X-Forwarded-For` header
+the proxy sets; a proxy that sets neither makes all its clients share one
+allowance.
+
 ### Example `.env`
 
 ```sh
@@ -424,9 +487,13 @@ them. We provide:
 - `script/fmt` — format all code (gofmt -s, goimports)
 - `script/fmt-check` — check formatting (read-only)
 - `script/check` — run test, lint, and fmt-check
-- `script/docker` — build the Docker image tagged via
+- `script/docker` — build the Docker image tagged via `script/projectname`, with
-  `script/projectname`
+  `--no-cache-filter=lint,builder` so the lint stage and the builder stage,
-- `script/cibuild` — CI entrypoint: plain `docker build .`
+  which runs the tests, run on every invocation
+- `script/cibuild` — CI entrypoint: `docker build` with
+  `--no-cache-filter=lint,builder`, so the lint stage and the builder stage,
+  which runs the tests, run on every invocation, because a cached build lints
+  nothing and queries no DNS
 - `script/precommit` — run by the git pre-commit hook; `go mod tidy`
   guard, then `script/check`
 - `script/install-precommit` — install the git pre-commit hook
@@ -467,11 +534,47 @@ docker run -d \
 
 ---
 
+## Running under upaas
+
+[upaas](https://git.eeqj.de/sneak/upaas) builds the image from this
+repository's `Dockerfile` and runs it. The app needs:
+
+- **Branch:** `prod`. `prod` is cut from `main`, and merging a `main` to
+  `prod` pull request is a deploy.
+- **Volume:** one host directory mounted at `/var/lib/dnswatcher`, where
+  the state file lives.
+- **Network and port:** the dashboard is unauthenticated and shows every
+  watched name and recent alert, and upaas publishes every mapped port on
+  all interfaces of the host
+  ([upaas issue 113](https://git.eeqj.de/sneak/upaas/issues/113)). Add a
+  port mapping to container port `8080` only if the dashboard should be
+  public. Otherwise add none: set the app's Docker network in upaas to
+  your reverse proxy's Docker network, and the proxy reaches the app at
+  `upaas-` followed by the app name, port `8080`.
+- **Required environment:** `DNSWATCHER_TARGETS`, a comma-separated list
+  of the domains and hostnames to watch. dnswatcher refuses to start
+  without it.
+- **Recommended environment:** at least one notification endpoint
+  (`DNSWATCHER_SLACK_WEBHOOK`, `DNSWATCHER_MATTERMOST_WEBHOOK`,
+  `DNSWATCHER_NTFY_TOPIC`); without one, changes show only on the
+  dashboard. `DNSWATCHER_METRICS_USERNAME` and
+  `DNSWATCHER_METRICS_PASSWORD` serve `/metrics` behind basic auth.
+- **Leave unset:** `DNSWATCHER_DATA_DIR`, which the image sets to
+  `/var/lib/dnswatcher`, and `PORT`, which defaults to `8080`. Every
+  setting comes from the environment; the image holds no config file.
+- **Health check:** the image's own, which requests
+  `/.well-known/healthcheck` every 10 seconds. upaas reads the
+  container's health 60 seconds after a deploy and marks the deploy
+  failed unless it is `healthy`.
+
+---
+
 ## Monitoring Lifecycle
 
-1. **Startup**: Load state from disk. If no state file exists, start
+1. **Startup**: Check that the data directory can be written, and exit
-   with empty state (first check will establish baseline without
+   with an error naming it if not. Load state from disk. If no state
-   triggering change notifications).
+   file exists, start with empty state (first check will establish
+   baseline without triggering change notifications).
 2. **Initial check**: Immediately perform all DNS, port, and TLS checks
    on startup.
 3. **Periodic checks** (DNS always runs first):

TESTING.md

@@ -2,8 +2,12 @@
 
 ## DNS Resolution Tests
 
-All resolver tests **MUST** use live queries against real DNS servers.
+DNS is never mocked in this project, not in tests and not anywhere
-No mocking of the DNS client layer is permitted.
+else; see the README section "No DNS mocking. Ever." Every test that
+looks something up in DNS **MUST** query live DNS servers, never a
+stand-in. Logic that works on record data, such as comparing or
+formatting records, may be tested on that data directly with no
+lookup.
 
 ### Rationale
 
@@ -21,14 +25,18 @@ real servers ensures the resolver works correctly in production.
 - Query timeout is calibrated to 3× maximum antipodal RTT (~300ms)
   plus processing margin
 - Root server fan-out is limited to reduce parallel query load
+- Live lookups that expect an answer go through `internal/livednstest`,
+  which limits how many run at once in a test binary and retries a
+  lookup that got none
 - Flaky failures from transient network issues are acceptable and
   should be investigated as potential resolver bugs, not papered over
   with mocks or skip flags
 
 ### What NOT to do
 
-- **Do not mock `DNSClient`** for resolver tests (the mock constructor
+- **Do not mock, fake or stub DNS** anywhere: no stand-in `DNSClient`,
-  exists for unit-testing other packages that consume the resolver)
+  no stand-in for the watcher's `DNSResolver`, no fake DNS server, no
+  canned responses
 - **Do not add `-short` flags** to skip slow tests
 - **Do not increase `-timeout`** to hide hanging queries
 - **Do not remove `-count=1` from `script/test`** — Go's test cache

TODO.md

@@ -1,237 +1,112 @@
 # Workflow
 
-* branch (from `main`)
+* branch (from `next`)
 * do the work in Next Step
 * move Next Step to the top of Completed Steps
 * move the top item of Future Steps into Next Step
 * commit (`TODO.md` changes in the same commit as the work)
-* merge to `main` if the branch is not protected, otherwise open a PR
 * push
+* open a PR against `next`
 
 # Status
 
-pre-1.0. No git tags. Core resolver work in flight on feature/resolver
+pre-1.0. No git tags. Work lands on `next` by PR. Open work for 1.0 is tracked
-(dirty: internal/resolver/resolver_test.go). Local checkout has diverged
+on the 1.0 milestone: https://git.eeqj.de/sneak/dnswatcher/milestone/7
-from origin: origin/main is 8 commits ahead (watcher orchestrator,
-unified TARGETS) and origin/feature/resolver already contains the full
-iterative resolver implementation with hermetic mocked tests.
 
 # Next Step
 
-Add the README sections required by policy (Description, Getting Started,
+NS failure and NS recovery notifications:
-Rationale, Design, TODO, License, Author) if any are still missing.
+https://git.eeqj.de/sneak/dnswatcher/issues/104
 
 # Completed Steps
 
+- 2026-10-01: `/metrics` allows each client address 30 requests a minute,
+  counted before Basic Auth, and answers 429 beyond that (closes #101).
+- 2026-10-01: `TODO.md` brought up to date: open issues listed by URL, every
+  Completed Steps entry cut to at most two lines (closes #146).
+- 2026-10-01: wildcard CORS now applies only to the public routes, not to
+  `/metrics`, and allows only the methods they serve (closes #100).
+- 2026-10-01: `internal/state` and `internal/watcher` no longer export test-only
+  constructors: two moved to `export_test.go`, one is deleted (closes #111).
+- 2026-10-01: notify shutdown tests use one timing constant per meaning, name
+  the bound they check, and require the drain's debug line (closes #116).
+- 2026-09-29: the entrypoint chowns the data directory to `dnswatcher` and runs
+  dnswatcher as that user, so a host bind mount needs no chown (closes #166).
+- 2026-09-29: the live-DNS test package is renamed `internal/livednstest`;
+  `make lint` fails when program code imports it (closes #164).
+- 2026-09-29: `.golangci.yml` re-fetched from `sneak/prompts`, with
+  `gomodguard_v2` and the org `depguard` `test-support` rule (closes #123).
+- 2026-09-29: watcher and resolver tests that look something up in DNS use the
+  real resolver against live DNS servers (closes #159).
+- 2026-09-28: the inconsistency alert is sent once, when two nameservers start
+  to disagree; every pair of nameservers is compared (closes #158).
+- 2026-09-28: DNS names in record values (CNAME, MX, SRV and NS targets) are
+  lower-cased, so letter case alone is not a change (closes #157).
+- 2026-09-28: lint and tests run on every build: `script/cibuild` and
+  `script/docker` pass `--no-cache-filter=lint,builder` (closes #115).
+- 2026-09-28: the server timeout test drives `Run` and checks the timeouts on
+  the `http.Server` it serves (closes #120).
+- 2026-09-28: upaas deploy readiness: the image runs as user `dnswatcher` with a
+  `HEALTHCHECK`; README "Running under upaas" (closes #147).
+- 2026-09-21: added behavioural tests for `internal/globals`,
+  `internal/healthcheck`, and `internal/logger` (closes #110).
 - 2026-09-21: `go mod tidy` dropped the redundant `golang.org/x/sync`
   `// indirect` line so `script/bootstrap` leaves a clean tree (#132)
-- 2026-08-10: comment-only corrections to `script/bootstrap`,
+- 2026-08-10: comment-only corrections to `script/bootstrap`, `script/cibuild`
-  `script/cibuild`, and `Dockerfile.lint`. The `goimports` pin in
+  and `Dockerfile.lint`; no behaviour changed.
-  `script/bootstrap` was justified by a claim that `script/fmt-check`
+- 2026-08-10: MIT `LICENSE` added at the repository root; the README's first
-  runs it on the host; it does not (it runs `gofmt -l .` only), so the
+  line and License section name the licence.
-  header now credits `script/fmt` alone. `script/cibuild` still claimed
+- 2026-08-10: policy scaffold present: `REPO_POLICIES.md`, `.editorconfig`,
-  the `Dockerfile` runs `make check`, which stopped being true when
+  `.dockerignore`, CI workflow, `make fmt-check`, `make docker`, `make hooks`.
-  linting moved to its own stage; it now describes the lint stage
+- 2026-08-10: Go's test cache disabled in `script/test` (`-count=1`), so every
-  (`make fmt-check` plus `golangci-lint`) and the builder stage
+  run queries live DNS; a failed run is rerun with `-v`.
-  (`make test`, `make build`). The `docker`-missing warning in
+- 2026-08-10: live-DNS tests made robust rather than gated (#93): a limit on
-  `script/bootstrap` reads as one sentence instead of three fragments
+  concurrent lookups, retries, and a quorum across nameservers.
-  each re-prefixed with `bootstrap:`. `Dockerfile.lint` now records the
+- 2026-08-10: all linting moved into Docker: `script/lint` builds
-  residual risk of omitting `golangci-lint config verify`: unknown
+  `Dockerfile.lint`, and the root `Dockerfile` has its own lint stage.
-  top-level keys in `.golangci.yml` are silently ignored, so a mistyped
+- 2026-08-09: in-flight notification deliveries are drained at shutdown, bounded
-  key lints clean while applying nothing. No behaviour changed
+  by the shutdown deadline (#106).
-- 2026-08-10: MIT `LICENSE` added at the repository root, closing the
+- 2026-08-09: `http.Server` sets all four socket timeouts; `WriteTimeout` stays
-  last gap in `REPO_POLICIES.md`'s required-minimum file list and
+  above the 60s handler timeout (#99).
-  removing the all-rights-reserved default that would otherwise have
+- 2026-08-09: `SecurityHeaders()` middleware sets HSTS, CSP and the other
-  shipped with a 1.0 tag. The licence choice is the standing org policy
+  security headers `REPO_POLICIES.md` requires on every response.
-  (any public repo lacking a licence gets MIT; a private repo with no
+- 2026-08-07: golangci-lint bumped to v2.12.2 and `.golangci.yml` set to the org
-  licence is already all-rights-reserved), and this repo is public. The
+  config; fixed the resulting `goconst`, `dupl` and `lll` findings.
-  file holds the canonical MIT text byte-for-byte with only the
+- 2026-07-07 Adopted scripts-to-rule-them-all: `script/` entrypoints, Makefile
-  copyright line filled in (`Copyright (c) 2026 sneak`); no clauses were
+  shims, README Entrypoints section
-  added, removed, or reflowed. `README.md`'s first line now names the
+- 2026-02-20: iterative DNS resolver implemented
-  licence, as the Description requirement demands, and the License
+- 2026-02-20: CI actions and go install refs pinned to commit SHAs; Gitea
-  section states MIT and points at the file instead of saying the choice
+  Actions workflow added
-  is pending. `make fmt` covers only Go sources (`gofmt -s`,
-  `goimports`), so it cannot reflow `LICENSE`
-- 2026-08-10: the policy scaffold (`REPO_POLICIES.md`, `.editorconfig`,
-  `.dockerignore`, `.gitea/workflows/check.yml`, and the `fmt-check`,
-  `docker`, and hooks Makefile targets) is present; it landed piecemeal
-  across the scripts-to-rule-them-all and policy commits rather than as
-  the single commit this file once planned
-- 2026-08-10: Go's test cache disabled for `script/test` via `-count=1`,
-  so every invocation actually executes. A cached pass replays an
-  earlier run's output without querying DNS at all, which in this repo
-  means the suite's entire premise goes unexercised while the run
-  reports green in under a second. The conditional verbose rerun that
-  `REPO_POLICIES.md` mandates was added at the same time (the primary
-  run had been unconditionally `-v`): quiet first, `-v` only on
-  failure, `-count=1` on both, and exit 1 forced regardless of the
-  rerun's result so a flake passing the second time cannot turn the
-  build green. `-timeout 90s` left alone as the deliberate backstop
-  above the 60s hard cap. Uncached suite runs ~4s, well inside the 20s
-  target
-- 2026-08-10: live-DNS test flakiness addressed by robustness rather
-  than gating, per the owner's ruling on #93: new
-  `internal/resolver/livedns_test.go` adds a package-wide concurrency
-  gate (so parallel tests stop bursting at the first root server),
-  retry with exponential backoff on transport failures only, and
-  quorum instead of unanimity for multi-nameserver assertions. Quorum
-  tolerates silence only: every per-nameserver status must be in a
-  closed allowlist (`ok`/`timeout`/`error`, or
-  `nxdomain`/`timeout`/`error`), so a wrong answer from a minority —
-  `nodata` today, any status added later — fails the test instead of
-  sliding through under the majority. The
-  `make test` cap moved to the new org-wide 60s hard cap / 20s target
-  with a 90s `-timeout` backstop; `REPO_POLICIES.md` re-vendored
-  byte-identical from `sneak/prompts`. No mocks, no `-short`, no build
-  tags, no skips, and no change to production resolver behaviour
-- 2026-08-10: all linting moved into Docker: new root `Dockerfile.lint`
-  on the digest-pinned `golangci/golangci-lint:v2.12.2` image,
-  `script/lint` reduced to a thin wrapper that builds it with
-  `--no-cache-filter=lint` so the linter actually executes every run,
-  golangci-lint install dropped from `script/bootstrap` (goimports
-  stays, `script/fmt` needs it on the host), and the root `Dockerfile`
-  given its own lint stage so its build no longer recurses through
-  `make check` into `script/lint`. `golangci-lint config verify` is
-  deliberately omitted: it fetches its schema over an unpinned live
-  HTTPS call
-- 2026-08-09: in-flight notification deliveries are now drained at
-  shutdown (#106): `notify.New` registers an fx `OnStop` hook that waits
-  on a `sync.WaitGroup` of tracked delivery goroutines, bounded by the
-  `OnStop` context; on expiry the outstanding count is logged at warn
-  level and parked retry backoffs are released instead of being dropped
-  silently, and deliveries submitted after the drain begins are refused
-  so shutdown cannot be extended indefinitely; an `OnStop` context that
-  is already expired on entry with nothing outstanding drains quietly
-  rather than warning about deliveries that were never abandoned
-- 2026-08-09: `http.Server` now sets all four socket-level timeouts
-  (`ReadTimeout` 15s, `ReadHeaderTimeout` 10s, `WriteTimeout` 75s,
-  `IdleTimeout` 120s) as named constants in `internal/server/server.go`,
-  closing the slowloris / unreaped-keep-alive exposure required by
-  `REPO_POLICIES.md` before 1.0; `WriteTimeout` is deliberately greater
-  than the 60s `chimw.Timeout` handler budget so that budget stays
-  reachable, and tests in `internal/server` pin both the non-zero
-  values and that relationship (#99)
-- 2026-08-07: golangci-lint bumped to v2.12.2 (commit-pinned installs
-  in `Dockerfile` and `script/bootstrap`); `.golangci.yml` set to the
-  org-standard v2-schema config used across the org's repos
-  (owner-authorized; same file is being landed as canonical via prompts
-  PR #24), with settings under `linters.settings` so the
-  lll/funlen/cyclop/dupl thresholds apply; fixed the resulting
-  `goconst`, `dupl`, and `lll` findings; the informational `gomodguard`
-  deprecation warning under this config is accepted
-- 2026-07-07 Adopted scripts-to-rule-them-all: `script/` entrypoints,
-  Makefile shims, README Entrypoints section
-- 2026-02-20: iterative DNS resolver implemented; tests made hermetic
-  with mocked DNS (origin/feature/resolver, unmerged)
-- 2026-02-20: CI actions and go install refs pinned to commit SHAs;
-  Gitea Actions workflow for make check (origin/ci/make-check, unmerged)
 - 2026-02-20: watcher monitoring orchestrator merged to main (#8)
 - 2026-02-20: DOMAINS/HOSTNAMES unified into single TARGETS config (#11)
 - 2026-02-19: TCP port connectivity checker, made concurrent with port
   validation; gosec G704 SSRF findings fixed without suppression
-  (feature branches, unmerged)
+- 2026-02-19: TLS certificate inspector with no-peer-certificates error path and
-- 2026-02-19: TLS certificate inspector with no-peer-certificates error
+  IP SANs
-  path and IP SANs (feature branch, unmerged)
 - 2026-02-19: gosec SSRF and formatting fixes on main
 - 2026-02-19: initial scaffold with per-nameserver DNS monitoring model
 
 # Future Steps
 
-Compliance:
+- nameserver IP address changes: https://git.eeqj.de/sneak/dnswatcher/issues/105
-
+- `DNSWATCHER_SENTRY_DSN` does nothing:
-- Pin Dockerfile base images by sha256 and ensure the Docker build runs
+  https://git.eeqj.de/sneak/dnswatcher/issues/107
-  make check
+- invalid DNS or TLS interval silently replaced by the default:
-
+  https://git.eeqj.de/sneak/dnswatcher/issues/177
-Branch reconciliation:
+- images report version `dev`: https://git.eeqj.de/sneak/dnswatcher/issues/109
-
+- trial run of the finished image:
-- Sync local checkout with origin: local main is 8 commits behind
+  https://git.eeqj.de/sneak/dnswatcher/issues/149
-  origin/main; local feature/resolver has diverged from
+- 1.0 readiness: run it with a real config and read the logs:
-  origin/feature/resolver, which already implements the resolver
+  https://git.eeqj.de/sneak/dnswatcher/issues/66
-- Merge in-flight branches to main once green: feature/resolver,
+- `goimports` in `make fmt-check`, Markdown formatting:
-  ci/make-check, feature/portcheck-implementation,
+  https://git.eeqj.de/sneak/dnswatcher/issues/119
-  feature/tlscheck-implementation
+- final state save at shutdown: https://git.eeqj.de/sneak/dnswatcher/issues/114
-
+- `internal/notify` shutdown tests hang when a drain returns early:
-Resolver (plan from untracked TODO.md; largely implemented on
+  https://git.eeqj.de/sneak/dnswatcher/issues/176
-origin/feature/resolver, verify each item before closing):
+- README accuracy sweep: https://git.eeqj.de/sneak/dnswatcher/issues/108
-
+- README sections required by policy:
-- Add github.com/miekg/dns dependency
+  https://git.eeqj.de/sneak/dnswatcher/issues/173
-- roots.go: hardcoded IANA root server list (a through m, IPv4/IPv6),
+- `script/install-precommit` in a linked worktree:
-  rootServers() returning ip:53 strings
+  https://git.eeqj.de/sneak/dnswatcher/issues/129
-- query.go: low-level query(ctx, server, name, qtype): UDP with TCP
+- fixed root server order: https://git.eeqj.de/sneak/dnswatcher/issues/138
-  fallback on truncation, RD=0, context respected, 5s per-query timeout,
+- review toward 1.0: https://git.eeqj.de/sneak/dnswatcher/issues/144
-  returns raw *dns.Msg
-- trace.go: iterative delegation chasing from roots: referral detection
-  (NOERROR, empty answer, NS in authority), glue extraction with
-  bailiwick check, out-of-bailiwick NS resolved with recursion guard,
-  delegation depth limit (20), retry across nameservers on failure, do
-  not chase CNAMEs inside trace
-- FindAuthoritativeNameservers: NS set via trace, sorted, FQDN
-  normalized, trailing dot handled; must pass its 9 tests
-- QueryNameserver: resolve NS host to IPs, query A/AAAA/CNAME/MX/TXT/
-  SRV/CAA/NS, build NameserverResponse with status mapping (OK,
-  NXDomain, NoData, Error), documented record formatting, sorted values,
-  lame delegation detection; must pass its 16 tests
-- QueryAllNameservers: find NS set for parent domain (public suffix
-  list), query all NS in parallel with bounded concurrency, return map
-  even when all fail, context cancellation; must pass its 4 tests
-- LookupNS: thin wrapper over FindAuthoritativeNameservers, sorted,
-  identical results; must pass its 3 tests
-- ResolveIPAddresses: collect A/AAAA from all NS, follow CNAME chains
-  with MaxCNAMEDepth, dedupe, sort, NXDOMAIN returns empty slice with
-  nil error; must pass its 9 tests
-- All 39 resolver tests pass, make check green, merge to main
-
-Watcher (internal/watcher/watcher.go):
-
-- Scheduling loop in Run(ctx): initial check on startup, separate
-  tickers for DNS/port and TLS intervals, persist state via state.Save()
-  after each cycle, clean shutdown on context cancel
-- Domain check: LookupNS, compare to stored state, store silently on
-  first run, notify with old/new NS lists on change
-- Hostname check: QueryAllNameservers, compare per-NS records; notify on
-  record changes, NS failure, NS recovery, inconsistency detected,
-  inconsistency resolved, empty response; store silently on first run
-- Port check: ResolveIPAddresses, check ports 80 and 443 per IP, notify
-  on open/closed transitions, handle new and disappeared IPs
-- TLS check: for each open IP:443, CheckCertificate; notify on expiry
-  warning, certificate change (CN/issuer/SANs), TLS failure/recovery
-
-Port checker (internal/portcheck/portcheck.go):
-
-- Tests against known-open ports and RFC documentation IPs
-- CheckPort: net.DialTimeout (5s), context respected; (true, nil) open,
-  (false, nil) closed/timeout/refused, error only for unexpected
-  failures
-
-TLS checker (internal/tlscheck/tlscheck.go):
-
-- Tests against known public HTTPS servers, verify fields populated
-- CheckCertificate: tls.Dial to specific IP:443 with hostname as SNI;
-  extract subject CN, issuer CN and org, NotAfter, SANs; error on
-  handshake failure
-
-Notification service (internal/notify/notify.go, Slack/Mattermost/ntfy
-backends exist):
-
-- Structured notification types: DNS change, port change, TLS expiry,
-  TLS change, NS failure, NS recovery, NS inconsistency
-- Per-backend formatting: Slack/Mattermost attachment colors (red
-  failures/expiry, yellow warnings, green recoveries, blue info); ntfy
-  priorities (urgent failures, high warnings, default changes, low
-  recoveries); include hostname, nameserver, old/new values, timestamps
-
-HTTP API handlers:
-
-- Wire *state.State and *watcher.Watcher into handler params
-- GET /api/v1/status: full state snapshot as JSON
-- GET /api/v1/domains: domain states with NS records and last-checked
-- GET /api/v1/hostnames: hostname states with per-NS record data
-
-Infrastructure notes (from untracked TODO.md):
-
-- Module path sneak.berlin/go/dnswatcher differs from the git.eeqj.de
-  remote intentionally; do not "fix" it
-- Dependencies: github.com/miekg/dns, golang.org/x/net/publicsuffix
-- Resolver tests originally used live DNS against *.dns.sneak.cloud
-  (required records documented in the test file header); origin now has
-  mocked hermetic tests, keep them hermetic

deploy/docker-entrypoint.sh

@@ -0,0 +1,17 @@
+#!/bin/sh
+# deploy/docker-entrypoint.sh: the Docker image's ENTRYPOINT. It runs as
+# root only to give the data directory to the dnswatcher user: a host
+# directory bind-mounted there keeps its host owner, often root, and may
+# hold a state file left by another uid, which dnswatcher could neither
+# read nor replace. dnswatcher itself always runs as the dnswatcher user.
+set -eu
+
+main() {
+    dir="${DNSWATCHER_DATA_DIR:-/var/lib/dnswatcher}"
+    mkdir -p "$dir"
+    chown -R dnswatcher:dnswatcher "$dir"
+    chmod 700 "$dir"
+    exec su-exec dnswatcher /usr/local/bin/dnswatcher "$@"
+}
+
+main "$@"

go.mod

@@ -6,6 +6,7 @@ require (
 	github.com/99designs/basicauth-go v0.0.0-20230316000542-bf6f9cbbf0f8
 	github.com/go-chi/chi/v5 v5.2.5
 	github.com/go-chi/cors v1.2.2
+	github.com/go-chi/httprate v0.16.0
 	github.com/joho/godotenv v1.5.1
 	github.com/miekg/dns v1.1.72
 	github.com/prometheus/client_golang v1.23.2
@@ -22,6 +23,7 @@ require (
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/fsnotify/fsnotify v1.9.0 // indirect
 	github.com/go-viper/mapstructure/v2 v2.4.0 // indirect
+	github.com/klauspost/cpuid/v2 v2.2.10 // indirect
 	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
 	github.com/pelletier/go-toml/v2 v2.2.4 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
@@ -34,6 +36,7 @@ require (
 	github.com/spf13/cast v1.10.0 // indirect
 	github.com/spf13/pflag v1.0.10 // indirect
 	github.com/subosito/gotenv v1.6.0 // indirect
+	github.com/zeebo/xxh3 v1.0.2 // indirect
 	go.uber.org/dig v1.19.0 // indirect
 	go.uber.org/multierr v1.10.0 // indirect
 	go.uber.org/zap v1.26.0 // indirect

go.sum

@@ -14,6 +14,8 @@ github.com/go-chi/chi/v5 v5.2.5 h1:Eg4myHZBjyvJmAFjFvWgrqDTXFyOzjj7YIm3L3mu6Ug=
 github.com/go-chi/chi/v5 v5.2.5/go.mod h1:X7Gx4mteadT3eDOMTsXzmI4/rwUpOwBHLpAfupzFJP0=
 github.com/go-chi/cors v1.2.2 h1:Jmey33TE+b+rB7fT8MUy1u0I4L+NARQlK6LhzKPSyQE=
 github.com/go-chi/cors v1.2.2/go.mod h1:sSbTewc+6wYHBBCW7ytsFSn836hqM7JxpglAy2Vzc58=
+github.com/go-chi/httprate v0.16.0 h1:8V5DH9j6pSK6UQoBsTpvMyFxycqaKEIToyPKzHJjUa8=
+github.com/go-chi/httprate v0.16.0/go.mod h1:A8lo+qRhk+s9LiuP5saS7XCGDXRXMcrueq0NfIuCa/I=
 github.com/go-viper/mapstructure/v2 v2.4.0 h1:EBsztssimR/CONLSZZ04E8qAkxNYq4Qp9LvH92wZUgs=
 github.com/go-viper/mapstructure/v2 v2.4.0/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM=
 github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
@@ -22,6 +24,8 @@ github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0=
 github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4=
 github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo=
 github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ=
+github.com/klauspost/cpuid/v2 v2.2.10 h1:tBs3QSyvjDyFTq3uoc/9xFpCuOsJQFNPiAhYdw2skhE=
+github.com/klauspost/cpuid/v2 v2.2.10/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0=
 github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
 github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
 github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
@@ -62,6 +66,10 @@ github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu
 github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=
 github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8=
 github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU=
+github.com/zeebo/assert v1.3.0 h1:g7C04CbJuIDKNPFHmsk4hwZDO5O+kntRxzaUoNXj+IQ=
+github.com/zeebo/assert v1.3.0/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0=
+github.com/zeebo/xxh3 v1.0.2 h1:xZmwmqxHZA8AI603jOQ0tMqmBr9lPeFwGg6d+xy9DC0=
+github.com/zeebo/xxh3 v1.0.2/go.mod h1:5NWz9Sef7zIDm2JHfFlcQvNekmcEl9ekUZQQKCYaDcA=
 go.uber.org/dig v1.19.0 h1:BACLhebsYdpQ7IROQ1AGPjrXcP5dF80U3gKoFzbaq/4=
 go.uber.org/dig v1.19.0/go.mod h1:Us0rSJiThwCv2GteUN0Q7OKvU7n5J4dxZ9JKUXozFdE=
 go.uber.org/fx v1.24.0 h1:wE8mruvpg2kiiL1Vqd0CC+tr0/24XIB10Iwp2lLWzkg=

internal/globals/globals_test.go

@@ -0,0 +1,50 @@
+package globals_test
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"sneak.berlin/go/dnswatcher/internal/globals"
+)
+
+// TestGlobals exercises the package-level version and appname
+// variables through their setters and read-back via New. These are
+// shared package state, so the test mutates a global and must run
+// sequentially; it cannot use t.Parallel().
+//
+//nolint:paralleltest // mutates shared package-level globals, must run sequentially
+func TestGlobals(t *testing.T) {
+	versions := []string{"v1.2.3", "dev", "", "v1.2.3-4-gabcdef"}
+	for _, want := range versions {
+		globals.SetVersion(want)
+
+		g, err := globals.New(nil)
+		require.NoError(t, err)
+		assert.Equal(t, want, g.Version,
+			"New must surface the version set by SetVersion")
+	}
+
+	names := []string{"dnswatcher", "other", ""}
+	for _, want := range names {
+		globals.SetAppname(want)
+
+		g, err := globals.New(nil)
+		require.NoError(t, err)
+		assert.Equal(t, want, g.Appname,
+			"New must surface the appname set by SetAppname")
+	}
+
+	// New returns a snapshot: a later SetVersion must not mutate a
+	// Globals handed out earlier.
+	globals.SetVersion("first")
+
+	g, err := globals.New(nil)
+	require.NoError(t, err)
+
+	globals.SetVersion("second")
+	assert.Equal(t, "first", g.Version,
+		"a Globals returned by New must not change when the "+
+			"package variable is set again")
+}

internal/healthcheck/healthcheck_test.go

@@ -0,0 +1,134 @@
+package healthcheck_test
+
+import (
+	"context"
+	"encoding/json"
+	"testing"
+	"time"
+
+	"go.uber.org/fx"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"sneak.berlin/go/dnswatcher/internal/config"
+	"sneak.berlin/go/dnswatcher/internal/globals"
+	"sneak.berlin/go/dnswatcher/internal/healthcheck"
+	"sneak.berlin/go/dnswatcher/internal/logger"
+)
+
+// recordingLifecycle is a minimal fx.Lifecycle that records the hooks
+// appended to it, so healthcheck.New can be exercised through its real
+// constructor without standing up a whole fx application.
+type recordingLifecycle struct {
+	hooks []fx.Hook
+}
+
+func (l *recordingLifecycle) Append(hook fx.Hook) {
+	l.hooks = append(l.hooks, hook)
+}
+
+// newHealthcheck builds a Healthcheck through the real constructor and
+// runs the registered OnStart hook so StartupTime is set the same way
+// the fx lifecycle would set it.
+func newHealthcheck(
+	t *testing.T,
+	maintenance bool,
+	version string,
+) *healthcheck.Healthcheck {
+	t.Helper()
+
+	g := &globals.Globals{Appname: "dnswatcher", Version: version}
+
+	log, err := logger.New(nil, logger.Params{Globals: g})
+	require.NoError(t, err)
+
+	lifecycle := &recordingLifecycle{}
+
+	hc, err := healthcheck.New(lifecycle, healthcheck.Params{
+		Globals: g,
+		Config:  &config.Config{MaintenanceMode: maintenance},
+		Logger:  log,
+	})
+	require.NoError(t, err)
+
+	require.Len(t, lifecycle.hooks, 1,
+		"New must register exactly one lifecycle hook")
+	require.NotNil(t, lifecycle.hooks[0].OnStart)
+	require.NoError(t, lifecycle.hooks[0].OnStart(context.Background()))
+
+	return hc
+}
+
+func TestCheckStatusAndPayloadShape(t *testing.T) {
+	t.Parallel()
+
+	hc := newHealthcheck(t, false, "v9.9.9")
+	resp := hc.Check()
+
+	assert.Equal(t, "ok", resp.Status)
+
+	// The JSON shape and field names are part of the contract for the
+	// /health and /.well-known/healthcheck routes, so assert on the
+	// exact set of keys the response marshals to.
+	raw, err := json.Marshal(resp)
+	require.NoError(t, err)
+
+	var fields map[string]json.RawMessage
+	require.NoError(t, json.Unmarshal(raw, &fields))
+
+	wantKeys := []string{
+		"status",
+		"now",
+		"uptimeSeconds",
+		"uptimeHuman",
+		"version",
+		"appname",
+		"maintenanceMode",
+	}
+	assert.Len(t, fields, len(wantKeys),
+		"response must marshal to exactly the documented fields")
+
+	for _, key := range wantKeys {
+		assert.Contains(t, fields, key, "missing JSON field %q", key)
+	}
+
+	// The Now field is documented as RFC3339Nano; a change to the
+	// format constant should turn this red.
+	_, err = time.Parse(time.RFC3339Nano, resp.Now)
+	assert.NoError(t, err, "Now must be RFC3339Nano")
+}
+
+func TestCheckMaintenanceModeReflectsConfig(t *testing.T) {
+	t.Parallel()
+
+	tests := []struct {
+		name        string
+		maintenance bool
+	}{
+		{"maintenance off", false},
+		{"maintenance on", true},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			t.Parallel()
+
+			hc := newHealthcheck(t, tt.maintenance, "test")
+			resp := hc.Check()
+			assert.Equal(t, tt.maintenance, resp.Maintenance,
+				"maintenanceMode must mirror Config.MaintenanceMode")
+		})
+	}
+}
+
+func TestCheckSurfacesVersionAndAppname(t *testing.T) {
+	t.Parallel()
+
+	hc := newHealthcheck(t, false, "surfaced-version-123")
+	resp := hc.Check()
+
+	assert.Equal(t, "surfaced-version-123", resp.Version,
+		"version from globals must appear in the payload")
+	assert.Equal(t, "dnswatcher", resp.Appname)
+}

internal/livednstest/livednstest.go

@@ -0,0 +1,121 @@
+// Package livednstest runs the live DNS operations of tests. Tests that
+// look something up in DNS query live DNS servers, never a stand-in —
+// see TESTING.md. Nothing here mocks, fakes, stubs, records or replays
+// DNS, and nothing here skips a test: it only changes *how* the live
+// queries are issued, so that a single dropped UDP packet or one slow
+// authoritative server does not turn correct code into a red build.
+//
+// Two mechanisms:
+//
+//  1. Bounded concurrency. Tests run in parallel and the build hosts
+//     have many cores, so without a limit every test starts its own
+//     iterative resolution at the same instant and they all hit the
+//     first root server within a few milliseconds of each other. Root
+//     servers rate-limit that, which shows up as a different arbitrary
+//     subset of tests failing on each run. Run caps how many live
+//     operations are in flight at once in one test binary.
+//
+//  2. Retry with exponential backoff. Each live operation gets several
+//     attempts with its own timeout. An attempt is retried when it
+//     obtained nothing to check, never because of what the test
+//     asserts about the result, so a wrong result still fails on the
+//     first attempt. A fault in the code under test that leaves
+//     nothing to check looks the same as live DNS not answering, and
+//     fails only after the last attempt.
+package livednstest
+
+import (
+	"context"
+	"errors"
+	"testing"
+	"time"
+)
+
+const (
+	// attempts is how many times a live DNS operation is attempted
+	// before the test fails.
+	attempts = 3
+
+	// AttemptTimeout bounds one attempt. Worst case for an operation
+	// is attempts * AttemptTimeout plus the backoff — about 26
+	// seconds, well inside the 90-second `go test -timeout` backstop
+	// even when several operations exhaust their attempts.
+	AttemptTimeout = 8 * time.Second
+
+	// backoffBase is the delay after the first failed attempt; it is
+	// multiplied by backoffFactor each time.
+	backoffBase = 500 * time.Millisecond
+
+	// backoffFactor is the exponential backoff multiplier.
+	backoffFactor = 2
+
+	// Concurrency caps how many live operations may be in flight
+	// across one test binary at once.
+	Concurrency = 6
+)
+
+// gate bounds concurrent live operations. It has to be package scoped:
+// the whole point is that it is shared by every parallel test in the
+// test binary.
+//
+//nolint:gochecknoglobals // package-wide live query rate limit
+var gate = make(chan struct{}, Concurrency)
+
+// ErrNoAnswer reports that a live operation produced no usable answer,
+// which is retried rather than asserted on.
+var ErrNoAnswer = errors.New("no answer from live DNS")
+
+// Run executes one attempt of a live operation, holding a slot in gate
+// for its duration and bounding it with its own timeout.
+func Run(op func(ctx context.Context) error) error {
+	gate <- struct{}{}
+	defer func() { <-gate }()
+
+	ctx, cancel := context.WithTimeout(
+		context.Background(), AttemptTimeout,
+	)
+	defer cancel()
+
+	return op(ctx)
+}
+
+// Retry runs op until it reports success, retrying failures with
+// exponential backoff, and fails the test if every attempt fails. op
+// returns an error only for a failure to obtain an answer — never for
+// an answer the test disagrees with, which belongs in an assertion so
+// that it fails immediately. op stores whatever it obtained where its
+// caller can find it.
+func Retry(
+	t *testing.T,
+	what string,
+	op func(ctx context.Context) error,
+) {
+	t.Helper()
+
+	var last error
+
+	backoff := backoffBase
+
+	for attempt := range attempts {
+		if attempt > 0 {
+			t.Logf(
+				"%s: attempt %d of %d failed (%v), "+
+					"retrying in %s",
+				what, attempt, attempts, last, backoff,
+			)
+			time.Sleep(backoff)
+
+			backoff *= backoffFactor
+		}
+
+		last = Run(op)
+		if last == nil {
+			return
+		}
+	}
+
+	t.Fatalf(
+		"%s: all %d live attempts failed: %v",
+		what, attempts, last,
+	)
+}

internal/livednstest/livednstest_test.go

@@ -0,0 +1,103 @@
+package livednstest_test
+
+import (
+	"context"
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+
+	"sneak.berlin/go/dnswatcher/internal/livednstest"
+)
+
+// Tests for the retry and the concurrency limit themselves. They
+// perform no DNS resolution of any kind.
+
+func TestRetryRecoversFromTransientFailure(t *testing.T) {
+	t.Parallel()
+
+	const wantAttempts = 2
+
+	attempts := 0
+
+	livednstest.Retry(t, "transient", func(_ context.Context) error {
+		attempts++
+
+		if attempts < wantAttempts {
+			return livednstest.ErrNoAnswer
+		}
+
+		return nil
+	})
+
+	assert.Equal(t, wantAttempts, attempts)
+}
+
+func TestRetryGivesEachAttemptADeadline(t *testing.T) {
+	t.Parallel()
+
+	livednstest.Retry(t, "deadline", func(ctx context.Context) error {
+		deadline, ok := ctx.Deadline()
+		assert.True(t, ok, "attempt should carry a deadline")
+
+		remaining := time.Until(deadline)
+
+		assert.LessOrEqual(t, remaining, livednstest.AttemptTimeout)
+
+		// Lower bound too: without one this passes for a
+		// deadline far shorter than intended, which would
+		// silently turn every live attempt into an instant
+		// timeout.
+		assert.Greater(t, remaining, livednstest.AttemptTimeout/2)
+
+		return nil
+	})
+}
+
+func TestRunBoundsConcurrency(t *testing.T) {
+	t.Parallel()
+
+	const workers = 24
+
+	var (
+		mu       sync.Mutex
+		wg       sync.WaitGroup
+		inFlight int
+		maxSeen  int
+	)
+
+	wg.Add(workers)
+
+	for range workers {
+		go func() {
+			defer wg.Done()
+
+			_ = livednstest.Run(func(_ context.Context) error {
+				mu.Lock()
+				inFlight++
+
+				if inFlight > maxSeen {
+					maxSeen = inFlight
+				}
+				mu.Unlock()
+
+				time.Sleep(time.Millisecond)
+
+				mu.Lock()
+				inFlight--
+				mu.Unlock()
+
+				return nil
+			})
+		}()
+	}
+
+	wg.Wait()
+
+	assert.Positive(t, maxSeen)
+	assert.LessOrEqual(
+		t, maxSeen, livednstest.Concurrency,
+		"live queries must stay under the package-wide gate",
+	)
+}

internal/logger/logger_test.go

@@ -0,0 +1,66 @@
+package logger_test
+
+import (
+	"context"
+	"log/slog"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"sneak.berlin/go/dnswatcher/internal/globals"
+	"sneak.berlin/go/dnswatcher/internal/logger"
+)
+
+func newTestLogger(t *testing.T) *logger.Logger {
+	t.Helper()
+
+	g := &globals.Globals{Appname: "dnswatcher", Version: "test"}
+
+	l, err := logger.New(nil, logger.Params{Globals: g})
+	require.NoError(t, err)
+
+	return l
+}
+
+// TestNewReturnsUsableLogger checks that the constructor yields a
+// working *slog.Logger.
+func TestNewReturnsUsableLogger(t *testing.T) {
+	t.Parallel()
+
+	l := newTestLogger(t)
+	require.NotNil(t, l.Get(), "Get must return a non-nil logger")
+}
+
+// TestDefaultLevelExcludesDebug verifies the default configuration
+// logs at info: debug records are suppressed, info records pass.
+func TestDefaultLevelExcludesDebug(t *testing.T) {
+	t.Parallel()
+
+	log := newTestLogger(t).Get()
+	ctx := context.Background()
+
+	assert.False(t, log.Enabled(ctx, slog.LevelDebug),
+		"debug must be suppressed at the default level")
+	assert.True(t, log.Enabled(ctx, slog.LevelInfo),
+		"info must be enabled at the default level")
+}
+
+// TestEnableDebugLoggingChangesLevel verifies the debug and non-debug
+// configurations differ as intended: enabling debug makes debug
+// records pass where they previously did not.
+func TestEnableDebugLoggingChangesLevel(t *testing.T) {
+	t.Parallel()
+
+	l := newTestLogger(t)
+	log := l.Get()
+	ctx := context.Background()
+
+	require.False(t, log.Enabled(ctx, slog.LevelDebug),
+		"debug must start disabled")
+
+	l.EnableDebugLogging()
+
+	assert.True(t, log.Enabled(ctx, slog.LevelDebug),
+		"debug must be enabled after EnableDebugLogging")
+}

internal/middleware/export_test.go

@@ -0,0 +1,10 @@
+package middleware
+
+import "time"
+
+// The /metrics rate limit, exported so the tests can count requests
+// against it.
+const (
+	MetricsRequestLimit                = metricsRequestLimit
+	MetricsRequestWindow time.Duration = metricsRequestWindow
+)

internal/middleware/middleware.go

@@ -11,6 +11,7 @@ import (
 	"github.com/99designs/basicauth-go"
 	"github.com/go-chi/chi/v5/middleware"
 	"github.com/go-chi/cors"
+	"github.com/go-chi/httprate"
 	"go.uber.org/fx"
 
 	"sneak.berlin/go/dnswatcher/internal/config"
@@ -21,6 +22,71 @@ import (
 // corsMaxAge is the maximum age for CORS preflight responses.
 const corsMaxAge = 300
 
+// Rate limit for /metrics: each client address may send
+// metricsRequestLimit requests per metricsRequestWindow. Every request
+// counts, so password guessing gets at most 30 tries a minute per
+// address. One Prometheus server scraping every 15 seconds sends 4
+// requests a minute, and two scraping every 5 seconds from one address
+// send 24, so normal scraping stays under the limit.
+const (
+	metricsRequestLimit  = 30
+	metricsRequestWindow = time.Minute
+)
+
+// Security response header values applied to every response.
+//
+// The CSP is as strict as the dashboard allows: the template ships no
+// JavaScript, no inline styles, no inline event handlers and no images,
+// and its only subresource is the embedded stylesheet at
+// /s/css/tailwind.min.css, which style-src 'self' permits. Neither
+// unsafe-inline nor unsafe-eval is used. frame-ancestors 'none' is the
+// primary anti-framing control; X-Frame-Options is the legacy fallback.
+const (
+	// hstsValue is emitted unconditionally, including over plain HTTP,
+	// because the service runs behind a TLS-terminating proxy and the
+	// browser must still enforce HTTPS end to end.
+	hstsValue = "max-age=31536000; includeSubDomains"
+
+	cspValue = "default-src 'self'; " +
+		"script-src 'none'; " +
+		"style-src 'self'; " +
+		"img-src 'self'; " +
+		"font-src 'none'; " +
+		"connect-src 'none'; " +
+		"object-src 'none'; " +
+		"base-uri 'none'; " +
+		"form-action 'none'; " +
+		"frame-ancestors 'none'"
+
+	frameOptionsValue = "DENY"
+
+	contentTypeOptionsValue = "nosniff"
+
+	// referrerPolicyValue is stricter than the policy minimum of
+	// strict-origin-when-cross-origin: the dashboard has no
+	// cross-origin navigation needs and its URL may name internal
+	// hosts.
+	referrerPolicyValue = "no-referrer"
+
+	permissionsPolicyValue = "accelerometer=(), " +
+		"autoplay=(), " +
+		"camera=(), " +
+		"display-capture=(), " +
+		"encrypted-media=(), " +
+		"fullscreen=(), " +
+		"geolocation=(), " +
+		"gyroscope=(), " +
+		"magnetometer=(), " +
+		"microphone=(), " +
+		"midi=(), " +
+		"payment=(), " +
+		"picture-in-picture=(), " +
+		"publickey-credentials-get=(), " +
+		"screen-wake-lock=(), " +
+		"usb=(), " +
+		"xr-spatial-tracking=()"
+)
+
 // Params contains dependencies for Middleware.
 type Params struct {
 	fx.In
@@ -169,23 +235,68 @@ func realIP(r *http.Request) string {
 	return addr
 }
 
-// CORS returns CORS middleware.
+// CORS returns middleware that lets any origin read a response. It is
+// for the public, read-only routes only, so it allows only the
+// methods those routes serve and no Authorization header.
 func (m *Middleware) CORS() func(http.Handler) http.Handler {
 	return cors.Handler(cors.Options{
 		AllowedOrigins:   []string{"*"},
-		AllowedMethods: []string{
+		AllowedMethods:   []string{"GET", "OPTIONS"},
-			"GET", "POST", "PUT", "DELETE", "OPTIONS",
+		AllowedHeaders:   []string{"Accept", "Content-Type"},
-		},
-		AllowedHeaders: []string{
-			"Accept", "Authorization",
-			"Content-Type", "X-CSRF-Token",
-		},
 		ExposedHeaders:   []string{"Link"},
 		AllowCredentials: false,
 		MaxAge:           corsMaxAge,
 	})
 }
 
+// SecurityHeaders returns middleware that sets the security response
+// headers required for production internet exposure on every response.
+//
+// The headers are set before the request reaches the next handler so
+// that they are present on every response, including panics recovered
+// by chi's Recoverer and timeouts produced by chi's Timeout.
+func (m *Middleware) SecurityHeaders() func(http.Handler) http.Handler {
+	return func(next http.Handler) http.Handler {
+		return http.HandlerFunc(func(
+			writer http.ResponseWriter,
+			request *http.Request,
+		) {
+			header := writer.Header()
+			header.Set("Strict-Transport-Security", hstsValue)
+			header.Set("Content-Security-Policy", cspValue)
+			header.Set("X-Frame-Options", frameOptionsValue)
+			header.Set(
+				"X-Content-Type-Options",
+				contentTypeOptionsValue,
+			)
+			header.Set("Referrer-Policy", referrerPolicyValue)
+			header.Set(
+				"Permissions-Policy",
+				permissionsPolicyValue,
+			)
+
+			next.ServeHTTP(writer, request)
+		})
+	}
+}
+
+// MetricsRateLimit returns middleware for /metrics that answers 429
+// Too Many Requests to a client address over the rate limit. The
+// address is the one realIP works out, so a client that is not a
+// trusted proxy cannot get a fresh allowance by sending its own
+// X-Real-IP or X-Forwarded-For. CanonicalizeIP counts all IPv6
+// addresses in one /64 as one client, since a client usually holds a
+// whole /64.
+func (m *Middleware) MetricsRateLimit() func(http.Handler) http.Handler {
+	return httprate.LimitBy(
+		metricsRequestLimit,
+		metricsRequestWindow,
+		func(request *http.Request) (string, error) {
+			return httprate.CanonicalizeIP(realIP(request)), nil
+		},
+	)
+}
+
 // MetricsAuth returns basic auth middleware for /metrics.
 func (m *Middleware) MetricsAuth() func(http.Handler) http.Handler {
 	if m.params.Config.MetricsUsername == "" {

internal/middleware/middleware_test.go

@@ -0,0 +1,478 @@
+package middleware_test
+
+import (
+	"net/http"
+	"net/http/httptest"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/go-chi/chi/v5"
+	"go.uber.org/fx/fxtest"
+
+	"sneak.berlin/go/dnswatcher/internal/config"
+	"sneak.berlin/go/dnswatcher/internal/globals"
+	"sneak.berlin/go/dnswatcher/internal/handlers"
+	"sneak.berlin/go/dnswatcher/internal/logger"
+	"sneak.berlin/go/dnswatcher/internal/middleware"
+	"sneak.berlin/go/dnswatcher/internal/notify"
+	"sneak.berlin/go/dnswatcher/internal/state"
+)
+
+// Expected security header values, spelled out literally so that any
+// change to the middleware has to be made deliberately here as well.
+const (
+	wantHSTS = "max-age=31536000; includeSubDomains"
+
+	wantCSP = "default-src 'self'; " +
+		"script-src 'none'; " +
+		"style-src 'self'; " +
+		"img-src 'self'; " +
+		"font-src 'none'; " +
+		"connect-src 'none'; " +
+		"object-src 'none'; " +
+		"base-uri 'none'; " +
+		"form-action 'none'; " +
+		"frame-ancestors 'none'"
+
+	wantFrameOptions = "DENY"
+
+	wantContentTypeOptions = "nosniff"
+
+	wantReferrerPolicy = "no-referrer"
+
+	wantPermissionsPolicy = "accelerometer=(), " +
+		"autoplay=(), " +
+		"camera=(), " +
+		"display-capture=(), " +
+		"encrypted-media=(), " +
+		"fullscreen=(), " +
+		"geolocation=(), " +
+		"gyroscope=(), " +
+		"magnetometer=(), " +
+		"microphone=(), " +
+		"midi=(), " +
+		"payment=(), " +
+		"picture-in-picture=(), " +
+		"publickey-credentials-get=(), " +
+		"screen-wake-lock=(), " +
+		"usb=(), " +
+		"xr-spatial-tracking=()"
+)
+
+// stylesheetPath is the only subresource the dashboard loads.
+const stylesheetPath = "/s/css/tailwind.min.css"
+
+// newTestLogger builds a logger for direct component construction.
+func newTestLogger(t *testing.T) *logger.Logger {
+	t.Helper()
+
+	glob, err := globals.New(nil)
+	if err != nil {
+		t.Fatalf("globals.New: %v", err)
+	}
+
+	log, err := logger.New(nil, logger.Params{Globals: glob})
+	if err != nil {
+		t.Fatalf("logger.New: %v", err)
+	}
+
+	return log
+}
+
+// newTestMiddleware builds a Middleware without an fx application.
+func newTestMiddleware(t *testing.T) *middleware.Middleware {
+	t.Helper()
+
+	glob, err := globals.New(nil)
+	if err != nil {
+		t.Fatalf("globals.New: %v", err)
+	}
+
+	mw, err := middleware.New(nil, middleware.Params{
+		Logger:  newTestLogger(t),
+		Globals: glob,
+		Config:  &config.Config{},
+	})
+	if err != nil {
+		t.Fatalf("middleware.New: %v", err)
+	}
+
+	return mw
+}
+
+// serveWithSecurityHeaders runs a GET through SecurityHeaders and
+// returns the recorded response.
+func serveWithSecurityHeaders(
+	t *testing.T,
+	target string,
+	handler http.Handler,
+) *httptest.ResponseRecorder {
+	t.Helper()
+
+	mw := newTestMiddleware(t)
+	rec := httptest.NewRecorder()
+	req := httptest.NewRequestWithContext(
+		t.Context(), http.MethodGet, target, nil,
+	)
+
+	mw.SecurityHeaders()(handler).ServeHTTP(rec, req)
+
+	return rec
+}
+
+// okHandler writes a trivial 200 response.
+func okHandler() http.Handler {
+	return http.HandlerFunc(func(
+		writer http.ResponseWriter,
+		_ *http.Request,
+	) {
+		writer.WriteHeader(http.StatusOK)
+	})
+}
+
+func TestSecurityHeaders(t *testing.T) {
+	t.Parallel()
+
+	tests := []struct {
+		name   string
+		header string
+		want   string
+	}{
+		{
+			"hsts",
+			"Strict-Transport-Security",
+			wantHSTS,
+		},
+		{
+			"csp",
+			"Content-Security-Policy",
+			wantCSP,
+		},
+		{
+			"frame options",
+			"X-Frame-Options",
+			wantFrameOptions,
+		},
+		{
+			"content type options",
+			"X-Content-Type-Options",
+			wantContentTypeOptions,
+		},
+		{
+			"referrer policy",
+			"Referrer-Policy",
+			wantReferrerPolicy,
+		},
+		{
+			"permissions policy",
+			"Permissions-Policy",
+			wantPermissionsPolicy,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			t.Parallel()
+
+			rec := serveWithSecurityHeaders(t, "/", okHandler())
+
+			got := rec.Header().Get(tt.header)
+			if got != tt.want {
+				t.Errorf(
+					"%s = %q, want %q",
+					tt.header, got, tt.want,
+				)
+			}
+		})
+	}
+}
+
+// TestSecurityHeadersCSPDirectives guards the properties the repo
+// policy requires of the content security policy itself.
+func TestSecurityHeadersCSPDirectives(t *testing.T) {
+	t.Parallel()
+
+	rec := serveWithSecurityHeaders(t, "/", okHandler())
+	csp := rec.Header().Get("Content-Security-Policy")
+
+	forbidden := []string{"unsafe-inline", "unsafe-eval"}
+	for _, directive := range forbidden {
+		if strings.Contains(csp, directive) {
+			t.Errorf("CSP must not contain %q: %q", directive, csp)
+		}
+	}
+
+	required := []string{
+		"default-src 'self'",
+		"script-src 'none'",
+		"style-src 'self'",
+		"frame-ancestors 'none'",
+	}
+	for _, directive := range required {
+		if !strings.Contains(csp, directive) {
+			t.Errorf("CSP must contain %q: %q", directive, csp)
+		}
+	}
+}
+
+// TestSecurityHeadersOnErrorResponse verifies the headers are emitted
+// even when the wrapped handler fails, since they are set before the
+// handler runs.
+func TestSecurityHeadersOnErrorResponse(t *testing.T) {
+	t.Parallel()
+
+	failing := http.HandlerFunc(func(
+		writer http.ResponseWriter,
+		_ *http.Request,
+	) {
+		http.Error(
+			writer,
+			"boom",
+			http.StatusInternalServerError,
+		)
+	})
+
+	rec := serveWithSecurityHeaders(t, "/api/v1/status", failing)
+
+	if rec.Code != http.StatusInternalServerError {
+		t.Fatalf("status = %d, want 500", rec.Code)
+	}
+
+	if got := rec.Header().Get(
+		"X-Content-Type-Options",
+	); got != wantContentTypeOptions {
+		t.Errorf(
+			"X-Content-Type-Options = %q, want %q",
+			got, wantContentTypeOptions,
+		)
+	}
+
+	if got := rec.Header().Get(
+		"Strict-Transport-Security",
+	); got != wantHSTS {
+		t.Errorf(
+			"Strict-Transport-Security = %q, want %q",
+			got, wantHSTS,
+		)
+	}
+}
+
+// newTestHandlers builds real Handlers with empty monitoring state.
+func newTestHandlers(t *testing.T) *handlers.Handlers {
+	t.Helper()
+
+	glob, err := globals.New(nil)
+	if err != nil {
+		t.Fatalf("globals.New: %v", err)
+	}
+
+	log := newTestLogger(t)
+
+	notifier, err := notify.New(fxtest.NewLifecycle(t), notify.Params{
+		Logger: log,
+		Config: &config.Config{},
+	})
+	if err != nil {
+		t.Fatalf("notify.New: %v", err)
+	}
+
+	st, err := state.New(fxtest.NewLifecycle(t), state.Params{
+		Logger: log,
+		Config: &config.Config{DataDir: t.TempDir()},
+	})
+	if err != nil {
+		t.Fatalf("state.New: %v", err)
+	}
+
+	hnd, err := handlers.New(nil, handlers.Params{
+		Logger:  log,
+		Globals: glob,
+		State:   st,
+		Notify:  notifier,
+	})
+	if err != nil {
+		t.Fatalf("handlers.New: %v", err)
+	}
+
+	return hnd
+}
+
+// TestDashboardRendersWithSecurityHeaders renders the real dashboard
+// through the middleware and checks that the policy still permits the
+// one stylesheet the page loads.
+func TestDashboardRendersWithSecurityHeaders(t *testing.T) {
+	t.Parallel()
+
+	mw := newTestMiddleware(t)
+	hnd := newTestHandlers(t)
+
+	router := chi.NewRouter()
+	router.Use(mw.SecurityHeaders())
+	router.Get("/", hnd.HandleDashboard())
+
+	rec := httptest.NewRecorder()
+	req := httptest.NewRequestWithContext(
+		t.Context(), http.MethodGet, "/", nil,
+	)
+
+	router.ServeHTTP(rec, req)
+
+	if rec.Code != http.StatusOK {
+		t.Fatalf("status = %d, want 200", rec.Code)
+	}
+
+	body := rec.Body.String()
+	if !strings.Contains(body, stylesheetPath) {
+		t.Errorf("dashboard does not reference %q", stylesheetPath)
+	}
+
+	if !strings.Contains(body, "dnswatcher") {
+		t.Errorf("dashboard body looks empty: %d bytes", len(body))
+	}
+
+	csp := rec.Header().Get("Content-Security-Policy")
+	if csp != wantCSP {
+		t.Errorf("CSP = %q, want %q", csp, wantCSP)
+	}
+
+	// The stylesheet is same-origin, so style-src 'self' allows it.
+	if !strings.Contains(csp, "style-src 'self'") {
+		t.Errorf("CSP would block %q: %q", stylesheetPath, csp)
+	}
+}
+
+// Addresses for the rate limit tests: a client connecting directly, a
+// trusted proxy, and a client behind that proxy as its X-Real-IP
+// header names it.
+const (
+	directClient  = "198.51.100.1:4000"
+	trustedProxy  = "10.0.0.1:4000"
+	proxiedClient = "203.0.113.1"
+)
+
+// statusFrom sends a GET through handler as if from remoteAddr, with
+// an X-Real-IP header when xRealIP is not empty, and returns the
+// response status.
+func statusFrom(
+	t *testing.T,
+	handler http.Handler,
+	remoteAddr string,
+	xRealIP string,
+) int {
+	t.Helper()
+
+	req := httptest.NewRequestWithContext(
+		t.Context(), http.MethodGet, "/metrics", nil,
+	)
+	req.RemoteAddr = remoteAddr
+
+	if xRealIP != "" {
+		req.Header.Set("X-Real-IP", xRealIP)
+	}
+
+	rec := httptest.NewRecorder()
+	handler.ServeHTTP(rec, req)
+
+	return rec.Code
+}
+
+// TestMetricsRateLimitAllowsScraping checks that one address can send,
+// within one window, what two Prometheus servers scraping every 5
+// seconds send in that time, without being turned away.
+func TestMetricsRateLimitAllowsScraping(t *testing.T) {
+	t.Parallel()
+
+	const scrapeInterval = 5 * time.Second
+
+	scrapes := 2 * int(middleware.MetricsRequestWindow/scrapeInterval)
+	limited := newTestMiddleware(t).MetricsRateLimit()(okHandler())
+
+	for i := range scrapes {
+		got := statusFrom(t, limited, directClient, "")
+		if got != http.StatusOK {
+			t.Fatalf(
+				"scrape %d of %d: status = %d, want 200",
+				i+1, scrapes, got,
+			)
+		}
+	}
+}
+
+// TestMetricsRateLimitKeysOnClientAddress checks which requests share
+// an allowance. Each case uses up the allowance of one client, then
+// sends one more request.
+func TestMetricsRateLimitKeysOnClientAddress(t *testing.T) {
+	t.Parallel()
+
+	tests := []struct {
+		name           string
+		usedRemoteAddr string
+		usedXRealIP    string
+		nextRemoteAddr string
+		nextXRealIP    string
+		want           int
+	}{
+		{
+			"same address",
+			directClient, "",
+			directClient, "",
+			http.StatusTooManyRequests,
+		},
+		{
+			"another address",
+			directClient, "",
+			"198.51.100.2:4000", "",
+			http.StatusOK,
+		},
+		{
+			"own X-Real-IP from an untrusted address",
+			directClient, "",
+			directClient, "203.0.113.9",
+			http.StatusTooManyRequests,
+		},
+		{
+			"same client behind the proxy",
+			trustedProxy, proxiedClient,
+			trustedProxy, proxiedClient,
+			http.StatusTooManyRequests,
+		},
+		{
+			"another client behind the proxy",
+			trustedProxy, proxiedClient,
+			trustedProxy, "203.0.113.2",
+			http.StatusOK,
+		},
+		{
+			"same IPv6 /64",
+			"[2001:db8::1]:4000", "",
+			"[2001:db8::2]:4000", "",
+			http.StatusTooManyRequests,
+		},
+		{
+			"another IPv6 /64",
+			"[2001:db8::1]:4000", "",
+			"[2001:db8:0:1::1]:4000", "",
+			http.StatusOK,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			t.Parallel()
+
+			limited := newTestMiddleware(t).MetricsRateLimit()(okHandler())
+
+			for range middleware.MetricsRequestLimit {
+				statusFrom(t, limited, tt.usedRemoteAddr, tt.usedXRealIP)
+			}
+
+			got := statusFrom(
+				t, limited, tt.nextRemoteAddr, tt.nextXRealIP,
+			)
+			if got != tt.want {
+				t.Errorf("status = %d, want %d", got, tt.want)
+			}
+		})
+	}
+}

internal/notify/shutdown_test.go

@@ -33,10 +33,29 @@ const (
 	// out.
 	drainDeadline = 50 * time.Millisecond
 
-	// drainSlack is the upper bound on how long a bounded
+	// timeoutDrainBound is how long a drain given drainDeadline
-	// drain may take; generous enough for a loaded CI box,
+	// may take to return before the test gives up on it.  At
-	// still far below the 20s test ceiling.
+	// forty times drainDeadline it leaves ample room for
-	drainSlack = 2 * time.Second
+	// scheduling delay on a loaded box under -race, yet it is far
+	// below the test binary's -timeout, so a drain that its
+	// deadline does not bound fails that one test instead of
+	// hanging the package.
+	timeoutDrainBound = 2 * time.Second
+
+	// longDrainDeadline is the deadline given to a drain that is
+	// expected to finish well before it: when the in-flight
+	// delivery completes after inFlightHold, or at once when
+	// nothing is in flight.  It is far above inFlightHold, so
+	// those drains never reach it, and four times
+	// idleDrainBound, so an idle drain that waited for its
+	// deadline instead of returning fails that bound.
+	longDrainDeadline = 2 * time.Second
+
+	// reachEndpointTimeout is how long a submitted delivery may
+	// take to reach the test server.  That normally takes a few
+	// milliseconds; the margin is for a loaded box under -race,
+	// and only a failing run ever waits this long.
+	reachEndpointTimeout = 2 * time.Second
 
 	// settleDelay is how long to wait before asserting that
 	// something did *not* happen.
@@ -46,10 +65,11 @@ const (
 	// nothing in flight.  It is deliberately far above the cost
 	// of the goroutine hop through inFlight.Wait() — which
 	// reached 57ms on a loaded box under -race with the package's
-	// parallel tests — and far below drainSlack, the deadline
+	// parallel tests — and far below longDrainDeadline, the
-	// such a drain is given.  A drain that blocked until its
+	// deadline such a drain is given.  A drain that blocked until
-	// deadline instead of returning on the WaitGroup therefore
+	// its deadline instead of returning on the WaitGroup
-	// still fails this bound, but scheduling delay alone cannot.
+	// therefore still fails this bound, but scheduling delay
+	// alone cannot.
 	idleDrainBound = 500 * time.Millisecond
 )
 
@@ -75,12 +95,14 @@ func (sb *syncBuffer) String() string {
 }
 
 // newLoggingService returns a Service writing JSON logs into
-// the returned buffer.
+// the returned buffer, debug level included.
 func newLoggingService(
 	transport http.RoundTripper,
 ) (*notify.Service, *syncBuffer) {
 	logs := &syncBuffer{}
-	handler := slog.NewJSONHandler(logs, nil)
+	handler := slog.NewJSONHandler(
+		logs, &slog.HandlerOptions{Level: slog.LevelDebug},
+	)
 
 	return notify.NewTestServiceWithLogger(transport, handler),
 		logs
@@ -134,7 +156,7 @@ func TestDrainWaitsForInFlightDelivery(t *testing.T) {
 	// drain begins.
 	select {
 	case <-entered:
-	case <-time.After(drainSlack):
+	case <-time.After(reachEndpointTimeout):
 		t.Fatal("delivery never reached the endpoint")
 	}
 
@@ -151,7 +173,7 @@ func TestDrainWaitsForInFlightDelivery(t *testing.T) {
 	defer timer.Stop()
 
 	ctx, cancel := context.WithTimeout(
-		context.Background(), drainSlack,
+		context.Background(), longDrainDeadline,
 	)
 	defer cancel()
 
@@ -257,11 +279,11 @@ func TestDrainBoundedByContextDeadline(t *testing.T) {
 
 	select {
 	case <-returned:
-	case <-time.After(drainSlack):
+	case <-time.After(timeoutDrainBound):
 		t.Fatalf(
 			"drain did not return within %v; its %v deadline "+
 				"did not bound it",
-			drainSlack, drainDeadline,
+			timeoutDrainBound, drainDeadline,
 		)
 	}
 
@@ -333,7 +355,7 @@ func TestDrainRefusesNewDeliveries(t *testing.T) {
 	svc.SetMattermostWebhookURL(target)
 
 	ctx, cancel := context.WithTimeout(
-		context.Background(), drainSlack,
+		context.Background(), longDrainDeadline,
 	)
 	defer cancel()
 
@@ -446,7 +468,7 @@ func TestNewRegistersDrainingStopHook(t *testing.T) {
 
 	select {
 	case <-entered:
-	case <-time.After(drainSlack):
+	case <-time.After(reachEndpointTimeout):
 		t.Fatal("delivery never reached the endpoint")
 	}
 
@@ -456,7 +478,7 @@ func TestNewRegistersDrainingStopHook(t *testing.T) {
 	defer timer.Stop()
 
 	ctx, cancel := context.WithTimeout(
-		context.Background(), drainSlack,
+		context.Background(), longDrainDeadline,
 	)
 	defer cancel()
 
@@ -487,7 +509,7 @@ func TestDrainWithoutDeliveriesReturnsImmediately(t *testing.T) {
 	start := time.Now()
 
 	ctx, cancel := context.WithTimeout(
-		context.Background(), drainSlack,
+		context.Background(), longDrainDeadline,
 	)
 	defer cancel()
 
@@ -495,9 +517,9 @@ func TestDrainWithoutDeliveriesReturnsImmediately(t *testing.T) {
 
 	if elapsed := time.Since(start); elapsed > idleDrainBound {
 		t.Errorf(
-			"drain of an idle service took %v, want well "+
+			"drain of an idle service took %v, want at most "+
-				"under its %v deadline",
+				"%v; its deadline was %v",
-			elapsed, drainSlack,
+			elapsed, idleDrainBound, longDrainDeadline,
 		)
 	}
 }
@@ -505,10 +527,11 @@ func TestDrainWithoutDeliveriesReturnsImmediately(t *testing.T) {
 // TestDrainWithCancelledContextDoesNotWarn verifies that an
 // OnStop context that is already dead on entry does not produce
 // an "abandoning them" warning when there was nothing in flight
-// to abandon.  The expired context wins the select immediately,
+// to abandon, and that the drain returns and says at debug level
-// so only the outstanding count can tell the difference between
+// that nothing was in flight.  The expired context wins the
-// a genuine timeout and a shutdown that had simply already run
+// select immediately, so only the outstanding count can tell the
-// out of time with no work left.
+// difference between a genuine timeout and a shutdown that had
+// simply already run out of time with no work left.
 func TestDrainWithCancelledContextDoesNotWarn(t *testing.T) {
 	t.Parallel()
 
@@ -517,11 +540,43 @@ func TestDrainWithCancelledContextDoesNotWarn(t *testing.T) {
 	ctx, cancel := context.WithCancel(context.Background())
 	cancel()
 
-	svc.Drain(ctx)
+	// A watchdog, as in TestDrainBoundedByContextDeadline, so
+	// that a drain which never returns fails here instead of
+	// hanging the package.
+	returned := make(chan struct{})
 
-	if output := logs.String(); strings.Contains(
+	go func() {
-		output, `"level":"WARN"`,
+		defer close(returned)
+
+		svc.Drain(ctx)
+	}()
+
+	select {
+	case <-returned:
+	case <-time.After(idleDrainBound):
+		t.Fatalf(
+			"drain with nothing in flight and a cancelled "+
+				"context did not return within %v",
+			idleDrainBound,
+		)
+	}
+
+	output := logs.String()
+
+	// The absence of a warning alone would also pass if the drain
+	// logged nothing at all, so require the debug line it writes
+	// when it finds nothing outstanding.
+	if !strings.Contains(
+		output, "all in-flight notifications completed",
 	) {
+		t.Errorf(
+			"drain did not log that nothing was in flight; "+
+				"log output: %s",
+			output,
+		)
+	}
+
+	if strings.Contains(output, `"level":"WARN"`) {
 		t.Errorf(
 			"drain with nothing in flight warned about "+
 				"abandoned deliveries; log output: %s",

internal/resolver/dns_client.go

@@ -7,8 +7,8 @@ import (
 	"github.com/miekg/dns"
 )
 
-// DNSClient abstracts DNS wire-protocol exchanges so the resolver
+// DNSClient sends one DNS message to a nameserver and returns the
-// can be tested without hitting real nameservers.
+// reply. The resolver holds one for UDP and one for TCP.
 type DNSClient interface {
 	ExchangeContext(
 		ctx context.Context,

internal/resolver/export_test.go

@@ -0,0 +1,8 @@
+package resolver
+
+import "github.com/miekg/dns"
+
+// ExtractRecordValue exports extractRecordValue for testing.
+func ExtractRecordValue(rr dns.RR) string {
+	return extractRecordValue(rr)
+}

internal/resolver/iterative.go

@@ -608,7 +608,10 @@ func classifyResponse(resp *NameserverResponse, state queryState) {
 	}
 }
 
-// extractRecordValue formats a DNS RR value as a string.
+// extractRecordValue formats a DNS RR value as a string. DNS names
+// are case-insensitive and nameservers may answer in any letter case,
+// so names are lower-cased to compare equal. TXT and CAA values keep
+// their letter case.
 func extractRecordValue(rr dns.RR) string {
 	switch r := rr.(type) {
 	case *dns.A:
@@ -616,22 +619,22 @@ func extractRecordValue(rr dns.RR) string {
 	case *dns.AAAA:
 		return r.AAAA.String()
 	case *dns.CNAME:
-		return r.Target
+		return strings.ToLower(r.Target)
 	case *dns.MX:
-		return fmt.Sprintf("%d %s", r.Preference, r.Mx)
+		return fmt.Sprintf("%d %s", r.Preference, strings.ToLower(r.Mx))
 	case *dns.TXT:
 		return strings.Join(r.Txt, "")
 	case *dns.SRV:
 		return fmt.Sprintf(
 			"%d %d %d %s",
-			r.Priority, r.Weight, r.Port, r.Target,
+			r.Priority, r.Weight, r.Port, strings.ToLower(r.Target),
 		)
 	case *dns.CAA:
 		return fmt.Sprintf(
 			"%d %s \"%s\"", r.Flag, r.Tag, r.Value,
 		)
 	case *dns.NS:
-		return r.Ns
+		return strings.ToLower(r.Ns)
 	default:
 		return ""
 	}

internal/resolver/iterative_test.go

@@ -0,0 +1,62 @@
+package resolver_test
+
+import (
+	"testing"
+
+	"github.com/miekg/dns"
+	"github.com/stretchr/testify/assert"
+
+	"sneak.berlin/go/dnswatcher/internal/resolver"
+)
+
+func TestExtractRecordValue_LetterCase(t *testing.T) {
+	t.Parallel()
+
+	tests := []struct {
+		name string
+		rr   dns.RR
+		want string
+	}{
+		{
+			name: "MX target lower-cased",
+			rr:   &dns.MX{Preference: 1, Mx: "ASPMX.L.GOOGLE.COM."},
+			want: "1 aspmx.l.google.com.",
+		},
+		{
+			name: "NS target lower-cased",
+			rr:   &dns.NS{Ns: "x.ns.joker.COM."},
+			want: "x.ns.joker.com.",
+		},
+		{
+			name: "CNAME target lower-cased",
+			rr:   &dns.CNAME{Target: "WWW.Example.Com."},
+			want: "www.example.com.",
+		},
+		{
+			name: "SRV target lower-cased",
+			rr: &dns.SRV{
+				Priority: 10, Weight: 5, Port: 443,
+				Target: "SIP.Example.Com.",
+			},
+			want: "10 5 443 sip.example.com.",
+		},
+		{
+			name: "TXT value keeps its case",
+			rr:   &dns.TXT{Txt: []string{"Verify=AbC123"}},
+			want: "Verify=AbC123",
+		},
+		{
+			name: "CAA value keeps its case",
+			rr:   &dns.CAA{Flag: 0, Tag: "issue", Value: "LetsEncrypt.org"},
+			want: `0 issue "LetsEncrypt.org"`,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			t.Parallel()
+
+			assert.Equal(t, tt.want, resolver.ExtractRecordValue(tt.rr))
+		})
+	}
+}

internal/resolver/livedns_harness_test.go

@@ -1,10 +1,7 @@
 package resolver_test
 
 import (
-	"context"
-	"sync"
 	"testing"
-	"time"
 
 	"github.com/stretchr/testify/assert"
 
@@ -12,9 +9,8 @@ import (
 )
 
 // Tests for the live-DNS harness in livedns_test.go itself. These
-// exercise pure logic and the retry/concurrency plumbing; they
+// exercise pure logic; they perform no DNS resolution of any kind, so
-// perform no DNS resolution of any kind, so they neither mock DNS
+// they neither mock DNS nor depend on it.
-// nor depend on it.
 
 // Names for the synthetic status maps below. Nothing is ever queried
 // at them: they are map keys handed to the package's pure counting
@@ -90,47 +86,6 @@ func TestStatusCountingIgnoresSilentNameservers(t *testing.T) {
 	)
 }
 
-func TestRetryLiveRecoversFromTransientFailure(t *testing.T) {
-	t.Parallel()
-
-	const wantAttempts = 2
-
-	attempts := 0
-
-	retryLive(t, "transient", func(_ context.Context) error {
-		attempts++
-
-		if attempts < wantAttempts {
-			return errLiveNoAnswer
-		}
-
-		return nil
-	})
-
-	assert.Equal(t, wantAttempts, attempts)
-}
-
-func TestRetryLiveGivesEachAttemptADeadline(t *testing.T) {
-	t.Parallel()
-
-	retryLive(t, "deadline", func(ctx context.Context) error {
-		deadline, ok := ctx.Deadline()
-		assert.True(t, ok, "attempt should carry a deadline")
-
-		remaining := time.Until(deadline)
-
-		assert.LessOrEqual(t, remaining, liveAttemptTimeout)
-
-		// Lower bound too: without one this passes for a
-		// deadline far shorter than intended, which would
-		// silently turn every live attempt into an instant
-		// timeout.
-		assert.Greater(t, remaining, liveAttemptTimeout/2)
-
-		return nil
-	})
-}
-
 // TestUnsanctionedStatusesRejectsWrongAnswers is the regression test
 // for the defect this allowlist exists to prevent: a minority of
 // nameservers answering WRONGLY while quorum keeps the suite green.
@@ -235,50 +190,3 @@ func TestUnsanctionedStatusesToleratesSilenceOnly(t *testing.T) {
 		unsanctionedStatuses(results, allowed...),
 	)
 }
-
-func TestRunLiveBoundsConcurrency(t *testing.T) {
-	t.Parallel()
-
-	const workers = 24
-
-	var (
-		mu       sync.Mutex
-		wg       sync.WaitGroup
-		inFlight int
-		maxSeen  int
-	)
-
-	wg.Add(workers)
-
-	for range workers {
-		go func() {
-			defer wg.Done()
-
-			_ = runLive(func(_ context.Context) error {
-				mu.Lock()
-				inFlight++
-
-				if inFlight > maxSeen {
-					maxSeen = inFlight
-				}
-				mu.Unlock()
-
-				time.Sleep(time.Millisecond)
-
-				mu.Lock()
-				inFlight--
-				mu.Unlock()
-
-				return nil
-			})
-		}()
-	}
-
-	wg.Wait()
-
-	assert.Positive(t, maxSeen)
-	assert.LessOrEqual(
-		t, maxSeen, liveConcurrency,
-		"live queries must stay under the package-wide gate",
-	)
-}

internal/resolver/livedns_test.go

@@ -8,8 +8,8 @@ import (
 	"sort"
 	"strings"
 	"testing"
-	"time"
 
+	"sneak.berlin/go/dnswatcher/internal/livednstest"
 	"sneak.berlin/go/dnswatcher/internal/resolver"
 )
 
@@ -17,144 +17,34 @@ import (
 // Live DNS test support
 // ----------------------------------------------------------------
 //
-// Every test in this package resolves against the real, live DNS —
+// Tests that look something up in DNS query live DNS servers, never a
-// see TESTING.md. Nothing here mocks, fakes, stubs, records or
+// stand-in; logic that works on record data may be tested on that
-// replays DNS, and nothing here skips or gates a test: the helpers
+// data with no lookup (see TESTING.md). Each live operation below goes
-// below only change *how* the live queries are issued, so that a
+// through livednstest.Retry, which bounds how many resolutions are in
-// single dropped UDP packet or one slow authoritative server does
+// flight at once and retries an operation that got no answer (see
-// not turn a correct resolver into a red build.
+// package livednstest).
 //
-// Three mechanisms, all test-side:
+// Where an assertion spans several independent nameservers, a quorum
+// is enough: a strict majority answering as expected. A server that
+// fails to answer is tolerated, while a server that answers *wrongly*
+// still fails the test.
 //
-//  1. Bounded concurrency. The package's tests are parallel and the
+// That tolerance is expressed as an ALLOWLIST of sanctioned statuses,
-//     build hosts have many cores, so without a limit every test
+// never as a blocklist of known-bad ones. A blocklist bans the one
-//     starts its own iterative resolution at the same instant and
+// wrong answer its author thought of and silently admits every other
-//     they all hit the first root server in rootServerList() within
+// status, including any added to the resolver later; an allowlist
-//     a few milliseconds of each other. Root servers rate-limit
+// fails on anything nobody explicitly sanctioned. Silence (timeout,
-//     that, which shows up as a different arbitrary subset of tests
+// error) is the only thing quorum exists to tolerate. A *wrong
-//     failing on each run. liveGate caps how many resolutions are
+// answer* — nxdomain for a name that exists, ok for one that does
-//     in flight at once.
+// not, nodata for either — is never tolerated at any count.
-//
-//  2. Retry with exponential backoff. Each live operation gets
-//     several attempts with its own timeout. The retry predicate is
-//     strictly transport-level — "did a nameserver answer at all" —
-//     never the assertion the test is making. A resolver that
-//     answers incorrectly still fails on the first attempt.
-//
-//  3. Quorum. Where an assertion spans several independent
-//     nameservers, a strict majority answering as expected is
-//     enough; a server that fails to answer is tolerated, while a
-//     server that answers *wrongly* still fails the test.
-//
-// The tolerance in (3) is expressed as an ALLOWLIST of sanctioned
-// statuses, never as a blocklist of known-bad ones. A blocklist bans
-// the one wrong answer its author thought of and silently admits
-// every other status, including any added to the resolver later; an
-// allowlist fails on anything nobody explicitly sanctioned. Silence
-// (timeout, error) is the only thing quorum exists to tolerate. A
-// *wrong answer* — nxdomain for a name that exists, ok for one that
-// does not, nodata for either — is never tolerated at any count.
 
-const (
+// minNameservers is the smallest nameserver count a well-run zone is
-	// liveAttempts is how many times a live DNS operation is
+// expected to publish.
-	// attempted before the test fails.
+const minNameservers = 2
-	liveAttempts = 3
 
-	// liveAttemptTimeout bounds one attempt. Worst case for an
+// errLiveNoQuorum reports that too few of a domain's nameservers
-	// operation is liveAttempts * liveAttemptTimeout plus the
+// answered for a quorum assertion to be made.
-	// backoff — about 26 seconds, well inside the 90-second
+var errLiveNoQuorum = errors.New("no nameserver quorum")
-	// `go test -timeout` backstop even when several operations
-	// exhaust their attempts.
-	liveAttemptTimeout = 8 * time.Second
-
-	// liveBackoffBase is the delay after the first failed
-	// attempt; it is multiplied by liveBackoffFactor each time.
-	liveBackoffBase = 500 * time.Millisecond
-
-	// liveBackoffFactor is the exponential backoff multiplier.
-	liveBackoffFactor = 2
-
-	// liveConcurrency caps how many live resolutions may be in
-	// flight across the whole package at once.
-	liveConcurrency = 6
-
-	// minNameservers is the smallest nameserver count a
-	// well-run zone is expected to publish.
-	minNameservers = 2
-)
-
-// liveGate bounds concurrent live resolutions package-wide. It has
-// to be package scoped: the whole point is that it is shared by
-// every parallel test in the package.
-//
-//nolint:gochecknoglobals // package-wide live query rate limit
-var liveGate = make(chan struct{}, liveConcurrency)
-
-var (
-	// errLiveNoAnswer reports that a live operation produced no
-	// usable answer, which is retried rather than asserted on.
-	errLiveNoAnswer = errors.New("no answer from live DNS")
-
-	// errLiveNoQuorum reports that too few of a domain's
-	// nameservers answered for a quorum assertion to be made.
-	errLiveNoQuorum = errors.New("no nameserver quorum")
-)
-
-// runLive executes one attempt of a live operation, holding a slot
-// in liveGate for its duration and bounding it with its own
-// timeout.
-func runLive(op func(ctx context.Context) error) error {
-	liveGate <- struct{}{}
-	defer func() { <-liveGate }()
-
-	ctx, cancel := context.WithTimeout(
-		context.Background(), liveAttemptTimeout,
-	)
-	defer cancel()
-
-	return op(ctx)
-}
-
-// retryLive runs op until it reports success, retrying transport
-// failures with exponential backoff, and fails the test if every
-// attempt fails. op returns an error only for a failure to obtain
-// an answer — never for an answer the test disagrees with, which
-// belongs in an assertion so that it fails immediately. op stores
-// whatever it obtained where its caller can find it.
-func retryLive(
-	t *testing.T,
-	what string,
-	op func(ctx context.Context) error,
-) {
-	t.Helper()
-
-	var last error
-
-	backoff := liveBackoffBase
-
-	for attempt := range liveAttempts {
-		if attempt > 0 {
-			t.Logf(
-				"%s: attempt %d of %d failed (%v), "+
-					"retrying in %s",
-				what, attempt, liveAttempts, last, backoff,
-			)
-			time.Sleep(backoff)
-
-			backoff *= liveBackoffFactor
-		}
-
-		last = runLive(op)
-		if last == nil {
-			return
-		}
-	}
-
-	t.Fatalf(
-		"%s: no answer after %d live attempts: %v",
-		what, liveAttempts, last,
-	)
-}
 
 // liveQuorum is how many of total nameservers must agree for a
 // multi-nameserver assertion to hold: a strict majority.
@@ -272,7 +162,7 @@ func liveFindAuthoritative(
 
 	var out []string
 
-	retryLive(
+	livednstest.Retry(
 		t,
 		"FindAuthoritativeNameservers("+domain+")",
 		func(ctx context.Context) error {
@@ -284,7 +174,7 @@ func liveFindAuthoritative(
 			if len(ns) == 0 {
 				return fmt.Errorf(
 					"%w: %s has no nameservers",
-					errLiveNoAnswer, domain,
+					livednstest.ErrNoAnswer, domain,
 				)
 			}
 
@@ -308,7 +198,7 @@ func liveLookupNS(
 
 	var out []string
 
-	retryLive(
+	livednstest.Retry(
 		t,
 		"LookupNS("+domain+")",
 		func(ctx context.Context) error {
@@ -320,7 +210,7 @@ func liveLookupNS(
 			if len(ns) == 0 {
 				return fmt.Errorf(
 					"%w: %s has no nameservers",
-					errLiveNoAnswer, domain,
+					livednstest.ErrNoAnswer, domain,
 				)
 			}
 
@@ -350,7 +240,7 @@ func liveQueryNameserver(
 
 	var out *resolver.NameserverResponse
 
-	retryLive(
+	livednstest.Retry(
 		t,
 		what,
 		func(ctx context.Context) error {
@@ -365,7 +255,7 @@ func liveQueryNameserver(
 				resp.Status == resolver.StatusError {
 				return fmt.Errorf(
 					"%w: %s returned %s: %s",
-					errLiveNoAnswer, nameserver,
+					livednstest.ErrNoAnswer, nameserver,
 					resp.Status, resp.Error,
 				)
 			}
@@ -392,7 +282,7 @@ func liveQueryAllNameservers(
 
 	var out map[string]*resolver.NameserverResponse
 
-	retryLive(
+	livednstest.Retry(
 		t,
 		"QueryAllNameservers("+hostname+")",
 		func(ctx context.Context) error {
@@ -404,7 +294,7 @@ func liveQueryAllNameservers(
 			if len(results) == 0 {
 				return fmt.Errorf(
 					"%w: no nameservers queried for %s",
-					errLiveNoAnswer, hostname,
+					livednstest.ErrNoAnswer, hostname,
 				)
 			}
 
@@ -437,7 +327,7 @@ func liveResolveIPs(
 
 	var out []string
 
-	retryLive(
+	livednstest.Retry(
 		t,
 		"ResolveIPAddresses("+hostname+")",
 		func(ctx context.Context) error {
@@ -449,7 +339,7 @@ func liveResolveIPs(
 			if len(ips) == 0 {
 				return fmt.Errorf(
 					"%w: no addresses for %s",
-					errLiveNoAnswer, hostname,
+					livednstest.ErrNoAnswer, hostname,
 				)
 			}
 
@@ -476,7 +366,7 @@ func liveResolveIPsAllowingEmpty(
 
 	var out []string
 
-	retryLive(
+	livednstest.Retry(
 		t,
 		"ResolveIPAddresses("+hostname+")",
 		func(ctx context.Context) error {

internal/resolver/resolver.go

@@ -67,17 +67,4 @@ func NewFromLogger(log *slog.Logger) *Resolver {
 	}
 }
 
-// NewFromLoggerWithClient creates a Resolver with a custom DNS
-// client, useful for testing with mock DNS responses.
-func NewFromLoggerWithClient(
-	log *slog.Logger,
-	client DNSClient,
-) *Resolver {
-	return &Resolver{
-		log:    log,
-		client: client,
-		tcp:    client,
-	}
-}
-
 // Method implementations are in iterative.go.

internal/resolver/resolver_test.go

@@ -10,7 +10,6 @@ import (
 	"testing"
 	"time"
 
-	"github.com/miekg/dns"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 
@@ -33,8 +32,8 @@ func newTestResolver(t *testing.T) *resolver.Resolver {
 }
 
 // findOneNSForDomain picks one authoritative nameserver to aim a
-// test at. Live-DNS retry, concurrency and quorum handling live in
+// test at. Quorum handling lives in livedns_test.go, and the live-DNS
-// livedns_test.go.
+// retry and concurrency limit in package livednstest.
 func findOneNSForDomain(
 	t *testing.T,
 	r *resolver.Resolver,
@@ -526,21 +525,18 @@ func TestQueryAllNameservers_ContextCanceled(t *testing.T) {
 func TestQueryNameserverIP_Timeout(t *testing.T) {
 	t.Parallel()
 
-	log := slog.New(slog.NewTextHandler(
+	r := newTestResolver(t)
-		os.Stderr,
-		&slog.HandlerOptions{Level: slog.LevelDebug},
-	))
-
-	r := resolver.NewFromLoggerWithClient(
-		log, &timeoutClient{},
-	)
 
+	// Nothing answers at 192.0.2.1, a documentation address. The
+	// resolver tries each query twice, and the first try gives up
+	// after two seconds. A deadline that ends during the first try
+	// makes the status vary from run to run between nodata and
+	// timeout, so the deadline must outlast the first try.
 	ctx, cancel := context.WithTimeout(
-		context.Background(), 10*time.Second,
+		context.Background(), 3*time.Second,
 	)
 	t.Cleanup(cancel)
 
-	// Query any IP — the client always returns a timeout error.
 	resp, err := r.QueryNameserverIP(
 		ctx, "unreachable.test.", "192.0.2.1",
 		"example.com",
@@ -551,27 +547,6 @@ func TestQueryNameserverIP_Timeout(t *testing.T) {
 	assert.NotEmpty(t, resp.Error)
 }
 
-// timeoutClient simulates DNS timeout errors for testing.
-type timeoutClient struct{}
-
-func (c *timeoutClient) ExchangeContext(
-	_ context.Context,
-	_ *dns.Msg,
-	_ string,
-) (*dns.Msg, time.Duration, error) {
-	return nil, 0, &net.OpError{
-		Op:  "read",
-		Net: "udp",
-		Err: &timeoutError{},
-	}
-}
-
-type timeoutError struct{}
-
-func (e *timeoutError) Error() string   { return "i/o timeout" }
-func (e *timeoutError) Timeout() bool   { return true }
-func (e *timeoutError) Temporary() bool { return true }
-
 func TestResolveIPAddresses_ContextCanceled(t *testing.T) {
 	t.Parallel()
 

internal/server/export_test.go

@@ -5,15 +5,20 @@ import (
 	"time"
 )
 
-// NewHTTPServer exports newHTTPServer for testing.
-func NewHTTPServer(
-	listenAddr string,
-	handler http.Handler,
-) *http.Server {
-	return newHTTPServer(listenAddr, handler)
-}
-
 // RequestTimeout exports the handler execution budget applied by
 // chimw.Timeout in SetupRoutes, so tests can assert the relationship
 // between it and the server's WriteTimeout.
 const RequestTimeout time.Duration = requestTimeout
+
+// SetListenPort overrides the port Run binds. A test uses it to hand
+// Run an unbindable port so ListenAndServe fails immediately and Run
+// returns after storing its http.Server.
+func SetListenPort(s *Server, port int) {
+	s.port = port
+}
+
+// HTTPServerOf returns the http.Server that Run built and stored, so a
+// test can inspect the timeouts the running server actually carries.
+func HTTPServerOf(s *Server) *http.Server {
+	return s.httpServer
+}

internal/server/routes.go

@@ -21,15 +21,22 @@ func (s *Server) SetupRoutes() {
 	// Global middleware
 	s.router.Use(chimw.Recoverer)
 	s.router.Use(chimw.RequestID)
+	s.router.Use(s.mw.SecurityHeaders())
 	s.router.Use(s.mw.Logging())
-	s.router.Use(s.mw.CORS())
 	s.router.Use(chimw.Timeout(requestTimeout))
 
+	// Public, unauthenticated, read-only routes, the only ones
+	// REPO_POLICIES.md allows wildcard CORS on. CORS is middleware of
+	// this whole router, not of a Group, so that it also answers
+	// OPTIONS preflight requests, which no route here registers.
+	public := chi.NewRouter()
+	public.Use(s.mw.CORS())
+
 	// Dashboard (read-only web UI)
-	s.router.Get("/", s.handlers.HandleDashboard())
+	public.Get("/", s.handlers.HandleDashboard())
 
 	// Static assets (embedded CSS/JS)
-	s.router.Mount(
+	public.Mount(
 		"/s",
 		http.StripPrefix(
 			"/s",
@@ -38,27 +45,33 @@ func (s *Server) SetupRoutes() {
 	)
 
 	// Health check (standard well-known path)
-	s.router.Get(
+	public.Get(
 		"/.well-known/healthcheck",
 		s.handlers.HandleHealthCheck(),
 	)
 
 	// Legacy health check (keep for backward compatibility)
-	s.router.Get("/health", s.handlers.HandleHealthCheck())
+	public.Get("/health", s.handlers.HandleHealthCheck())
 
 	// API v1 routes
-	s.router.Route("/api/v1", func(r chi.Router) {
+	public.Route("/api/v1", func(r chi.Router) {
 		r.Get("/status", s.handlers.HandleStatus())
 	})
 
-	// Metrics endpoint (optional, with basic auth)
+	s.router.Mount("/", public)
+
+	// Metrics endpoint (optional, with basic auth) and no CORS: a
+	// Prometheus scraper is not a browser. It is mounted rather than
+	// added with Get so that every method on /metrics, OPTIONS
+	// included, ends here instead of falling through to the public
+	// router and its CORS. The rate limit comes before Basic Auth, so
+	// failed logins count against it and a request over the limit
+	// never reaches the password check.
 	if s.params.Config.MetricsUsername != "" {
-		s.router.Group(func(r chi.Router) {
+		metrics := chi.NewRouter()
-			r.Use(s.mw.MetricsAuth())
+		metrics.Use(s.mw.MetricsRateLimit())
-			r.Get(
+		metrics.Use(s.mw.MetricsAuth())
-				"/metrics",
+		metrics.Get("/", promhttp.Handler().ServeHTTP)
-				promhttp.Handler().ServeHTTP,
+		s.router.Mount("/metrics", metrics)
-			)
-		})
 	}
 }

internal/server/routes_test.go

@@ -0,0 +1,290 @@
+package server_test
+
+import (
+	"net/http"
+	"net/http/httptest"
+	"testing"
+
+	"github.com/spf13/viper"
+
+	"sneak.berlin/go/dnswatcher/internal/server"
+)
+
+// Credentials for /metrics, which is only routed when a username is set.
+const (
+	metricsUsername = "scraper"
+	metricsPassword = "scrape-secret"
+)
+
+// The tests below set env vars and touch viper global state, so like
+// the config tests they cannot use t.Parallel.
+
+// routedServer builds the server with its routes set up, ready to serve
+// test requests. The caller must first configure viper.
+func routedServer(t *testing.T) *server.Server {
+	t.Helper()
+
+	srv := buildServer(t)
+	srv.SetupRoutes()
+
+	return srv
+}
+
+// crossOriginRequest builds a request as a browser sends it from a page
+// on another site.
+func crossOriginRequest(
+	t *testing.T,
+	method string,
+	target string,
+) *http.Request {
+	t.Helper()
+
+	req := httptest.NewRequestWithContext(t.Context(), method, target, nil)
+	req.Header.Set("Origin", "https://example.net")
+
+	return req
+}
+
+// preflightRequest builds the OPTIONS request a browser sends before a
+// cross-origin request with the given method and request headers.
+func preflightRequest(
+	t *testing.T,
+	target string,
+	method string,
+	headers string,
+) *http.Request {
+	t.Helper()
+
+	req := crossOriginRequest(t, http.MethodOptions, target)
+	req.Header.Set("Access-Control-Request-Method", method)
+
+	if headers != "" {
+		req.Header.Set("Access-Control-Request-Headers", headers)
+	}
+
+	return req
+}
+
+func serve(
+	srv *server.Server,
+	req *http.Request,
+) *httptest.ResponseRecorder {
+	rec := httptest.NewRecorder()
+	srv.ServeHTTP(rec, req)
+
+	return rec
+}
+
+// publicPaths returns one path on each public route.
+func publicPaths() []string {
+	return []string{
+		"/",
+		"/s/css/tailwind.min.css",
+		"/api/v1/status",
+		"/health",
+		"/.well-known/healthcheck",
+	}
+}
+
+// TestPublicRoutesAllowAnyOrigin checks that every public route answers
+// a cross-origin GET with the CORS wildcard.
+func TestPublicRoutesAllowAnyOrigin(t *testing.T) {
+	viper.Reset()
+	t.Setenv("DNSWATCHER_TARGETS", "example.com")
+	t.Setenv("DNSWATCHER_METRICS_USERNAME", metricsUsername)
+	t.Setenv("DNSWATCHER_METRICS_PASSWORD", metricsPassword)
+
+	srv := routedServer(t)
+
+	for _, path := range publicPaths() {
+		rec := serve(srv, crossOriginRequest(t, http.MethodGet, path))
+
+		if rec.Code != http.StatusOK {
+			t.Errorf("GET %s: status = %d, want 200", path, rec.Code)
+		}
+
+		got := rec.Header().Get("Access-Control-Allow-Origin")
+		if got != "*" {
+			t.Errorf(
+				"GET %s: Access-Control-Allow-Origin = %q, want %q",
+				path, got, "*",
+			)
+		}
+	}
+}
+
+// TestMetricsHasNoCORS checks that no request to the Basic-Auth
+// protected /metrics, preflight included, gets a CORS header.
+func TestMetricsHasNoCORS(t *testing.T) {
+	viper.Reset()
+	t.Setenv("DNSWATCHER_TARGETS", "example.com")
+	t.Setenv("DNSWATCHER_METRICS_USERNAME", metricsUsername)
+	t.Setenv("DNSWATCHER_METRICS_PASSWORD", metricsPassword)
+
+	srv := routedServer(t)
+
+	authenticated := crossOriginRequest(t, http.MethodGet, "/metrics")
+	authenticated.SetBasicAuth(metricsUsername, metricsPassword)
+
+	tests := []struct {
+		name       string
+		req        *http.Request
+		wantStatus int
+	}{
+		{
+			"authenticated GET",
+			authenticated,
+			http.StatusOK,
+		},
+		{
+			"unauthenticated GET",
+			crossOriginRequest(t, http.MethodGet, "/metrics"),
+			http.StatusUnauthorized,
+		},
+		{
+			"preflight",
+			preflightRequest(t, "/metrics", http.MethodGet, ""),
+			http.StatusUnauthorized,
+		},
+	}
+
+	for _, tt := range tests {
+		rec := serve(srv, tt.req)
+
+		if rec.Code != tt.wantStatus {
+			t.Errorf(
+				"%s: status = %d, want %d",
+				tt.name, rec.Code, tt.wantStatus,
+			)
+		}
+
+		got := rec.Header().Get("Access-Control-Allow-Origin")
+		if got != "" {
+			t.Errorf(
+				"%s: Access-Control-Allow-Origin = %q, want none",
+				tt.name, got,
+			)
+		}
+	}
+}
+
+// TestPreflightAllowsOnlyWhatPublicRoutesServe checks what each public
+// route agrees to in a CORS preflight: GET, but not POST, PUT or
+// DELETE, which no route serves, and not the Authorization or
+// X-CSRF-Token headers, which no public route reads. It checks every
+// public route because one added with Get, such as /health, answers a
+// preflight only while CORS is middleware of a whole router; in a
+// Group, chi would answer it with 405 and no CORS headers.
+func TestPreflightAllowsOnlyWhatPublicRoutesServe(t *testing.T) {
+	viper.Reset()
+	t.Setenv("DNSWATCHER_TARGETS", "example.com")
+	t.Setenv("DNSWATCHER_METRICS_USERNAME", metricsUsername)
+	t.Setenv("DNSWATCHER_METRICS_PASSWORD", metricsPassword)
+
+	srv := routedServer(t)
+
+	tests := []struct {
+		method  string
+		headers string
+		allowed bool
+	}{
+		{http.MethodGet, "", true},
+		{http.MethodGet, "Content-Type", true},
+		{http.MethodPost, "", false},
+		{http.MethodPut, "", false},
+		{http.MethodDelete, "", false},
+		{http.MethodGet, "Authorization", false},
+		{http.MethodGet, "X-CSRF-Token", false},
+	}
+
+	for _, path := range publicPaths() {
+		for _, tt := range tests {
+			rec := serve(srv, preflightRequest(
+				t, path, tt.method, tt.headers,
+			))
+
+			want := ""
+			if tt.allowed {
+				want = tt.method
+			}
+
+			got := rec.Header().Get("Access-Control-Allow-Methods")
+			if got != want {
+				t.Errorf(
+					"preflight to %s for %s with headers %q: "+
+						"Access-Control-Allow-Methods = %q, want %q",
+					path, tt.method, tt.headers, got, want,
+				)
+			}
+		}
+	}
+}
+
+// metricsRequest builds a GET for /metrics from remoteAddr that logs
+// in with the given password.
+func metricsRequest(
+	t *testing.T,
+	remoteAddr string,
+	password string,
+) *http.Request {
+	t.Helper()
+
+	req := httptest.NewRequestWithContext(
+		t.Context(), http.MethodGet, "/metrics", nil,
+	)
+	req.RemoteAddr = remoteAddr
+	req.SetBasicAuth(metricsUsername, password)
+
+	return req
+}
+
+// TestMetricsRateLimitComesBeforeAuth checks that failed logins to
+// /metrics count against the rate limit; that once an address is over
+// it, even the right password gets 429, with the same body as a wrong
+// one; and that another address still gets in.
+func TestMetricsRateLimitComesBeforeAuth(t *testing.T) {
+	viper.Reset()
+	t.Setenv("DNSWATCHER_TARGETS", "example.com")
+	t.Setenv("DNSWATCHER_METRICS_USERNAME", metricsUsername)
+	t.Setenv("DNSWATCHER_METRICS_PASSWORD", metricsPassword)
+
+	const (
+		guesser = "198.51.100.1:4000"
+		other   = "198.51.100.2:4000"
+
+		// Far more guesses than the rate limit allows.
+		maxGuesses = 1000
+	)
+
+	srv := routedServer(t)
+
+	var guess *httptest.ResponseRecorder
+
+	for range maxGuesses {
+		guess = serve(srv, metricsRequest(t, guesser, "wrong"))
+		if guess.Code != http.StatusUnauthorized {
+			break
+		}
+	}
+
+	if guess.Code != http.StatusTooManyRequests {
+		t.Fatalf("wrong password: status = %d, want 429", guess.Code)
+	}
+
+	right := serve(srv, metricsRequest(t, guesser, metricsPassword))
+	if right.Code != http.StatusTooManyRequests {
+		t.Errorf("right password: status = %d, want 429", right.Code)
+	}
+
+	if right.Body.String() != guess.Body.String() {
+		t.Errorf(
+			"429 body with right password = %q, with wrong one = %q",
+			right.Body.String(), guess.Body.String(),
+		)
+	}
+
+	rec := serve(srv, metricsRequest(t, other, metricsPassword))
+	if rec.Code != http.StatusOK {
+		t.Errorf("another address: status = %d, want 200", rec.Code)
+	}
+}

internal/server/server_test.go

@@ -1,112 +1,130 @@
 package server_test
 
 import (
-	"net/http"
 	"testing"
 
+	"github.com/spf13/viper"
+	"go.uber.org/fx"
+
+	"sneak.berlin/go/dnswatcher/internal/config"
+	"sneak.berlin/go/dnswatcher/internal/globals"
+	"sneak.berlin/go/dnswatcher/internal/handlers"
+	"sneak.berlin/go/dnswatcher/internal/healthcheck"
+	"sneak.berlin/go/dnswatcher/internal/logger"
+	"sneak.berlin/go/dnswatcher/internal/middleware"
+	"sneak.berlin/go/dnswatcher/internal/notify"
 	"sneak.berlin/go/dnswatcher/internal/server"
+	"sneak.berlin/go/dnswatcher/internal/state"
 )
 
-// noopHandler stands in for the router; newHTTPServer only stores it.
+// buildServer wires a *server.Server exactly as cmd/dnswatcher does,
-func noopHandler() http.Handler {
+// minus the watcher/resolver subtree that would touch live DNS. fx
-	return http.HandlerFunc(
+// builds the object graph but the lifecycle is never started, so no
-		func(w http.ResponseWriter, _ *http.Request) {
+// OnStart hook runs and nothing listens or resolves. The caller must
-			w.WriteHeader(http.StatusOK)
+// first configure viper (config.New reads it), which is also why the
-		},
+// caller cannot run in parallel.
+func buildServer(t *testing.T) *server.Server {
+	t.Helper()
+
+	var srv *server.Server
+
+	app := fx.New(
+		fx.NopLogger,
+		fx.Provide(
+			globals.New,
+			logger.New,
+			config.New,
+			state.New,
+			healthcheck.New,
+			notify.New,
+			middleware.New,
+			handlers.New,
+			server.New,
+		),
+		fx.Populate(&srv),
 	)
+
+	err := app.Err()
+	if err != nil {
+		t.Fatalf("building server graph: %v", err)
 	}
 
-// TestHTTPServerTimeoutsAreSet asserts that every socket-level
+	return srv
-// timeout is configured. A zero value in net/http means "no limit",
+}
-// so a refactor that silently drops one of these reintroduces the
+
-// slowloris / unreaped-keep-alive exposure this guards against.
+// TestRunWiresSocketTimeouts pins that the http.Server the running
+// server actually serves — the one Run builds and hands to
+// ListenAndServe — carries every socket-level timeout, plus the two
+// relationships the values must satisfy.
 //
-// The assertions are on the configured field values only; nothing
+// Run is driven to completion with an unbindable port: it builds and
-// here measures elapsed time, so the test cannot flake on timing.
+// stores s.httpServer, then ListenAndServe fails at once and Run
-func TestHTTPServerTimeoutsAreSet(t *testing.T) {
+// returns without ever listening. The assertions run in the same
-	t.Parallel()
+// goroutine after Run returns, so reading s.httpServer is free of any
+// data race. Nothing here measures elapsed time.
+//
+// ReadTimeout must be at least ReadHeaderTimeout. net/http reads the
+// headers under ReadHeaderTimeout, then sets the read deadline for the
+// rest of the request to ReadTimeout, counted from when it started
+// reading the request. If ReadTimeout were smaller, a request whose
+// headers arrived after ReadTimeout but within ReadHeaderTimeout would
+// get a read deadline that had already passed, so reading its body
+// would fail at once.
+func TestRunWiresSocketTimeouts(t *testing.T) {
+	// Sets an env var and touches viper global state, so like the
+	// config tests it cannot use t.Parallel.
+	viper.Reset()
+	t.Setenv("DNSWATCHER_TARGETS", "example.com")
 
-	srv := server.NewHTTPServer(":8080", noopHandler())
+	srv := buildServer(t)
+	server.SetListenPort(srv, -1)
 
-	if srv.ReadTimeout <= 0 {
+	srv.Run()
-		t.Errorf(
+
-			"ReadTimeout must be non-zero, got %v",
+	hs := server.HTTPServerOf(srv)
-			srv.ReadTimeout,
+	if hs == nil {
-		)
+		t.Fatal("Run did not build an http.Server")
 	}
 
-	if srv.ReadHeaderTimeout <= 0 {
+	if hs.ReadTimeout <= 0 {
+		t.Errorf("ReadTimeout must be non-zero, got %v", hs.ReadTimeout)
+	}
+
+	if hs.ReadHeaderTimeout <= 0 {
 		t.Errorf(
 			"ReadHeaderTimeout must be non-zero, got %v",
-			srv.ReadHeaderTimeout,
+			hs.ReadHeaderTimeout,
 		)
 	}
 
-	if srv.WriteTimeout <= 0 {
+	if hs.WriteTimeout <= 0 {
-		t.Errorf(
+		t.Errorf("WriteTimeout must be non-zero, got %v", hs.WriteTimeout)
-			"WriteTimeout must be non-zero, got %v",
-			srv.WriteTimeout,
-		)
 	}
 
-	if srv.IdleTimeout <= 0 {
+	if hs.IdleTimeout <= 0 {
-		t.Errorf(
+		t.Errorf("IdleTimeout must be non-zero, got %v", hs.IdleTimeout)
-			"IdleTimeout must be non-zero, got %v",
-			srv.IdleTimeout,
-		)
-	}
 	}
 
-// TestWriteTimeoutExceedsHandlerBudget pins the one relationship the
+	if hs.WriteTimeout <= server.RequestTimeout {
-// values must satisfy. net/http arms the write deadline once request
-// headers are read, so it covers handler execution plus the response
-// flush. If WriteTimeout were not greater than the chimw.Timeout
-// handler budget, the connection would be severed before a handler
-// that used its full budget could respond, making that budget
-// unreachable.
-func TestWriteTimeoutExceedsHandlerBudget(t *testing.T) {
-	t.Parallel()
-
-	srv := server.NewHTTPServer(":8080", noopHandler())
-
-	if srv.WriteTimeout <= server.RequestTimeout {
 		t.Errorf(
 			"WriteTimeout (%v) must exceed handler budget (%v)",
-			srv.WriteTimeout,
+			hs.WriteTimeout,
 			server.RequestTimeout,
 		)
 	}
-}
 
-// TestReadTimeoutCoversHeaderTimeout asserts the read deadline for
+	if hs.ReadTimeout < hs.ReadHeaderTimeout {
-// the whole request is at least as long as the header-only deadline;
-// a smaller ReadTimeout would make ReadHeaderTimeout unreachable.
-func TestReadTimeoutCoversHeaderTimeout(t *testing.T) {
-	t.Parallel()
-
-	srv := server.NewHTTPServer(":8080", noopHandler())
-
-	if srv.ReadTimeout < srv.ReadHeaderTimeout {
 		t.Errorf(
 			"ReadTimeout (%v) must be >= ReadHeaderTimeout (%v)",
-			srv.ReadTimeout,
+			hs.ReadTimeout,
-			srv.ReadHeaderTimeout,
+			hs.ReadHeaderTimeout,
+		)
+	}
+
+	if hs.Handler != srv {
+		t.Errorf(
+			"Run wired handler %T, want the *server.Server",
+			hs.Handler,
 		)
 	}
 }
-
-// TestHTTPServerAddrAndHandler covers the rest of the constructor so
-// a future edit cannot drop the listen address or the handler.
-func TestHTTPServerAddrAndHandler(t *testing.T) {
-	t.Parallel()
-
-	srv := server.NewHTTPServer(":9999", noopHandler())
-
-	if srv.Addr != ":9999" {
-		t.Errorf("Addr = %q, want %q", srv.Addr, ":9999")
-	}
-
-	if srv.Handler == nil {
-		t.Error("Handler must not be nil")
-	}
-}

internal/state/export_test.go

@@ -0,0 +1,23 @@
+package state
+
+import (
+	"log/slog"
+
+	"sneak.berlin/go/dnswatcher/internal/config"
+)
+
+// NewForTestWithDataDir creates an empty State that saves to dataDir,
+// without the fx lifecycle.
+func NewForTestWithDataDir(dataDir string) *State {
+	return &State{
+		log: slog.Default(),
+		snapshot: &Snapshot{
+			Version:      stateVersion,
+			Domains:      make(map[string]*DomainState),
+			Hostnames:    make(map[string]*HostnameState),
+			Ports:        make(map[string]*PortState),
+			Certificates: make(map[string]*CertificateState),
+		},
+		config: &config.Config{DataDir: dataDir},
+	}
+}

internal/state/state.go

@@ -148,6 +148,11 @@ func New(
 
 	lifecycle.Append(fx.Hook{
 		OnStart: func(_ context.Context) error {
+			err := state.checkDataDirWritable()
+			if err != nil {
+				return err
+			}
+
 			return state.Load()
 		},
 		OnStop: func(_ context.Context) error {
@@ -345,3 +350,27 @@ func (s *State) GetCertificateState(
 
 	return cs, ok
 }
+
+// checkDataDirWritable creates the data directory if needed, then writes
+// and removes the temp file that Save uses. It runs at startup so that an
+// unwritable directory stops the process, instead of the process running
+// with every save failing and only logged.
+func (s *State) checkDataDirWritable() error {
+	dir := s.config.DataDir
+	tmpPath := s.config.StatePath() + ".tmp"
+
+	err := os.MkdirAll(dir, dirPermissions)
+	if err == nil {
+		err = os.WriteFile(tmpPath, nil, filePermissions)
+	}
+
+	if err == nil {
+		err = os.Remove(tmpPath)
+	}
+
+	if err != nil {
+		return fmt.Errorf("data directory %s is not writable: %w", dir, err)
+	}
+
+	return nil
+}

internal/state/state_test.go

@@ -4,10 +4,16 @@ import (
 	"encoding/json"
 	"os"
 	"path/filepath"
+	"strings"
 	"sync"
 	"testing"
 	"time"
 
+	"go.uber.org/fx/fxtest"
+
+	"sneak.berlin/go/dnswatcher/internal/config"
+	"sneak.berlin/go/dnswatcher/internal/globals"
+	"sneak.berlin/go/dnswatcher/internal/logger"
 	"sneak.berlin/go/dnswatcher/internal/state"
 )
 
@@ -493,6 +499,107 @@ func TestSaveWritePermissionError(t *testing.T) {
 	}
 }
 
+// startState builds a State through the real constructor and runs its
+// startup hook against dataDir, returning the startup error.
+func startState(t *testing.T, dataDir string) error {
+	t.Helper()
+
+	g, err := globals.New(nil)
+	if err != nil {
+		t.Fatalf("globals.New: %v", err)
+	}
+
+	log, err := logger.New(nil, logger.Params{Globals: g})
+	if err != nil {
+		t.Fatalf("logger.New: %v", err)
+	}
+
+	lifecycle := fxtest.NewLifecycle(t)
+
+	_, err = state.New(lifecycle, state.Params{
+		Logger: log,
+		Config: &config.Config{DataDir: dataDir},
+	})
+	if err != nil {
+		t.Fatalf("state.New: %v", err)
+	}
+
+	return lifecycle.Start(t.Context())
+}
+
+// TestStartupFailsWhenDataDirNotWritable verifies that startup stops
+// with an error naming the data directory when it cannot be written.
+// The directory's parent is a regular file, which also fails as root.
+func TestStartupFailsWhenDataDirNotWritable(t *testing.T) {
+	t.Parallel()
+
+	parent := filepath.Join(t.TempDir(), "file")
+
+	err := os.WriteFile(parent, nil, 0o600)
+	if err != nil {
+		t.Fatalf("writing file: %v", err)
+	}
+
+	dataDir := filepath.Join(parent, "data")
+
+	err = startState(t, dataDir)
+	if err == nil {
+		t.Fatal("startup should fail when the data directory is not writable")
+	}
+
+	want := "data directory " + dataDir + " is not writable"
+	if !strings.Contains(err.Error(), want) {
+		t.Errorf("startup error %q does not contain %q", err, want)
+	}
+}
+
+// TestStartupFailsWhenExistingDataDirNotWritable verifies that startup
+// stops when the data directory exists but the temp file that saving uses
+// cannot be written in it. A directory sitting at the temp file's path
+// makes that write fail, which also holds as root.
+func TestStartupFailsWhenExistingDataDirNotWritable(t *testing.T) {
+	t.Parallel()
+
+	dataDir := t.TempDir()
+
+	err := os.Mkdir(filepath.Join(dataDir, "state.json.tmp"), 0o700)
+	if err != nil {
+		t.Fatalf("creating directory: %v", err)
+	}
+
+	err = startState(t, dataDir)
+	if err == nil {
+		t.Fatal("startup should fail when the data directory is not writable")
+	}
+
+	want := "data directory " + dataDir + " is not writable"
+	if !strings.Contains(err.Error(), want) {
+		t.Errorf("startup error %q does not contain %q", err, want)
+	}
+}
+
+// TestStartupCreatesDataDir verifies that startup creates a missing
+// data directory and leaves nothing behind in it.
+func TestStartupCreatesDataDir(t *testing.T) {
+	t.Parallel()
+
+	dataDir := filepath.Join(t.TempDir(), "data")
+
+	err := startState(t, dataDir)
+	if err != nil {
+		t.Fatalf("startup error: %v", err)
+	}
+
+	entries, err := os.ReadDir(dataDir)
+	if err != nil {
+		t.Fatalf("reading data directory: %v", err)
+	}
+
+	if len(entries) != 0 {
+		t.Errorf("startup left %d entries in the data directory", len(entries))
+	}
+}
+
 // TestPortStateUnmarshalJSON_NewFormat verifies deserialization of the
 // current multi-hostname format.
 func TestPortStateUnmarshalJSON_NewFormat(t *testing.T) {
@@ -632,7 +739,7 @@ func TestPortStateUnmarshalJSON_BothFormats(t *testing.T) {
 func TestGetSnapshot_ReturnsCopy(t *testing.T) {
 	t.Parallel()
 
-	s := state.NewForTest()
+	s := state.NewForTestWithDataDir(t.TempDir())
 
 	populateState(t, s)
 
@@ -654,7 +761,7 @@ func TestGetSnapshot_ReturnsCopy(t *testing.T) {
 func TestDomainState_GetSet(t *testing.T) {
 	t.Parallel()
 
-	s := state.NewForTest()
+	s := state.NewForTestWithDataDir(t.TempDir())
 
 	// Get on missing key returns false.
 	_, ok := s.GetDomainState("nonexistent.com")
@@ -705,7 +812,7 @@ func TestDomainState_GetSet(t *testing.T) {
 func TestHostnameState_GetSet(t *testing.T) {
 	t.Parallel()
 
-	s := state.NewForTest()
+	s := state.NewForTestWithDataDir(t.TempDir())
 
 	_, ok := s.GetHostnameState("missing.example.com")
 	if ok {
@@ -750,7 +857,7 @@ func TestHostnameState_GetSet(t *testing.T) {
 func TestPortState_GetSetDelete(t *testing.T) {
 	t.Parallel()
 
-	s := state.NewForTest()
+	s := state.NewForTestWithDataDir(t.TempDir())
 
 	_, ok := s.GetPortState("1.2.3.4:80")
 	if ok {
@@ -788,7 +895,7 @@ func TestPortState_GetSetDelete(t *testing.T) {
 func TestGetAllPortKeys(t *testing.T) {
 	t.Parallel()
 
-	s := state.NewForTest()
+	s := state.NewForTestWithDataDir(t.TempDir())
 
 	keys := s.GetAllPortKeys()
 	if len(keys) != 0 {
@@ -830,7 +937,7 @@ func TestGetAllPortKeys(t *testing.T) {
 func TestCertificateState_GetSet(t *testing.T) {
 	t.Parallel()
 
-	s := state.NewForTest()
+	s := state.NewForTestWithDataDir(t.TempDir())
 
 	_, ok := s.GetCertificateState("1.2.3.4:443:www.example.com")
 	if ok {
@@ -1051,7 +1158,7 @@ func TestLoadPreservesExistingStateOnMissingFile(t *testing.T) {
 func TestConcurrentGetSet(t *testing.T) {
 	t.Parallel()
 
-	s := state.NewForTest()
+	s := state.NewForTestWithDataDir(t.TempDir())
 
 	const goroutines = 20
 
@@ -1261,35 +1368,6 @@ func TestMultipleSavesOverwrite(t *testing.T) {
 	}
 }
 
-// TestNewForTest verifies the test helper creates a valid empty state.
-func TestNewForTest(t *testing.T) {
-	t.Parallel()
-
-	s := state.NewForTest()
-
-	snap := s.GetSnapshot()
-
-	if snap.Version != 1 {
-		t.Errorf("version: got %d, want 1", snap.Version)
-	}
-
-	if snap.Domains == nil {
-		t.Error("Domains map should be initialized")
-	}
-
-	if snap.Hostnames == nil {
-		t.Error("Hostnames map should be initialized")
-	}
-
-	if snap.Ports == nil {
-		t.Error("Ports map should be initialized")
-	}
-
-	if snap.Certificates == nil {
-		t.Error("Certificates map should be initialized")
-	}
-}
-
 // TestSaveFilePermissions verifies the saved file has restricted permissions.
 func TestSaveFilePermissions(t *testing.T) {
 	t.Parallel()

internal/state/state_test_helper.go

@@ -1,38 +0,0 @@
-package state
-
-import (
-	"log/slog"
-
-	"sneak.berlin/go/dnswatcher/internal/config"
-)
-
-// NewForTest creates a State for unit testing with no persistence.
-func NewForTest() *State {
-	return &State{
-		log: slog.Default(),
-		snapshot: &Snapshot{
-			Version:      stateVersion,
-			Domains:      make(map[string]*DomainState),
-			Hostnames:    make(map[string]*HostnameState),
-			Ports:        make(map[string]*PortState),
-			Certificates: make(map[string]*CertificateState),
-		},
-		config: &config.Config{DataDir: ""},
-	}
-}
-
-// NewForTestWithDataDir creates a State backed by the given directory
-// for tests that need file persistence.
-func NewForTestWithDataDir(dataDir string) *State {
-	return &State{
-		log: slog.Default(),
-		snapshot: &Snapshot{
-			Version:      stateVersion,
-			Domains:      make(map[string]*DomainState),
-			Hostnames:    make(map[string]*HostnameState),
-			Ports:        make(map[string]*PortState),
-			Certificates: make(map[string]*CertificateState),
-		},
-		config: &config.Config{DataDir: dataDir},
-	}
-}

internal/watcher/export_test.go

@@ -0,0 +1,50 @@
+package watcher
+
+import (
+	"context"
+	"log/slog"
+	"time"
+
+	"sneak.berlin/go/dnswatcher/internal/config"
+	"sneak.berlin/go/dnswatcher/internal/state"
+)
+
+// NewForTest creates a Watcher without fx for unit testing.
+func NewForTest(
+	cfg *config.Config,
+	st *state.State,
+	res DNSResolver,
+	pc PortChecker,
+	tc TLSChecker,
+	n Notifier,
+) *Watcher {
+	return &Watcher{
+		log:            slog.Default(),
+		config:         cfg,
+		state:          st,
+		resolver:       res,
+		portCheck:      pc,
+		tlsCheck:       tc,
+		notify:         n,
+		firstRun:       true,
+		expiryNotified: make(map[string]time.Time),
+	}
+}
+
+// NewlyDisagreeingPairs exports newlyDisagreeingPairs for testing.
+func NewlyDisagreeingPairs(
+	prev *state.HostnameState,
+	current map[string]map[string][]string,
+) [][2]string {
+	return newlyDisagreeingPairs(prev, current)
+}
+
+// DetectHostnameChanges exports detectHostnameChanges for testing.
+func (w *Watcher) DetectHostnameChanges(
+	ctx context.Context,
+	hostname string,
+	prev *state.HostnameState,
+	current map[string]map[string][]string,
+) {
+	w.detectHostnameChanges(ctx, hostname, prev, current)
+}

internal/watcher/inconsistency_test.go

@@ -0,0 +1,182 @@
+package watcher_test
+
+import (
+	"slices"
+	"testing"
+
+	"sneak.berlin/go/dnswatcher/internal/state"
+	"sneak.berlin/go/dnswatcher/internal/watcher"
+)
+
+const (
+	host = "www.example.net"
+	nsA  = "a.ns.example.net."
+	nsB  = "b.ns.example.net."
+	nsC  = "c.ns.example.net."
+	ip1  = "192.0.2.1"
+	ip2  = "192.0.2.2"
+	ip3  = "192.0.2.3"
+)
+
+// hostnameState builds the state a check with these records leaves behind.
+func hostnameState(
+	records map[string]map[string][]string,
+) *state.HostnameState {
+	hs := &state.HostnameState{
+		RecordsByNameserver: make(map[string]*state.NameserverRecordState),
+	}
+
+	for ns, recs := range records {
+		hs.RecordsByNameserver[ns] = &state.NameserverRecordState{
+			Records: recs,
+			Status:  "ok",
+		}
+	}
+
+	return hs
+}
+
+func TestNewlyDisagreeingPairs(t *testing.T) {
+	t.Parallel()
+
+	onlyA := map[string]map[string][]string{nsA: {"A": {ip1}}}
+	agree := map[string]map[string][]string{nsA: {"A": {ip1}}, nsB: {"A": {ip1}}}
+	disagree := map[string]map[string][]string{nsA: {"A": {ip1}}, nsB: {"A": {ip2}}}
+	alert := [][2]string{{nsA, nsB}}
+
+	// b already disagrees with a and c; then c changes, so a and c,
+	// which agreed, now differ.
+	bDiffers := map[string]map[string][]string{
+		nsA: {"A": {ip1}}, nsB: {"A": {ip2}}, nsC: {"A": {ip1}},
+	}
+	cChanges := map[string]map[string][]string{
+		nsA: {"A": {ip1}}, nsB: {"A": {ip2}}, nsC: {"A": {ip3}},
+	}
+
+	// Each case starts from the state loaded at startup and runs the
+	// checks in order; want[i] is what check i alerts for.
+	tests := []struct {
+		name   string
+		loaded map[string]map[string][]string
+		checks []map[string]map[string][]string
+		want   [][][2]string
+	}{
+		{
+			name:   "disagreement persisting across checks alerts once",
+			loaded: agree,
+			checks: []map[string]map[string][]string{disagree, disagree, disagree},
+			want:   [][][2]string{alert, nil, nil},
+		},
+		{
+			name:   "disagreement starting on a later check alerts on it",
+			loaded: agree,
+			checks: []map[string]map[string][]string{agree, agree, disagree},
+			want:   [][][2]string{nil, nil, alert},
+		},
+		{
+			name:   "disagreement in the loaded state does not alert",
+			loaded: disagree,
+			checks: []map[string]map[string][]string{disagree, disagree},
+			want:   [][][2]string{nil, nil},
+		},
+		{
+			name:   "nameserver new on the first check and disagreeing alerts once",
+			loaded: onlyA,
+			checks: []map[string]map[string][]string{disagree, disagree},
+			want:   [][][2]string{alert, nil},
+		},
+		{
+			name:   "disagreement after agreeing again alerts again",
+			loaded: agree,
+			checks: []map[string]map[string][]string{disagree, agree, disagree},
+			want:   [][][2]string{alert, nil, alert},
+		},
+		{
+			name:   "new disagreement while another nameserver differs alerts",
+			loaded: bDiffers,
+			checks: []map[string]map[string][]string{cChanges, cChanges},
+			want:   [][][2]string{{{nsA, nsC}}, nil},
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			t.Parallel()
+
+			prev := hostnameState(tt.loaded)
+
+			for i, current := range tt.checks {
+				got := watcher.NewlyDisagreeingPairs(prev, current)
+				if !slices.Equal(got, tt.want[i]) {
+					t.Errorf(
+						"check %d: alerted for %v, want %v",
+						i, got, tt.want[i],
+					)
+				}
+
+				prev = hostnameState(current)
+			}
+		})
+	}
+}
+
+func TestInconsistencyAlert(t *testing.T) {
+	t.Parallel()
+
+	onlyA := map[string]map[string][]string{nsA: {"A": {ip1}}}
+	agree := map[string]map[string][]string{nsA: {"A": {ip1}}, nsB: {"A": {ip1}}}
+	disagree := map[string]map[string][]string{nsA: {"A": {ip1}}, nsB: {"A": {ip2}}}
+
+	// Each case starts from the state loaded at startup and then sees
+	// the nameservers disagree on three checks in a row.
+	tests := []struct {
+		name   string
+		loaded map[string]map[string][]string
+		want   int
+	}{
+		{
+			name:   "disagreement lasting several checks alerts once",
+			loaded: agree,
+			want:   1,
+		},
+		{
+			name:   "disagreement in the loaded state does not alert",
+			loaded: disagree,
+			want:   0,
+		},
+		{
+			name:   "nameserver new on the first check and disagreeing alerts once",
+			loaded: onlyA,
+			want:   1,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			t.Parallel()
+
+			// The hostname change detection uses only the notifier.
+			notifier := &mockNotifier{}
+			w := watcher.NewForTest(nil, nil, nil, nil, nil, notifier)
+
+			prev := hostnameState(tt.loaded)
+
+			for range 3 {
+				w.DetectHostnameChanges(t.Context(), host, prev, disagree)
+				prev = hostnameState(disagree)
+			}
+
+			got := 0
+
+			for _, n := range notifier.getNotifications() {
+				if n.Title == "Inconsistency: "+host {
+					got++
+				}
+			}
+
+			if got != tt.want {
+				t.Errorf("sent %d inconsistency alerts, want %d", got, tt.want)
+			}
+		})
+	}
+}

internal/watcher/watcher.go

@@ -102,28 +102,6 @@ func New(
 	return w, nil
 }
 
-// NewForTest creates a Watcher without fx for unit testing.
-func NewForTest(
-	cfg *config.Config,
-	st *state.State,
-	res DNSResolver,
-	pc PortChecker,
-	tc TLSChecker,
-	n Notifier,
-) *Watcher {
-	return &Watcher{
-		log:            slog.Default(),
-		config:         cfg,
-		state:          st,
-		resolver:       res,
-		portCheck:      pc,
-		tlsCheck:       tc,
-		notify:         n,
-		firstRun:       true,
-		expiryNotified: make(map[string]time.Time),
-	}
-}
-
 // Run starts the monitoring loop with periodic scheduling.
 func (w *Watcher) Run(ctx context.Context) {
 	w.log.Info(
@@ -366,7 +344,7 @@ func (w *Watcher) detectHostnameChanges(
 ) {
 	w.detectRecordChanges(ctx, hostname, prev, current)
 	w.detectNSDisappearances(ctx, hostname, prev, current)
-	w.detectInconsistencies(ctx, hostname, current)
+	w.detectInconsistencies(ctx, hostname, prev, current)
 }
 
 func (w *Watcher) detectRecordChanges(
@@ -448,22 +426,11 @@ func (w *Watcher) detectNSDisappearances(
 func (w *Watcher) detectInconsistencies(
 	ctx context.Context,
 	hostname string,
+	prev *state.HostnameState,
 	current map[string]map[string][]string,
 ) {
-	nameservers := make([]string, 0, len(current))
+	for _, pair := range newlyDisagreeingPairs(prev, current) {
-	for ns := range current {
+		ns1, ns2 := pair[0], pair[1]
-		nameservers = append(nameservers, ns)
-	}
-
-	sort.Strings(nameservers)
-
-	for i := range len(nameservers) - 1 {
-		ns1 := nameservers[i]
-		ns2 := nameservers[i+1]
-
-		if recordsEqual(current[ns1], current[ns2]) {
-			continue
-		}
 
 		msg := fmt.Sprintf(
 			"Hostname: %s\n%s: %v\n%s: %v",
@@ -481,6 +448,43 @@ func (w *Watcher) detectInconsistencies(
 	}
 }
 
+// newlyDisagreeingPairs returns every pair of nameservers whose records
+// differ in current, in sorted order of name, except pairs where both
+// nameservers were in prev and already differed there. A nameserver
+// missing from prev is paired with every nameserver it differs from.
+func newlyDisagreeingPairs(
+	prev *state.HostnameState,
+	current map[string]map[string][]string,
+) [][2]string {
+	nameservers := make([]string, 0, len(current))
+	for ns := range current {
+		nameservers = append(nameservers, ns)
+	}
+
+	sort.Strings(nameservers)
+
+	var pairs [][2]string
+
+	for i, ns1 := range nameservers {
+		for _, ns2 := range nameservers[i+1:] {
+			if recordsEqual(current[ns1], current[ns2]) {
+				continue
+			}
+
+			prev1, ok1 := prev.RecordsByNameserver[ns1]
+			prev2, ok2 := prev.RecordsByNameserver[ns2]
+
+			if ok1 && ok2 && !recordsEqual(prev1.Records, prev2.Records) {
+				continue
+			}
+
+			pairs = append(pairs, [2]string{ns1, ns2})
+		}
+	}
+
+	return pairs
+}
+
 func (w *Watcher) checkAllPorts(ctx context.Context) {
 	// Phase 1: Build current IP:port → hostname associations
 	// from fresh DNS data.

internal/watcher/watcher_test.go

@@ -2,151 +2,81 @@ package watcher_test
 
 import (
 	"context"
-	"errors"
 	"fmt"
+	"log/slog"
+	"slices"
 	"sync"
 	"testing"
 	"time"
 
+	"go.uber.org/fx/fxtest"
+
 	"sneak.berlin/go/dnswatcher/internal/config"
+	"sneak.berlin/go/dnswatcher/internal/globals"
+	"sneak.berlin/go/dnswatcher/internal/livednstest"
+	"sneak.berlin/go/dnswatcher/internal/logger"
 	"sneak.berlin/go/dnswatcher/internal/portcheck"
+	"sneak.berlin/go/dnswatcher/internal/resolver"
 	"sneak.berlin/go/dnswatcher/internal/state"
 	"sneak.berlin/go/dnswatcher/internal/tlscheck"
 	"sneak.berlin/go/dnswatcher/internal/watcher"
 )
 
-// errNotFound is returned when mock data is missing.
+// The watcher looks these names up in live DNS with the real resolver,
-var errNotFound = errors.New("not found")
+// so tests assert on what the watcher does with the answers, never on
-
+// the records these zones publish. testHost's nameservers and addresses
-// Fixture values shared across tests.
+// stay the same from one check to the next, which the tests that check
+// it twice rely on.
 const (
-	testDomain = "example.com"
+	testDomain = "google.com"
-	testHost   = "www.example.com"
+	testHost   = "cloudflare.com"
-	testNS1    = "ns1.example.com."
-	testNS2    = "ns2.example.com."
-	testIPv4   = "93.184.216.34"
-	testIP     = "1.2.3.4"
 	testIssuer = "DigiCert"
 )
 
-// --- Mock implementations ---
+// Saved-state values that live DNS never returns: nameserver names
-
+// under .invalid and a documentation address.
-type mockResolver struct {
+const (
-	mu             sync.Mutex
+	oldNS1 = "ns1.example.invalid."
-	nsRecords      map[string][]string
+	oldNS2 = "ns2.example.invalid."
-	allRecords     map[string]map[string]map[string][]string
+	oldIP  = "192.0.2.1"
-	ipAddresses    map[string][]string
-	lookupNSErr    error
-	allRecordsErr  error
-	resolveIPErr   error
-	lookupNSCalls  int
-	allRecordCalls int
-}
-
-func (m *mockResolver) LookupNS(
-	_ context.Context,
-	domain string,
-) ([]string, error) {
-	m.mu.Lock()
-	defer m.mu.Unlock()
-
-	m.lookupNSCalls++
-
-	if m.lookupNSErr != nil {
-		return nil, m.lookupNSErr
-	}
-
-	ns, ok := m.nsRecords[domain]
-	if !ok {
-		return nil, fmt.Errorf(
-			"%w: NS for %s", errNotFound, domain,
 )
-	}
 
-	return ns, nil
+// --- Stand-ins for the port checker, TLS checker and notifier ---
-}
+//
-
+// DNS has none: the watchers built here use the real resolver (see
-func (m *mockResolver) LookupAllRecords(
+// TESTING.md).
-	_ context.Context,
-	hostname string,
-) (map[string]map[string][]string, error) {
-	m.mu.Lock()
-	defer m.mu.Unlock()
-
-	m.allRecordCalls++
-
-	if m.allRecordsErr != nil {
-		return nil, m.allRecordsErr
-	}
-
-	recs, ok := m.allRecords[hostname]
-	if !ok {
-		return nil, fmt.Errorf(
-			"%w: records for %s", errNotFound, hostname,
-		)
-	}
-
-	return recs, nil
-}
-
-func (m *mockResolver) ResolveIPAddresses(
-	_ context.Context,
-	hostname string,
-) ([]string, error) {
-	m.mu.Lock()
-	defer m.mu.Unlock()
-
-	if m.resolveIPErr != nil {
-		return nil, m.resolveIPErr
-	}
-
-	ips, ok := m.ipAddresses[hostname]
-	if !ok {
-		return nil, fmt.Errorf(
-			"%w: IPs for %s", errNotFound, hostname,
-		)
-	}
-
-	return ips, nil
-}
 
+// mockPortChecker reports every port open until closed is set.
 type mockPortChecker struct {
 	mu     sync.Mutex
-	results map[string]bool
+	closed bool
-	err     error
 	calls  int
 }
 
 func (m *mockPortChecker) CheckPort(
 	_ context.Context,
-	address string,
+	_ string,
-	port int,
+	_ int,
 ) (*portcheck.PortResult, error) {
 	m.mu.Lock()
 	defer m.mu.Unlock()
 
 	m.calls++
 
-	if m.err != nil {
+	return &portcheck.PortResult{Open: !m.closed}, nil
-		return nil, m.err
-	}
-
-	key := fmt.Sprintf("%s:%d", address, port)
-	open := m.results[key]
-
-	return &portcheck.PortResult{Open: open}, nil
 }
 
+// mockTLSChecker returns a certificate for the requested hostname that
+// expires at notAfter.
 type mockTLSChecker struct {
 	mu       sync.Mutex
-	certs map[string]*tlscheck.CertificateInfo
+	notAfter time.Time
-	err   error
 	calls    int
 }
 
 func (m *mockTLSChecker) CheckCertificate(
 	_ context.Context,
-	ip string,
+	_ string,
 	hostname string,
 ) (*tlscheck.CertificateInfo, error) {
 	m.mu.Lock()
@@ -154,20 +84,12 @@ func (m *mockTLSChecker) CheckCertificate(
 
 	m.calls++
 
-	if m.err != nil {
+	return &tlscheck.CertificateInfo{
-		return nil, m.err
+		CommonName:              hostname,
-	}
+		Issuer:                  testIssuer,
-
+		NotAfter:                m.notAfter,
-	key := fmt.Sprintf("%s:%s", ip, hostname)
+		SubjectAlternativeNames: []string{hostname},
-	cert, ok := m.certs[key]
+	}, nil
-
-	if !ok {
-		return nil, fmt.Errorf(
-			"%w: cert for %s", errNotFound, key,
-		)
-	}
-
-	return cert, nil
 }
 
 type notification struct {
@@ -205,10 +127,9 @@ func (m *mockNotifier) getNotifications() []notification {
 	return result
 }
 
-// --- Helper to build a Watcher for testing ---
+// --- Helpers to build a Watcher and run its checks against live DNS ---
 
 type testDeps struct {
-	resolver    *mockResolver
 	portChecker *mockPortChecker
 	tlsChecker  *mockTLSChecker
 	notifier    *mockNotifier
@@ -223,27 +144,37 @@ func newTestWatcher(
 	t.Helper()
 
 	deps := &testDeps{
-		resolver: &mockResolver{
+		portChecker: &mockPortChecker{},
-			nsRecords:   make(map[string][]string),
-			allRecords:  make(map[string]map[string]map[string][]string),
-			ipAddresses: make(map[string][]string),
-		},
-		portChecker: &mockPortChecker{
-			results: make(map[string]bool),
-		},
 		tlsChecker: &mockTLSChecker{
-			certs: make(map[string]*tlscheck.CertificateInfo),
+			notAfter: time.Now().Add(90 * 24 * time.Hour),
 		},
 		notifier: &mockNotifier{},
 		config:   cfg,
 	}
 
-	deps.state = state.NewForTest()
+	g, err := globals.New(nil)
+	if err != nil {
+		t.Fatalf("globals.New: %v", err)
+	}
+
+	log, err := logger.New(nil, logger.Params{Globals: g})
+	if err != nil {
+		t.Fatalf("logger.New: %v", err)
+	}
+
+	// The watcher saves state after every check, into cfg.DataDir.
+	deps.state, err = state.New(fxtest.NewLifecycle(t), state.Params{
+		Logger: log,
+		Config: cfg,
+	})
+	if err != nil {
+		t.Fatalf("state.New: %v", err)
+	}
 
 	w := watcher.NewForTest(
 		deps.config,
 		deps.state,
-		deps.resolver,
+		resolver.NewFromLogger(slog.Default()),
 		deps.portChecker,
 		deps.tlsChecker,
 		deps.notifier,
@@ -263,6 +194,122 @@ func defaultTestConfig(t *testing.T) *config.Config {
 	}
 }
 
+// checkOnce runs the watcher's checks once and returns an error when a
+// configured name has no hostname state saved by this check, or that
+// state holds no address. Either live DNS gave no answer for the name,
+// or the watcher saved no fresh result for it.
+func checkOnce(
+	ctx context.Context,
+	w *watcher.Watcher,
+	deps *testDeps,
+) error {
+	started := time.Now()
+
+	w.RunOnce(ctx)
+
+	names := slices.Concat(deps.config.Domains, deps.config.Hostnames)
+
+	for _, name := range names {
+		hs, ok := deps.state.GetHostnameState(name)
+		if !ok || hs.LastChecked.Before(started) ||
+			len(addresses(hs)) == 0 {
+			return fmt.Errorf(
+				"%s: %w, or the watcher saved no fresh "+
+					"result for it",
+				name, livednstest.ErrNoAnswer,
+			)
+		}
+	}
+
+	return nil
+}
+
+// runChecks builds a watcher, lets prepare set up the saved state and
+// stand-ins it starts from, and runs its checks once against live DNS.
+// If change is not nil, change then alters the saved state or stand-ins
+// and the checks run a second time. When either check finds no fresh
+// address for a name (see checkOnce), the watcher is thrown away and
+// all of this runs again on a new one, so a failed attempt leaves
+// nothing behind in the saved state, the stand-ins or the notifications.
+func runChecks(
+	t *testing.T,
+	cfg *config.Config,
+	prepare, change func(deps *testDeps),
+) *testDeps {
+	t.Helper()
+
+	var deps *testDeps
+
+	livednstest.Retry(t, "watcher checks", func(ctx context.Context) error {
+		var w *watcher.Watcher
+
+		w, deps = newTestWatcher(t, cfg)
+
+		if prepare != nil {
+			prepare(deps)
+		}
+
+		err := checkOnce(ctx, w, deps)
+		if err != nil || change == nil {
+			return err
+		}
+
+		change(deps)
+
+		return checkOnce(ctx, w, deps)
+	})
+
+	return deps
+}
+
+// addresses returns the A and AAAA values saved for a hostname.
+func addresses(hs *state.HostnameState) []string {
+	var ips []string
+
+	for _, nsState := range hs.RecordsByNameserver {
+		ips = append(ips, nsState.Records["A"]...)
+		ips = append(ips, nsState.Records["AAAA"]...)
+	}
+
+	return ips
+}
+
+// assertNotified checks that a notification with this title and
+// priority was sent.
+func assertNotified(
+	t *testing.T,
+	deps *testDeps,
+	title, priority string,
+) {
+	t.Helper()
+
+	notifications := deps.notifier.getNotifications()
+
+	for _, n := range notifications {
+		if n.Title == title && n.Priority == priority {
+			return
+		}
+	}
+
+	t.Errorf(
+		"expected %s notification %q, got: %v",
+		priority, title, notifications,
+	)
+}
+
+// countNotifications counts the notifications sent with this title.
+func countNotifications(deps *testDeps, title string) int {
+	count := 0
+
+	for _, n := range deps.notifier.getNotifications() {
+		if n.Title == title {
+			count++
+		}
+	}
+
+	return count
+}
+
 func TestFirstRunBaseline(t *testing.T) {
 	t.Parallel()
 
@@ -270,51 +317,12 @@ func TestFirstRunBaseline(t *testing.T) {
 	cfg.Domains = []string{testDomain}
 	cfg.Hostnames = []string{testHost}
 
-	w, deps := newTestWatcher(t, cfg)
+	deps := runChecks(t, cfg, nil, nil)
-	setupBaselineMocks(deps)
-
-	w.RunOnce(t.Context())
 
 	assertNoNotifications(t, deps)
 	assertStatePopulated(t, deps)
 }
 
-func setupBaselineMocks(deps *testDeps) {
-	deps.resolver.nsRecords[testDomain] = []string{
-		testNS1,
-		testNS2,
-	}
-	deps.resolver.allRecords[testDomain] = map[string]map[string][]string{
-		testNS1: {"A": {testIPv4}},
-		testNS2: {"A": {testIPv4}},
-	}
-	deps.resolver.allRecords[testHost] = map[string]map[string][]string{
-		testNS1: {"A": {testIPv4}},
-		testNS2: {"A": {testIPv4}},
-	}
-	deps.resolver.ipAddresses[testHost] = []string{
-		testIPv4,
-	}
-	deps.portChecker.results["93.184.216.34:80"] = true
-	deps.portChecker.results["93.184.216.34:443"] = true
-	deps.tlsChecker.certs["93.184.216.34:www.example.com"] = &tlscheck.CertificateInfo{
-		CommonName: testHost,
-		Issuer:     testIssuer,
-		NotAfter:   time.Now().Add(90 * 24 * time.Hour),
-		SubjectAlternativeNames: []string{
-			testHost,
-		},
-	}
-	deps.tlsChecker.certs["93.184.216.34:example.com"] = &tlscheck.CertificateInfo{
-		CommonName: testDomain,
-		Issuer:     testIssuer,
-		NotAfter:   time.Now().Add(90 * 24 * time.Hour),
-		SubjectAlternativeNames: []string{
-			testDomain,
-		},
-	}
-}
-
 func assertNoNotifications(
 	t *testing.T,
 	deps *testDeps,
@@ -361,26 +369,7 @@ func TestDomainPortAndTLSChecks(t *testing.T) {
 	cfg := defaultTestConfig(t)
 	cfg.Domains = []string{testDomain}
 
-	w, deps := newTestWatcher(t, cfg)
+	deps := runChecks(t, cfg, nil, nil)
-
-	deps.resolver.nsRecords[testDomain] = []string{
-		testNS1,
-	}
-	deps.resolver.allRecords[testDomain] = map[string]map[string][]string{
-		testNS1: {"A": {testIPv4}},
-	}
-	deps.portChecker.results["93.184.216.34:80"] = true
-	deps.portChecker.results["93.184.216.34:443"] = true
-	deps.tlsChecker.certs["93.184.216.34:example.com"] = &tlscheck.CertificateInfo{
-		CommonName: testDomain,
-		Issuer:     testIssuer,
-		NotAfter:   time.Now().Add(90 * 24 * time.Hour),
-		SubjectAlternativeNames: []string{
-			testDomain,
-		},
-	}
-
-	w.RunOnce(t.Context())
 
 	snap := deps.state.GetSnapshot()
 
@@ -419,50 +408,18 @@ func TestNSChangeDetection(t *testing.T) {
 	cfg := defaultTestConfig(t)
 	cfg.Domains = []string{testDomain}
 
-	w, deps := newTestWatcher(t, cfg)
+	// The saved state lists nameservers that live DNS does not.
+	deps := runChecks(t, cfg, func(deps *testDeps) {
+		deps.state.SetDomainState(testDomain, &state.DomainState{
+			Nameservers: []string{oldNS1, oldNS2},
+		})
+	}, nil)
 
-	deps.resolver.nsRecords[testDomain] = []string{
+	assertNotified(t, deps, "NS Change: "+testDomain, "warning")
-		testNS1,
-		testNS2,
-	}
-	deps.resolver.allRecords[testDomain] = map[string]map[string][]string{
-		testNS1: {"A": {testIP}},
-		testNS2: {"A": {testIP}},
-	}
-	deps.portChecker.results["1.2.3.4:80"] = false
-	deps.portChecker.results["1.2.3.4:443"] = false
 
-	ctx := t.Context()
+	ds, _ := deps.state.GetDomainState(testDomain)
-	w.RunOnce(ctx)
+	if slices.Contains(ds.Nameservers, oldNS1) {
-
+		t.Errorf("saved nameservers not updated: %v", ds.Nameservers)
-	deps.resolver.mu.Lock()
-	deps.resolver.nsRecords[testDomain] = []string{
-		testNS1,
-		"ns3.example.com.",
-	}
-	deps.resolver.allRecords[testDomain] = map[string]map[string][]string{
-		testNS1:            {"A": {testIP}},
-		"ns3.example.com.": {"A": {testIP}},
-	}
-	deps.resolver.mu.Unlock()
-
-	w.RunOnce(ctx)
-
-	notifications := deps.notifier.getNotifications()
-	if len(notifications) == 0 {
-		t.Error("expected notification for NS change")
-	}
-
-	found := false
-
-	for _, n := range notifications {
-		if n.Priority == "warning" {
-			found = true
-		}
-	}
-
-	if !found {
-		t.Error("expected warning-priority NS change notification")
 	}
 }
 
@@ -472,40 +429,18 @@ func TestRecordChangeDetection(t *testing.T) {
 	cfg := defaultTestConfig(t)
 	cfg.Hostnames = []string{testHost}
 
-	w, deps := newTestWatcher(t, cfg)
+	// Between the checks, save for every nameserver an address live DNS
-
+	// never returns.
-	deps.resolver.allRecords[testHost] = map[string]map[string][]string{
+	deps := runChecks(t, cfg, nil, func(deps *testDeps) {
-		testNS1: {"A": {testIPv4}},
+		hs, _ := deps.state.GetHostnameState(testHost)
+		for _, nsState := range hs.RecordsByNameserver {
+			nsState.Records = map[string][]string{"A": {oldIP}}
 		}
-	deps.resolver.ipAddresses[testHost] = []string{
-		testIPv4,
-	}
-	deps.portChecker.results["93.184.216.34:80"] = false
-	deps.portChecker.results["93.184.216.34:443"] = false
 
-	ctx := t.Context()
+		deps.state.SetHostnameState(testHost, hs)
-	w.RunOnce(ctx)
+	})
 
-	deps.resolver.mu.Lock()
+	assertNotified(t, deps, "Record Change: "+testHost, "warning")
-	deps.resolver.allRecords[testHost] = map[string]map[string][]string{
-		testNS1: {"A": {"93.184.216.35"}},
-	}
-	deps.resolver.ipAddresses[testHost] = []string{
-		"93.184.216.35",
-	}
-	deps.resolver.mu.Unlock()
-
-	deps.portChecker.mu.Lock()
-	deps.portChecker.results["93.184.216.35:80"] = false
-	deps.portChecker.results["93.184.216.35:443"] = false
-	deps.portChecker.mu.Unlock()
-
-	w.RunOnce(ctx)
-
-	notifications := deps.notifier.getNotifications()
-	if len(notifications) == 0 {
-		t.Error("expected notification for record change")
-	}
 }
 
 func TestPortStateChange(t *testing.T) {
@@ -514,38 +449,23 @@ func TestPortStateChange(t *testing.T) {
 	cfg := defaultTestConfig(t)
 	cfg.Hostnames = []string{testHost}
 
-	w, deps := newTestWatcher(t, cfg)
+	// Between the checks, every port closes.
-
+	deps := runChecks(t, cfg, nil, func(deps *testDeps) {
-	deps.resolver.allRecords[testHost] = map[string]map[string][]string{
-		testNS1: {"A": {testIP}},
-	}
-	deps.resolver.ipAddresses[testHost] = []string{
-		testIP,
-	}
-	deps.portChecker.results["1.2.3.4:80"] = true
-	deps.portChecker.results["1.2.3.4:443"] = true
-	deps.tlsChecker.certs["1.2.3.4:www.example.com"] = &tlscheck.CertificateInfo{
-		CommonName: testHost,
-		Issuer:     testIssuer,
-		NotAfter:   time.Now().Add(90 * 24 * time.Hour),
-		SubjectAlternativeNames: []string{
-			testHost,
-		},
-	}
-
-	ctx := t.Context()
-	w.RunOnce(ctx)
-
 		deps.portChecker.mu.Lock()
-	deps.portChecker.results["1.2.3.4:443"] = false
+		deps.portChecker.closed = true
 		deps.portChecker.mu.Unlock()
+	})
 
-	w.RunOnce(ctx)
+	hs, _ := deps.state.GetHostnameState(testHost)
-
+	assertNotified(
-	notifications := deps.notifier.getNotifications()
+		t, deps, "Port Change: "+addresses(hs)[0]+":443", "warning",
-	if len(notifications) == 0 {
+	)
-		t.Error("expected notification for port state change")
 }
+
+// expiresInThreeDays makes the TLS checker return certificates that
+// expire within the seven-day warning period.
+func expiresInThreeDays(deps *testDeps) {
+	deps.tlsChecker.notAfter = time.Now().Add(3 * 24 * time.Hour)
 }
 
 func TestTLSExpiryWarning(t *testing.T) {
@@ -554,49 +474,9 @@ func TestTLSExpiryWarning(t *testing.T) {
 	cfg := defaultTestConfig(t)
 	cfg.Hostnames = []string{testHost}
 
-	w, deps := newTestWatcher(t, cfg)
+	deps := runChecks(t, cfg, expiresInThreeDays, nil)
 
-	deps.resolver.allRecords[testHost] = map[string]map[string][]string{
+	assertNotified(t, deps, "TLS Expiry Warning: "+testHost, "warning")
-		testNS1: {"A": {testIP}},
-	}
-	deps.resolver.ipAddresses[testHost] = []string{
-		testIP,
-	}
-	deps.portChecker.results["1.2.3.4:80"] = true
-	deps.portChecker.results["1.2.3.4:443"] = true
-	deps.tlsChecker.certs["1.2.3.4:www.example.com"] = &tlscheck.CertificateInfo{
-		CommonName: testHost,
-		Issuer:     testIssuer,
-		NotAfter:   time.Now().Add(3 * 24 * time.Hour),
-		SubjectAlternativeNames: []string{
-			testHost,
-		},
-	}
-
-	ctx := t.Context()
-
-	// First run = baseline
-	w.RunOnce(ctx)
-
-	// Second run should warn about expiry
-	w.RunOnce(ctx)
-
-	notifications := deps.notifier.getNotifications()
-
-	found := false
-
-	for _, n := range notifications {
-		if n.Priority == "warning" {
-			found = true
-		}
-	}
-
-	if !found {
-		t.Errorf(
-			"expected expiry warning, got: %v",
-			notifications,
-		)
-	}
 }
 
 func TestTLSExpiryWarningDedup(t *testing.T) {
@@ -606,50 +486,25 @@ func TestTLSExpiryWarningDedup(t *testing.T) {
 	cfg.Hostnames = []string{testHost}
 	cfg.TLSInterval = 24 * time.Hour
 
-	w, deps := newTestWatcher(t, cfg)
+	title := "TLS Expiry Warning: " + testHost
 
-	deps.resolver.allRecords[testHost] = map[string]map[string][]string{
+	// The second check comes within the TLS interval of the first,
-		testNS1: {"A": {testIP}},
+	// so it must not warn again.
-	}
+	var warnings int
-	deps.resolver.ipAddresses[testHost] = []string{
+
-		testIP,
+	deps := runChecks(t, cfg, expiresInThreeDays, func(deps *testDeps) {
-	}
+		warnings = countNotifications(deps, title)
-	deps.portChecker.results["1.2.3.4:80"] = true
+	})
-	deps.portChecker.results["1.2.3.4:443"] = true
+
-	deps.tlsChecker.certs["1.2.3.4:www.example.com"] = &tlscheck.CertificateInfo{
+	if warnings == 0 {
-		CommonName: testHost,
+		t.Fatal("expected expiry warnings from the first check")
-		Issuer:     testIssuer,
-		NotAfter:   time.Now().Add(3 * 24 * time.Hour),
-		SubjectAlternativeNames: []string{
-			testHost,
-		},
 	}
 
-	ctx := t.Context()
+	got := countNotifications(deps, title)
-
+	if got != warnings {
-	// First run = baseline, no notifications
-	w.RunOnce(ctx)
-
-	// Second run should fire one expiry warning
-	w.RunOnce(ctx)
-
-	// Third run should NOT fire another warning (dedup)
-	w.RunOnce(ctx)
-
-	notifications := deps.notifier.getNotifications()
-
-	expiryCount := 0
-
-	for _, n := range notifications {
-		if n.Title == "TLS Expiry Warning: www.example.com" {
-			expiryCount++
-		}
-	}
-
-	if expiryCount != 1 {
 		t.Errorf(
-			"expected exactly 1 expiry warning (dedup), got %d",
+			"expected %d expiry warnings (dedup), got %d",
-			expiryCount,
+			warnings, got,
 		)
 	}
 }
@@ -657,21 +512,12 @@ func TestTLSExpiryWarningDedup(t *testing.T) {
 func TestGracefulShutdown(t *testing.T) {
 	t.Parallel()
 
+	// No domains or hostnames: stopping does not involve DNS.
 	cfg := defaultTestConfig(t)
-	cfg.Domains = []string{testDomain}
 	cfg.DNSInterval = 100 * time.Millisecond
 	cfg.TLSInterval = 100 * time.Millisecond
 
-	w, deps := newTestWatcher(t, cfg)
+	w, _ := newTestWatcher(t, cfg)
-
-	deps.resolver.nsRecords[testDomain] = []string{
-		testNS1,
-	}
-	deps.resolver.allRecords[testDomain] = map[string]map[string][]string{
-		testNS1: {"A": {testIP}},
-	}
-	deps.portChecker.results["1.2.3.4:80"] = false
-	deps.portChecker.results["1.2.3.4:443"] = false
 
 	ctx, cancel := context.WithCancel(t.Context())
 
@@ -693,90 +539,52 @@ func TestGracefulShutdown(t *testing.T) {
 	}
 }
 
-func setupHostnameIP(
-	deps *testDeps,
-	hostname, ip string,
-) {
-	deps.resolver.allRecords[hostname] = map[string]map[string][]string{
-		testNS1: {"A": {ip}},
-	}
-	deps.portChecker.results[ip+":80"] = true
-	deps.portChecker.results[ip+":443"] = true
-	deps.tlsChecker.certs[ip+":"+hostname] = &tlscheck.CertificateInfo{
-		CommonName:              hostname,
-		Issuer:                  testIssuer,
-		NotAfter:                time.Now().Add(90 * 24 * time.Hour),
-		SubjectAlternativeNames: []string{hostname},
-	}
-}
-
-func updateHostnameIP(deps *testDeps, hostname, ip string) {
-	deps.resolver.mu.Lock()
-	deps.resolver.allRecords[hostname] = map[string]map[string][]string{
-		testNS1: {"A": {ip}},
-	}
-	deps.resolver.mu.Unlock()
-
-	deps.portChecker.mu.Lock()
-	deps.portChecker.results[ip+":80"] = true
-	deps.portChecker.results[ip+":443"] = true
-	deps.portChecker.mu.Unlock()
-
-	deps.tlsChecker.mu.Lock()
-	deps.tlsChecker.certs[ip+":"+hostname] = &tlscheck.CertificateInfo{
-		CommonName:              hostname,
-		Issuer:                  testIssuer,
-		NotAfter:                time.Now().Add(90 * 24 * time.Hour),
-		SubjectAlternativeNames: []string{hostname},
-	}
-	deps.tlsChecker.mu.Unlock()
-}
-
 func TestDNSRunsBeforePortAndTLSChecks(t *testing.T) {
 	t.Parallel()
 
 	cfg := defaultTestConfig(t)
 	cfg.Hostnames = []string{testHost}
 
-	w, deps := newTestWatcher(t, cfg)
+	// The saved state says the last check found testHost at oldIP.
-
+	deps := runChecks(t, cfg, func(deps *testDeps) {
-	setupHostnameIP(deps, testHost, "10.0.0.1")
+		deps.state.SetHostnameState(testHost, &state.HostnameState{
-
+			RecordsByNameserver: map[string]*state.NameserverRecordState{
-	ctx := t.Context()
+				oldNS1: {
-	w.RunOnce(ctx)
+					Records: map[string][]string{"A": {oldIP}},
+					Status:  "ok",
+				},
+			},
+		})
+	}, nil)
 
 	snap := deps.state.GetSnapshot()
-	if _, ok := snap.Ports["10.0.0.1:80"]; !ok {
+
-		t.Fatal("expected port state for 10.0.0.1:80")
+	if _, ok := snap.Ports[oldIP+":80"]; ok {
+		t.Error("port check used stale DNS: found " + oldIP + ":80")
 	}
 
-	// DNS changes to a new IP; port and TLS must pick it up.
+	// Port and TLS checks must use the addresses this check found.
-	updateHostnameIP(deps, testHost, "10.0.0.2")
+	for _, ip := range addresses(snap.Hostnames[testHost]) {
-
+		if _, ok := snap.Ports[ip+":80"]; !ok {
-	w.RunOnce(ctx)
+			t.Error("port check used stale DNS: missing " + ip + ":80")
-
-	snap = deps.state.GetSnapshot()
-
-	if _, ok := snap.Ports["10.0.0.2:80"]; !ok {
-		t.Error("port check used stale DNS: missing 10.0.0.2:80")
 		}
 
-	certKey := "10.0.0.2:443:www.example.com"
+		certKey := ip + ":443:" + testHost
 		if _, ok := snap.Certificates[certKey]; !ok {
 			t.Error("TLS check used stale DNS: missing " + certKey)
 		}
 	}
+}
 
 func TestSendTestNotification_Enabled(t *testing.T) {
 	t.Parallel()
 
+	// No domains or hostnames: the startup notification does not
+	// involve DNS.
 	cfg := defaultTestConfig(t)
-	cfg.Domains = []string{testDomain}
-	cfg.Hostnames = []string{testHost}
 	cfg.SendTestNotification = true
 
 	w, deps := newTestWatcher(t, cfg)
-	setupBaselineMocks(deps)
 
 	w.RunOnce(t.Context())
 
@@ -797,14 +605,11 @@ func TestSendTestNotification_ViaRun(t *testing.T) {
 	t.Parallel()
 
 	cfg := defaultTestConfig(t)
-	cfg.Domains = []string{testDomain}
-	cfg.Hostnames = []string{testHost}
 	cfg.SendTestNotification = true
 	cfg.DNSInterval = 24 * time.Hour
 	cfg.TLSInterval = 24 * time.Hour
 
 	w, deps := newTestWatcher(t, cfg)
-	setupBaselineMocks(deps)
 
 	ctx, cancel := context.WithCancel(t.Context())
 
@@ -844,14 +649,11 @@ func TestSendTestNotification_Disabled(t *testing.T) {
 	t.Parallel()
 
 	cfg := defaultTestConfig(t)
-	cfg.Domains = []string{testDomain}
-	cfg.Hostnames = []string{testHost}
 	cfg.SendTestNotification = false
 	cfg.DNSInterval = 24 * time.Hour
 	cfg.TLSInterval = 24 * time.Hour
 
 	w, deps := newTestWatcher(t, cfg)
-	setupBaselineMocks(deps)
 
 	ctx, cancel := context.WithCancel(t.Context())
 
@@ -884,32 +686,22 @@ func TestNSFailureAndRecovery(t *testing.T) {
 	cfg := defaultTestConfig(t)
 	cfg.Hostnames = []string{testHost}
 
-	w, deps := newTestWatcher(t, cfg)
+	// Between the checks, save every nameserver the first check found
-
+	// as failed, and add, as answering, one that live DNS does not list.
-	deps.resolver.allRecords[testHost] = map[string]map[string][]string{
+	deps := runChecks(t, cfg, nil, func(deps *testDeps) {
-		testNS1: {"A": {testIP}},
+		hs, _ := deps.state.GetHostnameState(testHost)
-		testNS2: {"A": {testIP}},
+		for _, nsState := range hs.RecordsByNameserver {
+			nsState.Status = "error"
 		}
-	deps.resolver.ipAddresses[testHost] = []string{
+
-		testIP,
+		hs.RecordsByNameserver[oldNS1] = &state.NameserverRecordState{
+			Records: map[string][]string{"A": {oldIP}},
+			Status:  "ok",
 		}
-	deps.portChecker.results["1.2.3.4:80"] = false
-	deps.portChecker.results["1.2.3.4:443"] = false
 
-	ctx := t.Context()
+		deps.state.SetHostnameState(testHost, hs)
+	})
 
-	w.RunOnce(ctx)
+	assertNotified(t, deps, "NS Failure: "+testHost, "error")
-
+	assertNotified(t, deps, "NS Recovery: "+testHost, "success")
-	deps.resolver.mu.Lock()
-	deps.resolver.allRecords[testHost] = map[string]map[string][]string{
-		testNS1: {"A": {testIP}},
-	}
-	deps.resolver.mu.Unlock()
-
-	w.RunOnce(ctx)
-
-	notifications := deps.notifier.getNotifications()
-	if len(notifications) == 0 {
-		t.Error("expected notification for NS disappearance")
-	}
 }

script/cibuild

@@ -1,14 +1,18 @@
 #!/bin/sh
 # script/cibuild: run the CI build. The Dockerfile's lint stage runs
 # make fmt-check and golangci-lint; its builder stage runs make test
-# and make build. A successful build implies all of those passed.
+# and make build.
+#
+# --no-cache-filter=lint,builder runs both stages on every invocation;
+# otherwise an unchanged tree is served from the layer cache and passes
+# without linting or querying live DNS.
 set -eu
 
 ROOT="$(cd "$(dirname "$0")/.." && pwd -P)"
 
 main() {
     cd "$ROOT"
-    docker build .
+    docker build --no-cache-filter=lint,builder .
 }
 
 main "$@"

script/docker

@@ -1,6 +1,10 @@
 #!/bin/sh
 # script/docker: build the Docker image tagged with the project name.
-# Identical in all repos; the tag comes from script/projectname.
+# The tag comes from script/projectname.
+#
+# --no-cache-filter=lint,builder runs the lint stage and the builder
+# stage (make test) on every invocation; otherwise an unchanged tree is
+# served from the layer cache without linting or querying live DNS.
 set -eu
 
 SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd -P)"
@@ -8,7 +12,7 @@ ROOT="$(cd "$SCRIPT_DIR/.." && pwd -P)"
 
 main() {
     cd "$ROOT"
-    docker build -t "$("$SCRIPT_DIR/projectname")" .
+    docker build --no-cache-filter=lint,builder -t "$("$SCRIPT_DIR/projectname")" .
 }
 
 main "$@"