docker: a plain docker build . stamps the git version (closes #210)

A plain `docker build .`, which is how upaas builds, stamped `dev`:
`.dockerignore` left out `.git` and the builder declared
`ARG VERSION=dev`. `.dockerignore` now sends `.git` without
`.git/config`, which can hold a credential, and lists no tracked file,
which git would count as deleted. `ARG VERSION` has no default. The
Makefile takes a non-empty `VERSION` from the command line or the
environment, so a build arg still wins; otherwise `git describe` runs in
the builder, which trusts the checkout whoever owns it, as a context
sent as a tar archive keeps its owners. A new `make version` prints the
version; the build fails when the context carries `.git` and it comes
out empty, `dev` or `unknown`.

Model: opus-5-5

.dockerignore

@@ -1,6 +1,9 @@
-.git/
+# .git is sent, without its config: the builder stage derives the version it
+# stamps into the binary from it, and `git describe` does not need the config,
+# which can hold a credential (a password in the remote URL, a CI token). No
+# tracked file may be listed here: git in the build would see it as deleted
+# and mark the version -dirty, and an excluded .md would silently drop out of
+# the prettier check in Dockerfile.fmt.
+.git/config
 bin/
-*.md
+node_modules/
-LICENSE
-.editorconfig
-.gitignore

.gitignore

@@ -1,4 +1,5 @@
 bin/
+node_modules/
 vendor/
 data/
 .env

.golangci.yml

@@ -10,14 +10,20 @@ run:
 
 linters:
   default: all
+  enable:
+    # Successor to the deprecated gomodguard. Named explicitly, rather than
+    # left to `default: all`, because it carries the module policy below.
+    - gomodguard_v2
   disable:
     # Genuinely incompatible with project patterns
     - exhaustruct      # Requires all struct fields
-    - depguard         # Dependency allow/block lists
     - godot            # Requires comments to end with periods
-    - wsl              # Deprecated, replaced by wsl_v5
     - wrapcheck        # Too verbose for internal packages
     - varnamelen       # Short names like db, id are idiomatic Go
+    # Deprecated: the warning is attached to the old name, so it is
+    # silenced by disabling that name, not by enabling the successor.
+    - wsl              # Deprecated, replaced by wsl_v5
+    - gomodguard       # Deprecated, replaced by gomodguard_v2
   settings:
     lll:
       line-length: 88
@@ -28,6 +34,68 @@ linters:
       max-complexity: 15
     dupl:
       threshold: 100
+    depguard:
+      # Test-support code must not be compiled into the shipped binary. A
+      # test-support package exists to hand a test privileges the program
+      # itself must never have, so a file that is not a test must not import
+      # one. Test files, and the files inside a package whose directory name
+      # ends in `test`, are where that code belongs, and are exempt.
+      #
+      # The deny list below is the one part of this file a repository is
+      # expected to extend, and the only part it may. depguard matches an
+      # import path against a list of prefixes, so it cannot be told "any path
+      # whose last segment ends in test"; a repository's own test-support
+      # packages have to be named here one at a time, by full import path,
+      # under a module path that differs from repository to repository. Add
+      # them; change nothing else.
+      rules:
+        test-support:
+          list-mode: lax
+          files:
+            - "$all"
+            - "!$test"
+            - "!**/*test/**"
+          deny:
+            - pkg: net/http/httptest
+              desc: >-
+                Test-support code belongs in test files and in packages whose
+                directory name ends in test, not in the shipped binary.
+            - pkg: sneak.berlin/go/dnswatcher/internal/livednstest
+              desc: >-
+                Live-DNS test support belongs in test files and in packages
+                whose directory name ends in test, not in the shipped binary.
+    # Only decisions already recorded in the Go package defaults are
+    # listed here. Every entry matches the module path exactly.
+    gomodguard_v2:
+      blocked:
+        - module: github.com/rs/zerolog
+          recommendations:
+            - log/slog
+          reason: "Structured logging is stdlib log/slog."
+        # One entry per pre-fork module path, because the later releases
+        # are separate paths. A prefix match would be shorter but would
+        # also reach github.com/go-redis/redismock, the test double for
+        # the successor these entries recommend.
+        - module: github.com/go-redis/redis
+          recommendations:
+            - github.com/redis/go-redis/v9
+          reason: "Pre-fork module; use the maintained go-redis v9."
+        - module: github.com/go-redis/redis/v7
+          recommendations:
+            - github.com/redis/go-redis/v9
+          reason: "Pre-fork module; use the maintained go-redis v9."
+        - module: github.com/go-redis/redis/v8
+          recommendations:
+            - github.com/redis/go-redis/v9
+          reason: "Pre-fork module; use the maintained go-redis v9."
+        - module: github.com/sergi/go-diff
+          recommendations:
+            - github.com/aymanbagabas/go-udiff
+          reason: "No unified diff output; use go-udiff."
+        - module: github.com/hexops/gotextdiff
+          recommendations:
+            - github.com/aymanbagabas/go-udiff
+          reason: "Unmaintained fork; use go-udiff."
 
 issues:
   max-issues-per-linter: 0

.prettierignore

@@ -0,0 +1,5 @@
+bin/
+data/
+node_modules/
+.claude/
+static/css/tailwind.min.css

.prettierrc

@@ -0,0 +1,4 @@
+{
+    "tabWidth": 4,
+    "proseWrap": "always"
+}

Dockerfile

@@ -1,7 +1,10 @@
 # Lint stage - fast feedback on lint issues, before the build starts.
 # The linter is invoked directly rather than through `make lint`: that
 # target shells out to `docker build -f Dockerfile.lint`, and there is
-# no docker daemon inside a docker build.
+# no docker daemon inside a docker build. For the same reason this stage
+# runs only the Go half of `make fmt-check`; script/cibuild runs the
+# markdown half after this build.
+# script/cibuild and script/docker name this stage in --no-cache-filter.
 # golangci/golangci-lint:v2.12.2 (Debian-based), 2026-08-10
 FROM golangci/golangci-lint:v2.12.2@sha256:5cceeef04e53efe1470638d4b4b4f5ceefd574955ab3941b2d9a68a8c9ad5240 AS lint
 
@@ -11,15 +14,21 @@ RUN go mod download
 
 COPY . .
 
-RUN make fmt-check
+RUN script/fmt-check-go
 RUN golangci-lint run --config .golangci.yml ./...
 
 # Build stage
+# script/cibuild and script/docker name this stage in --no-cache-filter.
 # golang 1.25-alpine, 2026-02-28
 FROM golang@sha256:f6751d823c26342f9506c03797d2527668d095b0a15f1862cddb4d927a7a4ced AS builder
 
 RUN apk add --no-cache git make gcc musl-dev binutils-gold
 
+# A build context sent as a tar archive keeps its files' owners, and git
+# refuses to read a checkout owned by another user. Trust this one
+# whoever owns it.
+RUN git config --system --add safe.directory /src
+
 # Force BuildKit to run the lint stage before proceeding
 COPY --from=lint /src/go.sum /dev/null
 
@@ -32,24 +41,58 @@ COPY . .
 # Run the tests - build fails if any test fails
 RUN make test
 
-# Build the binary
+# Version stamped into the binary: the VERSION build arg when one is
+# given and not empty (script/docker passes one), otherwise what
+# `git describe` says of the .git in the build context, so a plain
+# `docker build .` of a clone stamps its tag or short commit. The build
+# arg reaches make through the environment.
+ARG VERSION
+
+# A context that carries .git, as a directory or as a file, must yield a
+# real version: one that is empty, `dev` or `unknown` cannot be traced
+# back to a commit.
+RUN version="$(make version)"; \
+    if [ -e .git ]; then \
+        case "$version" in \
+            "" | dev | unknown) \
+                echo "version is \"$version\" although the build context carries .git" >&2; \
+                exit 1 ;; \
+        esac; \
+    fi
+
 RUN make build
 
 # Runtime stage
 # alpine 3.21, 2026-02-28
 FROM alpine@sha256:c3f8e73fdb79deaebaa2037150150191b9dcbfba68b4a46d70103204c53f4709
 
-RUN apk add --no-cache ca-certificates tzdata
+RUN apk add --no-cache ca-certificates tzdata su-exec
 
-WORKDIR /app
+COPY --from=builder /src/bin/dnswatcher /usr/local/bin/dnswatcher
+COPY deploy/docker-entrypoint.sh /usr/local/bin/docker-entrypoint.sh
 
-COPY --from=builder /src/bin/dnswatcher /app/dnswatcher
+# dnswatcher runs as this unprivileged user. The entrypoint creates the
-
+# data directory and gives it to this user on every start.
-# Create data directory
+RUN addgroup -S -g 10001 dnswatcher \
-RUN mkdir -p /var/lib/dnswatcher
+    && adduser -S -G dnswatcher -u 10001 dnswatcher
 
 ENV DNSWATCHER_DATA_DIR=/var/lib/dnswatcher
 
+# Config loading also reads a `.env` file and a file named `dnswatcher`
+# (any config extension, or none) from the working directory. `/` holds
+# neither, so every setting comes from the environment. Do not make the
+# data directory, or the binary's directory, the working directory.
+WORKDIR /
+
+# No USER: the entrypoint must start as root to set up the data
+# directory; it then runs dnswatcher as the dnswatcher user.
+
 EXPOSE 8080
 
-ENTRYPOINT ["/app/dnswatcher"]
+# busybox wget (already in alpine) probes the health endpoint every 10
+# seconds, so the container is healthy well before upaas reads its health
+# 60 seconds after a deploy and fails the deploy unless it is healthy.
+HEALTHCHECK --interval=10s --timeout=5s --start-period=10s --retries=3 \
+    CMD wget -q -O /dev/null "http://127.0.0.1:${PORT:-8080}/.well-known/healthcheck" || exit 1
+
+ENTRYPOINT ["/usr/local/bin/docker-entrypoint.sh"]

Dockerfile.fmt

@@ -0,0 +1,54 @@
+# prettier over the markdown, in a container, so it is never installed
+# on the host. script/fmt-check-markdown builds the fmt-check stage;
+# script/fmt builds fmt-out and takes the formatted files back.
+
+# node:22-bookworm-slim, 2026-09-05
+FROM node:22-bookworm-slim@sha256:83f487e0a63425e5b4d146fb5e5be574bcbe1b7b843d3ebafdd95eaf7767a7e5 AS nodedeps
+
+# prettier lives outside /src so that a `COPY . .` of the repo cannot
+# overwrite it, and so that node_modules never appears in the tree
+# prettier is about to walk.
+WORKDIR /tools
+
+# package.json pins the version and yarn.lock pins the bytes:
+# --frozen-lockfile installs exactly the lockfile's resolution and fails
+# if package.json disagrees with it, so the tool cannot float between
+# runs. yarn is the one in the image above.
+COPY package.json yarn.lock ./
+RUN yarn install --frozen-lockfile --non-interactive --no-progress
+
+ENV PATH="/tools/node_modules/.bin:${PATH}"
+
+WORKDIR /src
+
+# Read-only markdown check. Must match $stage in
+# script/fmt-check-markdown.
+FROM nodedeps AS fmt-check
+
+COPY . .
+
+# --config, not discovery: a .prettierrc that failed to arrive would
+# otherwise leave prettier on its defaults, where proseWrap is "preserve"
+# and every wrap this check exists to enforce passes. Missing the file is
+# a hard error instead. --no-editorconfig so that .prettierrc alone sets
+# the style.
+RUN prettier --config .prettierrc --no-editorconfig --check "**/*.md"
+
+# Write path. Not a check: script/fmt builds this and takes the files.
+FROM nodedeps AS fmt
+
+COPY . .
+
+RUN prettier --config .prettierrc --no-editorconfig --write "**/*.md"
+
+# Only the markdown leaves, with its paths intact, so that the export
+# below cannot put anything else back over the caller's working tree.
+RUN mkdir -p /out && cd /src && \
+    find . -name '*.md' -type f -exec cp --parents '{}' /out/ ';'
+
+# Export target: `docker build --target fmt-out --output type=local`
+# writes /out's tree into a directory on the client, which is how
+# script/fmt gets formatted markdown back without a bind mount.
+# Must match $stage in script/fmt.
+FROM scratch AS fmt-out
+COPY --from=fmt /out/ /

Makefile

@@ -1,7 +1,13 @@
-.PHONY: all bootstrap setup build lint fmt fmt-check test check clean hooks docker
+.PHONY: all bootstrap setup build version lint fmt fmt-check test check clean hooks docker
 
 BINARY := dnswatcher
-VERSION := $(shell git describe --tags --always --dirty 2>/dev/null || echo "dev")
+# VERSION given on the command line (`make build VERSION=...`) or in the
+# environment, which is how the Dockerfile's VERSION build arg arrives,
+# wins over what `git describe` says of this checkout. An empty one counts
+# as not given; `override` is what replaces an empty command-line value.
+ifeq ($(VERSION),)
+override VERSION := $(shell git describe --tags --always --dirty 2>/dev/null || echo "dev")
+endif
 LDFLAGS := -X main.Version=$(VERSION)
 
 # Standard targets are thin shims; the implementations live in script/
@@ -19,6 +25,10 @@ setup:
 build:
 	go build -ldflags "$(LDFLAGS)" -o bin/$(BINARY) ./cmd/dnswatcher
 
+# Prints the version `make build` stamps; the Dockerfile checks it.
+version:
+	@echo "$(VERSION)"
+
 test:
 	@script/test
 

README.md

@@ -1,16 +1,20 @@
 # dnswatcher
 
-dnswatcher is an MIT-licensed, pre-1.0 Go daemon by [@sneak](https://sneak.berlin) that monitors DNS records, TCP port availability, and TLS certificates, delivering real-time change notifications via Slack, Mattermost, and ntfy webhooks.
+dnswatcher is an MIT-licensed, pre-1.0 Go daemon by
+[@sneak](https://sneak.berlin) that monitors DNS records, TCP port availability,
+and TLS certificates, delivering real-time change notifications via Slack,
+Mattermost, and ntfy webhooks.
 
-> ⚠️ Pre-1.0 software. APIs, configuration, and behavior may change without notice.
+> ⚠️ Pre-1.0 software. APIs, configuration, and behavior may change without
+> notice.
 
-dnswatcher watches configured DNS domains and hostnames for changes, monitors TCP
+dnswatcher watches configured DNS domains and hostnames for changes, monitors
-port availability, tracks TLS certificate expiry, and delivers real-time
+TCP port availability, tracks TLS certificate expiry, and delivers real-time
 notifications via Slack, Mattermost, and/or ntfy webhooks.
 
-It performs all DNS resolution itself via iterative (non-recursive) queries,
+It resolves the names it watches itself via iterative (non-recursive) queries,
-tracing from root nameservers to authoritative servers directly—never relying
+tracing from root nameservers to authoritative servers directly—never relying on
-on upstream recursive resolvers.
+upstream recursive resolvers.
 
 State is persisted to a local JSON file so that monitoring survives restarts
 without requiring an external database.
@@ -19,21 +23,43 @@ without requiring an external database.
 
 ## No DNS mocking. Ever.
 
-**DNS is never mocked in this project — not in tests, not anywhere else.**
+**DNS is never mocked in this project — not in tests, not anywhere else.** No
-No mock resolvers, no fake DNS servers, no stubbed lookups.
+mock resolvers, no fake DNS servers, no stubbed lookups.
 
-dnswatcher's entire purpose is correct behavior against the real DNS.
+dnswatcher's entire purpose is correct behavior against the real DNS. Tests
-Tests exercise real iterative resolution against live nameservers by
+exercise real iterative resolution against live nameservers by design; a test
-design; a test suite that passes against a mock proves nothing about the
+suite that passes against a mock proves nothing about the one thing this program
-one thing this program exists to do.
+exists to do.
 
-When live tests are flaky, that is a robustness problem, and it gets
+When live tests are flaky, that is a robustness problem, and it gets fixed with
-fixed with robustness: retries with backoff, querying multiple
+robustness: retries with backoff, querying multiple independent nameservers,
-independent nameservers, longer timeouts — or explicit opt-in gating
+longer timeouts — or explicit opt-in gating decided by the project owner. Never
-decided by the project owner. Never with mocks.
+with mocks.
 
-Contributions that introduce mocked, faked, or stubbed DNS will be
+Contributions that introduce mocked, faked, or stubbed DNS will be rejected.
-rejected.
+
+---
+
+## Getting Started
+
+You need git and Docker. This builds the image and runs dnswatcher watching
+`example.com` and `www.example.com`:
+
+```sh
+git clone https://git.eeqj.de/sneak/dnswatcher.git
+cd dnswatcher
+docker build -t dnswatcher .
+docker run -d --name dnswatcher \
+  -p 8080:8080 \
+  -v dnswatcher-data:/var/lib/dnswatcher \
+  -e DNSWATCHER_TARGETS=example.com,www.example.com \
+  dnswatcher
+```
+
+The build also runs the linter and the test suite, which queries live DNS. Once
+the container is running, the dashboard is at <http://localhost:8080/>. With no
+notification endpoint set, changes show only on the dashboard; see
+[Configuration](#configuration) to add one.
 
 ---
 
@@ -43,74 +69,101 @@ rejected.
 
 - Accepts a list of DNS domain names (apex domains, identified via the
   [Public Suffix List](https://publicsuffix.org/)).
-- Every **1 hour**, performs a full iterative trace from root servers to
+- Every **1 hour** by default, performs a full iterative trace from root servers
-  discover all authoritative nameservers (NS records) for each domain.
+  to discover all authoritative nameservers (NS records) for each domain.
 - Queries **every** discovered authoritative nameserver independently.
-- Stores the NS record set as observed by the delegation chain.
+- Stores the domain's NS record set, as its parent zone's servers delegate it,
+  and the IPv4 and IPv6 addresses each nameserver's name resolves to.
 - Any change triggers a notification:
-  - NS added to or removed from the delegation.
+    - NS added to or removed from that set.
-  - NS IP address changed (glue record change).
+    - NS address change: a nameserver that stays in the set resolves to
+      different addresses than on the previous check. A nameserver added or
+      removed gets only the NS change notification. When the lookup of a
+      nameserver's addresses fails or finds none, its previous addresses are
+      kept and nothing is sent.
 
 ### DNS Hostname Monitoring (Subdomains)
 
-- Accepts a list of DNS hostnames (subdomains, distinguished from apex
+- Accepts a list of DNS hostnames (subdomains, distinguished from apex domains
-  domains via the Public Suffix List).
+  via the Public Suffix List).
-- Every **1 hour**, performs a full iterative trace to discover the
+- Every **1 hour** by default, performs a full iterative trace to discover the
-  authoritative nameservers for the hostname's parent domain.
+  authoritative nameservers of the zone the hostname is in, which is not always
-- Queries **each** authoritative nameserver independently for **all**
+  its last two labels (a name under `co.uk`, or in a delegated subdomain).
-  record types: A, AAAA, CNAME, MX, TXT, SRV, CAA, NS.
+- Queries **each** authoritative nameserver independently for **all** record
-- Stores results **per nameserver**. The state for a hostname is not a
+  types: A, AAAA, CNAME, MX, TXT, SRV, CAA, NS.
-  merged view — it is a map from nameserver to record set.
+- Stores results **per nameserver**. The state for a hostname is not a merged
-- Any observable change in any nameserver's response triggers a
+  view — it is a map from nameserver to record set.
-  notification. This includes:
+- DNS names inside record values (CNAME, MX, SRV and NS targets) are stored in
-  - **Record change**: A nameserver returns different records than it
+  lower case, because names are case-insensitive and nameservers may answer in
-    did on the previous check (additions, removals, value changes).
+  any letter case. TXT and CAA values keep their letter case; they are not
-  - **NS query failure**: A nameserver that previously responded
+  lower-cased.
-    becomes unreachable (timeout, SERVFAIL, REFUSED, network error).
+- Any observable change in any nameserver's response triggers a notification.
-    This is distinct from "responded with no records."
+  This includes:
-  - **NS recovery**: A previously-unreachable nameserver starts
+    - **Record change**: A nameserver returns different records than it did on
-    responding again.
+      the previous check (additions, removals, value changes).
-  - **Inconsistency detected**: Two nameservers that previously agreed
+    - **NS query failure**: A nameserver that previously responded becomes
-    now return different record sets for the same hostname.
+      unreachable (timeout, SERVFAIL, REFUSED, network error). This is distinct
+      from "responded with no records": a nameserver that answers NXDOMAIN or
+      with no records has responded. The alert is sent once, on the check where
+      it starts failing. A failing nameserver gives no records, so it is not
+      reported as a record change or compared for inconsistency. A nameserver
+      that is already failing on the first check that sees it is recorded
+      silently.
+    - **NS recovery**: A previously-unreachable nameserver starts responding
+      again. Its records are not compared with those from before it failed, so a
+      change made while it was failing is not reported as a record change.
+    - **Inconsistency detected**: Two nameservers return different record sets
+      for the same hostname and did not already differ on the previous check.
+      Every pair of nameservers is compared. The alert is sent once for each
+      such pair, on the check where they start to disagree, and not again while
+      they keep disagreeing, including after a restart. A nameserver that was
+      not in the previous check (newly added, or back after dropping out), or
+      failed on it, and answers differently is reported on the check where it
+      answers. If a pair agrees again and later disagrees, the alert is sent
+      again.
 
 ### TCP Port Monitoring
 
-- For every configured domain and hostname, constructs a deduplicated list
+- For every configured domain and hostname, constructs a deduplicated list of
-  of all IPv4 and IPv6 addresses resolved via A, AAAA, and CNAME chain
+  the IPv4 and IPv6 addresses in the A and AAAA records its authoritative
-  resolution across all authoritative nameservers.
+  nameservers returned. A CNAME is not followed: a name whose CNAME points into
+  another zone usually has no addresses here, so its ports and certificate are
+  not checked.
 - Checks TCP connectivity on ports **80** and **443** for each IP address.
-- Every **1 hour**, re-checks all ports.
+- Every **1 hour** by default, re-checks all ports.
 - Any change in port availability triggers a notification:
     - Port transitioned from open to closed (or vice versa).
-  - New IP appeared (from DNS change) and its port state was recorded.
+    - New IP appeared (from DNS change): its port state is recorded without a
-  - IP disappeared (from DNS change) — noted in the DNS change
+      port notification; the DNS change notification shows the new address.
-    notification; port state for that IP is removed.
+    - IP disappeared (from DNS change) — noted in the DNS change notification;
+      port state for that IP is removed. When none of a name's nameservers
+      answered, its addresses are not known, so the port state saved for them is
+      kept.
 
 ### TLS Certificate Monitoring
 
-- Every **12 hours**, for each IP address listening on port 443, connects
+- Every **12 hours** by default, for each IP address listening on port 443,
-  via TLS using the correct SNI hostname.
+  connects via TLS using the correct SNI hostname.
 - Records the certificate's Subject CN, SANs, issuer, and expiry date.
 - Any change triggers a notification:
-  - Certificate is expiring within **7 days** (warning, repeated each
+    - Certificate is expiring within **7 days** by default (warning, repeated
-    check until renewed or expired).
+      each check until renewed or expired).
-  - Certificate CN, issuer, or SANs changed (replacement detected,
+    - Certificate CN, issuer, or SANs changed (replacement detected, reports old
-    reports old and new values).
+      and new CN and issuer).
-  - TLS connection failure to a previously-reachable IP:443 (handshake
+    - TLS connection failure to a previously-reachable IP:443 (handshake error,
-    error, timeout, connection refused after previously succeeding).
+      timeout, connection refused after previously succeeding).
-  - TLS recovery: a previously-failing IP:443 now completes a
+    - TLS recovery: a previously-failing IP:443 now completes a handshake again.
-    handshake again.
 
 ### Notifications
 
 **Every observable state change produces a notification.** dnswatcher is
-designed as a real-time change feed — degradations, failures, recoveries,
+designed as a real-time change feed — degradations, failures, recoveries, and
-and routine changes are all reported equally.
+routine changes are all reported equally.
 
 Supported notification backends:
 
 | Backend        | Configuration                              | Payload Format               |
-|----------------|--------------------------|------------------------------|
+| -------------- | ------------------------------------------ | ---------------------------- |
 | **Slack**      | Incoming Webhook URL                       | Attachments with color       |
 | **Mattermost** | Incoming Webhook URL                       | Slack-compatible attachments |
 | **ntfy**       | Topic URL (e.g. `https://ntfy.sh/mytopic`) | Title + body + priority      |
@@ -118,27 +171,39 @@ Supported notification backends:
 All configured endpoints receive every notification. Notification content
 includes:
 
-- **DNS record changes**: Which hostname, which nameserver, what record
+- **DNS record changes**: Which hostname, which nameserver, what record type,
-  type, old values, new values.
+  old values, new values.
 - **DNS NS changes**: Which domain, which nameservers were added/removed.
-- **NS query failures**: Which nameserver failed, error type (timeout,
+- **NS address changes**: Which domain, which nameserver, its old and new
-  SERVFAIL, REFUSED, network error), which hostname/domain affected.
+  addresses.
+- **NS query failures**: Which nameserver failed, error type (timeout, SERVFAIL,
+  REFUSED, network error), which hostname/domain affected.
 - **NS recoveries**: Which nameserver recovered, which hostname/domain.
-- **NS inconsistencies**: Which nameservers disagree, what each one
+- **NS inconsistencies**: Which nameservers disagree, what each one returned,
-  returned, which hostname affected.
+  which hostname affected.
-- **Port changes**: Which IP:port, old state, new state, all associated
+- **Port changes**: Which IP:port, its new state, all associated hostnames.
-  hostnames.
+- **TLS expiry warnings**: Expiry date and days remaining, CN, associated
-- **TLS expiry warnings**: Which certificate, days remaining, CN,
-  issuer, associated hostname and IP.
-- **TLS certificate changes**: Old and new CN/issuer/SANs, associated
   hostname and IP.
+- **TLS certificate changes**: Old and new CN and issuer, associated hostname
+  and IP. A change to the SANs alone is notified, but the SANs are not listed.
 - **TLS connection failures/recoveries**: Which IP:port, error details,
   associated hostname.
 
+Each endpoint is sent each notification on its own, in the background. A
+delivery that fails (a network error, no reply within 10 seconds, or an HTTP
+status of 400 or more) is retried up to 5 times: the first retry after about 1
+second, each wait after that twice as long up to 60 seconds, every wait varied
+at random by up to 25%. A delivery still failing after that is logged and
+dropped.
+
+The last 100 notifications, delivered or not, are kept in memory for the
+dashboard's Recent alerts. They are not saved to the state file, so a restart
+clears them.
+
 ### State Management
 
-- All monitoring state is kept in memory and persisted to a JSON file on
+- All monitoring state is kept in memory and persisted to a JSON file on disk
-  disk (`DATA_DIR/state.json`).
+  (`DATA_DIR/state.json`).
 - State is loaded on startup to resume monitoring without triggering
   false-positive change notifications.
 - State is written atomically (write to temp file, then rename) to prevent
@@ -146,87 +211,116 @@ includes:
 
 ### Web Dashboard
 
-dnswatcher includes an unauthenticated, read-only web dashboard at the
+dnswatcher includes an unauthenticated, read-only web dashboard at the root URL
-root URL (`/`). It displays:
+(`/`). It displays:
 
-- **Summary counts** for monitored domains, hostnames, ports, and
+- **Summary counts** for monitored domains, hostnames, ports, and certificates.
-  certificates.
 - **Domains** with their discovered nameservers.
 - **Hostnames** with per-nameserver DNS records and status.
 - **Ports** with open/closed state and associated hostnames.
 - **TLS certificates** with CN, issuer, expiry, and status.
-- **Recent alerts** (last 100 notifications sent since the process
+- **Recent alerts** (last 100 notifications sent since the process started),
-  started), displayed in reverse chronological order.
+  displayed in reverse chronological order.
 
-Every data point shows its age (e.g. "5m ago") so you can tell at a
+Every data point shows its age (e.g. "5m ago") so you can tell at a glance how
-glance how fresh the information is. The page auto-refreshes every 30
+fresh the information is. The page auto-refreshes every 30 seconds.
-seconds.
 
-The dashboard intentionally does not expose any configuration details
+The dashboard intentionally does not expose any configuration details such as
-such as webhook URLs, notification endpoints, or API tokens.
+webhook URLs, notification endpoints, or API tokens.
 
-All assets (CSS) are embedded in the binary and served from the
+All assets (CSS) are embedded in the binary and served from the application
-application itself. The dashboard makes zero external HTTP requests —
+itself. The dashboard makes zero external HTTP requests — no CDN dependencies or
-no CDN dependencies or third-party resources are loaded at runtime.
+third-party resources are loaded at runtime.
 
 ### HTTP API
 
 dnswatcher exposes a lightweight HTTP API for operational visibility:
 
 | Endpoint                       | Description                   |
-|---------------------------------------|--------------------------------|
+| ------------------------------ | ----------------------------- |
 | `GET /`                        | Web dashboard (HTML)          |
 | `GET /s/...`                   | Static assets (embedded CSS)  |
 | `GET /.well-known/healthcheck` | Health check (JSON)           |
 | `GET /health`                  | Health check (JSON, legacy)   |
 | `GET /api/v1/status`           | Current monitoring state      |
-| `GET /metrics`                        | Prometheus metrics (optional)  |
+| `GET /metrics`                 | Prometheus metrics, see below |
 
----
+`/metrics` is served only when `DNSWATCHER_METRICS_USERNAME` is set, behind
+Basic Auth. It has the Prometheus Go client's default metrics only (Go runtime,
+process, and counts of `/metrics` requests); dnswatcher records no metrics of
+its own.
 
-## Architecture
+Every route but `/metrics` may be read from a page on any origin: a cross-origin
+`GET` gets `Access-Control-Allow-Origin: *`. Only `GET` is allowed cross-origin,
+and without credentials. `/metrics` sends no CORS headers.
+
+#### Server timeouts
+
+The HTTP server sets all four socket-level timeouts. These are compile-time
+constants in `internal/server/server.go`, not configurable via environment
+variables.
+
+| Timeout             | Value | Purpose                                       |
+| ------------------- | ----- | --------------------------------------------- |
+| `ReadHeaderTimeout` | 10s   | Bounds the request header read (slowloris)    |
+| `ReadTimeout`       | 15s   | Bounds the whole request read, headers + body |
+| `WriteTimeout`      | 75s   | Bounds handler execution plus response flush  |
+| `IdleTimeout`       | 120s  | Reaps idle keep-alive connections             |
+
+These are distinct from the 60s per-request handler budget applied by
+`chimw.Timeout` in `internal/server/routes.go`, which cancels the request
+context but does not touch the socket. `WriteTimeout` is deliberately larger
+than that budget: the write deadline is armed once request headers are read, so
+a smaller value would sever the connection before a handler using its full
+budget could respond. `IdleTimeout` exceeds common Prometheus scrape intervals
+so the scraper reuses its connection.
+
+### Security Headers
+
+Every response — the dashboard, the static assets under `/s/...`, the
+healthchecks, the JSON API, and `/metrics` — carries the following headers, set
+by a global middleware:
+
+| Header                      | Value                                 |
+| --------------------------- | ------------------------------------- |
+| `Strict-Transport-Security` | `max-age=31536000; includeSubDomains` |
+| `Content-Security-Policy`   | see below                             |
+| `X-Frame-Options`           | `DENY`                                |
+| `X-Content-Type-Options`    | `nosniff`                             |
+| `Referrer-Policy`           | `no-referrer`                         |
+| `Permissions-Policy`        | all unused browser features denied    |
+
+The content security policy is:
 
 ```
-cmd/dnswatcher/main.go         Entry point (uber/fx bootstrap)
+default-src 'self'; script-src 'none'; style-src 'self'; img-src 'self';
-
+font-src 'none'; connect-src 'none'; object-src 'none'; base-uri 'none';
-internal/
+form-action 'none'; frame-ancestors 'none'
-  config/config.go             Viper-based configuration
-  globals/globals.go           Build-time variables (version)
-  logger/logger.go             slog structured logging (TTY detection)
-  healthcheck/healthcheck.go   Health check service
-  middleware/middleware.go      HTTP middleware (logging, CORS, metrics auth)
-  handlers/handlers.go         HTTP request handlers
-  server/
-    server.go                  HTTP server lifecycle
-    routes.go                  Route definitions
-  state/state.go               JSON file state persistence
-  resolver/resolver.go         Iterative DNS resolution engine
-  portcheck/portcheck.go       TCP port connectivity checker
-  tlscheck/tlscheck.go        TLS certificate inspector
-  notify/notify.go             Notification service (Slack, Mattermost, ntfy)
-  watcher/watcher.go           Main monitoring orchestrator and scheduler
 ```
 
-### Design Principles
+The dashboard ships no JavaScript (the 30-second refresh is a
+`<meta http-equiv="refresh">`), no inline styles, no inline event handlers, and
+no images; its only subresource is the embedded stylesheet at
+`/s/css/tailwind.min.css`, which `style-src 'self'` permits. The policy
+therefore needs neither `unsafe-inline` nor `unsafe-eval`.
+`frame-ancestors 'none'` is the primary anti-framing control, with
+`X-Frame-Options: DENY` retained as the legacy fallback.
 
-- **No recursive resolvers**: All DNS resolution is performed iteratively,
+HSTS is emitted unconditionally, including over plain HTTP. dnswatcher is
-  tracing from root nameservers through the delegation chain to
+expected to run behind a TLS-terminating reverse proxy, and the browser must
-  authoritative servers.
+still be told to enforce HTTPS end to end, so the header is never gated on
-- **No external database**: State is persisted as a single JSON file.
+whether the request itself arrived over TLS.
-- **Dependency injection**: All components are wired via
+
-  [uber/fx](https://github.com/uber-go/fx).
+`Referrer-Policy: no-referrer` is stricter than the
-- **Structured logging**: All logs use `log/slog` with JSON output in
+`strict-origin-when-cross-origin` baseline: the dashboard has no cross-origin
-  production (TTY detection for development).
+navigation needs, and its URL may name internal hosts.
-- **Graceful shutdown**: All background goroutines respect context
-  cancellation and the fx lifecycle. In-flight notification deliveries
-  are drained on shutdown, bounded by the shutdown timeout.
 
 ---
 
 ## Configuration
 
-Configuration is loaded via [Viper](https://github.com/spf13/viper) with
+Configuration is loaded via [Viper](https://github.com/spf13/viper) with the
-the following precedence (highest to lowest):
+following precedence (highest to lowest):
 
 1. Environment variables (prefixed with `DNSWATCHER_`)
 2. `.env` file (loaded via godotenv)
@@ -237,7 +331,7 @@ the following precedence (highest to lowest):
 ### Environment Variables
 
 | Variable                            | Description                                                                                                 | Default               |
-|---------------------------------|--------------------------------------------|-------------|
+| ----------------------------------- | ----------------------------------------------------------------------------------------------------------- | --------------------- |
 | `PORT`                              | HTTP listen port                                                                                            | `8080`                |
 | `DNSWATCHER_DEBUG`                  | Enable debug logging                                                                                        | `false`               |
 | `DNSWATCHER_DATA_DIR`               | Directory for state file                                                                                    | `/var/lib/dnswatcher` |
@@ -245,20 +339,44 @@ the following precedence (highest to lowest):
 | `DNSWATCHER_SLACK_WEBHOOK`          | Slack incoming webhook URL                                                                                  | `""`                  |
 | `DNSWATCHER_MATTERMOST_WEBHOOK`     | Mattermost incoming webhook URL                                                                             | `""`                  |
 | `DNSWATCHER_NTFY_TOPIC`             | ntfy topic URL                                                                                              | `""`                  |
-| `DNSWATCHER_DNS_INTERVAL`       | DNS check interval                         | `1h`        |
+| `DNSWATCHER_DNS_INTERVAL`           | DNS check interval, a positive duration such as `30m`; empty means the default, anything else stops startup | `1h`                  |
-| `DNSWATCHER_TLS_INTERVAL`       | TLS check interval                         | `12h`       |
+| `DNSWATCHER_TLS_INTERVAL`           | TLS check interval, a positive duration such as `6h`; empty means the default, anything else stops startup  | `12h`                 |
 | `DNSWATCHER_TLS_EXPIRY_WARNING`     | Days before expiry to warn                                                                                  | `7`                   |
 | `DNSWATCHER_SENTRY_DSN`             | Sentry DSN for error reporting                                                                              | `""`                  |
-| `DNSWATCHER_MAINTENANCE_MODE`   | Enable maintenance mode                    | `false`     |
+| `DNSWATCHER_MAINTENANCE_MODE`       | Only sets `maintenanceMode` in the health check response; changes nothing else                              | `false`               |
-| `DNSWATCHER_METRICS_USERNAME`   | Basic auth username for /metrics           | `""`        |
+| `DNSWATCHER_METRICS_USERNAME`       | Basic auth username for /metrics, which is served only when this is set                                     | `""`                  |
 | `DNSWATCHER_METRICS_PASSWORD`       | Basic auth password for /metrics                                                                            | `""`                  |
 | `DNSWATCHER_SEND_TEST_NOTIFICATION` | Send a test notification after first scan completes                                                         | `false`               |
 
 **`DNSWATCHER_TARGETS` is required.** dnswatcher will refuse to start if no
 monitoring targets are configured. A monitoring daemon with nothing to monitor
 is a misconfiguration, so dnswatcher fails fast with a clear error message
-rather than running silently. Set `DNSWATCHER_TARGETS` to a comma-separated
+rather than running silently. Set `DNSWATCHER_TARGETS` to a comma-separated list
-list of DNS names before starting.
+of DNS names before starting. A name listed more than once, in any letter case
+or with a trailing dot, is watched once.
+
+**`/metrics` is rate limited.** Each client address may send it 30 requests a
+minute, failed logins included; beyond that it answers `429 Too Many Requests`
+without checking the password. A Prometheus server scraping every 15 seconds
+sends 4 a minute. IPv6 addresses in one /64 count as one client. When the
+request comes from a private or loopback address, such as a reverse proxy's, the
+client address is taken from the `X-Real-IP` header the proxy sets, or else from
+`X-Forwarded-For`, as the last address in it that is not private or loopback. A
+proxy that sets neither makes all its clients share one allowance.
+
+**`DNSWATCHER_DNS_INTERVAL` and `DNSWATCHER_TLS_INTERVAL`** take a positive
+duration: a number followed by a unit such as `s`, `m` or `h`, for example
+`90s`, `30m`, `1h` or `1h30m`. There is no unit for days; write `24h`. An unset
+or empty variable (`DNSWATCHER_DNS_INTERVAL=`) means the default. If either is
+set to anything else, including a bare number or a zero or negative duration,
+dnswatcher refuses to start with an error naming the variable and the value.
+
+**`DNSWATCHER_SENTRY_DSN` reports crashes in HTTP requests to Sentry.** When it
+is set, a panic in an HTTP request handler is sent to Sentry, and the request
+still gets a `500 Internal Server Error` answer. Nothing else is sent to Sentry:
+DNS, port and TLS problems are reported as notifications. A value Sentry cannot
+parse stops dnswatcher at startup. At shutdown, reports not yet sent are sent,
+waiting at most 2 seconds.
 
 ### Example `.env`
 
@@ -277,33 +395,45 @@ DNSWATCHER_SEND_TEST_NOTIFICATION=true
 
 ## DNS Resolution Strategy
 
-dnswatcher never uses the system's configured recursive resolver.  Instead,
+dnswatcher never uses the system's configured recursive resolver for the names
-it performs full iterative resolution:
+it watches. Instead, it performs full iterative resolution:
 
-1. **Root servers**: Starts from the IANA root nameserver list (hardcoded,
+1. **Root servers**: Starts from the IPv4 addresses of the 13 root servers,
-   with periodic refresh).
+   built into the binary; the list is not refreshed.
 2. **TLD delegation**: Queries root servers for the TLD NS records.
-3. **Domain delegation**: Queries TLD nameservers for the domain's NS
+3. **Domain delegation**: Queries TLD nameservers for the domain's NS records.
-   records.
+   The delegation they give, from the domain's parent zone, is the domain's NS
-4. **Authoritative query**: Queries all discovered authoritative
+   record set.
-   nameservers directly for the requested records.
+4. **Authoritative query**: Queries all discovered authoritative nameservers
+   directly for the requested records.
+
+In steps 2 and 3 the servers are asked one at a time in a random order, chosen
+anew each time, so no one root server gets every first query. A server that does
+not reply, refuses the query, or gives an error reply such as SERVFAIL or a
+referral that leads no closer to the name is passed over for the next one. When
+a referral names a zone's nameservers without their addresses, the addresses of
+all of them are looked up, so that each can be asked.
 
 This approach ensures:
-- Independence from any upstream resolver's cache or filtering.
-- Ability to detect split-horizon or inconsistent responses across
-  authoritative servers.
-- Visibility into the full delegation chain.
 
-For hostname monitoring, the resolver follows CNAME chains (with a
+- Independence from any upstream resolver's cache or filtering.
-depth limit to prevent loops) before collecting terminal A/AAAA records.
+- Ability to detect split-horizon or inconsistent responses across authoritative
+  servers.
+
+CNAME chains are followed (with a depth limit to prevent loops) only to find the
+addresses of nameservers. A watched name's records are stored as its nameservers
+return them, CNAME included, without following it.
+
+Sending a notification or a Sentry report is the one use of the system's
+resolver: the HTTP client looks up the webhook's or Sentry's host name with it.
 
 ---
 
 ## State File Format
 
 The state file (`DATA_DIR/state.json`) contains the complete monitoring
-snapshot.  Hostname records are stored **per authoritative nameserver**,
+snapshot. Hostname records are stored **per authoritative nameserver**, not as a
-not as a merged view, to enable inconsistency detection.
+merged view, to enable inconsistency detection.
 
 ```json
 {
@@ -312,6 +442,10 @@ not as a merged view, to enable inconsistency detection.
     "domains": {
         "example.com": {
             "nameservers": ["ns1.example.com.", "ns2.example.com."],
+            "nameserverAddresses": {
+                "ns1.example.com.": ["192.0.2.53", "2001:db8::53"],
+                "ns2.example.com.": ["198.51.100.53"]
+            },
             "lastChecked": "2026-02-19T12:00:00Z"
         }
     },
@@ -363,13 +497,26 @@ not as a merged view, to enable inconsistency detection.
 }
 ```
 
-The `status` field for each per-nameserver entry and certificate entry
+The `status` field for each per-nameserver entry and certificate entry tracks
-tracks reachability:
+reachability:
 
 | Status  | Meaning                                                  |
-|-------------|-------------------------------------------------|
+| ------- | -------------------------------------------------------- |
 | `ok`    | Query succeeded, records are current                     |
-| `error`     | Query failed (timeout, SERVFAIL, network error)  |
+| `error` | Query failed (timeout, SERVFAIL, REFUSED, network error) |
+
+A nameserver that answers NXDOMAIN or with no records has status `ok` and empty
+`records`. A nameserver whose query failed, or that only referred it to other
+nameservers, has status `error`, empty `records`, and the reason in `error`. A
+certificate entry whose TLS connection or handshake failed likewise has status
+`error`, the reason in `error`, and the certificate fields left empty or zero.
+
+`nameserverAddresses` lists, by nameserver, the sorted addresses its name
+resolves to. A state file without it loads, and the next check fills it in
+without a notification.
+
+A port entry in the older format, with one `hostname` instead of the `hostnames`
+list, loads as a list of that one name.
 
 ---
 
@@ -378,56 +525,86 @@ tracks reachability:
 This repository adheres to the
 [Scripts to Rule Them All](https://github.com/github/scripts-to-rule-them-all)
 standard: normalized scripts in `script/` are the entrypoints for the
-development workflow, and the Makefile targets are thin shims that call
+development workflow, and the Makefile targets are thin shims that call them. We
-them. We provide:
+provide:
 
-- `script/bootstrap` — install all dependencies (go, pinned goimports,
+- `script/bootstrap` — install all dependencies (go, `go mod download`). It does
-  `go mod download`). It does not install golangci-lint: see
+  not install golangci-lint or prettier: both run in Docker, see `script/lint`
-  `script/lint` below.
+  and `script/fmt` below.
-- `script/setup` — make a fresh clone ready for development: bootstrap
+- `script/setup` — make a fresh clone ready for development: bootstrap plus the
-  plus the git pre-commit hook
+  git pre-commit hook
-- `script/projectname` — print the project name (used for the Docker
+- `script/projectname` — print the project name (used for the Docker image tag)
-  image tag)
+- `script/test` — run the test suite (race detector, coverage). Caching is
-- `script/test` — run the test suite (race detector, coverage). Caching
+  waived for testing, exactly as it is for linting: `-count=1` forces every
-  is waived for testing, exactly as it is for linting: `-count=1`
+  invocation to execute, because the suite queries live DNS and a cached pass
-  forces every invocation to execute, because the suite queries live
+  queries nothing. Failures are rerun with `-v` automatically, and the build
-  DNS and a cached pass queries nothing. Failures are rerun with `-v`
+  fails even if that rerun passes.
-  automatically, and the build fails even if that rerun passes.
 - `script/lint` — run golangci-lint, always inside Docker: it builds
-  `Dockerfile.lint`, which COPYs the repo into the digest-pinned
+  `Dockerfile.lint`, which COPYs the repo into the digest-pinned `golangci-lint`
-  `golangci-lint` image and lints as a build step, so a successful
+  image and lints as a build step, so a successful build is a clean lint. The
-  build is a clean lint. The linter is never installed or run on the
+  linter is never installed or run on the host, and Docker is the only
-  host, and Docker is the only prerequisite. Caching is waived for
+  prerequisite. Caching is waived for linting: the lint stage is forced to
-  linting: the lint stage is forced to execute on every run with
+  execute on every run with `--no-cache-filter`, because a cached build lints
-  `--no-cache-filter`, because a cached build lints nothing.
+  nothing.
-- `script/fmt` — format all code (gofmt -s, goimports)
+- `script/fmt` — format all code (gofmt -s, goimports) and all Markdown
-- `script/fmt-check` — check formatting (read-only)
+  (prettier). goimports runs with `go run` at a pinned commit, never from your
+  `PATH`. prettier runs inside Docker, built from `Dockerfile.fmt` on a
+  digest-pinned node image, at the version pinned by `package.json` and
+  `yarn.lock`; it is never installed on the host.
+- `script/fmt-check` — check formatting (read-only) with the same tools, failing
+  on any file `script/fmt` would change. It runs the two scripts below.
+- `script/fmt-check-go` — the gofmt and goimports half, on the host. The
+  `Dockerfile` lint stage runs it.
+- `script/fmt-check-markdown` — the prettier half, inside Docker, forced to
+  execute on every run with `--no-cache-filter`
 - `script/check` — run test, lint, and fmt-check
-- `script/docker` — build the Docker image tagged via
+- `script/docker` — build the Docker image tagged via `script/projectname`, with
-  `script/projectname`
+  `--no-cache-filter=lint,builder` so the lint stage and the builder stage,
-- `script/cibuild` — CI entrypoint: plain `docker build .`
+  which runs the tests, run on every invocation, and with the version from
-- `script/precommit` — run by the git pre-commit hook; `go mod tidy`
+  `git describe` passed as `--build-arg VERSION`
-  guard, then `script/check`
+- `script/cibuild` — CI entrypoint: `docker build` with
+  `--no-cache-filter=lint,builder`, so the lint stage and the builder stage,
+  which runs the tests, run on every invocation, because a cached build lints
+  nothing and queries no DNS; then `script/fmt-check-markdown`
+- `script/precommit` — run by the git pre-commit hook; `go mod tidy` guard, then
+  `script/check`
 - `script/install-precommit` — install the git pre-commit hook
 
 ## Building
 
 ```sh
 make build      # Build binary to bin/dnswatcher
+make version    # Print the version make build stamps
 make test       # Run tests with race detector
 make lint       # Run golangci-lint in Docker (requires docker)
-make fmt        # Format code
+make fmt        # Format code and Markdown (requires docker)
 make check      # Run all checks (test, lint, fmt-check)
 make clean      # Remove build artifacts
 ```
 
 ### Build-Time Variables
 
-Version is injected via `-ldflags`:
+`make build` sets the version with `-ldflags "-X main.Version=..."`, taking it
+from `VERSION` when given on the command line (`make build VERSION=1.2.3`) or in
+the environment, otherwise from `git describe --tags --always --dirty`, and
+`dev` without git metadata. An empty `VERSION` counts as not given. The version
+appears in the startup log and in the health check response.
 
-```sh
+The image takes it the same way, from the `.git` the build context carries, so a
-go build -ldflags "-X main.Version=$(git describe --tags --always)" ./cmd/dnswatcher
+plain `docker build .` of a clone stamps the commit it was built from; a clone
-```
+without tags stamps the short commit. A clone made with `--depth 1` carries at
+most a tag on its own commit, so such a clone of an untagged commit stamps the
+short commit. In a build from a directory, `.dockerignore` keeps out
+`.git/config`, which `git describe` does not need and which can hold a
+credential. Docker does not apply `.dockerignore` to a context sent as a tar
+archive, as upaas sends it, so that context carries `.git/config` into the
+build. It also keeps its files' owners, so git in the build trusts the checkout
+whoever owns it. A non-empty `--build-arg VERSION=...` takes precedence;
+`make docker` passes the version `git describe` gives on the host. The build
+fails when the context carries `.git`, as a directory or as a file, and the
+version comes out empty, `dev` or `unknown`. `.dockerignore` must list no
+tracked file: git in the build would see it as deleted and mark the version
+`-dirty`.
 
 ---
 
@@ -446,48 +623,167 @@ docker run -d \
 
 ---
 
+## Running under upaas
+
+[upaas](https://git.eeqj.de/sneak/upaas) builds the image from this repository's
+`Dockerfile` and runs it. The app needs:
+
+- **Branch:** `prod`. `prod` is cut from `main`, and merging a `main` to `prod`
+  pull request is a deploy.
+- **Volume:** one host directory mounted at `/var/lib/dnswatcher`, where the
+  state file lives.
+- **Network and port:** the dashboard is unauthenticated and shows every watched
+  name and recent alert, and upaas publishes every mapped port on all interfaces
+  of the host ([upaas issue 113](https://git.eeqj.de/sneak/upaas/issues/113)).
+  Add a port mapping to container port `8080` only if the dashboard should be
+  public. Otherwise add none: set the app's Docker network in upaas to your
+  reverse proxy's Docker network, and the proxy reaches the app at `upaas-`
+  followed by the app name, port `8080`.
+- **Required environment:** `DNSWATCHER_TARGETS`, a comma-separated list of the
+  domains and hostnames to watch. dnswatcher refuses to start without it.
+- **Recommended environment:** at least one notification endpoint
+  (`DNSWATCHER_SLACK_WEBHOOK`, `DNSWATCHER_MATTERMOST_WEBHOOK`,
+  `DNSWATCHER_NTFY_TOPIC`); without one, changes show only on the dashboard.
+  `DNSWATCHER_METRICS_USERNAME` and `DNSWATCHER_METRICS_PASSWORD` serve
+  `/metrics` behind basic auth.
+- **Leave unset:** `DNSWATCHER_DATA_DIR`, which the image sets to
+  `/var/lib/dnswatcher`, and `PORT`, which defaults to `8080`. Every setting
+  comes from the environment; the image holds no config file.
+- **Health check:** the image's own, which requests `/.well-known/healthcheck`
+  every 10 seconds. upaas reads the container's health 60 seconds after a deploy
+  and marks the deploy failed unless it is `healthy`.
+
+---
+
 ## Monitoring Lifecycle
 
-1. **Startup**: Load state from disk. If no state file exists, start
+1. **Startup**: Check that the data directory can be written, and exit with an
-   with empty state (first check will establish baseline without
+   error naming it if not. Load state from disk. If no state file exists, start
-   triggering change notifications).
+   with empty state (first check will establish baseline without triggering
-2. **Initial check**: Immediately perform all DNS, port, and TLS checks
+   change notifications).
-   on startup.
+2. **Initial check**: Immediately perform all DNS, port, and TLS checks on
+   startup.
 3. **Periodic checks** (DNS always runs first):
-   - DNS checks: every `DNSWATCHER_DNS_INTERVAL` (default 1h). Also
+    - DNS checks: every `DNSWATCHER_DNS_INTERVAL` (default 1h). Also re-run
-     re-run before every TLS check cycle to ensure fresh IPs.
+      before every TLS check cycle to ensure fresh IPs.
     - Port checks: every `DNSWATCHER_DNS_INTERVAL`, after DNS completes.
-   - TLS checks: every `DNSWATCHER_TLS_INTERVAL` (default 12h), after
+    - TLS checks: every `DNSWATCHER_TLS_INTERVAL` (default 12h), after DNS
-     DNS completes.
+      completes.
-   - Port and TLS checks always use freshly resolved IP addresses from
+    - Port and TLS checks use the IP addresses found by the DNS phase that
-     the DNS phase that immediately precedes them — never stale IPs
+      immediately precedes them. When that phase cannot find a name's
-     from a previous cycle.
+      nameservers at all, the addresses an earlier check saved for the name are
-4. **On change detection**: Send notifications to all configured
+      used.
-   endpoints, update in-memory state, persist to disk.
+4. **On change detection**: Send notifications to all configured endpoints,
-5. **Shutdown**: Persist final state to disk, wait for in-flight
+   update in-memory state, persist to disk.
-   notification deliveries to complete, stop gracefully. The wait is
+5. **Shutdown**: The watcher stops checking and saves the final state to disk,
-   bounded by the fx shutdown timeout (15s by default): deliveries still
+   and shutdown waits for that save before it goes on. Then it waits for
-   retrying against an unreachable endpoint when that expires are
+   in-flight notification deliveries to complete. Both waits share the fx
-   abandoned, and the number abandoned is logged at warn level rather
+   shutdown timeout (15s by default): deliveries still retrying against an
-   than dropped silently. Notifications generated after shutdown has
+   unreachable endpoint when that expires are abandoned, and the number
-   begun are refused and logged, so a late burst cannot extend the
+   abandoned is logged at warn level rather than dropped silently. Notifications
-   shutdown.
+   generated after shutdown has begun are refused and logged, so a late burst
+   cannot extend the shutdown. A DNS lookup, port check or TLS check that
+   shutdown cuts short saves nothing and sends no notification.
 
 ---
 
 ## Planned Future Features (Post-1.0)
 
-- **DNSSEC validation**: Validate the DNSSEC chain of trust during
+- **DNSSEC validation**: Validate the DNSSEC chain of trust during iterative
-  iterative resolution and report DNSSEC failures as notifications.
+  resolution and report DNSSEC failures as notifications.
 
 ---
 
 ## Project Structure
 
 Follows the conventions defined in `REPO_POLICIES.md`, adapted from the
-[upaas](https://git.eeqj.de/sneak/upaas) project template.  Uses uber/fx
+[upaas](https://git.eeqj.de/sneak/upaas) project template. Uses uber/fx for
-for dependency injection, go-chi for HTTP routing, slog for logging, and
+dependency injection, go-chi for HTTP routing, slog for logging, and Viper for
-Viper for configuration.
+configuration.
+
+---
+
+## Rationale
+
+dnswatcher exists to report changes to the DNS records, TCP port availability
+and TLS certificates of its configured domains and hostnames, failures and
+recoveries included: it is designed as a real-time change feed. It queries the
+authoritative nameservers directly, tracing from the root, instead of a
+recursive resolver, so no resolver's cache or filtering hides a change and
+nameservers that disagree with each other are seen. Its state is a single JSON
+file, so it survives a restart without an external database.
+
+---
+
+## Design
+
+```
+cmd/dnswatcher/main.go         Entry point (uber/fx bootstrap)
+
+internal/
+  config/
+    config.go                  Viper-based configuration
+    classify.go                Splits targets into domains and hostnames
+                               (Public Suffix List)
+  globals/globals.go           Build-time variables (version)
+  logger/logger.go             slog structured logging (TTY detection)
+  healthcheck/healthcheck.go   Health check service
+  middleware/middleware.go     HTTP middleware (logging, CORS, security
+                               headers, metrics auth and rate limit)
+  handlers/
+    handlers.go                Shared handler setup and JSON responses
+    dashboard.go               Web dashboard
+    templates/dashboard.html   Dashboard template (embedded)
+    status.go                  /api/v1/status
+    healthcheck.go             Health check handler
+  server/
+    server.go                  HTTP server lifecycle
+    routes.go                  Route definitions
+  state/state.go               JSON file state persistence
+  resolver/
+    resolver.go                Resolver setup and query status values
+    iterative.go               Iterative DNS resolution engine
+    dns_client.go              UDP and TCP DNS clients
+    errors.go                  Resolver errors
+  portcheck/portcheck.go       TCP port connectivity checker
+  tlscheck/tlscheck.go         TLS certificate inspector
+  notify/
+    notify.go                  Notification service (Slack, Mattermost, ntfy)
+    retry.go                   Delivery retries with backoff
+    history.go                 Last 100 notifications, for the dashboard
+    shutdown.go                Waits for deliveries at shutdown
+  watcher/
+    watcher.go                 Main monitoring orchestrator and scheduler
+    interfaces.go              The resolver, checkers and notifier it uses
+  livednstest/livednstest.go   Retry and concurrency limit for tests
+                               against live DNS (imported only by tests)
+
+static/
+  static.go                    Embeds the CSS served under /s/
+  css/tailwind.min.css         Dashboard stylesheet
+```
+
+### Design Principles
+
+- **No recursive resolvers**: The watched names are resolved iteratively,
+  tracing from root nameservers through the delegation chain to authoritative
+  servers.
+- **No external database**: State is persisted as a single JSON file.
+- **Dependency injection**: All components are wired via
+  [uber/fx](https://github.com/uber-go/fx).
+- **Structured logging**: All logs use `log/slog` with JSON output in production
+  (TTY detection for development).
+- **Graceful shutdown**: All background goroutines respect context cancellation
+  and the fx lifecycle. In-flight notification deliveries are drained on
+  shutdown, bounded by the shutdown timeout.
+
+---
+
+## TODO
+
+[`TODO.md`](./TODO.md) names the next step and the steps planned after it. The
+work for 1.0 is tracked as issues on the
+[1.0 milestone](https://git.eeqj.de/sneak/dnswatcher/milestone/7).
 
 ---
 

TESTING.md

@@ -2,36 +2,43 @@
 
 ## DNS Resolution Tests
 
-All resolver tests **MUST** use live queries against real DNS servers.
+DNS is never mocked in this project, not in tests and not anywhere else; see the
-No mocking of the DNS client layer is permitted.
+README section "No DNS mocking. Ever." Every test that looks something up in DNS
+**MUST** query live DNS servers, never a stand-in. Logic that works on record
+data, such as comparing or formatting records, may be tested on that data
+directly with no lookup.
 
 ### Rationale
 
-The resolver performs iterative resolution from root nameservers through
+The resolver performs iterative resolution from root nameservers through the
-the full delegation chain. Mocked responses cannot faithfully represent
+full delegation chain. Mocked responses cannot faithfully represent the variety
-the variety of real-world DNS behavior (truncation, referrals, glue
+of real-world DNS behavior (truncation, referrals, glue records, DNSSEC, varied
-records, DNSSEC, varied response times, EDNS, etc.). Testing against
+response times, EDNS, etc.). Testing against real servers ensures the resolver
-real servers ensures the resolver works correctly in production.
+works correctly in production.
 
 ### Constraints
 
 - Tests hit real DNS infrastructure and require network access
-- Test duration depends on network conditions; timeout tuning keeps
+- Test duration depends on network conditions; timeout tuning keeps the suite
-  the suite within the 60-second target
+  within the 60-second target
-- Query timeout is calibrated to 3× maximum antipodal RTT (~300ms)
+- Query timeout is calibrated to 3× maximum antipodal RTT (~300ms) plus
-  plus processing margin
+  processing margin
 - Root server fan-out is limited to reduce parallel query load
-- Flaky failures from transient network issues are acceptable and
+- Live lookups that expect an answer go through `internal/livednstest`, which
-  should be investigated as potential resolver bugs, not papered over
+  limits how many run at once in a test binary and retries a lookup that got
-  with mocks or skip flags
+  none
+- Flaky failures from transient network issues are acceptable and should be
+  investigated as potential resolver bugs, not papered over with mocks or skip
+  flags
 
 ### What NOT to do
 
-- **Do not mock `DNSClient`** for resolver tests (the mock constructor
+- **Do not mock, fake or stub DNS** anywhere: no stand-in `DNSClient`, no
-  exists for unit-testing other packages that consume the resolver)
+  stand-in for the watcher's `DNSResolver`, no fake DNS server, no canned
+  responses
 - **Do not add `-short` flags** to skip slow tests
 - **Do not increase `-timeout`** to hide hanging queries
-- **Do not remove `-count=1` from `script/test`** — Go's test cache
+- **Do not remove `-count=1` from `script/test`** — Go's test cache replays a
-  replays a previous run's output without querying anything, so a
+  previous run's output without querying anything, so a cached pass is not
-  cached pass is not evidence that live resolution works
+  evidence that live resolution works
 - **Do not modify linter configuration** to suppress findings

TODO.md

@@ -1,227 +1,138 @@
 # Workflow
 
-* branch (from `main`)
+- branch (from `next`)
-* do the work in Next Step
+- do the work in Next Step
-* move Next Step to the top of Completed Steps
+- move Next Step to the top of Completed Steps
-* move the top item of Future Steps into Next Step
+- move the top item of Future Steps into Next Step
-* commit (`TODO.md` changes in the same commit as the work)
+- commit (`TODO.md` changes in the same commit as the work)
-* merge to `main` if the branch is not protected, otherwise open a PR
+- push
-* push
+- open a PR against `next`
 
 # Status
 
-pre-1.0. No git tags. Core resolver work in flight on feature/resolver
+pre-1.0. No git tags. Work lands on `next` by PR. Open work for 1.0 is tracked
-(dirty: internal/resolver/resolver_test.go). Local checkout has diverged
+on the 1.0 milestone: https://git.eeqj.de/sneak/dnswatcher/milestone/7
-from origin: origin/main is 8 commits ahead (watcher orchestrator,
-unified TARGETS) and origin/feature/resolver already contains the full
-iterative resolver implementation with hermetic mocked tests.
 
 # Next Step
 
-Add the README sections required by policy (Description, Getting Started,
+trial run of the finished image: https://git.eeqj.de/sneak/dnswatcher/issues/149
-Rationale, Design, TODO, License, Author) if any are still missing.
 
 # Completed Steps
 
-- 2026-08-10: comment-only corrections to `script/bootstrap`,
+- 2026-10-02: a plain `docker build .` of a clone stamps its tag or short
-  `script/cibuild`, and `Dockerfile.lint`. The `goimports` pin in
+  commit, not `dev`: the build context now carries `.git` (closes #210).
-  `script/bootstrap` was justified by a claim that `script/fmt-check`
+- 2026-10-02: the resolver tries root servers, and every other server list it
-  runs it on the host; it does not (it runs `gofmt -l .` only), so the
+  walks, in a random order each time, not always from the top (closes #138).
-  header now credits `script/fmt` alone. `script/cibuild` still claimed
+- 2026-10-02: a name listed more than once in `DNSWATCHER_TARGETS`, in any
-  the `Dockerfile` runs `make check`, which stopped being true when
+  letter case or with a trailing dot, is watched once (closes #207).
-  linting moved to its own stage; it now describes the lint stage
+- 2026-10-01: README checked against the code and corrected: metrics, CORS,
-  (`make fmt-check` plus `golangci-lint`) and the builder stage
+  notification retries, CNAMEs, state file fields, Design tree (closes #108).
-  (`make test`, `make build`). The `docker`-missing warning in
+- 2026-10-01: a certificate within the expiry warning period is warned about on
-  `script/bootstrap` reads as one sentence instead of three fragments
+  every TLS check, where some checks used to skip it at random (closes #204).
-  each re-prefixed with `bootstrap:`. `Dockerfile.lint` now records the
+- 2026-10-01: a domain's NS set is its delegation from the parent zone's
-  residual risk of omitting `golangci-lint config verify`: unknown
+  servers, not whichever of its own servers answered first (closes #200).
-  top-level keys in `.golangci.yml` are silently ignored, so a mistyped
+- 2026-10-01: README has Getting Started, Rationale and TODO sections, and its
-  key lints clean while applying nothing. No behaviour changed
+  Architecture section is now Design, in the order policy sets (closes #173).
-- 2026-08-10: MIT `LICENSE` added at the repository root, closing the
+- 2026-10-01: a zone's server that answers SERVFAIL or a referral leading no
-  last gap in `REPO_POLICIES.md`'s required-minimum file list and
+  closer is passed over for the next, as one that times out is (closes #197).
-  removing the all-rights-reserved default that would otherwise have
+- 2026-10-01: when none of a configured name's nameservers answered, the port
-  shipped with a 1.0 tag. The licence choice is the standing org policy
+  state saved for its addresses is kept, not removed (closes #193).
-  (any public repo lacking a licence gets MIT; a private repo with no
+- 2026-10-01: `ResolveIPAddresses` returns an error, not no addresses, when no
-  licence is already all-rights-reserved), and this repo is public. The
+  nameserver of the name's zone answered (closes #190).
-  file holds the canonical MIT text byte-for-byte with only the
+- 2026-10-01: `make fmt` and `make fmt-check` cover Markdown with prettier, run
-  copyright line filled in (`Copyright (c) 2026 sneak`); no clauses were
+  in Docker at the version pinned by `yarn.lock` (closes #119).
-  added, removed, or reflowed. `README.md`'s first line now names the
+- 2026-10-01: `make fmt-check` fails on a file `goimports` would change; both
-  licence, as the Description requirement demands, and the License
+  format scripts run `goimports` at its pinned commit, not from `PATH` (#119).
-  section states MIT and points at the file instead of saying the choice
+- 2026-10-01: a hostname is queried at the servers of the zone it is in, found
-  is pending. `make fmt` covers only Go sources (`gofmt -s`,
+  by following delegations for the name, not its last two labels (closes #189).
-  `goimports`), so it cannot reflow `LICENSE`
+- 2026-10-01: each nameserver's addresses are saved with its domain, and a
-- 2026-08-10: the policy scaffold (`REPO_POLICIES.md`, `.editorconfig`,
+  change while it stays in the delegation is notified (closes #105).
-  `.dockerignore`, `.gitea/workflows/check.yml`, and the `fmt-check`,
+- 2026-10-01: the watcher saves state when it stops, and shutdown waits for that
-  `docker`, and hooks Makefile targets) is present; it landed piecemeal
+  save, so it no longer relies on the state's own stop hook (closes #114).
-  across the scripts-to-rule-them-all and policy commits rather than as
+- 2026-10-01: `DNSWATCHER_SENTRY_DSN` reports panics in HTTP handlers to Sentry,
-  the single commit this file once planned
+  and a DSN Sentry cannot parse stops startup (closes #107).
-- 2026-08-10: Go's test cache disabled for `script/test` via `-count=1`,
+- 2026-10-01: a port or TLS check that shutdown cuts short saves nothing and
-  so every invocation actually executes. A cached pass replays an
+  sends no notification, as a cut-short DNS lookup already did (closes #185).
-  earlier run's output without querying DNS at all, which in this repo
+- 2026-10-01: the client address from `X-Forwarded-For` is the last entry that
-  means the suite's entire premise goes unexercised while the run
+  is not a trusted proxy, not the first, which the client sets (closes #181).
-  reports green in under a second. The conditional verbose rerun that
+- 2026-10-01: a nameserver that does not answer is saved as `error` with the
-  `REPO_POLICIES.md` mandates was added at the same time (the primary
+  reason, and NS failure and NS recovery are notified (closes #104).
-  run had been unconditionally `-v`): quiet first, `-v` only on
+- 2026-10-01: a `DNSWATCHER_DNS_INTERVAL` or `DNSWATCHER_TLS_INTERVAL` that is
-  failure, `-count=1` on both, and exit 1 forced regardless of the
+  not a positive duration stops startup; empty means the default (closes #177).
-  rerun's result so a flake passing the second time cannot turn the
+- 2026-10-01: `/metrics` allows each client address 30 requests a minute,
-  build green. `-timeout 90s` left alone as the deliberate backstop
+  counted before Basic Auth, and answers 429 beyond that (closes #101).
-  above the 60s hard cap. Uncached suite runs ~4s, well inside the 20s
+- 2026-10-01: the image built by `make docker` reports the `git describe`
-  target
+  version, not `dev`, and the startup log now shows it (closes #109).
-- 2026-08-10: live-DNS test flakiness addressed by robustness rather
+- 2026-10-01: two notify shutdown tests always release the delivery they hold,
-  than gating, per the owner's ruling on #93: new
+  so a drain that returns early fails them instead of hanging (closes #176).
-  `internal/resolver/livedns_test.go` adds a package-wide concurrency
+- 2026-10-01: `script/install-precommit` asks git for the repository's git
-  gate (so parallel tests stop bursting at the first root server),
+  directory, so `make hooks` also works where `.git` is a file (closes #129).
-  retry with exponential backoff on transport failures only, and
+- 2026-10-01: `TODO.md` brought up to date: open issues listed by URL, every
-  quorum instead of unanimity for multi-nameserver assertions. Quorum
+  Completed Steps entry cut to at most two lines (closes #146).
-  tolerates silence only: every per-nameserver status must be in a
+- 2026-10-01: wildcard CORS now applies only to the public routes, not to
-  closed allowlist (`ok`/`timeout`/`error`, or
+  `/metrics`, and allows only the methods they serve (closes #100).
-  `nxdomain`/`timeout`/`error`), so a wrong answer from a minority —
+- 2026-10-01: `internal/state` and `internal/watcher` no longer export test-only
-  `nodata` today, any status added later — fails the test instead of
+  constructors: two moved to `export_test.go`, one is deleted (closes #111).
-  sliding through under the majority. The
+- 2026-10-01: notify shutdown tests use one timing constant per meaning, name
-  `make test` cap moved to the new org-wide 60s hard cap / 20s target
+  the bound they check, and require the drain's debug line (closes #116).
-  with a 90s `-timeout` backstop; `REPO_POLICIES.md` re-vendored
+- 2026-09-29: the entrypoint chowns the data directory to `dnswatcher` and runs
-  byte-identical from `sneak/prompts`. No mocks, no `-short`, no build
+  dnswatcher as that user, so a host bind mount needs no chown (closes #166).
-  tags, no skips, and no change to production resolver behaviour
+- 2026-09-29: the live-DNS test package is renamed `internal/livednstest`;
-- 2026-08-10: all linting moved into Docker: new root `Dockerfile.lint`
+  `make lint` fails when program code imports it (closes #164).
-  on the digest-pinned `golangci/golangci-lint:v2.12.2` image,
+- 2026-09-29: `.golangci.yml` re-fetched from `sneak/prompts`, with
-  `script/lint` reduced to a thin wrapper that builds it with
+  `gomodguard_v2` and the org `depguard` `test-support` rule (closes #123).
-  `--no-cache-filter=lint` so the linter actually executes every run,
+- 2026-09-29: watcher and resolver tests that look something up in DNS use the
-  golangci-lint install dropped from `script/bootstrap` (goimports
+  real resolver against live DNS servers (closes #159).
-  stays, `script/fmt` needs it on the host), and the root `Dockerfile`
+- 2026-09-28: the inconsistency alert is sent once, when two nameservers start
-  given its own lint stage so its build no longer recurses through
+  to disagree; every pair of nameservers is compared (closes #158).
-  `make check` into `script/lint`. `golangci-lint config verify` is
+- 2026-09-28: DNS names in record values (CNAME, MX, SRV and NS targets) are
-  deliberately omitted: it fetches its schema over an unpinned live
+  lower-cased, so letter case alone is not a change (closes #157).
-  HTTPS call
+- 2026-09-28: lint and tests run on every build: `script/cibuild` and
-- 2026-08-09: in-flight notification deliveries are now drained at
+  `script/docker` pass `--no-cache-filter=lint,builder` (closes #115).
-  shutdown (#106): `notify.New` registers an fx `OnStop` hook that waits
+- 2026-09-28: the server timeout test drives `Run` and checks the timeouts on
-  on a `sync.WaitGroup` of tracked delivery goroutines, bounded by the
+  the `http.Server` it serves (closes #120).
-  `OnStop` context; on expiry the outstanding count is logged at warn
+- 2026-09-28: upaas deploy readiness: the image runs as user `dnswatcher` with a
-  level and parked retry backoffs are released instead of being dropped
+  `HEALTHCHECK`; README "Running under upaas" (closes #147).
-  silently, and deliveries submitted after the drain begins are refused
+- 2026-09-21: added behavioural tests for `internal/globals`,
-  so shutdown cannot be extended indefinitely; an `OnStop` context that
+  `internal/healthcheck`, and `internal/logger` (closes #110).
-  is already expired on entry with nothing outstanding drains quietly
+- 2026-09-21: `go mod tidy` dropped the redundant `golang.org/x/sync`
-  rather than warning about deliveries that were never abandoned
+  `// indirect` line so `script/bootstrap` leaves a clean tree (#132)
-- 2026-08-07: golangci-lint bumped to v2.12.2 (commit-pinned installs
+- 2026-08-10: comment-only corrections to `script/bootstrap`, `script/cibuild`
-  in `Dockerfile` and `script/bootstrap`); `.golangci.yml` set to the
+  and `Dockerfile.lint`; no behaviour changed.
-  org-standard v2-schema config used across the org's repos
+- 2026-08-10: MIT `LICENSE` added at the repository root; the README's first
-  (owner-authorized; same file is being landed as canonical via prompts
+  line and License section name the licence.
-  PR #24), with settings under `linters.settings` so the
+- 2026-08-10: policy scaffold present: `REPO_POLICIES.md`, `.editorconfig`,
-  lll/funlen/cyclop/dupl thresholds apply; fixed the resulting
+  `.dockerignore`, CI workflow, `make fmt-check`, `make docker`, `make hooks`.
-  `goconst`, `dupl`, and `lll` findings; the informational `gomodguard`
+- 2026-08-10: Go's test cache disabled in `script/test` (`-count=1`), so every
-  deprecation warning under this config is accepted
+  run queries live DNS; a failed run is rerun with `-v`.
-- 2026-07-07 Adopted scripts-to-rule-them-all: `script/` entrypoints,
+- 2026-08-10: live-DNS tests made robust rather than gated (#93): a limit on
-  Makefile shims, README Entrypoints section
+  concurrent lookups, retries, and a quorum across nameservers.
-- 2026-02-20: iterative DNS resolver implemented; tests made hermetic
+- 2026-08-10: all linting moved into Docker: `script/lint` builds
-  with mocked DNS (origin/feature/resolver, unmerged)
+  `Dockerfile.lint`, and the root `Dockerfile` has its own lint stage.
-- 2026-02-20: CI actions and go install refs pinned to commit SHAs;
+- 2026-08-09: in-flight notification deliveries are drained at shutdown, bounded
-  Gitea Actions workflow for make check (origin/ci/make-check, unmerged)
+  by the shutdown deadline (#106).
+- 2026-08-09: `http.Server` sets all four socket timeouts; `WriteTimeout` stays
+  above the 60s handler timeout (#99).
+- 2026-08-09: `SecurityHeaders()` middleware sets HSTS, CSP and the other
+  security headers `REPO_POLICIES.md` requires on every response.
+- 2026-08-07: golangci-lint bumped to v2.12.2 and `.golangci.yml` set to the org
+  config; fixed the resulting `goconst`, `dupl` and `lll` findings.
+- 2026-07-07 Adopted scripts-to-rule-them-all: `script/` entrypoints, Makefile
+  shims, README Entrypoints section
+- 2026-02-20: iterative DNS resolver implemented
+- 2026-02-20: CI actions and go install refs pinned to commit SHAs; Gitea
+  Actions workflow added
 - 2026-02-20: watcher monitoring orchestrator merged to main (#8)
 - 2026-02-20: DOMAINS/HOSTNAMES unified into single TARGETS config (#11)
 - 2026-02-19: TCP port connectivity checker, made concurrent with port
   validation; gosec G704 SSRF findings fixed without suppression
-  (feature branches, unmerged)
+- 2026-02-19: TLS certificate inspector with no-peer-certificates error path and
-- 2026-02-19: TLS certificate inspector with no-peer-certificates error
+  IP SANs
-  path and IP SANs (feature branch, unmerged)
 - 2026-02-19: gosec SSRF and formatting fixes on main
 - 2026-02-19: initial scaffold with per-nameserver DNS monitoring model
 
 # Future Steps
 
-Compliance:
+- 1.0 readiness: run it with a real config and read the logs:
-
+  https://git.eeqj.de/sneak/dnswatcher/issues/66
-- Pin Dockerfile base images by sha256 and ensure the Docker build runs
+- review toward 1.0: https://git.eeqj.de/sneak/dnswatcher/issues/144
-  make check
-
-Branch reconciliation:
-
-- Sync local checkout with origin: local main is 8 commits behind
-  origin/main; local feature/resolver has diverged from
-  origin/feature/resolver, which already implements the resolver
-- Merge in-flight branches to main once green: feature/resolver,
-  ci/make-check, feature/portcheck-implementation,
-  feature/tlscheck-implementation
-
-Resolver (plan from untracked TODO.md; largely implemented on
-origin/feature/resolver, verify each item before closing):
-
-- Add github.com/miekg/dns dependency
-- roots.go: hardcoded IANA root server list (a through m, IPv4/IPv6),
-  rootServers() returning ip:53 strings
-- query.go: low-level query(ctx, server, name, qtype): UDP with TCP
-  fallback on truncation, RD=0, context respected, 5s per-query timeout,
-  returns raw *dns.Msg
-- trace.go: iterative delegation chasing from roots: referral detection
-  (NOERROR, empty answer, NS in authority), glue extraction with
-  bailiwick check, out-of-bailiwick NS resolved with recursion guard,
-  delegation depth limit (20), retry across nameservers on failure, do
-  not chase CNAMEs inside trace
-- FindAuthoritativeNameservers: NS set via trace, sorted, FQDN
-  normalized, trailing dot handled; must pass its 9 tests
-- QueryNameserver: resolve NS host to IPs, query A/AAAA/CNAME/MX/TXT/
-  SRV/CAA/NS, build NameserverResponse with status mapping (OK,
-  NXDomain, NoData, Error), documented record formatting, sorted values,
-  lame delegation detection; must pass its 16 tests
-- QueryAllNameservers: find NS set for parent domain (public suffix
-  list), query all NS in parallel with bounded concurrency, return map
-  even when all fail, context cancellation; must pass its 4 tests
-- LookupNS: thin wrapper over FindAuthoritativeNameservers, sorted,
-  identical results; must pass its 3 tests
-- ResolveIPAddresses: collect A/AAAA from all NS, follow CNAME chains
-  with MaxCNAMEDepth, dedupe, sort, NXDOMAIN returns empty slice with
-  nil error; must pass its 9 tests
-- All 39 resolver tests pass, make check green, merge to main
-
-Watcher (internal/watcher/watcher.go):
-
-- Scheduling loop in Run(ctx): initial check on startup, separate
-  tickers for DNS/port and TLS intervals, persist state via state.Save()
-  after each cycle, clean shutdown on context cancel
-- Domain check: LookupNS, compare to stored state, store silently on
-  first run, notify with old/new NS lists on change
-- Hostname check: QueryAllNameservers, compare per-NS records; notify on
-  record changes, NS failure, NS recovery, inconsistency detected,
-  inconsistency resolved, empty response; store silently on first run
-- Port check: ResolveIPAddresses, check ports 80 and 443 per IP, notify
-  on open/closed transitions, handle new and disappeared IPs
-- TLS check: for each open IP:443, CheckCertificate; notify on expiry
-  warning, certificate change (CN/issuer/SANs), TLS failure/recovery
-
-Port checker (internal/portcheck/portcheck.go):
-
-- Tests against known-open ports and RFC documentation IPs
-- CheckPort: net.DialTimeout (5s), context respected; (true, nil) open,
-  (false, nil) closed/timeout/refused, error only for unexpected
-  failures
-
-TLS checker (internal/tlscheck/tlscheck.go):
-
-- Tests against known public HTTPS servers, verify fields populated
-- CheckCertificate: tls.Dial to specific IP:443 with hostname as SNI;
-  extract subject CN, issuer CN and org, NotAfter, SANs; error on
-  handshake failure
-
-Notification service (internal/notify/notify.go, Slack/Mattermost/ntfy
-backends exist):
-
-- Structured notification types: DNS change, port change, TLS expiry,
-  TLS change, NS failure, NS recovery, NS inconsistency
-- Per-backend formatting: Slack/Mattermost attachment colors (red
-  failures/expiry, yellow warnings, green recoveries, blue info); ntfy
-  priorities (urgent failures, high warnings, default changes, low
-  recoveries); include hostname, nameserver, old/new values, timestamps
-
-HTTP API handlers:
-
-- Wire *state.State and *watcher.Watcher into handler params
-- GET /api/v1/status: full state snapshot as JSON
-- GET /api/v1/domains: domain states with NS records and last-checked
-- GET /api/v1/hostnames: hostname states with per-NS record data
-
-Infrastructure notes (from untracked TODO.md):
-
-- Module path sneak.berlin/go/dnswatcher differs from the git.eeqj.de
-  remote intentionally; do not "fix" it
-- Dependencies: github.com/miekg/dns, golang.org/x/net/publicsuffix
-- Resolver tests originally used live DNS against *.dns.sneak.cloud
-  (required records documented in the test file header); origin now has
-  mocked hermetic tests, keep them hermetic

cmd/dnswatcher/main.go

@@ -63,6 +63,7 @@ func main() {
 				return n
 			},
 		),
+		fx.Invoke(func(l *logger.Logger) { l.Identify() }),
 		fx.Invoke(func(*server.Server, *watcher.Watcher) {}),
 	).Run()
 }

deploy/docker-entrypoint.sh

@@ -0,0 +1,17 @@
+#!/bin/sh
+# deploy/docker-entrypoint.sh: the Docker image's ENTRYPOINT. It runs as
+# root only to give the data directory to the dnswatcher user: a host
+# directory bind-mounted there keeps its host owner, often root, and may
+# hold a state file left by another uid, which dnswatcher could neither
+# read nor replace. dnswatcher itself always runs as the dnswatcher user.
+set -eu
+
+main() {
+    dir="${DNSWATCHER_DATA_DIR:-/var/lib/dnswatcher}"
+    mkdir -p "$dir"
+    chown -R dnswatcher:dnswatcher "$dir"
+    chmod 700 "$dir"
+    exec su-exec dnswatcher /usr/local/bin/dnswatcher "$@"
+}
+
+main "$@"

go.mod

@@ -4,27 +4,30 @@ go 1.25.5
 
 require (
 	github.com/99designs/basicauth-go v0.0.0-20230316000542-bf6f9cbbf0f8
+	github.com/getsentry/sentry-go v0.49.0
 	github.com/go-chi/chi/v5 v5.2.5
 	github.com/go-chi/cors v1.2.2
+	github.com/go-chi/httprate v0.16.0
 	github.com/joho/godotenv v1.5.1
 	github.com/miekg/dns v1.1.72
 	github.com/prometheus/client_golang v1.23.2
 	github.com/spf13/viper v1.21.0
 	github.com/stretchr/testify v1.11.1
 	go.uber.org/fx v1.24.0
-	golang.org/x/net v0.50.0
+	golang.org/x/net v0.56.0
-	golang.org/x/sync v0.19.0
+	golang.org/x/sync v0.21.0
 )
 
 require (
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/cespare/xxhash/v2 v2.3.0 // indirect
-	github.com/davecgh/go-spew v1.1.1 // indirect
+	github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
 	github.com/fsnotify/fsnotify v1.9.0 // indirect
 	github.com/go-viper/mapstructure/v2 v2.4.0 // indirect
+	github.com/klauspost/cpuid/v2 v2.2.10 // indirect
 	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
 	github.com/pelletier/go-toml/v2 v2.2.4 // indirect
-	github.com/pmezard/go-difflib v1.0.0 // indirect
+	github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
 	github.com/prometheus/client_model v0.6.2 // indirect
 	github.com/prometheus/common v0.66.1 // indirect
 	github.com/prometheus/procfs v0.16.1 // indirect
@@ -34,16 +37,16 @@ require (
 	github.com/spf13/cast v1.10.0 // indirect
 	github.com/spf13/pflag v1.0.10 // indirect
 	github.com/subosito/gotenv v1.6.0 // indirect
+	github.com/zeebo/xxh3 v1.0.2 // indirect
 	go.uber.org/dig v1.19.0 // indirect
 	go.uber.org/multierr v1.10.0 // indirect
 	go.uber.org/zap v1.26.0 // indirect
 	go.yaml.in/yaml/v2 v2.4.2 // indirect
 	go.yaml.in/yaml/v3 v3.0.4 // indirect
-	golang.org/x/mod v0.32.0 // indirect
+	golang.org/x/mod v0.37.0 // indirect
-	golang.org/x/sync v0.19.0 // indirect
+	golang.org/x/sys v0.46.0 // indirect
-	golang.org/x/sys v0.41.0 // indirect
+	golang.org/x/text v0.39.0 // indirect
-	golang.org/x/text v0.34.0 // indirect
+	golang.org/x/tools v0.47.0 // indirect
-	golang.org/x/tools v0.41.0 // indirect
 	google.golang.org/protobuf v1.36.8 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 )

go.sum

@@ -4,16 +4,22 @@ github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
 github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
 github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
 github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
-github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
+github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM=
-github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8=
 github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0=
 github.com/fsnotify/fsnotify v1.9.0 h1:2Ml+OJNzbYCTzsxtv8vKSFD9PbJjmhYF14k/jKC7S9k=
 github.com/fsnotify/fsnotify v1.9.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0=
+github.com/getsentry/sentry-go v0.49.0 h1:Ehejknu1l023Ub7QoRBVLAI7g3Jnhqku4oWx4B4Sh5s=
+github.com/getsentry/sentry-go v0.49.0/go.mod h1:nuMJAoCfe1u0Bts2ocyNI+TW8HT84vRMqwA5Qq/SKUI=
 github.com/go-chi/chi/v5 v5.2.5 h1:Eg4myHZBjyvJmAFjFvWgrqDTXFyOzjj7YIm3L3mu6Ug=
 github.com/go-chi/chi/v5 v5.2.5/go.mod h1:X7Gx4mteadT3eDOMTsXzmI4/rwUpOwBHLpAfupzFJP0=
 github.com/go-chi/cors v1.2.2 h1:Jmey33TE+b+rB7fT8MUy1u0I4L+NARQlK6LhzKPSyQE=
 github.com/go-chi/cors v1.2.2/go.mod h1:sSbTewc+6wYHBBCW7ytsFSn836hqM7JxpglAy2Vzc58=
+github.com/go-chi/httprate v0.16.0 h1:8V5DH9j6pSK6UQoBsTpvMyFxycqaKEIToyPKzHJjUa8=
+github.com/go-chi/httprate v0.16.0/go.mod h1:A8lo+qRhk+s9LiuP5saS7XCGDXRXMcrueq0NfIuCa/I=
+github.com/go-errors/errors v1.4.2 h1:J6MZopCL4uSllY1OfXM374weqZFFItUbrImctkmUxIA=
+github.com/go-errors/errors v1.4.2/go.mod h1:sIVyrIiJhuEF+Pj9Ebtd6P/rEYROXFi3BopGUQ5a5Og=
 github.com/go-viper/mapstructure/v2 v2.4.0 h1:EBsztssimR/CONLSZZ04E8qAkxNYq4Qp9LvH92wZUgs=
 github.com/go-viper/mapstructure/v2 v2.4.0/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM=
 github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
@@ -22,6 +28,8 @@ github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0=
 github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4=
 github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo=
 github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ=
+github.com/klauspost/cpuid/v2 v2.2.10 h1:tBs3QSyvjDyFTq3uoc/9xFpCuOsJQFNPiAhYdw2skhE=
+github.com/klauspost/cpuid/v2 v2.2.10/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0=
 github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
 github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
 github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
@@ -34,8 +42,12 @@ github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq
 github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
 github.com/pelletier/go-toml/v2 v2.2.4 h1:mye9XuhQ6gvn5h28+VilKrrPoQVanw5PMw/TB0t5Ec4=
 github.com/pelletier/go-toml/v2 v2.2.4/go.mod h1:2gIqNv+qfxSVS7cM2xJQKtLSTLUE9V8t9Stt+h56mCY=
-github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/pingcap/errors v0.11.4 h1:lFuQV/oaUMGcD2tqt+01ROSmJs75VG1ToEOkZIZ4nE4=
-github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/pingcap/errors v0.11.4/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8=
+github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
+github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U=
+github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/prometheus/client_golang v1.23.2 h1:Je96obch5RDVy3FDMndoUsjAhG5Edi49h0RJWRi/o0o=
 github.com/prometheus/client_golang v1.23.2/go.mod h1:Tb1a6LWHB3/SPIzCoaDXI4I8UHKeFTEQ1YCr+0Gyqmg=
 github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNwqPLxwZyk=
@@ -44,8 +56,8 @@ github.com/prometheus/common v0.66.1 h1:h5E0h5/Y8niHc5DlaLlWLArTQI7tMrsfQjHV+d9Z
 github.com/prometheus/common v0.66.1/go.mod h1:gcaUsgf3KfRSwHY4dIMXLPV0K/Wg1oZ8+SbZk/HH/dA=
 github.com/prometheus/procfs v0.16.1 h1:hZ15bTNuirocR6u0JZ6BAHHmwS1p8B4P6MRqxtzMyRg=
 github.com/prometheus/procfs v0.16.1/go.mod h1:teAbpZRB1iIAJYREa1LsoWUXykVXA1KlTmWl8x/U+Is=
-github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ=
+github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ=
-github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog=
+github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc=
 github.com/sagikazarmark/locafero v0.11.0 h1:1iurJgmM9G3PA/I+wWYIOw/5SyBtxapeHDcg+AAIFXc=
 github.com/sagikazarmark/locafero v0.11.0/go.mod h1:nVIGvgyzw595SUSUE6tvCp3YYTeHs15MvlmU87WwIik=
 github.com/sourcegraph/conc v0.3.1-0.20240121214520-5f936abd7ae8 h1:+jumHNA0Wrelhe64i8F6HNlS8pkoyMv5sreGx2Ry5Rw=
@@ -62,6 +74,10 @@ github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu
 github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=
 github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8=
 github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU=
+github.com/zeebo/assert v1.3.0 h1:g7C04CbJuIDKNPFHmsk4hwZDO5O+kntRxzaUoNXj+IQ=
+github.com/zeebo/assert v1.3.0/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0=
+github.com/zeebo/xxh3 v1.0.2 h1:xZmwmqxHZA8AI603jOQ0tMqmBr9lPeFwGg6d+xy9DC0=
+github.com/zeebo/xxh3 v1.0.2/go.mod h1:5NWz9Sef7zIDm2JHfFlcQvNekmcEl9ekUZQQKCYaDcA=
 go.uber.org/dig v1.19.0 h1:BACLhebsYdpQ7IROQ1AGPjrXcP5dF80U3gKoFzbaq/4=
 go.uber.org/dig v1.19.0/go.mod h1:Us0rSJiThwCv2GteUN0Q7OKvU7n5J4dxZ9JKUXozFdE=
 go.uber.org/fx v1.24.0 h1:wE8mruvpg2kiiL1Vqd0CC+tr0/24XIB10Iwp2lLWzkg=
@@ -76,18 +92,18 @@ go.yaml.in/yaml/v2 v2.4.2 h1:DzmwEr2rDGHl7lsFgAHxmNz/1NlQ7xLIrlN2h5d1eGI=
 go.yaml.in/yaml/v2 v2.4.2/go.mod h1:081UH+NErpNdqlCXm3TtEran0rJZGxAYx9hb/ELlsPU=
 go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc=
 go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg=
-golang.org/x/mod v0.32.0 h1:9F4d3PHLljb6x//jOyokMv3eX+YDeepZSEo3mFJy93c=
+golang.org/x/mod v0.37.0 h1:vF1DjpVEshcIqoEaauuHebaLk1O1forxjxBaVn884JQ=
-golang.org/x/mod v0.32.0/go.mod h1:SgipZ/3h2Ci89DlEtEXWUk/HteuRin+HHhN+WbNhguU=
+golang.org/x/mod v0.37.0/go.mod h1:m8S8VeM9r4dzDwjrKO0a1sZP3YjeMamRRlD+fmR2Q/0=
-golang.org/x/net v0.50.0 h1:ucWh9eiCGyDR3vtzso0WMQinm2Dnt8cFMuQa9K33J60=
+golang.org/x/net v0.56.0 h1:Rw8j/hFzGvJUZwNBXnAtf5sVDVt+65SK2C7IxCxZt5o=
-golang.org/x/net v0.50.0/go.mod h1:UgoSli3F/pBgdJBHCTc+tp3gmrU4XswgGRgtnwWTfyM=
+golang.org/x/net v0.56.0/go.mod h1:D3Ku6r+V6JROoZK144D2XfMHFcMq/0zSfLelVTCFKec=
-golang.org/x/sync v0.19.0 h1:vV+1eWNmZ5geRlYjzm2adRgW2/mcpevXNg50YZtPCE4=
+golang.org/x/sync v0.21.0 h1:HLII4xRRTtCRkxYp4HNFF0Js/Og6q2i++KXbg0gHCwM=
-golang.org/x/sync v0.19.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI=
+golang.org/x/sync v0.21.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0=
-golang.org/x/sys v0.41.0 h1:Ivj+2Cp/ylzLiEU89QhWblYnOE9zerudt9Ftecq2C6k=
+golang.org/x/sys v0.46.0 h1:noSf2Fq6F8DBgS+LysIkx7rIExoNHJsxOAtPp4rthXw=
-golang.org/x/sys v0.41.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks=
+golang.org/x/sys v0.46.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
-golang.org/x/text v0.34.0 h1:oL/Qq0Kdaqxa1KbNeMKwQq0reLCCaFtqu2eNuSeNHbk=
+golang.org/x/text v0.39.0 h1:UbZz4pLOvn600D6Oh6GGEI6VAmndrEBLv8/6BEXzyus=
-golang.org/x/text v0.34.0/go.mod h1:homfLqTYRFyVYemLBFl5GgL/DWEiH5wcsQ5gSh1yziA=
+golang.org/x/text v0.39.0/go.mod h1:3UwRclnC2g0TU9x8PZiyfOajCd1zaUNHF9cvqcQZ+ZM=
-golang.org/x/tools v0.41.0 h1:a9b8iMweWG+S0OBnlU36rzLp20z1Rp10w+IY2czHTQc=
+golang.org/x/tools v0.47.0 h1:7Kn5x/d1svx/PzryTsqeoZN4TZwqeH5pGWjefhLi/1Q=
-golang.org/x/tools v0.41.0/go.mod h1:XSY6eDqxVNiYgezAVqqCeihT4j1U2CCsqvH3WhQpnlg=
+golang.org/x/tools v0.47.0/go.mod h1:dFHnyTvFWY212G+h7ZY4Vsp/K3U4/7W9TyVaAul8uCA=
 google.golang.org/protobuf v1.36.8 h1:xHScyCOEuuwZEc6UtSOvPbAT4zRh0xcNRYekJwfqyMc=
 google.golang.org/protobuf v1.36.8/go.mod h1:fuxRtAxBytpl4zzqUh6/eyUujkJdNiuEkXntxiD/uRU=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=

internal/config/classify.go

@@ -57,17 +57,22 @@ func ClassifyDNSName(name string) (DNSNameType, error) {
 
 // ClassifyTargets splits a list of DNS names into apex domains and
 // hostnames using the Public Suffix List. It returns an error if any
-// name cannot be classified.
+// name cannot be classified. A name given more than once, in any letter
+// case or with a trailing dot, is kept once.
 func ClassifyTargets(targets []string) ([]string, []string, error) {
 	var domains, hostnames []string
 
+	seen := make(map[string]bool)
+
 	for _, t := range targets {
 		normalized := strings.ToLower(strings.TrimSuffix(strings.TrimSpace(t), "."))
 
-		if normalized == "" {
+		if normalized == "" || seen[normalized] {
 			continue
 		}
 
+		seen[normalized] = true
+
 		typ, classErr := ClassifyDNSName(normalized)
 		if classErr != nil {
 			return nil, nil, classErr

internal/config/classify_test.go

@@ -1,6 +1,7 @@
 package config_test
 
 import (
+	"slices"
 	"testing"
 
 	"sneak.berlin/go/dnswatcher/internal/config"
@@ -93,6 +94,29 @@ func TestClassifyTargets(t *testing.T) {
 	}
 }
 
+func TestClassifyTargetsKeepsEachNameOnce(t *testing.T) {
+	t.Parallel()
+
+	domains, hostnames, err := config.ClassifyTargets([]string{
+		"example.org",
+		"Example.org.",
+		"www.example.org",
+		"EXAMPLE.ORG",
+		"WWW.Example.org.",
+	})
+	if err != nil {
+		t.Fatalf("unexpected error: %v", err)
+	}
+
+	if !slices.Equal(domains, []string{"example.org"}) {
+		t.Errorf("domains = %v, want [example.org]", domains)
+	}
+
+	if !slices.Equal(hostnames, []string{"www.example.org"}) {
+		t.Errorf("hostnames = %v, want [www.example.org]", hostnames)
+	}
+}
+
 func TestClassifyTargetsRejectsPublicSuffix(t *testing.T) {
 	t.Parallel()
 

internal/config/config.go

@@ -28,6 +28,13 @@ var ErrNoTargets = errors.New(
 	"no monitoring targets configured: set DNSWATCHER_TARGETS environment variable",
 )
 
+// ErrInvalidInterval is returned when DNSWATCHER_DNS_INTERVAL or
+// DNSWATCHER_TLS_INTERVAL is set but is not a positive duration. An empty
+// value counts as unset and means the default.
+var ErrInvalidInterval = errors.New(
+	"interval must be a positive duration such as 30m or 1h",
+)
+
 // Params contains dependencies for Config.
 type Params struct {
 	fx.In
@@ -125,18 +132,14 @@ func buildConfig(
 		}
 	}
 
-	dnsInterval, err := time.ParseDuration(
+	dnsInterval, err := parseInterval("DNS_INTERVAL")
-		viper.GetString("DNS_INTERVAL"),
-	)
 	if err != nil {
-		dnsInterval = defaultDNSInterval
+		return nil, err
 	}
 
-	tlsInterval, err := time.ParseDuration(
+	tlsInterval, err := parseInterval("TLS_INTERVAL")
-		viper.GetString("TLS_INTERVAL"),
-	)
 	if err != nil {
-		tlsInterval = defaultTLSInterval
+		return nil, err
 	}
 
 	domains, hostnames, err := parseAndValidateTargets()
@@ -168,6 +171,22 @@ func buildConfig(
 	return cfg, nil
 }
 
+// parseInterval reads the DNSWATCHER_-prefixed setting key as a duration. A
+// value that does not parse, or is zero or negative, is an error naming the
+// variable and the value; an unset variable has its default from setupViper.
+func parseInterval(key string) (time.Duration, error) {
+	value := viper.GetString(key)
+
+	interval, err := time.ParseDuration(value)
+	if err != nil || interval <= 0 {
+		return 0, fmt.Errorf(
+			"invalid DNSWATCHER_%s %q: %w", key, value, ErrInvalidInterval,
+		)
+	}
+
+	return interval, nil
+}
+
 func parseAndValidateTargets() ([]string, []string, error) {
 	domains, hostnames, err := ClassifyTargets(
 		parseCSV(viper.GetString("TARGETS")),

internal/config/config_test.go

@@ -1,6 +1,7 @@
 package config_test
 
 import (
+	"strconv"
 	"testing"
 	"time"
 
@@ -113,33 +114,40 @@ func TestNew_OnlyEmptyCSVSegments(t *testing.T) {
 	assert.ErrorIs(t, err, config.ErrNoTargets)
 }
 
-func TestNew_InvalidDNSInterval_FallsBackToDefault(t *testing.T) {
+// TestNew_InvalidIntervalStopsStartup checks values that must stop startup;
+// TestNew_DefaultValues and TestNew_EmptyIntervalMeansDefault check that an
+// unset or empty interval means the default.
+func TestNew_InvalidIntervalStopsStartup(t *testing.T) {
+	variables := []string{"DNSWATCHER_DNS_INTERVAL", "DNSWATCHER_TLS_INTERVAL"}
+	values := []string{
+		"banana", // not a duration
+		"5",      // no unit
+		"1d",     // days are not a unit time.ParseDuration knows
+		"0",      // zero
+		"-1h",    // negative
+	}
+
+	for _, variable := range variables {
+		for _, value := range values {
+			t.Run(variable+"="+value, func(t *testing.T) {
 				viper.Reset()
 				t.Setenv("DNSWATCHER_TARGETS", "example.com")
-	t.Setenv("DNSWATCHER_DNS_INTERVAL", "banana")
+				t.Setenv(variable, value)
 
-	cfg, err := config.New(nil, newTestParams(t))
+				_, err := config.New(nil, newTestParams(t))
-	require.NoError(t, err)
+				require.ErrorIs(t, err, config.ErrInvalidInterval)
-	assert.Equal(t, time.Hour, cfg.DNSInterval,
+				require.ErrorContains(t, err, variable)
-		"invalid DNS interval should fall back to 1h default")
+				require.ErrorContains(t, err, strconv.Quote(value))
+			})
+		}
+	}
 }
 
-func TestNew_InvalidTLSInterval_FallsBackToDefault(t *testing.T) {
+func TestNew_EmptyIntervalMeansDefault(t *testing.T) {
 	viper.Reset()
 	t.Setenv("DNSWATCHER_TARGETS", "example.com")
-	t.Setenv("DNSWATCHER_TLS_INTERVAL", "notaduration")
+	t.Setenv("DNSWATCHER_DNS_INTERVAL", "")
-
+	t.Setenv("DNSWATCHER_TLS_INTERVAL", "")
-	cfg, err := config.New(nil, newTestParams(t))
-	require.NoError(t, err)
-	assert.Equal(t, 12*time.Hour, cfg.TLSInterval,
-		"invalid TLS interval should fall back to 12h default")
-}
-
-func TestNew_BothIntervalsInvalid(t *testing.T) {
-	viper.Reset()
-	t.Setenv("DNSWATCHER_TARGETS", "example.com")
-	t.Setenv("DNSWATCHER_DNS_INTERVAL", "xyz")
-	t.Setenv("DNSWATCHER_TLS_INTERVAL", "abc")
 
 	cfg, err := config.New(nil, newTestParams(t))
 	require.NoError(t, err)

internal/globals/globals_test.go

@@ -0,0 +1,50 @@
+package globals_test
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"sneak.berlin/go/dnswatcher/internal/globals"
+)
+
+// TestGlobals exercises the package-level version and appname
+// variables through their setters and read-back via New. These are
+// shared package state, so the test mutates a global and must run
+// sequentially; it cannot use t.Parallel().
+//
+//nolint:paralleltest // mutates shared package-level globals, must run sequentially
+func TestGlobals(t *testing.T) {
+	versions := []string{"v1.2.3", "dev", "", "v1.2.3-4-gabcdef"}
+	for _, want := range versions {
+		globals.SetVersion(want)
+
+		g, err := globals.New(nil)
+		require.NoError(t, err)
+		assert.Equal(t, want, g.Version,
+			"New must surface the version set by SetVersion")
+	}
+
+	names := []string{"dnswatcher", "other", ""}
+	for _, want := range names {
+		globals.SetAppname(want)
+
+		g, err := globals.New(nil)
+		require.NoError(t, err)
+		assert.Equal(t, want, g.Appname,
+			"New must surface the appname set by SetAppname")
+	}
+
+	// New returns a snapshot: a later SetVersion must not mutate a
+	// Globals handed out earlier.
+	globals.SetVersion("first")
+
+	g, err := globals.New(nil)
+	require.NoError(t, err)
+
+	globals.SetVersion("second")
+	assert.Equal(t, "first", g.Version,
+		"a Globals returned by New must not change when the "+
+			"package variable is set again")
+}

internal/healthcheck/healthcheck_test.go

@@ -0,0 +1,134 @@
+package healthcheck_test
+
+import (
+	"context"
+	"encoding/json"
+	"testing"
+	"time"
+
+	"go.uber.org/fx"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"sneak.berlin/go/dnswatcher/internal/config"
+	"sneak.berlin/go/dnswatcher/internal/globals"
+	"sneak.berlin/go/dnswatcher/internal/healthcheck"
+	"sneak.berlin/go/dnswatcher/internal/logger"
+)
+
+// recordingLifecycle is a minimal fx.Lifecycle that records the hooks
+// appended to it, so healthcheck.New can be exercised through its real
+// constructor without standing up a whole fx application.
+type recordingLifecycle struct {
+	hooks []fx.Hook
+}
+
+func (l *recordingLifecycle) Append(hook fx.Hook) {
+	l.hooks = append(l.hooks, hook)
+}
+
+// newHealthcheck builds a Healthcheck through the real constructor and
+// runs the registered OnStart hook so StartupTime is set the same way
+// the fx lifecycle would set it.
+func newHealthcheck(
+	t *testing.T,
+	maintenance bool,
+	version string,
+) *healthcheck.Healthcheck {
+	t.Helper()
+
+	g := &globals.Globals{Appname: "dnswatcher", Version: version}
+
+	log, err := logger.New(nil, logger.Params{Globals: g})
+	require.NoError(t, err)
+
+	lifecycle := &recordingLifecycle{}
+
+	hc, err := healthcheck.New(lifecycle, healthcheck.Params{
+		Globals: g,
+		Config:  &config.Config{MaintenanceMode: maintenance},
+		Logger:  log,
+	})
+	require.NoError(t, err)
+
+	require.Len(t, lifecycle.hooks, 1,
+		"New must register exactly one lifecycle hook")
+	require.NotNil(t, lifecycle.hooks[0].OnStart)
+	require.NoError(t, lifecycle.hooks[0].OnStart(context.Background()))
+
+	return hc
+}
+
+func TestCheckStatusAndPayloadShape(t *testing.T) {
+	t.Parallel()
+
+	hc := newHealthcheck(t, false, "v9.9.9")
+	resp := hc.Check()
+
+	assert.Equal(t, "ok", resp.Status)
+
+	// The JSON shape and field names are part of the contract for the
+	// /health and /.well-known/healthcheck routes, so assert on the
+	// exact set of keys the response marshals to.
+	raw, err := json.Marshal(resp)
+	require.NoError(t, err)
+
+	var fields map[string]json.RawMessage
+	require.NoError(t, json.Unmarshal(raw, &fields))
+
+	wantKeys := []string{
+		"status",
+		"now",
+		"uptimeSeconds",
+		"uptimeHuman",
+		"version",
+		"appname",
+		"maintenanceMode",
+	}
+	assert.Len(t, fields, len(wantKeys),
+		"response must marshal to exactly the documented fields")
+
+	for _, key := range wantKeys {
+		assert.Contains(t, fields, key, "missing JSON field %q", key)
+	}
+
+	// The Now field is documented as RFC3339Nano; a change to the
+	// format constant should turn this red.
+	_, err = time.Parse(time.RFC3339Nano, resp.Now)
+	assert.NoError(t, err, "Now must be RFC3339Nano")
+}
+
+func TestCheckMaintenanceModeReflectsConfig(t *testing.T) {
+	t.Parallel()
+
+	tests := []struct {
+		name        string
+		maintenance bool
+	}{
+		{"maintenance off", false},
+		{"maintenance on", true},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			t.Parallel()
+
+			hc := newHealthcheck(t, tt.maintenance, "test")
+			resp := hc.Check()
+			assert.Equal(t, tt.maintenance, resp.Maintenance,
+				"maintenanceMode must mirror Config.MaintenanceMode")
+		})
+	}
+}
+
+func TestCheckSurfacesVersionAndAppname(t *testing.T) {
+	t.Parallel()
+
+	hc := newHealthcheck(t, false, "surfaced-version-123")
+	resp := hc.Check()
+
+	assert.Equal(t, "surfaced-version-123", resp.Version,
+		"version from globals must appear in the payload")
+	assert.Equal(t, "dnswatcher", resp.Appname)
+}

internal/livednstest/livednstest.go

@@ -0,0 +1,121 @@
+// Package livednstest runs the live DNS operations of tests. Tests that
+// look something up in DNS query live DNS servers, never a stand-in —
+// see TESTING.md. Nothing here mocks, fakes, stubs, records or replays
+// DNS, and nothing here skips a test: it only changes *how* the live
+// queries are issued, so that a single dropped UDP packet or one slow
+// authoritative server does not turn correct code into a red build.
+//
+// Two mechanisms:
+//
+//  1. Bounded concurrency. Tests run in parallel and the build hosts
+//     have many cores, so without a limit every test starts its own
+//     iterative resolution at the same instant and they all send their
+//     first queries to the root servers within a few milliseconds of
+//     each other. Root servers rate-limit that, which shows up as a
+//     different arbitrary subset of tests failing on each run. Run caps
+//     how many live operations are in flight at once in one test binary.
+//
+//  2. Retry with exponential backoff. Each live operation gets several
+//     attempts with its own timeout. An attempt is retried when it
+//     obtained nothing to check, never because of what the test
+//     asserts about the result, so a wrong result still fails on the
+//     first attempt. A fault in the code under test that leaves
+//     nothing to check looks the same as live DNS not answering, and
+//     fails only after the last attempt.
+package livednstest
+
+import (
+	"context"
+	"errors"
+	"testing"
+	"time"
+)
+
+const (
+	// attempts is how many times a live DNS operation is attempted
+	// before the test fails.
+	attempts = 3
+
+	// AttemptTimeout bounds one attempt. Worst case for an operation
+	// is attempts * AttemptTimeout plus the backoff — about 26
+	// seconds, well inside the 90-second `go test -timeout` backstop
+	// even when several operations exhaust their attempts.
+	AttemptTimeout = 8 * time.Second
+
+	// backoffBase is the delay after the first failed attempt; it is
+	// multiplied by backoffFactor each time.
+	backoffBase = 500 * time.Millisecond
+
+	// backoffFactor is the exponential backoff multiplier.
+	backoffFactor = 2
+
+	// Concurrency caps how many live operations may be in flight
+	// across one test binary at once.
+	Concurrency = 6
+)
+
+// gate bounds concurrent live operations. It has to be package scoped:
+// the whole point is that it is shared by every parallel test in the
+// test binary.
+//
+//nolint:gochecknoglobals // package-wide live query rate limit
+var gate = make(chan struct{}, Concurrency)
+
+// ErrNoAnswer reports that a live operation produced no usable answer,
+// which is retried rather than asserted on.
+var ErrNoAnswer = errors.New("no answer from live DNS")
+
+// Run executes one attempt of a live operation, holding a slot in gate
+// for its duration and bounding it with its own timeout.
+func Run(op func(ctx context.Context) error) error {
+	gate <- struct{}{}
+	defer func() { <-gate }()
+
+	ctx, cancel := context.WithTimeout(
+		context.Background(), AttemptTimeout,
+	)
+	defer cancel()
+
+	return op(ctx)
+}
+
+// Retry runs op until it reports success, retrying failures with
+// exponential backoff, and fails the test if every attempt fails. op
+// returns an error only for a failure to obtain an answer — never for
+// an answer the test disagrees with, which belongs in an assertion so
+// that it fails immediately. op stores whatever it obtained where its
+// caller can find it.
+func Retry(
+	t *testing.T,
+	what string,
+	op func(ctx context.Context) error,
+) {
+	t.Helper()
+
+	var last error
+
+	backoff := backoffBase
+
+	for attempt := range attempts {
+		if attempt > 0 {
+			t.Logf(
+				"%s: attempt %d of %d failed (%v), "+
+					"retrying in %s",
+				what, attempt, attempts, last, backoff,
+			)
+			time.Sleep(backoff)
+
+			backoff *= backoffFactor
+		}
+
+		last = Run(op)
+		if last == nil {
+			return
+		}
+	}
+
+	t.Fatalf(
+		"%s: all %d live attempts failed: %v",
+		what, attempts, last,
+	)
+}

internal/livednstest/livednstest_test.go

@@ -0,0 +1,103 @@
+package livednstest_test
+
+import (
+	"context"
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+
+	"sneak.berlin/go/dnswatcher/internal/livednstest"
+)
+
+// Tests for the retry and the concurrency limit themselves. They
+// perform no DNS resolution of any kind.
+
+func TestRetryRecoversFromTransientFailure(t *testing.T) {
+	t.Parallel()
+
+	const wantAttempts = 2
+
+	attempts := 0
+
+	livednstest.Retry(t, "transient", func(_ context.Context) error {
+		attempts++
+
+		if attempts < wantAttempts {
+			return livednstest.ErrNoAnswer
+		}
+
+		return nil
+	})
+
+	assert.Equal(t, wantAttempts, attempts)
+}
+
+func TestRetryGivesEachAttemptADeadline(t *testing.T) {
+	t.Parallel()
+
+	livednstest.Retry(t, "deadline", func(ctx context.Context) error {
+		deadline, ok := ctx.Deadline()
+		assert.True(t, ok, "attempt should carry a deadline")
+
+		remaining := time.Until(deadline)
+
+		assert.LessOrEqual(t, remaining, livednstest.AttemptTimeout)
+
+		// Lower bound too: without one this passes for a
+		// deadline far shorter than intended, which would
+		// silently turn every live attempt into an instant
+		// timeout.
+		assert.Greater(t, remaining, livednstest.AttemptTimeout/2)
+
+		return nil
+	})
+}
+
+func TestRunBoundsConcurrency(t *testing.T) {
+	t.Parallel()
+
+	const workers = 24
+
+	var (
+		mu       sync.Mutex
+		wg       sync.WaitGroup
+		inFlight int
+		maxSeen  int
+	)
+
+	wg.Add(workers)
+
+	for range workers {
+		go func() {
+			defer wg.Done()
+
+			_ = livednstest.Run(func(_ context.Context) error {
+				mu.Lock()
+				inFlight++
+
+				if inFlight > maxSeen {
+					maxSeen = inFlight
+				}
+				mu.Unlock()
+
+				time.Sleep(time.Millisecond)
+
+				mu.Lock()
+				inFlight--
+				mu.Unlock()
+
+				return nil
+			})
+		}()
+	}
+
+	wg.Wait()
+
+	assert.Positive(t, maxSeen)
+	assert.LessOrEqual(
+		t, maxSeen, livednstest.Concurrency,
+		"live queries must stay under the package-wide gate",
+	)
+}

internal/logger/logger_test.go

@@ -0,0 +1,66 @@
+package logger_test
+
+import (
+	"context"
+	"log/slog"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"sneak.berlin/go/dnswatcher/internal/globals"
+	"sneak.berlin/go/dnswatcher/internal/logger"
+)
+
+func newTestLogger(t *testing.T) *logger.Logger {
+	t.Helper()
+
+	g := &globals.Globals{Appname: "dnswatcher", Version: "test"}
+
+	l, err := logger.New(nil, logger.Params{Globals: g})
+	require.NoError(t, err)
+
+	return l
+}
+
+// TestNewReturnsUsableLogger checks that the constructor yields a
+// working *slog.Logger.
+func TestNewReturnsUsableLogger(t *testing.T) {
+	t.Parallel()
+
+	l := newTestLogger(t)
+	require.NotNil(t, l.Get(), "Get must return a non-nil logger")
+}
+
+// TestDefaultLevelExcludesDebug verifies the default configuration
+// logs at info: debug records are suppressed, info records pass.
+func TestDefaultLevelExcludesDebug(t *testing.T) {
+	t.Parallel()
+
+	log := newTestLogger(t).Get()
+	ctx := context.Background()
+
+	assert.False(t, log.Enabled(ctx, slog.LevelDebug),
+		"debug must be suppressed at the default level")
+	assert.True(t, log.Enabled(ctx, slog.LevelInfo),
+		"info must be enabled at the default level")
+}
+
+// TestEnableDebugLoggingChangesLevel verifies the debug and non-debug
+// configurations differ as intended: enabling debug makes debug
+// records pass where they previously did not.
+func TestEnableDebugLoggingChangesLevel(t *testing.T) {
+	t.Parallel()
+
+	l := newTestLogger(t)
+	log := l.Get()
+	ctx := context.Background()
+
+	require.False(t, log.Enabled(ctx, slog.LevelDebug),
+		"debug must start disabled")
+
+	l.EnableDebugLogging()
+
+	assert.True(t, log.Enabled(ctx, slog.LevelDebug),
+		"debug must be enabled after EnableDebugLogging")
+}

internal/middleware/export_test.go

@@ -0,0 +1,19 @@
+package middleware
+
+import (
+	"net/http"
+	"time"
+)
+
+// The /metrics rate limit, exported so the tests can count requests
+// against it.
+const (
+	MetricsRequestLimit                = metricsRequestLimit
+	MetricsRequestWindow time.Duration = metricsRequestWindow
+)
+
+// RealIP is realIP, exported so the tests can check which address it
+// takes as the client's.
+func RealIP(r *http.Request) string {
+	return realIP(r)
+}

internal/middleware/middleware.go

@@ -5,12 +5,14 @@ import (
 	"log/slog"
 	"net"
 	"net/http"
+	"net/netip"
 	"strings"
 	"time"
 
 	"github.com/99designs/basicauth-go"
 	"github.com/go-chi/chi/v5/middleware"
 	"github.com/go-chi/cors"
+	"github.com/go-chi/httprate"
 	"go.uber.org/fx"
 
 	"sneak.berlin/go/dnswatcher/internal/config"
@@ -21,6 +23,71 @@ import (
 // corsMaxAge is the maximum age for CORS preflight responses.
 const corsMaxAge = 300
 
+// Rate limit for /metrics: each client address may send
+// metricsRequestLimit requests per metricsRequestWindow. Every request
+// counts, so password guessing gets at most 30 tries a minute per
+// address. One Prometheus server scraping every 15 seconds sends 4
+// requests a minute, and two scraping every 5 seconds from one address
+// send 24, so normal scraping stays under the limit.
+const (
+	metricsRequestLimit  = 30
+	metricsRequestWindow = time.Minute
+)
+
+// Security response header values applied to every response.
+//
+// The CSP is as strict as the dashboard allows: the template ships no
+// JavaScript, no inline styles, no inline event handlers and no images,
+// and its only subresource is the embedded stylesheet at
+// /s/css/tailwind.min.css, which style-src 'self' permits. Neither
+// unsafe-inline nor unsafe-eval is used. frame-ancestors 'none' is the
+// primary anti-framing control; X-Frame-Options is the legacy fallback.
+const (
+	// hstsValue is emitted unconditionally, including over plain HTTP,
+	// because the service runs behind a TLS-terminating proxy and the
+	// browser must still enforce HTTPS end to end.
+	hstsValue = "max-age=31536000; includeSubDomains"
+
+	cspValue = "default-src 'self'; " +
+		"script-src 'none'; " +
+		"style-src 'self'; " +
+		"img-src 'self'; " +
+		"font-src 'none'; " +
+		"connect-src 'none'; " +
+		"object-src 'none'; " +
+		"base-uri 'none'; " +
+		"form-action 'none'; " +
+		"frame-ancestors 'none'"
+
+	frameOptionsValue = "DENY"
+
+	contentTypeOptionsValue = "nosniff"
+
+	// referrerPolicyValue is stricter than the policy minimum of
+	// strict-origin-when-cross-origin: the dashboard has no
+	// cross-origin navigation needs and its URL may name internal
+	// hosts.
+	referrerPolicyValue = "no-referrer"
+
+	permissionsPolicyValue = "accelerometer=(), " +
+		"autoplay=(), " +
+		"camera=(), " +
+		"display-capture=(), " +
+		"encrypted-media=(), " +
+		"fullscreen=(), " +
+		"geolocation=(), " +
+		"gyroscope=(), " +
+		"magnetometer=(), " +
+		"microphone=(), " +
+		"midi=(), " +
+		"payment=(), " +
+		"picture-in-picture=(), " +
+		"publickey-credentials-get=(), " +
+		"screen-wake-lock=(), " +
+		"usb=(), " +
+		"xr-spatial-tracking=()"
+)
+
 // Params contains dependencies for Middleware.
 type Params struct {
 	fx.In
@@ -142,6 +209,12 @@ func isTrustedProxy(ip net.IP) bool {
 
 // realIP extracts the client's real IP address from the request.
 // Proxy headers are only trusted from RFC1918/loopback addresses.
+//
+// Each proxy adds to the end of X-Forwarded-For the address it got the
+// request from, so the client can write every entry before the one the
+// first trusted proxy added. The client address is therefore the
+// rightmost entry that is not a trusted proxy, or the leftmost entry
+// when they all are.
 func realIP(r *http.Request) string {
 	addr := ipFromHostPort(r.RemoteAddr)
 	remoteIP := net.ParseIP(addr)
@@ -156,36 +229,100 @@ func realIP(r *http.Request) string {
 		return ip
 	}
 
-	if xff := r.Header.Get("X-Forwarded-For"); xff != "" {
+	// A proxy may add its entry as a header line of its own instead of
-		if parts := strings.SplitN(
+	// appending to the line the client sent, so all lines form one list.
-			xff, ",", 2, //nolint:mnd
+	entries := strings.Split(
-		); len(parts) > 0 {
+		strings.Join(r.Header.Values("X-Forwarded-For"), ","), ",",
-			if ip := strings.TrimSpace(parts[0]); ip != "" {
+	)
-				return ip
+	client := strings.TrimSpace(entries[0])
+
+	for i := len(entries) - 1; i > 0; i-- {
+		entry := strings.TrimSpace(entries[i])
+		if !isTrustedProxy(net.ParseIP(entry)) {
+			client = entry
+
+			break
 		}
 	}
+
+	if client != "" {
+		return client
 	}
 
 	return addr
 }
 
-// CORS returns CORS middleware.
+// CORS returns middleware that lets any origin read a response. It is
+// for the public, read-only routes only, so it allows only the
+// methods those routes serve and no Authorization header.
 func (m *Middleware) CORS() func(http.Handler) http.Handler {
 	return cors.Handler(cors.Options{
 		AllowedOrigins:   []string{"*"},
-		AllowedMethods: []string{
+		AllowedMethods:   []string{"GET", "OPTIONS"},
-			"GET", "POST", "PUT", "DELETE", "OPTIONS",
+		AllowedHeaders:   []string{"Accept", "Content-Type"},
-		},
-		AllowedHeaders: []string{
-			"Accept", "Authorization",
-			"Content-Type", "X-CSRF-Token",
-		},
 		ExposedHeaders:   []string{"Link"},
 		AllowCredentials: false,
 		MaxAge:           corsMaxAge,
 	})
 }
 
+// SecurityHeaders returns middleware that sets the security response
+// headers required for production internet exposure on every response.
+//
+// The headers are set before the request reaches the next handler so
+// that they are present on every response, including panics recovered
+// by chi's Recoverer and timeouts produced by chi's Timeout.
+func (m *Middleware) SecurityHeaders() func(http.Handler) http.Handler {
+	return func(next http.Handler) http.Handler {
+		return http.HandlerFunc(func(
+			writer http.ResponseWriter,
+			request *http.Request,
+		) {
+			header := writer.Header()
+			header.Set("Strict-Transport-Security", hstsValue)
+			header.Set("Content-Security-Policy", cspValue)
+			header.Set("X-Frame-Options", frameOptionsValue)
+			header.Set(
+				"X-Content-Type-Options",
+				contentTypeOptionsValue,
+			)
+			header.Set("Referrer-Policy", referrerPolicyValue)
+			header.Set(
+				"Permissions-Policy",
+				permissionsPolicyValue,
+			)
+
+			next.ServeHTTP(writer, request)
+		})
+	}
+}
+
+// MetricsRateLimit returns middleware for /metrics that answers 429
+// Too Many Requests to a client address over the rate limit. The
+// address is the one realIP works out, so a client that is not a
+// trusted proxy cannot get a fresh allowance by sending its own
+// X-Real-IP or X-Forwarded-For. CanonicalizeIP counts all IPv6
+// addresses in one /64 as one client, since a client usually holds a
+// whole /64. An IPv4 address a proxy reports in IPv6-mapped form
+// (::ffff:203.0.113.1) is turned back into plain IPv4 first, as every
+// such address is in the same /64.
+func (m *Middleware) MetricsRateLimit() func(http.Handler) http.Handler {
+	return httprate.LimitBy(
+		metricsRequestLimit,
+		metricsRequestWindow,
+		func(request *http.Request) (string, error) {
+			ip := realIP(request)
+
+			addr, err := netip.ParseAddr(ip)
+			if err == nil {
+				ip = addr.Unmap().String()
+			}
+
+			return httprate.CanonicalizeIP(ip), nil
+		},
+	)
+}
+
 // MetricsAuth returns basic auth middleware for /metrics.
 func (m *Middleware) MetricsAuth() func(http.Handler) http.Handler {
 	if m.params.Config.MetricsUsername == "" {

internal/middleware/middleware_test.go

@@ -0,0 +1,565 @@
+package middleware_test
+
+import (
+	"net/http"
+	"net/http/httptest"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/go-chi/chi/v5"
+	"go.uber.org/fx/fxtest"
+
+	"sneak.berlin/go/dnswatcher/internal/config"
+	"sneak.berlin/go/dnswatcher/internal/globals"
+	"sneak.berlin/go/dnswatcher/internal/handlers"
+	"sneak.berlin/go/dnswatcher/internal/logger"
+	"sneak.berlin/go/dnswatcher/internal/middleware"
+	"sneak.berlin/go/dnswatcher/internal/notify"
+	"sneak.berlin/go/dnswatcher/internal/state"
+)
+
+// Expected security header values, spelled out literally so that any
+// change to the middleware has to be made deliberately here as well.
+const (
+	wantHSTS = "max-age=31536000; includeSubDomains"
+
+	wantCSP = "default-src 'self'; " +
+		"script-src 'none'; " +
+		"style-src 'self'; " +
+		"img-src 'self'; " +
+		"font-src 'none'; " +
+		"connect-src 'none'; " +
+		"object-src 'none'; " +
+		"base-uri 'none'; " +
+		"form-action 'none'; " +
+		"frame-ancestors 'none'"
+
+	wantFrameOptions = "DENY"
+
+	wantContentTypeOptions = "nosniff"
+
+	wantReferrerPolicy = "no-referrer"
+
+	wantPermissionsPolicy = "accelerometer=(), " +
+		"autoplay=(), " +
+		"camera=(), " +
+		"display-capture=(), " +
+		"encrypted-media=(), " +
+		"fullscreen=(), " +
+		"geolocation=(), " +
+		"gyroscope=(), " +
+		"magnetometer=(), " +
+		"microphone=(), " +
+		"midi=(), " +
+		"payment=(), " +
+		"picture-in-picture=(), " +
+		"publickey-credentials-get=(), " +
+		"screen-wake-lock=(), " +
+		"usb=(), " +
+		"xr-spatial-tracking=()"
+)
+
+// stylesheetPath is the only subresource the dashboard loads.
+const stylesheetPath = "/s/css/tailwind.min.css"
+
+// newTestLogger builds a logger for direct component construction.
+func newTestLogger(t *testing.T) *logger.Logger {
+	t.Helper()
+
+	glob, err := globals.New(nil)
+	if err != nil {
+		t.Fatalf("globals.New: %v", err)
+	}
+
+	log, err := logger.New(nil, logger.Params{Globals: glob})
+	if err != nil {
+		t.Fatalf("logger.New: %v", err)
+	}
+
+	return log
+}
+
+// newTestMiddleware builds a Middleware without an fx application.
+func newTestMiddleware(t *testing.T) *middleware.Middleware {
+	t.Helper()
+
+	glob, err := globals.New(nil)
+	if err != nil {
+		t.Fatalf("globals.New: %v", err)
+	}
+
+	mw, err := middleware.New(nil, middleware.Params{
+		Logger:  newTestLogger(t),
+		Globals: glob,
+		Config:  &config.Config{},
+	})
+	if err != nil {
+		t.Fatalf("middleware.New: %v", err)
+	}
+
+	return mw
+}
+
+// serveWithSecurityHeaders runs a GET through SecurityHeaders and
+// returns the recorded response.
+func serveWithSecurityHeaders(
+	t *testing.T,
+	target string,
+	handler http.Handler,
+) *httptest.ResponseRecorder {
+	t.Helper()
+
+	mw := newTestMiddleware(t)
+	rec := httptest.NewRecorder()
+	req := httptest.NewRequestWithContext(
+		t.Context(), http.MethodGet, target, nil,
+	)
+
+	mw.SecurityHeaders()(handler).ServeHTTP(rec, req)
+
+	return rec
+}
+
+// okHandler writes a trivial 200 response.
+func okHandler() http.Handler {
+	return http.HandlerFunc(func(
+		writer http.ResponseWriter,
+		_ *http.Request,
+	) {
+		writer.WriteHeader(http.StatusOK)
+	})
+}
+
+func TestSecurityHeaders(t *testing.T) {
+	t.Parallel()
+
+	tests := []struct {
+		name   string
+		header string
+		want   string
+	}{
+		{
+			"hsts",
+			"Strict-Transport-Security",
+			wantHSTS,
+		},
+		{
+			"csp",
+			"Content-Security-Policy",
+			wantCSP,
+		},
+		{
+			"frame options",
+			"X-Frame-Options",
+			wantFrameOptions,
+		},
+		{
+			"content type options",
+			"X-Content-Type-Options",
+			wantContentTypeOptions,
+		},
+		{
+			"referrer policy",
+			"Referrer-Policy",
+			wantReferrerPolicy,
+		},
+		{
+			"permissions policy",
+			"Permissions-Policy",
+			wantPermissionsPolicy,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			t.Parallel()
+
+			rec := serveWithSecurityHeaders(t, "/", okHandler())
+
+			got := rec.Header().Get(tt.header)
+			if got != tt.want {
+				t.Errorf(
+					"%s = %q, want %q",
+					tt.header, got, tt.want,
+				)
+			}
+		})
+	}
+}
+
+// TestSecurityHeadersCSPDirectives guards the properties the repo
+// policy requires of the content security policy itself.
+func TestSecurityHeadersCSPDirectives(t *testing.T) {
+	t.Parallel()
+
+	rec := serveWithSecurityHeaders(t, "/", okHandler())
+	csp := rec.Header().Get("Content-Security-Policy")
+
+	forbidden := []string{"unsafe-inline", "unsafe-eval"}
+	for _, directive := range forbidden {
+		if strings.Contains(csp, directive) {
+			t.Errorf("CSP must not contain %q: %q", directive, csp)
+		}
+	}
+
+	required := []string{
+		"default-src 'self'",
+		"script-src 'none'",
+		"style-src 'self'",
+		"frame-ancestors 'none'",
+	}
+	for _, directive := range required {
+		if !strings.Contains(csp, directive) {
+			t.Errorf("CSP must contain %q: %q", directive, csp)
+		}
+	}
+}
+
+// TestSecurityHeadersOnErrorResponse verifies the headers are emitted
+// even when the wrapped handler fails, since they are set before the
+// handler runs.
+func TestSecurityHeadersOnErrorResponse(t *testing.T) {
+	t.Parallel()
+
+	failing := http.HandlerFunc(func(
+		writer http.ResponseWriter,
+		_ *http.Request,
+	) {
+		http.Error(
+			writer,
+			"boom",
+			http.StatusInternalServerError,
+		)
+	})
+
+	rec := serveWithSecurityHeaders(t, "/api/v1/status", failing)
+
+	if rec.Code != http.StatusInternalServerError {
+		t.Fatalf("status = %d, want 500", rec.Code)
+	}
+
+	if got := rec.Header().Get(
+		"X-Content-Type-Options",
+	); got != wantContentTypeOptions {
+		t.Errorf(
+			"X-Content-Type-Options = %q, want %q",
+			got, wantContentTypeOptions,
+		)
+	}
+
+	if got := rec.Header().Get(
+		"Strict-Transport-Security",
+	); got != wantHSTS {
+		t.Errorf(
+			"Strict-Transport-Security = %q, want %q",
+			got, wantHSTS,
+		)
+	}
+}
+
+// newTestHandlers builds real Handlers with empty monitoring state.
+func newTestHandlers(t *testing.T) *handlers.Handlers {
+	t.Helper()
+
+	glob, err := globals.New(nil)
+	if err != nil {
+		t.Fatalf("globals.New: %v", err)
+	}
+
+	log := newTestLogger(t)
+
+	notifier, err := notify.New(fxtest.NewLifecycle(t), notify.Params{
+		Logger: log,
+		Config: &config.Config{},
+	})
+	if err != nil {
+		t.Fatalf("notify.New: %v", err)
+	}
+
+	st, err := state.New(fxtest.NewLifecycle(t), state.Params{
+		Logger: log,
+		Config: &config.Config{DataDir: t.TempDir()},
+	})
+	if err != nil {
+		t.Fatalf("state.New: %v", err)
+	}
+
+	hnd, err := handlers.New(nil, handlers.Params{
+		Logger:  log,
+		Globals: glob,
+		State:   st,
+		Notify:  notifier,
+	})
+	if err != nil {
+		t.Fatalf("handlers.New: %v", err)
+	}
+
+	return hnd
+}
+
+// TestDashboardRendersWithSecurityHeaders renders the real dashboard
+// through the middleware and checks that the policy still permits the
+// one stylesheet the page loads.
+func TestDashboardRendersWithSecurityHeaders(t *testing.T) {
+	t.Parallel()
+
+	mw := newTestMiddleware(t)
+	hnd := newTestHandlers(t)
+
+	router := chi.NewRouter()
+	router.Use(mw.SecurityHeaders())
+	router.Get("/", hnd.HandleDashboard())
+
+	rec := httptest.NewRecorder()
+	req := httptest.NewRequestWithContext(
+		t.Context(), http.MethodGet, "/", nil,
+	)
+
+	router.ServeHTTP(rec, req)
+
+	if rec.Code != http.StatusOK {
+		t.Fatalf("status = %d, want 200", rec.Code)
+	}
+
+	body := rec.Body.String()
+	if !strings.Contains(body, stylesheetPath) {
+		t.Errorf("dashboard does not reference %q", stylesheetPath)
+	}
+
+	if !strings.Contains(body, "dnswatcher") {
+		t.Errorf("dashboard body looks empty: %d bytes", len(body))
+	}
+
+	csp := rec.Header().Get("Content-Security-Policy")
+	if csp != wantCSP {
+		t.Errorf("CSP = %q, want %q", csp, wantCSP)
+	}
+
+	// The stylesheet is same-origin, so style-src 'self' allows it.
+	if !strings.Contains(csp, "style-src 'self'") {
+		t.Errorf("CSP would block %q: %q", stylesheetPath, csp)
+	}
+}
+
+// Addresses for the rate limit and realIP tests: a client connecting
+// directly, a trusted proxy, and a client behind that proxy as the
+// proxy's X-Real-IP or X-Forwarded-For header names it.
+const (
+	directClient  = "198.51.100.1:4000"
+	trustedProxy  = "10.0.0.1:4000"
+	proxiedClient = "203.0.113.1"
+)
+
+// statusFrom sends a GET through handler as if from remoteAddr, with
+// an X-Real-IP header when xRealIP is not empty, and returns the
+// response status.
+func statusFrom(
+	t *testing.T,
+	handler http.Handler,
+	remoteAddr string,
+	xRealIP string,
+) int {
+	t.Helper()
+
+	req := httptest.NewRequestWithContext(
+		t.Context(), http.MethodGet, "/metrics", nil,
+	)
+	req.RemoteAddr = remoteAddr
+
+	if xRealIP != "" {
+		req.Header.Set("X-Real-IP", xRealIP)
+	}
+
+	rec := httptest.NewRecorder()
+	handler.ServeHTTP(rec, req)
+
+	return rec.Code
+}
+
+// TestMetricsRateLimitAllowsScraping checks that one address can send,
+// within one window, what two Prometheus servers scraping every 5
+// seconds send in that time, without being turned away.
+func TestMetricsRateLimitAllowsScraping(t *testing.T) {
+	t.Parallel()
+
+	const scrapeInterval = 5 * time.Second
+
+	scrapes := 2 * int(middleware.MetricsRequestWindow/scrapeInterval)
+	limited := newTestMiddleware(t).MetricsRateLimit()(okHandler())
+
+	for i := range scrapes {
+		got := statusFrom(t, limited, directClient, "")
+		if got != http.StatusOK {
+			t.Fatalf(
+				"scrape %d of %d: status = %d, want 200",
+				i+1, scrapes, got,
+			)
+		}
+	}
+}
+
+// TestMetricsRateLimitKeysOnClientAddress checks which requests share
+// an allowance. Each case uses up the allowance of one client, then
+// sends one more request.
+func TestMetricsRateLimitKeysOnClientAddress(t *testing.T) {
+	t.Parallel()
+
+	tests := []struct {
+		name           string
+		usedRemoteAddr string
+		usedXRealIP    string
+		nextRemoteAddr string
+		nextXRealIP    string
+		want           int
+	}{
+		{
+			"same address",
+			directClient, "",
+			directClient, "",
+			http.StatusTooManyRequests,
+		},
+		{
+			"another address",
+			directClient, "",
+			"198.51.100.2:4000", "",
+			http.StatusOK,
+		},
+		{
+			"own X-Real-IP from an untrusted address",
+			directClient, "",
+			directClient, "203.0.113.9",
+			http.StatusTooManyRequests,
+		},
+		{
+			"same client behind the proxy",
+			trustedProxy, proxiedClient,
+			trustedProxy, proxiedClient,
+			http.StatusTooManyRequests,
+		},
+		{
+			"another client behind the proxy",
+			trustedProxy, proxiedClient,
+			trustedProxy, "203.0.113.2",
+			http.StatusOK,
+		},
+		{
+			"another client behind the proxy, IPv6-mapped",
+			trustedProxy, "::ffff:203.0.113.1",
+			trustedProxy, "::ffff:203.0.113.2",
+			http.StatusOK,
+		},
+		{
+			"same IPv6 /64",
+			"[2001:db8::1]:4000", "",
+			"[2001:db8::2]:4000", "",
+			http.StatusTooManyRequests,
+		},
+		{
+			"another IPv6 /64",
+			"[2001:db8::1]:4000", "",
+			"[2001:db8:0:1::1]:4000", "",
+			http.StatusOK,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			t.Parallel()
+
+			limited := newTestMiddleware(t).MetricsRateLimit()(okHandler())
+
+			for range middleware.MetricsRequestLimit {
+				statusFrom(t, limited, tt.usedRemoteAddr, tt.usedXRealIP)
+			}
+
+			got := statusFrom(
+				t, limited, tt.nextRemoteAddr, tt.nextXRealIP,
+			)
+			if got != tt.want {
+				t.Errorf("status = %d, want %d", got, tt.want)
+			}
+		})
+	}
+}
+
+// TestRealIP checks which address realIP takes as the client's. Each
+// element of forwardedFor is sent as an X-Forwarded-For header line of
+// its own, and 198.51.100.9 is always an entry the client wrote itself.
+func TestRealIP(t *testing.T) {
+	t.Parallel()
+
+	tests := []struct {
+		name         string
+		remoteAddr   string
+		xRealIP      string
+		forwardedFor []string
+		want         string
+	}{
+		{
+			"untrusted peer, both headers ignored",
+			directClient, proxiedClient, []string{"198.51.100.9"},
+			"198.51.100.1",
+		},
+		{
+			"X-Real-IP from a trusted proxy wins",
+			trustedProxy, proxiedClient, []string{"203.0.113.8"},
+			proxiedClient,
+		},
+		{
+			"client's own entry, then the one the proxy added",
+			trustedProxy, "", []string{"198.51.100.9, 203.0.113.1"},
+			proxiedClient,
+		},
+		{
+			"several trusted proxies",
+			trustedProxy, "",
+			[]string{"198.51.100.9, 203.0.113.1, 10.0.0.3, 10.0.0.2"},
+			proxiedClient,
+		},
+		{
+			"proxy adds a header line of its own",
+			trustedProxy, "", []string{"198.51.100.9", proxiedClient},
+			proxiedClient,
+		},
+		{
+			"every entry a trusted proxy",
+			trustedProxy, "", []string{"10.0.0.3, 10.0.0.2"},
+			"10.0.0.3",
+		},
+		{
+			"empty where the client address belongs",
+			trustedProxy, "", []string{"203.0.113.1, , 10.0.0.2"},
+			"10.0.0.1",
+		},
+		{
+			"no headers from a trusted proxy",
+			trustedProxy, "", nil,
+			"10.0.0.1",
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			t.Parallel()
+
+			req := httptest.NewRequestWithContext(
+				t.Context(), http.MethodGet, "/", nil,
+			)
+			req.RemoteAddr = tt.remoteAddr
+
+			if tt.xRealIP != "" {
+				req.Header.Set("X-Real-IP", tt.xRealIP)
+			}
+
+			for _, line := range tt.forwardedFor {
+				req.Header.Add("X-Forwarded-For", line)
+			}
+
+			got := middleware.RealIP(req)
+			if got != tt.want {
+				t.Errorf("realIP = %q, want %q", got, tt.want)
+			}
+		})
+	}
+}

internal/notify/shutdown_test.go

@@ -33,10 +33,29 @@ const (
 	// out.
 	drainDeadline = 50 * time.Millisecond
 
-	// drainSlack is the upper bound on how long a bounded
+	// timeoutDrainBound is how long a drain given drainDeadline
-	// drain may take; generous enough for a loaded CI box,
+	// may take to return before the test gives up on it.  At
-	// still far below the 20s test ceiling.
+	// forty times drainDeadline it leaves ample room for
-	drainSlack = 2 * time.Second
+	// scheduling delay on a loaded box under -race, yet it is far
+	// below the test binary's -timeout, so a drain that its
+	// deadline does not bound fails that one test instead of
+	// hanging the package.
+	timeoutDrainBound = 2 * time.Second
+
+	// longDrainDeadline is the deadline given to a drain that is
+	// expected to finish well before it: when the in-flight
+	// delivery completes after inFlightHold, or at once when
+	// nothing is in flight.  It is far above inFlightHold, so
+	// those drains never reach it, and four times
+	// idleDrainBound, so an idle drain that waited for its
+	// deadline instead of returning fails that bound.
+	longDrainDeadline = 2 * time.Second
+
+	// reachEndpointTimeout is how long a submitted delivery may
+	// take to reach the test server.  That normally takes a few
+	// milliseconds; the margin is for a loaded box under -race,
+	// and only a failing run ever waits this long.
+	reachEndpointTimeout = 2 * time.Second
 
 	// settleDelay is how long to wait before asserting that
 	// something did *not* happen.
@@ -46,10 +65,11 @@ const (
 	// nothing in flight.  It is deliberately far above the cost
 	// of the goroutine hop through inFlight.Wait() — which
 	// reached 57ms on a loaded box under -race with the package's
-	// parallel tests — and far below drainSlack, the deadline
+	// parallel tests — and far below longDrainDeadline, the
-	// such a drain is given.  A drain that blocked until its
+	// deadline such a drain is given.  A drain that blocked until
-	// deadline instead of returning on the WaitGroup therefore
+	// its deadline instead of returning on the WaitGroup
-	// still fails this bound, but scheduling delay alone cannot.
+	// therefore still fails this bound, but scheduling delay
+	// alone cannot.
 	idleDrainBound = 500 * time.Millisecond
 )
 
@@ -75,12 +95,14 @@ func (sb *syncBuffer) String() string {
 }
 
 // newLoggingService returns a Service writing JSON logs into
-// the returned buffer.
+// the returned buffer, debug level included.
 func newLoggingService(
 	transport http.RoundTripper,
 ) (*notify.Service, *syncBuffer) {
 	logs := &syncBuffer{}
-	handler := slog.NewJSONHandler(logs, nil)
+	handler := slog.NewJSONHandler(
+		logs, &slog.HandlerOptions{Level: slog.LevelDebug},
+	)
 
 	return notify.NewTestServiceWithLogger(transport, handler),
 		logs
@@ -121,6 +143,12 @@ func TestDrainWaitsForInFlightDelivery(t *testing.T) {
 	srv := blockingNtfyServer(entered, release, &served)
 	defer srv.Close()
 
+	// srv.Close waits for the handler, so release it however the
+	// test ends; otherwise a drain that returns early hangs the
+	// package instead of failing this test.
+	releaseHandler := sync.OnceFunc(func() { close(release) })
+	defer releaseHandler()
+
 	topicURL, _ := url.Parse(srv.URL)
 
 	svc := notify.NewTestService(http.DefaultTransport)
@@ -134,7 +162,7 @@ func TestDrainWaitsForInFlightDelivery(t *testing.T) {
 	// drain begins.
 	select {
 	case <-entered:
-	case <-time.After(drainSlack):
+	case <-time.After(reachEndpointTimeout):
 		t.Fatal("delivery never reached the endpoint")
 	}
 
@@ -145,13 +173,11 @@ func TestDrainWaitsForInFlightDelivery(t *testing.T) {
 	// delay alone.
 	start := time.Now()
 
-	timer := time.AfterFunc(inFlightHold, func() {
+	timer := time.AfterFunc(inFlightHold, releaseHandler)
-		close(release)
-	})
 	defer timer.Stop()
 
 	ctx, cancel := context.WithTimeout(
-		context.Background(), drainSlack,
+		context.Background(), longDrainDeadline,
 	)
 	defer cancel()
 
@@ -246,7 +272,7 @@ func TestDrainBoundedByContextDeadline(t *testing.T) {
 	// all never returns here (the delivery is parked in a backoff
 	// that never fires), so an unbounded drain must fail this
 	// test promptly instead of hanging the package until the test
-	// binary's 30s timeout.
+	// binary's -timeout.
 	returned := make(chan struct{})
 
 	go func() {
@@ -257,11 +283,11 @@ func TestDrainBoundedByContextDeadline(t *testing.T) {
 
 	select {
 	case <-returned:
-	case <-time.After(drainSlack):
+	case <-time.After(timeoutDrainBound):
 		t.Fatalf(
 			"drain did not return within %v; its %v deadline "+
 				"did not bound it",
-			drainSlack, drainDeadline,
+			timeoutDrainBound, drainDeadline,
 		)
 	}
 
@@ -333,7 +359,7 @@ func TestDrainRefusesNewDeliveries(t *testing.T) {
 	svc.SetMattermostWebhookURL(target)
 
 	ctx, cancel := context.WithTimeout(
-		context.Background(), drainSlack,
+		context.Background(), longDrainDeadline,
 	)
 	defer cancel()
 
@@ -425,6 +451,11 @@ func TestNewRegistersDrainingStopHook(t *testing.T) {
 	srv := blockingNtfyServer(entered, release, &served)
 	defer srv.Close()
 
+	// As in TestDrainWaitsForInFlightDelivery: release the handler
+	// however the test ends, before srv.Close waits for it.
+	releaseHandler := sync.OnceFunc(func() { close(release) })
+	defer releaseHandler()
+
 	lifecycle := &recordingLifecycle{}
 	svc := newNotifyService(t, lifecycle, srv.URL)
 
@@ -446,17 +477,15 @@ func TestNewRegistersDrainingStopHook(t *testing.T) {
 
 	select {
 	case <-entered:
-	case <-time.After(drainSlack):
+	case <-time.After(reachEndpointTimeout):
 		t.Fatal("delivery never reached the endpoint")
 	}
 
-	timer := time.AfterFunc(inFlightHold, func() {
+	timer := time.AfterFunc(inFlightHold, releaseHandler)
-		close(release)
-	})
 	defer timer.Stop()
 
 	ctx, cancel := context.WithTimeout(
-		context.Background(), drainSlack,
+		context.Background(), longDrainDeadline,
 	)
 	defer cancel()
 
@@ -487,7 +516,7 @@ func TestDrainWithoutDeliveriesReturnsImmediately(t *testing.T) {
 	start := time.Now()
 
 	ctx, cancel := context.WithTimeout(
-		context.Background(), drainSlack,
+		context.Background(), longDrainDeadline,
 	)
 	defer cancel()
 
@@ -495,9 +524,9 @@ func TestDrainWithoutDeliveriesReturnsImmediately(t *testing.T) {
 
 	if elapsed := time.Since(start); elapsed > idleDrainBound {
 		t.Errorf(
-			"drain of an idle service took %v, want well "+
+			"drain of an idle service took %v, want at most "+
-				"under its %v deadline",
+				"%v; its deadline was %v",
-			elapsed, drainSlack,
+			elapsed, idleDrainBound, longDrainDeadline,
 		)
 	}
 }
@@ -505,10 +534,11 @@ func TestDrainWithoutDeliveriesReturnsImmediately(t *testing.T) {
 // TestDrainWithCancelledContextDoesNotWarn verifies that an
 // OnStop context that is already dead on entry does not produce
 // an "abandoning them" warning when there was nothing in flight
-// to abandon.  The expired context wins the select immediately,
+// to abandon, and that the drain returns and says at debug level
-// so only the outstanding count can tell the difference between
+// that nothing was in flight.  The expired context wins the
-// a genuine timeout and a shutdown that had simply already run
+// select immediately, so only the outstanding count can tell the
-// out of time with no work left.
+// difference between a genuine timeout and a shutdown that had
+// simply already run out of time with no work left.
 func TestDrainWithCancelledContextDoesNotWarn(t *testing.T) {
 	t.Parallel()
 
@@ -517,11 +547,43 @@ func TestDrainWithCancelledContextDoesNotWarn(t *testing.T) {
 	ctx, cancel := context.WithCancel(context.Background())
 	cancel()
 
-	svc.Drain(ctx)
+	// A watchdog, as in TestDrainBoundedByContextDeadline, so
+	// that a drain which never returns fails here instead of
+	// hanging the package.
+	returned := make(chan struct{})
 
-	if output := logs.String(); strings.Contains(
+	go func() {
-		output, `"level":"WARN"`,
+		defer close(returned)
+
+		svc.Drain(ctx)
+	}()
+
+	select {
+	case <-returned:
+	case <-time.After(idleDrainBound):
+		t.Fatalf(
+			"drain with nothing in flight and a cancelled "+
+				"context did not return within %v",
+			idleDrainBound,
+		)
+	}
+
+	output := logs.String()
+
+	// The absence of a warning alone would also pass if the drain
+	// logged nothing at all, so require the debug line it writes
+	// when it finds nothing outstanding.
+	if !strings.Contains(
+		output, "all in-flight notifications completed",
 	) {
+		t.Errorf(
+			"drain did not log that nothing was in flight; "+
+				"log output: %s",
+			output,
+		)
+	}
+
+	if strings.Contains(output, `"level":"WARN"`) {
 		t.Errorf(
 			"drain with nothing in flight warned about "+
 				"abandoned deliveries; log output: %s",

internal/resolver/dns_client.go

@@ -7,8 +7,8 @@ import (
 	"github.com/miekg/dns"
 )
 
-// DNSClient abstracts DNS wire-protocol exchanges so the resolver
+// DNSClient sends one DNS message to a nameserver and returns the
-// can be tested without hitting real nameservers.
+// reply. The resolver holds one for UDP and one for TCP.
 type DNSClient interface {
 	ExchangeContext(
 		ctx context.Context,

internal/resolver/errors.go

@@ -10,6 +10,18 @@ var (
 		"no authoritative nameservers found",
 	)
 
+	// ErrNoNameserverAnswered is returned when every nameserver
+	// asked about a name timed out, failed or returned a referral,
+	// so whether the name has addresses is unknown.
+	ErrNoNameserverAnswered = errors.New("no nameserver answered")
+
+	// ErrUnusableReply is returned when a server replied with an
+	// error such as SERVFAIL, or with a referral that leads no
+	// closer to the name asked about.
+	ErrUnusableReply = errors.New(
+		"reply is an error or a referral that leads no closer",
+	)
+
 	// ErrCNAMEDepthExceeded is returned when a CNAME chain
 	// exceeds MaxCNAMEDepth.
 	ErrCNAMEDepthExceeded = errors.New(

internal/resolver/export_test.go

@@ -0,0 +1,59 @@
+package resolver
+
+import (
+	"context"
+
+	"github.com/miekg/dns"
+)
+
+// ExtractRecordValue exports extractRecordValue for testing.
+func ExtractRecordValue(rr dns.RR) string {
+	return extractRecordValue(rr)
+}
+
+// UsableReply exports usableReply for testing.
+func UsableReply(resp *dns.Msg, zone string, name string) bool {
+	return usableReply(resp, zone, name)
+}
+
+// NSSetFrom exports nsSetFrom for testing.
+func NSSetFrom(resp *dns.Msg, domain string) []string {
+	return nsSetFrom(resp, domain)
+}
+
+// CollectIPs exports collectIPs for testing.
+func CollectIPs(
+	results map[string]*NameserverResponse,
+) ([]string, string, error) {
+	return collectIPs(results)
+}
+
+// QueryEachNS exports queryEachNS for testing.
+func (r *Resolver) QueryEachNS(
+	ctx context.Context,
+	nameservers []string,
+	hostname string,
+) (map[string]*NameserverResponse, error) {
+	return r.queryEachNS(ctx, nameservers, hostname)
+}
+
+// ResolveNSIPs exports resolveNSIPs for testing.
+func (r *Resolver) ResolveNSIPs(
+	ctx context.Context,
+	nsNames []string,
+) []string {
+	return r.resolveNSIPs(ctx, nsNames)
+}
+
+// RootServerList exports rootServerList for testing.
+func RootServerList() []string {
+	return rootServerList()
+}
+
+// Shuffled exports shuffled for testing.
+func Shuffled(
+	servers []string,
+	shuffle func(n int, swap func(i, j int)),
+) []string {
+	return shuffled(servers, shuffle)
+}

internal/resolver/iterative.go

@@ -4,7 +4,9 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"math/rand/v2"
 	"net"
+	"slices"
 	"sort"
 	"strings"
 	"time"
@@ -17,7 +19,6 @@ const (
 	maxRetries           = 2
 	maxDelegation        = 20
 	timeoutMultiplier    = 2
-	minDomainLabels      = 2
 )
 
 // ErrRefused is returned when a DNS server refuses a query.
@@ -208,21 +209,33 @@ func (r *Resolver) followDelegation(
 	domain string,
 	servers []string,
 ) ([]string, error) {
+	// servers are the root servers, the servers of zone ".".
+	zone := "."
+
 	for range maxDelegation {
 		if checkCtx(ctx) != nil {
 			return nil, ErrContextCanceled
 		}
 
 		resp, err := r.queryServers(
-			ctx, servers, domain, dns.TypeNS,
+			ctx, servers, zone, domain, dns.TypeNS,
 		)
 		if err != nil {
 			return nil, err
 		}
 
-		ansNS := extractNSSet(resp.Answer)
+		nsSet := nsSetFrom(resp, domain)
-		if len(ansNS) > 0 {
+		if len(nsSet) > 0 {
-			return ansNS, nil
+			return nsSet, nil
+		}
+
+		// An authoritative reply comes from the servers of the zone
+		// domain is in; it is not a referral, even when its authority
+		// section lists that zone's NS records. Without NS records in
+		// the answer, domain is not the zone's apex and has no
+		// nameservers of its own.
+		if resp.Authoritative {
+			return nil, ErrNoNameservers
 		}
 
 		authNS := extractNSSet(resp.Ns)
@@ -242,25 +255,52 @@ func (r *Resolver) followDelegation(
 		}
 
 		servers = nextServers
+		zone = referralZone(resp)
 	}
 
 	return nil, ErrNoNameservers
 }
 
+// shuffled returns a copy of servers in the order shuffle puts them
+// in. The resolver passes rand.Shuffle, so each time it walks a list of
+// servers it starts at a random one, and no one server gets every
+// first query.
+func shuffled(
+	servers []string,
+	shuffle func(n int, swap func(i, j int)),
+) []string {
+	order := slices.Clone(servers)
+	shuffle(len(order), func(i, j int) {
+		order[i], order[j] = order[j], order[i]
+	})
+
+	return order
+}
+
+// queryServers asks servers, the servers of zone, about name in a random
+// order until one gives a usable reply. A server that times out, refuses
+// or gives a reply that is not usable is passed over for the next.
 func (r *Resolver) queryServers(
 	ctx context.Context,
 	servers []string,
+	zone string,
 	name string,
 	qtype uint16,
 ) (*dns.Msg, error) {
 	var lastErr error
 
-	for _, ip := range servers {
+	for _, ip := range shuffled(servers, rand.Shuffle) {
 		if checkCtx(ctx) != nil {
 			return nil, ErrContextCanceled
 		}
 
 		resp, err := r.queryDNS(ctx, ip, name, qtype)
+		if err == nil && !usableReply(resp, zone, name) {
+			err = fmt.Errorf(
+				"query %s @%s: %w", name, ip, ErrUnusableReply,
+			)
+		}
+
 		if err == nil {
 			return resp, nil
 		}
@@ -271,6 +311,57 @@ func (r *Resolver) queryServers(
 	return nil, fmt.Errorf("all servers failed: %w", lastErr)
 }
 
+// usableReply reports whether resp, a reply from one of the servers of
+// zone to a query about name, is usable. An error reply such as SERVFAIL
+// is not. Nor is a referral, unless it refers the query to a zone below
+// zone that name is in: a server that refers it back to zone, up or
+// sideways does not serve zone as it should.
+func usableReply(resp *dns.Msg, zone string, name string) bool {
+	if resp.Rcode != dns.RcodeSuccess && resp.Rcode != dns.RcodeNameError {
+		return false
+	}
+
+	child := referralZone(resp)
+	if resp.Authoritative || len(resp.Answer) > 0 || child == "" {
+		return true
+	}
+
+	return child != zone && dns.IsSubDomain(zone, child) &&
+		dns.IsSubDomain(child, name)
+}
+
+// referralZone returns the zone a referral refers the query to: the
+// owner name of the NS records in resp's authority section, or "" when
+// there are none.
+func referralZone(resp *dns.Msg) string {
+	for _, rr := range resp.Ns {
+		if ns, ok := rr.(*dns.NS); ok {
+			return strings.ToLower(ns.Hdr.Name)
+		}
+	}
+
+	return ""
+}
+
+// nsSetFrom returns the NS set of domain that resp, a reply to a query
+// for domain's NS records, gives: the delegation in a referral to domain
+// itself, or else the NS records in the answer; empty when it gives
+// neither. A referral to domain comes from its parent zone's servers,
+// which all hold the same delegation, so the set does not depend on
+// which of them answered. domain's own servers, which can disagree about
+// their NS records, are then not asked.
+func nsSetFrom(resp *dns.Msg, domain string) []string {
+	if referralZone(resp) == domain {
+		return extractNSSet(resp.Ns)
+	}
+
+	return extractNSSet(resp.Answer)
+}
+
+// resolveNSIPs returns the addresses of every nameserver in nsNames
+// whose name resolves, for a referral that carries none. The walk can
+// then go on to the zone's other nameservers when one gives no usable
+// reply.
 func (r *Resolver) resolveNSIPs(
 	ctx context.Context,
 	nsNames []string,
@@ -282,10 +373,6 @@ func (r *Resolver) resolveNSIPs(
 		if err == nil {
 			ips = append(ips, resolved...)
 		}
-
-		if len(ips) > 0 {
-			break
-		}
 	}
 
 	return ips
@@ -304,6 +391,7 @@ func (r *Resolver) resolveNSIterative(
 
 	domain = dns.Fqdn(domain)
 	servers := rootServerList()
+	zone := "."
 
 	for range maxDelegation {
 		if checkCtx(ctx) != nil {
@@ -311,13 +399,13 @@ func (r *Resolver) resolveNSIterative(
 		}
 
 		resp, err := r.queryServers(
-			ctx, servers, domain, dns.TypeNS,
+			ctx, servers, zone, domain, dns.TypeNS,
 		)
 		if err != nil {
 			return nil, err
 		}
 
-		nsNames := extractNSSet(resp.Answer)
+		nsNames := nsSetFrom(resp, domain)
 		if len(nsNames) > 0 {
 			return nsNames, nil
 		}
@@ -336,6 +424,7 @@ func (r *Resolver) resolveNSIterative(
 		}
 
 		servers = nextServers
+		zone = referralZone(resp)
 	}
 
 	return nil, ErrNoNameservers
@@ -353,6 +442,7 @@ func (r *Resolver) resolveARecord(
 
 	hostname = dns.Fqdn(hostname)
 	servers := rootServerList()
+	zone := "."
 
 	for range maxDelegation {
 		if checkCtx(ctx) != nil {
@@ -360,7 +450,7 @@ func (r *Resolver) resolveARecord(
 		}
 
 		resp, err := r.queryServers(
-			ctx, servers, hostname, dns.TypeA,
+			ctx, servers, zone, hostname, dns.TypeA,
 		)
 		if err != nil {
 			return nil, fmt.Errorf(
@@ -398,6 +488,7 @@ func (r *Resolver) resolveARecord(
 		}
 
 		servers = nextServers
+		zone = referralZone(resp)
 	}
 
 	return nil, fmt.Errorf(
@@ -407,7 +498,10 @@ func (r *Resolver) resolveARecord(
 
 // FindAuthoritativeNameservers traces the delegation chain from
 // root servers to discover all authoritative nameservers for the
-// given domain. Walks up the label hierarchy for subdomains.
+// given domain, as the delegation from its parent zone's servers lists
+// them. For a name that is not a zone apex it tries each
+// parent name in turn, so it returns the nameservers of the zone the
+// name is in.
 func (r *Resolver) FindAuthoritativeNameservers(
 	ctx context.Context,
 	domain string,
@@ -504,7 +598,10 @@ func (r *Resolver) queryAllTypes(
 type queryState struct {
 	gotNXDomain bool
 	gotSERVFAIL bool
+	gotRefused  bool
 	gotTimeout  bool
+	gotReferral bool
+	netErr      error
 	hasRecords  bool
 }
 
@@ -542,8 +639,13 @@ func (r *Resolver) querySingleType(
 ) {
 	msg, err := r.queryDNS(ctx, nsIP, hostname, qtype)
 	if err != nil {
-		if isTimeout(err) {
+		switch {
+		case isTimeout(err):
 			state.gotTimeout = true
+		case errors.Is(err, ErrRefused):
+			state.gotRefused = true
+		default:
+			state.netErr = err
 		}
 
 		return
@@ -561,6 +663,19 @@ func (r *Resolver) querySingleType(
 		return
 	}
 
+	// A reply with no answer that lists other nameservers, from a server
+	// that does not hold the name's zone, is a referral and says nothing
+	// about the name's records. A server named in the delegation that
+	// does not hold the zone may send one, as do a parent zone's servers
+	// when FindAuthoritativeNameservers found no delegation for the
+	// name's zone and moved on to a parent name.
+	if !msg.Authoritative && len(msg.Answer) == 0 &&
+		len(extractNSSet(msg.Ns)) > 0 {
+		state.gotReferral = true
+
+		return
+	}
+
 	collectAnswerRecords(msg, resp, state)
 }
 
@@ -603,12 +718,24 @@ func classifyResponse(resp *NameserverResponse, state queryState) {
 	case state.gotSERVFAIL && !state.hasRecords:
 		resp.Status = StatusError
 		resp.Error = "server returned SERVFAIL"
+	case state.gotRefused && !state.hasRecords:
+		resp.Status = StatusError
+		resp.Error = "server returned REFUSED"
+	case state.netErr != nil && !state.hasRecords:
+		resp.Status = StatusError
+		resp.Error = "network error: " + state.netErr.Error()
+	case state.gotReferral && !state.hasRecords:
+		resp.Status = StatusError
+		resp.Error = "server returned a referral"
 	case !state.hasRecords && !state.gotNXDomain:
 		resp.Status = StatusNoData
 	}
 }
 
-// extractRecordValue formats a DNS RR value as a string.
+// extractRecordValue formats a DNS RR value as a string. DNS names
+// are case-insensitive and nameservers may answer in any letter case,
+// so names are lower-cased to compare equal. TXT and CAA values keep
+// their letter case.
 func extractRecordValue(rr dns.RR) string {
 	switch r := rr.(type) {
 	case *dns.A:
@@ -616,43 +743,29 @@ func extractRecordValue(rr dns.RR) string {
 	case *dns.AAAA:
 		return r.AAAA.String()
 	case *dns.CNAME:
-		return r.Target
+		return strings.ToLower(r.Target)
 	case *dns.MX:
-		return fmt.Sprintf("%d %s", r.Preference, r.Mx)
+		return fmt.Sprintf("%d %s", r.Preference, strings.ToLower(r.Mx))
 	case *dns.TXT:
 		return strings.Join(r.Txt, "")
 	case *dns.SRV:
 		return fmt.Sprintf(
 			"%d %d %d %s",
-			r.Priority, r.Weight, r.Port, r.Target,
+			r.Priority, r.Weight, r.Port, strings.ToLower(r.Target),
 		)
 	case *dns.CAA:
 		return fmt.Sprintf(
 			"%d %s \"%s\"", r.Flag, r.Tag, r.Value,
 		)
 	case *dns.NS:
-		return r.Ns
+		return strings.ToLower(r.Ns)
 	default:
 		return ""
 	}
 }
 
-// parentDomain returns the registerable parent domain.
+// QueryAllNameservers discovers the auth NSes of the zone the
-func parentDomain(hostname string) string {
+// hostname is in, then queries each one independently.
-	hostname = dns.Fqdn(strings.ToLower(hostname))
-	labels := dns.SplitDomainName(hostname)
-
-	if len(labels) <= minDomainLabels {
-		return strings.Join(labels, ".") + "."
-	}
-
-	return strings.Join(
-		labels[len(labels)-minDomainLabels:], ".",
-	) + "."
-}
-
-// QueryAllNameservers discovers auth NSes for the hostname's
-// parent domain, then queries each one independently.
 func (r *Resolver) QueryAllNameservers(
 	ctx context.Context,
 	hostname string,
@@ -661,9 +774,7 @@ func (r *Resolver) QueryAllNameservers(
 		return nil, ErrContextCanceled
 	}
 
-	parent := parentDomain(hostname)
+	nameservers, err := r.FindAuthoritativeNameservers(ctx, hostname)
-
-	nameservers, err := r.FindAuthoritativeNameservers(ctx, parent)
 	if err != nil {
 		return nil, err
 	}
@@ -679,11 +790,14 @@ func (r *Resolver) queryEachNS(
 	results := make(map[string]*NameserverResponse)
 
 	for _, ns := range nameservers {
+		resp, err := r.QueryNameserver(ctx, ns, hostname)
+
+		// A query the context cut short says nothing about the
+		// nameserver, so it must not be returned as its failure.
 		if checkCtx(ctx) != nil {
 			return nil, ErrContextCanceled
 		}
 
-		resp, err := r.QueryNameserver(ctx, ns, hostname)
 		if err != nil {
 			results[ns] = &NameserverResponse{
 				Nameserver: ns,
@@ -711,25 +825,19 @@ func (r *Resolver) LookupNS(
 
 // LookupAllRecords performs iterative resolution to find all DNS
 // records for the given hostname, keyed by authoritative nameserver.
+// Each nameserver's response carries its status and error with its
+// records.
 func (r *Resolver) LookupAllRecords(
 	ctx context.Context,
 	hostname string,
-) (map[string]map[string][]string, error) {
+) (map[string]*NameserverResponse, error) {
-	results, err := r.QueryAllNameservers(ctx, hostname)
+	return r.QueryAllNameservers(ctx, hostname)
-	if err != nil {
-		return nil, err
-	}
-
-	out := make(map[string]map[string][]string, len(results))
-	for ns, resp := range results {
-		out[ns] = resp.Records
-	}
-
-	return out, nil
 }
 
 // ResolveIPAddresses resolves a hostname to all IPv4 and IPv6
-// addresses, following CNAME chains up to MaxCNAMEDepth.
+// addresses, following CNAME chains up to MaxCNAMEDepth. When no
+// nameserver of the name's zone answered, it returns an error rather
+// than no addresses.
 func (r *Resolver) ResolveIPAddresses(
 	ctx context.Context,
 	hostname string,
@@ -755,7 +863,10 @@ func (r *Resolver) resolveIPWithCNAME(
 		return nil, err
 	}
 
-	ips, cnameTarget := collectIPs(results)
+	ips, cnameTarget, err := collectIPs(results)
+	if err != nil {
+		return nil, fmt.Errorf("resolving %s: %w", hostname, err)
+	}
 
 	if len(ips) == 0 && cnameTarget != "" {
 		return r.resolveIPWithCNAME(ctx, cnameTarget, depth+1)
@@ -766,16 +877,28 @@ func (r *Resolver) resolveIPWithCNAME(
 	return ips, nil
 }
 
+// collectIPs returns the addresses in the nameservers' answers and the
+// first CNAME target among them. It returns ErrNoNameserverAnswered when
+// every nameserver timed out, failed or returned a referral: that is not
+// a name with no addresses.
 func collectIPs(
 	results map[string]*NameserverResponse,
-) ([]string, string) {
+) ([]string, string, error) {
 	seen := make(map[string]bool)
 
 	var ips []string
 
 	var cnameTarget string
 
+	answered := false
+
 	for _, resp := range results {
+		if resp.Status == StatusTimeout || resp.Status == StatusError {
+			continue
+		}
+
+		answered = true
+
 		if resp.Status == StatusNXDomain {
 			continue
 		}
@@ -799,5 +922,9 @@ func collectIPs(
 		}
 	}
 
-	return ips, cnameTarget
+	if !answered {
+		return nil, "", ErrNoNameserverAnswered
+	}
+
+	return ips, cnameTarget, nil
 }

internal/resolver/iterative_test.go

@@ -0,0 +1,266 @@
+package resolver_test
+
+import (
+	"math/rand/v2"
+	"slices"
+	"testing"
+
+	"github.com/miekg/dns"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"sneak.berlin/go/dnswatcher/internal/resolver"
+)
+
+// TestCollectIPs_OneAnswerIsEnough checks that one nameserver answering
+// NXDOMAIN says the name has no addresses, though the other timed out.
+func TestCollectIPs_OneAnswerIsEnough(t *testing.T) {
+	t.Parallel()
+
+	ips, _, err := resolver.CollectIPs(
+		map[string]*resolver.NameserverResponse{
+			"ns1.example.": {Status: resolver.StatusTimeout},
+			"ns2.example.": {Status: resolver.StatusNXDomain},
+		},
+	)
+	require.NoError(t, err)
+	assert.Empty(t, ips)
+}
+
+// TestCollectIPs_FailedIsNoAnswer checks that nameservers that all have
+// status error, from a refusal, a server failure, a network error or a
+// referral, are no answer rather than a name with no addresses.
+func TestCollectIPs_FailedIsNoAnswer(t *testing.T) {
+	t.Parallel()
+
+	ips, _, err := resolver.CollectIPs(
+		map[string]*resolver.NameserverResponse{
+			"ns1.example.": {Status: resolver.StatusError},
+			"ns2.example.": {Status: resolver.StatusError},
+		},
+	)
+	require.ErrorIs(t, err, resolver.ErrNoNameserverAnswered)
+	assert.Empty(t, ips)
+}
+
+const (
+	// exampleCom is the zone most cases of TestUsableReply and
+	// TestNSSetFrom are about, and wwwExampleCom a name in it.
+	exampleCom    = "example.com."
+	wwwExampleCom = "www.example.com."
+
+	// exampleNS is the server the NS records nsRecord builds name.
+	exampleNS = "ns1.example.net."
+)
+
+// nsRecord builds an NS record that names a server of zone.
+func nsRecord(zone string) *dns.NS {
+	return &dns.NS{
+		Hdr: dns.RR_Header{
+			Name: zone, Rrtype: dns.TypeNS, Class: dns.ClassINET,
+		},
+		Ns: exampleNS,
+	}
+}
+
+// referralTo builds a reply that refers the query to the servers of
+// zone.
+func referralTo(zone string) *dns.Msg {
+	msg := new(dns.Msg)
+	msg.Ns = []dns.RR{nsRecord(zone)}
+
+	return msg
+}
+
+// TestUsableReply checks which replies from one of a zone's servers are
+// used. A reply that is not usable moves the query on to the zone's
+// next server.
+func TestUsableReply(t *testing.T) {
+	t.Parallel()
+
+	servfail := new(dns.Msg)
+	servfail.Rcode = dns.RcodeServerFailure
+
+	answer := new(dns.Msg)
+	answer.Authoritative = true
+	answer.Answer = []dns.RR{nsRecord(exampleCom)}
+
+	nxdomain := new(dns.Msg)
+	nxdomain.Authoritative = true
+	nxdomain.Rcode = dns.RcodeNameError
+
+	tests := []struct {
+		name  string
+		resp  *dns.Msg
+		zone  string
+		query string
+		want  bool
+	}{
+		{
+			name: "SERVFAIL", resp: servfail,
+			zone: exampleCom, query: exampleCom, want: false,
+		},
+		{
+			name: "answer", resp: answer,
+			zone: exampleCom, query: exampleCom, want: true,
+		},
+		{
+			name: "NXDOMAIN", resp: nxdomain,
+			zone: ".", query: exampleCom, want: true,
+		},
+		{
+			name: "root refers to com", resp: referralTo("com."),
+			zone: ".", query: exampleCom, want: true,
+		},
+		{
+			name: "com refers to example.com", resp: referralTo(exampleCom),
+			zone: "com.", query: wwwExampleCom, want: true,
+		},
+		{
+			name: "referral back to the zone", resp: referralTo(exampleCom),
+			zone: exampleCom, query: exampleCom, want: false,
+		},
+		{
+			name: "referral up to the root", resp: referralTo("."),
+			zone: exampleCom, query: exampleCom, want: false,
+		},
+		{
+			name: "referral sideways", resp: referralTo("net."),
+			zone: ".", query: exampleCom, want: false,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			t.Parallel()
+
+			assert.Equal(t, tt.want,
+				resolver.UsableReply(tt.resp, tt.zone, tt.query),
+			)
+		})
+	}
+}
+
+// TestNSSetFrom checks which NS set a reply gives for a domain; a set
+// that is not empty ends the walk. The referral to example.com that
+// com's servers all send alike gives its delegation, so the set is the
+// same whichever of them answered, and example.com's own servers, which
+// can disagree, are not asked.
+func TestNSSetFrom(t *testing.T) {
+	t.Parallel()
+
+	answer := new(dns.Msg)
+	answer.Authoritative = true
+	answer.Answer = []dns.RR{nsRecord(exampleCom)}
+
+	tests := []struct {
+		name   string
+		resp   *dns.Msg
+		domain string
+		want   []string
+	}{
+		{
+			name: "com refers to example.com", resp: referralTo(exampleCom),
+			domain: exampleCom, want: []string{exampleNS},
+		},
+		{
+			name: "com refers on, for www.example.com",
+			resp: referralTo(exampleCom), domain: wwwExampleCom,
+			want: nil,
+		},
+		{
+			name: "answer from a server that holds example.com",
+			resp: answer, domain: exampleCom,
+			want: []string{exampleNS},
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			t.Parallel()
+
+			assert.ElementsMatch(t, tt.want,
+				resolver.NSSetFrom(tt.resp, tt.domain),
+			)
+		})
+	}
+}
+
+func TestExtractRecordValue_LetterCase(t *testing.T) {
+	t.Parallel()
+
+	tests := []struct {
+		name string
+		rr   dns.RR
+		want string
+	}{
+		{
+			name: "MX target lower-cased",
+			rr:   &dns.MX{Preference: 1, Mx: "ASPMX.L.GOOGLE.COM."},
+			want: "1 aspmx.l.google.com.",
+		},
+		{
+			name: "NS target lower-cased",
+			rr:   &dns.NS{Ns: "x.ns.joker.COM."},
+			want: "x.ns.joker.com.",
+		},
+		{
+			name: "CNAME target lower-cased",
+			rr:   &dns.CNAME{Target: "WWW.Example.Com."},
+			want: "www.example.com.",
+		},
+		{
+			name: "SRV target lower-cased",
+			rr: &dns.SRV{
+				Priority: 10, Weight: 5, Port: 443,
+				Target: "SIP.Example.Com.",
+			},
+			want: "10 5 443 sip.example.com.",
+		},
+		{
+			name: "TXT value keeps its case",
+			rr:   &dns.TXT{Txt: []string{"Verify=AbC123"}},
+			want: "Verify=AbC123",
+		},
+		{
+			name: "CAA value keeps its case",
+			rr:   &dns.CAA{Flag: 0, Tag: "issue", Value: "LetsEncrypt.org"},
+			want: `0 issue "LetsEncrypt.org"`,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			t.Parallel()
+
+			assert.Equal(t, tt.want, resolver.ExtractRecordValue(tt.rr))
+		})
+	}
+}
+
+// TestShuffled shuffles the root servers with many seeds. Every order
+// must hold each root server once, so each is tried before a
+// resolution fails; each root server must come first for some seed, so
+// no one root server gets every first query; and the list passed in
+// must be left as it was.
+func TestShuffled(t *testing.T) {
+	t.Parallel()
+
+	const seeds = 1000
+
+	roots := resolver.RootServerList()
+	before := slices.Clone(roots)
+	first := make(map[string]bool)
+
+	for seed := range uint64(seeds) {
+		rng := rand.New(rand.NewPCG(seed, 0)) //nolint:gosec // seeded on purpose
+		order := resolver.Shuffled(roots, rng.Shuffle)
+
+		assert.ElementsMatch(t, roots, order)
+
+		first[order[0]] = true
+	}
+
+	assert.Len(t, first, len(roots))
+	assert.Equal(t, before, roots)
+}

internal/resolver/livedns_harness_test.go

@@ -1,10 +1,7 @@
 package resolver_test
 
 import (
-	"context"
-	"sync"
 	"testing"
-	"time"
 
 	"github.com/stretchr/testify/assert"
 
@@ -12,9 +9,8 @@ import (
 )
 
 // Tests for the live-DNS harness in livedns_test.go itself. These
-// exercise pure logic and the retry/concurrency plumbing; they
+// exercise pure logic; they perform no DNS resolution of any kind, so
-// perform no DNS resolution of any kind, so they neither mock DNS
+// they neither mock DNS nor depend on it.
-// nor depend on it.
 
 // Names for the synthetic status maps below. Nothing is ever queried
 // at them: they are map keys handed to the package's pure counting
@@ -90,47 +86,6 @@ func TestStatusCountingIgnoresSilentNameservers(t *testing.T) {
 	)
 }
 
-func TestRetryLiveRecoversFromTransientFailure(t *testing.T) {
-	t.Parallel()
-
-	const wantAttempts = 2
-
-	attempts := 0
-
-	retryLive(t, "transient", func(_ context.Context) error {
-		attempts++
-
-		if attempts < wantAttempts {
-			return errLiveNoAnswer
-		}
-
-		return nil
-	})
-
-	assert.Equal(t, wantAttempts, attempts)
-}
-
-func TestRetryLiveGivesEachAttemptADeadline(t *testing.T) {
-	t.Parallel()
-
-	retryLive(t, "deadline", func(ctx context.Context) error {
-		deadline, ok := ctx.Deadline()
-		assert.True(t, ok, "attempt should carry a deadline")
-
-		remaining := time.Until(deadline)
-
-		assert.LessOrEqual(t, remaining, liveAttemptTimeout)
-
-		// Lower bound too: without one this passes for a
-		// deadline far shorter than intended, which would
-		// silently turn every live attempt into an instant
-		// timeout.
-		assert.Greater(t, remaining, liveAttemptTimeout/2)
-
-		return nil
-	})
-}
-
 // TestUnsanctionedStatusesRejectsWrongAnswers is the regression test
 // for the defect this allowlist exists to prevent: a minority of
 // nameservers answering WRONGLY while quorum keeps the suite green.
@@ -235,50 +190,3 @@ func TestUnsanctionedStatusesToleratesSilenceOnly(t *testing.T) {
 		unsanctionedStatuses(results, allowed...),
 	)
 }
-
-func TestRunLiveBoundsConcurrency(t *testing.T) {
-	t.Parallel()
-
-	const workers = 24
-
-	var (
-		mu       sync.Mutex
-		wg       sync.WaitGroup
-		inFlight int
-		maxSeen  int
-	)
-
-	wg.Add(workers)
-
-	for range workers {
-		go func() {
-			defer wg.Done()
-
-			_ = runLive(func(_ context.Context) error {
-				mu.Lock()
-				inFlight++
-
-				if inFlight > maxSeen {
-					maxSeen = inFlight
-				}
-				mu.Unlock()
-
-				time.Sleep(time.Millisecond)
-
-				mu.Lock()
-				inFlight--
-				mu.Unlock()
-
-				return nil
-			})
-		}()
-	}
-
-	wg.Wait()
-
-	assert.Positive(t, maxSeen)
-	assert.LessOrEqual(
-		t, maxSeen, liveConcurrency,
-		"live queries must stay under the package-wide gate",
-	)
-}

internal/resolver/livedns_test.go

@@ -8,8 +8,8 @@ import (
 	"sort"
 	"strings"
 	"testing"
-	"time"
 
+	"sneak.berlin/go/dnswatcher/internal/livednstest"
 	"sneak.berlin/go/dnswatcher/internal/resolver"
 )
 
@@ -17,144 +17,34 @@ import (
 // Live DNS test support
 // ----------------------------------------------------------------
 //
-// Every test in this package resolves against the real, live DNS —
+// Tests that look something up in DNS query live DNS servers, never a
-// see TESTING.md. Nothing here mocks, fakes, stubs, records or
+// stand-in; logic that works on record data may be tested on that
-// replays DNS, and nothing here skips or gates a test: the helpers
+// data with no lookup (see TESTING.md). Each live operation below goes
-// below only change *how* the live queries are issued, so that a
+// through livednstest.Retry, which bounds how many resolutions are in
-// single dropped UDP packet or one slow authoritative server does
+// flight at once and retries an operation that got no answer (see
-// not turn a correct resolver into a red build.
+// package livednstest).
 //
-// Three mechanisms, all test-side:
+// Where an assertion spans several independent nameservers, a quorum
+// is enough: a strict majority answering as expected. A server that
+// fails to answer is tolerated, while a server that answers *wrongly*
+// still fails the test.
 //
-//  1. Bounded concurrency. The package's tests are parallel and the
+// That tolerance is expressed as an ALLOWLIST of sanctioned statuses,
-//     build hosts have many cores, so without a limit every test
+// never as a blocklist of known-bad ones. A blocklist bans the one
-//     starts its own iterative resolution at the same instant and
+// wrong answer its author thought of and silently admits every other
-//     they all hit the first root server in rootServerList() within
+// status, including any added to the resolver later; an allowlist
-//     a few milliseconds of each other. Root servers rate-limit
+// fails on anything nobody explicitly sanctioned. Silence (timeout,
-//     that, which shows up as a different arbitrary subset of tests
+// error) is the only thing quorum exists to tolerate. A *wrong
-//     failing on each run. liveGate caps how many resolutions are
+// answer* — nxdomain for a name that exists, ok for one that does
-//     in flight at once.
+// not, nodata for either — is never tolerated at any count.
-//
-//  2. Retry with exponential backoff. Each live operation gets
-//     several attempts with its own timeout. The retry predicate is
-//     strictly transport-level — "did a nameserver answer at all" —
-//     never the assertion the test is making. A resolver that
-//     answers incorrectly still fails on the first attempt.
-//
-//  3. Quorum. Where an assertion spans several independent
-//     nameservers, a strict majority answering as expected is
-//     enough; a server that fails to answer is tolerated, while a
-//     server that answers *wrongly* still fails the test.
-//
-// The tolerance in (3) is expressed as an ALLOWLIST of sanctioned
-// statuses, never as a blocklist of known-bad ones. A blocklist bans
-// the one wrong answer its author thought of and silently admits
-// every other status, including any added to the resolver later; an
-// allowlist fails on anything nobody explicitly sanctioned. Silence
-// (timeout, error) is the only thing quorum exists to tolerate. A
-// *wrong answer* — nxdomain for a name that exists, ok for one that
-// does not, nodata for either — is never tolerated at any count.
 
-const (
+// minNameservers is the smallest nameserver count a well-run zone is
-	// liveAttempts is how many times a live DNS operation is
+// expected to publish.
-	// attempted before the test fails.
+const minNameservers = 2
-	liveAttempts = 3
 
-	// liveAttemptTimeout bounds one attempt. Worst case for an
+// errLiveNoQuorum reports that too few of a domain's nameservers
-	// operation is liveAttempts * liveAttemptTimeout plus the
+// answered for a quorum assertion to be made.
-	// backoff — about 26 seconds, well inside the 90-second
+var errLiveNoQuorum = errors.New("no nameserver quorum")
-	// `go test -timeout` backstop even when several operations
-	// exhaust their attempts.
-	liveAttemptTimeout = 8 * time.Second
-
-	// liveBackoffBase is the delay after the first failed
-	// attempt; it is multiplied by liveBackoffFactor each time.
-	liveBackoffBase = 500 * time.Millisecond
-
-	// liveBackoffFactor is the exponential backoff multiplier.
-	liveBackoffFactor = 2
-
-	// liveConcurrency caps how many live resolutions may be in
-	// flight across the whole package at once.
-	liveConcurrency = 6
-
-	// minNameservers is the smallest nameserver count a
-	// well-run zone is expected to publish.
-	minNameservers = 2
-)
-
-// liveGate bounds concurrent live resolutions package-wide. It has
-// to be package scoped: the whole point is that it is shared by
-// every parallel test in the package.
-//
-//nolint:gochecknoglobals // package-wide live query rate limit
-var liveGate = make(chan struct{}, liveConcurrency)
-
-var (
-	// errLiveNoAnswer reports that a live operation produced no
-	// usable answer, which is retried rather than asserted on.
-	errLiveNoAnswer = errors.New("no answer from live DNS")
-
-	// errLiveNoQuorum reports that too few of a domain's
-	// nameservers answered for a quorum assertion to be made.
-	errLiveNoQuorum = errors.New("no nameserver quorum")
-)
-
-// runLive executes one attempt of a live operation, holding a slot
-// in liveGate for its duration and bounding it with its own
-// timeout.
-func runLive(op func(ctx context.Context) error) error {
-	liveGate <- struct{}{}
-	defer func() { <-liveGate }()
-
-	ctx, cancel := context.WithTimeout(
-		context.Background(), liveAttemptTimeout,
-	)
-	defer cancel()
-
-	return op(ctx)
-}
-
-// retryLive runs op until it reports success, retrying transport
-// failures with exponential backoff, and fails the test if every
-// attempt fails. op returns an error only for a failure to obtain
-// an answer — never for an answer the test disagrees with, which
-// belongs in an assertion so that it fails immediately. op stores
-// whatever it obtained where its caller can find it.
-func retryLive(
-	t *testing.T,
-	what string,
-	op func(ctx context.Context) error,
-) {
-	t.Helper()
-
-	var last error
-
-	backoff := liveBackoffBase
-
-	for attempt := range liveAttempts {
-		if attempt > 0 {
-			t.Logf(
-				"%s: attempt %d of %d failed (%v), "+
-					"retrying in %s",
-				what, attempt, liveAttempts, last, backoff,
-			)
-			time.Sleep(backoff)
-
-			backoff *= liveBackoffFactor
-		}
-
-		last = runLive(op)
-		if last == nil {
-			return
-		}
-	}
-
-	t.Fatalf(
-		"%s: no answer after %d live attempts: %v",
-		what, liveAttempts, last,
-	)
-}
 
 // liveQuorum is how many of total nameservers must agree for a
 // multi-nameserver assertion to hold: a strict majority.
@@ -272,7 +162,7 @@ func liveFindAuthoritative(
 
 	var out []string
 
-	retryLive(
+	livednstest.Retry(
 		t,
 		"FindAuthoritativeNameservers("+domain+")",
 		func(ctx context.Context) error {
@@ -284,7 +174,7 @@ func liveFindAuthoritative(
 			if len(ns) == 0 {
 				return fmt.Errorf(
 					"%w: %s has no nameservers",
-					errLiveNoAnswer, domain,
+					livednstest.ErrNoAnswer, domain,
 				)
 			}
 
@@ -308,7 +198,7 @@ func liveLookupNS(
 
 	var out []string
 
-	retryLive(
+	livednstest.Retry(
 		t,
 		"LookupNS("+domain+")",
 		func(ctx context.Context) error {
@@ -320,7 +210,7 @@ func liveLookupNS(
 			if len(ns) == 0 {
 				return fmt.Errorf(
 					"%w: %s has no nameservers",
-					errLiveNoAnswer, domain,
+					livednstest.ErrNoAnswer, domain,
 				)
 			}
 
@@ -350,7 +240,7 @@ func liveQueryNameserver(
 
 	var out *resolver.NameserverResponse
 
-	retryLive(
+	livednstest.Retry(
 		t,
 		what,
 		func(ctx context.Context) error {
@@ -365,7 +255,7 @@ func liveQueryNameserver(
 				resp.Status == resolver.StatusError {
 				return fmt.Errorf(
 					"%w: %s returned %s: %s",
-					errLiveNoAnswer, nameserver,
+					livednstest.ErrNoAnswer, nameserver,
 					resp.Status, resp.Error,
 				)
 			}
@@ -392,7 +282,7 @@ func liveQueryAllNameservers(
 
 	var out map[string]*resolver.NameserverResponse
 
-	retryLive(
+	livednstest.Retry(
 		t,
 		"QueryAllNameservers("+hostname+")",
 		func(ctx context.Context) error {
@@ -404,7 +294,7 @@ func liveQueryAllNameservers(
 			if len(results) == 0 {
 				return fmt.Errorf(
 					"%w: no nameservers queried for %s",
-					errLiveNoAnswer, hostname,
+					livednstest.ErrNoAnswer, hostname,
 				)
 			}
 
@@ -437,7 +327,7 @@ func liveResolveIPs(
 
 	var out []string
 
-	retryLive(
+	livednstest.Retry(
 		t,
 		"ResolveIPAddresses("+hostname+")",
 		func(ctx context.Context) error {
@@ -449,7 +339,7 @@ func liveResolveIPs(
 			if len(ips) == 0 {
 				return fmt.Errorf(
 					"%w: no addresses for %s",
-					errLiveNoAnswer, hostname,
+					livednstest.ErrNoAnswer, hostname,
 				)
 			}
 
@@ -476,7 +366,7 @@ func liveResolveIPsAllowingEmpty(
 
 	var out []string
 
-	retryLive(
+	livednstest.Retry(
 		t,
 		"ResolveIPAddresses("+hostname+")",
 		func(ctx context.Context) error {
@@ -493,3 +383,37 @@ func liveResolveIPsAllowingEmpty(
 
 	return out
 }
+
+// liveResolveNSIPs looks up the addresses of the nameservers named
+// names, retrying until there are at least atLeast of them: a name
+// whose lookup got no reply is left out of the result, not an error.
+func liveResolveNSIPs(
+	t *testing.T,
+	r *resolver.Resolver,
+	names []string,
+	atLeast int,
+) []string {
+	t.Helper()
+
+	var out []string
+
+	livednstest.Retry(
+		t,
+		"ResolveNSIPs("+strings.Join(names, ", ")+")",
+		func(ctx context.Context) error {
+			ips := r.ResolveNSIPs(ctx, names)
+			if len(ips) < atLeast {
+				return fmt.Errorf(
+					"%w: %d addresses, expected at least %d",
+					livednstest.ErrNoAnswer, len(ips), atLeast,
+				)
+			}
+
+			out = ips
+
+			return nil
+		},
+	)
+
+	return out
+}

internal/resolver/resolver.go

@@ -67,17 +67,4 @@ func NewFromLogger(log *slog.Logger) *Resolver {
 	}
 }
 
-// NewFromLoggerWithClient creates a Resolver with a custom DNS
-// client, useful for testing with mock DNS responses.
-func NewFromLoggerWithClient(
-	log *slog.Logger,
-	client DNSClient,
-) *Resolver {
-	return &Resolver{
-		log:    log,
-		client: client,
-		tcp:    client,
-	}
-}
-
 // Method implementations are in iterative.go.

internal/resolver/resolver_test.go

@@ -2,6 +2,7 @@ package resolver_test
 
 import (
 	"context"
+	"fmt"
 	"log/slog"
 	"net"
 	"os"
@@ -10,10 +11,10 @@ import (
 	"testing"
 	"time"
 
-	"github.com/miekg/dns"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 
+	"sneak.berlin/go/dnswatcher/internal/livednstest"
 	"sneak.berlin/go/dnswatcher/internal/resolver"
 )
 
@@ -33,8 +34,8 @@ func newTestResolver(t *testing.T) *resolver.Resolver {
 }
 
 // findOneNSForDomain picks one authoritative nameserver to aim a
-// test at. Live-DNS retry, concurrency and quorum handling live in
+// test at. Quorum handling lives in livedns_test.go, and the live-DNS
-// livedns_test.go.
+// retry and concurrency limit in package livednstest.
 func findOneNSForDomain(
 	t *testing.T,
 	r *resolver.Resolver,
@@ -78,9 +79,10 @@ func TestFindAuthoritativeNameservers_Subdomain(
 	t.Parallel()
 
 	r := newTestResolver(t)
-	nameservers := liveFindAuthoritative(t, r, "www.google.com")
+	fromHost := liveFindAuthoritative(t, r, "www.google.com")
+	fromZone := liveFindAuthoritative(t, r, "google.com")
 
-	assert.NotEmpty(t, nameservers)
+	assert.Equal(t, fromZone, fromHost)
 }
 
 func TestFindAuthoritativeNameservers_ReturnsSorted(
@@ -137,6 +139,28 @@ func TestFindAuthoritativeNameservers_CloudflareDomain(
 	}
 }
 
+// TestResolveNSIPs_EveryNameserver looks up the addresses of two of
+// google.com's nameservers together, as the walk does when a referral
+// names a zone's nameservers without their addresses, and compares them
+// with each looked up alone. Together they must give the addresses of
+// both, not only of the first that resolves, so that when one gives no
+// usable reply the walk goes on to the other.
+func TestResolveNSIPs_EveryNameserver(t *testing.T) {
+	t.Parallel()
+
+	r := newTestResolver(t)
+	names := []string{"ns3.google.com.", "ns4.google.com."}
+	want := make([]string, 0, len(names))
+
+	for _, name := range names {
+		want = append(want, liveResolveNSIPs(t, r, []string{name}, 1)...)
+	}
+
+	got := liveResolveNSIPs(t, r, names, len(want))
+
+	assert.ElementsMatch(t, want, got)
+}
+
 // ----------------------------------------------------------------
 // QueryNameserver tests
 // ----------------------------------------------------------------
@@ -232,6 +256,45 @@ func TestQueryNameserver_NXDomain(t *testing.T) {
 	assert.Equal(t, resolver.StatusNXDomain, resp.Status)
 }
 
+// TestQueryNameserver_Refused asks a google.com nameserver about
+// cloudflare.com, a zone it does not serve, which it refuses. Refusing
+// is a failure to answer, not an answer with no records.
+func TestQueryNameserver_Refused(t *testing.T) {
+	t.Parallel()
+
+	r := newTestResolver(t)
+	ns := findOneNSForDomain(t, r, "google.com")
+
+	var resp *resolver.NameserverResponse
+
+	livednstest.Retry(
+		t,
+		"QueryNameserver("+ns+", cloudflare.com)",
+		func(ctx context.Context) error {
+			var err error
+
+			resp, err = r.QueryNameserver(ctx, ns, "cloudflare.com")
+			if err != nil {
+				return err
+			}
+
+			// A timeout or a network error is no reply at all.
+			if resp.Status == resolver.StatusTimeout ||
+				strings.HasPrefix(resp.Error, "network error") {
+				return fmt.Errorf(
+					"%w: %s: %s",
+					livednstest.ErrNoAnswer, ns, resp.Error,
+				)
+			}
+
+			return nil
+		},
+	)
+
+	assert.Equal(t, resolver.StatusError, resp.Status)
+	assert.Equal(t, "server returned REFUSED", resp.Error)
+}
+
 func TestQueryNameserver_RecordsSorted(t *testing.T) {
 	t.Parallel()
 
@@ -310,12 +373,29 @@ func TestQueryAllNameservers_ReturnsAllNS(t *testing.T) {
 func TestQueryAllNameservers_AllReturnOK(t *testing.T) {
 	t.Parallel()
 
-	r := newTestResolver(t)
+	// The last two names are in zones other than their last two
-	results := liveQueryAllNameservers(t, r, "google.com")
+	// labels: google.co.uk, under the two-label suffix co.uk, and
+	// compute-1.amazonaws.com, which amazonaws.com delegates to other
+	// servers and which has a host name for each of its addresses.
+	// Servers above a name's zone only refer onward, which gives
+	// nodata, so ok shows the name was asked at its own zone's
+	// servers.
+	hostnames := []string{
+		"google.com",
+		"www.google.co.uk",
+		"ec2-3-80-0-1.compute-1.amazonaws.com",
+	}
 
-	// A quorum, not unanimity: one authoritative server being
+	for _, hostname := range hostnames {
-	// slow or rate-limiting us is a property of the live
+		t.Run(hostname, func(t *testing.T) {
-	// internet, not a resolver defect.
+			t.Parallel()
+
+			r := newTestResolver(t)
+			results := liveQueryAllNameservers(t, r, hostname)
+
+			// A quorum, not unanimity: one authoritative server
+			// being slow or rate-limiting us is a property of the
+			// live internet, not a resolver defect.
 			assert.GreaterOrEqual(
 				t,
 				countStatus(results, resolver.StatusOK),
@@ -324,12 +404,13 @@ func TestQueryAllNameservers_AllReturnOK(t *testing.T) {
 				describeStatuses(results),
 			)
 
-	// Quorum tolerates SILENCE only. Every individual result must
+			// Quorum tolerates SILENCE only. Every individual
-	// be either the expected answer or a non-answer: ok, timeout
+			// result must be either the expected answer or a
-	// or error, and nothing else. Stated as a closed allowlist so
+			// non-answer: ok, timeout or error, and nothing else.
-	// that a wrong answer no one thought to ban — nxdomain and
+			// Stated as a closed allowlist so that a wrong answer
-	// nodata today, any status added later — fails here rather
+			// no one thought to ban — nxdomain and nodata today,
-	// than sliding through under the quorum.
+			// any status added later — fails here rather than
+			// sliding through under the quorum.
 			assert.Empty(
 				t,
 				unsanctionedStatuses(
@@ -338,9 +419,12 @@ func TestQueryAllNameservers_AllReturnOK(t *testing.T) {
 					resolver.StatusTimeout,
 					resolver.StatusError,
 				),
-		"every nameserver must answer OK or not answer at all: %s",
+				"every nameserver must answer OK or not answer "+
+					"at all: %s",
 				describeStatuses(results),
 			)
+		})
+	}
 }
 
 func TestQueryAllNameservers_NXDomainFromAllNS(
@@ -519,6 +603,29 @@ func TestQueryAllNameservers_ContextCanceled(t *testing.T) {
 	assert.Error(t, err)
 }
 
+// TestQueryEachNS_CanceledDuringQuery cancels the context while a
+// nameserver is being queried, as shutdown does. A lookup cut short
+// says nothing about the nameserver, so it must return an error, not a
+// failed response for it.
+func TestQueryEachNS_CanceledDuringQuery(t *testing.T) {
+	t.Parallel()
+
+	r := newTestResolver(t)
+	ctx, cancel := context.WithCancel(context.Background())
+	t.Cleanup(cancel)
+
+	// Finding the nameserver's address alone starts at the root
+	// servers and takes several round trips, so a cancel a few
+	// milliseconds in lands during the query.
+	time.AfterFunc(5*time.Millisecond, cancel)
+
+	results, err := r.QueryEachNS(
+		ctx, []string{"ns1.google.com."}, "google.com",
+	)
+	require.ErrorIs(t, err, resolver.ErrContextCanceled)
+	assert.Nil(t, results)
+}
+
 // ----------------------------------------------------------------
 // Timeout tests
 // ----------------------------------------------------------------
@@ -526,21 +633,18 @@ func TestQueryAllNameservers_ContextCanceled(t *testing.T) {
 func TestQueryNameserverIP_Timeout(t *testing.T) {
 	t.Parallel()
 
-	log := slog.New(slog.NewTextHandler(
+	r := newTestResolver(t)
-		os.Stderr,
-		&slog.HandlerOptions{Level: slog.LevelDebug},
-	))
-
-	r := resolver.NewFromLoggerWithClient(
-		log, &timeoutClient{},
-	)
 
+	// Nothing answers at 192.0.2.1, a documentation address. The
+	// resolver tries each query twice, and the first try gives up
+	// after two seconds. A deadline that ends during the first try
+	// makes the status vary from run to run between error and
+	// timeout, so the deadline must outlast the first try.
 	ctx, cancel := context.WithTimeout(
-		context.Background(), 10*time.Second,
+		context.Background(), 3*time.Second,
 	)
 	t.Cleanup(cancel)
 
-	// Query any IP — the client always returns a timeout error.
 	resp, err := r.QueryNameserverIP(
 		ctx, "unreachable.test.", "192.0.2.1",
 		"example.com",
@@ -551,26 +655,81 @@ func TestQueryNameserverIP_Timeout(t *testing.T) {
 	assert.NotEmpty(t, resp.Error)
 }
 
-// timeoutClient simulates DNS timeout errors for testing.
+// TestCollectIPs_NoNameserverAnswered takes the response of a
-type timeoutClient struct{}
+// nameserver at 192.0.2.1, where nothing answers, as
+// TestQueryNameserverIP_Timeout does. Addresses collected from
+// nameservers that all failed to answer are an error, not none.
+func TestCollectIPs_NoNameserverAnswered(t *testing.T) {
+	t.Parallel()
 
-func (c *timeoutClient) ExchangeContext(
+	r := newTestResolver(t)
-	_ context.Context,
+
-	_ *dns.Msg,
+	// The deadline outlasts the first try, as in
-	_ string,
+	// TestQueryNameserverIP_Timeout.
-) (*dns.Msg, time.Duration, error) {
+	ctx, cancel := context.WithTimeout(
-	return nil, 0, &net.OpError{
+		context.Background(), 3*time.Second,
-		Op:  "read",
+	)
-		Net: "udp",
+	t.Cleanup(cancel)
-		Err: &timeoutError{},
+
-	}
+	resp, err := r.QueryNameserverIP(
+		ctx, "unreachable.test.", "192.0.2.1",
+		"example.com",
+	)
+	require.NoError(t, err)
+
+	ips, _, err := resolver.CollectIPs(
+		map[string]*resolver.NameserverResponse{resp.Nameserver: resp},
+	)
+	require.ErrorIs(t, err, resolver.ErrNoNameserverAnswered)
+	assert.Empty(t, ips)
 }
 
-type timeoutError struct{}
+// TestCollectIPs_ReferralIsNoAnswer asks a root server about
+// example.com, which the root zone does not hold, so it only refers the
+// query to the com servers. That reply is no answer, as is a parent
+// zone's when every server of the name's own zone failed.
+func TestCollectIPs_ReferralIsNoAnswer(t *testing.T) {
+	t.Parallel()
 
-func (e *timeoutError) Error() string   { return "i/o timeout" }
+	r := newTestResolver(t)
-func (e *timeoutError) Timeout() bool   { return true }
+
-func (e *timeoutError) Temporary() bool { return true }
+	var resp *resolver.NameserverResponse
+
+	livednstest.Retry(
+		t,
+		"QueryNameserverIP(a.root-servers.net, example.com)",
+		func(ctx context.Context) error {
+			var err error
+
+			resp, err = r.QueryNameserverIP(
+				ctx, "a.root-servers.net.", "198.41.0.4",
+				"example.com",
+			)
+			if err != nil {
+				return err
+			}
+
+			// A timeout or a network error is no reply at all.
+			if resp.Status == resolver.StatusTimeout ||
+				strings.HasPrefix(resp.Error, "network error") {
+				return fmt.Errorf(
+					"%w: %s", livednstest.ErrNoAnswer, resp.Error,
+				)
+			}
+
+			return nil
+		},
+	)
+
+	assert.Equal(t, resolver.StatusError, resp.Status)
+	assert.Equal(t, "server returned a referral", resp.Error)
+
+	ips, _, err := resolver.CollectIPs(
+		map[string]*resolver.NameserverResponse{resp.Nameserver: resp},
+	)
+	require.ErrorIs(t, err, resolver.ErrNoNameserverAnswered)
+	assert.Empty(t, ips)
+}
 
 func TestResolveIPAddresses_ContextCanceled(t *testing.T) {
 	t.Parallel()

internal/server/export_test.go

@@ -0,0 +1,38 @@
+package server
+
+import (
+	"net/http"
+	"time"
+
+	"github.com/go-chi/chi/v5"
+)
+
+// RequestTimeout exports the handler execution budget applied by
+// chimw.Timeout in SetupRoutes, so tests can assert the relationship
+// between it and the server's WriteTimeout.
+const RequestTimeout time.Duration = requestTimeout
+
+// SetListenPort overrides the port Run binds. A test uses it to hand
+// Run an unbindable port so ListenAndServe fails immediately and Run
+// returns after storing its http.Server.
+func SetListenPort(s *Server, port int) {
+	s.port = port
+}
+
+// HTTPServerOf returns the http.Server that Run built and stored, so a
+// test can inspect the timeouts the running server actually carries.
+func HTTPServerOf(s *Server) *http.Server {
+	return s.httpServer
+}
+
+// EnableSentry runs the Sentry setup that the start hook runs, without
+// starting the HTTP server.
+func EnableSentry(s *Server) error {
+	return s.enableSentry()
+}
+
+// RouterOf returns the router SetupRoutes built, so a test can add a
+// route that panics.
+func RouterOf(s *Server) *chi.Mux {
+	return s.router
+}

internal/server/routes.go

@@ -4,6 +4,7 @@ import (
 	"net/http"
 	"time"
 
+	sentryhttp "github.com/getsentry/sentry-go/http"
 	"github.com/go-chi/chi/v5"
 	chimw "github.com/go-chi/chi/v5/middleware"
 	"github.com/prometheus/client_golang/prometheus/promhttp"
@@ -21,15 +22,32 @@ func (s *Server) SetupRoutes() {
 	// Global middleware
 	s.router.Use(chimw.Recoverer)
 	s.router.Use(chimw.RequestID)
+	s.router.Use(s.mw.SecurityHeaders())
 	s.router.Use(s.mw.Logging())
-	s.router.Use(s.mw.CORS())
 	s.router.Use(chimw.Timeout(requestTimeout))
 
+	// Report panics in handlers to Sentry when DNSWATCHER_SENTRY_DSN is
+	// set. Repanic passes each panic on to chimw.Recoverer above, which
+	// still answers the request.
+	if s.sentryEnabled {
+		sentryHandler := sentryhttp.New(sentryhttp.Options{
+			Repanic: true,
+		})
+		s.router.Use(sentryHandler.Handle)
+	}
+
+	// Public, unauthenticated, read-only routes, the only ones
+	// REPO_POLICIES.md allows wildcard CORS on. CORS is middleware of
+	// this whole router, not of a Group, so that it also answers
+	// OPTIONS preflight requests, which no route here registers.
+	public := chi.NewRouter()
+	public.Use(s.mw.CORS())
+
 	// Dashboard (read-only web UI)
-	s.router.Get("/", s.handlers.HandleDashboard())
+	public.Get("/", s.handlers.HandleDashboard())
 
 	// Static assets (embedded CSS/JS)
-	s.router.Mount(
+	public.Mount(
 		"/s",
 		http.StripPrefix(
 			"/s",
@@ -38,27 +56,33 @@ func (s *Server) SetupRoutes() {
 	)
 
 	// Health check (standard well-known path)
-	s.router.Get(
+	public.Get(
 		"/.well-known/healthcheck",
 		s.handlers.HandleHealthCheck(),
 	)
 
 	// Legacy health check (keep for backward compatibility)
-	s.router.Get("/health", s.handlers.HandleHealthCheck())
+	public.Get("/health", s.handlers.HandleHealthCheck())
 
 	// API v1 routes
-	s.router.Route("/api/v1", func(r chi.Router) {
+	public.Route("/api/v1", func(r chi.Router) {
 		r.Get("/status", s.handlers.HandleStatus())
 	})
 
-	// Metrics endpoint (optional, with basic auth)
+	s.router.Mount("/", public)
+
+	// Metrics endpoint (optional, with basic auth) and no CORS: a
+	// Prometheus scraper is not a browser. It is mounted rather than
+	// added with Get so that every method on /metrics, OPTIONS
+	// included, ends here instead of falling through to the public
+	// router and its CORS. The rate limit comes before Basic Auth, so
+	// failed logins count against it and a request over the limit
+	// never reaches the password check.
 	if s.params.Config.MetricsUsername != "" {
-		s.router.Group(func(r chi.Router) {
+		metrics := chi.NewRouter()
-			r.Use(s.mw.MetricsAuth())
+		metrics.Use(s.mw.MetricsRateLimit())
-			r.Get(
+		metrics.Use(s.mw.MetricsAuth())
-				"/metrics",
+		metrics.Get("/", promhttp.Handler().ServeHTTP)
-				promhttp.Handler().ServeHTTP,
+		s.router.Mount("/metrics", metrics)
-			)
-		})
 	}
 }

internal/server/routes_test.go

@@ -0,0 +1,290 @@
+package server_test
+
+import (
+	"net/http"
+	"net/http/httptest"
+	"testing"
+
+	"github.com/spf13/viper"
+
+	"sneak.berlin/go/dnswatcher/internal/server"
+)
+
+// Credentials for /metrics, which is only routed when a username is set.
+const (
+	metricsUsername = "scraper"
+	metricsPassword = "scrape-secret"
+)
+
+// The tests below set env vars and touch viper global state, so like
+// the config tests they cannot use t.Parallel.
+
+// routedServer builds the server with its routes set up, ready to serve
+// test requests. The caller must first configure viper.
+func routedServer(t *testing.T) *server.Server {
+	t.Helper()
+
+	srv := buildServer(t)
+	srv.SetupRoutes()
+
+	return srv
+}
+
+// crossOriginRequest builds a request as a browser sends it from a page
+// on another site.
+func crossOriginRequest(
+	t *testing.T,
+	method string,
+	target string,
+) *http.Request {
+	t.Helper()
+
+	req := httptest.NewRequestWithContext(t.Context(), method, target, nil)
+	req.Header.Set("Origin", "https://example.net")
+
+	return req
+}
+
+// preflightRequest builds the OPTIONS request a browser sends before a
+// cross-origin request with the given method and request headers.
+func preflightRequest(
+	t *testing.T,
+	target string,
+	method string,
+	headers string,
+) *http.Request {
+	t.Helper()
+
+	req := crossOriginRequest(t, http.MethodOptions, target)
+	req.Header.Set("Access-Control-Request-Method", method)
+
+	if headers != "" {
+		req.Header.Set("Access-Control-Request-Headers", headers)
+	}
+
+	return req
+}
+
+func serve(
+	srv *server.Server,
+	req *http.Request,
+) *httptest.ResponseRecorder {
+	rec := httptest.NewRecorder()
+	srv.ServeHTTP(rec, req)
+
+	return rec
+}
+
+// publicPaths returns one path on each public route.
+func publicPaths() []string {
+	return []string{
+		"/",
+		"/s/css/tailwind.min.css",
+		"/api/v1/status",
+		"/health",
+		"/.well-known/healthcheck",
+	}
+}
+
+// TestPublicRoutesAllowAnyOrigin checks that every public route answers
+// a cross-origin GET with the CORS wildcard.
+func TestPublicRoutesAllowAnyOrigin(t *testing.T) {
+	viper.Reset()
+	t.Setenv("DNSWATCHER_TARGETS", "example.com")
+	t.Setenv("DNSWATCHER_METRICS_USERNAME", metricsUsername)
+	t.Setenv("DNSWATCHER_METRICS_PASSWORD", metricsPassword)
+
+	srv := routedServer(t)
+
+	for _, path := range publicPaths() {
+		rec := serve(srv, crossOriginRequest(t, http.MethodGet, path))
+
+		if rec.Code != http.StatusOK {
+			t.Errorf("GET %s: status = %d, want 200", path, rec.Code)
+		}
+
+		got := rec.Header().Get("Access-Control-Allow-Origin")
+		if got != "*" {
+			t.Errorf(
+				"GET %s: Access-Control-Allow-Origin = %q, want %q",
+				path, got, "*",
+			)
+		}
+	}
+}
+
+// TestMetricsHasNoCORS checks that no request to the Basic-Auth
+// protected /metrics, preflight included, gets a CORS header.
+func TestMetricsHasNoCORS(t *testing.T) {
+	viper.Reset()
+	t.Setenv("DNSWATCHER_TARGETS", "example.com")
+	t.Setenv("DNSWATCHER_METRICS_USERNAME", metricsUsername)
+	t.Setenv("DNSWATCHER_METRICS_PASSWORD", metricsPassword)
+
+	srv := routedServer(t)
+
+	authenticated := crossOriginRequest(t, http.MethodGet, "/metrics")
+	authenticated.SetBasicAuth(metricsUsername, metricsPassword)
+
+	tests := []struct {
+		name       string
+		req        *http.Request
+		wantStatus int
+	}{
+		{
+			"authenticated GET",
+			authenticated,
+			http.StatusOK,
+		},
+		{
+			"unauthenticated GET",
+			crossOriginRequest(t, http.MethodGet, "/metrics"),
+			http.StatusUnauthorized,
+		},
+		{
+			"preflight",
+			preflightRequest(t, "/metrics", http.MethodGet, ""),
+			http.StatusUnauthorized,
+		},
+	}
+
+	for _, tt := range tests {
+		rec := serve(srv, tt.req)
+
+		if rec.Code != tt.wantStatus {
+			t.Errorf(
+				"%s: status = %d, want %d",
+				tt.name, rec.Code, tt.wantStatus,
+			)
+		}
+
+		got := rec.Header().Get("Access-Control-Allow-Origin")
+		if got != "" {
+			t.Errorf(
+				"%s: Access-Control-Allow-Origin = %q, want none",
+				tt.name, got,
+			)
+		}
+	}
+}
+
+// TestPreflightAllowsOnlyWhatPublicRoutesServe checks what each public
+// route agrees to in a CORS preflight: GET, but not POST, PUT or
+// DELETE, which no route serves, and not the Authorization or
+// X-CSRF-Token headers, which no public route reads. It checks every
+// public route because one added with Get, such as /health, answers a
+// preflight only while CORS is middleware of a whole router; in a
+// Group, chi would answer it with 405 and no CORS headers.
+func TestPreflightAllowsOnlyWhatPublicRoutesServe(t *testing.T) {
+	viper.Reset()
+	t.Setenv("DNSWATCHER_TARGETS", "example.com")
+	t.Setenv("DNSWATCHER_METRICS_USERNAME", metricsUsername)
+	t.Setenv("DNSWATCHER_METRICS_PASSWORD", metricsPassword)
+
+	srv := routedServer(t)
+
+	tests := []struct {
+		method  string
+		headers string
+		allowed bool
+	}{
+		{http.MethodGet, "", true},
+		{http.MethodGet, "Content-Type", true},
+		{http.MethodPost, "", false},
+		{http.MethodPut, "", false},
+		{http.MethodDelete, "", false},
+		{http.MethodGet, "Authorization", false},
+		{http.MethodGet, "X-CSRF-Token", false},
+	}
+
+	for _, path := range publicPaths() {
+		for _, tt := range tests {
+			rec := serve(srv, preflightRequest(
+				t, path, tt.method, tt.headers,
+			))
+
+			want := ""
+			if tt.allowed {
+				want = tt.method
+			}
+
+			got := rec.Header().Get("Access-Control-Allow-Methods")
+			if got != want {
+				t.Errorf(
+					"preflight to %s for %s with headers %q: "+
+						"Access-Control-Allow-Methods = %q, want %q",
+					path, tt.method, tt.headers, got, want,
+				)
+			}
+		}
+	}
+}
+
+// metricsRequest builds a GET for /metrics from remoteAddr that logs
+// in with the given password.
+func metricsRequest(
+	t *testing.T,
+	remoteAddr string,
+	password string,
+) *http.Request {
+	t.Helper()
+
+	req := httptest.NewRequestWithContext(
+		t.Context(), http.MethodGet, "/metrics", nil,
+	)
+	req.RemoteAddr = remoteAddr
+	req.SetBasicAuth(metricsUsername, password)
+
+	return req
+}
+
+// TestMetricsRateLimitComesBeforeAuth checks that failed logins to
+// /metrics count against the rate limit; that once an address is over
+// it, even the right password gets 429, with the same body as a wrong
+// one; and that another address still gets in.
+func TestMetricsRateLimitComesBeforeAuth(t *testing.T) {
+	viper.Reset()
+	t.Setenv("DNSWATCHER_TARGETS", "example.com")
+	t.Setenv("DNSWATCHER_METRICS_USERNAME", metricsUsername)
+	t.Setenv("DNSWATCHER_METRICS_PASSWORD", metricsPassword)
+
+	const (
+		guesser = "198.51.100.1:4000"
+		other   = "198.51.100.2:4000"
+
+		// Far more guesses than the rate limit allows.
+		maxGuesses = 1000
+	)
+
+	srv := routedServer(t)
+
+	var guess *httptest.ResponseRecorder
+
+	for range maxGuesses {
+		guess = serve(srv, metricsRequest(t, guesser, "wrong"))
+		if guess.Code != http.StatusUnauthorized {
+			break
+		}
+	}
+
+	if guess.Code != http.StatusTooManyRequests {
+		t.Fatalf("wrong password: status = %d, want 429", guess.Code)
+	}
+
+	right := serve(srv, metricsRequest(t, guesser, metricsPassword))
+	if right.Code != http.StatusTooManyRequests {
+		t.Errorf("right password: status = %d, want 429", right.Code)
+	}
+
+	if right.Body.String() != guess.Body.String() {
+		t.Errorf(
+			"429 body with right password = %q, with wrong one = %q",
+			right.Body.String(), guess.Body.String(),
+		)
+	}
+
+	rec := serve(srv, metricsRequest(t, other, metricsPassword))
+	if rec.Code != http.StatusOK {
+		t.Errorf("another address: status = %d, want 200", rec.Code)
+	}
+}

internal/server/sentry_test.go

@@ -0,0 +1,190 @@
+package server_test
+
+import (
+	"io"
+	"net/http"
+	"net/http/httptest"
+	"net/url"
+	"strings"
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/getsentry/sentry-go"
+	"github.com/spf13/viper"
+	"go.uber.org/fx"
+
+	"sneak.berlin/go/dnswatcher/internal/server"
+)
+
+// The tests below set env vars and touch the global state of viper and
+// of Sentry, so they cannot use t.Parallel.
+
+// standInDelay is how long the Sentry stand-in takes to answer. It
+// records a report only then, so a report that Shutdown did not wait
+// for has not been recorded yet when Shutdown returns.
+const standInDelay = 100 * time.Millisecond
+
+// sentryStandIn is a local HTTP server in place of Sentry's, so that
+// nothing a test reports leaves the host. It keeps the body of every
+// request it receives.
+type sentryStandIn struct {
+	server *httptest.Server
+	mu     sync.Mutex
+	bodies []string
+}
+
+func newSentryStandIn(t *testing.T) *sentryStandIn {
+	t.Helper()
+
+	standIn := &sentryStandIn{}
+	standIn.server = httptest.NewServer(http.HandlerFunc(
+		func(_ http.ResponseWriter, r *http.Request) {
+			body, err := io.ReadAll(r.Body)
+			if err != nil {
+				t.Errorf("reading request to the Sentry stand-in: %v", err)
+			}
+
+			time.Sleep(standInDelay)
+
+			standIn.mu.Lock()
+			defer standIn.mu.Unlock()
+
+			standIn.bodies = append(standIn.bodies, string(body))
+		},
+	))
+	t.Cleanup(standIn.server.Close)
+
+	return standIn
+}
+
+// dsn returns a DSN that points Sentry at the stand-in.
+func (s *sentryStandIn) dsn(t *testing.T) string {
+	t.Helper()
+
+	dsn, err := url.Parse(s.server.URL)
+	if err != nil {
+		t.Fatalf("parsing the stand-in URL: %v", err)
+	}
+
+	dsn.User = url.User("public-key")
+	dsn.Path = "/1"
+
+	return dsn.String()
+}
+
+// received reports whether a request to the stand-in contained text.
+func (s *sentryStandIn) received(text string) bool {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+
+	for _, body := range s.bodies {
+		if strings.Contains(body, text) {
+			return true
+		}
+	}
+
+	return false
+}
+
+func TestSentryUnsetDoesNothing(t *testing.T) {
+	viper.Reset()
+	t.Setenv("DNSWATCHER_TARGETS", "example.com")
+	t.Setenv("DNSWATCHER_SENTRY_DSN", "")
+
+	srv := buildServer(t)
+
+	err := server.EnableSentry(srv)
+	if err != nil {
+		t.Fatalf("Sentry setup with no DSN: %v", err)
+	}
+
+	if sentry.CurrentHub().Client() != nil {
+		t.Error("Sentry was set up with no DSN configured")
+	}
+}
+
+// TestSentryReportsHandlerPanic checks that with a valid DSN a panic in
+// a handler is reported to Sentry, still reaches chimw.Recoverer, and
+// has been sent by the time Shutdown returns, and that nothing else is
+// sent to Sentry.
+func TestSentryReportsHandlerPanic(t *testing.T) {
+	standIn := newSentryStandIn(t)
+
+	viper.Reset()
+	t.Setenv("DNSWATCHER_TARGETS", "example.com")
+	t.Setenv("DNSWATCHER_SENTRY_DSN", standIn.dsn(t))
+
+	srv := buildServer(t)
+
+	err := server.EnableSentry(srv)
+	if err != nil {
+		t.Fatalf("Sentry setup with a valid DSN: %v", err)
+	}
+
+	// Sentry's client is global: close it so later tests find none.
+	t.Cleanup(func() {
+		sentry.CurrentHub().Client().Close()
+		sentry.CurrentHub().BindClient(nil)
+	})
+
+	const panicMessage = "handler panic in the Sentry test"
+
+	srv.SetupRoutes()
+	server.RouterOf(srv).Get(
+		"/panic",
+		func(http.ResponseWriter, *http.Request) {
+			panic(panicMessage)
+		},
+	)
+
+	// An ordinary request first: with client reports on, Sentry would
+	// add a count of its dropped transaction to the panic report.
+	serve(srv, httptest.NewRequestWithContext(
+		t.Context(), http.MethodGet, "/.well-known/healthcheck", nil,
+	))
+
+	rec := serve(srv, httptest.NewRequestWithContext(
+		t.Context(), http.MethodGet, "/panic", nil,
+	))
+	if rec.Code != http.StatusInternalServerError {
+		t.Errorf(
+			"status %d, want %d from the recoverer",
+			rec.Code, http.StatusInternalServerError,
+		)
+	}
+
+	err = srv.Shutdown(t.Context())
+	if err != nil {
+		t.Fatalf("Shutdown: %v", err)
+	}
+
+	if !standIn.received(panicMessage) {
+		t.Error("the panic had not been sent to Sentry when Shutdown returned")
+	}
+
+	if standIn.received("client_report") {
+		t.Error("Sentry was sent a client report, not only the panic")
+	}
+}
+
+func TestSentryInvalidDSNStopsStartup(t *testing.T) {
+	viper.Reset()
+	t.Setenv("DNSWATCHER_TARGETS", "example.com")
+	t.Setenv("DNSWATCHER_DATA_DIR", t.TempDir())
+	// Sentry cannot parse this: it has no public key before the host.
+	t.Setenv("DNSWATCHER_SENTRY_DSN", "https://sentry.test/1")
+
+	app := newServerApp(fx.Invoke(func(*server.Server) {}))
+
+	err := app.Start(t.Context())
+	if err == nil {
+		_ = app.Stop(t.Context())
+
+		t.Fatal("startup succeeded with an invalid DSN")
+	}
+
+	if !strings.Contains(err.Error(), "invalid DNSWATCHER_SENTRY_DSN") {
+		t.Errorf("startup error does not name the setting: %v", err)
+	}
+}

internal/server/server.go

@@ -9,6 +9,7 @@ import (
 	"net/http"
 	"time"
 
+	"github.com/getsentry/sentry-go"
 	"github.com/go-chi/chi/v5"
 	"go.uber.org/fx"
 
@@ -33,13 +34,62 @@ type Params struct {
 // shutdownTimeout is how long to wait for graceful shutdown.
 const shutdownTimeout = 30 * time.Second
 
-// readHeaderTimeout is the max duration for reading request headers.
+// sentryFlushTimeout is how long shutdown waits for Sentry to send the
-const readHeaderTimeout = 10 * time.Second
+// error reports it still holds.
+const sentryFlushTimeout = 2 * time.Second
+
+// Socket-level timeouts for the HTTP server.
+//
+// These bound time spent on the connection itself and are a distinct
+// control from the per-request handler budget enforced by
+// chimw.Timeout(requestTimeout) in routes.go: that one cancels the
+// request context after requestTimeout but never touches the socket,
+// so without the values below a peer can hold a connection open
+// forever (slowloris, unreaped keep-alives).
+//
+// The one hard constraint between the two controls is
+// writeTimeout > requestTimeout. net/http arms the write deadline
+// once the request headers have been read, so on a plaintext
+// connection it covers handler execution AND the response flush. If
+// writeTimeout were <= requestTimeout the server would sever the
+// connection before a handler that legitimately consumed its full
+// budget could emit anything, making the 60s budget unreachable in
+// practice. The margin between them is the response-flush allowance.
+//
+// The only clients of this service are browsers loading the dashboard
+// and a Prometheus scraper; the values are sized for those.
+const (
+	// readHeaderTimeout is the max duration for reading request
+	// headers.
+	readHeaderTimeout = 10 * time.Second
+
+	// readTimeout bounds reading the entire request, headers plus
+	// body. Every route here is a GET with no body, so this only
+	// ever needs to cover headers; the extra 5s over
+	// readHeaderTimeout is slack, not a real allowance, and keeps a
+	// body dribbled one byte at a time from holding the read side
+	// open indefinitely.
+	readTimeout = 15 * time.Second
+
+	// writeTimeout must exceed the requestTimeout handler budget
+	// (60s) per the note above. The 15s difference is the allowance
+	// for flushing a completed response to a slow client.
+	writeTimeout = 75 * time.Second
+
+	// idleTimeout reaps keep-alive connections between requests. It
+	// is deliberately longer than the common Prometheus scrape
+	// intervals (15s/30s/60s) so the scraper reuses its connection
+	// rather than reconnecting every cycle, while a browser tab
+	// left open on the dashboard stops occupying a connection
+	// within two minutes of going quiet.
+	idleTimeout = 120 * time.Second
+)
 
 // Server is the HTTP server.
 type Server struct {
 	startupTime   time.Time
 	port          int
+	sentryEnabled bool
 	log           *slog.Logger
 	router        *chi.Mux
 	httpServer    *http.Server
@@ -64,6 +114,12 @@ func New(
 	lifecycle.Append(fx.Hook{
 		OnStart: func(_ context.Context) error {
 			srv.startupTime = time.Now()
+
+			err := srv.enableSentry()
+			if err != nil {
+				return err
+			}
+
 			go srv.Run()
 
 			return nil
@@ -76,16 +132,29 @@ func New(
 	return srv, nil
 }
 
+// newHTTPServer builds the listening http.Server with every
+// socket-level timeout set. All four are set deliberately: a zero
+// value in net/http means "no limit", not "some default".
+func newHTTPServer(
+	listenAddr string,
+	handler http.Handler,
+) *http.Server {
+	return &http.Server{
+		Addr:              listenAddr,
+		Handler:           handler,
+		ReadTimeout:       readTimeout,
+		ReadHeaderTimeout: readHeaderTimeout,
+		WriteTimeout:      writeTimeout,
+		IdleTimeout:       idleTimeout,
+	}
+}
+
 // Run starts the HTTP server.
 func (s *Server) Run() {
 	s.SetupRoutes()
 
 	listenAddr := fmt.Sprintf(":%d", s.port)
-	s.httpServer = &http.Server{
+	s.httpServer = newHTTPServer(listenAddr, s)
-		Addr:              listenAddr,
-		Handler:           s,
-		ReadHeaderTimeout: readHeaderTimeout,
-	}
 
 	s.log.Info("http server starting", "addr", listenAddr)
 
@@ -95,8 +164,11 @@ func (s *Server) Run() {
 	}
 }
 
-// Shutdown gracefully shuts down the server.
+// Shutdown gracefully shuts down the server, then sends the error
+// reports Sentry still holds.
 func (s *Server) Shutdown(ctx context.Context) error {
+	defer s.flushSentry()
+
 	if s.httpServer == nil {
 		return nil
 	}
@@ -127,3 +199,45 @@ func (s *Server) ServeHTTP(
 ) {
 	s.router.ServeHTTP(writer, request)
 }
+
+// enableSentry turns on Sentry error reporting when
+// DNSWATCHER_SENTRY_DSN is set, and does nothing when it is not. A DSN
+// that Sentry cannot parse is an error, so that startup stops instead
+// of running without the error reporting the operator asked for.
+func (s *Server) enableSentry() error {
+	if s.params.Config.SentryDSN == "" {
+		return nil
+	}
+
+	err := sentry.Init(sentry.ClientOptions{
+		Dsn:     s.params.Config.SentryDSN,
+		Release: s.params.Globals.Appname + "-" + s.params.Globals.Version,
+		// Use the transport that queues each report as it is made. With
+		// the default one, Flush can return before sending a report made
+		// just before it, such as one from the last request at shutdown.
+		DisableTelemetryBuffer: true,
+		// Send panic reports only, not Sentry's counts of what it dropped,
+		// such as the transaction it starts for every request.
+		DisableClientReports: true,
+	})
+	if err != nil {
+		return fmt.Errorf("invalid DNSWATCHER_SENTRY_DSN: %w", err)
+	}
+
+	s.log.Info("sentry error reporting activated")
+	s.sentryEnabled = true
+
+	return nil
+}
+
+// flushSentry sends the error reports Sentry still holds, waiting at
+// most sentryFlushTimeout.
+func (s *Server) flushSentry() {
+	if !s.sentryEnabled {
+		return
+	}
+
+	if !sentry.Flush(sentryFlushTimeout) {
+		s.log.Warn("sentry flush timed out; some error reports were not sent")
+	}
+}

internal/server/server_test.go

@@ -0,0 +1,135 @@
+package server_test
+
+import (
+	"testing"
+
+	"github.com/spf13/viper"
+	"go.uber.org/fx"
+
+	"sneak.berlin/go/dnswatcher/internal/config"
+	"sneak.berlin/go/dnswatcher/internal/globals"
+	"sneak.berlin/go/dnswatcher/internal/handlers"
+	"sneak.berlin/go/dnswatcher/internal/healthcheck"
+	"sneak.berlin/go/dnswatcher/internal/logger"
+	"sneak.berlin/go/dnswatcher/internal/middleware"
+	"sneak.berlin/go/dnswatcher/internal/notify"
+	"sneak.berlin/go/dnswatcher/internal/server"
+	"sneak.berlin/go/dnswatcher/internal/state"
+)
+
+// newServerApp builds an fx app holding a *server.Server wired exactly
+// as cmd/dnswatcher wires it, minus the watcher/resolver subtree that
+// would touch live DNS, plus the given option. config.New reads viper,
+// so the caller must first configure it, which is also why the caller
+// cannot run in parallel.
+func newServerApp(option fx.Option) *fx.App {
+	return fx.New(
+		fx.NopLogger,
+		fx.Provide(
+			globals.New,
+			logger.New,
+			config.New,
+			state.New,
+			healthcheck.New,
+			notify.New,
+			middleware.New,
+			handlers.New,
+			server.New,
+		),
+		option,
+	)
+}
+
+// buildServer builds the server without starting the app's lifecycle,
+// so no OnStart hook runs and nothing listens or resolves.
+func buildServer(t *testing.T) *server.Server {
+	t.Helper()
+
+	var srv *server.Server
+
+	app := newServerApp(fx.Populate(&srv))
+
+	err := app.Err()
+	if err != nil {
+		t.Fatalf("building server graph: %v", err)
+	}
+
+	return srv
+}
+
+// TestRunWiresSocketTimeouts pins that the http.Server the running
+// server actually serves — the one Run builds and hands to
+// ListenAndServe — carries every socket-level timeout, plus the two
+// relationships the values must satisfy.
+//
+// Run is driven to completion with an unbindable port: it builds and
+// stores s.httpServer, then ListenAndServe fails at once and Run
+// returns without ever listening. The assertions run in the same
+// goroutine after Run returns, so reading s.httpServer is free of any
+// data race. Nothing here measures elapsed time.
+//
+// ReadTimeout must be at least ReadHeaderTimeout. net/http reads the
+// headers under ReadHeaderTimeout, then sets the read deadline for the
+// rest of the request to ReadTimeout, counted from when it started
+// reading the request. If ReadTimeout were smaller, a request whose
+// headers arrived after ReadTimeout but within ReadHeaderTimeout would
+// get a read deadline that had already passed, so reading its body
+// would fail at once.
+func TestRunWiresSocketTimeouts(t *testing.T) {
+	// Sets an env var and touches viper global state, so like the
+	// config tests it cannot use t.Parallel.
+	viper.Reset()
+	t.Setenv("DNSWATCHER_TARGETS", "example.com")
+
+	srv := buildServer(t)
+	server.SetListenPort(srv, -1)
+
+	srv.Run()
+
+	hs := server.HTTPServerOf(srv)
+	if hs == nil {
+		t.Fatal("Run did not build an http.Server")
+	}
+
+	if hs.ReadTimeout <= 0 {
+		t.Errorf("ReadTimeout must be non-zero, got %v", hs.ReadTimeout)
+	}
+
+	if hs.ReadHeaderTimeout <= 0 {
+		t.Errorf(
+			"ReadHeaderTimeout must be non-zero, got %v",
+			hs.ReadHeaderTimeout,
+		)
+	}
+
+	if hs.WriteTimeout <= 0 {
+		t.Errorf("WriteTimeout must be non-zero, got %v", hs.WriteTimeout)
+	}
+
+	if hs.IdleTimeout <= 0 {
+		t.Errorf("IdleTimeout must be non-zero, got %v", hs.IdleTimeout)
+	}
+
+	if hs.WriteTimeout <= server.RequestTimeout {
+		t.Errorf(
+			"WriteTimeout (%v) must exceed handler budget (%v)",
+			hs.WriteTimeout,
+			server.RequestTimeout,
+		)
+	}
+
+	if hs.ReadTimeout < hs.ReadHeaderTimeout {
+		t.Errorf(
+			"ReadTimeout (%v) must be >= ReadHeaderTimeout (%v)",
+			hs.ReadTimeout,
+			hs.ReadHeaderTimeout,
+		)
+	}
+
+	if hs.Handler != srv {
+		t.Errorf(
+			"Run wired handler %T, want the *server.Server",
+			hs.Handler,
+		)
+	}
+}

internal/state/export_test.go

@@ -0,0 +1,23 @@
+package state
+
+import (
+	"log/slog"
+
+	"sneak.berlin/go/dnswatcher/internal/config"
+)
+
+// NewForTestWithDataDir creates an empty State that saves to dataDir,
+// without the fx lifecycle.
+func NewForTestWithDataDir(dataDir string) *State {
+	return &State{
+		log: slog.Default(),
+		snapshot: &Snapshot{
+			Version:      stateVersion,
+			Domains:      make(map[string]*DomainState),
+			Hostnames:    make(map[string]*HostnameState),
+			Ports:        make(map[string]*PortState),
+			Certificates: make(map[string]*CertificateState),
+		},
+		config: &config.Config{DataDir: dataDir},
+	}
+}

internal/state/state.go

@@ -35,8 +35,12 @@ type Params struct {
 }
 
 // DomainState holds the monitoring state for an apex domain.
+// NameserverAddresses holds the sorted addresses each nameserver's name
+// resolves to, by nameserver name. A state file written before it
+// existed loads with it nil.
 type DomainState struct {
 	Nameservers         []string            `json:"nameservers"`
+	NameserverAddresses map[string][]string `json:"nameserverAddresses"`
 	LastChecked         time.Time           `json:"lastChecked"`
 }
 
@@ -148,6 +152,11 @@ func New(
 
 	lifecycle.Append(fx.Hook{
 		OnStart: func(_ context.Context) error {
+			err := state.checkDataDirWritable()
+			if err != nil {
+				return err
+			}
+
 			return state.Load()
 		},
 		OnStop: func(_ context.Context) error {
@@ -345,3 +354,27 @@ func (s *State) GetCertificateState(
 
 	return cs, ok
 }
+
+// checkDataDirWritable creates the data directory if needed, then writes
+// and removes the temp file that Save uses. It runs at startup so that an
+// unwritable directory stops the process, instead of the process running
+// with every save failing and only logged.
+func (s *State) checkDataDirWritable() error {
+	dir := s.config.DataDir
+	tmpPath := s.config.StatePath() + ".tmp"
+
+	err := os.MkdirAll(dir, dirPermissions)
+	if err == nil {
+		err = os.WriteFile(tmpPath, nil, filePermissions)
+	}
+
+	if err == nil {
+		err = os.Remove(tmpPath)
+	}
+
+	if err != nil {
+		return fmt.Errorf("data directory %s is not writable: %w", dir, err)
+	}
+
+	return nil
+}

internal/state/state_test.go

@@ -4,10 +4,17 @@ import (
 	"encoding/json"
 	"os"
 	"path/filepath"
+	"reflect"
+	"strings"
 	"sync"
 	"testing"
 	"time"
 
+	"go.uber.org/fx/fxtest"
+
+	"sneak.berlin/go/dnswatcher/internal/config"
+	"sneak.berlin/go/dnswatcher/internal/globals"
+	"sneak.berlin/go/dnswatcher/internal/logger"
 	"sneak.berlin/go/dnswatcher/internal/state"
 )
 
@@ -31,6 +38,10 @@ func populateState(t *testing.T, s *state.State) {
 
 	s.SetDomainState("example.com", &state.DomainState{
 		Nameservers: []string{testNS1, testNS2},
+		NameserverAddresses: map[string][]string{
+			testNS1: {testIP, testIPv4},
+			testNS2: {testIPv4},
+		},
 		LastChecked: now,
 	})
 
@@ -117,6 +128,64 @@ func TestSaveLoadRoundTrip_Domains(t *testing.T) {
 	if len(dom.Nameservers) != 2 {
 		t.Errorf("expected 2 nameservers, got %d", len(dom.Nameservers))
 	}
+
+	want := map[string][]string{
+		testNS1: {testIP, testIPv4},
+		testNS2: {testIPv4},
+	}
+	if !reflect.DeepEqual(dom.NameserverAddresses, want) {
+		t.Errorf(
+			"nameserver addresses: got %v, want %v",
+			dom.NameserverAddresses, want,
+		)
+	}
+}
+
+// TestLoadStateFromBeforeNameserverAddresses loads a state file written
+// before nameserver addresses were saved.
+func TestLoadStateFromBeforeNameserverAddresses(t *testing.T) {
+	t.Parallel()
+
+	dir := t.TempDir()
+
+	data := []byte(`{
+		"version": 1,
+		"lastUpdated": "2026-02-19T12:00:00Z",
+		"domains": {
+			"example.com": {
+				"nameservers": ["ns1.example.com.", "ns2.example.com."],
+				"lastChecked": "2026-02-19T12:00:00Z"
+			}
+		}
+	}`)
+
+	err := os.WriteFile(filepath.Join(dir, "state.json"), data, 0o600)
+	if err != nil {
+		t.Fatalf("writing state file: %v", err)
+	}
+
+	s := state.NewForTestWithDataDir(dir)
+
+	err = s.Load()
+	if err != nil {
+		t.Fatalf("Load() error: %v", err)
+	}
+
+	dom, ok := s.GetDomainState("example.com")
+	if !ok {
+		t.Fatal("missing domain example.com")
+	}
+
+	if !reflect.DeepEqual(dom.Nameservers, []string{testNS1, testNS2}) {
+		t.Errorf("nameservers: got %v", dom.Nameservers)
+	}
+
+	if dom.NameserverAddresses != nil {
+		t.Errorf(
+			"nameserver addresses: got %v, want none",
+			dom.NameserverAddresses,
+		)
+	}
 }
 
 // TestSaveLoadRoundTrip_Hostnames verifies hostname data survives a save/load cycle.
@@ -493,6 +562,107 @@ func TestSaveWritePermissionError(t *testing.T) {
 	}
 }
 
+// startState builds a State through the real constructor and runs its
+// startup hook against dataDir, returning the startup error.
+func startState(t *testing.T, dataDir string) error {
+	t.Helper()
+
+	g, err := globals.New(nil)
+	if err != nil {
+		t.Fatalf("globals.New: %v", err)
+	}
+
+	log, err := logger.New(nil, logger.Params{Globals: g})
+	if err != nil {
+		t.Fatalf("logger.New: %v", err)
+	}
+
+	lifecycle := fxtest.NewLifecycle(t)
+
+	_, err = state.New(lifecycle, state.Params{
+		Logger: log,
+		Config: &config.Config{DataDir: dataDir},
+	})
+	if err != nil {
+		t.Fatalf("state.New: %v", err)
+	}
+
+	return lifecycle.Start(t.Context())
+}
+
+// TestStartupFailsWhenDataDirNotWritable verifies that startup stops
+// with an error naming the data directory when it cannot be written.
+// The directory's parent is a regular file, which also fails as root.
+func TestStartupFailsWhenDataDirNotWritable(t *testing.T) {
+	t.Parallel()
+
+	parent := filepath.Join(t.TempDir(), "file")
+
+	err := os.WriteFile(parent, nil, 0o600)
+	if err != nil {
+		t.Fatalf("writing file: %v", err)
+	}
+
+	dataDir := filepath.Join(parent, "data")
+
+	err = startState(t, dataDir)
+	if err == nil {
+		t.Fatal("startup should fail when the data directory is not writable")
+	}
+
+	want := "data directory " + dataDir + " is not writable"
+	if !strings.Contains(err.Error(), want) {
+		t.Errorf("startup error %q does not contain %q", err, want)
+	}
+}
+
+// TestStartupFailsWhenExistingDataDirNotWritable verifies that startup
+// stops when the data directory exists but the temp file that saving uses
+// cannot be written in it. A directory sitting at the temp file's path
+// makes that write fail, which also holds as root.
+func TestStartupFailsWhenExistingDataDirNotWritable(t *testing.T) {
+	t.Parallel()
+
+	dataDir := t.TempDir()
+
+	err := os.Mkdir(filepath.Join(dataDir, "state.json.tmp"), 0o700)
+	if err != nil {
+		t.Fatalf("creating directory: %v", err)
+	}
+
+	err = startState(t, dataDir)
+	if err == nil {
+		t.Fatal("startup should fail when the data directory is not writable")
+	}
+
+	want := "data directory " + dataDir + " is not writable"
+	if !strings.Contains(err.Error(), want) {
+		t.Errorf("startup error %q does not contain %q", err, want)
+	}
+}
+
+// TestStartupCreatesDataDir verifies that startup creates a missing
+// data directory and leaves nothing behind in it.
+func TestStartupCreatesDataDir(t *testing.T) {
+	t.Parallel()
+
+	dataDir := filepath.Join(t.TempDir(), "data")
+
+	err := startState(t, dataDir)
+	if err != nil {
+		t.Fatalf("startup error: %v", err)
+	}
+
+	entries, err := os.ReadDir(dataDir)
+	if err != nil {
+		t.Fatalf("reading data directory: %v", err)
+	}
+
+	if len(entries) != 0 {
+		t.Errorf("startup left %d entries in the data directory", len(entries))
+	}
+}
+
 // TestPortStateUnmarshalJSON_NewFormat verifies deserialization of the
 // current multi-hostname format.
 func TestPortStateUnmarshalJSON_NewFormat(t *testing.T) {
@@ -632,7 +802,7 @@ func TestPortStateUnmarshalJSON_BothFormats(t *testing.T) {
 func TestGetSnapshot_ReturnsCopy(t *testing.T) {
 	t.Parallel()
 
-	s := state.NewForTest()
+	s := state.NewForTestWithDataDir(t.TempDir())
 
 	populateState(t, s)
 
@@ -654,7 +824,7 @@ func TestGetSnapshot_ReturnsCopy(t *testing.T) {
 func TestDomainState_GetSet(t *testing.T) {
 	t.Parallel()
 
-	s := state.NewForTest()
+	s := state.NewForTestWithDataDir(t.TempDir())
 
 	// Get on missing key returns false.
 	_, ok := s.GetDomainState("nonexistent.com")
@@ -705,7 +875,7 @@ func TestDomainState_GetSet(t *testing.T) {
 func TestHostnameState_GetSet(t *testing.T) {
 	t.Parallel()
 
-	s := state.NewForTest()
+	s := state.NewForTestWithDataDir(t.TempDir())
 
 	_, ok := s.GetHostnameState("missing.example.com")
 	if ok {
@@ -750,7 +920,7 @@ func TestHostnameState_GetSet(t *testing.T) {
 func TestPortState_GetSetDelete(t *testing.T) {
 	t.Parallel()
 
-	s := state.NewForTest()
+	s := state.NewForTestWithDataDir(t.TempDir())
 
 	_, ok := s.GetPortState("1.2.3.4:80")
 	if ok {
@@ -788,7 +958,7 @@ func TestPortState_GetSetDelete(t *testing.T) {
 func TestGetAllPortKeys(t *testing.T) {
 	t.Parallel()
 
-	s := state.NewForTest()
+	s := state.NewForTestWithDataDir(t.TempDir())
 
 	keys := s.GetAllPortKeys()
 	if len(keys) != 0 {
@@ -830,7 +1000,7 @@ func TestGetAllPortKeys(t *testing.T) {
 func TestCertificateState_GetSet(t *testing.T) {
 	t.Parallel()
 
-	s := state.NewForTest()
+	s := state.NewForTestWithDataDir(t.TempDir())
 
 	_, ok := s.GetCertificateState("1.2.3.4:443:www.example.com")
 	if ok {
@@ -1051,7 +1221,7 @@ func TestLoadPreservesExistingStateOnMissingFile(t *testing.T) {
 func TestConcurrentGetSet(t *testing.T) {
 	t.Parallel()
 
-	s := state.NewForTest()
+	s := state.NewForTestWithDataDir(t.TempDir())
 
 	const goroutines = 20
 
@@ -1261,35 +1431,6 @@ func TestMultipleSavesOverwrite(t *testing.T) {
 	}
 }
 
-// TestNewForTest verifies the test helper creates a valid empty state.
-func TestNewForTest(t *testing.T) {
-	t.Parallel()
-
-	s := state.NewForTest()
-
-	snap := s.GetSnapshot()
-
-	if snap.Version != 1 {
-		t.Errorf("version: got %d, want 1", snap.Version)
-	}
-
-	if snap.Domains == nil {
-		t.Error("Domains map should be initialized")
-	}
-
-	if snap.Hostnames == nil {
-		t.Error("Hostnames map should be initialized")
-	}
-
-	if snap.Ports == nil {
-		t.Error("Ports map should be initialized")
-	}
-
-	if snap.Certificates == nil {
-		t.Error("Certificates map should be initialized")
-	}
-}
-
 // TestSaveFilePermissions verifies the saved file has restricted permissions.
 func TestSaveFilePermissions(t *testing.T) {
 	t.Parallel()

internal/state/state_test_helper.go

@@ -1,38 +0,0 @@
-package state
-
-import (
-	"log/slog"
-
-	"sneak.berlin/go/dnswatcher/internal/config"
-)
-
-// NewForTest creates a State for unit testing with no persistence.
-func NewForTest() *State {
-	return &State{
-		log: slog.Default(),
-		snapshot: &Snapshot{
-			Version:      stateVersion,
-			Domains:      make(map[string]*DomainState),
-			Hostnames:    make(map[string]*HostnameState),
-			Ports:        make(map[string]*PortState),
-			Certificates: make(map[string]*CertificateState),
-		},
-		config: &config.Config{DataDir: ""},
-	}
-}
-
-// NewForTestWithDataDir creates a State backed by the given directory
-// for tests that need file persistence.
-func NewForTestWithDataDir(dataDir string) *State {
-	return &State{
-		log: slog.Default(),
-		snapshot: &Snapshot{
-			Version:      stateVersion,
-			Domains:      make(map[string]*DomainState),
-			Hostnames:    make(map[string]*HostnameState),
-			Ports:        make(map[string]*PortState),
-			Certificates: make(map[string]*CertificateState),
-		},
-		config: &config.Config{DataDir: dataDir},
-	}
-}

internal/watcher/cancelled_test.go

@@ -0,0 +1,81 @@
+package watcher_test
+
+import (
+	"context"
+	"log/slog"
+	"reflect"
+	"testing"
+
+	"sneak.berlin/go/dnswatcher/internal/portcheck"
+	"sneak.berlin/go/dnswatcher/internal/resolver"
+	"sneak.berlin/go/dnswatcher/internal/state"
+	"sneak.berlin/go/dnswatcher/internal/tlscheck"
+	"sneak.berlin/go/dnswatcher/internal/watcher"
+)
+
+// TestCancelledCheckSavesNothing runs a check with its context already
+// cancelled, which is how the rest of a check runs once shutdown cuts it
+// short. The real resolver drops the DNS lookup without sending a query,
+// and the real port and TLS checkers fail without connecting. The port
+// and certificate state the last check saved must stay as it was, and
+// nothing may be notified.
+func TestCancelledCheckSavesNothing(t *testing.T) {
+	t.Parallel()
+
+	cfg := defaultTestConfig(t)
+	cfg.Hostnames = []string{host}
+
+	// newTestWatcher's watcher has stand-in checkers. This one, on the
+	// same state and notifier, has the real ones.
+	_, deps := newTestWatcher(t, cfg)
+	w := watcher.NewForTest(
+		cfg,
+		deps.state,
+		resolver.NewFromLogger(slog.Default()),
+		portcheck.NewStandalone(),
+		tlscheck.NewStandalone(),
+		deps.notifier,
+	)
+
+	// The last check found host at a local address, with both ports
+	// open and a good certificate.
+	const localIP = "127.0.0.1"
+
+	deps.state.SetHostnameState(host, hostnameState(
+		map[string]map[string][]string{nsA: {"A": {localIP}}},
+	))
+
+	ports := map[string]*state.PortState{
+		localIP + ":80":  {Open: true, Hostnames: []string{host}},
+		localIP + ":443": {Open: true, Hostnames: []string{host}},
+	}
+	for key, ps := range ports {
+		deps.state.SetPortState(key, ps)
+	}
+
+	certKey := localIP + ":443:" + host
+	cert := &state.CertificateState{CommonName: host, Status: "ok"}
+	deps.state.SetCertificateState(certKey, cert)
+
+	ctx, cancel := context.WithCancel(t.Context())
+	cancel()
+
+	w.RunOnce(ctx)
+
+	for key, want := range ports {
+		got, _ := deps.state.GetPortState(key)
+		if !reflect.DeepEqual(got, want) {
+			t.Errorf("port %s saved as %+v, want %+v", key, got, want)
+		}
+	}
+
+	got, _ := deps.state.GetCertificateState(certKey)
+	if !reflect.DeepEqual(got, cert) {
+		t.Errorf("certificate saved as %+v, want %+v", got, cert)
+	}
+
+	notifications := deps.notifier.getNotifications()
+	if len(notifications) != 0 {
+		t.Errorf("sent %v, want no notifications", notifications)
+	}
+}

internal/watcher/export_test.go

@@ -0,0 +1,85 @@
+package watcher
+
+import (
+	"context"
+	"log/slog"
+	"time"
+
+	"sneak.berlin/go/dnswatcher/internal/config"
+	"sneak.berlin/go/dnswatcher/internal/resolver"
+	"sneak.berlin/go/dnswatcher/internal/state"
+)
+
+// NewForTest creates a Watcher without fx for unit testing.
+func NewForTest(
+	cfg *config.Config,
+	st *state.State,
+	res DNSResolver,
+	pc PortChecker,
+	tc TLSChecker,
+	n Notifier,
+) *Watcher {
+	return &Watcher{
+		log:       slog.Default(),
+		config:    cfg,
+		state:     st,
+		resolver:  res,
+		portCheck: pc,
+		tlsCheck:  tc,
+		notify:    n,
+		firstRun:  true,
+	}
+}
+
+// NewlyDisagreeingPairs exports newlyDisagreeingPairs for testing.
+func NewlyDisagreeingPairs(
+	prev, current *state.HostnameState,
+) [][2]string {
+	return newlyDisagreeingPairs(prev, current)
+}
+
+// DetectHostnameChanges exports detectHostnameChanges for testing.
+func (w *Watcher) DetectHostnameChanges(
+	ctx context.Context,
+	hostname string,
+	prev, current *state.HostnameState,
+) {
+	w.detectHostnameChanges(ctx, hostname, prev, current)
+}
+
+// ResolveNameserverAddresses exports resolveNameserverAddresses for
+// testing.
+func (w *Watcher) ResolveNameserverAddresses(
+	ctx context.Context,
+	nameservers []string,
+	prev map[string][]string,
+) map[string][]string {
+	return w.resolveNameserverAddresses(ctx, nameservers, prev)
+}
+
+// DetectNSAddressChanges exports detectNSAddressChanges for testing.
+func (w *Watcher) DetectNSAddressChanges(
+	ctx context.Context,
+	domain string,
+	prev, current map[string][]string,
+) {
+	w.detectNSAddressChanges(ctx, domain, prev, current)
+}
+
+// CheckAllPorts exports checkAllPorts for testing.
+func (w *Watcher) CheckAllPorts(ctx context.Context) {
+	w.checkAllPorts(ctx)
+}
+
+// RunTLSChecks exports runTLSChecks for testing.
+func (w *Watcher) RunTLSChecks(ctx context.Context) {
+	w.runTLSChecks(ctx)
+}
+
+// BuildHostnameState exports buildHostnameState for testing.
+func BuildHostnameState(
+	results map[string]*resolver.NameserverResponse,
+	now time.Time,
+) *state.HostnameState {
+	return buildHostnameState(results, now)
+}

internal/watcher/inconsistency_test.go

@@ -0,0 +1,185 @@
+package watcher_test
+
+import (
+	"slices"
+	"testing"
+
+	"sneak.berlin/go/dnswatcher/internal/state"
+	"sneak.berlin/go/dnswatcher/internal/watcher"
+)
+
+const (
+	host = "www.example.net"
+	nsA  = "a.ns.example.net."
+	nsB  = "b.ns.example.net."
+	nsC  = "c.ns.example.net."
+	ip1  = "192.0.2.1"
+	ip2  = "192.0.2.2"
+	ip3  = "192.0.2.3"
+)
+
+// hostnameState builds the state a check with these records leaves behind.
+func hostnameState(
+	records map[string]map[string][]string,
+) *state.HostnameState {
+	hs := &state.HostnameState{
+		RecordsByNameserver: make(map[string]*state.NameserverRecordState),
+	}
+
+	for ns, recs := range records {
+		hs.RecordsByNameserver[ns] = &state.NameserverRecordState{
+			Records: recs,
+			Status:  "ok",
+		}
+	}
+
+	return hs
+}
+
+func TestNewlyDisagreeingPairs(t *testing.T) {
+	t.Parallel()
+
+	onlyA := map[string]map[string][]string{nsA: {"A": {ip1}}}
+	agree := map[string]map[string][]string{nsA: {"A": {ip1}}, nsB: {"A": {ip1}}}
+	disagree := map[string]map[string][]string{nsA: {"A": {ip1}}, nsB: {"A": {ip2}}}
+	alert := [][2]string{{nsA, nsB}}
+
+	// b already disagrees with a and c; then c changes, so a and c,
+	// which agreed, now differ.
+	bDiffers := map[string]map[string][]string{
+		nsA: {"A": {ip1}}, nsB: {"A": {ip2}}, nsC: {"A": {ip1}},
+	}
+	cChanges := map[string]map[string][]string{
+		nsA: {"A": {ip1}}, nsB: {"A": {ip2}}, nsC: {"A": {ip3}},
+	}
+
+	// Each case starts from the state loaded at startup and runs the
+	// checks in order; want[i] is what check i alerts for.
+	tests := []struct {
+		name   string
+		loaded map[string]map[string][]string
+		checks []map[string]map[string][]string
+		want   [][][2]string
+	}{
+		{
+			name:   "disagreement persisting across checks alerts once",
+			loaded: agree,
+			checks: []map[string]map[string][]string{disagree, disagree, disagree},
+			want:   [][][2]string{alert, nil, nil},
+		},
+		{
+			name:   "disagreement starting on a later check alerts on it",
+			loaded: agree,
+			checks: []map[string]map[string][]string{agree, agree, disagree},
+			want:   [][][2]string{nil, nil, alert},
+		},
+		{
+			name:   "disagreement in the loaded state does not alert",
+			loaded: disagree,
+			checks: []map[string]map[string][]string{disagree, disagree},
+			want:   [][][2]string{nil, nil},
+		},
+		{
+			name:   "nameserver new on the first check and disagreeing alerts once",
+			loaded: onlyA,
+			checks: []map[string]map[string][]string{disagree, disagree},
+			want:   [][][2]string{alert, nil},
+		},
+		{
+			name:   "disagreement after agreeing again alerts again",
+			loaded: agree,
+			checks: []map[string]map[string][]string{disagree, agree, disagree},
+			want:   [][][2]string{alert, nil, alert},
+		},
+		{
+			name:   "new disagreement while another nameserver differs alerts",
+			loaded: bDiffers,
+			checks: []map[string]map[string][]string{cChanges, cChanges},
+			want:   [][][2]string{{{nsA, nsC}}, nil},
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			t.Parallel()
+
+			prev := hostnameState(tt.loaded)
+
+			for i, records := range tt.checks {
+				current := hostnameState(records)
+
+				got := watcher.NewlyDisagreeingPairs(prev, current)
+				if !slices.Equal(got, tt.want[i]) {
+					t.Errorf(
+						"check %d: alerted for %v, want %v",
+						i, got, tt.want[i],
+					)
+				}
+
+				prev = current
+			}
+		})
+	}
+}
+
+func TestInconsistencyAlert(t *testing.T) {
+	t.Parallel()
+
+	onlyA := map[string]map[string][]string{nsA: {"A": {ip1}}}
+	agree := map[string]map[string][]string{nsA: {"A": {ip1}}, nsB: {"A": {ip1}}}
+	disagree := map[string]map[string][]string{nsA: {"A": {ip1}}, nsB: {"A": {ip2}}}
+
+	// Each case starts from the state loaded at startup and then sees
+	// the nameservers disagree on three checks in a row.
+	tests := []struct {
+		name   string
+		loaded map[string]map[string][]string
+		want   int
+	}{
+		{
+			name:   "disagreement lasting several checks alerts once",
+			loaded: agree,
+			want:   1,
+		},
+		{
+			name:   "disagreement in the loaded state does not alert",
+			loaded: disagree,
+			want:   0,
+		},
+		{
+			name:   "nameserver new on the first check and disagreeing alerts once",
+			loaded: onlyA,
+			want:   1,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			t.Parallel()
+
+			// The hostname change detection uses only the notifier.
+			notifier := &mockNotifier{}
+			w := watcher.NewForTest(nil, nil, nil, nil, nil, notifier)
+
+			prev := hostnameState(tt.loaded)
+
+			for range 3 {
+				current := hostnameState(disagree)
+				w.DetectHostnameChanges(t.Context(), host, prev, current)
+				prev = current
+			}
+
+			got := 0
+
+			for _, n := range notifier.getNotifications() {
+				if n.Title == "Inconsistency: "+host {
+					got++
+				}
+			}
+
+			if got != tt.want {
+				t.Errorf("sent %d inconsistency alerts, want %d", got, tt.want)
+			}
+		})
+	}
+}

internal/watcher/interfaces.go

@@ -5,6 +5,7 @@ import (
 	"context"
 
 	"sneak.berlin/go/dnswatcher/internal/portcheck"
+	"sneak.berlin/go/dnswatcher/internal/resolver"
 	"sneak.berlin/go/dnswatcher/internal/tlscheck"
 )
 
@@ -17,11 +18,11 @@ type DNSResolver interface {
 	) ([]string, error)
 
 	// LookupAllRecords queries all record types for a hostname,
-	// returning results keyed by nameserver then record type.
+	// returning each nameserver's response keyed by nameserver.
 	LookupAllRecords(
 		ctx context.Context,
 		hostname string,
-	) (map[string]map[string][]string, error)
+	) (map[string]*resolver.NameserverResponse, error)
 
 	// ResolveIPAddresses resolves a hostname to all IP addresses.
 	ResolveIPAddresses(

internal/watcher/nsaddress_test.go

@@ -0,0 +1,151 @@
+package watcher_test
+
+import (
+	"context"
+	"log/slog"
+	"reflect"
+	"testing"
+
+	"sneak.berlin/go/dnswatcher/internal/livednstest"
+	"sneak.berlin/go/dnswatcher/internal/resolver"
+	"sneak.berlin/go/dnswatcher/internal/watcher"
+)
+
+const domain = "example.net"
+
+func TestNSAddressChangeAlerts(t *testing.T) {
+	t.Parallel()
+
+	// Each case is the nameserver addresses saved by the previous check
+	// and by the current one.
+	tests := []struct {
+		name          string
+		prev, current map[string][]string
+		want          int
+	}{
+		{
+			"same addresses",
+			map[string][]string{nsA: {ip1, ip2}},
+			map[string][]string{nsA: {ip1, ip2}},
+			0,
+		},
+		{
+			"same addresses in another order",
+			map[string][]string{nsA: {ip2, ip1}},
+			map[string][]string{nsA: {ip1, ip2}},
+			0,
+		},
+		{
+			"address replaced",
+			map[string][]string{nsA: {ip1}},
+			map[string][]string{nsA: {ip2}},
+			1,
+		},
+		{
+			"address added",
+			map[string][]string{nsA: {ip1}},
+			map[string][]string{nsA: {ip1, ip2}},
+			1,
+		},
+		{
+			"two nameservers changed",
+			map[string][]string{nsA: {ip1}, nsB: {ip2}},
+			map[string][]string{nsA: {ip3}, nsB: {ip3}},
+			2,
+		},
+		{
+			"nameserver added",
+			map[string][]string{nsA: {ip1}},
+			map[string][]string{nsA: {ip1}, nsB: {ip2}},
+			0,
+		},
+		{
+			"nameserver removed",
+			map[string][]string{nsA: {ip1}, nsB: {ip2}},
+			map[string][]string{nsA: {ip1}},
+			0,
+		},
+		{
+			"state file from before addresses were saved",
+			nil,
+			map[string][]string{nsA: {ip1}, nsB: {ip2}},
+			0,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			t.Parallel()
+
+			notifier := &mockNotifier{}
+			w := watcher.NewForTest(nil, nil, nil, nil, nil, notifier)
+
+			w.DetectNSAddressChanges(t.Context(), domain, tt.prev, tt.current)
+
+			got := len(notifier.getNotifications())
+			if got != tt.want {
+				t.Errorf("sent %d address changes, want %d", got, tt.want)
+			}
+		})
+	}
+}
+
+func TestNSAddressChangeAlertNamesDomainNameserverAndAddresses(
+	t *testing.T,
+) {
+	t.Parallel()
+
+	notifier := &mockNotifier{}
+	w := watcher.NewForTest(nil, nil, nil, nil, nil, notifier)
+
+	w.DetectNSAddressChanges(
+		t.Context(), domain,
+		map[string][]string{nsA: {ip1}},
+		map[string][]string{nsA: {ip2, ip3}},
+	)
+
+	want := notification{
+		Title: "NS Address Change: " + domain,
+		Message: "Domain: " + domain + "\nNameserver: " + nsA +
+			"\nOld: " + ip1 + "\nNew: " + ip2 + ", " + ip3,
+		Priority: "warning",
+	}
+
+	got := notifier.getNotifications()
+	if len(got) != 1 || got[0] != want {
+		t.Errorf("sent %v, want %v", got, want)
+	}
+}
+
+// TestNameserverWithNoAddressKeepsPrevious looks up nameserver names
+// with no address: two under .invalid, whose lookup fails with an
+// error, and one that does not exist under a real zone, which live DNS
+// answers with no address and no error. Each one with addresses saved
+// by the previous check keeps them; the one without gets none.
+func TestNameserverWithNoAddressKeepsPrevious(t *testing.T) {
+	t.Parallel()
+
+	w := watcher.NewForTest(
+		nil, nil, resolver.NewFromLogger(slog.Default()), nil, nil, nil,
+	)
+
+	nonexistentNS := "this-surely-does-not-exist-xyz." + testSmallDomain + "."
+
+	prev := map[string][]string{oldNS1: {oldIP}, nonexistentNS: {oldIP}}
+
+	var got map[string][]string
+
+	// The result is the same whether or not live DNS answers, so the
+	// lookup is not retried.
+	_ = livednstest.Run(func(ctx context.Context) error {
+		got = w.ResolveNameserverAddresses(
+			ctx, []string{oldNS1, oldNS2, nonexistentNS}, prev,
+		)
+
+		return nil
+	})
+
+	if !reflect.DeepEqual(got, prev) {
+		t.Errorf("saved %v, want %v", got, prev)
+	}
+}

internal/watcher/nsfailure_test.go

@@ -0,0 +1,448 @@
+package watcher_test
+
+import (
+	"context"
+	"fmt"
+	"log/slog"
+	"strings"
+	"testing"
+	"time"
+
+	"sneak.berlin/go/dnswatcher/internal/livednstest"
+	"sneak.berlin/go/dnswatcher/internal/resolver"
+	"sneak.berlin/go/dnswatcher/internal/state"
+	"sneak.berlin/go/dnswatcher/internal/watcher"
+)
+
+// answered is what a check saves for a nameserver that answered with
+// these records.
+func answered(records map[string][]string) *state.NameserverRecordState {
+	return &state.NameserverRecordState{Records: records, Status: "ok"}
+}
+
+// failed is what a check saves for a nameserver that did not answer.
+func failed() *state.NameserverRecordState {
+	return &state.NameserverRecordState{
+		Records: map[string][]string{},
+		Status:  "error",
+		Error:   "all queries timed out",
+	}
+}
+
+// saved builds the hostname state a check saves.
+func saved(
+	byNameserver map[string]*state.NameserverRecordState,
+) *state.HostnameState {
+	return &state.HostnameState{RecordsByNameserver: byNameserver}
+}
+
+// alertCounts counts the hostname alerts sent, by kind.
+type alertCounts struct {
+	failures, recoveries, recordChanges, inconsistencies int
+}
+
+// countAlerts runs the hostname change detection from the state loaded
+// at startup through each check in turn, and counts the alerts sent.
+func countAlerts(
+	t *testing.T,
+	loaded *state.HostnameState,
+	checks []*state.HostnameState,
+) alertCounts {
+	t.Helper()
+
+	// The hostname change detection uses only the notifier.
+	notifier := &mockNotifier{}
+	w := watcher.NewForTest(nil, nil, nil, nil, nil, notifier)
+
+	prev := loaded
+
+	for _, current := range checks {
+		w.DetectHostnameChanges(t.Context(), host, prev, current)
+		prev = current
+	}
+
+	var got alertCounts
+
+	for _, n := range notifier.getNotifications() {
+		kind, _, _ := strings.Cut(n.Title, ":")
+
+		switch kind {
+		case "NS Failure":
+			got.failures++
+		case "NS Recovery":
+			got.recoveries++
+		case "Record Change":
+			got.recordChanges++
+		case "Inconsistency":
+			got.inconsistencies++
+		}
+	}
+
+	return got
+}
+
+func TestNSFailureAndRecoveryAlerts(t *testing.T) {
+	t.Parallel()
+
+	records := map[string][]string{"A": {ip1}}
+
+	bothAnswer := saved(map[string]*state.NameserverRecordState{
+		nsA: answered(records), nsB: answered(records),
+	})
+	bFails := saved(map[string]*state.NameserverRecordState{
+		nsA: answered(records), nsB: failed(),
+	})
+	onlyA := saved(map[string]*state.NameserverRecordState{
+		nsA: answered(records),
+	})
+	bAnswersNoRecords := saved(map[string]*state.NameserverRecordState{
+		nsA: answered(records), nsB: answered(map[string][]string{}),
+	})
+	bAnswersDifferently := saved(map[string]*state.NameserverRecordState{
+		nsA: answered(records), nsB: answered(map[string][]string{"A": {ip2}}),
+	})
+
+	// Each case starts from the state loaded at startup and runs the
+	// checks in order.
+	tests := []struct {
+		name   string
+		loaded *state.HostnameState
+		checks []*state.HostnameState
+		want   alertCounts
+	}{
+		{
+			"failure lasting several checks alerts once",
+			bothAnswer, []*state.HostnameState{bFails, bFails, bFails},
+			alertCounts{failures: 1},
+		},
+		{
+			"recovery alerts once",
+			bFails, []*state.HostnameState{bothAnswer, bothAnswer},
+			alertCounts{recoveries: 1},
+		},
+		{
+			"failing again after recovering alerts again",
+			bothAnswer, []*state.HostnameState{bFails, bothAnswer, bFails},
+			alertCounts{failures: 2, recoveries: 1},
+		},
+		{
+			"nameserver failing when first seen does not alert",
+			onlyA, []*state.HostnameState{bFails, bFails},
+			alertCounts{},
+		},
+		{
+			"answer with no records is a record change, not a failure",
+			bothAnswer, []*state.HostnameState{bAnswersNoRecords},
+			alertCounts{recordChanges: 1, inconsistencies: 1},
+		},
+		{
+			"recovered nameserver that answers differently disagrees",
+			bFails, []*state.HostnameState{bAnswersDifferently},
+			alertCounts{recoveries: 1, inconsistencies: 1},
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			t.Parallel()
+
+			got := countAlerts(t, tt.loaded, tt.checks)
+			if got != tt.want {
+				t.Errorf("sent %+v, want %+v", got, tt.want)
+			}
+		})
+	}
+}
+
+func TestNSFailureAlertNamesHostnameNameserverAndReason(t *testing.T) {
+	t.Parallel()
+
+	records := map[string][]string{"A": {ip1}}
+
+	notifier := &mockNotifier{}
+	w := watcher.NewForTest(nil, nil, nil, nil, nil, notifier)
+
+	w.DetectHostnameChanges(
+		t.Context(), host,
+		saved(map[string]*state.NameserverRecordState{nsA: answered(records)}),
+		saved(map[string]*state.NameserverRecordState{nsA: failed()}),
+	)
+
+	notifications := notifier.getNotifications()
+	if len(notifications) != 1 {
+		t.Fatalf("sent %v, want one NS Failure", notifications)
+	}
+
+	msg := notifications[0].Message
+	if !strings.Contains(msg, host) || !strings.Contains(msg, nsA) ||
+		!strings.Contains(msg, failed().Error) {
+		t.Errorf(
+			"message %q does not name %s, %s and the reason",
+			msg, host, nsA,
+		)
+	}
+}
+
+// TestNameserverThatNeverAnswers asks a nameserver address where
+// nothing answers, 192.0.2.1, and checks what the watcher saves for it.
+// The deadline outlasts the resolver's first two-second try, as in the
+// resolver's timeout test.
+func TestNameserverThatNeverAnswers(t *testing.T) {
+	t.Parallel()
+
+	ctx, cancel := context.WithTimeout(t.Context(), 3*time.Second)
+	t.Cleanup(cancel)
+
+	res := resolver.NewFromLogger(slog.Default())
+
+	resp, err := res.QueryNameserverIP(ctx, nsA, "192.0.2.1", host)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	hs := watcher.BuildHostnameState(
+		map[string]*resolver.NameserverResponse{nsA: resp}, time.Now(),
+	)
+
+	got := hs.RecordsByNameserver[nsA]
+	if got.Status != "error" || got.Error == "" {
+		t.Errorf(
+			"saved status %q, error %q; want status error with a reason",
+			got.Status, got.Error,
+		)
+	}
+}
+
+// TestNameserverThatAnswersNXDOMAIN asks a real nameserver about a name
+// that does not exist and checks what the watcher saves for it: NXDOMAIN
+// is an answer, so the nameserver is saved as ok with no error.
+func TestNameserverThatAnswersNXDOMAIN(t *testing.T) {
+	t.Parallel()
+
+	res := resolver.NewFromLogger(slog.Default())
+	name := "this-surely-does-not-exist-xyz." + testDomain
+
+	var (
+		ns   string
+		resp *resolver.NameserverResponse
+	)
+
+	livednstest.Retry(t, "QueryNameserver("+name+")", func(ctx context.Context) error {
+		nameservers, err := res.LookupNS(ctx, testDomain)
+		if err != nil {
+			return err
+		}
+
+		ns = nameservers[0]
+
+		resp, err = res.QueryNameserver(ctx, ns, name)
+		if err != nil {
+			return err
+		}
+
+		// A timeout or a failure is no answer to check.
+		if resp.Status == resolver.StatusTimeout ||
+			resp.Status == resolver.StatusError {
+			return fmt.Errorf(
+				"%w: %s: %s", livednstest.ErrNoAnswer, ns, resp.Error,
+			)
+		}
+
+		return nil
+	})
+
+	if resp.Status != resolver.StatusNXDomain {
+		t.Fatalf("%s answered %q for %s, want NXDOMAIN", ns, resp.Status, name)
+	}
+
+	hs := watcher.BuildHostnameState(
+		map[string]*resolver.NameserverResponse{ns: resp}, time.Now(),
+	)
+
+	got := hs.RecordsByNameserver[ns]
+	if got.Status != "ok" || got.Error != "" {
+		t.Errorf(
+			"saved status %q, error %q; want status ok with no error",
+			got.Status, got.Error,
+		)
+	}
+}
+
+// TestNameserverThatRefuses asks a google.com nameserver about
+// cloudflare.com, a zone it does not serve, which it refuses, and checks
+// what the watcher saves for it: REFUSED is no answer, so the nameserver
+// is saved as error with the reason.
+func TestNameserverThatRefuses(t *testing.T) {
+	t.Parallel()
+
+	const reason = "server returned REFUSED"
+
+	res := resolver.NewFromLogger(slog.Default())
+
+	var (
+		ns   string
+		resp *resolver.NameserverResponse
+	)
+
+	livednstest.Retry(
+		t,
+		"QueryNameserver(cloudflare.com)",
+		func(ctx context.Context) error {
+			nameservers, err := res.LookupNS(ctx, testDomain)
+			if err != nil {
+				return err
+			}
+
+			ns = nameservers[0]
+
+			resp, err = res.QueryNameserver(ctx, ns, "cloudflare.com")
+			if err != nil {
+				return err
+			}
+
+			// A timeout or a network error is no reply at all.
+			if resp.Status == resolver.StatusTimeout ||
+				strings.HasPrefix(resp.Error, "network error") {
+				return fmt.Errorf(
+					"%w: %s: %s", livednstest.ErrNoAnswer, ns, resp.Error,
+				)
+			}
+
+			return nil
+		},
+	)
+
+	if resp.Error != reason {
+		t.Fatalf(
+			"%s answered %q (%s) for cloudflare.com, want REFUSED",
+			ns, resp.Status, resp.Error,
+		)
+	}
+
+	hs := watcher.BuildHostnameState(
+		map[string]*resolver.NameserverResponse{ns: resp}, time.Now(),
+	)
+
+	got := hs.RecordsByNameserver[ns]
+	if got.Status != failed().Status || got.Error != reason {
+		t.Errorf(
+			"saved status %q, error %q; want status %q, error %q",
+			got.Status, got.Error, failed().Status, reason,
+		)
+	}
+}
+
+// TestPortStateWhenNoNameserverAnswered runs the port checks on
+// hostname state built here, which gives the name no address. The port
+// state saved for its old address is kept only when the name is a
+// configured hostname or domain and none of its nameservers answered.
+func TestPortStateWhenNoNameserverAnswered(t *testing.T) {
+	t.Parallel()
+
+	noneAnswered := saved(map[string]*state.NameserverRecordState{
+		nsA: failed(), nsB: failed(),
+	})
+	oneAnsweredNoAddress := saved(map[string]*state.NameserverRecordState{
+		nsA: answered(map[string][]string{}), nsB: failed(),
+	})
+
+	configured := []string{host}
+
+	tests := []struct {
+		name      string
+		hostname  *state.HostnameState
+		hostnames []string
+		domains   []string
+		wantKept  bool
+	}{
+		{"no nameserver answered", noneAnswered, configured, nil, true},
+		{
+			"no nameserver answered, configured as a domain",
+			noneAnswered, nil, configured, true,
+		},
+		{
+			"one answered with no address",
+			oneAnsweredNoAddress, configured, nil, false,
+		},
+		{"no nameserver answered, not configured", noneAnswered, nil, nil, false},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			t.Parallel()
+
+			cfg := defaultTestConfig(t)
+			cfg.Hostnames = tt.hostnames
+			cfg.Domains = tt.domains
+
+			// The port checks read the saved hostname state and look
+			// nothing up, so the watcher has no resolver.
+			deps := newTestDeps(t, cfg)
+			w := watcher.NewForTest(
+				cfg, deps.state, nil,
+				deps.portChecker, deps.tlsChecker, deps.notifier,
+			)
+
+			key := ip1 + ":443"
+
+			deps.state.SetHostnameState(host, tt.hostname)
+			deps.state.SetPortState(key, &state.PortState{
+				Open: true, Hostnames: []string{host},
+			})
+
+			w.CheckAllPorts(t.Context())
+
+			_, kept := deps.state.GetPortState(key)
+			if kept != tt.wantKept {
+				t.Errorf("port state %s kept: %v, want %v", key, kept, tt.wantKept)
+			}
+		})
+	}
+}
+
+// TestPortStateWhenNoNameserverAnsweredAndOtherNameMovesAway saves the
+// port state of an address two configured hostnames resolve to. While
+// none of the first one's nameservers answer, the port checks run with
+// the other one still at that address, then after it moved away; the
+// port state is kept both times.
+func TestPortStateWhenNoNameserverAnsweredAndOtherNameMovesAway(
+	t *testing.T,
+) {
+	t.Parallel()
+
+	const other = "mail.example.net"
+
+	cfg := defaultTestConfig(t)
+	cfg.Hostnames = []string{host, other}
+
+	// The port checks read the saved hostname state and look nothing
+	// up, so the watcher has no resolver.
+	deps := newTestDeps(t, cfg)
+	w := watcher.NewForTest(
+		cfg, deps.state, nil,
+		deps.portChecker, deps.tlsChecker, deps.notifier,
+	)
+
+	key := ip1 + ":443"
+
+	deps.state.SetPortState(key, &state.PortState{
+		Open: true, Hostnames: []string{host, other},
+	})
+	deps.state.SetHostnameState(host, saved(
+		map[string]*state.NameserverRecordState{nsA: failed(), nsB: failed()},
+	))
+
+	for _, otherIP := range []string{ip1, ip2} {
+		deps.state.SetHostnameState(other, saved(
+			map[string]*state.NameserverRecordState{
+				nsA: answered(map[string][]string{"A": {otherIP}}),
+			},
+		))
+
+		w.CheckAllPorts(t.Context())
+
+		if _, kept := deps.state.GetPortState(key); !kept {
+			t.Fatalf("port state %s removed with %s at %s", key, other, otherIP)
+		}
+	}
+}

internal/watcher/watcher.go

@@ -4,15 +4,16 @@ import (
 	"context"
 	"fmt"
 	"log/slog"
+	"slices"
 	"sort"
 	"strings"
-	"sync"
 	"time"
 
 	"go.uber.org/fx"
 
 	"sneak.berlin/go/dnswatcher/internal/config"
 	"sneak.berlin/go/dnswatcher/internal/logger"
+	"sneak.berlin/go/dnswatcher/internal/resolver"
 	"sneak.berlin/go/dnswatcher/internal/state"
 	"sneak.berlin/go/dnswatcher/internal/tlscheck"
 )
@@ -55,9 +56,8 @@ type Watcher struct {
 	tlsCheck  TLSChecker
 	notify    Notifier
 	cancel    context.CancelFunc
+	done      chan struct{} // closed when Run returns
 	firstRun  bool
-	expiryNotifiedMu sync.Mutex
-	expiryNotified   map[string]time.Time
 }
 
 // New creates a new Watcher instance wired into the fx lifecycle.
@@ -74,57 +74,50 @@ func New(
 		tlsCheck:  params.TLSCheck,
 		notify:    params.Notify,
 		firstRun:  true,
-		expiryNotified: make(map[string]time.Time),
 	}
 
 	lifecycle.Append(fx.Hook{
-		OnStart: func(_ context.Context) error {
+		OnStart: func(startCtx context.Context) error {
-			// Use context.Background() — the fx startup context
+			// The fx startup context expires after startup
-			// expires after startup completes, so deriving from it
+			// completes, so the watcher's context drops its
-			// would cancel the watcher immediately. The watcher's
+			// cancellation. The watcher's lifetime is controlled
-			// lifetime is controlled by w.cancel in OnStop.
+			// by w.cancel in OnStop.
-			ctx, cancel := context.WithCancel(context.Background())
+			ctx, cancel := context.WithCancel(
+				context.WithoutCancel(startCtx),
+			)
 			w.cancel = cancel
+			w.done = make(chan struct{})
 
-			go w.Run(ctx) //nolint:contextcheck // intentionally not derived from startCtx
+			go func() {
+				defer close(w.done)
+
+				w.Run(ctx)
+			}()
 
 			return nil
 		},
-		OnStop: func(_ context.Context) error {
+		OnStop: func(ctx context.Context) error {
-			if w.cancel != nil {
 			w.cancel()
-			}
 
+			// Run saves state as it returns. Waiting for it here
+			// means the save is done before shutdown goes on.
+			select {
+			case <-w.done:
 				return nil
+			case <-ctx.Done():
+				return fmt.Errorf(
+					"waiting for the watcher to stop: %w",
+					ctx.Err(),
+				)
+			}
 		},
 	})
 
 	return w, nil
 }
 
-// NewForTest creates a Watcher without fx for unit testing.
+// Run starts the monitoring loop with periodic scheduling. When ctx
-func NewForTest(
+// is cancelled, it saves state and returns.
-	cfg *config.Config,
-	st *state.State,
-	res DNSResolver,
-	pc PortChecker,
-	tc TLSChecker,
-	n Notifier,
-) *Watcher {
-	return &Watcher{
-		log:            slog.Default(),
-		config:         cfg,
-		state:          st,
-		resolver:       res,
-		portCheck:      pc,
-		tlsCheck:       tc,
-		notify:         n,
-		firstRun:       true,
-		expiryNotified: make(map[string]time.Time),
-	}
-}
-
-// Run starts the monitoring loop with periodic scheduling.
 func (w *Watcher) Run(ctx context.Context) {
 	w.log.Info(
 		"watcher starting",
@@ -146,6 +139,7 @@ func (w *Watcher) Run(ctx context.Context) {
 	for {
 		select {
 		case <-ctx.Done():
+			w.saveState()
 			w.log.Info("watcher stopped")
 
 			return
@@ -237,19 +231,31 @@ func (w *Watcher) checkDomain(
 	now := time.Now().UTC()
 
 	prev, hasPrev := w.state.GetDomainState(domain)
+
+	var prevAddresses map[string][]string
+	if hasPrev {
+		prevAddresses = prev.NameserverAddresses
+	}
+
+	addresses := w.resolveNameserverAddresses(
+		ctx, nameservers, prevAddresses,
+	)
+
 	if hasPrev && !w.firstRun {
 		w.detectNSChanges(ctx, domain, prev.Nameservers, nameservers)
+		w.detectNSAddressChanges(ctx, domain, prevAddresses, addresses)
 	}
 
 	w.state.SetDomainState(domain, &state.DomainState{
 		Nameservers:         nameservers,
+		NameserverAddresses: addresses,
 		LastChecked:         now,
 	})
 
 	// Also look up A/AAAA records for the apex domain so that
 	// port and TLS checks (which read HostnameState) can find
 	// the domain's IP addresses.
-	records, err := w.resolver.LookupAllRecords(ctx, domain)
+	results, err := w.resolver.LookupAllRecords(ctx, domain)
 	if err != nil {
 		w.log.Error(
 			"failed to lookup records for domain",
@@ -260,12 +266,13 @@ func (w *Watcher) checkDomain(
 		return
 	}
 
+	newState := buildHostnameState(results, now)
+
 	prevHS, hasPrevHS := w.state.GetHostnameState(domain)
 	if hasPrevHS && !w.firstRun {
-		w.detectHostnameChanges(ctx, domain, prevHS, records)
+		w.detectHostnameChanges(ctx, domain, prevHS, newState)
 	}
 
-	newState := buildHostnameState(records, now)
 	w.state.SetHostnameState(domain, newState)
 }
 
@@ -310,11 +317,77 @@ func (w *Watcher) detectNSChanges(
 	)
 }
 
+// resolveNameserverAddresses returns the sorted addresses each
+// nameserver's name resolves to. A nameserver whose lookup fails, as it
+// does when no nameserver of the name's zone answers, or finds no
+// address keeps its addresses from prev and is not an address change.
+func (w *Watcher) resolveNameserverAddresses(
+	ctx context.Context,
+	nameservers []string,
+	prev map[string][]string,
+) map[string][]string {
+	addresses := make(map[string][]string, len(nameservers))
+
+	for _, ns := range nameservers {
+		ips, err := w.resolver.ResolveIPAddresses(ctx, ns)
+		if err == nil && len(ips) > 0 {
+			sort.Strings(ips)
+			addresses[ns] = ips
+
+			continue
+		}
+
+		w.log.Error(
+			"no addresses found for nameserver",
+			"nameserver", ns,
+			"error", err,
+		)
+
+		if prevIPs, ok := prev[ns]; ok {
+			addresses[ns] = prevIPs
+		}
+	}
+
+	return addresses
+}
+
+// detectNSAddressChanges notifies when a nameserver in both checks
+// resolves to different addresses. A nameserver added or removed is
+// reported by detectNSChanges alone, and one with no addresses saved by
+// the previous check, as in a state file from before they were saved,
+// is not compared.
+func (w *Watcher) detectNSAddressChanges(
+	ctx context.Context,
+	domain string,
+	prev, current map[string][]string,
+) {
+	for ns, cur := range current {
+		old, ok := prev[ns]
+		if !ok || sliceEqual(old, cur) {
+			continue
+		}
+
+		msg := fmt.Sprintf(
+			"Domain: %s\nNameserver: %s\nOld: %s\nNew: %s",
+			domain, ns,
+			strings.Join(old, ", "),
+			strings.Join(cur, ", "),
+		)
+
+		w.notify.SendNotification(
+			ctx,
+			"NS Address Change: "+domain,
+			msg,
+			"warning",
+		)
+	}
+}
+
 func (w *Watcher) checkHostname(
 	ctx context.Context,
 	hostname string,
 ) {
-	records, err := w.resolver.LookupAllRecords(ctx, hostname)
+	results, err := w.resolver.LookupAllRecords(ctx, hostname)
 	if err != nil {
 		w.log.Error(
 			"failed to lookup records",
@@ -325,19 +398,22 @@ func (w *Watcher) checkHostname(
 		return
 	}
 
-	now := time.Now().UTC()
+	newState := buildHostnameState(results, time.Now().UTC())
-	prev, hasPrev := w.state.GetHostnameState(hostname)
 
+	prev, hasPrev := w.state.GetHostnameState(hostname)
 	if hasPrev && !w.firstRun {
-		w.detectHostnameChanges(ctx, hostname, prev, records)
+		w.detectHostnameChanges(ctx, hostname, prev, newState)
 	}
 
-	newState := buildHostnameState(records, now)
 	w.state.SetHostnameState(hostname, newState)
 }
 
+// buildHostnameState saves each nameserver's response. A nameserver
+// that answered, even with NXDOMAIN or no records, is saved as ok; one
+// that timed out or failed is saved as error with the reason, and its
+// empty record set is not an answer.
 func buildHostnameState(
-	records map[string]map[string][]string,
+	results map[string]*resolver.NameserverResponse,
 	now time.Time,
 ) *state.HostnameState {
 	hs := &state.HostnameState{
@@ -347,12 +423,20 @@ func buildHostnameState(
 		LastChecked: now,
 	}
 
-	for ns, recs := range records {
+	for ns, resp := range results {
-		hs.RecordsByNameserver[ns] = &state.NameserverRecordState{
+		nsState := &state.NameserverRecordState{
-			Records:     recs,
+			Records:     resp.Records,
 			Status:      statusOK,
 			LastChecked: now,
 		}
+
+		if resp.Status == resolver.StatusTimeout ||
+			resp.Status == resolver.StatusError {
+			nsState.Status = statusError
+			nsState.Error = resp.Error
+		}
+
+		hs.RecordsByNameserver[ns] = nsState
 	}
 
 	return hs
@@ -361,27 +445,29 @@ func buildHostnameState(
 func (w *Watcher) detectHostnameChanges(
 	ctx context.Context,
 	hostname string,
-	prev *state.HostnameState,
+	prev, current *state.HostnameState,
-	current map[string]map[string][]string,
 ) {
 	w.detectRecordChanges(ctx, hostname, prev, current)
 	w.detectNSDisappearances(ctx, hostname, prev, current)
-	w.detectInconsistencies(ctx, hostname, current)
+	w.detectNSFailures(ctx, hostname, prev, current)
+	w.detectInconsistencies(ctx, hostname, prev, current)
 }
 
+// detectRecordChanges compares each nameserver's records with those of
+// the previous check. Only answers are compared: a nameserver that
+// failed on either check has no records to compare.
 func (w *Watcher) detectRecordChanges(
 	ctx context.Context,
 	hostname string,
-	prev *state.HostnameState,
+	prev, current *state.HostnameState,
-	current map[string]map[string][]string,
 ) {
-	for ns, recs := range current {
+	for ns, cur := range current.RecordsByNameserver {
 		prevNS, ok := prev.RecordsByNameserver[ns]
-		if !ok {
+		if !ok || prevNS.Status != statusOK || cur.Status != statusOK {
 			continue
 		}
 
-		if recordsEqual(prevNS.Records, recs) {
+		if recordsEqual(prevNS.Records, cur.Records) {
 			continue
 		}
 
@@ -389,7 +475,7 @@ func (w *Watcher) detectRecordChanges(
 			"Hostname: %s\nNameserver: %s\n"+
 				"Old: %v\nNew: %v",
 			hostname, ns,
-			prevNS.Records, recs,
+			prevNS.Records, cur.Records,
 		)
 
 		w.notify.SendNotification(
@@ -404,11 +490,10 @@ func (w *Watcher) detectRecordChanges(
 func (w *Watcher) detectNSDisappearances(
 	ctx context.Context,
 	hostname string,
-	prev *state.HostnameState,
+	prev, current *state.HostnameState,
-	current map[string]map[string][]string,
 ) {
 	for ns, prevNS := range prev.RecordsByNameserver {
-		if _, ok := current[ns]; ok || prevNS.Status != statusOK {
+		if _, ok := current.RecordsByNameserver[ns]; ok || prevNS.Status != statusOK {
 			continue
 		}
 
@@ -424,13 +509,36 @@ func (w *Watcher) detectNSDisappearances(
 			"error",
 		)
 	}
+}
 
-	for ns := range current {
+// detectNSFailures notifies when a nameserver that answered on the
+// previous check fails, and when one that failed answers again. A
+// nameserver missing from the previous check is not compared.
+func (w *Watcher) detectNSFailures(
+	ctx context.Context,
+	hostname string,
+	prev, current *state.HostnameState,
+) {
+	for ns, cur := range current.RecordsByNameserver {
 		prevNS, ok := prev.RecordsByNameserver[ns]
-		if !ok || prevNS.Status != statusError {
+		if !ok {
 			continue
 		}
 
+		switch {
+		case prevNS.Status == statusOK && cur.Status == statusError:
+			msg := fmt.Sprintf(
+				"Hostname: %s\nNameserver: %s\nError: %s",
+				hostname, ns, cur.Error,
+			)
+
+			w.notify.SendNotification(
+				ctx,
+				"NS Failure: "+hostname,
+				msg,
+				"error",
+			)
+		case prevNS.Status == statusError && cur.Status == statusOK:
 			msg := fmt.Sprintf(
 				"Hostname: %s\nNameserver: %s recovered",
 				hostname, ns,
@@ -443,33 +551,22 @@ func (w *Watcher) detectNSDisappearances(
 				"success",
 			)
 		}
+	}
 }
 
 func (w *Watcher) detectInconsistencies(
 	ctx context.Context,
 	hostname string,
-	current map[string]map[string][]string,
+	prev, current *state.HostnameState,
 ) {
-	nameservers := make([]string, 0, len(current))
+	for _, pair := range newlyDisagreeingPairs(prev, current) {
-	for ns := range current {
+		ns1, ns2 := pair[0], pair[1]
-		nameservers = append(nameservers, ns)
-	}
-
-	sort.Strings(nameservers)
-
-	for i := range len(nameservers) - 1 {
-		ns1 := nameservers[i]
-		ns2 := nameservers[i+1]
-
-		if recordsEqual(current[ns1], current[ns2]) {
-			continue
-		}
 
 		msg := fmt.Sprintf(
 			"Hostname: %s\n%s: %v\n%s: %v",
 			hostname,
-			ns1, current[ns1],
+			ns1, current.RecordsByNameserver[ns1].Records,
-			ns2, current[ns2],
+			ns2, current.RecordsByNameserver[ns2].Records,
 		)
 
 		w.notify.SendNotification(
@@ -481,6 +578,51 @@ func (w *Watcher) detectInconsistencies(
 	}
 }
 
+// newlyDisagreeingPairs returns every pair of nameservers that answered
+// in current and whose records differ there, in sorted order of name,
+// except pairs where both nameservers answered in prev and already
+// differed there. A nameserver missing from prev, or that failed there,
+// is paired with every nameserver it differs from. A nameserver that
+// failed in current has no records to compare and is in no pair.
+func newlyDisagreeingPairs(
+	prev, current *state.HostnameState,
+) [][2]string {
+	nameservers := make([]string, 0, len(current.RecordsByNameserver))
+	for ns, cur := range current.RecordsByNameserver {
+		if cur.Status == statusOK {
+			nameservers = append(nameservers, ns)
+		}
+	}
+
+	sort.Strings(nameservers)
+
+	var pairs [][2]string
+
+	for i, ns1 := range nameservers {
+		for _, ns2 := range nameservers[i+1:] {
+			if recordsEqual(
+				current.RecordsByNameserver[ns1].Records,
+				current.RecordsByNameserver[ns2].Records,
+			) {
+				continue
+			}
+
+			prev1, ok1 := prev.RecordsByNameserver[ns1]
+			prev2, ok2 := prev.RecordsByNameserver[ns2]
+
+			if ok1 && ok2 &&
+				prev1.Status == statusOK && prev2.Status == statusOK &&
+				!recordsEqual(prev1.Records, prev2.Records) {
+				continue
+			}
+
+			pairs = append(pairs, [2]string{ns1, ns2})
+		}
+	}
+
+	return pairs
+}
+
 func (w *Watcher) checkAllPorts(ctx context.Context) {
 	// Phase 1: Build current IP:port → hostname associations
 	// from fresh DNS data.
@@ -563,15 +705,46 @@ func parsePortKey(key string) (string, int) {
 }
 
 // cleanupStalePorts removes port state entries that are no
-// longer referenced by any hostname in the current DNS data.
+// longer referenced by any hostname in the current DNS data. An
+// entry saved for a configured name none of whose nameservers
+// answered is kept: that name's addresses are not known, not gone.
 func (w *Watcher) cleanupStalePorts(
 	currentAssociations map[string][]string,
 ) {
 	for _, key := range w.state.GetAllPortKeys() {
-		if _, exists := currentAssociations[key]; !exists {
+		if _, exists := currentAssociations[key]; exists {
+			continue
+		}
+
+		ps, ok := w.state.GetPortState(key)
+		if ok && slices.ContainsFunc(ps.Hostnames, w.noNameserverAnswered) {
+			continue
+		}
+
 		w.state.DeletePortState(key)
 	}
+}
+
+// noNameserverAnswered reports whether name is a configured domain or
+// hostname and none of its nameservers answered on its last check.
+func (w *Watcher) noNameserverAnswered(name string) bool {
+	if !slices.Contains(w.config.Hostnames, name) &&
+		!slices.Contains(w.config.Domains, name) {
+		return false
 	}
+
+	hs, ok := w.state.GetHostnameState(name)
+	if !ok {
+		return false
+	}
+
+	for _, nsState := range hs.RecordsByNameserver {
+		if nsState.Status == statusOK {
+			return false
+		}
+	}
+
+	return true
 }
 
 func (w *Watcher) collectIPs(hostname string) []string {
@@ -609,6 +782,13 @@ func (w *Watcher) checkSinglePort(
 	hostnames []string,
 ) {
 	result, err := w.portCheck.CheckPort(ctx, ip, port)
+
+	// A check the context cut short says nothing about the port, so it
+	// is neither saved nor notified.
+	if ctx.Err() != nil {
+		return
+	}
+
 	if err != nil {
 		w.log.Error(
 			"port check failed",
@@ -643,9 +823,24 @@ func (w *Watcher) checkSinglePort(
 		)
 	}
 
+	// A configured name on the saved list none of whose nameservers
+	// answered stays on it, so the entry is kept when the other names
+	// stop resolving to this address.
+	savedHostnames := slices.Clone(hostnames)
+
+	if hasPrev {
+		for _, name := range prev.Hostnames {
+			if !slices.Contains(hostnames, name) && w.noNameserverAnswered(name) {
+				savedHostnames = append(savedHostnames, name)
+			}
+		}
+
+		sort.Strings(savedHostnames)
+	}
+
 	w.state.SetPortState(key, &state.PortState{
 		Open:        result.Open,
-		Hostnames:   hostnames,
+		Hostnames:   savedHostnames,
 		LastChecked: now,
 	})
 }
@@ -684,6 +879,13 @@ func (w *Watcher) checkTLSCert(
 	hostname string,
 ) {
 	cert, err := w.tlsCheck.CheckCertificate(ctx, ip, hostname)
+
+	// A check the context cut short says nothing about the certificate,
+	// so it is neither saved nor notified.
+	if ctx.Err() != nil {
+		return
+	}
+
 	certKey := fmt.Sprintf("%s:%d:%s", ip, tlsPort, hostname)
 	now := time.Now().UTC()
 	prev, hasPrev := w.state.GetCertificateState(certKey)
@@ -822,22 +1024,6 @@ func (w *Watcher) checkTLSExpiry(
 		return
 	}
 
-	// Deduplicate expiry warnings: don't re-notify for the same
-	// hostname within the TLS check interval.
-	dedupKey := fmt.Sprintf("expiry:%s:%s", hostname, ip)
-
-	w.expiryNotifiedMu.Lock()
-
-	lastNotified, seen := w.expiryNotified[dedupKey]
-	if seen && time.Since(lastNotified) < w.config.TLSInterval {
-		w.expiryNotifiedMu.Unlock()
-
-		return
-	}
-
-	w.expiryNotified[dedupKey] = time.Now()
-	w.expiryNotifiedMu.Unlock()
-
 	msg := fmt.Sprintf(
 		"Host: %s\nIP: %s\nCN: %s\n"+
 			"Expires: %s (%.0f days)",

package.json

@@ -0,0 +1,9 @@
+{
+    "name": "dnswatcher-tooling",
+    "version": "0.0.0",
+    "private": true,
+    "description": "Pins the prettier that script/fmt and script/fmt-check run against this repo's markdown. Not a JavaScript project; nothing here is imported, published, or shipped.",
+    "devDependencies": {
+        "prettier": "3.9.6"
+    }
+}

script/bootstrap

@@ -3,20 +3,16 @@
 # this repo. Idempotent: every install is guarded by a check so already
 # installed tools are skipped. Base tooling comes from nix, apt, brew,
 # or apk (detected in that order); assumes nothing is present.
-# goimports is installed via `go install` at a pinned commit (never
+# goimports is not installed here: script/fmt and script/fmt-check-go
-# "latest") because script/fmt runs it on the host; script/fmt-check
+# run it with `go run` at a pinned commit.
-# does not (it runs gofmt only).
 # The linter is NOT installed here: golangci-lint runs via docker only
 # (script/lint), pinned by image digest, so its only prerequisite is a
-# working docker.
+# working docker. Nor is prettier: script/fmt and
+# script/fmt-check-markdown run it in a container from Dockerfile.fmt.
 set -eu
 
 ROOT="$(cd "$(dirname "$0")/.." && pwd -P)"
 
-# Pinned version, 2026-08-07 (same pin as the Dockerfile)
-# goimports v0.42.0
-GOIMPORTS_REF="golang.org/x/tools/cmd/goimports@009367f5c17a8d4c45a961a3a509277190a9a6f0"
-
 PKGMGR=""
 SUDO=""
 APT_UPDATED=""
@@ -71,15 +67,12 @@ main() {
     if missing make; then pkg_install gnumake make make make; fi
     if missing go; then pkg_install go golang go go; fi
 
-    # Format tools, pinned via go install (installs into
+    # Linting and the markdown formatter run via docker only. Warn,
-    # "$(go env GOPATH)/bin"; ensure that is on your PATH).
+    # don't fail: building and testing work without it.
-    if missing goimports; then go install "$GOIMPORTS_REF"; fi
-
-    # Linting runs via docker only (script/lint). Warn, don't fail:
-    # everything except `make lint` works without it.
     if missing docker; then
         echo "bootstrap: WARNING: docker not found; install it to" \
-            "run make lint and make docker." >&2
+            "run make lint, make fmt, make fmt-check, make check" \
+            "and make docker." >&2
     fi
 
     go mod download

script/cibuild

@@ -1,14 +1,23 @@
 #!/bin/sh
 # script/cibuild: run the CI build. The Dockerfile's lint stage runs
-# make fmt-check and golangci-lint; its builder stage runs make test
+# the Go half of make fmt-check and golangci-lint; its builder stage
-# and make build. A successful build implies all of those passed.
+# runs make test and make build. The markdown half of make fmt-check
+# runs after that build, as its own build of Dockerfile.fmt, because
+# there is no docker inside a docker build.
+#
+# --no-cache-filter=lint,builder runs both stages on every invocation;
+# otherwise an unchanged tree is served from the layer cache and passes
+# without linting or querying live DNS. script/fmt-check-markdown busts
+# its own cache the same way.
 set -eu
 
-ROOT="$(cd "$(dirname "$0")/.." && pwd -P)"
+SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd -P)"
+ROOT="$(cd "$SCRIPT_DIR/.." && pwd -P)"
 
 main() {
     cd "$ROOT"
-    docker build .
+    docker build --no-cache-filter=lint,builder .
+    "$SCRIPT_DIR/fmt-check-markdown"
 }
 
 main "$@"

script/docker

@@ -1,6 +1,10 @@
 #!/bin/sh
 # script/docker: build the Docker image tagged with the project name.
-# Identical in all repos; the tag comes from script/projectname.
+# The tag comes from script/projectname.
+#
+# --no-cache-filter=lint,builder runs the lint stage and the builder
+# stage (make test) on every invocation; otherwise an unchanged tree is
+# served from the layer cache without linting or querying live DNS.
 set -eu
 
 SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd -P)"
@@ -8,7 +12,15 @@ ROOT="$(cd "$SCRIPT_DIR/.." && pwd -P)"
 
 main() {
     cd "$ROOT"
-    docker build -t "$("$SCRIPT_DIR/projectname")" .
+    # Own line: a failing command substitution inside an argument does
+    # not trip `set -e`, so the inline form degrades silently to an
+    # empty constant. The VERSION build arg takes precedence over what
+    # the build would derive from the .git in its context.
+    version="$(git describe --tags --always --dirty 2>/dev/null || true)"
+    [ -n "$version" ] || version="unknown"
+    docker build --no-cache-filter=lint,builder \
+        --build-arg VERSION="$version" \
+        -t "$("$SCRIPT_DIR/projectname")" .
 }
 
 main "$@"

script/fmt

@@ -1,13 +1,65 @@
 #!/bin/sh
-# script/fmt: format all files (writes).
+# script/fmt: format all files (writes). Go with gofmt and goimports on
+# the host, markdown with the prettier pinned by Dockerfile.fmt.
+#
+# goimports runs with `go run` at a pinned commit, never from PATH, so
+# every machine formats with the same version and nothing installs it.
+#
+# The markdown pass is a `docker build --output type=local` rather than a
+# `docker run -v`, so it needs no bind mount and behaves the same against
+# a remote daemon; the formatted documents come back out of the build and
+# are copied over the tree here.
+#
+# Unlike script/fmt-check-markdown this does not bust the cache: it is
+# not a gate, and any edit to a document changes the COPY layer above the
+# prettier step, so a cached result is a result over this exact tree.
 set -eu
 
 ROOT="$(cd "$(dirname "$0")/.." && pwd -P)"
 
+# goimports v0.42.0, 2026-08-07. Must match script/fmt-check-go.
+GOIMPORTS_REF="golang.org/x/tools/cmd/goimports@009367f5c17a8d4c45a961a3a509277190a9a6f0"
+
+# Must match the export stage name in Dockerfile.fmt.
+stage=fmt-out
+
+die() {
+    echo "script/fmt: $*" >&2
+    exit 1
+}
+
 main() {
     cd "$ROOT"
     gofmt -s -w .
-    goimports -w .
+    go run "$GOIMPORTS_REF" -w .
+
+    tmp="$(mktemp -d "${TMPDIR:-/tmp}/dnswatcher-fmt.XXXXXX")"
+    trap 'rm -rf "$tmp"' EXIT INT TERM
+
+    docker build \
+        --target "$stage" \
+        --output "type=local,dest=$tmp/out" \
+        -f Dockerfile.fmt .
+
+    # An empty export means prettier was handed nothing, which must not
+    # read as "already formatted".
+    (cd "$tmp/out" && find . -type f -name '*.md') |
+        sed 's|^\./||' | LC_ALL=C sort >"$tmp/files"
+
+    [ -s "$tmp/files" ] ||
+        die "the formatting build produced no markdown; the build" \
+            "context reached prettier empty"
+
+    # Copied only where the bytes differ, so an already-formatted tree
+    # keeps its timestamps and says nothing.
+    while IFS= read -r f; do
+        [ -n "$f" ] || continue
+        if [ -f "$f" ] && cmp -s "$tmp/out/$f" "$f"; then
+            continue
+        fi
+        cp "$tmp/out/$f" "$f"
+        echo "prettier: reformatted $f"
+    done <"$tmp/files"
 }
 
 main "$@"

script/fmt-check

@@ -1,18 +1,14 @@
 #!/bin/sh
-# script/fmt-check: check formatting (read-only). Same scope as
+# script/fmt-check: check formatting (read-only). Same tools and scope
-# script/fmt, but fails instead of writing.
+# as script/fmt, but fails instead of writing: the Go on the host, the
+# markdown with prettier in a container.
 set -eu
 
-ROOT="$(cd "$(dirname "$0")/.." && pwd -P)"
+SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd -P)"
 
 main() {
-    cd "$ROOT"
+    "$SCRIPT_DIR/fmt-check-go"
-    files="$(gofmt -l .)"
+    "$SCRIPT_DIR/fmt-check-markdown"
-    if [ -n "$files" ]; then
-        echo "gofmt: files not formatted:" >&2
-        echo "$files" >&2
-        exit 1
-    fi
 }
 
 main "$@"

script/fmt-check-go

@@ -0,0 +1,31 @@
+#!/bin/sh
+# script/fmt-check-go: fail unless every Go source is formatted the way
+# script/fmt would leave it, and name the files that are not. Read-only.
+#
+# Its own script because the Dockerfile's lint stage runs this half
+# alone: there is no docker inside a docker build to run the markdown
+# half in.
+set -eu
+
+ROOT="$(cd "$(dirname "$0")/.." && pwd -P)"
+
+# goimports v0.42.0, 2026-08-07. Must match script/fmt.
+GOIMPORTS_REF="golang.org/x/tools/cmd/goimports@009367f5c17a8d4c45a961a3a509277190a9a6f0"
+
+main() {
+    cd "$ROOT"
+    files="$(gofmt -s -l .)"
+    if [ -n "$files" ]; then
+        echo "gofmt: files not formatted:" >&2
+        echo "$files" >&2
+        exit 1
+    fi
+    files="$(go run "$GOIMPORTS_REF" -l .)"
+    if [ -n "$files" ]; then
+        echo "goimports: files not formatted:" >&2
+        echo "$files" >&2
+        exit 1
+    fi
+}
+
+main "$@"

script/fmt-check-markdown

@@ -0,0 +1,29 @@
+#!/bin/sh
+# script/fmt-check-markdown: fail unless every .md is formatted the way
+# script/fmt would leave it. Read-only.
+#
+# prettier is never installed on the host: it runs in a container built
+# from Dockerfile.fmt, pinned by package.json and yarn.lock.
+# --no-cache-filter is here for the reason script/lint gives: a cached
+# build checks nothing.
+#
+# Its own script because script/cibuild runs this half alone, after the
+# Dockerfile's lint stage has checked the Go.
+set -eu
+
+ROOT="$(cd "$(dirname "$0")/.." && pwd -P)"
+
+# Must match the markdown check stage name in Dockerfile.fmt.
+stage=fmt-check
+
+main() {
+    cd "$ROOT"
+    docker build \
+        --progress=plain \
+        --no-cache-filter="$stage" \
+        --target "$stage" \
+        -f Dockerfile.fmt \
+        .
+}
+
+main "$@"

script/install-precommit

@@ -7,7 +7,20 @@ ROOT="$(cd "$(dirname "$0")/.." && pwd -P)"
 
 main() {
     cd "$ROOT"
-    hook=".git/hooks/pre-commit"
+    # Stop if this directory is not the top of its own git checkout, for
+    # example a copy inside another repository, whose hook must not be
+    # replaced.
+    if [ "$(git rev-parse --show-toplevel)" != "$ROOT" ]; then
+        echo "install-precommit: $ROOT is not the top of a git checkout" >&2
+        exit 1
+    fi
+    # Ask git for the repository's own git directory: .git is a file, not
+    # a directory, in some checkouts (for example a clone made with
+    # --separate-git-dir). core.hooksPath is deliberately not followed, so
+    # the hook is never written outside this repository.
+    hooks="$(git rev-parse --git-common-dir)/hooks"
+    mkdir -p "$hooks"
+    hook="$hooks/pre-commit"
     printf '#!/bin/sh\nset -e\nscript/precommit\n' > "$hook"
     chmod +x "$hook"
     echo "pre-commit hook installed: runs script/precommit"

yarn.lock

@@ -0,0 +1,8 @@
+# THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY.
+# yarn lockfile v1
+
+
+prettier@3.9.6:
+  version "3.9.6"
+  resolved "https://registry.yarnpkg.com/prettier/-/prettier-3.9.6.tgz#b3ea5146515d40fc53f18aa63f74dfab1e10dbf6"
+  integrity sha512-OpN0zzVdiaiAhxpuuj5efpIS4sY9j7bY6uR5mnj5yPzGkdkjNKSJeUThPb60Jw29QuAZgA4o+/iB49kFiaBX6g==