notify: a failed Mattermost delivery's error names Mattermost (closes #227)

Mattermost is sent by the Slack sender, which wrapped every HTTP error
status in ErrSlackFailed, so a Mattermost endpoint answering 503 was
logged as "slack notification failed". The sender now takes the error
to wrap: the Slack endpoint passes ErrSlackFailed and the Mattermost
endpoint passes ErrMattermostFailed, which was defined but unused.

A new delivery test sets both endpoints to a stand-in server answering
503 and checks the error logged for each names its own endpoint.

Model: opus-5-5

.dockerignore

@@ -0,0 +1,9 @@
+# .git is sent, without its config: the builder stage derives the version it
+# stamps into the binary from it, and `git describe` does not need the config,
+# which can hold a credential (a password in the remote URL, a CI token). No
+# tracked file may be listed here: git in the build would see it as deleted
+# and mark the version -dirty, and an excluded .md would silently drop out of
+# the prettier check in Dockerfile.fmt.
+.git/config
+bin/
+node_modules/

.editorconfig

@@ -0,0 +1,12 @@
+root = true
+
+[*]
+indent_style = space
+indent_size = 4
+end_of_line = lf
+charset = utf-8
+trim_trailing_whitespace = true
+insert_final_newline = true
+
+[Makefile]
+indent_style = tab

.gitea/workflows/check.yml

@@ -1,26 +1,17 @@
-name: Check
-
-on:
-  push:
-    branches: [main]
-  pull_request:
-    branches: [main]
-
+name: check
+on: [push]
+# A new push to a branch cancels that branch's older run, queued or running;
+# runs on other branches, `next` and `main` among them, are left alone.
+concurrency:
+    group: ${{ github.workflow }}-${{ github.ref }}
+    cancel-in-progress: true
 jobs:
-  check:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
-
-      - uses: actions/setup-go@40f1582b2485089dde7abd97c1529aa768e1baff # v5
-        with:
-          go-version-file: go.mod
-
-      - name: Install golangci-lint
-        run: go install github.com/golangci/golangci-lint/v2/cmd/golangci-lint@5d1e709b7be35cb2025444e19de266b056b7b7ee # v2.10.1
-
-      - name: Install goimports
-        run: go install golang.org/x/tools/cmd/goimports@009367f5c17a8d4c45a961a3a509277190a9a6f0 # v0.42.0
-
-      - name: Run make check
-        run: make check
+    check:
+        runs-on: ubuntu-latest
+        steps:
+            # actions/checkout v4.2.2, 2026-02-28
+            - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683
+              # script/cibuild needs no token, so none is left in .git/config.
+              with:
+                  persist-credentials: false
+            - run: script/cibuild

.gitignore

@@ -1,4 +1,5 @@
 bin/
+node_modules/
 vendor/
 data/
 .env

.golangci.yml

@@ -1,32 +1,102 @@
 version: "2"
 
+# Config schema uses the golangci-lint v2 layout (settings live under
+# linters.settings, not top-level linters-settings) so that the
+# thresholds below are actually applied by golangci-lint >= v2.
+
 run:
   timeout: 5m
   modules-download-mode: readonly
 
 linters:
   default: all
+  enable:
+    # Successor to the deprecated gomodguard. Named explicitly, rather than
+    # left to `default: all`, because it carries the module policy below.
+    - gomodguard_v2
   disable:
     # Genuinely incompatible with project patterns
     - exhaustruct      # Requires all struct fields
-    - depguard         # Dependency allow/block lists
     - godot            # Requires comments to end with periods
-    - wsl              # Deprecated, replaced by wsl_v5
     - wrapcheck        # Too verbose for internal packages
     - varnamelen       # Short names like db, id are idiomatic Go
-
-linters-settings:
-  lll:
-    line-length: 88
-  funlen:
-    lines: 80
-    statements: 50
-  cyclop:
-    max-complexity: 15
-  dupl:
-    threshold: 100
+    # Deprecated: the warning is attached to the old name, so it is
+    # silenced by disabling that name, not by enabling the successor.
+    - wsl              # Deprecated, replaced by wsl_v5
+    - gomodguard       # Deprecated, replaced by gomodguard_v2
+  settings:
+    lll:
+      line-length: 88
+    funlen:
+      lines: 80
+      statements: 50
+    cyclop:
+      max-complexity: 15
+    dupl:
+      threshold: 100
+    depguard:
+      # Test-support code must not be compiled into the shipped binary. A
+      # test-support package exists to hand a test privileges the program
+      # itself must never have, so a file that is not a test must not import
+      # one. Test files, and the files inside a package whose directory name
+      # ends in `test`, are where that code belongs, and are exempt.
+      #
+      # The deny list below is the one part of this file a repository is
+      # expected to extend, and the only part it may. depguard matches an
+      # import path against a list of prefixes, so it cannot be told "any path
+      # whose last segment ends in test"; a repository's own test-support
+      # packages have to be named here one at a time, by full import path,
+      # under a module path that differs from repository to repository. Add
+      # them; change nothing else.
+      rules:
+        test-support:
+          list-mode: lax
+          files:
+            - "$all"
+            - "!$test"
+            - "!**/*test/**"
+          deny:
+            - pkg: net/http/httptest
+              desc: >-
+                Test-support code belongs in test files and in packages whose
+                directory name ends in test, not in the shipped binary.
+            - pkg: sneak.berlin/go/dnswatcher/internal/livednstest
+              desc: >-
+                Live-DNS test support belongs in test files and in packages
+                whose directory name ends in test, not in the shipped binary.
+    # Only decisions already recorded in the Go package defaults are
+    # listed here. Every entry matches the module path exactly.
+    gomodguard_v2:
+      blocked:
+        - module: github.com/rs/zerolog
+          recommendations:
+            - log/slog
+          reason: "Structured logging is stdlib log/slog."
+        # One entry per pre-fork module path, because the later releases
+        # are separate paths. A prefix match would be shorter but would
+        # also reach github.com/go-redis/redismock, the test double for
+        # the successor these entries recommend.
+        - module: github.com/go-redis/redis
+          recommendations:
+            - github.com/redis/go-redis/v9
+          reason: "Pre-fork module; use the maintained go-redis v9."
+        - module: github.com/go-redis/redis/v7
+          recommendations:
+            - github.com/redis/go-redis/v9
+          reason: "Pre-fork module; use the maintained go-redis v9."
+        - module: github.com/go-redis/redis/v8
+          recommendations:
+            - github.com/redis/go-redis/v9
+          reason: "Pre-fork module; use the maintained go-redis v9."
+        - module: github.com/sergi/go-diff
+          recommendations:
+            - github.com/aymanbagabas/go-udiff
+          reason: "No unified diff output; use go-udiff."
+        - module: github.com/hexops/gotextdiff
+          recommendations:
+            - github.com/aymanbagabas/go-udiff
+          reason: "Unmaintained fork; use go-udiff."
 
 issues:
-  exclude-use-default: false
   max-issues-per-linter: 0
   max-same-issues: 0

.prettierignore

@@ -0,0 +1,5 @@
+bin/
+data/
+node_modules/
+.claude/
+static/css/tailwind.min.css

.prettierrc

@@ -0,0 +1,4 @@
+{
+    "tabWidth": 4,
+    "proseWrap": "always"
+}

CLAUDE.md

@@ -1,68 +0,0 @@
-# Repository Rules
-
-Last Updated 2026-01-08
-
-These rules MUST be followed at all times, it is very important.
-
-* Never use `git add -A` - add specific changes to a deliberate commit. A
-  commit should contain one change. After each change, make a commit with a
-  good one-line summary.
-
-* NEVER modify the linter config without asking first.
-
-* NEVER modify tests to exclude special cases or otherwise get them to pass
-  without asking first. In almost all cases, the code should be changed,
-  NOT the tests. If you think the test needs to be changed, make your case
-  for that and ask for permission to proceed, then stop. You need explicit
-  user approval to modify existing tests. (You do not need user approval
-  for writing NEW tests.)
-
-* When linting, assume the linter config is CORRECT, and that each item
-  output by the linter is something that legitimately needs fixing in the
-  code.
-
-* When running tests, use `make test`.
-
-* Before commits, run `make check`. This runs `make lint` and `make test`
-  and `make check-fmt`. Any issues discovered MUST be resolved before
-  committing unless explicitly told otherwise.
-
-* When fixing a bug, write a failing test for the bug FIRST. Add
-  appropriate logging to the test to ensure it is written correctly. Commit
-  that. Then go about fixing the bug until the test passes (without
-  modifying the test further). Then commit that.
-
-* When adding a new feature, do the same - implement a test first (TDD). It
-  doesn't have to be super complex. Commit the test, then commit the
-  feature.
-
-* When adding a new feature, use a feature branch. When the feature is
-  completely finished and the code is up to standards (passes `make check`)
-  then and only then can the feature branch be merged into `main` and the
-  branch deleted.
-
-* Write godoc documentation comments for all exported types and functions as
-  you go along.
-
-* ALWAYS be consistent in naming. If you name something one thing in one
-  place, name it the EXACT SAME THING in another place.
-
-* Be descriptive and specific in naming. `wl` is bad;
-  `SourceHostWhitelist` is good. `ConnsPerHost` is bad;
-  `MaxConnectionsPerHost` is good.
-
-* This is not prototype or teaching code - this is designed for production.
-  Any security issues (such as denial of service) or other web
-  vulnerabilities are P1 bugs and must be added to TODO.md at the top.
-
-* As this is production code, no stubbing of implementations unless
-  specifically instructed. We need working implementations.
-
-* Avoid vendoring deps unless specifically instructed to.  NEVER commit
-  the vendor directory, NEVER commit compiled binaries.  If these
-  directories or files exist, add them to .gitignore (and commit the
-  .gitignore) if they are not already in there.  Keep the entire git
-  repository (with history) small - under 20MiB, unless you specifically
-  must commit larger files (e.g. test fixture example media files).  Only
-  OUR source code and immediately supporting files (such as test examples)
-  goes into the repo/history.

Dockerfile

@@ -1,11 +1,12 @@
-# Build stage
-FROM golang:1.25-alpine AS builder
-
-RUN apk add --no-cache git make gcc musl-dev
-
-# Install golangci-lint v2
-RUN go install github.com/golangci/golangci-lint/v2/cmd/golangci-lint@latest
-RUN go install golang.org/x/tools/cmd/goimports@latest
+# Lint stage - fast feedback on lint issues, before the build starts.
+# The linter is invoked directly rather than through `make lint`: that
+# target shells out to `docker build -f Dockerfile.lint`, and there is
+# no docker daemon inside a docker build. For the same reason this stage
+# runs only the Go half of `make fmt-check`; script/cibuild runs the
+# markdown half after this build.
+# script/cibuild and script/docker name this stage in --no-cache-filter.
+# golangci/golangci-lint:v2.12.2 (Debian-based), 2026-08-10
+FROM golangci/golangci-lint:v2.12.2@sha256:5cceeef04e53efe1470638d4b4b4f5ceefd574955ab3941b2d9a68a8c9ad5240 AS lint
 
 WORKDIR /src
 COPY go.mod go.sum ./
@@ -13,26 +14,85 @@ RUN go mod download
 
 COPY . .
 
-# Run all checks - build fails if any check fails
-RUN make check
+RUN script/fmt-check-go
+RUN golangci-lint run --config .golangci.yml ./...
+
+# Build stage
+# script/cibuild and script/docker name this stage in --no-cache-filter.
+# golang 1.25-alpine, 2026-02-28
+FROM golang@sha256:f6751d823c26342f9506c03797d2527668d095b0a15f1862cddb4d927a7a4ced AS builder
+
+RUN apk add --no-cache git make gcc musl-dev binutils-gold
+
+# A build context sent as a tar archive keeps its files' owners, and git
+# refuses to read a checkout owned by another user. Trust this one
+# whoever owns it.
+RUN git config --system --add safe.directory /src
+
+# Force BuildKit to run the lint stage before proceeding
+COPY --from=lint /src/go.sum /dev/null
+
+WORKDIR /src
+COPY go.mod go.sum ./
+RUN go mod download
+
+COPY . .
+
+# Run the tests - build fails if any test fails
+RUN make test
+
+# Version stamped into the binary: the VERSION build arg when one is
+# given and not empty (script/docker passes one), otherwise what
+# `git describe` says of the .git in the build context, so a plain
+# `docker build .` of a clone stamps its tag or short commit. The build
+# arg reaches make through the environment.
+ARG VERSION
+
+# A context that carries .git, as a directory or as a file, must yield a
+# real version: one that is empty, `dev` or `unknown` cannot be traced
+# back to a commit.
+RUN version="$(make version)"; \
+    if [ -e .git ]; then \
+        case "$version" in \
+            "" | dev | unknown) \
+                echo "version is \"$version\" although the build context carries .git" >&2; \
+                exit 1 ;; \
+        esac; \
+    fi
 
-# Build the binary
 RUN make build
 
 # Runtime stage
-FROM alpine:3.21
+# alpine 3.21, 2026-02-28
+FROM alpine@sha256:c3f8e73fdb79deaebaa2037150150191b9dcbfba68b4a46d70103204c53f4709
 
-RUN apk add --no-cache ca-certificates tzdata
+RUN apk add --no-cache ca-certificates tzdata su-exec
 
-WORKDIR /app
+COPY --from=builder /src/bin/dnswatcher /usr/local/bin/dnswatcher
+COPY deploy/docker-entrypoint.sh /usr/local/bin/docker-entrypoint.sh
 
-COPY --from=builder /src/bin/dnswatcher /app/dnswatcher
-
-# Create data directory
-RUN mkdir -p /var/lib/dnswatcher
+# dnswatcher runs as this unprivileged user. The entrypoint creates the
+# data directory and gives it to this user on every start.
+RUN addgroup -S -g 10001 dnswatcher \
+    && adduser -S -G dnswatcher -u 10001 dnswatcher
 
 ENV DNSWATCHER_DATA_DIR=/var/lib/dnswatcher
 
+# Config loading also reads a `.env` file and a file named `dnswatcher`
+# (any config extension, or none) from the working directory. `/` holds
+# neither, so every setting comes from the environment. Do not make the
+# data directory, or the binary's directory, the working directory.
+WORKDIR /
+
+# No USER: the entrypoint must start as root to set up the data
+# directory; it then runs dnswatcher as the dnswatcher user.
+
 EXPOSE 8080
 
-ENTRYPOINT ["/app/dnswatcher"]
+# busybox wget (already in alpine) probes the health endpoint every 10
+# seconds, so the container is healthy well before upaas reads its health
+# 60 seconds after a deploy and fails the deploy unless it is healthy.
+HEALTHCHECK --interval=10s --timeout=5s --start-period=10s --retries=3 \
+    CMD wget -q -O /dev/null "http://127.0.0.1:${PORT:-8080}/.well-known/healthcheck" || exit 1
+
+ENTRYPOINT ["/usr/local/bin/docker-entrypoint.sh"]

Dockerfile.fmt

@@ -0,0 +1,54 @@
+# prettier over the markdown, in a container, so it is never installed
+# on the host. script/fmt-check-markdown builds the fmt-check stage;
+# script/fmt builds fmt-out and takes the formatted files back.
+
+# node:22-bookworm-slim, 2026-09-05
+FROM node:22-bookworm-slim@sha256:83f487e0a63425e5b4d146fb5e5be574bcbe1b7b843d3ebafdd95eaf7767a7e5 AS nodedeps
+
+# prettier lives outside /src so that a `COPY . .` of the repo cannot
+# overwrite it, and so that node_modules never appears in the tree
+# prettier is about to walk.
+WORKDIR /tools
+
+# package.json pins the version and yarn.lock pins the bytes:
+# --frozen-lockfile installs exactly the lockfile's resolution and fails
+# if package.json disagrees with it, so the tool cannot float between
+# runs. yarn is the one in the image above.
+COPY package.json yarn.lock ./
+RUN yarn install --frozen-lockfile --non-interactive --no-progress
+
+ENV PATH="/tools/node_modules/.bin:${PATH}"
+
+WORKDIR /src
+
+# Read-only markdown check. Must match $stage in
+# script/fmt-check-markdown.
+FROM nodedeps AS fmt-check
+
+COPY . .
+
+# --config, not discovery: a .prettierrc that failed to arrive would
+# otherwise leave prettier on its defaults, where proseWrap is "preserve"
+# and every wrap this check exists to enforce passes. Missing the file is
+# a hard error instead. --no-editorconfig so that .prettierrc alone sets
+# the style.
+RUN prettier --config .prettierrc --no-editorconfig --check "**/*.md"
+
+# Write path. Not a check: script/fmt builds this and takes the files.
+FROM nodedeps AS fmt
+
+COPY . .
+
+RUN prettier --config .prettierrc --no-editorconfig --write "**/*.md"
+
+# Only the markdown leaves, with its paths intact, so that the export
+# below cannot put anything else back over the caller's working tree.
+RUN mkdir -p /out && cd /src && \
+    find . -name '*.md' -type f -exec cp --parents '{}' /out/ ';'
+
+# Export target: `docker build --target fmt-out --output type=local`
+# writes /out's tree into a directory on the client, which is how
+# script/fmt gets formatted markdown back without a bind mount.
+# Must match $stage in script/fmt.
+FROM scratch AS fmt-out
+COPY --from=fmt /out/ /

Dockerfile.lint

@@ -0,0 +1,29 @@
+# Lint-only image: used by script/lint. golangci-lint is never run on
+# the host — the repo is COPYed into the build context and the linter
+# runs as a build step, so a successful build IS a clean lint. This
+# also works where the docker daemon is remote and bind mounts are
+# impossible.
+#
+# `golangci-lint config verify` is deliberately NOT run here: it
+# fetches its JSON schema over a live, unpinned HTTPS call, which would
+# make linting network-dependent and defeat hash-pinning. The cost of
+# that: unknown top-level keys in .golangci.yml are silently ignored,
+# so a mistyped or wrong-schema key lints clean while applying nothing.
+#
+# golangci/golangci-lint:v2.12.2 (Debian-based), 2026-08-10
+FROM golangci/golangci-lint:v2.12.2@sha256:5cceeef04e53efe1470638d4b4b4f5ceefd574955ab3941b2d9a68a8c9ad5240 AS deps
+
+WORKDIR /src
+
+# Dependencies first, so this stage stays cached across lint runs.
+COPY go.mod go.sum ./
+RUN go mod download
+
+# Everything below is invalidated on every run by the
+# --no-cache-filter=lint that script/lint passes: caching is explicitly
+# waived for linting, and a cached build lints nothing.
+FROM deps AS lint
+
+COPY . .
+
+RUN golangci-lint run --config .golangci.yml ./...

LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 sneak
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

Makefile

@@ -1,37 +1,54 @@
-.PHONY: all build lint fmt test check clean
+.PHONY: all bootstrap setup build version lint fmt fmt-check test check clean hooks docker
 
 BINARY := dnswatcher
-VERSION := $(shell git describe --tags --always --dirty 2>/dev/null || echo "dev")
-BUILDARCH := $(shell go env GOARCH)
-LDFLAGS := -X main.Version=$(VERSION) -X main.Buildarch=$(BUILDARCH)
+# VERSION given on the command line (`make build VERSION=...`) or in the
+# environment, which is how the Dockerfile's VERSION build arg arrives,
+# wins over what `git describe` says of this checkout. An empty one counts
+# as not given; `override` is what replaces an empty command-line value.
+ifeq ($(VERSION),)
+override VERSION := $(shell git describe --tags --always --dirty 2>/dev/null || echo "dev")
+endif
+LDFLAGS := -X main.Version=$(VERSION)
+
+# Standard targets are thin shims; the implementations live in script/
+# per the scripts-to-rule-them-all pattern (see the Entrypoints section
+# of README.md).
 
 all: check build
 
+bootstrap:
+	@script/bootstrap
+
+setup:
+	@script/setup
+
 build:
 	go build -ldflags "$(LDFLAGS)" -o bin/$(BINARY) ./cmd/dnswatcher
 
-lint:
-	golangci-lint run --config .golangci.yml ./...
-
-fmt:
-	gofmt -s -w .
-	goimports -w .
+# Prints the version `make build` stamps; the Dockerfile checks it.
+version:
+	@echo "$(VERSION)"
 
 test:
-	go test -v -race -cover ./...
+	@script/test
+
+lint:
+	@script/lint
+
+fmt:
+	@script/fmt
+
+fmt-check:
+	@script/fmt-check
 
-# Check runs all validation without making changes
-# Used by CI and Docker build - fails if anything is wrong
 check:
-	@echo "==> Checking formatting..."
-	@test -z "$$(gofmt -l .)" || (echo "Files not formatted:" && gofmt -l . && exit 1)
-	@echo "==> Running linter..."
-	golangci-lint run --config .golangci.yml ./...
-	@echo "==> Running tests..."
-	go test -v -race ./...
-	@echo "==> Building..."
-	go build -ldflags "$(LDFLAGS)" -o /dev/null ./cmd/dnswatcher
-	@echo "==> All checks passed!"
+	@script/check
+
+docker:
+	@script/docker
+
+hooks:
+	@script/install-precommit
 
 clean:
 	rm -rf bin/

REPO_POLICIES.md

@@ -0,0 +1,416 @@
+---
+title: Repository Policies
+last_modified: 2026-08-07
+---
+
+This document covers repository structure, tooling, and workflow standards. Code
+style conventions are in separate documents:
+
+- [Code Styleguide](https://git.eeqj.de/sneak/prompts/raw/branch/main/prompts/CODE_STYLEGUIDE.md)
+  (general, bash, Docker)
+- [Go](https://git.eeqj.de/sneak/prompts/raw/branch/main/prompts/CODE_STYLEGUIDE_GO.md)
+- [JavaScript](https://git.eeqj.de/sneak/prompts/raw/branch/main/prompts/CODE_STYLEGUIDE_JS.md)
+- [Python](https://git.eeqj.de/sneak/prompts/raw/branch/main/prompts/CODE_STYLEGUIDE_PYTHON.md)
+- [Go HTTP Server Conventions](https://git.eeqj.de/sneak/prompts/raw/branch/main/prompts/GO_HTTP_SERVER_CONVENTIONS.md)
+
+---
+
+- Cross-project documentation (such as this file) must include
+  `last_modified: YYYY-MM-DD` in the YAML front matter so it can be kept in sync
+  with the authoritative source as policies evolve.
+
+- **ALL external references must be pinned by cryptographic hash.** This
+  includes Docker base images, Go modules, npm packages, GitHub Actions, and
+  anything else fetched from a remote source. Version tags (`@v4`, `@latest`,
+  `:3.21`, etc.) are server-mutable and therefore remote code execution
+  vulnerabilities. The ONLY acceptable way to reference an external dependency
+  is by its content hash (Docker `@sha256:...`, Go module hash in `go.sum`, npm
+  integrity hash in lockfile, GitHub Actions `@<commit-sha>`). No exceptions.
+  This also means never `curl | bash` to install tools like pyenv, nvm, rustup,
+  etc. Instead, download a specific release archive from GitHub, verify its hash
+  (hardcoded in the Dockerfile or script), and only then install. Unverified
+  install scripts are arbitrary remote code execution. This is the single most
+  important rule in this document. Double-check every external reference in
+  every file before committing. There are zero exceptions to this rule.
+
+- Every repo with software must have a root `Makefile` with these targets:
+  `make bootstrap`, `make setup`, `make test`, `make lint`, `make fmt` (writes),
+  `make fmt-check` (read-only), `make check` (runs `test`, `lint`, `fmt-check`),
+  `make docker`, and `make hooks` (installs pre-commit hook). A model Makefile
+  is at `https://git.eeqj.de/sneak/prompts/raw/branch/main/Makefile`.
+
+- Repos follow the
+  [Scripts to Rule Them All](https://github.com/github/scripts-to-rule-them-all)
+  pattern: the implementation of each Makefile target lives in an executable
+  script in `script/` (`script/bootstrap`, `script/setup`, `script/test`,
+  `script/lint`, `script/fmt`, `script/fmt-check`, `script/check`,
+  `script/docker`), and the Makefile targets are thin shims that call them. The
+  scripts must be POSIX sh (`#!/bin/sh`, `set -eu`, no bashisms) so they run in
+  minimal containers (e.g. alpine images have no bash); locate the repo root
+  with `$(cd "$(dirname "$0")/.." && pwd -P)` and `cd` there before acting. From
+  the standard's canonical set we use `bootstrap`, `setup` (make the repo ready
+  for development after a fresh clone: runs `bootstrap`, then
+  `install-precommit`, plus any repo-specific initialization), `test`, and
+  `cibuild`. `script/bootstrap` installs all dependencies idempotently and
+  assumes nothing is present: base tools come from nix, apt, brew, or apk
+  (detected in that order; apt runs noninteractive). For node it uses the
+  installed node if present; otherwise it installs a PINNED node version via
+  nvm, first installing nvm itself if missing — from a hash-verified GitHub
+  release archive (never `curl | sh`), with bash installed as an explicit
+  prerequisite since nvm requires bash. yarn is then pinned via
+  `corepack prepare yarn@<version> --activate`. Never install "latest" or "lts";
+  always exact versions. `script/cibuild` runs the CI build: it changes to the
+  repo root and runs `docker build .`; the Gitea workflow calls it. Four further
+  scripts are our own extensions to the standard: `script/check` runs
+  `script/test`, `script/lint`, and `script/fmt-check`; `script/precommit` is
+  what the git pre-commit hook runs, and it calls `script/check`;
+  `script/install-precommit` installs the git pre-commit hook (the `make hooks`
+  target shims to it); and `script/projectname` (literally that filename) simply
+  outputs the project's name. Scripts that need the name call
+  `script/projectname` — e.g. `script/docker` assembles its image tag from it —
+  so those scripts stay byte-identical across all repos. Repo-type-specific
+  pre-commit extras (e.g. `go mod tidy` verification in Go repos) belong in
+  `script/precommit`, not in the hook itself. Model scripts are at
+  `https://git.eeqj.de/sneak/prompts/raw/branch/main/script/<name>`. The README
+  must document the provided scripts in an **Entrypoints** section (see the
+  README requirements below).
+
+- Always use Makefile targets (`make fmt`, `make test`, `make lint`, etc.)
+  instead of invoking the underlying tools directly. The Makefile is the single
+  source of truth for how these operations are run.
+
+- The Makefile is authoritative documentation for how the repo is used. Beyond
+  the required targets above, it should have targets for every common operation:
+  running a local development server (`make run`, `make dev`), re-initializing
+  or migrating the database (`make db-reset`, `make migrate`), building
+  artifacts (`make build`), generating code, seeding data, or anything else a
+  developer would do regularly. If someone checks out the repo and types
+  `make<tab>`, they should see every meaningful operation available. A new
+  contributor should be able to understand the entire development workflow by
+  reading the Makefile.
+
+- Every repo should have a `Dockerfile`. All Dockerfiles must run `make check`
+  as a build step so the build fails if the branch is not green. For non-server
+  repos, the Dockerfile should bring up a development environment and run
+  `make check`. For server repos, `make check` should run as an early build
+  stage before the final image is assembled. Dockerfiles install development
+  prerequisites by running `script/bootstrap` rather than duplicating installs
+  inline; COPY `script/` and the dependency manifests (`package.json` +
+  `yarn.lock`, `go.mod` + `go.sum`, etc.) before running it so the bootstrap
+  layer stays cached until dependencies change.
+
+- **Dockerfiles must use a separate lint stage for fail-fast feedback.** Go
+  repos use a multistage build where linting runs in an independent stage based
+  on the `golangci/golangci-lint` image (pinned by hash). This stage runs
+  `make fmt-check` and `make lint` before the full build begins. The build stage
+  then declares an explicit dependency on the lint stage via
+  `COPY --from=lint /src/go.sum /dev/null`, which forces BuildKit to complete
+  linting before proceeding to compilation and tests. This ensures lint failures
+  surface in seconds rather than minutes, without blocking on dependency
+  download or compilation in the build stage.
+
+    The standard pattern for a Go repo Dockerfile is:
+
+    ```dockerfile
+    # Lint stage — fast feedback on formatting and lint issues
+    # golangci/golangci-lint:v2.x.x, YYYY-MM-DD
+    FROM golangci/golangci-lint@sha256:... AS lint
+    WORKDIR /src
+    COPY go.mod go.sum ./
+    RUN go mod download
+    COPY . .
+    RUN make fmt-check
+    RUN make lint
+
+    # Build stage
+    # golang:1.x-alpine, YYYY-MM-DD
+    FROM golang@sha256:... AS builder
+    WORKDIR /src
+
+    # Force BuildKit to run the lint stage before proceeding
+    COPY --from=lint /src/go.sum /dev/null
+
+    COPY go.mod go.sum ./
+    RUN go mod download
+    COPY . .
+    RUN make test
+
+    ARG VERSION=dev
+    RUN CGO_ENABLED=0 go build -trimpath \
+        -ldflags="-s -w -X main.Version=${VERSION}" \
+        -o /app ./cmd/app/
+
+    # Runtime stage
+    FROM alpine@sha256:...
+    COPY --from=builder /app /usr/local/bin/app
+    ENTRYPOINT ["app"]
+    ```
+
+    Key points:
+    - The lint stage uses the `golangci/golangci-lint` image directly (it
+      includes both Go and the linter), so there is no need to install the
+      linter separately.
+    - `COPY --from=lint /src/go.sum /dev/null` is a no-op file copy that creates
+      a stage dependency. BuildKit runs stages in parallel by default; without
+      this line, the build stage would not wait for lint to finish and a lint
+      failure might not fail the overall build.
+    - If the project uses `//go:embed` directives that reference build artifacts
+      (e.g. a web frontend compiled in a separate stage), the lint stage must
+      create placeholder files so the embed directives resolve. Example:
+      `RUN mkdir -p web/dist && touch web/dist/index.html web/dist/style.css`.
+      The lint stage should not depend on the actual build output — it exists to
+      fail fast.
+    - If the project requires CGO or system libraries for linting (e.g.
+      `vips-dev`), install them in the lint stage with `apk add`.
+    - The build stage runs `make test` after compilation setup. Tests run in the
+      build stage, not the lint stage, because they may require compiled
+      artifacts or heavier dependencies.
+
+- Every repo should have a Gitea Actions workflow (`.gitea/workflows/`) that
+  runs `script/cibuild` (which runs `docker build .`) on push. Since the
+  Dockerfile already runs `make check`, a successful build implies all checks
+  pass.
+
+- Use platform-standard formatters: `black` for Python, `prettier` for
+  JS/CSS/Markdown/HTML, `go fmt` for Go. Always use default configuration with
+  two exceptions: four-space indents (except Go), and `proseWrap: always` for
+  Markdown (hard-wrap at 80 columns). Documentation and writing repos (Markdown,
+  HTML, CSS) should also have `.prettierrc` and `.prettierignore`.
+
+- Pre-commit hook: runs `script/precommit`, which calls `script/check`. If local
+  testing is not possible in the repo, `script/precommit` may skip `script/test`
+  and run only `script/lint` and `script/fmt-check`. The hook is installed by
+  `script/install-precommit`; the Makefile must provide a `make hooks` target
+  that shims to it.
+
+- All repos with software must have tests that run via the platform-standard
+  test framework (`go test`, `pytest`, `jest`/`vitest`, etc.). If no meaningful
+  tests exist yet, add the most minimal test possible — e.g. importing the
+  module under test to verify it compiles/parses. There is no excuse for
+  `make test` to be a no-op.
+
+- `make test` must complete in under 60 seconds. That is the hard cap, and a
+  suite that exceeds it fails. Under 20 seconds is the target. A suite between
+  20 and 60 seconds is still green, but the overage must be filed as an
+  improvement bug against that repo. Add a 90-second timeout to the test
+  invocation in the Makefile (`go test -timeout 90s`). The backstop deliberately
+  sits above the hard cap so that it catches a genuinely hung test rather than a
+  merely slow one.
+
+- **`make test` should use the conditional verbose rerun pattern.** Run tests
+  without `-v` (verbose) first. If tests fail, automatically rerun with `-v` to
+  show full output. This keeps CI logs and `docker build` output clean on
+  success (just package/suite summaries) while providing full diagnostic detail
+  on failure (every test case, every assertion). The general shell pattern:
+
+    ```makefile
+    test:
+    	@<test-command> || \
+    		{ echo "--- Rerunning with -v for details ---"; \
+    		  <test-command-with-v>; exit 1; }
+    ```
+
+    Go example:
+
+    ```makefile
+    test:
+    	@go test -timeout 90s -race -cover ./... || \
+    		{ echo "--- Rerunning with -v for details ---"; \
+    		  go test -timeout 90s -race -v ./...; exit 1; }
+    ```
+
+    Python example:
+
+    ```makefile
+    test:
+    	@python -m pytest || \
+    		{ echo "--- Rerunning with -v for details ---"; \
+    		  python -m pytest -v; exit 1; }
+    ```
+
+    The `exit 1` ensures the target always fails after a rerun — the first run
+    already proved the tests are broken, so the build must not pass even if a
+    flaky test happens to succeed on the second attempt. The rerun exists solely
+    for diagnostic output.
+
+- Docker builds must complete in under 5 minutes.
+
+- `make check` must not modify any files in the repo. Tests may use temporary
+  directories.
+
+- `main` must always pass `make check`, no exceptions.
+
+- Never commit secrets. `.env` files, credentials, API keys, and private keys
+  must be in `.gitignore`. No exceptions.
+
+- `.gitignore` should be comprehensive from the start: OS files (`.DS_Store`),
+  editor files (`.swp`, `*~`), language build artifacts, and `node_modules/`.
+  Fetch the standard `.gitignore` from
+  `https://git.eeqj.de/sneak/prompts/raw/branch/main/.gitignore` when setting up
+  a new repo.
+
+- **No build artifacts in version control.** Code-derived data (compiled
+  bundles, minified output, generated assets) must never be committed to the
+  repository if it can be avoided. The build process (e.g. Dockerfile, Makefile)
+  should generate these at build time. Notable exception: Go protobuf generated
+  files (`.pb.go`) ARE committed because repos need to work with `go get`, which
+  downloads code but does not execute code generation.
+
+- Never use `git add -A` or `git add .`. Always stage files explicitly by name.
+
+- Never force-push to `main`.
+
+- Make all changes on a feature branch. You can do whatever you want on a
+  feature branch.
+
+- `.golangci.yml` is standardized and must _NEVER_ be modified by an agent, only
+  manually by the user. Fetch from
+  `https://git.eeqj.de/sneak/prompts/raw/branch/main/.golangci.yml`. The
+  canonical golangci-lint version is v2.12.2 (released 2026-05-06), installed
+  commit-pinned via
+  `go install github.com/golangci/golangci-lint/v2/cmd/golangci-lint@c0d3ddc9cf3faa61a4e378e879ece580256d76e5`.
+
+- When pinning images or packages by hash, add a comment above the reference
+  with the version and date (YYYY-MM-DD).
+
+- Use `yarn`, not `npm`.
+
+- Write all dates as YYYY-MM-DD (ISO 8601).
+
+- Simple projects should be configured with environment variables.
+
+- Dockerized web services listen on port 8080 by default, overridable with
+  `PORT`.
+
+- **HTTP/web services must be hardened for production internet exposure before
+  tagging 1.0.** This means full compliance with security best practices
+  including, without limitation, all of the following:
+    - **Security headers** on every response:
+        - `Strict-Transport-Security` (HSTS) with `max-age` of at least one year
+          and `includeSubDomains`.
+        - `Content-Security-Policy` (CSP) with a restrictive default policy
+          (`default-src 'self'` as a baseline, tightened per-resource as
+          needed). Never use `unsafe-inline` or `unsafe-eval` unless
+          unavoidable, and document the reason.
+        - `X-Frame-Options: DENY` (or `SAMEORIGIN` if framing is required).
+          Prefer the `frame-ancestors` CSP directive as the primary control.
+        - `X-Content-Type-Options: nosniff`.
+        - `Referrer-Policy: strict-origin-when-cross-origin` (or stricter).
+        - `Permissions-Policy` restricting access to browser features the
+          application does not use (camera, microphone, geolocation, etc.).
+    - **Request and response limits:**
+        - Maximum request body size enforced on all endpoints (e.g. Go
+          `http.MaxBytesReader`). Choose a sane default per-route; never accept
+          unbounded input.
+        - Maximum response body size where applicable (e.g. paginated APIs).
+        - `ReadTimeout` and `ReadHeaderTimeout` on the `http.Server` to defend
+          against slowloris attacks.
+        - `WriteTimeout` on the `http.Server`.
+        - `IdleTimeout` on the `http.Server`.
+        - Per-handler execution time limits via `context.WithTimeout` or
+          chi/stdlib `middleware.Timeout`.
+    - **Authentication and session security:**
+        - Rate limiting on password-based authentication endpoints. API keys are
+          high-entropy and not susceptible to brute force, so they are exempt.
+        - CSRF tokens on all state-mutating HTML forms. API endpoints
+          authenticated via `Authorization` header (Bearer token, API key) are
+          exempt because the browser does not attach these automatically.
+        - Passwords stored using bcrypt, scrypt, or argon2 — never plain-text,
+          MD5, or SHA.
+        - Session cookies set with `HttpOnly`, `Secure`, and `SameSite=Lax` (or
+          `Strict`) attributes.
+    - **Reverse proxy awareness:**
+        - True client IP detection when behind a reverse proxy
+          (`X-Forwarded-For`, `X-Real-IP`). The application must accept
+          forwarded headers only from a configured set of trusted proxy
+          addresses — never trust `X-Forwarded-For` unconditionally.
+    - **CORS:**
+        - Authenticated endpoints must restrict `Access-Control-Allow-Origin` to
+          an explicit allowlist of known origins. Wildcard (`*`) is acceptable
+          only for public, unauthenticated read-only APIs.
+    - **Error handling:**
+        - Internal errors must never leak stack traces, SQL queries, file paths,
+          or other implementation details to the client. Return generic error
+          messages in production; detailed errors only when `DEBUG` is enabled.
+    - **TLS:**
+        - Services never terminate TLS directly. They are always deployed behind
+          a TLS-terminating reverse proxy. The service itself listens on plain
+          HTTP. However, HSTS headers and `Secure` cookie flags must still be
+          set by the application so that the browser enforces HTTPS end-to-end.
+
+    This list is non-exhaustive. Apply defense-in-depth: if a standard security
+    hardening measure exists for HTTP services and is not listed here, it is
+    still expected. When in doubt, harden.
+
+- `README.md` is the primary documentation. Required sections:
+    - **Description**: First line must include the project name, purpose,
+      category (web server, SPA, CLI tool, etc.), license, and author. Example:
+      "µPaaS is an MIT-licensed Go web application by @sneak that receives
+      git-frontend webhooks and deploys applications via Docker in realtime."
+    - **Getting Started**: Copy-pasteable install/usage code block.
+    - **Entrypoints**: Opens by stating that the repo adheres to the
+      [Scripts to Rule Them All](https://github.com/github/scripts-to-rule-them-all)
+      standard (with that link), then documents each provided `script/`
+      entrypoint and its purpose.
+    - **Rationale**: Why does this exist?
+    - **Design**: How is the program structured?
+    - **TODO**: Update meticulously, even between commits. When planning, put
+      the todo list in the README so a new agent can pick up where the last one
+      left off.
+    - **License**: MIT, GPL, or WTFPL. Ask the user for new projects. Include a
+      `LICENSE` file in the repo root and a License section in the README.
+    - **Author**: [@sneak](https://sneak.berlin).
+
+- First commit of a new repo should contain only `README.md`.
+
+- Go module root: `sneak.berlin/go/<name>`. Always run `go mod tidy` before
+  committing.
+
+- Use SemVer.
+
+- Database migrations live in `internal/db/migrations/` and must be embedded in
+  the binary.
+    - `000_migration.sql` — contains ONLY the creation of the migrations
+      tracking table itself. Nothing else.
+    - `001_schema.sql` — the full application schema.
+    - **Pre-1.0.0:** never add additional migration files (002, 003, etc.).
+      There is no installed base to migrate. Edit `001_schema.sql` directly.
+    - **Post-1.0.0:** add new numbered migration files for each schema change.
+      Never edit existing migrations after release.
+
+- All repos should have an `.editorconfig` enforcing the project's indentation
+  settings.
+
+- Avoid putting files in the repo root unless necessary. Root should contain
+  only project-level config files (`README.md`, `Makefile`, `Dockerfile`,
+  `LICENSE`, `.gitignore`, `.editorconfig`, `REPO_POLICIES.md`, and
+  language-specific config). Everything else goes in a subdirectory. Canonical
+  subdirectory names:
+    - `bin/` — executable scripts and tools
+    - `cmd/` — Go command entrypoints
+    - `configs/` — configuration templates and examples
+    - `deploy/` — deployment manifests (k8s, compose, terraform)
+    - `docs/` — documentation and markdown (README.md stays in root)
+    - `internal/` — Go internal packages
+    - `internal/db/migrations/` — database migrations
+    - `pkg/` — Go library packages
+    - `share/` — systemd units, data files
+    - `static/` — static assets (images, fonts, etc.)
+    - `web/` — web frontend source
+
+- When setting up a new repo, files from the `prompts` repo may be used as
+  templates. Fetch them from
+  `https://git.eeqj.de/sneak/prompts/raw/branch/main/<path>`.
+
+- New repos must contain at minimum:
+    - `README.md`, `.git`, `.gitignore`, `.editorconfig`
+    - `LICENSE`, `REPO_POLICIES.md` (copy from the `prompts` repo)
+    - `Makefile`
+    - `script/` entrypoints (`bootstrap`, `setup`, `projectname`, `test`,
+      `lint`, `fmt`, `fmt-check`, `check`, `docker`, `cibuild`, `precommit`,
+      `install-precommit`)
+    - `Dockerfile`, `.dockerignore`
+    - `.gitea/workflows/check.yml`
+    - Go: `go.mod`, `go.sum`, `.golangci.yml`
+    - JS: `package.json`, `yarn.lock`, `.prettierrc`, `.prettierignore`
+    - Python: `pyproject.toml`

TESTING.md

@@ -2,33 +2,43 @@
 
 ## DNS Resolution Tests
 
-All resolver tests **MUST** use live queries against real DNS servers.
-No mocking of the DNS client layer is permitted.
+DNS is never mocked in this project, not in tests and not anywhere else; see the
+README section "No DNS mocking. Ever." Every test that looks something up in DNS
+**MUST** query live DNS servers, never a stand-in. Logic that works on record
+data, such as comparing or formatting records, may be tested on that data
+directly with no lookup.
 
 ### Rationale
 
-The resolver performs iterative resolution from root nameservers through
-the full delegation chain. Mocked responses cannot faithfully represent
-the variety of real-world DNS behavior (truncation, referrals, glue
-records, DNSSEC, varied response times, EDNS, etc.). Testing against
-real servers ensures the resolver works correctly in production.
+The resolver performs iterative resolution from root nameservers through the
+full delegation chain. Mocked responses cannot faithfully represent the variety
+of real-world DNS behavior (truncation, referrals, glue records, DNSSEC, varied
+response times, EDNS, etc.). Testing against real servers ensures the resolver
+works correctly in production.
 
 ### Constraints
 
 - Tests hit real DNS infrastructure and require network access
-- Test duration depends on network conditions; timeout tuning keeps
-  the suite within the 30-second target
-- Query timeout is calibrated to 3× maximum antipodal RTT (~300ms)
-  plus processing margin
+- Test duration depends on network conditions; timeout tuning keeps the suite
+  within the 60-second target
+- Query timeout is calibrated to 3× maximum antipodal RTT (~300ms) plus
+  processing margin
 - Root server fan-out is limited to reduce parallel query load
-- Flaky failures from transient network issues are acceptable and
-  should be investigated as potential resolver bugs, not papered over
-  with mocks or skip flags
+- Live lookups that expect an answer go through `internal/livednstest`, which
+  limits how many run at once in a test binary and retries a lookup that got
+  none
+- Flaky failures from transient network issues are acceptable and should be
+  investigated as potential resolver bugs, not papered over with mocks or skip
+  flags
 
 ### What NOT to do
 
-- **Do not mock `DNSClient`** for resolver tests (the mock constructor
-  exists for unit-testing other packages that consume the resolver)
+- **Do not mock, fake or stub DNS** anywhere: no stand-in `DNSClient`, no
+  stand-in for the watcher's `DNSResolver`, no fake DNS server, no canned
+  responses
 - **Do not add `-short` flags** to skip slow tests
 - **Do not increase `-timeout`** to hide hanging queries
+- **Do not remove `-count=1` from `script/test`** — Go's test cache replays a
+  previous run's output without querying anything, so a cached pass is not
+  evidence that live resolution works
 - **Do not modify linter configuration** to suppress findings

TODO.md

@@ -0,0 +1,152 @@
+# Workflow
+
+- branch (from `next`)
+- do the work in Next Step
+- move Next Step to the top of Completed Steps
+- move the top item of Future Steps into Next Step
+- commit (`TODO.md` changes in the same commit as the work)
+- push
+- open a PR against `next`
+
+# Status
+
+pre-1.0. No git tags. Work lands on `next` by PR. Open work for 1.0 is tracked
+on the 1.0 milestone: https://git.eeqj.de/sneak/dnswatcher/milestone/7
+
+# Next Step
+
+trial run of the finished image: https://git.eeqj.de/sneak/dnswatcher/issues/149
+
+# Completed Steps
+
+- 2026-10-02: a Mattermost webhook that answers an HTTP error is logged as
+  `mattermost notification failed`, not as a Slack failure (closes #227).
+- 2026-10-02: durations in the log are written as text such as `2m0s`, not as a
+  bare count of nanoseconds (closes #228).
+- 2026-10-02: a watched name whose nameservers answer with a CNAME and no
+  address gets port and TLS checks at the end of its CNAME chain (closes #203).
+- 2026-10-02: a resolver test that reads one record type from a nameserver's
+  answer asks again when that type is missing from it (closes #218).
+- 2026-10-02: a plain `docker build .` of a clone stamps its tag or short
+  commit, not `dev`: the build context now carries `.git` (closes #210).
+- 2026-10-02: a query a server refuses is not resent asking for recursion, and
+  every root server refusing is reported as DNS interception (closes #206).
+- 2026-10-02: a push to a branch cancels that branch's older CI run, and the
+  checkout leaves no token in `.git/config` (closes #216).
+- 2026-10-02: watcher tests send far fewer queries and a live attempt may take
+  18s; nameserver addresses are asked only for A, AAAA, CNAME (closes #214).
+- 2026-10-02: the resolver tries root servers, and every other server list it
+  walks, in a random order each time, not always from the top (closes #138).
+- 2026-10-02: a name listed more than once in `DNSWATCHER_TARGETS`, in any
+  letter case or with a trailing dot, is watched once (closes #207).
+- 2026-10-01: README checked against the code and corrected: metrics, CORS,
+  notification retries, CNAMEs, state file fields, Design tree (closes #108).
+- 2026-10-01: a certificate within the expiry warning period is warned about on
+  every TLS check, where some checks used to skip it at random (closes #204).
+- 2026-10-01: a domain's NS set is its delegation from the parent zone's
+  servers, not whichever of its own servers answered first (closes #200).
+- 2026-10-01: README has Getting Started, Rationale and TODO sections, and its
+  Architecture section is now Design, in the order policy sets (closes #173).
+- 2026-10-01: a zone's server that answers SERVFAIL or a referral leading no
+  closer is passed over for the next, as one that times out is (closes #197).
+- 2026-10-01: when none of a configured name's nameservers answered, the port
+  state saved for its addresses is kept, not removed (closes #193).
+- 2026-10-01: `ResolveIPAddresses` returns an error, not no addresses, when no
+  nameserver of the name's zone answered (closes #190).
+- 2026-10-01: `make fmt` and `make fmt-check` cover Markdown with prettier, run
+  in Docker at the version pinned by `yarn.lock` (closes #119).
+- 2026-10-01: `make fmt-check` fails on a file `goimports` would change; both
+  format scripts run `goimports` at its pinned commit, not from `PATH` (#119).
+- 2026-10-01: a hostname is queried at the servers of the zone it is in, found
+  by following delegations for the name, not its last two labels (closes #189).
+- 2026-10-01: each nameserver's addresses are saved with its domain, and a
+  change while it stays in the delegation is notified (closes #105).
+- 2026-10-01: the watcher saves state when it stops, and shutdown waits for that
+  save, so it no longer relies on the state's own stop hook (closes #114).
+- 2026-10-01: `DNSWATCHER_SENTRY_DSN` reports panics in HTTP handlers to Sentry,
+  and a DSN Sentry cannot parse stops startup (closes #107).
+- 2026-10-01: a port or TLS check that shutdown cuts short saves nothing and
+  sends no notification, as a cut-short DNS lookup already did (closes #185).
+- 2026-10-01: the client address from `X-Forwarded-For` is the last entry that
+  is not a trusted proxy, not the first, which the client sets (closes #181).
+- 2026-10-01: a nameserver that does not answer is saved as `error` with the
+  reason, and NS failure and NS recovery are notified (closes #104).
+- 2026-10-01: a `DNSWATCHER_DNS_INTERVAL` or `DNSWATCHER_TLS_INTERVAL` that is
+  not a positive duration stops startup; empty means the default (closes #177).
+- 2026-10-01: `/metrics` allows each client address 30 requests a minute,
+  counted before Basic Auth, and answers 429 beyond that (closes #101).
+- 2026-10-01: the image built by `make docker` reports the `git describe`
+  version, not `dev`, and the startup log now shows it (closes #109).
+- 2026-10-01: two notify shutdown tests always release the delivery they hold,
+  so a drain that returns early fails them instead of hanging (closes #176).
+- 2026-10-01: `script/install-precommit` asks git for the repository's git
+  directory, so `make hooks` also works where `.git` is a file (closes #129).
+- 2026-10-01: `TODO.md` brought up to date: open issues listed by URL, every
+  Completed Steps entry cut to at most two lines (closes #146).
+- 2026-10-01: wildcard CORS now applies only to the public routes, not to
+  `/metrics`, and allows only the methods they serve (closes #100).
+- 2026-10-01: `internal/state` and `internal/watcher` no longer export test-only
+  constructors: two moved to `export_test.go`, one is deleted (closes #111).
+- 2026-10-01: notify shutdown tests use one timing constant per meaning, name
+  the bound they check, and require the drain's debug line (closes #116).
+- 2026-09-29: the entrypoint chowns the data directory to `dnswatcher` and runs
+  dnswatcher as that user, so a host bind mount needs no chown (closes #166).
+- 2026-09-29: the live-DNS test package is renamed `internal/livednstest`;
+  `make lint` fails when program code imports it (closes #164).
+- 2026-09-29: `.golangci.yml` re-fetched from `sneak/prompts`, with
+  `gomodguard_v2` and the org `depguard` `test-support` rule (closes #123).
+- 2026-09-29: watcher and resolver tests that look something up in DNS use the
+  real resolver against live DNS servers (closes #159).
+- 2026-09-28: the inconsistency alert is sent once, when two nameservers start
+  to disagree; every pair of nameservers is compared (closes #158).
+- 2026-09-28: DNS names in record values (CNAME, MX, SRV and NS targets) are
+  lower-cased, so letter case alone is not a change (closes #157).
+- 2026-09-28: lint and tests run on every build: `script/cibuild` and
+  `script/docker` pass `--no-cache-filter=lint,builder` (closes #115).
+- 2026-09-28: the server timeout test drives `Run` and checks the timeouts on
+  the `http.Server` it serves (closes #120).
+- 2026-09-28: upaas deploy readiness: the image runs as user `dnswatcher` with a
+  `HEALTHCHECK`; README "Running under upaas" (closes #147).
+- 2026-09-21: added behavioural tests for `internal/globals`,
+  `internal/healthcheck`, and `internal/logger` (closes #110).
+- 2026-09-21: `go mod tidy` dropped the redundant `golang.org/x/sync`
+  `// indirect` line so `script/bootstrap` leaves a clean tree (#132)
+- 2026-08-10: comment-only corrections to `script/bootstrap`, `script/cibuild`
+  and `Dockerfile.lint`; no behaviour changed.
+- 2026-08-10: MIT `LICENSE` added at the repository root; the README's first
+  line and License section name the licence.
+- 2026-08-10: policy scaffold present: `REPO_POLICIES.md`, `.editorconfig`,
+  `.dockerignore`, CI workflow, `make fmt-check`, `make docker`, `make hooks`.
+- 2026-08-10: Go's test cache disabled in `script/test` (`-count=1`), so every
+  run queries live DNS; a failed run is rerun with `-v`.
+- 2026-08-10: live-DNS tests made robust rather than gated (#93): a limit on
+  concurrent lookups, retries, and a quorum across nameservers.
+- 2026-08-10: all linting moved into Docker: `script/lint` builds
+  `Dockerfile.lint`, and the root `Dockerfile` has its own lint stage.
+- 2026-08-09: in-flight notification deliveries are drained at shutdown, bounded
+  by the shutdown deadline (#106).
+- 2026-08-09: `http.Server` sets all four socket timeouts; `WriteTimeout` stays
+  above the 60s handler timeout (#99).
+- 2026-08-09: `SecurityHeaders()` middleware sets HSTS, CSP and the other
+  security headers `REPO_POLICIES.md` requires on every response.
+- 2026-08-07: golangci-lint bumped to v2.12.2 and `.golangci.yml` set to the org
+  config; fixed the resulting `goconst`, `dupl` and `lll` findings.
+- 2026-07-07 Adopted scripts-to-rule-them-all: `script/` entrypoints, Makefile
+  shims, README Entrypoints section
+- 2026-02-20: iterative DNS resolver implemented
+- 2026-02-20: CI actions and go install refs pinned to commit SHAs; Gitea
+  Actions workflow added
+- 2026-02-20: watcher monitoring orchestrator merged to main (#8)
+- 2026-02-20: DOMAINS/HOSTNAMES unified into single TARGETS config (#11)
+- 2026-02-19: TCP port connectivity checker, made concurrent with port
+  validation; gosec G704 SSRF findings fixed without suppression
+- 2026-02-19: TLS certificate inspector with no-peer-certificates error path and
+  IP SANs
+- 2026-02-19: gosec SSRF and formatting fixes on main
+- 2026-02-19: initial scaffold with per-nameserver DNS monitoring model
+
+# Future Steps
+
+- 1.0 readiness: run it with a real config and read the logs:
+  https://git.eeqj.de/sneak/dnswatcher/issues/66
+- review toward 1.0: https://git.eeqj.de/sneak/dnswatcher/issues/144

cmd/dnswatcher/main.go

@@ -25,15 +25,13 @@ import (
 //
 //nolint:gochecknoglobals // build-time variables
 var (
-	Appname   = "dnswatcher"
-	Version   string
-	Buildarch string
+	Appname = "dnswatcher"
+	Version string
 )
 
 func main() {
 	globals.SetAppname(Appname)
 	globals.SetVersion(Version)
-	globals.SetBuildarch(Buildarch)
 
 	fx.New(
 		fx.Provide(
@@ -65,6 +63,7 @@ func main() {
 				return n
 			},
 		),
+		fx.Invoke(func(l *logger.Logger) { l.Identify() }),
 		fx.Invoke(func(*server.Server, *watcher.Watcher) {}),
 	).Run()
 }

deploy/docker-entrypoint.sh

@@ -0,0 +1,17 @@
+#!/bin/sh
+# deploy/docker-entrypoint.sh: the Docker image's ENTRYPOINT. It runs as
+# root only to give the data directory to the dnswatcher user: a host
+# directory bind-mounted there keeps its host owner, often root, and may
+# hold a state file left by another uid, which dnswatcher could neither
+# read nor replace. dnswatcher itself always runs as the dnswatcher user.
+set -eu
+
+main() {
+    dir="${DNSWATCHER_DATA_DIR:-/var/lib/dnswatcher}"
+    mkdir -p "$dir"
+    chown -R dnswatcher:dnswatcher "$dir"
+    chmod 700 "$dir"
+    exec su-exec dnswatcher /usr/local/bin/dnswatcher "$@"
+}
+
+main "$@"

go.mod

@@ -4,27 +4,30 @@ go 1.25.5
 
 require (
 	github.com/99designs/basicauth-go v0.0.0-20230316000542-bf6f9cbbf0f8
+	github.com/getsentry/sentry-go v0.49.0
 	github.com/go-chi/chi/v5 v5.2.5
 	github.com/go-chi/cors v1.2.2
+	github.com/go-chi/httprate v0.16.0
 	github.com/joho/godotenv v1.5.1
 	github.com/miekg/dns v1.1.72
 	github.com/prometheus/client_golang v1.23.2
 	github.com/spf13/viper v1.21.0
 	github.com/stretchr/testify v1.11.1
 	go.uber.org/fx v1.24.0
-	golang.org/x/net v0.50.0
-	golang.org/x/sync v0.19.0
+	golang.org/x/net v0.56.0
+	golang.org/x/sync v0.21.0
 )
 
 require (
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/cespare/xxhash/v2 v2.3.0 // indirect
-	github.com/davecgh/go-spew v1.1.1 // indirect
+	github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
 	github.com/fsnotify/fsnotify v1.9.0 // indirect
 	github.com/go-viper/mapstructure/v2 v2.4.0 // indirect
+	github.com/klauspost/cpuid/v2 v2.2.10 // indirect
 	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
 	github.com/pelletier/go-toml/v2 v2.2.4 // indirect
-	github.com/pmezard/go-difflib v1.0.0 // indirect
+	github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
 	github.com/prometheus/client_model v0.6.2 // indirect
 	github.com/prometheus/common v0.66.1 // indirect
 	github.com/prometheus/procfs v0.16.1 // indirect
@@ -34,16 +37,16 @@ require (
 	github.com/spf13/cast v1.10.0 // indirect
 	github.com/spf13/pflag v1.0.10 // indirect
 	github.com/subosito/gotenv v1.6.0 // indirect
+	github.com/zeebo/xxh3 v1.0.2 // indirect
 	go.uber.org/dig v1.19.0 // indirect
 	go.uber.org/multierr v1.10.0 // indirect
 	go.uber.org/zap v1.26.0 // indirect
 	go.yaml.in/yaml/v2 v2.4.2 // indirect
 	go.yaml.in/yaml/v3 v3.0.4 // indirect
-	golang.org/x/mod v0.32.0 // indirect
-	golang.org/x/sync v0.19.0 // indirect
-	golang.org/x/sys v0.41.0 // indirect
-	golang.org/x/text v0.34.0 // indirect
-	golang.org/x/tools v0.41.0 // indirect
+	golang.org/x/mod v0.37.0 // indirect
+	golang.org/x/sys v0.46.0 // indirect
+	golang.org/x/text v0.39.0 // indirect
+	golang.org/x/tools v0.47.0 // indirect
 	google.golang.org/protobuf v1.36.8 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 )

go.sum

@@ -4,16 +4,22 @@ github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
 github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
 github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
 github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
-github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
-github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM=
+github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8=
 github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0=
 github.com/fsnotify/fsnotify v1.9.0 h1:2Ml+OJNzbYCTzsxtv8vKSFD9PbJjmhYF14k/jKC7S9k=
 github.com/fsnotify/fsnotify v1.9.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0=
+github.com/getsentry/sentry-go v0.49.0 h1:Ehejknu1l023Ub7QoRBVLAI7g3Jnhqku4oWx4B4Sh5s=
+github.com/getsentry/sentry-go v0.49.0/go.mod h1:nuMJAoCfe1u0Bts2ocyNI+TW8HT84vRMqwA5Qq/SKUI=
 github.com/go-chi/chi/v5 v5.2.5 h1:Eg4myHZBjyvJmAFjFvWgrqDTXFyOzjj7YIm3L3mu6Ug=
 github.com/go-chi/chi/v5 v5.2.5/go.mod h1:X7Gx4mteadT3eDOMTsXzmI4/rwUpOwBHLpAfupzFJP0=
 github.com/go-chi/cors v1.2.2 h1:Jmey33TE+b+rB7fT8MUy1u0I4L+NARQlK6LhzKPSyQE=
 github.com/go-chi/cors v1.2.2/go.mod h1:sSbTewc+6wYHBBCW7ytsFSn836hqM7JxpglAy2Vzc58=
+github.com/go-chi/httprate v0.16.0 h1:8V5DH9j6pSK6UQoBsTpvMyFxycqaKEIToyPKzHJjUa8=
+github.com/go-chi/httprate v0.16.0/go.mod h1:A8lo+qRhk+s9LiuP5saS7XCGDXRXMcrueq0NfIuCa/I=
+github.com/go-errors/errors v1.4.2 h1:J6MZopCL4uSllY1OfXM374weqZFFItUbrImctkmUxIA=
+github.com/go-errors/errors v1.4.2/go.mod h1:sIVyrIiJhuEF+Pj9Ebtd6P/rEYROXFi3BopGUQ5a5Og=
 github.com/go-viper/mapstructure/v2 v2.4.0 h1:EBsztssimR/CONLSZZ04E8qAkxNYq4Qp9LvH92wZUgs=
 github.com/go-viper/mapstructure/v2 v2.4.0/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM=
 github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
@@ -22,6 +28,8 @@ github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0=
 github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4=
 github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo=
 github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ=
+github.com/klauspost/cpuid/v2 v2.2.10 h1:tBs3QSyvjDyFTq3uoc/9xFpCuOsJQFNPiAhYdw2skhE=
+github.com/klauspost/cpuid/v2 v2.2.10/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0=
 github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
 github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
 github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
@@ -34,8 +42,12 @@ github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq
 github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
 github.com/pelletier/go-toml/v2 v2.2.4 h1:mye9XuhQ6gvn5h28+VilKrrPoQVanw5PMw/TB0t5Ec4=
 github.com/pelletier/go-toml/v2 v2.2.4/go.mod h1:2gIqNv+qfxSVS7cM2xJQKtLSTLUE9V8t9Stt+h56mCY=
-github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
-github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/pingcap/errors v0.11.4 h1:lFuQV/oaUMGcD2tqt+01ROSmJs75VG1ToEOkZIZ4nE4=
+github.com/pingcap/errors v0.11.4/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8=
+github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
+github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U=
+github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/prometheus/client_golang v1.23.2 h1:Je96obch5RDVy3FDMndoUsjAhG5Edi49h0RJWRi/o0o=
 github.com/prometheus/client_golang v1.23.2/go.mod h1:Tb1a6LWHB3/SPIzCoaDXI4I8UHKeFTEQ1YCr+0Gyqmg=
 github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNwqPLxwZyk=
@@ -44,8 +56,8 @@ github.com/prometheus/common v0.66.1 h1:h5E0h5/Y8niHc5DlaLlWLArTQI7tMrsfQjHV+d9Z
 github.com/prometheus/common v0.66.1/go.mod h1:gcaUsgf3KfRSwHY4dIMXLPV0K/Wg1oZ8+SbZk/HH/dA=
 github.com/prometheus/procfs v0.16.1 h1:hZ15bTNuirocR6u0JZ6BAHHmwS1p8B4P6MRqxtzMyRg=
 github.com/prometheus/procfs v0.16.1/go.mod h1:teAbpZRB1iIAJYREa1LsoWUXykVXA1KlTmWl8x/U+Is=
-github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ=
-github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog=
+github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ=
+github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc=
 github.com/sagikazarmark/locafero v0.11.0 h1:1iurJgmM9G3PA/I+wWYIOw/5SyBtxapeHDcg+AAIFXc=
 github.com/sagikazarmark/locafero v0.11.0/go.mod h1:nVIGvgyzw595SUSUE6tvCp3YYTeHs15MvlmU87WwIik=
 github.com/sourcegraph/conc v0.3.1-0.20240121214520-5f936abd7ae8 h1:+jumHNA0Wrelhe64i8F6HNlS8pkoyMv5sreGx2Ry5Rw=
@@ -62,6 +74,10 @@ github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu
 github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=
 github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8=
 github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU=
+github.com/zeebo/assert v1.3.0 h1:g7C04CbJuIDKNPFHmsk4hwZDO5O+kntRxzaUoNXj+IQ=
+github.com/zeebo/assert v1.3.0/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0=
+github.com/zeebo/xxh3 v1.0.2 h1:xZmwmqxHZA8AI603jOQ0tMqmBr9lPeFwGg6d+xy9DC0=
+github.com/zeebo/xxh3 v1.0.2/go.mod h1:5NWz9Sef7zIDm2JHfFlcQvNekmcEl9ekUZQQKCYaDcA=
 go.uber.org/dig v1.19.0 h1:BACLhebsYdpQ7IROQ1AGPjrXcP5dF80U3gKoFzbaq/4=
 go.uber.org/dig v1.19.0/go.mod h1:Us0rSJiThwCv2GteUN0Q7OKvU7n5J4dxZ9JKUXozFdE=
 go.uber.org/fx v1.24.0 h1:wE8mruvpg2kiiL1Vqd0CC+tr0/24XIB10Iwp2lLWzkg=
@@ -76,18 +92,18 @@ go.yaml.in/yaml/v2 v2.4.2 h1:DzmwEr2rDGHl7lsFgAHxmNz/1NlQ7xLIrlN2h5d1eGI=
 go.yaml.in/yaml/v2 v2.4.2/go.mod h1:081UH+NErpNdqlCXm3TtEran0rJZGxAYx9hb/ELlsPU=
 go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc=
 go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg=
-golang.org/x/mod v0.32.0 h1:9F4d3PHLljb6x//jOyokMv3eX+YDeepZSEo3mFJy93c=
-golang.org/x/mod v0.32.0/go.mod h1:SgipZ/3h2Ci89DlEtEXWUk/HteuRin+HHhN+WbNhguU=
-golang.org/x/net v0.50.0 h1:ucWh9eiCGyDR3vtzso0WMQinm2Dnt8cFMuQa9K33J60=
-golang.org/x/net v0.50.0/go.mod h1:UgoSli3F/pBgdJBHCTc+tp3gmrU4XswgGRgtnwWTfyM=
-golang.org/x/sync v0.19.0 h1:vV+1eWNmZ5geRlYjzm2adRgW2/mcpevXNg50YZtPCE4=
-golang.org/x/sync v0.19.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI=
-golang.org/x/sys v0.41.0 h1:Ivj+2Cp/ylzLiEU89QhWblYnOE9zerudt9Ftecq2C6k=
-golang.org/x/sys v0.41.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks=
-golang.org/x/text v0.34.0 h1:oL/Qq0Kdaqxa1KbNeMKwQq0reLCCaFtqu2eNuSeNHbk=
-golang.org/x/text v0.34.0/go.mod h1:homfLqTYRFyVYemLBFl5GgL/DWEiH5wcsQ5gSh1yziA=
-golang.org/x/tools v0.41.0 h1:a9b8iMweWG+S0OBnlU36rzLp20z1Rp10w+IY2czHTQc=
-golang.org/x/tools v0.41.0/go.mod h1:XSY6eDqxVNiYgezAVqqCeihT4j1U2CCsqvH3WhQpnlg=
+golang.org/x/mod v0.37.0 h1:vF1DjpVEshcIqoEaauuHebaLk1O1forxjxBaVn884JQ=
+golang.org/x/mod v0.37.0/go.mod h1:m8S8VeM9r4dzDwjrKO0a1sZP3YjeMamRRlD+fmR2Q/0=
+golang.org/x/net v0.56.0 h1:Rw8j/hFzGvJUZwNBXnAtf5sVDVt+65SK2C7IxCxZt5o=
+golang.org/x/net v0.56.0/go.mod h1:D3Ku6r+V6JROoZK144D2XfMHFcMq/0zSfLelVTCFKec=
+golang.org/x/sync v0.21.0 h1:HLII4xRRTtCRkxYp4HNFF0Js/Og6q2i++KXbg0gHCwM=
+golang.org/x/sync v0.21.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0=
+golang.org/x/sys v0.46.0 h1:noSf2Fq6F8DBgS+LysIkx7rIExoNHJsxOAtPp4rthXw=
+golang.org/x/sys v0.46.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
+golang.org/x/text v0.39.0 h1:UbZz4pLOvn600D6Oh6GGEI6VAmndrEBLv8/6BEXzyus=
+golang.org/x/text v0.39.0/go.mod h1:3UwRclnC2g0TU9x8PZiyfOajCd1zaUNHF9cvqcQZ+ZM=
+golang.org/x/tools v0.47.0 h1:7Kn5x/d1svx/PzryTsqeoZN4TZwqeH5pGWjefhLi/1Q=
+golang.org/x/tools v0.47.0/go.mod h1:dFHnyTvFWY212G+h7ZY4Vsp/K3U4/7W9TyVaAul8uCA=
 google.golang.org/protobuf v1.36.8 h1:xHScyCOEuuwZEc6UtSOvPbAT4zRh0xcNRYekJwfqyMc=
 google.golang.org/protobuf v1.36.8/go.mod h1:fuxRtAxBytpl4zzqUh6/eyUujkJdNiuEkXntxiD/uRU=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=

internal/config/classify.go

@@ -57,17 +57,22 @@ func ClassifyDNSName(name string) (DNSNameType, error) {
 
 // ClassifyTargets splits a list of DNS names into apex domains and
 // hostnames using the Public Suffix List. It returns an error if any
-// name cannot be classified.
+// name cannot be classified. A name given more than once, in any letter
+// case or with a trailing dot, is kept once.
 func ClassifyTargets(targets []string) ([]string, []string, error) {
 	var domains, hostnames []string
 
+	seen := make(map[string]bool)
+
 	for _, t := range targets {
 		normalized := strings.ToLower(strings.TrimSuffix(strings.TrimSpace(t), "."))
 
-		if normalized == "" {
+		if normalized == "" || seen[normalized] {
 			continue
 		}
 
+		seen[normalized] = true
+
 		typ, classErr := ClassifyDNSName(normalized)
 		if classErr != nil {
 			return nil, nil, classErr

internal/config/classify_test.go

@@ -1,6 +1,7 @@
 package config_test
 
 import (
+	"slices"
 	"testing"
 
 	"sneak.berlin/go/dnswatcher/internal/config"
@@ -17,13 +18,33 @@ func TestClassifyDNSName(t *testing.T) {
 	}{
 		{name: "apex domain simple", input: "example.com", want: config.DNSNameTypeDomain},
 		{name: "hostname simple", input: "www.example.com", want: config.DNSNameTypeHostname},
-		{name: "apex domain multi-part TLD", input: "example.co.uk", want: config.DNSNameTypeDomain},
-		{name: "hostname multi-part TLD", input: "api.example.co.uk", want: config.DNSNameTypeHostname},
+		{
+			name:  "apex domain multi-part TLD",
+			input: "example.co.uk",
+			want:  config.DNSNameTypeDomain,
+		},
+		{
+			name:  "hostname multi-part TLD",
+			input: "api.example.co.uk",
+			want:  config.DNSNameTypeHostname,
+		},
 		{name: "public suffix itself", input: "co.uk", wantErr: true},
 		{name: "empty string", input: "", wantErr: true},
-		{name: "deeply nested hostname", input: "a.b.c.example.com", want: config.DNSNameTypeHostname},
-		{name: "trailing dot stripped", input: "example.com.", want: config.DNSNameTypeDomain},
-		{name: "uppercase normalized", input: "WWW.Example.COM", want: config.DNSNameTypeHostname},
+		{
+			name:  "deeply nested hostname",
+			input: "a.b.c.example.com",
+			want:  config.DNSNameTypeHostname,
+		},
+		{
+			name:  "trailing dot stripped",
+			input: "example.com.",
+			want:  config.DNSNameTypeDomain,
+		},
+		{
+			name:  "uppercase normalized",
+			input: "WWW.Example.COM",
+			want:  config.DNSNameTypeHostname,
+		},
 	}
 
 	for _, tt := range tests {
@@ -73,6 +94,29 @@ func TestClassifyTargets(t *testing.T) {
 	}
 }
 
+func TestClassifyTargetsKeepsEachNameOnce(t *testing.T) {
+	t.Parallel()
+
+	domains, hostnames, err := config.ClassifyTargets([]string{
+		"example.org",
+		"Example.org.",
+		"www.example.org",
+		"EXAMPLE.ORG",
+		"WWW.Example.org.",
+	})
+	if err != nil {
+		t.Fatalf("unexpected error: %v", err)
+	}
+
+	if !slices.Equal(domains, []string{"example.org"}) {
+		t.Errorf("domains = %v, want [example.org]", domains)
+	}
+
+	if !slices.Equal(hostnames, []string{"www.example.org"}) {
+		t.Errorf("hostnames = %v, want [www.example.org]", hostnames)
+	}
+}
+
 func TestClassifyTargetsRejectsPublicSuffix(t *testing.T) {
 	t.Parallel()
 

internal/config/config.go

@@ -23,6 +23,18 @@ const (
 	defaultTLSExpiryWarning = 7
 )
 
+// ErrNoTargets is returned when no monitoring targets are configured.
+var ErrNoTargets = errors.New(
+	"no monitoring targets configured: set DNSWATCHER_TARGETS environment variable",
+)
+
+// ErrInvalidInterval is returned when DNSWATCHER_DNS_INTERVAL or
+// DNSWATCHER_TLS_INTERVAL is set but is not a positive duration. An empty
+// value counts as unset and means the default.
+var ErrInvalidInterval = errors.New(
+	"interval must be a positive duration such as 30m or 1h",
+)
+
 // Params contains dependencies for Config.
 type Params struct {
 	fx.In
@@ -33,23 +45,24 @@ type Params struct {
 
 // Config holds application configuration.
 type Config struct {
-	Port              int
-	Debug             bool
-	DataDir           string
-	Domains           []string
-	Hostnames         []string
-	SlackWebhook      string
-	MattermostWebhook string
-	NtfyTopic         string
-	DNSInterval       time.Duration
-	TLSInterval       time.Duration
-	TLSExpiryWarning  int
-	SentryDSN         string
-	MaintenanceMode   bool
-	MetricsUsername   string
-	MetricsPassword   string
-	params            *Params
-	log               *slog.Logger
+	Port                 int
+	Debug                bool
+	DataDir              string
+	Domains              []string
+	Hostnames            []string
+	SlackWebhook         string
+	MattermostWebhook    string
+	NtfyTopic            string
+	DNSInterval          time.Duration
+	TLSInterval          time.Duration
+	TLSExpiryWarning     int
+	SentryDSN            string
+	MaintenanceMode      bool
+	MetricsUsername      string
+	MetricsPassword      string
+	SendTestNotification bool
+	params               *Params
+	log                  *slog.Logger
 }
 
 // New creates a new Config instance from environment and config files.
@@ -88,7 +101,7 @@ func setupViper(name string) {
 
 	viper.SetDefault("PORT", defaultPort)
 	viper.SetDefault("DEBUG", false)
-	viper.SetDefault("DATA_DIR", "./data")
+	viper.SetDefault("DATA_DIR", "/var/lib/"+name)
 	viper.SetDefault("TARGETS", "")
 	viper.SetDefault("SLACK_WEBHOOK", "")
 	viper.SetDefault("MATTERMOST_WEBHOOK", "")
@@ -100,6 +113,7 @@ func setupViper(name string) {
 	viper.SetDefault("MAINTENANCE_MODE", false)
 	viper.SetDefault("METRICS_USERNAME", "")
 	viper.SetDefault("METRICS_PASSWORD", "")
+	viper.SetDefault("SEND_TEST_NOTIFICATION", false)
 }
 
 func buildConfig(
@@ -118,48 +132,76 @@ func buildConfig(
 		}
 	}
 
-	dnsInterval, err := time.ParseDuration(
-		viper.GetString("DNS_INTERVAL"),
-	)
+	dnsInterval, err := parseInterval("DNS_INTERVAL")
 	if err != nil {
-		dnsInterval = defaultDNSInterval
+		return nil, err
 	}
 
-	tlsInterval, err := time.ParseDuration(
-		viper.GetString("TLS_INTERVAL"),
-	)
+	tlsInterval, err := parseInterval("TLS_INTERVAL")
 	if err != nil {
-		tlsInterval = defaultTLSInterval
+		return nil, err
 	}
 
+	domains, hostnames, err := parseAndValidateTargets()
+	if err != nil {
+		return nil, err
+	}
+
+	cfg := &Config{
+		Port:                 viper.GetInt("PORT"),
+		Debug:                viper.GetBool("DEBUG"),
+		DataDir:              viper.GetString("DATA_DIR"),
+		Domains:              domains,
+		Hostnames:            hostnames,
+		SlackWebhook:         viper.GetString("SLACK_WEBHOOK"),
+		MattermostWebhook:    viper.GetString("MATTERMOST_WEBHOOK"),
+		NtfyTopic:            viper.GetString("NTFY_TOPIC"),
+		DNSInterval:          dnsInterval,
+		TLSInterval:          tlsInterval,
+		TLSExpiryWarning:     viper.GetInt("TLS_EXPIRY_WARNING"),
+		SentryDSN:            viper.GetString("SENTRY_DSN"),
+		MaintenanceMode:      viper.GetBool("MAINTENANCE_MODE"),
+		MetricsUsername:      viper.GetString("METRICS_USERNAME"),
+		MetricsPassword:      viper.GetString("METRICS_PASSWORD"),
+		SendTestNotification: viper.GetBool("SEND_TEST_NOTIFICATION"),
+		params:               params,
+		log:                  log,
+	}
+
+	return cfg, nil
+}
+
+// parseInterval reads the DNSWATCHER_-prefixed setting key as a duration. A
+// value that does not parse, or is zero or negative, is an error naming the
+// variable and the value; an unset variable has its default from setupViper.
+func parseInterval(key string) (time.Duration, error) {
+	value := viper.GetString(key)
+
+	interval, err := time.ParseDuration(value)
+	if err != nil || interval <= 0 {
+		return 0, fmt.Errorf(
+			"invalid DNSWATCHER_%s %q: %w", key, value, ErrInvalidInterval,
+		)
+	}
+
+	return interval, nil
+}
+
+func parseAndValidateTargets() ([]string, []string, error) {
 	domains, hostnames, err := ClassifyTargets(
 		parseCSV(viper.GetString("TARGETS")),
 	)
 	if err != nil {
-		return nil, fmt.Errorf("invalid targets configuration: %w", err)
+		return nil, nil, fmt.Errorf(
+			"invalid targets configuration: %w", err,
+		)
 	}
 
-	cfg := &Config{
-		Port:              viper.GetInt("PORT"),
-		Debug:             viper.GetBool("DEBUG"),
-		DataDir:           viper.GetString("DATA_DIR"),
-		Domains:           domains,
-		Hostnames:         hostnames,
-		SlackWebhook:      viper.GetString("SLACK_WEBHOOK"),
-		MattermostWebhook: viper.GetString("MATTERMOST_WEBHOOK"),
-		NtfyTopic:         viper.GetString("NTFY_TOPIC"),
-		DNSInterval:       dnsInterval,
-		TLSInterval:       tlsInterval,
-		TLSExpiryWarning:  viper.GetInt("TLS_EXPIRY_WARNING"),
-		SentryDSN:         viper.GetString("SENTRY_DSN"),
-		MaintenanceMode:   viper.GetBool("MAINTENANCE_MODE"),
-		MetricsUsername:   viper.GetString("METRICS_USERNAME"),
-		MetricsPassword:   viper.GetString("METRICS_PASSWORD"),
-		params:            params,
-		log:               log,
+	if len(domains) == 0 && len(hostnames) == 0 {
+		return nil, nil, ErrNoTargets
 	}
 
-	return cfg, nil
+	return domains, hostnames, nil
 }
 
 func parseCSV(input string) []string {

internal/config/config_test.go

@@ -0,0 +1,270 @@
+package config_test
+
+import (
+	"strconv"
+	"testing"
+	"time"
+
+	"github.com/spf13/viper"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"sneak.berlin/go/dnswatcher/internal/config"
+	"sneak.berlin/go/dnswatcher/internal/globals"
+	"sneak.berlin/go/dnswatcher/internal/logger"
+)
+
+// newTestParams creates config.Params suitable for testing
+// without requiring the fx dependency injection framework.
+func newTestParams(t *testing.T) config.Params {
+	t.Helper()
+
+	g := &globals.Globals{
+		Appname: "dnswatcher",
+		Version: "test",
+	}
+
+	l, err := logger.New(nil, logger.Params{Globals: g})
+	require.NoError(t, err, "failed to create logger")
+
+	return config.Params{
+		Globals: g,
+		Logger:  l,
+	}
+}
+
+// These tests exercise viper global state and MUST NOT use
+// t.Parallel(). Each test resets viper for isolation.
+
+func TestNew_DefaultValues(t *testing.T) {
+	viper.Reset()
+	t.Setenv("DNSWATCHER_TARGETS", "example.com,www.example.com")
+
+	cfg, err := config.New(nil, newTestParams(t))
+	require.NoError(t, err)
+
+	assert.Equal(t, 8080, cfg.Port)
+	assert.False(t, cfg.Debug)
+	assert.Equal(t, "/var/lib/dnswatcher", cfg.DataDir)
+	assert.Equal(t, time.Hour, cfg.DNSInterval)
+	assert.Equal(t, 12*time.Hour, cfg.TLSInterval)
+	assert.Equal(t, 7, cfg.TLSExpiryWarning)
+	assert.False(t, cfg.MaintenanceMode)
+	assert.Empty(t, cfg.SlackWebhook)
+	assert.Empty(t, cfg.MattermostWebhook)
+	assert.Empty(t, cfg.NtfyTopic)
+	assert.Empty(t, cfg.SentryDSN)
+	assert.Empty(t, cfg.MetricsUsername)
+	assert.Empty(t, cfg.MetricsPassword)
+	assert.False(t, cfg.SendTestNotification)
+}
+
+func TestNew_EnvironmentOverrides(t *testing.T) {
+	viper.Reset()
+	t.Setenv("DNSWATCHER_TARGETS", "example.com")
+	t.Setenv("PORT", "9090")
+	t.Setenv("DNSWATCHER_DEBUG", "true")
+	t.Setenv("DNSWATCHER_DATA_DIR", "/tmp/test-data")
+	t.Setenv("DNSWATCHER_DNS_INTERVAL", "30m")
+	t.Setenv("DNSWATCHER_TLS_INTERVAL", "6h")
+	t.Setenv("DNSWATCHER_TLS_EXPIRY_WARNING", "14")
+	t.Setenv("DNSWATCHER_SLACK_WEBHOOK", "https://hooks.slack.com/t")
+	t.Setenv("DNSWATCHER_MATTERMOST_WEBHOOK", "https://mm.test/hooks/t")
+	t.Setenv("DNSWATCHER_NTFY_TOPIC", "https://ntfy.sh/test")
+	t.Setenv("DNSWATCHER_SENTRY_DSN", "https://sentry.test/1")
+	t.Setenv("DNSWATCHER_MAINTENANCE_MODE", "true")
+	t.Setenv("DNSWATCHER_METRICS_USERNAME", "admin")
+	t.Setenv("DNSWATCHER_METRICS_PASSWORD", "secret")
+	t.Setenv("DNSWATCHER_SEND_TEST_NOTIFICATION", "true")
+
+	cfg, err := config.New(nil, newTestParams(t))
+	require.NoError(t, err)
+
+	assert.Equal(t, 9090, cfg.Port)
+	assert.True(t, cfg.Debug)
+	assert.Equal(t, "/tmp/test-data", cfg.DataDir)
+	assert.Equal(t, 30*time.Minute, cfg.DNSInterval)
+	assert.Equal(t, 6*time.Hour, cfg.TLSInterval)
+	assert.Equal(t, 14, cfg.TLSExpiryWarning)
+	assert.Equal(t, "https://hooks.slack.com/t", cfg.SlackWebhook)
+	assert.Equal(t, "https://mm.test/hooks/t", cfg.MattermostWebhook)
+	assert.Equal(t, "https://ntfy.sh/test", cfg.NtfyTopic)
+	assert.Equal(t, "https://sentry.test/1", cfg.SentryDSN)
+	assert.True(t, cfg.MaintenanceMode)
+	assert.Equal(t, "admin", cfg.MetricsUsername)
+	assert.Equal(t, "secret", cfg.MetricsPassword)
+	assert.True(t, cfg.SendTestNotification)
+}
+
+func TestNew_NoTargetsError(t *testing.T) {
+	viper.Reset()
+	t.Setenv("DNSWATCHER_TARGETS", "")
+
+	_, err := config.New(nil, newTestParams(t))
+	require.Error(t, err)
+	assert.ErrorIs(t, err, config.ErrNoTargets)
+}
+
+func TestNew_OnlyEmptyCSVSegments(t *testing.T) {
+	viper.Reset()
+	t.Setenv("DNSWATCHER_TARGETS", " , , ")
+
+	_, err := config.New(nil, newTestParams(t))
+	require.Error(t, err)
+	assert.ErrorIs(t, err, config.ErrNoTargets)
+}
+
+// TestNew_InvalidIntervalStopsStartup checks values that must stop startup;
+// TestNew_DefaultValues and TestNew_EmptyIntervalMeansDefault check that an
+// unset or empty interval means the default.
+func TestNew_InvalidIntervalStopsStartup(t *testing.T) {
+	variables := []string{"DNSWATCHER_DNS_INTERVAL", "DNSWATCHER_TLS_INTERVAL"}
+	values := []string{
+		"banana", // not a duration
+		"5",      // no unit
+		"1d",     // days are not a unit time.ParseDuration knows
+		"0",      // zero
+		"-1h",    // negative
+	}
+
+	for _, variable := range variables {
+		for _, value := range values {
+			t.Run(variable+"="+value, func(t *testing.T) {
+				viper.Reset()
+				t.Setenv("DNSWATCHER_TARGETS", "example.com")
+				t.Setenv(variable, value)
+
+				_, err := config.New(nil, newTestParams(t))
+				require.ErrorIs(t, err, config.ErrInvalidInterval)
+				require.ErrorContains(t, err, variable)
+				require.ErrorContains(t, err, strconv.Quote(value))
+			})
+		}
+	}
+}
+
+func TestNew_EmptyIntervalMeansDefault(t *testing.T) {
+	viper.Reset()
+	t.Setenv("DNSWATCHER_TARGETS", "example.com")
+	t.Setenv("DNSWATCHER_DNS_INTERVAL", "")
+	t.Setenv("DNSWATCHER_TLS_INTERVAL", "")
+
+	cfg, err := config.New(nil, newTestParams(t))
+	require.NoError(t, err)
+	assert.Equal(t, time.Hour, cfg.DNSInterval)
+	assert.Equal(t, 12*time.Hour, cfg.TLSInterval)
+}
+
+func TestNew_DebugEnablesDebugLogging(t *testing.T) {
+	viper.Reset()
+	t.Setenv("DNSWATCHER_TARGETS", "example.com")
+	t.Setenv("DNSWATCHER_DEBUG", "true")
+
+	cfg, err := config.New(nil, newTestParams(t))
+	require.NoError(t, err)
+	assert.True(t, cfg.Debug)
+}
+
+func TestNew_PortEnvNotPrefixed(t *testing.T) {
+	viper.Reset()
+	t.Setenv("DNSWATCHER_TARGETS", "example.com")
+	t.Setenv("PORT", "3000")
+
+	cfg, err := config.New(nil, newTestParams(t))
+	require.NoError(t, err)
+	assert.Equal(t, 3000, cfg.Port,
+		"PORT env should work without DNSWATCHER_ prefix")
+}
+
+func TestNew_TargetClassification(t *testing.T) {
+	viper.Reset()
+	t.Setenv("DNSWATCHER_TARGETS",
+		"example.com,www.example.com,api.example.com,example.org")
+
+	cfg, err := config.New(nil, newTestParams(t))
+	require.NoError(t, err)
+
+	// example.com and example.org are apex domains
+	assert.Len(t, cfg.Domains, 2)
+	// www.example.com and api.example.com are hostnames
+	assert.Len(t, cfg.Hostnames, 2)
+}
+
+func TestNew_InvalidTargetPublicSuffix(t *testing.T) {
+	viper.Reset()
+	t.Setenv("DNSWATCHER_TARGETS", "co.uk")
+
+	_, err := config.New(nil, newTestParams(t))
+	require.Error(t, err, "public suffix should be rejected")
+}
+
+func TestNew_EmptyAppnameDefaultsToDnswatcher(t *testing.T) {
+	viper.Reset()
+	t.Setenv("DNSWATCHER_TARGETS", "example.com")
+
+	g := &globals.Globals{Appname: "", Version: "test"}
+
+	l, err := logger.New(nil, logger.Params{Globals: g})
+	require.NoError(t, err)
+
+	cfg, err := config.New(
+		nil, config.Params{Globals: g, Logger: l},
+	)
+	require.NoError(t, err)
+	assert.Equal(t, 8080, cfg.Port,
+		"defaults should load when appname is empty")
+}
+
+func TestNew_TargetsWithWhitespace(t *testing.T) {
+	viper.Reset()
+	t.Setenv("DNSWATCHER_TARGETS", " example.com , www.example.com ")
+
+	cfg, err := config.New(nil, newTestParams(t))
+	require.NoError(t, err)
+	assert.Equal(t, 2, len(cfg.Domains)+len(cfg.Hostnames),
+		"whitespace around targets should be trimmed")
+}
+
+func TestNew_TargetsWithTrailingComma(t *testing.T) {
+	viper.Reset()
+	t.Setenv("DNSWATCHER_TARGETS", "example.com,www.example.com,")
+
+	cfg, err := config.New(nil, newTestParams(t))
+	require.NoError(t, err)
+	assert.Equal(t, 2, len(cfg.Domains)+len(cfg.Hostnames),
+		"trailing comma should be ignored")
+}
+
+func TestNew_CustomDNSIntervalDuration(t *testing.T) {
+	viper.Reset()
+	t.Setenv("DNSWATCHER_TARGETS", "example.com")
+	t.Setenv("DNSWATCHER_DNS_INTERVAL", "5s")
+
+	cfg, err := config.New(nil, newTestParams(t))
+	require.NoError(t, err)
+	assert.Equal(t, 5*time.Second, cfg.DNSInterval)
+}
+
+func TestStatePath(t *testing.T) {
+	t.Parallel()
+
+	tests := []struct {
+		name    string
+		dataDir string
+		want    string
+	}{
+		{"default", "/var/lib/dnswatcher", "/var/lib/dnswatcher/state.json"},
+		{"absolute", "/var/lib/dw", "/var/lib/dw/state.json"},
+		{"nested", "/opt/app/data", "/opt/app/data/state.json"},
+		{"empty", "", "/state.json"},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			t.Parallel()
+
+			cfg := &config.Config{DataDir: tt.dataDir}
+			assert.Equal(t, tt.want, cfg.StatePath())
+		})
+	}
+}

internal/config/export_test.go

@@ -0,0 +1,6 @@
+package config
+
+// ParseCSVForTest exports parseCSV for use in external tests.
+func ParseCSVForTest(input string) []string {
+	return parseCSV(input)
+}

internal/config/parsecsv_test.go

@@ -0,0 +1,44 @@
+package config_test
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"sneak.berlin/go/dnswatcher/internal/config"
+)
+
+func TestParseCSV(t *testing.T) {
+	t.Parallel()
+
+	tests := []struct {
+		name  string
+		input string
+		want  []string
+	}{
+		{"empty string", "", nil},
+		{"single value", "a", []string{"a"}},
+		{"multiple values", "a,b,c", []string{"a", "b", "c"}},
+		{"whitespace trimmed", " a , b ", []string{"a", "b"}},
+		{"trailing comma", "a,b,", []string{"a", "b"}},
+		{"leading comma", ",a,b", []string{"a", "b"}},
+		{"consecutive commas", "a,,b", []string{"a", "b"}},
+		{"all empty segments", ",,,", nil},
+		{"whitespace only", " , , ", nil},
+		{"tabs", "\ta\t,\tb\t", []string{"a", "b"}},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			t.Parallel()
+
+			got := config.ParseCSVForTest(tt.input)
+			require.Len(t, got, len(tt.want))
+
+			for i, w := range tt.want {
+				assert.Equal(t, w, got[i])
+			}
+		})
+	}
+}

internal/globals/globals.go

@@ -12,17 +12,15 @@ import (
 //
 //nolint:gochecknoglobals // Required for ldflags injection at build time
 var (
-	mu        sync.RWMutex
-	appname   string
-	version   string
-	buildarch string
+	mu      sync.RWMutex
+	appname string
+	version string
 )
 
 // Globals holds build-time variables for dependency injection.
 type Globals struct {
-	Appname   string
-	Version   string
-	Buildarch string
+	Appname string
+	Version string
 }
 
 // New creates a new Globals instance from package-level variables.
@@ -31,9 +29,8 @@ func New(_ fx.Lifecycle) (*Globals, error) {
 	defer mu.RUnlock()
 
 	return &Globals{
-		Appname:   appname,
-		Version:   version,
-		Buildarch: buildarch,
+		Appname: appname,
+		Version: version,
 	}, nil
 }
 
@@ -52,11 +49,3 @@ func SetVersion(ver string) {
 
 	version = ver
 }
-
-// SetBuildarch sets the build architecture.
-func SetBuildarch(arch string) {
-	mu.Lock()
-	defer mu.Unlock()
-
-	buildarch = arch
-}

internal/globals/globals_test.go

@@ -0,0 +1,50 @@
+package globals_test
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"sneak.berlin/go/dnswatcher/internal/globals"
+)
+
+// TestGlobals exercises the package-level version and appname
+// variables through their setters and read-back via New. These are
+// shared package state, so the test mutates a global and must run
+// sequentially; it cannot use t.Parallel().
+//
+//nolint:paralleltest // mutates shared package-level globals, must run sequentially
+func TestGlobals(t *testing.T) {
+	versions := []string{"v1.2.3", "dev", "", "v1.2.3-4-gabcdef"}
+	for _, want := range versions {
+		globals.SetVersion(want)
+
+		g, err := globals.New(nil)
+		require.NoError(t, err)
+		assert.Equal(t, want, g.Version,
+			"New must surface the version set by SetVersion")
+	}
+
+	names := []string{"dnswatcher", "other", ""}
+	for _, want := range names {
+		globals.SetAppname(want)
+
+		g, err := globals.New(nil)
+		require.NoError(t, err)
+		assert.Equal(t, want, g.Appname,
+			"New must surface the appname set by SetAppname")
+	}
+
+	// New returns a snapshot: a later SetVersion must not mutate a
+	// Globals handed out earlier.
+	globals.SetVersion("first")
+
+	g, err := globals.New(nil)
+	require.NoError(t, err)
+
+	globals.SetVersion("second")
+	assert.Equal(t, "first", g.Version,
+		"a Globals returned by New must not change when the "+
+			"package variable is set again")
+}

internal/handlers/dashboard.go

@@ -0,0 +1,151 @@
+package handlers
+
+import (
+	"embed"
+	"fmt"
+	"html/template"
+	"math"
+	"net/http"
+	"strings"
+	"time"
+
+	"sneak.berlin/go/dnswatcher/internal/notify"
+	"sneak.berlin/go/dnswatcher/internal/state"
+)
+
+//go:embed templates/dashboard.html
+var dashboardFS embed.FS
+
+// Time unit constants for relative time calculations.
+const (
+	secondsPerMinute = 60
+	minutesPerHour   = 60
+	hoursPerDay      = 24
+)
+
+// newDashboardTemplate parses the embedded dashboard HTML
+// template with helper functions.
+func newDashboardTemplate() *template.Template {
+	funcs := template.FuncMap{
+		"relTime":       relTime,
+		"joinStrings":   joinStrings,
+		"formatRecords": formatRecords,
+		"expiryDays":    expiryDays,
+	}
+
+	return template.Must(
+		template.New("dashboard.html").
+			Funcs(funcs).
+			ParseFS(dashboardFS, "templates/dashboard.html"),
+	)
+}
+
+// dashboardData is the data passed to the dashboard template.
+type dashboardData struct {
+	Snapshot    state.Snapshot
+	Alerts      []notify.AlertEntry
+	StateAge    string
+	GeneratedAt string
+}
+
+// HandleDashboard returns the dashboard page handler.
+func (h *Handlers) HandleDashboard() http.HandlerFunc {
+	tmpl := newDashboardTemplate()
+
+	return func(
+		writer http.ResponseWriter,
+		_ *http.Request,
+	) {
+		snap := h.state.GetSnapshot()
+		alerts := h.notifyHistory.Recent()
+
+		data := dashboardData{
+			Snapshot:    snap,
+			Alerts:      alerts,
+			StateAge:    relTime(snap.LastUpdated),
+			GeneratedAt: time.Now().UTC().Format("2006-01-02 15:04:05"),
+		}
+
+		writer.Header().Set(
+			"Content-Type", "text/html; charset=utf-8",
+		)
+
+		err := tmpl.Execute(writer, data)
+		if err != nil {
+			h.log.Error(
+				"dashboard template error",
+				"error", err,
+			)
+		}
+	}
+}
+
+// relTime returns a human-readable relative time string such
+// as "2 minutes ago" or "never" for zero times.
+func relTime(t time.Time) string {
+	if t.IsZero() {
+		return "never"
+	}
+
+	d := time.Since(t)
+	if d < 0 {
+		return "just now"
+	}
+
+	seconds := int(math.Round(d.Seconds()))
+	if seconds < secondsPerMinute {
+		return fmt.Sprintf("%ds ago", seconds)
+	}
+
+	minutes := seconds / secondsPerMinute
+	if minutes < minutesPerHour {
+		return fmt.Sprintf("%dm ago", minutes)
+	}
+
+	hours := minutes / minutesPerHour
+	if hours < hoursPerDay {
+		return fmt.Sprintf(
+			"%dh %dm ago", hours, minutes%minutesPerHour,
+		)
+	}
+
+	days := hours / hoursPerDay
+
+	return fmt.Sprintf(
+		"%dd %dh ago", days, hours%hoursPerDay,
+	)
+}
+
+// joinStrings joins a string slice with a separator.
+func joinStrings(items []string, sep string) string {
+	return strings.Join(items, sep)
+}
+
+// formatRecords formats a map of record type → values into a
+// compact display string.
+func formatRecords(records map[string][]string) string {
+	if len(records) == 0 {
+		return "-"
+	}
+
+	var parts []string
+
+	for rtype, values := range records {
+		for _, v := range values {
+			parts = append(parts, rtype+": "+v)
+		}
+	}
+
+	return strings.Join(parts, ", ")
+}
+
+// expiryDays returns the number of days until the given time,
+// rounded down. Returns 0 if already expired.
+func expiryDays(t time.Time) int {
+	d := time.Until(t).Hours() / hoursPerDay
+	if d < 0 {
+		return 0
+	}
+
+	return int(d)
+}

internal/handlers/dashboard_test.go

@@ -0,0 +1,80 @@
+package handlers_test
+
+import (
+	"testing"
+	"time"
+
+	"sneak.berlin/go/dnswatcher/internal/handlers"
+)
+
+func TestRelTime(t *testing.T) {
+	t.Parallel()
+
+	tests := []struct {
+		name string
+		dur  time.Duration
+		want string
+	}{
+		{"zero", 0, "never"},
+		{"seconds", 30 * time.Second, "30s ago"},
+		{"minutes", 5 * time.Minute, "5m ago"},
+		{"hours", 2*time.Hour + 15*time.Minute, "2h 15m ago"},
+		{"days", 48*time.Hour + 3*time.Hour, "2d 3h ago"},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			t.Parallel()
+
+			var input time.Time
+			if tt.dur > 0 {
+				input = time.Now().Add(-tt.dur)
+			}
+
+			got := handlers.RelTime(input)
+			if got != tt.want {
+				t.Errorf(
+					"RelTime(%v) = %q, want %q",
+					tt.dur, got, tt.want,
+				)
+			}
+		})
+	}
+}
+
+func TestExpiryDays(t *testing.T) {
+	t.Parallel()
+
+	// 10 days from now.
+	future := time.Now().Add(10 * 24 * time.Hour)
+
+	days := handlers.ExpiryDays(future)
+	if days < 9 || days > 10 {
+		t.Errorf("expected ~10 days, got %d", days)
+	}
+
+	// Already expired.
+	past := time.Now().Add(-24 * time.Hour)
+
+	days = handlers.ExpiryDays(past)
+	if days != 0 {
+		t.Errorf("expected 0 for expired, got %d", days)
+	}
+}
+
+func TestFormatRecords(t *testing.T) {
+	t.Parallel()
+
+	got := handlers.FormatRecords(nil)
+	if got != "-" {
+		t.Errorf("expected -, got %q", got)
+	}
+
+	got = handlers.FormatRecords(map[string][]string{
+		"A": {"1.2.3.4"},
+	})
+
+	if got != "A: 1.2.3.4" {
+		t.Errorf("unexpected format: %q", got)
+	}
+}

internal/handlers/export_test.go

@@ -0,0 +1,18 @@
+package handlers
+
+import "time"
+
+// RelTime exports relTime for testing.
+func RelTime(t time.Time) string {
+	return relTime(t)
+}
+
+// ExpiryDays exports expiryDays for testing.
+func ExpiryDays(t time.Time) int {
+	return expiryDays(t)
+}
+
+// FormatRecords exports formatRecords for testing.
+func FormatRecords(records map[string][]string) string {
+	return formatRecords(records)
+}

internal/handlers/handlers.go

@@ -11,6 +11,8 @@ import (
 	"sneak.berlin/go/dnswatcher/internal/globals"
 	"sneak.berlin/go/dnswatcher/internal/healthcheck"
 	"sneak.berlin/go/dnswatcher/internal/logger"
+	"sneak.berlin/go/dnswatcher/internal/notify"
+	"sneak.berlin/go/dnswatcher/internal/state"
 )
 
 // Params contains dependencies for Handlers.
@@ -20,23 +22,29 @@ type Params struct {
 	Logger      *logger.Logger
 	Globals     *globals.Globals
 	Healthcheck *healthcheck.Healthcheck
+	State       *state.State
+	Notify      *notify.Service
 }
 
 // Handlers provides HTTP request handlers.
 type Handlers struct {
-	log     *slog.Logger
-	params  *Params
-	globals *globals.Globals
-	hc      *healthcheck.Healthcheck
+	log           *slog.Logger
+	params        *Params
+	globals       *globals.Globals
+	hc            *healthcheck.Healthcheck
+	state         *state.State
+	notifyHistory *notify.AlertHistory
 }
 
 // New creates a new Handlers instance.
 func New(_ fx.Lifecycle, params Params) (*Handlers, error) {
 	return &Handlers{
-		log:     params.Logger.Get(),
-		params:  &params,
-		globals: params.Globals,
-		hc:      params.Healthcheck,
+		log:           params.Logger.Get(),
+		params:        &params,
+		globals:       params.Globals,
+		hc:            params.Healthcheck,
+		state:         params.State,
+		notifyHistory: params.Notify.History(),
 	}, nil
 }
 

internal/handlers/status.go

@@ -2,22 +2,217 @@ package handlers
 
 import (
 	"net/http"
+	"sort"
+	"time"
+
+	"sneak.berlin/go/dnswatcher/internal/state"
 )
 
+// statusDomainInfo holds status information for a monitored domain.
+type statusDomainInfo struct {
+	Nameservers []string  `json:"nameservers"`
+	LastChecked time.Time `json:"lastChecked"`
+}
+
+// statusHostnameNSInfo holds per-nameserver status for a hostname.
+type statusHostnameNSInfo struct {
+	Records     map[string][]string `json:"records"`
+	Status      string              `json:"status"`
+	LastChecked time.Time           `json:"lastChecked"`
+}
+
+// statusHostnameInfo holds status information for a monitored hostname.
+type statusHostnameInfo struct {
+	Nameservers map[string]*statusHostnameNSInfo `json:"nameservers"`
+	LastChecked time.Time                        `json:"lastChecked"`
+}
+
+// statusPortInfo holds status information for a monitored port.
+type statusPortInfo struct {
+	Open        bool      `json:"open"`
+	Hostnames   []string  `json:"hostnames"`
+	LastChecked time.Time `json:"lastChecked"`
+}
+
+// statusCertificateInfo holds status information for a TLS certificate.
+type statusCertificateInfo struct {
+	CommonName              string    `json:"commonName"`
+	Issuer                  string    `json:"issuer"`
+	NotAfter                time.Time `json:"notAfter"`
+	SubjectAlternativeNames []string  `json:"subjectAlternativeNames"`
+	Status                  string    `json:"status"`
+	LastChecked             time.Time `json:"lastChecked"`
+}
+
+// statusCounts holds summary counts of monitored resources.
+type statusCounts struct {
+	Domains      int `json:"domains"`
+	Hostnames    int `json:"hostnames"`
+	Ports        int `json:"ports"`
+	PortsOpen    int `json:"portsOpen"`
+	Certificates int `json:"certificates"`
+	CertsOK      int `json:"certificatesOk"`
+	CertsError   int `json:"certificatesError"`
+}
+
+// statusResponse is the full /api/v1/status response.
+type statusResponse struct {
+	Status       string                            `json:"status"`
+	LastUpdated  time.Time                         `json:"lastUpdated"`
+	Counts       statusCounts                      `json:"counts"`
+	Domains      map[string]*statusDomainInfo      `json:"domains"`
+	Hostnames    map[string]*statusHostnameInfo    `json:"hostnames"`
+	Ports        map[string]*statusPortInfo        `json:"ports"`
+	Certificates map[string]*statusCertificateInfo `json:"certificates"`
+}
+
 // HandleStatus returns the monitoring status handler.
 func (h *Handlers) HandleStatus() http.HandlerFunc {
-	type response struct {
-		Status string `json:"status"`
-	}
-
 	return func(
 		writer http.ResponseWriter,
 		request *http.Request,
 	) {
+		snap := h.state.GetSnapshot()
+
+		resp := buildStatusResponse(snap)
+
 		h.respondJSON(
 			writer, request,
-			&response{Status: "ok"},
+			resp,
 			http.StatusOK,
 		)
 	}
 }
+
+// buildStatusResponse constructs the full status response from
+// the current monitoring snapshot.
+func buildStatusResponse(
+	snap state.Snapshot,
+) *statusResponse {
+	resp := &statusResponse{
+		Status:       "ok",
+		LastUpdated:  snap.LastUpdated,
+		Domains:      make(map[string]*statusDomainInfo),
+		Hostnames:    make(map[string]*statusHostnameInfo),
+		Ports:        make(map[string]*statusPortInfo),
+		Certificates: make(map[string]*statusCertificateInfo),
+	}
+
+	buildDomains(snap, resp)
+	buildHostnames(snap, resp)
+	buildPorts(snap, resp)
+	buildCertificates(snap, resp)
+	buildCounts(resp)
+
+	return resp
+}
+
+func buildDomains(
+	snap state.Snapshot,
+	resp *statusResponse,
+) {
+	for name, ds := range snap.Domains {
+		ns := make([]string, len(ds.Nameservers))
+		copy(ns, ds.Nameservers)
+		sort.Strings(ns)
+
+		resp.Domains[name] = &statusDomainInfo{
+			Nameservers: ns,
+			LastChecked: ds.LastChecked,
+		}
+	}
+}
+
+func buildHostnames(
+	snap state.Snapshot,
+	resp *statusResponse,
+) {
+	for name, hs := range snap.Hostnames {
+		info := &statusHostnameInfo{
+			Nameservers: make(map[string]*statusHostnameNSInfo),
+			LastChecked: hs.LastChecked,
+		}
+
+		for ns, nsState := range hs.RecordsByNameserver {
+			recs := make(map[string][]string, len(nsState.Records))
+			for rtype, vals := range nsState.Records {
+				copied := make([]string, len(vals))
+				copy(copied, vals)
+				recs[rtype] = copied
+			}
+
+			info.Nameservers[ns] = &statusHostnameNSInfo{
+				Records:     recs,
+				Status:      nsState.Status,
+				LastChecked: nsState.LastChecked,
+			}
+		}
+
+		resp.Hostnames[name] = info
+	}
+}
+
+func buildPorts(
+	snap state.Snapshot,
+	resp *statusResponse,
+) {
+	for key, ps := range snap.Ports {
+		hostnames := make([]string, len(ps.Hostnames))
+		copy(hostnames, ps.Hostnames)
+		sort.Strings(hostnames)
+
+		resp.Ports[key] = &statusPortInfo{
+			Open:        ps.Open,
+			Hostnames:   hostnames,
+			LastChecked: ps.LastChecked,
+		}
+	}
+}
+
+func buildCertificates(
+	snap state.Snapshot,
+	resp *statusResponse,
+) {
+	for key, cs := range snap.Certificates {
+		sans := make([]string, len(cs.SubjectAlternativeNames))
+		copy(sans, cs.SubjectAlternativeNames)
+
+		resp.Certificates[key] = &statusCertificateInfo{
+			CommonName:              cs.CommonName,
+			Issuer:                  cs.Issuer,
+			NotAfter:                cs.NotAfter,
+			SubjectAlternativeNames: sans,
+			Status:                  cs.Status,
+			LastChecked:             cs.LastChecked,
+		}
+	}
+}
+
+func buildCounts(resp *statusResponse) {
+	var portsOpen, certsOK, certsError int
+
+	for _, ps := range resp.Ports {
+		if ps.Open {
+			portsOpen++
+		}
+	}
+
+	for _, cs := range resp.Certificates {
+		switch cs.Status {
+		case "ok":
+			certsOK++
+		case "error":
+			certsError++
+		}
+	}
+
+	resp.Counts = statusCounts{
+		Domains:      len(resp.Domains),
+		Hostnames:    len(resp.Hostnames),
+		Ports:        len(resp.Ports),
+		PortsOpen:    portsOpen,
+		Certificates: len(resp.Certificates),
+		CertsOK:      certsOK,
+		CertsError:   certsError,
+	}
+}

internal/handlers/templates/dashboard.html

@@ -0,0 +1,370 @@
+<!doctype html>
+<html lang="en" class="bg-slate-950">
+  <head>
+    <meta charset="utf-8" />
+    <meta http-equiv="refresh" content="30" />
+    <meta name="viewport" content="width=device-width, initial-scale=1" />
+    <title>dnswatcher</title>
+    <link rel="stylesheet" href="/s/css/tailwind.min.css" />
+  </head>
+  <body
+    class="bg-surface-950 text-slate-300 font-mono text-sm min-h-screen antialiased"
+  >
+    <div class="max-w-6xl mx-auto px-4 py-8">
+      {{/* ---- Header ---- */}}
+      <div class="mb-8">
+        <h1 class="text-2xl font-bold text-teal-400 tracking-tight">
+          dnswatcher
+        </h1>
+        <p class="text-xs text-slate-500 mt-1">
+          state updated {{ .StateAge }} &middot; page generated
+          {{ .GeneratedAt }} UTC &middot; auto-refresh 30s
+        </p>
+      </div>
+
+      {{/* ---- Summary bar ---- */}}
+      <div
+        class="grid grid-cols-2 sm:grid-cols-4 gap-3 mb-8"
+      >
+        <div class="bg-surface-800 border border-slate-700/50 rounded-lg p-4">
+          <div class="text-xs text-slate-500 uppercase tracking-wider">
+            Domains
+          </div>
+          <div class="text-2xl font-bold text-teal-400 mt-1">
+            {{ len .Snapshot.Domains }}
+          </div>
+        </div>
+        <div class="bg-surface-800 border border-slate-700/50 rounded-lg p-4">
+          <div class="text-xs text-slate-500 uppercase tracking-wider">
+            Hostnames
+          </div>
+          <div class="text-2xl font-bold text-teal-400 mt-1">
+            {{ len .Snapshot.Hostnames }}
+          </div>
+        </div>
+        <div class="bg-surface-800 border border-slate-700/50 rounded-lg p-4">
+          <div class="text-xs text-slate-500 uppercase tracking-wider">
+            Ports
+          </div>
+          <div class="text-2xl font-bold text-teal-400 mt-1">
+            {{ len .Snapshot.Ports }}
+          </div>
+        </div>
+        <div class="bg-surface-800 border border-slate-700/50 rounded-lg p-4">
+          <div class="text-xs text-slate-500 uppercase tracking-wider">
+            Certificates
+          </div>
+          <div class="text-2xl font-bold text-teal-400 mt-1">
+            {{ len .Snapshot.Certificates }}
+          </div>
+        </div>
+      </div>
+
+      {{/* ---- Domains ---- */}}
+      <section class="mb-8">
+        <h2
+          class="text-sm font-semibold text-teal-300 uppercase tracking-wider mb-3 border-b border-slate-700/50 pb-2"
+        >
+          Domains
+        </h2>
+        {{ if .Snapshot.Domains }}
+        <div class="overflow-x-auto">
+          <table class="w-full text-left text-xs">
+            <thead>
+              <tr class="text-slate-500 uppercase tracking-wider">
+                <th class="py-2 px-3">Domain</th>
+                <th class="py-2 px-3">Nameservers</th>
+                <th class="py-2 px-3">Checked</th>
+              </tr>
+            </thead>
+            <tbody class="divide-y divide-slate-800">
+              {{ range $name, $ds := .Snapshot.Domains }}
+              <tr class="hover:bg-surface-800/50">
+                <td class="py-2 px-3 text-slate-200 font-medium">
+                  {{ $name }}
+                </td>
+                <td class="py-2 px-3 text-slate-400 break-all">
+                  {{ joinStrings $ds.Nameservers ", " }}
+                </td>
+                <td class="py-2 px-3 text-slate-500 whitespace-nowrap">
+                  {{ relTime $ds.LastChecked }}
+                </td>
+              </tr>
+              {{ end }}
+            </tbody>
+          </table>
+        </div>
+        {{ else }}
+        <p class="text-slate-600 italic text-xs">
+          No domains configured.
+        </p>
+        {{ end }}
+      </section>
+
+      {{/* ---- Hostnames ---- */}}
+      <section class="mb-8">
+        <h2
+          class="text-sm font-semibold text-teal-300 uppercase tracking-wider mb-3 border-b border-slate-700/50 pb-2"
+        >
+          Hostnames
+        </h2>
+        {{ if .Snapshot.Hostnames }}
+        <div class="overflow-x-auto">
+          <table class="w-full text-left text-xs">
+            <thead>
+              <tr class="text-slate-500 uppercase tracking-wider">
+                <th class="py-2 px-3">Hostname</th>
+                <th class="py-2 px-3">NS</th>
+                <th class="py-2 px-3">Status</th>
+                <th class="py-2 px-3">Records</th>
+                <th class="py-2 px-3">Checked</th>
+              </tr>
+            </thead>
+            <tbody class="divide-y divide-slate-800">
+              {{ range $name, $hs := .Snapshot.Hostnames }}
+              {{ range $ns, $nsr := $hs.RecordsByNameserver }}
+              <tr class="hover:bg-surface-800/50">
+                <td class="py-2 px-3 text-slate-200 font-medium">
+                  {{ $name }}
+                </td>
+                <td class="py-2 px-3 text-slate-400 break-all">
+                  {{ $ns }}
+                </td>
+                <td class="py-2 px-3">
+                  {{ if eq $nsr.Status "ok" }}
+                  <span
+                    class="inline-block px-1.5 py-0.5 rounded text-[10px] font-bold uppercase bg-teal-900/50 text-teal-400 border border-teal-700/30"
+                    >ok</span
+                  >
+                  {{ else }}
+                  <span
+                    class="inline-block px-1.5 py-0.5 rounded text-[10px] font-bold uppercase bg-red-900/50 text-red-400 border border-red-700/30"
+                    >{{ $nsr.Status }}</span
+                  >
+                  {{ end }}
+                </td>
+                <td
+                  class="py-2 px-3 text-slate-400 break-all max-w-xs"
+                >
+                  {{ formatRecords $nsr.Records }}
+                </td>
+                <td class="py-2 px-3 text-slate-500 whitespace-nowrap">
+                  {{ relTime $nsr.LastChecked }}
+                </td>
+              </tr>
+              {{ end }}
+              {{ end }}
+            </tbody>
+          </table>
+        </div>
+        {{ else }}
+        <p class="text-slate-600 italic text-xs">
+          No hostnames configured.
+        </p>
+        {{ end }}
+      </section>
+
+      {{/* ---- Ports ---- */}}
+      <section class="mb-8">
+        <h2
+          class="text-sm font-semibold text-teal-300 uppercase tracking-wider mb-3 border-b border-slate-700/50 pb-2"
+        >
+          Ports
+        </h2>
+        {{ if .Snapshot.Ports }}
+        <div class="overflow-x-auto">
+          <table class="w-full text-left text-xs">
+            <thead>
+              <tr class="text-slate-500 uppercase tracking-wider">
+                <th class="py-2 px-3">Address</th>
+                <th class="py-2 px-3">State</th>
+                <th class="py-2 px-3">Hostnames</th>
+                <th class="py-2 px-3">Checked</th>
+              </tr>
+            </thead>
+            <tbody class="divide-y divide-slate-800">
+              {{ range $key, $ps := .Snapshot.Ports }}
+              <tr class="hover:bg-surface-800/50">
+                <td class="py-2 px-3 text-slate-200 font-medium">
+                  {{ $key }}
+                </td>
+                <td class="py-2 px-3">
+                  {{ if $ps.Open }}
+                  <span
+                    class="inline-block px-1.5 py-0.5 rounded text-[10px] font-bold uppercase bg-teal-900/50 text-teal-400 border border-teal-700/30"
+                    >open</span
+                  >
+                  {{ else }}
+                  <span
+                    class="inline-block px-1.5 py-0.5 rounded text-[10px] font-bold uppercase bg-red-900/50 text-red-400 border border-red-700/30"
+                    >closed</span
+                  >
+                  {{ end }}
+                </td>
+                <td class="py-2 px-3 text-slate-400 break-all">
+                  {{ joinStrings $ps.Hostnames ", " }}
+                </td>
+                <td class="py-2 px-3 text-slate-500 whitespace-nowrap">
+                  {{ relTime $ps.LastChecked }}
+                </td>
+              </tr>
+              {{ end }}
+            </tbody>
+          </table>
+        </div>
+        {{ else }}
+        <p class="text-slate-600 italic text-xs">
+          No port data yet.
+        </p>
+        {{ end }}
+      </section>
+
+      {{/* ---- Certificates ---- */}}
+      <section class="mb-8">
+        <h2
+          class="text-sm font-semibold text-teal-300 uppercase tracking-wider mb-3 border-b border-slate-700/50 pb-2"
+        >
+          Certificates
+        </h2>
+        {{ if .Snapshot.Certificates }}
+        <div class="overflow-x-auto">
+          <table class="w-full text-left text-xs">
+            <thead>
+              <tr class="text-slate-500 uppercase tracking-wider">
+                <th class="py-2 px-3">Endpoint</th>
+                <th class="py-2 px-3">Status</th>
+                <th class="py-2 px-3">CN</th>
+                <th class="py-2 px-3">Issuer</th>
+                <th class="py-2 px-3">Expires</th>
+                <th class="py-2 px-3">Checked</th>
+              </tr>
+            </thead>
+            <tbody class="divide-y divide-slate-800">
+              {{ range $key, $cs := .Snapshot.Certificates }}
+              <tr class="hover:bg-surface-800/50">
+                <td class="py-2 px-3 text-slate-400 break-all">
+                  {{ $key }}
+                </td>
+                <td class="py-2 px-3">
+                  {{ if eq $cs.Status "ok" }}
+                  <span
+                    class="inline-block px-1.5 py-0.5 rounded text-[10px] font-bold uppercase bg-teal-900/50 text-teal-400 border border-teal-700/30"
+                    >ok</span
+                  >
+                  {{ else }}
+                  <span
+                    class="inline-block px-1.5 py-0.5 rounded text-[10px] font-bold uppercase bg-red-900/50 text-red-400 border border-red-700/30"
+                    >{{ $cs.Status }}</span
+                  >
+                  {{ end }}
+                </td>
+                <td class="py-2 px-3 text-slate-200">
+                  {{ $cs.CommonName }}
+                </td>
+                <td class="py-2 px-3 text-slate-400 break-all">
+                  {{ $cs.Issuer }}
+                </td>
+                <td class="py-2 px-3 whitespace-nowrap">
+                  {{ if not $cs.NotAfter.IsZero }}
+                    {{ $days := expiryDays $cs.NotAfter }}
+                    {{ if lt $days 7 }}
+                    <span class="text-red-400 font-medium"
+                      >{{ $cs.NotAfter.Format "2006-01-02" }}
+                      ({{ $days }}d)</span
+                    >
+                    {{ else if lt $days 30 }}
+                    <span class="text-amber-400"
+                      >{{ $cs.NotAfter.Format "2006-01-02" }}
+                      ({{ $days }}d)</span
+                    >
+                    {{ else }}
+                    <span class="text-slate-400"
+                      >{{ $cs.NotAfter.Format "2006-01-02" }}
+                      ({{ $days }}d)</span
+                    >
+                    {{ end }}
+                  {{ end }}
+                </td>
+                <td class="py-2 px-3 text-slate-500 whitespace-nowrap">
+                  {{ relTime $cs.LastChecked }}
+                </td>
+              </tr>
+              {{ end }}
+            </tbody>
+          </table>
+        </div>
+        {{ else }}
+        <p class="text-slate-600 italic text-xs">
+          No certificate data yet.
+        </p>
+        {{ end }}
+      </section>
+
+      {{/* ---- Recent Alerts ---- */}}
+      <section class="mb-8">
+        <h2
+          class="text-sm font-semibold text-teal-300 uppercase tracking-wider mb-3 border-b border-slate-700/50 pb-2"
+        >
+          Recent Alerts ({{ len .Alerts }})
+        </h2>
+        {{ if .Alerts }}
+        <div class="space-y-2">
+          {{ range .Alerts }}
+          <div
+            class="bg-surface-800 border rounded-lg px-4 py-3 {{ if eq .Priority "error" }}border-red-700/40{{ else if eq .Priority "warning" }}border-amber-700/40{{ else if eq .Priority "success" }}border-teal-700/40{{ else }}border-blue-700/40{{ end }}"
+          >
+            <div class="flex items-center gap-3 mb-1">
+              {{ if eq .Priority "error" }}
+              <span
+                class="inline-block px-1.5 py-0.5 rounded text-[10px] font-bold uppercase bg-red-900/50 text-red-400 border border-red-700/30"
+                >error</span
+              >
+              {{ else if eq .Priority "warning" }}
+              <span
+                class="inline-block px-1.5 py-0.5 rounded text-[10px] font-bold uppercase bg-amber-900/50 text-amber-400 border border-amber-700/30"
+                >warning</span
+              >
+              {{ else if eq .Priority "success" }}
+              <span
+                class="inline-block px-1.5 py-0.5 rounded text-[10px] font-bold uppercase bg-teal-900/50 text-teal-400 border border-teal-700/30"
+                >success</span
+              >
+              {{ else }}
+              <span
+                class="inline-block px-1.5 py-0.5 rounded text-[10px] font-bold uppercase bg-blue-900/50 text-blue-400 border border-blue-700/30"
+                >info</span
+              >
+              {{ end }}
+              <span class="text-slate-200 text-xs font-medium">
+                {{ .Title }}
+              </span>
+              <span class="text-slate-600 text-[11px] ml-auto whitespace-nowrap">
+                {{ .Timestamp.Format "2006-01-02 15:04:05" }} UTC
+                ({{ relTime .Timestamp }})
+              </span>
+            </div>
+            <p
+              class="text-slate-400 text-xs whitespace-pre-line pl-0.5"
+            >
+              {{ .Message }}
+            </p>
+          </div>
+          {{ end }}
+        </div>
+        {{ else }}
+        <p class="text-slate-600 italic text-xs">
+          No alerts recorded since last restart.
+        </p>
+        {{ end }}
+      </section>
+
+      {{/* ---- Footer ---- */}}
+      <div
+        class="text-[11px] text-slate-700 border-t border-slate-800 pt-4 mt-8"
+      >
+        dnswatcher &middot; monitoring {{ len .Snapshot.Domains }} domains +
+        {{ len .Snapshot.Hostnames }} hostnames
+      </div>
+    </div>
+  </body>
+</html>

internal/healthcheck/healthcheck_test.go

@@ -0,0 +1,134 @@
+package healthcheck_test
+
+import (
+	"context"
+	"encoding/json"
+	"testing"
+	"time"
+
+	"go.uber.org/fx"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"sneak.berlin/go/dnswatcher/internal/config"
+	"sneak.berlin/go/dnswatcher/internal/globals"
+	"sneak.berlin/go/dnswatcher/internal/healthcheck"
+	"sneak.berlin/go/dnswatcher/internal/logger"
+)
+
+// recordingLifecycle is a minimal fx.Lifecycle that records the hooks
+// appended to it, so healthcheck.New can be exercised through its real
+// constructor without standing up a whole fx application.
+type recordingLifecycle struct {
+	hooks []fx.Hook
+}
+
+func (l *recordingLifecycle) Append(hook fx.Hook) {
+	l.hooks = append(l.hooks, hook)
+}
+
+// newHealthcheck builds a Healthcheck through the real constructor and
+// runs the registered OnStart hook so StartupTime is set the same way
+// the fx lifecycle would set it.
+func newHealthcheck(
+	t *testing.T,
+	maintenance bool,
+	version string,
+) *healthcheck.Healthcheck {
+	t.Helper()
+
+	g := &globals.Globals{Appname: "dnswatcher", Version: version}
+
+	log, err := logger.New(nil, logger.Params{Globals: g})
+	require.NoError(t, err)
+
+	lifecycle := &recordingLifecycle{}
+
+	hc, err := healthcheck.New(lifecycle, healthcheck.Params{
+		Globals: g,
+		Config:  &config.Config{MaintenanceMode: maintenance},
+		Logger:  log,
+	})
+	require.NoError(t, err)
+
+	require.Len(t, lifecycle.hooks, 1,
+		"New must register exactly one lifecycle hook")
+	require.NotNil(t, lifecycle.hooks[0].OnStart)
+	require.NoError(t, lifecycle.hooks[0].OnStart(context.Background()))
+
+	return hc
+}
+
+func TestCheckStatusAndPayloadShape(t *testing.T) {
+	t.Parallel()
+
+	hc := newHealthcheck(t, false, "v9.9.9")
+	resp := hc.Check()
+
+	assert.Equal(t, "ok", resp.Status)
+
+	// The JSON shape and field names are part of the contract for the
+	// /health and /.well-known/healthcheck routes, so assert on the
+	// exact set of keys the response marshals to.
+	raw, err := json.Marshal(resp)
+	require.NoError(t, err)
+
+	var fields map[string]json.RawMessage
+	require.NoError(t, json.Unmarshal(raw, &fields))
+
+	wantKeys := []string{
+		"status",
+		"now",
+		"uptimeSeconds",
+		"uptimeHuman",
+		"version",
+		"appname",
+		"maintenanceMode",
+	}
+	assert.Len(t, fields, len(wantKeys),
+		"response must marshal to exactly the documented fields")
+
+	for _, key := range wantKeys {
+		assert.Contains(t, fields, key, "missing JSON field %q", key)
+	}
+
+	// The Now field is documented as RFC3339Nano; a change to the
+	// format constant should turn this red.
+	_, err = time.Parse(time.RFC3339Nano, resp.Now)
+	assert.NoError(t, err, "Now must be RFC3339Nano")
+}
+
+func TestCheckMaintenanceModeReflectsConfig(t *testing.T) {
+	t.Parallel()
+
+	tests := []struct {
+		name        string
+		maintenance bool
+	}{
+		{"maintenance off", false},
+		{"maintenance on", true},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			t.Parallel()
+
+			hc := newHealthcheck(t, tt.maintenance, "test")
+			resp := hc.Check()
+			assert.Equal(t, tt.maintenance, resp.Maintenance,
+				"maintenanceMode must mirror Config.MaintenanceMode")
+		})
+	}
+}
+
+func TestCheckSurfacesVersionAndAppname(t *testing.T) {
+	t.Parallel()
+
+	hc := newHealthcheck(t, false, "surfaced-version-123")
+	resp := hc.Check()
+
+	assert.Equal(t, "surfaced-version-123", resp.Version,
+		"version from globals must appear in the payload")
+	assert.Equal(t, "dnswatcher", resp.Appname)
+}

internal/livednstest/livednstest.go

@@ -0,0 +1,128 @@
+// Package livednstest runs the live DNS operations of tests. Tests that
+// look something up in DNS query live DNS servers, never a stand-in —
+// see TESTING.md. Nothing here mocks, fakes, stubs, records or replays
+// DNS, and nothing here skips a test: it only changes *how* the live
+// queries are issued, so that a single dropped UDP packet or one slow
+// authoritative server does not turn correct code into a red build.
+//
+// Two mechanisms:
+//
+//  1. Bounded concurrency. Tests run in parallel and the build hosts
+//     have many cores, so without a limit every test starts its own
+//     iterative resolution at the same instant and they all send their
+//     first queries to the root servers within a few milliseconds of
+//     each other. Root servers rate-limit that, which shows up as a
+//     different arbitrary subset of tests failing on each run. Run caps
+//     how many live operations are in flight at once in one test binary.
+//
+//  2. Retry with exponential backoff. Each live operation gets several
+//     attempts with its own timeout. An attempt is retried when it
+//     obtained nothing to check, never because of what the test
+//     asserts about the result, so a wrong result still fails on the
+//     first attempt. A fault in the code under test that leaves
+//     nothing to check looks the same as live DNS not answering, and
+//     fails only after the last attempt.
+package livednstest
+
+import (
+	"context"
+	"errors"
+	"testing"
+	"time"
+)
+
+const (
+	// attempts is how many times a live DNS operation is attempted
+	// before the test fails.
+	attempts = 3
+
+	// AttemptTimeout bounds one attempt. It must fit the longest
+	// operation, a watcher check, which sends over a hundred queries one
+	// after another and on a slow build host takes several times as long
+	// as the few seconds it takes on a fast one. An operation whose
+	// every attempt fails takes attempts * AttemptTimeout plus the
+	// backoff, about 56 seconds, after it waits for one of the
+	// Concurrency slots that every live operation in the test binary
+	// shares. So when live DNS does not answer at all, a test binary
+	// with more live operations than slots runs into the 90-second
+	// `go test -timeout` backstop instead of each test failing on its
+	// own.
+	AttemptTimeout = 18 * time.Second
+
+	// backoffBase is the delay after the first failed attempt; it is
+	// multiplied by backoffFactor each time.
+	backoffBase = 500 * time.Millisecond
+
+	// backoffFactor is the exponential backoff multiplier.
+	backoffFactor = 2
+
+	// Concurrency caps how many live operations may be in flight
+	// across one test binary at once.
+	Concurrency = 6
+)
+
+// gate bounds concurrent live operations. It has to be package scoped:
+// the whole point is that it is shared by every parallel test in the
+// test binary.
+//
+//nolint:gochecknoglobals // package-wide live query rate limit
+var gate = make(chan struct{}, Concurrency)
+
+// ErrNoAnswer reports that a live operation produced no usable answer,
+// which is retried rather than asserted on.
+var ErrNoAnswer = errors.New("no answer from live DNS")
+
+// Run executes one attempt of a live operation, holding a slot in gate
+// for its duration and bounding it with its own timeout.
+func Run(op func(ctx context.Context) error) error {
+	gate <- struct{}{}
+	defer func() { <-gate }()
+
+	ctx, cancel := context.WithTimeout(
+		context.Background(), AttemptTimeout,
+	)
+	defer cancel()
+
+	return op(ctx)
+}
+
+// Retry runs op until it reports success, retrying failures with
+// exponential backoff, and fails the test if every attempt fails. op
+// returns an error only for a failure to obtain an answer — never for
+// an answer the test disagrees with, which belongs in an assertion so
+// that it fails immediately. op stores whatever it obtained where its
+// caller can find it.
+func Retry(
+	t *testing.T,
+	what string,
+	op func(ctx context.Context) error,
+) {
+	t.Helper()
+
+	var last error
+
+	backoff := backoffBase
+
+	for attempt := range attempts {
+		if attempt > 0 {
+			t.Logf(
+				"%s: attempt %d of %d failed (%v), "+
+					"retrying in %s",
+				what, attempt, attempts, last, backoff,
+			)
+			time.Sleep(backoff)
+
+			backoff *= backoffFactor
+		}
+
+		last = Run(op)
+		if last == nil {
+			return
+		}
+	}
+
+	t.Fatalf(
+		"%s: all %d live attempts failed: %v",
+		what, attempts, last,
+	)
+}

internal/livednstest/livednstest_test.go

@@ -0,0 +1,103 @@
+package livednstest_test
+
+import (
+	"context"
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+
+	"sneak.berlin/go/dnswatcher/internal/livednstest"
+)
+
+// Tests for the retry and the concurrency limit themselves. They
+// perform no DNS resolution of any kind.
+
+func TestRetryRecoversFromTransientFailure(t *testing.T) {
+	t.Parallel()
+
+	const wantAttempts = 2
+
+	attempts := 0
+
+	livednstest.Retry(t, "transient", func(_ context.Context) error {
+		attempts++
+
+		if attempts < wantAttempts {
+			return livednstest.ErrNoAnswer
+		}
+
+		return nil
+	})
+
+	assert.Equal(t, wantAttempts, attempts)
+}
+
+func TestRetryGivesEachAttemptADeadline(t *testing.T) {
+	t.Parallel()
+
+	livednstest.Retry(t, "deadline", func(ctx context.Context) error {
+		deadline, ok := ctx.Deadline()
+		assert.True(t, ok, "attempt should carry a deadline")
+
+		remaining := time.Until(deadline)
+
+		assert.LessOrEqual(t, remaining, livednstest.AttemptTimeout)
+
+		// Lower bound too: without one this passes for a
+		// deadline far shorter than intended, which would
+		// silently turn every live attempt into an instant
+		// timeout.
+		assert.Greater(t, remaining, livednstest.AttemptTimeout/2)
+
+		return nil
+	})
+}
+
+func TestRunBoundsConcurrency(t *testing.T) {
+	t.Parallel()
+
+	const workers = 24
+
+	var (
+		mu       sync.Mutex
+		wg       sync.WaitGroup
+		inFlight int
+		maxSeen  int
+	)
+
+	wg.Add(workers)
+
+	for range workers {
+		go func() {
+			defer wg.Done()
+
+			_ = livednstest.Run(func(_ context.Context) error {
+				mu.Lock()
+				inFlight++
+
+				if inFlight > maxSeen {
+					maxSeen = inFlight
+				}
+				mu.Unlock()
+
+				time.Sleep(time.Millisecond)
+
+				mu.Lock()
+				inFlight--
+				mu.Unlock()
+
+				return nil
+			})
+		}()
+	}
+
+	wg.Wait()
+
+	assert.Positive(t, maxSeen)
+	assert.LessOrEqual(
+		t, maxSeen, livednstest.Concurrency,
+		"live queries must stay under the package-wide gate",
+	)
+}

internal/logger/logger.go

@@ -78,6 +78,5 @@ func (l *Logger) Identify() {
 	l.log.Info("starting",
 		"appname", l.params.Globals.Appname,
 		"version", l.params.Globals.Version,
-		"buildarch", l.params.Globals.Buildarch,
 	)
 }

internal/logger/logger_test.go

@@ -0,0 +1,66 @@
+package logger_test
+
+import (
+	"context"
+	"log/slog"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"sneak.berlin/go/dnswatcher/internal/globals"
+	"sneak.berlin/go/dnswatcher/internal/logger"
+)
+
+func newTestLogger(t *testing.T) *logger.Logger {
+	t.Helper()
+
+	g := &globals.Globals{Appname: "dnswatcher", Version: "test"}
+
+	l, err := logger.New(nil, logger.Params{Globals: g})
+	require.NoError(t, err)
+
+	return l
+}
+
+// TestNewReturnsUsableLogger checks that the constructor yields a
+// working *slog.Logger.
+func TestNewReturnsUsableLogger(t *testing.T) {
+	t.Parallel()
+
+	l := newTestLogger(t)
+	require.NotNil(t, l.Get(), "Get must return a non-nil logger")
+}
+
+// TestDefaultLevelExcludesDebug verifies the default configuration
+// logs at info: debug records are suppressed, info records pass.
+func TestDefaultLevelExcludesDebug(t *testing.T) {
+	t.Parallel()
+
+	log := newTestLogger(t).Get()
+	ctx := context.Background()
+
+	assert.False(t, log.Enabled(ctx, slog.LevelDebug),
+		"debug must be suppressed at the default level")
+	assert.True(t, log.Enabled(ctx, slog.LevelInfo),
+		"info must be enabled at the default level")
+}
+
+// TestEnableDebugLoggingChangesLevel verifies the debug and non-debug
+// configurations differ as intended: enabling debug makes debug
+// records pass where they previously did not.
+func TestEnableDebugLoggingChangesLevel(t *testing.T) {
+	t.Parallel()
+
+	l := newTestLogger(t)
+	log := l.Get()
+	ctx := context.Background()
+
+	require.False(t, log.Enabled(ctx, slog.LevelDebug),
+		"debug must start disabled")
+
+	l.EnableDebugLogging()
+
+	assert.True(t, log.Enabled(ctx, slog.LevelDebug),
+		"debug must be enabled after EnableDebugLogging")
+}

internal/middleware/export_test.go

@@ -0,0 +1,19 @@
+package middleware
+
+import (
+	"net/http"
+	"time"
+)
+
+// The /metrics rate limit, exported so the tests can count requests
+// against it.
+const (
+	MetricsRequestLimit                = metricsRequestLimit
+	MetricsRequestWindow time.Duration = metricsRequestWindow
+)
+
+// RealIP is realIP, exported so the tests can check which address it
+// takes as the client's.
+func RealIP(r *http.Request) string {
+	return realIP(r)
+}

internal/middleware/middleware.go

@@ -5,12 +5,14 @@ import (
 	"log/slog"
 	"net"
 	"net/http"
+	"net/netip"
 	"strings"
 	"time"
 
 	"github.com/99designs/basicauth-go"
 	"github.com/go-chi/chi/v5/middleware"
 	"github.com/go-chi/cors"
+	"github.com/go-chi/httprate"
 	"go.uber.org/fx"
 
 	"sneak.berlin/go/dnswatcher/internal/config"
@@ -21,6 +23,71 @@ import (
 // corsMaxAge is the maximum age for CORS preflight responses.
 const corsMaxAge = 300
 
+// Rate limit for /metrics: each client address may send
+// metricsRequestLimit requests per metricsRequestWindow. Every request
+// counts, so password guessing gets at most 30 tries a minute per
+// address. One Prometheus server scraping every 15 seconds sends 4
+// requests a minute, and two scraping every 5 seconds from one address
+// send 24, so normal scraping stays under the limit.
+const (
+	metricsRequestLimit  = 30
+	metricsRequestWindow = time.Minute
+)
+
+// Security response header values applied to every response.
+//
+// The CSP is as strict as the dashboard allows: the template ships no
+// JavaScript, no inline styles, no inline event handlers and no images,
+// and its only subresource is the embedded stylesheet at
+// /s/css/tailwind.min.css, which style-src 'self' permits. Neither
+// unsafe-inline nor unsafe-eval is used. frame-ancestors 'none' is the
+// primary anti-framing control; X-Frame-Options is the legacy fallback.
+const (
+	// hstsValue is emitted unconditionally, including over plain HTTP,
+	// because the service runs behind a TLS-terminating proxy and the
+	// browser must still enforce HTTPS end to end.
+	hstsValue = "max-age=31536000; includeSubDomains"
+
+	cspValue = "default-src 'self'; " +
+		"script-src 'none'; " +
+		"style-src 'self'; " +
+		"img-src 'self'; " +
+		"font-src 'none'; " +
+		"connect-src 'none'; " +
+		"object-src 'none'; " +
+		"base-uri 'none'; " +
+		"form-action 'none'; " +
+		"frame-ancestors 'none'"
+
+	frameOptionsValue = "DENY"
+
+	contentTypeOptionsValue = "nosniff"
+
+	// referrerPolicyValue is stricter than the policy minimum of
+	// strict-origin-when-cross-origin: the dashboard has no
+	// cross-origin navigation needs and its URL may name internal
+	// hosts.
+	referrerPolicyValue = "no-referrer"
+
+	permissionsPolicyValue = "accelerometer=(), " +
+		"autoplay=(), " +
+		"camera=(), " +
+		"display-capture=(), " +
+		"encrypted-media=(), " +
+		"fullscreen=(), " +
+		"geolocation=(), " +
+		"gyroscope=(), " +
+		"magnetometer=(), " +
+		"microphone=(), " +
+		"midi=(), " +
+		"payment=(), " +
+		"picture-in-picture=(), " +
+		"publickey-credentials-get=(), " +
+		"screen-wake-lock=(), " +
+		"usb=(), " +
+		"xr-spatial-tracking=()"
+)
+
 // Params contains dependencies for Middleware.
 type Params struct {
 	fx.In
@@ -142,6 +209,12 @@ func isTrustedProxy(ip net.IP) bool {
 
 // realIP extracts the client's real IP address from the request.
 // Proxy headers are only trusted from RFC1918/loopback addresses.
+//
+// Each proxy adds to the end of X-Forwarded-For the address it got the
+// request from, so the client can write every entry before the one the
+// first trusted proxy added. The client address is therefore the
+// rightmost entry that is not a trusted proxy, or the leftmost entry
+// when they all are.
 func realIP(r *http.Request) string {
 	addr := ipFromHostPort(r.RemoteAddr)
 	remoteIP := net.ParseIP(addr)
@@ -156,36 +229,100 @@ func realIP(r *http.Request) string {
 		return ip
 	}
 
-	if xff := r.Header.Get("X-Forwarded-For"); xff != "" {
-		if parts := strings.SplitN(
-			xff, ",", 2, //nolint:mnd
-		); len(parts) > 0 {
-			if ip := strings.TrimSpace(parts[0]); ip != "" {
-				return ip
-			}
+	// A proxy may add its entry as a header line of its own instead of
+	// appending to the line the client sent, so all lines form one list.
+	entries := strings.Split(
+		strings.Join(r.Header.Values("X-Forwarded-For"), ","), ",",
+	)
+	client := strings.TrimSpace(entries[0])
+
+	for i := len(entries) - 1; i > 0; i-- {
+		entry := strings.TrimSpace(entries[i])
+		if !isTrustedProxy(net.ParseIP(entry)) {
+			client = entry
+
+			break
 		}
 	}
 
+	if client != "" {
+		return client
+	}
+
 	return addr
 }
 
-// CORS returns CORS middleware.
+// CORS returns middleware that lets any origin read a response. It is
+// for the public, read-only routes only, so it allows only the
+// methods those routes serve and no Authorization header.
 func (m *Middleware) CORS() func(http.Handler) http.Handler {
 	return cors.Handler(cors.Options{
-		AllowedOrigins: []string{"*"},
-		AllowedMethods: []string{
-			"GET", "POST", "PUT", "DELETE", "OPTIONS",
-		},
-		AllowedHeaders: []string{
-			"Accept", "Authorization",
-			"Content-Type", "X-CSRF-Token",
-		},
+		AllowedOrigins:   []string{"*"},
+		AllowedMethods:   []string{"GET", "OPTIONS"},
+		AllowedHeaders:   []string{"Accept", "Content-Type"},
 		ExposedHeaders:   []string{"Link"},
 		AllowCredentials: false,
 		MaxAge:           corsMaxAge,
 	})
 }
 
+// SecurityHeaders returns middleware that sets the security response
+// headers required for production internet exposure on every response.
+//
+// The headers are set before the request reaches the next handler so
+// that they are present on every response, including panics recovered
+// by chi's Recoverer and timeouts produced by chi's Timeout.
+func (m *Middleware) SecurityHeaders() func(http.Handler) http.Handler {
+	return func(next http.Handler) http.Handler {
+		return http.HandlerFunc(func(
+			writer http.ResponseWriter,
+			request *http.Request,
+		) {
+			header := writer.Header()
+			header.Set("Strict-Transport-Security", hstsValue)
+			header.Set("Content-Security-Policy", cspValue)
+			header.Set("X-Frame-Options", frameOptionsValue)
+			header.Set(
+				"X-Content-Type-Options",
+				contentTypeOptionsValue,
+			)
+			header.Set("Referrer-Policy", referrerPolicyValue)
+			header.Set(
+				"Permissions-Policy",
+				permissionsPolicyValue,
+			)
+
+			next.ServeHTTP(writer, request)
+		})
+	}
+}
+
+// MetricsRateLimit returns middleware for /metrics that answers 429
+// Too Many Requests to a client address over the rate limit. The
+// address is the one realIP works out, so a client that is not a
+// trusted proxy cannot get a fresh allowance by sending its own
+// X-Real-IP or X-Forwarded-For. CanonicalizeIP counts all IPv6
+// addresses in one /64 as one client, since a client usually holds a
+// whole /64. An IPv4 address a proxy reports in IPv6-mapped form
+// (::ffff:203.0.113.1) is turned back into plain IPv4 first, as every
+// such address is in the same /64.
+func (m *Middleware) MetricsRateLimit() func(http.Handler) http.Handler {
+	return httprate.LimitBy(
+		metricsRequestLimit,
+		metricsRequestWindow,
+		func(request *http.Request) (string, error) {
+			ip := realIP(request)
+
+			addr, err := netip.ParseAddr(ip)
+			if err == nil {
+				ip = addr.Unmap().String()
+			}
+
+			return httprate.CanonicalizeIP(ip), nil
+		},
+	)
+}
+
 // MetricsAuth returns basic auth middleware for /metrics.
 func (m *Middleware) MetricsAuth() func(http.Handler) http.Handler {
 	if m.params.Config.MetricsUsername == "" {

internal/middleware/middleware_test.go

@@ -0,0 +1,565 @@
+package middleware_test
+
+import (
+	"net/http"
+	"net/http/httptest"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/go-chi/chi/v5"
+	"go.uber.org/fx/fxtest"
+
+	"sneak.berlin/go/dnswatcher/internal/config"
+	"sneak.berlin/go/dnswatcher/internal/globals"
+	"sneak.berlin/go/dnswatcher/internal/handlers"
+	"sneak.berlin/go/dnswatcher/internal/logger"
+	"sneak.berlin/go/dnswatcher/internal/middleware"
+	"sneak.berlin/go/dnswatcher/internal/notify"
+	"sneak.berlin/go/dnswatcher/internal/state"
+)
+
+// Expected security header values, spelled out literally so that any
+// change to the middleware has to be made deliberately here as well.
+const (
+	wantHSTS = "max-age=31536000; includeSubDomains"
+
+	wantCSP = "default-src 'self'; " +
+		"script-src 'none'; " +
+		"style-src 'self'; " +
+		"img-src 'self'; " +
+		"font-src 'none'; " +
+		"connect-src 'none'; " +
+		"object-src 'none'; " +
+		"base-uri 'none'; " +
+		"form-action 'none'; " +
+		"frame-ancestors 'none'"
+
+	wantFrameOptions = "DENY"
+
+	wantContentTypeOptions = "nosniff"
+
+	wantReferrerPolicy = "no-referrer"
+
+	wantPermissionsPolicy = "accelerometer=(), " +
+		"autoplay=(), " +
+		"camera=(), " +
+		"display-capture=(), " +
+		"encrypted-media=(), " +
+		"fullscreen=(), " +
+		"geolocation=(), " +
+		"gyroscope=(), " +
+		"magnetometer=(), " +
+		"microphone=(), " +
+		"midi=(), " +
+		"payment=(), " +
+		"picture-in-picture=(), " +
+		"publickey-credentials-get=(), " +
+		"screen-wake-lock=(), " +
+		"usb=(), " +
+		"xr-spatial-tracking=()"
+)
+
+// stylesheetPath is the only subresource the dashboard loads.
+const stylesheetPath = "/s/css/tailwind.min.css"
+
+// newTestLogger builds a logger for direct component construction.
+func newTestLogger(t *testing.T) *logger.Logger {
+	t.Helper()
+
+	glob, err := globals.New(nil)
+	if err != nil {
+		t.Fatalf("globals.New: %v", err)
+	}
+
+	log, err := logger.New(nil, logger.Params{Globals: glob})
+	if err != nil {
+		t.Fatalf("logger.New: %v", err)
+	}
+
+	return log
+}
+
+// newTestMiddleware builds a Middleware without an fx application.
+func newTestMiddleware(t *testing.T) *middleware.Middleware {
+	t.Helper()
+
+	glob, err := globals.New(nil)
+	if err != nil {
+		t.Fatalf("globals.New: %v", err)
+	}
+
+	mw, err := middleware.New(nil, middleware.Params{
+		Logger:  newTestLogger(t),
+		Globals: glob,
+		Config:  &config.Config{},
+	})
+	if err != nil {
+		t.Fatalf("middleware.New: %v", err)
+	}
+
+	return mw
+}
+
+// serveWithSecurityHeaders runs a GET through SecurityHeaders and
+// returns the recorded response.
+func serveWithSecurityHeaders(
+	t *testing.T,
+	target string,
+	handler http.Handler,
+) *httptest.ResponseRecorder {
+	t.Helper()
+
+	mw := newTestMiddleware(t)
+	rec := httptest.NewRecorder()
+	req := httptest.NewRequestWithContext(
+		t.Context(), http.MethodGet, target, nil,
+	)
+
+	mw.SecurityHeaders()(handler).ServeHTTP(rec, req)
+
+	return rec
+}
+
+// okHandler writes a trivial 200 response.
+func okHandler() http.Handler {
+	return http.HandlerFunc(func(
+		writer http.ResponseWriter,
+		_ *http.Request,
+	) {
+		writer.WriteHeader(http.StatusOK)
+	})
+}
+
+func TestSecurityHeaders(t *testing.T) {
+	t.Parallel()
+
+	tests := []struct {
+		name   string
+		header string
+		want   string
+	}{
+		{
+			"hsts",
+			"Strict-Transport-Security",
+			wantHSTS,
+		},
+		{
+			"csp",
+			"Content-Security-Policy",
+			wantCSP,
+		},
+		{
+			"frame options",
+			"X-Frame-Options",
+			wantFrameOptions,
+		},
+		{
+			"content type options",
+			"X-Content-Type-Options",
+			wantContentTypeOptions,
+		},
+		{
+			"referrer policy",
+			"Referrer-Policy",
+			wantReferrerPolicy,
+		},
+		{
+			"permissions policy",
+			"Permissions-Policy",
+			wantPermissionsPolicy,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			t.Parallel()
+
+			rec := serveWithSecurityHeaders(t, "/", okHandler())
+
+			got := rec.Header().Get(tt.header)
+			if got != tt.want {
+				t.Errorf(
+					"%s = %q, want %q",
+					tt.header, got, tt.want,
+				)
+			}
+		})
+	}
+}
+
+// TestSecurityHeadersCSPDirectives guards the properties the repo
+// policy requires of the content security policy itself.
+func TestSecurityHeadersCSPDirectives(t *testing.T) {
+	t.Parallel()
+
+	rec := serveWithSecurityHeaders(t, "/", okHandler())
+	csp := rec.Header().Get("Content-Security-Policy")
+
+	forbidden := []string{"unsafe-inline", "unsafe-eval"}
+	for _, directive := range forbidden {
+		if strings.Contains(csp, directive) {
+			t.Errorf("CSP must not contain %q: %q", directive, csp)
+		}
+	}
+
+	required := []string{
+		"default-src 'self'",
+		"script-src 'none'",
+		"style-src 'self'",
+		"frame-ancestors 'none'",
+	}
+	for _, directive := range required {
+		if !strings.Contains(csp, directive) {
+			t.Errorf("CSP must contain %q: %q", directive, csp)
+		}
+	}
+}
+
+// TestSecurityHeadersOnErrorResponse verifies the headers are emitted
+// even when the wrapped handler fails, since they are set before the
+// handler runs.
+func TestSecurityHeadersOnErrorResponse(t *testing.T) {
+	t.Parallel()
+
+	failing := http.HandlerFunc(func(
+		writer http.ResponseWriter,
+		_ *http.Request,
+	) {
+		http.Error(
+			writer,
+			"boom",
+			http.StatusInternalServerError,
+		)
+	})
+
+	rec := serveWithSecurityHeaders(t, "/api/v1/status", failing)
+
+	if rec.Code != http.StatusInternalServerError {
+		t.Fatalf("status = %d, want 500", rec.Code)
+	}
+
+	if got := rec.Header().Get(
+		"X-Content-Type-Options",
+	); got != wantContentTypeOptions {
+		t.Errorf(
+			"X-Content-Type-Options = %q, want %q",
+			got, wantContentTypeOptions,
+		)
+	}
+
+	if got := rec.Header().Get(
+		"Strict-Transport-Security",
+	); got != wantHSTS {
+		t.Errorf(
+			"Strict-Transport-Security = %q, want %q",
+			got, wantHSTS,
+		)
+	}
+}
+
+// newTestHandlers builds real Handlers with empty monitoring state.
+func newTestHandlers(t *testing.T) *handlers.Handlers {
+	t.Helper()
+
+	glob, err := globals.New(nil)
+	if err != nil {
+		t.Fatalf("globals.New: %v", err)
+	}
+
+	log := newTestLogger(t)
+
+	notifier, err := notify.New(fxtest.NewLifecycle(t), notify.Params{
+		Logger: log,
+		Config: &config.Config{},
+	})
+	if err != nil {
+		t.Fatalf("notify.New: %v", err)
+	}
+
+	st, err := state.New(fxtest.NewLifecycle(t), state.Params{
+		Logger: log,
+		Config: &config.Config{DataDir: t.TempDir()},
+	})
+	if err != nil {
+		t.Fatalf("state.New: %v", err)
+	}
+
+	hnd, err := handlers.New(nil, handlers.Params{
+		Logger:  log,
+		Globals: glob,
+		State:   st,
+		Notify:  notifier,
+	})
+	if err != nil {
+		t.Fatalf("handlers.New: %v", err)
+	}
+
+	return hnd
+}
+
+// TestDashboardRendersWithSecurityHeaders renders the real dashboard
+// through the middleware and checks that the policy still permits the
+// one stylesheet the page loads.
+func TestDashboardRendersWithSecurityHeaders(t *testing.T) {
+	t.Parallel()
+
+	mw := newTestMiddleware(t)
+	hnd := newTestHandlers(t)
+
+	router := chi.NewRouter()
+	router.Use(mw.SecurityHeaders())
+	router.Get("/", hnd.HandleDashboard())
+
+	rec := httptest.NewRecorder()
+	req := httptest.NewRequestWithContext(
+		t.Context(), http.MethodGet, "/", nil,
+	)
+
+	router.ServeHTTP(rec, req)
+
+	if rec.Code != http.StatusOK {
+		t.Fatalf("status = %d, want 200", rec.Code)
+	}
+
+	body := rec.Body.String()
+	if !strings.Contains(body, stylesheetPath) {
+		t.Errorf("dashboard does not reference %q", stylesheetPath)
+	}
+
+	if !strings.Contains(body, "dnswatcher") {
+		t.Errorf("dashboard body looks empty: %d bytes", len(body))
+	}
+
+	csp := rec.Header().Get("Content-Security-Policy")
+	if csp != wantCSP {
+		t.Errorf("CSP = %q, want %q", csp, wantCSP)
+	}
+
+	// The stylesheet is same-origin, so style-src 'self' allows it.
+	if !strings.Contains(csp, "style-src 'self'") {
+		t.Errorf("CSP would block %q: %q", stylesheetPath, csp)
+	}
+}
+
+// Addresses for the rate limit and realIP tests: a client connecting
+// directly, a trusted proxy, and a client behind that proxy as the
+// proxy's X-Real-IP or X-Forwarded-For header names it.
+const (
+	directClient  = "198.51.100.1:4000"
+	trustedProxy  = "10.0.0.1:4000"
+	proxiedClient = "203.0.113.1"
+)
+
+// statusFrom sends a GET through handler as if from remoteAddr, with
+// an X-Real-IP header when xRealIP is not empty, and returns the
+// response status.
+func statusFrom(
+	t *testing.T,
+	handler http.Handler,
+	remoteAddr string,
+	xRealIP string,
+) int {
+	t.Helper()
+
+	req := httptest.NewRequestWithContext(
+		t.Context(), http.MethodGet, "/metrics", nil,
+	)
+	req.RemoteAddr = remoteAddr
+
+	if xRealIP != "" {
+		req.Header.Set("X-Real-IP", xRealIP)
+	}
+
+	rec := httptest.NewRecorder()
+	handler.ServeHTTP(rec, req)
+
+	return rec.Code
+}
+
+// TestMetricsRateLimitAllowsScraping checks that one address can send,
+// within one window, what two Prometheus servers scraping every 5
+// seconds send in that time, without being turned away.
+func TestMetricsRateLimitAllowsScraping(t *testing.T) {
+	t.Parallel()
+
+	const scrapeInterval = 5 * time.Second
+
+	scrapes := 2 * int(middleware.MetricsRequestWindow/scrapeInterval)
+	limited := newTestMiddleware(t).MetricsRateLimit()(okHandler())
+
+	for i := range scrapes {
+		got := statusFrom(t, limited, directClient, "")
+		if got != http.StatusOK {
+			t.Fatalf(
+				"scrape %d of %d: status = %d, want 200",
+				i+1, scrapes, got,
+			)
+		}
+	}
+}
+
+// TestMetricsRateLimitKeysOnClientAddress checks which requests share
+// an allowance. Each case uses up the allowance of one client, then
+// sends one more request.
+func TestMetricsRateLimitKeysOnClientAddress(t *testing.T) {
+	t.Parallel()
+
+	tests := []struct {
+		name           string
+		usedRemoteAddr string
+		usedXRealIP    string
+		nextRemoteAddr string
+		nextXRealIP    string
+		want           int
+	}{
+		{
+			"same address",
+			directClient, "",
+			directClient, "",
+			http.StatusTooManyRequests,
+		},
+		{
+			"another address",
+			directClient, "",
+			"198.51.100.2:4000", "",
+			http.StatusOK,
+		},
+		{
+			"own X-Real-IP from an untrusted address",
+			directClient, "",
+			directClient, "203.0.113.9",
+			http.StatusTooManyRequests,
+		},
+		{
+			"same client behind the proxy",
+			trustedProxy, proxiedClient,
+			trustedProxy, proxiedClient,
+			http.StatusTooManyRequests,
+		},
+		{
+			"another client behind the proxy",
+			trustedProxy, proxiedClient,
+			trustedProxy, "203.0.113.2",
+			http.StatusOK,
+		},
+		{
+			"another client behind the proxy, IPv6-mapped",
+			trustedProxy, "::ffff:203.0.113.1",
+			trustedProxy, "::ffff:203.0.113.2",
+			http.StatusOK,
+		},
+		{
+			"same IPv6 /64",
+			"[2001:db8::1]:4000", "",
+			"[2001:db8::2]:4000", "",
+			http.StatusTooManyRequests,
+		},
+		{
+			"another IPv6 /64",
+			"[2001:db8::1]:4000", "",
+			"[2001:db8:0:1::1]:4000", "",
+			http.StatusOK,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			t.Parallel()
+
+			limited := newTestMiddleware(t).MetricsRateLimit()(okHandler())
+
+			for range middleware.MetricsRequestLimit {
+				statusFrom(t, limited, tt.usedRemoteAddr, tt.usedXRealIP)
+			}
+
+			got := statusFrom(
+				t, limited, tt.nextRemoteAddr, tt.nextXRealIP,
+			)
+			if got != tt.want {
+				t.Errorf("status = %d, want %d", got, tt.want)
+			}
+		})
+	}
+}
+
+// TestRealIP checks which address realIP takes as the client's. Each
+// element of forwardedFor is sent as an X-Forwarded-For header line of
+// its own, and 198.51.100.9 is always an entry the client wrote itself.
+func TestRealIP(t *testing.T) {
+	t.Parallel()
+
+	tests := []struct {
+		name         string
+		remoteAddr   string
+		xRealIP      string
+		forwardedFor []string
+		want         string
+	}{
+		{
+			"untrusted peer, both headers ignored",
+			directClient, proxiedClient, []string{"198.51.100.9"},
+			"198.51.100.1",
+		},
+		{
+			"X-Real-IP from a trusted proxy wins",
+			trustedProxy, proxiedClient, []string{"203.0.113.8"},
+			proxiedClient,
+		},
+		{
+			"client's own entry, then the one the proxy added",
+			trustedProxy, "", []string{"198.51.100.9, 203.0.113.1"},
+			proxiedClient,
+		},
+		{
+			"several trusted proxies",
+			trustedProxy, "",
+			[]string{"198.51.100.9, 203.0.113.1, 10.0.0.3, 10.0.0.2"},
+			proxiedClient,
+		},
+		{
+			"proxy adds a header line of its own",
+			trustedProxy, "", []string{"198.51.100.9", proxiedClient},
+			proxiedClient,
+		},
+		{
+			"every entry a trusted proxy",
+			trustedProxy, "", []string{"10.0.0.3, 10.0.0.2"},
+			"10.0.0.3",
+		},
+		{
+			"empty where the client address belongs",
+			trustedProxy, "", []string{"203.0.113.1, , 10.0.0.2"},
+			"10.0.0.1",
+		},
+		{
+			"no headers from a trusted proxy",
+			trustedProxy, "", nil,
+			"10.0.0.1",
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			t.Parallel()
+
+			req := httptest.NewRequestWithContext(
+				t.Context(), http.MethodGet, "/", nil,
+			)
+			req.RemoteAddr = tt.remoteAddr
+
+			if tt.xRealIP != "" {
+				req.Header.Set("X-Real-IP", tt.xRealIP)
+			}
+
+			for _, line := range tt.forwardedFor {
+				req.Header.Add("X-Forwarded-For", line)
+			}
+
+			got := middleware.RealIP(req)
+			if got != tt.want {
+				t.Errorf("realIP = %q, want %q", got, tt.want)
+			}
+		})
+	}
+}

internal/notify/export_test.go

@@ -0,0 +1,122 @@
+package notify
+
+import (
+	"context"
+	"io"
+	"log/slog"
+	"net/http"
+	"net/url"
+	"time"
+)
+
+// NtfyPriority exports ntfyPriority for testing.
+func NtfyPriority(priority string) string {
+	return ntfyPriority(priority)
+}
+
+// SlackColor exports slackColor for testing.
+func SlackColor(priority string) string {
+	return slackColor(priority)
+}
+
+// NewRequestForTest exports newRequest for testing.
+func NewRequestForTest(
+	ctx context.Context,
+	method string,
+	target *url.URL,
+	body io.Reader,
+) *http.Request {
+	return newRequest(ctx, method, target, body)
+}
+
+// NewTestService creates a Service suitable for unit testing.
+// It discards log output and uses the given transport.
+func NewTestService(transport http.RoundTripper) *Service {
+	return newService(slog.New(slog.DiscardHandler), transport)
+}
+
+// NewTestServiceWithLogger creates a Service that writes to the
+// given handler, so tests can assert on emitted log records.
+func NewTestServiceWithLogger(
+	transport http.RoundTripper,
+	handler slog.Handler,
+) *Service {
+	return newService(slog.New(handler), transport)
+}
+
+// Drain exports drain for testing.
+func (svc *Service) Drain(ctx context.Context) {
+	svc.drain(ctx)
+}
+
+// OutstandingDeliveries reports how many delivery goroutines
+// are currently tracked as in flight.
+func (svc *Service) OutstandingDeliveries() int64 {
+	return svc.outstanding.Load()
+}
+
+// SetNtfyURL sets the ntfy URL on a Service for testing.
+func (svc *Service) SetNtfyURL(u *url.URL) {
+	svc.ntfyURL = u
+}
+
+// SetSlackWebhookURL sets the Slack webhook URL on a
+// Service for testing.
+func (svc *Service) SetSlackWebhookURL(u *url.URL) {
+	svc.slackWebhookURL = u
+}
+
+// SetMattermostWebhookURL sets the Mattermost webhook URL on
+// a Service for testing.
+func (svc *Service) SetMattermostWebhookURL(u *url.URL) {
+	svc.mattermostWebhookURL = u
+}
+
+// SendNtfy exports sendNtfy for testing.
+func (svc *Service) SendNtfy(
+	ctx context.Context,
+	topicURL *url.URL,
+	title, message, priority string,
+) error {
+	return svc.sendNtfy(ctx, topicURL, title, message, priority)
+}
+
+// SendSlack exports sendSlack for testing.
+func (svc *Service) SendSlack(
+	ctx context.Context,
+	webhookURL *url.URL,
+	failed error,
+	title, message, priority string,
+) error {
+	return svc.sendSlack(
+		ctx, webhookURL, failed, title, message, priority,
+	)
+}
+
+// SetRetryConfig overrides the retry configuration for
+// testing.
+func (svc *Service) SetRetryConfig(cfg RetryConfig) {
+	svc.retryConfig = cfg
+}
+
+// SetSleepFunc overrides the sleep function so tests can
+// eliminate real delays.
+func (svc *Service) SetSleepFunc(
+	fn func(time.Duration) <-chan time.Time,
+) {
+	svc.sleepFn = fn
+}
+
+// DeliverWithRetry exports deliverWithRetry for testing.
+func (svc *Service) DeliverWithRetry(
+	ctx context.Context,
+	endpoint string,
+	fn func(context.Context) error,
+) error {
+	return svc.deliverWithRetry(ctx, endpoint, fn)
+}
+
+// BackoffDuration exports RetryConfig.backoff for testing.
+func (rc RetryConfig) BackoffDuration(attempt int) time.Duration {
+	return rc.defaults().backoff(attempt)
+}

internal/notify/history.go

@@ -0,0 +1,62 @@
+package notify
+
+import (
+	"sync"
+	"time"
+)
+
+// maxAlertHistory is the maximum number of alerts to retain.
+const maxAlertHistory = 100
+
+// AlertEntry represents a single notification that was sent.
+type AlertEntry struct {
+	Timestamp time.Time
+	Title     string
+	Message   string
+	Priority  string
+}
+
+// AlertHistory is a thread-safe ring buffer that stores
+// the most recent alerts.
+type AlertHistory struct {
+	mu      sync.RWMutex
+	entries [maxAlertHistory]AlertEntry
+	count   int
+	index   int
+}
+
+// NewAlertHistory creates a new empty AlertHistory.
+func NewAlertHistory() *AlertHistory {
+	return &AlertHistory{}
+}
+
+// Add records a new alert entry in the ring buffer.
+func (h *AlertHistory) Add(entry AlertEntry) {
+	h.mu.Lock()
+	defer h.mu.Unlock()
+
+	h.entries[h.index] = entry
+	h.index = (h.index + 1) % maxAlertHistory
+
+	if h.count < maxAlertHistory {
+		h.count++
+	}
+}
+
+// Recent returns the stored alerts in reverse chronological
+// order (newest first). Returns at most maxAlertHistory entries.
+func (h *AlertHistory) Recent() []AlertEntry {
+	h.mu.RLock()
+	defer h.mu.RUnlock()
+
+	result := make([]AlertEntry, h.count)
+
+	for i := range h.count {
+		// Walk backwards from the most recent entry.
+		idx := (h.index - 1 - i + maxAlertHistory) %
+			maxAlertHistory
+		result[i] = h.entries[idx]
+	}
+
+	return result
+}

internal/notify/history_test.go

@@ -0,0 +1,88 @@
+package notify_test
+
+import (
+	"testing"
+	"time"
+
+	"sneak.berlin/go/dnswatcher/internal/notify"
+)
+
+func TestAlertHistoryEmpty(t *testing.T) {
+	t.Parallel()
+
+	h := notify.NewAlertHistory()
+
+	entries := h.Recent()
+	if len(entries) != 0 {
+		t.Fatalf("expected 0 entries, got %d", len(entries))
+	}
+}
+
+func TestAlertHistoryAddAndRecent(t *testing.T) {
+	t.Parallel()
+
+	h := notify.NewAlertHistory()
+
+	now := time.Now().UTC()
+
+	h.Add(notify.AlertEntry{
+		Timestamp: now.Add(-2 * time.Minute),
+		Title:     "first",
+		Message:   "msg1",
+		Priority:  prioInfo,
+	})
+
+	h.Add(notify.AlertEntry{
+		Timestamp: now.Add(-1 * time.Minute),
+		Title:     "second",
+		Message:   "msg2",
+		Priority:  prioWarning,
+	})
+
+	entries := h.Recent()
+	if len(entries) != 2 {
+		t.Fatalf("expected 2 entries, got %d", len(entries))
+	}
+
+	// Newest first.
+	if entries[0].Title != "second" {
+		t.Errorf(
+			"expected newest first, got %q", entries[0].Title,
+		)
+	}
+
+	if entries[1].Title != "first" {
+		t.Errorf(
+			"expected oldest second, got %q", entries[1].Title,
+		)
+	}
+}
+
+func TestAlertHistoryOverflow(t *testing.T) {
+	t.Parallel()
+
+	h := notify.NewAlertHistory()
+
+	const totalEntries = 110
+
+	// Fill beyond capacity.
+	for i := range totalEntries {
+		h.Add(notify.AlertEntry{
+			Timestamp: time.Now().UTC(),
+			Title:     "alert",
+			Message:   "msg",
+			Priority:  string(rune('0' + i%10)),
+		})
+	}
+
+	entries := h.Recent()
+
+	const maxHistory = 100
+
+	if len(entries) != maxHistory {
+		t.Fatalf(
+			"expected %d entries, got %d",
+			maxHistory, len(entries),
+		)
+	}
+}

internal/notify/notify.go

@@ -12,6 +12,8 @@ import (
 	"log/slog"
 	"net/http"
 	"net/url"
+	"sync"
+	"sync/atomic"
 	"time"
 
 	"go.uber.org/fx"
@@ -112,18 +114,44 @@ type Service struct {
 	ntfyURL              *url.URL
 	slackWebhookURL      *url.URL
 	mattermostWebhookURL *url.URL
+	history              *AlertHistory
+	retryConfig          RetryConfig
+	sleepFn              func(time.Duration) <-chan time.Time
+
+	// Shutdown draining state.  drainMu guards draining and
+	// serialises it against the counter increment in
+	// startDelivery; inFlight tracks the delivery goroutines
+	// themselves and outstanding mirrors its count so a timed
+	// out drain can report how many were abandoned.
+	drainMu     sync.Mutex
+	draining    bool
+	inFlight    sync.WaitGroup
+	outstanding atomic.Int64
+	abandon     chan struct{}
+	abandonOnce sync.Once
+}
+
+// newService builds a Service with the fields every Service
+// needs regardless of how it was constructed.
+func newService(
+	log *slog.Logger,
+	transport http.RoundTripper,
+) *Service {
+	return &Service{
+		log:       log,
+		transport: transport,
+		history:   NewAlertHistory(),
+		abandon:   make(chan struct{}),
+	}
 }
 
 // New creates a new notify Service.
 func New(
-	_ fx.Lifecycle,
+	lifecycle fx.Lifecycle,
 	params Params,
 ) (*Service, error) {
-	svc := &Service{
-		log:       params.Logger.Get(),
-		transport: http.DefaultTransport,
-		config:    params.Config,
-	}
+	svc := newService(params.Logger.Get(), http.DefaultTransport)
+	svc.config = params.Config
 
 	if params.Config.NtfyTopic != "" {
 		u, err := ValidateWebhookURL(
@@ -164,68 +192,114 @@ func New(
 		svc.mattermostWebhookURL = u
 	}
 
+	lifecycle.Append(fx.Hook{
+		OnStop: func(ctx context.Context) error {
+			svc.drain(ctx)
+
+			return nil
+		},
+	})
+
 	return svc, nil
 }
 
+// History returns the alert history for reading recent alerts.
+func (svc *Service) History() *AlertHistory {
+	return svc.history
+}
+
 // SendNotification sends a notification to all configured
-// endpoints.
+// endpoints and records it in the alert history.
 func (svc *Service) SendNotification(
 	ctx context.Context,
 	title, message, priority string,
 ) {
-	if svc.ntfyURL != nil {
-		go func() {
-			notifyCtx := context.WithoutCancel(ctx)
+	svc.history.Add(AlertEntry{
+		Timestamp: time.Now().UTC(),
+		Title:     title,
+		Message:   message,
+		Priority:  priority,
+	})
 
-			err := svc.sendNtfy(
-				notifyCtx,
-				svc.ntfyURL,
-				title, message, priority,
+	svc.dispatchNtfy(ctx, title, message, priority)
+	svc.dispatchSlack(ctx, title, message, priority)
+	svc.dispatchMattermost(ctx, title, message, priority)
+}
+
+// dispatch delivers a notification to one endpoint on a
+// tracked background goroutine.
+//
+// The delivery context is detached from ctx with
+// context.WithoutCancel so that a cancelled caller does not
+// kill a delivery already under way; the shutdown drain, not
+// the caller, decides how long deliveries may keep running.
+func (svc *Service) dispatch(
+	ctx context.Context,
+	endpoint string,
+	send func(context.Context) error,
+) {
+	notifyCtx := context.WithoutCancel(ctx)
+
+	svc.startDelivery(endpoint, func() {
+		err := svc.deliverWithRetry(notifyCtx, endpoint, send)
+		if err != nil {
+			svc.log.Error(
+				"failed to send notification after retries",
+				"endpoint", endpoint,
+				"error", err,
 			)
-			if err != nil {
-				svc.log.Error(
-					"failed to send ntfy notification",
-					"error", err,
-				)
-			}
-		}()
+		}
+	})
+}
+
+func (svc *Service) dispatchNtfy(
+	ctx context.Context,
+	title, message, priority string,
+) {
+	if svc.ntfyURL == nil {
+		return
 	}
 
-	if svc.slackWebhookURL != nil {
-		go func() {
-			notifyCtx := context.WithoutCancel(ctx)
+	svc.dispatch(ctx, "ntfy", func(c context.Context) error {
+		return svc.sendNtfy(
+			c, svc.ntfyURL, title, message, priority,
+		)
+	})
+}
 
-			err := svc.sendSlack(
-				notifyCtx,
-				svc.slackWebhookURL,
-				title, message, priority,
-			)
-			if err != nil {
-				svc.log.Error(
-					"failed to send slack notification",
-					"error", err,
-				)
-			}
-		}()
+func (svc *Service) dispatchSlack(
+	ctx context.Context,
+	title, message, priority string,
+) {
+	if svc.slackWebhookURL == nil {
+		return
 	}
 
-	if svc.mattermostWebhookURL != nil {
-		go func() {
-			notifyCtx := context.WithoutCancel(ctx)
+	svc.dispatch(ctx, "slack", func(c context.Context) error {
+		return svc.sendSlack(
+			c, svc.slackWebhookURL, ErrSlackFailed,
+			title, message, priority,
+		)
+	})
+}
 
-			err := svc.sendSlack(
-				notifyCtx,
-				svc.mattermostWebhookURL,
+func (svc *Service) dispatchMattermost(
+	ctx context.Context,
+	title, message, priority string,
+) {
+	if svc.mattermostWebhookURL == nil {
+		return
+	}
+
+	svc.dispatch(
+		ctx, "mattermost",
+		func(c context.Context) error {
+			return svc.sendSlack(
+				c, svc.mattermostWebhookURL, ErrMattermostFailed,
 				title, message, priority,
 			)
-			if err != nil {
-				svc.log.Error(
-					"failed to send mattermost notification",
-					"error", err,
-				)
-			}
-		}()
-	}
+		},
+	)
 }
 
 func (svc *Service) sendNtfy(
@@ -297,9 +371,14 @@ type SlackAttachment struct {
 	Text  string `json:"text"`
 }
 
+// sendSlack posts to a Slack or Mattermost incoming webhook, which
+// take the same payload. An HTTP error status is returned wrapped in
+// failed, ErrSlackFailed or ErrMattermostFailed, so the error names
+// the endpoint.
 func (svc *Service) sendSlack(
 	ctx context.Context,
 	webhookURL *url.URL,
+	failed error,
 	title, message, priority string,
 ) error {
 	ctx, cancel := context.WithTimeout(
@@ -347,7 +426,7 @@ func (svc *Service) sendSlack(
 	if resp.StatusCode >= httpStatusClientError {
 		return fmt.Errorf(
 			"%w: status %d",
-			ErrSlackFailed, resp.StatusCode,
+			failed, resp.StatusCode,
 		)
 	}
 

internal/notify/retry.go

@@ -0,0 +1,150 @@
+package notify
+
+import (
+	"context"
+	"fmt"
+	"math"
+	"math/rand/v2"
+	"time"
+)
+
+// Retry defaults.
+const (
+	// DefaultMaxRetries is the number of additional attempts
+	// after the first failure.
+	DefaultMaxRetries = 5
+
+	// DefaultBaseDelay is the initial delay before the first
+	// retry attempt.
+	DefaultBaseDelay = 1 * time.Second
+
+	// DefaultMaxDelay caps the computed backoff delay.
+	DefaultMaxDelay = 60 * time.Second
+
+	// backoffMultiplier is the exponential growth factor.
+	backoffMultiplier = 2
+
+	// jitterFraction controls the ±random spread applied
+	// to each delay (0.25 = ±25%).
+	jitterFraction = 0.25
+)
+
+// RetryConfig holds tuning knobs for the retry loop.
+// Zero values fall back to the package defaults above.
+type RetryConfig struct {
+	MaxRetries int
+	BaseDelay  time.Duration
+	MaxDelay   time.Duration
+}
+
+// defaults returns a copy with zero fields replaced by
+// package defaults.
+func (rc RetryConfig) defaults() RetryConfig {
+	if rc.MaxRetries <= 0 {
+		rc.MaxRetries = DefaultMaxRetries
+	}
+
+	if rc.BaseDelay <= 0 {
+		rc.BaseDelay = DefaultBaseDelay
+	}
+
+	if rc.MaxDelay <= 0 {
+		rc.MaxDelay = DefaultMaxDelay
+	}
+
+	return rc
+}
+
+// backoff computes the delay for attempt n (0-indexed) with
+// jitter.  The raw delay is BaseDelay * 2^n, capped at
+// MaxDelay, then randomised by ±jitterFraction.
+func (rc RetryConfig) backoff(attempt int) time.Duration {
+	raw := float64(rc.BaseDelay) *
+		math.Pow(backoffMultiplier, float64(attempt))
+
+	if raw > float64(rc.MaxDelay) {
+		raw = float64(rc.MaxDelay)
+	}
+
+	// Apply jitter: uniform in [raw*(1-j), raw*(1+j)].
+	lo := raw * (1 - jitterFraction)
+	hi := raw * (1 + jitterFraction)
+
+	jittered := lo + rand.Float64()*(hi-lo) //nolint:gosec // jitter needs no crypto/rand
+
+	return time.Duration(jittered)
+}
+
+// deliverWithRetry calls fn, retrying on error with
+// exponential backoff.  It logs every failed attempt and
+// returns the last error if all attempts are exhausted.
+func (svc *Service) deliverWithRetry(
+	ctx context.Context,
+	endpoint string,
+	fn func(context.Context) error,
+) error {
+	cfg := svc.retryConfig.defaults()
+
+	var lastErr error
+
+	// attempt 0 is the initial call; attempts 1..MaxRetries
+	// are retries.
+	for attempt := range cfg.MaxRetries + 1 {
+		lastErr = fn(ctx)
+		if lastErr == nil {
+			if attempt > 0 {
+				svc.log.Info(
+					"notification delivered after retry",
+					"endpoint", endpoint,
+					"attempt", attempt+1,
+				)
+			}
+
+			return nil
+		}
+
+		// Last attempt — don't sleep, just return.
+		if attempt == cfg.MaxRetries {
+			break
+		}
+
+		delay := cfg.backoff(attempt)
+
+		svc.log.Warn(
+			"notification delivery failed, retrying",
+			"endpoint", endpoint,
+			"attempt", attempt+1,
+			"maxAttempts", cfg.MaxRetries+1,
+			// As text: the JSON log writes a time.Duration as
+			// bare nanoseconds.
+			"retryIn", delay.String(),
+			"error", lastErr,
+		)
+
+		select {
+		case <-ctx.Done():
+			return ctx.Err()
+		case <-svc.abandon:
+			// Shutdown drained past its deadline; stop
+			// sleeping rather than outlive the process.
+			// A nil channel (Service built without a
+			// constructor) simply never fires.
+			return fmt.Errorf(
+				"%w: %s", ErrDeliveryAbandoned, endpoint,
+			)
+		case <-svc.sleepFunc(delay):
+		}
+	}
+
+	return lastErr
+}
+
+// sleepFunc returns a channel that closes after d.
+// It is a field-level indirection so tests can override it.
+func (svc *Service) sleepFunc(d time.Duration) <-chan time.Time {
+	if svc.sleepFn != nil {
+		return svc.sleepFn(d)
+	}
+
+	return time.After(d)
+}

internal/notify/retry_test.go

@@ -0,0 +1,538 @@
+package notify_test
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"net/http"
+	"net/http/httptest"
+	"net/url"
+	"sync"
+	"sync/atomic"
+	"testing"
+	"time"
+
+	"sneak.berlin/go/dnswatcher/internal/notify"
+)
+
+// Static test errors (err113).
+var (
+	errTransient = errors.New("transient failure")
+	errPermanent = errors.New("permanent failure")
+	errFail      = errors.New("fail")
+)
+
+// instantSleep returns a closed channel immediately, removing
+// real delays from tests.
+func instantSleep(_ time.Duration) <-chan time.Time {
+	ch := make(chan time.Time, 1)
+	ch <- time.Now()
+
+	return ch
+}
+
+// ── backoff calculation ───────────────────────────────────
+
+func TestBackoffDurationIncreases(t *testing.T) {
+	t.Parallel()
+
+	cfg := notify.RetryConfig{
+		MaxRetries: 5,
+		BaseDelay:  1 * time.Second,
+		MaxDelay:   30 * time.Second,
+	}
+
+	prev := time.Duration(0)
+
+	// With jitter the exact value varies, but the trend
+	// should be increasing for the first few attempts.
+	for attempt := range 4 {
+		d := cfg.BackoffDuration(attempt)
+		if d <= 0 {
+			t.Fatalf(
+				"attempt %d: backoff must be positive, got %v",
+				attempt, d,
+			)
+		}
+
+		// Allow jitter to occasionally flatten a step, but
+		// the midpoint (no-jitter) should be strictly higher.
+		midpoint := cfg.BaseDelay * (1 << attempt)
+		if attempt > 0 && midpoint <= prev {
+			t.Fatalf(
+				"midpoint should grow: attempt %d midpoint=%v prev=%v",
+				attempt, midpoint, prev,
+			)
+		}
+
+		prev = midpoint
+	}
+}
+
+func TestBackoffDurationCappedAtMax(t *testing.T) {
+	t.Parallel()
+
+	cfg := notify.RetryConfig{
+		MaxRetries: 5,
+		BaseDelay:  1 * time.Second,
+		MaxDelay:   5 * time.Second,
+	}
+
+	// Attempt 10 would be 1024s without capping.
+	d := cfg.BackoffDuration(10)
+
+	// With ±25% jitter on a 5s cap: max is 6.25s.
+	const maxWithJitter = 5*time.Second +
+		5*time.Second/4 +
+		time.Millisecond // rounding margin
+
+	if d > maxWithJitter {
+		t.Errorf(
+			"backoff %v exceeds max+jitter %v",
+			d, maxWithJitter,
+		)
+	}
+}
+
+// ── deliverWithRetry ──────────────────────────────────────
+
+func TestDeliverWithRetrySucceedsFirstAttempt(t *testing.T) {
+	t.Parallel()
+
+	svc := notify.NewTestService(http.DefaultTransport)
+	svc.SetSleepFunc(instantSleep)
+
+	var calls atomic.Int32
+
+	err := svc.DeliverWithRetry(
+		context.Background(), "test",
+		func(_ context.Context) error {
+			calls.Add(1)
+
+			return nil
+		},
+	)
+	if err != nil {
+		t.Fatalf("unexpected error: %v", err)
+	}
+
+	if calls.Load() != 1 {
+		t.Errorf("expected 1 call, got %d", calls.Load())
+	}
+}
+
+func TestDeliverWithRetryRetriesOnFailure(t *testing.T) {
+	t.Parallel()
+
+	svc := notify.NewTestService(http.DefaultTransport)
+	svc.SetSleepFunc(instantSleep)
+	svc.SetRetryConfig(notify.RetryConfig{
+		MaxRetries: 3,
+		BaseDelay:  time.Millisecond,
+		MaxDelay:   10 * time.Millisecond,
+	})
+
+	var calls atomic.Int32
+
+	// Fail twice, then succeed on the third attempt.
+	err := svc.DeliverWithRetry(
+		context.Background(), "test",
+		func(_ context.Context) error {
+			n := calls.Add(1)
+			if n <= 2 {
+				return errTransient
+			}
+
+			return nil
+		},
+	)
+	if err != nil {
+		t.Fatalf("expected success after retries: %v", err)
+	}
+
+	if calls.Load() != 3 {
+		t.Errorf("expected 3 calls, got %d", calls.Load())
+	}
+}
+
+func TestDeliverWithRetryExhaustsAttempts(t *testing.T) {
+	t.Parallel()
+
+	svc := notify.NewTestService(http.DefaultTransport)
+	svc.SetSleepFunc(instantSleep)
+	svc.SetRetryConfig(notify.RetryConfig{
+		MaxRetries: 2,
+		BaseDelay:  time.Millisecond,
+		MaxDelay:   10 * time.Millisecond,
+	})
+
+	var calls atomic.Int32
+
+	err := svc.DeliverWithRetry(
+		context.Background(), "test",
+		func(_ context.Context) error {
+			calls.Add(1)
+
+			return errPermanent
+		},
+	)
+	if err == nil {
+		t.Fatal("expected error when all retries exhausted")
+	}
+
+	if !errors.Is(err, errPermanent) {
+		t.Errorf("expected permanent failure, got: %v", err)
+	}
+
+	// 1 initial + 2 retries = 3 total.
+	if calls.Load() != 3 {
+		t.Errorf("expected 3 calls, got %d", calls.Load())
+	}
+}
+
+// TestDeliverWithRetryLogsRetryInAsText checks that the wait
+// before a retry is logged as text such as "1.02s", not as a
+// count of nanoseconds.
+func TestDeliverWithRetryLogsRetryInAsText(t *testing.T) {
+	t.Parallel()
+
+	svc, logs := newLoggingService(http.DefaultTransport)
+	svc.SetRetryConfig(notify.RetryConfig{
+		MaxRetries: 1,
+		BaseDelay:  time.Second,
+		MaxDelay:   time.Second,
+	})
+
+	var waited time.Duration
+
+	svc.SetSleepFunc(func(d time.Duration) <-chan time.Time {
+		waited = d
+
+		return instantSleep(d)
+	})
+
+	_ = svc.DeliverWithRetry(
+		context.Background(), "test",
+		func(_ context.Context) error {
+			return errFail
+		},
+	)
+
+	// With one retry, only the first failure is logged.
+	var record map[string]any
+
+	err := json.Unmarshal([]byte(logs.String()), &record)
+	if err != nil {
+		t.Fatalf("log is not one JSON record: %v\n%s", err, logs)
+	}
+
+	if record["retryIn"] != waited.String() {
+		t.Errorf(
+			"retryIn logged as %v, want %q",
+			record["retryIn"], waited.String(),
+		)
+	}
+}
+
+func TestDeliverWithRetryRespectsContextCancellation(
+	t *testing.T,
+) {
+	t.Parallel()
+
+	svc := notify.NewTestService(http.DefaultTransport)
+	svc.SetRetryConfig(notify.RetryConfig{
+		MaxRetries: 5,
+		BaseDelay:  time.Millisecond,
+		MaxDelay:   10 * time.Millisecond,
+	})
+
+	// Use a blocking sleep so the context cancellation is
+	// the only way out.
+	svc.SetSleepFunc(func(_ time.Duration) <-chan time.Time {
+		return make(chan time.Time) // never fires
+	})
+
+	ctx, cancel := context.WithCancel(context.Background())
+
+	done := make(chan error, 1)
+
+	go func() {
+		done <- svc.DeliverWithRetry(
+			ctx, "test",
+			func(_ context.Context) error {
+				return errFail
+			},
+		)
+	}()
+
+	// Wait for the first failure + retry sleep to be
+	// entered, then cancel.
+	time.Sleep(50 * time.Millisecond)
+	cancel()
+
+	select {
+	case err := <-done:
+		if !errors.Is(err, context.Canceled) {
+			t.Errorf(
+				"expected context.Canceled, got: %v", err,
+			)
+		}
+	case <-time.After(2 * time.Second):
+		t.Fatal("deliverWithRetry did not return after cancel")
+	}
+}
+
+// ── integration: SendNotification with retry ──────────────
+
+func TestSendNotificationRetriesTransientFailure(
+	t *testing.T,
+) {
+	t.Parallel()
+
+	var (
+		mu       sync.Mutex
+		attempts int
+	)
+
+	srv := httptest.NewServer(
+		http.HandlerFunc(
+			func(w http.ResponseWriter, _ *http.Request) {
+				mu.Lock()
+				attempts++
+				n := attempts
+				mu.Unlock()
+
+				if n <= 2 {
+					w.WriteHeader(
+						http.StatusInternalServerError,
+					)
+
+					return
+				}
+
+				w.WriteHeader(http.StatusOK)
+			}),
+	)
+	defer srv.Close()
+
+	svc := newRetryTestService(srv.URL, "ntfy")
+
+	svc.SendNotification(
+		context.Background(),
+		"Retry Test", "body", "warning",
+	)
+
+	waitForCondition(t, func() bool {
+		mu.Lock()
+		defer mu.Unlock()
+
+		return attempts >= 3
+	})
+}
+
+// newRetryTestService creates a test service with instant
+// sleep and low retry delays for the named endpoint.
+func newRetryTestService(
+	rawURL, endpoint string,
+) *notify.Service {
+	svc := notify.NewTestService(http.DefaultTransport)
+	svc.SetSleepFunc(instantSleep)
+	svc.SetRetryConfig(notify.RetryConfig{
+		MaxRetries: 3,
+		BaseDelay:  time.Millisecond,
+		MaxDelay:   10 * time.Millisecond,
+	})
+
+	u, _ := url.Parse(rawURL)
+
+	switch endpoint {
+	case "ntfy":
+		svc.SetNtfyURL(u)
+	case "slack":
+		svc.SetSlackWebhookURL(u)
+	case "mattermost":
+		svc.SetMattermostWebhookURL(u)
+	}
+
+	return svc
+}
+
+func TestSendNotificationAllEndpointsRetrySetup(
+	t *testing.T,
+) {
+	t.Parallel()
+
+	result := newEndpointRetryResult()
+	ntfySrv, slackSrv, mmSrv := newRetryServers(result)
+
+	defer ntfySrv.Close()
+	defer slackSrv.Close()
+	defer mmSrv.Close()
+
+	svc := buildAllEndpointRetryService(
+		ntfySrv.URL, slackSrv.URL, mmSrv.URL,
+	)
+
+	svc.SendNotification(
+		context.Background(),
+		"Multi-Retry", "testing", "error",
+	)
+
+	assertAllEndpointsRetried(t, result)
+}
+
+// endpointRetryResult tracks per-endpoint retry state.
+type endpointRetryResult struct {
+	mu            sync.Mutex
+	ntfyAttempts  int
+	slackAttempts int
+	mmAttempts    int
+	ntfyOK        bool
+	slackOK       bool
+	mmOK          bool
+}
+
+func newEndpointRetryResult() *endpointRetryResult {
+	return &endpointRetryResult{}
+}
+
+func newRetryServers(
+	r *endpointRetryResult,
+) (*httptest.Server, *httptest.Server, *httptest.Server) {
+	mk := func(
+		attempts *int, ok *bool,
+	) *httptest.Server {
+		return httptest.NewServer(
+			http.HandlerFunc(
+				func(w http.ResponseWriter, _ *http.Request) {
+					r.mu.Lock()
+					*attempts++
+					n := *attempts
+					r.mu.Unlock()
+
+					if n == 1 {
+						w.WriteHeader(
+							http.StatusServiceUnavailable,
+						)
+
+						return
+					}
+
+					r.mu.Lock()
+					*ok = true
+					r.mu.Unlock()
+
+					w.WriteHeader(http.StatusOK)
+				}),
+		)
+	}
+
+	return mk(&r.ntfyAttempts, &r.ntfyOK),
+		mk(&r.slackAttempts, &r.slackOK),
+		mk(&r.mmAttempts, &r.mmOK)
+}
+
+func buildAllEndpointRetryService(
+	ntfyURL, slackURL, mmURL string,
+) *notify.Service {
+	svc := notify.NewTestService(http.DefaultTransport)
+	svc.SetSleepFunc(instantSleep)
+	svc.SetRetryConfig(notify.RetryConfig{
+		MaxRetries: 3,
+		BaseDelay:  time.Millisecond,
+		MaxDelay:   10 * time.Millisecond,
+	})
+
+	nu, _ := url.Parse(ntfyURL)
+	su, _ := url.Parse(slackURL)
+	mu, _ := url.Parse(mmURL)
+
+	svc.SetNtfyURL(nu)
+	svc.SetSlackWebhookURL(su)
+	svc.SetMattermostWebhookURL(mu)
+
+	return svc
+}
+
+func assertAllEndpointsRetried(
+	t *testing.T,
+	r *endpointRetryResult,
+) {
+	t.Helper()
+
+	waitForCondition(t, func() bool {
+		r.mu.Lock()
+		defer r.mu.Unlock()
+
+		return r.ntfyOK && r.slackOK && r.mmOK
+	})
+
+	r.mu.Lock()
+	defer r.mu.Unlock()
+
+	if r.ntfyAttempts < 2 {
+		t.Errorf(
+			"ntfy: expected >= 2 attempts, got %d",
+			r.ntfyAttempts,
+		)
+	}
+
+	if r.slackAttempts < 2 {
+		t.Errorf(
+			"slack: expected >= 2 attempts, got %d",
+			r.slackAttempts,
+		)
+	}
+
+	if r.mmAttempts < 2 {
+		t.Errorf(
+			"mattermost: expected >= 2 attempts, got %d",
+			r.mmAttempts,
+		)
+	}
+}
+
+func TestSendNotificationPermanentFailureLogsError(
+	t *testing.T,
+) {
+	t.Parallel()
+
+	var (
+		mu       sync.Mutex
+		attempts int
+	)
+
+	srv := httptest.NewServer(
+		http.HandlerFunc(
+			func(w http.ResponseWriter, _ *http.Request) {
+				mu.Lock()
+				attempts++
+				mu.Unlock()
+
+				w.WriteHeader(
+					http.StatusInternalServerError,
+				)
+			}),
+	)
+	defer srv.Close()
+
+	svc := newRetryTestService(srv.URL, "slack")
+	svc.SetRetryConfig(notify.RetryConfig{
+		MaxRetries: 2,
+		BaseDelay:  time.Millisecond,
+		MaxDelay:   10 * time.Millisecond,
+	})
+
+	svc.SendNotification(
+		context.Background(),
+		"Permanent Fail", "body", "error",
+	)
+
+	// 1 initial + 2 retries = 3 total.
+	waitForCondition(t, func() bool {
+		mu.Lock()
+		defer mu.Unlock()
+
+		return attempts >= 3
+	})
+}

internal/notify/shutdown.go

@@ -0,0 +1,119 @@
+package notify
+
+import (
+	"context"
+	"errors"
+)
+
+// ErrDeliveryAbandoned is returned by a retry loop that was
+// cut short because shutdown drained past its deadline.
+var ErrDeliveryAbandoned = errors.New(
+	"notification delivery abandoned at shutdown",
+)
+
+// startDelivery runs fn on its own goroutine while tracking it,
+// so that drain can wait for it during shutdown.
+//
+// The WaitGroup counter is incremented here, on the caller's
+// goroutine, before the worker exists: incrementing it inside
+// the worker would race with drain's Wait and could let
+// shutdown sail past a delivery that had not started yet.
+//
+// Once draining has begun the delivery is refused outright
+// rather than queued, so a steady stream of newly submitted
+// notifications cannot keep extending the drain.
+func (svc *Service) startDelivery(endpoint string, fn func()) {
+	svc.drainMu.Lock()
+
+	if svc.draining {
+		svc.drainMu.Unlock()
+
+		svc.log.Warn(
+			"notification not dispatched: shutdown in progress",
+			"endpoint", endpoint,
+		)
+
+		return
+	}
+
+	svc.outstanding.Add(1)
+
+	// WaitGroup.Go increments the counter synchronously, here,
+	// and only then starts the goroutine.
+	svc.inFlight.Go(func() {
+		// Runs before the WaitGroup counter is decremented, so
+		// a drain that times out reports an accurate count.
+		defer svc.outstanding.Add(-1)
+
+		fn()
+	})
+
+	svc.drainMu.Unlock()
+}
+
+// drain waits for in-flight notification deliveries to finish.
+//
+// It first stops accepting new deliveries, then waits until
+// either every outstanding delivery has completed or ctx
+// expires — whichever comes first.  ctx is the context fx
+// passes to the OnStop hook, so a permanently dead webhook
+// cannot hang shutdown indefinitely.
+//
+// When the deadline arrives with deliveries still outstanding,
+// the count is logged at warn level and the abandon channel is
+// closed, which releases any retry loop sleeping in backoff.
+// Deliveries already inside an HTTP round trip are bounded by
+// the existing httpClientTimeout instead.
+//
+// A ctx that is already expired on entry is not by itself cause
+// for alarm: if nothing is outstanding there is nothing to
+// abandon, and the drain says so at debug level rather than
+// warning about deliveries that do not exist.
+func (svc *Service) drain(ctx context.Context) {
+	svc.drainMu.Lock()
+	svc.draining = true
+	svc.drainMu.Unlock()
+
+	done := make(chan struct{})
+
+	go func() {
+		svc.inFlight.Wait()
+		close(done)
+	}()
+
+	select {
+	case <-done:
+		svc.log.Debug(
+			"all in-flight notifications completed",
+		)
+	case <-ctx.Done():
+		// outstanding is decremented before the WaitGroup
+		// counter, and startDelivery can no longer add to it
+		// now that draining is set, so a zero here means every
+		// delivery really did finish.  ctx expiring in that
+		// state (an OnStop context that was already cancelled
+		// on entry is the usual way) abandons nothing, so it
+		// must not close abandon or warn about it.
+		abandoned := svc.outstanding.Load()
+		if abandoned == 0 {
+			svc.log.Debug(
+				"all in-flight notifications completed",
+			)
+
+			return
+		}
+
+		svc.abandonOnce.Do(func() {
+			if svc.abandon != nil {
+				close(svc.abandon)
+			}
+		})
+
+		svc.log.Warn(
+			"shutdown deadline reached with notifications "+
+				"still in flight; abandoning them",
+			"abandoned", abandoned,
+			"error", ctx.Err(),
+		)
+	}
+}

internal/notify/shutdown_test.go

@@ -0,0 +1,593 @@
+package notify_test
+
+import (
+	"bytes"
+	"context"
+	"log/slog"
+	"net/http"
+	"net/http/httptest"
+	"net/url"
+	"strings"
+	"sync"
+	"sync/atomic"
+	"testing"
+	"time"
+
+	"go.uber.org/fx"
+
+	"sneak.berlin/go/dnswatcher/internal/config"
+	"sneak.berlin/go/dnswatcher/internal/globals"
+	"sneak.berlin/go/dnswatcher/internal/logger"
+	"sneak.berlin/go/dnswatcher/internal/notify"
+)
+
+// Timings used by the drain tests.  They stay in the same
+// 10-100ms band as the retry tests so the suite never waits on
+// a real backoff delay.
+const (
+	// inFlightHold is how long a delivery is kept mid-request
+	// before the handler is released.
+	inFlightHold = 30 * time.Millisecond
+
+	// drainDeadline bounds a drain that is expected to time
+	// out.
+	drainDeadline = 50 * time.Millisecond
+
+	// timeoutDrainBound is how long a drain given drainDeadline
+	// may take to return before the test gives up on it.  At
+	// forty times drainDeadline it leaves ample room for
+	// scheduling delay on a loaded box under -race, yet it is far
+	// below the test binary's -timeout, so a drain that its
+	// deadline does not bound fails that one test instead of
+	// hanging the package.
+	timeoutDrainBound = 2 * time.Second
+
+	// longDrainDeadline is the deadline given to a drain that is
+	// expected to finish well before it: when the in-flight
+	// delivery completes after inFlightHold, or at once when
+	// nothing is in flight.  It is far above inFlightHold, so
+	// those drains never reach it, and four times
+	// idleDrainBound, so an idle drain that waited for its
+	// deadline instead of returning fails that bound.
+	longDrainDeadline = 2 * time.Second
+
+	// reachEndpointTimeout is how long a submitted delivery may
+	// take to reach the test server.  That normally takes a few
+	// milliseconds; the margin is for a loaded box under -race,
+	// and only a failing run ever waits this long.
+	reachEndpointTimeout = 2 * time.Second
+
+	// settleDelay is how long to wait before asserting that
+	// something did *not* happen.
+	settleDelay = 50 * time.Millisecond
+
+	// idleDrainBound is the upper bound on a drain that has
+	// nothing in flight.  It is deliberately far above the cost
+	// of the goroutine hop through inFlight.Wait() — which
+	// reached 57ms on a loaded box under -race with the package's
+	// parallel tests — and far below longDrainDeadline, the
+	// deadline such a drain is given.  A drain that blocked until
+	// its deadline instead of returning on the WaitGroup
+	// therefore still fails this bound, but scheduling delay
+	// alone cannot.
+	idleDrainBound = 500 * time.Millisecond
+)
+
+// syncBuffer is an io.Writer safe for concurrent use, so log
+// output written from delivery goroutines can be inspected.
+type syncBuffer struct {
+	mu  sync.Mutex
+	buf bytes.Buffer
+}
+
+func (sb *syncBuffer) Write(p []byte) (int, error) {
+	sb.mu.Lock()
+	defer sb.mu.Unlock()
+
+	return sb.buf.Write(p) //nolint:wrapcheck // test helper
+}
+
+func (sb *syncBuffer) String() string {
+	sb.mu.Lock()
+	defer sb.mu.Unlock()
+
+	return sb.buf.String()
+}
+
+// newLoggingService returns a Service writing JSON logs into
+// the returned buffer, debug level included.
+func newLoggingService(
+	transport http.RoundTripper,
+) (*notify.Service, *syncBuffer) {
+	logs := &syncBuffer{}
+	handler := slog.NewJSONHandler(
+		logs, &slog.HandlerOptions{Level: slog.LevelDebug},
+	)
+
+	return notify.NewTestServiceWithLogger(transport, handler),
+		logs
+}
+
+// blockingNtfyServer returns a server whose handler signals on
+// entered, waits for release, and then responds 200.
+func blockingNtfyServer(
+	entered chan<- struct{},
+	release <-chan struct{},
+	served *atomic.Bool,
+) *httptest.Server {
+	var once sync.Once
+
+	return httptest.NewServer(
+		http.HandlerFunc(
+			func(w http.ResponseWriter, _ *http.Request) {
+				once.Do(func() { close(entered) })
+				<-release
+
+				served.Store(true)
+
+				w.WriteHeader(http.StatusOK)
+			}),
+	)
+}
+
+// TestDrainWaitsForInFlightDelivery verifies that a delivery
+// already under way when shutdown starts is allowed to finish.
+func TestDrainWaitsForInFlightDelivery(t *testing.T) {
+	t.Parallel()
+
+	var served atomic.Bool
+
+	entered := make(chan struct{})
+	release := make(chan struct{})
+
+	srv := blockingNtfyServer(entered, release, &served)
+	defer srv.Close()
+
+	// srv.Close waits for the handler, so release it however the
+	// test ends; otherwise a drain that returns early hangs the
+	// package instead of failing this test.
+	releaseHandler := sync.OnceFunc(func() { close(release) })
+	defer releaseHandler()
+
+	topicURL, _ := url.Parse(srv.URL)
+
+	svc := notify.NewTestService(http.DefaultTransport)
+	svc.SetNtfyURL(topicURL)
+
+	svc.SendNotification(
+		context.Background(), "t", "m", prioInfo,
+	)
+
+	// Make sure the delivery really is mid-request before the
+	// drain begins.
+	select {
+	case <-entered:
+	case <-time.After(reachEndpointTimeout):
+		t.Fatal("delivery never reached the endpoint")
+	}
+
+	// As in TestDrainBoundedByContextDeadline: start is captured
+	// before the clock it is compared against, here the timer
+	// holding the delivery open, so elapsed covers the whole hold
+	// and the lower bound cannot come out short from scheduling
+	// delay alone.
+	start := time.Now()
+
+	timer := time.AfterFunc(inFlightHold, releaseHandler)
+	defer timer.Stop()
+
+	ctx, cancel := context.WithTimeout(
+		context.Background(), longDrainDeadline,
+	)
+	defer cancel()
+
+	svc.Drain(ctx)
+
+	elapsed := time.Since(start)
+
+	if !served.Load() {
+		t.Error(
+			"drain returned before the in-flight delivery " +
+				"completed",
+		)
+	}
+
+	if elapsed < inFlightHold {
+		t.Errorf(
+			"drain took %v, want at least %v",
+			elapsed, inFlightHold,
+		)
+	}
+
+	if got := svc.OutstandingDeliveries(); got != 0 {
+		t.Errorf("outstanding deliveries = %d, want 0", got)
+	}
+}
+
+// neverFires returns a channel that never delivers, standing in
+// for a long backoff sleep without actually sleeping.
+func neverFires(_ time.Duration) <-chan time.Time {
+	return make(chan time.Time)
+}
+
+// TestDrainBoundedByContextDeadline verifies that a delivery
+// stuck retrying against a dead endpoint does not hold shutdown
+// past the OnStop context deadline, and that the abandoned
+// deliveries are logged at warn level rather than dropped
+// silently.
+func TestDrainBoundedByContextDeadline(t *testing.T) {
+	t.Parallel()
+
+	var requests atomic.Int64
+
+	srv := httptest.NewServer(
+		http.HandlerFunc(
+			func(w http.ResponseWriter, _ *http.Request) {
+				requests.Add(1)
+
+				w.WriteHeader(http.StatusInternalServerError)
+			}),
+	)
+	defer srv.Close()
+
+	topicURL, _ := url.Parse(srv.URL)
+
+	svc, logs := newLoggingService(http.DefaultTransport)
+	svc.SetNtfyURL(topicURL)
+	// Never let the backoff sleep complete: the delivery is
+	// parked in its retry wait until shutdown releases it.
+	svc.SetSleepFunc(neverFires)
+	svc.SetRetryConfig(notify.RetryConfig{
+		MaxRetries: 5,
+		BaseDelay:  time.Hour,
+		MaxDelay:   time.Hour,
+	})
+
+	svc.SendNotification(
+		context.Background(), "t", "m", prioError,
+	)
+
+	waitForCondition(t, func() bool {
+		return requests.Load() >= 1 &&
+			svc.OutstandingDeliveries() == 1
+	})
+
+	// start must be captured *before* the deadline clock starts,
+	// so that the measured interval is a superset of the deadline
+	// interval.  Capturing it after context.WithTimeout would
+	// make elapsed structurally smaller than drainDeadline and
+	// the lower bound below unfalsifiable-by-luck: it would fail
+	// whenever the two statements were separated by any
+	// scheduling delay, and pass otherwise, regardless of what
+	// the drain did.
+	start := time.Now()
+
+	ctx, cancel := context.WithTimeout(
+		context.Background(), drainDeadline,
+	)
+	defer cancel()
+
+	// The upper bound is enforced by a watchdog rather than by
+	// measuring after the fact: a drain that is not bounded at
+	// all never returns here (the delivery is parked in a backoff
+	// that never fires), so an unbounded drain must fail this
+	// test promptly instead of hanging the package until the test
+	// binary's -timeout.
+	returned := make(chan struct{})
+
+	go func() {
+		defer close(returned)
+
+		svc.Drain(ctx)
+	}()
+
+	select {
+	case <-returned:
+	case <-time.After(timeoutDrainBound):
+		t.Fatalf(
+			"drain did not return within %v; its %v deadline "+
+				"did not bound it",
+			timeoutDrainBound, drainDeadline,
+		)
+	}
+
+	// The lower bound is the real assertion: the drain must have
+	// waited for its whole deadline rather than giving up on the
+	// outstanding delivery early.  With start captured above, an
+	// early return is the only thing that can make it fail.
+	if elapsed := time.Since(start); elapsed < drainDeadline {
+		t.Errorf(
+			"drain returned after %v, before its %v deadline",
+			elapsed, drainDeadline,
+		)
+	}
+
+	assertAbandonLogged(t, logs.String())
+
+	// The abandoned delivery must stop retrying rather than
+	// outlive the drain.
+	waitForCondition(t, func() bool {
+		return svc.OutstandingDeliveries() == 0
+	})
+}
+
+// assertAbandonLogged checks that the drain logged the
+// abandoned deliveries at warn level with a count.
+func assertAbandonLogged(t *testing.T, output string) {
+	t.Helper()
+
+	if !strings.Contains(output, `"level":"WARN"`) {
+		t.Errorf(
+			"abandoned deliveries not logged at warn level; "+
+				"log output: %s",
+			output,
+		)
+	}
+
+	if !strings.Contains(output, `"abandoned":1`) {
+		t.Errorf(
+			"abandoned delivery count not logged; "+
+				"log output: %s",
+			output,
+		)
+	}
+}
+
+// TestDrainRefusesNewDeliveries verifies that notifications
+// submitted after the drain has begun are refused and logged,
+// so a stream of new work cannot extend shutdown indefinitely.
+func TestDrainRefusesNewDeliveries(t *testing.T) {
+	t.Parallel()
+
+	var requests atomic.Int64
+
+	srv := httptest.NewServer(
+		http.HandlerFunc(
+			func(w http.ResponseWriter, _ *http.Request) {
+				requests.Add(1)
+
+				w.WriteHeader(http.StatusOK)
+			}),
+	)
+	defer srv.Close()
+
+	target, _ := url.Parse(srv.URL)
+
+	svc, logs := newLoggingService(http.DefaultTransport)
+	svc.SetNtfyURL(target)
+	svc.SetSlackWebhookURL(target)
+	svc.SetMattermostWebhookURL(target)
+
+	ctx, cancel := context.WithTimeout(
+		context.Background(), longDrainDeadline,
+	)
+	defer cancel()
+
+	// Nothing is in flight, so this returns immediately and
+	// leaves the service refusing further deliveries.
+	svc.Drain(ctx)
+
+	for range 3 {
+		svc.SendNotification(
+			context.Background(), "t", "m", prioInfo,
+		)
+	}
+
+	time.Sleep(settleDelay)
+
+	if got := requests.Load(); got != 0 {
+		t.Errorf(
+			"%d requests reached the endpoint after drain, "+
+				"want 0",
+			got,
+		)
+	}
+
+	if got := svc.OutstandingDeliveries(); got != 0 {
+		t.Errorf("outstanding deliveries = %d, want 0", got)
+	}
+
+	output := logs.String()
+	if !strings.Contains(output, "shutdown in progress") {
+		t.Errorf(
+			"refused deliveries not logged; log output: %s",
+			output,
+		)
+	}
+}
+
+// recordingLifecycle is a minimal fx.Lifecycle that records the
+// hooks appended to it, so the wiring done by notify.New can be
+// inspected without standing up a whole fx application.
+type recordingLifecycle struct {
+	hooks []fx.Hook
+}
+
+func (l *recordingLifecycle) Append(hook fx.Hook) {
+	l.hooks = append(l.hooks, hook)
+}
+
+// newNotifyService builds a Service through the real
+// constructor, wired to the given lifecycle.
+func newNotifyService(
+	t *testing.T,
+	lifecycle fx.Lifecycle,
+	ntfyTopic string,
+) *notify.Service {
+	t.Helper()
+
+	g, err := globals.New(nil)
+	if err != nil {
+		t.Fatalf("globals.New: %v", err)
+	}
+
+	log, err := logger.New(nil, logger.Params{Globals: g})
+	if err != nil {
+		t.Fatalf("logger.New: %v", err)
+	}
+
+	svc, err := notify.New(lifecycle, notify.Params{
+		Logger: log,
+		Config: &config.Config{NtfyTopic: ntfyTopic},
+	})
+	if err != nil {
+		t.Fatalf("notify.New: %v", err)
+	}
+
+	return svc
+}
+
+// TestNewRegistersDrainingStopHook verifies that notify.New
+// wires an OnStop hook into the fx lifecycle and that the hook
+// waits for in-flight deliveries.
+func TestNewRegistersDrainingStopHook(t *testing.T) {
+	t.Parallel()
+
+	var served atomic.Bool
+
+	entered := make(chan struct{})
+	release := make(chan struct{})
+
+	srv := blockingNtfyServer(entered, release, &served)
+	defer srv.Close()
+
+	// As in TestDrainWaitsForInFlightDelivery: release the handler
+	// however the test ends, before srv.Close waits for it.
+	releaseHandler := sync.OnceFunc(func() { close(release) })
+	defer releaseHandler()
+
+	lifecycle := &recordingLifecycle{}
+	svc := newNotifyService(t, lifecycle, srv.URL)
+
+	if len(lifecycle.hooks) != 1 {
+		t.Fatalf(
+			"appended %d lifecycle hooks, want 1",
+			len(lifecycle.hooks),
+		)
+	}
+
+	stop := lifecycle.hooks[0].OnStop
+	if stop == nil {
+		t.Fatal("lifecycle hook has no OnStop function")
+	}
+
+	svc.SendNotification(
+		context.Background(), "t", "m", prioInfo,
+	)
+
+	select {
+	case <-entered:
+	case <-time.After(reachEndpointTimeout):
+		t.Fatal("delivery never reached the endpoint")
+	}
+
+	timer := time.AfterFunc(inFlightHold, releaseHandler)
+	defer timer.Stop()
+
+	ctx, cancel := context.WithTimeout(
+		context.Background(), longDrainDeadline,
+	)
+	defer cancel()
+
+	err := stop(ctx)
+	if err != nil {
+		t.Fatalf("OnStop returned error: %v", err)
+	}
+
+	if !served.Load() {
+		t.Error(
+			"OnStop returned before the in-flight delivery " +
+				"completed",
+		)
+	}
+}
+
+// TestDrainWithoutDeliveriesReturnsImmediately verifies the
+// common case: nothing in flight, shutdown is not delayed.
+func TestDrainWithoutDeliveriesReturnsImmediately(t *testing.T) {
+	t.Parallel()
+
+	svc := notify.NewTestService(http.DefaultTransport)
+
+	// Captured before the deadline clock, as elsewhere in this
+	// file; for an upper bound that is the conservative
+	// direction, since the measured interval can then only be
+	// longer than the drain itself.
+	start := time.Now()
+
+	ctx, cancel := context.WithTimeout(
+		context.Background(), longDrainDeadline,
+	)
+	defer cancel()
+
+	svc.Drain(ctx)
+
+	if elapsed := time.Since(start); elapsed > idleDrainBound {
+		t.Errorf(
+			"drain of an idle service took %v, want at most "+
+				"%v; its deadline was %v",
+			elapsed, idleDrainBound, longDrainDeadline,
+		)
+	}
+}
+
+// TestDrainWithCancelledContextDoesNotWarn verifies that an
+// OnStop context that is already dead on entry does not produce
+// an "abandoning them" warning when there was nothing in flight
+// to abandon, and that the drain returns and says at debug level
+// that nothing was in flight.  The expired context wins the
+// select immediately, so only the outstanding count can tell the
+// difference between a genuine timeout and a shutdown that had
+// simply already run out of time with no work left.
+func TestDrainWithCancelledContextDoesNotWarn(t *testing.T) {
+	t.Parallel()
+
+	svc, logs := newLoggingService(http.DefaultTransport)
+
+	ctx, cancel := context.WithCancel(context.Background())
+	cancel()
+
+	// A watchdog, as in TestDrainBoundedByContextDeadline, so
+	// that a drain which never returns fails here instead of
+	// hanging the package.
+	returned := make(chan struct{})
+
+	go func() {
+		defer close(returned)
+
+		svc.Drain(ctx)
+	}()
+
+	select {
+	case <-returned:
+	case <-time.After(idleDrainBound):
+		t.Fatalf(
+			"drain with nothing in flight and a cancelled "+
+				"context did not return within %v",
+			idleDrainBound,
+		)
+	}
+
+	output := logs.String()
+
+	// The absence of a warning alone would also pass if the drain
+	// logged nothing at all, so require the debug line it writes
+	// when it finds nothing outstanding.
+	if !strings.Contains(
+		output, "all in-flight notifications completed",
+	) {
+		t.Errorf(
+			"drain did not log that nothing was in flight; "+
+				"log output: %s",
+			output,
+		)
+	}
+
+	if strings.Contains(output, `"level":"WARN"`) {
+		t.Errorf(
+			"drain with nothing in flight warned about "+
+				"abandoned deliveries; log output: %s",
+			output,
+		)
+	}
+}

internal/portcheck/portcheck.go

@@ -193,7 +193,9 @@ func (c *Checker) checkConnection(
 	c.log.Debug(
 		"port check succeeded",
 		"target", target,
-		"latency", latency,
+		// As text: the JSON log writes a time.Duration as bare
+		// nanoseconds.
+		"latency", latency.String(),
 	)
 
 	return &PortResult{

internal/resolver/dns_client.go

@@ -7,8 +7,8 @@ import (
 	"github.com/miekg/dns"
 )
 
-// DNSClient abstracts DNS wire-protocol exchanges so the resolver
-// can be tested without hitting real nameservers.
+// DNSClient sends one DNS message to a nameserver and returns the
+// reply. The resolver holds one for UDP and one for TCP.
 type DNSClient interface {
 	ExchangeContext(
 		ctx context.Context,

internal/resolver/errors.go

@@ -4,17 +4,29 @@ import "errors"
 
 // Sentinel errors returned by the resolver.
 var (
-	// ErrNotImplemented indicates a method is stubbed out.
-	ErrNotImplemented = errors.New(
-		"resolver not yet implemented",
-	)
-
 	// ErrNoNameservers is returned when no authoritative NS
 	// could be discovered for a domain.
 	ErrNoNameservers = errors.New(
 		"no authoritative nameservers found",
 	)
 
+	// ErrNoNameserverAnswered is returned when every nameserver
+	// asked about a name timed out, failed or returned a referral,
+	// so whether the name has addresses is unknown.
+	ErrNoNameserverAnswered = errors.New("no nameserver answered")
+
+	// ErrUnusableReply is returned when a server replied with an
+	// error such as SERVFAIL, or with a referral that leads no
+	// closer to the name asked about.
+	ErrUnusableReply = errors.New(
+		"reply is an error or a referral that leads no closer",
+	)
+
+	// ErrIntercepted is returned when every root server refused a
+	// query. Root servers refuse no query, so the refusals came from
+	// something on the network answering in their place.
+	ErrIntercepted = errors.New("this network intercepts DNS queries")
+
 	// ErrCNAMEDepthExceeded is returned when a CNAME chain
 	// exceeds MaxCNAMEDepth.
 	ErrCNAMEDepthExceeded = errors.New(

internal/resolver/export_test.go

@@ -0,0 +1,70 @@
+package resolver
+
+import (
+	"context"
+
+	"github.com/miekg/dns"
+)
+
+// ExtractRecordValue exports extractRecordValue for testing.
+func ExtractRecordValue(rr dns.RR) string {
+	return extractRecordValue(rr)
+}
+
+// UsableReply exports usableReply for testing.
+func UsableReply(resp *dns.Msg, zone string, name string) bool {
+	return usableReply(resp, zone, name)
+}
+
+// NSSetFrom exports nsSetFrom for testing.
+func NSSetFrom(resp *dns.Msg, domain string) []string {
+	return nsSetFrom(resp, domain)
+}
+
+// CollectIPs exports collectIPs for testing.
+func CollectIPs(
+	results map[string]*NameserverResponse,
+) ([]string, string, error) {
+	return collectIPs(results)
+}
+
+// QueryServers exports queryServers for testing.
+func (r *Resolver) QueryServers(
+	ctx context.Context,
+	servers []string,
+	zone string,
+	name string,
+	qtype uint16,
+) (*dns.Msg, error) {
+	return r.queryServers(ctx, servers, zone, name, qtype)
+}
+
+// QueryEachNS exports queryEachNS for testing.
+func (r *Resolver) QueryEachNS(
+	ctx context.Context,
+	nameservers []string,
+	hostname string,
+) (map[string]*NameserverResponse, error) {
+	return r.queryEachNS(ctx, nameservers, hostname, recordTypes())
+}
+
+// ResolveNSIPs exports resolveNSIPs for testing.
+func (r *Resolver) ResolveNSIPs(
+	ctx context.Context,
+	nsNames []string,
+) []string {
+	return r.resolveNSIPs(ctx, nsNames)
+}
+
+// RootServerList exports rootServerList for testing.
+func RootServerList() []string {
+	return rootServerList()
+}
+
+// Shuffled exports shuffled for testing.
+func Shuffled(
+	servers []string,
+	shuffle func(n int, swap func(i, j int)),
+) []string {
+	return shuffled(servers, shuffle)
+}

internal/resolver/iterative.go

@@ -4,8 +4,9 @@ import (
 	"context"
 	"errors"
 	"fmt"
-	"math/rand"
+	"math/rand/v2"
 	"net"
+	"slices"
 	"sort"
 	"strings"
 	"time"
@@ -18,7 +19,6 @@ const (
 	maxRetries           = 2
 	maxDelegation        = 20
 	timeoutMultiplier    = 2
-	minDomainLabels      = 2
 )
 
 // ErrRefused is returned when a DNS server refuses a query.
@@ -42,22 +42,6 @@ func rootServerList() []string {
 	}
 }
 
-const maxRootServers = 3
-
-// randomRootServers returns a shuffled subset of root servers.
-func randomRootServers() []string {
-	all := rootServerList()
-	rand.Shuffle(len(all), func(i, j int) {
-		all[i], all[j] = all[j], all[i]
-	})
-
-	if len(all) > maxRootServers {
-		return all[:maxRootServers]
-	}
-
-	return all
-}
-
 func checkCtx(ctx context.Context) error {
 	err := ctx.Err()
 	if err != nil {
@@ -123,9 +107,8 @@ func (r *Resolver) retryTCP(
 	return resp
 }
 
-// queryDNS sends a DNS query to a specific server IP.
-// Tries non-recursive first, falls back to recursive on
-// REFUSED (handles DNS interception environments).
+// queryDNS sends a DNS query to a specific server IP, never asking it
+// for recursion. A reply of REFUSED is returned as ErrRefused.
 func (r *Resolver) queryDNS(
 	ctx context.Context,
 	serverIP string,
@@ -149,25 +132,12 @@ func (r *Resolver) queryDNS(
 	}
 
 	if resp.Rcode == dns.RcodeRefused {
-		msg.RecursionDesired = true
-
-		resp, err = r.tryExchange(ctx, msg, addr)
-		if err != nil {
-			return nil, fmt.Errorf(
-				"query %s @%s: %w", name, serverIP, err,
-			)
-		}
-
-		if resp.Rcode == dns.RcodeRefused {
-			return nil, fmt.Errorf(
-				"query %s @%s: %w", name, serverIP, ErrRefused,
-			)
-		}
+		return nil, fmt.Errorf(
+			"query %s @%s: %w", name, serverIP, ErrRefused,
+		)
 	}
 
-	resp = r.retryTCP(ctx, msg, addr, resp)
-
-	return resp, nil
+	return r.retryTCP(ctx, msg, addr, resp), nil
 }
 
 func extractNSSet(rrs []dns.RR) []string {
@@ -225,26 +195,38 @@ func (r *Resolver) followDelegation(
 	domain string,
 	servers []string,
 ) ([]string, error) {
+	// servers are the root servers, the servers of zone ".".
+	zone := "."
+
 	for range maxDelegation {
 		if checkCtx(ctx) != nil {
 			return nil, ErrContextCanceled
 		}
 
 		resp, err := r.queryServers(
-			ctx, servers, domain, dns.TypeNS,
+			ctx, servers, zone, domain, dns.TypeNS,
 		)
 		if err != nil {
 			return nil, err
 		}
 
-		ansNS := extractNSSet(resp.Answer)
-		if len(ansNS) > 0 {
-			return ansNS, nil
+		nsSet := nsSetFrom(resp, domain)
+		if len(nsSet) > 0 {
+			return nsSet, nil
+		}
+
+		// An authoritative reply comes from the servers of the zone
+		// domain is in; it is not a referral, even when its authority
+		// section lists that zone's NS records. Without NS records in
+		// the answer, domain is not the zone's apex and has no
+		// nameservers of its own.
+		if resp.Authoritative {
+			return nil, ErrNoNameservers
 		}
 
 		authNS := extractNSSet(resp.Ns)
 		if len(authNS) == 0 {
-			return r.resolveNSRecursive(ctx, domain)
+			return r.resolveNSIterative(ctx, domain)
 		}
 
 		glue := extractGlue(resp.Extra)
@@ -259,35 +241,135 @@ func (r *Resolver) followDelegation(
 		}
 
 		servers = nextServers
+		zone = referralZone(resp)
 	}
 
 	return nil, ErrNoNameservers
 }
 
+// shuffled returns a copy of servers in the order shuffle puts them
+// in. The resolver passes rand.Shuffle, so each time it walks a list of
+// servers it starts at a random one, and no one server gets every
+// first query.
+func shuffled(
+	servers []string,
+	shuffle func(n int, swap func(i, j int)),
+) []string {
+	order := slices.Clone(servers)
+	shuffle(len(order), func(i, j int) {
+		order[i], order[j] = order[j], order[i]
+	})
+
+	return order
+}
+
+// queryServers asks servers, the servers of zone, about name in a random
+// order until one gives a usable reply. A server that times out, refuses
+// or gives a reply that is not usable is passed over for the next. When
+// every server refused, the error says so, and when they are the root
+// servers it is ErrIntercepted.
 func (r *Resolver) queryServers(
 	ctx context.Context,
 	servers []string,
+	zone string,
 	name string,
 	qtype uint16,
 ) (*dns.Msg, error) {
 	var lastErr error
 
-	for _, ip := range servers {
+	refused := 0
+
+	for _, ip := range shuffled(servers, rand.Shuffle) {
 		if checkCtx(ctx) != nil {
 			return nil, ErrContextCanceled
 		}
 
 		resp, err := r.queryDNS(ctx, ip, name, qtype)
+		if err == nil && !usableReply(resp, zone, name) {
+			err = fmt.Errorf(
+				"query %s @%s: %w", name, ip, ErrUnusableReply,
+			)
+		}
+
 		if err == nil {
 			return resp, nil
 		}
 
+		if errors.Is(err, ErrRefused) {
+			refused++
+		}
+
 		lastErr = err
 	}
 
+	if refused == len(servers) && zone == "." {
+		return nil, fmt.Errorf(
+			"every root server refused a query for %s: %w",
+			name, ErrIntercepted,
+		)
+	}
+
+	if refused == len(servers) {
+		return nil, fmt.Errorf(
+			"every server of %s refused a query for %s: %w",
+			zone, name, ErrRefused,
+		)
+	}
+
 	return nil, fmt.Errorf("all servers failed: %w", lastErr)
 }
 
+// usableReply reports whether resp, a reply from one of the servers of
+// zone to a query about name, is usable. An error reply such as SERVFAIL
+// is not. Nor is a referral, unless it refers the query to a zone below
+// zone that name is in: a server that refers it back to zone, up or
+// sideways does not serve zone as it should.
+func usableReply(resp *dns.Msg, zone string, name string) bool {
+	if resp.Rcode != dns.RcodeSuccess && resp.Rcode != dns.RcodeNameError {
+		return false
+	}
+
+	child := referralZone(resp)
+	if resp.Authoritative || len(resp.Answer) > 0 || child == "" {
+		return true
+	}
+
+	return child != zone && dns.IsSubDomain(zone, child) &&
+		dns.IsSubDomain(child, name)
+}
+
+// referralZone returns the zone a referral refers the query to: the
+// owner name of the NS records in resp's authority section, or "" when
+// there are none.
+func referralZone(resp *dns.Msg) string {
+	for _, rr := range resp.Ns {
+		if ns, ok := rr.(*dns.NS); ok {
+			return strings.ToLower(ns.Hdr.Name)
+		}
+	}
+
+	return ""
+}
+
+// nsSetFrom returns the NS set of domain that resp, a reply to a query
+// for domain's NS records, gives: the delegation in a referral to domain
+// itself, or else the NS records in the answer; empty when it gives
+// neither. A referral to domain comes from its parent zone's servers,
+// which all hold the same delegation, so the set does not depend on
+// which of them answered. domain's own servers, which can disagree about
+// their NS records, are then not asked.
+func nsSetFrom(resp *dns.Msg, domain string) []string {
+	if referralZone(resp) == domain {
+		return extractNSSet(resp.Ns)
+	}
+
+	return extractNSSet(resp.Answer)
+}
+
+// resolveNSIPs returns the addresses of every nameserver in nsNames
+// whose name resolves, for a referral that carries none. The walk can
+// then go on to the zone's other nameservers when one gives no usable
+// reply.
 func (r *Resolver) resolveNSIPs(
 	ctx context.Context,
 	nsNames []string,
@@ -299,69 +381,92 @@ func (r *Resolver) resolveNSIPs(
 		if err == nil {
 			ips = append(ips, resolved...)
 		}
-
-		if len(ips) > 0 {
-			break
-		}
 	}
 
 	return ips
 }
 
-// resolveNSRecursive queries for NS records using recursive
-// resolution as a fallback for intercepted environments.
-func (r *Resolver) resolveNSRecursive(
+// resolveNSIterative queries for NS records using iterative
+// resolution as a fallback when followDelegation finds no
+// authoritative answer in the delegation chain.
+func (r *Resolver) resolveNSIterative(
 	ctx context.Context,
 	domain string,
 ) ([]string, error) {
-	domain = dns.Fqdn(domain)
-	msg := new(dns.Msg)
-	msg.SetQuestion(domain, dns.TypeNS)
-	msg.RecursionDesired = true
+	if checkCtx(ctx) != nil {
+		return nil, ErrContextCanceled
+	}
 
-	for _, ip := range randomRootServers() {
+	domain = dns.Fqdn(domain)
+	servers := rootServerList()
+	zone := "."
+
+	for range maxDelegation {
 		if checkCtx(ctx) != nil {
 			return nil, ErrContextCanceled
 		}
 
-		addr := net.JoinHostPort(ip, "53")
-
-		resp, _, err := r.client.ExchangeContext(ctx, msg, addr)
+		resp, err := r.queryServers(
+			ctx, servers, zone, domain, dns.TypeNS,
+		)
 		if err != nil {
-			continue
+			return nil, err
 		}
 
-		nsNames := extractNSSet(resp.Answer)
+		nsNames := nsSetFrom(resp, domain)
 		if len(nsNames) > 0 {
 			return nsNames, nil
 		}
+
+		// Follow delegation.
+		authNS := extractNSSet(resp.Ns)
+		if len(authNS) == 0 {
+			break
+		}
+
+		glue := extractGlue(resp.Extra)
+		nextServers := glueIPs(authNS, glue)
+
+		if len(nextServers) == 0 {
+			break
+		}
+
+		servers = nextServers
+		zone = referralZone(resp)
 	}
 
 	return nil, ErrNoNameservers
 }
 
-// resolveARecord resolves a hostname to IPv4 addresses.
+// resolveARecord resolves a hostname to IPv4 addresses using
+// iterative resolution through the delegation chain.
 func (r *Resolver) resolveARecord(
 	ctx context.Context,
 	hostname string,
 ) ([]string, error) {
-	hostname = dns.Fqdn(hostname)
-	msg := new(dns.Msg)
-	msg.SetQuestion(hostname, dns.TypeA)
-	msg.RecursionDesired = true
+	if checkCtx(ctx) != nil {
+		return nil, ErrContextCanceled
+	}
 
-	for _, ip := range randomRootServers() {
+	hostname = dns.Fqdn(hostname)
+	servers := rootServerList()
+	zone := "."
+
+	for range maxDelegation {
 		if checkCtx(ctx) != nil {
 			return nil, ErrContextCanceled
 		}
 
-		addr := net.JoinHostPort(ip, "53")
-
-		resp, _, err := r.client.ExchangeContext(ctx, msg, addr)
+		resp, err := r.queryServers(
+			ctx, servers, zone, hostname, dns.TypeA,
+		)
 		if err != nil {
-			continue
+			return nil, fmt.Errorf(
+				"resolving %s: %w", hostname, err,
+			)
 		}
 
+		// Check for A records in the answer section.
 		var ips []string
 
 		for _, rr := range resp.Answer {
@@ -373,6 +478,25 @@ func (r *Resolver) resolveARecord(
 		if len(ips) > 0 {
 			return ips, nil
 		}
+
+		// Follow delegation if present.
+		authNS := extractNSSet(resp.Ns)
+		if len(authNS) == 0 {
+			break
+		}
+
+		glue := extractGlue(resp.Extra)
+		nextServers := glueIPs(authNS, glue)
+
+		if len(nextServers) == 0 {
+			// Resolve NS IPs iteratively — but guard
+			// against infinite recursion by using only
+			// already-resolved servers.
+			break
+		}
+
+		servers = nextServers
+		zone = referralZone(resp)
 	}
 
 	return nil, fmt.Errorf(
@@ -382,7 +506,10 @@ func (r *Resolver) resolveARecord(
 
 // FindAuthoritativeNameservers traces the delegation chain from
 // root servers to discover all authoritative nameservers for the
-// given domain. Walks up the label hierarchy for subdomains.
+// given domain, as the delegation from its parent zone's servers lists
+// them. For a name that is not a zone apex it tries each
+// parent name in turn, so it returns the nameservers of the zone the
+// name is in.
 func (r *Resolver) FindAuthoritativeNameservers(
 	ctx context.Context,
 	domain string,
@@ -402,24 +529,56 @@ func (r *Resolver) FindAuthoritativeNameservers(
 		candidate := strings.Join(labels[i:], ".") + "."
 
 		nsNames, err := r.followDelegation(
-			ctx, candidate, randomRootServers(),
+			ctx, candidate, rootServerList(),
 		)
 		if err == nil && len(nsNames) > 0 {
 			sort.Strings(nsNames)
 
 			return nsNames, nil
 		}
+
+		// The root servers would refuse every parent name too.
+		if errors.Is(err, ErrIntercepted) {
+			return nil, err
+		}
 	}
 
 	return nil, ErrNoNameservers
 }
 
+// recordTypes returns the record types a nameserver is asked for when a
+// name is checked.
+func recordTypes() []uint16 {
+	return []uint16{
+		dns.TypeA, dns.TypeAAAA, dns.TypeCNAME,
+		dns.TypeMX, dns.TypeTXT, dns.TypeSRV,
+		dns.TypeCAA, dns.TypeNS,
+	}
+}
+
+// addressTypes returns the record types ResolveIPAddresses asks for,
+// the only ones it reads.
+func addressTypes() []uint16 {
+	return []uint16{dns.TypeA, dns.TypeAAAA, dns.TypeCNAME}
+}
+
 // QueryNameserver queries a specific nameserver for all record
 // types and builds a NameserverResponse.
 func (r *Resolver) QueryNameserver(
 	ctx context.Context,
 	nsHostname string,
 	hostname string,
+) (*NameserverResponse, error) {
+	return r.queryNameserver(ctx, nsHostname, hostname, recordTypes())
+}
+
+// queryNameserver queries a specific nameserver for the record types
+// in qtypes and builds a NameserverResponse.
+func (r *Resolver) queryNameserver(
+	ctx context.Context,
+	nsHostname string,
+	hostname string,
+	qtypes []uint16,
 ) (*NameserverResponse, error) {
 	if checkCtx(ctx) != nil {
 		return nil, ErrContextCanceled
@@ -432,14 +591,32 @@ func (r *Resolver) QueryNameserver(
 
 	hostname = dns.Fqdn(hostname)
 
-	return r.queryAllTypes(ctx, nsHostname, nsIPs[0], hostname)
+	return r.queryTypes(ctx, nsHostname, nsIPs[0], hostname, qtypes)
 }
 
-func (r *Resolver) queryAllTypes(
+// QueryNameserverIP queries a nameserver by its IP address directly,
+// bypassing NS hostname resolution.
+func (r *Resolver) QueryNameserverIP(
 	ctx context.Context,
 	nsHostname string,
 	nsIP string,
 	hostname string,
+) (*NameserverResponse, error) {
+	if checkCtx(ctx) != nil {
+		return nil, ErrContextCanceled
+	}
+
+	hostname = dns.Fqdn(hostname)
+
+	return r.queryTypes(ctx, nsHostname, nsIP, hostname, recordTypes())
+}
+
+func (r *Resolver) queryTypes(
+	ctx context.Context,
+	nsHostname string,
+	nsIP string,
+	hostname string,
+	qtypes []uint16,
 ) (*NameserverResponse, error) {
 	resp := &NameserverResponse{
 		Nameserver: nsHostname,
@@ -447,12 +624,6 @@ func (r *Resolver) queryAllTypes(
 		Status:     StatusOK,
 	}
 
-	qtypes := []uint16{
-		dns.TypeA, dns.TypeAAAA, dns.TypeCNAME,
-		dns.TypeMX, dns.TypeTXT, dns.TypeSRV,
-		dns.TypeCAA, dns.TypeNS,
-	}
-
 	state := r.queryEachType(ctx, nsIP, hostname, qtypes, resp)
 	classifyResponse(resp, state)
 
@@ -462,6 +633,10 @@ func (r *Resolver) queryAllTypes(
 type queryState struct {
 	gotNXDomain bool
 	gotSERVFAIL bool
+	gotRefused  bool
+	gotTimeout  bool
+	gotReferral bool
+	netErr      error
 	hasRecords  bool
 }
 
@@ -499,6 +674,15 @@ func (r *Resolver) querySingleType(
 ) {
 	msg, err := r.queryDNS(ctx, nsIP, hostname, qtype)
 	if err != nil {
+		switch {
+		case isTimeout(err):
+			state.gotTimeout = true
+		case errors.Is(err, ErrRefused):
+			state.gotRefused = true
+		default:
+			state.netErr = err
+		}
+
 		return
 	}
 
@@ -514,6 +698,19 @@ func (r *Resolver) querySingleType(
 		return
 	}
 
+	// A reply with no answer that lists other nameservers, from a server
+	// that does not hold the name's zone, is a referral and says nothing
+	// about the name's records. A server named in the delegation that
+	// does not hold the zone may send one, as do a parent zone's servers
+	// when FindAuthoritativeNameservers found no delegation for the
+	// name's zone and moved on to a parent name.
+	if !msg.Authoritative && len(msg.Answer) == 0 &&
+		len(extractNSSet(msg.Ns)) > 0 {
+		state.gotReferral = true
+
+		return
+	}
+
 	collectAnswerRecords(msg, resp, state)
 }
 
@@ -536,18 +733,44 @@ func collectAnswerRecords(
 	}
 }
 
+// isTimeout checks whether an error is a network timeout.
+func isTimeout(err error) bool {
+	var netErr net.Error
+	if errors.As(err, &netErr) {
+		return netErr.Timeout()
+	}
+
+	return false
+}
+
 func classifyResponse(resp *NameserverResponse, state queryState) {
 	switch {
 	case state.gotNXDomain && !state.hasRecords:
 		resp.Status = StatusNXDomain
+	case state.gotTimeout && !state.hasRecords:
+		resp.Status = StatusTimeout
+		resp.Error = "all queries timed out"
 	case state.gotSERVFAIL && !state.hasRecords:
 		resp.Status = StatusError
+		resp.Error = "server returned SERVFAIL"
+	case state.gotRefused && !state.hasRecords:
+		resp.Status = StatusError
+		resp.Error = "server returned REFUSED"
+	case state.netErr != nil && !state.hasRecords:
+		resp.Status = StatusError
+		resp.Error = "network error: " + state.netErr.Error()
+	case state.gotReferral && !state.hasRecords:
+		resp.Status = StatusError
+		resp.Error = "server returned a referral"
 	case !state.hasRecords && !state.gotNXDomain:
 		resp.Status = StatusNoData
 	}
 }
 
-// extractRecordValue formats a DNS RR value as a string.
+// extractRecordValue formats a DNS RR value as a string. DNS names
+// are case-insensitive and nameservers may answer in any letter case,
+// so names are lower-cased to compare equal. TXT and CAA values keep
+// their letter case.
 func extractRecordValue(rr dns.RR) string {
 	switch r := rr.(type) {
 	case *dns.A:
@@ -555,43 +778,29 @@ func extractRecordValue(rr dns.RR) string {
 	case *dns.AAAA:
 		return r.AAAA.String()
 	case *dns.CNAME:
-		return r.Target
+		return strings.ToLower(r.Target)
 	case *dns.MX:
-		return fmt.Sprintf("%d %s", r.Preference, r.Mx)
+		return fmt.Sprintf("%d %s", r.Preference, strings.ToLower(r.Mx))
 	case *dns.TXT:
 		return strings.Join(r.Txt, "")
 	case *dns.SRV:
 		return fmt.Sprintf(
 			"%d %d %d %s",
-			r.Priority, r.Weight, r.Port, r.Target,
+			r.Priority, r.Weight, r.Port, strings.ToLower(r.Target),
 		)
 	case *dns.CAA:
 		return fmt.Sprintf(
 			"%d %s \"%s\"", r.Flag, r.Tag, r.Value,
 		)
 	case *dns.NS:
-		return r.Ns
+		return strings.ToLower(r.Ns)
 	default:
 		return ""
 	}
 }
 
-// parentDomain returns the registerable parent domain.
-func parentDomain(hostname string) string {
-	hostname = dns.Fqdn(strings.ToLower(hostname))
-	labels := dns.SplitDomainName(hostname)
-
-	if len(labels) <= minDomainLabels {
-		return strings.Join(labels, ".") + "."
-	}
-
-	return strings.Join(
-		labels[len(labels)-minDomainLabels:], ".",
-	) + "."
-}
-
-// QueryAllNameservers discovers auth NSes for the hostname's
-// parent domain, then queries each one independently.
+// QueryAllNameservers discovers the auth NSes of the zone the
+// hostname is in, then queries each one independently.
 func (r *Resolver) QueryAllNameservers(
 	ctx context.Context,
 	hostname string,
@@ -600,29 +809,31 @@ func (r *Resolver) QueryAllNameservers(
 		return nil, ErrContextCanceled
 	}
 
-	parent := parentDomain(hostname)
-
-	nameservers, err := r.FindAuthoritativeNameservers(ctx, parent)
+	nameservers, err := r.FindAuthoritativeNameservers(ctx, hostname)
 	if err != nil {
 		return nil, err
 	}
 
-	return r.queryEachNS(ctx, nameservers, hostname)
+	return r.queryEachNS(ctx, nameservers, hostname, recordTypes())
 }
 
 func (r *Resolver) queryEachNS(
 	ctx context.Context,
 	nameservers []string,
 	hostname string,
+	qtypes []uint16,
 ) (map[string]*NameserverResponse, error) {
 	results := make(map[string]*NameserverResponse)
 
 	for _, ns := range nameservers {
+		resp, err := r.queryNameserver(ctx, ns, hostname, qtypes)
+
+		// A query the context cut short says nothing about the
+		// nameserver, so it must not be returned as its failure.
 		if checkCtx(ctx) != nil {
 			return nil, ErrContextCanceled
 		}
 
-		resp, err := r.QueryNameserver(ctx, ns, hostname)
 		if err != nil {
 			results[ns] = &NameserverResponse{
 				Nameserver: ns,
@@ -650,25 +861,20 @@ func (r *Resolver) LookupNS(
 
 // LookupAllRecords performs iterative resolution to find all DNS
 // records for the given hostname, keyed by authoritative nameserver.
+// Each nameserver's response carries its status and error with its
+// records.
 func (r *Resolver) LookupAllRecords(
 	ctx context.Context,
 	hostname string,
-) (map[string]map[string][]string, error) {
-	results, err := r.QueryAllNameservers(ctx, hostname)
-	if err != nil {
-		return nil, err
-	}
-
-	out := make(map[string]map[string][]string, len(results))
-	for ns, resp := range results {
-		out[ns] = resp.Records
-	}
-
-	return out, nil
+) (map[string]*NameserverResponse, error) {
+	return r.QueryAllNameservers(ctx, hostname)
 }
 
 // ResolveIPAddresses resolves a hostname to all IPv4 and IPv6
-// addresses, following CNAME chains up to MaxCNAMEDepth.
+// addresses, following CNAME chains up to MaxCNAMEDepth. It asks each
+// nameserver of the name's zone for its A, AAAA and CNAME records only.
+// When no nameserver of the name's zone answered, it returns an error
+// rather than no addresses.
 func (r *Resolver) ResolveIPAddresses(
 	ctx context.Context,
 	hostname string,
@@ -689,12 +895,20 @@ func (r *Resolver) resolveIPWithCNAME(
 		return nil, ErrCNAMEDepthExceeded
 	}
 
-	results, err := r.QueryAllNameservers(ctx, hostname)
+	nameservers, err := r.FindAuthoritativeNameservers(ctx, hostname)
 	if err != nil {
 		return nil, err
 	}
 
-	ips, cnameTarget := collectIPs(results)
+	results, err := r.queryEachNS(ctx, nameservers, hostname, addressTypes())
+	if err != nil {
+		return nil, err
+	}
+
+	ips, cnameTarget, err := collectIPs(results)
+	if err != nil {
+		return nil, fmt.Errorf("resolving %s: %w", hostname, err)
+	}
 
 	if len(ips) == 0 && cnameTarget != "" {
 		return r.resolveIPWithCNAME(ctx, cnameTarget, depth+1)
@@ -705,16 +919,28 @@ func (r *Resolver) resolveIPWithCNAME(
 	return ips, nil
 }
 
+// collectIPs returns the addresses in the nameservers' answers and the
+// first CNAME target among them. It returns ErrNoNameserverAnswered when
+// every nameserver timed out, failed or returned a referral: that is not
+// a name with no addresses.
 func collectIPs(
 	results map[string]*NameserverResponse,
-) ([]string, string) {
+) ([]string, string, error) {
 	seen := make(map[string]bool)
 
 	var ips []string
 
 	var cnameTarget string
 
+	answered := false
+
 	for _, resp := range results {
+		if resp.Status == StatusTimeout || resp.Status == StatusError {
+			continue
+		}
+
+		answered = true
+
 		if resp.Status == StatusNXDomain {
 			continue
 		}
@@ -738,5 +964,9 @@ func collectIPs(
 		}
 	}
 
-	return ips, cnameTarget
+	if !answered {
+		return nil, "", ErrNoNameserverAnswered
+	}
+
+	return ips, cnameTarget, nil
 }

internal/resolver/iterative_test.go

@@ -0,0 +1,266 @@
+package resolver_test
+
+import (
+	"math/rand/v2"
+	"slices"
+	"testing"
+
+	"github.com/miekg/dns"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"sneak.berlin/go/dnswatcher/internal/resolver"
+)
+
+// TestCollectIPs_OneAnswerIsEnough checks that one nameserver answering
+// NXDOMAIN says the name has no addresses, though the other timed out.
+func TestCollectIPs_OneAnswerIsEnough(t *testing.T) {
+	t.Parallel()
+
+	ips, _, err := resolver.CollectIPs(
+		map[string]*resolver.NameserverResponse{
+			"ns1.example.": {Status: resolver.StatusTimeout},
+			"ns2.example.": {Status: resolver.StatusNXDomain},
+		},
+	)
+	require.NoError(t, err)
+	assert.Empty(t, ips)
+}
+
+// TestCollectIPs_FailedIsNoAnswer checks that nameservers that all have
+// status error, from a refusal, a server failure, a network error or a
+// referral, are no answer rather than a name with no addresses.
+func TestCollectIPs_FailedIsNoAnswer(t *testing.T) {
+	t.Parallel()
+
+	ips, _, err := resolver.CollectIPs(
+		map[string]*resolver.NameserverResponse{
+			"ns1.example.": {Status: resolver.StatusError},
+			"ns2.example.": {Status: resolver.StatusError},
+		},
+	)
+	require.ErrorIs(t, err, resolver.ErrNoNameserverAnswered)
+	assert.Empty(t, ips)
+}
+
+const (
+	// exampleCom is the zone most cases of TestUsableReply and
+	// TestNSSetFrom are about, and wwwExampleCom a name in it.
+	exampleCom    = "example.com."
+	wwwExampleCom = "www.example.com."
+
+	// exampleNS is the server the NS records nsRecord builds name.
+	exampleNS = "ns1.example.net."
+)
+
+// nsRecord builds an NS record that names a server of zone.
+func nsRecord(zone string) *dns.NS {
+	return &dns.NS{
+		Hdr: dns.RR_Header{
+			Name: zone, Rrtype: dns.TypeNS, Class: dns.ClassINET,
+		},
+		Ns: exampleNS,
+	}
+}
+
+// referralTo builds a reply that refers the query to the servers of
+// zone.
+func referralTo(zone string) *dns.Msg {
+	msg := new(dns.Msg)
+	msg.Ns = []dns.RR{nsRecord(zone)}
+
+	return msg
+}
+
+// TestUsableReply checks which replies from one of a zone's servers are
+// used. A reply that is not usable moves the query on to the zone's
+// next server.
+func TestUsableReply(t *testing.T) {
+	t.Parallel()
+
+	servfail := new(dns.Msg)
+	servfail.Rcode = dns.RcodeServerFailure
+
+	answer := new(dns.Msg)
+	answer.Authoritative = true
+	answer.Answer = []dns.RR{nsRecord(exampleCom)}
+
+	nxdomain := new(dns.Msg)
+	nxdomain.Authoritative = true
+	nxdomain.Rcode = dns.RcodeNameError
+
+	tests := []struct {
+		name  string
+		resp  *dns.Msg
+		zone  string
+		query string
+		want  bool
+	}{
+		{
+			name: "SERVFAIL", resp: servfail,
+			zone: exampleCom, query: exampleCom, want: false,
+		},
+		{
+			name: "answer", resp: answer,
+			zone: exampleCom, query: exampleCom, want: true,
+		},
+		{
+			name: "NXDOMAIN", resp: nxdomain,
+			zone: ".", query: exampleCom, want: true,
+		},
+		{
+			name: "root refers to com", resp: referralTo("com."),
+			zone: ".", query: exampleCom, want: true,
+		},
+		{
+			name: "com refers to example.com", resp: referralTo(exampleCom),
+			zone: "com.", query: wwwExampleCom, want: true,
+		},
+		{
+			name: "referral back to the zone", resp: referralTo(exampleCom),
+			zone: exampleCom, query: exampleCom, want: false,
+		},
+		{
+			name: "referral up to the root", resp: referralTo("."),
+			zone: exampleCom, query: exampleCom, want: false,
+		},
+		{
+			name: "referral sideways", resp: referralTo("net."),
+			zone: ".", query: exampleCom, want: false,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			t.Parallel()
+
+			assert.Equal(t, tt.want,
+				resolver.UsableReply(tt.resp, tt.zone, tt.query),
+			)
+		})
+	}
+}
+
+// TestNSSetFrom checks which NS set a reply gives for a domain; a set
+// that is not empty ends the walk. The referral to example.com that
+// com's servers all send alike gives its delegation, so the set is the
+// same whichever of them answered, and example.com's own servers, which
+// can disagree, are not asked.
+func TestNSSetFrom(t *testing.T) {
+	t.Parallel()
+
+	answer := new(dns.Msg)
+	answer.Authoritative = true
+	answer.Answer = []dns.RR{nsRecord(exampleCom)}
+
+	tests := []struct {
+		name   string
+		resp   *dns.Msg
+		domain string
+		want   []string
+	}{
+		{
+			name: "com refers to example.com", resp: referralTo(exampleCom),
+			domain: exampleCom, want: []string{exampleNS},
+		},
+		{
+			name: "com refers on, for www.example.com",
+			resp: referralTo(exampleCom), domain: wwwExampleCom,
+			want: nil,
+		},
+		{
+			name: "answer from a server that holds example.com",
+			resp: answer, domain: exampleCom,
+			want: []string{exampleNS},
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			t.Parallel()
+
+			assert.ElementsMatch(t, tt.want,
+				resolver.NSSetFrom(tt.resp, tt.domain),
+			)
+		})
+	}
+}
+
+func TestExtractRecordValue_LetterCase(t *testing.T) {
+	t.Parallel()
+
+	tests := []struct {
+		name string
+		rr   dns.RR
+		want string
+	}{
+		{
+			name: "MX target lower-cased",
+			rr:   &dns.MX{Preference: 1, Mx: "ASPMX.L.GOOGLE.COM."},
+			want: "1 aspmx.l.google.com.",
+		},
+		{
+			name: "NS target lower-cased",
+			rr:   &dns.NS{Ns: "x.ns.joker.COM."},
+			want: "x.ns.joker.com.",
+		},
+		{
+			name: "CNAME target lower-cased",
+			rr:   &dns.CNAME{Target: "WWW.Example.Com."},
+			want: "www.example.com.",
+		},
+		{
+			name: "SRV target lower-cased",
+			rr: &dns.SRV{
+				Priority: 10, Weight: 5, Port: 443,
+				Target: "SIP.Example.Com.",
+			},
+			want: "10 5 443 sip.example.com.",
+		},
+		{
+			name: "TXT value keeps its case",
+			rr:   &dns.TXT{Txt: []string{"Verify=AbC123"}},
+			want: "Verify=AbC123",
+		},
+		{
+			name: "CAA value keeps its case",
+			rr:   &dns.CAA{Flag: 0, Tag: "issue", Value: "LetsEncrypt.org"},
+			want: `0 issue "LetsEncrypt.org"`,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			t.Parallel()
+
+			assert.Equal(t, tt.want, resolver.ExtractRecordValue(tt.rr))
+		})
+	}
+}
+
+// TestShuffled shuffles the root servers with many seeds. Every order
+// must hold each root server once, so each is tried before a
+// resolution fails; each root server must come first for some seed, so
+// no one root server gets every first query; and the list passed in
+// must be left as it was.
+func TestShuffled(t *testing.T) {
+	t.Parallel()
+
+	const seeds = 1000
+
+	roots := resolver.RootServerList()
+	before := slices.Clone(roots)
+	first := make(map[string]bool)
+
+	for seed := range uint64(seeds) {
+		rng := rand.New(rand.NewPCG(seed, 0)) //nolint:gosec // seeded on purpose
+		order := resolver.Shuffled(roots, rng.Shuffle)
+
+		assert.ElementsMatch(t, roots, order)
+
+		first[order[0]] = true
+	}
+
+	assert.Len(t, first, len(roots))
+	assert.Equal(t, before, roots)
+}

internal/resolver/livedns_harness_test.go

@@ -0,0 +1,192 @@
+package resolver_test
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+
+	"sneak.berlin/go/dnswatcher/internal/resolver"
+)
+
+// Tests for the live-DNS harness in livedns_test.go itself. These
+// exercise pure logic; they perform no DNS resolution of any kind, so
+// they neither mock DNS nor depend on it.
+
+// Names for the synthetic status maps below. Nothing is ever queried
+// at them: they are map keys handed to the package's pure counting
+// helpers, not a stand-in for a nameserver.
+const (
+	nsExample1 = "ns1.example."
+	nsExample2 = "ns2.example."
+	nsExample3 = "ns3.example."
+	nsExample4 = "ns4.example."
+)
+
+func TestLiveQuorumIsStrictMajority(t *testing.T) {
+	t.Parallel()
+
+	cases := map[int]int{
+		0:  1,
+		1:  1,
+		2:  2,
+		3:  2,
+		4:  3,
+		5:  3,
+		13: 7,
+	}
+
+	for total, want := range cases {
+		assert.Equal(
+			t, want, liveQuorum(total),
+			"liveQuorum(%d)", total,
+		)
+	}
+}
+
+func TestStatusCountingIgnoresSilentNameservers(t *testing.T) {
+	t.Parallel()
+
+	results := map[string]*resolver.NameserverResponse{
+		nsExample1: {
+			Nameserver: nsExample1,
+			Status:     resolver.StatusOK,
+		},
+		nsExample2: {
+			Nameserver: nsExample2,
+			Status:     resolver.StatusOK,
+		},
+		nsExample3: {
+			Nameserver: nsExample3,
+			Status:     resolver.StatusTimeout,
+		},
+		nsExample4: {
+			Nameserver: nsExample4,
+			Status:     resolver.StatusError,
+		},
+	}
+
+	assert.Equal(
+		t, 2, countStatus(results, resolver.StatusOK),
+	)
+	assert.Equal(
+		t, 0, countStatus(results, resolver.StatusNXDomain),
+	)
+
+	// Two of four answered, which is short of the quorum of
+	// three: this is the state that triggers a retry rather
+	// than an assertion failure.
+	assert.Equal(t, 2, answeredCount(results))
+	assert.Less(t, answeredCount(results), liveQuorum(len(results)))
+
+	assert.Equal(
+		t,
+		"ns1.example.=ok ns2.example.=ok "+
+			"ns3.example.=timeout ns4.example.=error",
+		describeStatuses(results),
+	)
+}
+
+// TestUnsanctionedStatusesRejectsWrongAnswers is the regression test
+// for the defect this allowlist exists to prevent: a minority of
+// nameservers answering WRONGLY while quorum keeps the suite green.
+// nodata is the case that motivated it — it is a wrong answer, not
+// silence, and it was previously banned by neither test.
+func TestUnsanctionedStatusesRejectsWrongAnswers(t *testing.T) {
+	t.Parallel()
+
+	// Four nameservers, three OK and one answering nodata: a
+	// quorum of three is satisfied and no NXDOMAIN is present, so
+	// the old blocklist assertions both passed on this input.
+	results := map[string]*resolver.NameserverResponse{
+		nsExample1: {
+			Nameserver: nsExample1,
+			Status:     resolver.StatusOK,
+		},
+		nsExample2: {
+			Nameserver: nsExample2,
+			Status:     resolver.StatusOK,
+		},
+		nsExample3: {
+			Nameserver: nsExample3,
+			Status:     resolver.StatusOK,
+		},
+		nsExample4: {
+			Nameserver: nsExample4,
+			Status:     resolver.StatusNoData,
+		},
+	}
+
+	assert.GreaterOrEqual(
+		t,
+		countStatus(results, resolver.StatusOK),
+		liveQuorum(len(results)),
+	)
+	assert.Zero(t, countStatus(results, resolver.StatusNXDomain))
+
+	// nodata is an ANSWER, so it never triggers a retry: nothing
+	// but the allowlist stands between it and a false green.
+	assert.Equal(t, len(results), answeredCount(results))
+
+	assert.Equal(
+		t,
+		[]string{nsExample4 + "=nodata"},
+		unsanctionedStatuses(
+			results,
+			resolver.StatusOK,
+			resolver.StatusTimeout,
+			resolver.StatusError,
+		),
+		"nodata must be reported as an unsanctioned status",
+	)
+}
+
+func TestUnsanctionedStatusesToleratesSilenceOnly(t *testing.T) {
+	t.Parallel()
+
+	results := map[string]*resolver.NameserverResponse{
+		nsExample1: {
+			Nameserver: nsExample1,
+			Status:     resolver.StatusNXDomain,
+		},
+		nsExample2: {
+			Nameserver: nsExample2,
+			Status:     resolver.StatusTimeout,
+		},
+		nsExample3: {
+			Nameserver: nsExample3,
+			Status:     resolver.StatusError,
+		},
+	}
+
+	allowed := []string{
+		resolver.StatusNXDomain,
+		resolver.StatusTimeout,
+		resolver.StatusError,
+	}
+
+	assert.Empty(
+		t,
+		unsanctionedStatuses(results, allowed...),
+		"timeout and error are non-answers and are tolerated",
+	)
+
+	// The same silent nameservers do not count towards a quorum.
+	assert.Equal(t, 1, answeredCount(results))
+
+	// An unknown status is treated as silence by answeredCount —
+	// so it retries and fails loudly — and is unsanctioned by the
+	// allowlist rather than quietly permitted.
+	const laterStatus = "some-status-added-later"
+
+	results[nsExample4] = &resolver.NameserverResponse{
+		Nameserver: nsExample4,
+		Status:     laterStatus,
+	}
+
+	assert.Equal(t, 1, answeredCount(results))
+	assert.Equal(
+		t,
+		[]string{nsExample4 + "=" + laterStatus},
+		unsanctionedStatuses(results, allowed...),
+	)
+}

internal/resolver/livedns_test.go

@@ -0,0 +1,437 @@
+package resolver_test
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"slices"
+	"sort"
+	"strings"
+	"testing"
+
+	"sneak.berlin/go/dnswatcher/internal/livednstest"
+	"sneak.berlin/go/dnswatcher/internal/resolver"
+)
+
+// ----------------------------------------------------------------
+// Live DNS test support
+// ----------------------------------------------------------------
+//
+// Tests that look something up in DNS query live DNS servers, never a
+// stand-in; logic that works on record data may be tested on that
+// data with no lookup (see TESTING.md). Each live operation below goes
+// through livednstest.Retry, which bounds how many resolutions are in
+// flight at once and retries an operation that got no answer (see
+// package livednstest).
+//
+// Where an assertion spans several independent nameservers, a quorum
+// is enough: a strict majority answering as expected. A server that
+// fails to answer is tolerated, while a server that answers *wrongly*
+// still fails the test.
+//
+// That tolerance is expressed as an ALLOWLIST of sanctioned statuses,
+// never as a blocklist of known-bad ones. A blocklist bans the one
+// wrong answer its author thought of and silently admits every other
+// status, including any added to the resolver later; an allowlist
+// fails on anything nobody explicitly sanctioned. Silence (timeout,
+// error) is the only thing quorum exists to tolerate. A *wrong
+// answer* — nxdomain for a name that exists, ok for one that does
+// not, nodata for either — is never tolerated at any count.
+
+// minNameservers is the smallest nameserver count a well-run zone is
+// expected to publish.
+const minNameservers = 2
+
+// errLiveNoQuorum reports that too few of a domain's nameservers
+// answered for a quorum assertion to be made.
+var errLiveNoQuorum = errors.New("no nameserver quorum")
+
+// liveQuorum is how many of total nameservers must agree for a
+// multi-nameserver assertion to hold: a strict majority.
+func liveQuorum(total int) int {
+	if total < 1 {
+		return 1
+	}
+
+	return total/2 + 1
+}
+
+// countStatus counts the responses carrying the given status.
+func countStatus(
+	results map[string]*resolver.NameserverResponse,
+	status string,
+) int {
+	n := 0
+
+	for _, resp := range results {
+		if resp.Status == status {
+			n++
+		}
+	}
+
+	return n
+}
+
+// liveAnswerStatuses is the closed set of statuses that count as a
+// nameserver having ANSWERED at all, whether or not the test agrees
+// with the answer. It is deliberately an allowlist: a status added
+// to the resolver later is treated as silence, so it can only ever
+// cause a retry and then a loud failure, never a quiet pass.
+func liveAnswerStatuses() []string {
+	return []string{
+		resolver.StatusOK,
+		resolver.StatusNXDomain,
+		resolver.StatusNoData,
+	}
+}
+
+// answeredCount counts the nameservers that produced an answer of
+// any kind, as opposed to failing or timing out.
+func answeredCount(
+	results map[string]*resolver.NameserverResponse,
+) int {
+	answers := liveAnswerStatuses()
+
+	n := 0
+
+	for _, resp := range results {
+		if slices.Contains(answers, resp.Status) {
+			n++
+		}
+	}
+
+	return n
+}
+
+// unsanctionedStatuses returns "nameserver=status" for every result
+// whose status the caller did not explicitly sanction, sorted for a
+// stable failure message. Callers pass the full closed set they will
+// accept — the expected answer plus whichever non-answers (timeout,
+// error) quorum is allowed to tolerate — so that any status outside
+// it fails the test by name.
+func unsanctionedStatuses(
+	results map[string]*resolver.NameserverResponse,
+	allowed ...string,
+) []string {
+	offenders := make([]string, 0, len(results))
+
+	for ns, resp := range results {
+		if slices.Contains(allowed, resp.Status) {
+			continue
+		}
+
+		offenders = append(
+			offenders, fmt.Sprintf("%s=%s", ns, resp.Status),
+		)
+	}
+
+	sort.Strings(offenders)
+
+	return offenders
+}
+
+// describeStatuses renders per-nameserver statuses for use in
+// assertion failure messages.
+func describeStatuses(
+	results map[string]*resolver.NameserverResponse,
+) string {
+	parts := make([]string, 0, len(results))
+	for ns, resp := range results {
+		parts = append(
+			parts, fmt.Sprintf("%s=%s", ns, resp.Status),
+		)
+	}
+
+	sort.Strings(parts)
+
+	return strings.Join(parts, " ")
+}
+
+// ----------------------------------------------------------------
+// Live operation wrappers
+// ----------------------------------------------------------------
+
+// liveFindAuthoritative resolves a domain's authoritative
+// nameservers, retrying until the delegation chain can be walked.
+func liveFindAuthoritative(
+	t *testing.T,
+	r *resolver.Resolver,
+	domain string,
+) []string {
+	t.Helper()
+
+	var out []string
+
+	livednstest.Retry(
+		t,
+		"FindAuthoritativeNameservers("+domain+")",
+		func(ctx context.Context) error {
+			ns, err := r.FindAuthoritativeNameservers(ctx, domain)
+			if err != nil {
+				return err
+			}
+
+			if len(ns) == 0 {
+				return fmt.Errorf(
+					"%w: %s has no nameservers",
+					livednstest.ErrNoAnswer, domain,
+				)
+			}
+
+			out = ns
+
+			return nil
+		},
+	)
+
+	return out
+}
+
+// liveLookupNS is liveFindAuthoritative through the LookupNS entry
+// point, so that both entry points stay independently exercised.
+func liveLookupNS(
+	t *testing.T,
+	r *resolver.Resolver,
+	domain string,
+) []string {
+	t.Helper()
+
+	var out []string
+
+	livednstest.Retry(
+		t,
+		"LookupNS("+domain+")",
+		func(ctx context.Context) error {
+			ns, err := r.LookupNS(ctx, domain)
+			if err != nil {
+				return err
+			}
+
+			if len(ns) == 0 {
+				return fmt.Errorf(
+					"%w: %s has no nameservers",
+					livednstest.ErrNoAnswer, domain,
+				)
+			}
+
+			out = ns
+
+			return nil
+		},
+	)
+
+	return out
+}
+
+// liveQueryNameserver queries one nameserver, retrying while that
+// nameserver fails to answer. NXDOMAIN and NODATA are answers and
+// are returned to the caller to assert on.
+//
+// QueryNameserver sends one query per record type, so one lost query
+// leaves its type out of an answer that is otherwise fine. A test names
+// in types the record types it reads; an answer holding records of none
+// of them is retried too.
+func liveQueryNameserver(
+	t *testing.T,
+	r *resolver.Resolver,
+	nameserver string,
+	hostname string,
+	types ...string,
+) *resolver.NameserverResponse {
+	t.Helper()
+
+	what := fmt.Sprintf(
+		"QueryNameserver(%s, %s)", nameserver, hostname,
+	)
+
+	var out *resolver.NameserverResponse
+
+	livednstest.Retry(
+		t,
+		what,
+		func(ctx context.Context) error {
+			resp, err := r.QueryNameserver(
+				ctx, nameserver, hostname,
+			)
+			if err != nil {
+				return err
+			}
+
+			if resp.Status == resolver.StatusTimeout ||
+				resp.Status == resolver.StatusError {
+				return fmt.Errorf(
+					"%w: %s returned %s: %s",
+					livednstest.ErrNoAnswer, nameserver,
+					resp.Status, resp.Error,
+				)
+			}
+
+			hasRecords := func(recordType string) bool {
+				return len(resp.Records[recordType]) > 0
+			}
+
+			if len(types) > 0 && !slices.ContainsFunc(types, hasRecords) {
+				return fmt.Errorf(
+					"%w: %s returned no %s records",
+					livednstest.ErrNoAnswer, nameserver,
+					strings.Join(types, " or "),
+				)
+			}
+
+			out = resp
+
+			return nil
+		},
+	)
+
+	return out
+}
+
+// liveQueryAllNameservers queries every authoritative nameserver
+// for a hostname, retrying until a quorum of them has answered.
+// Individual nameservers that stay silent are left in the result
+// for the caller to account for.
+func liveQueryAllNameservers(
+	t *testing.T,
+	r *resolver.Resolver,
+	hostname string,
+) map[string]*resolver.NameserverResponse {
+	t.Helper()
+
+	var out map[string]*resolver.NameserverResponse
+
+	livednstest.Retry(
+		t,
+		"QueryAllNameservers("+hostname+")",
+		func(ctx context.Context) error {
+			results, err := r.QueryAllNameservers(ctx, hostname)
+			if err != nil {
+				return err
+			}
+
+			if len(results) == 0 {
+				return fmt.Errorf(
+					"%w: no nameservers queried for %s",
+					livednstest.ErrNoAnswer, hostname,
+				)
+			}
+
+			answered := answeredCount(results)
+			if answered < liveQuorum(len(results)) {
+				return fmt.Errorf(
+					"%w: %d of %d answered: %s",
+					errLiveNoQuorum, answered,
+					len(results), describeStatuses(results),
+				)
+			}
+
+			out = results
+
+			return nil
+		},
+	)
+
+	return out
+}
+
+// liveResolveIPs resolves a hostname that is expected to have
+// addresses, retrying until at least one is returned.
+func liveResolveIPs(
+	t *testing.T,
+	r *resolver.Resolver,
+	hostname string,
+) []string {
+	t.Helper()
+
+	var out []string
+
+	livednstest.Retry(
+		t,
+		"ResolveIPAddresses("+hostname+")",
+		func(ctx context.Context) error {
+			ips, err := r.ResolveIPAddresses(ctx, hostname)
+			if err != nil {
+				return err
+			}
+
+			if len(ips) == 0 {
+				return fmt.Errorf(
+					"%w: no addresses for %s",
+					livednstest.ErrNoAnswer, hostname,
+				)
+			}
+
+			out = ips
+
+			return nil
+		},
+	)
+
+	return out
+}
+
+// liveResolveIPsAllowingEmpty resolves a hostname that may legitimately
+// have no addresses, so the empty result is returned rather than
+// retried. Used for names that must not exist; the corresponding
+// QueryAllNameservers test is what proves the nameservers actively
+// said NXDOMAIN rather than merely staying silent.
+func liveResolveIPsAllowingEmpty(
+	t *testing.T,
+	r *resolver.Resolver,
+	hostname string,
+) []string {
+	t.Helper()
+
+	var out []string
+
+	livednstest.Retry(
+		t,
+		"ResolveIPAddresses("+hostname+")",
+		func(ctx context.Context) error {
+			ips, err := r.ResolveIPAddresses(ctx, hostname)
+			if err != nil {
+				return err
+			}
+
+			out = ips
+
+			return nil
+		},
+	)
+
+	return out
+}
+
+// liveResolveNSIPs looks up the addresses of the nameservers named
+// names, retrying until there are at least atLeast of them: a name
+// whose lookup got no reply is left out of the result, not an error.
+func liveResolveNSIPs(
+	t *testing.T,
+	r *resolver.Resolver,
+	names []string,
+	atLeast int,
+) []string {
+	t.Helper()
+
+	var out []string
+
+	livednstest.Retry(
+		t,
+		"ResolveNSIPs("+strings.Join(names, ", ")+")",
+		func(ctx context.Context) error {
+			ips := r.ResolveNSIPs(ctx, names)
+			if len(ips) < atLeast {
+				return fmt.Errorf(
+					"%w: %d addresses, expected at least %d",
+					livednstest.ErrNoAnswer, len(ips), atLeast,
+				)
+			}
+
+			out = ips
+
+			return nil
+		},
+	)
+
+	return out
+}

internal/resolver/resolver.go

@@ -17,6 +17,7 @@ const (
 	StatusError    = "error"
 	StatusNXDomain = "nxdomain"
 	StatusNoData   = "nodata"
+	StatusTimeout  = "timeout"
 )
 
 // MaxCNAMEDepth is the maximum CNAME chain depth to follow.
@@ -66,17 +67,4 @@ func NewFromLogger(log *slog.Logger) *Resolver {
 	}
 }
 
-// NewFromLoggerWithClient creates a Resolver with a custom DNS
-// client, useful for testing with mock DNS responses.
-func NewFromLoggerWithClient(
-	log *slog.Logger,
-	client DNSClient,
-) *Resolver {
-	return &Resolver{
-		log:    log,
-		client: client,
-		tcp:    client,
-	}
-}
-
 // Method implementations are in iterative.go.

internal/resolver/resolver_test.go

@@ -2,6 +2,7 @@ package resolver_test
 
 import (
 	"context"
+	"fmt"
 	"log/slog"
 	"net"
 	"os"
@@ -10,9 +11,11 @@ import (
 	"testing"
 	"time"
 
+	"github.com/miekg/dns"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 
+	"sneak.berlin/go/dnswatcher/internal/livednstest"
 	"sneak.berlin/go/dnswatcher/internal/resolver"
 )
 
@@ -31,32 +34,17 @@ func newTestResolver(t *testing.T) *resolver.Resolver {
 	return resolver.NewFromLogger(log)
 }
 
-func testContext(t *testing.T) context.Context {
-	t.Helper()
-
-	ctx, cancel := context.WithTimeout(
-		context.Background(), 60*time.Second,
-	)
-	t.Cleanup(cancel)
-
-	return ctx
-}
-
+// findOneNSForDomain picks one authoritative nameserver to aim a
+// test at. Quorum handling lives in livedns_test.go, and the live-DNS
+// retry and concurrency limit in package livednstest.
 func findOneNSForDomain(
 	t *testing.T,
 	r *resolver.Resolver,
-	ctx context.Context, //nolint:revive // test helper
 	domain string,
 ) string {
 	t.Helper()
 
-	nameservers, err := r.FindAuthoritativeNameservers(
-		ctx, domain,
-	)
-	require.NoError(t, err)
-	require.NotEmpty(t, nameservers)
-
-	return nameservers[0]
+	return liveFindAuthoritative(t, r, domain)[0]
 }
 
 // ----------------------------------------------------------------
@@ -69,13 +57,7 @@ func TestFindAuthoritativeNameservers_ValidDomain(
 	t.Parallel()
 
 	r := newTestResolver(t)
-	ctx := testContext(t)
-
-	nameservers, err := r.FindAuthoritativeNameservers(
-		ctx, "google.com",
-	)
-	require.NoError(t, err)
-	require.NotEmpty(t, nameservers)
+	nameservers := liveFindAuthoritative(t, r, "google.com")
 
 	hasGoogleNS := false
 
@@ -98,13 +80,10 @@ func TestFindAuthoritativeNameservers_Subdomain(
 	t.Parallel()
 
 	r := newTestResolver(t)
-	ctx := testContext(t)
+	fromHost := liveFindAuthoritative(t, r, "www.google.com")
+	fromZone := liveFindAuthoritative(t, r, "google.com")
 
-	nameservers, err := r.FindAuthoritativeNameservers(
-		ctx, "www.google.com",
-	)
-	require.NoError(t, err)
-	require.NotEmpty(t, nameservers)
+	assert.Equal(t, fromZone, fromHost)
 }
 
 func TestFindAuthoritativeNameservers_ReturnsSorted(
@@ -113,12 +92,7 @@ func TestFindAuthoritativeNameservers_ReturnsSorted(
 	t.Parallel()
 
 	r := newTestResolver(t)
-	ctx := testContext(t)
-
-	nameservers, err := r.FindAuthoritativeNameservers(
-		ctx, "google.com",
-	)
-	require.NoError(t, err)
+	nameservers := liveFindAuthoritative(t, r, "google.com")
 
 	assert.True(
 		t,
@@ -133,17 +107,8 @@ func TestFindAuthoritativeNameservers_Deterministic(
 	t.Parallel()
 
 	r := newTestResolver(t)
-	ctx := testContext(t)
-
-	first, err := r.FindAuthoritativeNameservers(
-		ctx, "google.com",
-	)
-	require.NoError(t, err)
-
-	second, err := r.FindAuthoritativeNameservers(
-		ctx, "google.com",
-	)
-	require.NoError(t, err)
+	first := liveFindAuthoritative(t, r, "google.com")
+	second := liveFindAuthoritative(t, r, "google.com")
 
 	assert.Equal(t, first, second)
 }
@@ -154,17 +119,8 @@ func TestFindAuthoritativeNameservers_TrailingDot(
 	t.Parallel()
 
 	r := newTestResolver(t)
-	ctx := testContext(t)
-
-	ns1, err := r.FindAuthoritativeNameservers(
-		ctx, "google.com",
-	)
-	require.NoError(t, err)
-
-	ns2, err := r.FindAuthoritativeNameservers(
-		ctx, "google.com.",
-	)
-	require.NoError(t, err)
+	ns1 := liveFindAuthoritative(t, r, "google.com")
+	ns2 := liveFindAuthoritative(t, r, "google.com.")
 
 	assert.Equal(t, ns1, ns2)
 }
@@ -175,13 +131,7 @@ func TestFindAuthoritativeNameservers_CloudflareDomain(
 	t.Parallel()
 
 	r := newTestResolver(t)
-	ctx := testContext(t)
-
-	nameservers, err := r.FindAuthoritativeNameservers(
-		ctx, "cloudflare.com",
-	)
-	require.NoError(t, err)
-	require.NotEmpty(t, nameservers)
+	nameservers := liveFindAuthoritative(t, r, "cloudflare.com")
 
 	for _, ns := range nameservers {
 		assert.True(t, strings.HasSuffix(ns, "."),
@@ -190,6 +140,28 @@ func TestFindAuthoritativeNameservers_CloudflareDomain(
 	}
 }
 
+// TestResolveNSIPs_EveryNameserver looks up the addresses of two of
+// google.com's nameservers together, as the walk does when a referral
+// names a zone's nameservers without their addresses, and compares them
+// with each looked up alone. Together they must give the addresses of
+// both, not only of the first that resolves, so that when one gives no
+// usable reply the walk goes on to the other.
+func TestResolveNSIPs_EveryNameserver(t *testing.T) {
+	t.Parallel()
+
+	r := newTestResolver(t)
+	names := []string{"ns3.google.com.", "ns4.google.com."}
+	want := make([]string, 0, len(names))
+
+	for _, name := range names {
+		want = append(want, liveResolveNSIPs(t, r, []string{name}, 1)...)
+	}
+
+	got := liveResolveNSIPs(t, r, names, len(want))
+
+	assert.ElementsMatch(t, want, got)
+}
+
 // ----------------------------------------------------------------
 // QueryNameserver tests
 // ----------------------------------------------------------------
@@ -198,13 +170,9 @@ func TestQueryNameserver_BasicA(t *testing.T) {
 	t.Parallel()
 
 	r := newTestResolver(t)
-	ctx := testContext(t)
-	ns := findOneNSForDomain(t, r, ctx, "google.com")
+	ns := findOneNSForDomain(t, r, "google.com")
+	resp := liveQueryNameserver(t, r, ns, "www.google.com", "A", "CNAME")
 
-	resp, err := r.QueryNameserver(
-		ctx, ns, "www.google.com",
-	)
-	require.NoError(t, err)
 	require.NotNil(t, resp)
 
 	assert.Equal(t, resolver.StatusOK, resp.Status)
@@ -221,13 +189,8 @@ func TestQueryNameserver_AAAA(t *testing.T) {
 	t.Parallel()
 
 	r := newTestResolver(t)
-	ctx := testContext(t)
-	ns := findOneNSForDomain(t, r, ctx, "cloudflare.com")
-
-	resp, err := r.QueryNameserver(
-		ctx, ns, "cloudflare.com",
-	)
-	require.NoError(t, err)
+	ns := findOneNSForDomain(t, r, "cloudflare.com")
+	resp := liveQueryNameserver(t, r, ns, "cloudflare.com", "AAAA")
 
 	aaaaRecords := resp.Records["AAAA"]
 	require.NotEmpty(t, aaaaRecords,
@@ -246,13 +209,8 @@ func TestQueryNameserver_MX(t *testing.T) {
 	t.Parallel()
 
 	r := newTestResolver(t)
-	ctx := testContext(t)
-	ns := findOneNSForDomain(t, r, ctx, "google.com")
-
-	resp, err := r.QueryNameserver(
-		ctx, ns, "google.com",
-	)
-	require.NoError(t, err)
+	ns := findOneNSForDomain(t, r, "google.com")
+	resp := liveQueryNameserver(t, r, ns, "google.com", "MX")
 
 	mxRecords := resp.Records["MX"]
 	require.NotEmpty(t, mxRecords,
@@ -264,13 +222,8 @@ func TestQueryNameserver_TXT(t *testing.T) {
 	t.Parallel()
 
 	r := newTestResolver(t)
-	ctx := testContext(t)
-	ns := findOneNSForDomain(t, r, ctx, "google.com")
-
-	resp, err := r.QueryNameserver(
-		ctx, ns, "google.com",
-	)
-	require.NoError(t, err)
+	ns := findOneNSForDomain(t, r, "google.com")
+	resp := liveQueryNameserver(t, r, ns, "google.com", "TXT")
 
 	txtRecords := resp.Records["TXT"]
 	require.NotEmpty(t, txtRecords,
@@ -296,29 +249,240 @@ func TestQueryNameserver_NXDomain(t *testing.T) {
 	t.Parallel()
 
 	r := newTestResolver(t)
-	ctx := testContext(t)
-	ns := findOneNSForDomain(t, r, ctx, "google.com")
-
-	resp, err := r.QueryNameserver(
-		ctx, ns,
-		"this-surely-does-not-exist-xyz.google.com",
+	ns := findOneNSForDomain(t, r, "google.com")
+	resp := liveQueryNameserver(
+		t, r, ns, "this-surely-does-not-exist-xyz.google.com",
 	)
-	require.NoError(t, err)
 
 	assert.Equal(t, resolver.StatusNXDomain, resp.Status)
 }
 
+// TestQueryNameserver_Refused asks a google.com nameserver about
+// cloudflare.com, a zone it does not serve, which it refuses. Refusing
+// is a failure to answer, not an answer with no records.
+func TestQueryNameserver_Refused(t *testing.T) {
+	t.Parallel()
+
+	r := newTestResolver(t)
+	ns := findOneNSForDomain(t, r, "google.com")
+
+	var resp *resolver.NameserverResponse
+
+	livednstest.Retry(
+		t,
+		"QueryNameserver("+ns+", cloudflare.com)",
+		func(ctx context.Context) error {
+			var err error
+
+			resp, err = r.QueryNameserver(ctx, ns, "cloudflare.com")
+			if err != nil {
+				return err
+			}
+
+			// A timeout or a network error is no reply at all.
+			if resp.Status == resolver.StatusTimeout ||
+				strings.HasPrefix(resp.Error, "network error") {
+				return fmt.Errorf(
+					"%w: %s: %s",
+					livednstest.ErrNoAnswer, ns, resp.Error,
+				)
+			}
+
+			return nil
+		},
+	)
+
+	assert.Equal(t, resolver.StatusError, resp.Status)
+	assert.Equal(t, "server returned REFUSED", resp.Error)
+}
+
+// TestQueryNameserverIP_RecursiveResolverRefused asks Quad9, a public
+// recursive resolver, about google.com at both of its addresses. Quad9
+// refuses a query that does not ask for recursion and answers one that
+// does. The resolver never asks for recursion, so it must be reported
+// as refusing, never as answering.
+func TestQueryNameserverIP_RecursiveResolverRefused(t *testing.T) {
+	t.Parallel()
+
+	r := newTestResolver(t)
+
+	for _, ip := range []string{"9.9.9.9", "149.112.112.112"} {
+		var resp *resolver.NameserverResponse
+
+		livednstest.Retry(
+			t,
+			"QueryNameserverIP("+ip+", google.com)",
+			func(ctx context.Context) error {
+				var err error
+
+				resp, err = r.QueryNameserverIP(
+					ctx, ip, ip, "google.com",
+				)
+				if err != nil {
+					return err
+				}
+
+				// A timeout or a network error is no reply at all.
+				if resp.Status == resolver.StatusTimeout ||
+					strings.HasPrefix(resp.Error, "network error") {
+					return fmt.Errorf(
+						"%w: %s: %s",
+						livednstest.ErrNoAnswer, ip, resp.Error,
+					)
+				}
+
+				return nil
+			},
+		)
+
+		assert.Equal(t, resolver.StatusError, resp.Status, ip)
+		assert.Equal(t, "server returned REFUSED", resp.Error, ip)
+	}
+}
+
+// googleNameserverIPv4s returns the IPv4 addresses of google.com's
+// nameservers, the only addresses the resolver asks servers at.
+func googleNameserverIPv4s(t *testing.T, r *resolver.Resolver) []string {
+	t.Helper()
+
+	names := liveFindAuthoritative(t, r, "google.com")
+
+	return liveResolveNSIPs(t, r, names, len(names))
+}
+
+// TestQueryServers_EveryServerRefused asks all of google.com's
+// nameservers about cloudflare.com, a zone they do not serve, which
+// they all refuse. The error says every server refused; it is not
+// ErrIntercepted, which only the root servers refusing shows.
+func TestQueryServers_EveryServerRefused(t *testing.T) {
+	t.Parallel()
+
+	r := newTestResolver(t)
+	servers := googleNameserverIPv4s(t, r)
+
+	var err error
+
+	livednstest.Retry(
+		t,
+		"QueryServers(google.com servers, cloudflare.com)",
+		func(ctx context.Context) error {
+			_, err = r.QueryServers(
+				ctx, servers, "google.com.", "cloudflare.com.",
+				dns.TypeNS,
+			)
+
+			// When not every server refused, one may have given no
+			// reply at all, so the attempt is tried again.
+			if err != nil &&
+				!strings.HasPrefix(err.Error(), "every server of") {
+				return fmt.Errorf(
+					"%w: %w", livednstest.ErrNoAnswer, err,
+				)
+			}
+
+			return nil
+		},
+	)
+
+	require.ErrorIs(t, err, resolver.ErrRefused)
+	require.NotErrorIs(t, err, resolver.ErrIntercepted)
+	require.EqualError(
+		t, err,
+		"every server of google.com. refused a query for "+
+			"cloudflare.com.: dns query refused",
+	)
+}
+
+// TestQueryServers_EveryRootServerRefused passes google.com's
+// nameservers to QueryServers as the servers of the root zone. They
+// refuse a query about cloudflare.com, as root servers would if
+// something on the network answered in their place, so the error is
+// ErrIntercepted.
+func TestQueryServers_EveryRootServerRefused(t *testing.T) {
+	t.Parallel()
+
+	r := newTestResolver(t)
+	servers := googleNameserverIPv4s(t, r)
+
+	var err error
+
+	livednstest.Retry(
+		t,
+		"QueryServers(google.com servers as root servers, cloudflare.com)",
+		func(ctx context.Context) error {
+			_, err = r.QueryServers(
+				ctx, servers, ".", "cloudflare.com.", dns.TypeNS,
+			)
+
+			// When not every server refused, one may have given no
+			// reply at all, so the attempt is tried again. Both errors
+			// for every server refusing say "refused a query for".
+			if err != nil &&
+				!strings.Contains(err.Error(), "refused a query for") {
+				return fmt.Errorf(
+					"%w: %w", livednstest.ErrNoAnswer, err,
+				)
+			}
+
+			return nil
+		},
+	)
+
+	require.ErrorIs(t, err, resolver.ErrIntercepted)
+	require.EqualError(
+		t, err,
+		"every root server refused a query for cloudflare.com.: "+
+			"this network intercepts DNS queries",
+	)
+}
+
+// TestQueryServers_NotEveryRootServerRefused passes google.com's
+// nameservers and 192.0.2.1 to QueryServers as the servers of the root
+// zone. The google.com nameservers refuse a query about cloudflare.com,
+// but nothing answers at 192.0.2.1, a documentation address, so not
+// every server refused, wherever 192.0.2.1 falls in the random order:
+// the error is not ErrIntercepted and does not say every server refused.
+func TestQueryServers_NotEveryRootServerRefused(t *testing.T) {
+	t.Parallel()
+
+	r := newTestResolver(t)
+	servers := googleNameserverIPv4s(t, r)
+	servers = append(servers, "192.0.2.1")
+
+	var err error
+
+	livednstest.Retry(
+		t,
+		"QueryServers(google.com servers and 192.0.2.1, cloudflare.com)",
+		func(ctx context.Context) error {
+			_, err = r.QueryServers(
+				ctx, servers, ".", "cloudflare.com.", dns.TypeNS,
+			)
+
+			// An attempt that ran out of time may not have asked every
+			// server, so it is tried again.
+			if ctx.Err() != nil {
+				return fmt.Errorf(
+					"%w: %w", livednstest.ErrNoAnswer, err,
+				)
+			}
+
+			return nil
+		},
+	)
+
+	require.Error(t, err)
+	require.NotErrorIs(t, err, resolver.ErrIntercepted)
+	// Both errors for every server refusing say "refused a query for".
+	require.NotContains(t, err.Error(), "refused a query for")
+}
+
 func TestQueryNameserver_RecordsSorted(t *testing.T) {
 	t.Parallel()
 
 	r := newTestResolver(t)
-	ctx := testContext(t)
-	ns := findOneNSForDomain(t, r, ctx, "google.com")
-
-	resp, err := r.QueryNameserver(
-		ctx, ns, "google.com",
-	)
-	require.NoError(t, err)
+	ns := findOneNSForDomain(t, r, "google.com")
+	resp := liveQueryNameserver(t, r, ns, "google.com")
 
 	for recordType, values := range resp.Records {
 		assert.True(
@@ -335,13 +499,8 @@ func TestQueryNameserver_ResponseIncludesNameserver(
 	t.Parallel()
 
 	r := newTestResolver(t)
-	ctx := testContext(t)
-	ns := findOneNSForDomain(t, r, ctx, "cloudflare.com")
-
-	resp, err := r.QueryNameserver(
-		ctx, ns, "cloudflare.com",
-	)
-	require.NoError(t, err)
+	ns := findOneNSForDomain(t, r, "cloudflare.com")
+	resp := liveQueryNameserver(t, r, ns, "cloudflare.com")
 
 	assert.Equal(t, ns, resp.Nameserver)
 }
@@ -352,14 +511,10 @@ func TestQueryNameserver_EmptyRecordsOnNXDomain(
 	t.Parallel()
 
 	r := newTestResolver(t)
-	ctx := testContext(t)
-	ns := findOneNSForDomain(t, r, ctx, "google.com")
-
-	resp, err := r.QueryNameserver(
-		ctx, ns,
-		"this-surely-does-not-exist-xyz.google.com",
+	ns := findOneNSForDomain(t, r, "google.com")
+	resp := liveQueryNameserver(
+		t, r, ns, "this-surely-does-not-exist-xyz.google.com",
 	)
-	require.NoError(t, err)
 
 	totalRecords := 0
 	for _, values := range resp.Records {
@@ -373,18 +528,9 @@ func TestQueryNameserver_TrailingDotHandling(t *testing.T) {
 	t.Parallel()
 
 	r := newTestResolver(t)
-	ctx := testContext(t)
-	ns := findOneNSForDomain(t, r, ctx, "google.com")
-
-	resp1, err := r.QueryNameserver(
-		ctx, ns, "google.com",
-	)
-	require.NoError(t, err)
-
-	resp2, err := r.QueryNameserver(
-		ctx, ns, "google.com.",
-	)
-	require.NoError(t, err)
+	ns := findOneNSForDomain(t, r, "google.com")
+	resp1 := liveQueryNameserver(t, r, ns, "google.com")
+	resp2 := liveQueryNameserver(t, r, ns, "google.com.")
 
 	assert.Equal(t, resp1.Status, resp2.Status)
 }
@@ -397,15 +543,9 @@ func TestQueryAllNameservers_ReturnsAllNS(t *testing.T) {
 	t.Parallel()
 
 	r := newTestResolver(t)
-	ctx := testContext(t)
+	results := liveQueryAllNameservers(t, r, "google.com")
 
-	results, err := r.QueryAllNameservers(
-		ctx, "google.com",
-	)
-	require.NoError(t, err)
-	require.NotEmpty(t, results)
-
-	assert.GreaterOrEqual(t, len(results), 2)
+	assert.GreaterOrEqual(t, len(results), minNameservers)
 
 	for ns, resp := range results {
 		assert.Equal(t, ns, resp.Nameserver)
@@ -415,19 +555,57 @@ func TestQueryAllNameservers_ReturnsAllNS(t *testing.T) {
 func TestQueryAllNameservers_AllReturnOK(t *testing.T) {
 	t.Parallel()
 
-	r := newTestResolver(t)
-	ctx := testContext(t)
+	// The last two names are in zones other than their last two
+	// labels: google.co.uk, under the two-label suffix co.uk, and
+	// compute-1.amazonaws.com, which amazonaws.com delegates to other
+	// servers and which has a host name for each of its addresses.
+	// Servers above a name's zone only refer onward, which gives
+	// nodata, so ok shows the name was asked at its own zone's
+	// servers.
+	hostnames := []string{
+		"google.com",
+		"www.google.co.uk",
+		"ec2-3-80-0-1.compute-1.amazonaws.com",
+	}
 
-	results, err := r.QueryAllNameservers(
-		ctx, "google.com",
-	)
-	require.NoError(t, err)
+	for _, hostname := range hostnames {
+		t.Run(hostname, func(t *testing.T) {
+			t.Parallel()
 
-	for ns, resp := range results {
-		assert.Equal(
-			t, resolver.StatusOK, resp.Status,
-			"NS %s should return OK", ns,
-		)
+			r := newTestResolver(t)
+			results := liveQueryAllNameservers(t, r, hostname)
+
+			// A quorum, not unanimity: one authoritative server
+			// being slow or rate-limiting us is a property of the
+			// live internet, not a resolver defect.
+			assert.GreaterOrEqual(
+				t,
+				countStatus(results, resolver.StatusOK),
+				liveQuorum(len(results)),
+				"a quorum of nameservers should answer OK: %s",
+				describeStatuses(results),
+			)
+
+			// Quorum tolerates SILENCE only. Every individual
+			// result must be either the expected answer or a
+			// non-answer: ok, timeout or error, and nothing else.
+			// Stated as a closed allowlist so that a wrong answer
+			// no one thought to ban — nxdomain and nodata today,
+			// any status added later — fails here rather than
+			// sliding through under the quorum.
+			assert.Empty(
+				t,
+				unsanctionedStatuses(
+					results,
+					resolver.StatusOK,
+					resolver.StatusTimeout,
+					resolver.StatusError,
+				),
+				"every nameserver must answer OK or not answer "+
+					"at all: %s",
+				describeStatuses(results),
+			)
+		})
 	}
 }
 
@@ -437,20 +615,34 @@ func TestQueryAllNameservers_NXDomainFromAllNS(
 	t.Parallel()
 
 	r := newTestResolver(t)
-	ctx := testContext(t)
-
-	results, err := r.QueryAllNameservers(
-		ctx,
-		"this-surely-does-not-exist-xyz.google.com",
+	results := liveQueryAllNameservers(
+		t, r, "this-surely-does-not-exist-xyz.google.com",
 	)
-	require.NoError(t, err)
 
-	for ns, resp := range results {
-		assert.Equal(
-			t, resolver.StatusNXDomain, resp.Status,
-			"NS %s should return nxdomain", ns,
-		)
-	}
+	assert.GreaterOrEqual(
+		t,
+		countStatus(results, resolver.StatusNXDomain),
+		liveQuorum(len(results)),
+		"a quorum of nameservers should report NXDOMAIN: %s",
+		describeStatuses(results),
+	)
+
+	// Silence is tolerated; any actual answer other than NXDOMAIN
+	// is not. Closed allowlist for the same reason as above: a
+	// server answering `ok` or `nodata` for a name that must not
+	// exist is a wrong answer, not a slow one.
+	assert.Empty(
+		t,
+		unsanctionedStatuses(
+			results,
+			resolver.StatusNXDomain,
+			resolver.StatusTimeout,
+			resolver.StatusError,
+		),
+		"every nameserver must report NXDOMAIN or not answer "+
+			"at all: %s",
+		describeStatuses(results),
+	)
 }
 
 // ----------------------------------------------------------------
@@ -461,11 +653,7 @@ func TestLookupNS_ValidDomain(t *testing.T) {
 	t.Parallel()
 
 	r := newTestResolver(t)
-	ctx := testContext(t)
-
-	nameservers, err := r.LookupNS(ctx, "google.com")
-	require.NoError(t, err)
-	require.NotEmpty(t, nameservers)
+	nameservers := liveLookupNS(t, r, "google.com")
 
 	for _, ns := range nameservers {
 		assert.True(t, strings.HasSuffix(ns, "."),
@@ -478,10 +666,7 @@ func TestLookupNS_Sorted(t *testing.T) {
 	t.Parallel()
 
 	r := newTestResolver(t)
-	ctx := testContext(t)
-
-	nameservers, err := r.LookupNS(ctx, "google.com")
-	require.NoError(t, err)
+	nameservers := liveLookupNS(t, r, "google.com")
 
 	assert.True(t, sort.StringsAreSorted(nameservers))
 }
@@ -490,15 +675,8 @@ func TestLookupNS_MatchesFindAuthoritative(t *testing.T) {
 	t.Parallel()
 
 	r := newTestResolver(t)
-	ctx := testContext(t)
-
-	fromLookup, err := r.LookupNS(ctx, "google.com")
-	require.NoError(t, err)
-
-	fromFind, err := r.FindAuthoritativeNameservers(
-		ctx, "google.com",
-	)
-	require.NoError(t, err)
+	fromLookup := liveLookupNS(t, r, "google.com")
+	fromFind := liveFindAuthoritative(t, r, "google.com")
 
 	assert.Equal(t, fromFind, fromLookup)
 }
@@ -511,11 +689,7 @@ func TestResolveIPAddresses_ReturnsIPs(t *testing.T) {
 	t.Parallel()
 
 	r := newTestResolver(t)
-	ctx := testContext(t)
-
-	ips, err := r.ResolveIPAddresses(ctx, "google.com")
-	require.NoError(t, err)
-	require.NotEmpty(t, ips)
+	ips := liveResolveIPs(t, r, "google.com")
 
 	for _, ip := range ips {
 		parsed := net.ParseIP(ip)
@@ -529,10 +703,7 @@ func TestResolveIPAddresses_Deduplicated(t *testing.T) {
 	t.Parallel()
 
 	r := newTestResolver(t)
-	ctx := testContext(t)
-
-	ips, err := r.ResolveIPAddresses(ctx, "google.com")
-	require.NoError(t, err)
+	ips := liveResolveIPs(t, r, "google.com")
 
 	seen := make(map[string]bool)
 
@@ -546,10 +717,7 @@ func TestResolveIPAddresses_Sorted(t *testing.T) {
 	t.Parallel()
 
 	r := newTestResolver(t)
-	ctx := testContext(t)
-
-	ips, err := r.ResolveIPAddresses(ctx, "google.com")
-	require.NoError(t, err)
+	ips := liveResolveIPs(t, r, "google.com")
 
 	assert.True(t, sort.StringsAreSorted(ips))
 }
@@ -560,13 +728,10 @@ func TestResolveIPAddresses_NXDomainReturnsEmpty(
 	t.Parallel()
 
 	r := newTestResolver(t)
-	ctx := testContext(t)
-
-	ips, err := r.ResolveIPAddresses(
-		ctx,
-		"this-surely-does-not-exist-xyz.google.com",
+	ips := liveResolveIPsAllowingEmpty(
+		t, r, "this-surely-does-not-exist-xyz.google.com",
 	)
-	require.NoError(t, err)
+
 	assert.Empty(t, ips)
 }
 
@@ -574,11 +739,37 @@ func TestResolveIPAddresses_CloudflareDomain(t *testing.T) {
 	t.Parallel()
 
 	r := newTestResolver(t)
-	ctx := testContext(t)
+	ips := liveResolveIPs(t, r, "cloudflare.com")
 
-	ips, err := r.ResolveIPAddresses(ctx, "cloudflare.com")
-	require.NoError(t, err)
-	require.NotEmpty(t, ips)
+	assert.NotEmpty(t, ips)
+}
+
+// TestResolveIPAddresses_NameserverIPv4AndIPv6 looks up the addresses of
+// one of cloudflare.com's nameservers, as a domain check does for each
+// nameserver. That name has A and AAAA records, so both kinds of address
+// come back.
+func TestResolveIPAddresses_NameserverIPv4AndIPv6(t *testing.T) {
+	t.Parallel()
+
+	r := newTestResolver(t)
+	ns := findOneNSForDomain(t, r, "cloudflare.com")
+	ips := liveResolveIPs(t, r, ns)
+
+	var ipv4, ipv6 int
+
+	for _, ip := range ips {
+		parsed := net.ParseIP(ip)
+		require.NotNil(t, parsed, "should be valid IP: %s", ip)
+
+		if parsed.To4() != nil {
+			ipv4++
+		} else {
+			ipv6++
+		}
+	}
+
+	assert.Positive(t, ipv4, "no IPv4 address for %s: %v", ns, ips)
+	assert.Positive(t, ipv6, "no IPv6 address for %s: %v", ns, ips)
 }
 
 // ----------------------------------------------------------------
@@ -622,6 +813,134 @@ func TestQueryAllNameservers_ContextCanceled(t *testing.T) {
 	assert.Error(t, err)
 }
 
+// TestQueryEachNS_CanceledDuringQuery cancels the context while a
+// nameserver is being queried, as shutdown does. A lookup cut short
+// says nothing about the nameserver, so it must return an error, not a
+// failed response for it.
+func TestQueryEachNS_CanceledDuringQuery(t *testing.T) {
+	t.Parallel()
+
+	r := newTestResolver(t)
+	ctx, cancel := context.WithCancel(context.Background())
+	t.Cleanup(cancel)
+
+	// Finding the nameserver's address alone starts at the root
+	// servers and takes several round trips, so a cancel a few
+	// milliseconds in lands during the query.
+	time.AfterFunc(5*time.Millisecond, cancel)
+
+	results, err := r.QueryEachNS(
+		ctx, []string{"ns1.google.com."}, "google.com",
+	)
+	require.ErrorIs(t, err, resolver.ErrContextCanceled)
+	assert.Nil(t, results)
+}
+
+// ----------------------------------------------------------------
+// Timeout tests
+// ----------------------------------------------------------------
+
+func TestQueryNameserverIP_Timeout(t *testing.T) {
+	t.Parallel()
+
+	r := newTestResolver(t)
+
+	// Nothing answers at 192.0.2.1, a documentation address. The
+	// resolver tries each query twice, and the first try gives up
+	// after two seconds. A deadline that ends during the first try
+	// makes the status vary from run to run between error and
+	// timeout, so the deadline must outlast the first try.
+	ctx, cancel := context.WithTimeout(
+		context.Background(), 3*time.Second,
+	)
+	t.Cleanup(cancel)
+
+	resp, err := r.QueryNameserverIP(
+		ctx, "unreachable.test.", "192.0.2.1",
+		"example.com",
+	)
+	require.NoError(t, err)
+
+	assert.Equal(t, resolver.StatusTimeout, resp.Status)
+	assert.NotEmpty(t, resp.Error)
+}
+
+// TestCollectIPs_NoNameserverAnswered takes the response of a
+// nameserver at 192.0.2.1, where nothing answers, as
+// TestQueryNameserverIP_Timeout does. Addresses collected from
+// nameservers that all failed to answer are an error, not none.
+func TestCollectIPs_NoNameserverAnswered(t *testing.T) {
+	t.Parallel()
+
+	r := newTestResolver(t)
+
+	// The deadline outlasts the first try, as in
+	// TestQueryNameserverIP_Timeout.
+	ctx, cancel := context.WithTimeout(
+		context.Background(), 3*time.Second,
+	)
+	t.Cleanup(cancel)
+
+	resp, err := r.QueryNameserverIP(
+		ctx, "unreachable.test.", "192.0.2.1",
+		"example.com",
+	)
+	require.NoError(t, err)
+
+	ips, _, err := resolver.CollectIPs(
+		map[string]*resolver.NameserverResponse{resp.Nameserver: resp},
+	)
+	require.ErrorIs(t, err, resolver.ErrNoNameserverAnswered)
+	assert.Empty(t, ips)
+}
+
+// TestCollectIPs_ReferralIsNoAnswer asks a root server about
+// example.com, which the root zone does not hold, so it only refers the
+// query to the com servers. That reply is no answer, as is a parent
+// zone's when every server of the name's own zone failed.
+func TestCollectIPs_ReferralIsNoAnswer(t *testing.T) {
+	t.Parallel()
+
+	r := newTestResolver(t)
+
+	var resp *resolver.NameserverResponse
+
+	livednstest.Retry(
+		t,
+		"QueryNameserverIP(a.root-servers.net, example.com)",
+		func(ctx context.Context) error {
+			var err error
+
+			resp, err = r.QueryNameserverIP(
+				ctx, "a.root-servers.net.", "198.41.0.4",
+				"example.com",
+			)
+			if err != nil {
+				return err
+			}
+
+			// A timeout or a network error is no reply at all.
+			if resp.Status == resolver.StatusTimeout ||
+				strings.HasPrefix(resp.Error, "network error") {
+				return fmt.Errorf(
+					"%w: %s", livednstest.ErrNoAnswer, resp.Error,
+				)
+			}
+
+			return nil
+		},
+	)
+
+	assert.Equal(t, resolver.StatusError, resp.Status)
+	assert.Equal(t, "server returned a referral", resp.Error)
+
+	ips, _, err := resolver.CollectIPs(
+		map[string]*resolver.NameserverResponse{resp.Nameserver: resp},
+	)
+	require.ErrorIs(t, err, resolver.ErrNoNameserverAnswered)
+	assert.Empty(t, ips)
+}
+
 func TestResolveIPAddresses_ContextCanceled(t *testing.T) {
 	t.Parallel()
 

internal/server/export_test.go

@@ -0,0 +1,38 @@
+package server
+
+import (
+	"net/http"
+	"time"
+
+	"github.com/go-chi/chi/v5"
+)
+
+// RequestTimeout exports the handler execution budget applied by
+// chimw.Timeout in SetupRoutes, so tests can assert the relationship
+// between it and the server's WriteTimeout.
+const RequestTimeout time.Duration = requestTimeout
+
+// SetListenPort overrides the port Run binds. A test uses it to hand
+// Run an unbindable port so ListenAndServe fails immediately and Run
+// returns after storing its http.Server.
+func SetListenPort(s *Server, port int) {
+	s.port = port
+}
+
+// HTTPServerOf returns the http.Server that Run built and stored, so a
+// test can inspect the timeouts the running server actually carries.
+func HTTPServerOf(s *Server) *http.Server {
+	return s.httpServer
+}
+
+// EnableSentry runs the Sentry setup that the start hook runs, without
+// starting the HTTP server.
+func EnableSentry(s *Server) error {
+	return s.enableSentry()
+}
+
+// RouterOf returns the router SetupRoutes built, so a test can add a
+// route that panics.
+func RouterOf(s *Server) *chi.Mux {
+	return s.router
+}

internal/server/routes.go

@@ -1,11 +1,15 @@
 package server
 
 import (
+	"net/http"
 	"time"
 
+	sentryhttp "github.com/getsentry/sentry-go/http"
 	"github.com/go-chi/chi/v5"
 	chimw "github.com/go-chi/chi/v5/middleware"
 	"github.com/prometheus/client_golang/prometheus/promhttp"
+
+	"sneak.berlin/go/dnswatcher/static"
 )
 
 // requestTimeout is the maximum duration for handling a request.
@@ -18,26 +22,67 @@ func (s *Server) SetupRoutes() {
 	// Global middleware
 	s.router.Use(chimw.Recoverer)
 	s.router.Use(chimw.RequestID)
+	s.router.Use(s.mw.SecurityHeaders())
 	s.router.Use(s.mw.Logging())
-	s.router.Use(s.mw.CORS())
 	s.router.Use(chimw.Timeout(requestTimeout))
 
-	// Health check
-	s.router.Get("/health", s.handlers.HandleHealthCheck())
+	// Report panics in handlers to Sentry when DNSWATCHER_SENTRY_DSN is
+	// set. Repanic passes each panic on to chimw.Recoverer above, which
+	// still answers the request.
+	if s.sentryEnabled {
+		sentryHandler := sentryhttp.New(sentryhttp.Options{
+			Repanic: true,
+		})
+		s.router.Use(sentryHandler.Handle)
+	}
+
+	// Public, unauthenticated, read-only routes, the only ones
+	// REPO_POLICIES.md allows wildcard CORS on. CORS is middleware of
+	// this whole router, not of a Group, so that it also answers
+	// OPTIONS preflight requests, which no route here registers.
+	public := chi.NewRouter()
+	public.Use(s.mw.CORS())
+
+	// Dashboard (read-only web UI)
+	public.Get("/", s.handlers.HandleDashboard())
+
+	// Static assets (embedded CSS/JS)
+	public.Mount(
+		"/s",
+		http.StripPrefix(
+			"/s",
+			http.FileServer(http.FS(static.Static)),
+		),
+	)
+
+	// Health check (standard well-known path)
+	public.Get(
+		"/.well-known/healthcheck",
+		s.handlers.HandleHealthCheck(),
+	)
+
+	// Legacy health check (keep for backward compatibility)
+	public.Get("/health", s.handlers.HandleHealthCheck())
 
 	// API v1 routes
-	s.router.Route("/api/v1", func(r chi.Router) {
+	public.Route("/api/v1", func(r chi.Router) {
 		r.Get("/status", s.handlers.HandleStatus())
 	})
 
-	// Metrics endpoint (optional, with basic auth)
+	s.router.Mount("/", public)
+
+	// Metrics endpoint (optional, with basic auth) and no CORS: a
+	// Prometheus scraper is not a browser. It is mounted rather than
+	// added with Get so that every method on /metrics, OPTIONS
+	// included, ends here instead of falling through to the public
+	// router and its CORS. The rate limit comes before Basic Auth, so
+	// failed logins count against it and a request over the limit
+	// never reaches the password check.
 	if s.params.Config.MetricsUsername != "" {
-		s.router.Group(func(r chi.Router) {
-			r.Use(s.mw.MetricsAuth())
-			r.Get(
-				"/metrics",
-				promhttp.Handler().ServeHTTP,
-			)
-		})
+		metrics := chi.NewRouter()
+		metrics.Use(s.mw.MetricsRateLimit())
+		metrics.Use(s.mw.MetricsAuth())
+		metrics.Get("/", promhttp.Handler().ServeHTTP)
+		s.router.Mount("/metrics", metrics)
 	}
 }

internal/server/routes_test.go

@@ -0,0 +1,290 @@
+package server_test
+
+import (
+	"net/http"
+	"net/http/httptest"
+	"testing"
+
+	"github.com/spf13/viper"
+
+	"sneak.berlin/go/dnswatcher/internal/server"
+)
+
+// Credentials for /metrics, which is only routed when a username is set.
+const (
+	metricsUsername = "scraper"
+	metricsPassword = "scrape-secret"
+)
+
+// The tests below set env vars and touch viper global state, so like
+// the config tests they cannot use t.Parallel.
+
+// routedServer builds the server with its routes set up, ready to serve
+// test requests. The caller must first configure viper.
+func routedServer(t *testing.T) *server.Server {
+	t.Helper()
+
+	srv := buildServer(t)
+	srv.SetupRoutes()
+
+	return srv
+}
+
+// crossOriginRequest builds a request as a browser sends it from a page
+// on another site.
+func crossOriginRequest(
+	t *testing.T,
+	method string,
+	target string,
+) *http.Request {
+	t.Helper()
+
+	req := httptest.NewRequestWithContext(t.Context(), method, target, nil)
+	req.Header.Set("Origin", "https://example.net")
+
+	return req
+}
+
+// preflightRequest builds the OPTIONS request a browser sends before a
+// cross-origin request with the given method and request headers.
+func preflightRequest(
+	t *testing.T,
+	target string,
+	method string,
+	headers string,
+) *http.Request {
+	t.Helper()
+
+	req := crossOriginRequest(t, http.MethodOptions, target)
+	req.Header.Set("Access-Control-Request-Method", method)
+
+	if headers != "" {
+		req.Header.Set("Access-Control-Request-Headers", headers)
+	}
+
+	return req
+}
+
+func serve(
+	srv *server.Server,
+	req *http.Request,
+) *httptest.ResponseRecorder {
+	rec := httptest.NewRecorder()
+	srv.ServeHTTP(rec, req)
+
+	return rec
+}
+
+// publicPaths returns one path on each public route.
+func publicPaths() []string {
+	return []string{
+		"/",
+		"/s/css/tailwind.min.css",
+		"/api/v1/status",
+		"/health",
+		"/.well-known/healthcheck",
+	}
+}
+
+// TestPublicRoutesAllowAnyOrigin checks that every public route answers
+// a cross-origin GET with the CORS wildcard.
+func TestPublicRoutesAllowAnyOrigin(t *testing.T) {
+	viper.Reset()
+	t.Setenv("DNSWATCHER_TARGETS", "example.com")
+	t.Setenv("DNSWATCHER_METRICS_USERNAME", metricsUsername)
+	t.Setenv("DNSWATCHER_METRICS_PASSWORD", metricsPassword)
+
+	srv := routedServer(t)
+
+	for _, path := range publicPaths() {
+		rec := serve(srv, crossOriginRequest(t, http.MethodGet, path))
+
+		if rec.Code != http.StatusOK {
+			t.Errorf("GET %s: status = %d, want 200", path, rec.Code)
+		}
+
+		got := rec.Header().Get("Access-Control-Allow-Origin")
+		if got != "*" {
+			t.Errorf(
+				"GET %s: Access-Control-Allow-Origin = %q, want %q",
+				path, got, "*",
+			)
+		}
+	}
+}
+
+// TestMetricsHasNoCORS checks that no request to the Basic-Auth
+// protected /metrics, preflight included, gets a CORS header.
+func TestMetricsHasNoCORS(t *testing.T) {
+	viper.Reset()
+	t.Setenv("DNSWATCHER_TARGETS", "example.com")
+	t.Setenv("DNSWATCHER_METRICS_USERNAME", metricsUsername)
+	t.Setenv("DNSWATCHER_METRICS_PASSWORD", metricsPassword)
+
+	srv := routedServer(t)
+
+	authenticated := crossOriginRequest(t, http.MethodGet, "/metrics")
+	authenticated.SetBasicAuth(metricsUsername, metricsPassword)
+
+	tests := []struct {
+		name       string
+		req        *http.Request
+		wantStatus int
+	}{
+		{
+			"authenticated GET",
+			authenticated,
+			http.StatusOK,
+		},
+		{
+			"unauthenticated GET",
+			crossOriginRequest(t, http.MethodGet, "/metrics"),
+			http.StatusUnauthorized,
+		},
+		{
+			"preflight",
+			preflightRequest(t, "/metrics", http.MethodGet, ""),
+			http.StatusUnauthorized,
+		},
+	}
+
+	for _, tt := range tests {
+		rec := serve(srv, tt.req)
+
+		if rec.Code != tt.wantStatus {
+			t.Errorf(
+				"%s: status = %d, want %d",
+				tt.name, rec.Code, tt.wantStatus,
+			)
+		}
+
+		got := rec.Header().Get("Access-Control-Allow-Origin")
+		if got != "" {
+			t.Errorf(
+				"%s: Access-Control-Allow-Origin = %q, want none",
+				tt.name, got,
+			)
+		}
+	}
+}
+
+// TestPreflightAllowsOnlyWhatPublicRoutesServe checks what each public
+// route agrees to in a CORS preflight: GET, but not POST, PUT or
+// DELETE, which no route serves, and not the Authorization or
+// X-CSRF-Token headers, which no public route reads. It checks every
+// public route because one added with Get, such as /health, answers a
+// preflight only while CORS is middleware of a whole router; in a
+// Group, chi would answer it with 405 and no CORS headers.
+func TestPreflightAllowsOnlyWhatPublicRoutesServe(t *testing.T) {
+	viper.Reset()
+	t.Setenv("DNSWATCHER_TARGETS", "example.com")
+	t.Setenv("DNSWATCHER_METRICS_USERNAME", metricsUsername)
+	t.Setenv("DNSWATCHER_METRICS_PASSWORD", metricsPassword)
+
+	srv := routedServer(t)
+
+	tests := []struct {
+		method  string
+		headers string
+		allowed bool
+	}{
+		{http.MethodGet, "", true},
+		{http.MethodGet, "Content-Type", true},
+		{http.MethodPost, "", false},
+		{http.MethodPut, "", false},
+		{http.MethodDelete, "", false},
+		{http.MethodGet, "Authorization", false},
+		{http.MethodGet, "X-CSRF-Token", false},
+	}
+
+	for _, path := range publicPaths() {
+		for _, tt := range tests {
+			rec := serve(srv, preflightRequest(
+				t, path, tt.method, tt.headers,
+			))
+
+			want := ""
+			if tt.allowed {
+				want = tt.method
+			}
+
+			got := rec.Header().Get("Access-Control-Allow-Methods")
+			if got != want {
+				t.Errorf(
+					"preflight to %s for %s with headers %q: "+
+						"Access-Control-Allow-Methods = %q, want %q",
+					path, tt.method, tt.headers, got, want,
+				)
+			}
+		}
+	}
+}
+
+// metricsRequest builds a GET for /metrics from remoteAddr that logs
+// in with the given password.
+func metricsRequest(
+	t *testing.T,
+	remoteAddr string,
+	password string,
+) *http.Request {
+	t.Helper()
+
+	req := httptest.NewRequestWithContext(
+		t.Context(), http.MethodGet, "/metrics", nil,
+	)
+	req.RemoteAddr = remoteAddr
+	req.SetBasicAuth(metricsUsername, password)
+
+	return req
+}
+
+// TestMetricsRateLimitComesBeforeAuth checks that failed logins to
+// /metrics count against the rate limit; that once an address is over
+// it, even the right password gets 429, with the same body as a wrong
+// one; and that another address still gets in.
+func TestMetricsRateLimitComesBeforeAuth(t *testing.T) {
+	viper.Reset()
+	t.Setenv("DNSWATCHER_TARGETS", "example.com")
+	t.Setenv("DNSWATCHER_METRICS_USERNAME", metricsUsername)
+	t.Setenv("DNSWATCHER_METRICS_PASSWORD", metricsPassword)
+
+	const (
+		guesser = "198.51.100.1:4000"
+		other   = "198.51.100.2:4000"
+
+		// Far more guesses than the rate limit allows.
+		maxGuesses = 1000
+	)
+
+	srv := routedServer(t)
+
+	var guess *httptest.ResponseRecorder
+
+	for range maxGuesses {
+		guess = serve(srv, metricsRequest(t, guesser, "wrong"))
+		if guess.Code != http.StatusUnauthorized {
+			break
+		}
+	}
+
+	if guess.Code != http.StatusTooManyRequests {
+		t.Fatalf("wrong password: status = %d, want 429", guess.Code)
+	}
+
+	right := serve(srv, metricsRequest(t, guesser, metricsPassword))
+	if right.Code != http.StatusTooManyRequests {
+		t.Errorf("right password: status = %d, want 429", right.Code)
+	}
+
+	if right.Body.String() != guess.Body.String() {
+		t.Errorf(
+			"429 body with right password = %q, with wrong one = %q",
+			right.Body.String(), guess.Body.String(),
+		)
+	}
+
+	rec := serve(srv, metricsRequest(t, other, metricsPassword))
+	if rec.Code != http.StatusOK {
+		t.Errorf("another address: status = %d, want 200", rec.Code)
+	}
+}

internal/server/sentry_test.go

@@ -0,0 +1,190 @@
+package server_test
+
+import (
+	"io"
+	"net/http"
+	"net/http/httptest"
+	"net/url"
+	"strings"
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/getsentry/sentry-go"
+	"github.com/spf13/viper"
+	"go.uber.org/fx"
+
+	"sneak.berlin/go/dnswatcher/internal/server"
+)
+
+// The tests below set env vars and touch the global state of viper and
+// of Sentry, so they cannot use t.Parallel.
+
+// standInDelay is how long the Sentry stand-in takes to answer. It
+// records a report only then, so a report that Shutdown did not wait
+// for has not been recorded yet when Shutdown returns.
+const standInDelay = 100 * time.Millisecond
+
+// sentryStandIn is a local HTTP server in place of Sentry's, so that
+// nothing a test reports leaves the host. It keeps the body of every
+// request it receives.
+type sentryStandIn struct {
+	server *httptest.Server
+	mu     sync.Mutex
+	bodies []string
+}
+
+func newSentryStandIn(t *testing.T) *sentryStandIn {
+	t.Helper()
+
+	standIn := &sentryStandIn{}
+	standIn.server = httptest.NewServer(http.HandlerFunc(
+		func(_ http.ResponseWriter, r *http.Request) {
+			body, err := io.ReadAll(r.Body)
+			if err != nil {
+				t.Errorf("reading request to the Sentry stand-in: %v", err)
+			}
+
+			time.Sleep(standInDelay)
+
+			standIn.mu.Lock()
+			defer standIn.mu.Unlock()
+
+			standIn.bodies = append(standIn.bodies, string(body))
+		},
+	))
+	t.Cleanup(standIn.server.Close)
+
+	return standIn
+}
+
+// dsn returns a DSN that points Sentry at the stand-in.
+func (s *sentryStandIn) dsn(t *testing.T) string {
+	t.Helper()
+
+	dsn, err := url.Parse(s.server.URL)
+	if err != nil {
+		t.Fatalf("parsing the stand-in URL: %v", err)
+	}
+
+	dsn.User = url.User("public-key")
+	dsn.Path = "/1"
+
+	return dsn.String()
+}
+
+// received reports whether a request to the stand-in contained text.
+func (s *sentryStandIn) received(text string) bool {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+
+	for _, body := range s.bodies {
+		if strings.Contains(body, text) {
+			return true
+		}
+	}
+
+	return false
+}
+
+func TestSentryUnsetDoesNothing(t *testing.T) {
+	viper.Reset()
+	t.Setenv("DNSWATCHER_TARGETS", "example.com")
+	t.Setenv("DNSWATCHER_SENTRY_DSN", "")
+
+	srv := buildServer(t)
+
+	err := server.EnableSentry(srv)
+	if err != nil {
+		t.Fatalf("Sentry setup with no DSN: %v", err)
+	}
+
+	if sentry.CurrentHub().Client() != nil {
+		t.Error("Sentry was set up with no DSN configured")
+	}
+}
+
+// TestSentryReportsHandlerPanic checks that with a valid DSN a panic in
+// a handler is reported to Sentry, still reaches chimw.Recoverer, and
+// has been sent by the time Shutdown returns, and that nothing else is
+// sent to Sentry.
+func TestSentryReportsHandlerPanic(t *testing.T) {
+	standIn := newSentryStandIn(t)
+
+	viper.Reset()
+	t.Setenv("DNSWATCHER_TARGETS", "example.com")
+	t.Setenv("DNSWATCHER_SENTRY_DSN", standIn.dsn(t))
+
+	srv := buildServer(t)
+
+	err := server.EnableSentry(srv)
+	if err != nil {
+		t.Fatalf("Sentry setup with a valid DSN: %v", err)
+	}
+
+	// Sentry's client is global: close it so later tests find none.
+	t.Cleanup(func() {
+		sentry.CurrentHub().Client().Close()
+		sentry.CurrentHub().BindClient(nil)
+	})
+
+	const panicMessage = "handler panic in the Sentry test"
+
+	srv.SetupRoutes()
+	server.RouterOf(srv).Get(
+		"/panic",
+		func(http.ResponseWriter, *http.Request) {
+			panic(panicMessage)
+		},
+	)
+
+	// An ordinary request first: with client reports on, Sentry would
+	// add a count of its dropped transaction to the panic report.
+	serve(srv, httptest.NewRequestWithContext(
+		t.Context(), http.MethodGet, "/.well-known/healthcheck", nil,
+	))
+
+	rec := serve(srv, httptest.NewRequestWithContext(
+		t.Context(), http.MethodGet, "/panic", nil,
+	))
+	if rec.Code != http.StatusInternalServerError {
+		t.Errorf(
+			"status %d, want %d from the recoverer",
+			rec.Code, http.StatusInternalServerError,
+		)
+	}
+
+	err = srv.Shutdown(t.Context())
+	if err != nil {
+		t.Fatalf("Shutdown: %v", err)
+	}
+
+	if !standIn.received(panicMessage) {
+		t.Error("the panic had not been sent to Sentry when Shutdown returned")
+	}
+
+	if standIn.received("client_report") {
+		t.Error("Sentry was sent a client report, not only the panic")
+	}
+}
+
+func TestSentryInvalidDSNStopsStartup(t *testing.T) {
+	viper.Reset()
+	t.Setenv("DNSWATCHER_TARGETS", "example.com")
+	t.Setenv("DNSWATCHER_DATA_DIR", t.TempDir())
+	// Sentry cannot parse this: it has no public key before the host.
+	t.Setenv("DNSWATCHER_SENTRY_DSN", "https://sentry.test/1")
+
+	app := newServerApp(fx.Invoke(func(*server.Server) {}))
+
+	err := app.Start(t.Context())
+	if err == nil {
+		_ = app.Stop(t.Context())
+
+		t.Fatal("startup succeeded with an invalid DSN")
+	}
+
+	if !strings.Contains(err.Error(), "invalid DNSWATCHER_SENTRY_DSN") {
+		t.Errorf("startup error does not name the setting: %v", err)
+	}
+}

internal/server/server.go

@@ -9,6 +9,7 @@ import (
 	"net/http"
 	"time"
 
+	"github.com/getsentry/sentry-go"
 	"github.com/go-chi/chi/v5"
 	"go.uber.org/fx"
 
@@ -33,19 +34,68 @@ type Params struct {
 // shutdownTimeout is how long to wait for graceful shutdown.
 const shutdownTimeout = 30 * time.Second
 
-// readHeaderTimeout is the max duration for reading request headers.
-const readHeaderTimeout = 10 * time.Second
+// sentryFlushTimeout is how long shutdown waits for Sentry to send the
+// error reports it still holds.
+const sentryFlushTimeout = 2 * time.Second
+
+// Socket-level timeouts for the HTTP server.
+//
+// These bound time spent on the connection itself and are a distinct
+// control from the per-request handler budget enforced by
+// chimw.Timeout(requestTimeout) in routes.go: that one cancels the
+// request context after requestTimeout but never touches the socket,
+// so without the values below a peer can hold a connection open
+// forever (slowloris, unreaped keep-alives).
+//
+// The one hard constraint between the two controls is
+// writeTimeout > requestTimeout. net/http arms the write deadline
+// once the request headers have been read, so on a plaintext
+// connection it covers handler execution AND the response flush. If
+// writeTimeout were <= requestTimeout the server would sever the
+// connection before a handler that legitimately consumed its full
+// budget could emit anything, making the 60s budget unreachable in
+// practice. The margin between them is the response-flush allowance.
+//
+// The only clients of this service are browsers loading the dashboard
+// and a Prometheus scraper; the values are sized for those.
+const (
+	// readHeaderTimeout is the max duration for reading request
+	// headers.
+	readHeaderTimeout = 10 * time.Second
+
+	// readTimeout bounds reading the entire request, headers plus
+	// body. Every route here is a GET with no body, so this only
+	// ever needs to cover headers; the extra 5s over
+	// readHeaderTimeout is slack, not a real allowance, and keeps a
+	// body dribbled one byte at a time from holding the read side
+	// open indefinitely.
+	readTimeout = 15 * time.Second
+
+	// writeTimeout must exceed the requestTimeout handler budget
+	// (60s) per the note above. The 15s difference is the allowance
+	// for flushing a completed response to a slow client.
+	writeTimeout = 75 * time.Second
+
+	// idleTimeout reaps keep-alive connections between requests. It
+	// is deliberately longer than the common Prometheus scrape
+	// intervals (15s/30s/60s) so the scraper reuses its connection
+	// rather than reconnecting every cycle, while a browser tab
+	// left open on the dashboard stops occupying a connection
+	// within two minutes of going quiet.
+	idleTimeout = 120 * time.Second
+)
 
 // Server is the HTTP server.
 type Server struct {
-	startupTime time.Time
-	port        int
-	log         *slog.Logger
-	router      *chi.Mux
-	httpServer  *http.Server
-	params      Params
-	mw          *middleware.Middleware
-	handlers    *handlers.Handlers
+	startupTime   time.Time
+	port          int
+	sentryEnabled bool
+	log           *slog.Logger
+	router        *chi.Mux
+	httpServer    *http.Server
+	params        Params
+	mw            *middleware.Middleware
+	handlers      *handlers.Handlers
 }
 
 // New creates a new Server instance.
@@ -64,6 +114,12 @@ func New(
 	lifecycle.Append(fx.Hook{
 		OnStart: func(_ context.Context) error {
 			srv.startupTime = time.Now()
+
+			err := srv.enableSentry()
+			if err != nil {
+				return err
+			}
+
 			go srv.Run()
 
 			return nil
@@ -76,16 +132,29 @@ func New(
 	return srv, nil
 }
 
+// newHTTPServer builds the listening http.Server with every
+// socket-level timeout set. All four are set deliberately: a zero
+// value in net/http means "no limit", not "some default".
+func newHTTPServer(
+	listenAddr string,
+	handler http.Handler,
+) *http.Server {
+	return &http.Server{
+		Addr:              listenAddr,
+		Handler:           handler,
+		ReadTimeout:       readTimeout,
+		ReadHeaderTimeout: readHeaderTimeout,
+		WriteTimeout:      writeTimeout,
+		IdleTimeout:       idleTimeout,
+	}
+}
+
 // Run starts the HTTP server.
 func (s *Server) Run() {
 	s.SetupRoutes()
 
 	listenAddr := fmt.Sprintf(":%d", s.port)
-	s.httpServer = &http.Server{
-		Addr:              listenAddr,
-		Handler:           s,
-		ReadHeaderTimeout: readHeaderTimeout,
-	}
+	s.httpServer = newHTTPServer(listenAddr, s)
 
 	s.log.Info("http server starting", "addr", listenAddr)
 
@@ -95,8 +164,11 @@ func (s *Server) Run() {
 	}
 }
 
-// Shutdown gracefully shuts down the server.
+// Shutdown gracefully shuts down the server, then sends the error
+// reports Sentry still holds.
 func (s *Server) Shutdown(ctx context.Context) error {
+	defer s.flushSentry()
+
 	if s.httpServer == nil {
 		return nil
 	}
@@ -127,3 +199,45 @@ func (s *Server) ServeHTTP(
 ) {
 	s.router.ServeHTTP(writer, request)
 }
+
+// enableSentry turns on Sentry error reporting when
+// DNSWATCHER_SENTRY_DSN is set, and does nothing when it is not. A DSN
+// that Sentry cannot parse is an error, so that startup stops instead
+// of running without the error reporting the operator asked for.
+func (s *Server) enableSentry() error {
+	if s.params.Config.SentryDSN == "" {
+		return nil
+	}
+
+	err := sentry.Init(sentry.ClientOptions{
+		Dsn:     s.params.Config.SentryDSN,
+		Release: s.params.Globals.Appname + "-" + s.params.Globals.Version,
+		// Use the transport that queues each report as it is made. With
+		// the default one, Flush can return before sending a report made
+		// just before it, such as one from the last request at shutdown.
+		DisableTelemetryBuffer: true,
+		// Send panic reports only, not Sentry's counts of what it dropped,
+		// such as the transaction it starts for every request.
+		DisableClientReports: true,
+	})
+	if err != nil {
+		return fmt.Errorf("invalid DNSWATCHER_SENTRY_DSN: %w", err)
+	}
+
+	s.log.Info("sentry error reporting activated")
+	s.sentryEnabled = true
+
+	return nil
+}
+
+// flushSentry sends the error reports Sentry still holds, waiting at
+// most sentryFlushTimeout.
+func (s *Server) flushSentry() {
+	if !s.sentryEnabled {
+		return
+	}
+
+	if !sentry.Flush(sentryFlushTimeout) {
+		s.log.Warn("sentry flush timed out; some error reports were not sent")
+	}
+}

internal/server/server_test.go

@@ -0,0 +1,135 @@
+package server_test
+
+import (
+	"testing"
+
+	"github.com/spf13/viper"
+	"go.uber.org/fx"
+
+	"sneak.berlin/go/dnswatcher/internal/config"
+	"sneak.berlin/go/dnswatcher/internal/globals"
+	"sneak.berlin/go/dnswatcher/internal/handlers"
+	"sneak.berlin/go/dnswatcher/internal/healthcheck"
+	"sneak.berlin/go/dnswatcher/internal/logger"
+	"sneak.berlin/go/dnswatcher/internal/middleware"
+	"sneak.berlin/go/dnswatcher/internal/notify"
+	"sneak.berlin/go/dnswatcher/internal/server"
+	"sneak.berlin/go/dnswatcher/internal/state"
+)
+
+// newServerApp builds an fx app holding a *server.Server wired exactly
+// as cmd/dnswatcher wires it, minus the watcher/resolver subtree that
+// would touch live DNS, plus the given option. config.New reads viper,
+// so the caller must first configure it, which is also why the caller
+// cannot run in parallel.
+func newServerApp(option fx.Option) *fx.App {
+	return fx.New(
+		fx.NopLogger,
+		fx.Provide(
+			globals.New,
+			logger.New,
+			config.New,
+			state.New,
+			healthcheck.New,
+			notify.New,
+			middleware.New,
+			handlers.New,
+			server.New,
+		),
+		option,
+	)
+}
+
+// buildServer builds the server without starting the app's lifecycle,
+// so no OnStart hook runs and nothing listens or resolves.
+func buildServer(t *testing.T) *server.Server {
+	t.Helper()
+
+	var srv *server.Server
+
+	app := newServerApp(fx.Populate(&srv))
+
+	err := app.Err()
+	if err != nil {
+		t.Fatalf("building server graph: %v", err)
+	}
+
+	return srv
+}
+
+// TestRunWiresSocketTimeouts pins that the http.Server the running
+// server actually serves — the one Run builds and hands to
+// ListenAndServe — carries every socket-level timeout, plus the two
+// relationships the values must satisfy.
+//
+// Run is driven to completion with an unbindable port: it builds and
+// stores s.httpServer, then ListenAndServe fails at once and Run
+// returns without ever listening. The assertions run in the same
+// goroutine after Run returns, so reading s.httpServer is free of any
+// data race. Nothing here measures elapsed time.
+//
+// ReadTimeout must be at least ReadHeaderTimeout. net/http reads the
+// headers under ReadHeaderTimeout, then sets the read deadline for the
+// rest of the request to ReadTimeout, counted from when it started
+// reading the request. If ReadTimeout were smaller, a request whose
+// headers arrived after ReadTimeout but within ReadHeaderTimeout would
+// get a read deadline that had already passed, so reading its body
+// would fail at once.
+func TestRunWiresSocketTimeouts(t *testing.T) {
+	// Sets an env var and touches viper global state, so like the
+	// config tests it cannot use t.Parallel.
+	viper.Reset()
+	t.Setenv("DNSWATCHER_TARGETS", "example.com")
+
+	srv := buildServer(t)
+	server.SetListenPort(srv, -1)
+
+	srv.Run()
+
+	hs := server.HTTPServerOf(srv)
+	if hs == nil {
+		t.Fatal("Run did not build an http.Server")
+	}
+
+	if hs.ReadTimeout <= 0 {
+		t.Errorf("ReadTimeout must be non-zero, got %v", hs.ReadTimeout)
+	}
+
+	if hs.ReadHeaderTimeout <= 0 {
+		t.Errorf(
+			"ReadHeaderTimeout must be non-zero, got %v",
+			hs.ReadHeaderTimeout,
+		)
+	}
+
+	if hs.WriteTimeout <= 0 {
+		t.Errorf("WriteTimeout must be non-zero, got %v", hs.WriteTimeout)
+	}
+
+	if hs.IdleTimeout <= 0 {
+		t.Errorf("IdleTimeout must be non-zero, got %v", hs.IdleTimeout)
+	}
+
+	if hs.WriteTimeout <= server.RequestTimeout {
+		t.Errorf(
+			"WriteTimeout (%v) must exceed handler budget (%v)",
+			hs.WriteTimeout,
+			server.RequestTimeout,
+		)
+	}
+
+	if hs.ReadTimeout < hs.ReadHeaderTimeout {
+		t.Errorf(
+			"ReadTimeout (%v) must be >= ReadHeaderTimeout (%v)",
+			hs.ReadTimeout,
+			hs.ReadHeaderTimeout,
+		)
+	}
+
+	if hs.Handler != srv {
+		t.Errorf(
+			"Run wired handler %T, want the *server.Server",
+			hs.Handler,
+		)
+	}
+}

internal/state/export_test.go

@@ -6,8 +6,9 @@ import (
 	"sneak.berlin/go/dnswatcher/internal/config"
 )
 
-// NewForTest creates a State for unit testing with no persistence.
-func NewForTest() *State {
+// NewForTestWithDataDir creates an empty State that saves to dataDir,
+// without the fx lifecycle.
+func NewForTestWithDataDir(dataDir string) *State {
 	return &State{
 		log: slog.Default(),
 		snapshot: &Snapshot{
@@ -17,6 +18,6 @@ func NewForTest() *State {
 			Ports:        make(map[string]*PortState),
 			Certificates: make(map[string]*CertificateState),
 		},
-		config: &config.Config{DataDir: ""},
+		config: &config.Config{DataDir: dataDir},
 	}
 }

internal/state/state.go

@@ -35,9 +35,13 @@ type Params struct {
 }
 
 // DomainState holds the monitoring state for an apex domain.
+// NameserverAddresses holds the sorted addresses each nameserver's name
+// resolves to, by nameserver name. A state file written before it
+// existed loads with it nil.
 type DomainState struct {
-	Nameservers []string  `json:"nameservers"`
-	LastChecked time.Time `json:"lastChecked"`
+	Nameservers         []string            `json:"nameservers"`
+	NameserverAddresses map[string][]string `json:"nameserverAddresses"`
+	LastChecked         time.Time           `json:"lastChecked"`
 }
 
 // NameserverRecordState holds one NS's response for a hostname.
@@ -49,18 +53,62 @@ type NameserverRecordState struct {
 }
 
 // HostnameState holds per-nameserver monitoring state for a hostname.
+// CNAMEAddresses holds the sorted addresses at the end of the name's
+// CNAME chain, found when its nameservers answered with a CNAME and no
+// address; it is empty otherwise. It is nil when they are not known: a
+// state file written before it existed loads with it nil.
 type HostnameState struct {
 	RecordsByNameserver map[string]*NameserverRecordState `json:"recordsByNameserver"`
+	CNAMEAddresses      []string                          `json:"cnameAddresses"`
 	LastChecked         time.Time                         `json:"lastChecked"`
 }
 
 // PortState holds the monitoring state for a port.
 type PortState struct {
 	Open        bool      `json:"open"`
-	Hostname    string    `json:"hostname"`
+	Hostnames   []string  `json:"hostnames"`
 	LastChecked time.Time `json:"lastChecked"`
 }
 
+// UnmarshalJSON implements custom unmarshaling to handle both
+// the old single-hostname format and the new multi-hostname
+// format for backward compatibility with existing state files.
+func (ps *PortState) UnmarshalJSON(data []byte) error {
+	// Use an alias to prevent infinite recursion.
+	type portStateAlias struct {
+		Open        bool      `json:"open"`
+		Hostnames   []string  `json:"hostnames"`
+		LastChecked time.Time `json:"lastChecked"`
+	}
+
+	var alias portStateAlias
+
+	err := json.Unmarshal(data, &alias)
+	if err != nil {
+		return fmt.Errorf("unmarshaling port state: %w", err)
+	}
+
+	ps.Open = alias.Open
+	ps.Hostnames = alias.Hostnames
+	ps.LastChecked = alias.LastChecked
+
+	// If Hostnames is empty, try reading the old single-hostname
+	// format for backward compatibility.
+	if len(ps.Hostnames) == 0 {
+		var old struct {
+			Hostname string `json:"hostname"`
+		}
+
+		// Best-effort: ignore errors since the main unmarshal
+		// already succeeded.
+		if json.Unmarshal(data, &old) == nil && old.Hostname != "" {
+			ps.Hostnames = []string{old.Hostname}
+		}
+	}
+
+	return nil
+}
+
 // CertificateState holds TLS certificate monitoring state.
 type CertificateState struct {
 	CommonName              string    `json:"commonName"`
@@ -109,6 +157,11 @@ func New(
 
 	lifecycle.Append(fx.Hook{
 		OnStart: func(_ context.Context) error {
+			err := state.checkDataDirWritable()
+			if err != nil {
+				return err
+			}
+
 			return state.Load()
 		},
 		OnStop: func(_ context.Context) error {
@@ -263,6 +316,27 @@ func (s *State) GetPortState(key string) (*PortState, bool) {
 	return ps, ok
 }
 
+// DeletePortState removes a port state entry.
+func (s *State) DeletePortState(key string) {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+
+	delete(s.snapshot.Ports, key)
+}
+
+// GetAllPortKeys returns all port state keys.
+func (s *State) GetAllPortKeys() []string {
+	s.mu.RLock()
+	defer s.mu.RUnlock()
+
+	keys := make([]string, 0, len(s.snapshot.Ports))
+	for k := range s.snapshot.Ports {
+		keys = append(keys, k)
+	}
+
+	return keys
+}
+
 // SetCertificateState updates the state for a certificate.
 func (s *State) SetCertificateState(
 	key string,
@@ -285,3 +359,27 @@ func (s *State) GetCertificateState(
 
 	return cs, ok
 }
+
+// checkDataDirWritable creates the data directory if needed, then writes
+// and removes the temp file that Save uses. It runs at startup so that an
+// unwritable directory stops the process, instead of the process running
+// with every save failing and only logged.
+func (s *State) checkDataDirWritable() error {
+	dir := s.config.DataDir
+	tmpPath := s.config.StatePath() + ".tmp"
+
+	err := os.MkdirAll(dir, dirPermissions)
+	if err == nil {
+		err = os.WriteFile(tmpPath, nil, filePermissions)
+	}
+
+	if err == nil {
+		err = os.Remove(tmpPath)
+	}
+
+	if err != nil {
+		return fmt.Errorf("data directory %s is not writable: %w", dir, err)
+	}
+
+	return nil
+}

internal/watcher/cancelled_test.go

@@ -0,0 +1,81 @@
+package watcher_test
+
+import (
+	"context"
+	"log/slog"
+	"reflect"
+	"testing"
+
+	"sneak.berlin/go/dnswatcher/internal/portcheck"
+	"sneak.berlin/go/dnswatcher/internal/resolver"
+	"sneak.berlin/go/dnswatcher/internal/state"
+	"sneak.berlin/go/dnswatcher/internal/tlscheck"
+	"sneak.berlin/go/dnswatcher/internal/watcher"
+)
+
+// TestCancelledCheckSavesNothing runs a check with its context already
+// cancelled, which is how the rest of a check runs once shutdown cuts it
+// short. The real resolver drops the DNS lookup without sending a query,
+// and the real port and TLS checkers fail without connecting. The port
+// and certificate state the last check saved must stay as it was, and
+// nothing may be notified.
+func TestCancelledCheckSavesNothing(t *testing.T) {
+	t.Parallel()
+
+	cfg := defaultTestConfig(t)
+	cfg.Hostnames = []string{host}
+
+	// newTestWatcher's watcher has stand-in checkers. This one, on the
+	// same state and notifier, has the real ones.
+	_, deps := newTestWatcher(t, cfg)
+	w := watcher.NewForTest(
+		cfg,
+		deps.state,
+		resolver.NewFromLogger(slog.Default()),
+		portcheck.NewStandalone(),
+		tlscheck.NewStandalone(),
+		deps.notifier,
+	)
+
+	// The last check found host at a local address, with both ports
+	// open and a good certificate.
+	const localIP = "127.0.0.1"
+
+	deps.state.SetHostnameState(host, hostnameState(
+		map[string]map[string][]string{nsA: {"A": {localIP}}},
+	))
+
+	ports := map[string]*state.PortState{
+		localIP + ":80":  {Open: true, Hostnames: []string{host}},
+		localIP + ":443": {Open: true, Hostnames: []string{host}},
+	}
+	for key, ps := range ports {
+		deps.state.SetPortState(key, ps)
+	}
+
+	certKey := localIP + ":443:" + host
+	cert := &state.CertificateState{CommonName: host, Status: "ok"}
+	deps.state.SetCertificateState(certKey, cert)
+
+	ctx, cancel := context.WithCancel(t.Context())
+	cancel()
+
+	w.RunOnce(ctx)
+
+	for key, want := range ports {
+		got, _ := deps.state.GetPortState(key)
+		if !reflect.DeepEqual(got, want) {
+			t.Errorf("port %s saved as %+v, want %+v", key, got, want)
+		}
+	}
+
+	got, _ := deps.state.GetCertificateState(certKey)
+	if !reflect.DeepEqual(got, cert) {
+		t.Errorf("certificate saved as %+v, want %+v", got, cert)
+	}
+
+	notifications := deps.notifier.getNotifications()
+	if len(notifications) != 0 {
+		t.Errorf("sent %v, want no notifications", notifications)
+	}
+}

internal/watcher/cname_test.go

@@ -0,0 +1,336 @@
+package watcher_test
+
+import (
+	"context"
+	"log/slog"
+	"slices"
+	"testing"
+
+	"sneak.berlin/go/dnswatcher/internal/livednstest"
+	"sneak.berlin/go/dnswatcher/internal/resolver"
+	"sneak.berlin/go/dnswatcher/internal/state"
+	"sneak.berlin/go/dnswatcher/internal/watcher"
+)
+
+// TestCNAMEIntoAnotherZonePortAndTLSChecks runs the port and TLS
+// checks on hostname state built here: the name's nameserver answered
+// with a CNAME into another zone, and following it found ip1. Both
+// checks must use ip1. They look nothing up, so the watcher has no
+// resolver.
+func TestCNAMEIntoAnotherZonePortAndTLSChecks(t *testing.T) {
+	t.Parallel()
+
+	cfg := defaultTestConfig(t)
+	cfg.Hostnames = []string{host}
+
+	deps := newTestDeps(t, cfg)
+	w := watcher.NewForTest(
+		cfg, deps.state, nil,
+		deps.portChecker, deps.tlsChecker, deps.notifier,
+	)
+
+	deps.state.SetHostnameState(host, cnameState(ip1))
+
+	w.CheckAllPorts(t.Context())
+	w.RunTLSChecks(t.Context())
+
+	snap := deps.state.GetSnapshot()
+
+	ps, ok := snap.Ports[ip1+":443"]
+	if !ok || !slices.Contains(ps.Hostnames, host) {
+		t.Errorf("no port state for %s at %s:443", host, ip1)
+	}
+
+	certKey := ip1 + ":443:" + host
+	if _, ok := snap.Certificates[certKey]; !ok {
+		t.Errorf("no certificate state %s", certKey)
+	}
+}
+
+// TestCNAMEThatCannotBeFollowedKeepsPrevious runs a check of a name, not
+// the watcher's first, from the point where its records have been looked
+// up: they hold a CNAME to a target under .invalid, whose lookup fails.
+// The previous check found the same records, and oldIP at the end of the
+// CNAME. The check must keep oldIP and send nothing.
+func TestCNAMEThatCannotBeFollowedKeepsPrevious(t *testing.T) {
+	t.Parallel()
+
+	w, deps := newTestWatcher(t, defaultTestConfig(t))
+	w.SetFirstRun(false)
+
+	records := map[string]map[string][]string{
+		nsA: cnameTo("target.example.invalid."),
+	}
+
+	prev := hostnameState(records)
+	prev.CNAMEAddresses = []string{oldIP}
+	deps.state.SetHostnameState(host, prev)
+
+	// The result is the same whether or not live DNS answers, so the
+	// lookup is not retried.
+	_ = livednstest.Run(func(ctx context.Context) error {
+		w.UpdateHostnameState(ctx, host, hostnameState(records))
+
+		return nil
+	})
+
+	hs, _ := deps.state.GetHostnameState(host)
+	if !slices.Equal(hs.CNAMEAddresses, prev.CNAMEAddresses) {
+		t.Errorf(
+			"saved %v, want %v",
+			hs.CNAMEAddresses, prev.CNAMEAddresses,
+		)
+	}
+
+	notifications := deps.notifier.getNotifications()
+	if len(notifications) != 0 {
+		t.Errorf("sent %v, want no notifications", notifications)
+	}
+}
+
+// followLive follows in live DNS the CNAMEs in a name's records, built
+// from records, and returns the addresses saved for the name. The
+// previous check saved oldIP, which is kept when a target cannot be
+// followed; that is retried. The tests point CNAMEs only at names in
+// zones with two nameservers, to keep queries few (see the top of
+// watcher_test.go).
+func followLive(
+	t *testing.T,
+	records map[string]map[string][]string,
+) []string {
+	t.Helper()
+
+	w := watcher.NewForTest(
+		nil, nil, resolver.NewFromLogger(slog.Default()), nil, nil, nil,
+	)
+	prev := cnameState(oldIP)
+
+	var current *state.HostnameState
+
+	livednstest.Retry(t, "following CNAMEs", func(ctx context.Context) error {
+		current = hostnameState(records)
+
+		w.ResolveCNAMEAddresses(ctx, host, current, prev)
+
+		if slices.Equal(current.CNAMEAddresses, prev.CNAMEAddresses) {
+			return livednstest.ErrNoAnswer
+		}
+
+		return nil
+	})
+
+	return current.CNAMEAddresses
+}
+
+// TestCNAMEAddressesOfEveryTarget gives a name's two nameservers
+// different CNAME targets, as when a secondary still serves an old one.
+// The addresses at the end of both are saved, whichever answer is read
+// first: one.one.one.one has 1.1.1.1, and dns.adguard-dns.com has
+// 94.140.14.14.
+func TestCNAMEAddressesOfEveryTarget(t *testing.T) {
+	t.Parallel()
+
+	found := followLive(t, map[string]map[string][]string{
+		nsA: cnameTo("one.one.one.one."),
+		nsB: cnameTo("dns.adguard-dns.com."),
+	})
+
+	for _, ip := range []string{"1.1.1.1", "94.140.14.14"} {
+		if !slices.Contains(found, ip) {
+			t.Errorf("saved %v, want %s among them", found, ip)
+		}
+	}
+}
+
+// TestCNAMEChainEndingInNoAddressSavesEmptyList follows a CNAME to a
+// name live DNS answers with NXDOMAIN. An empty list is saved, not nil,
+// which would mean the addresses are not known.
+func TestCNAMEChainEndingInNoAddressSavesEmptyList(t *testing.T) {
+	t.Parallel()
+
+	found := followLive(t, map[string]map[string][]string{
+		nsA: cnameTo("this-surely-does-not-exist-xyz.example.org."),
+	})
+
+	if found == nil || len(found) != 0 {
+		t.Errorf("saved %#v, want an empty list", found)
+	}
+}
+
+// TestCNAMEBesideAnAddressNotFollowed gives one nameserver of a name an
+// address and another a CNAME. The CNAME is not followed: an empty list
+// is saved, not nil, and nothing is looked up, the watcher having no
+// resolver.
+func TestCNAMEBesideAnAddressNotFollowed(t *testing.T) {
+	t.Parallel()
+
+	w := watcher.NewForTest(nil, nil, nil, nil, nil, nil)
+
+	current := hostnameState(map[string]map[string][]string{
+		nsA: {"A": {ip1}},
+		nsB: cnameTo("target.example.org."),
+	})
+
+	w.ResolveCNAMEAddresses(t.Context(), host, current, nil)
+
+	if current.CNAMEAddresses == nil || len(current.CNAMEAddresses) != 0 {
+		t.Errorf("saved %#v, want an empty list", current.CNAMEAddresses)
+	}
+}
+
+// TestCNAMEWhoseNameserversAllFailedKeepsPrevious checks a name none of
+// whose nameservers answered. The addresses the previous check saved
+// from following its CNAME are kept, and nothing is looked up: the
+// watcher has no resolver.
+func TestCNAMEWhoseNameserversAllFailedKeepsPrevious(t *testing.T) {
+	t.Parallel()
+
+	w := watcher.NewForTest(nil, nil, nil, nil, nil, nil)
+
+	current := saved(map[string]*state.NameserverRecordState{
+		nsA: failed(), nsB: failed(),
+	})
+	prev := cnameState(oldIP)
+
+	w.ResolveCNAMEAddresses(t.Context(), host, current, prev)
+
+	if !slices.Equal(current.CNAMEAddresses, prev.CNAMEAddresses) {
+		t.Errorf(
+			"saved %v, want %v",
+			current.CNAMEAddresses, prev.CNAMEAddresses,
+		)
+	}
+}
+
+// cnameTo builds the records of a nameserver that answered with a CNAME
+// to target and no address.
+func cnameTo(target string) map[string][]string {
+	return map[string][]string{"CNAME": {target}}
+}
+
+// cnameState builds the state a check leaves behind for a name whose
+// nameserver answered with a CNAME and no address, when following the
+// CNAME found these addresses, which may be none.
+func cnameState(addresses ...string) *state.HostnameState {
+	hs := hostnameState(map[string]map[string][]string{
+		nsA: cnameTo("target.example.org."),
+	})
+
+	hs.CNAMEAddresses = append([]string{}, addresses...)
+
+	return hs
+}
+
+func TestCNAMEAddressChangeAlerts(t *testing.T) {
+	t.Parallel()
+
+	// A state file written before the addresses were saved loads with
+	// them nil.
+	olderStateFile := cnameState()
+	olderStateFile.CNAMEAddresses = nil
+
+	// Each case is the state saved by the previous check and by the
+	// current one. The name's records are the same in both.
+	tests := []struct {
+		name          string
+		prev, current *state.HostnameState
+		want          int
+	}{
+		{
+			"same addresses",
+			cnameState(ip1, ip2), cnameState(ip1, ip2), 0,
+		},
+		{
+			"same addresses in another order",
+			cnameState(ip2, ip1), cnameState(ip1, ip2), 0,
+		},
+		{
+			"address replaced",
+			cnameState(ip1), cnameState(ip2), 1,
+		},
+		{
+			"address added",
+			cnameState(ip1), cnameState(ip1, ip2), 1,
+		},
+		{
+			"no address at the end of the chain now",
+			cnameState(ip1), cnameState(), 1,
+		},
+		{
+			"addresses at the end of the chain again",
+			cnameState(), cnameState(ip1), 1,
+		},
+		{
+			"state file from before addresses were saved",
+			olderStateFile, cnameState(ip1), 0,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			t.Parallel()
+
+			notifier := &mockNotifier{}
+			w := watcher.NewForTest(nil, nil, nil, nil, nil, notifier)
+
+			w.DetectHostnameChanges(t.Context(), host, tt.prev, tt.current)
+
+			got := len(notifier.getNotifications())
+			if got != tt.want {
+				t.Errorf("sent %d notifications, want %d", got, tt.want)
+			}
+		})
+	}
+}
+
+func TestCNAMEAddressChangeAlertNamesHostnameAndAddresses(t *testing.T) {
+	t.Parallel()
+
+	notifier := &mockNotifier{}
+	w := watcher.NewForTest(nil, nil, nil, nil, nil, notifier)
+
+	w.DetectHostnameChanges(
+		t.Context(), host, cnameState(ip1), cnameState(ip2, ip3),
+	)
+
+	want := notification{
+		Title: "CNAME Address Change: " + host,
+		Message: "Hostname: " + host +
+			"\nOld: " + ip1 + "\nNew: " + ip2 + ", " + ip3,
+		Priority: "warning",
+	}
+
+	got := notifier.getNotifications()
+	if len(got) != 1 || got[0] != want {
+		t.Errorf("sent %v, want %v", got, want)
+	}
+}
+
+// TestNameMovedFromARecordsToCNAMEAlerts checks a name that answers
+// with an A record and then with a CNAME whose chain ends in ip2. The
+// second check is notified as a CNAME address change from no addresses,
+// beside the record change. Nothing is looked up: the watcher has no
+// resolver.
+func TestNameMovedFromARecordsToCNAMEAlerts(t *testing.T) {
+	t.Parallel()
+
+	notifier := &mockNotifier{}
+	w := watcher.NewForTest(nil, nil, nil, nil, nil, notifier)
+
+	prev := hostnameState(map[string]map[string][]string{
+		nsA: {"A": {ip1}},
+	})
+	w.ResolveCNAMEAddresses(t.Context(), host, prev, nil)
+
+	w.DetectHostnameChanges(t.Context(), host, prev, cnameState(ip2))
+
+	title := "CNAME Address Change: " + host
+	message := "Hostname: " + host + "\nOld: \nNew: " + ip2
+
+	got := notifier.getNotifications()
+	if !slices.ContainsFunc(got, func(n notification) bool {
+		return n.Title == title && n.Message == message
+	}) {
+		t.Errorf("sent %v, want %q with %q among them", got, title, message)
+	}
+}

internal/watcher/export_test.go

@@ -0,0 +1,109 @@
+package watcher
+
+import (
+	"context"
+	"log/slog"
+	"time"
+
+	"sneak.berlin/go/dnswatcher/internal/config"
+	"sneak.berlin/go/dnswatcher/internal/resolver"
+	"sneak.berlin/go/dnswatcher/internal/state"
+)
+
+// NewForTest creates a Watcher without fx for unit testing.
+func NewForTest(
+	cfg *config.Config,
+	st *state.State,
+	res DNSResolver,
+	pc PortChecker,
+	tc TLSChecker,
+	n Notifier,
+) *Watcher {
+	return &Watcher{
+		log:       slog.Default(),
+		config:    cfg,
+		state:     st,
+		resolver:  res,
+		portCheck: pc,
+		tlsCheck:  tc,
+		notify:    n,
+		firstRun:  true,
+	}
+}
+
+// NewlyDisagreeingPairs exports newlyDisagreeingPairs for testing.
+func NewlyDisagreeingPairs(
+	prev, current *state.HostnameState,
+) [][2]string {
+	return newlyDisagreeingPairs(prev, current)
+}
+
+// SetFirstRun sets whether the watcher is on its first check, in which
+// nothing is compared with the previous check. NewForTest's watcher is.
+func (w *Watcher) SetFirstRun(firstRun bool) {
+	w.firstRun = firstRun
+}
+
+// UpdateHostnameState exports updateHostnameState for testing.
+func (w *Watcher) UpdateHostnameState(
+	ctx context.Context,
+	hostname string,
+	newState *state.HostnameState,
+) {
+	w.updateHostnameState(ctx, hostname, newState)
+}
+
+// DetectHostnameChanges exports detectHostnameChanges for testing.
+func (w *Watcher) DetectHostnameChanges(
+	ctx context.Context,
+	hostname string,
+	prev, current *state.HostnameState,
+) {
+	w.detectHostnameChanges(ctx, hostname, prev, current)
+}
+
+// ResolveNameserverAddresses exports resolveNameserverAddresses for
+// testing.
+func (w *Watcher) ResolveNameserverAddresses(
+	ctx context.Context,
+	nameservers []string,
+	prev map[string][]string,
+) map[string][]string {
+	return w.resolveNameserverAddresses(ctx, nameservers, prev)
+}
+
+// ResolveCNAMEAddresses exports resolveCNAMEAddresses for testing.
+func (w *Watcher) ResolveCNAMEAddresses(
+	ctx context.Context,
+	hostname string,
+	current, prev *state.HostnameState,
+) {
+	w.resolveCNAMEAddresses(ctx, hostname, current, prev)
+}
+
+// DetectNSAddressChanges exports detectNSAddressChanges for testing.
+func (w *Watcher) DetectNSAddressChanges(
+	ctx context.Context,
+	domain string,
+	prev, current map[string][]string,
+) {
+	w.detectNSAddressChanges(ctx, domain, prev, current)
+}
+
+// CheckAllPorts exports checkAllPorts for testing.
+func (w *Watcher) CheckAllPorts(ctx context.Context) {
+	w.checkAllPorts(ctx)
+}
+
+// RunTLSChecks exports runTLSChecks for testing.
+func (w *Watcher) RunTLSChecks(ctx context.Context) {
+	w.runTLSChecks(ctx)
+}
+
+// BuildHostnameState exports buildHostnameState for testing.
+func BuildHostnameState(
+	results map[string]*resolver.NameserverResponse,
+	now time.Time,
+) *state.HostnameState {
+	return buildHostnameState(results, now)
+}

internal/watcher/inconsistency_test.go

@@ -0,0 +1,185 @@
+package watcher_test
+
+import (
+	"slices"
+	"testing"
+
+	"sneak.berlin/go/dnswatcher/internal/state"
+	"sneak.berlin/go/dnswatcher/internal/watcher"
+)
+
+const (
+	host = "www.example.net"
+	nsA  = "a.ns.example.net."
+	nsB  = "b.ns.example.net."
+	nsC  = "c.ns.example.net."
+	ip1  = "192.0.2.1"
+	ip2  = "192.0.2.2"
+	ip3  = "192.0.2.3"
+)
+
+// hostnameState builds the state a check with these records leaves behind.
+func hostnameState(
+	records map[string]map[string][]string,
+) *state.HostnameState {
+	hs := &state.HostnameState{
+		RecordsByNameserver: make(map[string]*state.NameserverRecordState),
+	}
+
+	for ns, recs := range records {
+		hs.RecordsByNameserver[ns] = &state.NameserverRecordState{
+			Records: recs,
+			Status:  "ok",
+		}
+	}
+
+	return hs
+}
+
+func TestNewlyDisagreeingPairs(t *testing.T) {
+	t.Parallel()
+
+	onlyA := map[string]map[string][]string{nsA: {"A": {ip1}}}
+	agree := map[string]map[string][]string{nsA: {"A": {ip1}}, nsB: {"A": {ip1}}}
+	disagree := map[string]map[string][]string{nsA: {"A": {ip1}}, nsB: {"A": {ip2}}}
+	alert := [][2]string{{nsA, nsB}}
+
+	// b already disagrees with a and c; then c changes, so a and c,
+	// which agreed, now differ.
+	bDiffers := map[string]map[string][]string{
+		nsA: {"A": {ip1}}, nsB: {"A": {ip2}}, nsC: {"A": {ip1}},
+	}
+	cChanges := map[string]map[string][]string{
+		nsA: {"A": {ip1}}, nsB: {"A": {ip2}}, nsC: {"A": {ip3}},
+	}
+
+	// Each case starts from the state loaded at startup and runs the
+	// checks in order; want[i] is what check i alerts for.
+	tests := []struct {
+		name   string
+		loaded map[string]map[string][]string
+		checks []map[string]map[string][]string
+		want   [][][2]string
+	}{
+		{
+			name:   "disagreement persisting across checks alerts once",
+			loaded: agree,
+			checks: []map[string]map[string][]string{disagree, disagree, disagree},
+			want:   [][][2]string{alert, nil, nil},
+		},
+		{
+			name:   "disagreement starting on a later check alerts on it",
+			loaded: agree,
+			checks: []map[string]map[string][]string{agree, agree, disagree},
+			want:   [][][2]string{nil, nil, alert},
+		},
+		{
+			name:   "disagreement in the loaded state does not alert",
+			loaded: disagree,
+			checks: []map[string]map[string][]string{disagree, disagree},
+			want:   [][][2]string{nil, nil},
+		},
+		{
+			name:   "nameserver new on the first check and disagreeing alerts once",
+			loaded: onlyA,
+			checks: []map[string]map[string][]string{disagree, disagree},
+			want:   [][][2]string{alert, nil},
+		},
+		{
+			name:   "disagreement after agreeing again alerts again",
+			loaded: agree,
+			checks: []map[string]map[string][]string{disagree, agree, disagree},
+			want:   [][][2]string{alert, nil, alert},
+		},
+		{
+			name:   "new disagreement while another nameserver differs alerts",
+			loaded: bDiffers,
+			checks: []map[string]map[string][]string{cChanges, cChanges},
+			want:   [][][2]string{{{nsA, nsC}}, nil},
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			t.Parallel()
+
+			prev := hostnameState(tt.loaded)
+
+			for i, records := range tt.checks {
+				current := hostnameState(records)
+
+				got := watcher.NewlyDisagreeingPairs(prev, current)
+				if !slices.Equal(got, tt.want[i]) {
+					t.Errorf(
+						"check %d: alerted for %v, want %v",
+						i, got, tt.want[i],
+					)
+				}
+
+				prev = current
+			}
+		})
+	}
+}
+
+func TestInconsistencyAlert(t *testing.T) {
+	t.Parallel()
+
+	onlyA := map[string]map[string][]string{nsA: {"A": {ip1}}}
+	agree := map[string]map[string][]string{nsA: {"A": {ip1}}, nsB: {"A": {ip1}}}
+	disagree := map[string]map[string][]string{nsA: {"A": {ip1}}, nsB: {"A": {ip2}}}
+
+	// Each case starts from the state loaded at startup and then sees
+	// the nameservers disagree on three checks in a row.
+	tests := []struct {
+		name   string
+		loaded map[string]map[string][]string
+		want   int
+	}{
+		{
+			name:   "disagreement lasting several checks alerts once",
+			loaded: agree,
+			want:   1,
+		},
+		{
+			name:   "disagreement in the loaded state does not alert",
+			loaded: disagree,
+			want:   0,
+		},
+		{
+			name:   "nameserver new on the first check and disagreeing alerts once",
+			loaded: onlyA,
+			want:   1,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			t.Parallel()
+
+			// The hostname change detection uses only the notifier.
+			notifier := &mockNotifier{}
+			w := watcher.NewForTest(nil, nil, nil, nil, nil, notifier)
+
+			prev := hostnameState(tt.loaded)
+
+			for range 3 {
+				current := hostnameState(disagree)
+				w.DetectHostnameChanges(t.Context(), host, prev, current)
+				prev = current
+			}
+
+			got := 0
+
+			for _, n := range notifier.getNotifications() {
+				if n.Title == "Inconsistency: "+host {
+					got++
+				}
+			}
+
+			if got != tt.want {
+				t.Errorf("sent %d inconsistency alerts, want %d", got, tt.want)
+			}
+		})
+	}
+}

internal/watcher/interfaces.go

@@ -5,6 +5,7 @@ import (
 	"context"
 
 	"sneak.berlin/go/dnswatcher/internal/portcheck"
+	"sneak.berlin/go/dnswatcher/internal/resolver"
 	"sneak.berlin/go/dnswatcher/internal/tlscheck"
 )
 
@@ -17,11 +18,11 @@ type DNSResolver interface {
 	) ([]string, error)
 
 	// LookupAllRecords queries all record types for a hostname,
-	// returning results keyed by nameserver then record type.
+	// returning each nameserver's response keyed by nameserver.
 	LookupAllRecords(
 		ctx context.Context,
 		hostname string,
-	) (map[string]map[string][]string, error)
+	) (map[string]*resolver.NameserverResponse, error)
 
 	// ResolveIPAddresses resolves a hostname to all IP addresses.
 	ResolveIPAddresses(

internal/watcher/nsaddress_test.go

@@ -0,0 +1,151 @@
+package watcher_test
+
+import (
+	"context"
+	"log/slog"
+	"reflect"
+	"testing"
+
+	"sneak.berlin/go/dnswatcher/internal/livednstest"
+	"sneak.berlin/go/dnswatcher/internal/resolver"
+	"sneak.berlin/go/dnswatcher/internal/watcher"
+)
+
+const domain = "example.net"
+
+func TestNSAddressChangeAlerts(t *testing.T) {
+	t.Parallel()
+
+	// Each case is the nameserver addresses saved by the previous check
+	// and by the current one.
+	tests := []struct {
+		name          string
+		prev, current map[string][]string
+		want          int
+	}{
+		{
+			"same addresses",
+			map[string][]string{nsA: {ip1, ip2}},
+			map[string][]string{nsA: {ip1, ip2}},
+			0,
+		},
+		{
+			"same addresses in another order",
+			map[string][]string{nsA: {ip2, ip1}},
+			map[string][]string{nsA: {ip1, ip2}},
+			0,
+		},
+		{
+			"address replaced",
+			map[string][]string{nsA: {ip1}},
+			map[string][]string{nsA: {ip2}},
+			1,
+		},
+		{
+			"address added",
+			map[string][]string{nsA: {ip1}},
+			map[string][]string{nsA: {ip1, ip2}},
+			1,
+		},
+		{
+			"two nameservers changed",
+			map[string][]string{nsA: {ip1}, nsB: {ip2}},
+			map[string][]string{nsA: {ip3}, nsB: {ip3}},
+			2,
+		},
+		{
+			"nameserver added",
+			map[string][]string{nsA: {ip1}},
+			map[string][]string{nsA: {ip1}, nsB: {ip2}},
+			0,
+		},
+		{
+			"nameserver removed",
+			map[string][]string{nsA: {ip1}, nsB: {ip2}},
+			map[string][]string{nsA: {ip1}},
+			0,
+		},
+		{
+			"state file from before addresses were saved",
+			nil,
+			map[string][]string{nsA: {ip1}, nsB: {ip2}},
+			0,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			t.Parallel()
+
+			notifier := &mockNotifier{}
+			w := watcher.NewForTest(nil, nil, nil, nil, nil, notifier)
+
+			w.DetectNSAddressChanges(t.Context(), domain, tt.prev, tt.current)
+
+			got := len(notifier.getNotifications())
+			if got != tt.want {
+				t.Errorf("sent %d address changes, want %d", got, tt.want)
+			}
+		})
+	}
+}
+
+func TestNSAddressChangeAlertNamesDomainNameserverAndAddresses(
+	t *testing.T,
+) {
+	t.Parallel()
+
+	notifier := &mockNotifier{}
+	w := watcher.NewForTest(nil, nil, nil, nil, nil, notifier)
+
+	w.DetectNSAddressChanges(
+		t.Context(), domain,
+		map[string][]string{nsA: {ip1}},
+		map[string][]string{nsA: {ip2, ip3}},
+	)
+
+	want := notification{
+		Title: "NS Address Change: " + domain,
+		Message: "Domain: " + domain + "\nNameserver: " + nsA +
+			"\nOld: " + ip1 + "\nNew: " + ip2 + ", " + ip3,
+		Priority: "warning",
+	}
+
+	got := notifier.getNotifications()
+	if len(got) != 1 || got[0] != want {
+		t.Errorf("sent %v, want %v", got, want)
+	}
+}
+
+// TestNameserverWithNoAddressKeepsPrevious looks up nameserver names
+// with no address: two under .invalid, whose lookup fails with an
+// error, and one that does not exist under a real zone, which live DNS
+// answers with no address and no error. Each one with addresses saved
+// by the previous check keeps them; the one without gets none.
+func TestNameserverWithNoAddressKeepsPrevious(t *testing.T) {
+	t.Parallel()
+
+	w := watcher.NewForTest(
+		nil, nil, resolver.NewFromLogger(slog.Default()), nil, nil, nil,
+	)
+
+	nonexistentNS := "this-surely-does-not-exist-xyz." + testSmallDomain + "."
+
+	prev := map[string][]string{oldNS1: {oldIP}, nonexistentNS: {oldIP}}
+
+	var got map[string][]string
+
+	// The result is the same whether or not live DNS answers, so the
+	// lookup is not retried.
+	_ = livednstest.Run(func(ctx context.Context) error {
+		got = w.ResolveNameserverAddresses(
+			ctx, []string{oldNS1, oldNS2, nonexistentNS}, prev,
+		)
+
+		return nil
+	})
+
+	if !reflect.DeepEqual(got, prev) {
+		t.Errorf("saved %v, want %v", got, prev)
+	}
+}

internal/watcher/nsfailure_test.go

@@ -0,0 +1,448 @@
+package watcher_test
+
+import (
+	"context"
+	"fmt"
+	"log/slog"
+	"strings"
+	"testing"
+	"time"
+
+	"sneak.berlin/go/dnswatcher/internal/livednstest"
+	"sneak.berlin/go/dnswatcher/internal/resolver"
+	"sneak.berlin/go/dnswatcher/internal/state"
+	"sneak.berlin/go/dnswatcher/internal/watcher"
+)
+
+// answered is what a check saves for a nameserver that answered with
+// these records.
+func answered(records map[string][]string) *state.NameserverRecordState {
+	return &state.NameserverRecordState{Records: records, Status: "ok"}
+}
+
+// failed is what a check saves for a nameserver that did not answer.
+func failed() *state.NameserverRecordState {
+	return &state.NameserverRecordState{
+		Records: map[string][]string{},
+		Status:  "error",
+		Error:   "all queries timed out",
+	}
+}
+
+// saved builds the hostname state a check saves.
+func saved(
+	byNameserver map[string]*state.NameserverRecordState,
+) *state.HostnameState {
+	return &state.HostnameState{RecordsByNameserver: byNameserver}
+}
+
+// alertCounts counts the hostname alerts sent, by kind.
+type alertCounts struct {
+	failures, recoveries, recordChanges, inconsistencies int
+}
+
+// countAlerts runs the hostname change detection from the state loaded
+// at startup through each check in turn, and counts the alerts sent.
+func countAlerts(
+	t *testing.T,
+	loaded *state.HostnameState,
+	checks []*state.HostnameState,
+) alertCounts {
+	t.Helper()
+
+	// The hostname change detection uses only the notifier.
+	notifier := &mockNotifier{}
+	w := watcher.NewForTest(nil, nil, nil, nil, nil, notifier)
+
+	prev := loaded
+
+	for _, current := range checks {
+		w.DetectHostnameChanges(t.Context(), host, prev, current)
+		prev = current
+	}
+
+	var got alertCounts
+
+	for _, n := range notifier.getNotifications() {
+		kind, _, _ := strings.Cut(n.Title, ":")
+
+		switch kind {
+		case "NS Failure":
+			got.failures++
+		case "NS Recovery":
+			got.recoveries++
+		case "Record Change":
+			got.recordChanges++
+		case "Inconsistency":
+			got.inconsistencies++
+		}
+	}
+
+	return got
+}
+
+func TestNSFailureAndRecoveryAlerts(t *testing.T) {
+	t.Parallel()
+
+	records := map[string][]string{"A": {ip1}}
+
+	bothAnswer := saved(map[string]*state.NameserverRecordState{
+		nsA: answered(records), nsB: answered(records),
+	})
+	bFails := saved(map[string]*state.NameserverRecordState{
+		nsA: answered(records), nsB: failed(),
+	})
+	onlyA := saved(map[string]*state.NameserverRecordState{
+		nsA: answered(records),
+	})
+	bAnswersNoRecords := saved(map[string]*state.NameserverRecordState{
+		nsA: answered(records), nsB: answered(map[string][]string{}),
+	})
+	bAnswersDifferently := saved(map[string]*state.NameserverRecordState{
+		nsA: answered(records), nsB: answered(map[string][]string{"A": {ip2}}),
+	})
+
+	// Each case starts from the state loaded at startup and runs the
+	// checks in order.
+	tests := []struct {
+		name   string
+		loaded *state.HostnameState
+		checks []*state.HostnameState
+		want   alertCounts
+	}{
+		{
+			"failure lasting several checks alerts once",
+			bothAnswer, []*state.HostnameState{bFails, bFails, bFails},
+			alertCounts{failures: 1},
+		},
+		{
+			"recovery alerts once",
+			bFails, []*state.HostnameState{bothAnswer, bothAnswer},
+			alertCounts{recoveries: 1},
+		},
+		{
+			"failing again after recovering alerts again",
+			bothAnswer, []*state.HostnameState{bFails, bothAnswer, bFails},
+			alertCounts{failures: 2, recoveries: 1},
+		},
+		{
+			"nameserver failing when first seen does not alert",
+			onlyA, []*state.HostnameState{bFails, bFails},
+			alertCounts{},
+		},
+		{
+			"answer with no records is a record change, not a failure",
+			bothAnswer, []*state.HostnameState{bAnswersNoRecords},
+			alertCounts{recordChanges: 1, inconsistencies: 1},
+		},
+		{
+			"recovered nameserver that answers differently disagrees",
+			bFails, []*state.HostnameState{bAnswersDifferently},
+			alertCounts{recoveries: 1, inconsistencies: 1},
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			t.Parallel()
+
+			got := countAlerts(t, tt.loaded, tt.checks)
+			if got != tt.want {
+				t.Errorf("sent %+v, want %+v", got, tt.want)
+			}
+		})
+	}
+}
+
+func TestNSFailureAlertNamesHostnameNameserverAndReason(t *testing.T) {
+	t.Parallel()
+
+	records := map[string][]string{"A": {ip1}}
+
+	notifier := &mockNotifier{}
+	w := watcher.NewForTest(nil, nil, nil, nil, nil, notifier)
+
+	w.DetectHostnameChanges(
+		t.Context(), host,
+		saved(map[string]*state.NameserverRecordState{nsA: answered(records)}),
+		saved(map[string]*state.NameserverRecordState{nsA: failed()}),
+	)
+
+	notifications := notifier.getNotifications()
+	if len(notifications) != 1 {
+		t.Fatalf("sent %v, want one NS Failure", notifications)
+	}
+
+	msg := notifications[0].Message
+	if !strings.Contains(msg, host) || !strings.Contains(msg, nsA) ||
+		!strings.Contains(msg, failed().Error) {
+		t.Errorf(
+			"message %q does not name %s, %s and the reason",
+			msg, host, nsA,
+		)
+	}
+}
+
+// TestNameserverThatNeverAnswers asks a nameserver address where
+// nothing answers, 192.0.2.1, and checks what the watcher saves for it.
+// The deadline outlasts the resolver's first two-second try, as in the
+// resolver's timeout test.
+func TestNameserverThatNeverAnswers(t *testing.T) {
+	t.Parallel()
+
+	ctx, cancel := context.WithTimeout(t.Context(), 3*time.Second)
+	t.Cleanup(cancel)
+
+	res := resolver.NewFromLogger(slog.Default())
+
+	resp, err := res.QueryNameserverIP(ctx, nsA, "192.0.2.1", host)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	hs := watcher.BuildHostnameState(
+		map[string]*resolver.NameserverResponse{nsA: resp}, time.Now(),
+	)
+
+	got := hs.RecordsByNameserver[nsA]
+	if got.Status != "error" || got.Error == "" {
+		t.Errorf(
+			"saved status %q, error %q; want status error with a reason",
+			got.Status, got.Error,
+		)
+	}
+}
+
+// TestNameserverThatAnswersNXDOMAIN asks a real nameserver about a name
+// that does not exist and checks what the watcher saves for it: NXDOMAIN
+// is an answer, so the nameserver is saved as ok with no error.
+func TestNameserverThatAnswersNXDOMAIN(t *testing.T) {
+	t.Parallel()
+
+	res := resolver.NewFromLogger(slog.Default())
+	name := "this-surely-does-not-exist-xyz." + testDomain
+
+	var (
+		ns   string
+		resp *resolver.NameserverResponse
+	)
+
+	livednstest.Retry(t, "QueryNameserver("+name+")", func(ctx context.Context) error {
+		nameservers, err := res.LookupNS(ctx, testDomain)
+		if err != nil {
+			return err
+		}
+
+		ns = nameservers[0]
+
+		resp, err = res.QueryNameserver(ctx, ns, name)
+		if err != nil {
+			return err
+		}
+
+		// A timeout or a failure is no answer to check.
+		if resp.Status == resolver.StatusTimeout ||
+			resp.Status == resolver.StatusError {
+			return fmt.Errorf(
+				"%w: %s: %s", livednstest.ErrNoAnswer, ns, resp.Error,
+			)
+		}
+
+		return nil
+	})
+
+	if resp.Status != resolver.StatusNXDomain {
+		t.Fatalf("%s answered %q for %s, want NXDOMAIN", ns, resp.Status, name)
+	}
+
+	hs := watcher.BuildHostnameState(
+		map[string]*resolver.NameserverResponse{ns: resp}, time.Now(),
+	)
+
+	got := hs.RecordsByNameserver[ns]
+	if got.Status != "ok" || got.Error != "" {
+		t.Errorf(
+			"saved status %q, error %q; want status ok with no error",
+			got.Status, got.Error,
+		)
+	}
+}
+
+// TestNameserverThatRefuses asks a google.com nameserver about
+// cloudflare.com, a zone it does not serve, which it refuses, and checks
+// what the watcher saves for it: REFUSED is no answer, so the nameserver
+// is saved as error with the reason.
+func TestNameserverThatRefuses(t *testing.T) {
+	t.Parallel()
+
+	const reason = "server returned REFUSED"
+
+	res := resolver.NewFromLogger(slog.Default())
+
+	var (
+		ns   string
+		resp *resolver.NameserverResponse
+	)
+
+	livednstest.Retry(
+		t,
+		"QueryNameserver(cloudflare.com)",
+		func(ctx context.Context) error {
+			nameservers, err := res.LookupNS(ctx, testDomain)
+			if err != nil {
+				return err
+			}
+
+			ns = nameservers[0]
+
+			resp, err = res.QueryNameserver(ctx, ns, "cloudflare.com")
+			if err != nil {
+				return err
+			}
+
+			// A timeout or a network error is no reply at all.
+			if resp.Status == resolver.StatusTimeout ||
+				strings.HasPrefix(resp.Error, "network error") {
+				return fmt.Errorf(
+					"%w: %s: %s", livednstest.ErrNoAnswer, ns, resp.Error,
+				)
+			}
+
+			return nil
+		},
+	)
+
+	if resp.Error != reason {
+		t.Fatalf(
+			"%s answered %q (%s) for cloudflare.com, want REFUSED",
+			ns, resp.Status, resp.Error,
+		)
+	}
+
+	hs := watcher.BuildHostnameState(
+		map[string]*resolver.NameserverResponse{ns: resp}, time.Now(),
+	)
+
+	got := hs.RecordsByNameserver[ns]
+	if got.Status != failed().Status || got.Error != reason {
+		t.Errorf(
+			"saved status %q, error %q; want status %q, error %q",
+			got.Status, got.Error, failed().Status, reason,
+		)
+	}
+}
+
+// TestPortStateWhenNoNameserverAnswered runs the port checks on
+// hostname state built here, which gives the name no address. The port
+// state saved for its old address is kept only when the name is a
+// configured hostname or domain and none of its nameservers answered.
+func TestPortStateWhenNoNameserverAnswered(t *testing.T) {
+	t.Parallel()
+
+	noneAnswered := saved(map[string]*state.NameserverRecordState{
+		nsA: failed(), nsB: failed(),
+	})
+	oneAnsweredNoAddress := saved(map[string]*state.NameserverRecordState{
+		nsA: answered(map[string][]string{}), nsB: failed(),
+	})
+
+	configured := []string{host}
+
+	tests := []struct {
+		name      string
+		hostname  *state.HostnameState
+		hostnames []string
+		domains   []string
+		wantKept  bool
+	}{
+		{"no nameserver answered", noneAnswered, configured, nil, true},
+		{
+			"no nameserver answered, configured as a domain",
+			noneAnswered, nil, configured, true,
+		},
+		{
+			"one answered with no address",
+			oneAnsweredNoAddress, configured, nil, false,
+		},
+		{"no nameserver answered, not configured", noneAnswered, nil, nil, false},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			t.Parallel()
+
+			cfg := defaultTestConfig(t)
+			cfg.Hostnames = tt.hostnames
+			cfg.Domains = tt.domains
+
+			// The port checks read the saved hostname state and look
+			// nothing up, so the watcher has no resolver.
+			deps := newTestDeps(t, cfg)
+			w := watcher.NewForTest(
+				cfg, deps.state, nil,
+				deps.portChecker, deps.tlsChecker, deps.notifier,
+			)
+
+			key := ip1 + ":443"
+
+			deps.state.SetHostnameState(host, tt.hostname)
+			deps.state.SetPortState(key, &state.PortState{
+				Open: true, Hostnames: []string{host},
+			})
+
+			w.CheckAllPorts(t.Context())
+
+			_, kept := deps.state.GetPortState(key)
+			if kept != tt.wantKept {
+				t.Errorf("port state %s kept: %v, want %v", key, kept, tt.wantKept)
+			}
+		})
+	}
+}
+
+// TestPortStateWhenNoNameserverAnsweredAndOtherNameMovesAway saves the
+// port state of an address two configured hostnames resolve to. While
+// none of the first one's nameservers answer, the port checks run with
+// the other one still at that address, then after it moved away; the
+// port state is kept both times.
+func TestPortStateWhenNoNameserverAnsweredAndOtherNameMovesAway(
+	t *testing.T,
+) {
+	t.Parallel()
+
+	const other = "mail.example.net"
+
+	cfg := defaultTestConfig(t)
+	cfg.Hostnames = []string{host, other}
+
+	// The port checks read the saved hostname state and look nothing
+	// up, so the watcher has no resolver.
+	deps := newTestDeps(t, cfg)
+	w := watcher.NewForTest(
+		cfg, deps.state, nil,
+		deps.portChecker, deps.tlsChecker, deps.notifier,
+	)
+
+	key := ip1 + ":443"
+
+	deps.state.SetPortState(key, &state.PortState{
+		Open: true, Hostnames: []string{host, other},
+	})
+	deps.state.SetHostnameState(host, saved(
+		map[string]*state.NameserverRecordState{nsA: failed(), nsB: failed()},
+	))
+
+	for _, otherIP := range []string{ip1, ip2} {
+		deps.state.SetHostnameState(other, saved(
+			map[string]*state.NameserverRecordState{
+				nsA: answered(map[string][]string{"A": {otherIP}}),
+			},
+		))
+
+		w.CheckAllPorts(t.Context())
+
+		if _, kept := deps.state.GetPortState(key); !kept {
+			t.Fatalf("port state %s removed with %s at %s", key, other, otherIP)
+		}
+	}
+}

package.json

@@ -0,0 +1,9 @@
+{
+    "name": "dnswatcher-tooling",
+    "version": "0.0.0",
+    "private": true,
+    "description": "Pins the prettier that script/fmt and script/fmt-check run against this repo's markdown. Not a JavaScript project; nothing here is imported, published, or shipped.",
+    "devDependencies": {
+        "prettier": "3.9.6"
+    }
+}

script/bootstrap

@@ -0,0 +1,83 @@
+#!/bin/sh
+# script/bootstrap: install all dependencies needed to build and develop
+# this repo. Idempotent: every install is guarded by a check so already
+# installed tools are skipped. Base tooling comes from nix, apt, brew,
+# or apk (detected in that order); assumes nothing is present.
+# goimports is not installed here: script/fmt and script/fmt-check-go
+# run it with `go run` at a pinned commit.
+# The linter is NOT installed here: golangci-lint runs via docker only
+# (script/lint), pinned by image digest, so its only prerequisite is a
+# working docker. Nor is prettier: script/fmt and
+# script/fmt-check-markdown run it in a container from Dockerfile.fmt.
+set -eu
+
+ROOT="$(cd "$(dirname "$0")/.." && pwd -P)"
+
+PKGMGR=""
+SUDO=""
+APT_UPDATED=""
+
+detect_pkgmgr() {
+    [ -n "$PKGMGR" ] && return 0
+    if command -v nix-env >/dev/null 2>&1; then
+        PKGMGR="nix"
+    elif command -v apt-get >/dev/null 2>&1; then
+        PKGMGR="apt"
+    elif command -v brew >/dev/null 2>&1; then
+        PKGMGR="brew"
+    elif command -v apk >/dev/null 2>&1; then
+        PKGMGR="apk"
+    else
+        echo "bootstrap: no supported package manager (nix, apt, brew, apk)" >&2
+        exit 1
+    fi
+    if [ "$PKGMGR" = "apt" ]; then
+        export DEBIAN_FRONTEND=noninteractive
+        if [ "$(id -u)" != "0" ]; then
+            SUDO="sudo"
+        fi
+    fi
+}
+
+# pkg_install <nix-attr> <apt-pkg> <brew-formula> <apk-pkg>
+pkg_install() {
+    detect_pkgmgr
+    case "$PKGMGR" in
+        nix) nix-env -iA "nixpkgs.$1" ;;
+        apt)
+            if [ -z "$APT_UPDATED" ]; then
+                $SUDO env DEBIAN_FRONTEND=noninteractive apt-get update
+                APT_UPDATED=1
+            fi
+            $SUDO env DEBIAN_FRONTEND=noninteractive apt-get install -y "$2"
+            ;;
+        brew) brew install "$3" ;;
+        apk) apk add --no-cache "$4" ;;
+    esac
+}
+
+missing() {
+    ! command -v "$1" >/dev/null 2>&1
+}
+
+main() {
+    cd "$ROOT"
+
+    if missing git; then pkg_install git git git git; fi
+    if missing make; then pkg_install gnumake make make make; fi
+    if missing go; then pkg_install go golang go go; fi
+
+    # Linting and the markdown formatter run via docker only. Warn,
+    # don't fail: building and testing work without it.
+    if missing docker; then
+        echo "bootstrap: WARNING: docker not found; install it to" \
+            "run make lint, make fmt, make fmt-check, make check" \
+            "and make docker." >&2
+    fi
+
+    go mod download
+
+    echo "bootstrap complete"
+}
+
+main "$@"

script/check

@@ -0,0 +1,14 @@
+#!/bin/sh
+# script/check: run all checks (test, lint, fmt-check). Our own
+# extension to scripts-to-rule-them-all. Must not modify any files.
+set -eu
+
+SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd -P)"
+
+main() {
+    "$SCRIPT_DIR/test"
+    "$SCRIPT_DIR/lint"
+    "$SCRIPT_DIR/fmt-check"
+}
+
+main "$@"

script/cibuild

@@ -0,0 +1,23 @@
+#!/bin/sh
+# script/cibuild: run the CI build. The Dockerfile's lint stage runs
+# the Go half of make fmt-check and golangci-lint; its builder stage
+# runs make test and make build. The markdown half of make fmt-check
+# runs after that build, as its own build of Dockerfile.fmt, because
+# there is no docker inside a docker build.
+#
+# --no-cache-filter=lint,builder runs both stages on every invocation;
+# otherwise an unchanged tree is served from the layer cache and passes
+# without linting or querying live DNS. script/fmt-check-markdown busts
+# its own cache the same way.
+set -eu
+
+SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd -P)"
+ROOT="$(cd "$SCRIPT_DIR/.." && pwd -P)"
+
+main() {
+    cd "$ROOT"
+    docker build --no-cache-filter=lint,builder .
+    "$SCRIPT_DIR/fmt-check-markdown"
+}
+
+main "$@"

script/docker

@@ -0,0 +1,26 @@
+#!/bin/sh
+# script/docker: build the Docker image tagged with the project name.
+# The tag comes from script/projectname.
+#
+# --no-cache-filter=lint,builder runs the lint stage and the builder
+# stage (make test) on every invocation; otherwise an unchanged tree is
+# served from the layer cache without linting or querying live DNS.
+set -eu
+
+SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd -P)"
+ROOT="$(cd "$SCRIPT_DIR/.." && pwd -P)"
+
+main() {
+    cd "$ROOT"
+    # Own line: a failing command substitution inside an argument does
+    # not trip `set -e`, so the inline form degrades silently to an
+    # empty constant. The VERSION build arg takes precedence over what
+    # the build would derive from the .git in its context.
+    version="$(git describe --tags --always --dirty 2>/dev/null || true)"
+    [ -n "$version" ] || version="unknown"
+    docker build --no-cache-filter=lint,builder \
+        --build-arg VERSION="$version" \
+        -t "$("$SCRIPT_DIR/projectname")" .
+}
+
+main "$@"

script/fmt

@@ -0,0 +1,65 @@
+#!/bin/sh
+# script/fmt: format all files (writes). Go with gofmt and goimports on
+# the host, markdown with the prettier pinned by Dockerfile.fmt.
+#
+# goimports runs with `go run` at a pinned commit, never from PATH, so
+# every machine formats with the same version and nothing installs it.
+#
+# The markdown pass is a `docker build --output type=local` rather than a
+# `docker run -v`, so it needs no bind mount and behaves the same against
+# a remote daemon; the formatted documents come back out of the build and
+# are copied over the tree here.
+#
+# Unlike script/fmt-check-markdown this does not bust the cache: it is
+# not a gate, and any edit to a document changes the COPY layer above the
+# prettier step, so a cached result is a result over this exact tree.
+set -eu
+
+ROOT="$(cd "$(dirname "$0")/.." && pwd -P)"
+
+# goimports v0.42.0, 2026-08-07. Must match script/fmt-check-go.
+GOIMPORTS_REF="golang.org/x/tools/cmd/goimports@009367f5c17a8d4c45a961a3a509277190a9a6f0"
+
+# Must match the export stage name in Dockerfile.fmt.
+stage=fmt-out
+
+die() {
+    echo "script/fmt: $*" >&2
+    exit 1
+}
+
+main() {
+    cd "$ROOT"
+    gofmt -s -w .
+    go run "$GOIMPORTS_REF" -w .
+
+    tmp="$(mktemp -d "${TMPDIR:-/tmp}/dnswatcher-fmt.XXXXXX")"
+    trap 'rm -rf "$tmp"' EXIT INT TERM
+
+    docker build \
+        --target "$stage" \
+        --output "type=local,dest=$tmp/out" \
+        -f Dockerfile.fmt .
+
+    # An empty export means prettier was handed nothing, which must not
+    # read as "already formatted".
+    (cd "$tmp/out" && find . -type f -name '*.md') |
+        sed 's|^\./||' | LC_ALL=C sort >"$tmp/files"
+
+    [ -s "$tmp/files" ] ||
+        die "the formatting build produced no markdown; the build" \
+            "context reached prettier empty"
+
+    # Copied only where the bytes differ, so an already-formatted tree
+    # keeps its timestamps and says nothing.
+    while IFS= read -r f; do
+        [ -n "$f" ] || continue
+        if [ -f "$f" ] && cmp -s "$tmp/out/$f" "$f"; then
+            continue
+        fi
+        cp "$tmp/out/$f" "$f"
+        echo "prettier: reformatted $f"
+    done <"$tmp/files"
+}
+
+main "$@"

script/fmt-check

@@ -0,0 +1,14 @@
+#!/bin/sh
+# script/fmt-check: check formatting (read-only). Same tools and scope
+# as script/fmt, but fails instead of writing: the Go on the host, the
+# markdown with prettier in a container.
+set -eu
+
+SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd -P)"
+
+main() {
+    "$SCRIPT_DIR/fmt-check-go"
+    "$SCRIPT_DIR/fmt-check-markdown"
+}
+
+main "$@"

script/fmt-check-go

@@ -0,0 +1,31 @@
+#!/bin/sh
+# script/fmt-check-go: fail unless every Go source is formatted the way
+# script/fmt would leave it, and name the files that are not. Read-only.
+#
+# Its own script because the Dockerfile's lint stage runs this half
+# alone: there is no docker inside a docker build to run the markdown
+# half in.
+set -eu
+
+ROOT="$(cd "$(dirname "$0")/.." && pwd -P)"
+
+# goimports v0.42.0, 2026-08-07. Must match script/fmt.
+GOIMPORTS_REF="golang.org/x/tools/cmd/goimports@009367f5c17a8d4c45a961a3a509277190a9a6f0"
+
+main() {
+    cd "$ROOT"
+    files="$(gofmt -s -l .)"
+    if [ -n "$files" ]; then
+        echo "gofmt: files not formatted:" >&2
+        echo "$files" >&2
+        exit 1
+    fi
+    files="$(go run "$GOIMPORTS_REF" -l .)"
+    if [ -n "$files" ]; then
+        echo "goimports: files not formatted:" >&2
+        echo "$files" >&2
+        exit 1
+    fi
+}
+
+main "$@"

script/fmt-check-markdown

@@ -0,0 +1,29 @@
+#!/bin/sh
+# script/fmt-check-markdown: fail unless every .md is formatted the way
+# script/fmt would leave it. Read-only.
+#
+# prettier is never installed on the host: it runs in a container built
+# from Dockerfile.fmt, pinned by package.json and yarn.lock.
+# --no-cache-filter is here for the reason script/lint gives: a cached
+# build checks nothing.
+#
+# Its own script because script/cibuild runs this half alone, after the
+# Dockerfile's lint stage has checked the Go.
+set -eu
+
+ROOT="$(cd "$(dirname "$0")/.." && pwd -P)"
+
+# Must match the markdown check stage name in Dockerfile.fmt.
+stage=fmt-check
+
+main() {
+    cd "$ROOT"
+    docker build \
+        --progress=plain \
+        --no-cache-filter="$stage" \
+        --target "$stage" \
+        -f Dockerfile.fmt \
+        .
+}
+
+main "$@"

script/install-precommit

@@ -0,0 +1,29 @@
+#!/bin/sh
+# script/install-precommit: install the git pre-commit hook that runs
+# script/precommit. Our own extension to scripts-to-rule-them-all.
+set -eu
+
+ROOT="$(cd "$(dirname "$0")/.." && pwd -P)"
+
+main() {
+    cd "$ROOT"
+    # Stop if this directory is not the top of its own git checkout, for
+    # example a copy inside another repository, whose hook must not be
+    # replaced.
+    if [ "$(git rev-parse --show-toplevel)" != "$ROOT" ]; then
+        echo "install-precommit: $ROOT is not the top of a git checkout" >&2
+        exit 1
+    fi
+    # Ask git for the repository's own git directory: .git is a file, not
+    # a directory, in some checkouts (for example a clone made with
+    # --separate-git-dir). core.hooksPath is deliberately not followed, so
+    # the hook is never written outside this repository.
+    hooks="$(git rev-parse --git-common-dir)/hooks"
+    mkdir -p "$hooks"
+    hook="$hooks/pre-commit"
+    printf '#!/bin/sh\nset -e\nscript/precommit\n' > "$hook"
+    chmod +x "$hook"
+    echo "pre-commit hook installed: runs script/precommit"
+}
+
+main "$@"

script/lint

@@ -0,0 +1,28 @@
+#!/bin/sh
+# script/lint: run the linter. golangci-lint is never installed or run
+# on the host: it runs via docker only, one way, everywhere. This
+# builds Dockerfile.lint, which COPYs the repo into the digest-pinned
+# golangci-lint image and lints as a build step, so a successful build
+# means a clean lint.
+#
+# --no-cache-filter=lint forces the lint stage (source copy + linter
+# run) to execute on every invocation. Without it an unchanged tree
+# returns success in well under a second having linted nothing. The
+# deps stage (base image + go mod download) stays cached, and no global
+# cache invalidation is performed. --progress=plain keeps the linter's
+# own output visible.
+set -eu
+
+ROOT="$(cd "$(dirname "$0")/.." && pwd -P)"
+
+main() {
+    cd "$ROOT"
+    docker build \
+        --progress=plain \
+        --no-cache-filter=lint \
+        --target lint \
+        -f Dockerfile.lint \
+        .
+}
+
+main "$@"

script/precommit

@@ -0,0 +1,21 @@
+#!/bin/sh
+# script/precommit: run by the git pre-commit hook; fails the commit if
+# checks fail. Our own extension to scripts-to-rule-them-all. Go extra:
+# go mod tidy must be a no-op before the checks run.
+set -eu
+
+SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd -P)"
+ROOT="$(cd "$SCRIPT_DIR/.." && pwd -P)"
+
+main() {
+    cd "$ROOT"
+    go mod tidy
+    if ! git diff --exit-code -- go.mod go.sum; then
+        echo "precommit: go mod tidy changed go.mod/go.sum;" \
+            "stage the changes and retry" >&2
+        exit 1
+    fi
+    "$SCRIPT_DIR/check"
+}
+
+main "$@"

script/projectname

@@ -0,0 +1,12 @@
+#!/bin/sh
+# script/projectname: output the name of this project. Our own
+# extension to scripts-to-rule-them-all. Other scripts that need the
+# name (e.g. script/docker) call this, so they can stay identical
+# across all repos.
+set -eu
+
+main() {
+    echo "dnswatcher"
+}
+
+main "$@"

script/setup

@@ -0,0 +1,13 @@
+#!/bin/sh
+# script/setup: set up the repo for development after a fresh clone:
+# installs dependencies and the git pre-commit hook.
+set -eu
+
+SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd -P)"
+
+main() {
+    "$SCRIPT_DIR/bootstrap"
+    "$SCRIPT_DIR/install-precommit"
+}
+
+main "$@"

script/test

@@ -0,0 +1,35 @@
+#!/bin/sh
+# script/test: run the test suite.
+#
+# -count=1 disables Go's test cache, and is load-bearing here. This
+# suite queries live DNS on every run by policy (TESTING.md); a cached
+# result is a replay of an earlier run's output with no query made at
+# all. On an unchanged tree the whole suite would return success in
+# under a second having resolved nothing, which makes the repeated-run
+# green that is used as evidence for flakiness fixes worthless. Do not
+# remove it.
+#
+# Conditional verbose rerun per REPO_POLICIES.md: run quiet first so
+# CI and docker build logs stay readable, and rerun with -v only on
+# failure. The rerun also carries -count=1 (a cached replay of the
+# failure would show nothing new), and the exit status is forced to 1
+# no matter how the rerun ends: the first failure already proved the
+# suite broken, so a flaky test that passes the second time must not
+# turn the build green.
+#
+# -timeout 90s is a deliberate backstop above the 60s hard cap on
+# suite duration. Do not lower it.
+set -eu
+
+ROOT="$(cd "$(dirname "$0")/.." && pwd -P)"
+
+main() {
+    cd "$ROOT"
+    go test -count=1 -race -timeout 90s -cover ./... || {
+        echo "--- Rerunning with -v for details ---" >&2
+        go test -count=1 -race -timeout 90s -v ./... || true
+        exit 1
+    }
+}
+
+main "$@"

static/static.go

@@ -0,0 +1,10 @@
+// Package static provides embedded static assets.
+package static
+
+import "embed"
+
+// Static contains the embedded static assets (CSS, JS) served
+// at the /s/ URL prefix.
+//
+//go:embed css
+var Static embed.FS

yarn.lock

@@ -0,0 +1,8 @@
+# THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY.
+# yarn lockfile v1
+
+
+prettier@3.9.6:
+  version "3.9.6"
+  resolved "https://registry.yarnpkg.com/prettier/-/prettier-3.9.6.tgz#b3ea5146515d40fc53f18aa63f74dfab1e10dbf6"
+  integrity sha512-OpN0zzVdiaiAhxpuuj5efpIS4sY9j7bY6uR5mnj5yPzGkdkjNKSJeUThPb60Jw29QuAZgA4o+/iB49kFiaBX6g==