diff --git a/.dockerignore b/.dockerignore index bae18ae..322166a 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,6 +1,75 @@ -.git/ -bin/ -*.md -LICENSE -.editorconfig -.gitignore +# .dockerignore does NOT use .gitignore semantics. Docker matches with +# moby/patternmatcher: filepath.Match plus `**`, so `*` does not cross +# `/` and an unprefixed pattern is anchored at the context root. Every +# depth-independent pattern therefore needs `**/`, or `config/.env` and +# `certs/server.key` still ship while this file reads as solved. Only +# genuinely root-anchored entries go unprefixed. Never transplant these +# into .gitignore, where `**/` is wrong. +# +# Matching is case-sensitive, so secrets use character ranges rather +# than an ALL-CAPS twin, which would still miss `Server.Key`. +# +# Extend with this repo's own host-built artifacts, written anchored: +# `/myapp`, never `**/myapp`, which also matches `cmd/myapp/` and +# deletes the package directory from the context. + +# .git is sent without its config. Without a VERSION build argument the +# stage that compiles runs `git describe --tags --always` on .git, which +# does not need .git/config; that file can hold a credential, such as a +# password in a remote URL or the token the CI checkout step stores there. +# Each submodule keeps a config with the same exposure in its git directory +# under .git/modules/, nested again for a submodule's own submodules, or in +# its own .git directory when it keeps one. +# KNOWN GAP: a submodule whose name has a `config` segment (`config`, +# `deploy/config`, `config/lib`) loses its whole git directory, because +# `**/.git/modules/**/config` also matches that segment's directory +# under .git/modules/. Go's version stamping then fails the build; +# nothing leaks. Name such a submodule without that segment: +# `git submodule add --name`. +**/.git/config +**/.git/modules/**/config + +# Agent scratch: one full checkout of the repo per in-flight agent. +# Anchored because it occurs once where agents run at the repo root. +# KNOWN GAP: a repo running agents in subdirectories still ships +# `services/api/.claude/` and must add its own anchored entry. +.claude + +# Environment files. `*.env` covers bare `.env` and the `prod.env` +# convention. Re-include a committed template with a negation if the +# build needs one: `!docs/example.env`. +**/*.[eE][nN][vV] +**/.[eE][nN][vV].* +**/.[eE][nN][vV][rR][cC] + +# Private keys and the bundles carrying them. Public certificates +# (*.crt, *.cer) are deliberately absent: they are legitimate inputs. +**/*.[pP][eE][mM] +**/*.[kK][eE][yY] +**/*.[pP]12 +**/*.[pP][fF][xX] +**/[iI][dD]_[rR][sS][aA] +**/[iI][dD]_[dD][sS][aA] +**/[iI][dD]_[eE][cC][dD][sS][aA] +**/[iI][dD]_[eE][cC][dD][sS][aA]_[sS][kK] +**/[iI][dD]_[eE][dD]25519 +**/[iI][dD]_[eE][dD]25519_[sS][kK] + +# Dependencies: restored inside the image, never copied in. +**/node_modules + +# OS metadata. +**/.DS_Store +**/Thumbs.db + +# Editor state: never a build input, and it churns COPY. +**/*.swp +**/*.swo +**/*~ +**/*.bak +**/.idea +**/.vscode +**/*.sublime-* + +# This repository's host-built artifacts: `make build` writes bin/. +/bin diff --git a/.editorconfig b/.editorconfig index 2fe0ce0..92ec261 100644 --- a/.editorconfig +++ b/.editorconfig @@ -10,3 +10,6 @@ insert_final_newline = true [Makefile] indent_style = tab + +[*.go] +indent_style = tab diff --git a/.gitea/workflows/check.yml b/.gitea/workflows/check.yml index a55bc55..78de800 100644 --- a/.gitea/workflows/check.yml +++ b/.gitea/workflows/check.yml @@ -1,9 +1,17 @@ name: check on: [push] +# A new push to a branch cancels that branch's older run, queued or running; +# runs on other branches, `next` and `main` among them, are left alone. +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true jobs: check: runs-on: ubuntu-latest steps: - # actions/checkout v4.2.2, 2026-02-28 + # actions/checkout v4.2.2, 2026-02-22 - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 + # script/cibuild needs no token, so none is left in .git/config. + with: + persist-credentials: false - run: script/cibuild diff --git a/.gitignore b/.gitignore index 9c22cd8..79453f6 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,57 @@ +# OS +.DS_Store +Thumbs.db + +# Editors +*.swp +*.swo +*~ +*.bak +.idea/ +.vscode/ +*.sublime-* + +# Agent scratch (worktrees of this repo, created and destroyed by +# in-flight tooling). Unanchored: .gitignore patterns already match at +# every depth, so no prefix is wanted here. This is not a .dockerignore +# entry and must not be given a `**/` prefix on the way into one. +.claude/ + +# Node +node_modules/ + +# Secrets. Unanchored like every entry above, so each matches at every +# depth. Matching is case-sensitive on Linux, so names use character +# ranges rather than a lowercase form that misses `Server.Key`. + +# Environment files. `*.env` covers bare `.env` and the `prod.env` +# convention. Only the templates `example.env` and `sample.env` are +# re-included below. A repository that commits any other template adds +# its own negation after these lines, for example `!.env.example`. +*.[eE][nN][vV] +.[eE][nN][vV].* +.[eE][nN][vV][rR][cC] +!example.env +!sample.env + +# Private keys and the bundles carrying them. +*.[pP][eE][mM] +*.[kK][eE][yY] +*.[pP]12 +*.[pP][fF][xX] +[iI][dD]_[rR][sS][aA] +[iI][dD]_[dD][sS][aA] +[iI][dD]_[eE][cC][dD][sS][aA] +[iI][dD]_[eE][cC][dD][sS][aA]_[sS][kK] +[iI][dD]_[eE][dD]25519 +[iI][dD]_[eE][dD]25519_[sS][kK] + +# This repository's own entries, kept after the canonical content above. +*.log +*.out +*.test bin/ vendor/ data/ -.env *.exe /dnswatcher diff --git a/.golangci.yml b/.golangci.yml index 26b1610..786cedd 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -10,14 +10,21 @@ run: linters: default: all + enable: + # Successor to the deprecated gomodguard. Named explicitly, rather than + # left to `default: all`, because it carries the module policy below. + - gomodguard_v2 disable: # Genuinely incompatible with project patterns - exhaustruct # Requires all struct fields - - depguard # Dependency allow/block lists + - exhaustruct_v5 # Requires all struct fields (successor to exhaustruct) - godot # Requires comments to end with periods - - wsl # Deprecated, replaced by wsl_v5 - wrapcheck # Too verbose for internal packages - varnamelen # Short names like db, id are idiomatic Go + # Deprecated: the warning is attached to the old name, so it is + # silenced by disabling that name, not by enabling the successor. + - wsl # Deprecated, replaced by wsl_v5 + - gomodguard # Deprecated, replaced by gomodguard_v2 settings: lll: line-length: 88 @@ -28,6 +35,68 @@ linters: max-complexity: 15 dupl: threshold: 100 + depguard: + # Test-support code must not be compiled into the shipped binary. A + # test-support package exists to hand a test privileges the program + # itself must never have, so a file that is not a test must not import + # one. Test files, and the files inside a package whose directory name + # ends in `test`, are where that code belongs, and are exempt. + # + # The deny list below is the one part of this file a repository is + # expected to extend, and the only part it may. depguard matches an + # import path against a list of prefixes, so it cannot be told "any path + # whose last segment ends in test"; a repository's own test-support + # packages have to be named here one at a time, by full import path, + # under a module path that differs from repository to repository. Add + # them; change nothing else. + rules: + test-support: + list-mode: lax + files: + - "$all" + - "!$test" + - "!**/*test/**" + deny: + - pkg: net/http/httptest + desc: >- + Test-support code belongs in test files and in packages whose + directory name ends in test, not in the shipped binary. + - pkg: sneak.berlin/go/dnswatcher/internal/livednstest + desc: >- + Live-DNS test support belongs in test files and in packages + whose directory name ends in test, not in the shipped binary. + # Only decisions already recorded in the Go package defaults are + # listed here. Every entry matches the module path exactly. + gomodguard_v2: + blocked: + - module: github.com/rs/zerolog + recommendations: + - log/slog + reason: "Structured logging is stdlib log/slog." + # One entry per pre-fork module path, because the later releases + # are separate paths. A prefix match would be shorter but would + # also reach github.com/go-redis/redismock, the test double for + # the successor these entries recommend. + - module: github.com/go-redis/redis + recommendations: + - github.com/redis/go-redis/v9 + reason: "Pre-fork module; use the maintained go-redis v9." + - module: github.com/go-redis/redis/v7 + recommendations: + - github.com/redis/go-redis/v9 + reason: "Pre-fork module; use the maintained go-redis v9." + - module: github.com/go-redis/redis/v8 + recommendations: + - github.com/redis/go-redis/v9 + reason: "Pre-fork module; use the maintained go-redis v9." + - module: github.com/sergi/go-diff + recommendations: + - github.com/aymanbagabas/go-udiff + reason: "No unified diff output; use go-udiff." + - module: github.com/hexops/gotextdiff + recommendations: + - github.com/aymanbagabas/go-udiff + reason: "Unmaintained fork; use go-udiff." issues: max-issues-per-linter: 0 diff --git a/.prettierignore b/.prettierignore new file mode 100644 index 0000000..23d67fc --- /dev/null +++ b/.prettierignore @@ -0,0 +1,2 @@ +node_modules/ +yarn.lock diff --git a/.prettierrc b/.prettierrc new file mode 100644 index 0000000..8af31cd --- /dev/null +++ b/.prettierrc @@ -0,0 +1,4 @@ +{ + "tabWidth": 4, + "proseWrap": "always" +} diff --git a/Dockerfile b/Dockerfile index 94b5b57..79729c6 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,55 +1,132 @@ -# Lint stage - fast feedback on lint issues, before the build starts. -# The linter is invoked directly rather than through `make lint`: that -# target shells out to `docker build -f Dockerfile.lint`, and there is -# no docker daemon inside a docker build. -# golangci/golangci-lint:v2.12.2 (Debian-based), 2026-08-10 -FROM golangci/golangci-lint:v2.12.2@sha256:5cceeef04e53efe1470638d4b4b4f5ceefd574955ab3941b2d9a68a8c9ad5240 AS lint - +# Lint phase, built alone by script/lint. The linter is invoked directly +# rather than through `make lint`, which is itself a docker build and +# would recurse into a daemon that does not exist in a build step. +# golangci/golangci-lint:v2.14.0 (Debian-based), 2026-10-06 +FROM golangci/golangci-lint@sha256:ad862ba6b3798cbe0fd9fd7408d498fd74fbd2623a92406b2fd3898faf0bf98f AS lint WORKDIR /src COPY go.mod go.sum ./ RUN go mod download - COPY . . - -RUN make fmt-check RUN golangci-lint run --config .golangci.yml ./... -# Build stage -# golang 1.25-alpine, 2026-02-28 -FROM golang@sha256:f6751d823c26342f9506c03797d2527668d095b0a15f1862cddb4d927a7a4ced AS builder - -RUN apk add --no-cache git make gcc musl-dev binutils-gold - -# Force BuildKit to run the lint stage before proceeding -COPY --from=lint /src/go.sum /dev/null - +# Test phase, built alone by script/test. -race needs cgo and so a C +# compiler, which the Debian Go image ships and the alpine one does not. +# The tests query live DNS (TESTING.md), so this step needs the network. +# -count=1 keeps Go's test result cache out of both runs, as TESTING.md +# requires. -timeout 90s is a backstop above the 60-second cap on the +# suite. The rerun with -v only shows details: the build fails however +# it ends, because the first run already failed. +# golang 1.25.7-trixie, 2026-10-06 +FROM golang@sha256:2b174ffcf56c7ad0c47d30d2630693265639ddf2a5141149c2da34db921791b4 AS test +# The file permission tests skip themselves as root, so the tests run as +# an ordinary user, whose home directory holds Go's build cache. +RUN useradd --create-home tester +USER tester WORKDIR /src COPY go.mod go.sum ./ RUN go mod download +COPY . . +RUN go test -count=1 -race -timeout 90s -cover ./... || \ + { echo "--- Rerunning with -v for details ---"; \ + go test -count=1 -race -timeout 90s -v ./...; exit 1; } +# Markdown formatting with prettier, at the version package.json and +# yarn.lock pin, so it is never installed on the host. script/fmt-check +# builds the fmt-check stage and script/fmt the fmt-out stage; the image +# does not depend on any of these stages, so a plain `docker build .` +# skips them. +# node:22-bookworm-slim, 2026-09-05 +FROM node@sha256:83f487e0a63425e5b4d146fb5e5be574bcbe1b7b843d3ebafdd95eaf7767a7e5 AS nodedeps +# prettier lives outside /src, so the COPY of the repo cannot overwrite +# it and node_modules is not in the tree prettier walks. +WORKDIR /tools +COPY package.json yarn.lock ./ +RUN yarn install --frozen-lockfile --non-interactive --no-progress +ENV PATH="/tools/node_modules/.bin:${PATH}" +WORKDIR /src + +# --config rather than discovery: a missing .prettierrc is then an error +# instead of prettier's defaults, under which every wrap passes. +# --no-editorconfig so .prettierrc alone sets the style. +FROM nodedeps AS fmt-check +COPY . . +RUN prettier --config .prettierrc --no-editorconfig --check "**/*.md" + +# Only the markdown is copied out, with its paths, so the export cannot +# put anything else back over the working tree. +FROM nodedeps AS fmt +COPY . . +RUN prettier --config .prettierrc --no-editorconfig --write "**/*.md" && \ + mkdir -p /out && \ + find . -name '*.md' -type f -exec cp --parents '{}' /out/ ';' + +FROM scratch AS fmt-out +COPY --from=fmt /out/ / + +# Build stage. Nothing is wanted from the lint and test phases; the +# copies are what make BuildKit build them first, so this stage cannot +# run unless lint and test passed. +# golang 1.25-alpine, 2026-02-28 +FROM golang@sha256:f6751d823c26342f9506c03797d2527668d095b0a15f1862cddb4d927a7a4ced AS builder +COPY --from=lint /src/go.sum /dev/null +COPY --from=test /src/go.sum /dev/null +RUN apk add --no-cache git +# A tar-stream context keeps the sender's file owners, which git refuses. +RUN git config --system --add safe.directory /src +WORKDIR /src +COPY go.mod go.sum ./ +RUN go mod download COPY . . -# Run the tests - build fails if any test fails -RUN make test +# The VERSION build arg when one is given (script/docker and +# script/cibuild pass one), otherwise `git describe --tags --always` on +# the .git in the build context. With .git present, a version that is +# still empty, dev or unknown fails the build: git is missing or could +# not read the checkout. +ARG VERSION +RUN VERSION="${VERSION:-$(git describe --tags --always)}"; \ + if [ -e .git ]; then \ + case "$VERSION" in ""|dev|unknown) \ + echo "version is '$VERSION' although .git is present" >&2; \ + exit 1 ;; \ + esac; \ + fi; \ + CGO_ENABLED=0 go build -trimpath \ + -ldflags="-s -w -X main.Version=${VERSION}" \ + -o /src/bin/dnswatcher ./cmd/dnswatcher/ -# Build the binary -RUN make build - -# Runtime stage +# Runtime stage, and the last one: a plain `docker build .` builds this +# stage's chain and nothing else. # alpine 3.21, 2026-02-28 FROM alpine@sha256:c3f8e73fdb79deaebaa2037150150191b9dcbfba68b4a46d70103204c53f4709 -RUN apk add --no-cache ca-certificates tzdata +RUN apk add --no-cache ca-certificates tzdata su-exec -WORKDIR /app +COPY --from=builder /src/bin/dnswatcher /usr/local/bin/dnswatcher +COPY deploy/docker-entrypoint.sh /usr/local/bin/docker-entrypoint.sh -COPY --from=builder /src/bin/dnswatcher /app/dnswatcher - -# Create data directory -RUN mkdir -p /var/lib/dnswatcher +# dnswatcher runs as this unprivileged user. The entrypoint creates the +# data directory and gives it to this user on every start. +RUN addgroup -S -g 10001 dnswatcher \ + && adduser -S -G dnswatcher -u 10001 dnswatcher ENV DNSWATCHER_DATA_DIR=/var/lib/dnswatcher +# Config loading also reads a `.env` file and a file named `dnswatcher` +# (any config extension, or none) from the working directory. `/` holds +# neither, so every setting comes from the environment. Do not make the +# data directory, or the binary's directory, the working directory. +WORKDIR / + +# No USER: the entrypoint must start as root to set up the data +# directory; it then runs dnswatcher as the dnswatcher user. + EXPOSE 8080 -ENTRYPOINT ["/app/dnswatcher"] +# busybox wget (already in alpine) probes the health endpoint every 10 +# seconds, so the container is healthy well before upaas reads its health +# 60 seconds after a deploy and fails the deploy unless it is healthy. +HEALTHCHECK --interval=10s --timeout=5s --start-period=10s --retries=3 \ + CMD wget -q -O /dev/null "http://127.0.0.1:${PORT:-8080}/.well-known/healthcheck" || exit 1 + +ENTRYPOINT ["/usr/local/bin/docker-entrypoint.sh"] diff --git a/Dockerfile.lint b/Dockerfile.lint deleted file mode 100644 index 4dcb14e..0000000 --- a/Dockerfile.lint +++ /dev/null @@ -1,29 +0,0 @@ -# Lint-only image: used by script/lint. golangci-lint is never run on -# the host — the repo is COPYed into the build context and the linter -# runs as a build step, so a successful build IS a clean lint. This -# also works where the docker daemon is remote and bind mounts are -# impossible. -# -# `golangci-lint config verify` is deliberately NOT run here: it -# fetches its JSON schema over a live, unpinned HTTPS call, which would -# make linting network-dependent and defeat hash-pinning. The cost of -# that: unknown top-level keys in .golangci.yml are silently ignored, -# so a mistyped or wrong-schema key lints clean while applying nothing. -# -# golangci/golangci-lint:v2.12.2 (Debian-based), 2026-08-10 -FROM golangci/golangci-lint:v2.12.2@sha256:5cceeef04e53efe1470638d4b4b4f5ceefd574955ab3941b2d9a68a8c9ad5240 AS deps - -WORKDIR /src - -# Dependencies first, so this stage stays cached across lint runs. -COPY go.mod go.sum ./ -RUN go mod download - -# Everything below is invalidated on every run by the -# --no-cache-filter=lint that script/lint passes: caching is explicitly -# waived for linting, and a cached build lints nothing. -FROM deps AS lint - -COPY . . - -RUN golangci-lint run --config .golangci.yml ./... diff --git a/Makefile b/Makefile index 7b2447f..251a12c 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,13 @@ -.PHONY: all bootstrap setup build lint fmt fmt-check test check clean hooks docker +.PHONY: all bootstrap setup build version lint fmt fmt-check test check clean hooks docker BINARY := dnswatcher -VERSION := $(shell git describe --tags --always --dirty 2>/dev/null || echo "dev") +# VERSION given on the command line (`make build VERSION=...`) or in the +# environment wins over what `git describe` says of this checkout. An +# empty one counts as not given; `override` is what replaces an empty +# command-line value. +ifeq ($(VERSION),) +override VERSION := $(shell git describe --tags --always --dirty 2>/dev/null || echo "dev") +endif LDFLAGS := -X main.Version=$(VERSION) # Standard targets are thin shims; the implementations live in script/ @@ -19,6 +25,10 @@ setup: build: go build -ldflags "$(LDFLAGS)" -o bin/$(BINARY) ./cmd/dnswatcher +# Prints the version `make build` stamps. +version: + @echo "$(VERSION)" + test: @script/test diff --git a/README.md b/README.md index 9e854ff..7fd92fa 100644 --- a/README.md +++ b/README.md @@ -1,16 +1,20 @@ # dnswatcher -dnswatcher is an MIT-licensed, pre-1.0 Go daemon by [@sneak](https://sneak.berlin) that monitors DNS records, TCP port availability, and TLS certificates, delivering real-time change notifications via Slack, Mattermost, and ntfy webhooks. +dnswatcher is an MIT-licensed, pre-1.0 Go daemon by +[@sneak](https://sneak.berlin) that monitors DNS records, TCP port availability, +and TLS certificates, delivering real-time change notifications via Slack, +Mattermost, and ntfy webhooks. -> ⚠️ Pre-1.0 software. APIs, configuration, and behavior may change without notice. +> ⚠️ Pre-1.0 software. APIs, configuration, and behavior may change without +> notice. -dnswatcher watches configured DNS domains and hostnames for changes, monitors TCP -port availability, tracks TLS certificate expiry, and delivers real-time +dnswatcher watches configured DNS domains and hostnames for changes, monitors +TCP port availability, tracks TLS certificate expiry, and delivers real-time notifications via Slack, Mattermost, and/or ntfy webhooks. -It performs all DNS resolution itself via iterative (non-recursive) queries, -tracing from root nameservers to authoritative servers directly—never relying -on upstream recursive resolvers. +It resolves the names it watches itself via iterative (non-recursive) queries, +tracing from root nameservers to authoritative servers directly—never relying on +upstream recursive resolvers. State is persisted to a local JSON file so that monitoring survives restarts without requiring an external database. @@ -19,21 +23,43 @@ without requiring an external database. ## No DNS mocking. Ever. -**DNS is never mocked in this project — not in tests, not anywhere else.** -No mock resolvers, no fake DNS servers, no stubbed lookups. +**DNS is never mocked in this project — not in tests, not anywhere else.** No +mock resolvers, no fake DNS servers, no stubbed lookups. -dnswatcher's entire purpose is correct behavior against the real DNS. -Tests exercise real iterative resolution against live nameservers by -design; a test suite that passes against a mock proves nothing about the -one thing this program exists to do. +dnswatcher's entire purpose is correct behavior against the real DNS. Tests +exercise real iterative resolution against live nameservers by design; a test +suite that passes against a mock proves nothing about the one thing this program +exists to do. -When live tests are flaky, that is a robustness problem, and it gets -fixed with robustness: retries with backoff, querying multiple -independent nameservers, longer timeouts — or explicit opt-in gating -decided by the project owner. Never with mocks. +When live tests are flaky, that is a robustness problem, and it gets fixed with +robustness: retries with backoff, querying multiple independent nameservers, +longer timeouts — or explicit opt-in gating decided by the project owner. Never +with mocks. -Contributions that introduce mocked, faked, or stubbed DNS will be -rejected. +Contributions that introduce mocked, faked, or stubbed DNS will be rejected. + +--- + +## Getting Started + +You need git and Docker. This builds the image and runs dnswatcher watching +`example.com` and `www.example.com`: + +```sh +git clone https://git.eeqj.de/sneak/dnswatcher.git +cd dnswatcher +docker build -t dnswatcher . +docker run -d --name dnswatcher \ + -p 8080:8080 \ + -v dnswatcher-data:/var/lib/dnswatcher \ + -e DNSWATCHER_TARGETS=example.com,www.example.com \ + dnswatcher +``` + +The build also runs the linter and the test suite, which queries live DNS. Once +the container is running, the dashboard is at . With no +notification endpoint set, changes show only on the dashboard; see +[Configuration](#configuration) to add one. --- @@ -43,190 +69,378 @@ rejected. - Accepts a list of DNS domain names (apex domains, identified via the [Public Suffix List](https://publicsuffix.org/)). -- Every **1 hour**, performs a full iterative trace from root servers to - discover all authoritative nameservers (NS records) for each domain. +- Every **1 hour** by default, performs a full iterative trace from root servers + to discover all authoritative nameservers (NS records) for each domain. - Queries **every** discovered authoritative nameserver independently. -- Stores the NS record set as observed by the delegation chain. +- Stores the domain's NS record set, as its parent zone's servers delegate it, + and the IPv4 and IPv6 addresses each nameserver's name resolves to. The set is + only ever the domain's own delegation. A domain whose parent zone's servers + answer NXDOMAIN, that it does not exist, has no nameservers and is shown as + not existing (see Web Dashboard and HTTP API). A domain that exists but has no + delegation of its own, such as `octocat.github.io`, has no nameservers either. + When the parent zone's servers do not answer, or the lookup of the domain's + own records (below) fails, the check fails and the set from the previous check + is kept. - Any change triggers a notification: - - NS added to or removed from the delegation. - - NS IP address changed (glue record change). + - NS added to or removed from that set. A domain that had nameservers on the + previous check and no longer exists gets one with all of them removed. + After an upgrade, a domain with no delegation of its own, for which an + earlier version saved its parent zone's nameservers, also gets one with + all of them removed, on its first check. That one does not mean the domain + stopped existing: it is not shown as not existing, and its records are + still watched. + - NS address change: a nameserver that stays in the set resolves to + different addresses than on the previous check. A nameserver added or + removed gets only the NS change notification. When the lookup of a + nameserver's addresses fails or finds none, its previous addresses are + kept and nothing is sent. The lookup fails when no nameserver it asks + answers every one of its queries, for A, AAAA and CNAME. +- Also watches the domain's own records as a hostname's are watched (see DNS + Hostname Monitoring below): its A, AAAA, CNAME, MX, TXT, SRV, CAA and NS + records, stored per nameserver. Their changes are notified as a hostname's + are, as a record change, NS query failure, NS recovery, inconsistency or CNAME + address change, in a message that starts `Domain:` where a hostname's starts + `Hostname:`. A nameserver removed from the domain's delegation gets only the + NS change notification above. A domain listed in + `DNSWATCHER_SKIP_RECORD_NOTIFICATIONS` gets no record change or inconsistency + notification. A domain with no delegation of its own has these records asked + at the servers of the zone it is in, as a hostname has. When none of those + servers answers, the lookup of its NS record set fails, and they are saved as + failed and notified as a hostname's are (see below). A nameserver the domain's + records were asked at on the previous check but not on this one gets a + hostname's NS change notification (see below), unless it was in the domain's + delegation on the previous check, as when that delegation is withdrawn. A + domain that does not exist has none: they are not asked for, and those saved + by an earlier check are removed without a notification. ### DNS Hostname Monitoring (Subdomains) -- Accepts a list of DNS hostnames (subdomains, distinguished from apex - domains via the Public Suffix List). -- Every **1 hour**, performs a full iterative trace to discover the - authoritative nameservers for the hostname's parent domain. -- Queries **each** authoritative nameserver independently for **all** - record types: A, AAAA, CNAME, MX, TXT, SRV, CAA, NS. -- Stores results **per nameserver**. The state for a hostname is not a - merged view — it is a map from nameserver to record set. -- Any observable change in any nameserver's response triggers a - notification. This includes: - - **Record change**: A nameserver returns different records than it - did on the previous check (additions, removals, value changes). - - **NS query failure**: A nameserver that previously responded - becomes unreachable (timeout, SERVFAIL, REFUSED, network error). - This is distinct from "responded with no records." - - **NS recovery**: A previously-unreachable nameserver starts - responding again. - - **Inconsistency detected**: Two nameservers that previously agreed - now return different record sets for the same hostname. +- Accepts a list of DNS hostnames (subdomains, distinguished from apex domains + via the Public Suffix List). +- Every **1 hour** by default, performs a full iterative trace to discover the + authoritative nameservers of the zone the hostname is in, which is not always + its last two labels (a name under `co.uk`, or in a delegated subdomain). The + trace moves from a name to its parent only when the servers asked answer that + the name has no delegation of its own, or does not exist. When they do not + answer, the check fails and the hostname's records from the previous check are + kept, unless they are the nameservers the previous check saved for the + hostname. Each of those is then saved as failed, with the reason, as a + domain's nameservers are when none of them answers, so it gets an NS query + failure notification, and an NS recovery notification when it answers again + (see below). A trace that gets no answer before it reaches them, as when the + host running dnswatcher loses its network, fails the check without a + notification. +- Queries **each** authoritative nameserver independently for **all** record + types: A, AAAA, CNAME, MX, TXT, SRV, CAA, NS. +- Each record type is a query of its own. When a nameserver answers some types + but the query for another gets no usable reply at any of its addresses (no + reply after two tries, an error reply such as SERVFAIL, a referral, or a reply + too large for UDP whose retry over TCP fails), the failure is logged with the + reason, and the type is listed in the nameserver's `failedTypes` and keeps the + records saved for the nameserver by the previous check. On that check those + records are not compared with the other nameservers', so no record change or + inconsistency is reported for the type; on the next check they are compared + with the nameserver's answer as usual. When the previous check did not know + the type's records either, because the nameserver was new or failing then or + the type was already listed in `unknownTypes`, the type is also listed in + `unknownTypes` and left out of every comparison until it answers. A nameserver + none of whose queries got a usable reply has failed (see NS query failure + below). +- Stores results **per nameserver**. The state for a hostname is not a merged + view — it is a map from nameserver to record set. +- DNS names inside record values (CNAME, MX, SRV and NS targets) are stored in + lower case, because names are case-insensitive and nameservers may answer in + any letter case. TXT and CAA values keep their letter case; they are not + lower-cased. +- Any observable change in any nameserver's response triggers a notification, + except a record change or an inconsistency for a domain or hostname listed in + `DNSWATCHER_SKIP_RECORD_NOTIFICATIONS`. This includes: + - **Record change**: A nameserver returns different records than it did on + the previous check (additions, removals, value changes). For a domain or + hostname listed in `DNSWATCHER_SKIP_RECORD_NOTIFICATIONS`, neither a + record change nor an inconsistency is notified; its records are still + checked and saved, and its other notifications are sent. + - **NS query failure**: A nameserver that previously responded becomes + unreachable (timeout, SERVFAIL, REFUSED, network error). This is distinct + from "responded with no records": a nameserver that answers NXDOMAIN or + with no records has responded. The alert is sent once, on the check where + it starts failing. A failing nameserver gives no records, so it is not + reported as a record change or compared for inconsistency. A nameserver + that is already failing on the first check that sees it is recorded + silently. + - **NS recovery**: A previously-unreachable nameserver starts responding + again. Its records are not compared with those from before it failed, so a + change made while it was failing is not reported as a record change. + - **NS change**: A nameserver of the zone the hostname is in on the previous + check is not one now, whether or not it answered. One notification lists + every such nameserver on a `Removed:` line. Nothing is sent for a + nameserver that joins the zone. + - **Inconsistency detected**: Two nameservers return different record sets + for the same hostname and did not already differ on the previous check. + Every pair of nameservers is compared. The alert is sent once for each + such pair, on the check where they start to disagree, and not again while + they keep disagreeing, including after a restart. A nameserver that was + not in the previous check (newly added, or back after dropping out), or + failed on it, and answers differently is reported on the check where it + answers. So is a pair that differs in a record type whose query to either + nameserver failed on the previous check. If a pair agrees again and later + disagrees, the alert is sent again. For a domain or hostname listed in + `DNSWATCHER_SKIP_RECORD_NOTIFICATIONS`, no inconsistency is notified. + - **CNAME address change**: The addresses at the end of a name's CNAME chain + differ from those of the previous check. They are found when its + nameservers answer with a CNAME and no address; a name that answers with + an address has none. A change from or to no addresses is sent too, as when + a name moves between A records and a CNAME. Nothing is sent when the + previous addresses were kept because a chain could not be followed or none + of the name's nameservers answered. The first check after loading a state + file without `cnameAddresses` sends nothing: it saves the addresses it + finds for the next check to compare. ### TCP Port Monitoring -- For every configured domain and hostname, constructs a deduplicated list - of all IPv4 and IPv6 addresses resolved via A, AAAA, and CNAME chain - resolution across all authoritative nameservers. +- For every configured domain and hostname, constructs a deduplicated list of + the IPv4 and IPv6 addresses in the A and AAAA records its authoritative + nameservers returned. When they returned a CNAME and no address, the CNAME + chain is followed and the addresses at its end are used, and a change in those + is notified as a CNAME address change. When the nameservers gave different + CNAME targets, each is followed and the addresses of all are used. When a + chain cannot be followed, or none of the name's nameservers answered, the + addresses the last check found at its end are used. - Checks TCP connectivity on ports **80** and **443** for each IP address. -- Every **1 hour**, re-checks all ports. +- Every **1 hour** by default, re-checks all ports. - Any change in port availability triggers a notification: - - Port transitioned from open to closed (or vice versa). - - New IP appeared (from DNS change) and its port state was recorded. - - IP disappeared (from DNS change) — noted in the DNS change - notification; port state for that IP is removed. + - Port transitioned from open to closed (or vice versa). + - New IP appeared (from DNS change): its port state is recorded without a + port notification; the DNS change notification shows the new address. A + domain or hostname listed in `DNSWATCHER_SKIP_RECORD_NOTIFICATIONS` gets + no notification for an address added to its own A or AAAA records. + - IP disappeared (from DNS change) — noted in the DNS change notification; + port state for that IP is removed. A domain or hostname listed in + `DNSWATCHER_SKIP_RECORD_NOTIFICATIONS` gets no notification for an address + removed from its own A or AAAA records. When none of a name's nameservers + answered, its addresses are not known, so the port state saved for them is + kept. ### TLS Certificate Monitoring -- Every **12 hours**, for each IP address listening on port 443, connects - via TLS using the correct SNI hostname. +- Every **12 hours** by default, for each IP address listening on port 443, + connects via TLS using the correct SNI hostname. - Records the certificate's Subject CN, SANs, issuer, and expiry date. - Any change triggers a notification: - - Certificate is expiring within **7 days** (warning, repeated each - check until renewed or expired). - - Certificate CN, issuer, or SANs changed (replacement detected, - reports old and new values). - - TLS connection failure to a previously-reachable IP:443 (handshake - error, timeout, connection refused after previously succeeding). - - TLS recovery: a previously-failing IP:443 now completes a - handshake again. + - Certificate is expiring within **7 days** by default (warning, repeated + each check until renewed or expired). + - Certificate CN, issuer, or SANs changed (replacement detected, reports old + and new CN and issuer). + - TLS connection failure to a previously-reachable IP:443 (handshake error, + timeout, or connection refused when port 443 closed after the port check + that found it open). + - TLS recovery: a previously-failing IP:443 now completes a handshake again. +- A port check that finds port 443 closed removes the certificate entries for + that IP address, without a notification of its own: the port change + notification reports the closed port. When the port opens again, the next TLS + check saves the certificate as new, without a certificate change or recovery + notification. ### Notifications **Every observable state change produces a notification.** dnswatcher is -designed as a real-time change feed — degradations, failures, recoveries, -and routine changes are all reported equally. +designed as a real-time change feed — degradations, failures, recoveries, and +routine changes are all reported equally. A domain or hostname listed in +`DNSWATCHER_SKIP_RECORD_NOTIFICATIONS` gets no record change or inconsistency +notification. Supported notification backends: -| Backend | Configuration | Payload Format | -|----------------|--------------------------|------------------------------| -| **Slack** | Incoming Webhook URL | Attachments with color | -| **Mattermost** | Incoming Webhook URL | Slack-compatible attachments | -| **ntfy** | Topic URL (e.g. `https://ntfy.sh/mytopic`) | Title + body + priority | +| Backend | Configuration | Payload Format | +| -------------- | ------------------------------------------ | ---------------------------- | +| **Slack** | Incoming Webhook URL | Attachments with color | +| **Mattermost** | Incoming Webhook URL | Slack-compatible attachments | +| **ntfy** | Topic URL (e.g. `https://ntfy.sh/mytopic`) | Title + body + priority | All configured endpoints receive every notification. Notification content includes: -- **DNS record changes**: Which hostname, which nameserver, what record - type, old values, new values. -- **DNS NS changes**: Which domain, which nameservers were added/removed. -- **NS query failures**: Which nameserver failed, error type (timeout, - SERVFAIL, REFUSED, network error), which hostname/domain affected. +- **DNS record changes**: Which hostname or domain, which nameserver, what + record type, old values, new values. +- **DNS NS changes**: Which domain, which nameservers were added to or removed + from its delegation. For a hostname, or a domain with no delegation of its + own, which nameservers left the zone it is in. +- **NS address changes**: Which domain, which nameserver, its old and new + addresses. +- **CNAME address changes**: Which hostname or domain, the old and new addresses + at the end of its CNAME chain. +- **NS query failures**: Which nameserver failed, error type (timeout, SERVFAIL, + REFUSED, network error), which hostname/domain affected. - **NS recoveries**: Which nameserver recovered, which hostname/domain. -- **NS inconsistencies**: Which nameservers disagree, what each one - returned, which hostname affected. -- **Port changes**: Which IP:port, old state, new state, all associated - hostnames. -- **TLS expiry warnings**: Which certificate, days remaining, CN, - issuer, associated hostname and IP. -- **TLS certificate changes**: Old and new CN/issuer/SANs, associated +- **NS inconsistencies**: Which nameservers disagree, what each one returned, + which hostname or domain affected. +- **Port changes**: Which IP:port, its new state, and the domains and the + hostnames that resolve to it, on a `Domains:` line and a `Hostnames:` line. A + line that would name nothing is left out. +- **TLS expiry warnings**: Expiry date and days remaining, CN, associated hostname and IP. +- **TLS certificate changes**: Old and new CN and issuer, associated hostname + and IP. A change to the SANs alone is notified, but the SANs are not listed. - **TLS connection failures/recoveries**: Which IP:port, error details, associated hostname. +Each endpoint is sent each notification on its own, in the background. A +delivery that fails (a network error, no reply within 10 seconds, or an HTTP +status other than 2xx) is retried up to 5 times: the first retry after about 1 +second, each wait after that twice as long up to 60 seconds, every wait varied +at random by up to 25%. A delivery still failing after that is logged and +dropped. + +The last 100 notifications, delivered or not, are kept in memory for the +dashboard's Recent alerts. They are not saved to the state file, so a restart +clears them. + ### State Management -- All monitoring state is kept in memory and persisted to a JSON file on - disk (`DATA_DIR/state.json`). +- All monitoring state is kept in memory and persisted to a JSON file on disk + (`DATA_DIR/state.json`). - State is loaded on startup to resume monitoring without triggering false-positive change notifications. - State is written atomically (write to temp file, then rename) to prevent corruption. +- A name removed from `DNSWATCHER_TARGETS` is removed from the state at startup, + before the first check, without a notification: its domain, hostname and + certificate entries go, it is taken off each port entry's list of names, and a + port entry left with no name goes, so the dashboard and `/api/v1/status` no + longer list or count it. The first check's port checks remove the port entries + of addresses no configured name has. +- Each port check also removes, without a notification, the certificate entries + for an address whose port 443 it found closed, and those for an address their + name no longer resolves to, except while none of the name's nameservers + answer. ### Web Dashboard -dnswatcher includes an unauthenticated, read-only web dashboard at the -root URL (`/`). It displays: +dnswatcher includes an unauthenticated, read-only web dashboard at the root URL +(`/`). It displays: -- **Summary counts** for monitored domains, hostnames, ports, and - certificates. -- **Domains** with their discovered nameservers. -- **Hostnames** with per-nameserver DNS records and status. -- **Ports** with open/closed state and associated hostnames. -- **TLS certificates** with CN, issuer, expiry, and status. -- **Recent alerts** (last 100 notifications sent since the process - started), displayed in reverse chronological order. +- **Summary counts** for monitored domains, hostnames, ports, and certificates. +- **Domains** with their discovered nameservers, or "does not exist" for a + domain whose parent zone's servers answered NXDOMAIN, and each domain's own + records per nameserver and status, shown as a hostname's are. +- **Hostnames** with per-nameserver DNS records and status. For a nameserver + whose query failed, the reason is shown in place of the records. +- **Ports** with open/closed state and the domains and hostnames that resolve to + each address, in separate columns. +- **TLS certificates** with CN, issuer, expiry, and status. For a failed check, + the reason is shown in place of CN, issuer and expiry. +- **Recent alerts** (last 100 notifications sent since the process started), + displayed in reverse chronological order. -Every data point shows its age (e.g. "5m ago") so you can tell at a -glance how fresh the information is. The page auto-refreshes every 30 -seconds. +Every data point shows its age (e.g. "5m ago") so you can tell at a glance how +fresh the information is. The page auto-refreshes every 30 seconds. -The dashboard intentionally does not expose any configuration details -such as webhook URLs, notification endpoints, or API tokens. +The dashboard intentionally does not expose any configuration details such as +webhook URLs, notification endpoints, or API tokens. -All assets (CSS) are embedded in the binary and served from the -application itself. The dashboard makes zero external HTTP requests — -no CDN dependencies or third-party resources are loaded at runtime. +All assets (CSS) are embedded in the binary and served from the application +itself. The dashboard makes zero external HTTP requests — no CDN dependencies or +third-party resources are loaded at runtime. ### HTTP API dnswatcher exposes a lightweight HTTP API for operational visibility: -| Endpoint | Description | -|---------------------------------------|--------------------------------| -| `GET /` | Web dashboard (HTML) | -| `GET /s/...` | Static assets (embedded CSS) | -| `GET /.well-known/healthcheck` | Health check (JSON) | -| `GET /health` | Health check (JSON, legacy) | -| `GET /api/v1/status` | Current monitoring state | -| `GET /metrics` | Prometheus metrics (optional) | +| Endpoint | Description | +| ------------------------------ | ----------------------------- | +| `GET /` | Web dashboard (HTML) | +| `GET /s/...` | Static assets (embedded CSS) | +| `GET /.well-known/healthcheck` | Health check (JSON) | +| `GET /health` | Health check (JSON, legacy) | +| `GET /api/v1/status` | Current monitoring state | +| `GET /metrics` | Prometheus metrics, see below | ---- +In `/api/v1/status`, each nameserver entry and certificate entry whose `status` +is `error` also has `error`, the reason, as in the state file (see State File +Format). A domain's own records are in its entry in `domains`, under +`recordsByNameserver`, in the form a hostname's entry in `hostnames` has them +under `nameservers`; `hostnames` and `counts.hostnames` hold no domain. A domain +entry's `nxdomain` is `true` when the domain's parent zone's servers answered +NXDOMAIN, that it does not exist; its `nameservers` and `recordsByNameserver` +are then empty. A port entry lists the domains that resolve to its address in +`domains`, and the hostnames in `hostnames`. -## Architecture +`/metrics` is served only when `DNSWATCHER_METRICS_USERNAME` is set, behind +Basic Auth. It has the Prometheus Go client's default metrics only (Go runtime, +process, and counts of `/metrics` requests); dnswatcher records no metrics of +its own. + +Every route but `/metrics` may be read from a page on any origin: a cross-origin +`GET` gets `Access-Control-Allow-Origin: *`. Only `GET` is allowed cross-origin, +and without credentials. `/metrics` sends no CORS headers. + +#### Server timeouts + +The HTTP server sets all four socket-level timeouts. These are compile-time +constants in `internal/server/server.go`, not configurable via environment +variables. + +| Timeout | Value | Purpose | +| ------------------- | ----- | --------------------------------------------- | +| `ReadHeaderTimeout` | 10s | Bounds the request header read (slowloris) | +| `ReadTimeout` | 15s | Bounds the whole request read, headers + body | +| `WriteTimeout` | 75s | Bounds handler execution plus response flush | +| `IdleTimeout` | 120s | Reaps idle keep-alive connections | + +These are distinct from the 60s per-request handler budget applied by +`chimw.Timeout` in `internal/server/routes.go`, which cancels the request +context but does not touch the socket. `WriteTimeout` is deliberately larger +than that budget: the write deadline is armed once request headers are read, so +a smaller value would sever the connection before a handler using its full +budget could respond. `IdleTimeout` exceeds common Prometheus scrape intervals +so the scraper reuses its connection. + +### Security Headers + +Every response — the dashboard, the static assets under `/s/...`, the +healthchecks, the JSON API, and `/metrics` — carries the following headers, set +by a global middleware: + +| Header | Value | +| --------------------------- | ------------------------------------- | +| `Strict-Transport-Security` | `max-age=31536000; includeSubDomains` | +| `Content-Security-Policy` | see below | +| `X-Frame-Options` | `DENY` | +| `X-Content-Type-Options` | `nosniff` | +| `Referrer-Policy` | `no-referrer` | +| `Permissions-Policy` | all unused browser features denied | + +The content security policy is: ``` -cmd/dnswatcher/main.go Entry point (uber/fx bootstrap) - -internal/ - config/config.go Viper-based configuration - globals/globals.go Build-time variables (version) - logger/logger.go slog structured logging (TTY detection) - healthcheck/healthcheck.go Health check service - middleware/middleware.go HTTP middleware (logging, CORS, metrics auth) - handlers/handlers.go HTTP request handlers - server/ - server.go HTTP server lifecycle - routes.go Route definitions - state/state.go JSON file state persistence - resolver/resolver.go Iterative DNS resolution engine - portcheck/portcheck.go TCP port connectivity checker - tlscheck/tlscheck.go TLS certificate inspector - notify/notify.go Notification service (Slack, Mattermost, ntfy) - watcher/watcher.go Main monitoring orchestrator and scheduler +default-src 'self'; script-src 'none'; style-src 'self'; img-src 'self'; +font-src 'none'; connect-src 'none'; object-src 'none'; base-uri 'none'; +form-action 'none'; frame-ancestors 'none' ``` -### Design Principles +The dashboard ships no JavaScript (the 30-second refresh is a +``), no inline styles, no inline event handlers, and +no images; its only subresource is the embedded stylesheet at +`/s/css/tailwind.min.css`, which `style-src 'self'` permits. The policy +therefore needs neither `unsafe-inline` nor `unsafe-eval`. +`frame-ancestors 'none'` is the primary anti-framing control, with +`X-Frame-Options: DENY` retained as the legacy fallback. -- **No recursive resolvers**: All DNS resolution is performed iteratively, - tracing from root nameservers through the delegation chain to - authoritative servers. -- **No external database**: State is persisted as a single JSON file. -- **Dependency injection**: All components are wired via - [uber/fx](https://github.com/uber-go/fx). -- **Structured logging**: All logs use `log/slog` with JSON output in - production (TTY detection for development). -- **Graceful shutdown**: All background goroutines respect context - cancellation and the fx lifecycle. In-flight notification deliveries - are drained on shutdown, bounded by the shutdown timeout. +HSTS is emitted unconditionally, including over plain HTTP. dnswatcher is +expected to run behind a TLS-terminating reverse proxy, and the browser must +still be told to enforce HTTPS end to end, so the header is never gated on +whether the request itself arrived over TLS. + +`Referrer-Policy: no-referrer` is stricter than the +`strict-origin-when-cross-origin` baseline: the dashboard has no cross-origin +navigation needs, and its URL may name internal hosts. --- ## Configuration -Configuration is loaded via [Viper](https://github.com/spf13/viper) with -the following precedence (highest to lowest): +Configuration is loaded via [Viper](https://github.com/spf13/viper) with the +following precedence (highest to lowest): 1. Environment variables (prefixed with `DNSWATCHER_`) 2. `.env` file (loaded via godotenv) @@ -236,29 +450,64 @@ the following precedence (highest to lowest): ### Environment Variables -| Variable | Description | Default | -|---------------------------------|--------------------------------------------|-------------| -| `PORT` | HTTP listen port | `8080` | -| `DNSWATCHER_DEBUG` | Enable debug logging | `false` | -| `DNSWATCHER_DATA_DIR` | Directory for state file | `/var/lib/dnswatcher` | -| `DNSWATCHER_TARGETS` | Comma-separated DNS names (auto-classified via PSL) | `""` | -| `DNSWATCHER_SLACK_WEBHOOK` | Slack incoming webhook URL | `""` | -| `DNSWATCHER_MATTERMOST_WEBHOOK` | Mattermost incoming webhook URL | `""` | -| `DNSWATCHER_NTFY_TOPIC` | ntfy topic URL | `""` | -| `DNSWATCHER_DNS_INTERVAL` | DNS check interval | `1h` | -| `DNSWATCHER_TLS_INTERVAL` | TLS check interval | `12h` | -| `DNSWATCHER_TLS_EXPIRY_WARNING` | Days before expiry to warn | `7` | -| `DNSWATCHER_SENTRY_DSN` | Sentry DSN for error reporting | `""` | -| `DNSWATCHER_MAINTENANCE_MODE` | Enable maintenance mode | `false` | -| `DNSWATCHER_METRICS_USERNAME` | Basic auth username for /metrics | `""` | -| `DNSWATCHER_METRICS_PASSWORD` | Basic auth password for /metrics | `""` | -| `DNSWATCHER_SEND_TEST_NOTIFICATION` | Send a test notification after first scan completes | `false` | +| Variable | Description | Default | +| -------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------- | --------------------- | +| `PORT` | HTTP listen port; a port that cannot be bound, such as one already in use, stops startup | `8080` | +| `DNSWATCHER_DEBUG` | Enable debug logging | `false` | +| `DNSWATCHER_DATA_DIR` | Directory for state file | `/var/lib/dnswatcher` | +| `DNSWATCHER_TARGETS` | Comma-separated DNS names (auto-classified via PSL) | `""` | +| `DNSWATCHER_SKIP_RECORD_NOTIFICATIONS` | Comma-separated names from `DNSWATCHER_TARGETS` for which no record change or inconsistency is notified; any other name stops startup | `""` | +| `DNSWATCHER_SLACK_WEBHOOK` | Slack incoming webhook URL | `""` | +| `DNSWATCHER_MATTERMOST_WEBHOOK` | Mattermost incoming webhook URL | `""` | +| `DNSWATCHER_NTFY_TOPIC` | ntfy topic URL | `""` | +| `DNSWATCHER_DNS_INTERVAL` | DNS check interval, a positive duration such as `30m`; empty means the default, anything else stops startup | `1h` | +| `DNSWATCHER_TLS_INTERVAL` | TLS check interval, a positive duration such as `6h`; empty means the default, anything else stops startup | `12h` | +| `DNSWATCHER_TLS_EXPIRY_WARNING` | Days before expiry to warn | `7` | +| `DNSWATCHER_SENTRY_DSN` | Sentry DSN for error reporting | `""` | +| `DNSWATCHER_MAINTENANCE_MODE` | Only sets `maintenanceMode` in the health check response; changes nothing else | `false` | +| `DNSWATCHER_METRICS_USERNAME` | Basic auth username for /metrics, which is served only when this is set | `""` | +| `DNSWATCHER_METRICS_PASSWORD` | Basic auth password for /metrics; required when `DNSWATCHER_METRICS_USERNAME` is set, and startup stops without it | `""` | +| `DNSWATCHER_SEND_TEST_NOTIFICATION` | Send a test notification after first scan completes | `false` | **`DNSWATCHER_TARGETS` is required.** dnswatcher will refuse to start if no monitoring targets are configured. A monitoring daemon with nothing to monitor is a misconfiguration, so dnswatcher fails fast with a clear error message -rather than running silently. Set `DNSWATCHER_TARGETS` to a comma-separated -list of DNS names before starting. +rather than running silently. Set `DNSWATCHER_TARGETS` to a comma-separated list +of DNS names before starting. A name listed more than once, in any letter case +or with a trailing dot, is watched once. + +**`/metrics` is rate limited.** Each client address may send it 30 requests a +minute, failed logins included; beyond that it answers `429 Too Many Requests` +without checking the password. A Prometheus server scraping every 15 seconds +sends 4 a minute. IPv6 addresses in one /64 count as one client. When the +request comes from a private or loopback address, such as a reverse proxy's, the +client address is taken from the `X-Real-IP` header the proxy sets, or else from +`X-Forwarded-For`, as the last address in it that is not private or loopback. A +proxy that sets neither makes all its clients share one allowance. + +**`DNSWATCHER_DNS_INTERVAL` and `DNSWATCHER_TLS_INTERVAL`** take a positive +duration: a number followed by a unit such as `s`, `m` or `h`, for example +`90s`, `30m`, `1h` or `1h30m`. There is no unit for days; write `24h`. An unset +or empty variable (`DNSWATCHER_DNS_INTERVAL=`) means the default. If either is +set to anything else, including a bare number or a zero or negative duration, +dnswatcher refuses to start with an error naming the variable and the value. + +**`PORT` and `DNSWATCHER_TLS_EXPIRY_WARNING`** take a whole number: 1 to 65535 +for `PORT`, and a number of days, 1 or more, for +`DNSWATCHER_TLS_EXPIRY_WARNING`. `DNSWATCHER_DEBUG`, +`DNSWATCHER_MAINTENANCE_MODE` and `DNSWATCHER_SEND_TEST_NOTIFICATION` take +`true` or `false`; `1`, `t`, `T`, `TRUE` and `True` also mean true, and `0`, +`f`, `F`, `FALSE` and `False` also mean false. An unset or empty variable means +the default. Any other value, such as `PORT=http`, +`DNSWATCHER_TLS_EXPIRY_WARNING=7d` or `DNSWATCHER_DEBUG=yes`, makes dnswatcher +refuse to start with an error naming the variable and the value. + +**`DNSWATCHER_SENTRY_DSN` reports crashes in HTTP requests to Sentry.** When it +is set, a panic in an HTTP request handler is sent to Sentry, and the request +still gets a `500 Internal Server Error` answer. Nothing else is sent to Sentry: +DNS, port and TLS problems are reported as notifications. A value Sentry cannot +parse stops dnswatcher at startup. At shutdown, reports not yet sent are sent, +waiting at most 2 seconds. ### Example `.env` @@ -277,99 +526,185 @@ DNSWATCHER_SEND_TEST_NOTIFICATION=true ## DNS Resolution Strategy -dnswatcher never uses the system's configured recursive resolver. Instead, -it performs full iterative resolution: +dnswatcher never uses the system's configured recursive resolver for the names +it watches. Instead, it performs full iterative resolution: -1. **Root servers**: Starts from the IANA root nameserver list (hardcoded, - with periodic refresh). +1. **Root servers**: Starts from the IPv4 addresses of the 13 root servers, + built into the binary; the list is not refreshed. 2. **TLD delegation**: Queries root servers for the TLD NS records. -3. **Domain delegation**: Queries TLD nameservers for the domain's NS - records. -4. **Authoritative query**: Queries all discovered authoritative - nameservers directly for the requested records. +3. **Domain delegation**: Queries TLD nameservers for the domain's NS records. + The delegation they give, from the domain's parent zone, is the domain's NS + record set. +4. **Authoritative query**: Queries all discovered authoritative nameservers + directly for the requested records. + +In steps 2 and 3 the servers are asked one at a time in a random order, chosen +anew each time, so no one root server gets every first query. A server that does +not reply, refuses the query, or gives an error reply such as SERVFAIL or a +referral that leads no closer to the name is passed over for the next one. When +a referral names a zone's nameservers without their addresses, the addresses of +all of them are looked up, so that each can be asked. When it gives addresses +for only some of them, those are asked first, and the others are looked up and +asked only if none of those gives a usable reply. Both hold in the walk to a +name's nameservers and in the lookup of a nameserver's own address. Such a +lookup can need others in turn; lookups go at most three deep, one inside +another, so delegations that point at each other still end. + +In step 4 each record type is asked at the IPv4 addresses the nameserver's name +resolves to, one at a time, until one gives a usable reply; the next type is +asked first at that address. A type's query fails only when none of them gives +one, and the nameserver has failed only when that is so for every type. This approach ensures: -- Independence from any upstream resolver's cache or filtering. -- Ability to detect split-horizon or inconsistent responses across - authoritative servers. -- Visibility into the full delegation chain. -For hostname monitoring, the resolver follows CNAME chains (with a -depth limit to prevent loops) before collecting terminal A/AAAA records. +- Independence from any upstream resolver's cache or filtering. +- Ability to detect split-horizon or inconsistent responses across authoritative + servers. + +A root server never offers recursion, so a reply marked "recursion available" +from a root server's address came from a recursive resolver on the network +answering in its place, as on a network where a router rule, a VPN or a DNS +filter sends all DNS to one resolver. The first such reply from a root server's +address, a refusal included, stops the lookup at once with the error "this +network intercepts DNS queries", naming that address, and no other root server +is asked. A lookup that every root server refuses without that mark fails with +the same error. Every lookup starts at the root servers, so on such a network +every lookup fails with that error, whether or not that resolver has the name +cached, and dnswatcher's tests fail there too. That is by design: nothing asks +again another way or otherwise works around the interception. + +A watched name's records are stored as its nameservers return them, CNAME +included. When they return a CNAME and no address, the chain of every CNAME +target they gave is followed (with a depth limit to prevent loops) to the A and +AAAA records at its end, and the port and TLS checks use those addresses. +Nameservers' addresses are also found by following CNAME chains. + +Sending a notification or a Sentry report is the one use of the system's +resolver: the HTTP client looks up the webhook's or Sentry's host name with it. --- ## State File Format The state file (`DATA_DIR/state.json`) contains the complete monitoring -snapshot. Hostname records are stored **per authoritative nameserver**, -not as a merged view, to enable inconsistency detection. +snapshot. Hostname records are stored **per authoritative nameserver**, not as a +merged view, to enable inconsistency detection. `hostnames` also holds each +domain's own records, under the domain's name. ```json { - "version": 1, - "lastUpdated": "2026-02-19T12:00:00Z", - "domains": { - "example.com": { - "nameservers": ["ns1.example.com.", "ns2.example.com."], - "lastChecked": "2026-02-19T12:00:00Z" - } - }, - "hostnames": { - "www.example.com": { - "recordsByNameserver": { - "ns1.example.com.": { - "records": { - "A": ["93.184.216.34"], - "AAAA": ["2606:2800:220:1:248:1893:25c8:1946"] - }, - "status": "ok", - "lastChecked": "2026-02-19T12:00:00Z" - }, - "ns2.example.com.": { - "records": { - "A": ["93.184.216.34"], - "AAAA": ["2606:2800:220:1:248:1893:25c8:1946"] - }, - "status": "ok", - "lastChecked": "2026-02-19T12:00:00Z" + "version": 1, + "lastUpdated": "2026-02-19T12:00:00Z", + "domains": { + "example.com": { + "nameservers": ["ns1.example.com.", "ns2.example.com."], + "nameserverAddresses": { + "ns1.example.com.": ["192.0.2.53", "2001:db8::53"], + "ns2.example.com.": ["198.51.100.53"] + }, + "lastChecked": "2026-02-19T12:00:00Z" } - }, - "lastChecked": "2026-02-19T12:00:00Z" - } - }, - "ports": { - "93.184.216.34:80": { - "open": true, - "hostnames": ["www.example.com"], - "lastChecked": "2026-02-19T12:00:00Z" }, - "93.184.216.34:443": { - "open": true, - "hostnames": ["www.example.com"], - "lastChecked": "2026-02-19T12:00:00Z" + "hostnames": { + "www.example.com": { + "recordsByNameserver": { + "ns1.example.com.": { + "records": { + "A": ["93.184.216.34"], + "AAAA": ["2606:2800:220:1:248:1893:25c8:1946"] + }, + "status": "ok", + "lastChecked": "2026-02-19T12:00:00Z" + }, + "ns2.example.com.": { + "records": { + "A": ["93.184.216.34"], + "AAAA": ["2606:2800:220:1:248:1893:25c8:1946"] + }, + "status": "ok", + "lastChecked": "2026-02-19T12:00:00Z" + } + }, + "cnameAddresses": [], + "lastChecked": "2026-02-19T12:00:00Z" + } + }, + "ports": { + "93.184.216.34:80": { + "open": true, + "hostnames": ["www.example.com"], + "lastChecked": "2026-02-19T12:00:00Z" + }, + "93.184.216.34:443": { + "open": true, + "hostnames": ["www.example.com"], + "lastChecked": "2026-02-19T12:00:00Z" + } + }, + "certificates": { + "93.184.216.34:443:www.example.com": { + "commonName": "www.example.com", + "issuer": "DigiCert TLS RSA SHA256 2020 CA1", + "notAfter": "2027-01-15T23:59:59Z", + "subjectAlternativeNames": ["www.example.com"], + "status": "ok", + "lastChecked": "2026-02-19T06:00:00Z" + } } - }, - "certificates": { - "93.184.216.34:443:www.example.com": { - "commonName": "www.example.com", - "issuer": "DigiCert TLS RSA SHA256 2020 CA1", - "notAfter": "2027-01-15T23:59:59Z", - "subjectAlternativeNames": ["www.example.com"], - "status": "ok", - "lastChecked": "2026-02-19T06:00:00Z" - } - } } ``` -The `status` field for each per-nameserver entry and certificate entry -tracks reachability: +The `status` field for each per-nameserver entry and certificate entry tracks +reachability: -| Status | Meaning | -|-------------|-------------------------------------------------| -| `ok` | Query succeeded, records are current | -| `error` | Query failed (timeout, SERVFAIL, network error) | +| Status | Meaning | +| ------- | -------------------------------------------------------- | +| `ok` | Query succeeded, records are current except as below | +| `error` | Query failed (timeout, SERVFAIL, REFUSED, network error) | + +A nameserver that answers NXDOMAIN or with no records has status `ok` and empty +`records`. A nameserver whose query failed, or that only referred it to other +nameservers, has status `error`, empty `records`, and the reason in `error`. A +certificate entry whose TLS connection or handshake failed likewise has status +`error`, the reason in `error`, and the certificate fields left empty or zero. + +A nameserver with status `ok` whose query for one record type failed lists that +type in `failedTypes` and holds its records from the previous check, which may +not be current. When the previous check did not know the type's records either, +because the nameserver was new or failing then or the type was already listed in +`unknownTypes`, the type is also listed in `unknownTypes`, and `records` holds +nothing for it. Both lists are left out when empty. + +`nameserverAddresses` lists, by nameserver, the sorted addresses its name +resolves to. A state file without it loads, and the next check fills it in +without a notification. + +A domain entry has `"nxdomain": true` when the domain's parent zone's servers +answered NXDOMAIN, that it does not exist. Its `nameservers` and +`nameserverAddresses` are then empty, and `hostnames` holds no entry for it. +`nxdomain` is left out when false. + +`cnameAddresses` lists the sorted addresses at the end of the chain of every +CNAME target a hostname's nameservers gave, found when they answered with a +CNAME and no address; it is empty when they answered with an address. When a +chain cannot be followed, or none of the name's nameservers answered its queries +for A, AAAA and CNAME, the previous check's list is kept, or `null` when no +earlier check saved one. A state file without it loads, and the first check +after that saves it without a notification. + +A port entry's key is its address and port, and a certificate entry's key its +address, port 443 and hostname, joined by colons. An IPv6 address is written in +brackets in these keys, as in `[2001:db8::1]:443` and +`[2001:db8::1]:443:www.example.com`. `/api/v1/status` and the dashboard show the +same keys. + +A port entry's `hostnames` lists every name that resolves to its address, +domains included. A port entry in the older format, with one `hostname` instead +of the `hostnames` list, loads as a list of that one name. + +A state file that leaves out any of `domains`, `hostnames`, `ports` and +`certificates`, or holds `null` for one, loads with that section empty. A file +holding `{}` loads as empty state. --- @@ -378,56 +713,79 @@ tracks reachability: This repository adheres to the [Scripts to Rule Them All](https://github.com/github/scripts-to-rule-them-all) standard: normalized scripts in `script/` are the entrypoints for the -development workflow, and the Makefile targets are thin shims that call -them. We provide: +development workflow, and the Makefile targets are thin shims that call them. We +provide: -- `script/bootstrap` — install all dependencies (go, pinned goimports, - `go mod download`). It does not install golangci-lint: see - `script/lint` below. -- `script/setup` — make a fresh clone ready for development: bootstrap - plus the git pre-commit hook -- `script/projectname` — print the project name (used for the Docker - image tag) -- `script/test` — run the test suite (race detector, coverage). Caching - is waived for testing, exactly as it is for linting: `-count=1` - forces every invocation to execute, because the suite queries live - DNS and a cached pass queries nothing. Failures are rerun with `-v` - automatically, and the build fails even if that rerun passes. -- `script/lint` — run golangci-lint, always inside Docker: it builds - `Dockerfile.lint`, which COPYs the repo into the digest-pinned - `golangci-lint` image and lints as a build step, so a successful - build is a clean lint. The linter is never installed or run on the - host, and Docker is the only prerequisite. Caching is waived for - linting: the lint stage is forced to execute on every run with - `--no-cache-filter`, because a cached build lints nothing. -- `script/fmt` — format all code (gofmt -s, goimports) -- `script/fmt-check` — check formatting (read-only) +- `script/bootstrap` — install all dependencies (go, `go mod download`). It does + not install golangci-lint or prettier: both run in Docker, see `script/lint` + and `script/fmt` below. +- `script/setup` — make a fresh clone ready for development: bootstrap plus the + git pre-commit hook +- `script/projectname` — print the project name (used for the Docker image tag) +- `script/test` — run the test suite (race detector, coverage) by building the + `Dockerfile`'s test phase, which queries live DNS. `-count=1` keeps Go's test + result cache out. Failures are rerun with `-v` automatically, and the build + fails even if that rerun passes. +- `script/lint` — run golangci-lint by building the `Dockerfile`'s lint phase, + on the digest-pinned `golangci-lint` image, so a successful build is a clean + lint. The linter is never installed or run on the host, and Docker is the only + prerequisite. +- `script/fmt` — format all code (gofmt -s, goimports) and all Markdown + (prettier). goimports runs with `go run` at a pinned commit, never from your + `PATH`. prettier runs inside Docker, in a stage of the `Dockerfile` on a + digest-pinned node image, at the version pinned by `package.json` and + `yarn.lock`; it is never installed on the host. +- `script/fmt-check` — check formatting (read-only) with the same tools, failing + on any file `script/fmt` would change: gofmt and goimports on the host, + prettier inside Docker - `script/check` — run test, lint, and fmt-check -- `script/docker` — build the Docker image tagged via - `script/projectname` -- `script/cibuild` — CI entrypoint: plain `docker build .` -- `script/precommit` — run by the git pre-commit hook; `go mod tidy` - guard, then `script/check` +- `script/docker` — build the Docker image tagged via `script/projectname`, with + the version from `git describe` passed as `--build-arg VERSION` +- `script/cibuild` — CI entrypoint: `script/bootstrap`, then `script/check`, + then the image build `script/docker` does, which runs the lint and test phases + again +- `script/precommit` — run by the git pre-commit hook; `go mod tidy` guard, then + `script/check` - `script/install-precommit` — install the git pre-commit hook +Linting and testing are phases of the `Dockerfile`, and the image is built only +when both pass. Every `docker build` in `script/` passes `--no-cache`, because a +cached build lints nothing and queries no DNS. + ## Building ```sh make build # Build binary to bin/dnswatcher -make test # Run tests with race detector +make version # Print the version make build stamps +make test # Run tests with race detector in Docker (requires docker) make lint # Run golangci-lint in Docker (requires docker) -make fmt # Format code +make fmt # Format code and Markdown (requires docker) make check # Run all checks (test, lint, fmt-check) make clean # Remove build artifacts ``` ### Build-Time Variables -Version is injected via `-ldflags`: +`make build` sets the version with `-ldflags "-X main.Version=..."`, taking it +from `VERSION` when given on the command line (`make build VERSION=1.2.3`) or in +the environment, otherwise from `git describe --tags --always --dirty`, and +`dev` without git metadata. An empty `VERSION` counts as not given. The version +appears in the startup log and in the health check response. -```sh -go build -ldflags "-X main.Version=$(git describe --tags --always)" ./cmd/dnswatcher -``` +The image takes it from `git describe --tags --always` on the `.git` the build +context carries, so a plain `docker build .` of a clone stamps the commit it was +built from; a clone without tags stamps the short commit. A clone made with +`--depth 1` carries at most a tag on its own commit, so such a clone of an +untagged commit stamps the short commit. In a build from a directory, +`.dockerignore` keeps out `.git/config` and every submodule's git config, which +`git describe` does not need and which can hold a credential. Docker does not +apply `.dockerignore` to a context sent as a tar archive, as upaas sends it, so +that context carries `.git/config` into the build. It also keeps its files' +owners, so git in the build trusts the checkout whoever owns it. A non-empty +`--build-arg VERSION=...` takes precedence; `make docker` passes the version +`git describe` gives on the host. The build fails when the context carries +`.git`, as a directory or as a file, and the version comes out empty, `dev` or +`unknown`. --- @@ -446,48 +804,173 @@ docker run -d \ --- +## Running under upaas + +[upaas](https://git.eeqj.de/sneak/upaas) builds the image from this repository's +`Dockerfile` and runs it. The app needs: + +- **Branch:** `prod`. `prod` is cut from `main`, and merging a `main` to `prod` + pull request is a deploy. +- **Volume:** one host directory mounted at `/var/lib/dnswatcher`, where the + state file lives. +- **Network and port:** the dashboard is unauthenticated and shows every watched + name and recent alert, and upaas publishes every mapped port on all interfaces + of the host ([upaas issue 113](https://git.eeqj.de/sneak/upaas/issues/113)). + Add a port mapping to container port `8080` only if the dashboard should be + public. Otherwise add none: set the app's Docker network in upaas to your + reverse proxy's Docker network, and the proxy reaches the app at `upaas-` + followed by the app name, port `8080`. +- **Required environment:** `DNSWATCHER_TARGETS`, a comma-separated list of the + domains and hostnames to watch. dnswatcher refuses to start without it. +- **Recommended environment:** at least one notification endpoint + (`DNSWATCHER_SLACK_WEBHOOK`, `DNSWATCHER_MATTERMOST_WEBHOOK`, + `DNSWATCHER_NTFY_TOPIC`); without one, changes show only on the dashboard. + `DNSWATCHER_METRICS_USERNAME` and `DNSWATCHER_METRICS_PASSWORD` serve + `/metrics` behind basic auth. +- **Leave unset:** `DNSWATCHER_DATA_DIR`, which the image sets to + `/var/lib/dnswatcher`, and `PORT`, which defaults to `8080`. Every setting + comes from the environment; the image holds no config file. +- **Health check:** the image's own, which requests `/.well-known/healthcheck` + every 10 seconds. upaas reads the container's health 60 seconds after a deploy + and marks the deploy failed unless it is `healthy`. + +--- + ## Monitoring Lifecycle -1. **Startup**: Load state from disk. If no state file exists, start - with empty state (first check will establish baseline without - triggering change notifications). -2. **Initial check**: Immediately perform all DNS, port, and TLS checks - on startup. +1. **Startup**: Check that the data directory can be written, and exit with an + error naming it if not. Load state from disk. If no state file exists, start + with empty state (first check will establish baseline without triggering + change notifications). Remove from the state the names no longer in + `DNSWATCHER_TARGETS` (see State Management). +2. **Initial check**: Immediately perform all DNS, port, and TLS checks on + startup. 3. **Periodic checks** (DNS always runs first): - - DNS checks: every `DNSWATCHER_DNS_INTERVAL` (default 1h). Also - re-run before every TLS check cycle to ensure fresh IPs. - - Port checks: every `DNSWATCHER_DNS_INTERVAL`, after DNS completes. - - TLS checks: every `DNSWATCHER_TLS_INTERVAL` (default 12h), after - DNS completes. - - Port and TLS checks always use freshly resolved IP addresses from - the DNS phase that immediately precedes them — never stale IPs - from a previous cycle. -4. **On change detection**: Send notifications to all configured - endpoints, update in-memory state, persist to disk. -5. **Shutdown**: Persist final state to disk, wait for in-flight - notification deliveries to complete, stop gracefully. The wait is - bounded by the fx shutdown timeout (15s by default): deliveries still - retrying against an unreachable endpoint when that expires are - abandoned, and the number abandoned is logged at warn level rather - than dropped silently. Notifications generated after shutdown has - begun are refused and logged, so a late burst cannot extend the - shutdown. + - DNS checks: every `DNSWATCHER_DNS_INTERVAL` (default 1h). Also re-run + before every TLS check cycle to ensure fresh IPs. + - Port checks: every `DNSWATCHER_DNS_INTERVAL`, after DNS completes. + - TLS checks: every `DNSWATCHER_TLS_INTERVAL` (default 12h), after DNS + completes. + - Port and TLS checks use the IP addresses found by the DNS phase that + immediately precedes them. When that phase cannot find a name's + nameservers at all, the addresses an earlier check saved for the name are + used. When it cannot follow a name's CNAME chain, or none of the name's + nameservers answered, the addresses an earlier check found at the end of + the chain are used. +4. **On change detection**: Send notifications to all configured endpoints, + update in-memory state, persist to disk. A record change or inconsistency for + a domain or hostname listed in `DNSWATCHER_SKIP_RECORD_NOTIFICATIONS` sends + no notification, but the state is still updated and saved. +5. **Shutdown**: The watcher stops checking and saves the final state to disk, + and shutdown waits for that save before it goes on. Then it waits for + in-flight notification deliveries to complete. Both waits share the fx + shutdown timeout (15s by default): deliveries still retrying against an + unreachable endpoint when that expires are abandoned, and the number + abandoned is logged at warn level rather than dropped silently. Notifications + generated after shutdown has begun are refused and logged, so a late burst + cannot extend the shutdown. A DNS lookup, port check or TLS check that + shutdown cuts short saves nothing and sends no notification. --- ## Planned Future Features (Post-1.0) -- **DNSSEC validation**: Validate the DNSSEC chain of trust during - iterative resolution and report DNSSEC failures as notifications. +- **DNSSEC validation**: Validate the DNSSEC chain of trust during iterative + resolution and report DNSSEC failures as notifications. --- ## Project Structure Follows the conventions defined in `REPO_POLICIES.md`, adapted from the -[upaas](https://git.eeqj.de/sneak/upaas) project template. Uses uber/fx -for dependency injection, go-chi for HTTP routing, slog for logging, and -Viper for configuration. +[upaas](https://git.eeqj.de/sneak/upaas) project template. Uses uber/fx for +dependency injection, go-chi for HTTP routing, slog for logging, and Viper for +configuration. + +--- + +## Rationale + +dnswatcher exists to report changes to the DNS records, TCP port availability +and TLS certificates of its configured domains and hostnames, failures and +recoveries included: it is designed as a real-time change feed. It queries the +authoritative nameservers directly, tracing from the root, instead of a +recursive resolver, so no resolver's cache or filtering hides a change and +nameservers that disagree with each other are seen. Its state is a single JSON +file, so it survives a restart without an external database. + +--- + +## Design + +``` +cmd/dnswatcher/main.go Entry point (uber/fx bootstrap) + +internal/ + config/ + config.go Viper-based configuration + classify.go Splits targets into domains and hostnames + (Public Suffix List) + globals/globals.go Build-time variables (version) + logger/logger.go slog structured logging (TTY detection) + healthcheck/healthcheck.go Health check service + middleware/middleware.go HTTP middleware (logging, CORS, security + headers, metrics auth and rate limit) + handlers/ + handlers.go Shared handler setup and JSON responses + dashboard.go Web dashboard + templates/dashboard.html Dashboard template (embedded) + status.go /api/v1/status + healthcheck.go Health check handler + static.go Static assets under /s/ + server/ + server.go HTTP server lifecycle + routes.go Route definitions + state/state.go JSON file state persistence + resolver/ + resolver.go Resolver setup and query status values + iterative.go Iterative DNS resolution engine + dns_client.go UDP and TCP DNS clients + errors.go Resolver errors + portcheck/portcheck.go TCP port connectivity checker + tlscheck/tlscheck.go TLS certificate inspector + notify/ + notify.go Notification service (Slack, Mattermost, ntfy) + retry.go Delivery retries with backoff + history.go Last 100 notifications, for the dashboard + shutdown.go Waits for deliveries at shutdown + watcher/ + watcher.go Main monitoring orchestrator and scheduler + interfaces.go The resolver, checkers and notifier it uses + livednstest/livednstest.go Retry and concurrency limit for tests + against live DNS (imported only by tests) + +static/ + static.go Embeds the CSS served under /s/ + css/tailwind.min.css Dashboard stylesheet +``` + +### Design Principles + +- **No recursive resolvers**: The watched names are resolved iteratively, + tracing from root nameservers through the delegation chain to authoritative + servers. +- **No external database**: State is persisted as a single JSON file. +- **Dependency injection**: All components are wired via + [uber/fx](https://github.com/uber-go/fx). +- **Structured logging**: All logs use `log/slog` with JSON output in production + (TTY detection for development). +- **Graceful shutdown**: All background goroutines respect context cancellation + and the fx lifecycle. In-flight notification deliveries are drained on + shutdown, bounded by the shutdown timeout. + +--- + +## TODO + +[`TODO.md`](./TODO.md) names the next step and the steps planned after it. The +work for 1.0 is tracked as issues on the +[1.0 milestone](https://git.eeqj.de/sneak/dnswatcher/milestone/7). --- diff --git a/REPO_POLICIES.md b/REPO_POLICIES.md index 9aba6b0..20382d1 100644 --- a/REPO_POLICIES.md +++ b/REPO_POLICIES.md @@ -1,6 +1,6 @@ --- title: Repository Policies -last_modified: 2026-08-07 +last_modified: 2026-10-04 --- This document covers repository structure, tooling, and workflow standards. Code @@ -60,17 +60,28 @@ style conventions are in separate documents: prerequisite since nvm requires bash. yarn is then pinned via `corepack prepare yarn@ --activate`. Never install "latest" or "lts"; always exact versions. `script/cibuild` runs the CI build: it changes to the - repo root and runs `docker build .`; the Gitea workflow calls it. Four further - scripts are our own extensions to the standard: `script/check` runs - `script/test`, `script/lint`, and `script/fmt-check`; `script/precommit` is - what the git pre-commit hook runs, and it calls `script/check`; - `script/install-precommit` installs the git pre-commit hook (the `make hooks` - target shims to it); and `script/projectname` (literally that filename) simply - outputs the project's name. Scripts that need the name call - `script/projectname` — e.g. `script/docker` assembles its image tag from it — - so those scripts stay byte-identical across all repos. Repo-type-specific - pre-commit extras (e.g. `go mod tidy` verification in Go repos) belong in - `script/precommit`, not in the hook itself. Model scripts are at + repo root, runs `script/bootstrap`, runs `script/check`, and builds the image + with the version; the Gitea workflow calls it. **`script/cibuild` runs + `script/bootstrap` first**, because the workflow checks out the repo and runs + nothing else, while `script/fmt-check` runs the formatter on the host: on a + pristine checkout with nothing installed the run dies there, after the + containerised gates have passed. **The bootstrap alone is not enough**: + `script/bootstrap` installs node and yarn under nvm and leaves neither on the + `PATH` of the shell that called it, so a bare `yarn` still exits 127. The host + entrypoints that need yarn — `script/fmt` and `script/fmt-check` — therefore + source nvm for the pinned node version before invoking it, exactly as + `script/bootstrap`'s own install step does. A runner carrying nothing but + docker and git then gets through `script/check`. Four further scripts are our + own extensions to the standard: `script/check` runs `script/test`, + `script/lint` and `script/fmt-check`; `script/precommit` is what the git + pre-commit hook runs, and it calls `script/check`; `script/install-precommit` + installs the git pre-commit hook (the `make hooks` target shims to it); and + `script/projectname` (literally that filename) simply outputs the project's + name. Scripts that need the name call `script/projectname` — e.g. + `script/docker` assembles its image tag from it — so those scripts stay + byte-identical across all repos. Repo-type-specific pre-commit extras (e.g. + `go mod tidy` verification in Go repos) belong in `script/precommit`, not in + the hook itself. Model scripts are at `https://git.eeqj.de/sneak/prompts/raw/branch/main/script/`. The README must document the provided scripts in an **Entrypoints** section (see the README requirements below). @@ -89,87 +100,198 @@ style conventions are in separate documents: contributor should be able to understand the entire development workflow by reading the Makefile. -- Every repo should have a `Dockerfile`. All Dockerfiles must run `make check` - as a build step so the build fails if the branch is not green. For non-server - repos, the Dockerfile should bring up a development environment and run - `make check`. For server repos, `make check` should run as an early build - stage before the final image is assembled. Dockerfiles install development - prerequisites by running `script/bootstrap` rather than duplicating installs - inline; COPY `script/` and the dependency manifests (`package.json` + - `yarn.lock`, `go.mod` + `go.sum`, etc.) before running it so the bootstrap - layer stays cached until dependencies change. +- Every repo should have a `Dockerfile`, and it carries the repo's gates: a + `lint` phase and a `test` phase, with the final stage depending on both so the + image cannot be built unless they pass. For non-server repos the final stage + brings up a development environment; for server repos it is the runtime image. + The gate phases and the build stage start from their pinned base images and + install what those images lack either inline, as the canonical Go `Dockerfile` + below does for `git`, or by running `script/bootstrap`, as the `prompts` + repo's own `Dockerfile` does for its yarn packages. The development + environment stage installs development prerequisites by running + `script/bootstrap` rather than duplicating its installs inline. A stage that + runs `script/bootstrap` COPYs `script/` and the dependency manifests + (`package.json` + `yarn.lock`, `go.mod` + `go.sum`, etc.) before running it. -- **Dockerfiles must use a separate lint stage for fail-fast feedback.** Go - repos use a multistage build where linting runs in an independent stage based - on the `golangci/golangci-lint` image (pinned by hash). This stage runs - `make fmt-check` and `make lint` before the full build begins. The build stage - then declares an explicit dependency on the lint stage via - `COPY --from=lint /src/go.sum /dev/null`, which forces BuildKit to complete - linting before proceeding to compilation and tests. This ensures lint failures - surface in seconds rather than minutes, without blocking on dependency - download or compilation in the build stage. +- **Linting and testing run in Docker, as phases of the `Dockerfile`.** There is + no separate lint file. `script/lint` and `script/test` each build one phase + and nothing else: - The standard pattern for a Go repo Dockerfile is: + ```sh + docker build --no-cache --target lint -t "$(script/projectname)-lint" . + docker build --no-cache --target test -t "$(script/projectname)-test" . + ``` + + **A stage that is not the last one in the file is built only when the final + stage's chain depends on it, or when `--target` names it.** That is why the + two gates are always invoked by name here, and why the final stage carries a + `COPY --from=` of a harmless file from each of them: without that edge a + plain `docker build .` builds the last stage alone and exits 0 having linted + and tested nothing. + + **Every `docker build` in `script/` is tagged**, here and in + `script/cibuild` and `script/docker`. An untagged build leaves a dangling + image behind on every invocation, on every developer host and every CI + runner; a tagged one replaces the previous image. + + Inside a phase the tool is invoked directly — `golangci-lint`, `go test`, + `eslint`, `prettier` — never through `make lint` or `script/test`, which are + themselves a `docker build` and would recurse into a daemon that does not + exist in a build step. Formatting is the exception and stays on the host: + `script/fmt` writes the working tree, and `script/fmt-check` is its + read-only twin. + + **No lint verdict may come from a host invocation of the linter.** On a + shared host golangci-lint reads a result cache keyed on file content rather + than location, so a second checkout of the same content is served the first + one's findings, and a host-global lock in `$TMPDIR` makes concurrent runs + exit non-zero with `parallel golangci-lint is running` — a status a caller + cannot tell from real findings. Both have produced wrong verdicts in this + org, in both directions. A container has its own cache, its own `TMPDIR` and + a digest-pinned binary, so neither is reachable. + +- **Any build that runs checks is built with `--no-cache`.** Docker invalidates + a `COPY` layer only when the copied content changes, so on an unchanged tree + the check `RUN` is served from cache, nothing executes, and the build still + exits 0. Every `docker build` in `script/` therefore passes `--no-cache`: + `script/lint`, `script/test`, `script/cibuild` and `script/docker` are the + four, and there is no fifth — `script/check` runs the two gate phases and + `script/fmt-check`, and builds no image of its own. A bare `docker build .` is + not evidence that anything ran: a sub-second build reporting success is a + cache hit, not a result. Never invalidate by pruning — `docker builder prune` + and friends destroy a build cache shared with every other build on the host. + When a check is added or changed, prove it works by planting a defect it must + catch and watching the run fail on it, then revert the defect. A green run + alone shows neither that the check ran nor that it covers what it should. + +- **The gate phases are separate stages, and the build stage depends on both.** + The lint phase is based on the `golangci/golangci-lint` image (pinned by + hash), so lint failures surface in seconds rather than after a full compile, + and the test phase is based on the Debian Go image. The canonical Go repo + `Dockerfile`: ```dockerfile - # Lint stage — fast feedback on formatting and lint issues + # Lint phase # golangci/golangci-lint:v2.x.x, YYYY-MM-DD FROM golangci/golangci-lint@sha256:... AS lint WORKDIR /src COPY go.mod go.sum ./ RUN go mod download COPY . . - RUN make fmt-check - RUN make lint + RUN golangci-lint run --config .golangci.yml ./... - # Build stage - # golang:1.x-alpine, YYYY-MM-DD - FROM golang@sha256:... AS builder + # Test phase. -race needs cgo and so a C compiler, which the Debian Go + # image ships and the alpine one does not. + # golang:1.x, YYYY-MM-DD + FROM golang@sha256:... AS test WORKDIR /src - - # Force BuildKit to run the lint stage before proceeding - COPY --from=lint /src/go.sum /dev/null - COPY go.mod go.sum ./ RUN go mod download COPY . . - RUN make test + RUN go test -timeout 90s -race -cover ./... || \ + { echo "--- Rerunning with -v for details ---"; \ + go test -timeout 90s -race -v ./...; exit 1; } - ARG VERSION=dev - RUN CGO_ENABLED=0 go build -trimpath \ - -ldflags="-s -w -X main.Version=${VERSION}" \ - -o /app ./cmd/app/ + # Build stage. Nothing is wanted from either phase above; the copies + # are what make BuildKit build them first, so this stage cannot run + # unless lint and test passed. + # golang:1.x-alpine, YYYY-MM-DD + FROM golang@sha256:... AS builder + COPY --from=lint /src/go.sum /dev/null + COPY --from=test /src/go.sum /dev/null + RUN apk add --no-cache git + # A tar-stream context keeps the sender's file owners, which git refuses. + RUN git config --system --add safe.directory /src + WORKDIR /src + COPY go.mod go.sum ./ + RUN go mod download + COPY . . - # Runtime stage + # The VERSION build arg when one is given, otherwise + # `git describe --tags --always` on the .git in the build context. With + # .git present, a version that is still empty, dev or unknown fails the + # build: git is missing or could not read the checkout. + ARG VERSION + RUN VERSION="${VERSION:-$(git describe --tags --always)}"; \ + if [ -e .git ]; then \ + case "$VERSION" in ""|dev|unknown) \ + echo "version is '$VERSION' although .git is present" >&2; \ + exit 1 ;; \ + esac; \ + fi; \ + CGO_ENABLED=0 go build -trimpath \ + -ldflags="-s -w -X main.Version=${VERSION}" \ + -o /app ./cmd/app/ + + # Runtime stage, and the last one FROM alpine@sha256:... COPY --from=builder /app /usr/local/bin/app ENTRYPOINT ["app"] ``` Key points: - - The lint stage uses the `golangci/golangci-lint` image directly (it - includes both Go and the linter), so there is no need to install the - linter separately. - - `COPY --from=lint /src/go.sum /dev/null` is a no-op file copy that creates - a stage dependency. BuildKit runs stages in parallel by default; without - this line, the build stage would not wait for lint to finish and a lint - failure might not fail the overall build. + - The lint phase uses the `golangci/golangci-lint` image directly (it has + both Go and the linter), so nothing needs installing. + - `COPY --from= /src/go.sum /dev/null` is a no-op copy whose only + purpose is the ordering edge. BuildKit runs stages in parallel by default, + and a stage nothing depends on is not built at all, so without these two + lines a red gate would not fail the build. + - Keep the runtime stage last, and if you add a stage after it, give it the + same two copies. A plain `docker build .` builds the last stage's chain + and nothing else. - If the project uses `//go:embed` directives that reference build artifacts - (e.g. a web frontend compiled in a separate stage), the lint stage must + (e.g. a web frontend compiled in a separate stage), the lint phase must create placeholder files so the embed directives resolve. Example: `RUN mkdir -p web/dist && touch web/dist/index.html web/dist/style.css`. - The lint stage should not depend on the actual build output — it exists to - fail fast. - - If the project requires CGO or system libraries for linting (e.g. - `vips-dev`), install them in the lint stage with `apk add`. - - The build stage runs `make test` after compilation setup. Tests run in the - build stage, not the lint stage, because they may require compiled - artifacts or heavier dependencies. + - If the project requires CGO or system libraries for linting, install them + in the lint phase. The `golangci/golangci-lint` image is Debian-based and + has no `apk`, so install with `apt-get` under the Debian package name + (`libvips-dev`, where alpine says `vips-dev`), and delete the package + lists in the same `RUN`, so the layer does not keep them: + + ```dockerfile + RUN apt-get update \ + && apt-get install -y --no-install-recommends libvips-dev \ + && rm -rf /var/lib/apt/lists/* + ``` + + - `.dockerignore` lets `.git` into the build context. It keeps out every git + `config` at any depth (`**/.git/config`, `**/.git/modules/**/config`): the + repository's own, each submodule's under `.git/modules/`, and that of a + submodule keeping its own `.git` directory. `git describe` does not need + them, and each can hold a credential: a password in a remote URL, or the + token the CI checkout step stores there. A submodule whose name has a + `config` segment (`config`, `deploy/config`, `config/lib`) loses its whole + git directory to `**/.git/modules/**/config`, and Go's version stamping + then fails the build: give it a name without that segment + (`git submodule add --name`). The stage that compiles has `git` (the + Debian Go image has it; an alpine one needs `apk add --no-cache git`) and + takes the version from the `VERSION` build argument when one is given, + otherwise from `git describe --tags --always`. That gives the tag on a + tagged commit; on a later commit, the tag, the number of commits since it + and the short commit (`v1.2.3-4-gabc1234`); and the short commit when no + tag is reachable. The stage that compiles also marks its working directory + safe for git (`git config --system --add safe.directory /src`): a context + sent as a tar stream keeps the sender's file owners, and git refuses a + checkout owned by another user, so the version would come out empty. + `ARG VERSION` has no default, and the build fails if the context carries + `.git` and the version still comes out empty, `dev` or `unknown`. A plain + `docker build .` with no build arguments must succeed; a Dockerfile that + refuses an empty build argument drops that refusal and keeps the argument. - Every repo should have a Gitea Actions workflow (`.gitea/workflows/`) that - runs `script/cibuild` (which runs `docker build .`) on push. Since the - Dockerfile already runs `make check`, a successful build implies all checks - pass. + runs `script/cibuild` on push, and checks out the repo as its only other step. + That script bootstraps, runs the gate phases, and then builds the image, so a + successful run means every check passed; a bare `docker build .` does not + carry the same guarantee, because its gate phases may come from the cache. The + image build is uncached and so runs the gate phases a second time. That is the + price of the rule above, and it is worth paying: the image that ships is built + from a run of its own gates rather than from a cache entry. A separate + workflow limited to `main` by a `branches` list under `on: push` cannot be + checked by review: to try a change to it, add the feature branch to that list + and push, then remove the branch from the list again before merging. Keep any + job in it that publishes behind `if: github.ref_name == 'main'`, so the run + from the feature branch publishes nothing. - Use platform-standard formatters: `black` for Python, `prettier` for JS/CSS/Markdown/HTML, `go fmt` for Go. Always use default configuration with @@ -193,15 +315,17 @@ style conventions are in separate documents: suite that exceeds it fails. Under 20 seconds is the target. A suite between 20 and 60 seconds is still green, but the overage must be filed as an improvement bug against that repo. Add a 90-second timeout to the test - invocation in the Makefile (`go test -timeout 90s`). The backstop deliberately - sits above the hard cap so that it catches a genuinely hung test rather than a - merely slow one. + invocation (`go test -timeout 90s`). The backstop deliberately sits above the + hard cap so that it catches a genuinely hung test rather than a merely slow + one. -- **`make test` should use the conditional verbose rerun pattern.** Run tests - without `-v` (verbose) first. If tests fail, automatically rerun with `-v` to - show full output. This keeps CI logs and `docker build` output clean on - success (just package/suite summaries) while providing full diagnostic detail - on failure (every test case, every assertion). The general shell pattern: +- **The test command should use the conditional verbose rerun pattern.** Run + tests without `-v` (verbose) first. If tests fail, automatically rerun with + `-v` to show full output. This keeps CI logs and `docker build` output clean + on success (just package/suite summaries) while providing full diagnostic + detail on failure (every test case, every assertion). The command lives in the + `test` phase of the `Dockerfile`, since `script/test` builds that phase; the + Makefile form below is the same pattern for any repo-local invocation: ```makefile test: @@ -214,11 +338,26 @@ style conventions are in separate documents: ```makefile test: - @go test -timeout 90s -race -cover ./... || \ + @go test -count=1 -timeout 90s -race -cover ./... || \ { echo "--- Rerunning with -v for details ---"; \ - go test -timeout 90s -race -v ./...; exit 1; } + go test -count=1 -timeout 90s -race -v ./...; exit 1; } ``` + `-count=1` is required on both invocations: it defeats Go's test _result_ + cache, so neither run can report a stored pass in place of running the + tests. It leaves the build cache alone, so it costs the runtime of the suite + and no recompilation. + + That cache is Go's own, separate from Docker's layer cache. Go stores a + passing result in its cache directory (`GOCACHE`), and when the same tests + run again on unchanged code it prints that result, marked `(cached)`, + without running them. That matters on a developer's machine, where this + target runs and the directory lasts from one run to the next. The `test` + phase of the `Dockerfile` needs no `-count=1`: its base image holds no + result for this repo's tests and nothing before its `go test` step runs a + test, so there is nothing to replay. `--no-cache` (above) is what makes that + step run on an unchanged tree. + Python example: ```makefile @@ -244,10 +383,84 @@ style conventions are in separate documents: must be in `.gitignore`. No exceptions. - `.gitignore` should be comprehensive from the start: OS files (`.DS_Store`), - editor files (`.swp`, `*~`), language build artifacts, and `node_modules/`. - Fetch the standard `.gitignore` from - `https://git.eeqj.de/sneak/prompts/raw/branch/main/.gitignore` when setting up - a new repo. + editor files (`.swp`, `*~`), in-repo agent scratch directories (`.claude/`), + language build artifacts, and `node_modules/`. Fetch the standard `.gitignore` + from `https://git.eeqj.de/sneak/prompts/raw/branch/main/.gitignore` when + setting up a new repo. These patterns are written to `.gitignore`'s own + semantics, in which an unanchored pattern already matches at every depth; they + are not a `.dockerignore` and must not be transplanted into one unmodified. + +- **`.dockerignore` does not use `.gitignore` semantics, and copying patterns + across unmodified leaves secrets in the build context.** Docker matches with + `moby/patternmatcher`: `filepath.Match` semantics plus a `**` extension, so + `*` does not cross `/` and a pattern without a leading `**/` is anchored at + the build-context root. A `.dockerignore` listing `.env`, `*.pem` and `*.key` + therefore excludes only the copies at the repository root, while `config/.env` + and `certs/server.key` still reach the context and can land in an image layer + — which is more dangerous than a short file with no secret patterns at all, + because it reads as solved and stops anyone looking. Give every + depth-independent pattern the `**/` prefix and leave only genuinely + root-anchored entries unprefixed: `.claude`, and the repo's own host-built + binary, written `/myapp` and never `**/myapp`, which would also match + `cmd/myapp/` and delete the package directory from the context. Matching is + case-sensitive, and an ALL-CAPS twin per pattern still misses `Server.Key`, so + secret names use character ranges — `**/*.[kK][eE][yY]`, `**/*.[pP][eE][mM]`, + and likewise for `.envrc` and the extensionless SSH keys. Where such a pattern + also catches something the build needs, re-include it with a negation + (`!docs/example.env`); deleting the pattern reopens the exposure for every + other file it covers. Fetch the standard `.dockerignore` from + `https://git.eeqj.de/sneak/prompts/raw/branch/main/.dockerignore` and extend + it with the repo's own artifacts. + +- **In-repo agent scratch belongs in both files, written to each file's own + semantics.** `.claude/` holds one worktree per in-flight agent — an entire + additional checkout of the repo — so under `COPY . .` the build context + inflates by a multiple of the repo and another session's unreviewed work can + be copied into an image layer. In `.gitignore` the entry is `.claude/`, + unanchored. In `.dockerignore` it is `.claude`, anchored and with **no** `**/` + prefix, because the prefixed form would also delete any nested directory of + that name from the build. Anchoring carries a known gap that the canonical + `.dockerignore` states in its own comment, since consuming repos receive the + file and not the tracker: the directory is created in the agent's working + directory, so a repo running agents in subdirectories still ships + `services/api/.claude/` and must add its own anchored entry there. + +- **A plain `docker build .` of a clone stamps the version that + `git describe --tags --always` gives**, derived from the `.git` in the build + context as the canonical `Dockerfile` above shows. Without its failure check, + a missing `git` or an unreadable checkout would leave `-X main.Version=` empty + and the build would still exit 0. `script/docker` and `script/cibuild` pass + the version they compute on the host; it takes precedence. They do this + byte-identically across repos: + + ```sh + # Own line: a failing command substitution inside an argument does not + # trip `set -e`, so the inline form degrades to an empty constant. + version="$(git describe --tags --always --dirty 2>/dev/null || true)" + [ -n "$version" ] || version="unknown" + docker build --no-cache \ + --build-arg VERSION="$version" \ + -t "$(script/projectname)" . + ``` + + `--always` makes an untagged repo yield an abbreviated commit hash rather + than failing, and the `[ -n "$version" ]` line is the single place the + fallback is applied — a live check that fires on a build from an export with + no `.git` and on a repository with no commits yet. Do not fold it into the + substitution as `|| echo unknown`, which makes the guard unreachable. The + Dockerfile's side is `ARG VERSION` in the stage that compiles, declared + there because `ARG` is stage-scoped; passing `VERSION` to a repo whose + Dockerfile declares no such `ARG` is ignored and costs nothing, which is why + the scripts stay byte-identical. One consequence for CI: the standard + checkout action clones shallow and fetches no tags, so a repo that embeds a + tag-derived version must set `fetch-depth: 0` on its checkout step. + +- **Verify `.dockerignore` by enumerating the image, not by reading the + patterns.** Plant files at the root _and_ at least two directories deep, build + a probe image that does `COPY . .`, and list what actually landed + (`docker run --rm --entrypoint find IMAGE /app`). The `transferring context` + size is not a substitute: a nested secret is a few bytes, and BuildKit + transfers only the delta from the previous build. - **No build artifacts in version control.** Code-derived data (compiled bundles, minified output, generated assets) must never be committed to the @@ -263,12 +476,56 @@ style conventions are in separate documents: - Make all changes on a feature branch. You can do whatever you want on a feature branch. -- `.golangci.yml` is standardized and must _NEVER_ be modified by an agent, only - manually by the user. Fetch from - `https://git.eeqj.de/sneak/prompts/raw/branch/main/.golangci.yml`. The - canonical golangci-lint version is v2.12.2 (released 2026-05-06), installed - commit-pinned via - `go install github.com/golangci/golangci-lint/v2/cmd/golangci-lint@c0d3ddc9cf3faa61a4e378e879ece580256d76e5`. +- `.golangci.yml` is standardized. The vendored copy in a consuming repo must + _NEVER_ be modified by an agent: fetch it from + `https://git.eeqj.de/sneak/prompts/raw/branch/main/.golangci.yml` and keep it + byte-identical, so that no repo can quietly loosen its own linting. Linter + configuration changes are made to the canonical copy in the `prompts` repo and + reach consuming repos by re-vendoring; an agent may open a PR against + canonical, which only the user merges. One list is exempt from byte-identity, + because it cannot be written once for every repo: the `deny` list of the + `test-support` depguard rule, where a repo names its own test-support packages + by full import path. A repo adds entries there and changes nothing else, and a + re-vendor carries its entries forward. The canonical golangci-lint version is + v2.14.0 (released 2026-09-24), pinned as the digest of the lint phase's base + image + (`golangci/golangci-lint@sha256:ad862ba6b3798cbe0fd9fd7408d498fd74fbd2623a92406b2fd3898faf0bf98f`, + which reports `2.14.0 built with go1.27.0 from 114493f9`). A module's `go` + directive must not name a newer Go minor version than the one golangci-lint + was built with, or golangci-lint refuses to lint it: this release lints + `go 1.27.1` but not `go 1.28`. That digest is the only pin, since no repo + installs golangci-lint on the host. A repo sets the lint phase digest to the + one named here and re-vendors `.golangci.yml` in the same commit, whichever of + the two prompted the change: the canonical copy can name linters that an older + golangci-lint rejects, and a newer golangci-lint can add linters that + `default: all` switches on until the canonical copy disables them. + +- **`script/bootstrap` installs a pinned tool by comparing versions, never by + testing presence.** An `if ! command -v ; then install; fi` guard tests + `PATH` only, so on an already-provisioned machine the pin is inert and a + version bump is a silent no-op — while the Dockerfile, installing into a clean + image, gets the pinned version, so a local `make check` and `make docker` can + disagree about what the tool even is. The canonical form: + - compares the installed version against the pin over the **whole** version + token; a parser that stops at the first `-` reports `2.12.2` for a host + running `2.12.2-rc1` and skips the install; + - treats absent, non-zero, empty or unrecognised `--version` output as a + mismatch, so the failure direction is a redundant install and never a + skipped one; + - after installing, re-resolves the binary the way callers do — `hash -r`, + then through `PATH`, not through the directory the installer wrote to — + and fails naming the resolved path, since an install that a shadowing + binary hides succeeds while changing nothing any caller sees; + - is actually called, and prints the version on both success paths: a + function defined and never invoked has the same exit status and the same + empty output as one that worked. + + Keep it POSIX sh: no arrays, no `[[`, no `grep -P`. + + A Go tool a repo needs on the host is installed with `go install` pinned to + a commit hash (`go install @`). It is never tracked as + a `go.mod` tool dependency or through a `tools.go` file, either of which + pulls the tool's own dependencies into the repo's `go.mod` and `go.sum`. - When pinning images or packages by hash, add a comment above the reference with the version and date (YYYY-MM-DD). @@ -382,12 +639,14 @@ style conventions are in separate documents: settings. - Avoid putting files in the repo root unless necessary. Root should contain - only project-level config files (`README.md`, `Makefile`, `Dockerfile`, - `LICENSE`, `.gitignore`, `.editorconfig`, `REPO_POLICIES.md`, and - language-specific config). Everything else goes in a subdirectory. Canonical - subdirectory names: + only project-level config files (`README.md`, `AGENTS.md`, `Makefile`, + `Dockerfile`, `LICENSE`, `.gitignore`, `.editorconfig`, `REPO_POLICIES.md`, + and language-specific config). Everything else goes in a subdirectory. + Canonical subdirectory names: - `bin/` — executable scripts and tools - - `cmd/` — Go command entrypoints + - `cmd/` — Go command entrypoints; thin only: one `main.go` per binary whose + body is a single call into `internal/` or `pkg/`, no project logic in + `cmd/` - `configs/` — configuration templates and examples - `deploy/` — deployment manifests (k8s, compose, terraform) - `docs/` — documentation and markdown (README.md stays in root) @@ -414,3 +673,7 @@ style conventions are in separate documents: - Go: `go.mod`, `go.sum`, `.golangci.yml` - JS: `package.json`, `yarn.lock`, `.prettierrc`, `.prettierignore` - Python: `pyproject.toml` + +- Guidance for coding agents lives in one `AGENTS.md` at the repository root. It + is never committed under a file or directory named after one agent tool, such + as `CLAUDE.md` or `.claude/`, and never split into separate memory files. diff --git a/TESTING.md b/TESTING.md index 659db13..c256ba3 100644 --- a/TESTING.md +++ b/TESTING.md @@ -2,36 +2,45 @@ ## DNS Resolution Tests -All resolver tests **MUST** use live queries against real DNS servers. -No mocking of the DNS client layer is permitted. +DNS is never mocked in this project, not in tests and not anywhere else; see the +README section "No DNS mocking. Ever." Every test that looks something up in DNS +**MUST** query live DNS servers, never a stand-in. Logic that works on record +data, such as comparing or formatting records, may be tested on that data +directly with no lookup. ### Rationale -The resolver performs iterative resolution from root nameservers through -the full delegation chain. Mocked responses cannot faithfully represent -the variety of real-world DNS behavior (truncation, referrals, glue -records, DNSSEC, varied response times, EDNS, etc.). Testing against -real servers ensures the resolver works correctly in production. +The resolver performs iterative resolution from root nameservers through the +full delegation chain. Mocked responses cannot faithfully represent the variety +of real-world DNS behavior (truncation, referrals, glue records, DNSSEC, varied +response times, EDNS, etc.). Testing against real servers ensures the resolver +works correctly in production. ### Constraints - Tests hit real DNS infrastructure and require network access -- Test duration depends on network conditions; timeout tuning keeps - the suite within the 60-second target -- Query timeout is calibrated to 3× maximum antipodal RTT (~300ms) - plus processing margin -- Root server fan-out is limited to reduce parallel query load -- Flaky failures from transient network issues are acceptable and - should be investigated as potential resolver bugs, not papered over - with mocks or skip flags +- Test duration depends on network conditions. `make test` must finish in under + 60 seconds, the hard cap in `REPO_POLICIES.md`; under 20 seconds is the target +- Each query to a nameserver has a 2-second timeout, and a query over UDP that + gets no reply is sent once more (`queryTimeoutDuration` and `maxRetries` in + `internal/resolver/iterative.go`) +- In the walk to a name's nameservers, the servers of each zone are asked one at + a time in a random order, as the README's DNS Resolution Strategy describes +- Live lookups that expect an answer go through `internal/livednstest`, which + limits how many run at once in a test binary and retries a lookup that got + none +- A flaky live test is a robustness problem, fixed with retries with backoff, + several independent nameservers or longer timeouts, never with mocks or skip + flags ### What NOT to do -- **Do not mock `DNSClient`** for resolver tests (the mock constructor - exists for unit-testing other packages that consume the resolver) +- **Do not mock, fake or stub DNS** anywhere: no stand-in `DNSClient`, no + stand-in for the watcher's `DNSResolver`, no fake DNS server, no canned + responses - **Do not add `-short` flags** to skip slow tests - **Do not increase `-timeout`** to hide hanging queries -- **Do not remove `-count=1` from `script/test`** — Go's test cache - replays a previous run's output without querying anything, so a +- **Do not remove `-count=1` from the test phase of the `Dockerfile`** — Go's + test cache replays a previous run's output without querying anything, so a cached pass is not evidence that live resolution works - **Do not modify linter configuration** to suppress findings diff --git a/TODO.md b/TODO.md index 91ea4e5..f2674ea 100644 --- a/TODO.md +++ b/TODO.md @@ -1,227 +1,233 @@ # Workflow -* branch (from `main`) -* do the work in Next Step -* move Next Step to the top of Completed Steps -* move the top item of Future Steps into Next Step -* commit (`TODO.md` changes in the same commit as the work) -* merge to `main` if the branch is not protected, otherwise open a PR -* push +- branch (from `next`) +- do the work in Next Step +- move Next Step to the top of Completed Steps +- move the top item of Future Steps into Next Step +- commit (`TODO.md` changes in the same commit as the work) +- push +- open a PR against `next` # Status -pre-1.0. No git tags. Core resolver work in flight on feature/resolver -(dirty: internal/resolver/resolver_test.go). Local checkout has diverged -from origin: origin/main is 8 commits ahead (watcher orchestrator, -unified TARGETS) and origin/feature/resolver already contains the full -iterative resolver implementation with hermetic mocked tests. +pre-1.0. Work lands on `next` by PR. Open work for 1.0 is tracked on the 1.0 +milestone: https://git.eeqj.de/sneak/dnswatcher/milestone/7 # Next Step -Add the README sections required by policy (Description, Getting Started, -Rationale, Design, TODO, License, Author) if any are still missing. +1.0 tracking issue: https://git.eeqj.de/sneak/dnswatcher/issues/66 # Completed Steps -- 2026-08-10: comment-only corrections to `script/bootstrap`, - `script/cibuild`, and `Dockerfile.lint`. The `goimports` pin in - `script/bootstrap` was justified by a claim that `script/fmt-check` - runs it on the host; it does not (it runs `gofmt -l .` only), so the - header now credits `script/fmt` alone. `script/cibuild` still claimed - the `Dockerfile` runs `make check`, which stopped being true when - linting moved to its own stage; it now describes the lint stage - (`make fmt-check` plus `golangci-lint`) and the builder stage - (`make test`, `make build`). The `docker`-missing warning in - `script/bootstrap` reads as one sentence instead of three fragments - each re-prefixed with `bootstrap:`. `Dockerfile.lint` now records the - residual risk of omitting `golangci-lint config verify`: unknown - top-level keys in `.golangci.yml` are silently ignored, so a mistyped - key lints clean while applying nothing. No behaviour changed -- 2026-08-10: MIT `LICENSE` added at the repository root, closing the - last gap in `REPO_POLICIES.md`'s required-minimum file list and - removing the all-rights-reserved default that would otherwise have - shipped with a 1.0 tag. The licence choice is the standing org policy - (any public repo lacking a licence gets MIT; a private repo with no - licence is already all-rights-reserved), and this repo is public. The - file holds the canonical MIT text byte-for-byte with only the - copyright line filled in (`Copyright (c) 2026 sneak`); no clauses were - added, removed, or reflowed. `README.md`'s first line now names the - licence, as the Description requirement demands, and the License - section states MIT and points at the file instead of saying the choice - is pending. `make fmt` covers only Go sources (`gofmt -s`, - `goimports`), so it cannot reflow `LICENSE` -- 2026-08-10: the policy scaffold (`REPO_POLICIES.md`, `.editorconfig`, - `.dockerignore`, `.gitea/workflows/check.yml`, and the `fmt-check`, - `docker`, and hooks Makefile targets) is present; it landed piecemeal - across the scripts-to-rule-them-all and policy commits rather than as - the single commit this file once planned -- 2026-08-10: Go's test cache disabled for `script/test` via `-count=1`, - so every invocation actually executes. A cached pass replays an - earlier run's output without querying DNS at all, which in this repo - means the suite's entire premise goes unexercised while the run - reports green in under a second. The conditional verbose rerun that - `REPO_POLICIES.md` mandates was added at the same time (the primary - run had been unconditionally `-v`): quiet first, `-v` only on - failure, `-count=1` on both, and exit 1 forced regardless of the - rerun's result so a flake passing the second time cannot turn the - build green. `-timeout 90s` left alone as the deliberate backstop - above the 60s hard cap. Uncached suite runs ~4s, well inside the 20s - target -- 2026-08-10: live-DNS test flakiness addressed by robustness rather - than gating, per the owner's ruling on #93: new - `internal/resolver/livedns_test.go` adds a package-wide concurrency - gate (so parallel tests stop bursting at the first root server), - retry with exponential backoff on transport failures only, and - quorum instead of unanimity for multi-nameserver assertions. Quorum - tolerates silence only: every per-nameserver status must be in a - closed allowlist (`ok`/`timeout`/`error`, or - `nxdomain`/`timeout`/`error`), so a wrong answer from a minority — - `nodata` today, any status added later — fails the test instead of - sliding through under the majority. The - `make test` cap moved to the new org-wide 60s hard cap / 20s target - with a 90s `-timeout` backstop; `REPO_POLICIES.md` re-vendored - byte-identical from `sneak/prompts`. No mocks, no `-short`, no build - tags, no skips, and no change to production resolver behaviour -- 2026-08-10: all linting moved into Docker: new root `Dockerfile.lint` - on the digest-pinned `golangci/golangci-lint:v2.12.2` image, - `script/lint` reduced to a thin wrapper that builds it with - `--no-cache-filter=lint` so the linter actually executes every run, - golangci-lint install dropped from `script/bootstrap` (goimports - stays, `script/fmt` needs it on the host), and the root `Dockerfile` - given its own lint stage so its build no longer recurses through - `make check` into `script/lint`. `golangci-lint config verify` is - deliberately omitted: it fetches its schema over an unpinned live - HTTPS call -- 2026-08-09: in-flight notification deliveries are now drained at - shutdown (#106): `notify.New` registers an fx `OnStop` hook that waits - on a `sync.WaitGroup` of tracked delivery goroutines, bounded by the - `OnStop` context; on expiry the outstanding count is logged at warn - level and parked retry backoffs are released instead of being dropped - silently, and deliveries submitted after the drain begins are refused - so shutdown cannot be extended indefinitely; an `OnStop` context that - is already expired on entry with nothing outstanding drains quietly - rather than warning about deliveries that were never abandoned -- 2026-08-07: golangci-lint bumped to v2.12.2 (commit-pinned installs - in `Dockerfile` and `script/bootstrap`); `.golangci.yml` set to the - org-standard v2-schema config used across the org's repos - (owner-authorized; same file is being landed as canonical via prompts - PR #24), with settings under `linters.settings` so the - lll/funlen/cyclop/dupl thresholds apply; fixed the resulting - `goconst`, `dupl`, and `lll` findings; the informational `gomodguard` - deprecation warning under this config is accepted -- 2026-07-07 Adopted scripts-to-rule-them-all: `script/` entrypoints, - Makefile shims, README Entrypoints section -- 2026-02-20: iterative DNS resolver implemented; tests made hermetic - with mocked DNS (origin/feature/resolver, unmerged) -- 2026-02-20: CI actions and go install refs pinned to commit SHAs; - Gitea Actions workflow for make check (origin/ci/make-check, unmerged) +- 2026-10-16: the first reply marked "recursion available" from a root server's + address, a refusal included, stops the lookup with "this network intercepts + DNS queries", so on a network that sends DNS to a resolver dnswatcher and its + tests fail, by design (closes #300). +- 2026-10-15: a query its caller's context ends before it is sent records no + reason about the nameserver, and a try the context's deadline cuts short is a + timeout, so a nameserver whose only failures were timeouts is `timeout`, not a + network error or `ok` (closes #298). +- 2026-10-15: the resolver no longer says its query timeout grows on each try; + every try waits the same 2 seconds, and the unused `timeoutMultiplier` is gone + (closes #296). +- 2026-10-15: `TESTING.md` Constraints give the resolver's 2-second query + timeout and second try, say a zone's servers are asked one at a time in a + random order, give the 60-second cap and 20-second target for `make test`, and + say a flaky live test is fixed with robustness, not with mocks (closes #289). +- 2026-10-15: a request under `/s/` that names a directory, such as `/s/` or + `/s/css`, answers 404 instead of a listing of its files (closes #286). +- 2026-10-15: a notification endpoint that answers with a redirect, or any + status other than 2xx, is a failed delivery, retried and logged (closes #287). +- 2026-10-15: a nameserver is asked at each of its IPv4 addresses in turn, so it + has failed only when none of them gives a usable reply (closes #285). +- 2026-10-15: the log names a notification endpoint (`slack`, `mattermost`, + `ntfy`) and never shows its URL's path or query (closes #288). +- 2026-10-13: a hostname or a domain whose zone's nameservers all stop answering + gets an NS Failure for each, then an NS Recovery (closes #284). +- 2026-10-13: an IPv6 address is written in brackets before its port, as in + `[2001:db8::1]:443`, in keys and Port Change notifications (closes #270). +- 2026-10-13: `TODO.md` names only open issues as next and future steps, and its + Status says nothing about git tags (closes #281). +- 2026-10-13: a port check that finds port 443 closed removes that address's + certificate entries, so none is left shown as `ok` (closes #265). +- 2026-10-13: a state file holding `{}`, or leaving out a section, loads with + that section empty; the first check no longer crashes on it (closes #267). +- 2026-10-13: a stop before the first scan completes sends no startup + notification; a scan that ran to the end still sends it (closes #271). +- 2026-10-13: a nameserver leaving a domain's delegation sends only NS Change; + one leaving a hostname's zone sends NS Change, not NS Failure (closes #262). +- 2026-10-13: `DNSWATCHER_METRICS_USERNAME` set without + `DNSWATCHER_METRICS_PASSWORD` stops startup (closes #268). +- 2026-10-13: a save syncs the state file to disk before renaming it into place, + so a crash cannot leave it empty and stop the next start (closes #264). +- 2026-10-13: a `PORT` that cannot be bound stops startup with an error naming + the address, instead of running without the HTTP server (closes #266). +- 2026-10-13: a bad `PORT`, `DNSWATCHER_TLS_EXPIRY_WARNING` or true/false + setting stops startup instead of being read as 0 or false (closes #269). +- 2026-10-13: a check saving the state while the dashboard or `/api/v1/status` + is read cannot crash the process (closes #261). +- 2026-10-06: canonical files re-fetched from `sneak/prompts` at `dd4027b`; lint + and test are phases of the `Dockerfile`, every scripted build uncached (closes + #257). +- 2026-10-05: a name listed in `DNSWATCHER_SKIP_RECORD_NOTIFICATIONS` gets no + Record Change or Inconsistency notification; a name listed there that is not + in `DNSWATCHER_TARGETS` stops startup (closes #255). +- 2026-10-02: a nameserver whose query for one record type failed while the + others answered with no records is `ok`, not `nodata` (closes #253). +- 2026-10-02: a domain that does not exist is shown so, with no nameservers; no + name gets a parent's nameservers when its own did not answer (closes #222). +- 2026-10-02: the refused-query test sends one query to four operators' public + resolvers in turn until one replies, not eight to one operator (closes #251). +- 2026-10-02: a name removed from `DNSWATCHER_TARGETS` leaves the state, and so + the dashboard and API, at startup, before the first check (closes #223). +- 2026-10-02: a record type whose query to a nameserver fails keeps its previous + records and alerts nothing; the other types are still saved (closes #231). +- 2026-10-02: a Port Change notification lists the port's domains on a + `Domains:` line and its hostnames on a `Hostnames:` line (closes #248). +- 2026-10-02: the dashboard's Ports table and `/api/v1/status` port entries list + a port's domains apart from its hostnames (closes #245). +- 2026-10-02: nameservers a referral names without addresses are looked up, + three deep at most; `pool.ntp.org`'s nameservers resolve (closes #221). +- 2026-10-02: an apex domain is not counted or listed as a hostname; its records + show under Domains, and notifications about them say `Domain:` (closes #224). +- 2026-10-02: the dashboard lists each nameserver's record types in one fixed + order, the README's, then any other type, not a random one (closes #226). +- 2026-10-02: the dashboard and `/api/v1/status` show why a nameserver query or + a certificate check failed, which only the state file showed (closes #225). +- 2026-10-02: a name's CNAME is stored once per nameserver, not once per record + type asked for; a state file with repeats loads each value once (closes #220). +- 2026-10-02: a DNS lookup that shutdown cuts short logs no error; one that + fails otherwise, or runs out of time, still does (closes #229). +- 2026-10-02: Record Change and Inconsistency notifications list only the record + types that differ, each with its values as plain text (closes #219). +- 2026-10-02: the startup notification no longer says every notification + endpoint works; it says it is a test sent to each of them (closes #230). +- 2026-10-02: a Mattermost webhook that answers an HTTP error is logged as + `mattermost notification failed`, not as a Slack failure (closes #227). +- 2026-10-02: durations in the log are written as text such as `2m0s`, not as a + bare count of nanoseconds (closes #228). +- 2026-10-02: a watched name whose nameservers answer with a CNAME and no + address gets port and TLS checks at the end of its CNAME chain (closes #203). +- 2026-10-02: a resolver test that reads one record type from a nameserver's + answer asks again when that type is missing from it (closes #218). +- 2026-10-02: a plain `docker build .` of a clone stamps its tag or short + commit, not `dev`: the build context now carries `.git` (closes #210). +- 2026-10-02: a query a server refuses is not resent asking for recursion, and + every root server refusing is reported as DNS interception (closes #206). +- 2026-10-02: a push to a branch cancels that branch's older CI run, and the + checkout leaves no token in `.git/config` (closes #216). +- 2026-10-02: watcher tests send far fewer queries and a live attempt may take + 18s; nameserver addresses are asked only for A, AAAA, CNAME (closes #214). +- 2026-10-02: the resolver tries root servers, and every other server list it + walks, in a random order each time, not always from the top (closes #138). +- 2026-10-02: a name listed more than once in `DNSWATCHER_TARGETS`, in any + letter case or with a trailing dot, is watched once (closes #207). +- 2026-10-01: README checked against the code and corrected: metrics, CORS, + notification retries, CNAMEs, state file fields, Design tree (closes #108). +- 2026-10-01: a certificate within the expiry warning period is warned about on + every TLS check, where some checks used to skip it at random (closes #204). +- 2026-10-01: a domain's NS set is its delegation from the parent zone's + servers, not whichever of its own servers answered first (closes #200). +- 2026-10-01: README has Getting Started, Rationale and TODO sections, and its + Architecture section is now Design, in the order policy sets (closes #173). +- 2026-10-01: a zone's server that answers SERVFAIL or a referral leading no + closer is passed over for the next, as one that times out is (closes #197). +- 2026-10-01: when none of a configured name's nameservers answered, the port + state saved for its addresses is kept, not removed (closes #193). +- 2026-10-01: `ResolveIPAddresses` returns an error, not no addresses, when no + nameserver of the name's zone answered (closes #190). +- 2026-10-01: `make fmt` and `make fmt-check` cover Markdown with prettier, run + in Docker at the version pinned by `yarn.lock` (closes #119). +- 2026-10-01: `make fmt-check` fails on a file `goimports` would change; both + format scripts run `goimports` at its pinned commit, not from `PATH` (#119). +- 2026-10-01: a hostname is queried at the servers of the zone it is in, found + by following delegations for the name, not its last two labels (closes #189). +- 2026-10-01: each nameserver's addresses are saved with its domain, and a + change while it stays in the delegation is notified (closes #105). +- 2026-10-01: the watcher saves state when it stops, and shutdown waits for that + save, so it no longer relies on the state's own stop hook (closes #114). +- 2026-10-01: `DNSWATCHER_SENTRY_DSN` reports panics in HTTP handlers to Sentry, + and a DSN Sentry cannot parse stops startup (closes #107). +- 2026-10-01: a port or TLS check that shutdown cuts short saves nothing and + sends no notification, as a cut-short DNS lookup already did (closes #185). +- 2026-10-01: the client address from `X-Forwarded-For` is the last entry that + is not a trusted proxy, not the first, which the client sets (closes #181). +- 2026-10-01: a nameserver that does not answer is saved as `error` with the + reason, and NS failure and NS recovery are notified (closes #104). +- 2026-10-01: a `DNSWATCHER_DNS_INTERVAL` or `DNSWATCHER_TLS_INTERVAL` that is + not a positive duration stops startup; empty means the default (closes #177). +- 2026-10-01: `/metrics` allows each client address 30 requests a minute, + counted before Basic Auth, and answers 429 beyond that (closes #101). +- 2026-10-01: the image built by `make docker` reports the `git describe` + version, not `dev`, and the startup log now shows it (closes #109). +- 2026-10-01: two notify shutdown tests always release the delivery they hold, + so a drain that returns early fails them instead of hanging (closes #176). +- 2026-10-01: `script/install-precommit` asks git for the repository's git + directory, so `make hooks` also works where `.git` is a file (closes #129). +- 2026-10-01: `TODO.md` brought up to date: open issues listed by URL, every + Completed Steps entry cut to at most two lines (closes #146). +- 2026-10-01: wildcard CORS now applies only to the public routes, not to + `/metrics`, and allows only the methods they serve (closes #100). +- 2026-10-01: `internal/state` and `internal/watcher` no longer export test-only + constructors: two moved to `export_test.go`, one is deleted (closes #111). +- 2026-10-01: notify shutdown tests use one timing constant per meaning, name + the bound they check, and require the drain's debug line (closes #116). +- 2026-09-29: the entrypoint chowns the data directory to `dnswatcher` and runs + dnswatcher as that user, so a host bind mount needs no chown (closes #166). +- 2026-09-29: the live-DNS test package is renamed `internal/livednstest`; + `make lint` fails when program code imports it (closes #164). +- 2026-09-29: `.golangci.yml` re-fetched from `sneak/prompts`, with + `gomodguard_v2` and the org `depguard` `test-support` rule (closes #123). +- 2026-09-29: watcher and resolver tests that look something up in DNS use the + real resolver against live DNS servers (closes #159). +- 2026-09-28: the inconsistency alert is sent once, when two nameservers start + to disagree; every pair of nameservers is compared (closes #158). +- 2026-09-28: DNS names in record values (CNAME, MX, SRV and NS targets) are + lower-cased, so letter case alone is not a change (closes #157). +- 2026-09-28: lint and tests run on every build: `script/cibuild` and + `script/docker` pass `--no-cache-filter=lint,builder` (closes #115). +- 2026-09-28: the server timeout test drives `Run` and checks the timeouts on + the `http.Server` it serves (closes #120). +- 2026-09-28: upaas deploy readiness: the image runs as user `dnswatcher` with a + `HEALTHCHECK`; README "Running under upaas" (closes #147). +- 2026-09-21: added behavioural tests for `internal/globals`, + `internal/healthcheck`, and `internal/logger` (closes #110). +- 2026-09-21: `go mod tidy` dropped the redundant `golang.org/x/sync` + `// indirect` line so `script/bootstrap` leaves a clean tree (#132) +- 2026-08-10: comment-only corrections to `script/bootstrap`, `script/cibuild` + and `Dockerfile.lint`; no behaviour changed. +- 2026-08-10: MIT `LICENSE` added at the repository root; the README's first + line and License section name the licence. +- 2026-08-10: policy scaffold present: `REPO_POLICIES.md`, `.editorconfig`, + `.dockerignore`, CI workflow, `make fmt-check`, `make docker`, `make hooks`. +- 2026-08-10: Go's test cache disabled in `script/test` (`-count=1`), so every + run queries live DNS; a failed run is rerun with `-v`. +- 2026-08-10: live-DNS tests made robust rather than gated (#93): a limit on + concurrent lookups, retries, and a quorum across nameservers. +- 2026-08-10: all linting moved into Docker: `script/lint` builds + `Dockerfile.lint`, and the root `Dockerfile` has its own lint stage. +- 2026-08-09: in-flight notification deliveries are drained at shutdown, bounded + by the shutdown deadline (#106). +- 2026-08-09: `http.Server` sets all four socket timeouts; `WriteTimeout` stays + above the 60s handler timeout (#99). +- 2026-08-09: `SecurityHeaders()` middleware sets HSTS, CSP and the other + security headers `REPO_POLICIES.md` requires on every response. +- 2026-08-07: golangci-lint bumped to v2.12.2 and `.golangci.yml` set to the org + config; fixed the resulting `goconst`, `dupl` and `lll` findings. +- 2026-07-07 Adopted scripts-to-rule-them-all: `script/` entrypoints, Makefile + shims, README Entrypoints section +- 2026-02-20: iterative DNS resolver implemented +- 2026-02-20: CI actions and go install refs pinned to commit SHAs; Gitea + Actions workflow added - 2026-02-20: watcher monitoring orchestrator merged to main (#8) - 2026-02-20: DOMAINS/HOSTNAMES unified into single TARGETS config (#11) - 2026-02-19: TCP port connectivity checker, made concurrent with port validation; gosec G704 SSRF findings fixed without suppression - (feature branches, unmerged) -- 2026-02-19: TLS certificate inspector with no-peer-certificates error - path and IP SANs (feature branch, unmerged) +- 2026-02-19: TLS certificate inspector with no-peer-certificates error path and + IP SANs - 2026-02-19: gosec SSRF and formatting fixes on main - 2026-02-19: initial scaffold with per-nameserver DNS monitoring model # Future Steps -Compliance: - -- Pin Dockerfile base images by sha256 and ensure the Docker build runs - make check - -Branch reconciliation: - -- Sync local checkout with origin: local main is 8 commits behind - origin/main; local feature/resolver has diverged from - origin/feature/resolver, which already implements the resolver -- Merge in-flight branches to main once green: feature/resolver, - ci/make-check, feature/portcheck-implementation, - feature/tlscheck-implementation - -Resolver (plan from untracked TODO.md; largely implemented on -origin/feature/resolver, verify each item before closing): - -- Add github.com/miekg/dns dependency -- roots.go: hardcoded IANA root server list (a through m, IPv4/IPv6), - rootServers() returning ip:53 strings -- query.go: low-level query(ctx, server, name, qtype): UDP with TCP - fallback on truncation, RD=0, context respected, 5s per-query timeout, - returns raw *dns.Msg -- trace.go: iterative delegation chasing from roots: referral detection - (NOERROR, empty answer, NS in authority), glue extraction with - bailiwick check, out-of-bailiwick NS resolved with recursion guard, - delegation depth limit (20), retry across nameservers on failure, do - not chase CNAMEs inside trace -- FindAuthoritativeNameservers: NS set via trace, sorted, FQDN - normalized, trailing dot handled; must pass its 9 tests -- QueryNameserver: resolve NS host to IPs, query A/AAAA/CNAME/MX/TXT/ - SRV/CAA/NS, build NameserverResponse with status mapping (OK, - NXDomain, NoData, Error), documented record formatting, sorted values, - lame delegation detection; must pass its 16 tests -- QueryAllNameservers: find NS set for parent domain (public suffix - list), query all NS in parallel with bounded concurrency, return map - even when all fail, context cancellation; must pass its 4 tests -- LookupNS: thin wrapper over FindAuthoritativeNameservers, sorted, - identical results; must pass its 3 tests -- ResolveIPAddresses: collect A/AAAA from all NS, follow CNAME chains - with MaxCNAMEDepth, dedupe, sort, NXDOMAIN returns empty slice with - nil error; must pass its 9 tests -- All 39 resolver tests pass, make check green, merge to main - -Watcher (internal/watcher/watcher.go): - -- Scheduling loop in Run(ctx): initial check on startup, separate - tickers for DNS/port and TLS intervals, persist state via state.Save() - after each cycle, clean shutdown on context cancel -- Domain check: LookupNS, compare to stored state, store silently on - first run, notify with old/new NS lists on change -- Hostname check: QueryAllNameservers, compare per-NS records; notify on - record changes, NS failure, NS recovery, inconsistency detected, - inconsistency resolved, empty response; store silently on first run -- Port check: ResolveIPAddresses, check ports 80 and 443 per IP, notify - on open/closed transitions, handle new and disappeared IPs -- TLS check: for each open IP:443, CheckCertificate; notify on expiry - warning, certificate change (CN/issuer/SANs), TLS failure/recovery - -Port checker (internal/portcheck/portcheck.go): - -- Tests against known-open ports and RFC documentation IPs -- CheckPort: net.DialTimeout (5s), context respected; (true, nil) open, - (false, nil) closed/timeout/refused, error only for unexpected - failures - -TLS checker (internal/tlscheck/tlscheck.go): - -- Tests against known public HTTPS servers, verify fields populated -- CheckCertificate: tls.Dial to specific IP:443 with hostname as SNI; - extract subject CN, issuer CN and org, NotAfter, SANs; error on - handshake failure - -Notification service (internal/notify/notify.go, Slack/Mattermost/ntfy -backends exist): - -- Structured notification types: DNS change, port change, TLS expiry, - TLS change, NS failure, NS recovery, NS inconsistency -- Per-backend formatting: Slack/Mattermost attachment colors (red - failures/expiry, yellow warnings, green recoveries, blue info); ntfy - priorities (urgent failures, high warnings, default changes, low - recoveries); include hostname, nameserver, old/new values, timestamps - -HTTP API handlers: - -- Wire *state.State and *watcher.Watcher into handler params -- GET /api/v1/status: full state snapshot as JSON -- GET /api/v1/domains: domain states with NS records and last-checked -- GET /api/v1/hostnames: hostname states with per-NS record data - -Infrastructure notes (from untracked TODO.md): - -- Module path sneak.berlin/go/dnswatcher differs from the git.eeqj.de - remote intentionally; do not "fix" it -- Dependencies: github.com/miekg/dns, golang.org/x/net/publicsuffix -- Resolver tests originally used live DNS against *.dns.sneak.cloud - (required records documented in the test file header); origin now has - mocked hermetic tests, keep them hermetic +- DNSSEC validation, after 1.0: https://git.eeqj.de/sneak/dnswatcher/issues/59 diff --git a/cmd/dnswatcher/main.go b/cmd/dnswatcher/main.go index baff13b..11e6780 100644 --- a/cmd/dnswatcher/main.go +++ b/cmd/dnswatcher/main.go @@ -63,6 +63,7 @@ func main() { return n }, ), + fx.Invoke(func(l *logger.Logger) { l.Identify() }), fx.Invoke(func(*server.Server, *watcher.Watcher) {}), ).Run() } diff --git a/deploy/docker-entrypoint.sh b/deploy/docker-entrypoint.sh new file mode 100755 index 0000000..62b95cd --- /dev/null +++ b/deploy/docker-entrypoint.sh @@ -0,0 +1,17 @@ +#!/bin/sh +# deploy/docker-entrypoint.sh: the Docker image's ENTRYPOINT. It runs as +# root only to give the data directory to the dnswatcher user: a host +# directory bind-mounted there keeps its host owner, often root, and may +# hold a state file left by another uid, which dnswatcher could neither +# read nor replace. dnswatcher itself always runs as the dnswatcher user. +set -eu + +main() { + dir="${DNSWATCHER_DATA_DIR:-/var/lib/dnswatcher}" + mkdir -p "$dir" + chown -R dnswatcher:dnswatcher "$dir" + chmod 700 "$dir" + exec su-exec dnswatcher /usr/local/bin/dnswatcher "$@" +} + +main "$@" diff --git a/go.mod b/go.mod index 53638c9..da8e0b7 100644 --- a/go.mod +++ b/go.mod @@ -4,27 +4,30 @@ go 1.25.5 require ( github.com/99designs/basicauth-go v0.0.0-20230316000542-bf6f9cbbf0f8 + github.com/getsentry/sentry-go v0.49.0 github.com/go-chi/chi/v5 v5.2.5 github.com/go-chi/cors v1.2.2 + github.com/go-chi/httprate v0.16.0 github.com/joho/godotenv v1.5.1 github.com/miekg/dns v1.1.72 github.com/prometheus/client_golang v1.23.2 github.com/spf13/viper v1.21.0 github.com/stretchr/testify v1.11.1 go.uber.org/fx v1.24.0 - golang.org/x/net v0.50.0 - golang.org/x/sync v0.19.0 + golang.org/x/net v0.56.0 + golang.org/x/sync v0.21.0 ) require ( github.com/beorn7/perks v1.0.1 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect - github.com/davecgh/go-spew v1.1.1 // indirect + github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/fsnotify/fsnotify v1.9.0 // indirect github.com/go-viper/mapstructure/v2 v2.4.0 // indirect + github.com/klauspost/cpuid/v2 v2.2.10 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/pelletier/go-toml/v2 v2.2.4 // indirect - github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/prometheus/client_model v0.6.2 // indirect github.com/prometheus/common v0.66.1 // indirect github.com/prometheus/procfs v0.16.1 // indirect @@ -34,16 +37,16 @@ require ( github.com/spf13/cast v1.10.0 // indirect github.com/spf13/pflag v1.0.10 // indirect github.com/subosito/gotenv v1.6.0 // indirect + github.com/zeebo/xxh3 v1.0.2 // indirect go.uber.org/dig v1.19.0 // indirect go.uber.org/multierr v1.10.0 // indirect go.uber.org/zap v1.26.0 // indirect go.yaml.in/yaml/v2 v2.4.2 // indirect go.yaml.in/yaml/v3 v3.0.4 // indirect - golang.org/x/mod v0.32.0 // indirect - golang.org/x/sync v0.19.0 // indirect - golang.org/x/sys v0.41.0 // indirect - golang.org/x/text v0.34.0 // indirect - golang.org/x/tools v0.41.0 // indirect + golang.org/x/mod v0.37.0 // indirect + golang.org/x/sys v0.46.0 // indirect + golang.org/x/text v0.39.0 // indirect + golang.org/x/tools v0.47.0 // indirect google.golang.org/protobuf v1.36.8 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go.sum b/go.sum index 720b18f..eb081e2 100644 --- a/go.sum +++ b/go.sum @@ -4,16 +4,22 @@ github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= -github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= -github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8= github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= github.com/fsnotify/fsnotify v1.9.0 h1:2Ml+OJNzbYCTzsxtv8vKSFD9PbJjmhYF14k/jKC7S9k= github.com/fsnotify/fsnotify v1.9.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0= +github.com/getsentry/sentry-go v0.49.0 h1:Ehejknu1l023Ub7QoRBVLAI7g3Jnhqku4oWx4B4Sh5s= +github.com/getsentry/sentry-go v0.49.0/go.mod h1:nuMJAoCfe1u0Bts2ocyNI+TW8HT84vRMqwA5Qq/SKUI= github.com/go-chi/chi/v5 v5.2.5 h1:Eg4myHZBjyvJmAFjFvWgrqDTXFyOzjj7YIm3L3mu6Ug= github.com/go-chi/chi/v5 v5.2.5/go.mod h1:X7Gx4mteadT3eDOMTsXzmI4/rwUpOwBHLpAfupzFJP0= github.com/go-chi/cors v1.2.2 h1:Jmey33TE+b+rB7fT8MUy1u0I4L+NARQlK6LhzKPSyQE= github.com/go-chi/cors v1.2.2/go.mod h1:sSbTewc+6wYHBBCW7ytsFSn836hqM7JxpglAy2Vzc58= +github.com/go-chi/httprate v0.16.0 h1:8V5DH9j6pSK6UQoBsTpvMyFxycqaKEIToyPKzHJjUa8= +github.com/go-chi/httprate v0.16.0/go.mod h1:A8lo+qRhk+s9LiuP5saS7XCGDXRXMcrueq0NfIuCa/I= +github.com/go-errors/errors v1.4.2 h1:J6MZopCL4uSllY1OfXM374weqZFFItUbrImctkmUxIA= +github.com/go-errors/errors v1.4.2/go.mod h1:sIVyrIiJhuEF+Pj9Ebtd6P/rEYROXFi3BopGUQ5a5Og= github.com/go-viper/mapstructure/v2 v2.4.0 h1:EBsztssimR/CONLSZZ04E8qAkxNYq4Qp9LvH92wZUgs= github.com/go-viper/mapstructure/v2 v2.4.0/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM= github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= @@ -22,6 +28,8 @@ github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0= github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4= github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo= github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ= +github.com/klauspost/cpuid/v2 v2.2.10 h1:tBs3QSyvjDyFTq3uoc/9xFpCuOsJQFNPiAhYdw2skhE= +github.com/klauspost/cpuid/v2 v2.2.10/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= @@ -34,8 +42,12 @@ github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/pelletier/go-toml/v2 v2.2.4 h1:mye9XuhQ6gvn5h28+VilKrrPoQVanw5PMw/TB0t5Ec4= github.com/pelletier/go-toml/v2 v2.2.4/go.mod h1:2gIqNv+qfxSVS7cM2xJQKtLSTLUE9V8t9Stt+h56mCY= -github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= -github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/pingcap/errors v0.11.4 h1:lFuQV/oaUMGcD2tqt+01ROSmJs75VG1ToEOkZIZ4nE4= +github.com/pingcap/errors v0.11.4/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/prometheus/client_golang v1.23.2 h1:Je96obch5RDVy3FDMndoUsjAhG5Edi49h0RJWRi/o0o= github.com/prometheus/client_golang v1.23.2/go.mod h1:Tb1a6LWHB3/SPIzCoaDXI4I8UHKeFTEQ1YCr+0Gyqmg= github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNwqPLxwZyk= @@ -44,8 +56,8 @@ github.com/prometheus/common v0.66.1 h1:h5E0h5/Y8niHc5DlaLlWLArTQI7tMrsfQjHV+d9Z github.com/prometheus/common v0.66.1/go.mod h1:gcaUsgf3KfRSwHY4dIMXLPV0K/Wg1oZ8+SbZk/HH/dA= github.com/prometheus/procfs v0.16.1 h1:hZ15bTNuirocR6u0JZ6BAHHmwS1p8B4P6MRqxtzMyRg= github.com/prometheus/procfs v0.16.1/go.mod h1:teAbpZRB1iIAJYREa1LsoWUXykVXA1KlTmWl8x/U+Is= -github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= -github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= +github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= +github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= github.com/sagikazarmark/locafero v0.11.0 h1:1iurJgmM9G3PA/I+wWYIOw/5SyBtxapeHDcg+AAIFXc= github.com/sagikazarmark/locafero v0.11.0/go.mod h1:nVIGvgyzw595SUSUE6tvCp3YYTeHs15MvlmU87WwIik= github.com/sourcegraph/conc v0.3.1-0.20240121214520-5f936abd7ae8 h1:+jumHNA0Wrelhe64i8F6HNlS8pkoyMv5sreGx2Ry5Rw= @@ -62,6 +74,10 @@ github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8= github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU= +github.com/zeebo/assert v1.3.0 h1:g7C04CbJuIDKNPFHmsk4hwZDO5O+kntRxzaUoNXj+IQ= +github.com/zeebo/assert v1.3.0/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0= +github.com/zeebo/xxh3 v1.0.2 h1:xZmwmqxHZA8AI603jOQ0tMqmBr9lPeFwGg6d+xy9DC0= +github.com/zeebo/xxh3 v1.0.2/go.mod h1:5NWz9Sef7zIDm2JHfFlcQvNekmcEl9ekUZQQKCYaDcA= go.uber.org/dig v1.19.0 h1:BACLhebsYdpQ7IROQ1AGPjrXcP5dF80U3gKoFzbaq/4= go.uber.org/dig v1.19.0/go.mod h1:Us0rSJiThwCv2GteUN0Q7OKvU7n5J4dxZ9JKUXozFdE= go.uber.org/fx v1.24.0 h1:wE8mruvpg2kiiL1Vqd0CC+tr0/24XIB10Iwp2lLWzkg= @@ -76,18 +92,18 @@ go.yaml.in/yaml/v2 v2.4.2 h1:DzmwEr2rDGHl7lsFgAHxmNz/1NlQ7xLIrlN2h5d1eGI= go.yaml.in/yaml/v2 v2.4.2/go.mod h1:081UH+NErpNdqlCXm3TtEran0rJZGxAYx9hb/ELlsPU= go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc= go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= -golang.org/x/mod v0.32.0 h1:9F4d3PHLljb6x//jOyokMv3eX+YDeepZSEo3mFJy93c= -golang.org/x/mod v0.32.0/go.mod h1:SgipZ/3h2Ci89DlEtEXWUk/HteuRin+HHhN+WbNhguU= -golang.org/x/net v0.50.0 h1:ucWh9eiCGyDR3vtzso0WMQinm2Dnt8cFMuQa9K33J60= -golang.org/x/net v0.50.0/go.mod h1:UgoSli3F/pBgdJBHCTc+tp3gmrU4XswgGRgtnwWTfyM= -golang.org/x/sync v0.19.0 h1:vV+1eWNmZ5geRlYjzm2adRgW2/mcpevXNg50YZtPCE4= -golang.org/x/sync v0.19.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= -golang.org/x/sys v0.41.0 h1:Ivj+2Cp/ylzLiEU89QhWblYnOE9zerudt9Ftecq2C6k= -golang.org/x/sys v0.41.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= -golang.org/x/text v0.34.0 h1:oL/Qq0Kdaqxa1KbNeMKwQq0reLCCaFtqu2eNuSeNHbk= -golang.org/x/text v0.34.0/go.mod h1:homfLqTYRFyVYemLBFl5GgL/DWEiH5wcsQ5gSh1yziA= -golang.org/x/tools v0.41.0 h1:a9b8iMweWG+S0OBnlU36rzLp20z1Rp10w+IY2czHTQc= -golang.org/x/tools v0.41.0/go.mod h1:XSY6eDqxVNiYgezAVqqCeihT4j1U2CCsqvH3WhQpnlg= +golang.org/x/mod v0.37.0 h1:vF1DjpVEshcIqoEaauuHebaLk1O1forxjxBaVn884JQ= +golang.org/x/mod v0.37.0/go.mod h1:m8S8VeM9r4dzDwjrKO0a1sZP3YjeMamRRlD+fmR2Q/0= +golang.org/x/net v0.56.0 h1:Rw8j/hFzGvJUZwNBXnAtf5sVDVt+65SK2C7IxCxZt5o= +golang.org/x/net v0.56.0/go.mod h1:D3Ku6r+V6JROoZK144D2XfMHFcMq/0zSfLelVTCFKec= +golang.org/x/sync v0.21.0 h1:HLII4xRRTtCRkxYp4HNFF0Js/Og6q2i++KXbg0gHCwM= +golang.org/x/sync v0.21.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= +golang.org/x/sys v0.46.0 h1:noSf2Fq6F8DBgS+LysIkx7rIExoNHJsxOAtPp4rthXw= +golang.org/x/sys v0.46.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +golang.org/x/text v0.39.0 h1:UbZz4pLOvn600D6Oh6GGEI6VAmndrEBLv8/6BEXzyus= +golang.org/x/text v0.39.0/go.mod h1:3UwRclnC2g0TU9x8PZiyfOajCd1zaUNHF9cvqcQZ+ZM= +golang.org/x/tools v0.47.0 h1:7Kn5x/d1svx/PzryTsqeoZN4TZwqeH5pGWjefhLi/1Q= +golang.org/x/tools v0.47.0/go.mod h1:dFHnyTvFWY212G+h7ZY4Vsp/K3U4/7W9TyVaAul8uCA= google.golang.org/protobuf v1.36.8 h1:xHScyCOEuuwZEc6UtSOvPbAT4zRh0xcNRYekJwfqyMc= google.golang.org/protobuf v1.36.8/go.mod h1:fuxRtAxBytpl4zzqUh6/eyUujkJdNiuEkXntxiD/uRU= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= diff --git a/internal/config/classify.go b/internal/config/classify.go index 1076215..ef10036 100644 --- a/internal/config/classify.go +++ b/internal/config/classify.go @@ -57,17 +57,22 @@ func ClassifyDNSName(name string) (DNSNameType, error) { // ClassifyTargets splits a list of DNS names into apex domains and // hostnames using the Public Suffix List. It returns an error if any -// name cannot be classified. +// name cannot be classified. A name given more than once, in any letter +// case or with a trailing dot, is kept once. func ClassifyTargets(targets []string) ([]string, []string, error) { var domains, hostnames []string + seen := make(map[string]bool) + for _, t := range targets { normalized := strings.ToLower(strings.TrimSuffix(strings.TrimSpace(t), ".")) - if normalized == "" { + if normalized == "" || seen[normalized] { continue } + seen[normalized] = true + typ, classErr := ClassifyDNSName(normalized) if classErr != nil { return nil, nil, classErr diff --git a/internal/config/classify_test.go b/internal/config/classify_test.go index a9c03af..a82c853 100644 --- a/internal/config/classify_test.go +++ b/internal/config/classify_test.go @@ -1,6 +1,7 @@ package config_test import ( + "slices" "testing" "sneak.berlin/go/dnswatcher/internal/config" @@ -93,6 +94,29 @@ func TestClassifyTargets(t *testing.T) { } } +func TestClassifyTargetsKeepsEachNameOnce(t *testing.T) { + t.Parallel() + + domains, hostnames, err := config.ClassifyTargets([]string{ + "example.org", + "Example.org.", + "www.example.org", + "EXAMPLE.ORG", + "WWW.Example.org.", + }) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + + if !slices.Equal(domains, []string{"example.org"}) { + t.Errorf("domains = %v, want [example.org]", domains) + } + + if !slices.Equal(hostnames, []string{"www.example.org"}) { + t.Errorf("hostnames = %v, want [www.example.org]", hostnames) + } +} + func TestClassifyTargetsRejectsPublicSuffix(t *testing.T) { t.Parallel() diff --git a/internal/config/config.go b/internal/config/config.go index 264b90b..4f1d6b7 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -5,6 +5,8 @@ import ( "errors" "fmt" "log/slog" + "slices" + "strconv" "strings" "time" @@ -23,11 +25,49 @@ const ( defaultTLSExpiryWarning = 7 ) +// maxPort is the highest TCP port number. +const maxPort = 65535 + // ErrNoTargets is returned when no monitoring targets are configured. var ErrNoTargets = errors.New( "no monitoring targets configured: set DNSWATCHER_TARGETS environment variable", ) +// ErrInvalidInterval is returned when DNSWATCHER_DNS_INTERVAL or +// DNSWATCHER_TLS_INTERVAL is set but is not a positive duration. An empty +// value counts as unset and means the default. +var ErrInvalidInterval = errors.New( + "interval must be a positive duration such as 30m or 1h", +) + +// ErrNotInTargets is returned when DNSWATCHER_SKIP_RECORD_NOTIFICATIONS +// lists a name that is not in DNSWATCHER_TARGETS. +var ErrNotInTargets = errors.New("name is not in DNSWATCHER_TARGETS") + +// ErrInvalidPort is returned when PORT is set but is not a whole number +// from 1 to 65535. An empty value counts as unset and means the default. +var ErrInvalidPort = errors.New("port must be a whole number from 1 to 65535") + +// ErrInvalidTLSExpiryWarning is returned when DNSWATCHER_TLS_EXPIRY_WARNING +// is set but is not a whole number, 1 or more. An empty value counts as unset +// and means the default. +var ErrInvalidTLSExpiryWarning = errors.New( + "TLS expiry warning must be a whole number of days, 1 or more", +) + +// ErrInvalidBool is returned when DNSWATCHER_DEBUG, +// DNSWATCHER_MAINTENANCE_MODE or DNSWATCHER_SEND_TEST_NOTIFICATION is set +// but is not a value strconv.ParseBool accepts. An empty value counts as +// unset and means the default. +var ErrInvalidBool = errors.New("value must be true or false") + +// ErrNoMetricsPassword is returned when DNSWATCHER_METRICS_USERNAME is set and +// DNSWATCHER_METRICS_PASSWORD is empty. Basic Auth on /metrics would otherwise +// let in anyone who sends the username with an empty password. +var ErrNoMetricsPassword = errors.New( + "DNSWATCHER_METRICS_PASSWORD must be set when DNSWATCHER_METRICS_USERNAME is set", +) + // Params contains dependencies for Config. type Params struct { fx.In @@ -38,24 +78,25 @@ type Params struct { // Config holds application configuration. type Config struct { - Port int - Debug bool - DataDir string - Domains []string - Hostnames []string - SlackWebhook string - MattermostWebhook string - NtfyTopic string - DNSInterval time.Duration - TLSInterval time.Duration - TLSExpiryWarning int - SentryDSN string - MaintenanceMode bool - MetricsUsername string - MetricsPassword string - SendTestNotification bool - params *Params - log *slog.Logger + Port int + Debug bool + DataDir string + Domains []string + Hostnames []string + SkipRecordNotifications []string + SlackWebhook string + MattermostWebhook string + NtfyTopic string + DNSInterval time.Duration + TLSInterval time.Duration + TLSExpiryWarning int + SentryDSN string + MaintenanceMode bool + MetricsUsername string + MetricsPassword string + SendTestNotification bool + params *Params + log *slog.Logger } // New creates a new Config instance from environment and config files. @@ -74,6 +115,10 @@ func New(_ fx.Lifecycle, params Params) (*Config, error) { return nil, err } + if cfg.MetricsUsername != "" && cfg.MetricsPassword == "" { + return nil, ErrNoMetricsPassword + } + configureDebugLogging(cfg, params) return cfg, nil @@ -96,6 +141,7 @@ func setupViper(name string) { viper.SetDefault("DEBUG", false) viper.SetDefault("DATA_DIR", "/var/lib/"+name) viper.SetDefault("TARGETS", "") + viper.SetDefault("SKIP_RECORD_NOTIFICATIONS", "") viper.SetDefault("SLACK_WEBHOOK", "") viper.SetDefault("MATTERMOST_WEBHOOK", "") viper.SetDefault("NTFY_TOPIC", "") @@ -113,30 +159,19 @@ func buildConfig( log *slog.Logger, params *Params, ) (*Config, error) { - err := viper.ReadInConfig() + err := readConfigFile(log) if err != nil { - var notFound viper.ConfigFileNotFoundError - if !errors.As(err, ¬Found) { - log.Error("config file malformed", "error", err) - - return nil, fmt.Errorf( - "config file malformed: %w", err, - ) - } + return nil, err } - dnsInterval, err := time.ParseDuration( - viper.GetString("DNS_INTERVAL"), - ) + dnsInterval, err := parseInterval("DNS_INTERVAL") if err != nil { - dnsInterval = defaultDNSInterval + return nil, err } - tlsInterval, err := time.ParseDuration( - viper.GetString("TLS_INTERVAL"), - ) + tlsInterval, err := parseInterval("TLS_INTERVAL") if err != nil { - tlsInterval = defaultTLSInterval + return nil, err } domains, hostnames, err := parseAndValidateTargets() @@ -144,30 +179,141 @@ func buildConfig( return nil, err } + skipRecordNotifications, err := parseSkipRecordNotifications( + domains, hostnames, + ) + if err != nil { + return nil, err + } + + port, err := parsePort() + if err != nil { + return nil, err + } + + tlsExpiryWarning, err := parseTLSExpiryWarning() + if err != nil { + return nil, err + } + + debug, err := parseBool("DEBUG") + if err != nil { + return nil, err + } + + maintenanceMode, err := parseBool("MAINTENANCE_MODE") + if err != nil { + return nil, err + } + + sendTestNotification, err := parseBool("SEND_TEST_NOTIFICATION") + if err != nil { + return nil, err + } + cfg := &Config{ - Port: viper.GetInt("PORT"), - Debug: viper.GetBool("DEBUG"), - DataDir: viper.GetString("DATA_DIR"), - Domains: domains, - Hostnames: hostnames, - SlackWebhook: viper.GetString("SLACK_WEBHOOK"), - MattermostWebhook: viper.GetString("MATTERMOST_WEBHOOK"), - NtfyTopic: viper.GetString("NTFY_TOPIC"), - DNSInterval: dnsInterval, - TLSInterval: tlsInterval, - TLSExpiryWarning: viper.GetInt("TLS_EXPIRY_WARNING"), - SentryDSN: viper.GetString("SENTRY_DSN"), - MaintenanceMode: viper.GetBool("MAINTENANCE_MODE"), - MetricsUsername: viper.GetString("METRICS_USERNAME"), - MetricsPassword: viper.GetString("METRICS_PASSWORD"), - SendTestNotification: viper.GetBool("SEND_TEST_NOTIFICATION"), - params: params, - log: log, + Port: port, + Debug: debug, + DataDir: viper.GetString("DATA_DIR"), + Domains: domains, + Hostnames: hostnames, + SkipRecordNotifications: skipRecordNotifications, + SlackWebhook: viper.GetString("SLACK_WEBHOOK"), + MattermostWebhook: viper.GetString("MATTERMOST_WEBHOOK"), + NtfyTopic: viper.GetString("NTFY_TOPIC"), + DNSInterval: dnsInterval, + TLSInterval: tlsInterval, + TLSExpiryWarning: tlsExpiryWarning, + SentryDSN: viper.GetString("SENTRY_DSN"), + MaintenanceMode: maintenanceMode, + MetricsUsername: viper.GetString("METRICS_USERNAME"), + MetricsPassword: viper.GetString("METRICS_PASSWORD"), + SendTestNotification: sendTestNotification, + params: params, + log: log, } return cfg, nil } +// readConfigFile reads the config file, if there is one. +func readConfigFile(log *slog.Logger) error { + err := viper.ReadInConfig() + if err != nil { + var notFound viper.ConfigFileNotFoundError + if !errors.As(err, ¬Found) { + log.Error("config file malformed", "error", err) + + return fmt.Errorf("config file malformed: %w", err) + } + } + + return nil +} + +// parseInterval reads the DNSWATCHER_-prefixed setting key as a duration. A +// value that does not parse, or is zero or negative, is an error naming the +// variable and the value; an unset variable has its default from setupViper. +func parseInterval(key string) (time.Duration, error) { + value := viper.GetString(key) + + interval, err := time.ParseDuration(value) + if err != nil || interval <= 0 { + return 0, fmt.Errorf( + "invalid DNSWATCHER_%s %q: %w", key, value, ErrInvalidInterval, + ) + } + + return interval, nil +} + +// parsePort reads PORT, which has no DNSWATCHER_ prefix, as a port number. A +// value that is not a whole number from 1 to 65535 is an error naming the +// variable and the value; an unset variable has its default from setupViper. +func parsePort() (int, error) { + value := viper.GetString("PORT") + + port, err := strconv.Atoi(value) + if err != nil || port < 1 || port > maxPort { + return 0, fmt.Errorf("invalid PORT %q: %w", value, ErrInvalidPort) + } + + return port, nil +} + +// parseTLSExpiryWarning reads DNSWATCHER_TLS_EXPIRY_WARNING as a number of +// days. A value that is not a whole number, 1 or more, is an error naming the +// variable and the value; an unset variable has its default from setupViper. +func parseTLSExpiryWarning() (int, error) { + value := viper.GetString("TLS_EXPIRY_WARNING") + + days, err := strconv.Atoi(value) + if err != nil || days < 1 { + return 0, fmt.Errorf( + "invalid DNSWATCHER_TLS_EXPIRY_WARNING %q: %w", + value, ErrInvalidTLSExpiryWarning, + ) + } + + return days, nil +} + +// parseBool reads the DNSWATCHER_-prefixed setting key as true or false. A +// value strconv.ParseBool does not accept is an error naming the variable and +// the value; an unset variable has its default from setupViper. +func parseBool(key string) (bool, error) { + value := viper.GetString(key) + + result, err := strconv.ParseBool(value) + if err != nil { + return false, fmt.Errorf( + "invalid DNSWATCHER_%s %q: %w", key, value, ErrInvalidBool, + ) + } + + return result, nil +} + func parseAndValidateTargets() ([]string, []string, error) { domains, hostnames, err := ClassifyTargets( parseCSV(viper.GetString("TARGETS")), @@ -185,6 +331,36 @@ func parseAndValidateTargets() ([]string, []string, error) { return domains, hostnames, nil } +// parseSkipRecordNotifications reads DNSWATCHER_SKIP_RECORD_NOTIFICATIONS, +// a comma-separated list of names from the targets. Each name is written +// as ClassifyTargets writes a target, in lower case without a trailing +// dot, and a name listed more than once is kept once. A name that is +// not one of domains or hostnames is an error naming it. +func parseSkipRecordNotifications( + domains, hostnames []string, +) ([]string, error) { + value := viper.GetString("SKIP_RECORD_NOTIFICATIONS") + + var names []string + + for _, listed := range parseCSV(value) { + name := strings.ToLower(strings.TrimSuffix(listed, ".")) + + if !slices.Contains(domains, name) && !slices.Contains(hostnames, name) { + return nil, fmt.Errorf( + "invalid DNSWATCHER_SKIP_RECORD_NOTIFICATIONS %q: %w", + listed, ErrNotInTargets, + ) + } + + if !slices.Contains(names, name) { + names = append(names, name) + } + } + + return names, nil +} + func parseCSV(input string) []string { if input == "" { return nil diff --git a/internal/config/config_test.go b/internal/config/config_test.go index 2917818..35b4307 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -1,6 +1,7 @@ package config_test import ( + "strconv" "testing" "time" @@ -56,6 +57,7 @@ func TestNew_DefaultValues(t *testing.T) { assert.Empty(t, cfg.MetricsUsername) assert.Empty(t, cfg.MetricsPassword) assert.False(t, cfg.SendTestNotification) + assert.Empty(t, cfg.SkipRecordNotifications) } func TestNew_EnvironmentOverrides(t *testing.T) { @@ -113,38 +115,136 @@ func TestNew_OnlyEmptyCSVSegments(t *testing.T) { assert.ErrorIs(t, err, config.ErrNoTargets) } -func TestNew_InvalidDNSInterval_FallsBackToDefault(t *testing.T) { - viper.Reset() - t.Setenv("DNSWATCHER_TARGETS", "example.com") - t.Setenv("DNSWATCHER_DNS_INTERVAL", "banana") +// TestNew_InvalidIntervalStopsStartup checks values that must stop startup; +// TestNew_DefaultValues and TestNew_EmptyValueMeansDefault check that an +// unset or empty interval means the default. +func TestNew_InvalidIntervalStopsStartup(t *testing.T) { + variables := []string{"DNSWATCHER_DNS_INTERVAL", "DNSWATCHER_TLS_INTERVAL"} + values := []string{ + "banana", // not a duration + "5", // no unit + "1d", // days are not a unit time.ParseDuration knows + "0", // zero + "-1h", // negative + } - cfg, err := config.New(nil, newTestParams(t)) - require.NoError(t, err) - assert.Equal(t, time.Hour, cfg.DNSInterval, - "invalid DNS interval should fall back to 1h default") + for _, variable := range variables { + for _, value := range values { + t.Run(variable+"="+value, func(t *testing.T) { + viper.Reset() + t.Setenv("DNSWATCHER_TARGETS", "example.com") + t.Setenv(variable, value) + + _, err := config.New(nil, newTestParams(t)) + require.ErrorIs(t, err, config.ErrInvalidInterval) + require.ErrorContains(t, err, variable) + require.ErrorContains(t, err, strconv.Quote(value)) + }) + } + } } -func TestNew_InvalidTLSInterval_FallsBackToDefault(t *testing.T) { +func TestNew_EmptyValueMeansDefault(t *testing.T) { viper.Reset() t.Setenv("DNSWATCHER_TARGETS", "example.com") - t.Setenv("DNSWATCHER_TLS_INTERVAL", "notaduration") - - cfg, err := config.New(nil, newTestParams(t)) - require.NoError(t, err) - assert.Equal(t, 12*time.Hour, cfg.TLSInterval, - "invalid TLS interval should fall back to 12h default") -} - -func TestNew_BothIntervalsInvalid(t *testing.T) { - viper.Reset() - t.Setenv("DNSWATCHER_TARGETS", "example.com") - t.Setenv("DNSWATCHER_DNS_INTERVAL", "xyz") - t.Setenv("DNSWATCHER_TLS_INTERVAL", "abc") + t.Setenv("DNSWATCHER_DNS_INTERVAL", "") + t.Setenv("DNSWATCHER_TLS_INTERVAL", "") + t.Setenv("PORT", "") + t.Setenv("DNSWATCHER_TLS_EXPIRY_WARNING", "") + t.Setenv("DNSWATCHER_DEBUG", "") + t.Setenv("DNSWATCHER_MAINTENANCE_MODE", "") + t.Setenv("DNSWATCHER_SEND_TEST_NOTIFICATION", "") cfg, err := config.New(nil, newTestParams(t)) require.NoError(t, err) assert.Equal(t, time.Hour, cfg.DNSInterval) assert.Equal(t, 12*time.Hour, cfg.TLSInterval) + assert.Equal(t, 8080, cfg.Port) + assert.Equal(t, 7, cfg.TLSExpiryWarning) + assert.False(t, cfg.Debug) + assert.False(t, cfg.MaintenanceMode) + assert.False(t, cfg.SendTestNotification) +} + +// TestNew_InvalidNumberOrBoolStopsStartup checks values that must stop +// startup; TestNew_EnvironmentOverrides, the two tests after this one and +// TestNew_EmptyValueMeansDefault check values that must not. +func TestNew_InvalidNumberOrBoolStopsStartup(t *testing.T) { + notBools := []string{"yes", "on", "2", "tRUE"} + tests := []struct { + variable string + values []string + want error + }{ + { + variable: "PORT", + values: []string{"notaport", "0", "-1", "65536", "8080.5"}, + want: config.ErrInvalidPort, + }, + { + variable: "DNSWATCHER_TLS_EXPIRY_WARNING", + values: []string{"abc", "7d", "0", "-5"}, + want: config.ErrInvalidTLSExpiryWarning, + }, + { + variable: "DNSWATCHER_DEBUG", + values: notBools, + want: config.ErrInvalidBool, + }, + { + variable: "DNSWATCHER_MAINTENANCE_MODE", + values: notBools, + want: config.ErrInvalidBool, + }, + { + variable: "DNSWATCHER_SEND_TEST_NOTIFICATION", + values: notBools, + want: config.ErrInvalidBool, + }, + } + + for _, tt := range tests { + for _, value := range tt.values { + t.Run(tt.variable+"="+value, func(t *testing.T) { + viper.Reset() + t.Setenv("DNSWATCHER_TARGETS", "example.com") + t.Setenv(tt.variable, value) + + _, err := config.New(nil, newTestParams(t)) + require.ErrorIs(t, err, tt.want) + require.ErrorContains(t, err, tt.variable) + require.ErrorContains(t, err, strconv.Quote(value)) + }) + } + } +} + +func TestNew_LowestNumbersAndOtherSpellingsOfTrue(t *testing.T) { + viper.Reset() + t.Setenv("DNSWATCHER_TARGETS", "example.com") + t.Setenv("PORT", "1") + t.Setenv("DNSWATCHER_TLS_EXPIRY_WARNING", "1") + t.Setenv("DNSWATCHER_DEBUG", "1") + t.Setenv("DNSWATCHER_MAINTENANCE_MODE", "T") + t.Setenv("DNSWATCHER_SEND_TEST_NOTIFICATION", "True") + + cfg, err := config.New(nil, newTestParams(t)) + require.NoError(t, err) + assert.Equal(t, 1, cfg.Port) + assert.Equal(t, 1, cfg.TLSExpiryWarning) + assert.True(t, cfg.Debug) + assert.True(t, cfg.MaintenanceMode) + assert.True(t, cfg.SendTestNotification) +} + +func TestNew_HighestPort(t *testing.T) { + viper.Reset() + t.Setenv("DNSWATCHER_TARGETS", "example.com") + t.Setenv("PORT", "65535") + + cfg, err := config.New(nil, newTestParams(t)) + require.NoError(t, err) + assert.Equal(t, 65535, cfg.Port) } func TestNew_DebugEnablesDebugLogging(t *testing.T) { @@ -227,6 +327,51 @@ func TestNew_TargetsWithTrailingComma(t *testing.T) { "trailing comma should be ignored") } +func TestNew_SkipRecordNotifications(t *testing.T) { + viper.Reset() + t.Setenv("DNSWATCHER_TARGETS", "example.net,www.example.net,example.org") + t.Setenv("DNSWATCHER_SKIP_RECORD_NOTIFICATIONS", + " WWW.Example.net. , example.net,www.example.net") + + cfg, err := config.New(nil, newTestParams(t)) + require.NoError(t, err) + assert.Equal(t, + []string{"www.example.net", "example.net"}, + cfg.SkipRecordNotifications, + "names are written as targets are, each once", + ) +} + +func TestNew_SkipRecordNotificationsNotInTargetsStopsStartup(t *testing.T) { + viper.Reset() + t.Setenv("DNSWATCHER_TARGETS", "example.net") + t.Setenv("DNSWATCHER_SKIP_RECORD_NOTIFICATIONS", + "example.net,www.example.net") + + _, err := config.New(nil, newTestParams(t)) + require.ErrorIs(t, err, config.ErrNotInTargets) + require.ErrorContains(t, err, "DNSWATCHER_SKIP_RECORD_NOTIFICATIONS") + require.ErrorContains(t, err, `"www.example.net"`) +} + +// TestNew_MetricsUsernameWithoutPasswordStopsStartup checks an unset and an +// empty password; TestNew_EnvironmentOverrides checks a username with one. +func TestNew_MetricsUsernameWithoutPasswordStopsStartup(t *testing.T) { + viper.Reset() + t.Setenv("DNSWATCHER_TARGETS", "example.com") + t.Setenv("DNSWATCHER_METRICS_USERNAME", "prom") + + _, err := config.New(nil, newTestParams(t)) + require.ErrorIs(t, err, config.ErrNoMetricsPassword, "password unset") + require.ErrorContains(t, err, "DNSWATCHER_METRICS_PASSWORD") + + viper.Reset() + t.Setenv("DNSWATCHER_METRICS_PASSWORD", "") + + _, err = config.New(nil, newTestParams(t)) + require.ErrorIs(t, err, config.ErrNoMetricsPassword, "password empty") +} + func TestNew_CustomDNSIntervalDuration(t *testing.T) { viper.Reset() t.Setenv("DNSWATCHER_TARGETS", "example.com") diff --git a/internal/globals/globals_test.go b/internal/globals/globals_test.go new file mode 100644 index 0000000..f44174e --- /dev/null +++ b/internal/globals/globals_test.go @@ -0,0 +1,50 @@ +package globals_test + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "sneak.berlin/go/dnswatcher/internal/globals" +) + +// TestGlobals exercises the package-level version and appname +// variables through their setters and read-back via New. These are +// shared package state, so the test mutates a global and must run +// sequentially; it cannot use t.Parallel(). +// +//nolint:paralleltest // mutates shared package-level globals, must run sequentially +func TestGlobals(t *testing.T) { + versions := []string{"v1.2.3", "dev", "", "v1.2.3-4-gabcdef"} + for _, want := range versions { + globals.SetVersion(want) + + g, err := globals.New(nil) + require.NoError(t, err) + assert.Equal(t, want, g.Version, + "New must surface the version set by SetVersion") + } + + names := []string{"dnswatcher", "other", ""} + for _, want := range names { + globals.SetAppname(want) + + g, err := globals.New(nil) + require.NoError(t, err) + assert.Equal(t, want, g.Appname, + "New must surface the appname set by SetAppname") + } + + // New returns a snapshot: a later SetVersion must not mutate a + // Globals handed out earlier. + globals.SetVersion("first") + + g, err := globals.New(nil) + require.NoError(t, err) + + globals.SetVersion("second") + assert.Equal(t, "first", g.Version, + "a Globals returned by New must not change when the "+ + "package variable is set again") +} diff --git a/internal/handlers/dashboard.go b/internal/handlers/dashboard.go index 6d4ac57..a5068c8 100644 --- a/internal/handlers/dashboard.go +++ b/internal/handlers/dashboard.go @@ -1,11 +1,14 @@ package handlers import ( + "cmp" "embed" "fmt" "html/template" + "maps" "math" "net/http" + "slices" "strings" "time" @@ -40,12 +43,19 @@ func newDashboardTemplate() *template.Template { ) } -// dashboardData is the data passed to the dashboard template. +// dashboardData is the data passed to the dashboard template. Hostnames +// and DomainRecords split the records in Snapshot.Hostnames, which also +// holds the apex domains' own (see splitHostnames). Ports holds +// Snapshot.Ports with each port's names split into domains and +// hostnames, as /api/v1/status gives them (see buildPorts). type dashboardData struct { - Snapshot state.Snapshot - Alerts []notify.AlertEntry - StateAge string - GeneratedAt string + Snapshot state.Snapshot + Hostnames map[string]*state.HostnameState + DomainRecords map[string]*state.HostnameState + Ports map[string]*statusPortInfo + Alerts []notify.AlertEntry + StateAge string + GeneratedAt string } // HandleDashboard returns the dashboard page handler. @@ -58,12 +68,16 @@ func (h *Handlers) HandleDashboard() http.HandlerFunc { ) { snap := h.state.GetSnapshot() alerts := h.notifyHistory.Recent() + hostnames, domainRecords := splitHostnames(snap) data := dashboardData{ - Snapshot: snap, - Alerts: alerts, - StateAge: relTime(snap.LastUpdated), - GeneratedAt: time.Now().UTC().Format("2006-01-02 15:04:05"), + Snapshot: snap, + Hostnames: hostnames, + DomainRecords: domainRecords, + Ports: buildPorts(snap), + Alerts: alerts, + StateAge: relTime(snap.LastUpdated), + GeneratedAt: time.Now().UTC().Format("2006-01-02 15:04:05"), } writer.Header().Set( @@ -122,16 +136,37 @@ func joinStrings(items []string, sep string) string { } // formatRecords formats a map of record type → values into a -// compact display string. +// compact display string. Record types are listed in the order the +// README lists them, any other type after them in alphabetical order, +// so rows of nameservers with the same records read the same. func formatRecords(records map[string][]string) string { if len(records) == 0 { return "-" } + order := []string{"A", "AAAA", "CNAME", "MX", "TXT", "SRV", "CAA", "NS"} + + position := func(rtype string) int { + i := slices.Index(order, rtype) + if i < 0 { + return len(order) + } + + return i + } + + rtypes := slices.Collect(maps.Keys(records)) + slices.SortFunc(rtypes, func(a, b string) int { + return cmp.Or( + cmp.Compare(position(a), position(b)), + strings.Compare(a, b), + ) + }) + var parts []string - for rtype, values := range records { - for _, v := range values { + for _, rtype := range rtypes { + for _, v := range records[rtype] { parts = append(parts, rtype+": "+v) } } diff --git a/internal/handlers/dashboard_test.go b/internal/handlers/dashboard_test.go index 554cf9d..57d4f9c 100644 --- a/internal/handlers/dashboard_test.go +++ b/internal/handlers/dashboard_test.go @@ -1,6 +1,8 @@ package handlers_test import ( + "regexp" + "strings" "testing" "time" @@ -78,3 +80,194 @@ func TestFormatRecords(t *testing.T) { t.Errorf("unexpected format: %q", got) } } + +// TestFormatRecordsTypeOrder checks that record types are listed in +// the README's order (A, AAAA, CNAME, MX, TXT, SRV, CAA, NS), with +// any other type after them in alphabetical order. +func TestFormatRecordsTypeOrder(t *testing.T) { + t.Parallel() + + got := handlers.FormatRecords(map[string][]string{ + "SOA": {"ns1.example.com. hostmaster.example.com. 1 2 3 4 5"}, + "NS": {"ns1.example.com.", "ns2.example.com."}, + "CAA": {`0 issue "letsencrypt.org"`}, + "DNAME": {"example.net."}, + "TXT": {"v=spf1 -all"}, + "SRV": {"10 5 443 www.example.com."}, + "MX": {"10 mail.example.com."}, + "CNAME": {"www.example.com."}, + "AAAA": {"2001:db8::1"}, + "A": {"192.0.2.1"}, + }) + + want := strings.Join([]string{ + "A: 192.0.2.1", + "AAAA: 2001:db8::1", + "CNAME: www.example.com.", + "MX: 10 mail.example.com.", + "TXT: v=spf1 -all", + "SRV: 10 5 443 www.example.com.", + `CAA: 0 issue "letsencrypt.org"`, + "NS: ns1.example.com.", + "NS: ns2.example.com.", + "DNAME: example.net.", + "SOA: ns1.example.com. hostmaster.example.com. 1 2 3 4 5", + }, ", ") + + if got != want { + t.Errorf("FormatRecords lists types out of order:\n got %q\nwant %q", + got, want) + } +} + +// dashboardRow returns the table row of page that contains name. +func dashboardRow(t *testing.T, page string, name string) string { + t.Helper() + + for row := range strings.SplitSeq(page, " "+heading+" ") { + return section + } + } + + t.Fatalf("dashboard has no section headed %q", heading) + + return "" +} + +// TestDashboardShowsDomainRecordsUnderDomains checks that the dashboard +// shows an apex domain's own records in the Domains section, and +// neither lists nor counts the domain as a hostname. +func TestDashboardShowsDomainRecordsUnderDomains(t *testing.T) { + t.Parallel() + + page := get(t, newHandlersWithFailures(t).HandleDashboard()) + + domains := dashboardSection(t, page, "Domains") + if !strings.Contains(dashboardRow(t, domains, domainAddress), testDomain) { + t.Errorf("row of %s does not name %s", domainAddress, testDomain) + } + + if strings.Contains(dashboardSection(t, page, "Hostnames"), testDomain) { + t.Errorf("Hostnames section lists the domain %s", testDomain) + } + + words := strings.Join(strings.Fields(page), " ") + + footer := "monitoring 2 domains + 1 hostnames" + if !strings.Contains(words, footer) { + t.Errorf("dashboard does not say %q", footer) + } + + // With the tags taken out, the summary bar starts "Domains 2 + // Hostnames 1". + text := regexp.MustCompile(`<[^>]*>`).ReplaceAllString(page, " ") + summary := "Domains 2 Hostnames 1" + + if !strings.Contains(strings.Join(strings.Fields(text), " "), summary) { + t.Errorf("summary bar does not say %q", summary) + } +} + +// TestDashboardMarksDomainThatDoesNotExist checks that the Domains +// section says a domain that does not exist does not exist, and does +// not say so of a domain that exists. +func TestDashboardMarksDomainThatDoesNotExist(t *testing.T) { + t.Parallel() + + page := get(t, newHandlersWithFailures(t).HandleDashboard()) + domains := dashboardSection(t, page, "Domains") + + if !strings.Contains(dashboardRow(t, domains, missingDomain), "does not exist") { + t.Errorf("row of %s does not say it does not exist", missingDomain) + } + + if strings.Contains(dashboardRow(t, domains, testDomain), "does not exist") { + t.Errorf("row of %s says it does not exist", testDomain) + } +} + +// rowCells returns the text of each cell of a dashboard table row +// whose cells start with tag, "]*>`) + parts := strings.Split(row, tag)[1:] + cells := make([]string, 0, len(parts)) + + for _, cell := range parts { + text := tags.ReplaceAllString(tag+cell, " ") + cells = append(cells, strings.Join(strings.Fields(text), " ")) + } + + return cells +} + +// TestDashboardPortsTellDomainsFromHostnames checks that the Ports +// table lists an apex domain under Domains and a hostname under +// Hostnames when both resolve to the port's address. +func TestDashboardPortsTellDomainsFromHostnames(t *testing.T) { + t.Parallel() + + page := get(t, newHandlersWithFailures(t).HandleDashboard()) + ports := dashboardSection(t, page, "Ports") + + headings := rowCells(dashboardRow(t, ports, "Address"), "
- {{ len .Snapshot.Hostnames }} + {{ len .Hostnames }}
@@ -84,7 +84,11 @@ {{ $name }} + {{ if $ds.NXDomain }} + does not exist + {{ else }} {{ joinStrings $ds.Nameservers ", " }} + {{ end }} {{ relTime $ds.LastChecked }} @@ -94,6 +98,24 @@
+ {{ if .DomainRecords }} +
+ + + + + + + + + + + + {{ template "records" .DomainRecords }} + +
DomainNSStatusRecordsChecked
+
+ {{ end }} {{ else }}

No domains configured. @@ -108,7 +130,7 @@ > Hostnames - {{ if .Snapshot.Hostnames }} + {{ if .Hostnames }}

@@ -121,39 +143,7 @@ - {{ range $name, $hs := .Snapshot.Hostnames }} - {{ range $ns, $nsr := $hs.RecordsByNameserver }} - - - - - - - - {{ end }} - {{ end }} + {{ template "records" .Hostnames }}
- {{ $name }} - - {{ $ns }} - - {{ if eq $nsr.Status "ok" }} - ok - {{ else }} - {{ $nsr.Status }} - {{ end }} - - {{ formatRecords $nsr.Records }} - - {{ relTime $nsr.LastChecked }} -
@@ -171,19 +161,20 @@ > Ports - {{ if .Snapshot.Ports }} + {{ if .Ports }}
+ - {{ range $key, $ps := .Snapshot.Ports }} + {{ range $key, $ps := .Ports }} + @@ -258,6 +252,11 @@ > {{ end }} + {{ if $cs.Error }} + + {{ else }} @@ -285,6 +284,7 @@ {{ end }} {{ end }} + {{ end }} @@ -363,8 +363,48 @@ class="text-[11px] text-slate-700 border-t border-slate-800 pt-4 mt-8" > dnswatcher · monitoring {{ len .Snapshot.Domains }} domains + - {{ len .Snapshot.Hostnames }} hostnames + {{ len .Hostnames }} hostnames +{{/* ---- One row per nameserver of each name in the map it is given ---- */}} +{{ define "records" }} +{{ range $name, $hs := . }} +{{ range $ns, $nsr := $hs.RecordsByNameserver }} + + + + + + + +{{ end }} +{{ end }} +{{ end }} diff --git a/internal/healthcheck/healthcheck_test.go b/internal/healthcheck/healthcheck_test.go new file mode 100644 index 0000000..b87c3d1 --- /dev/null +++ b/internal/healthcheck/healthcheck_test.go @@ -0,0 +1,134 @@ +package healthcheck_test + +import ( + "context" + "encoding/json" + "testing" + "time" + + "go.uber.org/fx" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "sneak.berlin/go/dnswatcher/internal/config" + "sneak.berlin/go/dnswatcher/internal/globals" + "sneak.berlin/go/dnswatcher/internal/healthcheck" + "sneak.berlin/go/dnswatcher/internal/logger" +) + +// recordingLifecycle is a minimal fx.Lifecycle that records the hooks +// appended to it, so healthcheck.New can be exercised through its real +// constructor without standing up a whole fx application. +type recordingLifecycle struct { + hooks []fx.Hook +} + +func (l *recordingLifecycle) Append(hook fx.Hook) { + l.hooks = append(l.hooks, hook) +} + +// newHealthcheck builds a Healthcheck through the real constructor and +// runs the registered OnStart hook so StartupTime is set the same way +// the fx lifecycle would set it. +func newHealthcheck( + t *testing.T, + maintenance bool, + version string, +) *healthcheck.Healthcheck { + t.Helper() + + g := &globals.Globals{Appname: "dnswatcher", Version: version} + + log, err := logger.New(nil, logger.Params{Globals: g}) + require.NoError(t, err) + + lifecycle := &recordingLifecycle{} + + hc, err := healthcheck.New(lifecycle, healthcheck.Params{ + Globals: g, + Config: &config.Config{MaintenanceMode: maintenance}, + Logger: log, + }) + require.NoError(t, err) + + require.Len(t, lifecycle.hooks, 1, + "New must register exactly one lifecycle hook") + require.NotNil(t, lifecycle.hooks[0].OnStart) + require.NoError(t, lifecycle.hooks[0].OnStart(context.Background())) + + return hc +} + +func TestCheckStatusAndPayloadShape(t *testing.T) { + t.Parallel() + + hc := newHealthcheck(t, false, "v9.9.9") + resp := hc.Check() + + assert.Equal(t, "ok", resp.Status) + + // The JSON shape and field names are part of the contract for the + // /health and /.well-known/healthcheck routes, so assert on the + // exact set of keys the response marshals to. + raw, err := json.Marshal(resp) + require.NoError(t, err) + + var fields map[string]json.RawMessage + require.NoError(t, json.Unmarshal(raw, &fields)) + + wantKeys := []string{ + "status", + "now", + "uptimeSeconds", + "uptimeHuman", + "version", + "appname", + "maintenanceMode", + } + assert.Len(t, fields, len(wantKeys), + "response must marshal to exactly the documented fields") + + for _, key := range wantKeys { + assert.Contains(t, fields, key, "missing JSON field %q", key) + } + + // The Now field is documented as RFC3339Nano; a change to the + // format constant should turn this red. + _, err = time.Parse(time.RFC3339Nano, resp.Now) + assert.NoError(t, err, "Now must be RFC3339Nano") +} + +func TestCheckMaintenanceModeReflectsConfig(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + maintenance bool + }{ + {"maintenance off", false}, + {"maintenance on", true}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + hc := newHealthcheck(t, tt.maintenance, "test") + resp := hc.Check() + assert.Equal(t, tt.maintenance, resp.Maintenance, + "maintenanceMode must mirror Config.MaintenanceMode") + }) + } +} + +func TestCheckSurfacesVersionAndAppname(t *testing.T) { + t.Parallel() + + hc := newHealthcheck(t, false, "surfaced-version-123") + resp := hc.Check() + + assert.Equal(t, "surfaced-version-123", resp.Version, + "version from globals must appear in the payload") + assert.Equal(t, "dnswatcher", resp.Appname) +} diff --git a/internal/livednstest/livednstest.go b/internal/livednstest/livednstest.go new file mode 100644 index 0000000..7ce20cc --- /dev/null +++ b/internal/livednstest/livednstest.go @@ -0,0 +1,128 @@ +// Package livednstest runs the live DNS operations of tests. Tests that +// look something up in DNS query live DNS servers, never a stand-in — +// see TESTING.md. Nothing here mocks, fakes, stubs, records or replays +// DNS, and nothing here skips a test: it only changes *how* the live +// queries are issued, so that a single dropped UDP packet or one slow +// authoritative server does not turn correct code into a red build. +// +// Two mechanisms: +// +// 1. Bounded concurrency. Tests run in parallel and the build hosts +// have many cores, so without a limit every test starts its own +// iterative resolution at the same instant and they all send their +// first queries to the root servers within a few milliseconds of +// each other. Root servers rate-limit that, which shows up as a +// different arbitrary subset of tests failing on each run. Run caps +// how many live operations are in flight at once in one test binary. +// +// 2. Retry with exponential backoff. Each live operation gets several +// attempts with its own timeout. An attempt is retried when it +// obtained nothing to check, never because of what the test +// asserts about the result, so a wrong result still fails on the +// first attempt. A fault in the code under test that leaves +// nothing to check looks the same as live DNS not answering, and +// fails only after the last attempt. +package livednstest + +import ( + "context" + "errors" + "testing" + "time" +) + +const ( + // attempts is how many times a live DNS operation is attempted + // before the test fails. + attempts = 3 + + // AttemptTimeout bounds one attempt. It must fit the longest + // operation, a watcher check, which sends over a hundred queries one + // after another and on a slow build host takes several times as long + // as the few seconds it takes on a fast one. An operation whose + // every attempt fails takes attempts * AttemptTimeout plus the + // backoff, about 56 seconds, after it waits for one of the + // Concurrency slots that every live operation in the test binary + // shares. So when live DNS does not answer at all, a test binary + // with more live operations than slots runs into the 90-second + // `go test -timeout` backstop instead of each test failing on its + // own. + AttemptTimeout = 18 * time.Second + + // backoffBase is the delay after the first failed attempt; it is + // multiplied by backoffFactor each time. + backoffBase = 500 * time.Millisecond + + // backoffFactor is the exponential backoff multiplier. + backoffFactor = 2 + + // Concurrency caps how many live operations may be in flight + // across one test binary at once. + Concurrency = 6 +) + +// gate bounds concurrent live operations. It has to be package scoped: +// the whole point is that it is shared by every parallel test in the +// test binary. +// +//nolint:gochecknoglobals // package-wide live query rate limit +var gate = make(chan struct{}, Concurrency) + +// ErrNoAnswer reports that a live operation produced no usable answer, +// which is retried rather than asserted on. +var ErrNoAnswer = errors.New("no answer from live DNS") + +// Run executes one attempt of a live operation, holding a slot in gate +// for its duration and bounding it with its own timeout. +func Run(op func(ctx context.Context) error) error { + gate <- struct{}{} + defer func() { <-gate }() + + ctx, cancel := context.WithTimeout( + context.Background(), AttemptTimeout, + ) + defer cancel() + + return op(ctx) +} + +// Retry runs op until it reports success, retrying failures with +// exponential backoff, and fails the test if every attempt fails. op +// returns an error only for a failure to obtain an answer — never for +// an answer the test disagrees with, which belongs in an assertion so +// that it fails immediately. op stores whatever it obtained where its +// caller can find it. +func Retry( + t *testing.T, + what string, + op func(ctx context.Context) error, +) { + t.Helper() + + var last error + + backoff := backoffBase + + for attempt := range attempts { + if attempt > 0 { + t.Logf( + "%s: attempt %d of %d failed (%v), "+ + "retrying in %s", + what, attempt, attempts, last, backoff, + ) + time.Sleep(backoff) + + backoff *= backoffFactor + } + + last = Run(op) + if last == nil { + return + } + } + + t.Fatalf( + "%s: all %d live attempts failed: %v", + what, attempts, last, + ) +} diff --git a/internal/livednstest/livednstest_test.go b/internal/livednstest/livednstest_test.go new file mode 100644 index 0000000..d7f2376 --- /dev/null +++ b/internal/livednstest/livednstest_test.go @@ -0,0 +1,103 @@ +package livednstest_test + +import ( + "context" + "sync" + "testing" + "time" + + "github.com/stretchr/testify/assert" + + "sneak.berlin/go/dnswatcher/internal/livednstest" +) + +// Tests for the retry and the concurrency limit themselves. They +// perform no DNS resolution of any kind. + +func TestRetryRecoversFromTransientFailure(t *testing.T) { + t.Parallel() + + const wantAttempts = 2 + + attempts := 0 + + livednstest.Retry(t, "transient", func(_ context.Context) error { + attempts++ + + if attempts < wantAttempts { + return livednstest.ErrNoAnswer + } + + return nil + }) + + assert.Equal(t, wantAttempts, attempts) +} + +func TestRetryGivesEachAttemptADeadline(t *testing.T) { + t.Parallel() + + livednstest.Retry(t, "deadline", func(ctx context.Context) error { + deadline, ok := ctx.Deadline() + assert.True(t, ok, "attempt should carry a deadline") + + remaining := time.Until(deadline) + + assert.LessOrEqual(t, remaining, livednstest.AttemptTimeout) + + // Lower bound too: without one this passes for a + // deadline far shorter than intended, which would + // silently turn every live attempt into an instant + // timeout. + assert.Greater(t, remaining, livednstest.AttemptTimeout/2) + + return nil + }) +} + +func TestRunBoundsConcurrency(t *testing.T) { + t.Parallel() + + const workers = 24 + + var ( + mu sync.Mutex + wg sync.WaitGroup + inFlight int + maxSeen int + ) + + wg.Add(workers) + + for range workers { + go func() { + defer wg.Done() + + _ = livednstest.Run(func(_ context.Context) error { + mu.Lock() + inFlight++ + + if inFlight > maxSeen { + maxSeen = inFlight + } + mu.Unlock() + + time.Sleep(time.Millisecond) + + mu.Lock() + inFlight-- + mu.Unlock() + + return nil + }) + }() + } + + wg.Wait() + + assert.Positive(t, maxSeen) + assert.LessOrEqual( + t, maxSeen, livednstest.Concurrency, + "live queries must stay under the package-wide gate", + ) +} diff --git a/internal/logger/logger_test.go b/internal/logger/logger_test.go new file mode 100644 index 0000000..80ec8c9 --- /dev/null +++ b/internal/logger/logger_test.go @@ -0,0 +1,66 @@ +package logger_test + +import ( + "context" + "log/slog" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "sneak.berlin/go/dnswatcher/internal/globals" + "sneak.berlin/go/dnswatcher/internal/logger" +) + +func newTestLogger(t *testing.T) *logger.Logger { + t.Helper() + + g := &globals.Globals{Appname: "dnswatcher", Version: "test"} + + l, err := logger.New(nil, logger.Params{Globals: g}) + require.NoError(t, err) + + return l +} + +// TestNewReturnsUsableLogger checks that the constructor yields a +// working *slog.Logger. +func TestNewReturnsUsableLogger(t *testing.T) { + t.Parallel() + + l := newTestLogger(t) + require.NotNil(t, l.Get(), "Get must return a non-nil logger") +} + +// TestDefaultLevelExcludesDebug verifies the default configuration +// logs at info: debug records are suppressed, info records pass. +func TestDefaultLevelExcludesDebug(t *testing.T) { + t.Parallel() + + log := newTestLogger(t).Get() + ctx := context.Background() + + assert.False(t, log.Enabled(ctx, slog.LevelDebug), + "debug must be suppressed at the default level") + assert.True(t, log.Enabled(ctx, slog.LevelInfo), + "info must be enabled at the default level") +} + +// TestEnableDebugLoggingChangesLevel verifies the debug and non-debug +// configurations differ as intended: enabling debug makes debug +// records pass where they previously did not. +func TestEnableDebugLoggingChangesLevel(t *testing.T) { + t.Parallel() + + l := newTestLogger(t) + log := l.Get() + ctx := context.Background() + + require.False(t, log.Enabled(ctx, slog.LevelDebug), + "debug must start disabled") + + l.EnableDebugLogging() + + assert.True(t, log.Enabled(ctx, slog.LevelDebug), + "debug must be enabled after EnableDebugLogging") +} diff --git a/internal/middleware/export_test.go b/internal/middleware/export_test.go new file mode 100644 index 0000000..93680e8 --- /dev/null +++ b/internal/middleware/export_test.go @@ -0,0 +1,19 @@ +package middleware + +import ( + "net/http" + "time" +) + +// The /metrics rate limit, exported so the tests can count requests +// against it. +const ( + MetricsRequestLimit = metricsRequestLimit + MetricsRequestWindow time.Duration = metricsRequestWindow +) + +// RealIP is realIP, exported so the tests can check which address it +// takes as the client's. +func RealIP(r *http.Request) string { + return realIP(r) +} diff --git a/internal/middleware/middleware.go b/internal/middleware/middleware.go index 0a05dd5..e558b0d 100644 --- a/internal/middleware/middleware.go +++ b/internal/middleware/middleware.go @@ -5,12 +5,14 @@ import ( "log/slog" "net" "net/http" + "net/netip" "strings" "time" "github.com/99designs/basicauth-go" "github.com/go-chi/chi/v5/middleware" "github.com/go-chi/cors" + "github.com/go-chi/httprate" "go.uber.org/fx" "sneak.berlin/go/dnswatcher/internal/config" @@ -21,6 +23,71 @@ import ( // corsMaxAge is the maximum age for CORS preflight responses. const corsMaxAge = 300 +// Rate limit for /metrics: each client address may send +// metricsRequestLimit requests per metricsRequestWindow. Every request +// counts, so password guessing gets at most 30 tries a minute per +// address. One Prometheus server scraping every 15 seconds sends 4 +// requests a minute, and two scraping every 5 seconds from one address +// send 24, so normal scraping stays under the limit. +const ( + metricsRequestLimit = 30 + metricsRequestWindow = time.Minute +) + +// Security response header values applied to every response. +// +// The CSP is as strict as the dashboard allows: the template ships no +// JavaScript, no inline styles, no inline event handlers and no images, +// and its only subresource is the embedded stylesheet at +// /s/css/tailwind.min.css, which style-src 'self' permits. Neither +// unsafe-inline nor unsafe-eval is used. frame-ancestors 'none' is the +// primary anti-framing control; X-Frame-Options is the legacy fallback. +const ( + // hstsValue is emitted unconditionally, including over plain HTTP, + // because the service runs behind a TLS-terminating proxy and the + // browser must still enforce HTTPS end to end. + hstsValue = "max-age=31536000; includeSubDomains" + + cspValue = "default-src 'self'; " + + "script-src 'none'; " + + "style-src 'self'; " + + "img-src 'self'; " + + "font-src 'none'; " + + "connect-src 'none'; " + + "object-src 'none'; " + + "base-uri 'none'; " + + "form-action 'none'; " + + "frame-ancestors 'none'" + + frameOptionsValue = "DENY" + + contentTypeOptionsValue = "nosniff" + + // referrerPolicyValue is stricter than the policy minimum of + // strict-origin-when-cross-origin: the dashboard has no + // cross-origin navigation needs and its URL may name internal + // hosts. + referrerPolicyValue = "no-referrer" + + permissionsPolicyValue = "accelerometer=(), " + + "autoplay=(), " + + "camera=(), " + + "display-capture=(), " + + "encrypted-media=(), " + + "fullscreen=(), " + + "geolocation=(), " + + "gyroscope=(), " + + "magnetometer=(), " + + "microphone=(), " + + "midi=(), " + + "payment=(), " + + "picture-in-picture=(), " + + "publickey-credentials-get=(), " + + "screen-wake-lock=(), " + + "usb=(), " + + "xr-spatial-tracking=()" +) + // Params contains dependencies for Middleware. type Params struct { fx.In @@ -142,6 +209,12 @@ func isTrustedProxy(ip net.IP) bool { // realIP extracts the client's real IP address from the request. // Proxy headers are only trusted from RFC1918/loopback addresses. +// +// Each proxy adds to the end of X-Forwarded-For the address it got the +// request from, so the client can write every entry before the one the +// first trusted proxy added. The client address is therefore the +// rightmost entry that is not a trusted proxy, or the leftmost entry +// when they all are. func realIP(r *http.Request) string { addr := ipFromHostPort(r.RemoteAddr) remoteIP := net.ParseIP(addr) @@ -156,36 +229,100 @@ func realIP(r *http.Request) string { return ip } - if xff := r.Header.Get("X-Forwarded-For"); xff != "" { - if parts := strings.SplitN( - xff, ",", 2, //nolint:mnd - ); len(parts) > 0 { - if ip := strings.TrimSpace(parts[0]); ip != "" { - return ip - } + // A proxy may add its entry as a header line of its own instead of + // appending to the line the client sent, so all lines form one list. + entries := strings.Split( + strings.Join(r.Header.Values("X-Forwarded-For"), ","), ",", + ) + client := strings.TrimSpace(entries[0]) + + for i := len(entries) - 1; i > 0; i-- { + entry := strings.TrimSpace(entries[i]) + if !isTrustedProxy(net.ParseIP(entry)) { + client = entry + + break } } + if client != "" { + return client + } + return addr } -// CORS returns CORS middleware. +// CORS returns middleware that lets any origin read a response. It is +// for the public, read-only routes only, so it allows only the +// methods those routes serve and no Authorization header. func (m *Middleware) CORS() func(http.Handler) http.Handler { return cors.Handler(cors.Options{ - AllowedOrigins: []string{"*"}, - AllowedMethods: []string{ - "GET", "POST", "PUT", "DELETE", "OPTIONS", - }, - AllowedHeaders: []string{ - "Accept", "Authorization", - "Content-Type", "X-CSRF-Token", - }, + AllowedOrigins: []string{"*"}, + AllowedMethods: []string{"GET", "OPTIONS"}, + AllowedHeaders: []string{"Accept", "Content-Type"}, ExposedHeaders: []string{"Link"}, AllowCredentials: false, MaxAge: corsMaxAge, }) } +// SecurityHeaders returns middleware that sets the security response +// headers required for production internet exposure on every response. +// +// The headers are set before the request reaches the next handler so +// that they are present on every response, including panics recovered +// by chi's Recoverer and timeouts produced by chi's Timeout. +func (m *Middleware) SecurityHeaders() func(http.Handler) http.Handler { + return func(next http.Handler) http.Handler { + return http.HandlerFunc(func( + writer http.ResponseWriter, + request *http.Request, + ) { + header := writer.Header() + header.Set("Strict-Transport-Security", hstsValue) + header.Set("Content-Security-Policy", cspValue) + header.Set("X-Frame-Options", frameOptionsValue) + header.Set( + "X-Content-Type-Options", + contentTypeOptionsValue, + ) + header.Set("Referrer-Policy", referrerPolicyValue) + header.Set( + "Permissions-Policy", + permissionsPolicyValue, + ) + + next.ServeHTTP(writer, request) + }) + } +} + +// MetricsRateLimit returns middleware for /metrics that answers 429 +// Too Many Requests to a client address over the rate limit. The +// address is the one realIP works out, so a client that is not a +// trusted proxy cannot get a fresh allowance by sending its own +// X-Real-IP or X-Forwarded-For. CanonicalizeIP counts all IPv6 +// addresses in one /64 as one client, since a client usually holds a +// whole /64. An IPv4 address a proxy reports in IPv6-mapped form +// (::ffff:203.0.113.1) is turned back into plain IPv4 first, as every +// such address is in the same /64. +func (m *Middleware) MetricsRateLimit() func(http.Handler) http.Handler { + return httprate.LimitBy( + metricsRequestLimit, + metricsRequestWindow, + func(request *http.Request) (string, error) { + ip := realIP(request) + + addr, err := netip.ParseAddr(ip) + if err == nil { + ip = addr.Unmap().String() + } + + return httprate.CanonicalizeIP(ip), nil + }, + ) +} + // MetricsAuth returns basic auth middleware for /metrics. func (m *Middleware) MetricsAuth() func(http.Handler) http.Handler { if m.params.Config.MetricsUsername == "" { diff --git a/internal/middleware/middleware_test.go b/internal/middleware/middleware_test.go new file mode 100644 index 0000000..9e306a9 --- /dev/null +++ b/internal/middleware/middleware_test.go @@ -0,0 +1,565 @@ +package middleware_test + +import ( + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" + + "github.com/go-chi/chi/v5" + "go.uber.org/fx/fxtest" + + "sneak.berlin/go/dnswatcher/internal/config" + "sneak.berlin/go/dnswatcher/internal/globals" + "sneak.berlin/go/dnswatcher/internal/handlers" + "sneak.berlin/go/dnswatcher/internal/logger" + "sneak.berlin/go/dnswatcher/internal/middleware" + "sneak.berlin/go/dnswatcher/internal/notify" + "sneak.berlin/go/dnswatcher/internal/state" +) + +// Expected security header values, spelled out literally so that any +// change to the middleware has to be made deliberately here as well. +const ( + wantHSTS = "max-age=31536000; includeSubDomains" + + wantCSP = "default-src 'self'; " + + "script-src 'none'; " + + "style-src 'self'; " + + "img-src 'self'; " + + "font-src 'none'; " + + "connect-src 'none'; " + + "object-src 'none'; " + + "base-uri 'none'; " + + "form-action 'none'; " + + "frame-ancestors 'none'" + + wantFrameOptions = "DENY" + + wantContentTypeOptions = "nosniff" + + wantReferrerPolicy = "no-referrer" + + wantPermissionsPolicy = "accelerometer=(), " + + "autoplay=(), " + + "camera=(), " + + "display-capture=(), " + + "encrypted-media=(), " + + "fullscreen=(), " + + "geolocation=(), " + + "gyroscope=(), " + + "magnetometer=(), " + + "microphone=(), " + + "midi=(), " + + "payment=(), " + + "picture-in-picture=(), " + + "publickey-credentials-get=(), " + + "screen-wake-lock=(), " + + "usb=(), " + + "xr-spatial-tracking=()" +) + +// stylesheetPath is the only subresource the dashboard loads. +const stylesheetPath = "/s/css/tailwind.min.css" + +// newTestLogger builds a logger for direct component construction. +func newTestLogger(t *testing.T) *logger.Logger { + t.Helper() + + glob, err := globals.New(nil) + if err != nil { + t.Fatalf("globals.New: %v", err) + } + + log, err := logger.New(nil, logger.Params{Globals: glob}) + if err != nil { + t.Fatalf("logger.New: %v", err) + } + + return log +} + +// newTestMiddleware builds a Middleware without an fx application. +func newTestMiddleware(t *testing.T) *middleware.Middleware { + t.Helper() + + glob, err := globals.New(nil) + if err != nil { + t.Fatalf("globals.New: %v", err) + } + + mw, err := middleware.New(nil, middleware.Params{ + Logger: newTestLogger(t), + Globals: glob, + Config: &config.Config{}, + }) + if err != nil { + t.Fatalf("middleware.New: %v", err) + } + + return mw +} + +// serveWithSecurityHeaders runs a GET through SecurityHeaders and +// returns the recorded response. +func serveWithSecurityHeaders( + t *testing.T, + target string, + handler http.Handler, +) *httptest.ResponseRecorder { + t.Helper() + + mw := newTestMiddleware(t) + rec := httptest.NewRecorder() + req := httptest.NewRequestWithContext( + t.Context(), http.MethodGet, target, nil, + ) + + mw.SecurityHeaders()(handler).ServeHTTP(rec, req) + + return rec +} + +// okHandler writes a trivial 200 response. +func okHandler() http.Handler { + return http.HandlerFunc(func( + writer http.ResponseWriter, + _ *http.Request, + ) { + writer.WriteHeader(http.StatusOK) + }) +} + +func TestSecurityHeaders(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + header string + want string + }{ + { + "hsts", + "Strict-Transport-Security", + wantHSTS, + }, + { + "csp", + "Content-Security-Policy", + wantCSP, + }, + { + "frame options", + "X-Frame-Options", + wantFrameOptions, + }, + { + "content type options", + "X-Content-Type-Options", + wantContentTypeOptions, + }, + { + "referrer policy", + "Referrer-Policy", + wantReferrerPolicy, + }, + { + "permissions policy", + "Permissions-Policy", + wantPermissionsPolicy, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + rec := serveWithSecurityHeaders(t, "/", okHandler()) + + got := rec.Header().Get(tt.header) + if got != tt.want { + t.Errorf( + "%s = %q, want %q", + tt.header, got, tt.want, + ) + } + }) + } +} + +// TestSecurityHeadersCSPDirectives guards the properties the repo +// policy requires of the content security policy itself. +func TestSecurityHeadersCSPDirectives(t *testing.T) { + t.Parallel() + + rec := serveWithSecurityHeaders(t, "/", okHandler()) + csp := rec.Header().Get("Content-Security-Policy") + + forbidden := []string{"unsafe-inline", "unsafe-eval"} + for _, directive := range forbidden { + if strings.Contains(csp, directive) { + t.Errorf("CSP must not contain %q: %q", directive, csp) + } + } + + required := []string{ + "default-src 'self'", + "script-src 'none'", + "style-src 'self'", + "frame-ancestors 'none'", + } + for _, directive := range required { + if !strings.Contains(csp, directive) { + t.Errorf("CSP must contain %q: %q", directive, csp) + } + } +} + +// TestSecurityHeadersOnErrorResponse verifies the headers are emitted +// even when the wrapped handler fails, since they are set before the +// handler runs. +func TestSecurityHeadersOnErrorResponse(t *testing.T) { + t.Parallel() + + failing := http.HandlerFunc(func( + writer http.ResponseWriter, + _ *http.Request, + ) { + http.Error( + writer, + "boom", + http.StatusInternalServerError, + ) + }) + + rec := serveWithSecurityHeaders(t, "/api/v1/status", failing) + + if rec.Code != http.StatusInternalServerError { + t.Fatalf("status = %d, want 500", rec.Code) + } + + if got := rec.Header().Get( + "X-Content-Type-Options", + ); got != wantContentTypeOptions { + t.Errorf( + "X-Content-Type-Options = %q, want %q", + got, wantContentTypeOptions, + ) + } + + if got := rec.Header().Get( + "Strict-Transport-Security", + ); got != wantHSTS { + t.Errorf( + "Strict-Transport-Security = %q, want %q", + got, wantHSTS, + ) + } +} + +// newTestHandlers builds real Handlers with empty monitoring state. +func newTestHandlers(t *testing.T) *handlers.Handlers { + t.Helper() + + glob, err := globals.New(nil) + if err != nil { + t.Fatalf("globals.New: %v", err) + } + + log := newTestLogger(t) + + notifier, err := notify.New(fxtest.NewLifecycle(t), notify.Params{ + Logger: log, + Config: &config.Config{}, + }) + if err != nil { + t.Fatalf("notify.New: %v", err) + } + + st, err := state.New(fxtest.NewLifecycle(t), state.Params{ + Logger: log, + Config: &config.Config{DataDir: t.TempDir()}, + }) + if err != nil { + t.Fatalf("state.New: %v", err) + } + + hnd, err := handlers.New(nil, handlers.Params{ + Logger: log, + Globals: glob, + State: st, + Notify: notifier, + }) + if err != nil { + t.Fatalf("handlers.New: %v", err) + } + + return hnd +} + +// TestDashboardRendersWithSecurityHeaders renders the real dashboard +// through the middleware and checks that the policy still permits the +// one stylesheet the page loads. +func TestDashboardRendersWithSecurityHeaders(t *testing.T) { + t.Parallel() + + mw := newTestMiddleware(t) + hnd := newTestHandlers(t) + + router := chi.NewRouter() + router.Use(mw.SecurityHeaders()) + router.Get("/", hnd.HandleDashboard()) + + rec := httptest.NewRecorder() + req := httptest.NewRequestWithContext( + t.Context(), http.MethodGet, "/", nil, + ) + + router.ServeHTTP(rec, req) + + if rec.Code != http.StatusOK { + t.Fatalf("status = %d, want 200", rec.Code) + } + + body := rec.Body.String() + if !strings.Contains(body, stylesheetPath) { + t.Errorf("dashboard does not reference %q", stylesheetPath) + } + + if !strings.Contains(body, "dnswatcher") { + t.Errorf("dashboard body looks empty: %d bytes", len(body)) + } + + csp := rec.Header().Get("Content-Security-Policy") + if csp != wantCSP { + t.Errorf("CSP = %q, want %q", csp, wantCSP) + } + + // The stylesheet is same-origin, so style-src 'self' allows it. + if !strings.Contains(csp, "style-src 'self'") { + t.Errorf("CSP would block %q: %q", stylesheetPath, csp) + } +} + +// Addresses for the rate limit and realIP tests: a client connecting +// directly, a trusted proxy, and a client behind that proxy as the +// proxy's X-Real-IP or X-Forwarded-For header names it. +const ( + directClient = "198.51.100.1:4000" + trustedProxy = "10.0.0.1:4000" + proxiedClient = "203.0.113.1" +) + +// statusFrom sends a GET through handler as if from remoteAddr, with +// an X-Real-IP header when xRealIP is not empty, and returns the +// response status. +func statusFrom( + t *testing.T, + handler http.Handler, + remoteAddr string, + xRealIP string, +) int { + t.Helper() + + req := httptest.NewRequestWithContext( + t.Context(), http.MethodGet, "/metrics", nil, + ) + req.RemoteAddr = remoteAddr + + if xRealIP != "" { + req.Header.Set("X-Real-IP", xRealIP) + } + + rec := httptest.NewRecorder() + handler.ServeHTTP(rec, req) + + return rec.Code +} + +// TestMetricsRateLimitAllowsScraping checks that one address can send, +// within one window, what two Prometheus servers scraping every 5 +// seconds send in that time, without being turned away. +func TestMetricsRateLimitAllowsScraping(t *testing.T) { + t.Parallel() + + const scrapeInterval = 5 * time.Second + + scrapes := 2 * int(middleware.MetricsRequestWindow/scrapeInterval) + limited := newTestMiddleware(t).MetricsRateLimit()(okHandler()) + + for i := range scrapes { + got := statusFrom(t, limited, directClient, "") + if got != http.StatusOK { + t.Fatalf( + "scrape %d of %d: status = %d, want 200", + i+1, scrapes, got, + ) + } + } +} + +// TestMetricsRateLimitKeysOnClientAddress checks which requests share +// an allowance. Each case uses up the allowance of one client, then +// sends one more request. +func TestMetricsRateLimitKeysOnClientAddress(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + usedRemoteAddr string + usedXRealIP string + nextRemoteAddr string + nextXRealIP string + want int + }{ + { + "same address", + directClient, "", + directClient, "", + http.StatusTooManyRequests, + }, + { + "another address", + directClient, "", + "198.51.100.2:4000", "", + http.StatusOK, + }, + { + "own X-Real-IP from an untrusted address", + directClient, "", + directClient, "203.0.113.9", + http.StatusTooManyRequests, + }, + { + "same client behind the proxy", + trustedProxy, proxiedClient, + trustedProxy, proxiedClient, + http.StatusTooManyRequests, + }, + { + "another client behind the proxy", + trustedProxy, proxiedClient, + trustedProxy, "203.0.113.2", + http.StatusOK, + }, + { + "another client behind the proxy, IPv6-mapped", + trustedProxy, "::ffff:203.0.113.1", + trustedProxy, "::ffff:203.0.113.2", + http.StatusOK, + }, + { + "same IPv6 /64", + "[2001:db8::1]:4000", "", + "[2001:db8::2]:4000", "", + http.StatusTooManyRequests, + }, + { + "another IPv6 /64", + "[2001:db8::1]:4000", "", + "[2001:db8:0:1::1]:4000", "", + http.StatusOK, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + limited := newTestMiddleware(t).MetricsRateLimit()(okHandler()) + + for range middleware.MetricsRequestLimit { + statusFrom(t, limited, tt.usedRemoteAddr, tt.usedXRealIP) + } + + got := statusFrom( + t, limited, tt.nextRemoteAddr, tt.nextXRealIP, + ) + if got != tt.want { + t.Errorf("status = %d, want %d", got, tt.want) + } + }) + } +} + +// TestRealIP checks which address realIP takes as the client's. Each +// element of forwardedFor is sent as an X-Forwarded-For header line of +// its own, and 198.51.100.9 is always an entry the client wrote itself. +func TestRealIP(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + remoteAddr string + xRealIP string + forwardedFor []string + want string + }{ + { + "untrusted peer, both headers ignored", + directClient, proxiedClient, []string{"198.51.100.9"}, + "198.51.100.1", + }, + { + "X-Real-IP from a trusted proxy wins", + trustedProxy, proxiedClient, []string{"203.0.113.8"}, + proxiedClient, + }, + { + "client's own entry, then the one the proxy added", + trustedProxy, "", []string{"198.51.100.9, 203.0.113.1"}, + proxiedClient, + }, + { + "several trusted proxies", + trustedProxy, "", + []string{"198.51.100.9, 203.0.113.1, 10.0.0.3, 10.0.0.2"}, + proxiedClient, + }, + { + "proxy adds a header line of its own", + trustedProxy, "", []string{"198.51.100.9", proxiedClient}, + proxiedClient, + }, + { + "every entry a trusted proxy", + trustedProxy, "", []string{"10.0.0.3, 10.0.0.2"}, + "10.0.0.3", + }, + { + "empty where the client address belongs", + trustedProxy, "", []string{"203.0.113.1, , 10.0.0.2"}, + "10.0.0.1", + }, + { + "no headers from a trusted proxy", + trustedProxy, "", nil, + "10.0.0.1", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + req := httptest.NewRequestWithContext( + t.Context(), http.MethodGet, "/", nil, + ) + req.RemoteAddr = tt.remoteAddr + + if tt.xRealIP != "" { + req.Header.Set("X-Real-IP", tt.xRealIP) + } + + for _, line := range tt.forwardedFor { + req.Header.Add("X-Forwarded-For", line) + } + + got := middleware.RealIP(req) + if got != tt.want { + t.Errorf("realIP = %q, want %q", got, tt.want) + } + }) + } +} diff --git a/internal/notify/delivery_test.go b/internal/notify/delivery_test.go index fdccd73..61a0e15 100644 --- a/internal/notify/delivery_test.go +++ b/internal/notify/delivery_test.go @@ -6,9 +6,13 @@ import ( "encoding/json" "errors" "io" + "log/slog" + "maps" "net/http" "net/http/httptest" "net/url" + "slices" + "strings" "sync" "testing" "time" @@ -36,6 +40,13 @@ const ( prioUrgent = "urgent" ) +// Endpoint names, as the log gives them, used across multiple tests. +const ( + endpointNtfy = "ntfy" + endpointSlack = "slack" + endpointMattermost = "mattermost" +) + // testHost is the hostname used in request construction tests. const testHost = "example.com" @@ -413,7 +424,8 @@ func sendSlackInfo( svc *notify.Service, target *url.URL, ) error { return svc.SendSlack( - context.Background(), target, "t", "m", prioInfo, + context.Background(), target, notify.ErrSlackFailed, + "t", "m", prioInfo, ) } @@ -506,6 +518,7 @@ func TestSendSlackPayloadFields(t *testing.T) { err := svc.SendSlack( context.Background(), webhookURL, + notify.ErrSlackFailed, "Alert Title", "Alert body text", "warning", @@ -608,7 +621,8 @@ func TestSendSlackAllColors(t *testing.T) { err := svc.SendSlack( context.Background(), - webhookURL, "t", "m", tc.priority, + webhookURL, notify.ErrSlackFailed, + "t", "m", tc.priority, ) if err != nil { t.Fatalf("SendSlack error: %v", err) @@ -659,7 +673,8 @@ func TestSendSlackNetworkError(t *testing.T) { ) err := svc.SendSlack( - context.Background(), webhookURL, "t", "m", "info", + context.Background(), webhookURL, notify.ErrSlackFailed, + "t", "m", "info", ) if err == nil { t.Fatal("expected error for network failure") @@ -1028,6 +1043,242 @@ func TestSendNotificationMattermostError(t *testing.T) { ) } +// TestSendNotificationErrorNamesEndpoint verifies that, with both +// Slack and Mattermost set, a failed delivery's logged error names +// the endpoint that failed. Both are sent by the Slack sender. +func TestSendNotificationErrorNamesEndpoint(t *testing.T) { + t.Parallel() + + srv := httptest.NewServer( + http.HandlerFunc( + func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusServiceUnavailable) + }), + ) + defer srv.Close() + + target, _ := url.Parse(srv.URL) + + svc, logs := newLoggingService(http.DefaultTransport) + svc.SetSlackWebhookURL(target) + svc.SetMattermostWebhookURL(target) + svc.SetSleepFunc(instantSleep) + svc.SetRetryConfig(notify.RetryConfig{ + MaxRetries: 1, + BaseDelay: time.Millisecond, + MaxDelay: time.Millisecond, + }) + + svc.SendNotification( + context.Background(), "t", "m", prioError, + ) + + waitForCondition(t, func() bool { + return svc.OutstandingDeliveries() == 0 + }) + + got := map[string]string{} + + for line := range strings.Lines(logs.String()) { + var record struct { + Msg string `json:"msg"` + Endpoint string `json:"endpoint"` + Error string `json:"error"` + } + + _ = json.Unmarshal([]byte(line), &record) + + if record.Msg == "failed to send notification after retries" { + got[record.Endpoint] = record.Error + } + } + + want := map[string]string{ + "slack": "slack notification failed: status 503", + "mattermost": "mattermost notification failed: status 503", + } + + if !maps.Equal(got, want) { + t.Errorf("logged errors = %v, want %v", got, want) + } +} + +// ── webhook URLs kept out of the log ────────────────────── + +// webhookPathAndQuery follows the server's address in every +// endpoint's URL in the tests below. A webhook URL's path or query +// lets anyone post to the channel or topic, so neither word marked +// secret in it may reach the log. +const webhookPathAndQuery = "/hooks/secret-in-path?token=secret-in-query" + +// assertNoWebhookSecret fails the test if text holds either word +// marked secret in webhookPathAndQuery. +func assertNoWebhookSecret(t *testing.T, text string) { + t.Helper() + + for _, secret := range []string{"secret-in-path", "secret-in-query"} { + if strings.Contains(text, secret) { + t.Errorf("%q from the webhook URL is in:\n%s", secret, text) + } + } +} + +// logRecord is the part of a JSON log line the tests below read. +type logRecord struct { + Level string `json:"level"` + Msg string `json:"msg"` + Endpoint string `json:"endpoint"` + Title string `json:"title"` + Error string `json:"error"` +} + +// parseLog decodes each line of a JSON log. +func parseLog(t *testing.T, logs string) []logRecord { + t.Helper() + + var records []logRecord + + for line := range strings.Lines(logs) { + var record logRecord + + err := json.Unmarshal([]byte(line), &record) + if err != nil { + t.Fatalf("decoding log line %q: %v", line, err) + } + + records = append(records, record) + } + + return records +} + +// sendToEachEndpoint sends one notification with the given title to +// ntfy, Slack and Mattermost, each at baseURL followed by +// webhookPathAndQuery, and retries a failed delivery once. When +// every delivery has finished it returns the JSON log written, +// debug lines included. +func sendToEachEndpoint(t *testing.T, baseURL, title string) string { + t.Helper() + + target, err := url.Parse(baseURL + webhookPathAndQuery) + if err != nil { + t.Fatalf("parsing webhook URL: %v", err) + } + + svc, logs := newLoggingService(http.DefaultTransport) + svc.SetNtfyURL(target) + svc.SetSlackWebhookURL(target) + svc.SetMattermostWebhookURL(target) + svc.SetSleepFunc(instantSleep) + svc.SetRetryConfig(notify.RetryConfig{ + MaxRetries: 1, + BaseDelay: time.Millisecond, + MaxDelay: time.Millisecond, + }) + + svc.SendNotification(context.Background(), title, "m", prioInfo) + + waitForCondition(t, func() bool { + return svc.OutstandingDeliveries() == 0 + }) + + return logs.String() +} + +// assertEachEndpointRetriedThenFailed fails the test unless, for each +// of ntfy, Slack and Mattermost, the lines of logs whose error +// contains reason are the retry line and then the final failure line. +func assertEachEndpointRetriedThenFailed( + t *testing.T, logs, reason string, +) { + t.Helper() + + // For each endpoint, the messages of its lines whose error + // contains reason, in the order they were logged. + got := map[string][]string{} + + for _, record := range parseLog(t, logs) { + if strings.Contains(record.Error, reason) { + got[record.Endpoint] = append( + got[record.Endpoint], record.Msg, + ) + } + } + + lines := []string{ + "notification delivery failed, retrying", + "failed to send notification after retries", + } + want := map[string][]string{ + endpointNtfy: lines, + endpointSlack: lines, + endpointMattermost: lines, + } + + if !maps.EqualFunc(got, want, slices.Equal) { + t.Errorf("lines whose error contains %q = %v, want %v", + reason, got, want) + } +} + +// TestSendNotificationDebugLogOmitsWebhookURL checks that, with debug +// logging on, a delivery to each endpoint logs the endpoint's name and +// the title, and no part of the URL's path or query. +func TestSendNotificationDebugLogOmitsWebhookURL(t *testing.T) { + t.Parallel() + + const title = "Port Change" + + srv := httptest.NewServer( + http.HandlerFunc( + func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusOK) + }), + ) + defer srv.Close() + + logs := sendToEachEndpoint(t, srv.URL, title) + + assertNoWebhookSecret(t, logs) + + got := map[string]string{} + + for _, record := range parseLog(t, logs) { + if record.Level == slog.LevelDebug.String() { + got[record.Endpoint] = record.Title + } + } + + want := map[string]string{ + endpointNtfy: title, + endpointSlack: title, + endpointMattermost: title, + } + + if !maps.Equal(got, want) { + t.Errorf("debug lines' endpoint and title = %v, want %v", got, want) + } +} + +// TestSendNotificationErrorLogOmitsWebhookURL checks that a delivery +// to each endpoint that cannot connect logs the transport's error, +// which names the refused connection, on its retry line and on its +// final failure line, and that no line holds any part of the URL's +// path or query. +func TestSendNotificationErrorLogOmitsWebhookURL(t *testing.T) { + t.Parallel() + + // Closed before anything is sent, so every attempt fails to + // connect. + srv := httptest.NewServer(http.NotFoundHandler()) + srv.Close() + + logs := sendToEachEndpoint(t, srv.URL, "t") + + assertNoWebhookSecret(t, logs) + assertEachEndpointRetriedThenFailed(t, logs, "connection refused") +} + // ── SlackPayload JSON marshaling ────────────────────────── func TestSlackPayloadJSON(t *testing.T) { diff --git a/internal/notify/export_test.go b/internal/notify/export_test.go index 723fee6..1324e3c 100644 --- a/internal/notify/export_test.go +++ b/internal/notify/export_test.go @@ -81,14 +81,16 @@ func (svc *Service) SendNtfy( return svc.sendNtfy(ctx, topicURL, title, message, priority) } -// SendSlack exports sendSlack for testing. +// SendSlack exports sendSlack for testing. It sends as the +// "slack" endpoint. func (svc *Service) SendSlack( ctx context.Context, webhookURL *url.URL, + failed error, title, message, priority string, ) error { return svc.sendSlack( - ctx, webhookURL, title, message, priority, + ctx, webhookURL, "slack", failed, title, message, priority, ) } diff --git a/internal/notify/notify.go b/internal/notify/notify.go index 1185ed6..e17e615 100644 --- a/internal/notify/notify.go +++ b/internal/notify/notify.go @@ -25,9 +25,6 @@ import ( // HTTP client timeout. const httpClientTimeout = 10 * time.Second -// HTTP status code thresholds. -const httpStatusClientError = 400 - // Sentinel errors for notification failures. var ( // ErrNtfyFailed indicates the ntfy request failed. @@ -56,6 +53,15 @@ func IsAllowedScheme(scheme string) bool { func ValidateWebhookURL(raw string) (*url.URL, error) { u, err := url.ParseRequestURI(raw) if err != nil { + // A *url.Error quotes the whole URL, and this error is + // logged at startup. A webhook URL's path or query lets + // anyone post to the channel or topic, so keep only the + // reason. + var urlErr *url.Error + if errors.As(err, &urlErr) { + err = urlErr.Err + } + return nil, fmt.Errorf("invalid URL: %w", err) } @@ -277,7 +283,8 @@ func (svc *Service) dispatchSlack( svc.dispatch(ctx, "slack", func(c context.Context) error { return svc.sendSlack( - c, svc.slackWebhookURL, title, message, priority, + c, svc.slackWebhookURL, "slack", ErrSlackFailed, + title, message, priority, ) }) } @@ -294,8 +301,8 @@ func (svc *Service) dispatchMattermost( ctx, "mattermost", func(c context.Context) error { return svc.sendSlack( - c, svc.mattermostWebhookURL, - title, message, priority, + c, svc.mattermostWebhookURL, "mattermost", + ErrMattermostFailed, title, message, priority, ) }, ) @@ -306,9 +313,11 @@ func (svc *Service) sendNtfy( topicURL *url.URL, title, message, priority string, ) error { + // The topic URL is left out of the log: it lets anyone who has + // it read and post to the topic. svc.log.Debug( "sending ntfy notification", - "topic", topicURL.String(), + "endpoint", "ntfy", "title", title, ) @@ -332,7 +341,10 @@ func (svc *Service) sendNtfy( defer func() { _ = resp.Body.Close() }() - if resp.StatusCode >= httpStatusClientError { + // Only a 2xx reply means ntfy took the notification. A + // redirect is not followed, so it is a failure too. + if resp.StatusCode < http.StatusOK || + resp.StatusCode >= http.StatusMultipleChoices { return fmt.Errorf( "%w: status %d", ErrNtfyFailed, resp.StatusCode, @@ -370,9 +382,17 @@ type SlackAttachment struct { Text string `json:"text"` } +// sendSlack posts to a Slack or Mattermost incoming webhook, which +// take the same payload. endpoint, "slack" or "mattermost", names +// the webhook in the log in place of its URL, which lets anyone who +// has it post to the channel. A reply status other than 2xx is +// returned wrapped in failed, ErrSlackFailed or ErrMattermostFailed, +// so the error names the endpoint. func (svc *Service) sendSlack( ctx context.Context, webhookURL *url.URL, + endpoint string, + failed error, title, message, priority string, ) error { ctx, cancel := context.WithTimeout( @@ -382,7 +402,7 @@ func (svc *Service) sendSlack( svc.log.Debug( "sending webhook notification", - "url", webhookURL.String(), + "endpoint", endpoint, "title", title, ) @@ -417,10 +437,13 @@ func (svc *Service) sendSlack( defer func() { _ = resp.Body.Close() }() - if resp.StatusCode >= httpStatusClientError { + // Only a 2xx reply means the webhook took the notification. A + // redirect is not followed, so it is a failure too. + if resp.StatusCode < http.StatusOK || + resp.StatusCode >= http.StatusMultipleChoices { return fmt.Errorf( "%w: status %d", - ErrSlackFailed, resp.StatusCode, + failed, resp.StatusCode, ) } diff --git a/internal/notify/notify_test.go b/internal/notify/notify_test.go index 3ac34dd..d0e49a0 100644 --- a/internal/notify/notify_test.go +++ b/internal/notify/notify_test.go @@ -79,6 +79,23 @@ func TestValidateWebhookURLInvalid(t *testing.T) { } } +// TestValidateWebhookURLErrorOmitsURL checks that the error for a +// webhook URL that does not parse, here because a carriage return +// was left at its end, leaves out the URL's path and query: the +// error is logged at startup. +func TestValidateWebhookURLErrorOmitsURL(t *testing.T) { + t.Parallel() + + _, err := notify.ValidateWebhookURL( + "https://hooks.example.com" + webhookPathAndQuery + "\r", + ) + if err == nil { + t.Fatal("expected an error for a URL ending in a carriage return") + } + + assertNoWebhookSecret(t, err.Error()) +} + func TestIsAllowedScheme(t *testing.T) { t.Parallel() diff --git a/internal/notify/retry.go b/internal/notify/retry.go index 6745085..ccf6666 100644 --- a/internal/notify/retry.go +++ b/internal/notify/retry.go @@ -115,7 +115,9 @@ func (svc *Service) deliverWithRetry( "endpoint", endpoint, "attempt", attempt+1, "maxAttempts", cfg.MaxRetries+1, - "retryIn", delay, + // As text: the JSON log writes a time.Duration as + // bare nanoseconds. + "retryIn", delay.String(), "error", lastErr, ) diff --git a/internal/notify/retry_test.go b/internal/notify/retry_test.go index 878e1bf..2fd9f58 100644 --- a/internal/notify/retry_test.go +++ b/internal/notify/retry_test.go @@ -2,6 +2,7 @@ package notify_test import ( "context" + "encoding/json" "errors" "net/http" "net/http/httptest" @@ -189,6 +190,50 @@ func TestDeliverWithRetryExhaustsAttempts(t *testing.T) { } } +// TestDeliverWithRetryLogsRetryInAsText checks that the wait +// before a retry is logged as text such as "1.02s", not as a +// count of nanoseconds. +func TestDeliverWithRetryLogsRetryInAsText(t *testing.T) { + t.Parallel() + + svc, logs := newLoggingService(http.DefaultTransport) + svc.SetRetryConfig(notify.RetryConfig{ + MaxRetries: 1, + BaseDelay: time.Second, + MaxDelay: time.Second, + }) + + var waited time.Duration + + svc.SetSleepFunc(func(d time.Duration) <-chan time.Time { + waited = d + + return instantSleep(d) + }) + + _ = svc.DeliverWithRetry( + context.Background(), "test", + func(_ context.Context) error { + return errFail + }, + ) + + // With one retry, only the first failure is logged. + var record map[string]any + + err := json.Unmarshal([]byte(logs.String()), &record) + if err != nil { + t.Fatalf("log is not one JSON record: %v\n%s", err, logs) + } + + if record["retryIn"] != waited.String() { + t.Errorf( + "retryIn logged as %v, want %q", + record["retryIn"], waited.String(), + ) + } +} + func TestDeliverWithRetryRespectsContextCancellation( t *testing.T, ) { @@ -491,3 +536,33 @@ func TestSendNotificationPermanentFailureLogsError( return attempts >= 3 }) } + +// TestSendNotificationRedirectFails checks that a 302 reply is a +// failed delivery at each endpoint: the server is sent the first +// attempt and the retry, and each endpoint logs a retry line and +// then the final failure line, both naming the 302. +func TestSendNotificationRedirectFails(t *testing.T) { + t.Parallel() + + var requests atomic.Int64 + + srv := httptest.NewServer( + http.HandlerFunc( + func(w http.ResponseWriter, r *http.Request) { + requests.Add(1) + http.Redirect(w, r, "/moved", http.StatusFound) + }), + ) + defer srv.Close() + + logs := sendToEachEndpoint(t, srv.URL, "t") + + // ntfy, Slack and Mattermost each send once and retry once. + const wantRequests = 3 * 2 + + if sent := requests.Load(); sent != wantRequests { + t.Errorf("server got %d requests, want %d", sent, wantRequests) + } + + assertEachEndpointRetriedThenFailed(t, logs, "status 302") +} diff --git a/internal/notify/shutdown_test.go b/internal/notify/shutdown_test.go index f938b64..afc1048 100644 --- a/internal/notify/shutdown_test.go +++ b/internal/notify/shutdown_test.go @@ -33,10 +33,29 @@ const ( // out. drainDeadline = 50 * time.Millisecond - // drainSlack is the upper bound on how long a bounded - // drain may take; generous enough for a loaded CI box, - // still far below the 20s test ceiling. - drainSlack = 2 * time.Second + // timeoutDrainBound is how long a drain given drainDeadline + // may take to return before the test gives up on it. At + // forty times drainDeadline it leaves ample room for + // scheduling delay on a loaded box under -race, yet it is far + // below the test binary's -timeout, so a drain that its + // deadline does not bound fails that one test instead of + // hanging the package. + timeoutDrainBound = 2 * time.Second + + // longDrainDeadline is the deadline given to a drain that is + // expected to finish well before it: when the in-flight + // delivery completes after inFlightHold, or at once when + // nothing is in flight. It is far above inFlightHold, so + // those drains never reach it, and four times + // idleDrainBound, so an idle drain that waited for its + // deadline instead of returning fails that bound. + longDrainDeadline = 2 * time.Second + + // reachEndpointTimeout is how long a submitted delivery may + // take to reach the test server. That normally takes a few + // milliseconds; the margin is for a loaded box under -race, + // and only a failing run ever waits this long. + reachEndpointTimeout = 2 * time.Second // settleDelay is how long to wait before asserting that // something did *not* happen. @@ -46,10 +65,11 @@ const ( // nothing in flight. It is deliberately far above the cost // of the goroutine hop through inFlight.Wait() — which // reached 57ms on a loaded box under -race with the package's - // parallel tests — and far below drainSlack, the deadline - // such a drain is given. A drain that blocked until its - // deadline instead of returning on the WaitGroup therefore - // still fails this bound, but scheduling delay alone cannot. + // parallel tests — and far below longDrainDeadline, the + // deadline such a drain is given. A drain that blocked until + // its deadline instead of returning on the WaitGroup + // therefore still fails this bound, but scheduling delay + // alone cannot. idleDrainBound = 500 * time.Millisecond ) @@ -75,12 +95,14 @@ func (sb *syncBuffer) String() string { } // newLoggingService returns a Service writing JSON logs into -// the returned buffer. +// the returned buffer, debug level included. func newLoggingService( transport http.RoundTripper, ) (*notify.Service, *syncBuffer) { logs := &syncBuffer{} - handler := slog.NewJSONHandler(logs, nil) + handler := slog.NewJSONHandler( + logs, &slog.HandlerOptions{Level: slog.LevelDebug}, + ) return notify.NewTestServiceWithLogger(transport, handler), logs @@ -121,6 +143,12 @@ func TestDrainWaitsForInFlightDelivery(t *testing.T) { srv := blockingNtfyServer(entered, release, &served) defer srv.Close() + // srv.Close waits for the handler, so release it however the + // test ends; otherwise a drain that returns early hangs the + // package instead of failing this test. + releaseHandler := sync.OnceFunc(func() { close(release) }) + defer releaseHandler() + topicURL, _ := url.Parse(srv.URL) svc := notify.NewTestService(http.DefaultTransport) @@ -134,7 +162,7 @@ func TestDrainWaitsForInFlightDelivery(t *testing.T) { // drain begins. select { case <-entered: - case <-time.After(drainSlack): + case <-time.After(reachEndpointTimeout): t.Fatal("delivery never reached the endpoint") } @@ -145,13 +173,11 @@ func TestDrainWaitsForInFlightDelivery(t *testing.T) { // delay alone. start := time.Now() - timer := time.AfterFunc(inFlightHold, func() { - close(release) - }) + timer := time.AfterFunc(inFlightHold, releaseHandler) defer timer.Stop() ctx, cancel := context.WithTimeout( - context.Background(), drainSlack, + context.Background(), longDrainDeadline, ) defer cancel() @@ -246,7 +272,7 @@ func TestDrainBoundedByContextDeadline(t *testing.T) { // all never returns here (the delivery is parked in a backoff // that never fires), so an unbounded drain must fail this // test promptly instead of hanging the package until the test - // binary's 30s timeout. + // binary's -timeout. returned := make(chan struct{}) go func() { @@ -257,11 +283,11 @@ func TestDrainBoundedByContextDeadline(t *testing.T) { select { case <-returned: - case <-time.After(drainSlack): + case <-time.After(timeoutDrainBound): t.Fatalf( "drain did not return within %v; its %v deadline "+ "did not bound it", - drainSlack, drainDeadline, + timeoutDrainBound, drainDeadline, ) } @@ -333,7 +359,7 @@ func TestDrainRefusesNewDeliveries(t *testing.T) { svc.SetMattermostWebhookURL(target) ctx, cancel := context.WithTimeout( - context.Background(), drainSlack, + context.Background(), longDrainDeadline, ) defer cancel() @@ -425,6 +451,11 @@ func TestNewRegistersDrainingStopHook(t *testing.T) { srv := blockingNtfyServer(entered, release, &served) defer srv.Close() + // As in TestDrainWaitsForInFlightDelivery: release the handler + // however the test ends, before srv.Close waits for it. + releaseHandler := sync.OnceFunc(func() { close(release) }) + defer releaseHandler() + lifecycle := &recordingLifecycle{} svc := newNotifyService(t, lifecycle, srv.URL) @@ -446,17 +477,15 @@ func TestNewRegistersDrainingStopHook(t *testing.T) { select { case <-entered: - case <-time.After(drainSlack): + case <-time.After(reachEndpointTimeout): t.Fatal("delivery never reached the endpoint") } - timer := time.AfterFunc(inFlightHold, func() { - close(release) - }) + timer := time.AfterFunc(inFlightHold, releaseHandler) defer timer.Stop() ctx, cancel := context.WithTimeout( - context.Background(), drainSlack, + context.Background(), longDrainDeadline, ) defer cancel() @@ -487,7 +516,7 @@ func TestDrainWithoutDeliveriesReturnsImmediately(t *testing.T) { start := time.Now() ctx, cancel := context.WithTimeout( - context.Background(), drainSlack, + context.Background(), longDrainDeadline, ) defer cancel() @@ -495,9 +524,9 @@ func TestDrainWithoutDeliveriesReturnsImmediately(t *testing.T) { if elapsed := time.Since(start); elapsed > idleDrainBound { t.Errorf( - "drain of an idle service took %v, want well "+ - "under its %v deadline", - elapsed, drainSlack, + "drain of an idle service took %v, want at most "+ + "%v; its deadline was %v", + elapsed, idleDrainBound, longDrainDeadline, ) } } @@ -505,10 +534,11 @@ func TestDrainWithoutDeliveriesReturnsImmediately(t *testing.T) { // TestDrainWithCancelledContextDoesNotWarn verifies that an // OnStop context that is already dead on entry does not produce // an "abandoning them" warning when there was nothing in flight -// to abandon. The expired context wins the select immediately, -// so only the outstanding count can tell the difference between -// a genuine timeout and a shutdown that had simply already run -// out of time with no work left. +// to abandon, and that the drain returns and says at debug level +// that nothing was in flight. The expired context wins the +// select immediately, so only the outstanding count can tell the +// difference between a genuine timeout and a shutdown that had +// simply already run out of time with no work left. func TestDrainWithCancelledContextDoesNotWarn(t *testing.T) { t.Parallel() @@ -517,11 +547,43 @@ func TestDrainWithCancelledContextDoesNotWarn(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) cancel() - svc.Drain(ctx) + // A watchdog, as in TestDrainBoundedByContextDeadline, so + // that a drain which never returns fails here instead of + // hanging the package. + returned := make(chan struct{}) - if output := logs.String(); strings.Contains( - output, `"level":"WARN"`, + go func() { + defer close(returned) + + svc.Drain(ctx) + }() + + select { + case <-returned: + case <-time.After(idleDrainBound): + t.Fatalf( + "drain with nothing in flight and a cancelled "+ + "context did not return within %v", + idleDrainBound, + ) + } + + output := logs.String() + + // The absence of a warning alone would also pass if the drain + // logged nothing at all, so require the debug line it writes + // when it finds nothing outstanding. + if !strings.Contains( + output, "all in-flight notifications completed", ) { + t.Errorf( + "drain did not log that nothing was in flight; "+ + "log output: %s", + output, + ) + } + + if strings.Contains(output, `"level":"WARN"`) { t.Errorf( "drain with nothing in flight warned about "+ "abandoned deliveries; log output: %s", diff --git a/internal/portcheck/portcheck.go b/internal/portcheck/portcheck.go index a57b230..3d795b2 100644 --- a/internal/portcheck/portcheck.go +++ b/internal/portcheck/portcheck.go @@ -193,7 +193,9 @@ func (c *Checker) checkConnection( c.log.Debug( "port check succeeded", "target", target, - "latency", latency, + // As text: the JSON log writes a time.Duration as bare + // nanoseconds. + "latency", latency.String(), ) return &PortResult{ diff --git a/internal/resolver/dns_client.go b/internal/resolver/dns_client.go index 589c657..13135e5 100644 --- a/internal/resolver/dns_client.go +++ b/internal/resolver/dns_client.go @@ -7,8 +7,8 @@ import ( "github.com/miekg/dns" ) -// DNSClient abstracts DNS wire-protocol exchanges so the resolver -// can be tested without hitting real nameservers. +// DNSClient sends one DNS message to a nameserver and returns the +// reply. The resolver holds one for UDP and one for TCP. type DNSClient interface { ExchangeContext( ctx context.Context, diff --git a/internal/resolver/errors.go b/internal/resolver/errors.go index 3f203d4..5cf5b46 100644 --- a/internal/resolver/errors.go +++ b/internal/resolver/errors.go @@ -2,6 +2,23 @@ package resolver import "errors" +// NoAnswerError is returned when the walk to a name's nameservers +// reached a zone through a referral and none of that zone's +// nameservers, Nameservers as the referral names them, answered. Err +// says why they did not. +type NoAnswerError struct { + Nameservers []string + Err error +} + +func (e *NoAnswerError) Error() string { + return e.Err.Error() +} + +func (e *NoAnswerError) Unwrap() error { + return e.Err +} + // Sentinel errors returned by the resolver. var ( // ErrNoNameservers is returned when no authoritative NS @@ -10,12 +27,54 @@ var ( "no authoritative nameservers found", ) + // ErrNXDomain is returned when the servers of the zone a domain + // is in answer NXDOMAIN: the domain does not exist. + ErrNXDomain = errors.New("domain does not exist") + + // ErrNoNameserverAnswered is returned when every nameserver + // asked about a name timed out, failed or returned a referral, + // so whether the name has addresses is unknown. + ErrNoNameserverAnswered = errors.New("no nameserver answered") + + // ErrNoNameserverAddress is returned when a nameserver is to be + // asked and no address for it was given. + ErrNoNameserverAddress = errors.New("nameserver has no address") + + // ErrUnusableReply is returned when a server replied with an + // error such as SERVFAIL, or with a referral that leads no + // closer to the name asked about. + ErrUnusableReply = errors.New( + "reply is an error or a referral that leads no closer", + ) + + // ErrTruncated is the reason given for a reply too large for UDP + // whose retry over TCP failed. + ErrTruncated = errors.New( + "reply truncated and its retry over TCP failed", + ) + + // ErrIntercepted is returned when a root server's address gave a + // reply marked recursion available, a refusal included, which ends + // the lookup at once, or when every root server refused a query. + // Root servers do neither, so those replies came from something on + // the network answering in their place. On such a network + // dnswatcher fails with it, by design: nothing works around the + // interception. + ErrIntercepted = errors.New("this network intercepts DNS queries") + // ErrCNAMEDepthExceeded is returned when a CNAME chain // exceeds MaxCNAMEDepth. ErrCNAMEDepthExceeded = errors.New( "CNAME chain depth exceeded", ) + // ErrLookupDepthExceeded is returned when nameserver addresses + // were not looked up because lookups were already maxLookupDepth + // deep, one inside another. + ErrLookupDepthExceeded = errors.New( + "lookups of nameserver addresses go too deep", + ) + // ErrContextCanceled wraps context cancellation for the // resolver's iterative queries. ErrContextCanceled = errors.New("context canceled") diff --git a/internal/resolver/export_test.go b/internal/resolver/export_test.go new file mode 100644 index 0000000..58ea51b --- /dev/null +++ b/internal/resolver/export_test.go @@ -0,0 +1,159 @@ +package resolver + +import ( + "context" + "log/slog" + "time" + + "github.com/miekg/dns" +) + +// NewWithFailingTCP returns a Resolver whose TCP client gives up before +// it can connect, so the retry over TCP of every truncated reply fails. +func NewWithFailingTCP(log *slog.Logger) *Resolver { + r := NewFromLogger(log) + r.tcp = &tcpClient{timeout: time.Nanosecond} + + return r +} + +// NewWithQueryTimeout returns a Resolver whose queries over UDP give up +// after timeout, so a test that asks an address where nothing answers +// does not wait out the usual timeout. +func NewWithQueryTimeout(log *slog.Logger, timeout time.Duration) *Resolver { + r := NewFromLogger(log) + r.client = &udpClient{timeout: timeout} + + return r +} + +// FollowDelegation exports followDelegation for testing. +func (r *Resolver) FollowDelegation( + ctx context.Context, + domain string, + zone string, + nameservers []string, + servers []string, +) ([]string, error) { + return r.followDelegation(ctx, domain, zone, nameservers, servers) +} + +// FindAuthoritativeNameserversFrom exports findAuthoritativeNameservers +// for testing. +func (r *Resolver) FindAuthoritativeNameserversFrom( + ctx context.Context, + domain string, + servers []string, +) ([]string, error) { + return r.findAuthoritativeNameservers(ctx, domain, servers) +} + +// ResolveNSIterative exports resolveNSIterative for testing. +func (r *Resolver) ResolveNSIterative( + ctx context.Context, + domain string, +) ([]string, error) { + return r.resolveNSIterative(ctx, domain) +} + +// ExtractRecordValue exports extractRecordValue for testing. +func ExtractRecordValue(rr dns.RR) string { + return extractRecordValue(rr) +} + +// CollectAnswerRecords exports collectAnswerRecords for testing. +func CollectAnswerRecords(msg *dns.Msg, resp *NameserverResponse) { + var state queryState + + collectAnswerRecords(msg, resp, &state) +} + +// UsableReply exports usableReply for testing. +func UsableReply(resp *dns.Msg, zone string, name string) bool { + return usableReply(resp, zone, name) +} + +// NSSetFrom exports nsSetFrom for testing. +func NSSetFrom(resp *dns.Msg, domain string) []string { + return nsSetFrom(resp, domain) +} + +// CollectIPs exports collectIPs for testing. +func CollectIPs( + results map[string]*NameserverResponse, +) ([]string, string, error) { + return collectIPs(results) +} + +// QueryServers exports queryServers for testing. +func (r *Resolver) QueryServers( + ctx context.Context, + servers []string, + zone string, + name string, + qtype uint16, +) (*dns.Msg, error) { + return r.queryServers(ctx, servers, zone, name, qtype) +} + +// QueryTypes exports queryTypes for testing. +func (r *Resolver) QueryTypes( + ctx context.Context, + nsHostname string, + nsIPs []string, + hostname string, + qtypes []uint16, +) (*NameserverResponse, error) { + return r.queryTypes(ctx, nsHostname, nsIPs, hostname, qtypes) +} + +// QueryEachNS exports queryEachNS for testing. +func (r *Resolver) QueryEachNS( + ctx context.Context, + nameservers []string, + hostname string, +) (map[string]*NameserverResponse, error) { + return r.queryEachNS(ctx, nameservers, hostname, recordTypes()) +} + +// ResolveNSIPs exports resolveNSIPs for testing, looking each name up +// as a lookup that no other lookup started. +func (r *Resolver) ResolveNSIPs( + ctx context.Context, + nsNames []string, +) []string { + ips, _ := r.resolveNSIPs(ctx, nsNames, 1) + + return ips +} + +// MaxLookupDepth exports maxLookupDepth for testing. +const MaxLookupDepth = maxLookupDepth + +// QueryZone exports queryZone for testing. +func (r *Resolver) QueryZone( + ctx context.Context, + given []string, + withoutAddresses []string, + zone string, + name string, + qtype uint16, + depth int, +) (*dns.Msg, error) { + return r.queryZone( + ctx, given, withoutAddresses, zone, name, qtype, depth, + ) +} + +// RootServerList exports rootServerList for testing. +func RootServerList() []string { + return rootServerList() +} + +// Shuffled exports shuffled for testing. +func Shuffled( + servers []string, + shuffle func(n int, swap func(i, j int)), +) []string { + return shuffled(servers, shuffle) +} diff --git a/internal/resolver/iterative.go b/internal/resolver/iterative.go index eebab39..cfca76f 100644 --- a/internal/resolver/iterative.go +++ b/internal/resolver/iterative.go @@ -4,8 +4,11 @@ import ( "context" "errors" "fmt" + "math/rand/v2" "net" + "slices" "sort" + "strconv" "strings" "time" @@ -16,8 +19,16 @@ const ( queryTimeoutDuration = 2 * time.Second maxRetries = 2 maxDelegation = 20 - timeoutMultiplier = 2 - minDomainLabels = 2 + + // maxLookupDepth is how many lookups of nameserver addresses may be + // under way one inside another. Looking up a nameserver's address + // can meet a referral that names nameservers without their + // addresses, which are then looked up in turn; without a limit, + // delegations that point at each other would never end. Each level + // multiplies the queries sent. pool.ntp.org needs three: the + // address of its nameserver g.ntpns.org can need a.ntpns.org's, + // which needs a bitnames.com nameserver's. + maxLookupDepth = 3 ) // ErrRefused is returned when a DNS server refuses a query. @@ -50,19 +61,11 @@ func checkCtx(ctx context.Context) error { return nil } -func (r *Resolver) exchangeWithTimeout( - ctx context.Context, - msg *dns.Msg, - addr string, - attempt int, -) (*dns.Msg, error) { - _ = attempt // timeout escalation handled by client config - - resp, _, err := r.client.ExchangeContext(ctx, msg, addr) - - return resp, err -} - +// tryExchange sends msg to addr over UDP, up to maxRetries times, until +// a reply comes. Every try waits the same queryTimeoutDuration, or less +// when ctx's deadline comes first. When ctx has ended before a further +// try, it returns the error of the try already made, such as a timeout, +// or ErrContextCanceled when none was made. func (r *Resolver) tryExchange( ctx context.Context, msg *dns.Msg, @@ -72,14 +75,16 @@ func (r *Resolver) tryExchange( var err error - for attempt := range maxRetries { + for range maxRetries { if checkCtx(ctx) != nil { - return nil, ErrContextCanceled + if err == nil { + return nil, ErrContextCanceled + } + + return nil, err } - resp, err = r.exchangeWithTimeout( - ctx, msg, addr, attempt, - ) + resp, _, err = r.client.ExchangeContext(ctx, msg, addr) if err == nil { break } @@ -88,6 +93,9 @@ func (r *Resolver) tryExchange( return resp, err } +// retryTCP returns the reply to msg over TCP when resp, its reply over +// UDP, is truncated. When that fails it returns resp, still truncated, +// which holds only the records that fit. func (r *Resolver) retryTCP( ctx context.Context, msg *dns.Msg, @@ -106,9 +114,9 @@ func (r *Resolver) retryTCP( return resp } -// queryDNS sends a DNS query to a specific server IP. -// Tries non-recursive first, falls back to recursive on -// REFUSED (handles DNS interception environments). +// queryDNS sends a DNS query to a specific server IP, never asking it +// for recursion. A reply of REFUSED is returned together with +// ErrRefused, so that the caller can still read the reply's flags. func (r *Resolver) queryDNS( ctx context.Context, serverIP string, @@ -132,25 +140,12 @@ func (r *Resolver) queryDNS( } if resp.Rcode == dns.RcodeRefused { - msg.RecursionDesired = true - - resp, err = r.tryExchange(ctx, msg, addr) - if err != nil { - return nil, fmt.Errorf( - "query %s @%s: %w", name, serverIP, err, - ) - } - - if resp.Rcode == dns.RcodeRefused { - return nil, fmt.Errorf( - "query %s @%s: %w", name, serverIP, ErrRefused, - ) - } + return resp, fmt.Errorf( + "query %s @%s: %w", name, serverIP, ErrRefused, + ) } - resp = r.retryTCP(ctx, msg, addr, resp) - - return resp, nil + return r.retryTCP(ctx, msg, addr, resp), nil } func extractNSSet(rrs []dns.RR) []string { @@ -203,26 +198,61 @@ func glueIPs(nsNames []string, glue map[string][]net.IP) []string { return ips } +// followDelegation follows referrals to domain from servers, the +// addresses of the servers of zone, and returns the NS set of domain's +// delegation. nameservers are the names a referral gave the servers of +// zone; the walk to a name's nameservers starts at the root servers, the +// servers of zone ".", which no referral names. When the servers of the +// zone domain is in answer that domain does not exist, the error is +// ErrNXDomain. When they answer that it has no delegation of its own, +// because it is not the zone's apex, the set is empty and there is no +// error. Any other error means that no such answer came; it is a +// *NoAnswerError when the servers that did not answer are those of a +// zone a referral named. func (r *Resolver) followDelegation( ctx context.Context, domain string, + zone string, + nameservers []string, servers []string, ) ([]string, error) { + var withoutAddresses []string + for range maxDelegation { if checkCtx(ctx) != nil { return nil, ErrContextCanceled } - resp, err := r.queryServers( - ctx, servers, domain, dns.TypeNS, + resp, err := r.queryZone( + ctx, servers, withoutAddresses, zone, domain, dns.TypeNS, 0, ) + + // A query that ctx cut short says nothing about the servers. + if err != nil && len(nameservers) > 0 && checkCtx(ctx) == nil { + return nil, &NoAnswerError{Nameservers: nameservers, Err: err} + } + if err != nil { return nil, err } - ansNS := extractNSSet(resp.Answer) - if len(ansNS) > 0 { - return ansNS, nil + nsSet := nsSetFrom(resp, domain) + if len(nsSet) > 0 { + return nsSet, nil + } + + // An authoritative reply comes from the servers of the zone + // domain is in; it is not a referral, even when its authority + // section lists that zone's NS records. Without NS records in + // the answer, domain has no nameservers of its own: it does + // not exist, when the reply is NXDOMAIN, or else it is not the + // zone's apex. + if resp.Authoritative && resp.Rcode == dns.RcodeNameError { + return nil, ErrNXDomain + } + + if resp.Authoritative { + return []string{}, nil } authNS := extractNSSet(resp.Ns) @@ -230,70 +260,265 @@ func (r *Resolver) followDelegation( return r.resolveNSIterative(ctx, domain) } - glue := extractGlue(resp.Extra) - nextServers := glueIPs(authNS, glue) - - if len(nextServers) == 0 { - nextServers = r.resolveNSIPs(ctx, authNS) - } - - if len(nextServers) == 0 { - return nil, ErrNoNameservers - } - - servers = nextServers + nameservers = authNS + servers, withoutAddresses = referralNameservers(resp) + zone = referralZone(resp) } return nil, ErrNoNameservers } +// shuffled returns a copy of servers in the order shuffle puts them +// in. The resolver passes rand.Shuffle, so each time it walks a list of +// servers it starts at a random one, and no one server gets every +// first query. +func shuffled( + servers []string, + shuffle func(n int, swap func(i, j int)), +) []string { + order := slices.Clone(servers) + shuffle(len(order), func(i, j int) { + order[i], order[j] = order[j], order[i] + }) + + return order +} + +// queryServers asks servers, the servers of zone, about name in a random +// order until one gives a usable reply. When they are the root servers, +// the first reply marked recursion available, a refusal included, ends +// the lookup with ErrIntercepted, naming the address it came from, and +// no other root server is asked: root servers offer no recursion, so +// that reply came from something on the network answering in their +// place. Otherwise a server that times out, refuses or gives a reply +// that is not usable is passed over for the next. When every server +// refused, the error says so, and when they are the root servers it is +// ErrIntercepted. func (r *Resolver) queryServers( ctx context.Context, servers []string, + zone string, name string, qtype uint16, ) (*dns.Msg, error) { var lastErr error - for _, ip := range servers { + refused := 0 + + for _, ip := range shuffled(servers, rand.Shuffle) { if checkCtx(ctx) != nil { return nil, ErrContextCanceled } + // queryDNS returns a refusal together with its reply, so a + // refusal's mark is checked here too. resp, err := r.queryDNS(ctx, ip, name, qtype) + if resp != nil && zone == "." && resp.RecursionAvailable { + return nil, fmt.Errorf( + "query %s @%s: reply marked recursion available, "+ + "so not from a root server: %w", + name, ip, ErrIntercepted, + ) + } + + if err == nil && !usableReply(resp, zone, name) { + err = fmt.Errorf( + "query %s @%s: %w", name, ip, ErrUnusableReply, + ) + } + if err == nil { return resp, nil } + if errors.Is(err, ErrRefused) { + refused++ + } + lastErr = err } + if refused == len(servers) && zone == "." { + return nil, fmt.Errorf( + "every root server refused a query for %s: %w", + name, ErrIntercepted, + ) + } + + if refused == len(servers) { + return nil, fmt.Errorf( + "every server of %s refused a query for %s: %w", + zone, name, ErrRefused, + ) + } + return nil, fmt.Errorf("all servers failed: %w", lastErr) } -func (r *Resolver) resolveNSIPs( - ctx context.Context, - nsNames []string, -) []string { - var ips []string +// isErrorReply reports whether msg is an error reply: one with any code +// but NOERROR and NXDOMAIN, such as SERVFAIL, NOTIMP or FORMERR. An error +// reply says nothing about the name's records. +func isErrorReply(msg *dns.Msg) bool { + return msg.Rcode != dns.RcodeSuccess && msg.Rcode != dns.RcodeNameError +} - for _, ns := range nsNames { - resolved, err := r.resolveARecord(ctx, ns) - if err == nil { - ips = append(ips, resolved...) - } +// usableReply reports whether resp, a reply from one of the servers of +// zone to a query about name, is usable. An error reply such as SERVFAIL +// is not. Nor is a referral, unless it refers the query to a zone below +// zone that name is in: a server that refers it back to zone, up or +// sideways does not serve zone as it should. +func usableReply(resp *dns.Msg, zone string, name string) bool { + if isErrorReply(resp) { + return false + } - if len(ips) > 0 { - break + child := referralZone(resp) + if resp.Authoritative || len(resp.Answer) > 0 || child == "" { + return true + } + + return child != zone && dns.IsSubDomain(zone, child) && + dns.IsSubDomain(child, name) +} + +// referralZone returns the zone a referral refers the query to: the +// owner name of the NS records in resp's authority section, or "" when +// there are none. +func referralZone(resp *dns.Msg) string { + for _, rr := range resp.Ns { + if ns, ok := rr.(*dns.NS); ok { + return strings.ToLower(ns.Hdr.Name) } } - return ips + return "" +} + +// nsSetFrom returns the NS set of domain that resp, a reply to a query +// for domain's NS records, gives: the delegation in a referral to domain +// itself, or else the NS records in the answer; empty when it gives +// neither. A referral to domain comes from its parent zone's servers, +// which all hold the same delegation, so the set does not depend on +// which of them answered. domain's own servers, which can disagree about +// their NS records, are then not asked. +func nsSetFrom(resp *dns.Msg, domain string) []string { + if referralZone(resp) == domain { + return extractNSSet(resp.Ns) + } + + return extractNSSet(resp.Answer) +} + +// referralNameservers returns the IPv4 addresses that resp, a referral, +// gives for the nameservers it names, and the names of the nameservers +// it gives no address for. +func referralNameservers(resp *dns.Msg) ([]string, []string) { + glue := extractGlue(resp.Extra) + + var given, withoutAddresses []string + + for _, ns := range extractNSSet(resp.Ns) { + ips := glueIPs([]string{ns}, glue) + if len(ips) == 0 { + withoutAddresses = append(withoutAddresses, ns) + } + + given = append(given, ips...) + } + + return given, withoutAddresses +} + +// queryZone asks the servers of zone about name as queryServers does: +// first those at given, the addresses a referral gave, and only when +// none of them gives a usable reply, the nameservers named +// withoutAddresses, once their addresses are looked up. depth is how +// many lookups of a nameserver's address are under way, 0 in the walk +// to a domain's nameservers; at maxLookupDepth, no address is looked +// up. When the limit is why none was found, here or in a lookup this +// one started, the error is ErrLookupDepthExceeded. +func (r *Resolver) queryZone( + ctx context.Context, + given []string, + withoutAddresses []string, + zone string, + name string, + qtype uint16, + depth int, +) (*dns.Msg, error) { + err := fmt.Errorf( + "no address for any nameserver of %s: %w", zone, ErrNoNameservers, + ) + + if len(given) > 0 { + var resp *dns.Msg + + resp, err = r.queryServers(ctx, given, zone, name, qtype) + if err == nil { + return resp, nil + } + } + + if len(withoutAddresses) == 0 { + return nil, err + } + + if depth >= maxLookupDepth { + return nil, fmt.Errorf( + "addresses of the nameservers of %s not looked up: %w", + zone, ErrLookupDepthExceeded, + ) + } + + lookedUp, limitErr := r.resolveNSIPs(ctx, withoutAddresses, depth+1) + if limitErr != nil { + return nil, limitErr + } + + if len(lookedUp) == 0 { + return nil, err + } + + return r.queryServers(ctx, lookedUp, zone, name, qtype) +} + +// resolveNSIPs returns the addresses of every nameserver in nsNames +// whose name resolves, each looked up at depth (see resolveARecord). +// The walk can then go on to the zone's other nameservers when one +// gives no usable reply. When none resolves and the depth limit +// stopped one of the lookups, it returns that lookup's error. +func (r *Resolver) resolveNSIPs( + ctx context.Context, + nsNames []string, + depth int, +) ([]string, error) { + var ( + ips []string + limitErr error + ) + + for _, ns := range nsNames { + resolved, err := r.resolveARecord(ctx, ns, depth) + + switch { + case err == nil: + ips = append(ips, resolved...) + case errors.Is(err, ErrLookupDepthExceeded): + limitErr = err + } + } + + if len(ips) > 0 { + return ips, nil + } + + return nil, limitErr } // resolveNSIterative queries for NS records using iterative // resolution as a fallback when followDelegation finds no -// authoritative answer in the delegation chain. +// authoritative answer in the delegation chain. Its result means what +// followDelegation's does. func (r *Resolver) resolveNSIterative( ctx context.Context, domain string, @@ -304,6 +529,7 @@ func (r *Resolver) resolveNSIterative( domain = dns.Fqdn(domain) servers := rootServerList() + zone := "." for range maxDelegation { if checkCtx(ctx) != nil { @@ -311,17 +537,27 @@ func (r *Resolver) resolveNSIterative( } resp, err := r.queryServers( - ctx, servers, domain, dns.TypeNS, + ctx, servers, zone, domain, dns.TypeNS, ) if err != nil { return nil, err } - nsNames := extractNSSet(resp.Answer) + nsNames := nsSetFrom(resp, domain) if len(nsNames) > 0 { return nsNames, nil } + // As in followDelegation: domain has no nameservers of its + // own. + if resp.Authoritative && resp.Rcode == dns.RcodeNameError { + return nil, ErrNXDomain + } + + if resp.Authoritative { + return []string{}, nil + } + // Follow delegation. authNS := extractNSSet(resp.Ns) if len(authNS) == 0 { @@ -336,16 +572,20 @@ func (r *Resolver) resolveNSIterative( } servers = nextServers + zone = referralZone(resp) } return nil, ErrNoNameservers } -// resolveARecord resolves a hostname to IPv4 addresses using -// iterative resolution through the delegation chain. +// resolveARecord resolves a hostname, a nameserver's name, to IPv4 +// addresses using iterative resolution through the delegation chain. +// depth is how many lookups of a nameserver's address are under way, +// this one included: 1 for a lookup that no other lookup started. func (r *Resolver) resolveARecord( ctx context.Context, hostname string, + depth int, ) ([]string, error) { if checkCtx(ctx) != nil { return nil, ErrContextCanceled @@ -353,14 +593,18 @@ func (r *Resolver) resolveARecord( hostname = dns.Fqdn(hostname) servers := rootServerList() + zone := "." + + var withoutAddresses []string for range maxDelegation { if checkCtx(ctx) != nil { return nil, ErrContextCanceled } - resp, err := r.queryServers( - ctx, servers, hostname, dns.TypeA, + resp, err := r.queryZone( + ctx, servers, withoutAddresses, zone, hostname, dns.TypeA, + depth, ) if err != nil { return nil, fmt.Errorf( @@ -387,17 +631,8 @@ func (r *Resolver) resolveARecord( break } - glue := extractGlue(resp.Extra) - nextServers := glueIPs(authNS, glue) - - if len(nextServers) == 0 { - // Resolve NS IPs iteratively — but guard - // against infinite recursion by using only - // already-resolved servers. - break - } - - servers = nextServers + servers, withoutAddresses = referralNameservers(resp) + zone = referralZone(resp) } return nil, fmt.Errorf( @@ -407,10 +642,25 @@ func (r *Resolver) resolveARecord( // FindAuthoritativeNameservers traces the delegation chain from // root servers to discover all authoritative nameservers for the -// given domain. Walks up the label hierarchy for subdomains. +// given domain, as the delegation from its parent zone's servers lists +// them. When the servers asked answer that the name has no delegation +// of its own, or does not exist, it tries each parent name in turn, so +// it returns the nameservers of the zone the name is in. When they do +// not answer, it returns the error and tries no parent name; when a +// referral named them, the error is a *NoAnswerError naming them. func (r *Resolver) FindAuthoritativeNameservers( ctx context.Context, domain string, +) ([]string, error) { + return r.findAuthoritativeNameservers(ctx, domain, rootServerList()) +} + +// findAuthoritativeNameservers is FindAuthoritativeNameservers with each +// walk starting at servers, the root servers. +func (r *Resolver) findAuthoritativeNameservers( + ctx context.Context, + domain string, + servers []string, ) ([]string, error) { if checkCtx(ctx) != nil { return nil, ErrContextCanceled @@ -426,10 +676,12 @@ func (r *Resolver) FindAuthoritativeNameservers( candidate := strings.Join(labels[i:], ".") + "." - nsNames, err := r.followDelegation( - ctx, candidate, rootServerList(), - ) - if err == nil && len(nsNames) > 0 { + nsNames, err := r.followDelegation(ctx, candidate, ".", nil, servers) + if err != nil && !errors.Is(err, ErrNXDomain) { + return nil, err + } + + if len(nsNames) > 0 { sort.Strings(nsNames) return nsNames, nil @@ -439,25 +691,53 @@ func (r *Resolver) FindAuthoritativeNameservers( return nil, ErrNoNameservers } +// recordTypes returns the record types a nameserver is asked for when a +// name is checked. +func recordTypes() []uint16 { + return []uint16{ + dns.TypeA, dns.TypeAAAA, dns.TypeCNAME, + dns.TypeMX, dns.TypeTXT, dns.TypeSRV, + dns.TypeCAA, dns.TypeNS, + } +} + +// addressTypes returns the record types ResolveIPAddresses asks for, +// the only ones it reads. +func addressTypes() []uint16 { + return []uint16{dns.TypeA, dns.TypeAAAA, dns.TypeCNAME} +} + // QueryNameserver queries a specific nameserver for all record // types and builds a NameserverResponse. func (r *Resolver) QueryNameserver( ctx context.Context, nsHostname string, hostname string, +) (*NameserverResponse, error) { + return r.queryNameserver(ctx, nsHostname, hostname, recordTypes()) +} + +// queryNameserver queries a specific nameserver, at the IPv4 addresses +// its name resolves to, for the record types in qtypes and builds a +// NameserverResponse. +func (r *Resolver) queryNameserver( + ctx context.Context, + nsHostname string, + hostname string, + qtypes []uint16, ) (*NameserverResponse, error) { if checkCtx(ctx) != nil { return nil, ErrContextCanceled } - nsIPs, err := r.resolveARecord(ctx, nsHostname) + nsIPs, err := r.resolveARecord(ctx, nsHostname, 1) if err != nil { return nil, fmt.Errorf("resolving NS %s: %w", nsHostname, err) } hostname = dns.Fqdn(hostname) - return r.queryAllTypes(ctx, nsHostname, nsIPs[0], hostname) + return r.queryTypes(ctx, nsHostname, nsIPs, hostname, qtypes) } // QueryNameserverIP queries a nameserver by its IP address directly, @@ -474,55 +754,105 @@ func (r *Resolver) QueryNameserverIP( hostname = dns.Fqdn(hostname) - return r.queryAllTypes(ctx, nsHostname, nsIP, hostname) + return r.queryTypes( + ctx, nsHostname, []string{nsIP}, hostname, recordTypes(), + ) } -func (r *Resolver) queryAllTypes( +// queryTypes asks the nameserver nsHostname, at its addresses nsIPs, +// about hostname's records of each type in qtypes and builds a +// NameserverResponse. With no address there is nothing to ask, and it +// returns ErrNoNameserverAddress. +func (r *Resolver) queryTypes( ctx context.Context, nsHostname string, - nsIP string, + nsIPs []string, hostname string, + qtypes []uint16, ) (*NameserverResponse, error) { + if len(nsIPs) == 0 { + return nil, fmt.Errorf("%w: %s", ErrNoNameserverAddress, nsHostname) + } + resp := &NameserverResponse{ Nameserver: nsHostname, Records: make(map[string][]string), Status: StatusOK, } - qtypes := []uint16{ - dns.TypeA, dns.TypeAAAA, dns.TypeCNAME, - dns.TypeMX, dns.TypeTXT, dns.TypeSRV, - dns.TypeCAA, dns.TypeNS, - } - - state := r.queryEachType(ctx, nsIP, hostname, qtypes, resp) + state := r.queryEachType(ctx, nsIPs, hostname, qtypes, resp) classifyResponse(resp, state) return resp, nil } type queryState struct { - gotNXDomain bool - gotSERVFAIL bool - gotTimeout bool - hasRecords bool + gotNXDomain bool + gotErrorReply bool + errorReply string // its code, such as SERVFAIL, or number if unnamed + gotRefused bool + gotTimeout bool + gotReferral bool + netErr error + hasRecords bool + answered bool } +// queryEachType asks the nameserver about hostname's records of each type +// in qtypes, at its addresses nsIPs in turn until one gives a usable +// reply, and lists in resp.FailedTypes the types that got none at any of +// them, logging each with the reasons unless ctx was cancelled: shutdown +// cancels it, and a query it cut short did not fail. func (r *Resolver) queryEachType( ctx context.Context, - nsIP string, + nsIPs []string, hostname string, qtypes []uint16, resp *NameserverResponse, ) queryState { var state queryState + // Reordered below; the caller's list is left as it was. + nsIPs = slices.Clone(nsIPs) + for _, qtype := range qtypes { if checkCtx(ctx) != nil { break } - r.querySingleType(ctx, nsIP, hostname, qtype, resp, &state) + answeredAt, err := r.querySingleType( + ctx, nsIPs, hostname, qtype, resp, &state, + ) + if err == nil { + state.answered = true + + // The next type is asked first at the address that answered. + nsIPs[0], nsIPs[answeredAt] = nsIPs[answeredAt], nsIPs[0] + + continue + } + + rtype := dns.TypeToString[qtype] + resp.FailedTypes = append(resp.FailedTypes, rtype) + + if errors.Is(ctx.Err(), context.Canceled) { + continue + } + + r.log.Warn( + "record type query failed", + "hostname", hostname, + "nameserver", resp.Nameserver, + "type", rtype, + "error", err, + ) + } + + // The reply about another type can carry the name's CNAME. When the + // query for CNAME itself failed, that is left out too, so Records + // holds nothing for a failed type. + for _, rtype := range resp.FailedTypes { + delete(resp.Records, rtype) } for k := range resp.Records { @@ -532,38 +862,110 @@ func (r *Resolver) queryEachType( return state } +// querySingleType asks the nameserver about hostname's records of type +// qtype at its addresses nsIPs, of which there is at least one, one at a +// time, until one answers: with records, with none, or with NXDOMAIN. +// It returns the index in nsIPs of the address that answered; when none +// did, it returns why each did not, each reason naming its address. func (r *Resolver) querySingleType( ctx context.Context, - nsIP string, + nsIPs []string, hostname string, qtype uint16, resp *NameserverResponse, state *queryState, -) { - msg, err := r.queryDNS(ctx, nsIP, hostname, qtype) - if err != nil { - if isTimeout(err) { +) (int, error) { + var errs []error + + for i, nsIP := range nsIPs { + msg, err := r.queryDNS(ctx, nsIP, hostname, qtype) + + switch { + case err == nil: + err = readReply(msg, resp, state) + if err != nil { + err = fmt.Errorf("query %s @%s: %w", hostname, nsIP, err) + } + case errors.Is(err, ErrContextCanceled): + // The caller's context ended before the query was sent. + // That says nothing about the nameserver, so no reason is + // recorded. + case isTimeout(err): state.gotTimeout = true + case errors.Is(err, ErrRefused): + state.gotRefused = true + default: + state.netErr = err } - return + if err == nil { + return i, nil + } + + errs = append(errs, err) } + return 0, errors.Join(errs...) +} + +// readReply adds to resp the records in msg, a nameserver's reply to a +// query about one record type. It returns nil when the nameserver +// answered: with records, with none, or with NXDOMAIN; otherwise it +// returns why not. +func readReply( + msg *dns.Msg, + resp *NameserverResponse, + state *queryState, +) error { if msg.Rcode == dns.RcodeNameError { state.gotNXDomain = true - return + return nil } - if msg.Rcode == dns.RcodeServerFailure { - state.gotSERVFAIL = true + if isErrorReply(msg) { + state.gotErrorReply = true - return + code, named := dns.RcodeToString[msg.Rcode] + if !named { + code = strconv.Itoa(msg.Rcode) + } + + state.errorReply = code + + return fmt.Errorf( + "server returned %s: %w", state.errorReply, ErrUnusableReply, + ) + } + + // A reply with no answer that lists other nameservers, from a server + // that does not hold the name's zone, is a referral and says nothing + // about the name's records. A server named in the delegation that + // does not hold the zone may send one. + if !msg.Authoritative && len(msg.Answer) == 0 && + len(extractNSSet(msg.Ns)) > 0 { + state.gotReferral = true + + return fmt.Errorf("server returned a referral: %w", ErrUnusableReply) + } + + // A reply still truncated is one whose TCP retry failed, and holds + // only the records that fit. + if msg.Truncated { + state.netErr = ErrTruncated + + return ErrTruncated } collectAnswerRecords(msg, resp, state) + + return nil } +// collectAnswerRecords adds the records in msg's answer to resp, each +// value once per record type. For a name with a CNAME, a nameserver +// answers a query of any type with that CNAME, so the same value comes +// in the answer to every type asked for. func collectAnswerRecords( msg *dns.Msg, resp *NameserverResponse, @@ -576,9 +978,12 @@ func collectAnswerRecords( } typeName := dns.TypeToString[rr.Header().Rrtype] - resp.Records[typeName] = append( - resp.Records[typeName], val, - ) + if !slices.Contains(resp.Records[typeName], val) { + resp.Records[typeName] = append( + resp.Records[typeName], val, + ) + } + state.hasRecords = true } } @@ -593,22 +998,41 @@ func isTimeout(err error) bool { return false } +// classifyResponse sets the nameserver's status. One that answered no +// record type has failed, and Error says why; it says all queries timed +// out only when every query, at every address, did. One that answered +// some has the status of those answers. It has no data only when every type +// answered with no records: a type in FailedTypes may have records, so a +// nameserver with one stays ok. func classifyResponse(resp *NameserverResponse, state queryState) { switch { case state.gotNXDomain && !state.hasRecords: resp.Status = StatusNXDomain - case state.gotTimeout && !state.hasRecords: + case state.gotErrorReply && !state.answered: + resp.Status = StatusError + resp.Error = "server returned " + state.errorReply + case state.gotRefused && !state.answered: + resp.Status = StatusError + resp.Error = "server returned REFUSED" + case state.netErr != nil && !state.answered: + resp.Status = StatusError + resp.Error = "network error: " + state.netErr.Error() + case state.gotReferral && !state.answered: + resp.Status = StatusError + resp.Error = "server returned a referral" + case state.gotTimeout && !state.answered: resp.Status = StatusTimeout resp.Error = "all queries timed out" - case state.gotSERVFAIL && !state.hasRecords: - resp.Status = StatusError - resp.Error = "server returned SERVFAIL" - case !state.hasRecords && !state.gotNXDomain: + // An NXDOMAIN reply with no records was taken by the first case. + case !state.hasRecords && len(resp.FailedTypes) == 0: resp.Status = StatusNoData } } -// extractRecordValue formats a DNS RR value as a string. +// extractRecordValue formats a DNS RR value as a string. DNS names +// are case-insensitive and nameservers may answer in any letter case, +// so names are lower-cased to compare equal. TXT and CAA values keep +// their letter case. func extractRecordValue(rr dns.RR) string { switch r := rr.(type) { case *dns.A: @@ -616,43 +1040,29 @@ func extractRecordValue(rr dns.RR) string { case *dns.AAAA: return r.AAAA.String() case *dns.CNAME: - return r.Target + return strings.ToLower(r.Target) case *dns.MX: - return fmt.Sprintf("%d %s", r.Preference, r.Mx) + return fmt.Sprintf("%d %s", r.Preference, strings.ToLower(r.Mx)) case *dns.TXT: return strings.Join(r.Txt, "") case *dns.SRV: return fmt.Sprintf( "%d %d %d %s", - r.Priority, r.Weight, r.Port, r.Target, + r.Priority, r.Weight, r.Port, strings.ToLower(r.Target), ) case *dns.CAA: return fmt.Sprintf( "%d %s \"%s\"", r.Flag, r.Tag, r.Value, ) case *dns.NS: - return r.Ns + return strings.ToLower(r.Ns) default: return "" } } -// parentDomain returns the registerable parent domain. -func parentDomain(hostname string) string { - hostname = dns.Fqdn(strings.ToLower(hostname)) - labels := dns.SplitDomainName(hostname) - - if len(labels) <= minDomainLabels { - return strings.Join(labels, ".") + "." - } - - return strings.Join( - labels[len(labels)-minDomainLabels:], ".", - ) + "." -} - -// QueryAllNameservers discovers auth NSes for the hostname's -// parent domain, then queries each one independently. +// QueryAllNameservers discovers the auth NSes of the zone the +// hostname is in, then queries each one independently. func (r *Resolver) QueryAllNameservers( ctx context.Context, hostname string, @@ -661,29 +1071,31 @@ func (r *Resolver) QueryAllNameservers( return nil, ErrContextCanceled } - parent := parentDomain(hostname) - - nameservers, err := r.FindAuthoritativeNameservers(ctx, parent) + nameservers, err := r.FindAuthoritativeNameservers(ctx, hostname) if err != nil { return nil, err } - return r.queryEachNS(ctx, nameservers, hostname) + return r.queryEachNS(ctx, nameservers, hostname, recordTypes()) } func (r *Resolver) queryEachNS( ctx context.Context, nameservers []string, hostname string, + qtypes []uint16, ) (map[string]*NameserverResponse, error) { results := make(map[string]*NameserverResponse) for _, ns := range nameservers { + resp, err := r.queryNameserver(ctx, ns, hostname, qtypes) + + // A query the context cut short says nothing about the + // nameserver, so it must not be returned as its failure. if checkCtx(ctx) != nil { return nil, ErrContextCanceled } - resp, err := r.QueryNameserver(ctx, ns, hostname) if err != nil { results[ns] = &NameserverResponse{ Nameserver: ns, @@ -701,35 +1113,40 @@ func (r *Resolver) queryEachNS( return results, nil } -// LookupNS returns the NS record set for a domain. +// LookupNS returns the NS record set of a domain, as the delegation from +// its parent zone's servers lists it, and never a parent name's. When +// they answer that the domain does not exist, the error is ErrNXDomain. +// When they answer that it has no delegation of its own, the set is +// empty and there is no error. func (r *Resolver) LookupNS( ctx context.Context, domain string, ) ([]string, error) { - return r.FindAuthoritativeNameservers(ctx, domain) + if checkCtx(ctx) != nil { + return nil, ErrContextCanceled + } + + return r.followDelegation( + ctx, dns.Fqdn(strings.ToLower(domain)), ".", nil, rootServerList(), + ) } // LookupAllRecords performs iterative resolution to find all DNS // records for the given hostname, keyed by authoritative nameserver. +// Each nameserver's response carries its status and error with its +// records. func (r *Resolver) LookupAllRecords( ctx context.Context, hostname string, -) (map[string]map[string][]string, error) { - results, err := r.QueryAllNameservers(ctx, hostname) - if err != nil { - return nil, err - } - - out := make(map[string]map[string][]string, len(results)) - for ns, resp := range results { - out[ns] = resp.Records - } - - return out, nil +) (map[string]*NameserverResponse, error) { + return r.QueryAllNameservers(ctx, hostname) } // ResolveIPAddresses resolves a hostname to all IPv4 and IPv6 -// addresses, following CNAME chains up to MaxCNAMEDepth. +// addresses, following CNAME chains up to MaxCNAMEDepth. It asks each +// nameserver of the name's zone for its A, AAAA and CNAME records only. +// When no nameserver of the name's zone answered, it returns an error +// rather than no addresses. func (r *Resolver) ResolveIPAddresses( ctx context.Context, hostname string, @@ -750,12 +1167,20 @@ func (r *Resolver) resolveIPWithCNAME( return nil, ErrCNAMEDepthExceeded } - results, err := r.QueryAllNameservers(ctx, hostname) + nameservers, err := r.FindAuthoritativeNameservers(ctx, hostname) if err != nil { return nil, err } - ips, cnameTarget := collectIPs(results) + results, err := r.queryEachNS(ctx, nameservers, hostname, addressTypes()) + if err != nil { + return nil, err + } + + ips, cnameTarget, err := collectIPs(results) + if err != nil { + return nil, fmt.Errorf("resolving %s: %w", hostname, err) + } if len(ips) == 0 && cnameTarget != "" { return r.resolveIPWithCNAME(ctx, cnameTarget, depth+1) @@ -766,16 +1191,31 @@ func (r *Resolver) resolveIPWithCNAME( return ips, nil } +// collectIPs returns the addresses in the nameservers' answers and the +// first CNAME target among them. A nameserver whose query for one of the +// types failed gave only part of the addresses, and is left out. It +// returns ErrNoNameserverAnswered when every nameserver timed out, +// failed, returned a referral or was left out: that is not a name with +// no addresses. func collectIPs( results map[string]*NameserverResponse, -) ([]string, string) { +) ([]string, string, error) { seen := make(map[string]bool) var ips []string var cnameTarget string + answered := false + for _, resp := range results { + if resp.Status == StatusTimeout || resp.Status == StatusError || + len(resp.FailedTypes) > 0 { + continue + } + + answered = true + if resp.Status == StatusNXDomain { continue } @@ -799,5 +1239,9 @@ func collectIPs( } } - return ips, cnameTarget + if !answered { + return nil, "", ErrNoNameserverAnswered + } + + return ips, cnameTarget, nil } diff --git a/internal/resolver/iterative_internal_test.go b/internal/resolver/iterative_internal_test.go new file mode 100644 index 0000000..a804f33 --- /dev/null +++ b/internal/resolver/iterative_internal_test.go @@ -0,0 +1,166 @@ +package resolver + +import ( + "context" + "log/slog" + "strconv" + "syscall" + "testing" + + "github.com/miekg/dns" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// TestClassifyResponse sets a nameserver's status from the results of +// its queries and the record types whose query failed, built here. One +// that answered some record types, even with no records, has not failed +// when its query for another type got no usable reply, whatever the +// reason, and is ok, not nodata: that type may have records. One whose +// every query got none has failed, and its error says all queries timed +// out only when every one did: a reply such as a refusal, or a network +// error, at one of its addresses is given instead. Only one whose every +// type answered with no records is nodata. +func TestClassifyResponse(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + results queryState + failedTypes []string + wantStatus string + wantError string + }{ + {"every type answered with no records", + queryState{answered: true}, + nil, StatusNoData, ""}, + {"some types answered with no records, another timed out", + queryState{answered: true, gotTimeout: true}, + []string{"A"}, StatusOK, ""}, + {"some types answered with no records, another got SERVFAIL", + queryState{answered: true, gotErrorReply: true, errorReply: "SERVFAIL"}, + []string{"A"}, StatusOK, ""}, + {"some types answered with no records, another was refused", + queryState{answered: true, gotRefused: true}, + []string{"A"}, StatusOK, ""}, + {"some types answered with no records, another got a network error", + queryState{answered: true, netErr: syscall.ECONNREFUSED}, + []string{"A"}, StatusOK, ""}, + {"some types answered with no records, another's reply was " + + "truncated and its retry over TCP failed", + queryState{answered: true, netErr: ErrTruncated}, + []string{"TXT"}, StatusOK, ""}, + {"some types answered with no records, another got a referral", + queryState{answered: true, gotReferral: true}, + []string{"A"}, StatusOK, ""}, + {"every query timed out", + queryState{gotTimeout: true}, + []string{"A", "AAAA", "CNAME"}, StatusTimeout, "all queries timed out"}, + {"every query got NOTIMP", + queryState{gotErrorReply: true, errorReply: "NOTIMP"}, + []string{"A", "AAAA", "CNAME"}, StatusError, "server returned NOTIMP"}, + {"every query got SERVFAIL at one address, timed out at another", + queryState{gotErrorReply: true, errorReply: "SERVFAIL", gotTimeout: true}, + []string{"A"}, StatusError, "server returned SERVFAIL"}, + {"every query was refused at one address, timed out at another", + queryState{gotRefused: true, gotTimeout: true}, + []string{"A"}, StatusError, "server returned REFUSED"}, + {"every query got a network error at one address, timed out at another", + queryState{netErr: syscall.ECONNREFUSED, gotTimeout: true}, + []string{"A"}, StatusError, "network error: connection refused"}, + {"every query got a referral at one address, timed out at another", + queryState{gotReferral: true, gotTimeout: true}, + []string{"A"}, StatusError, "server returned a referral"}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + resp := &NameserverResponse{Status: StatusOK, FailedTypes: tt.failedTypes} + classifyResponse(resp, tt.results) + + assert.Equal(t, tt.wantStatus, resp.Status) + assert.Equal(t, tt.wantError, resp.Error) + assert.Equal(t, tt.failedTypes, resp.FailedTypes) + }) + } +} + +// TestReadReply checks which replies to a query about one record type, +// built here, are an answer: one with the code NOERROR or NXDOMAIN. A +// reply with any other code is not, and the type's query has failed; a +// nameserver whose only reply it is has failed, and Error gives the +// code, or its number when the code has no name. +func TestReadReply(t *testing.T) { + t.Parallel() + + tests := []struct { + rcode int + wantStatus string + wantError string + }{ + {dns.RcodeSuccess, StatusNoData, ""}, + {dns.RcodeNameError, StatusNXDomain, ""}, + {dns.RcodeServerFailure, StatusError, "server returned SERVFAIL"}, + {dns.RcodeNotImplemented, StatusError, "server returned NOTIMP"}, + {dns.RcodeFormatError, StatusError, "server returned FORMERR"}, + {12, StatusError, "server returned 12"}, // unassigned, no name + } + + for _, tt := range tests { + t.Run(strconv.Itoa(tt.rcode), func(t *testing.T) { + t.Parallel() + + msg := new(dns.Msg) + msg.Authoritative = true + msg.Rcode = tt.rcode + + resp := &NameserverResponse{Records: map[string][]string{}} + + var state queryState + + err := readReply(msg, resp, &state) + classifyResponse(resp, state) + + if tt.wantStatus == StatusError { + require.ErrorIs(t, err, ErrUnusableReply) + } else { + require.NoError(t, err) + } + + assert.Equal(t, tt.wantStatus, resp.Status) + assert.Equal(t, tt.wantError, resp.Error) + }) + } +} + +// TestQuerySingleType_ContextEnded asks a nameserver about one record +// type with a context that has already ended, so no query is sent. A +// query the context ended before it was sent says nothing about the +// nameserver and records no reason: a nameserver whose earlier query +// timed out is still reported as timed out, not as a network error. +func TestQuerySingleType_ContextEnded(t *testing.T) { + t.Parallel() + + r := NewFromLogger(slog.New(slog.DiscardHandler)) + + ctx, cancel := context.WithCancel(t.Context()) + cancel() + + resp := &NameserverResponse{Records: map[string][]string{}, Status: StatusOK} + + // The query for an earlier record type timed out. + state := queryState{gotTimeout: true} + + _, err := r.querySingleType( + ctx, []string{"192.0.2.1"}, "example.com.", dns.TypeA, resp, &state, + ) + require.ErrorIs(t, err, ErrContextCanceled) + assert.Equal(t, queryState{gotTimeout: true}, state) + + classifyResponse(resp, state) + + assert.Equal(t, StatusTimeout, resp.Status) + assert.Equal(t, "all queries timed out", resp.Error) +} diff --git a/internal/resolver/iterative_test.go b/internal/resolver/iterative_test.go new file mode 100644 index 0000000..836b555 --- /dev/null +++ b/internal/resolver/iterative_test.go @@ -0,0 +1,317 @@ +package resolver_test + +import ( + "math/rand/v2" + "slices" + "testing" + + "github.com/miekg/dns" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "sneak.berlin/go/dnswatcher/internal/resolver" +) + +// TestCollectIPs_OneAnswerIsEnough checks that one nameserver answering +// NXDOMAIN says the name has no addresses, though the other timed out. +func TestCollectIPs_OneAnswerIsEnough(t *testing.T) { + t.Parallel() + + ips, _, err := resolver.CollectIPs( + map[string]*resolver.NameserverResponse{ + "ns1.example.": {Status: resolver.StatusTimeout}, + "ns2.example.": {Status: resolver.StatusNXDomain}, + }, + ) + require.NoError(t, err) + assert.Empty(t, ips) +} + +// TestCollectIPs_FailedIsNoAnswer checks that nameservers that all have +// status error, from a refusal, a server failure, a network error or a +// referral, are no answer rather than a name with no addresses. +func TestCollectIPs_FailedIsNoAnswer(t *testing.T) { + t.Parallel() + + ips, _, err := resolver.CollectIPs( + map[string]*resolver.NameserverResponse{ + "ns1.example.": {Status: resolver.StatusError}, + "ns2.example.": {Status: resolver.StatusError}, + }, + ) + require.ErrorIs(t, err, resolver.ErrNoNameserverAnswered) + assert.Empty(t, ips) +} + +// TestCollectIPs_FailedTypeIsNoAnswer checks that a nameserver whose +// query for one of the types failed is no answer: its addresses are +// only part of them. +func TestCollectIPs_FailedTypeIsNoAnswer(t *testing.T) { + t.Parallel() + + ips, _, err := resolver.CollectIPs( + map[string]*resolver.NameserverResponse{ + nsExample1: { + Records: map[string][]string{"A": {"192.0.2.1"}}, + FailedTypes: []string{"AAAA"}, + Status: resolver.StatusOK, + }, + }, + ) + require.ErrorIs(t, err, resolver.ErrNoNameserverAnswered) + assert.Empty(t, ips) +} + +const ( + // exampleCom is the zone most cases of TestUsableReply and + // TestNSSetFrom are about, and wwwExampleCom a name in it. + exampleCom = "example.com." + wwwExampleCom = "www.example.com." + + // exampleNS is the server the NS records nsRecord builds name. + exampleNS = "ns1.example.net." +) + +// nsRecord builds an NS record that names a server of zone. +func nsRecord(zone string) *dns.NS { + return &dns.NS{ + Hdr: dns.RR_Header{ + Name: zone, Rrtype: dns.TypeNS, Class: dns.ClassINET, + }, + Ns: exampleNS, + } +} + +// referralTo builds a reply that refers the query to the servers of +// zone. +func referralTo(zone string) *dns.Msg { + msg := new(dns.Msg) + msg.Ns = []dns.RR{nsRecord(zone)} + + return msg +} + +// TestUsableReply checks which replies from one of a zone's servers are +// used. A reply that is not usable moves the query on to the zone's +// next server. +func TestUsableReply(t *testing.T) { + t.Parallel() + + servfail := new(dns.Msg) + servfail.Rcode = dns.RcodeServerFailure + + answer := new(dns.Msg) + answer.Authoritative = true + answer.Answer = []dns.RR{nsRecord(exampleCom)} + + nxdomain := new(dns.Msg) + nxdomain.Authoritative = true + nxdomain.Rcode = dns.RcodeNameError + + tests := []struct { + name string + resp *dns.Msg + zone string + query string + want bool + }{ + { + name: "SERVFAIL", resp: servfail, + zone: exampleCom, query: exampleCom, want: false, + }, + { + name: "answer", resp: answer, + zone: exampleCom, query: exampleCom, want: true, + }, + { + name: "NXDOMAIN", resp: nxdomain, + zone: ".", query: exampleCom, want: true, + }, + { + name: "root refers to com", resp: referralTo("com."), + zone: ".", query: exampleCom, want: true, + }, + { + name: "com refers to example.com", resp: referralTo(exampleCom), + zone: "com.", query: wwwExampleCom, want: true, + }, + { + name: "referral back to the zone", resp: referralTo(exampleCom), + zone: exampleCom, query: exampleCom, want: false, + }, + { + name: "referral up to the root", resp: referralTo("."), + zone: exampleCom, query: exampleCom, want: false, + }, + { + name: "referral sideways", resp: referralTo("net."), + zone: ".", query: exampleCom, want: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + assert.Equal(t, tt.want, + resolver.UsableReply(tt.resp, tt.zone, tt.query), + ) + }) + } +} + +// TestNSSetFrom checks which NS set a reply gives for a domain; a set +// that is not empty ends the walk. The referral to example.com that +// com's servers all send alike gives its delegation, so the set is the +// same whichever of them answered, and example.com's own servers, which +// can disagree, are not asked. +func TestNSSetFrom(t *testing.T) { + t.Parallel() + + answer := new(dns.Msg) + answer.Authoritative = true + answer.Answer = []dns.RR{nsRecord(exampleCom)} + + tests := []struct { + name string + resp *dns.Msg + domain string + want []string + }{ + { + name: "com refers to example.com", resp: referralTo(exampleCom), + domain: exampleCom, want: []string{exampleNS}, + }, + { + name: "com refers on, for www.example.com", + resp: referralTo(exampleCom), domain: wwwExampleCom, + want: nil, + }, + { + name: "answer from a server that holds example.com", + resp: answer, domain: exampleCom, + want: []string{exampleNS}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + assert.ElementsMatch(t, tt.want, + resolver.NSSetFrom(tt.resp, tt.domain), + ) + }) + } +} + +func TestExtractRecordValue_LetterCase(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + rr dns.RR + want string + }{ + { + name: "MX target lower-cased", + rr: &dns.MX{Preference: 1, Mx: "ASPMX.L.GOOGLE.COM."}, + want: "1 aspmx.l.google.com.", + }, + { + name: "NS target lower-cased", + rr: &dns.NS{Ns: "x.ns.joker.COM."}, + want: "x.ns.joker.com.", + }, + { + name: "CNAME target lower-cased", + rr: &dns.CNAME{Target: "WWW.Example.Com."}, + want: "www.example.com.", + }, + { + name: "SRV target lower-cased", + rr: &dns.SRV{ + Priority: 10, Weight: 5, Port: 443, + Target: "SIP.Example.Com.", + }, + want: "10 5 443 sip.example.com.", + }, + { + name: "TXT value keeps its case", + rr: &dns.TXT{Txt: []string{"Verify=AbC123"}}, + want: "Verify=AbC123", + }, + { + name: "CAA value keeps its case", + rr: &dns.CAA{Flag: 0, Tag: "issue", Value: "LetsEncrypt.org"}, + want: `0 issue "LetsEncrypt.org"`, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + assert.Equal(t, tt.want, resolver.ExtractRecordValue(tt.rr)) + }) + } +} + +// TestCollectAnswerRecords_CNAMEOnce collects the answers a nameserver +// gives for a name with a CNAME, one for each record type a check asks +// for. Each answer holds the CNAME, which must be stored once. +func TestCollectAnswerRecords_CNAMEOnce(t *testing.T) { + t.Parallel() + + cname := &dns.CNAME{ + Hdr: dns.RR_Header{ + Name: "git.eeqj.de.", Rrtype: dns.TypeCNAME, Class: dns.ClassINET, + }, + Target: "fsn1app1.datavi.be.", + } + + resp := &resolver.NameserverResponse{Records: map[string][]string{}} + + for _, qtype := range []uint16{ + dns.TypeA, dns.TypeAAAA, dns.TypeCNAME, dns.TypeMX, + dns.TypeTXT, dns.TypeSRV, dns.TypeCAA, dns.TypeNS, + } { + msg := new(dns.Msg) + msg.SetQuestion("git.eeqj.de.", qtype) + msg.Answer = []dns.RR{cname} + + resolver.CollectAnswerRecords(msg, resp) + } + + assert.Equal(t, + map[string][]string{"CNAME": {"fsn1app1.datavi.be."}}, + resp.Records, + ) +} + +// TestShuffled shuffles the root servers with many seeds. Every order +// must hold each root server once, so each is tried before a +// resolution fails; each root server must come first for some seed, so +// no one root server gets every first query; and the list passed in +// must be left as it was. +func TestShuffled(t *testing.T) { + t.Parallel() + + const seeds = 1000 + + roots := resolver.RootServerList() + before := slices.Clone(roots) + first := make(map[string]bool) + + for seed := range uint64(seeds) { + rng := rand.New(rand.NewPCG(seed, 0)) //nolint:gosec // seeded on purpose + order := resolver.Shuffled(roots, rng.Shuffle) + + assert.ElementsMatch(t, roots, order) + + first[order[0]] = true + } + + assert.Len(t, first, len(roots)) + assert.Equal(t, before, roots) +} diff --git a/internal/resolver/livedns_harness_test.go b/internal/resolver/livedns_harness_test.go index 87ce704..2efdb39 100644 --- a/internal/resolver/livedns_harness_test.go +++ b/internal/resolver/livedns_harness_test.go @@ -1,10 +1,7 @@ package resolver_test import ( - "context" - "sync" "testing" - "time" "github.com/stretchr/testify/assert" @@ -12,9 +9,8 @@ import ( ) // Tests for the live-DNS harness in livedns_test.go itself. These -// exercise pure logic and the retry/concurrency plumbing; they -// perform no DNS resolution of any kind, so they neither mock DNS -// nor depend on it. +// exercise pure logic; they perform no DNS resolution of any kind, so +// they neither mock DNS nor depend on it. // Names for the synthetic status maps below. Nothing is ever queried // at them: they are map keys handed to the package's pure counting @@ -90,47 +86,6 @@ func TestStatusCountingIgnoresSilentNameservers(t *testing.T) { ) } -func TestRetryLiveRecoversFromTransientFailure(t *testing.T) { - t.Parallel() - - const wantAttempts = 2 - - attempts := 0 - - retryLive(t, "transient", func(_ context.Context) error { - attempts++ - - if attempts < wantAttempts { - return errLiveNoAnswer - } - - return nil - }) - - assert.Equal(t, wantAttempts, attempts) -} - -func TestRetryLiveGivesEachAttemptADeadline(t *testing.T) { - t.Parallel() - - retryLive(t, "deadline", func(ctx context.Context) error { - deadline, ok := ctx.Deadline() - assert.True(t, ok, "attempt should carry a deadline") - - remaining := time.Until(deadline) - - assert.LessOrEqual(t, remaining, liveAttemptTimeout) - - // Lower bound too: without one this passes for a - // deadline far shorter than intended, which would - // silently turn every live attempt into an instant - // timeout. - assert.Greater(t, remaining, liveAttemptTimeout/2) - - return nil - }) -} - // TestUnsanctionedStatusesRejectsWrongAnswers is the regression test // for the defect this allowlist exists to prevent: a minority of // nameservers answering WRONGLY while quorum keeps the suite green. @@ -235,50 +190,3 @@ func TestUnsanctionedStatusesToleratesSilenceOnly(t *testing.T) { unsanctionedStatuses(results, allowed...), ) } - -func TestRunLiveBoundsConcurrency(t *testing.T) { - t.Parallel() - - const workers = 24 - - var ( - mu sync.Mutex - wg sync.WaitGroup - inFlight int - maxSeen int - ) - - wg.Add(workers) - - for range workers { - go func() { - defer wg.Done() - - _ = runLive(func(_ context.Context) error { - mu.Lock() - inFlight++ - - if inFlight > maxSeen { - maxSeen = inFlight - } - mu.Unlock() - - time.Sleep(time.Millisecond) - - mu.Lock() - inFlight-- - mu.Unlock() - - return nil - }) - }() - } - - wg.Wait() - - assert.Positive(t, maxSeen) - assert.LessOrEqual( - t, maxSeen, liveConcurrency, - "live queries must stay under the package-wide gate", - ) -} diff --git a/internal/resolver/livedns_test.go b/internal/resolver/livedns_test.go index b864460..53946d6 100644 --- a/internal/resolver/livedns_test.go +++ b/internal/resolver/livedns_test.go @@ -8,8 +8,8 @@ import ( "sort" "strings" "testing" - "time" + "sneak.berlin/go/dnswatcher/internal/livednstest" "sneak.berlin/go/dnswatcher/internal/resolver" ) @@ -17,144 +17,34 @@ import ( // Live DNS test support // ---------------------------------------------------------------- // -// Every test in this package resolves against the real, live DNS — -// see TESTING.md. Nothing here mocks, fakes, stubs, records or -// replays DNS, and nothing here skips or gates a test: the helpers -// below only change *how* the live queries are issued, so that a -// single dropped UDP packet or one slow authoritative server does -// not turn a correct resolver into a red build. +// Tests that look something up in DNS query live DNS servers, never a +// stand-in; logic that works on record data may be tested on that +// data with no lookup (see TESTING.md). Each live operation below goes +// through livednstest.Retry, which bounds how many resolutions are in +// flight at once and retries an operation that got no answer (see +// package livednstest). // -// Three mechanisms, all test-side: +// Where an assertion spans several independent nameservers, a quorum +// is enough: a strict majority answering as expected. A server that +// fails to answer is tolerated, while a server that answers *wrongly* +// still fails the test. // -// 1. Bounded concurrency. The package's tests are parallel and the -// build hosts have many cores, so without a limit every test -// starts its own iterative resolution at the same instant and -// they all hit the first root server in rootServerList() within -// a few milliseconds of each other. Root servers rate-limit -// that, which shows up as a different arbitrary subset of tests -// failing on each run. liveGate caps how many resolutions are -// in flight at once. -// -// 2. Retry with exponential backoff. Each live operation gets -// several attempts with its own timeout. The retry predicate is -// strictly transport-level — "did a nameserver answer at all" — -// never the assertion the test is making. A resolver that -// answers incorrectly still fails on the first attempt. -// -// 3. Quorum. Where an assertion spans several independent -// nameservers, a strict majority answering as expected is -// enough; a server that fails to answer is tolerated, while a -// server that answers *wrongly* still fails the test. -// -// The tolerance in (3) is expressed as an ALLOWLIST of sanctioned -// statuses, never as a blocklist of known-bad ones. A blocklist bans -// the one wrong answer its author thought of and silently admits -// every other status, including any added to the resolver later; an -// allowlist fails on anything nobody explicitly sanctioned. Silence -// (timeout, error) is the only thing quorum exists to tolerate. A -// *wrong answer* — nxdomain for a name that exists, ok for one that -// does not, nodata for either — is never tolerated at any count. +// That tolerance is expressed as an ALLOWLIST of sanctioned statuses, +// never as a blocklist of known-bad ones. A blocklist bans the one +// wrong answer its author thought of and silently admits every other +// status, including any added to the resolver later; an allowlist +// fails on anything nobody explicitly sanctioned. Silence (timeout, +// error) is the only thing quorum exists to tolerate. A *wrong +// answer* — nxdomain for a name that exists, ok for one that does +// not, nodata for either — is never tolerated at any count. -const ( - // liveAttempts is how many times a live DNS operation is - // attempted before the test fails. - liveAttempts = 3 +// minNameservers is the smallest nameserver count a well-run zone is +// expected to publish. +const minNameservers = 2 - // liveAttemptTimeout bounds one attempt. Worst case for an - // operation is liveAttempts * liveAttemptTimeout plus the - // backoff — about 26 seconds, well inside the 90-second - // `go test -timeout` backstop even when several operations - // exhaust their attempts. - liveAttemptTimeout = 8 * time.Second - - // liveBackoffBase is the delay after the first failed - // attempt; it is multiplied by liveBackoffFactor each time. - liveBackoffBase = 500 * time.Millisecond - - // liveBackoffFactor is the exponential backoff multiplier. - liveBackoffFactor = 2 - - // liveConcurrency caps how many live resolutions may be in - // flight across the whole package at once. - liveConcurrency = 6 - - // minNameservers is the smallest nameserver count a - // well-run zone is expected to publish. - minNameservers = 2 -) - -// liveGate bounds concurrent live resolutions package-wide. It has -// to be package scoped: the whole point is that it is shared by -// every parallel test in the package. -// -//nolint:gochecknoglobals // package-wide live query rate limit -var liveGate = make(chan struct{}, liveConcurrency) - -var ( - // errLiveNoAnswer reports that a live operation produced no - // usable answer, which is retried rather than asserted on. - errLiveNoAnswer = errors.New("no answer from live DNS") - - // errLiveNoQuorum reports that too few of a domain's - // nameservers answered for a quorum assertion to be made. - errLiveNoQuorum = errors.New("no nameserver quorum") -) - -// runLive executes one attempt of a live operation, holding a slot -// in liveGate for its duration and bounding it with its own -// timeout. -func runLive(op func(ctx context.Context) error) error { - liveGate <- struct{}{} - defer func() { <-liveGate }() - - ctx, cancel := context.WithTimeout( - context.Background(), liveAttemptTimeout, - ) - defer cancel() - - return op(ctx) -} - -// retryLive runs op until it reports success, retrying transport -// failures with exponential backoff, and fails the test if every -// attempt fails. op returns an error only for a failure to obtain -// an answer — never for an answer the test disagrees with, which -// belongs in an assertion so that it fails immediately. op stores -// whatever it obtained where its caller can find it. -func retryLive( - t *testing.T, - what string, - op func(ctx context.Context) error, -) { - t.Helper() - - var last error - - backoff := liveBackoffBase - - for attempt := range liveAttempts { - if attempt > 0 { - t.Logf( - "%s: attempt %d of %d failed (%v), "+ - "retrying in %s", - what, attempt, liveAttempts, last, backoff, - ) - time.Sleep(backoff) - - backoff *= liveBackoffFactor - } - - last = runLive(op) - if last == nil { - return - } - } - - t.Fatalf( - "%s: no answer after %d live attempts: %v", - what, liveAttempts, last, - ) -} +// errLiveNoQuorum reports that too few of a domain's nameservers +// answered for a quorum assertion to be made. +var errLiveNoQuorum = errors.New("no nameserver quorum") // liveQuorum is how many of total nameservers must agree for a // multi-nameserver assertion to hold: a strict majority. @@ -272,7 +162,7 @@ func liveFindAuthoritative( var out []string - retryLive( + livednstest.Retry( t, "FindAuthoritativeNameservers("+domain+")", func(ctx context.Context) error { @@ -284,7 +174,7 @@ func liveFindAuthoritative( if len(ns) == 0 { return fmt.Errorf( "%w: %s has no nameservers", - errLiveNoAnswer, domain, + livednstest.ErrNoAnswer, domain, ) } @@ -297,8 +187,8 @@ func liveFindAuthoritative( return out } -// liveLookupNS is liveFindAuthoritative through the LookupNS entry -// point, so that both entry points stay independently exercised. +// liveLookupNS looks up the NS record set of domain, a domain that has +// one, retrying until the delegation chain can be walked. func liveLookupNS( t *testing.T, r *resolver.Resolver, @@ -308,7 +198,7 @@ func liveLookupNS( var out []string - retryLive( + livednstest.Retry( t, "LookupNS("+domain+")", func(ctx context.Context) error { @@ -320,7 +210,7 @@ func liveLookupNS( if len(ns) == 0 { return fmt.Errorf( "%w: %s has no nameservers", - errLiveNoAnswer, domain, + livednstest.ErrNoAnswer, domain, ) } @@ -336,11 +226,17 @@ func liveLookupNS( // liveQueryNameserver queries one nameserver, retrying while that // nameserver fails to answer. NXDOMAIN and NODATA are answers and // are returned to the caller to assert on. +// +// QueryNameserver sends one query per record type, so one lost query +// leaves its type out of an answer that is otherwise fine. A test names +// in types the record types it reads; an answer holding records of none +// of them is retried too. func liveQueryNameserver( t *testing.T, r *resolver.Resolver, nameserver string, hostname string, + types ...string, ) *resolver.NameserverResponse { t.Helper() @@ -350,7 +246,7 @@ func liveQueryNameserver( var out *resolver.NameserverResponse - retryLive( + livednstest.Retry( t, what, func(ctx context.Context) error { @@ -365,11 +261,23 @@ func liveQueryNameserver( resp.Status == resolver.StatusError { return fmt.Errorf( "%w: %s returned %s: %s", - errLiveNoAnswer, nameserver, + livednstest.ErrNoAnswer, nameserver, resp.Status, resp.Error, ) } + hasRecords := func(recordType string) bool { + return len(resp.Records[recordType]) > 0 + } + + if len(types) > 0 && !slices.ContainsFunc(types, hasRecords) { + return fmt.Errorf( + "%w: %s returned no %s records", + livednstest.ErrNoAnswer, nameserver, + strings.Join(types, " or "), + ) + } + out = resp return nil @@ -392,7 +300,7 @@ func liveQueryAllNameservers( var out map[string]*resolver.NameserverResponse - retryLive( + livednstest.Retry( t, "QueryAllNameservers("+hostname+")", func(ctx context.Context) error { @@ -404,7 +312,7 @@ func liveQueryAllNameservers( if len(results) == 0 { return fmt.Errorf( "%w: no nameservers queried for %s", - errLiveNoAnswer, hostname, + livednstest.ErrNoAnswer, hostname, ) } @@ -437,7 +345,7 @@ func liveResolveIPs( var out []string - retryLive( + livednstest.Retry( t, "ResolveIPAddresses("+hostname+")", func(ctx context.Context) error { @@ -449,7 +357,7 @@ func liveResolveIPs( if len(ips) == 0 { return fmt.Errorf( "%w: no addresses for %s", - errLiveNoAnswer, hostname, + livednstest.ErrNoAnswer, hostname, ) } @@ -476,7 +384,7 @@ func liveResolveIPsAllowingEmpty( var out []string - retryLive( + livednstest.Retry( t, "ResolveIPAddresses("+hostname+")", func(ctx context.Context) error { @@ -493,3 +401,37 @@ func liveResolveIPsAllowingEmpty( return out } + +// liveResolveNSIPs looks up the addresses of the nameservers named +// names, retrying until there are at least atLeast of them: a name +// whose lookup got no reply is left out of the result, not an error. +func liveResolveNSIPs( + t *testing.T, + r *resolver.Resolver, + names []string, + atLeast int, +) []string { + t.Helper() + + var out []string + + livednstest.Retry( + t, + "ResolveNSIPs("+strings.Join(names, ", ")+")", + func(ctx context.Context) error { + ips := r.ResolveNSIPs(ctx, names) + if len(ips) < atLeast { + return fmt.Errorf( + "%w: %d addresses, expected at least %d", + livednstest.ErrNoAnswer, len(ips), atLeast, + ) + } + + out = ips + + return nil + }, + ) + + return out +} diff --git a/internal/resolver/resolver.go b/internal/resolver/resolver.go index aec9b89..1bdfdb9 100644 --- a/internal/resolver/resolver.go +++ b/internal/resolver/resolver.go @@ -31,11 +31,15 @@ type Params struct { } // NameserverResponse holds one nameserver's response for a query. +// FailedTypes lists the record types whose query got no usable reply, +// and Records holds nothing for them: their records are not known. When +// no record type got one, Status and Error say the nameserver failed. type NameserverResponse struct { - Nameserver string - Records map[string][]string - Status string - Error string + Nameserver string + Records map[string][]string + FailedTypes []string + Status string + Error string } // Resolver performs iterative DNS resolution from root servers. @@ -67,17 +71,4 @@ func NewFromLogger(log *slog.Logger) *Resolver { } } -// NewFromLoggerWithClient creates a Resolver with a custom DNS -// client, useful for testing with mock DNS responses. -func NewFromLoggerWithClient( - log *slog.Logger, - client DNSClient, -) *Resolver { - return &Resolver{ - log: log, - client: client, - tcp: client, - } -} - // Method implementations are in iterative.go. diff --git a/internal/resolver/resolver_test.go b/internal/resolver/resolver_test.go index 13bdcde..f67fb06 100644 --- a/internal/resolver/resolver_test.go +++ b/internal/resolver/resolver_test.go @@ -1,7 +1,10 @@ package resolver_test import ( + "bytes" "context" + "errors" + "fmt" "log/slog" "net" "os" @@ -14,6 +17,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "sneak.berlin/go/dnswatcher/internal/livednstest" "sneak.berlin/go/dnswatcher/internal/resolver" ) @@ -21,6 +25,13 @@ import ( // Test helpers // ---------------------------------------------------------------- +// nonexistentDomain is a .com domain that does not exist. +const nonexistentDomain = "dnswatcher-test-does-not-exist.com" + +// noAnswerAddress is 192.0.2.1, a documentation address: nothing +// answers there. +const noAnswerAddress = "192.0.2.1" + func newTestResolver(t *testing.T) *resolver.Resolver { t.Helper() @@ -33,8 +44,8 @@ func newTestResolver(t *testing.T) *resolver.Resolver { } // findOneNSForDomain picks one authoritative nameserver to aim a -// test at. Live-DNS retry, concurrency and quorum handling live in -// livedns_test.go. +// test at. Quorum handling lives in livedns_test.go, and the live-DNS +// retry and concurrency limit in package livednstest. func findOneNSForDomain( t *testing.T, r *resolver.Resolver, @@ -78,9 +89,56 @@ func TestFindAuthoritativeNameservers_Subdomain( t.Parallel() r := newTestResolver(t) - nameservers := liveFindAuthoritative(t, r, "www.google.com") + fromHost := liveFindAuthoritative(t, r, "www.google.com") + fromZone := liveFindAuthoritative(t, r, "google.com") - assert.NotEmpty(t, nameservers) + assert.Equal(t, fromZone, fromHost) +} + +// TestFindAuthoritativeNameservers_DelegatedSubdomain looks up the +// nameservers of www.cs.cmu.edu, a name in cs.cmu.edu, a zone that +// cmu.edu delegates to other servers. The servers of cs.cmu.edu answer +// that the name has no delegation of its own, so it gets their names, +// not those of the cmu.edu servers. Every referral on the way gives the +// nameservers' addresses, so the walk sends few queries. +func TestFindAuthoritativeNameservers_DelegatedSubdomain( + t *testing.T, +) { + t.Parallel() + + r := newTestResolver(t) + fromHost := liveFindAuthoritative(t, r, "www.cs.cmu.edu") + fromZone := liveLookupNS(t, r, "cs.cmu.edu") + fromParent := liveLookupNS(t, r, "cmu.edu") + + assert.Equal(t, fromZone, fromHost) + assert.NotEqual(t, fromParent, fromHost) +} + +// TestFindAuthoritativeNameservers_NoAnswer starts each walk for +// www.google.com at 192.0.2.1, a documentation address where nothing +// answers. A walk that got no answer does not say that the name has no +// delegation of its own, so the lookup returns that walk's error, about +// www.google.com, and tries no parent name: trying google.com and com +// would end in ErrNoNameservers, or in the error of a walk for one of +// them. The servers that did not answer are those the walk started at, +// which no referral named, so the error is no NoAnswerError, as when +// the host running dnswatcher has lost its network. +func TestFindAuthoritativeNameservers_NoAnswer(t *testing.T) { + t.Parallel() + + r := resolver.NewWithQueryTimeout(slog.Default(), 100*time.Millisecond) + + nameservers, err := r.FindAuthoritativeNameserversFrom( + t.Context(), "www.google.com", []string{noAnswerAddress}, + ) + require.Error(t, err) + require.NotErrorIs(t, err, resolver.ErrNoNameservers) + assert.Contains(t, err.Error(), "query www.google.com. @"+noAnswerAddress) + assert.Empty(t, nameservers) + + var noAnswer *resolver.NoAnswerError + assert.NotErrorAs(t, err, &noAnswer) } func TestFindAuthoritativeNameservers_ReturnsSorted( @@ -137,6 +195,134 @@ func TestFindAuthoritativeNameservers_CloudflareDomain( } } +// TestResolveNSIPs_EveryNameserver looks up the addresses of two of +// google.com's nameservers together, as the walk does when a referral +// names a zone's nameservers without their addresses, and compares them +// with each looked up alone. Together they must give the addresses of +// both, not only of the first that resolves, so that when one gives no +// usable reply the walk goes on to the other. +func TestResolveNSIPs_EveryNameserver(t *testing.T) { + t.Parallel() + + r := newTestResolver(t) + names := []string{"ns3.google.com.", "ns4.google.com."} + want := make([]string, 0, len(names)) + + for _, name := range names { + want = append(want, liveResolveNSIPs(t, r, []string{name}, 1)...) + } + + got := liveResolveNSIPs(t, r, names, len(want)) + + assert.ElementsMatch(t, want, got) +} + +// TestResolveNSIPs_ZoneDelegatedWithoutAddresses looks up the address +// of a.ntpns.org, a nameserver of pool.ntp.org. The org servers delegate +// ntpns.org to nameservers in other zones and give none of their +// addresses, so those are looked up on the way. +func TestResolveNSIPs_ZoneDelegatedWithoutAddresses(t *testing.T) { + t.Parallel() + + r := newTestResolver(t) + ips := liveResolveNSIPs(t, r, []string{"a.ntpns.org."}, 1) + + for _, ip := range ips { + assert.NotNil(t, net.ParseIP(ip), "should be valid IP: %s", ip) + } +} + +// TestQueryZone_GivenAddressesFail asks the servers of ntp.org about +// pool.ntp.org, as the walk to a name under ntp.org does after the org +// servers' referral. That referral names four nameservers and gives an +// address for ns1.everett.org alone; here the given address is +// 192.0.2.1, where nothing answers, so the other three must be looked +// up and asked. +func TestQueryZone_GivenAddressesFail(t *testing.T) { + t.Parallel() + + r := newTestResolver(t) + + var resp *dns.Msg + + livednstest.Retry( + t, + "QueryZone(192.0.2.1 and three ntp.org nameservers, pool.ntp.org)", + func(ctx context.Context) error { + var err error + + resp, err = r.QueryZone( + ctx, []string{"192.0.2.1"}, + []string{"anyns.pch.net.", "dns1.udel.edu.", "dns2.udel.edu."}, + "ntp.org.", "pool.ntp.org.", dns.TypeNS, 0, + ) + + return err + }, + ) + + assert.NotEmpty(t, resolver.NSSetFrom(resp, "pool.ntp.org.")) +} + +// TestQueryZone_LookupDepth asks the servers of g.ntpns.org, a +// nameserver of pool.ntp.org, for its address, as looking that address +// up does when anyns.pch.net, one of the servers of ntpns.org, gives the +// referral to g.ntpns.org without addresses. Their addresses are looked +// up (here only a.ntpns.org's), and that needs a bitnames.com +// nameserver's address, as the org servers delegate ntpns.org without +// addresses. From depth 1, where looking up g.ntpns.org's address +// starts, that makes three lookups and the address is found. From one +// below maxLookupDepth, the bitnames.com lookup would be past the limit, +// so nothing can be asked, and the error says the limit is why. +func TestQueryZone_LookupDepth(t *testing.T) { + t.Parallel() + + r := newTestResolver(t) + withoutAddresses := []string{"a.ntpns.org."} + + var resp *dns.Msg + + livednstest.Retry( + t, + "QueryZone(a.ntpns.org without its address, g.ntpns.org)", + func(ctx context.Context) error { + var err error + + resp, err = r.QueryZone( + ctx, nil, withoutAddresses, "g.ntpns.org.", "g.ntpns.org.", + dns.TypeA, 1, + ) + + return err + }, + ) + + assert.NotEmpty(t, resp.Answer) + + var limitErr error + + // Any other error is live DNS not answering, and is retried. + livednstest.Retry( + t, + "QueryZone(a.ntpns.org without its address, g.ntpns.org, "+ + "one below the limit)", + func(ctx context.Context) error { + _, limitErr = r.QueryZone( + ctx, nil, withoutAddresses, "g.ntpns.org.", "g.ntpns.org.", + dns.TypeA, resolver.MaxLookupDepth-1, + ) + if limitErr == nil || + errors.Is(limitErr, resolver.ErrLookupDepthExceeded) { + return nil + } + + return limitErr + }, + ) + + require.ErrorIs(t, limitErr, resolver.ErrLookupDepthExceeded) +} + // ---------------------------------------------------------------- // QueryNameserver tests // ---------------------------------------------------------------- @@ -146,7 +332,7 @@ func TestQueryNameserver_BasicA(t *testing.T) { r := newTestResolver(t) ns := findOneNSForDomain(t, r, "google.com") - resp := liveQueryNameserver(t, r, ns, "www.google.com") + resp := liveQueryNameserver(t, r, ns, "www.google.com", "A", "CNAME") require.NotNil(t, resp) @@ -160,12 +346,144 @@ func TestQueryNameserver_BasicA(t *testing.T) { ) } +// TestQueryNameserver_ZoneDelegatedWithoutAddresses asks a.ntpns.org, a +// nameserver of pool.ntp.org, about pool.ntp.org, as the watcher does. +// The org servers delegate ntpns.org without the addresses of its +// nameservers, so finding a.ntpns.org's address needs a lookup inside +// the one QueryNameserver starts. +func TestQueryNameserver_ZoneDelegatedWithoutAddresses(t *testing.T) { + t.Parallel() + + r := newTestResolver(t) + resp := liveQueryNameserver(t, r, "a.ntpns.org.", "pool.ntp.org", "A") + + assert.Equal(t, resolver.StatusOK, resp.Status) +} + +// TestQueryTypes_FirstAddressNeverAnswers asks a google.com nameserver +// about google.com's A and AAAA records at a list of addresses that puts +// 192.0.2.1, where nothing answers, before the nameserver's own. The +// nameserver answers at its own address, so it is ok, with its records. +// Each attempt has time for one round of tries at 192.0.2.1, two tries +// of two seconds, and not for two, so AAAA must be asked first at the +// address that answered A. +func TestQueryTypes_FirstAddressNeverAnswers(t *testing.T) { + t.Parallel() + + r := newTestResolver(t) + ns := findOneNSForDomain(t, r, "google.com") + nsIPs := append( + []string{noAnswerAddress}, liveResolveNSIPs(t, r, []string{ns}, 1)..., + ) + + var resp *resolver.NameserverResponse + + livednstest.Retry( + t, + "QueryTypes("+ns+" at "+strings.Join(nsIPs, ", ")+", google.com)", + func(ctx context.Context) error { + ctx, cancel := context.WithTimeout(ctx, 6*time.Second) + defer cancel() + + var err error + + resp, err = r.QueryTypes( + ctx, ns, nsIPs, "google.com.", + []uint16{dns.TypeA, dns.TypeAAAA}, + ) + if err != nil { + return err + } + + if len(resp.FailedTypes) > 0 { + return fmt.Errorf( + "%w: %s gave no usable reply for %s", + livednstest.ErrNoAnswer, ns, + strings.Join(resp.FailedTypes, ", "), + ) + } + + return nil + }, + ) + + assert.Equal(t, resolver.StatusOK, resp.Status) + assert.NotEmpty(t, resp.Records["A"]) + assert.NotEmpty(t, resp.Records["AAAA"]) +} + +// TestQueryTypes_EveryAddressFails asks a google.com nameserver about +// cloudflare.com, a zone it does not serve, at 192.0.2.1, where nothing +// answers, and then at its own address, which refuses. The nameserver +// has failed with the refusal, not a timeout, and the warning about the +// type gives the reason at each address. +func TestQueryTypes_EveryAddressFails(t *testing.T) { + t.Parallel() + + var logs bytes.Buffer + + r := resolver.NewFromLogger(slog.New(slog.NewTextHandler(&logs, nil))) + ns := findOneNSForDomain(t, r, "google.com") + nsIPs := append( + []string{noAnswerAddress}, liveResolveNSIPs(t, r, []string{ns}, 1)..., + ) + + var resp *resolver.NameserverResponse + + livednstest.Retry( + t, + "QueryTypes("+ns+" at "+strings.Join(nsIPs, ", ")+", cloudflare.com)", + func(ctx context.Context) error { + logs.Reset() + + var err error + + resp, err = r.QueryTypes( + ctx, ns, nsIPs, "cloudflare.com.", []uint16{dns.TypeA}, + ) + if err != nil { + return err + } + + // A timeout or a network error at its own address is no + // reply at all. + if resp.Status == resolver.StatusTimeout || + strings.HasPrefix(resp.Error, "network error") { + return fmt.Errorf( + "%w: %s: %s", livednstest.ErrNoAnswer, ns, resp.Error, + ) + } + + return nil + }, + ) + + assert.Equal(t, resolver.StatusError, resp.Status) + assert.Equal(t, "server returned REFUSED", resp.Error) + assert.Contains(t, logs.String(), "@"+noAnswerAddress+": ") + assert.Contains(t, logs.String(), "@"+nsIPs[1]+": ") +} + +// TestQueryTypes_NoAddress asks a nameserver given no address. There is +// nothing to ask it at, so it has failed, with an error that says it has +// no address; it is not ok. +func TestQueryTypes_NoAddress(t *testing.T) { + t.Parallel() + + resp, err := newTestResolver(t).QueryTypes( + t.Context(), "ns1.example.net.", nil, "www.example.net.", + []uint16{dns.TypeA}, + ) + require.ErrorIs(t, err, resolver.ErrNoNameserverAddress) + assert.Nil(t, resp) +} + func TestQueryNameserver_AAAA(t *testing.T) { t.Parallel() r := newTestResolver(t) ns := findOneNSForDomain(t, r, "cloudflare.com") - resp := liveQueryNameserver(t, r, ns, "cloudflare.com") + resp := liveQueryNameserver(t, r, ns, "cloudflare.com", "AAAA") aaaaRecords := resp.Records["AAAA"] require.NotEmpty(t, aaaaRecords, @@ -185,7 +503,7 @@ func TestQueryNameserver_MX(t *testing.T) { r := newTestResolver(t) ns := findOneNSForDomain(t, r, "google.com") - resp := liveQueryNameserver(t, r, ns, "google.com") + resp := liveQueryNameserver(t, r, ns, "google.com", "MX") mxRecords := resp.Records["MX"] require.NotEmpty(t, mxRecords, @@ -198,7 +516,7 @@ func TestQueryNameserver_TXT(t *testing.T) { r := newTestResolver(t) ns := findOneNSForDomain(t, r, "google.com") - resp := liveQueryNameserver(t, r, ns, "google.com") + resp := liveQueryNameserver(t, r, ns, "google.com", "TXT") txtRecords := resp.Records["TXT"] require.NotEmpty(t, txtRecords, @@ -220,6 +538,38 @@ func TestQueryNameserver_TXT(t *testing.T) { ) } +// TestQueryNameserver_TruncatedReplyWhoseTCPRetryFails asks a google.com +// nameserver about google.com with a resolver whose retries over TCP +// fail. google.com's TXT records do not fit in a reply over UDP, so TXT +// is reported as failed, holding none of the records that fit, and +// logged with the reason, which names the address whose reply was +// truncated, while the nameserver, which answered the other types, is +// ok. +func TestQueryNameserver_TruncatedReplyWhoseTCPRetryFails(t *testing.T) { + t.Parallel() + + ns := findOneNSForDomain(t, newTestResolver(t), "google.com") + + var logs bytes.Buffer + + r := resolver.NewWithFailingTCP(slog.New(slog.NewTextHandler(&logs, nil))) + resp := liveQueryNameserver(t, r, ns, "google.com") + + assert.Equal(t, resolver.StatusOK, resp.Status) + assert.Contains(t, resp.FailedTypes, "TXT") + assert.NotContains(t, resp.Records, "TXT") + assert.Contains(t, logs.String(), + "hostname=google.com. nameserver="+ns+" type=TXT error=", + ) + + nsIPs := liveResolveNSIPs(t, newTestResolver(t), []string{ns}, 1) + for _, ip := range nsIPs { + assert.Contains(t, logs.String(), + "@"+ip+": "+resolver.ErrTruncated.Error(), + ) + } +} + func TestQueryNameserver_NXDomain(t *testing.T) { t.Parallel() @@ -232,6 +582,342 @@ func TestQueryNameserver_NXDomain(t *testing.T) { assert.Equal(t, resolver.StatusNXDomain, resp.Status) } +// TestQueryNameserver_Refused asks a google.com nameserver about +// cloudflare.com, a zone it does not serve, which it refuses. Refusing +// is a failure to answer, not an answer with no records. +func TestQueryNameserver_Refused(t *testing.T) { + t.Parallel() + + r := newTestResolver(t) + ns := findOneNSForDomain(t, r, "google.com") + + var resp *resolver.NameserverResponse + + livednstest.Retry( + t, + "QueryNameserver("+ns+", cloudflare.com)", + func(ctx context.Context) error { + var err error + + resp, err = r.QueryNameserver(ctx, ns, "cloudflare.com") + if err != nil { + return err + } + + // A timeout or a network error is no reply at all. + if resp.Status == resolver.StatusTimeout || + strings.HasPrefix(resp.Error, "network error") { + return fmt.Errorf( + "%w: %s: %s", + livednstest.ErrNoAnswer, ns, resp.Error, + ) + } + + return nil + }, + ) + + assert.Equal(t, resolver.StatusError, resp.Status) + assert.Equal(t, "server returned REFUSED", resp.Error) +} + +// TestQueryNameserver_EveryAddressRefuses asks ada.ns.cloudflare.com, +// whose name has several IPv4 addresses, about google.com, a zone it +// does not serve, which it refuses. A record type is asked at each of +// the nameserver's addresses before it fails, so the warning about it +// names every one. +func TestQueryNameserver_EveryAddressRefuses(t *testing.T) { + t.Parallel() + + const ns = "ada.ns.cloudflare.com." + + var logs bytes.Buffer + + r := resolver.NewFromLogger(slog.New(slog.NewTextHandler(&logs, nil))) + nsIPs := liveResolveNSIPs(t, r, []string{ns}, 2) + + var resp *resolver.NameserverResponse + + livednstest.Retry( + t, + "QueryNameserver("+ns+", google.com)", + func(ctx context.Context) error { + logs.Reset() + + var err error + + resp, err = r.QueryNameserver(ctx, ns, "google.com") + if err != nil { + return err + } + + // A timeout or a network error at every address is no + // reply at all. + if resp.Status == resolver.StatusTimeout || + strings.HasPrefix(resp.Error, "network error") { + return fmt.Errorf( + "%w: %s: %s", livednstest.ErrNoAnswer, ns, resp.Error, + ) + } + + return nil + }, + ) + + assert.Equal(t, resolver.StatusError, resp.Status) + assert.Equal(t, "server returned REFUSED", resp.Error) + + for _, ip := range nsIPs { + assert.Contains(t, logs.String(), "@"+ip+": ") + } +} + +// TestQueryServers_RecursiveResolverRefused passes a public recursive +// resolver to QueryServers as the server of google.com. These resolvers +// refuse a query that does not ask for recursion and answer one that +// does. The resolver never asks for recursion, so the query must be +// reported as refused, never answered. Each resolver is run by a +// different operator, and they are asked in turn until one replies, so +// one operator not answering does not fail the test. +func TestQueryServers_RecursiveResolverRefused(t *testing.T) { + t.Parallel() + + r := newTestResolver(t) + resolvers := []string{ + "64.6.64.6", "185.222.222.222", "4.2.2.1", "9.9.9.9", + } + + var err error + + livednstest.Retry( + t, + "QueryServers(public recursive resolvers, google.com)", + func(ctx context.Context) error { + for _, ip := range resolvers { + _, err = r.QueryServers( + ctx, []string{ip}, "google.com.", "google.com.", + dns.TypeA, + ) + + // A refusal or an answer is a reply; anything else may + // be no reply at all, so the next resolver is asked. + if err == nil || errors.Is(err, resolver.ErrRefused) { + return nil + } + } + + return fmt.Errorf("%w: %w", livednstest.ErrNoAnswer, err) + }, + ) + + require.ErrorIs(t, err, resolver.ErrRefused) +} + +// googleNameserverIPv4s returns the IPv4 addresses of google.com's +// nameservers, the only addresses the resolver asks servers at. +func googleNameserverIPv4s(t *testing.T, r *resolver.Resolver) []string { + t.Helper() + + names := liveFindAuthoritative(t, r, "google.com") + + return liveResolveNSIPs(t, r, names, len(names)) +} + +// TestQueryServers_EveryServerRefused asks all of google.com's +// nameservers about cloudflare.com, a zone they do not serve, which +// they all refuse. The error says every server refused; it is not +// ErrIntercepted, which only the root servers refusing shows. +func TestQueryServers_EveryServerRefused(t *testing.T) { + t.Parallel() + + r := newTestResolver(t) + servers := googleNameserverIPv4s(t, r) + + var err error + + livednstest.Retry( + t, + "QueryServers(google.com servers, cloudflare.com)", + func(ctx context.Context) error { + _, err = r.QueryServers( + ctx, servers, "google.com.", "cloudflare.com.", + dns.TypeNS, + ) + + // When not every server refused, one may have given no + // reply at all, so the attempt is tried again. + if err != nil && + !strings.HasPrefix(err.Error(), "every server of") { + return fmt.Errorf( + "%w: %w", livednstest.ErrNoAnswer, err, + ) + } + + return nil + }, + ) + + require.ErrorIs(t, err, resolver.ErrRefused) + require.NotErrorIs(t, err, resolver.ErrIntercepted) + require.EqualError( + t, err, + "every server of google.com. refused a query for "+ + "cloudflare.com.: dns query refused", + ) +} + +// TestQueryServers_EveryRootServerRefused passes google.com's +// nameservers to QueryServers as the servers of the root zone. They +// refuse a query about cloudflare.com, as root servers would if +// something on the network answered in their place, so the error is +// ErrIntercepted. +func TestQueryServers_EveryRootServerRefused(t *testing.T) { + t.Parallel() + + r := newTestResolver(t) + servers := googleNameserverIPv4s(t, r) + + var err error + + livednstest.Retry( + t, + "QueryServers(google.com servers as root servers, cloudflare.com)", + func(ctx context.Context) error { + _, err = r.QueryServers( + ctx, servers, ".", "cloudflare.com.", dns.TypeNS, + ) + + // When not every server refused, one may have given no + // reply at all, so the attempt is tried again. Both errors + // for every server refusing say "refused a query for". + if err != nil && + !strings.Contains(err.Error(), "refused a query for") { + return fmt.Errorf( + "%w: %w", livednstest.ErrNoAnswer, err, + ) + } + + return nil + }, + ) + + require.ErrorIs(t, err, resolver.ErrIntercepted) + require.EqualError( + t, err, + "every root server refused a query for cloudflare.com.: "+ + "this network intercepts DNS queries", + ) +} + +// TestQueryServers_NotEveryRootServerRefused passes google.com's +// nameservers and 192.0.2.1 to QueryServers as the servers of the root +// zone. The google.com nameservers refuse a query about cloudflare.com, +// but nothing answers at 192.0.2.1, a documentation address, so not +// every server refused, wherever 192.0.2.1 falls in the random order: +// the error is not ErrIntercepted and does not say every server refused. +func TestQueryServers_NotEveryRootServerRefused(t *testing.T) { + t.Parallel() + + r := newTestResolver(t) + servers := googleNameserverIPv4s(t, r) + servers = append(servers, "192.0.2.1") + + var err error + + livednstest.Retry( + t, + "QueryServers(google.com servers and 192.0.2.1, cloudflare.com)", + func(ctx context.Context) error { + _, err = r.QueryServers( + ctx, servers, ".", "cloudflare.com.", dns.TypeNS, + ) + + // An attempt that ran out of time may not have asked every + // server, so it is tried again. + if ctx.Err() != nil { + return fmt.Errorf( + "%w: %w", livednstest.ErrNoAnswer, err, + ) + } + + return nil + }, + ) + + require.Error(t, err) + require.NotErrorIs(t, err, resolver.ErrIntercepted) + // Both errors for every server refusing say "refused a query for". + require.NotContains(t, err.Error(), "refused a query for") +} + +// TestQueryServers_RecursiveResolversAsRootServers passes public +// recursive resolvers to QueryServers as the servers of the root zone, +// as a network that sends every DNS query to a resolver would. Their +// replies are marked recursion available, which no root server's is: +// SERVFAIL, REFUSED, or an answer from a resolver's cache, as 1.1.1.1 +// gives for com., which would be used if a root server gave it. The +// first such reply ends the lookup with ErrIntercepted, naming the +// resolver that gave it. In the first group the replies differ; every +// resolver in the second refuses a query that does not ask for +// recursion, as 9.9.9.9 does, so there a refusal ends the lookup +// instead of being passed over. Each group has resolvers of four +// operators, and a resolver that gives no reply is passed over for the +// next, so one of them answering is enough. +func TestQueryServers_RecursiveResolversAsRootServers(t *testing.T) { + t.Parallel() + + r := newTestResolver(t) + groups := [][]string{ + {"1.1.1.1", "8.8.8.8", "9.9.9.9", "208.67.222.222"}, + {"64.6.64.6", "185.222.222.222", "4.2.2.1", "94.140.14.14"}, + } + + for _, resolvers := range groups { + for _, name := range []string{"com.", "desec.io."} { + var err error + + livednstest.Retry( + t, + "QueryServers("+strings.Join(resolvers, ", ")+ + " as root servers, "+name+")", + func(ctx context.Context) error { + _, err = r.QueryServers( + ctx, resolvers, ".", name, dns.TypeNS, + ) + + // When no resolver gave a reply marked recursion + // available, none may have replied at all, so the + // attempt is tried again. + if err != nil && !strings.Contains( + err.Error(), "marked recursion available", + ) { + return fmt.Errorf( + "%w: %w", livednstest.ErrNoAnswer, err, + ) + } + + return nil + }, + ) + + require.ErrorIs(t, err, resolver.ErrIntercepted) + + // The error names the resolver whose reply ended the + // lookup. + messages := make([]string, 0, len(resolvers)) + for _, ip := range resolvers { + messages = append(messages, + "query "+name+" @"+ip+": reply marked recursion "+ + "available, so not from a root server: "+ + "this network intercepts DNS queries", + ) + } + + require.Contains(t, messages, err.Error()) + } + } +} + func TestQueryNameserver_RecordsSorted(t *testing.T) { t.Parallel() @@ -310,37 +996,58 @@ func TestQueryAllNameservers_ReturnsAllNS(t *testing.T) { func TestQueryAllNameservers_AllReturnOK(t *testing.T) { t.Parallel() - r := newTestResolver(t) - results := liveQueryAllNameservers(t, r, "google.com") + // The last two names are in zones other than their last two + // labels: google.co.uk, under the two-label suffix co.uk, and + // compute-1.amazonaws.com, which amazonaws.com delegates to other + // servers and which has a host name for each of its addresses. + // Servers above a name's zone only refer onward, which gives + // nodata, so ok shows the name was asked at its own zone's + // servers. + hostnames := []string{ + "google.com", + "www.google.co.uk", + "ec2-3-80-0-1.compute-1.amazonaws.com", + } - // A quorum, not unanimity: one authoritative server being - // slow or rate-limiting us is a property of the live - // internet, not a resolver defect. - assert.GreaterOrEqual( - t, - countStatus(results, resolver.StatusOK), - liveQuorum(len(results)), - "a quorum of nameservers should answer OK: %s", - describeStatuses(results), - ) + for _, hostname := range hostnames { + t.Run(hostname, func(t *testing.T) { + t.Parallel() - // Quorum tolerates SILENCE only. Every individual result must - // be either the expected answer or a non-answer: ok, timeout - // or error, and nothing else. Stated as a closed allowlist so - // that a wrong answer no one thought to ban — nxdomain and - // nodata today, any status added later — fails here rather - // than sliding through under the quorum. - assert.Empty( - t, - unsanctionedStatuses( - results, - resolver.StatusOK, - resolver.StatusTimeout, - resolver.StatusError, - ), - "every nameserver must answer OK or not answer at all: %s", - describeStatuses(results), - ) + r := newTestResolver(t) + results := liveQueryAllNameservers(t, r, hostname) + + // A quorum, not unanimity: one authoritative server + // being slow or rate-limiting us is a property of the + // live internet, not a resolver defect. + assert.GreaterOrEqual( + t, + countStatus(results, resolver.StatusOK), + liveQuorum(len(results)), + "a quorum of nameservers should answer OK: %s", + describeStatuses(results), + ) + + // Quorum tolerates SILENCE only. Every individual + // result must be either the expected answer or a + // non-answer: ok, timeout or error, and nothing else. + // Stated as a closed allowlist so that a wrong answer + // no one thought to ban — nxdomain and nodata today, + // any status added later — fails here rather than + // sliding through under the quorum. + assert.Empty( + t, + unsanctionedStatuses( + results, + resolver.StatusOK, + resolver.StatusTimeout, + resolver.StatusError, + ), + "every nameserver must answer OK or not answer "+ + "at all: %s", + describeStatuses(results), + ) + }) + } } func TestQueryAllNameservers_NXDomainFromAllNS( @@ -415,6 +1122,167 @@ func TestLookupNS_MatchesFindAuthoritative(t *testing.T) { assert.Equal(t, fromFind, fromLookup) } +// TestLookupNS_ParentZoneDelegatedWithoutAddresses looks up the +// nameservers of g.ntpns.org. The org servers delegate its parent zone, +// ntpns.org, without the addresses of its nameservers, so the walk has +// to look them up to ask them. If it did not, the walk for g.ntpns.org +// would fail. +func TestLookupNS_ParentZoneDelegatedWithoutAddresses(t *testing.T) { + t.Parallel() + + r := newTestResolver(t) + nameservers := liveLookupNS(t, r, "g.ntpns.org") + + assert.Contains(t, nameservers, "a.ntpns.org.") +} + +// TestLookupNS_DomainThatDoesNotExist looks up the nameservers of a .com +// domain that does not exist. The .com servers answer NXDOMAIN, so the +// error is ErrNXDomain, and the domain does not get their names. +func TestLookupNS_DomainThatDoesNotExist(t *testing.T) { + t.Parallel() + + r := newTestResolver(t) + + var ( + nameservers []string + err error + ) + + livednstest.Retry( + t, + "LookupNS("+nonexistentDomain+")", + func(ctx context.Context) error { + nameservers, err = r.LookupNS(ctx, nonexistentDomain) + if errors.Is(err, resolver.ErrNXDomain) { + return nil + } + + return err + }, + ) + + require.ErrorIs(t, err, resolver.ErrNXDomain) + assert.Empty(t, nameservers) +} + +// TestLookupNS_NoDelegationOfItsOwn looks up the nameservers of +// www.google.com, a name in the google.com zone with no delegation of +// its own, as a domain such as octocat.github.io is. The google.com +// servers answer with no NS records for it: the set is empty, and it is +// not ErrNXDomain. +func TestLookupNS_NoDelegationOfItsOwn(t *testing.T) { + t.Parallel() + + r := newTestResolver(t) + + var nameservers []string + + livednstest.Retry( + t, + "LookupNS(www.google.com)", + func(ctx context.Context) error { + var err error + + nameservers, err = r.LookupNS(ctx, "www.google.com") + + return err + }, + ) + + assert.Empty(t, nameservers) +} + +// TestFollowDelegation_NoAnswer starts the walk LookupNS uses, for +// google.com, at 192.0.2.1, a documentation address where nothing +// answers. A walk that got no answer is an error, not an empty set, +// which the watcher would report as an NS Change with every nameserver +// removed. +func TestFollowDelegation_NoAnswer(t *testing.T) { + t.Parallel() + + r := resolver.NewWithQueryTimeout(slog.Default(), 100*time.Millisecond) + + nameservers, err := r.FollowDelegation( + t.Context(), "google.com.", ".", nil, []string{noAnswerAddress}, + ) + require.Error(t, err) + assert.Empty(t, nameservers) +} + +// TestFollowDelegation_NoAnswerAfterReferral starts the walk for +// www.example.net at the servers of example.net as a referral names +// them, ns1.example.net and ns2.example.net, at 192.0.2.1, where nothing +// answers. The error is a NoAnswerError naming them, which the watcher +// compares with the nameservers it saved for the name. +func TestFollowDelegation_NoAnswerAfterReferral(t *testing.T) { + t.Parallel() + + r := resolver.NewWithQueryTimeout(slog.Default(), 100*time.Millisecond) + nameservers := []string{"ns1.example.net.", "ns2.example.net."} + + _, err := r.FollowDelegation( + t.Context(), "www.example.net.", "example.net.", nameservers, + []string{noAnswerAddress}, + ) + + var noAnswer *resolver.NoAnswerError + require.ErrorAs(t, err, &noAnswer) + assert.Equal(t, nameservers, noAnswer.Nameservers) + assert.Contains(t, err.Error(), "query www.example.net. @"+noAnswerAddress) +} + +// TestResolveNSIterative_NoDelegationOfItsOwn walks to the nameservers +// of www.google.com as the fallback walk does. As in +// TestLookupNS_NoDelegationOfItsOwn, the set is empty, with no error. +func TestResolveNSIterative_NoDelegationOfItsOwn(t *testing.T) { + t.Parallel() + + r := newTestResolver(t) + + var nameservers []string + + livednstest.Retry( + t, + "ResolveNSIterative(www.google.com)", + func(ctx context.Context) error { + var err error + + nameservers, err = r.ResolveNSIterative(ctx, "www.google.com") + + return err + }, + ) + + assert.Empty(t, nameservers) +} + +// TestResolveNSIterative_DomainThatDoesNotExist walks to the nameservers +// of a .com domain that does not exist as the fallback walk does. As in +// TestLookupNS_DomainThatDoesNotExist, the error is ErrNXDomain. +func TestResolveNSIterative_DomainThatDoesNotExist(t *testing.T) { + t.Parallel() + + r := newTestResolver(t) + + var err error + + livednstest.Retry( + t, + "ResolveNSIterative("+nonexistentDomain+")", + func(ctx context.Context) error { + _, err = r.ResolveNSIterative(ctx, nonexistentDomain) + if errors.Is(err, resolver.ErrNXDomain) { + return nil + } + + return err + }, + ) + + require.ErrorIs(t, err, resolver.ErrNXDomain) +} + // ---------------------------------------------------------------- // ResolveIPAddresses tests // ---------------------------------------------------------------- @@ -478,6 +1346,34 @@ func TestResolveIPAddresses_CloudflareDomain(t *testing.T) { assert.NotEmpty(t, ips) } +// TestResolveIPAddresses_NameserverIPv4AndIPv6 looks up the addresses of +// one of cloudflare.com's nameservers, as a domain check does for each +// nameserver. That name has A and AAAA records, so both kinds of address +// come back. +func TestResolveIPAddresses_NameserverIPv4AndIPv6(t *testing.T) { + t.Parallel() + + r := newTestResolver(t) + ns := findOneNSForDomain(t, r, "cloudflare.com") + ips := liveResolveIPs(t, r, ns) + + var ipv4, ipv6 int + + for _, ip := range ips { + parsed := net.ParseIP(ip) + require.NotNil(t, parsed, "should be valid IP: %s", ip) + + if parsed.To4() != nil { + ipv4++ + } else { + ipv6++ + } + } + + assert.Positive(t, ipv4, "no IPv4 address for %s: %v", ns, ips) + assert.Positive(t, ipv6, "no IPv6 address for %s: %v", ns, ips) +} + // ---------------------------------------------------------------- // Context cancellation tests // ---------------------------------------------------------------- @@ -519,6 +1415,29 @@ func TestQueryAllNameservers_ContextCanceled(t *testing.T) { assert.Error(t, err) } +// TestQueryEachNS_CanceledDuringQuery cancels the context while a +// nameserver is being queried, as shutdown does. A lookup cut short +// says nothing about the nameserver, so it must return an error, not a +// failed response for it. +func TestQueryEachNS_CanceledDuringQuery(t *testing.T) { + t.Parallel() + + r := newTestResolver(t) + ctx, cancel := context.WithCancel(context.Background()) + t.Cleanup(cancel) + + // Finding the nameserver's address alone starts at the root + // servers and takes several round trips, so a cancel a few + // milliseconds in lands during the query. + time.AfterFunc(5*time.Millisecond, cancel) + + results, err := r.QueryEachNS( + ctx, []string{"ns1.google.com."}, "google.com", + ) + require.ErrorIs(t, err, resolver.ErrContextCanceled) + assert.Nil(t, results) +} + // ---------------------------------------------------------------- // Timeout tests // ---------------------------------------------------------------- @@ -526,21 +1445,18 @@ func TestQueryAllNameservers_ContextCanceled(t *testing.T) { func TestQueryNameserverIP_Timeout(t *testing.T) { t.Parallel() - log := slog.New(slog.NewTextHandler( - os.Stderr, - &slog.HandlerOptions{Level: slog.LevelDebug}, - )) - - r := resolver.NewFromLoggerWithClient( - log, &timeoutClient{}, - ) + r := newTestResolver(t) + // Nothing answers at 192.0.2.1, a documentation address. The + // resolver tries each query twice, and each try gives up after two + // seconds. This deadline ends during the second try of the first + // record type; TestQueryNameserverIP_DeadlineDuringFirstTry covers + // one that ends during the first. ctx, cancel := context.WithTimeout( - context.Background(), 10*time.Second, + context.Background(), 3*time.Second, ) t.Cleanup(cancel) - // Query any IP — the client always returns a timeout error. resp, err := r.QueryNameserverIP( ctx, "unreachable.test.", "192.0.2.1", "example.com", @@ -551,26 +1467,160 @@ func TestQueryNameserverIP_Timeout(t *testing.T) { assert.NotEmpty(t, resp.Error) } -// timeoutClient simulates DNS timeout errors for testing. -type timeoutClient struct{} +// TestQueryNameserverIP_DeadlineDuringFirstTry gives a query to +// 192.0.2.1, where nothing answers, a deadline that ends during its +// first try. That try ends at the deadline as a timeout, so the +// nameserver is reported as timed out, not as answering. +func TestQueryNameserverIP_DeadlineDuringFirstTry(t *testing.T) { + t.Parallel() -func (c *timeoutClient) ExchangeContext( - _ context.Context, - _ *dns.Msg, - _ string, -) (*dns.Msg, time.Duration, error) { - return nil, 0, &net.OpError{ - Op: "read", - Net: "udp", - Err: &timeoutError{}, - } + r := newTestResolver(t) + + // Well under the two seconds one try waits. + ctx, cancel := context.WithTimeout( + context.Background(), 500*time.Millisecond, + ) + t.Cleanup(cancel) + + resp, err := r.QueryNameserverIP( + ctx, "unreachable.test.", "192.0.2.1", + "example.com", + ) + require.NoError(t, err) + + assert.Equal(t, resolver.StatusTimeout, resp.Status) + assert.NotEmpty(t, resp.Error) } -type timeoutError struct{} +// TestQueryNameserverIP_CancelDuringFirstTry cancels the context while +// the first try of a query to 192.0.2.1, where nothing answers, waits +// for a reply. That try's timeout is all that is learned about the +// nameserver, so it is reported as timed out, not as answering. +func TestQueryNameserverIP_CancelDuringFirstTry(t *testing.T) { + t.Parallel() -func (e *timeoutError) Error() string { return "i/o timeout" } -func (e *timeoutError) Timeout() bool { return true } -func (e *timeoutError) Temporary() bool { return true } + r := newTestResolver(t) + + // The first try starts at once and waits two seconds for a reply; + // the cancel comes a second in. The DNS client cuts a try short at + // the context's deadline but not at a cancel, so the try still + // waits its full two seconds and ends as a timeout a second after + // the cancel. By then the context has certainly ended, so no second + // try is made. With a deadline instead, the try ends at the moment + // the context does, and whether a second try is made depends on + // which the resolver sees first. + ctx, cancel := context.WithCancel(context.Background()) + t.Cleanup(cancel) + time.AfterFunc(time.Second, cancel) + + resp, err := r.QueryNameserverIP( + ctx, "unreachable.test.", "192.0.2.1", + "example.com", + ) + require.NoError(t, err) + + assert.Equal(t, resolver.StatusTimeout, resp.Status) + assert.NotEmpty(t, resp.Error) +} + +// TestQueryNameserverIP_CancelledLogsNothing cancels the context while +// a query to 192.0.2.1, where nothing answers, is waiting for a reply, +// as shutdown does. The query was cut short, not failed, so nothing is +// logged. +func TestQueryNameserverIP_CancelledLogsNothing(t *testing.T) { + t.Parallel() + + var logs bytes.Buffer + + r := resolver.NewFromLogger(slog.New(slog.NewTextHandler(&logs, nil))) + + ctx, cancel := context.WithCancel(context.Background()) + t.Cleanup(cancel) + time.AfterFunc(100*time.Millisecond, cancel) + + _, err := r.QueryNameserverIP( + ctx, "unreachable.test.", "192.0.2.1", "example.com", + ) + require.NoError(t, err) + + assert.Empty(t, logs.String()) +} + +// TestCollectIPs_NoNameserverAnswered takes the response of a +// nameserver at 192.0.2.1, where nothing answers, as +// TestQueryNameserverIP_Timeout does. Addresses collected from +// nameservers that all failed to answer are an error, not none. +func TestCollectIPs_NoNameserverAnswered(t *testing.T) { + t.Parallel() + + r := newTestResolver(t) + + // The deadline outlasts the first try, as in + // TestQueryNameserverIP_Timeout. + ctx, cancel := context.WithTimeout( + context.Background(), 3*time.Second, + ) + t.Cleanup(cancel) + + resp, err := r.QueryNameserverIP( + ctx, "unreachable.test.", "192.0.2.1", + "example.com", + ) + require.NoError(t, err) + + ips, _, err := resolver.CollectIPs( + map[string]*resolver.NameserverResponse{resp.Nameserver: resp}, + ) + require.ErrorIs(t, err, resolver.ErrNoNameserverAnswered) + assert.Empty(t, ips) +} + +// TestCollectIPs_ReferralIsNoAnswer asks a root server about +// example.com, which the root zone does not hold, so it only refers the +// query to the com servers. That reply is no answer, as is a parent +// zone's when every server of the name's own zone failed. +func TestCollectIPs_ReferralIsNoAnswer(t *testing.T) { + t.Parallel() + + r := newTestResolver(t) + + var resp *resolver.NameserverResponse + + livednstest.Retry( + t, + "QueryNameserverIP(a.root-servers.net, example.com)", + func(ctx context.Context) error { + var err error + + resp, err = r.QueryNameserverIP( + ctx, "a.root-servers.net.", "198.41.0.4", + "example.com", + ) + if err != nil { + return err + } + + // A timeout or a network error is no reply at all. + if resp.Status == resolver.StatusTimeout || + strings.HasPrefix(resp.Error, "network error") { + return fmt.Errorf( + "%w: %s", livednstest.ErrNoAnswer, resp.Error, + ) + } + + return nil + }, + ) + + assert.Equal(t, resolver.StatusError, resp.Status) + assert.Equal(t, "server returned a referral", resp.Error) + + ips, _, err := resolver.CollectIPs( + map[string]*resolver.NameserverResponse{resp.Nameserver: resp}, + ) + require.ErrorIs(t, err, resolver.ErrNoNameserverAnswered) + assert.Empty(t, ips) +} func TestResolveIPAddresses_ContextCanceled(t *testing.T) { t.Parallel() diff --git a/internal/server/export_test.go b/internal/server/export_test.go new file mode 100644 index 0000000..0fecf2f --- /dev/null +++ b/internal/server/export_test.go @@ -0,0 +1,38 @@ +package server + +import ( + "net/http" + "time" + + "github.com/go-chi/chi/v5" +) + +// RequestTimeout exports the handler execution budget applied by +// chimw.Timeout in SetupRoutes, so tests can assert the relationship +// between it and the server's WriteTimeout. +const RequestTimeout time.Duration = requestTimeout + +// SetListenPort overrides the port Run binds. A test uses it to hand +// Run an unbindable port, so that Run stores its http.Server and then +// returns an error without listening. +func SetListenPort(s *Server, port int) { + s.port = port +} + +// HTTPServerOf returns the http.Server that Run built and stored, so a +// test can inspect the timeouts the running server actually carries. +func HTTPServerOf(s *Server) *http.Server { + return s.httpServer +} + +// EnableSentry runs the Sentry setup that the start hook runs, without +// starting the HTTP server. +func EnableSentry(s *Server) error { + return s.enableSentry() +} + +// RouterOf returns the router SetupRoutes built, so a test can add a +// route that panics. +func RouterOf(s *Server) *chi.Mux { + return s.router +} diff --git a/internal/server/routes.go b/internal/server/routes.go index fa99177..52778ee 100644 --- a/internal/server/routes.go +++ b/internal/server/routes.go @@ -4,11 +4,10 @@ import ( "net/http" "time" + sentryhttp "github.com/getsentry/sentry-go/http" "github.com/go-chi/chi/v5" chimw "github.com/go-chi/chi/v5/middleware" "github.com/prometheus/client_golang/prometheus/promhttp" - - "sneak.berlin/go/dnswatcher/static" ) // requestTimeout is the maximum duration for handling a request. @@ -21,44 +20,61 @@ func (s *Server) SetupRoutes() { // Global middleware s.router.Use(chimw.Recoverer) s.router.Use(chimw.RequestID) + s.router.Use(s.mw.SecurityHeaders()) s.router.Use(s.mw.Logging()) - s.router.Use(s.mw.CORS()) s.router.Use(chimw.Timeout(requestTimeout)) + // Report panics in handlers to Sentry when DNSWATCHER_SENTRY_DSN is + // set. Repanic passes each panic on to chimw.Recoverer above, which + // still answers the request. + if s.sentryEnabled { + sentryHandler := sentryhttp.New(sentryhttp.Options{ + Repanic: true, + }) + s.router.Use(sentryHandler.Handle) + } + + // Public, unauthenticated, read-only routes, the only ones + // REPO_POLICIES.md allows wildcard CORS on. CORS is middleware of + // this whole router, not of a Group, so that it also answers + // OPTIONS preflight requests, which no route here registers. + public := chi.NewRouter() + public.Use(s.mw.CORS()) + // Dashboard (read-only web UI) - s.router.Get("/", s.handlers.HandleDashboard()) + public.Get("/", s.handlers.HandleDashboard()) // Static assets (embedded CSS/JS) - s.router.Mount( - "/s", - http.StripPrefix( - "/s", - http.FileServer(http.FS(static.Static)), - ), - ) + public.Mount("/s", http.StripPrefix("/s", s.handlers.HandleStatic())) // Health check (standard well-known path) - s.router.Get( + public.Get( "/.well-known/healthcheck", s.handlers.HandleHealthCheck(), ) // Legacy health check (keep for backward compatibility) - s.router.Get("/health", s.handlers.HandleHealthCheck()) + public.Get("/health", s.handlers.HandleHealthCheck()) // API v1 routes - s.router.Route("/api/v1", func(r chi.Router) { + public.Route("/api/v1", func(r chi.Router) { r.Get("/status", s.handlers.HandleStatus()) }) - // Metrics endpoint (optional, with basic auth) + s.router.Mount("/", public) + + // Metrics endpoint (optional, with basic auth) and no CORS: a + // Prometheus scraper is not a browser. It is mounted rather than + // added with Get so that every method on /metrics, OPTIONS + // included, ends here instead of falling through to the public + // router and its CORS. The rate limit comes before Basic Auth, so + // failed logins count against it and a request over the limit + // never reaches the password check. if s.params.Config.MetricsUsername != "" { - s.router.Group(func(r chi.Router) { - r.Use(s.mw.MetricsAuth()) - r.Get( - "/metrics", - promhttp.Handler().ServeHTTP, - ) - }) + metrics := chi.NewRouter() + metrics.Use(s.mw.MetricsRateLimit()) + metrics.Use(s.mw.MetricsAuth()) + metrics.Get("/", promhttp.Handler().ServeHTTP) + s.router.Mount("/metrics", metrics) } } diff --git a/internal/server/routes_test.go b/internal/server/routes_test.go new file mode 100644 index 0000000..5513a9c --- /dev/null +++ b/internal/server/routes_test.go @@ -0,0 +1,330 @@ +package server_test + +import ( + "mime" + "net/http" + "net/http/httptest" + "testing" + + "github.com/spf13/viper" + + "sneak.berlin/go/dnswatcher/internal/server" +) + +// Credentials for /metrics, which is only routed when a username is set. +const ( + metricsUsername = "scraper" + metricsPassword = "scrape-secret" +) + +// The tests below set env vars and touch viper global state, so like +// the config tests they cannot use t.Parallel. + +// routedServer builds the server with its routes set up, ready to serve +// test requests. The caller must first configure viper. +func routedServer(t *testing.T) *server.Server { + t.Helper() + + srv := buildServer(t) + srv.SetupRoutes() + + return srv +} + +// crossOriginRequest builds a request as a browser sends it from a page +// on another site. +func crossOriginRequest( + t *testing.T, + method string, + target string, +) *http.Request { + t.Helper() + + req := httptest.NewRequestWithContext(t.Context(), method, target, nil) + req.Header.Set("Origin", "https://example.net") + + return req +} + +// preflightRequest builds the OPTIONS request a browser sends before a +// cross-origin request with the given method and request headers. +func preflightRequest( + t *testing.T, + target string, + method string, + headers string, +) *http.Request { + t.Helper() + + req := crossOriginRequest(t, http.MethodOptions, target) + req.Header.Set("Access-Control-Request-Method", method) + + if headers != "" { + req.Header.Set("Access-Control-Request-Headers", headers) + } + + return req +} + +func serve( + srv *server.Server, + req *http.Request, +) *httptest.ResponseRecorder { + rec := httptest.NewRecorder() + srv.ServeHTTP(rec, req) + + return rec +} + +// publicPaths returns one path on each public route. +func publicPaths() []string { + return []string{ + "/", + "/s/css/tailwind.min.css", + "/api/v1/status", + "/health", + "/.well-known/healthcheck", + } +} + +// TestPublicRoutesAllowAnyOrigin checks that every public route answers +// a cross-origin GET with the CORS wildcard. +func TestPublicRoutesAllowAnyOrigin(t *testing.T) { + viper.Reset() + t.Setenv("DNSWATCHER_TARGETS", "example.com") + t.Setenv("DNSWATCHER_METRICS_USERNAME", metricsUsername) + t.Setenv("DNSWATCHER_METRICS_PASSWORD", metricsPassword) + + srv := routedServer(t) + + for _, path := range publicPaths() { + rec := serve(srv, crossOriginRequest(t, http.MethodGet, path)) + + if rec.Code != http.StatusOK { + t.Errorf("GET %s: status = %d, want 200", path, rec.Code) + } + + got := rec.Header().Get("Access-Control-Allow-Origin") + if got != "*" { + t.Errorf( + "GET %s: Access-Control-Allow-Origin = %q, want %q", + path, got, "*", + ) + } + } +} + +// TestMetricsHasNoCORS checks that no request to the Basic-Auth +// protected /metrics, preflight included, gets a CORS header. +func TestMetricsHasNoCORS(t *testing.T) { + viper.Reset() + t.Setenv("DNSWATCHER_TARGETS", "example.com") + t.Setenv("DNSWATCHER_METRICS_USERNAME", metricsUsername) + t.Setenv("DNSWATCHER_METRICS_PASSWORD", metricsPassword) + + srv := routedServer(t) + + authenticated := crossOriginRequest(t, http.MethodGet, "/metrics") + authenticated.SetBasicAuth(metricsUsername, metricsPassword) + + tests := []struct { + name string + req *http.Request + wantStatus int + }{ + { + "authenticated GET", + authenticated, + http.StatusOK, + }, + { + "unauthenticated GET", + crossOriginRequest(t, http.MethodGet, "/metrics"), + http.StatusUnauthorized, + }, + { + "preflight", + preflightRequest(t, "/metrics", http.MethodGet, ""), + http.StatusUnauthorized, + }, + } + + for _, tt := range tests { + rec := serve(srv, tt.req) + + if rec.Code != tt.wantStatus { + t.Errorf( + "%s: status = %d, want %d", + tt.name, rec.Code, tt.wantStatus, + ) + } + + got := rec.Header().Get("Access-Control-Allow-Origin") + if got != "" { + t.Errorf( + "%s: Access-Control-Allow-Origin = %q, want none", + tt.name, got, + ) + } + } +} + +// TestPreflightAllowsOnlyWhatPublicRoutesServe checks what each public +// route agrees to in a CORS preflight: GET, but not POST, PUT or +// DELETE, which no route serves, and not the Authorization or +// X-CSRF-Token headers, which no public route reads. It checks every +// public route because one added with Get, such as /health, answers a +// preflight only while CORS is middleware of a whole router; in a +// Group, chi would answer it with 405 and no CORS headers. +func TestPreflightAllowsOnlyWhatPublicRoutesServe(t *testing.T) { + viper.Reset() + t.Setenv("DNSWATCHER_TARGETS", "example.com") + t.Setenv("DNSWATCHER_METRICS_USERNAME", metricsUsername) + t.Setenv("DNSWATCHER_METRICS_PASSWORD", metricsPassword) + + srv := routedServer(t) + + tests := []struct { + method string + headers string + allowed bool + }{ + {http.MethodGet, "", true}, + {http.MethodGet, "Content-Type", true}, + {http.MethodPost, "", false}, + {http.MethodPut, "", false}, + {http.MethodDelete, "", false}, + {http.MethodGet, "Authorization", false}, + {http.MethodGet, "X-CSRF-Token", false}, + } + + for _, path := range publicPaths() { + for _, tt := range tests { + rec := serve(srv, preflightRequest( + t, path, tt.method, tt.headers, + )) + + want := "" + if tt.allowed { + want = tt.method + } + + got := rec.Header().Get("Access-Control-Allow-Methods") + if got != want { + t.Errorf( + "preflight to %s for %s with headers %q: "+ + "Access-Control-Allow-Methods = %q, want %q", + path, tt.method, tt.headers, got, want, + ) + } + } + } +} + +// metricsRequest builds a GET for /metrics from remoteAddr that logs +// in with the given password. +func metricsRequest( + t *testing.T, + remoteAddr string, + password string, +) *http.Request { + t.Helper() + + req := httptest.NewRequestWithContext( + t.Context(), http.MethodGet, "/metrics", nil, + ) + req.RemoteAddr = remoteAddr + req.SetBasicAuth(metricsUsername, password) + + return req +} + +// TestMetricsRateLimitComesBeforeAuth checks that failed logins to +// /metrics count against the rate limit; that once an address is over +// it, even the right password gets 429, with the same body as a wrong +// one; and that another address still gets in. +func TestMetricsRateLimitComesBeforeAuth(t *testing.T) { + viper.Reset() + t.Setenv("DNSWATCHER_TARGETS", "example.com") + t.Setenv("DNSWATCHER_METRICS_USERNAME", metricsUsername) + t.Setenv("DNSWATCHER_METRICS_PASSWORD", metricsPassword) + + const ( + guesser = "198.51.100.1:4000" + other = "198.51.100.2:4000" + + // Far more guesses than the rate limit allows. + maxGuesses = 1000 + ) + + srv := routedServer(t) + + var guess *httptest.ResponseRecorder + + for range maxGuesses { + guess = serve(srv, metricsRequest(t, guesser, "wrong")) + if guess.Code != http.StatusUnauthorized { + break + } + } + + if guess.Code != http.StatusTooManyRequests { + t.Fatalf("wrong password: status = %d, want 429", guess.Code) + } + + right := serve(srv, metricsRequest(t, guesser, metricsPassword)) + if right.Code != http.StatusTooManyRequests { + t.Errorf("right password: status = %d, want 429", right.Code) + } + + if right.Body.String() != guess.Body.String() { + t.Errorf( + "429 body with right password = %q, with wrong one = %q", + right.Body.String(), guess.Body.String(), + ) + } + + rec := serve(srv, metricsRequest(t, other, metricsPassword)) + if rec.Code != http.StatusOK { + t.Errorf("another address: status = %d, want 200", rec.Code) + } +} + +// TestStaticDirectoriesAnswerNotFound checks that a request naming a +// directory of the static assets, with or without the trailing slash, +// answers 404 rather than a listing of it or a redirect to one, and +// that the stylesheet is still served as CSS. +func TestStaticDirectoriesAnswerNotFound(t *testing.T) { + viper.Reset() + t.Setenv("DNSWATCHER_TARGETS", "example.com") + + srv := routedServer(t) + + for _, path := range []string{"/s", "/s/", "/s/css", "/s/css/"} { + rec := serve(srv, httptest.NewRequestWithContext( + t.Context(), http.MethodGet, path, nil, + )) + + if rec.Code != http.StatusNotFound { + t.Errorf("GET %s: status = %d, want 404", path, rec.Code) + } + } + + rec := serve(srv, httptest.NewRequestWithContext( + t.Context(), http.MethodGet, "/s/css/tailwind.min.css", nil, + )) + + if rec.Code != http.StatusOK { + t.Errorf("GET stylesheet: status = %d, want 200", rec.Code) + } + + contentType := rec.Header().Get("Content-Type") + + mediaType, _, err := mime.ParseMediaType(contentType) + if err != nil || mediaType != "text/css" { + t.Errorf( + "GET stylesheet: Content-Type = %q, want text/css", + contentType, + ) + } +} diff --git a/internal/server/sentry_test.go b/internal/server/sentry_test.go new file mode 100644 index 0000000..1525d3f --- /dev/null +++ b/internal/server/sentry_test.go @@ -0,0 +1,190 @@ +package server_test + +import ( + "io" + "net/http" + "net/http/httptest" + "net/url" + "strings" + "sync" + "testing" + "time" + + "github.com/getsentry/sentry-go" + "github.com/spf13/viper" + "go.uber.org/fx" + + "sneak.berlin/go/dnswatcher/internal/server" +) + +// The tests below set env vars and touch the global state of viper and +// of Sentry, so they cannot use t.Parallel. + +// standInDelay is how long the Sentry stand-in takes to answer. It +// records a report only then, so a report that Shutdown did not wait +// for has not been recorded yet when Shutdown returns. +const standInDelay = 100 * time.Millisecond + +// sentryStandIn is a local HTTP server in place of Sentry's, so that +// nothing a test reports leaves the host. It keeps the body of every +// request it receives. +type sentryStandIn struct { + server *httptest.Server + mu sync.Mutex + bodies []string +} + +func newSentryStandIn(t *testing.T) *sentryStandIn { + t.Helper() + + standIn := &sentryStandIn{} + standIn.server = httptest.NewServer(http.HandlerFunc( + func(_ http.ResponseWriter, r *http.Request) { + body, err := io.ReadAll(r.Body) + if err != nil { + t.Errorf("reading request to the Sentry stand-in: %v", err) + } + + time.Sleep(standInDelay) + + standIn.mu.Lock() + defer standIn.mu.Unlock() + + standIn.bodies = append(standIn.bodies, string(body)) + }, + )) + t.Cleanup(standIn.server.Close) + + return standIn +} + +// dsn returns a DSN that points Sentry at the stand-in. +func (s *sentryStandIn) dsn(t *testing.T) string { + t.Helper() + + dsn, err := url.Parse(s.server.URL) + if err != nil { + t.Fatalf("parsing the stand-in URL: %v", err) + } + + dsn.User = url.User("public-key") + dsn.Path = "/1" + + return dsn.String() +} + +// received reports whether a request to the stand-in contained text. +func (s *sentryStandIn) received(text string) bool { + s.mu.Lock() + defer s.mu.Unlock() + + for _, body := range s.bodies { + if strings.Contains(body, text) { + return true + } + } + + return false +} + +func TestSentryUnsetDoesNothing(t *testing.T) { + viper.Reset() + t.Setenv("DNSWATCHER_TARGETS", "example.com") + t.Setenv("DNSWATCHER_SENTRY_DSN", "") + + srv := buildServer(t) + + err := server.EnableSentry(srv) + if err != nil { + t.Fatalf("Sentry setup with no DSN: %v", err) + } + + if sentry.CurrentHub().Client() != nil { + t.Error("Sentry was set up with no DSN configured") + } +} + +// TestSentryReportsHandlerPanic checks that with a valid DSN a panic in +// a handler is reported to Sentry, still reaches chimw.Recoverer, and +// has been sent by the time Shutdown returns, and that nothing else is +// sent to Sentry. +func TestSentryReportsHandlerPanic(t *testing.T) { + standIn := newSentryStandIn(t) + + viper.Reset() + t.Setenv("DNSWATCHER_TARGETS", "example.com") + t.Setenv("DNSWATCHER_SENTRY_DSN", standIn.dsn(t)) + + srv := buildServer(t) + + err := server.EnableSentry(srv) + if err != nil { + t.Fatalf("Sentry setup with a valid DSN: %v", err) + } + + // Sentry's client is global: close it so later tests find none. + t.Cleanup(func() { + sentry.CurrentHub().Client().Close() + sentry.CurrentHub().BindClient(nil) + }) + + const panicMessage = "handler panic in the Sentry test" + + srv.SetupRoutes() + server.RouterOf(srv).Get( + "/panic", + func(http.ResponseWriter, *http.Request) { + panic(panicMessage) + }, + ) + + // An ordinary request first: with client reports on, Sentry would + // add a count of its dropped transaction to the panic report. + serve(srv, httptest.NewRequestWithContext( + t.Context(), http.MethodGet, "/.well-known/healthcheck", nil, + )) + + rec := serve(srv, httptest.NewRequestWithContext( + t.Context(), http.MethodGet, "/panic", nil, + )) + if rec.Code != http.StatusInternalServerError { + t.Errorf( + "status %d, want %d from the recoverer", + rec.Code, http.StatusInternalServerError, + ) + } + + err = srv.Shutdown(t.Context()) + if err != nil { + t.Fatalf("Shutdown: %v", err) + } + + if !standIn.received(panicMessage) { + t.Error("the panic had not been sent to Sentry when Shutdown returned") + } + + if standIn.received("client_report") { + t.Error("Sentry was sent a client report, not only the panic") + } +} + +func TestSentryInvalidDSNStopsStartup(t *testing.T) { + viper.Reset() + t.Setenv("DNSWATCHER_TARGETS", "example.com") + t.Setenv("DNSWATCHER_DATA_DIR", t.TempDir()) + // Sentry cannot parse this: it has no public key before the host. + t.Setenv("DNSWATCHER_SENTRY_DSN", "https://sentry.test/1") + + app := newServerApp(fx.Invoke(func(*server.Server) {})) + + err := app.Start(t.Context()) + if err == nil { + _ = app.Stop(t.Context()) + + t.Fatal("startup succeeded with an invalid DSN") + } + + if !strings.Contains(err.Error(), "invalid DNSWATCHER_SENTRY_DSN") { + t.Errorf("startup error does not name the setting: %v", err) + } +} diff --git a/internal/server/server.go b/internal/server/server.go index 4cb30a2..7f333bf 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -6,9 +6,11 @@ import ( "errors" "fmt" "log/slog" + "net" "net/http" "time" + "github.com/getsentry/sentry-go" "github.com/go-chi/chi/v5" "go.uber.org/fx" @@ -33,19 +35,68 @@ type Params struct { // shutdownTimeout is how long to wait for graceful shutdown. const shutdownTimeout = 30 * time.Second -// readHeaderTimeout is the max duration for reading request headers. -const readHeaderTimeout = 10 * time.Second +// sentryFlushTimeout is how long shutdown waits for Sentry to send the +// error reports it still holds. +const sentryFlushTimeout = 2 * time.Second + +// Socket-level timeouts for the HTTP server. +// +// These bound time spent on the connection itself and are a distinct +// control from the per-request handler budget enforced by +// chimw.Timeout(requestTimeout) in routes.go: that one cancels the +// request context after requestTimeout but never touches the socket, +// so without the values below a peer can hold a connection open +// forever (slowloris, unreaped keep-alives). +// +// The one hard constraint between the two controls is +// writeTimeout > requestTimeout. net/http arms the write deadline +// once the request headers have been read, so on a plaintext +// connection it covers handler execution AND the response flush. If +// writeTimeout were <= requestTimeout the server would sever the +// connection before a handler that legitimately consumed its full +// budget could emit anything, making the 60s budget unreachable in +// practice. The margin between them is the response-flush allowance. +// +// The only clients of this service are browsers loading the dashboard +// and a Prometheus scraper; the values are sized for those. +const ( + // readHeaderTimeout is the max duration for reading request + // headers. + readHeaderTimeout = 10 * time.Second + + // readTimeout bounds reading the entire request, headers plus + // body. Every route here is a GET with no body, so this only + // ever needs to cover headers; the extra 5s over + // readHeaderTimeout is slack, not a real allowance, and keeps a + // body dribbled one byte at a time from holding the read side + // open indefinitely. + readTimeout = 15 * time.Second + + // writeTimeout must exceed the requestTimeout handler budget + // (60s) per the note above. The 15s difference is the allowance + // for flushing a completed response to a slow client. + writeTimeout = 75 * time.Second + + // idleTimeout reaps keep-alive connections between requests. It + // is deliberately longer than the common Prometheus scrape + // intervals (15s/30s/60s) so the scraper reuses its connection + // rather than reconnecting every cycle, while a browser tab + // left open on the dashboard stops occupying a connection + // within two minutes of going quiet. + idleTimeout = 120 * time.Second +) // Server is the HTTP server. type Server struct { - startupTime time.Time - port int - log *slog.Logger - router *chi.Mux - httpServer *http.Server - params Params - mw *middleware.Middleware - handlers *handlers.Handlers + startupTime time.Time + port int + sentryEnabled bool + log *slog.Logger + router *chi.Mux + httpServer *http.Server + params Params + mw *middleware.Middleware + handlers *handlers.Handlers } // New creates a new Server instance. @@ -62,11 +113,15 @@ func New( } lifecycle.Append(fx.Hook{ - OnStart: func(_ context.Context) error { + OnStart: func(ctx context.Context) error { srv.startupTime = time.Now() - go srv.Run() - return nil + err := srv.enableSentry() + if err != nil { + return err + } + + return srv.Run(ctx) }, OnStop: func(ctx context.Context) error { return srv.Shutdown(ctx) @@ -76,27 +131,56 @@ func New( return srv, nil } -// Run starts the HTTP server. -func (s *Server) Run() { - s.SetupRoutes() - - listenAddr := fmt.Sprintf(":%d", s.port) - s.httpServer = &http.Server{ +// newHTTPServer builds the listening http.Server with every +// socket-level timeout set. All four are set deliberately: a zero +// value in net/http means "no limit", not "some default". +func newHTTPServer( + listenAddr string, + handler http.Handler, +) *http.Server { + return &http.Server{ Addr: listenAddr, - Handler: s, + Handler: handler, + ReadTimeout: readTimeout, ReadHeaderTimeout: readHeaderTimeout, - } - - s.log.Info("http server starting", "addr", listenAddr) - - err := s.httpServer.ListenAndServe() - if err != nil && !errors.Is(err, http.ErrServerClosed) { - s.log.Error("http server error", "error", err) + WriteTimeout: writeTimeout, + IdleTimeout: idleTimeout, } } -// Shutdown gracefully shuts down the server. +// Run binds the listen port, then serves HTTP on it in a new goroutine +// and returns. A port that cannot be bound is returned as an error, so +// that startup stops instead of running without the HTTP server. +func (s *Server) Run(ctx context.Context) error { + s.SetupRoutes() + + listenAddr := fmt.Sprintf(":%d", s.port) + s.httpServer = newHTTPServer(listenAddr, s) + + s.log.Info("http server starting", "addr", listenAddr) + + lc := &net.ListenConfig{} + + listener, err := lc.Listen(ctx, "tcp", listenAddr) + if err != nil { + return fmt.Errorf("starting http server: %w", err) + } + + go func() { + err := s.httpServer.Serve(listener) + if err != nil && !errors.Is(err, http.ErrServerClosed) { + s.log.Error("http server error", "error", err) + } + }() + + return nil +} + +// Shutdown gracefully shuts down the server, then sends the error +// reports Sentry still holds. func (s *Server) Shutdown(ctx context.Context) error { + defer s.flushSentry() + if s.httpServer == nil { return nil } @@ -127,3 +211,45 @@ func (s *Server) ServeHTTP( ) { s.router.ServeHTTP(writer, request) } + +// enableSentry turns on Sentry error reporting when +// DNSWATCHER_SENTRY_DSN is set, and does nothing when it is not. A DSN +// that Sentry cannot parse is an error, so that startup stops instead +// of running without the error reporting the operator asked for. +func (s *Server) enableSentry() error { + if s.params.Config.SentryDSN == "" { + return nil + } + + err := sentry.Init(sentry.ClientOptions{ + Dsn: s.params.Config.SentryDSN, + Release: s.params.Globals.Appname + "-" + s.params.Globals.Version, + // Use the transport that queues each report as it is made. With + // the default one, Flush can return before sending a report made + // just before it, such as one from the last request at shutdown. + DisableTelemetryBuffer: true, + // Send panic reports only, not Sentry's counts of what it dropped, + // such as the transaction it starts for every request. + DisableClientReports: true, + }) + if err != nil { + return fmt.Errorf("invalid DNSWATCHER_SENTRY_DSN: %w", err) + } + + s.log.Info("sentry error reporting activated") + s.sentryEnabled = true + + return nil +} + +// flushSentry sends the error reports Sentry still holds, waiting at +// most sentryFlushTimeout. +func (s *Server) flushSentry() { + if !s.sentryEnabled { + return + } + + if !sentry.Flush(sentryFlushTimeout) { + s.log.Warn("sentry flush timed out; some error reports were not sent") + } +} diff --git a/internal/server/server_test.go b/internal/server/server_test.go new file mode 100644 index 0000000..297d505 --- /dev/null +++ b/internal/server/server_test.go @@ -0,0 +1,173 @@ +package server_test + +import ( + "net" + "strings" + "testing" + + "github.com/spf13/viper" + "go.uber.org/fx" + + "sneak.berlin/go/dnswatcher/internal/config" + "sneak.berlin/go/dnswatcher/internal/globals" + "sneak.berlin/go/dnswatcher/internal/handlers" + "sneak.berlin/go/dnswatcher/internal/healthcheck" + "sneak.berlin/go/dnswatcher/internal/logger" + "sneak.berlin/go/dnswatcher/internal/middleware" + "sneak.berlin/go/dnswatcher/internal/notify" + "sneak.berlin/go/dnswatcher/internal/server" + "sneak.berlin/go/dnswatcher/internal/state" +) + +// newServerApp builds an fx app holding a *server.Server wired exactly +// as cmd/dnswatcher wires it, minus the watcher/resolver subtree that +// would touch live DNS, plus the given option. config.New reads viper, +// so the caller must first configure it, which is also why the caller +// cannot run in parallel. +func newServerApp(option fx.Option) *fx.App { + return fx.New( + fx.NopLogger, + fx.Provide( + globals.New, + logger.New, + config.New, + state.New, + healthcheck.New, + notify.New, + middleware.New, + handlers.New, + server.New, + ), + option, + ) +} + +// buildServer builds the server without starting the app's lifecycle, +// so no OnStart hook runs and nothing listens or resolves. +func buildServer(t *testing.T) *server.Server { + t.Helper() + + var srv *server.Server + + app := newServerApp(fx.Populate(&srv)) + + err := app.Err() + if err != nil { + t.Fatalf("building server graph: %v", err) + } + + return srv +} + +// TestRunWiresSocketTimeouts pins that the http.Server the running +// server actually serves — the one Run builds and serves on the port +// it binds — carries every socket-level timeout, plus the two +// relationships the values must satisfy. +// +// Run is given an unbindable port: it builds and stores s.httpServer, +// then the bind fails and Run returns the error without ever +// listening. Nothing here measures elapsed time. +// +// ReadTimeout must be at least ReadHeaderTimeout. net/http reads the +// headers under ReadHeaderTimeout, then sets the read deadline for the +// rest of the request to ReadTimeout, counted from when it started +// reading the request. If ReadTimeout were smaller, a request whose +// headers arrived after ReadTimeout but within ReadHeaderTimeout would +// get a read deadline that had already passed, so reading its body +// would fail at once. +func TestRunWiresSocketTimeouts(t *testing.T) { + // Sets an env var and touches viper global state, so like the + // config tests it cannot use t.Parallel. + viper.Reset() + t.Setenv("DNSWATCHER_TARGETS", "example.com") + + srv := buildServer(t) + server.SetListenPort(srv, -1) + + err := srv.Run(t.Context()) + if err == nil { + t.Fatal("Run bound port -1") + } + + hs := server.HTTPServerOf(srv) + if hs == nil { + t.Fatal("Run did not build an http.Server") + } + + if hs.ReadTimeout <= 0 { + t.Errorf("ReadTimeout must be non-zero, got %v", hs.ReadTimeout) + } + + if hs.ReadHeaderTimeout <= 0 { + t.Errorf( + "ReadHeaderTimeout must be non-zero, got %v", + hs.ReadHeaderTimeout, + ) + } + + if hs.WriteTimeout <= 0 { + t.Errorf("WriteTimeout must be non-zero, got %v", hs.WriteTimeout) + } + + if hs.IdleTimeout <= 0 { + t.Errorf("IdleTimeout must be non-zero, got %v", hs.IdleTimeout) + } + + if hs.WriteTimeout <= server.RequestTimeout { + t.Errorf( + "WriteTimeout (%v) must exceed handler budget (%v)", + hs.WriteTimeout, + server.RequestTimeout, + ) + } + + if hs.ReadTimeout < hs.ReadHeaderTimeout { + t.Errorf( + "ReadTimeout (%v) must be >= ReadHeaderTimeout (%v)", + hs.ReadTimeout, + hs.ReadHeaderTimeout, + ) + } + + if hs.Handler != srv { + t.Errorf( + "Run wired handler %T, want the *server.Server", + hs.Handler, + ) + } +} + +func TestPortInUseStopsStartup(t *testing.T) { + // Bound on every address, as Run binds, so that Run's bind fails. + lc := &net.ListenConfig{} + + taken, err := lc.Listen(t.Context(), "tcp", ":0") + if err != nil { + t.Fatalf("binding a port for the test: %v", err) + } + + defer func() { _ = taken.Close() }() + + _, port, err := net.SplitHostPort(taken.Addr().String()) + if err != nil { + t.Fatalf("reading the bound port: %v", err) + } + + viper.Reset() + t.Setenv("DNSWATCHER_TARGETS", "example.com") + t.Setenv("DNSWATCHER_DATA_DIR", t.TempDir()) + t.Setenv("PORT", port) + + app := newServerApp(fx.Invoke(func(*server.Server) {})) + + err = app.Start(t.Context()) + if err == nil { + _ = app.Stop(t.Context()) + + t.Fatal("startup succeeded on a port that is already bound") + } + + if !strings.Contains(err.Error(), ":"+port) { + t.Errorf("startup error does not name port %s: %v", port, err) + } +} diff --git a/internal/state/export_test.go b/internal/state/export_test.go new file mode 100644 index 0000000..8defb0c --- /dev/null +++ b/internal/state/export_test.go @@ -0,0 +1,23 @@ +package state + +import ( + "log/slog" + + "sneak.berlin/go/dnswatcher/internal/config" +) + +// NewForTestWithDataDir creates an empty State that saves to dataDir, +// without the fx lifecycle. +func NewForTestWithDataDir(dataDir string) *State { + return &State{ + log: slog.Default(), + snapshot: &Snapshot{ + Version: stateVersion, + Domains: make(map[string]*DomainState), + Hostnames: make(map[string]*HostnameState), + Ports: make(map[string]*PortState), + Certificates: make(map[string]*CertificateState), + }, + config: &config.Config{DataDir: dataDir}, + } +} diff --git a/internal/state/state.go b/internal/state/state.go index efe681c..744f4e8 100644 --- a/internal/state/state.go +++ b/internal/state/state.go @@ -4,10 +4,13 @@ package state import ( "context" "encoding/json" + "errors" "fmt" "log/slog" + "maps" "os" "path/filepath" + "slices" "sync" "time" @@ -35,22 +38,41 @@ type Params struct { } // DomainState holds the monitoring state for an apex domain. +// NameserverAddresses holds the sorted addresses each nameserver's name +// resolves to, by nameserver name. A state file written before it +// existed loads with it nil. NXDomain is true when the domain's parent +// zone's servers answered that it does not exist; it then has no +// nameservers. type DomainState struct { - Nameservers []string `json:"nameservers"` - LastChecked time.Time `json:"lastChecked"` + Nameservers []string `json:"nameservers"` + NameserverAddresses map[string][]string `json:"nameserverAddresses"` + NXDomain bool `json:"nxdomain,omitempty"` + LastChecked time.Time `json:"lastChecked"` } // NameserverRecordState holds one NS's response for a hostname. +// FailedTypes lists the record types whose query to the nameserver +// failed on this check: Records holds for them the records saved by the +// previous check, which are kept. UnknownTypes lists those of them whose +// records the previous check did not know either, as when the +// nameserver was new or failing then: Records holds nothing for them. type NameserverRecordState struct { - Records map[string][]string `json:"records"` - Status string `json:"status"` - Error string `json:"error,omitempty"` - LastChecked time.Time `json:"lastChecked"` + Records map[string][]string `json:"records"` + FailedTypes []string `json:"failedTypes,omitempty"` + UnknownTypes []string `json:"unknownTypes,omitempty"` + Status string `json:"status"` + Error string `json:"error,omitempty"` + LastChecked time.Time `json:"lastChecked"` } // HostnameState holds per-nameserver monitoring state for a hostname. +// CNAMEAddresses holds the sorted addresses at the end of the name's +// CNAME chain, found when its nameservers answered with a CNAME and no +// address; it is empty otherwise. It is nil when they are not known: a +// state file written before it existed loads with it nil. type HostnameState struct { RecordsByNameserver map[string]*NameserverRecordState `json:"recordsByNameserver"` + CNAMEAddresses []string `json:"cnameAddresses"` LastChecked time.Time `json:"lastChecked"` } @@ -112,6 +134,8 @@ type CertificateState struct { } // Snapshot is the complete monitoring state persisted to disk. +// Hostnames also holds each apex domain's own records, under the +// domain's name, which has an entry in Domains too. type Snapshot struct { Version int `json:"version"` LastUpdated time.Time `json:"lastUpdated"` @@ -148,6 +172,11 @@ func New( lifecycle.Append(fx.Hook{ OnStart: func(_ context.Context) error { + err := state.checkDataDirWritable() + if err != nil { + return err + } + return state.Load() }, OnStop: func(_ context.Context) error { @@ -187,6 +216,37 @@ func (s *State) Load() error { return fmt.Errorf("parsing state file: %w", err) } + // A section the file leaves out, or holds as null, loads empty: the + // setters write into these maps, and writing into a nil map panics. + if snapshot.Domains == nil { + snapshot.Domains = make(map[string]*DomainState) + } + + if snapshot.Hostnames == nil { + snapshot.Hostnames = make(map[string]*HostnameState) + } + + if snapshot.Ports == nil { + snapshot.Ports = make(map[string]*PortState) + } + + if snapshot.Certificates == nil { + snapshot.Certificates = make(map[string]*CertificateState) + } + + // A state file saved before each record value was stored once can + // hold a hostname's CNAME once for every record type asked for. + // Each value is kept once, so the first check does not see a + // record change. + for _, hs := range snapshot.Hostnames { + for _, ns := range hs.RecordsByNameserver { + for recordType, values := range ns.Records { + slices.Sort(values) + ns.Records[recordType] = slices.Compact(values) + } + } + } + s.snapshot = &snapshot s.log.Info("loaded state from disk", "path", path) @@ -212,10 +272,20 @@ func (s *State) Save() error { return fmt.Errorf("creating data directory: %w", err) } - // Atomic write: write to temp file, then rename + // Atomic write: write to temp file, then rename. The temp file is + // synced before the rename and the directory after it, or a crash + // can leave the renamed state file empty. tmpPath := path + ".tmp" - err = os.WriteFile(tmpPath, data, filePermissions) + //nolint:gosec // path is from trusted config + tmpFile, err := os.OpenFile( + tmpPath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, filePermissions, + ) + if err == nil { + _, err = tmpFile.Write(data) + err = errors.Join(err, tmpFile.Sync(), tmpFile.Close()) + } + if err != nil { return fmt.Errorf("writing temp state file: %w", err) } @@ -225,17 +295,35 @@ func (s *State) Save() error { return fmt.Errorf("renaming state file: %w", err) } + dir, err := os.Open(filepath.Dir(path)) + if err == nil { + err = errors.Join(dir.Sync(), dir.Close()) + } + + if err != nil { + return fmt.Errorf("syncing data directory: %w", err) + } + s.log.Debug("state saved to disk", "path", path) return nil } -// GetSnapshot returns a copy of the current snapshot. +// GetSnapshot returns a copy of the current snapshot, with copies of +// its maps, so that a caller can range over them while the state is +// written. The entries in the maps are not copied: an entry saved in the +// state must be replaced, never changed in place. func (s *State) GetSnapshot() Snapshot { s.mu.RLock() defer s.mu.RUnlock() - return *s.snapshot + snap := *s.snapshot + snap.Domains = maps.Clone(snap.Domains) + snap.Hostnames = maps.Clone(snap.Hostnames) + snap.Ports = maps.Clone(snap.Ports) + snap.Certificates = maps.Clone(snap.Certificates) + + return snap } // SetDomainState updates the state for a domain. @@ -261,6 +349,27 @@ func (s *State) GetDomainState( return ds, ok } +// DeleteDomainState removes a domain state entry. +func (s *State) DeleteDomainState(domain string) { + s.mu.Lock() + defer s.mu.Unlock() + + delete(s.snapshot.Domains, domain) +} + +// GetAllDomainNames returns the names of all domain state entries. +func (s *State) GetAllDomainNames() []string { + s.mu.RLock() + defer s.mu.RUnlock() + + names := make([]string, 0, len(s.snapshot.Domains)) + for name := range s.snapshot.Domains { + names = append(names, name) + } + + return names +} + // SetHostnameState updates the state for a hostname. func (s *State) SetHostnameState( hostname string, @@ -284,6 +393,28 @@ func (s *State) GetHostnameState( return hs, ok } +// DeleteHostnameState removes a hostname state entry. +func (s *State) DeleteHostnameState(hostname string) { + s.mu.Lock() + defer s.mu.Unlock() + + delete(s.snapshot.Hostnames, hostname) +} + +// GetAllHostnames returns the names of all hostname state entries, +// which include each apex domain's own records. +func (s *State) GetAllHostnames() []string { + s.mu.RLock() + defer s.mu.RUnlock() + + names := make([]string, 0, len(s.snapshot.Hostnames)) + for name := range s.snapshot.Hostnames { + names = append(names, name) + } + + return names +} + // SetPortState updates the state for a port. func (s *State) SetPortState(key string, ps *PortState) { s.mu.Lock() @@ -345,3 +476,48 @@ func (s *State) GetCertificateState( return cs, ok } + +// DeleteCertificateState removes a certificate state entry. +func (s *State) DeleteCertificateState(key string) { + s.mu.Lock() + defer s.mu.Unlock() + + delete(s.snapshot.Certificates, key) +} + +// GetAllCertificateKeys returns all certificate state keys. +func (s *State) GetAllCertificateKeys() []string { + s.mu.RLock() + defer s.mu.RUnlock() + + keys := make([]string, 0, len(s.snapshot.Certificates)) + for k := range s.snapshot.Certificates { + keys = append(keys, k) + } + + return keys +} + +// checkDataDirWritable creates the data directory if needed, then writes +// and removes the temp file that Save uses. It runs at startup so that an +// unwritable directory stops the process, instead of the process running +// with every save failing and only logged. +func (s *State) checkDataDirWritable() error { + dir := s.config.DataDir + tmpPath := s.config.StatePath() + ".tmp" + + err := os.MkdirAll(dir, dirPermissions) + if err == nil { + err = os.WriteFile(tmpPath, nil, filePermissions) + } + + if err == nil { + err = os.Remove(tmpPath) + } + + if err != nil { + return fmt.Errorf("data directory %s is not writable: %w", dir, err) + } + + return nil +} diff --git a/internal/state/state_test.go b/internal/state/state_test.go index 699d17c..9b3d998 100644 --- a/internal/state/state_test.go +++ b/internal/state/state_test.go @@ -4,10 +4,17 @@ import ( "encoding/json" "os" "path/filepath" + "reflect" + "strings" "sync" "testing" "time" + "go.uber.org/fx/fxtest" + + "sneak.berlin/go/dnswatcher/internal/config" + "sneak.berlin/go/dnswatcher/internal/globals" + "sneak.berlin/go/dnswatcher/internal/logger" "sneak.berlin/go/dnswatcher/internal/state" ) @@ -31,6 +38,10 @@ func populateState(t *testing.T, s *state.State) { s.SetDomainState("example.com", &state.DomainState{ Nameservers: []string{testNS1, testNS2}, + NameserverAddresses: map[string][]string{ + testNS1: {testIP, testIPv4}, + testNS2: {testIPv4}, + }, LastChecked: now, }) @@ -117,6 +128,261 @@ func TestSaveLoadRoundTrip_Domains(t *testing.T) { if len(dom.Nameservers) != 2 { t.Errorf("expected 2 nameservers, got %d", len(dom.Nameservers)) } + + want := map[string][]string{ + testNS1: {testIP, testIPv4}, + testNS2: {testIPv4}, + } + if !reflect.DeepEqual(dom.NameserverAddresses, want) { + t.Errorf( + "nameserver addresses: got %v, want %v", + dom.NameserverAddresses, want, + ) + } +} + +// TestLoadStateFromBeforeNameserverAddresses loads a state file written +// before nameserver addresses were saved. +func TestLoadStateFromBeforeNameserverAddresses(t *testing.T) { + t.Parallel() + + dir := t.TempDir() + + data := []byte(`{ + "version": 1, + "lastUpdated": "2026-02-19T12:00:00Z", + "domains": { + "example.com": { + "nameservers": ["ns1.example.com.", "ns2.example.com."], + "lastChecked": "2026-02-19T12:00:00Z" + } + } + }`) + + err := os.WriteFile(filepath.Join(dir, "state.json"), data, 0o600) + if err != nil { + t.Fatalf("writing state file: %v", err) + } + + s := state.NewForTestWithDataDir(dir) + + err = s.Load() + if err != nil { + t.Fatalf("Load() error: %v", err) + } + + dom, ok := s.GetDomainState("example.com") + if !ok { + t.Fatal("missing domain example.com") + } + + if !reflect.DeepEqual(dom.Nameservers, []string{testNS1, testNS2}) { + t.Errorf("nameservers: got %v", dom.Nameservers) + } + + if dom.NameserverAddresses != nil { + t.Errorf( + "nameserver addresses: got %v, want none", + dom.NameserverAddresses, + ) + } +} + +// TestSaveLoadRoundTrip_CNAMEAddresses checks that no addresses at the +// end of a hostname's CNAME chain load as an empty list, and addresses +// that are not known load as nil: the watcher tells the two apart. +func TestSaveLoadRoundTrip_CNAMEAddresses(t *testing.T) { + t.Parallel() + + dir := t.TempDir() + s := state.NewForTestWithDataDir(dir) + + want := map[string][]string{ + "cname.example.com": {testIP}, + "none.example.com": {}, + "not-known.example.com": nil, + } + + for name, addresses := range want { + s.SetHostnameState(name, &state.HostnameState{ + CNAMEAddresses: addresses, + }) + } + + err := s.Save() + if err != nil { + t.Fatalf("Save() error: %v", err) + } + + loaded := state.NewForTestWithDataDir(dir) + + err = loaded.Load() + if err != nil { + t.Fatalf("Load() error: %v", err) + } + + for name, addresses := range want { + hs, ok := loaded.GetHostnameState(name) + if !ok { + t.Fatalf("missing hostname %s", name) + } + + if !reflect.DeepEqual(hs.CNAMEAddresses, addresses) { + t.Errorf( + "%s: loaded %#v, want %#v", + name, hs.CNAMEAddresses, addresses, + ) + } + } +} + +// TestSaveLoadRoundTrip_FailedTypes checks that a nameserver's +// failedTypes and unknownTypes survive a save and load. Without +// unknownTypes, a type whose records were not known would load as one +// with no records. +func TestSaveLoadRoundTrip_FailedTypes(t *testing.T) { + t.Parallel() + + dir := t.TempDir() + s := state.NewForTestWithDataDir(dir) + + failed := []string{"TXT", "CAA"} + unknown := []string{"CAA"} + + s.SetHostnameState(testHostname, &state.HostnameState{ + RecordsByNameserver: map[string]*state.NameserverRecordState{ + testNS1: { + Records: map[string][]string{"TXT": {"v=spf1 -all"}}, + FailedTypes: failed, + UnknownTypes: unknown, + Status: "ok", + }, + }, + }) + + err := s.Save() + if err != nil { + t.Fatalf("Save() error: %v", err) + } + + loaded := state.NewForTestWithDataDir(dir) + + err = loaded.Load() + if err != nil { + t.Fatalf("Load() error: %v", err) + } + + hs, ok := loaded.GetHostnameState(testHostname) + if !ok { + t.Fatal("missing hostname " + testHostname) + } + + ns1 := hs.RecordsByNameserver[testNS1] + if ns1 == nil { + t.Fatal("missing nameserver " + testNS1) + } + + if !reflect.DeepEqual(ns1.FailedTypes, failed) { + t.Errorf("failedTypes: got %#v", ns1.FailedTypes) + } + + if !reflect.DeepEqual(ns1.UnknownTypes, unknown) { + t.Errorf("unknownTypes: got %#v", ns1.UnknownTypes) + } +} + +// TestLoadStateFromBeforeCNAMEAddresses loads a state file written +// before the addresses at the end of a hostname's CNAME chain were +// saved. They load as not known (nil), not as none. +func TestLoadStateFromBeforeCNAMEAddresses(t *testing.T) { + t.Parallel() + + dir := t.TempDir() + + data := []byte(`{ + "version": 1, + "lastUpdated": "2026-02-19T12:00:00Z", + "hostnames": { + "www.example.com": { + "recordsByNameserver": {}, + "lastChecked": "2026-02-19T12:00:00Z" + } + } + }`) + + err := os.WriteFile(filepath.Join(dir, "state.json"), data, 0o600) + if err != nil { + t.Fatalf("writing state file: %v", err) + } + + s := state.NewForTestWithDataDir(dir) + + err = s.Load() + if err != nil { + t.Fatalf("Load() error: %v", err) + } + + hs, ok := s.GetHostnameState(testHostname) + if !ok { + t.Fatal("missing hostname " + testHostname) + } + + if hs.CNAMEAddresses != nil { + t.Errorf("CNAME addresses: got %#v, want nil", hs.CNAMEAddresses) + } +} + +// TestLoadStateWithRepeatedValues loads a state file saved when a +// hostname's CNAME was stored once for every record type asked for. +// Each value must load once, and every different value must load. +func TestLoadStateWithRepeatedValues(t *testing.T) { + t.Parallel() + + dir := t.TempDir() + + data := []byte(`{ + "version": 1, + "hostnames": { + "www.example.com": { + "recordsByNameserver": { + "ns1.example.com.": { + "records": { + "A": ["192.0.2.2", "192.0.2.1", "192.0.2.2", "192.0.2.1"], + "CNAME": ["a.example.net.", "a.example.net.", "a.example.net."] + }, + "status": "ok" + } + } + } + } + }`) + + err := os.WriteFile(filepath.Join(dir, "state.json"), data, 0o600) + if err != nil { + t.Fatalf("writing state file: %v", err) + } + + s := state.NewForTestWithDataDir(dir) + + err = s.Load() + if err != nil { + t.Fatalf("Load() error: %v", err) + } + + hs, ok := s.GetHostnameState(testHostname) + if !ok { + t.Fatal("missing hostname " + testHostname) + } + + want := map[string][]string{ + "A": {"192.0.2.1", "192.0.2.2"}, + "CNAME": {"a.example.net."}, + } + + got := hs.RecordsByNameserver[testNS1].Records + if !reflect.DeepEqual(got, want) { + t.Errorf("records: got %v, want %v", got, want) + } } // TestSaveLoadRoundTrip_Hostnames verifies hostname data survives a save/load cycle. @@ -352,6 +618,53 @@ func TestSaveAtomicWrite(t *testing.T) { } } +// TestSaveOverLeftoverTempFile verifies that a temp file left by a +// crash, longer than the state Save writes, leaves nothing of itself in +// the state file, and that no temp file remains after Save. +func TestSaveOverLeftoverTempFile(t *testing.T) { + t.Parallel() + + dir := t.TempDir() + statePath := filepath.Join(dir, "state.json") + tmpPath := statePath + ".tmp" + + err := os.WriteFile(tmpPath, []byte(strings.Repeat("x", 64*1024)), 0o600) + if err != nil { + t.Fatalf("writing leftover temp file: %v", err) + } + + s := state.NewForTestWithDataDir(dir) + + populateState(t, s) + + err = s.Save() + if err != nil { + t.Fatalf("Save() error: %v", err) + } + + //nolint:gosec // test file with known path + data, err := os.ReadFile(statePath) + if err != nil { + t.Fatalf("reading state file: %v", err) + } + + var snap state.Snapshot + + err = json.Unmarshal(data, &snap) + if err != nil { + t.Fatalf("state file is not complete JSON: %v", err) + } + + if len(snap.Domains) != 2 { + t.Errorf("state file has %d domains, want 2", len(snap.Domains)) + } + + _, err = os.Stat(tmpPath) + if !os.IsNotExist(err) { + t.Error("temp file should not exist after save") + } +} + // TestLoadMissingFile verifies that loading a nonexistent file is not an error. func TestLoadMissingFile(t *testing.T) { t.Parallel() @@ -382,6 +695,58 @@ func TestLoadMissingFile(t *testing.T) { } } +// TestLoadFileWithoutSections loads a state file holding {}, and files +// that each leave out one section. Every section loads as an empty map. +// TestFirstCheckAfterLoadingFileWithoutSections in the watcher tests +// runs the first check after such a load. +func TestLoadFileWithoutSections(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + file string + }{ + {"no sections", `{}`}, + {"no domains", `{"hostnames": {}, "ports": {}, "certificates": {}}`}, + {"no hostnames", `{"domains": {}, "ports": {}, "certificates": {}}`}, + {"no ports", `{"domains": {}, "hostnames": {}, "certificates": {}}`}, + {"no certificates", `{"domains": {}, "hostnames": {}, "ports": {}}`}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + dir := t.TempDir() + + err := os.WriteFile( + filepath.Join(dir, "state.json"), []byte(tt.file), 0o600, + ) + if err != nil { + t.Fatalf("writing state file: %v", err) + } + + s := state.NewForTestWithDataDir(dir) + + err = s.Load() + if err != nil { + t.Fatalf("Load() error: %v", err) + } + + want := state.Snapshot{ + Domains: map[string]*state.DomainState{}, + Hostnames: map[string]*state.HostnameState{}, + Ports: map[string]*state.PortState{}, + Certificates: map[string]*state.CertificateState{}, + } + + if got := s.GetSnapshot(); !reflect.DeepEqual(got, want) { + t.Errorf("loaded %#v, want %#v", got, want) + } + }) + } +} + // TestLoadCorruptFile verifies that a corrupt state file returns an error. func TestLoadCorruptFile(t *testing.T) { t.Parallel() @@ -493,6 +858,107 @@ func TestSaveWritePermissionError(t *testing.T) { } } +// startState builds a State through the real constructor and runs its +// startup hook against dataDir, returning the startup error. +func startState(t *testing.T, dataDir string) error { + t.Helper() + + g, err := globals.New(nil) + if err != nil { + t.Fatalf("globals.New: %v", err) + } + + log, err := logger.New(nil, logger.Params{Globals: g}) + if err != nil { + t.Fatalf("logger.New: %v", err) + } + + lifecycle := fxtest.NewLifecycle(t) + + _, err = state.New(lifecycle, state.Params{ + Logger: log, + Config: &config.Config{DataDir: dataDir}, + }) + if err != nil { + t.Fatalf("state.New: %v", err) + } + + return lifecycle.Start(t.Context()) +} + +// TestStartupFailsWhenDataDirNotWritable verifies that startup stops +// with an error naming the data directory when it cannot be written. +// The directory's parent is a regular file, which also fails as root. +func TestStartupFailsWhenDataDirNotWritable(t *testing.T) { + t.Parallel() + + parent := filepath.Join(t.TempDir(), "file") + + err := os.WriteFile(parent, nil, 0o600) + if err != nil { + t.Fatalf("writing file: %v", err) + } + + dataDir := filepath.Join(parent, "data") + + err = startState(t, dataDir) + if err == nil { + t.Fatal("startup should fail when the data directory is not writable") + } + + want := "data directory " + dataDir + " is not writable" + if !strings.Contains(err.Error(), want) { + t.Errorf("startup error %q does not contain %q", err, want) + } +} + +// TestStartupFailsWhenExistingDataDirNotWritable verifies that startup +// stops when the data directory exists but the temp file that saving uses +// cannot be written in it. A directory sitting at the temp file's path +// makes that write fail, which also holds as root. +func TestStartupFailsWhenExistingDataDirNotWritable(t *testing.T) { + t.Parallel() + + dataDir := t.TempDir() + + err := os.Mkdir(filepath.Join(dataDir, "state.json.tmp"), 0o700) + if err != nil { + t.Fatalf("creating directory: %v", err) + } + + err = startState(t, dataDir) + if err == nil { + t.Fatal("startup should fail when the data directory is not writable") + } + + want := "data directory " + dataDir + " is not writable" + if !strings.Contains(err.Error(), want) { + t.Errorf("startup error %q does not contain %q", err, want) + } +} + +// TestStartupCreatesDataDir verifies that startup creates a missing +// data directory and leaves nothing behind in it. +func TestStartupCreatesDataDir(t *testing.T) { + t.Parallel() + + dataDir := filepath.Join(t.TempDir(), "data") + + err := startState(t, dataDir) + if err != nil { + t.Fatalf("startup error: %v", err) + } + + entries, err := os.ReadDir(dataDir) + if err != nil { + t.Fatalf("reading data directory: %v", err) + } + + if len(entries) != 0 { + t.Errorf("startup left %d entries in the data directory", len(entries)) + } +} + // TestPortStateUnmarshalJSON_NewFormat verifies deserialization of the // current multi-hostname format. func TestPortStateUnmarshalJSON_NewFormat(t *testing.T) { @@ -628,25 +1094,78 @@ func TestPortStateUnmarshalJSON_BothFormats(t *testing.T) { } } -// TestGetSnapshot_ReturnsCopy verifies that GetSnapshot returns a value copy. +// TestGetSnapshot_ReturnsCopy verifies that the maps in the snapshot +// GetSnapshot returns are not the state's own. func TestGetSnapshot_ReturnsCopy(t *testing.T) { t.Parallel() - s := state.NewForTest() + s := state.NewForTestWithDataDir(t.TempDir()) populateState(t, s) snap := s.GetSnapshot() - // Mutate the returned snapshot's map. - snap.Domains["mutated.example.com"] = &state.DomainState{ - Nameservers: []string{"ns1.mutated.com."}, + const added = "added.example.com" + + snap.Domains[added] = &state.DomainState{} + snap.Hostnames[added] = &state.HostnameState{} + snap.Ports[added] = &state.PortState{} + snap.Certificates[added] = &state.CertificateState{} + + if _, ok := s.GetDomainState(added); ok { + t.Error("the snapshot's Domains map is the state's own") } - // The original domain data should still be accessible. - _, ok := s.GetDomainState("example.com") - if !ok { - t.Error("original domain should still exist") + if _, ok := s.GetHostnameState(added); ok { + t.Error("the snapshot's Hostnames map is the state's own") + } + + if _, ok := s.GetPortState(added); ok { + t.Error("the snapshot's Ports map is the state's own") + } + + if _, ok := s.GetCertificateState(added); ok { + t.Error("the snapshot's Certificates map is the state's own") + } +} + +// TestGetSnapshot_WhileWritten ranges over snapshots while another +// goroutine writes the state, as the dashboard does while a check runs. +// Under -race it fails when a snapshot's maps are the state's own. +func TestGetSnapshot_WhileWritten(t *testing.T) { + t.Parallel() + + s := state.NewForTestWithDataDir(t.TempDir()) + + const writes = 1000 + + written := make(chan struct{}) + + go func() { + defer close(written) + + for range writes { + s.SetDomainState("example.com", &state.DomainState{}) + s.SetHostnameState(testHostname, &state.HostnameState{}) + s.SetPortState(testIPv4+":443", &state.PortState{}) + s.SetCertificateState( + testIPv4+":443:"+testHostname, &state.CertificateState{}, + ) + } + }() + + for { + select { + case <-written: + return + default: + } + + // json.Marshal ranges over each of the snapshot's maps. + _, err := json.Marshal(s.GetSnapshot()) + if err != nil { + t.Fatal(err) + } } } @@ -654,7 +1173,7 @@ func TestGetSnapshot_ReturnsCopy(t *testing.T) { func TestDomainState_GetSet(t *testing.T) { t.Parallel() - s := state.NewForTest() + s := state.NewForTestWithDataDir(t.TempDir()) // Get on missing key returns false. _, ok := s.GetDomainState("nonexistent.com") @@ -705,7 +1224,7 @@ func TestDomainState_GetSet(t *testing.T) { func TestHostnameState_GetSet(t *testing.T) { t.Parallel() - s := state.NewForTest() + s := state.NewForTestWithDataDir(t.TempDir()) _, ok := s.GetHostnameState("missing.example.com") if ok { @@ -750,7 +1269,7 @@ func TestHostnameState_GetSet(t *testing.T) { func TestPortState_GetSetDelete(t *testing.T) { t.Parallel() - s := state.NewForTest() + s := state.NewForTestWithDataDir(t.TempDir()) _, ok := s.GetPortState("1.2.3.4:80") if ok { @@ -788,7 +1307,7 @@ func TestPortState_GetSetDelete(t *testing.T) { func TestGetAllPortKeys(t *testing.T) { t.Parallel() - s := state.NewForTest() + s := state.NewForTestWithDataDir(t.TempDir()) keys := s.GetAllPortKeys() if len(keys) != 0 { @@ -830,7 +1349,7 @@ func TestGetAllPortKeys(t *testing.T) { func TestCertificateState_GetSet(t *testing.T) { t.Parallel() - s := state.NewForTest() + s := state.NewForTestWithDataDir(t.TempDir()) _, ok := s.GetCertificateState("1.2.3.4:443:www.example.com") if ok { @@ -1051,7 +1570,7 @@ func TestLoadPreservesExistingStateOnMissingFile(t *testing.T) { func TestConcurrentGetSet(t *testing.T) { t.Parallel() - s := state.NewForTest() + s := state.NewForTestWithDataDir(t.TempDir()) const goroutines = 20 @@ -1261,35 +1780,6 @@ func TestMultipleSavesOverwrite(t *testing.T) { } } -// TestNewForTest verifies the test helper creates a valid empty state. -func TestNewForTest(t *testing.T) { - t.Parallel() - - s := state.NewForTest() - - snap := s.GetSnapshot() - - if snap.Version != 1 { - t.Errorf("version: got %d, want 1", snap.Version) - } - - if snap.Domains == nil { - t.Error("Domains map should be initialized") - } - - if snap.Hostnames == nil { - t.Error("Hostnames map should be initialized") - } - - if snap.Ports == nil { - t.Error("Ports map should be initialized") - } - - if snap.Certificates == nil { - t.Error("Certificates map should be initialized") - } -} - // TestSaveFilePermissions verifies the saved file has restricted permissions. func TestSaveFilePermissions(t *testing.T) { t.Parallel() diff --git a/internal/state/state_test_helper.go b/internal/state/state_test_helper.go deleted file mode 100644 index 7e9aac7..0000000 --- a/internal/state/state_test_helper.go +++ /dev/null @@ -1,38 +0,0 @@ -package state - -import ( - "log/slog" - - "sneak.berlin/go/dnswatcher/internal/config" -) - -// NewForTest creates a State for unit testing with no persistence. -func NewForTest() *State { - return &State{ - log: slog.Default(), - snapshot: &Snapshot{ - Version: stateVersion, - Domains: make(map[string]*DomainState), - Hostnames: make(map[string]*HostnameState), - Ports: make(map[string]*PortState), - Certificates: make(map[string]*CertificateState), - }, - config: &config.Config{DataDir: ""}, - } -} - -// NewForTestWithDataDir creates a State backed by the given directory -// for tests that need file persistence. -func NewForTestWithDataDir(dataDir string) *State { - return &State{ - log: slog.Default(), - snapshot: &Snapshot{ - Version: stateVersion, - Domains: make(map[string]*DomainState), - Hostnames: make(map[string]*HostnameState), - Ports: make(map[string]*PortState), - Certificates: make(map[string]*CertificateState), - }, - config: &config.Config{DataDir: dataDir}, - } -} diff --git a/internal/watcher/cancelled_test.go b/internal/watcher/cancelled_test.go new file mode 100644 index 0000000..247ab7c --- /dev/null +++ b/internal/watcher/cancelled_test.go @@ -0,0 +1,177 @@ +package watcher_test + +import ( + "bytes" + "context" + "log/slog" + "reflect" + "strings" + "testing" + "time" + + "sneak.berlin/go/dnswatcher/internal/portcheck" + "sneak.berlin/go/dnswatcher/internal/resolver" + "sneak.berlin/go/dnswatcher/internal/state" + "sneak.berlin/go/dnswatcher/internal/tlscheck" + "sneak.berlin/go/dnswatcher/internal/watcher" +) + +// TestCancelledCheckSavesNothing runs a check with its context already +// cancelled, which is how the rest of a check runs once shutdown cuts it +// short. The real resolver drops the DNS lookup without sending a query, +// and the real port and TLS checkers fail without connecting. The port +// and certificate state the last check saved must stay as it was, and +// nothing may be notified. +func TestCancelledCheckSavesNothing(t *testing.T) { + t.Parallel() + + cfg := defaultTestConfig(t) + cfg.Hostnames = []string{host} + + // newTestWatcher's watcher has stand-in checkers. This one, on the + // same state and notifier, has the real ones. + _, deps := newTestWatcher(t, cfg) + w := watcher.NewForTest( + cfg, + deps.state, + resolver.NewFromLogger(slog.Default()), + portcheck.NewStandalone(), + tlscheck.NewStandalone(), + deps.notifier, + ) + + // The last check found host at a local address, with both ports + // open and a good certificate. + const localIP = "127.0.0.1" + + deps.state.SetHostnameState(host, hostnameState( + map[string]map[string][]string{nsA: {"A": {localIP}}}, + )) + + ports := map[string]*state.PortState{ + localIP + ":80": {Open: true, Hostnames: []string{host}}, + localIP + ":443": {Open: true, Hostnames: []string{host}}, + } + for key, ps := range ports { + deps.state.SetPortState(key, ps) + } + + certKey := localIP + ":443:" + host + cert := &state.CertificateState{CommonName: host, Status: "ok"} + deps.state.SetCertificateState(certKey, cert) + + ctx, cancel := context.WithCancel(t.Context()) + cancel() + + w.RunOnce(ctx) + + for key, want := range ports { + got, _ := deps.state.GetPortState(key) + if !reflect.DeepEqual(got, want) { + t.Errorf("port %s saved as %+v, want %+v", key, got, want) + } + } + + got, _ := deps.state.GetCertificateState(certKey) + if !reflect.DeepEqual(got, cert) { + t.Errorf("certificate saved as %+v, want %+v", got, cert) + } + + notifications := deps.notifier.getNotifications() + if len(notifications) != 0 { + t.Errorf("sent %v, want no notifications", notifications) + } +} + +// TestStopBeforeFirstScanSendsNoStartupNotification runs a watcher with +// the startup notification on and its context already cancelled, so the +// first scan is cut short before its lookup is sent. The scan did not +// complete, so no startup notification may be sent. +func TestStopBeforeFirstScanSendsNoStartupNotification(t *testing.T) { + t.Parallel() + + cfg := defaultTestConfig(t) + cfg.Hostnames = []string{testHost} + cfg.SendTestNotification = true + + w, deps := newTestWatcher(t, cfg) + + ctx, cancel := context.WithCancel(t.Context()) + cancel() + + w.Run(ctx) + + notifications := deps.notifier.getNotifications() + if len(notifications) != 0 { + t.Errorf("sent %v, want no notifications", notifications) + } +} + +// newLoggingWatcher returns a watcher for a domain and a hostname, with +// the real resolver, that writes what it logs at warning level or above +// into the returned buffer. +func newLoggingWatcher(t *testing.T) (*watcher.Watcher, *bytes.Buffer) { + t.Helper() + + cfg := defaultTestConfig(t) + cfg.Domains = []string{testSmallDomain} + cfg.Hostnames = []string{host} + + w, _ := newTestWatcher(t, cfg) + + logs := &bytes.Buffer{} + w.SetLogger(slog.New(slog.NewJSONHandler( + logs, &slog.HandlerOptions{Level: slog.LevelWarn}, + ))) + + return w, logs +} + +// TestLookupCutShortIsNotLogged checks a domain and a hostname, looks +// up a nameserver's addresses and follows a CNAME, with the context +// cancelled, as shutdown leaves it. The real resolver fails each lookup +// without sending a query. Shutdown cutting a lookup short is not a +// failure, so nothing may be logged at warning level or above. +func TestLookupCutShortIsNotLogged(t *testing.T) { + t.Parallel() + + w, logs := newLoggingWatcher(t) + + ctx, cancel := context.WithCancel(t.Context()) + cancel() + + w.RunOnce(ctx) + w.ResolveNameserverAddresses(ctx, []string{nsA}, nil) + w.ResolveCNAMEAddresses(ctx, host, cnameState(), nil) + + if logs.Len() > 0 { + t.Errorf("logged at warning level or above:\n%s", logs) + } +} + +// TestLookupOutOfTimeIsLoggedAsError does what +// TestLookupCutShortIsNotLogged does, with the context's deadline passed +// instead. A lookup that ran out of time did fail, so the domain's NS +// lookup, the hostname's lookup, the nameserver's address lookup and the +// CNAME's are each logged as an error. +func TestLookupOutOfTimeIsLoggedAsError(t *testing.T) { + t.Parallel() + + w, logs := newLoggingWatcher(t) + + ctx, cancel := context.WithDeadline(t.Context(), time.Now()) + t.Cleanup(cancel) + + w.RunOnce(ctx) + w.ResolveNameserverAddresses(ctx, []string{nsA}, nil) + w.ResolveCNAMEAddresses(ctx, host, cnameState(), nil) + + const want = 4 + + lines := strings.Count(logs.String(), "\n") + errorLines := strings.Count(logs.String(), `"level":"ERROR"`) + + if lines != want || errorLines != want { + t.Errorf("logged:\n%s\nwant %d lines, each at error level", logs, want) + } +} diff --git a/internal/watcher/cname_test.go b/internal/watcher/cname_test.go new file mode 100644 index 0000000..9f492f1 --- /dev/null +++ b/internal/watcher/cname_test.go @@ -0,0 +1,372 @@ +package watcher_test + +import ( + "context" + "log/slog" + "slices" + "testing" + + "sneak.berlin/go/dnswatcher/internal/livednstest" + "sneak.berlin/go/dnswatcher/internal/resolver" + "sneak.berlin/go/dnswatcher/internal/state" + "sneak.berlin/go/dnswatcher/internal/watcher" +) + +// TestCNAMEIntoAnotherZonePortAndTLSChecks runs the port and TLS +// checks on hostname state built here: the name's nameserver answered +// with a CNAME into another zone, and following it found ip1. Both +// checks must use ip1. They look nothing up, so the watcher has no +// resolver. +func TestCNAMEIntoAnotherZonePortAndTLSChecks(t *testing.T) { + t.Parallel() + + cfg := defaultTestConfig(t) + cfg.Hostnames = []string{host} + + deps := newTestDeps(t, cfg) + w := watcher.NewForTest( + cfg, deps.state, nil, + deps.portChecker, deps.tlsChecker, deps.notifier, + ) + + deps.state.SetHostnameState(host, cnameState(ip1)) + + w.CheckAllPorts(t.Context()) + w.RunTLSChecks(t.Context()) + + snap := deps.state.GetSnapshot() + + ps, ok := snap.Ports[ip1+":443"] + if !ok || !slices.Contains(ps.Hostnames, host) { + t.Errorf("no port state for %s at %s:443", host, ip1) + } + + certKey := ip1 + ":443:" + host + if _, ok := snap.Certificates[certKey]; !ok { + t.Errorf("no certificate state %s", certKey) + } +} + +// TestCNAMEThatCannotBeFollowedKeepsPrevious runs a check of a name, not +// the watcher's first, from the point where its records have been looked +// up: they hold a CNAME to a target under .invalid, whose lookup fails. +// The previous check found the same records, and oldIP at the end of the +// CNAME. The check must keep oldIP and send nothing. +func TestCNAMEThatCannotBeFollowedKeepsPrevious(t *testing.T) { + t.Parallel() + + w, deps := newTestWatcher(t, defaultTestConfig(t)) + w.SetFirstRun(false) + + records := map[string]map[string][]string{ + nsA: cnameTo("target.example.invalid."), + } + + prev := hostnameState(records) + prev.CNAMEAddresses = []string{oldIP} + deps.state.SetHostnameState(host, prev) + + // The result is the same whether or not live DNS answers, so the + // lookup is not retried. + _ = livednstest.Run(func(ctx context.Context) error { + w.UpdateHostnameState(ctx, host, hostnameState(records)) + + return nil + }) + + hs, _ := deps.state.GetHostnameState(host) + if !slices.Equal(hs.CNAMEAddresses, prev.CNAMEAddresses) { + t.Errorf( + "saved %v, want %v", + hs.CNAMEAddresses, prev.CNAMEAddresses, + ) + } + + notifications := deps.notifier.getNotifications() + if len(notifications) != 0 { + t.Errorf("sent %v, want no notifications", notifications) + } +} + +// followLive follows in live DNS the CNAMEs in a name's records, built +// from records, and returns the addresses saved for the name. The +// previous check saved oldIP, which is kept when a target cannot be +// followed; that is retried. The tests point CNAMEs only at names in +// zones with two nameservers, to keep queries few (see the top of +// watcher_test.go). +func followLive( + t *testing.T, + records map[string]map[string][]string, +) []string { + t.Helper() + + w := watcher.NewForTest( + nil, nil, resolver.NewFromLogger(slog.Default()), nil, nil, nil, + ) + prev := cnameState(oldIP) + + var current *state.HostnameState + + livednstest.Retry(t, "following CNAMEs", func(ctx context.Context) error { + current = hostnameState(records) + + w.ResolveCNAMEAddresses(ctx, host, current, prev) + + if slices.Equal(current.CNAMEAddresses, prev.CNAMEAddresses) { + return livednstest.ErrNoAnswer + } + + return nil + }) + + return current.CNAMEAddresses +} + +// TestCNAMEAddressesOfEveryTarget gives a name's two nameservers +// different CNAME targets, as when a secondary still serves an old one. +// The addresses at the end of both are saved, whichever answer is read +// first: one.one.one.one has 1.1.1.1, and dns.adguard-dns.com has +// 94.140.14.14. +func TestCNAMEAddressesOfEveryTarget(t *testing.T) { + t.Parallel() + + found := followLive(t, map[string]map[string][]string{ + nsA: cnameTo("one.one.one.one."), + nsB: cnameTo("dns.adguard-dns.com."), + }) + + for _, ip := range []string{"1.1.1.1", "94.140.14.14"} { + if !slices.Contains(found, ip) { + t.Errorf("saved %v, want %s among them", found, ip) + } + } +} + +// TestCNAMEChainEndingInNoAddressSavesEmptyList follows a CNAME to a +// name live DNS answers with NXDOMAIN. An empty list is saved, not nil, +// which would mean the addresses are not known. +func TestCNAMEChainEndingInNoAddressSavesEmptyList(t *testing.T) { + t.Parallel() + + found := followLive(t, map[string]map[string][]string{ + nsA: cnameTo("this-surely-does-not-exist-xyz.example.org."), + }) + + if found == nil || len(found) != 0 { + t.Errorf("saved %#v, want an empty list", found) + } +} + +// TestCNAMEBesideAnAddressNotFollowed gives one nameserver of a name an +// address and another a CNAME. The CNAME is not followed: an empty list +// is saved, not nil, and nothing is looked up, the watcher having no +// resolver. +func TestCNAMEBesideAnAddressNotFollowed(t *testing.T) { + t.Parallel() + + w := watcher.NewForTest(nil, nil, nil, nil, nil, nil) + + current := hostnameState(map[string]map[string][]string{ + nsA: {"A": {ip1}}, + nsB: cnameTo("target.example.org."), + }) + + w.ResolveCNAMEAddresses(t.Context(), host, current, nil) + + if current.CNAMEAddresses == nil || len(current.CNAMEAddresses) != 0 { + t.Errorf("saved %#v, want an empty list", current.CNAMEAddresses) + } +} + +// TestCNAMEWhoseNameserversAllFailedKeepsPrevious checks a name none of +// whose nameservers answered. The addresses the previous check saved +// from following its CNAME are kept, and nothing is looked up: the +// watcher has no resolver. +func TestCNAMEWhoseNameserversAllFailedKeepsPrevious(t *testing.T) { + t.Parallel() + + w := watcher.NewForTest(nil, nil, nil, nil, nil, nil) + + current := saved(map[string]*state.NameserverRecordState{ + nsA: failed(), nsB: failed(), + }) + prev := cnameState(oldIP) + + w.ResolveCNAMEAddresses(t.Context(), host, current, prev) + + if !slices.Equal(current.CNAMEAddresses, prev.CNAMEAddresses) { + t.Errorf( + "saved %v, want %v", + current.CNAMEAddresses, prev.CNAMEAddresses, + ) + } +} + +// TestCNAMEWhoseAddressQueryFailedKeepsPrevious checks a name whose +// nameserver answered, but whose query for A, AAAA or CNAME failed with +// nothing kept for it. That is not an answer with no address: the +// addresses the previous check saved from following its CNAME are kept, +// and nothing is looked up, the watcher having no resolver. +func TestCNAMEWhoseAddressQueryFailedKeepsPrevious(t *testing.T) { + t.Parallel() + + for _, rtype := range []string{"A", "AAAA", "CNAME"} { + t.Run(rtype, func(t *testing.T) { + t.Parallel() + + w := watcher.NewForTest(nil, nil, nil, nil, nil, nil) + + current := saved(map[string]*state.NameserverRecordState{ + nsA: { + Records: map[string][]string{}, + FailedTypes: []string{rtype}, + UnknownTypes: []string{rtype}, + Status: "ok", + }, + }) + prev := cnameState(oldIP) + + w.ResolveCNAMEAddresses(t.Context(), host, current, prev) + + if !slices.Equal(current.CNAMEAddresses, prev.CNAMEAddresses) { + t.Errorf( + "saved %v, want %v", + current.CNAMEAddresses, prev.CNAMEAddresses, + ) + } + }) + } +} + +// cnameTo builds the records of a nameserver that answered with a CNAME +// to target and no address. +func cnameTo(target string) map[string][]string { + return map[string][]string{"CNAME": {target}} +} + +// cnameState builds the state a check leaves behind for a name whose +// nameserver answered with a CNAME and no address, when following the +// CNAME found these addresses, which may be none. +func cnameState(addresses ...string) *state.HostnameState { + hs := hostnameState(map[string]map[string][]string{ + nsA: cnameTo("target.example.org."), + }) + + hs.CNAMEAddresses = append([]string{}, addresses...) + + return hs +} + +func TestCNAMEAddressChangeAlerts(t *testing.T) { + t.Parallel() + + // A state file written before the addresses were saved loads with + // them nil. + olderStateFile := cnameState() + olderStateFile.CNAMEAddresses = nil + + // Each case is the state saved by the previous check and by the + // current one. The name's records are the same in both. + tests := []struct { + name string + prev, current *state.HostnameState + want int + }{ + { + "same addresses", + cnameState(ip1, ip2), cnameState(ip1, ip2), 0, + }, + { + "same addresses in another order", + cnameState(ip2, ip1), cnameState(ip1, ip2), 0, + }, + { + "address replaced", + cnameState(ip1), cnameState(ip2), 1, + }, + { + "address added", + cnameState(ip1), cnameState(ip1, ip2), 1, + }, + { + "no address at the end of the chain now", + cnameState(ip1), cnameState(), 1, + }, + { + "addresses at the end of the chain again", + cnameState(), cnameState(ip1), 1, + }, + { + "state file from before addresses were saved", + olderStateFile, cnameState(ip1), 0, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + notifier := &mockNotifier{} + w := watcher.NewForTest(nil, nil, nil, nil, nil, notifier) + + w.DetectHostnameChanges(t.Context(), host, tt.prev, tt.current) + + got := len(notifier.getNotifications()) + if got != tt.want { + t.Errorf("sent %d notifications, want %d", got, tt.want) + } + }) + } +} + +func TestCNAMEAddressChangeAlertNamesHostnameAndAddresses(t *testing.T) { + t.Parallel() + + notifier := &mockNotifier{} + w := watcher.NewForTest(nil, nil, nil, nil, nil, notifier) + + w.DetectHostnameChanges( + t.Context(), host, cnameState(ip1), cnameState(ip2, ip3), + ) + + want := notification{ + Title: "CNAME Address Change: " + host, + Message: "Hostname: " + host + + "\nOld: " + ip1 + "\nNew: " + ip2 + ", " + ip3, + Priority: "warning", + } + + got := notifier.getNotifications() + if len(got) != 1 || got[0] != want { + t.Errorf("sent %v, want %v", got, want) + } +} + +// TestNameMovedFromARecordsToCNAMEAlerts checks a name that answers +// with an A record and then with a CNAME whose chain ends in ip2. The +// second check is notified as a CNAME address change from no addresses, +// beside the record change. Nothing is looked up: the watcher has no +// resolver. +func TestNameMovedFromARecordsToCNAMEAlerts(t *testing.T) { + t.Parallel() + + notifier := &mockNotifier{} + w := watcher.NewForTest(nil, nil, nil, nil, nil, notifier) + + prev := hostnameState(map[string]map[string][]string{ + nsA: {"A": {ip1}}, + }) + w.ResolveCNAMEAddresses(t.Context(), host, prev, nil) + + w.DetectHostnameChanges(t.Context(), host, prev, cnameState(ip2)) + + title := "CNAME Address Change: " + host + message := "Hostname: " + host + "\nOld: \nNew: " + ip2 + + got := notifier.getNotifications() + if !slices.ContainsFunc(got, func(n notification) bool { + return n.Title == title && n.Message == message + }) { + t.Errorf("sent %v, want %q with %q among them", got, title, message) + } +} diff --git a/internal/watcher/export_test.go b/internal/watcher/export_test.go new file mode 100644 index 0000000..f13a0b0 --- /dev/null +++ b/internal/watcher/export_test.go @@ -0,0 +1,160 @@ +package watcher + +import ( + "context" + "log/slog" + "time" + + "sneak.berlin/go/dnswatcher/internal/config" + "sneak.berlin/go/dnswatcher/internal/resolver" + "sneak.berlin/go/dnswatcher/internal/state" +) + +// NewForTest creates a Watcher without fx for unit testing. A nil cfg +// is an empty configuration. +func NewForTest( + cfg *config.Config, + st *state.State, + res DNSResolver, + pc PortChecker, + tc TLSChecker, + n Notifier, +) *Watcher { + if cfg == nil { + cfg = &config.Config{} + } + + return &Watcher{ + log: slog.Default(), + config: cfg, + state: st, + resolver: res, + portCheck: pc, + tlsCheck: tc, + notify: n, + firstRun: true, + } +} + +// SetLogger replaces the watcher's logger, so a test can read what it +// logs. +func (w *Watcher) SetLogger(log *slog.Logger) { + w.log = log +} + +// NewlyDisagreeingPairs exports newlyDisagreeingPairs for testing. +func NewlyDisagreeingPairs( + prev, current *state.HostnameState, +) [][2]string { + return newlyDisagreeingPairs(prev, current) +} + +// SetFirstRun sets whether the watcher is on its first check, in which +// nothing is compared with the previous check. NewForTest's watcher is. +func (w *Watcher) SetFirstRun(firstRun bool) { + w.firstRun = firstRun +} + +// UpdateHostnameState exports updateHostnameState for testing. +func (w *Watcher) UpdateHostnameState( + ctx context.Context, + hostname string, + newState *state.HostnameState, +) { + w.updateHostnameState(ctx, hostname, newState) +} + +// DetectHostnameChanges exports detectHostnameChanges for testing. +func (w *Watcher) DetectHostnameChanges( + ctx context.Context, + hostname string, + prev, current *state.HostnameState, +) { + w.detectHostnameChanges(ctx, hostname, prev, current) +} + +// ResolveNameserverAddresses exports resolveNameserverAddresses for +// testing. +func (w *Watcher) ResolveNameserverAddresses( + ctx context.Context, + nameservers []string, + prev map[string][]string, +) map[string][]string { + return w.resolveNameserverAddresses(ctx, nameservers, prev) +} + +// ResolveCNAMEAddresses exports resolveCNAMEAddresses for testing. +func (w *Watcher) ResolveCNAMEAddresses( + ctx context.Context, + hostname string, + current, prev *state.HostnameState, +) { + w.resolveCNAMEAddresses(ctx, hostname, current, prev) +} + +// DetectNSAddressChanges exports detectNSAddressChanges for testing. +func (w *Watcher) DetectNSAddressChanges( + ctx context.Context, + domain string, + prev, current map[string][]string, +) { + w.detectNSAddressChanges(ctx, domain, prev, current) +} + +// MaybeSendTestNotification exports maybeSendTestNotification for +// testing. +func (w *Watcher) MaybeSendTestNotification(ctx context.Context) { + w.maybeSendTestNotification(ctx) +} + +// CleanupRemovedTargets exports cleanupRemovedTargets for testing. +func (w *Watcher) CleanupRemovedTargets() { + w.cleanupRemovedTargets() +} + +// CheckAllPorts exports checkAllPorts for testing. +func (w *Watcher) CheckAllPorts(ctx context.Context) { + w.checkAllPorts(ctx) +} + +// RunTLSChecks exports runTLSChecks for testing. +func (w *Watcher) RunTLSChecks(ctx context.Context) { + w.runTLSChecks(ctx) +} + +// CheckHostnameResult exports checkHostnameResult for testing. +func (w *Watcher) CheckHostnameResult( + ctx context.Context, + hostname string, + results map[string]*resolver.NameserverResponse, + err error, +) bool { + return w.checkHostnameResult(ctx, hostname, results, err) +} + +// CheckDomainResult exports checkDomainResult for testing. +func (w *Watcher) CheckDomainResult( + ctx context.Context, + domain string, + nameservers []string, + err error, +) { + w.checkDomainResult(ctx, domain, nameservers, err) +} + +// FailedNameservers exports failedNameservers for testing. +func FailedNameservers( + err error, + prev *state.HostnameState, +) map[string]*resolver.NameserverResponse { + return failedNameservers(err, prev) +} + +// BuildHostnameState exports buildHostnameState for testing. +func BuildHostnameState( + results map[string]*resolver.NameserverResponse, + prev *state.HostnameState, + now time.Time, +) *state.HostnameState { + return buildHostnameState(results, prev, now) +} diff --git a/internal/watcher/failedtype_test.go b/internal/watcher/failedtype_test.go new file mode 100644 index 0000000..832ebf2 --- /dev/null +++ b/internal/watcher/failedtype_test.go @@ -0,0 +1,364 @@ +package watcher_test + +import ( + "maps" + "slices" + "testing" + "time" + + "sneak.berlin/go/dnswatcher/internal/resolver" + "sneak.berlin/go/dnswatcher/internal/state" + "sneak.berlin/go/dnswatcher/internal/watcher" +) + +const ( + // txt is the record type whose query fails in these tests. + txt = "TXT" + spf1 = "v=spf1 -all" + spf2 = "v=spf1 include:example.net -all" +) + +// response is a nameserver's response with these records, whose queries +// for failedTypes failed. +func response( + records map[string][]string, + failedTypes ...string, +) *resolver.NameserverResponse { + return &resolver.NameserverResponse{ + Records: records, + FailedTypes: failedTypes, + Status: resolver.StatusOK, + } +} + +// savedChecks saves the state of each check in turn from the +// nameservers' responses, each from the state the check before saved. +func savedChecks( + checks ...map[string]*resolver.NameserverResponse, +) []*state.HostnameState { + states := make([]*state.HostnameState, 0, len(checks)) + + var prev *state.HostnameState + + for _, results := range checks { + prev = watcher.BuildHostnameState(results, prev, time.Now()) + states = append(states, prev) + } + + return states +} + +// TestFailedTypeKeepsPreviousRecords saves a check in which nsA's query +// for TXT failed, after previous checks of several kinds. TXT is always +// saved in FailedTypes, and in UnknownTypes when there was nothing to +// keep. +func TestFailedTypeKeepsPreviousRecords(t *testing.T) { + t.Parallel() + + aOnly := map[string][]string{"A": {ip1}} + withTXT := map[string][]string{"A": {ip1}, txt: {spf1}} + txtKept := &state.NameserverRecordState{ + Records: withTXT, FailedTypes: []string{txt}, Status: "ok", + } + txtNotKnown := &state.NameserverRecordState{ + Records: aOnly, + FailedTypes: []string{txt}, + UnknownTypes: []string{txt}, + Status: "ok", + } + + tests := []struct { + name string + prev *state.HostnameState + wantRecords map[string][]string + wantUnknown []string + }{ + { + "previous TXT records are kept", + saved(map[string]*state.NameserverRecordState{nsA: answered(withTXT)}), + withTXT, nil, + }, + { + "previous check had no TXT records", + saved(map[string]*state.NameserverRecordState{nsA: answered(aOnly)}), + aOnly, nil, + }, + { + "TXT failed on the previous check, which kept its records", + saved(map[string]*state.NameserverRecordState{nsA: txtKept}), + withTXT, nil, + }, + {"first check", nil, aOnly, []string{txt}}, + { + "nameserver new on this check", + saved(map[string]*state.NameserverRecordState{nsB: answered(withTXT)}), + aOnly, []string{txt}, + }, + { + "nameserver failed on the previous check", + saved(map[string]*state.NameserverRecordState{nsA: failed()}), + aOnly, []string{txt}, + }, + { + "TXT failed on the previous check with nothing to keep", + saved(map[string]*state.NameserverRecordState{nsA: txtNotKnown}), + aOnly, []string{txt}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + hs := watcher.BuildHostnameState( + map[string]*resolver.NameserverResponse{ + nsA: response(map[string][]string{"A": {ip1}}, txt), + }, + tt.prev, time.Now(), + ) + + got := hs.RecordsByNameserver[nsA] + if got.Status != "ok" || + !maps.EqualFunc(got.Records, tt.wantRecords, slices.Equal) || + !slices.Equal(got.FailedTypes, []string{txt}) || + !slices.Equal(got.UnknownTypes, tt.wantUnknown) { + t.Errorf( + "saved status %q, records %v, failed types %v, "+ + "unknown types %v; want ok, %v, [%s], %v", + got.Status, got.Records, got.FailedTypes, + got.UnknownTypes, tt.wantRecords, txt, tt.wantUnknown, + ) + } + }) + } +} + +// TestFailedTypeAlerts saves the checks of each case in turn from the +// nameservers' responses, the first being the state loaded at startup, +// and counts the alerts sent. nsB's TXT query fails on one check, and +// nothing changes. +func TestFailedTypeAlerts(t *testing.T) { + t.Parallel() + + records := map[string][]string{"A": {ip1}, txt: {spf1}} + aOnly := map[string][]string{"A": {ip1}} + + bothAnswer := map[string]*resolver.NameserverResponse{ + nsA: response(records), nsB: response(records), + } + bTXTFails := map[string]*resolver.NameserverResponse{ + nsA: response(records), nsB: response(aOnly, txt), + } + onlyA := map[string]*resolver.NameserverResponse{ + nsA: response(records), + } + bFails := map[string]*resolver.NameserverResponse{ + nsA: response(records), + nsB: { + Records: map[string][]string{}, + Status: resolver.StatusTimeout, + Error: "all queries timed out", + }, + } + + tests := []struct { + name string + checks []map[string]*resolver.NameserverResponse + want alertCounts + }{ + { + "type failing at one nameserver alerts nothing, nor its next answer", + []map[string]*resolver.NameserverResponse{ + bothAnswer, bTXTFails, bothAnswer, + }, + alertCounts{}, + }, + { + "type failing on the first check alerts nothing on the next", + []map[string]*resolver.NameserverResponse{bTXTFails, bothAnswer}, + alertCounts{}, + }, + { + "type failing at a nameserver new on that check alerts nothing", + []map[string]*resolver.NameserverResponse{ + onlyA, bTXTFails, bothAnswer, + }, + alertCounts{}, + }, + { + "type failing at a recovering nameserver alerts the recovery", + []map[string]*resolver.NameserverResponse{ + bFails, bTXTFails, bothAnswer, + }, + alertCounts{recoveries: 1}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + states := savedChecks(tt.checks...) + + got := countAlerts(t, states[0], states[1:]) + if got != tt.want { + t.Errorf("sent %+v, want %+v", got, tt.want) + } + }) + } +} + +// TestFailedTypeComparedOnceItAnswers saves the checks of each case in +// turn as TestFailedTypeAlerts does. nsB's TXT query fails on one check, +// and the TXT record changes: the change is sent as a Record Change for +// each nameserver on the check where it answers it, and an Inconsistency +// only when nsB still answers the old record. +func TestFailedTypeComparedOnceItAnswers(t *testing.T) { + t.Parallel() + + records := map[string][]string{"A": {ip1}, txt: {spf1}} + changed := map[string][]string{"A": {ip1}, txt: {spf2}} + aOnly := map[string][]string{"A": {ip1}} + + bothAnswer := map[string]*resolver.NameserverResponse{ + nsA: response(records), nsB: response(records), + } + bTXTFails := map[string]*resolver.NameserverResponse{ + nsA: response(records), nsB: response(aOnly, txt), + } + bothChange := map[string]*resolver.NameserverResponse{ + nsA: response(changed), nsB: response(changed), + } + aChangesBTXTFails := map[string]*resolver.NameserverResponse{ + nsA: response(changed), nsB: response(aOnly, txt), + } + bStillOld := map[string]*resolver.NameserverResponse{ + nsA: response(changed), nsB: response(records), + } + + tests := []struct { + name string + checks []map[string]*resolver.NameserverResponse + want alertCounts + }{ + { + "change made while the type failed", + []map[string]*resolver.NameserverResponse{ + bothAnswer, bTXTFails, bothChange, + }, + alertCounts{recordChanges: 2}, + }, + { + "change seen at one nameserver while the other's type failed", + []map[string]*resolver.NameserverResponse{ + bothAnswer, aChangesBTXTFails, bothChange, + }, + alertCounts{recordChanges: 2}, + }, + { + "old record answered after the type failed", + []map[string]*resolver.NameserverResponse{ + bothAnswer, aChangesBTXTFails, bStillOld, + }, + alertCounts{recordChanges: 1, inconsistencies: 1}, + }, + { + "change after the type failed on the first check and answered", + []map[string]*resolver.NameserverResponse{ + bTXTFails, bothAnswer, bothChange, + }, + alertCounts{recordChanges: 2}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + states := savedChecks(tt.checks...) + + got := countAlerts(t, states[0], states[1:]) + if got != tt.want { + t.Errorf("sent %+v, want %+v", got, tt.want) + } + }) + } +} + +// TestFailedTypeLeftOutOfMessages checks that a Record Change and an +// Inconsistency name only the record types they compared. nsB's TXT +// records are not known on the first check, and on the second either +// answered or still not known; nsB's A record changes, so both alerts +// are sent and name the A record alone. +func TestFailedTypeLeftOutOfMessages(t *testing.T) { + t.Parallel() + + withTXT := map[string][]string{"A": {ip1}, txt: {spf1}} + txtNotKnown := func(address string) *state.NameserverRecordState { + return &state.NameserverRecordState{ + Records: map[string][]string{"A": {address}}, + FailedTypes: []string{txt}, + UnknownTypes: []string{txt}, + Status: "ok", + } + } + + before := saved(map[string]*state.NameserverRecordState{ + nsA: answered(withTXT), nsB: txtNotKnown(ip1), + }) + + tests := []struct { + name string + after *state.HostnameState + }{ + { + "TXT answers", + saved(map[string]*state.NameserverRecordState{ + nsA: answered(withTXT), + nsB: answered(map[string][]string{"A": {ip2}, txt: {spf1}}), + }), + }, + { + "TXT still not known", + saved(map[string]*state.NameserverRecordState{ + nsA: answered(withTXT), nsB: txtNotKnown(ip2), + }), + }, + } + + want := map[string]string{ + "Record Change: " + host: "Hostname: " + host + + "\nNameserver: " + nsB + "\nType: A\nOld: " + ip1 + "\nNew: " + ip2, + "Inconsistency: " + host: "Hostname: " + host + + "\nType: A\n" + nsA + ": " + ip1 + "\n" + nsB + ": " + ip2, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + // The hostname change detection uses only the notifier. + notifier := &mockNotifier{} + w := watcher.NewForTest(nil, nil, nil, nil, nil, notifier) + + w.DetectHostnameChanges(t.Context(), host, before, tt.after) + + notifications := notifier.getNotifications() + if len(notifications) != len(want) { + t.Fatalf( + "sent %d notifications, want %d: %v", + len(notifications), len(want), notifications, + ) + } + + for _, n := range notifications { + if n.Message != want[n.Title] { + t.Errorf( + "%s message:\n%s\nwant:\n%s", + n.Title, n.Message, want[n.Title], + ) + } + } + }) + } +} diff --git a/internal/watcher/inconsistency_test.go b/internal/watcher/inconsistency_test.go new file mode 100644 index 0000000..43c129e --- /dev/null +++ b/internal/watcher/inconsistency_test.go @@ -0,0 +1,240 @@ +package watcher_test + +import ( + "slices" + "testing" + + "sneak.berlin/go/dnswatcher/internal/state" + "sneak.berlin/go/dnswatcher/internal/watcher" +) + +const ( + host = "www.example.net" + nsA = "a.ns.example.net." + nsB = "b.ns.example.net." + nsC = "c.ns.example.net." + ip1 = "192.0.2.1" + ip2 = "192.0.2.2" + ip3 = "192.0.2.3" +) + +// hostnameState builds the state a check with these records leaves behind. +func hostnameState( + records map[string]map[string][]string, +) *state.HostnameState { + hs := &state.HostnameState{ + RecordsByNameserver: make(map[string]*state.NameserverRecordState), + } + + for ns, recs := range records { + hs.RecordsByNameserver[ns] = &state.NameserverRecordState{ + Records: recs, + Status: "ok", + } + } + + return hs +} + +func TestNewlyDisagreeingPairs(t *testing.T) { + t.Parallel() + + onlyA := map[string]map[string][]string{nsA: {"A": {ip1}}} + agree := map[string]map[string][]string{nsA: {"A": {ip1}}, nsB: {"A": {ip1}}} + disagree := map[string]map[string][]string{nsA: {"A": {ip1}}, nsB: {"A": {ip2}}} + alert := [][2]string{{nsA, nsB}} + + // b already disagrees with a and c; then c changes, so a and c, + // which agreed, now differ. + bDiffers := map[string]map[string][]string{ + nsA: {"A": {ip1}}, nsB: {"A": {ip2}}, nsC: {"A": {ip1}}, + } + cChanges := map[string]map[string][]string{ + nsA: {"A": {ip1}}, nsB: {"A": {ip2}}, nsC: {"A": {ip3}}, + } + + // Each case starts from the state loaded at startup and runs the + // checks in order; want[i] is what check i alerts for. + tests := []struct { + name string + loaded map[string]map[string][]string + checks []map[string]map[string][]string + want [][][2]string + }{ + { + name: "disagreement persisting across checks alerts once", + loaded: agree, + checks: []map[string]map[string][]string{disagree, disagree, disagree}, + want: [][][2]string{alert, nil, nil}, + }, + { + name: "disagreement starting on a later check alerts on it", + loaded: agree, + checks: []map[string]map[string][]string{agree, agree, disagree}, + want: [][][2]string{nil, nil, alert}, + }, + { + name: "disagreement in the loaded state does not alert", + loaded: disagree, + checks: []map[string]map[string][]string{disagree, disagree}, + want: [][][2]string{nil, nil}, + }, + { + name: "nameserver new on the first check and disagreeing alerts once", + loaded: onlyA, + checks: []map[string]map[string][]string{disagree, disagree}, + want: [][][2]string{alert, nil}, + }, + { + name: "disagreement after agreeing again alerts again", + loaded: agree, + checks: []map[string]map[string][]string{disagree, agree, disagree}, + want: [][][2]string{alert, nil, alert}, + }, + { + name: "new disagreement while another nameserver differs alerts", + loaded: bDiffers, + checks: []map[string]map[string][]string{cChanges, cChanges}, + want: [][][2]string{{{nsA, nsC}}, nil}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + prev := hostnameState(tt.loaded) + + for i, records := range tt.checks { + current := hostnameState(records) + + got := watcher.NewlyDisagreeingPairs(prev, current) + if !slices.Equal(got, tt.want[i]) { + t.Errorf( + "check %d: alerted for %v, want %v", + i, got, tt.want[i], + ) + } + + prev = current + } + }) + } +} + +func TestInconsistencyAlert(t *testing.T) { + t.Parallel() + + onlyA := map[string]map[string][]string{nsA: {"A": {ip1}}} + agree := map[string]map[string][]string{nsA: {"A": {ip1}}, nsB: {"A": {ip1}}} + disagree := map[string]map[string][]string{nsA: {"A": {ip1}}, nsB: {"A": {ip2}}} + + // Each case starts from the state loaded at startup and then sees + // the nameservers disagree on three checks in a row. + tests := []struct { + name string + loaded map[string]map[string][]string + want int + }{ + { + name: "disagreement lasting several checks alerts once", + loaded: agree, + want: 1, + }, + { + name: "disagreement in the loaded state does not alert", + loaded: disagree, + want: 0, + }, + { + name: "nameserver new on the first check and disagreeing alerts once", + loaded: onlyA, + want: 1, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + // The hostname change detection uses only the notifier. + notifier := &mockNotifier{} + w := watcher.NewForTest(nil, nil, nil, nil, nil, notifier) + + prev := hostnameState(tt.loaded) + + for range 3 { + current := hostnameState(disagree) + w.DetectHostnameChanges(t.Context(), host, prev, current) + prev = current + } + + got := 0 + + for _, n := range notifier.getNotifications() { + if n.Title == "Inconsistency: "+host { + got++ + } + } + + if got != tt.want { + t.Errorf("sent %d inconsistency alerts, want %d", got, tt.want) + } + }) + } +} + +// TestFirstCheckAfterRepeatedValuesLoaded saves a state file holding a +// hostname's CNAME once for every record type asked for, as checks did +// before each value was stored once, and two addresses each repeated, +// and loads it. A check that then finds each value once at each +// nameserver must notify nothing. +func TestFirstCheckAfterRepeatedValuesLoaded(t *testing.T) { + t.Parallel() + + const ( + cnameType = "CNAME" + cname = "c.example.net." + ) + + cfg := defaultTestConfig(t) + repeated := map[string][]string{ + "A": {ip2, ip1, ip2, ip1}, + cnameType: {cname, cname, cname, cname, cname, cname, cname, cname}, + } + once := map[string][]string{"A": {ip1, ip2}, cnameType: {cname}} + + saved := newTestDeps(t, cfg).state + saved.SetHostnameState(host, hostnameState(map[string]map[string][]string{ + nsA: repeated, nsB: repeated, + })) + + err := saved.Save() + if err != nil { + t.Fatalf("saving the state file: %v", err) + } + + deps := newTestDeps(t, cfg) + + err = deps.state.Load() + if err != nil { + t.Fatalf("loading the state file: %v", err) + } + + prev, ok := deps.state.GetHostnameState(host) + if !ok { + t.Fatal("the state file has no state for " + host) + } + + current := hostnameState(map[string]map[string][]string{ + nsA: once, nsB: once, + }) + + // The hostname change detection uses only the notifier. + w := watcher.NewForTest(nil, nil, nil, nil, nil, deps.notifier) + w.DetectHostnameChanges(t.Context(), host, prev, current) + + if got := deps.notifier.getNotifications(); len(got) != 0 { + t.Errorf("sent %v, want no notification", got) + } +} diff --git a/internal/watcher/interfaces.go b/internal/watcher/interfaces.go index dd68017..c92d8d6 100644 --- a/internal/watcher/interfaces.go +++ b/internal/watcher/interfaces.go @@ -5,23 +5,29 @@ import ( "context" "sneak.berlin/go/dnswatcher/internal/portcheck" + "sneak.berlin/go/dnswatcher/internal/resolver" "sneak.berlin/go/dnswatcher/internal/tlscheck" ) // DNSResolver performs iterative DNS resolution. type DNSResolver interface { - // LookupNS discovers authoritative nameservers for a domain. + // LookupNS returns a domain's NS record set, as its parent zone's + // servers delegate it: empty when they answer that it has none, and + // resolver.ErrNXDomain when they answer that it does not exist. LookupNS( ctx context.Context, domain string, ) ([]string, error) // LookupAllRecords queries all record types for a hostname, - // returning results keyed by nameserver then record type. + // returning each nameserver's response keyed by nameserver. When + // the walk to the nameservers reached a zone through a referral and + // none of that zone's nameservers answered, the error is a + // *resolver.NoAnswerError naming them. LookupAllRecords( ctx context.Context, hostname string, - ) (map[string]map[string][]string, error) + ) (map[string]*resolver.NameserverResponse, error) // ResolveIPAddresses resolves a hostname to all IP addresses. ResolveIPAddresses( diff --git a/internal/watcher/ipv6_test.go b/internal/watcher/ipv6_test.go new file mode 100644 index 0000000..7b1608d --- /dev/null +++ b/internal/watcher/ipv6_test.go @@ -0,0 +1,83 @@ +package watcher_test + +import ( + "maps" + "slices" + "testing" + + "sneak.berlin/go/dnswatcher/internal/state" + "sneak.berlin/go/dnswatcher/internal/watcher" +) + +// TestIPv6AddressInBrackets runs the port checks, not the watcher's +// first, on hostname state built here for a hostname at an IPv6 address +// whose port 443 the previous check found closed. It is open now. Then +// it runs the TLS checks and the port checks again. The port and +// certificate keys and the Port Change notification write the address +// in brackets, so its last group cannot be read as the port, and the +// second port checks keep the certificate entry, whose address they read +// back from its key. When a last port check finds port 443 closed, the +// certificate entry is removed. Nothing is looked up. +func TestIPv6AddressInBrackets(t *testing.T) { + t.Parallel() + + const aaaaType = "AAAA" + + cfg := defaultTestConfig(t) + cfg.Hostnames = []string{host} + + deps := newTestDeps(t, cfg) + w := watcher.NewForTest( + cfg, deps.state, nil, + deps.portChecker, deps.tlsChecker, deps.notifier, + ) + w.SetFirstRun(false) + + deps.state.SetHostnameState(host, saved( + map[string]*state.NameserverRecordState{ + nsA: answered(map[string][]string{aaaaType: {"2001:db8::1"}}), + }, + )) + deps.state.SetPortState("[2001:db8::1]:443", &state.PortState{ + Hostnames: []string{host}, + }) + + w.CheckAllPorts(t.Context()) + w.RunTLSChecks(t.Context()) + w.CheckAllPorts(t.Context()) + + snap := deps.state.GetSnapshot() + + gotPorts := slices.Sorted(maps.Keys(snap.Ports)) + wantPorts := []string{"[2001:db8::1]:443", "[2001:db8::1]:80"} + + if !slices.Equal(gotPorts, wantPorts) { + t.Errorf("port entries %v, want %v", gotPorts, wantPorts) + } + + gotCerts := slices.Sorted(maps.Keys(snap.Certificates)) + wantCerts := []string{"[2001:db8::1]:443:www.example.net"} + + if !slices.Equal(gotCerts, wantCerts) { + t.Errorf("certificate entries %v, want %v", gotCerts, wantCerts) + } + + title := "Port Change: [2001:db8::1]:443" + want := `Hostnames: www.example.net +Address: [2001:db8::1]:443 +Port now open` + + got := deps.notifier.getNotifications() + if len(got) != 1 || got[0].Title != title || got[0].Message != want { + t.Errorf("sent %v, want one %q with message:\n%s", got, title, want) + } + + deps.portChecker.closed = true + + w.CheckAllPorts(t.Context()) + + gotCerts = slices.Sorted(maps.Keys(deps.state.GetSnapshot().Certificates)) + if len(gotCerts) != 0 { + t.Errorf("certificate entries %v with port 443 closed, want none", gotCerts) + } +} diff --git a/internal/watcher/message_test.go b/internal/watcher/message_test.go new file mode 100644 index 0000000..c9857f1 --- /dev/null +++ b/internal/watcher/message_test.go @@ -0,0 +1,294 @@ +package watcher_test + +import ( + "maps" + "strings" + "testing" + + "sneak.berlin/go/dnswatcher/internal/state" + "sneak.berlin/go/dnswatcher/internal/watcher" +) + +// When one nameserver's A record changes and its TXT record does not, +// the record change and the inconsistency it starts name the A record +// alone, with its values written as plain text. +func TestChangeMessagesNameTheChangedType(t *testing.T) { + t.Parallel() + + // A nameserver's records: this A address and the same TXT record. + records := func(address string) map[string][]string { + return map[string][]string{ + "A": {address}, + "TXT": {"v=spf1 -all"}, + } + } + + before := hostnameState(map[string]map[string][]string{ + nsA: records(ip1), + nsB: records(ip1), + }) + after := hostnameState(map[string]map[string][]string{ + nsA: records(ip1), + nsB: records(ip2), + }) + + // The hostname change detection uses only the notifier. + notifier := &mockNotifier{} + w := watcher.NewForTest(nil, nil, nil, nil, nil, notifier) + + w.DetectHostnameChanges(t.Context(), host, before, after) + + want := map[string]string{ + "Record Change: " + host: `Hostname: www.example.net +Nameserver: b.ns.example.net. +Type: A +Old: 192.0.2.1 +New: 192.0.2.2`, + "Inconsistency: " + host: `Hostname: www.example.net +Type: A +a.ns.example.net.: 192.0.2.1 +b.ns.example.net.: 192.0.2.2`, + } + + notifications := notifier.getNotifications() + if len(notifications) != len(want) { + t.Fatalf( + "sent %d notifications, want %d: %v", + len(notifications), len(want), notifications, + ) + } + + for _, n := range notifications { + if n.Message != want[n.Title] { + t.Errorf( + "%s message:\n%s\nwant:\n%s", + n.Title, n.Message, want[n.Title], + ) + } + } +} + +// Every kind of notification about a configured apex domain's own +// records names it as a domain. +func TestDomainRecordNotificationsNameTheDomain(t *testing.T) { + t.Parallel() + + cfg := defaultTestConfig(t) + cfg.Domains = []string{domain} + + deps := newTestDeps(t, cfg) + w := watcher.NewForTest(cfg, deps.state, nil, nil, nil, deps.notifier) + + // nsA's address changes, which also makes it differ from nsC; nsB + // fails; nsC answers again. + w.DetectHostnameChanges(t.Context(), domain, + saved(map[string]*state.NameserverRecordState{ + nsA: answered(map[string][]string{"A": {ip1}}), + nsB: answered(map[string][]string{"A": {ip1}}), + nsC: failed(), + }), + saved(map[string]*state.NameserverRecordState{ + nsA: answered(map[string][]string{"A": {ip2}}), + nsB: failed(), + nsC: answered(map[string][]string{"A": {ip1}}), + }), + ) + + // The address at the end of its CNAME chain changes. + w.DetectHostnameChanges( + t.Context(), domain, cnameState(ip1), cnameState(ip2), + ) + + want := map[string]int{ + "Record Change": 1, + "Inconsistency": 1, + "NS Failure": 1, + "NS Recovery": 1, + "CNAME Address Change": 1, + } + + sent := make(map[string]int) + + for _, n := range deps.notifier.getNotifications() { + kind, _, _ := strings.Cut(n.Title, ":") + sent[kind]++ + + if !strings.HasPrefix(n.Message, "Domain: "+domain+"\n") { + t.Errorf("%s message does not name the domain:\n%s", + n.Title, n.Message) + } + } + + if !maps.Equal(sent, want) { + t.Errorf("sent %v, want %v", sent, want) + } +} + +// nsB, which answered, and nsC, which did not, are no longer among the +// nameservers. One NS Change lists both, for the hostname and for the +// configured apex domain with no delegation of its own. Nothing is sent +// for the domain whose delegation on the previous check listed them: +// the check of its delegation sends its NS Change. +func TestRemovedNameserversAreAnNSChange(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + target string + // The domain's nameservers saved by the previous check. + delegation []string + // The NS Change sent, or "" for none. + want string + }{ + { + name: "hostname", + target: host, + want: `Hostname: www.example.net +Removed: b.ns.example.net., c.ns.example.net.`, + }, + { + name: "domain with no delegation of its own", + target: domain, + delegation: []string{}, + want: `Domain: example.net +Removed: b.ns.example.net., c.ns.example.net.`, + }, + { + name: "domain whose delegation listed them", + target: domain, + delegation: []string{nsA, nsB, nsC}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + cfg := defaultTestConfig(t) + cfg.Domains = []string{domain} + + deps := newTestDeps(t, cfg) + w := watcher.NewForTest( + cfg, deps.state, nil, nil, nil, deps.notifier, + ) + + deps.state.SetDomainState(domain, &state.DomainState{ + Nameservers: tt.delegation, + }) + + prev := saved(map[string]*state.NameserverRecordState{ + nsA: answered(map[string][]string{"A": {ip1}}), + nsB: answered(map[string][]string{"A": {ip1}}), + nsC: failed(), + }) + current := saved(map[string]*state.NameserverRecordState{ + nsA: answered(map[string][]string{"A": {ip1}}), + }) + + w.DetectHostnameChanges(t.Context(), tt.target, prev, current) + + got := deps.notifier.getNotifications() + + if tt.want == "" { + if len(got) != 0 { + t.Errorf("sent %v, want nothing", got) + } + + return + } + + title := "NS Change: " + tt.target + if len(got) != 1 || got[0].Title != title || + got[0].Message != tt.want { + t.Errorf("sent %v, want one %q with message:\n%s", + got, title, tt.want) + } + }) + } +} + +// The startup notification counts the configured domains and hostnames, +// although the state's hostnames also hold the apex domain's own +// records. Nothing is looked up: the watcher has no resolver. +func TestStartupNotificationCountsConfiguredNames(t *testing.T) { + t.Parallel() + + cfg := defaultTestConfig(t) + cfg.SendTestNotification = true + cfg.Domains = []string{domain} + cfg.Hostnames = []string{host} + + deps := newTestDeps(t, cfg) + w := watcher.NewForTest(cfg, deps.state, nil, nil, nil, deps.notifier) + + // The state a check of both names saves. + deps.state.SetDomainState(domain, &state.DomainState{ + Nameservers: []string{nsA}, + }) + + for _, name := range []string{domain, host} { + deps.state.SetHostnameState(name, saved( + map[string]*state.NameserverRecordState{ + nsA: answered(map[string][]string{"A": {ip1}}), + }, + )) + } + + w.MaybeSendTestNotification(t.Context()) + + notifications := deps.notifier.getNotifications() + + counts := "\nMonitoring 1 domain(s) and 1 hostname(s).\n" + if len(notifications) != 1 || + !strings.Contains(notifications[0].Message, counts) { + t.Errorf("sent %v, want one message with %q", notifications, counts) + } +} + +// A Port Change notification lists the configured apex domain and the +// hostname that resolve to the port's address on separate lines. The +// port checks read the saved hostname state and look nothing up, so the +// watcher has no resolver. +func TestPortChangeListsDomainsApartFromHostnames(t *testing.T) { + t.Parallel() + + cfg := defaultTestConfig(t) + cfg.Domains = []string{domain} + cfg.Hostnames = []string{host} + + deps := newTestDeps(t, cfg) + w := watcher.NewForTest( + cfg, deps.state, nil, + deps.portChecker, deps.tlsChecker, deps.notifier, + ) + w.SetFirstRun(false) + + // Both names resolve to ip1, whose port 443 the previous check + // found open. It is closed now. + for _, name := range []string{domain, host} { + deps.state.SetHostnameState(name, saved( + map[string]*state.NameserverRecordState{ + nsA: answered(map[string][]string{"A": {ip1}}), + }, + )) + } + + key := ip1 + ":443" + deps.state.SetPortState(key, &state.PortState{ + Open: true, Hostnames: []string{domain, host}, + }) + deps.portChecker.closed = true + + w.CheckAllPorts(t.Context()) + + title := "Port Change: " + key + want := `Domains: example.net +Hostnames: www.example.net +Address: 192.0.2.1:443 +Port now closed` + + got := deps.notifier.getNotifications() + if len(got) != 1 || got[0].Title != title || got[0].Message != want { + t.Errorf("sent %v, want one %q with message:\n%s", got, title, want) + } +} diff --git a/internal/watcher/nsaddress_test.go b/internal/watcher/nsaddress_test.go new file mode 100644 index 0000000..6eaf3de --- /dev/null +++ b/internal/watcher/nsaddress_test.go @@ -0,0 +1,151 @@ +package watcher_test + +import ( + "context" + "log/slog" + "reflect" + "testing" + + "sneak.berlin/go/dnswatcher/internal/livednstest" + "sneak.berlin/go/dnswatcher/internal/resolver" + "sneak.berlin/go/dnswatcher/internal/watcher" +) + +const domain = "example.net" + +func TestNSAddressChangeAlerts(t *testing.T) { + t.Parallel() + + // Each case is the nameserver addresses saved by the previous check + // and by the current one. + tests := []struct { + name string + prev, current map[string][]string + want int + }{ + { + "same addresses", + map[string][]string{nsA: {ip1, ip2}}, + map[string][]string{nsA: {ip1, ip2}}, + 0, + }, + { + "same addresses in another order", + map[string][]string{nsA: {ip2, ip1}}, + map[string][]string{nsA: {ip1, ip2}}, + 0, + }, + { + "address replaced", + map[string][]string{nsA: {ip1}}, + map[string][]string{nsA: {ip2}}, + 1, + }, + { + "address added", + map[string][]string{nsA: {ip1}}, + map[string][]string{nsA: {ip1, ip2}}, + 1, + }, + { + "two nameservers changed", + map[string][]string{nsA: {ip1}, nsB: {ip2}}, + map[string][]string{nsA: {ip3}, nsB: {ip3}}, + 2, + }, + { + "nameserver added", + map[string][]string{nsA: {ip1}}, + map[string][]string{nsA: {ip1}, nsB: {ip2}}, + 0, + }, + { + "nameserver removed", + map[string][]string{nsA: {ip1}, nsB: {ip2}}, + map[string][]string{nsA: {ip1}}, + 0, + }, + { + "state file from before addresses were saved", + nil, + map[string][]string{nsA: {ip1}, nsB: {ip2}}, + 0, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + notifier := &mockNotifier{} + w := watcher.NewForTest(nil, nil, nil, nil, nil, notifier) + + w.DetectNSAddressChanges(t.Context(), domain, tt.prev, tt.current) + + got := len(notifier.getNotifications()) + if got != tt.want { + t.Errorf("sent %d address changes, want %d", got, tt.want) + } + }) + } +} + +func TestNSAddressChangeAlertNamesDomainNameserverAndAddresses( + t *testing.T, +) { + t.Parallel() + + notifier := &mockNotifier{} + w := watcher.NewForTest(nil, nil, nil, nil, nil, notifier) + + w.DetectNSAddressChanges( + t.Context(), domain, + map[string][]string{nsA: {ip1}}, + map[string][]string{nsA: {ip2, ip3}}, + ) + + want := notification{ + Title: "NS Address Change: " + domain, + Message: "Domain: " + domain + "\nNameserver: " + nsA + + "\nOld: " + ip1 + "\nNew: " + ip2 + ", " + ip3, + Priority: "warning", + } + + got := notifier.getNotifications() + if len(got) != 1 || got[0] != want { + t.Errorf("sent %v, want %v", got, want) + } +} + +// TestNameserverWithNoAddressKeepsPrevious looks up nameserver names +// with no address: two under .invalid, whose lookup fails with an +// error, and one that does not exist under a real zone, which live DNS +// answers with no address and no error. Each one with addresses saved +// by the previous check keeps them; the one without gets none. +func TestNameserverWithNoAddressKeepsPrevious(t *testing.T) { + t.Parallel() + + w := watcher.NewForTest( + nil, nil, resolver.NewFromLogger(slog.Default()), nil, nil, nil, + ) + + nonexistentNS := "this-surely-does-not-exist-xyz." + testSmallDomain + "." + + prev := map[string][]string{oldNS1: {oldIP}, nonexistentNS: {oldIP}} + + var got map[string][]string + + // The result is the same whether or not live DNS answers, so the + // lookup is not retried. + _ = livednstest.Run(func(ctx context.Context) error { + got = w.ResolveNameserverAddresses( + ctx, []string{oldNS1, oldNS2, nonexistentNS}, prev, + ) + + return nil + }) + + if !reflect.DeepEqual(got, prev) { + t.Errorf("saved %v, want %v", got, prev) + } +} diff --git a/internal/watcher/nsfailure_test.go b/internal/watcher/nsfailure_test.go new file mode 100644 index 0000000..745a185 --- /dev/null +++ b/internal/watcher/nsfailure_test.go @@ -0,0 +1,581 @@ +package watcher_test + +import ( + "context" + "errors" + "fmt" + "log/slog" + "strings" + "testing" + "time" + + "sneak.berlin/go/dnswatcher/internal/livednstest" + "sneak.berlin/go/dnswatcher/internal/resolver" + "sneak.berlin/go/dnswatcher/internal/state" + "sneak.berlin/go/dnswatcher/internal/watcher" +) + +// errNoReply is why a walk that got no reply from a zone's servers +// failed. +var errNoReply = errors.New("all servers failed: i/o timeout") + +// answered is what a check saves for a nameserver that answered with +// these records. +func answered(records map[string][]string) *state.NameserverRecordState { + return &state.NameserverRecordState{Records: records, Status: "ok"} +} + +// failed is what a check saves for a nameserver that did not answer. +func failed() *state.NameserverRecordState { + return &state.NameserverRecordState{ + Records: map[string][]string{}, + Status: "error", + Error: "all queries timed out", + } +} + +// saved builds the hostname state a check saves. +func saved( + byNameserver map[string]*state.NameserverRecordState, +) *state.HostnameState { + return &state.HostnameState{RecordsByNameserver: byNameserver} +} + +// alertCounts counts the hostname alerts sent, by kind. +type alertCounts struct { + failures, recoveries, recordChanges, inconsistencies int +} + +// countAlerts runs the hostname change detection from the state loaded +// at startup through each check in turn, and counts the alerts sent. +func countAlerts( + t *testing.T, + loaded *state.HostnameState, + checks []*state.HostnameState, +) alertCounts { + t.Helper() + + // The hostname change detection uses only the notifier. + notifier := &mockNotifier{} + w := watcher.NewForTest(nil, nil, nil, nil, nil, notifier) + + prev := loaded + + for _, current := range checks { + w.DetectHostnameChanges(t.Context(), host, prev, current) + prev = current + } + + var got alertCounts + + for _, n := range notifier.getNotifications() { + kind, _, _ := strings.Cut(n.Title, ":") + + switch kind { + case "NS Failure": + got.failures++ + case "NS Recovery": + got.recoveries++ + case "Record Change": + got.recordChanges++ + case "Inconsistency": + got.inconsistencies++ + } + } + + return got +} + +func TestNSFailureAndRecoveryAlerts(t *testing.T) { + t.Parallel() + + records := map[string][]string{"A": {ip1}} + + bothAnswer := saved(map[string]*state.NameserverRecordState{ + nsA: answered(records), nsB: answered(records), + }) + bFails := saved(map[string]*state.NameserverRecordState{ + nsA: answered(records), nsB: failed(), + }) + onlyA := saved(map[string]*state.NameserverRecordState{ + nsA: answered(records), + }) + bAnswersNoRecords := saved(map[string]*state.NameserverRecordState{ + nsA: answered(records), nsB: answered(map[string][]string{}), + }) + bAnswersDifferently := saved(map[string]*state.NameserverRecordState{ + nsA: answered(records), nsB: answered(map[string][]string{"A": {ip2}}), + }) + + // Each case starts from the state loaded at startup and runs the + // checks in order. + tests := []struct { + name string + loaded *state.HostnameState + checks []*state.HostnameState + want alertCounts + }{ + { + "failure lasting several checks alerts once", + bothAnswer, []*state.HostnameState{bFails, bFails, bFails}, + alertCounts{failures: 1}, + }, + { + "recovery alerts once", + bFails, []*state.HostnameState{bothAnswer, bothAnswer}, + alertCounts{recoveries: 1}, + }, + { + "failing again after recovering alerts again", + bothAnswer, []*state.HostnameState{bFails, bothAnswer, bFails}, + alertCounts{failures: 2, recoveries: 1}, + }, + { + "nameserver failing when first seen does not alert", + onlyA, []*state.HostnameState{bFails, bFails}, + alertCounts{}, + }, + { + "answer with no records is a record change, not a failure", + bothAnswer, []*state.HostnameState{bAnswersNoRecords}, + alertCounts{recordChanges: 1, inconsistencies: 1}, + }, + { + "recovered nameserver that answers differently disagrees", + bFails, []*state.HostnameState{bAnswersDifferently}, + alertCounts{recoveries: 1, inconsistencies: 1}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + got := countAlerts(t, tt.loaded, tt.checks) + if got != tt.want { + t.Errorf("sent %+v, want %+v", got, tt.want) + } + }) + } +} + +func TestNSFailureAlertNamesHostnameNameserverAndReason(t *testing.T) { + t.Parallel() + + records := map[string][]string{"A": {ip1}} + + notifier := &mockNotifier{} + w := watcher.NewForTest(nil, nil, nil, nil, nil, notifier) + + w.DetectHostnameChanges( + t.Context(), host, + saved(map[string]*state.NameserverRecordState{nsA: answered(records)}), + saved(map[string]*state.NameserverRecordState{nsA: failed()}), + ) + + notifications := notifier.getNotifications() + if len(notifications) != 1 { + t.Fatalf("sent %v, want one NS Failure", notifications) + } + + msg := notifications[0].Message + if !strings.Contains(msg, host) || !strings.Contains(msg, nsA) || + !strings.Contains(msg, failed().Error) { + t.Errorf( + "message %q does not name %s, %s and the reason", + msg, host, nsA, + ) + } +} + +// TestZoneNameserversAllStopAnswering runs three checks of host from +// the point where its lookup has returned. On the first, nsA and nsB, +// the nameservers of its zone, answer. On the second, the lookup fails +// with the error the walk to them returns when neither answers (see +// the resolver's TestFollowDelegation_NoAnswerAfterReferral), given +// here because no live zone has nameservers that never answer. For a +// hostname it is the lookup of its records that fails, and for a domain +// with no delegation of its own the lookup of its delegation. Both are +// saved as failed, with the reason, and each gets an NS Failure. On the +// third, both answer, and each gets an NS Recovery. A domain's records +// are checked as a hostname's, so its first and third checks are too. +func TestZoneNameserversAllStopAnswering(t *testing.T) { + t.Parallel() + + records := map[string][]string{"A": {ip1}} + bothAnswer := map[string]*resolver.NameserverResponse{ + nsA: response(records), nsB: response(records), + } + noAnswer := &resolver.NoAnswerError{ + Nameservers: []string{nsA, nsB}, Err: errNoReply, + } + + tests := []struct { + name string + isDomain bool + }{ + {"hostname", false}, + {"domain with no delegation of its own", true}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + cfg := defaultTestConfig(t) + if tt.isDomain { + cfg.Domains = []string{host} + } else { + cfg.Hostnames = []string{host} + } + + // The watcher is given each lookup's result, so it has no + // resolver. + deps := newTestDeps(t, cfg) + w := watcher.NewForTest( + cfg, deps.state, nil, + deps.portChecker, deps.tlsChecker, deps.notifier, + ) + w.SetFirstRun(false) + + w.CheckHostnameResult(t.Context(), host, bothAnswer, nil) + + if tt.isDomain { + w.CheckDomainResult(t.Context(), host, nil, noAnswer) + } else { + w.CheckHostnameResult(t.Context(), host, nil, noAnswer) + } + + current, _ := deps.state.GetHostnameState(host) + + for _, ns := range []string{nsA, nsB} { + got := current.RecordsByNameserver[ns] + if got.Status != failed().Status || got.Error != errNoReply.Error() { + t.Errorf( + "%s saved with status %q, error %q; want %q, %q", + ns, got.Status, got.Error, failed().Status, errNoReply, + ) + } + } + + if n := countNotifications(deps, "NS Failure: "+host); n != 2 { + t.Errorf("sent %d NS Failure notifications, want 2", n) + } + + w.CheckHostnameResult(t.Context(), host, bothAnswer, nil) + + if n := countNotifications(deps, "NS Recovery: "+host); n != 2 { + t.Errorf("sent %d NS Recovery notifications, want 2", n) + } + }) + } +} + +// TestFailedNameserversOnlyForThoseOfThePreviousCheck gives +// FailedNameservers lookup errors after a check in which nsA and nsB +// answered. Only an error that names both makes them failed. +func TestFailedNameserversOnlyForThoseOfThePreviousCheck(t *testing.T) { + t.Parallel() + + records := map[string][]string{"A": {ip1}} + bothAnswered := saved(map[string]*state.NameserverRecordState{ + nsA: answered(records), nsB: answered(records), + }) + noAnswer := func(nameservers ...string) error { + return &resolver.NoAnswerError{Nameservers: nameservers, Err: errNoReply} + } + + tests := []struct { + name string + err error + prev *state.HostnameState + wantFailed bool + }{ + {"both did not answer", noAnswer(nsA, nsB), bothAnswered, true}, + { + "the servers of a parent zone did not answer", + noAnswer(nsC), bothAnswered, false, + }, + { + "the zone's delegation lists only nsA now", + noAnswer(nsA), bothAnswered, false, + }, + {"no root server answered", errNoReply, bothAnswered, false}, + {"no previous check", noAnswer(nsA, nsB), nil, false}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + got := watcher.FailedNameservers(tt.err, tt.prev) + if (got != nil) != tt.wantFailed { + t.Errorf("returned %v, want failed: %v", got, tt.wantFailed) + } + }) + } +} + +// TestNameserverThatNeverAnswers asks a nameserver address where +// nothing answers, 192.0.2.1, and checks what the watcher saves for it. +// The deadline outlasts the resolver's first two-second try, as in the +// resolver's timeout test. +func TestNameserverThatNeverAnswers(t *testing.T) { + t.Parallel() + + ctx, cancel := context.WithTimeout(t.Context(), 3*time.Second) + t.Cleanup(cancel) + + res := resolver.NewFromLogger(slog.Default()) + + resp, err := res.QueryNameserverIP(ctx, nsA, "192.0.2.1", host) + if err != nil { + t.Fatal(err) + } + + hs := watcher.BuildHostnameState( + map[string]*resolver.NameserverResponse{nsA: resp}, nil, time.Now(), + ) + + got := hs.RecordsByNameserver[nsA] + if got.Status != "error" || got.Error == "" { + t.Errorf( + "saved status %q, error %q; want status error with a reason", + got.Status, got.Error, + ) + } +} + +// TestNameserverThatAnswersNXDOMAIN asks a real nameserver about a name +// that does not exist and checks what the watcher saves for it: NXDOMAIN +// is an answer, so the nameserver is saved as ok with no error. +func TestNameserverThatAnswersNXDOMAIN(t *testing.T) { + t.Parallel() + + res := resolver.NewFromLogger(slog.Default()) + name := "this-surely-does-not-exist-xyz." + testDomain + + var ( + ns string + resp *resolver.NameserverResponse + ) + + livednstest.Retry(t, "QueryNameserver("+name+")", func(ctx context.Context) error { + nameservers, err := res.LookupNS(ctx, testDomain) + if err != nil { + return err + } + + ns = nameservers[0] + + resp, err = res.QueryNameserver(ctx, ns, name) + if err != nil { + return err + } + + // A timeout or a failure is no answer to check. + if resp.Status == resolver.StatusTimeout || + resp.Status == resolver.StatusError { + return fmt.Errorf( + "%w: %s: %s", livednstest.ErrNoAnswer, ns, resp.Error, + ) + } + + return nil + }) + + if resp.Status != resolver.StatusNXDomain { + t.Fatalf("%s answered %q for %s, want NXDOMAIN", ns, resp.Status, name) + } + + hs := watcher.BuildHostnameState( + map[string]*resolver.NameserverResponse{ns: resp}, nil, time.Now(), + ) + + got := hs.RecordsByNameserver[ns] + if got.Status != "ok" || got.Error != "" { + t.Errorf( + "saved status %q, error %q; want status ok with no error", + got.Status, got.Error, + ) + } +} + +// TestNameserverThatRefuses asks a google.com nameserver about +// cloudflare.com, a zone it does not serve, which it refuses, and checks +// what the watcher saves for it: REFUSED is no answer, so the nameserver +// is saved as error with the reason. +func TestNameserverThatRefuses(t *testing.T) { + t.Parallel() + + const reason = "server returned REFUSED" + + res := resolver.NewFromLogger(slog.Default()) + + var ( + ns string + resp *resolver.NameserverResponse + ) + + livednstest.Retry( + t, + "QueryNameserver(cloudflare.com)", + func(ctx context.Context) error { + nameservers, err := res.LookupNS(ctx, testDomain) + if err != nil { + return err + } + + ns = nameservers[0] + + resp, err = res.QueryNameserver(ctx, ns, "cloudflare.com") + if err != nil { + return err + } + + // A timeout or a network error is no reply at all. + if resp.Status == resolver.StatusTimeout || + strings.HasPrefix(resp.Error, "network error") { + return fmt.Errorf( + "%w: %s: %s", livednstest.ErrNoAnswer, ns, resp.Error, + ) + } + + return nil + }, + ) + + if resp.Error != reason { + t.Fatalf( + "%s answered %q (%s) for cloudflare.com, want REFUSED", + ns, resp.Status, resp.Error, + ) + } + + hs := watcher.BuildHostnameState( + map[string]*resolver.NameserverResponse{ns: resp}, nil, time.Now(), + ) + + got := hs.RecordsByNameserver[ns] + if got.Status != failed().Status || got.Error != reason { + t.Errorf( + "saved status %q, error %q; want status %q, error %q", + got.Status, got.Error, failed().Status, reason, + ) + } +} + +// TestPortStateWhenNoNameserverAnswered runs the port checks on +// hostname state built here, which gives the name no address. The port +// state saved for its old address is kept only when the name is a +// configured hostname or domain and none of its nameservers answered. +func TestPortStateWhenNoNameserverAnswered(t *testing.T) { + t.Parallel() + + noneAnswered := saved(map[string]*state.NameserverRecordState{ + nsA: failed(), nsB: failed(), + }) + oneAnsweredNoAddress := saved(map[string]*state.NameserverRecordState{ + nsA: answered(map[string][]string{}), nsB: failed(), + }) + + configured := []string{host} + + tests := []struct { + name string + hostname *state.HostnameState + hostnames []string + domains []string + wantKept bool + }{ + {"no nameserver answered", noneAnswered, configured, nil, true}, + { + "no nameserver answered, configured as a domain", + noneAnswered, nil, configured, true, + }, + { + "one answered with no address", + oneAnsweredNoAddress, configured, nil, false, + }, + {"no nameserver answered, not configured", noneAnswered, nil, nil, false}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + cfg := defaultTestConfig(t) + cfg.Hostnames = tt.hostnames + cfg.Domains = tt.domains + + // The port checks read the saved hostname state and look + // nothing up, so the watcher has no resolver. + deps := newTestDeps(t, cfg) + w := watcher.NewForTest( + cfg, deps.state, nil, + deps.portChecker, deps.tlsChecker, deps.notifier, + ) + + key := ip1 + ":443" + + deps.state.SetHostnameState(host, tt.hostname) + deps.state.SetPortState(key, &state.PortState{ + Open: true, Hostnames: []string{host}, + }) + + w.CheckAllPorts(t.Context()) + + _, kept := deps.state.GetPortState(key) + if kept != tt.wantKept { + t.Errorf("port state %s kept: %v, want %v", key, kept, tt.wantKept) + } + }) + } +} + +// TestPortStateWhenNoNameserverAnsweredAndOtherNameMovesAway saves the +// port state of an address two configured hostnames resolve to. While +// none of the first one's nameservers answer, the port checks run with +// the other one still at that address, then after it moved away; the +// port state is kept both times. +func TestPortStateWhenNoNameserverAnsweredAndOtherNameMovesAway( + t *testing.T, +) { + t.Parallel() + + const other = "mail.example.net" + + cfg := defaultTestConfig(t) + cfg.Hostnames = []string{host, other} + + // The port checks read the saved hostname state and look nothing + // up, so the watcher has no resolver. + deps := newTestDeps(t, cfg) + w := watcher.NewForTest( + cfg, deps.state, nil, + deps.portChecker, deps.tlsChecker, deps.notifier, + ) + + key := ip1 + ":443" + + deps.state.SetPortState(key, &state.PortState{ + Open: true, Hostnames: []string{host, other}, + }) + deps.state.SetHostnameState(host, saved( + map[string]*state.NameserverRecordState{nsA: failed(), nsB: failed()}, + )) + + for _, otherIP := range []string{ip1, ip2} { + deps.state.SetHostnameState(other, saved( + map[string]*state.NameserverRecordState{ + nsA: answered(map[string][]string{"A": {otherIP}}), + }, + )) + + w.CheckAllPorts(t.Context()) + + if _, kept := deps.state.GetPortState(key); !kept { + t.Fatalf("port state %s removed with %s at %s", key, other, otherIP) + } + } +} diff --git a/internal/watcher/removed_test.go b/internal/watcher/removed_test.go new file mode 100644 index 0000000..2549b18 --- /dev/null +++ b/internal/watcher/removed_test.go @@ -0,0 +1,291 @@ +package watcher_test + +import ( + "maps" + "net" + "slices" + "strings" + "testing" + + "sneak.berlin/go/dnswatcher/internal/state" + "sneak.berlin/go/dnswatcher/internal/watcher" +) + +// TestRemovedTargetsLeaveTheState loads a state saved while a domain +// and a hostname now removed from the configuration were still in it, +// and runs the removal that Run does before the first check. The +// removed names' domain, hostname and certificate entries are gone, +// the configured names' are kept, and nothing is notified. Nothing is +// looked up: the watcher has no resolver. +func TestRemovedTargetsLeaveTheState(t *testing.T) { + t.Parallel() + + const ( + removedDomain = "example.com" + removedHost = "www.example.com" + ) + + cfg := defaultTestConfig(t) + cfg.Domains = []string{domain} + cfg.Hostnames = []string{host} + + deps := newTestDeps(t, cfg) + w := watcher.NewForTest( + cfg, deps.state, nil, + deps.portChecker, deps.tlsChecker, deps.notifier, + ) + + // The state a check of all four names saves, each name at ip1. + for _, name := range []string{domain, removedDomain} { + deps.state.SetDomainState(name, &state.DomainState{ + Nameservers: []string{nsA}, + }) + } + + for _, name := range []string{domain, host, removedDomain, removedHost} { + deps.state.SetHostnameState(name, saved( + map[string]*state.NameserverRecordState{ + nsA: answered(map[string][]string{"A": {ip1}}), + }, + )) + deps.state.SetCertificateState( + ip1+":443:"+name, &state.CertificateState{Status: "ok"}, + ) + } + + err := deps.state.Save() + if err != nil { + t.Fatalf("saving the state: %v", err) + } + + err = deps.state.Load() + if err != nil { + t.Fatalf("loading the state: %v", err) + } + + w.CleanupRemovedTargets() + + snap := deps.state.GetSnapshot() + + got := slices.Sorted(maps.Keys(snap.Domains)) + if want := []string{domain}; !slices.Equal(got, want) { + t.Errorf("domain entries %v, want %v", got, want) + } + + got = slices.Sorted(maps.Keys(snap.Hostnames)) + if want := []string{domain, host}; !slices.Equal(got, want) { + t.Errorf("hostname entries %v, want %v", got, want) + } + + got = slices.Sorted(maps.Keys(snap.Certificates)) + if want := []string{ + ip1 + ":443:" + domain, ip1 + ":443:" + host, + }; !slices.Equal(got, want) { + t.Errorf("certificate entries %v, want %v", got, want) + } + + if sent := deps.notifier.getNotifications(); len(sent) != 0 { + t.Errorf("sent %v, want nothing", sent) + } +} + +// TestRemovedTargetsLeaveThePortEntries loads a state whose port +// entries name a domain and a hostname now removed from the +// configuration, and runs the removal that Run does before the first +// check. The removed names are off each port entry's list of names, the +// entry only they had is gone, the entry that also names configured +// names is kept for the port checks, and nothing is notified. +func TestRemovedTargetsLeaveThePortEntries(t *testing.T) { + t.Parallel() + + const ( + removedDomain = "example.com" + removedHost = "www.example.com" + ) + + cfg := defaultTestConfig(t) + cfg.Domains = []string{domain} + cfg.Hostnames = []string{host} + + deps := newTestDeps(t, cfg) + w := watcher.NewForTest( + cfg, deps.state, nil, + deps.portChecker, deps.tlsChecker, deps.notifier, + ) + + // The port 443 entries a check of all four names saves: each name + // at ip1, except the removed hostname, at ip2. + deps.state.SetPortState(ip1+":443", &state.PortState{ + Open: true, Hostnames: []string{removedDomain, domain, host}, + }) + deps.state.SetPortState(ip2+":443", &state.PortState{ + Open: true, Hostnames: []string{removedHost}, + }) + + err := deps.state.Save() + if err != nil { + t.Fatalf("saving the state: %v", err) + } + + err = deps.state.Load() + if err != nil { + t.Fatalf("loading the state: %v", err) + } + + w.CleanupRemovedTargets() + + if sent := deps.notifier.getNotifications(); len(sent) != 0 { + t.Errorf("sent %v, want nothing", sent) + } + + snap := deps.state.GetSnapshot() + + got := slices.Sorted(maps.Keys(snap.Ports)) + if want := []string{ip1 + ":443"}; !slices.Equal(got, want) { + t.Fatalf("port entries %v, want %v", got, want) + } + + got = snap.Ports[ip1+":443"].Hostnames + if want := []string{domain, host}; !slices.Equal(got, want) { + t.Errorf("names of port entry %s:443 %v, want %v", ip1, got, want) + } +} + +// TestCertificateStateForAnAddressGone runs the port checks on hostname +// state built here for a configured hostname, with certificate entries +// saved for it at ip1, ip2 and an IPv6 address. When its nameservers +// answered with ip1 and the IPv6 address, the entry for ip2 is removed. +// When none of them answered, its addresses are not known, and every +// entry is kept. Nothing is notified, and nothing is looked up. +func TestCertificateStateForAnAddressGone(t *testing.T) { + t.Parallel() + + const ip6 = "2001:db8::1" + + tests := []struct { + name string + hostname *state.HostnameState + want []string + }{ + { + "answered without ip2", + saved(map[string]*state.NameserverRecordState{ + nsA: answered(map[string][]string{ + "A": {ip1}, "AAAA": {ip6}, + }), + }), + []string{ip1 + ":443:" + host, "[" + ip6 + "]:443:" + host}, + }, + { + "no nameserver answered", + saved(map[string]*state.NameserverRecordState{ + nsA: failed(), nsB: failed(), + }), + []string{ + ip1 + ":443:" + host, + ip2 + ":443:" + host, + "[" + ip6 + "]:443:" + host, + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + cfg := defaultTestConfig(t) + cfg.Hostnames = []string{host} + + deps := newTestDeps(t, cfg) + w := watcher.NewForTest( + cfg, deps.state, nil, + deps.portChecker, deps.tlsChecker, deps.notifier, + ) + w.SetFirstRun(false) + + deps.state.SetHostnameState(host, tt.hostname) + + for _, ip := range []string{ip1, ip2, ip6} { + deps.state.SetCertificateState( + net.JoinHostPort(ip, "443")+":"+host, + &state.CertificateState{Status: "ok"}, + ) + } + + w.CheckAllPorts(t.Context()) + + got := slices.Sorted(maps.Keys(deps.state.GetSnapshot().Certificates)) + if !slices.Equal(got, tt.want) { + t.Errorf("certificate entries %v, want %v", got, tt.want) + } + + if sent := deps.notifier.getNotifications(); len(sent) != 0 { + t.Errorf("sent %v, want nothing", sent) + } + }) + } +} + +// TestCertificateStateForAPortClosed runs the port checks on hostname +// and port state built here for a configured hostname at ip1, whose +// port 443 the previous check found open and whose certificate entry +// is ok. The port is closed now: the certificate entry is removed, and +// only the Port Change is notified. When the port opens again, the next +// TLS check saves the certificate as new and notifies nothing about it. +// Nothing is looked up. +func TestCertificateStateForAPortClosed(t *testing.T) { + t.Parallel() + + cfg := defaultTestConfig(t) + cfg.Hostnames = []string{host} + + deps := newTestDeps(t, cfg) + w := watcher.NewForTest( + cfg, deps.state, nil, + deps.portChecker, deps.tlsChecker, deps.notifier, + ) + w.SetFirstRun(false) + + certKey := ip1 + ":443:" + host + + deps.state.SetHostnameState(host, saved( + map[string]*state.NameserverRecordState{ + nsA: answered(map[string][]string{"A": {ip1}}), + }, + )) + deps.state.SetPortState(ip1+":443", &state.PortState{ + Open: true, Hostnames: []string{host}, + }) + deps.state.SetCertificateState(certKey, &state.CertificateState{ + CommonName: host, Status: "ok", + }) + deps.portChecker.closed = true + + w.CheckAllPorts(t.Context()) + + if _, ok := deps.state.GetCertificateState(certKey); ok { + t.Errorf("certificate entry %s kept with port 443 closed", certKey) + } + + sent := deps.notifier.getNotifications() + if title := "Port Change: " + ip1 + ":443"; len(sent) != 1 || + sent[0].Title != title { + t.Errorf("sent %v, want one %q", sent, title) + } + + deps.portChecker.closed = false + + w.CheckAllPorts(t.Context()) + w.RunTLSChecks(t.Context()) + + cert, ok := deps.state.GetCertificateState(certKey) + if !ok || cert.Status != "ok" { + t.Errorf("certificate entry %s after port 443 opened: %+v", certKey, cert) + } + + for _, n := range deps.notifier.getNotifications() { + if strings.HasPrefix(n.Title, "TLS") { + t.Errorf("sent %v, want no TLS notification", n) + } + } +} diff --git a/internal/watcher/skiprecordnotifications_test.go b/internal/watcher/skiprecordnotifications_test.go new file mode 100644 index 0000000..b28afb5 --- /dev/null +++ b/internal/watcher/skiprecordnotifications_test.go @@ -0,0 +1,107 @@ +package watcher_test + +import ( + "slices" + "testing" + + "sneak.berlin/go/dnswatcher/internal/config" + "sneak.berlin/go/dnswatcher/internal/state" + "sneak.berlin/go/dnswatcher/internal/watcher" +) + +// TestSkipRecordNotificationsAlerts runs the hostname change detection +// for two names on the same check: nsA's address changes, so that nsA +// now differs from nsB, and nsC stops answering. The name in +// SkipRecordNotifications gets only the NS Failure; the other name also +// gets the Record Change and the Inconsistency. +func TestSkipRecordNotificationsAlerts(t *testing.T) { + t.Parallel() + + const skipped = "skipped.example.net" + + prev := saved(map[string]*state.NameserverRecordState{ + nsA: answered(map[string][]string{"A": {ip1}}), + nsB: answered(map[string][]string{"A": {ip1}}), + nsC: answered(map[string][]string{"A": {ip1}}), + }) + current := saved(map[string]*state.NameserverRecordState{ + nsA: answered(map[string][]string{"A": {ip2}}), + nsB: answered(map[string][]string{"A": {ip1}}), + nsC: failed(), + }) + + cfg := &config.Config{ + Hostnames: []string{host, skipped}, + SkipRecordNotifications: []string{skipped}, + } + + // The hostname change detection uses only the configuration and the + // notifier. + notifier := &mockNotifier{} + w := watcher.NewForTest(cfg, nil, nil, nil, nil, notifier) + + w.DetectHostnameChanges(t.Context(), host, prev, current) + w.DetectHostnameChanges(t.Context(), skipped, prev, current) + + sent := notifier.getNotifications() + titles := make([]string, 0, len(sent)) + + for _, n := range sent { + titles = append(titles, n.Title) + } + + slices.Sort(titles) + + want := []string{ + "Inconsistency: " + host, + "NS Failure: " + skipped, + "NS Failure: " + host, + "Record Change: " + host, + } + + if !slices.Equal(titles, want) { + t.Errorf("sent %v, want %v", titles, want) + } +} + +// TestSkipRecordNotificationsCheck checks testHost, which is in +// SkipRecordNotifications, from a saved state in which every nameserver +// live DNS lists answered with an address live DNS never returns. No +// Record Change and no Inconsistency is sent, and the check still saves +// what the nameservers answer. +func TestSkipRecordNotificationsCheck(t *testing.T) { + t.Parallel() + + cfg := defaultTestConfig(t) + cfg.Hostnames = []string{testHost} + cfg.SkipRecordNotifications = []string{testHost} + + nameservers := lookupNameservers(t, testHost) + + _, deps := runChecks(t, cfg, func(deps *testDeps) { + byNameserver := make(map[string]*state.NameserverRecordState) + for _, ns := range nameservers { + byNameserver[ns] = answered(map[string][]string{"A": {oldIP}}) + } + + deps.state.SetHostnameState(testHost, saved(byNameserver)) + }) + + for _, title := range []string{ + "Record Change: " + testHost, + "Inconsistency: " + testHost, + } { + if n := countNotifications(deps, title); n != 0 { + t.Errorf("sent %d %q, want 0", n, title) + } + } + + // A nameserver whose query for A failed keeps oldIP, so the check + // is that some address live DNS gave was saved. + hs, _ := deps.state.GetHostnameState(testHost) + fromLiveDNS := func(ip string) bool { return ip != oldIP } + + if !slices.ContainsFunc(addresses(hs), fromLiveDNS) { + t.Errorf("saved addresses %v, want those live DNS gave", addresses(hs)) + } +} diff --git a/internal/watcher/watcher.go b/internal/watcher/watcher.go index fe2c4fb..f0f26ff 100644 --- a/internal/watcher/watcher.go +++ b/internal/watcher/watcher.go @@ -2,17 +2,22 @@ package watcher import ( "context" + "errors" "fmt" "log/slog" + "maps" + "net" + "slices" "sort" + "strconv" "strings" - "sync" "time" "go.uber.org/fx" "sneak.berlin/go/dnswatcher/internal/config" "sneak.berlin/go/dnswatcher/internal/logger" + "sneak.berlin/go/dnswatcher/internal/resolver" "sneak.berlin/go/dnswatcher/internal/state" "sneak.berlin/go/dnswatcher/internal/tlscheck" ) @@ -47,17 +52,16 @@ type Params struct { // Watcher orchestrates all monitoring checks on a schedule. type Watcher struct { - log *slog.Logger - config *config.Config - state *state.State - resolver DNSResolver - portCheck PortChecker - tlsCheck TLSChecker - notify Notifier - cancel context.CancelFunc - firstRun bool - expiryNotifiedMu sync.Mutex - expiryNotified map[string]time.Time + log *slog.Logger + config *config.Config + state *state.State + resolver DNSResolver + portCheck PortChecker + tlsCheck TLSChecker + notify Notifier + cancel context.CancelFunc + done chan struct{} // closed when Run returns + firstRun bool } // New creates a new Watcher instance wired into the fx lifecycle. @@ -66,76 +70,77 @@ func New( params Params, ) (*Watcher, error) { w := &Watcher{ - log: params.Logger.Get(), - config: params.Config, - state: params.State, - resolver: params.Resolver, - portCheck: params.PortCheck, - tlsCheck: params.TLSCheck, - notify: params.Notify, - firstRun: true, - expiryNotified: make(map[string]time.Time), + log: params.Logger.Get(), + config: params.Config, + state: params.State, + resolver: params.Resolver, + portCheck: params.PortCheck, + tlsCheck: params.TLSCheck, + notify: params.Notify, + firstRun: true, } lifecycle.Append(fx.Hook{ - OnStart: func(_ context.Context) error { - // Use context.Background() — the fx startup context - // expires after startup completes, so deriving from it - // would cancel the watcher immediately. The watcher's - // lifetime is controlled by w.cancel in OnStop. - ctx, cancel := context.WithCancel(context.Background()) + OnStart: func(startCtx context.Context) error { + // The fx startup context expires after startup + // completes, so the watcher's context drops its + // cancellation. The watcher's lifetime is controlled + // by w.cancel in OnStop. + ctx, cancel := context.WithCancel( + context.WithoutCancel(startCtx), + ) w.cancel = cancel + w.done = make(chan struct{}) - go w.Run(ctx) //nolint:contextcheck // intentionally not derived from startCtx + go func() { + defer close(w.done) + + w.Run(ctx) + }() return nil }, - OnStop: func(_ context.Context) error { - if w.cancel != nil { - w.cancel() - } + OnStop: func(ctx context.Context) error { + w.cancel() - return nil + // Run saves state as it returns. Waiting for it here + // means the save is done before shutdown goes on. + select { + case <-w.done: + return nil + case <-ctx.Done(): + return fmt.Errorf( + "waiting for the watcher to stop: %w", + ctx.Err(), + ) + } }, }) return w, nil } -// NewForTest creates a Watcher without fx for unit testing. -func NewForTest( - cfg *config.Config, - st *state.State, - res DNSResolver, - pc PortChecker, - tc TLSChecker, - n Notifier, -) *Watcher { - return &Watcher{ - log: slog.Default(), - config: cfg, - state: st, - resolver: res, - portCheck: pc, - tlsCheck: tc, - notify: n, - firstRun: true, - expiryNotified: make(map[string]time.Time), - } -} - -// Run starts the monitoring loop with periodic scheduling. +// Run starts the monitoring loop with periodic scheduling. When ctx +// is cancelled, it saves state and returns. func (w *Watcher) Run(ctx context.Context) { w.log.Info( "watcher starting", "domains", len(w.config.Domains), "hostnames", len(w.config.Hostnames), - "dnsInterval", w.config.DNSInterval, - "tlsInterval", w.config.TLSInterval, + // As text: the JSON log writes a time.Duration as bare + // nanoseconds. + "dnsInterval", w.config.DNSInterval.String(), + "tlsInterval", w.config.TLSInterval.String(), ) + w.cleanupRemovedTargets() w.RunOnce(ctx) - w.maybeSendTestNotification(ctx) + + // RunOnce returns as usual when shutdown cuts the scan short, and + // the startup notification says the scan completed. + if ctx.Err() == nil { + w.maybeSendTestNotification(ctx) + } dnsTicker := time.NewTicker(w.config.DNSInterval) tlsTicker := time.NewTicker(w.config.TLSInterval) @@ -146,6 +151,7 @@ func (w *Watcher) Run(ctx context.Context) { for { select { case <-ctx.Done(): + w.saveState() w.log.Info("watcher stopped") return @@ -203,6 +209,59 @@ func (w *Watcher) detectFirstRun() { } } +// cleanupRemovedTargets removes from the loaded state the domain, +// hostname and certificate entries of names no longer in the +// configuration, which changes only at a restart, and takes those names +// off each port entry's list of names, removing a port entry left with +// none. Nothing is notified. A configured domain's own records are +// saved as a hostname entry under its name, which is kept. +func (w *Watcher) cleanupRemovedTargets() { + for _, name := range w.state.GetAllDomainNames() { + if !w.isDomain(name) { + w.state.DeleteDomainState(name) + } + } + + for _, name := range w.state.GetAllHostnames() { + if !w.isConfigured(name) { + w.state.DeleteHostnameState(name) + } + } + + for _, key := range w.state.GetAllCertificateKeys() { + if _, hostname := parseCertKey(key); !w.isConfigured(hostname) { + w.state.DeleteCertificateState(key) + } + } + + for _, key := range w.state.GetAllPortKeys() { + ps, ok := w.state.GetPortState(key) + if !ok { + continue + } + + var names []string + + for _, name := range ps.Hostnames { + if w.isConfigured(name) { + names = append(names, name) + } + } + + if len(names) == 0 { + w.state.DeletePortState(key) + + continue + } + + w.state.SetPortState(key, &state.PortState{ + Open: ps.Open, + Hostnames: names, + LastChecked: ps.LastChecked, + }) + } +} + // runDNSChecks performs DNS resolution for all configured domains // and hostnames, updating state with freshly resolved records. // This must complete before port or TLS checks run so those @@ -217,18 +276,60 @@ func (w *Watcher) runDNSChecks(ctx context.Context) { } } +// logFailedLookup logs a failed DNS lookup at error level, unless ctx +// was cancelled: shutdown cancels it, and a lookup it cut short did not +// fail. A lookup that ran out of time did fail, so it is logged. +func (w *Watcher) logFailedLookup( + ctx context.Context, + msg string, + args ...any, +) { + if errors.Is(ctx.Err(), context.Canceled) { + return + } + + w.log.Error(msg, args...) +} + +// checkDomain looks up domain's delegation and checks domain with it +// (see checkDomainResult). func (w *Watcher) checkDomain( ctx context.Context, domain string, ) { nameservers, err := w.resolver.LookupNS(ctx, domain) + + w.checkDomainResult(ctx, domain, nameservers, err) +} + +// checkDomainResult finishes a check of domain from nameservers and err, +// what the lookup of its delegation returned. +func (w *Watcher) checkDomainResult( + ctx context.Context, + domain string, + nameservers []string, + err error, +) { + // A domain that does not exist has no nameservers. + nxdomain := errors.Is(err, resolver.ErrNXDomain) + if nxdomain { + nameservers, err = []string{}, nil + } + if err != nil { - w.log.Error( + w.logFailedLookup( + ctx, "failed to lookup NS", "domain", domain, "error", err, ) + // For a domain with no delegation of its own, the servers that + // did not answer can be those of the zone it is in, at which + // its records are asked. They are then saved as failed, as a + // hostname's are. + w.checkHostnameResult(ctx, domain, nil, err) + return } @@ -237,36 +338,41 @@ func (w *Watcher) checkDomain( now := time.Now().UTC() prev, hasPrev := w.state.GetDomainState(domain) - if hasPrev && !w.firstRun { - w.detectNSChanges(ctx, domain, prev.Nameservers, nameservers) + + var prevAddresses map[string][]string + if hasPrev { + prevAddresses = prev.NameserverAddresses } - w.state.SetDomainState(domain, &state.DomainState{ - Nameservers: nameservers, - LastChecked: now, - }) - - // Also look up A/AAAA records for the apex domain so that - // port and TLS checks (which read HostnameState) can find - // the domain's IP addresses. - records, err := w.resolver.LookupAllRecords(ctx, domain) - if err != nil { - w.log.Error( - "failed to lookup records for domain", - "domain", domain, - "error", err, - ) + addresses := w.resolveNameserverAddresses( + ctx, nameservers, prevAddresses, + ) + // A domain that does not exist has no records of its own: none are + // asked for, and those saved by an earlier check are removed. + // Otherwise the apex domain's records are also checked and saved as + // a hostname's, so that the port and TLS checks find its addresses. + // Notifications about them name it as a domain (see nameLine). When + // their lookup fails, the check fails as when the NS lookup does, so + // the saved delegation and the saved records come from the same + // check, as detectNSDisappearances needs. + if nxdomain { + w.state.DeleteHostnameState(domain) + } else if !w.checkHostname(ctx, domain) { return } - prevHS, hasPrevHS := w.state.GetHostnameState(domain) - if hasPrevHS && !w.firstRun { - w.detectHostnameChanges(ctx, domain, prevHS, records) + if hasPrev && !w.firstRun { + w.detectNSChanges(ctx, domain, prev.Nameservers, nameservers) + w.detectNSAddressChanges(ctx, domain, prevAddresses, addresses) } - newState := buildHostnameState(records, now) - w.state.SetHostnameState(domain, newState) + w.state.SetDomainState(domain, &state.DomainState{ + Nameservers: nameservers, + NameserverAddresses: addresses, + NXDomain: nxdomain, + LastChecked: now, + }) } func (w *Watcher) detectNSChanges( @@ -310,34 +416,255 @@ func (w *Watcher) detectNSChanges( ) } +// resolveNameserverAddresses returns the sorted addresses each +// nameserver's name resolves to. A nameserver whose lookup fails, as it +// does when no nameserver of the name's zone answers, or finds no +// address keeps its addresses from prev and is not an address change. +func (w *Watcher) resolveNameserverAddresses( + ctx context.Context, + nameservers []string, + prev map[string][]string, +) map[string][]string { + addresses := make(map[string][]string, len(nameservers)) + + for _, ns := range nameservers { + ips, err := w.resolver.ResolveIPAddresses(ctx, ns) + if err == nil && len(ips) > 0 { + sort.Strings(ips) + addresses[ns] = ips + + continue + } + + w.logFailedLookup( + ctx, + "no addresses found for nameserver", + "nameserver", ns, + "error", err, + ) + + if prevIPs, ok := prev[ns]; ok { + addresses[ns] = prevIPs + } + } + + return addresses +} + +// detectNSAddressChanges notifies when a nameserver in both checks +// resolves to different addresses. A nameserver added or removed is +// reported by detectNSChanges alone, and one with no addresses saved by +// the previous check, as in a state file from before they were saved, +// is not compared. +func (w *Watcher) detectNSAddressChanges( + ctx context.Context, + domain string, + prev, current map[string][]string, +) { + for ns, cur := range current { + old, ok := prev[ns] + if !ok || sliceEqual(old, cur) { + continue + } + + msg := fmt.Sprintf( + "Domain: %s\nNameserver: %s\nOld: %s\nNew: %s", + domain, ns, + strings.Join(old, ", "), + strings.Join(cur, ", "), + ) + + w.notify.SendNotification( + ctx, + "NS Address Change: "+domain, + msg, + "warning", + ) + } +} + +// checkHostname looks up hostname's records and checks them (see +// checkHostnameResult). It reports whether it saved them. func (w *Watcher) checkHostname( ctx context.Context, hostname string, -) { - records, err := w.resolver.LookupAllRecords(ctx, hostname) +) bool { + results, err := w.resolver.LookupAllRecords(ctx, hostname) if err != nil { - w.log.Error( + w.logFailedLookup( + ctx, "failed to lookup records", "hostname", hostname, "error", err, ) + } + + return w.checkHostnameResult(ctx, hostname, results, err) +} + +// checkHostnameResult finishes a check of hostname from results and err, +// what the lookup of its records returned, and reports whether it saved +// them. When the lookup failed, it saves nothing, unless the nameservers +// that did not answer are those of the previous check (see +// failedNameservers). +func (w *Watcher) checkHostnameResult( + ctx context.Context, + hostname string, + results map[string]*resolver.NameserverResponse, + err error, +) bool { + prev, _ := w.state.GetHostnameState(hostname) + + if err != nil { + results = failedNameservers(err, prev) + if results == nil { + return false + } + } + + w.updateHostnameState( + ctx, hostname, buildHostnameState(results, prev, time.Now().UTC()), + ) + + return true +} + +// failedNameservers returns a failed response for each nameserver in +// prev, a name's state from the previous check, with err as the reason, +// when err says that the walk to the name's nameservers reached their +// zone and none of them answered. For any other error, as when the walk +// got no answer before it reached their zone, or when prev is nil, it +// returns nil. +func failedNameservers( + err error, + prev *state.HostnameState, +) map[string]*resolver.NameserverResponse { + var noAnswer *resolver.NoAnswerError + if !errors.As(err, &noAnswer) || prev == nil { + return nil + } + + saved := slices.Sorted(maps.Keys(prev.RecordsByNameserver)) + if !slices.Equal(noAnswer.Nameservers, saved) { + return nil + } + + results := make(map[string]*resolver.NameserverResponse) + + for _, ns := range saved { + results[ns] = &resolver.NameserverResponse{ + Nameserver: ns, + Records: make(map[string][]string), + Status: resolver.StatusError, + Error: err.Error(), + } + } + + return results +} + +// updateHostnameState finishes a check of hostname from newState, built +// from its nameservers' answers: it follows the CNAME in them, notifies +// what changed since the previous check, and saves newState. +func (w *Watcher) updateHostnameState( + ctx context.Context, + hostname string, + newState *state.HostnameState, +) { + prev, hasPrev := w.state.GetHostnameState(hostname) + + w.resolveCNAMEAddresses(ctx, hostname, newState, prev) + + if hasPrev && !w.firstRun { + w.detectHostnameChanges(ctx, hostname, prev, newState) + } + + w.state.SetHostnameState(hostname, newState) +} + +// resolveCNAMEAddresses saves in current the addresses at the end of +// hostname's CNAME chain, when the nameservers' answers in current hold +// a CNAME and no address, and an empty list otherwise. Every CNAME +// target the nameservers gave is followed with ResolveIPAddresses and +// the addresses found for all of them are saved, so nameservers that +// disagree on the target do not change the result from check to check. +// The addresses saved in prev, which may be nil, are kept when none of +// the name's nameservers answered its queries for A, AAAA and CNAME, +// and when a target cannot be followed, as when no nameserver of a zone +// in its chain answers. +func (w *Watcher) resolveCNAMEAddresses( + ctx context.Context, + hostname string, + current, prev *state.HostnameState, +) { + var prevAddresses []string + if prev != nil { + prevAddresses = prev.CNAMEAddresses + } + + // Empty, not nil: nil means the addresses are not known. + current.CNAMEAddresses = []string{} + + answered := false + targets := make(map[string]bool) + + for _, nsState := range current.RecordsByNameserver { + if nsState.Status != statusOK || + slices.Contains(nsState.FailedTypes, "A") || + slices.Contains(nsState.FailedTypes, "AAAA") || + slices.Contains(nsState.FailedTypes, "CNAME") { + continue + } + + answered = true + + if len(nsState.Records["A"]) > 0 || len(nsState.Records["AAAA"]) > 0 { + return + } + + for _, target := range nsState.Records["CNAME"] { + targets[target] = true + } + } + + if !answered { + current.CNAMEAddresses = prevAddresses return } - now := time.Now().UTC() - prev, hasPrev := w.state.GetHostnameState(hostname) + for target := range targets { + ips, err := w.resolver.ResolveIPAddresses(ctx, target) + if err != nil { + w.logFailedLookup( + ctx, + "failed to follow CNAME", + "hostname", hostname, + "target", target, + "error", err, + ) - if hasPrev && !w.firstRun { - w.detectHostnameChanges(ctx, hostname, prev, records) + current.CNAMEAddresses = prevAddresses + + return + } + + current.CNAMEAddresses = append(current.CNAMEAddresses, ips...) } - newState := buildHostnameState(records, now) - w.state.SetHostnameState(hostname, newState) + // Still the empty list when every chain ends in no address. + slices.Sort(current.CNAMEAddresses) + current.CNAMEAddresses = slices.Compact(current.CNAMEAddresses) } +// buildHostnameState saves each nameserver's response. A nameserver +// that answered, even with NXDOMAIN or no records, is saved as ok, with +// the record types whose query failed; one that timed out or failed is +// saved as error with the reason, and its empty record set is not an +// answer. prev is the hostname's state from the previous check, or nil. func buildHostnameState( - records map[string]map[string][]string, + results map[string]*resolver.NameserverResponse, + prev *state.HostnameState, now time.Time, ) *state.HostnameState { hs := &state.HostnameState{ @@ -347,49 +674,177 @@ func buildHostnameState( LastChecked: now, } - for ns, recs := range records { - hs.RecordsByNameserver[ns] = &state.NameserverRecordState{ - Records: recs, + for ns, resp := range results { + nsState := &state.NameserverRecordState{ + Records: maps.Clone(resp.Records), Status: statusOK, LastChecked: now, } + + if resp.Status == resolver.StatusTimeout || + resp.Status == resolver.StatusError { + nsState.Status = statusError + nsState.Error = resp.Error + } else { + nsState.FailedTypes = resp.FailedTypes + + var prevNS *state.NameserverRecordState + if prev != nil { + prevNS = prev.RecordsByNameserver[ns] + } + + keepFailedTypes(nsState, prevNS) + } + + hs.RecordsByNameserver[ns] = nsState } return hs } -func (w *Watcher) detectHostnameChanges( - ctx context.Context, - hostname string, - prev *state.HostnameState, - current map[string]map[string][]string, -) { - w.detectRecordChanges(ctx, hostname, prev, current) - w.detectNSDisappearances(ctx, hostname, prev, current) - w.detectInconsistencies(ctx, hostname, current) -} +// keepFailedTypes copies into nsState, for each record type in its +// FailedTypes, the records prevNS, the nameserver's state from the +// previous check, holds for that type, which may be none. When prevNS +// does not know them either, because the nameserver was new or failing +// then or the type was in its UnknownTypes, the type goes in +// nsState.UnknownTypes instead. +func keepFailedTypes(nsState, prevNS *state.NameserverRecordState) { + for _, rtype := range nsState.FailedTypes { + if prevNS == nil || prevNS.Status != statusOK || + slices.Contains(prevNS.UnknownTypes, rtype) { + nsState.UnknownTypes = append(nsState.UnknownTypes, rtype) -func (w *Watcher) detectRecordChanges( - ctx context.Context, - hostname string, - prev *state.HostnameState, - current map[string]map[string][]string, -) { - for ns, recs := range current { - prevNS, ok := prev.RecordsByNameserver[ns] - if !ok { continue } - if recordsEqual(prevNS.Records, recs) { + if records, ok := prevNS.Records[rtype]; ok { + nsState.Records[rtype] = records + } + } +} + +func (w *Watcher) detectHostnameChanges( + ctx context.Context, + hostname string, + prev, current *state.HostnameState, +) { + w.detectRecordChanges(ctx, hostname, prev, current) + w.detectNSDisappearances(ctx, hostname, prev, current) + w.detectNSFailures(ctx, hostname, prev, current) + w.detectInconsistencies(ctx, hostname, prev, current) + w.detectCNAMEAddressChanges(ctx, hostname, prev, current) +} + +// isDomain reports whether name is a configured apex domain, whose own +// records are checked and saved as a hostname's are. +func (w *Watcher) isDomain(name string) bool { + return slices.Contains(w.config.Domains, name) +} + +// nameLine is the line a notification about name's records starts with: +// "Domain: " and the name for a configured apex domain, and +// "Hostname: " otherwise. +func (w *Watcher) nameLine(name string) string { + if w.isDomain(name) { + return "Domain: " + name + } + + return "Hostname: " + name +} + +// portNameLines lists the names that resolve to a port's address, the +// configured apex domains on one line and the hostnames on the next, +// leaving out a line that would name nothing. +func (w *Watcher) portNameLines(names []string) string { + var domains, hostnames []string + + for _, name := range names { + if w.isDomain(name) { + domains = append(domains, name) + } else { + hostnames = append(hostnames, name) + } + } + + var lines []string + + if len(domains) > 0 { + lines = append(lines, "Domains: "+strings.Join(domains, ", ")) + } + + if len(hostnames) > 0 { + lines = append(lines, "Hostnames: "+strings.Join(hostnames, ", ")) + } + + return strings.Join(lines, "\n") +} + +// detectCNAMEAddressChanges notifies when the addresses at the end of +// hostname's CNAME chain differ from those the previous check saved, +// including a change from or to none. When the previous addresses are +// not known (nil), as on the first check after loading a state file +// written before they were saved, nothing is compared. +func (w *Watcher) detectCNAMEAddressChanges( + ctx context.Context, + hostname string, + prev, current *state.HostnameState, +) { + old, cur := prev.CNAMEAddresses, current.CNAMEAddresses + if old == nil || sliceEqual(old, cur) { + return + } + + msg := fmt.Sprintf( + "%s\nOld: %s\nNew: %s", + w.nameLine(hostname), + strings.Join(old, ", "), + strings.Join(cur, ", "), + ) + + w.notify.SendNotification( + ctx, + "CNAME Address Change: "+hostname, + msg, + "warning", + ) +} + +// detectRecordChanges compares each nameserver's records with those of +// the previous check. Only answers are compared: a nameserver that +// failed on either check has no records to compare. The records kept +// for a record type whose query failed are compared too, but not those +// of a type in UnknownTypes on either check, which the message leaves +// out as well. Nothing is sent for a name in SkipRecordNotifications. +func (w *Watcher) detectRecordChanges( + ctx context.Context, + hostname string, + prev, current *state.HostnameState, +) { + if slices.Contains(w.config.SkipRecordNotifications, hostname) { + return + } + + for ns, cur := range current.RecordsByNameserver { + prevNS, ok := prev.RecordsByNameserver[ns] + if !ok || prevNS.Status != statusOK || cur.Status != statusOK { + continue + } + + unknown := slices.Concat(prevNS.UnknownTypes, cur.UnknownTypes) + oldRecords := withoutTypes(prevNS.Records, unknown) + newRecords := withoutTypes(cur.Records, unknown) + + if recordsEqual(oldRecords, newRecords) { continue } msg := fmt.Sprintf( - "Hostname: %s\nNameserver: %s\n"+ - "Old: %v\nNew: %v", - hostname, ns, - prevNS.Records, recs, + "%s\nNameserver: %s\n%s", + w.nameLine(hostname), ns, + recordDifferences( + "Old", oldRecords, + "New", newRecords, + ), ) w.notify.SendNotification( @@ -401,75 +856,126 @@ func (w *Watcher) detectRecordChanges( } } +// detectNSDisappearances sends one NS Change listing the nameservers of +// the previous check that are no longer those of the zone hostname is +// in. For a configured apex domain it leaves out those of the domain's +// saved delegation, which detectNSChanges notifies when they are removed +// from it. checkDomain saves a new delegation only after this runs, and +// only together with the records, so the saved delegation and the +// nameservers in prev come from the same check. func (w *Watcher) detectNSDisappearances( ctx context.Context, hostname string, - prev *state.HostnameState, - current map[string]map[string][]string, + prev, current *state.HostnameState, ) { - for ns, prevNS := range prev.RecordsByNameserver { - if _, ok := current[ns]; ok || prevNS.Status != statusOK { - continue + var delegation []string + + if w.isDomain(hostname) { + if domainState, ok := w.state.GetDomainState(hostname); ok { + delegation = domainState.Nameservers } - - msg := fmt.Sprintf( - "Hostname: %s\nNameserver: %s disappeared", - hostname, ns, - ) - - w.notify.SendNotification( - ctx, - "NS Failure: "+hostname, - msg, - "error", - ) } - for ns := range current { + var removed []string + + for ns := range prev.RecordsByNameserver { + _, ok := current.RecordsByNameserver[ns] + if !ok && !slices.Contains(delegation, ns) { + removed = append(removed, ns) + } + } + + if len(removed) == 0 { + return + } + + slices.Sort(removed) + + msg := fmt.Sprintf( + "%s\nRemoved: %s", + w.nameLine(hostname), + strings.Join(removed, ", "), + ) + + w.notify.SendNotification( + ctx, + "NS Change: "+hostname, + msg, + "warning", + ) +} + +// detectNSFailures notifies when a nameserver that answered on the +// previous check fails, and when one that failed answers again. A +// nameserver missing from the previous check is not compared. +func (w *Watcher) detectNSFailures( + ctx context.Context, + hostname string, + prev, current *state.HostnameState, +) { + for ns, cur := range current.RecordsByNameserver { prevNS, ok := prev.RecordsByNameserver[ns] - if !ok || prevNS.Status != statusError { + if !ok { continue } - msg := fmt.Sprintf( - "Hostname: %s\nNameserver: %s recovered", - hostname, ns, - ) + switch { + case prevNS.Status == statusOK && cur.Status == statusError: + msg := fmt.Sprintf( + "%s\nNameserver: %s\nError: %s", + w.nameLine(hostname), ns, cur.Error, + ) - w.notify.SendNotification( - ctx, - "NS Recovery: "+hostname, - msg, - "success", - ) + w.notify.SendNotification( + ctx, + "NS Failure: "+hostname, + msg, + "error", + ) + case prevNS.Status == statusError && cur.Status == statusOK: + msg := fmt.Sprintf( + "%s\nNameserver: %s recovered", + w.nameLine(hostname), ns, + ) + + w.notify.SendNotification( + ctx, + "NS Recovery: "+hostname, + msg, + "success", + ) + } } } +// detectInconsistencies notifies each pair of nameservers that newly +// disagree (see newlyDisagreeingPairs). Nothing is sent for a name in +// SkipRecordNotifications. func (w *Watcher) detectInconsistencies( ctx context.Context, hostname string, - current map[string]map[string][]string, + prev, current *state.HostnameState, ) { - nameservers := make([]string, 0, len(current)) - for ns := range current { - nameservers = append(nameservers, ns) + if slices.Contains(w.config.SkipRecordNotifications, hostname) { + return } - sort.Strings(nameservers) + for _, pair := range newlyDisagreeingPairs(prev, current) { + ns1, ns2 := pair[0], pair[1] + state1 := current.RecordsByNameserver[ns1] + state2 := current.RecordsByNameserver[ns2] - for i := range len(nameservers) - 1 { - ns1 := nameservers[i] - ns2 := nameservers[i+1] - - if recordsEqual(current[ns1], current[ns2]) { - continue - } + // The record types left out of the comparison are left out of + // the message too. + failed := slices.Concat(state1.FailedTypes, state2.FailedTypes) msg := fmt.Sprintf( - "Hostname: %s\n%s: %v\n%s: %v", - hostname, - ns1, current[ns1], - ns2, current[ns2], + "%s\n%s", + w.nameLine(hostname), + recordDifferences( + ns1, withoutTypes(state1.Records, failed), + ns2, withoutTypes(state2.Records, failed), + ), ) w.notify.SendNotification( @@ -481,6 +987,53 @@ func (w *Watcher) detectInconsistencies( } } +// newlyDisagreeingPairs returns every pair of nameservers that answered +// in current and whose records differ there, in sorted order of name, +// except pairs where both nameservers answered in prev and already +// differed there. A nameserver missing from prev, or that failed there, +// is paired with every nameserver it differs from. A nameserver that +// failed in current has no records to compare and is in no pair. In +// both checks, a record type whose query failed at either nameserver is +// not compared. +func newlyDisagreeingPairs( + prev, current *state.HostnameState, +) [][2]string { + nameservers := make([]string, 0, len(current.RecordsByNameserver)) + for ns, cur := range current.RecordsByNameserver { + if cur.Status == statusOK { + nameservers = append(nameservers, ns) + } + } + + sort.Strings(nameservers) + + var pairs [][2]string + + for i, ns1 := range nameservers { + for _, ns2 := range nameservers[i+1:] { + if nameserversAgree( + current.RecordsByNameserver[ns1], + current.RecordsByNameserver[ns2], + ) { + continue + } + + prev1, ok1 := prev.RecordsByNameserver[ns1] + prev2, ok2 := prev.RecordsByNameserver[ns2] + + if ok1 && ok2 && + prev1.Status == statusOK && prev2.Status == statusOK && + !nameserversAgree(prev1, prev2) { + continue + } + + pairs = append(pairs, [2]string{ns1, ns2}) + } + } + + return pairs +} + func (w *Watcher) checkAllPorts(ctx context.Context) { // Phase 1: Build current IP:port → hostname associations // from fresh DNS data. @@ -498,8 +1051,11 @@ func (w *Watcher) checkAllPorts(ctx context.Context) { } // Phase 3: Remove port state entries that no longer have - // any hostname referencing them. + // any hostname referencing them, and certificate entries for + // an address whose port 443 is closed or that their name no + // longer has. w.cleanupStalePorts(associations) + w.cleanupStaleCertificates() } // buildPortAssociations constructs a map from IP:port keys to @@ -518,7 +1074,7 @@ func (w *Watcher) buildPortAssociations() map[string][]string { ips := w.collectIPs(name) for _, ip := range ips { for _, port := range monitoredPorts { - key := fmt.Sprintf("%s:%d", ip, port) + key := net.JoinHostPort(ip, strconv.Itoa(port)) if assoc[key] == nil { assoc[key] = make(map[string]bool) } @@ -543,18 +1099,15 @@ func (w *Watcher) buildPortAssociations() map[string][]string { return result } -// parsePortKey splits an "ip:port" key into its components. +// parsePortKey splits an "ip:port" key, written by net.JoinHostPort, +// into its components. func parsePortKey(key string) (string, int) { - lastColon := strings.LastIndex(key, ":") - if lastColon < 0 { + ip, port, err := net.SplitHostPort(key) + if err != nil { return key, 0 } - ip := key[:lastColon] - - var p int - - _, err := fmt.Sscanf(key[lastColon+1:], "%d", &p) + p, err := strconv.Atoi(port) if err != nil { return ip, 0 } @@ -563,17 +1116,93 @@ func parsePortKey(key string) (string, int) { } // cleanupStalePorts removes port state entries that are no -// longer referenced by any hostname in the current DNS data. +// longer referenced by any hostname in the current DNS data. An +// entry saved for a configured name none of whose nameservers +// answered is kept: that name's addresses are not known, not gone. func (w *Watcher) cleanupStalePorts( currentAssociations map[string][]string, ) { for _, key := range w.state.GetAllPortKeys() { - if _, exists := currentAssociations[key]; !exists { - w.state.DeletePortState(key) + if _, exists := currentAssociations[key]; exists { + continue } + + ps, ok := w.state.GetPortState(key) + if ok && slices.ContainsFunc(ps.Hostnames, w.noNameserverAnswered) { + continue + } + + w.state.DeletePortState(key) } } +// cleanupStaleCertificates removes the certificate entries for an +// address whose port 443 was found closed, and for an address their +// name no longer resolves to. An entry saved for a name none of whose +// nameservers answered is kept unless its port is closed: that name's +// addresses are not known, not gone. +func (w *Watcher) cleanupStaleCertificates() { + for _, key := range w.state.GetAllCertificateKeys() { + ip, hostname := parseCertKey(key) + + ps, ok := w.state.GetPortState(net.JoinHostPort(ip, strconv.Itoa(tlsPort))) + if ok && !ps.Open { + w.state.DeleteCertificateState(key) + + continue + } + + if slices.Contains(w.collectIPs(hostname), ip) || + w.noNameserverAnswered(hostname) { + continue + } + + w.state.DeleteCertificateState(key) + } +} + +// parseCertKey splits an "ip:port:hostname" certificate key into its +// address and hostname. +func parseCertKey(key string) (string, string) { + lastColon := strings.LastIndex(key, ":") + if lastColon < 0 { + return "", key + } + + ip, _ := parsePortKey(key[:lastColon]) + + return ip, key[lastColon+1:] +} + +// isConfigured reports whether name is a configured domain or hostname. +func (w *Watcher) isConfigured(name string) bool { + return w.isDomain(name) || slices.Contains(w.config.Hostnames, name) +} + +// noNameserverAnswered reports whether name is a configured domain or +// hostname and none of its nameservers answered on its last check. +func (w *Watcher) noNameserverAnswered(name string) bool { + if !w.isConfigured(name) { + return false + } + + hs, ok := w.state.GetHostnameState(name) + if !ok { + return false + } + + for _, nsState := range hs.RecordsByNameserver { + if nsState.Status == statusOK { + return false + } + } + + return true +} + +// collectIPs returns the addresses saved for hostname: those in its +// nameservers' A and AAAA records, and those at the end of its CNAME +// chain. func (w *Watcher) collectIPs(hostname string) []string { hs, ok := w.state.GetHostnameState(hostname) if !ok { @@ -592,6 +1221,10 @@ func (w *Watcher) collectIPs(hostname string) []string { } } + for _, ip := range hs.CNAMEAddresses { + ipSet[ip] = true + } + result := make([]string, 0, len(ipSet)) for ip := range ipSet { result = append(result, ip) @@ -609,6 +1242,13 @@ func (w *Watcher) checkSinglePort( hostnames []string, ) { result, err := w.portCheck.CheckPort(ctx, ip, port) + + // A check the context cut short says nothing about the port, so it + // is neither saved nor notified. + if ctx.Err() != nil { + return + } + if err != nil { w.log.Error( "port check failed", @@ -620,7 +1260,7 @@ func (w *Watcher) checkSinglePort( return } - key := fmt.Sprintf("%s:%d", ip, port) + key := net.JoinHostPort(ip, strconv.Itoa(port)) now := time.Now().UTC() prev, hasPrev := w.state.GetPortState(key) @@ -631,8 +1271,8 @@ func (w *Watcher) checkSinglePort( } msg := fmt.Sprintf( - "Hosts: %s\nAddress: %s\nPort now %s", - strings.Join(hostnames, ", "), key, stateStr, + "%s\nAddress: %s\nPort now %s", + w.portNameLines(hostnames), key, stateStr, ) w.notify.SendNotification( @@ -643,9 +1283,24 @@ func (w *Watcher) checkSinglePort( ) } + // A configured name on the saved list none of whose nameservers + // answered stays on it, so the entry is kept when the other names + // stop resolving to this address. + savedHostnames := slices.Clone(hostnames) + + if hasPrev { + for _, name := range prev.Hostnames { + if !slices.Contains(hostnames, name) && w.noNameserverAnswered(name) { + savedHostnames = append(savedHostnames, name) + } + } + + sort.Strings(savedHostnames) + } + w.state.SetPortState(key, &state.PortState{ Open: result.Open, - Hostnames: hostnames, + Hostnames: savedHostnames, LastChecked: now, }) } @@ -667,7 +1322,7 @@ func (w *Watcher) checkTLSForHostname( ips := w.collectIPs(hostname) for _, ip := range ips { - portKey := fmt.Sprintf("%s:%d", ip, tlsPort) + portKey := net.JoinHostPort(ip, strconv.Itoa(tlsPort)) ps, ok := w.state.GetPortState(portKey) if !ok || !ps.Open { @@ -684,7 +1339,14 @@ func (w *Watcher) checkTLSCert( hostname string, ) { cert, err := w.tlsCheck.CheckCertificate(ctx, ip, hostname) - certKey := fmt.Sprintf("%s:%d:%s", ip, tlsPort, hostname) + + // A check the context cut short says nothing about the certificate, + // so it is neither saved nor notified. + if ctx.Err() != nil { + return + } + + certKey := net.JoinHostPort(ip, strconv.Itoa(tlsPort)) + ":" + hostname now := time.Now().UTC() prev, hasPrev := w.state.GetCertificateState(certKey) @@ -822,22 +1484,6 @@ func (w *Watcher) checkTLSExpiry( return } - // Deduplicate expiry warnings: don't re-notify for the same - // hostname within the TLS check interval. - dedupKey := fmt.Sprintf("expiry:%s:%s", hostname, ip) - - w.expiryNotifiedMu.Lock() - - lastNotified, seen := w.expiryNotified[dedupKey] - if seen && time.Since(lastNotified) < w.config.TLSInterval { - w.expiryNotifiedMu.Unlock() - - return - } - - w.expiryNotified[dedupKey] = time.Now() - w.expiryNotifiedMu.Unlock() - msg := fmt.Sprintf( "Host: %s\nIP: %s\nCN: %s\n"+ "Expires: %s (%.0f days)", @@ -863,8 +1509,11 @@ func (w *Watcher) saveState() { // maybeSendTestNotification sends a startup status notification // after the first full scan completes, if SEND_TEST_NOTIFICATION -// is enabled. The message is clearly informational ("all ok") -// and not an error or anomaly alert. +// is enabled. The message is informational, not an error or anomaly +// alert. It is written before it reaches any endpoint, so it claims +// nothing about whether the endpoints work. Domains and hostnames are +// counted from the configuration: the state's hostnames also hold each +// apex domain's own records. func (w *Watcher) maybeSendTestNotification(ctx context.Context) { if !w.config.SendTestNotification { return @@ -876,9 +1525,10 @@ func (w *Watcher) maybeSendTestNotification(ctx context.Context) { "dnswatcher has started and completed its initial scan.\n"+ "Monitoring %d domain(s) and %d hostname(s).\n"+ "Tracking %d port endpoint(s) and %d TLS certificate(s).\n"+ - "All notification channels are working.", - len(snap.Domains), - len(snap.Hostnames), + "This is a test notification, sent to every configured "+ + "notification endpoint.", + len(w.config.Domains), + len(w.config.Hostnames), len(snap.Ports), len(snap.Certificates), ) @@ -904,6 +1554,33 @@ func toSet(items []string) map[string]bool { return set } +// nameserversAgree reports whether two nameservers' states from the same +// check hold the same records, leaving out the record types either lists +// in FailedTypes: the records held for those are kept from an earlier +// check, or not known. +func nameserversAgree(a, b *state.NameserverRecordState) bool { + failed := slices.Concat(a.FailedTypes, b.FailedTypes) + + return recordsEqual( + withoutTypes(a.Records, failed), withoutTypes(b.Records, failed), + ) +} + +// withoutTypes returns a copy of records without the record types in +// types. +func withoutTypes( + records map[string][]string, + types []string, +) map[string][]string { + records = maps.Clone(records) + + for _, rtype := range types { + delete(records, rtype) + } + + return records +} + func recordsEqual( a, b map[string][]string, ) bool { @@ -921,6 +1598,54 @@ func recordsEqual( return true } +// recordDifferences describes, in sorted order of type, each record +// type whose values differ between a and b: a line naming the type, +// then a line with a's values after labelA and one with b's after +// labelB. Types with the same values in both are left out. +func recordDifferences( + labelA string, a map[string][]string, + labelB string, b map[string][]string, +) string { + types := make([]string, 0, len(a)+len(b)) + + for recordType := range a { + types = append(types, recordType) + } + + for recordType := range b { + if _, ok := a[recordType]; !ok { + types = append(types, recordType) + } + } + + sort.Strings(types) + + var lines []string + + for _, recordType := range types { + if sliceEqual(a[recordType], b[recordType]) { + continue + } + + lines = append(lines, + "Type: "+recordType, + labelA+": "+joinValues(a[recordType]), + labelB+": "+joinValues(b[recordType]), + ) + } + + return strings.Join(lines, "\n") +} + +// joinValues lists record values separated by commas, or says none. +func joinValues(values []string) string { + if len(values) == 0 { + return "none" + } + + return strings.Join(values, ", ") +} + func sliceEqual(a, b []string) bool { if len(a) != len(b) { return false diff --git a/internal/watcher/watcher_test.go b/internal/watcher/watcher_test.go index 0069af2..79feb23 100644 --- a/internal/watcher/watcher_test.go +++ b/internal/watcher/watcher_test.go @@ -2,151 +2,94 @@ package watcher_test import ( "context" - "errors" "fmt" + "log/slog" + "net" + "os" + "slices" + "strings" "sync" "testing" "time" + "go.uber.org/fx/fxtest" + "sneak.berlin/go/dnswatcher/internal/config" + "sneak.berlin/go/dnswatcher/internal/globals" + "sneak.berlin/go/dnswatcher/internal/livednstest" + "sneak.berlin/go/dnswatcher/internal/logger" "sneak.berlin/go/dnswatcher/internal/portcheck" + "sneak.berlin/go/dnswatcher/internal/resolver" "sneak.berlin/go/dnswatcher/internal/state" "sneak.berlin/go/dnswatcher/internal/tlscheck" "sneak.berlin/go/dnswatcher/internal/watcher" ) -// errNotFound is returned when mock data is missing. -var errNotFound = errors.New("not found") - -// Fixture values shared across tests. +// The watcher looks these names up in live DNS with the real resolver, +// so tests assert on what the watcher does with the answers, never on +// the records these zones publish. The nameservers of testHost and +// testSmallDomain stay the same between a test looking them up and its +// check. Every query a check sends is one more that can be lost, so the +// tests keep them few. A check asks each of a name's nameservers about +// every record type, and both names have two. A domain check also looks +// up each nameserver's addresses at every nameserver of the zone that +// nameserver is in: testSmallDomain's nameservers are in zones with two +// nameservers, while a domain whose nameservers are in, say, +// cloudflare.com, which has five, makes each domain check much longer. +// A test checks a domain only when it is about domains, and checks once, +// from saved state it builds, rather than twice. The tests that query +// testDomain's nameservers directly do no domain check. const ( - testDomain = "example.com" - testHost = "www.example.com" - testNS1 = "ns1.example.com." - testNS2 = "ns2.example.com." - testIPv4 = "93.184.216.34" - testIP = "1.2.3.4" - testIssuer = "DigiCert" + testDomain = "google.com" + testSmallDomain = "desec.io" + testHost = "example.org" + testIssuer = "DigiCert" ) -// --- Mock implementations --- +// Saved-state values that live DNS never returns: nameserver names +// under .invalid and a documentation address. +const ( + oldNS1 = "ns1.example.invalid." + oldNS2 = "ns2.example.invalid." + oldIP = "192.0.2.1" +) -type mockResolver struct { - mu sync.Mutex - nsRecords map[string][]string - allRecords map[string]map[string]map[string][]string - ipAddresses map[string][]string - lookupNSErr error - allRecordsErr error - resolveIPErr error - lookupNSCalls int - allRecordCalls int -} - -func (m *mockResolver) LookupNS( - _ context.Context, - domain string, -) ([]string, error) { - m.mu.Lock() - defer m.mu.Unlock() - - m.lookupNSCalls++ - - if m.lookupNSErr != nil { - return nil, m.lookupNSErr - } - - ns, ok := m.nsRecords[domain] - if !ok { - return nil, fmt.Errorf( - "%w: NS for %s", errNotFound, domain, - ) - } - - return ns, nil -} - -func (m *mockResolver) LookupAllRecords( - _ context.Context, - hostname string, -) (map[string]map[string][]string, error) { - m.mu.Lock() - defer m.mu.Unlock() - - m.allRecordCalls++ - - if m.allRecordsErr != nil { - return nil, m.allRecordsErr - } - - recs, ok := m.allRecords[hostname] - if !ok { - return nil, fmt.Errorf( - "%w: records for %s", errNotFound, hostname, - ) - } - - return recs, nil -} - -func (m *mockResolver) ResolveIPAddresses( - _ context.Context, - hostname string, -) ([]string, error) { - m.mu.Lock() - defer m.mu.Unlock() - - if m.resolveIPErr != nil { - return nil, m.resolveIPErr - } - - ips, ok := m.ipAddresses[hostname] - if !ok { - return nil, fmt.Errorf( - "%w: IPs for %s", errNotFound, hostname, - ) - } - - return ips, nil -} +// --- Stand-ins for the port checker, TLS checker and notifier --- +// +// DNS has none: the watchers built here use the real resolver (see +// TESTING.md). +// mockPortChecker reports every port open until closed is set. type mockPortChecker struct { - mu sync.Mutex - results map[string]bool - err error - calls int + mu sync.Mutex + closed bool + calls int } func (m *mockPortChecker) CheckPort( _ context.Context, - address string, - port int, + _ string, + _ int, ) (*portcheck.PortResult, error) { m.mu.Lock() defer m.mu.Unlock() m.calls++ - if m.err != nil { - return nil, m.err - } - - key := fmt.Sprintf("%s:%d", address, port) - open := m.results[key] - - return &portcheck.PortResult{Open: open}, nil + return &portcheck.PortResult{Open: !m.closed}, nil } +// mockTLSChecker returns a certificate for the requested hostname that +// expires at notAfter. type mockTLSChecker struct { - mu sync.Mutex - certs map[string]*tlscheck.CertificateInfo - err error - calls int + mu sync.Mutex + notAfter time.Time + calls int } func (m *mockTLSChecker) CheckCertificate( _ context.Context, - ip string, + _ string, hostname string, ) (*tlscheck.CertificateInfo, error) { m.mu.Lock() @@ -154,20 +97,12 @@ func (m *mockTLSChecker) CheckCertificate( m.calls++ - if m.err != nil { - return nil, m.err - } - - key := fmt.Sprintf("%s:%s", ip, hostname) - cert, ok := m.certs[key] - - if !ok { - return nil, fmt.Errorf( - "%w: cert for %s", errNotFound, key, - ) - } - - return cert, nil + return &tlscheck.CertificateInfo{ + CommonName: hostname, + Issuer: testIssuer, + NotAfter: m.notAfter, + SubjectAlternativeNames: []string{hostname}, + }, nil } type notification struct { @@ -205,15 +140,31 @@ func (m *mockNotifier) getNotifications() []notification { return result } -// --- Helper to build a Watcher for testing --- +// cancelOnNotify is a mockNotifier that calls cancel at each +// notification, as if shutdown cut the check short there. +type cancelOnNotify struct { + *mockNotifier + + cancel context.CancelFunc +} + +func (n cancelOnNotify) SendNotification( + ctx context.Context, + title, message, priority string, +) { + n.mockNotifier.SendNotification(ctx, title, message, priority) + n.cancel() +} + +// --- Helpers to build a Watcher and run its checks against live DNS --- type testDeps struct { - resolver *mockResolver portChecker *mockPortChecker tlsChecker *mockTLSChecker notifier *mockNotifier state *state.State config *config.Config + log *logger.Logger } func newTestWatcher( @@ -222,28 +173,12 @@ func newTestWatcher( ) (*watcher.Watcher, *testDeps) { t.Helper() - deps := &testDeps{ - resolver: &mockResolver{ - nsRecords: make(map[string][]string), - allRecords: make(map[string]map[string]map[string][]string), - ipAddresses: make(map[string][]string), - }, - portChecker: &mockPortChecker{ - results: make(map[string]bool), - }, - tlsChecker: &mockTLSChecker{ - certs: make(map[string]*tlscheck.CertificateInfo), - }, - notifier: &mockNotifier{}, - config: cfg, - } - - deps.state = state.NewForTest() + deps := newTestDeps(t, cfg) w := watcher.NewForTest( deps.config, deps.state, - deps.resolver, + resolver.NewFromLogger(slog.Default()), deps.portChecker, deps.tlsChecker, deps.notifier, @@ -252,6 +187,40 @@ func newTestWatcher( return w, deps } +func newTestDeps(t *testing.T, cfg *config.Config) *testDeps { + t.Helper() + + deps := &testDeps{ + portChecker: &mockPortChecker{}, + tlsChecker: &mockTLSChecker{ + notAfter: time.Now().Add(90 * 24 * time.Hour), + }, + notifier: &mockNotifier{}, + config: cfg, + } + + g, err := globals.New(nil) + if err != nil { + t.Fatalf("globals.New: %v", err) + } + + deps.log, err = logger.New(nil, logger.Params{Globals: g}) + if err != nil { + t.Fatalf("logger.New: %v", err) + } + + // The watcher saves state after every check, into cfg.DataDir. + deps.state, err = state.New(fxtest.NewLifecycle(t), state.Params{ + Logger: deps.log, + Config: cfg, + }) + if err != nil { + t.Fatalf("state.New: %v", err) + } + + return deps +} + func defaultTestConfig(t *testing.T) *config.Config { t.Helper() @@ -263,55 +232,170 @@ func defaultTestConfig(t *testing.T) *config.Config { } } +// checkOnce runs the watcher's checks once and returns an error when a +// configured name has no hostname state saved by this check, or that +// state holds no address, or a configured domain's nameserver has no +// address saved or still has oldIP, which the tests save and live DNS +// never returns. Either live DNS gave no answer for the name, or the +// watcher saved no fresh result for it. +func checkOnce( + ctx context.Context, + w *watcher.Watcher, + deps *testDeps, +) error { + started := time.Now() + + w.RunOnce(ctx) + + names := slices.Concat(deps.config.Domains, deps.config.Hostnames) + + for _, name := range names { + hs, ok := deps.state.GetHostnameState(name) + if !ok || hs.LastChecked.Before(started) || + len(addresses(hs)) == 0 { + return fmt.Errorf( + "%s: %w, or the watcher saved no fresh "+ + "result for it", + name, livednstest.ErrNoAnswer, + ) + } + } + + for _, name := range deps.config.Domains { + ds, _ := deps.state.GetDomainState(name) + for _, ns := range ds.Nameservers { + ips := ds.NameserverAddresses[ns] + if len(ips) == 0 || slices.Contains(ips, oldIP) { + return fmt.Errorf( + "%s: nameserver %s: %w, or the watcher saved "+ + "no fresh addresses for it", + name, ns, livednstest.ErrNoAnswer, + ) + } + } + } + + return nil +} + +// runChecks builds a watcher, lets prepare set up the saved state and +// stand-ins it starts from, and runs its checks once against live DNS. +// When the check finds no fresh address for a name (see checkOnce), the +// watcher is thrown away and all of this runs again on a new one, so a +// failed attempt leaves nothing behind in the saved state, the +// stand-ins or the notifications. +func runChecks( + t *testing.T, + cfg *config.Config, + prepare func(deps *testDeps), +) (*watcher.Watcher, *testDeps) { + t.Helper() + + var ( + w *watcher.Watcher + deps *testDeps + ) + + livednstest.Retry(t, "watcher checks", func(ctx context.Context) error { + w, deps = newTestWatcher(t, cfg) + + if prepare != nil { + prepare(deps) + } + + return checkOnce(ctx, w, deps) + }) + + return w, deps +} + +// lookupNameservers returns the nameservers live DNS lists for name, +// for a test to save in the state its check starts from. +func lookupNameservers(t *testing.T, name string) []string { + t.Helper() + + res := resolver.NewFromLogger(slog.Default()) + + var nameservers []string + + livednstest.Retry(t, "LookupNS("+name+")", func(ctx context.Context) error { + var err error + + nameservers, err = res.LookupNS(ctx, name) + + return err + }) + + return nameservers +} + +// addresses returns the A and AAAA values saved for a hostname, and the +// addresses saved at the end of its CNAME chain. +func addresses(hs *state.HostnameState) []string { + var ips []string + + for _, nsState := range hs.RecordsByNameserver { + ips = append(ips, nsState.Records["A"]...) + ips = append(ips, nsState.Records["AAAA"]...) + } + + return append(ips, hs.CNAMEAddresses...) +} + +// assertNotified checks that a notification with this title and +// priority was sent. +func assertNotified( + t *testing.T, + deps *testDeps, + title, priority string, +) { + t.Helper() + + notifications := deps.notifier.getNotifications() + + for _, n := range notifications { + if n.Title == title && n.Priority == priority { + return + } + } + + t.Errorf( + "expected %s notification %q, got: %v", + priority, title, notifications, + ) +} + +// countNotifications counts the notifications sent with this title. +func countNotifications(deps *testDeps, title string) int { + count := 0 + + for _, n := range deps.notifier.getNotifications() { + if n.Title == title { + count++ + } + } + + return count +} + func TestFirstRunBaseline(t *testing.T) { t.Parallel() cfg := defaultTestConfig(t) - cfg.Domains = []string{testDomain} + cfg.Domains = []string{testSmallDomain} cfg.Hostnames = []string{testHost} - w, deps := newTestWatcher(t, cfg) - setupBaselineMocks(deps) - - w.RunOnce(t.Context()) + _, deps := runChecks(t, cfg, nil) assertNoNotifications(t, deps) assertStatePopulated(t, deps) -} -func setupBaselineMocks(deps *testDeps) { - deps.resolver.nsRecords[testDomain] = []string{ - testNS1, - testNS2, - } - deps.resolver.allRecords[testDomain] = map[string]map[string][]string{ - testNS1: {"A": {testIPv4}}, - testNS2: {"A": {testIPv4}}, - } - deps.resolver.allRecords[testHost] = map[string]map[string][]string{ - testNS1: {"A": {testIPv4}}, - testNS2: {"A": {testIPv4}}, - } - deps.resolver.ipAddresses[testHost] = []string{ - testIPv4, - } - deps.portChecker.results["93.184.216.34:80"] = true - deps.portChecker.results["93.184.216.34:443"] = true - deps.tlsChecker.certs["93.184.216.34:www.example.com"] = &tlscheck.CertificateInfo{ - CommonName: testHost, - Issuer: testIssuer, - NotAfter: time.Now().Add(90 * 24 * time.Hour), - SubjectAlternativeNames: []string{ - testHost, - }, - } - deps.tlsChecker.certs["93.184.216.34:example.com"] = &tlscheck.CertificateInfo{ - CommonName: testDomain, - Issuer: testIssuer, - NotAfter: time.Now().Add(90 * 24 * time.Hour), - SubjectAlternativeNames: []string{ - testDomain, - }, + // testHost answers with an address, so the check saves an empty list + // of CNAME addresses for it; nil would mean the check did not look + // at whether to follow a CNAME. + hs, _ := deps.state.GetHostnameState(testHost) + if hs.CNAMEAddresses == nil || len(hs.CNAMEAddresses) != 0 { + t.Errorf("saved CNAME addresses %#v, want []", hs.CNAMEAddresses) } } @@ -359,28 +443,9 @@ func TestDomainPortAndTLSChecks(t *testing.T) { t.Parallel() cfg := defaultTestConfig(t) - cfg.Domains = []string{testDomain} + cfg.Domains = []string{testSmallDomain} - w, deps := newTestWatcher(t, cfg) - - deps.resolver.nsRecords[testDomain] = []string{ - testNS1, - } - deps.resolver.allRecords[testDomain] = map[string]map[string][]string{ - testNS1: {"A": {testIPv4}}, - } - deps.portChecker.results["93.184.216.34:80"] = true - deps.portChecker.results["93.184.216.34:443"] = true - deps.tlsChecker.certs["93.184.216.34:example.com"] = &tlscheck.CertificateInfo{ - CommonName: testDomain, - Issuer: testIssuer, - NotAfter: time.Now().Add(90 * 24 * time.Hour), - SubjectAlternativeNames: []string{ - testDomain, - }, - } - - w.RunOnce(t.Context()) + _, deps := runChecks(t, cfg, nil) snap := deps.state.GetSnapshot() @@ -413,56 +478,310 @@ func TestDomainPortAndTLSChecks(t *testing.T) { } } +// TestFirstCheckAfterLoadingFileWithoutSections loads a state file +// holding {}, and one that leaves out the certificates section, and runs +// the first check. It saves domain, hostname, port and certificate +// entries; a section loaded as nil made it panic. +func TestFirstCheckAfterLoadingFileWithoutSections(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + file string + }{ + {"no sections", `{}`}, + {"no certificates", `{"domains": {}, "hostnames": {}, "ports": {}}`}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + cfg := defaultTestConfig(t) + cfg.Domains = []string{testSmallDomain} + + _, deps := runChecks(t, cfg, func(deps *testDeps) { + err := os.WriteFile(cfg.StatePath(), []byte(tt.file), 0o600) + if err != nil { + t.Fatalf("writing the state file: %v", err) + } + + err = deps.state.Load() + if err != nil { + t.Fatalf("loading the state file: %v", err) + } + }) + + snap := deps.state.GetSnapshot() + if len(snap.Domains) == 0 || len(snap.Hostnames) == 0 || + len(snap.Ports) == 0 || len(snap.Certificates) == 0 { + t.Errorf( + "saved %d domain, %d hostname, %d port and %d "+ + "certificate entries, want at least one of each", + len(snap.Domains), len(snap.Hostnames), + len(snap.Ports), len(snap.Certificates), + ) + } + }) + } +} + func TestNSChangeDetection(t *testing.T) { t.Parallel() cfg := defaultTestConfig(t) - cfg.Domains = []string{testDomain} + cfg.Domains = []string{testSmallDomain} - w, deps := newTestWatcher(t, cfg) + // The saved state lists nameservers that live DNS does not. + _, deps := runChecks(t, cfg, func(deps *testDeps) { + deps.state.SetDomainState(testSmallDomain, &state.DomainState{ + Nameservers: []string{oldNS1, oldNS2}, + }) + }) - deps.resolver.nsRecords[testDomain] = []string{ - testNS1, - testNS2, + assertNotified(t, deps, "NS Change: "+testSmallDomain, "warning") + + ds, _ := deps.state.GetDomainState(testSmallDomain) + if slices.Contains(ds.Nameservers, oldNS1) { + t.Errorf("saved nameservers not updated: %v", ds.Nameservers) } - deps.resolver.allRecords[testDomain] = map[string]map[string][]string{ - testNS1: {"A": {testIP}}, - testNS2: {"A": {testIP}}, - } - deps.portChecker.results["1.2.3.4:80"] = false - deps.portChecker.results["1.2.3.4:443"] = false +} - ctx := t.Context() - w.RunOnce(ctx) +// TestDomainThatDoesNotExist checks a .com domain that does not exist, +// with nameservers and records saved by an earlier check. The .com +// servers answer that it does not exist, so it is saved with nxdomain +// set and no nameservers, an NS Change removes them all, and its saved +// records are removed rather than asked for at the .com servers. +func TestDomainThatDoesNotExist(t *testing.T) { + t.Parallel() - deps.resolver.mu.Lock() - deps.resolver.nsRecords[testDomain] = []string{ - testNS1, - "ns3.example.com.", - } - deps.resolver.allRecords[testDomain] = map[string]map[string][]string{ - testNS1: {"A": {testIP}}, - "ns3.example.com.": {"A": {testIP}}, - } - deps.resolver.mu.Unlock() + const domain = "dnswatcher-test-does-not-exist.com" - w.RunOnce(ctx) + cfg := defaultTestConfig(t) + cfg.Domains = []string{domain} - notifications := deps.notifier.getNotifications() - if len(notifications) == 0 { - t.Error("expected notification for NS change") + var deps *testDeps + + livednstest.Retry(t, "watcher checks", func(ctx context.Context) error { + var w *watcher.Watcher + + w, deps = newTestWatcher(t, cfg) + + deps.state.SetDomainState(domain, &state.DomainState{ + Nameservers: []string{oldNS1, oldNS2}, + }) + deps.state.SetHostnameState(domain, &state.HostnameState{ + RecordsByNameserver: map[string]*state.NameserverRecordState{ + oldNS1: { + Records: map[string][]string{"A": {oldIP}}, + Status: "ok", + }, + }, + }) + + started := time.Now() + + w.RunOnce(ctx) + + // When no server answered, the domain's state is not saved. + ds, _ := deps.state.GetDomainState(domain) + if ds.LastChecked.Before(started) { + return fmt.Errorf("%s: %w", domain, livednstest.ErrNoAnswer) + } + + return nil + }) + + ds, _ := deps.state.GetDomainState(domain) + if !ds.NXDomain || len(ds.Nameservers) != 0 { + t.Errorf("saved nxdomain %v and nameservers %v, want true and none", + ds.NXDomain, ds.Nameservers) } - found := false + if hs, ok := deps.state.GetHostnameState(domain); ok { + t.Errorf("records saved for %s: %v", domain, hs.RecordsByNameserver) + } - for _, n := range notifications { - if n.Priority == "warning" { - found = true + assertNotified(t, deps, "NS Change: "+domain, "warning") + + // That is the only notification, and it removes both nameservers, + // in either order. + for _, n := range deps.notifier.getNotifications() { + removed := strings.TrimPrefix( + n.Message, "Domain: "+domain+"\nAdded: \nRemoved: ", + ) + if removed != oldNS1+", "+oldNS2 && removed != oldNS2+", "+oldNS1 { + t.Errorf("unexpected notification: %v", n) + } + } +} + +// TestDomainWithNoDelegationOfItsOwn checks a domain with no delegation +// of its own: codeberg.page is on the public suffix list, so +// docs.codeberg.page is a domain, but the .page servers delegate only +// codeberg.page, whose servers answer for it. It is saved with no +// nameservers and without nxdomain, and its records, asked at the +// codeberg.page servers, are saved. Those are testSmallDomain's two +// nameservers; github.io, the zone of the README's example, has eight. +// The previous check saved a delegation to oldNS1, now withdrawn, and +// records asked at oldNS1. A first check is cut short at its first +// notification, as shutdown would, and a second runs in full: across +// them the NS Change of its delegation removes oldNS1, and no other NS +// Change lists it again. +func TestDomainWithNoDelegationOfItsOwn(t *testing.T) { + t.Parallel() + + const domain = "docs.codeberg.page" + + cfg := defaultTestConfig(t) + cfg.Domains = []string{domain} + + var deps *testDeps + + livednstest.Retry(t, "watcher checks", func(ctx context.Context) error { + deps = newTestDeps(t, cfg) + + firstCtx, cancel := context.WithCancel(ctx) + defer cancel() + + w := watcher.NewForTest( + cfg, + deps.state, + resolver.NewFromLogger(slog.Default()), + deps.portChecker, + deps.tlsChecker, + cancelOnNotify{mockNotifier: deps.notifier, cancel: cancel}, + ) + + deps.state.SetDomainState(domain, &state.DomainState{ + Nameservers: []string{oldNS1}, + }) + deps.state.SetHostnameState(domain, saved( + map[string]*state.NameserverRecordState{ + oldNS1: answered(map[string][]string{"A": {oldIP}}), + }, + )) + + w.RunOnce(firstCtx) + + err := checkOnce(ctx, w, deps) + + // A domain saved as not existing has no records to wait for; + // the checks below fail on it. + if ds, ok := deps.state.GetDomainState(domain); ok && ds.NXDomain { + return nil + } + + return err + }) + + ds, _ := deps.state.GetDomainState(domain) + if ds.NXDomain || len(ds.Nameservers) != 0 { + t.Errorf("saved nxdomain %v and nameservers %v, want false and none", + ds.NXDomain, ds.Nameservers) + } + + if _, ok := deps.state.GetHostnameState(domain); !ok { + t.Errorf("no records saved for %s", domain) + } + + title := "NS Change: " + domain + want := "Domain: " + domain + "\nAdded: \nRemoved: " + oldNS1 + + var changes []notification + + for _, n := range deps.notifier.getNotifications() { + if n.Title == title { + changes = append(changes, n) } } - if !found { - t.Error("expected warning-priority NS change notification") + if len(changes) != 1 || changes[0].Message != want { + t.Errorf("sent %v, want one %q with message:\n%s", + changes, title, want) + } +} + +func TestNSAddressChangeDetection(t *testing.T) { + t.Parallel() + + cfg := defaultTestConfig(t) + cfg.Domains = []string{testSmallDomain} + + nameservers := lookupNameservers(t, testSmallDomain) + + // The saved state lists the nameservers live DNS lists, each at an + // address live DNS never returns. + _, deps := runChecks(t, cfg, func(deps *testDeps) { + nsAddresses := make(map[string][]string, len(nameservers)) + for _, ns := range nameservers { + nsAddresses[ns] = []string{oldIP} + } + + deps.state.SetDomainState(testSmallDomain, &state.DomainState{ + Nameservers: nameservers, + NameserverAddresses: nsAddresses, + }) + }) + + title := "NS Address Change: " + testSmallDomain + ds, _ := deps.state.GetDomainState(testSmallDomain) + + // One alert per nameserver, naming it and the address it had. + for _, ns := range ds.Nameservers { + prefix := "Domain: " + testSmallDomain + "\nNameserver: " + ns + + "\nOld: " + oldIP + "\nNew: " + + sent := 0 + + for _, n := range deps.notifier.getNotifications() { + if n.Title == title && strings.HasPrefix(n.Message, prefix) { + sent++ + } + } + + if sent != 1 { + t.Errorf("sent %d address changes for %s, want 1", sent, ns) + } + } + + if n := countNotifications(deps, title); n != len(ds.Nameservers) { + t.Errorf( + "sent %d address changes for %d nameservers", + n, len(ds.Nameservers), + ) + } + + if n := countNotifications(deps, "NS Change: "+testSmallDomain); n != 0 { + t.Errorf("sent %d NS changes, want 0", n) + } +} + +func TestNSAddedAndRemovedIsNoAddressChange(t *testing.T) { + t.Parallel() + + cfg := defaultTestConfig(t) + cfg.Domains = []string{testSmallDomain} + + nameservers := lookupNameservers(t, testSmallDomain) + + // The saved state lists oldNS1, which live DNS does not, in place of + // the first nameserver live DNS lists, so that the check finds that + // one added and oldNS1 removed. Only oldNS1 has addresses saved. + _, deps := runChecks(t, cfg, func(deps *testDeps) { + deps.state.SetDomainState(testSmallDomain, &state.DomainState{ + Nameservers: append([]string{oldNS1}, nameservers[1:]...), + NameserverAddresses: map[string][]string{oldNS1: {oldIP}}, + }) + }) + + if n := countNotifications(deps, "NS Change: "+testSmallDomain); n != 1 { + t.Errorf("sent %d NS changes, want 1", n) + } + + title := "NS Address Change: " + testSmallDomain + if n := countNotifications(deps, title); n != 0 { + t.Errorf("sent %d address changes, want 0", n) } } @@ -472,40 +791,20 @@ func TestRecordChangeDetection(t *testing.T) { cfg := defaultTestConfig(t) cfg.Hostnames = []string{testHost} - w, deps := newTestWatcher(t, cfg) + nameservers := lookupNameservers(t, testHost) - deps.resolver.allRecords[testHost] = map[string]map[string][]string{ - testNS1: {"A": {testIPv4}}, - } - deps.resolver.ipAddresses[testHost] = []string{ - testIPv4, - } - deps.portChecker.results["93.184.216.34:80"] = false - deps.portChecker.results["93.184.216.34:443"] = false + // The saved state has every nameserver live DNS lists answering + // with an address live DNS never returns. + _, deps := runChecks(t, cfg, func(deps *testDeps) { + byNameserver := make(map[string]*state.NameserverRecordState) + for _, ns := range nameservers { + byNameserver[ns] = answered(map[string][]string{"A": {oldIP}}) + } - ctx := t.Context() - w.RunOnce(ctx) + deps.state.SetHostnameState(testHost, saved(byNameserver)) + }) - deps.resolver.mu.Lock() - deps.resolver.allRecords[testHost] = map[string]map[string][]string{ - testNS1: {"A": {"93.184.216.35"}}, - } - deps.resolver.ipAddresses[testHost] = []string{ - "93.184.216.35", - } - deps.resolver.mu.Unlock() - - deps.portChecker.mu.Lock() - deps.portChecker.results["93.184.216.35:80"] = false - deps.portChecker.results["93.184.216.35:443"] = false - deps.portChecker.mu.Unlock() - - w.RunOnce(ctx) - - notifications := deps.notifier.getNotifications() - if len(notifications) == 0 { - t.Error("expected notification for record change") - } + assertNotified(t, deps, "Record Change: "+testHost, "warning") } func TestPortStateChange(t *testing.T) { @@ -514,38 +813,27 @@ func TestPortStateChange(t *testing.T) { cfg := defaultTestConfig(t) cfg.Hostnames = []string{testHost} - w, deps := newTestWatcher(t, cfg) - - deps.resolver.allRecords[testHost] = map[string]map[string][]string{ - testNS1: {"A": {testIP}}, - } - deps.resolver.ipAddresses[testHost] = []string{ - testIP, - } - deps.portChecker.results["1.2.3.4:80"] = true - deps.portChecker.results["1.2.3.4:443"] = true - deps.tlsChecker.certs["1.2.3.4:www.example.com"] = &tlscheck.CertificateInfo{ - CommonName: testHost, - Issuer: testIssuer, - NotAfter: time.Now().Add(90 * 24 * time.Hour), - SubjectAlternativeNames: []string{ - testHost, - }, - } - - ctx := t.Context() - w.RunOnce(ctx) + w, deps := runChecks(t, cfg, nil) + // Every port closes, and the port checks run again. They look + // nothing up. deps.portChecker.mu.Lock() - deps.portChecker.results["1.2.3.4:443"] = false + deps.portChecker.closed = true deps.portChecker.mu.Unlock() - w.RunOnce(ctx) + w.CheckAllPorts(t.Context()) - notifications := deps.notifier.getNotifications() - if len(notifications) == 0 { - t.Error("expected notification for port state change") - } + hs, _ := deps.state.GetHostnameState(testHost) + assertNotified( + t, deps, "Port Change: "+net.JoinHostPort(addresses(hs)[0], "443"), + "warning", + ) +} + +// expiresInThreeDays makes the TLS checker return certificates that +// expire within the seven-day warning period. +func expiresInThreeDays(deps *testDeps) { + deps.tlsChecker.notAfter = time.Now().Add(3 * 24 * time.Hour) } func TestTLSExpiryWarning(t *testing.T) { @@ -554,124 +842,71 @@ func TestTLSExpiryWarning(t *testing.T) { cfg := defaultTestConfig(t) cfg.Hostnames = []string{testHost} - w, deps := newTestWatcher(t, cfg) + _, deps := runChecks(t, cfg, expiresInThreeDays) - deps.resolver.allRecords[testHost] = map[string]map[string][]string{ - testNS1: {"A": {testIP}}, - } - deps.resolver.ipAddresses[testHost] = []string{ - testIP, - } - deps.portChecker.results["1.2.3.4:80"] = true - deps.portChecker.results["1.2.3.4:443"] = true - deps.tlsChecker.certs["1.2.3.4:www.example.com"] = &tlscheck.CertificateInfo{ - CommonName: testHost, - Issuer: testIssuer, - NotAfter: time.Now().Add(3 * 24 * time.Hour), - SubjectAlternativeNames: []string{ - testHost, - }, - } - - ctx := t.Context() - - // First run = baseline - w.RunOnce(ctx) - - // Second run should warn about expiry - w.RunOnce(ctx) - - notifications := deps.notifier.getNotifications() - - found := false - - for _, n := range notifications { - if n.Priority == "warning" { - found = true - } - } - - if !found { - t.Errorf( - "expected expiry warning, got: %v", - notifications, - ) - } + assertNotified(t, deps, "TLS Expiry Warning: "+testHost, "warning") } -func TestTLSExpiryWarningDedup(t *testing.T) { +// TestTLSExpiryWarningEachCheck runs the TLS checks three times in a +// row on hostname and port state built here, for a certificate that +// expires within the warning period. Each check warns once, whether the +// TLS interval is a nanosecond, shorter than the time between two +// checks, or a day, longer than it. +func TestTLSExpiryWarningEachCheck(t *testing.T) { t.Parallel() - cfg := defaultTestConfig(t) - cfg.Hostnames = []string{testHost} - cfg.TLSInterval = 24 * time.Hour + title := "TLS Expiry Warning: " + host - w, deps := newTestWatcher(t, cfg) + for _, interval := range []time.Duration{time.Nanosecond, 24 * time.Hour} { + t.Run(interval.String(), func(t *testing.T) { + t.Parallel() - deps.resolver.allRecords[testHost] = map[string]map[string][]string{ - testNS1: {"A": {testIP}}, - } - deps.resolver.ipAddresses[testHost] = []string{ - testIP, - } - deps.portChecker.results["1.2.3.4:80"] = true - deps.portChecker.results["1.2.3.4:443"] = true - deps.tlsChecker.certs["1.2.3.4:www.example.com"] = &tlscheck.CertificateInfo{ - CommonName: testHost, - Issuer: testIssuer, - NotAfter: time.Now().Add(3 * 24 * time.Hour), - SubjectAlternativeNames: []string{ - testHost, - }, - } + cfg := defaultTestConfig(t) + cfg.Hostnames = []string{host} + cfg.TLSInterval = interval - ctx := t.Context() + // The TLS checks read the saved hostname and port state and + // look nothing up, so the watcher has no resolver. + deps := newTestDeps(t, cfg) + w := watcher.NewForTest( + cfg, deps.state, nil, + deps.portChecker, deps.tlsChecker, deps.notifier, + ) - // First run = baseline, no notifications - w.RunOnce(ctx) + expiresInThreeDays(deps) + deps.state.SetHostnameState(host, saved( + map[string]*state.NameserverRecordState{ + nsA: answered(map[string][]string{"A": {ip1}}), + }, + )) + deps.state.SetPortState(ip1+":443", &state.PortState{ + Open: true, Hostnames: []string{host}, + }) - // Second run should fire one expiry warning - w.RunOnce(ctx) + for check := 1; check <= 3; check++ { + w.RunTLSChecks(t.Context()) - // Third run should NOT fire another warning (dedup) - w.RunOnce(ctx) - - notifications := deps.notifier.getNotifications() - - expiryCount := 0 - - for _, n := range notifications { - if n.Title == "TLS Expiry Warning: www.example.com" { - expiryCount++ - } - } - - if expiryCount != 1 { - t.Errorf( - "expected exactly 1 expiry warning (dedup), got %d", - expiryCount, - ) + got := countNotifications(deps, title) + if got != check { + t.Fatalf( + "after check %d: %d expiry warnings, want %d", + check, got, check, + ) + } + } + }) } } func TestGracefulShutdown(t *testing.T) { t.Parallel() + // No domains or hostnames: stopping does not involve DNS. cfg := defaultTestConfig(t) - cfg.Domains = []string{testDomain} cfg.DNSInterval = 100 * time.Millisecond cfg.TLSInterval = 100 * time.Millisecond - w, deps := newTestWatcher(t, cfg) - - deps.resolver.nsRecords[testDomain] = []string{ - testNS1, - } - deps.resolver.allRecords[testDomain] = map[string]map[string][]string{ - testNS1: {"A": {testIP}}, - } - deps.portChecker.results["1.2.3.4:80"] = false - deps.portChecker.results["1.2.3.4:443"] = false + w, _ := newTestWatcher(t, cfg) ctx, cancel := context.WithCancel(t.Context()) @@ -693,43 +928,83 @@ func TestGracefulShutdown(t *testing.T) { } } -func setupHostnameIP( - deps *testDeps, - hostname, ip string, -) { - deps.resolver.allRecords[hostname] = map[string]map[string][]string{ - testNS1: {"A": {ip}}, - } - deps.portChecker.results[ip+":80"] = true - deps.portChecker.results[ip+":443"] = true - deps.tlsChecker.certs[ip+":"+hostname] = &tlscheck.CertificateInfo{ - CommonName: hostname, - Issuer: testIssuer, - NotAfter: time.Now().Add(90 * 24 * time.Hour), - SubjectAlternativeNames: []string{hostname}, - } -} +// TestStopSavesState stops a watcher built by New the way fx stops it, +// and checks that a change made to the state after the last check is in +// the state file afterwards. The state's own stop hook never runs here, +// so only the watcher can have saved it. Nothing is configured to +// check, so no DNS is involved. +func TestStopSavesState(t *testing.T) { + t.Parallel() -func updateHostnameIP(deps *testDeps, hostname, ip string) { - deps.resolver.mu.Lock() - deps.resolver.allRecords[hostname] = map[string]map[string][]string{ - testNS1: {"A": {ip}}, - } - deps.resolver.mu.Unlock() + cfg := defaultTestConfig(t) + deps := newTestDeps(t, cfg) + lc := fxtest.NewLifecycle(t) - deps.portChecker.mu.Lock() - deps.portChecker.results[ip+":80"] = true - deps.portChecker.results[ip+":443"] = true - deps.portChecker.mu.Unlock() - - deps.tlsChecker.mu.Lock() - deps.tlsChecker.certs[ip+":"+hostname] = &tlscheck.CertificateInfo{ - CommonName: hostname, - Issuer: testIssuer, - NotAfter: time.Now().Add(90 * 24 * time.Hour), - SubjectAlternativeNames: []string{hostname}, + _, err := watcher.New(lc, watcher.Params{ + Logger: deps.log, + Config: cfg, + State: deps.state, + Resolver: resolver.NewFromLogger(slog.Default()), + PortCheck: deps.portChecker, + TLSCheck: deps.tlsChecker, + Notify: deps.notifier, + }) + if err != nil { + t.Fatalf("watcher.New: %v", err) + } + + lc.RequireStart() + + // The first check saves state once. Wait for that save before + // changing the state, so the change can reach the file only + // through the save made at stop. + deadline := time.Now().Add(5 * time.Second) + + for { + _, err = os.Stat(cfg.StatePath()) + if err == nil { + break + } + + if time.Now().After(deadline) { + t.Fatalf("the first check saved no state: %v", err) + } + + time.Sleep(10 * time.Millisecond) + } + + deps.state.SetDomainState(testDomain, &state.DomainState{ + Nameservers: []string{oldNS1}, + }) + + ctx, cancel := context.WithTimeout(t.Context(), 5*time.Second) + defer cancel() + + err = lc.Stop(ctx) + if err != nil { + t.Fatalf("stopping the watcher: %v", err) + } + + saved, err := state.New(fxtest.NewLifecycle(t), state.Params{ + Logger: deps.log, + Config: cfg, + }) + if err != nil { + t.Fatalf("state.New: %v", err) + } + + err = saved.Load() + if err != nil { + t.Fatalf("loading the state file: %v", err) + } + + ds, ok := saved.GetDomainState(testDomain) + if !ok || !slices.Equal(ds.Nameservers, []string{oldNS1}) { + t.Errorf( + "state file after stop has %+v for %s, want nameservers %v", + ds, testDomain, []string{oldNS1}, + ) } - deps.tlsChecker.mu.Unlock() } func TestDNSRunsBeforePortAndTLSChecks(t *testing.T) { @@ -738,45 +1013,47 @@ func TestDNSRunsBeforePortAndTLSChecks(t *testing.T) { cfg := defaultTestConfig(t) cfg.Hostnames = []string{testHost} - w, deps := newTestWatcher(t, cfg) - - setupHostnameIP(deps, testHost, "10.0.0.1") - - ctx := t.Context() - w.RunOnce(ctx) + // The saved state says the last check found testHost at oldIP. + _, deps := runChecks(t, cfg, func(deps *testDeps) { + deps.state.SetHostnameState(testHost, &state.HostnameState{ + RecordsByNameserver: map[string]*state.NameserverRecordState{ + oldNS1: { + Records: map[string][]string{"A": {oldIP}}, + Status: "ok", + }, + }, + }) + }) snap := deps.state.GetSnapshot() - if _, ok := snap.Ports["10.0.0.1:80"]; !ok { - t.Fatal("expected port state for 10.0.0.1:80") + + if _, ok := snap.Ports[oldIP+":80"]; ok { + t.Error("port check used stale DNS: found " + oldIP + ":80") } - // DNS changes to a new IP; port and TLS must pick it up. - updateHostnameIP(deps, testHost, "10.0.0.2") + // Port and TLS checks must use the addresses this check found. + for _, ip := range addresses(snap.Hostnames[testHost]) { + portKey := net.JoinHostPort(ip, "80") + if _, ok := snap.Ports[portKey]; !ok { + t.Error("port check used stale DNS: missing " + portKey) + } - w.RunOnce(ctx) - - snap = deps.state.GetSnapshot() - - if _, ok := snap.Ports["10.0.0.2:80"]; !ok { - t.Error("port check used stale DNS: missing 10.0.0.2:80") - } - - certKey := "10.0.0.2:443:www.example.com" - if _, ok := snap.Certificates[certKey]; !ok { - t.Error("TLS check used stale DNS: missing " + certKey) + certKey := net.JoinHostPort(ip, "443") + ":" + testHost + if _, ok := snap.Certificates[certKey]; !ok { + t.Error("TLS check used stale DNS: missing " + certKey) + } } } func TestSendTestNotification_Enabled(t *testing.T) { t.Parallel() + // No domains or hostnames: the startup notification does not + // involve DNS. cfg := defaultTestConfig(t) - cfg.Domains = []string{testDomain} - cfg.Hostnames = []string{testHost} cfg.SendTestNotification = true w, deps := newTestWatcher(t, cfg) - setupBaselineMocks(deps) w.RunOnce(t.Context()) @@ -797,14 +1074,11 @@ func TestSendTestNotification_ViaRun(t *testing.T) { t.Parallel() cfg := defaultTestConfig(t) - cfg.Domains = []string{testDomain} - cfg.Hostnames = []string{testHost} cfg.SendTestNotification = true cfg.DNSInterval = 24 * time.Hour cfg.TLSInterval = 24 * time.Hour w, deps := newTestWatcher(t, cfg) - setupBaselineMocks(deps) ctx, cancel := context.WithCancel(t.Context()) @@ -823,18 +1097,27 @@ func TestSendTestNotification_ViaRun(t *testing.T) { notifications := deps.notifier.getNotifications() + // No names are configured, so every count is 0. + wantMessage := "dnswatcher has started and completed its initial scan.\n" + + "Monitoring 0 domain(s) and 0 hostname(s).\n" + + "Tracking 0 port endpoint(s) and 0 TLS certificate(s).\n" + + "This is a test notification, sent to every configured " + + "notification endpoint." + found := false for _, n := range notifications { if n.Priority == "success" && - n.Title == "✅ dnswatcher startup complete" { + n.Title == "✅ dnswatcher startup complete" && + n.Message == wantMessage { found = true } } if !found { t.Errorf( - "expected startup test notification, got: %v", + "expected startup test notification with message %q, got: %v", + wantMessage, notifications, ) } @@ -844,14 +1127,11 @@ func TestSendTestNotification_Disabled(t *testing.T) { t.Parallel() cfg := defaultTestConfig(t) - cfg.Domains = []string{testDomain} - cfg.Hostnames = []string{testHost} cfg.SendTestNotification = false cfg.DNSInterval = 24 * time.Hour cfg.TLSInterval = 24 * time.Hour w, deps := newTestWatcher(t, cfg) - setupBaselineMocks(deps) ctx, cancel := context.WithCancel(t.Context()) @@ -878,38 +1158,34 @@ func TestSendTestNotification_Disabled(t *testing.T) { } } -func TestNSFailureAndRecovery(t *testing.T) { +func TestNSChangeAndRecovery(t *testing.T) { t.Parallel() cfg := defaultTestConfig(t) cfg.Hostnames = []string{testHost} - w, deps := newTestWatcher(t, cfg) + nameservers := lookupNameservers(t, testHost) - deps.resolver.allRecords[testHost] = map[string]map[string][]string{ - testNS1: {"A": {testIP}}, - testNS2: {"A": {testIP}}, - } - deps.resolver.ipAddresses[testHost] = []string{ - testIP, - } - deps.portChecker.results["1.2.3.4:80"] = false - deps.portChecker.results["1.2.3.4:443"] = false + // The saved state has every nameserver live DNS lists as one that + // did not answer, and, as answering, one that live DNS does not + // list, which the check then finds removed. + _, deps := runChecks(t, cfg, func(deps *testDeps) { + byNameserver := map[string]*state.NameserverRecordState{ + oldNS1: answered(map[string][]string{"A": {oldIP}}), + } + for _, ns := range nameservers { + byNameserver[ns] = failed() + } - ctx := t.Context() + deps.state.SetHostnameState(testHost, saved(byNameserver)) + }) - w.RunOnce(ctx) + assertNotified(t, deps, "NS Change: "+testHost, "warning") + assertNotified(t, deps, "NS Recovery: "+testHost, "success") - deps.resolver.mu.Lock() - deps.resolver.allRecords[testHost] = map[string]map[string][]string{ - testNS1: {"A": {testIP}}, - } - deps.resolver.mu.Unlock() - - w.RunOnce(ctx) - - notifications := deps.notifier.getNotifications() - if len(notifications) == 0 { - t.Error("expected notification for NS disappearance") + // A nameserver that did not answer has no records to compare, so + // its recovery is not also a record change. + if n := countNotifications(deps, "Record Change: "+testHost); n != 0 { + t.Errorf("sent %d record changes on recovery, want 0", n) } } diff --git a/package.json b/package.json new file mode 100644 index 0000000..53eca1f --- /dev/null +++ b/package.json @@ -0,0 +1,9 @@ +{ + "name": "dnswatcher-tooling", + "version": "0.0.0", + "private": true, + "description": "Pins the prettier that script/fmt and script/fmt-check run against this repo's markdown. Not a JavaScript project; nothing here is imported, published, or shipped.", + "devDependencies": { + "prettier": "3.9.6" + } +} diff --git a/script/bootstrap b/script/bootstrap index 06da5d0..b40a411 100755 --- a/script/bootstrap +++ b/script/bootstrap @@ -3,20 +3,16 @@ # this repo. Idempotent: every install is guarded by a check so already # installed tools are skipped. Base tooling comes from nix, apt, brew, # or apk (detected in that order); assumes nothing is present. -# goimports is installed via `go install` at a pinned commit (never -# "latest") because script/fmt runs it on the host; script/fmt-check -# does not (it runs gofmt only). +# goimports is not installed here: script/fmt and script/fmt-check run +# it with `go run` at a pinned commit. # The linter is NOT installed here: golangci-lint runs via docker only # (script/lint), pinned by image digest, so its only prerequisite is a -# working docker. +# working docker. Nor is prettier: script/fmt and script/fmt-check run +# it in a stage of the Dockerfile. set -eu ROOT="$(cd "$(dirname "$0")/.." && pwd -P)" -# Pinned version, 2026-08-07 (same pin as the Dockerfile) -# goimports v0.42.0 -GOIMPORTS_REF="golang.org/x/tools/cmd/goimports@009367f5c17a8d4c45a961a3a509277190a9a6f0" - PKGMGR="" SUDO="" APT_UPDATED="" @@ -71,15 +67,12 @@ main() { if missing make; then pkg_install gnumake make make make; fi if missing go; then pkg_install go golang go go; fi - # Format tools, pinned via go install (installs into - # "$(go env GOPATH)/bin"; ensure that is on your PATH). - if missing goimports; then go install "$GOIMPORTS_REF"; fi - - # Linting runs via docker only (script/lint). Warn, don't fail: - # everything except `make lint` works without it. + # Testing, linting and the markdown formatter run via docker only. + # Warn, don't fail: make build works without it. if missing docker; then echo "bootstrap: WARNING: docker not found; install it to" \ - "run make lint and make docker." >&2 + "run make test, make lint, make fmt, make fmt-check," \ + "make check and make docker." >&2 fi go mod download diff --git a/script/cibuild b/script/cibuild index 1b9e57d..d8d3200 100755 --- a/script/cibuild +++ b/script/cibuild @@ -1,14 +1,28 @@ #!/bin/sh -# script/cibuild: run the CI build. The Dockerfile's lint stage runs -# make fmt-check and golangci-lint; its builder stage runs make test -# and make build. A successful build implies all of those passed. +# script/cibuild: run the CI build. It bootstraps first: a CI runner +# checks out and runs this and nothing else, and script/fmt-check runs +# the formatter on the host, which a pristine checkout cannot do. +# --no-cache for the same reason as script/docker: the gate phases the +# final stage depends on are RUN steps, and a cached one is a check that +# did not run. set -eu -ROOT="$(cd "$(dirname "$0")/.." && pwd -P)" +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd -P)" +ROOT="$(cd "$SCRIPT_DIR/.." && pwd -P)" main() { cd "$ROOT" - docker build . + "$SCRIPT_DIR/bootstrap" + "$SCRIPT_DIR/check" + # Own line: a failing command substitution inside an argument does + # not trip `set -e`, so the inline form degrades silently to an + # empty constant. The VERSION build argument takes precedence over + # the version a build stage derives from the .git in the context. + version="$(git describe --tags --always --dirty 2>/dev/null || true)" + [ -n "$version" ] || version="unknown" + docker build --no-cache \ + --build-arg VERSION="$version" \ + -t "$("$SCRIPT_DIR/projectname")" . } main "$@" diff --git a/script/docker b/script/docker index 9b9ea86..07b626c 100755 --- a/script/docker +++ b/script/docker @@ -1,6 +1,8 @@ #!/bin/sh # script/docker: build the Docker image tagged with the project name. # Identical in all repos; the tag comes from script/projectname. +# --no-cache because the gate phases the final stage depends on are RUN +# steps, and a cached one is a check that did not run. set -eu SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd -P)" @@ -8,7 +10,15 @@ ROOT="$(cd "$SCRIPT_DIR/.." && pwd -P)" main() { cd "$ROOT" - docker build -t "$("$SCRIPT_DIR/projectname")" . + # Own line: a failing command substitution inside an argument does + # not trip `set -e`, so the inline form degrades silently to an + # empty constant. The VERSION build argument takes precedence over + # the version a build stage derives from the .git in the context. + version="$(git describe --tags --always --dirty 2>/dev/null || true)" + [ -n "$version" ] || version="unknown" + docker build --no-cache \ + --build-arg VERSION="$version" \ + -t "$("$SCRIPT_DIR/projectname")" . } main "$@" diff --git a/script/fmt b/script/fmt index b3235a8..9470b03 100755 --- a/script/fmt +++ b/script/fmt @@ -1,13 +1,58 @@ #!/bin/sh -# script/fmt: format all files (writes). +# script/fmt: format all files (writes). Go with gofmt and goimports on +# the host, markdown with prettier in the Dockerfile's fmt stage. +# +# goimports runs with `go run` at a pinned commit, never from PATH, so +# every machine formats with the same version and nothing installs it. +# +# The markdown pass is a `docker build --output type=local` rather than a +# `docker run -v`, so it needs no bind mount and behaves the same against +# a remote daemon; the formatted documents come back out of the build and +# are copied over the tree here. --output writes files and makes no +# image, so there is nothing to tag. set -eu ROOT="$(cd "$(dirname "$0")/.." && pwd -P)" +# goimports v0.42.0, 2026-08-07. Must match script/fmt-check. +GOIMPORTS_REF="golang.org/x/tools/cmd/goimports@009367f5c17a8d4c45a961a3a509277190a9a6f0" + +die() { + echo "script/fmt: $*" >&2 + exit 1 +} + main() { cd "$ROOT" gofmt -s -w . - goimports -w . + go run "$GOIMPORTS_REF" -w . + + tmp="$(mktemp -d "${TMPDIR:-/tmp}/dnswatcher-fmt.XXXXXX")" + trap 'rm -rf "$tmp"' EXIT INT TERM + + docker build --no-cache \ + --target fmt-out \ + --output "type=local,dest=$tmp/out" . + + # An empty export means prettier was handed nothing, which must not + # read as "already formatted". + (cd "$tmp/out" && find . -type f -name '*.md') | + sed 's|^\./||' | LC_ALL=C sort >"$tmp/files" + + [ -s "$tmp/files" ] || + die "the formatting build produced no markdown; the build" \ + "context reached prettier empty" + + # Copied only where the bytes differ, so an already-formatted tree + # keeps its timestamps and says nothing. + while IFS= read -r f; do + [ -n "$f" ] || continue + if [ -f "$f" ] && cmp -s "$tmp/out/$f" "$f"; then + continue + fi + cp "$tmp/out/$f" "$f" + echo "prettier: reformatted $f" + done <"$tmp/files" } main "$@" diff --git a/script/fmt-check b/script/fmt-check index b7438ef..b9b9ca9 100755 --- a/script/fmt-check +++ b/script/fmt-check @@ -1,18 +1,37 @@ #!/bin/sh -# script/fmt-check: check formatting (read-only). Same scope as -# script/fmt, but fails instead of writing. +# script/fmt-check: check formatting (read-only). Same tools and scope +# as script/fmt, but fails instead of writing and names what it would +# change: the Go on the host, the markdown with prettier in the +# Dockerfile's fmt-check stage. +# +# --no-cache because a cached check layer is a check that did not run. +# The tag makes each build replace the previous image instead of leaving +# a dangling one behind. set -eu -ROOT="$(cd "$(dirname "$0")/.." && pwd -P)" +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd -P)" +ROOT="$(cd "$SCRIPT_DIR/.." && pwd -P)" + +# goimports v0.42.0, 2026-08-07. Must match script/fmt. +GOIMPORTS_REF="golang.org/x/tools/cmd/goimports@009367f5c17a8d4c45a961a3a509277190a9a6f0" main() { cd "$ROOT" - files="$(gofmt -l .)" + files="$(gofmt -s -l .)" if [ -n "$files" ]; then echo "gofmt: files not formatted:" >&2 echo "$files" >&2 exit 1 fi + files="$(go run "$GOIMPORTS_REF" -l .)" + if [ -n "$files" ]; then + echo "goimports: files not formatted:" >&2 + echo "$files" >&2 + exit 1 + fi + docker build --no-cache \ + --target fmt-check \ + -t "$("$SCRIPT_DIR/projectname")-fmt-check" . } main "$@" diff --git a/script/install-precommit b/script/install-precommit index 723bae1..c005c5e 100755 --- a/script/install-precommit +++ b/script/install-precommit @@ -7,7 +7,20 @@ ROOT="$(cd "$(dirname "$0")/.." && pwd -P)" main() { cd "$ROOT" - hook=".git/hooks/pre-commit" + # Stop if this directory is not the top of its own git checkout, for + # example a copy inside another repository, whose hook must not be + # replaced. + if [ "$(git rev-parse --show-toplevel)" != "$ROOT" ]; then + echo "install-precommit: $ROOT is not the top of a git checkout" >&2 + exit 1 + fi + # Ask git for the repository's own git directory: .git is a file, not + # a directory, in some checkouts (for example a clone made with + # --separate-git-dir). core.hooksPath is deliberately not followed, so + # the hook is never written outside this repository. + hooks="$(git rev-parse --git-common-dir)/hooks" + mkdir -p "$hooks" + hook="$hooks/pre-commit" printf '#!/bin/sh\nset -e\nscript/precommit\n' > "$hook" chmod +x "$hook" echo "pre-commit hook installed: runs script/precommit" diff --git a/script/lint b/script/lint index 84a1972..2d8b075 100755 --- a/script/lint +++ b/script/lint @@ -1,28 +1,23 @@ #!/bin/sh -# script/lint: run the linter. golangci-lint is never installed or run -# on the host: it runs via docker only, one way, everywhere. This -# builds Dockerfile.lint, which COPYs the repo into the digest-pinned -# golangci-lint image and lints as a build step, so a successful build -# means a clean lint. +# script/lint: run the linter. Linting is a phase of the Dockerfile and +# this builds that phase alone; the linter is never installed or run on +# a developer host, where a shared result cache and a host-global lock +# make its answer untrustworthy. # -# --no-cache-filter=lint forces the lint stage (source copy + linter -# run) to execute on every invocation. Without it an unchanged tree -# returns success in well under a second having linted nothing. The -# deps stage (base image + go mod download) stays cached, and no global -# cache invalidation is performed. --progress=plain keeps the linter's -# own output visible. +# The phase is not the last stage in the file, so it is built only when +# --target names it. --no-cache because a cached lint layer is a lint +# that did not run. The tag makes each build replace the previous image +# instead of leaving a dangling one behind. set -eu -ROOT="$(cd "$(dirname "$0")/.." && pwd -P)" +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd -P)" +ROOT="$(cd "$SCRIPT_DIR/.." && pwd -P)" main() { cd "$ROOT" - docker build \ - --progress=plain \ - --no-cache-filter=lint \ + docker build --no-cache \ --target lint \ - -f Dockerfile.lint \ - . + -t "$("$SCRIPT_DIR/projectname")-lint" . } main "$@" diff --git a/script/test b/script/test index b4ef492..cd239f2 100755 --- a/script/test +++ b/script/test @@ -1,35 +1,19 @@ #!/bin/sh -# script/test: run the test suite. -# -# -count=1 disables Go's test cache, and is load-bearing here. This -# suite queries live DNS on every run by policy (TESTING.md); a cached -# result is a replay of an earlier run's output with no query made at -# all. On an unchanged tree the whole suite would return success in -# under a second having resolved nothing, which makes the repeated-run -# green that is used as evidence for flakiness fixes worthless. Do not -# remove it. -# -# Conditional verbose rerun per REPO_POLICIES.md: run quiet first so -# CI and docker build logs stay readable, and rerun with -v only on -# failure. The rerun also carries -count=1 (a cached replay of the -# failure would show nothing new), and the exit status is forced to 1 -# no matter how the rerun ends: the first failure already proved the -# suite broken, so a flaky test that passes the second time must not -# turn the build green. -# -# -timeout 90s is a deliberate backstop above the 60s hard cap on -# suite duration. Do not lower it. +# script/test: run the test suite. Testing is a phase of the Dockerfile +# and this builds that phase alone, on the same terms as script/lint: +# --target because a phase that is not the last stage is built only when +# named, --no-cache because a cached test layer is a test that did not +# run, and a tag so each build replaces the previous image. set -eu -ROOT="$(cd "$(dirname "$0")/.." && pwd -P)" +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd -P)" +ROOT="$(cd "$SCRIPT_DIR/.." && pwd -P)" main() { cd "$ROOT" - go test -count=1 -race -timeout 90s -cover ./... || { - echo "--- Rerunning with -v for details ---" >&2 - go test -count=1 -race -timeout 90s -v ./... || true - exit 1 - } + docker build --no-cache \ + --target test \ + -t "$("$SCRIPT_DIR/projectname")-test" . } main "$@" diff --git a/yarn.lock b/yarn.lock new file mode 100644 index 0000000..8f3c21a --- /dev/null +++ b/yarn.lock @@ -0,0 +1,8 @@ +# THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY. +# yarn lockfile v1 + + +prettier@3.9.6: + version "3.9.6" + resolved "https://registry.yarnpkg.com/prettier/-/prettier-3.9.6.tgz#b3ea5146515d40fc53f18aa63f74dfab1e10dbf6" + integrity sha512-OpN0zzVdiaiAhxpuuj5efpIS4sY9j7bY6uR5mnj5yPzGkdkjNKSJeUThPb60Jw29QuAZgA4o+/iB49kFiaBX6g==
Address StateDomains Hostnames Checked
{{ $key }} @@ -201,6 +192,9 @@ > {{ end }} + {{ joinStrings $ps.Domains ", " }} + {{ joinStrings $ps.Hostnames ", " }} +
{{ $cs.Error }}
+ 		{{ $cs.CommonName }} {{ relTime $cs.LastChecked }}
+ {{ $name }} + + {{ $ns }} + + {{ if eq $nsr.Status "ok" }} + ok + {{ else }} + {{ $nsr.Status }} + {{ end }} + + {{ if $nsr.Error }} + {{ $nsr.Error }} + {{ else }} + {{ formatRecords $nsr.Records }} + {{ end }} + + {{ relTime $nsr.LastChecked }} +