healthcheck: answer 503 when no root server gave a usable reply in the last round of DNS checks (closes #302)

The health check answered ok while every DNS lookup failed, as on a
network that intercepts DNS queries. A lookup whose root servers give no
usable reply now fails with ErrNoRootServerAnswered. At the end of each
round of DNS checks that shutdown did not cut short, the watcher records
whether every lookup failed with it or with ErrIntercepted; the health
check then gives status error, and /.well-known/healthcheck and /health
answer 503. Until the first round finishes it stays ok.

The health check now depends on the watcher, so fx starts the watcher
before the HTTP server and stops the server first. NewWithRootServers
lets tests start a resolver's walks at real servers that are not root
servers.

Model: opus-5-5

internal/watcher/dnsunreachable_test.go

@@ -0,0 +1,66 @@
+package watcher_test
+
+import (
+	"log/slog"
+	"testing"
+
+	"sneak.berlin/go/dnswatcher/internal/resolver"
+	"sneak.berlin/go/dnswatcher/internal/watcher"
+)
+
+// TestDNSUnreachableWhenNoRootServerGivesUsableReply runs one round of
+// checks of a domain and a hostname with every walk starting at real
+// servers that are not root servers. At 192.0.2.1, a documentation
+// address, nothing answers, as when no root server answers. Public
+// recursive resolvers of four operators mark their replies recursion
+// available, as on a network that intercepts DNS queries; one that gives
+// no reply is passed over for the next, and if none replies, no root
+// server answered. Either way every lookup failed because no root server
+// gave a usable reply, so the round could not reach the DNS.
+// TestFirstRunBaseline and TestDomainThatDoesNotExist check rounds in
+// which the root servers gave a usable reply.
+func TestDNSUnreachableWhenNoRootServerGivesUsableReply(t *testing.T) {
+	t.Parallel()
+
+	tests := []struct {
+		name    string
+		servers []string
+	}{
+		{"no root server answers", []string{"192.0.2.1"}},
+		{
+			"the network intercepts DNS queries",
+			[]string{"1.1.1.1", "8.8.8.8", "9.9.9.9", "208.67.222.222"},
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			t.Parallel()
+
+			cfg := defaultTestConfig(t)
+			cfg.Domains = []string{testSmallDomain}
+			cfg.Hostnames = []string{testHost}
+
+			deps := newTestDeps(t, cfg)
+			w := watcher.NewForTest(
+				cfg,
+				deps.state,
+				resolver.NewWithRootServers(slog.Default(), tt.servers),
+				deps.portChecker,
+				deps.tlsChecker,
+				deps.notifier,
+			)
+
+			if w.DNSUnreachable() {
+				t.Error("DNSUnreachable is true before the first round")
+			}
+
+			w.RunOnce(t.Context())
+
+			if !w.DNSUnreachable() {
+				t.Error("DNSUnreachable is false after a round in " +
+					"which no root server gave a usable reply")
+			}
+		})
+	}
+}

internal/watcher/watcher.go

@@ -11,6 +11,7 @@ import (
 	"sort"
 	"strconv"
 	"strings"
+	"sync/atomic"
 	"time"
 
 	"go.uber.org/fx"
@@ -62,6 +63,11 @@ type Watcher struct {
 	cancel    context.CancelFunc
 	done      chan struct{} // closed when Run returns
 	firstRun  bool
+
+	// dnsUnreachable is what DNSUnreachable reports. Each round of DNS
+	// checks sets it, and the HTTP server reads it from its own
+	// goroutines.
+	dnsUnreachable atomic.Bool
 }
 
 // New creates a new Watcher instance wired into the fx lifecycle.
@@ -197,6 +203,18 @@ func (w *Watcher) RunOnce(ctx context.Context) {
 	w.firstRun = false
 }
 
+// DNSUnreachable reports whether the most recent round of DNS checks
+// could not reach the DNS at all: every lookup in it failed because no
+// root server gave a usable reply, as when none of them answered or the
+// network intercepts DNS queries. A round reached the DNS even when a
+// name's own nameservers did not answer or a domain does not exist,
+// since the root servers gave a usable reply to those lookups. It is
+// false until the first round finishes, and a round that shutdown cut
+// short changes nothing. It may be called while a round runs.
+func (w *Watcher) DNSUnreachable() bool {
+	return w.dnsUnreachable.Load()
+}
+
 func (w *Watcher) detectFirstRun() {
 	snap := w.state.GetSnapshot()
 	hasState := len(snap.Domains) > 0 ||
@@ -265,15 +283,40 @@ func (w *Watcher) cleanupRemovedTargets() {
 // runDNSChecks performs DNS resolution for all configured domains
 // and hostnames, updating state with freshly resolved records.
 // This must complete before port or TLS checks run so those
-// checks operate on current IP addresses.
+// checks operate on current IP addresses. Unless ctx was cut short, it
+// then records whether this round could reach the DNS (see
+// DNSUnreachable).
 func (w *Watcher) runDNSChecks(ctx context.Context) {
+	errs := make([]error, 0, len(w.config.Domains)+len(w.config.Hostnames))
+
 	for _, domain := range w.config.Domains {
-		w.checkDomain(ctx, domain)
+		errs = append(errs, w.checkDomain(ctx, domain))
 	}
 
 	for _, hostname := range w.config.Hostnames {
-		w.checkHostname(ctx, hostname)
+		_, err := w.checkHostname(ctx, hostname)
+		errs = append(errs, err)
 	}
+
+	if ctx.Err() == nil {
+		w.dnsUnreachable.Store(noRootServerGaveUsableReply(errs))
+	}
+}
+
+// noRootServerGaveUsableReply reports whether errs, the errors of the
+// lookups in a round of DNS checks, nil for a lookup that succeeded,
+// show that every lookup failed because no root server gave a usable
+// reply: each is ErrNoRootServerAnswered or ErrIntercepted. A round with
+// no lookups shows nothing.
+func noRootServerGaveUsableReply(errs []error) bool {
+	for _, err := range errs {
+		if !errors.Is(err, resolver.ErrNoRootServerAnswered) &&
+			!errors.Is(err, resolver.ErrIntercepted) {
+			return false
+		}
+	}
+
+	return len(errs) > 0
 }
 
 // logFailedLookup logs a failed DNS lookup at error level, unless ctx
@@ -292,14 +335,16 @@ func (w *Watcher) logFailedLookup(
 }
 
 // checkDomain looks up domain's delegation and checks domain with it
-// (see checkDomainResult).
+// (see checkDomainResult). It returns the lookup's error.
 func (w *Watcher) checkDomain(
 	ctx context.Context,
 	domain string,
-) {
+) error {
 	nameservers, err := w.resolver.LookupNS(ctx, domain)
 
 	w.checkDomainResult(ctx, domain, nameservers, err)
+
+	return err
 }
 
 // checkDomainResult finishes a check of domain from nameservers and err,
@@ -358,7 +403,7 @@ func (w *Watcher) checkDomainResult(
 	// check, as detectNSDisappearances needs.
 	if nxdomain {
 		w.state.DeleteHostnameState(domain)
-	} else if !w.checkHostname(ctx, domain) {
+	} else if saved, _ := w.checkHostname(ctx, domain); !saved {
 		return
 	}
 
@@ -484,11 +529,12 @@ func (w *Watcher) detectNSAddressChanges(
 }
 
 // checkHostname looks up hostname's records and checks them (see
-// checkHostnameResult). It reports whether it saved them.
+// checkHostnameResult). It reports whether it saved them, and returns
+// the lookup's error.
 func (w *Watcher) checkHostname(
 	ctx context.Context,
 	hostname string,
-) bool {
+) (bool, error) {
 	results, err := w.resolver.LookupAllRecords(ctx, hostname)
 	if err != nil {
 		w.logFailedLookup(
@@ -499,7 +545,7 @@ func (w *Watcher) checkHostname(
 		)
 	}
 
-	return w.checkHostnameResult(ctx, hostname, results, err)
+	return w.checkHostnameResult(ctx, hostname, results, err), err
 }
 
 // checkHostnameResult finishes a check of hostname from results and err,

internal/watcher/watcher_test.go

@@ -385,11 +385,16 @@ func TestFirstRunBaseline(t *testing.T) {
 	cfg.Domains = []string{testSmallDomain}
 	cfg.Hostnames = []string{testHost}
 
-	_, deps := runChecks(t, cfg, nil)
+	w, deps := runChecks(t, cfg, nil)
 
 	assertNoNotifications(t, deps)
 	assertStatePopulated(t, deps)
 
+	if w.DNSUnreachable() {
+		t.Error("DNSUnreachable is true after a round whose lookups " +
+			"succeeded")
+	}
+
 	// testHost answers with an address, so the check saves an empty list
 	// of CNAME addresses for it; nil would mean the check did not look
 	// at whether to follow a CNAME.
@@ -551,7 +556,9 @@ func TestNSChangeDetection(t *testing.T) {
 // with nameservers and records saved by an earlier check. The .com
 // servers answer that it does not exist, so it is saved with nxdomain
 // set and no nameservers, an NS Change removes them all, and its saved
-// records are removed rather than asked for at the .com servers.
+// records are removed rather than asked for at the .com servers. The
+// round's only lookup failed, but the root servers and the .com servers
+// answered, so the round reached the DNS.
 func TestDomainThatDoesNotExist(t *testing.T) {
 	t.Parallel()
 
@@ -560,11 +567,12 @@ func TestDomainThatDoesNotExist(t *testing.T) {
 	cfg := defaultTestConfig(t)
 	cfg.Domains = []string{domain}
 
-	var deps *testDeps
+	var (
+		w    *watcher.Watcher
+		deps *testDeps
+	)
 
 	livednstest.Retry(t, "watcher checks", func(ctx context.Context) error {
-		var w *watcher.Watcher
-
 		w, deps = newTestWatcher(t, cfg)
 
 		deps.state.SetDomainState(domain, &state.DomainState{
@@ -602,6 +610,11 @@ func TestDomainThatDoesNotExist(t *testing.T) {
 		t.Errorf("records saved for %s: %v", domain, hs.RecordsByNameserver)
 	}
 
+	if w.DNSUnreachable() {
+		t.Error("DNSUnreachable is true after a round that reached " +
+			"the .com servers")
+	}
+
 	assertNotified(t, deps, "NS Change: "+domain, "warning")
 
 	// That is the only notification, and it removes both nameservers,