healthcheck: answer 503 when the last round of DNS checks reached no root server (closes #302)

The health check answered ok while every DNS lookup failed, as on a
network that intercepts DNS queries. A lookup whose root servers give no
usable reply now fails with ErrNoRootServerAnswered. At the end of each
round of DNS checks the watcher records whether every lookup failed with
it or with ErrIntercepted; the health check then gives status error, and
/.well-known/healthcheck and /health answer 503. Until the first round
finishes it stays ok.

The health check now depends on the watcher, so fx builds the watcher
first: its first round starts before the HTTP server listens, and the
server tests' fx graph includes it. NewWithRootServers lets tests start
a resolver's walks at real servers that are not root servers.

Model: opus-5-5

internal/watcher/watcher.go

@@ -11,6 +11,7 @@ import (
 	"sort"
 	"strconv"
 	"strings"
+	"sync/atomic"
 	"time"
 
 	"go.uber.org/fx"
@@ -62,6 +63,11 @@ type Watcher struct {
 	cancel    context.CancelFunc
 	done      chan struct{} // closed when Run returns
 	firstRun  bool
+
+	// dnsUnreachable is what DNSUnreachable reports. Each round of DNS
+	// checks sets it, and the HTTP server reads it from its own
+	// goroutines.
+	dnsUnreachable atomic.Bool
 }
 
 // New creates a new Watcher instance wired into the fx lifecycle.
@@ -197,6 +203,18 @@ func (w *Watcher) RunOnce(ctx context.Context) {
 	w.firstRun = false
 }
 
+// DNSUnreachable reports whether the most recent round of DNS checks
+// could not reach the DNS at all: no lookup in it got a reply from a
+// root server, because none of them answered or because the network
+// intercepts DNS queries. A round in which the root servers answered
+// reached the DNS, even when a name's own nameservers did not answer or
+// a domain does not exist. It is false until the first round finishes,
+// and a round that shutdown cut short changes nothing. It may be called
+// while a round runs.
+func (w *Watcher) DNSUnreachable() bool {
+	return w.dnsUnreachable.Load()
+}
+
 func (w *Watcher) detectFirstRun() {
 	snap := w.state.GetSnapshot()
 	hasState := len(snap.Domains) > 0 ||
@@ -265,15 +283,40 @@ func (w *Watcher) cleanupRemovedTargets() {
 // runDNSChecks performs DNS resolution for all configured domains
 // and hostnames, updating state with freshly resolved records.
 // This must complete before port or TLS checks run so those
-// checks operate on current IP addresses.
+// checks operate on current IP addresses. Unless ctx was cut short, it
+// then records whether this round could reach the DNS (see
+// DNSUnreachable).
 func (w *Watcher) runDNSChecks(ctx context.Context) {
+	errs := make([]error, 0, len(w.config.Domains)+len(w.config.Hostnames))
+
 	for _, domain := range w.config.Domains {
-		w.checkDomain(ctx, domain)
+		errs = append(errs, w.checkDomain(ctx, domain))
 	}
 
 	for _, hostname := range w.config.Hostnames {
-		w.checkHostname(ctx, hostname)
+		_, err := w.checkHostname(ctx, hostname)
+		errs = append(errs, err)
 	}
+
+	if ctx.Err() == nil {
+		w.dnsUnreachable.Store(noRootServerReplied(errs))
+	}
+}
+
+// noRootServerReplied reports whether errs, the errors of the lookups in
+// a round of DNS checks, nil for a lookup that succeeded, show that no
+// lookup got a reply from a root server: each failed because none of
+// them answered, or because the network intercepts DNS queries. A round
+// with no lookups shows nothing.
+func noRootServerReplied(errs []error) bool {
+	for _, err := range errs {
+		if !errors.Is(err, resolver.ErrNoRootServerAnswered) &&
+			!errors.Is(err, resolver.ErrIntercepted) {
+			return false
+		}
+	}
+
+	return len(errs) > 0
 }
 
 // logFailedLookup logs a failed DNS lookup at error level, unless ctx
@@ -292,14 +335,16 @@ func (w *Watcher) logFailedLookup(
 }
 
 // checkDomain looks up domain's delegation and checks domain with it
-// (see checkDomainResult).
+// (see checkDomainResult). It returns the lookup's error.
 func (w *Watcher) checkDomain(
 	ctx context.Context,
 	domain string,
-) {
+) error {
 	nameservers, err := w.resolver.LookupNS(ctx, domain)
 
 	w.checkDomainResult(ctx, domain, nameservers, err)
+
+	return err
 }
 
 // checkDomainResult finishes a check of domain from nameservers and err,
@@ -358,7 +403,7 @@ func (w *Watcher) checkDomainResult(
 	// check, as detectNSDisappearances needs.
 	if nxdomain {
 		w.state.DeleteHostnameState(domain)
-	} else if !w.checkHostname(ctx, domain) {
+	} else if saved, _ := w.checkHostname(ctx, domain); !saved {
 		return
 	}
 
@@ -484,11 +529,12 @@ func (w *Watcher) detectNSAddressChanges(
 }
 
 // checkHostname looks up hostname's records and checks them (see
-// checkHostnameResult). It reports whether it saved them.
+// checkHostnameResult). It reports whether it saved them, and returns
+// the lookup's error.
 func (w *Watcher) checkHostname(
 	ctx context.Context,
 	hostname string,
-) bool {
+) (bool, error) {
 	results, err := w.resolver.LookupAllRecords(ctx, hostname)
 	if err != nil {
 		w.logFailedLookup(
@@ -499,7 +545,7 @@ func (w *Watcher) checkHostname(
 		)
 	}
 
-	return w.checkHostnameResult(ctx, hostname, results, err)
+	return w.checkHostnameResult(ctx, hostname, results, err), err
 }
 
 // checkHostnameResult finishes a check of hostname from results and err,