healthcheck: answer 503 when the last round of DNS checks reached no root server (closes #302)

The health check answered ok while every DNS lookup failed, as on a
network that intercepts DNS queries. A lookup whose root servers give no
usable reply now fails with ErrNoRootServerAnswered. At the end of each
round of DNS checks the watcher records whether every lookup failed with
it or with ErrIntercepted; the health check then gives status error, and
/.well-known/healthcheck and /health answer 503. Until the first round
finishes it stays ok.

The health check now depends on the watcher, so fx builds the watcher
first: its first round starts before the HTTP server listens, and the
server tests' fx graph includes it. NewWithRootServers lets tests start
a resolver's walks at real servers that are not root servers.

Model: opus-5-5

internal/resolver/errors.go

@@ -36,6 +36,12 @@ var (
 	// so whether the name has addresses is unknown.
 	ErrNoNameserverAnswered = errors.New("no nameserver answered")
 
+	// ErrNoRootServerAnswered is returned when the root servers were
+	// asked and none of them gave a usable reply, as when the host
+	// running dnswatcher has lost its network. When they all refused,
+	// the error is ErrIntercepted instead.
+	ErrNoRootServerAnswered = errors.New("no root server answered")
+
 	// ErrNoNameserverAddress is returned when a nameserver is to be
 	// asked and no address for it was given.
 	ErrNoNameserverAddress = errors.New("nameserver has no address")

internal/resolver/iterative.go

@@ -293,7 +293,8 @@ func shuffled(
 // place. Otherwise a server that times out, refuses or gives a reply
 // that is not usable is passed over for the next. When every server
 // refused, the error says so, and when they are the root servers it is
-// ErrIntercepted.
+// ErrIntercepted. When the root servers gave no usable reply otherwise,
+// it is ErrNoRootServerAnswered.
 func (r *Resolver) queryServers(
 	ctx context.Context,
 	servers []string,
@@ -352,6 +353,10 @@ func (r *Resolver) queryServers(
 		)
 	}
 
+	if zone == "." {
+		return nil, fmt.Errorf("%w: %w", ErrNoRootServerAnswered, lastErr)
+	}
+
 	return nil, fmt.Errorf("all servers failed: %w", lastErr)
 }
 
@@ -528,7 +533,7 @@ func (r *Resolver) resolveNSIterative(
 	}
 
 	domain = dns.Fqdn(domain)
-	servers := rootServerList()
+	servers := r.rootServers
 	zone := "."
 
 	for range maxDelegation {
@@ -592,7 +597,7 @@ func (r *Resolver) resolveARecord(
 	}
 
 	hostname = dns.Fqdn(hostname)
-	servers := rootServerList()
+	servers := r.rootServers
 	zone := "."
 
 	var withoutAddresses []string
@@ -652,7 +657,7 @@ func (r *Resolver) FindAuthoritativeNameservers(
 	ctx context.Context,
 	domain string,
 ) ([]string, error) {
-	return r.findAuthoritativeNameservers(ctx, domain, rootServerList())
+	return r.findAuthoritativeNameservers(ctx, domain, r.rootServers)
 }
 
 // findAuthoritativeNameservers is FindAuthoritativeNameservers with each
@@ -1127,7 +1132,7 @@ func (r *Resolver) LookupNS(
 	}
 
 	return r.followDelegation(
-		ctx, dns.Fqdn(strings.ToLower(domain)), ".", nil, rootServerList(),
+		ctx, dns.Fqdn(strings.ToLower(domain)), ".", nil, r.rootServers,
 	)
 }
 

internal/resolver/resolver.go

@@ -43,10 +43,12 @@ type NameserverResponse struct {
 }
 
 // Resolver performs iterative DNS resolution from root servers.
+// rootServers are the addresses every walk from the root starts at.
 type Resolver struct {
-	log    *slog.Logger
-	client DNSClient
-	tcp    DNSClient
+	log         *slog.Logger
+	client      DNSClient
+	tcp         DNSClient
+	rootServers []string
 }
 
 // New creates a new Resolver instance for use with uber/fx.
@@ -55,9 +57,10 @@ func New(
 	params Params,
 ) (*Resolver, error) {
 	return &Resolver{
-		log:    params.Logger.Get(),
-		client: &udpClient{timeout: queryTimeoutDuration},
-		tcp:    &tcpClient{timeout: queryTimeoutDuration},
+		log:         params.Logger.Get(),
+		client:      &udpClient{timeout: queryTimeoutDuration},
+		tcp:         &tcpClient{timeout: queryTimeoutDuration},
+		rootServers: rootServerList(),
 	}, nil
 }
 
@@ -65,10 +68,22 @@ func New(
 // useful for testing without the fx lifecycle.
 func NewFromLogger(log *slog.Logger) *Resolver {
 	return &Resolver{
-		log:    log,
-		client: &udpClient{timeout: queryTimeoutDuration},
-		tcp:    &tcpClient{timeout: queryTimeoutDuration},
+		log:         log,
+		client:      &udpClient{timeout: queryTimeoutDuration},
+		tcp:         &tcpClient{timeout: queryTimeoutDuration},
+		rootServers: rootServerList(),
 	}
 }
 
+// NewWithRootServers creates a Resolver as NewFromLogger does, except
+// that every walk from the root starts at servers instead of at the root
+// servers. Tests use it to show what dnswatcher does when no root server
+// answers, or when something on the network answers in their place.
+func NewWithRootServers(log *slog.Logger, servers []string) *Resolver {
+	r := NewFromLogger(log)
+	r.rootServers = servers
+
+	return r
+}
+
 // Method implementations are in iterative.go.

internal/resolver/resolver_test.go

@@ -815,6 +815,7 @@ func TestQueryServers_EveryRootServerRefused(t *testing.T) {
 // but nothing answers at 192.0.2.1, a documentation address, so not
 // every server refused, wherever 192.0.2.1 falls in the random order:
 // the error is not ErrIntercepted and does not say every server refused.
+// No root server gave a usable reply, so it is ErrNoRootServerAnswered.
 func TestQueryServers_NotEveryRootServerRefused(t *testing.T) {
 	t.Parallel()
 
@@ -848,6 +849,34 @@ func TestQueryServers_NotEveryRootServerRefused(t *testing.T) {
 	require.NotErrorIs(t, err, resolver.ErrIntercepted)
 	// Both errors for every server refusing say "refused a query for".
 	require.NotContains(t, err.Error(), "refused a query for")
+	require.ErrorIs(t, err, resolver.ErrNoRootServerAnswered)
+}
+
+// TestLookupsWhenNoRootServerAnswers starts every walk at 192.0.2.1, a
+// documentation address where nothing answers, as if no root server
+// answered. The lookups the watcher makes, LookupNS for a domain and
+// LookupAllRecords for a hostname, both fail with
+// ErrNoRootServerAnswered, which the watcher reads.
+func TestLookupsWhenNoRootServerAnswers(t *testing.T) {
+	t.Parallel()
+
+	r := resolver.NewWithRootServers(slog.Default(), []string{noAnswerAddress})
+
+	t.Run("LookupNS", func(t *testing.T) {
+		t.Parallel()
+
+		_, err := r.LookupNS(t.Context(), "google.com")
+		require.ErrorIs(t, err, resolver.ErrNoRootServerAnswered)
+		require.NotErrorIs(t, err, resolver.ErrIntercepted)
+	})
+
+	t.Run("LookupAllRecords", func(t *testing.T) {
+		t.Parallel()
+
+		_, err := r.LookupAllRecords(t.Context(), "www.google.com")
+		require.ErrorIs(t, err, resolver.ErrNoRootServerAnswered)
+		require.NotErrorIs(t, err, resolver.ErrIntercepted)
+	})
 }
 
 // TestQueryServers_RecursiveResolversAsRootServers passes public