healthcheck: answer 503 when no root server gave a usable reply in the last round of DNS checks (closes #302)

The health check answered ok while every DNS lookup failed, as on a
network that intercepts DNS queries. A lookup whose root servers give no
usable reply now fails with ErrNoRootServerAnswered. At the end of each
round of DNS checks that shutdown did not cut short, the watcher records
whether every lookup failed with it or with ErrIntercepted; the health
check then gives status error, and /.well-known/healthcheck and /health
answer 503. Until the first round finishes it stays ok.

The health check now depends on the watcher, so fx starts the watcher
before the HTTP server and stops the server first. NewWithRootServers
lets tests start a resolver's walks at real servers that are not root
servers.

Model: opus-5-5

internal/healthcheck/healthcheck.go

@@ -11,6 +11,13 @@ import (
 	"sneak.berlin/go/dnswatcher/internal/config"
 	"sneak.berlin/go/dnswatcher/internal/globals"
 	"sneak.berlin/go/dnswatcher/internal/logger"
+	"sneak.berlin/go/dnswatcher/internal/watcher"
+)
+
+// Values of Response.Status.
+const (
+	StatusOK    = "ok"
+	StatusError = "error"
 )
 
 // Params contains dependencies for Healthcheck.
@@ -20,6 +27,7 @@ type Params struct {
 	Globals *globals.Globals
 	Config  *config.Config
 	Logger  *logger.Logger
+	Watcher *watcher.Watcher
 }
 
 // Healthcheck provides health status information.
@@ -61,10 +69,17 @@ func New(
 	return healthcheck, nil
 }
 
-// Check returns the current health status.
+// Check returns the current health status. Status is StatusError when
+// the watcher's most recent round of DNS checks could not reach the DNS
+// (see watcher.Watcher.DNSUnreachable), and StatusOK otherwise.
 func (h *Healthcheck) Check() *Response {
+	status := StatusOK
+	if h.params.Watcher.DNSUnreachable() {
+		status = StatusError
+	}
+
 	return &Response{
-		Status:        "ok",
+		Status:        status,
 		Now:           time.Now().UTC().Format(time.RFC3339Nano),
 		UptimeSeconds: int64(h.uptime().Seconds()),
 		UptimeHuman:   h.uptime().String(),

internal/healthcheck/healthcheck_test.go

@@ -3,10 +3,12 @@ package healthcheck_test
 import (
 	"context"
 	"encoding/json"
+	"log/slog"
 	"testing"
 	"time"
 
 	"go.uber.org/fx"
+	"go.uber.org/fx/fxtest"
 
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
@@ -15,6 +17,12 @@ import (
 	"sneak.berlin/go/dnswatcher/internal/globals"
 	"sneak.berlin/go/dnswatcher/internal/healthcheck"
 	"sneak.berlin/go/dnswatcher/internal/logger"
+	"sneak.berlin/go/dnswatcher/internal/notify"
+	"sneak.berlin/go/dnswatcher/internal/portcheck"
+	"sneak.berlin/go/dnswatcher/internal/resolver"
+	"sneak.berlin/go/dnswatcher/internal/state"
+	"sneak.berlin/go/dnswatcher/internal/tlscheck"
+	"sneak.berlin/go/dnswatcher/internal/watcher"
 )
 
 // recordingLifecycle is a minimal fx.Lifecycle that records the hooks
@@ -30,12 +38,14 @@ func (l *recordingLifecycle) Append(hook fx.Hook) {
 
 // newHealthcheck builds a Healthcheck through the real constructor and
 // runs the registered OnStart hook so StartupTime is set the same way
-// the fx lifecycle would set it.
+// the fx lifecycle would set it. It follows the watcher it returns,
+// which watches example.org with res and has run no round of checks.
 func newHealthcheck(
 	t *testing.T,
+	res *resolver.Resolver,
 	maintenance bool,
 	version string,
-) *healthcheck.Healthcheck {
+) (*healthcheck.Healthcheck, *watcher.Watcher) {
 	t.Helper()
 
 	g := &globals.Globals{Appname: "dnswatcher", Version: version}
@@ -43,12 +53,21 @@ func newHealthcheck(
 	log, err := logger.New(nil, logger.Params{Globals: g})
 	require.NoError(t, err)
 
+	cfg := &config.Config{
+		MaintenanceMode: maintenance,
+		Hostnames:       []string{"example.org"},
+		DataDir:         t.TempDir(),
+	}
+
+	w := newWatcher(t, log, cfg, res)
+
 	lifecycle := &recordingLifecycle{}
 
 	hc, err := healthcheck.New(lifecycle, healthcheck.Params{
 		Globals: g,
-		Config:  &config.Config{MaintenanceMode: maintenance},
+		Config:  cfg,
 		Logger:  log,
+		Watcher: w,
 	})
 	require.NoError(t, err)
 
@@ -57,16 +76,53 @@ func newHealthcheck(
 	require.NotNil(t, lifecycle.hooks[0].OnStart)
 	require.NoError(t, lifecycle.hooks[0].OnStart(context.Background()))
 
-	return hc
+	return hc, w
+}
+
+// newWatcher builds a watcher through the real constructor, with res as
+// its resolver. Its lifecycle is never started, so it runs a round of
+// checks only when the test calls RunOnce.
+func newWatcher(
+	t *testing.T,
+	log *logger.Logger,
+	cfg *config.Config,
+	res *resolver.Resolver,
+) *watcher.Watcher {
+	t.Helper()
+
+	lifecycle := fxtest.NewLifecycle(t)
+
+	st, err := state.New(lifecycle, state.Params{Logger: log, Config: cfg})
+	require.NoError(t, err)
+
+	notifier, err := notify.New(
+		lifecycle, notify.Params{Logger: log, Config: cfg},
+	)
+	require.NoError(t, err)
+
+	w, err := watcher.New(lifecycle, watcher.Params{
+		Logger:    log,
+		Config:    cfg,
+		State:     st,
+		Resolver:  res,
+		PortCheck: portcheck.NewStandalone(),
+		TLSCheck:  tlscheck.NewStandalone(),
+		Notify:    notifier,
+	})
+	require.NoError(t, err)
+
+	return w
 }
 
 func TestCheckStatusAndPayloadShape(t *testing.T) {
 	t.Parallel()
 
-	hc := newHealthcheck(t, false, "v9.9.9")
+	hc, _ := newHealthcheck(
+		t, resolver.NewFromLogger(slog.Default()), false, "v9.9.9",
+	)
 	resp := hc.Check()
 
-	assert.Equal(t, "ok", resp.Status)
+	assert.Equal(t, healthcheck.StatusOK, resp.Status)
 
 	// The JSON shape and field names are part of the contract for the
 	// /health and /.well-known/healthcheck routes, so assert on the
@@ -114,7 +170,10 @@ func TestCheckMaintenanceModeReflectsConfig(t *testing.T) {
 		t.Run(tt.name, func(t *testing.T) {
 			t.Parallel()
 
-			hc := newHealthcheck(t, tt.maintenance, "test")
+			hc, _ := newHealthcheck(
+				t, resolver.NewFromLogger(slog.Default()), tt.maintenance,
+				"test",
+			)
 			resp := hc.Check()
 			assert.Equal(t, tt.maintenance, resp.Maintenance,
 				"maintenanceMode must mirror Config.MaintenanceMode")
@@ -125,10 +184,38 @@ func TestCheckMaintenanceModeReflectsConfig(t *testing.T) {
 func TestCheckSurfacesVersionAndAppname(t *testing.T) {
 	t.Parallel()
 
-	hc := newHealthcheck(t, false, "surfaced-version-123")
+	hc, _ := newHealthcheck(
+		t, resolver.NewFromLogger(slog.Default()), false,
+		"surfaced-version-123",
+	)
 	resp := hc.Check()
 
 	assert.Equal(t, "surfaced-version-123", resp.Version,
 		"version from globals must appear in the payload")
 	assert.Equal(t, "dnswatcher", resp.Appname)
 }
+
+// TestCheckStatusFollowsTheDNSChecks checks that the status is ok
+// before the watcher's first round of DNS checks, and error after a
+// round that could not reach the DNS. The watcher's resolver starts
+// every walk at public recursive resolvers of four operators, whose
+// replies are marked recursion available, as on a network that
+// intercepts DNS queries. If none of them replies, no root server
+// answered, and the round could not reach the DNS either.
+func TestCheckStatusFollowsTheDNSChecks(t *testing.T) {
+	t.Parallel()
+
+	res := resolver.NewWithRootServers(
+		slog.Default(),
+		[]string{"1.1.1.1", "8.8.8.8", "9.9.9.9", "208.67.222.222"},
+	)
+	hc, w := newHealthcheck(t, res, false, "test")
+
+	assert.Equal(t, healthcheck.StatusOK, hc.Check().Status,
+		"status before the first round")
+
+	w.RunOnce(t.Context())
+
+	assert.Equal(t, healthcheck.StatusError, hc.Check().Status,
+		"status after a round that could not reach the DNS")
+}