healthcheck: answer 503 when no root server gave a usable reply in the last round of DNS checks (closes #302)

The health check answered ok while every DNS lookup failed, as on a
network that intercepts DNS queries. A lookup whose root servers give no
usable reply now fails with ErrNoRootServerAnswered. At the end of each
round of DNS checks that shutdown did not cut short, the watcher records
whether every lookup failed with it or with ErrIntercepted; the health
check then gives status error, and /.well-known/healthcheck and /health
answer 503. Until the first round finishes it stays ok.

The health check now depends on the watcher, so fx starts the watcher
before the HTTP server and stops the server first. NewWithRootServers
lets tests start a resolver's walks at real servers that are not root
servers.

Model: opus-5-5

internal/handlers/healthcheck.go

@@ -2,16 +2,25 @@ package handlers
 
 import (
 	"net/http"
+
+	"sneak.berlin/go/dnswatcher/internal/healthcheck"
 )
 
-// HandleHealthCheck returns the health check handler.
+// HandleHealthCheck returns the health check handler. It answers 503
+// Service Unavailable when the health check's status is error, so that
+// a container health check fails, and 200 OK otherwise.
 func (h *Handlers) HandleHealthCheck() http.HandlerFunc {
 	return func(
 		writer http.ResponseWriter,
 		request *http.Request,
 	) {
-		h.respondJSON(
-			writer, request, h.hc.Check(), http.StatusOK,
-		)
+		resp := h.hc.Check()
+
+		code := http.StatusOK
+		if resp.Status == healthcheck.StatusError {
+			code = http.StatusServiceUnavailable
+		}
+
+		h.respondJSON(writer, request, resp, code)
 	}
 }

internal/handlers/healthcheck_test.go

@@ -0,0 +1,158 @@
+package handlers_test
+
+import (
+	"encoding/json"
+	"log/slog"
+	"net/http"
+	"net/http/httptest"
+	"testing"
+
+	"go.uber.org/fx/fxtest"
+
+	"sneak.berlin/go/dnswatcher/internal/config"
+	"sneak.berlin/go/dnswatcher/internal/globals"
+	"sneak.berlin/go/dnswatcher/internal/handlers"
+	"sneak.berlin/go/dnswatcher/internal/healthcheck"
+	"sneak.berlin/go/dnswatcher/internal/logger"
+	"sneak.berlin/go/dnswatcher/internal/notify"
+	"sneak.berlin/go/dnswatcher/internal/portcheck"
+	"sneak.berlin/go/dnswatcher/internal/resolver"
+	"sneak.berlin/go/dnswatcher/internal/state"
+	"sneak.berlin/go/dnswatcher/internal/tlscheck"
+	"sneak.berlin/go/dnswatcher/internal/watcher"
+)
+
+// TestHealthCheckAnswers503WhenDNSUnreachable serves the health check
+// before the watcher's first round of DNS checks, when it answers 200
+// with status ok, and after a round that could not reach the DNS, when
+// it answers 503 with status error. The watcher's resolver starts every
+// walk at public recursive resolvers of four operators, whose replies
+// are marked recursion available, as on a network that intercepts DNS
+// queries. If none of them replies, no root server answered, and the
+// round could not reach the DNS either.
+func TestHealthCheckAnswers503WhenDNSUnreachable(t *testing.T) {
+	t.Parallel()
+
+	hnd, w := newHandlersWithWatcher(t, resolver.NewWithRootServers(
+		slog.Default(),
+		[]string{"1.1.1.1", "8.8.8.8", "9.9.9.9", "208.67.222.222"},
+	))
+
+	assertHealthCheck(t, hnd, http.StatusOK, healthcheck.StatusOK)
+
+	w.RunOnce(t.Context())
+
+	assertHealthCheck(
+		t, hnd, http.StatusServiceUnavailable, healthcheck.StatusError,
+	)
+}
+
+// newHandlersWithWatcher builds real Handlers whose health check
+// follows the watcher it returns, which watches example.org with res.
+// The lifecycle is never started, so the watcher runs a round of checks
+// only when the test calls RunOnce.
+func newHandlersWithWatcher(
+	t *testing.T,
+	res *resolver.Resolver,
+) (*handlers.Handlers, *watcher.Watcher) {
+	t.Helper()
+
+	glob, err := globals.New(nil)
+	if err != nil {
+		t.Fatalf("globals.New: %v", err)
+	}
+
+	log, err := logger.New(nil, logger.Params{Globals: glob})
+	if err != nil {
+		t.Fatalf("logger.New: %v", err)
+	}
+
+	cfg := &config.Config{
+		Hostnames: []string{"example.org"},
+		DataDir:   t.TempDir(),
+	}
+	lifecycle := fxtest.NewLifecycle(t)
+
+	st, err := state.New(lifecycle, state.Params{Logger: log, Config: cfg})
+	if err != nil {
+		t.Fatalf("state.New: %v", err)
+	}
+
+	notifier, err := notify.New(
+		lifecycle, notify.Params{Logger: log, Config: cfg},
+	)
+	if err != nil {
+		t.Fatalf("notify.New: %v", err)
+	}
+
+	w, err := watcher.New(lifecycle, watcher.Params{
+		Logger:    log,
+		Config:    cfg,
+		State:     st,
+		Resolver:  res,
+		PortCheck: portcheck.NewStandalone(),
+		TLSCheck:  tlscheck.NewStandalone(),
+		Notify:    notifier,
+	})
+	if err != nil {
+		t.Fatalf("watcher.New: %v", err)
+	}
+
+	hc, err := healthcheck.New(lifecycle, healthcheck.Params{
+		Globals: glob,
+		Config:  cfg,
+		Logger:  log,
+		Watcher: w,
+	})
+	if err != nil {
+		t.Fatalf("healthcheck.New: %v", err)
+	}
+
+	hnd, err := handlers.New(nil, handlers.Params{
+		Logger:      log,
+		Globals:     glob,
+		Healthcheck: hc,
+		State:       st,
+		Notify:      notifier,
+	})
+	if err != nil {
+		t.Fatalf("handlers.New: %v", err)
+	}
+
+	return hnd, w
+}
+
+// assertHealthCheck serves one request to the health check and checks
+// that it answers with code and a JSON body whose status is status.
+func assertHealthCheck(
+	t *testing.T,
+	hnd *handlers.Handlers,
+	code int,
+	status string,
+) {
+	t.Helper()
+
+	rec := httptest.NewRecorder()
+	req := httptest.NewRequestWithContext(
+		t.Context(), http.MethodGet, "/.well-known/healthcheck", nil,
+	)
+
+	hnd.HandleHealthCheck()(rec, req)
+
+	if rec.Code != code {
+		t.Errorf("code = %d, want %d", rec.Code, code)
+	}
+
+	var body struct {
+		Status string `json:"status"`
+	}
+
+	err := json.Unmarshal(rec.Body.Bytes(), &body)
+	if err != nil {
+		t.Fatalf("decoding response: %v", err)
+	}
+
+	if body.Status != status {
+		t.Errorf("status = %q, want %q", body.Status, status)
+	}
+}