watcher: a hostname whose zone's nameservers all stop answering gets NS Failure (closes #284)

When none of the nameservers of a hostname's zone answered, the walk to
them failed and the check was only logged. The walk now returns a
NoAnswerError naming the nameservers of the zone it reached when none
of them answers. When those are the nameservers the previous check
saved for the hostname, each is saved as failed with the reason, as a
domain's are, so it gets an NS Failure and later an NS Recovery. A walk
that fails before that zone, as when the host loses its network, is
still only logged.

Not tested live: the resolver naming a zone's nameservers that never
answer, as no live zone has such nameservers; the watcher test builds
that error.

Model: opus-5-5

internal/watcher/export_test.go

@@ -122,6 +122,14 @@ func (w *Watcher) RunTLSChecks(ctx context.Context) {
 	w.runTLSChecks(ctx)
 }
 
+// FailedNameservers exports failedNameservers for testing.
+func FailedNameservers(
+	err error,
+	prev *state.HostnameState,
+) map[string]*resolver.NameserverResponse {
+	return failedNameservers(err, prev)
+}
+
 // BuildHostnameState exports buildHostnameState for testing.
 func BuildHostnameState(
 	results map[string]*resolver.NameserverResponse,

internal/watcher/interfaces.go

@@ -20,7 +20,10 @@ type DNSResolver interface {
 	) ([]string, error)
 
 	// LookupAllRecords queries all record types for a hostname,
-	// returning each nameserver's response keyed by nameserver.
+	// returning each nameserver's response keyed by nameserver. When
+	// the walk to the nameservers reached a zone through a referral and
+	// none of that zone's nameservers answered, the error is a
+	// *resolver.NoAnswerError naming them.
 	LookupAllRecords(
 		ctx context.Context,
 		hostname string,

internal/watcher/nsfailure_test.go

@@ -2,6 +2,7 @@ package watcher_test
 
 import (
 	"context"
+	"errors"
 	"fmt"
 	"log/slog"
 	"strings"
@@ -14,6 +15,10 @@ import (
 	"sneak.berlin/go/dnswatcher/internal/watcher"
 )
 
+// errNoReply is why a walk that got no reply from a zone's servers
+// failed.
+var errNoReply = errors.New("all servers failed: i/o timeout")
+
 // answered is what a check saves for a nameserver that answered with
 // these records.
 func answered(records map[string][]string) *state.NameserverRecordState {
@@ -183,6 +188,116 @@ func TestNSFailureAlertNamesHostnameNameserverAndReason(t *testing.T) {
 	}
 }
 
+// TestHostnameWhoseNameserversAllStopAnswering starts from a check of
+// host in which nsA and nsB answered, then gives the watcher the error
+// the walk to host's nameservers returns when it reached their zone and
+// neither answered. No live zone has nameservers that never answer, so
+// the error is built here. Both are saved as failed, with the reason,
+// and each gets an NS Failure. On the next check both answer, and each
+// gets an NS Recovery.
+func TestHostnameWhoseNameserversAllStopAnswering(t *testing.T) {
+	t.Parallel()
+
+	cfg := defaultTestConfig(t)
+	cfg.Hostnames = []string{host}
+
+	// The watcher is given each check's lookup result, so it has no
+	// resolver.
+	deps := newTestDeps(t, cfg)
+	w := watcher.NewForTest(
+		cfg, deps.state, nil,
+		deps.portChecker, deps.tlsChecker, deps.notifier,
+	)
+	w.SetFirstRun(false)
+
+	records := map[string][]string{"A": {ip1}}
+	bothAnswer := map[string]*resolver.NameserverResponse{
+		nsA: response(records), nsB: response(records),
+	}
+
+	deps.state.SetHostnameState(
+		host, watcher.BuildHostnameState(bothAnswer, nil, time.Now()),
+	)
+
+	err := &resolver.NoAnswerError{
+		Nameservers: []string{nsA, nsB}, Err: errNoReply,
+	}
+
+	prev, _ := deps.state.GetHostnameState(host)
+	w.UpdateHostnameState(t.Context(), host, watcher.BuildHostnameState(
+		watcher.FailedNameservers(err, prev), prev, time.Now(),
+	))
+
+	current, _ := deps.state.GetHostnameState(host)
+
+	for _, ns := range []string{nsA, nsB} {
+		got := current.RecordsByNameserver[ns]
+		if got.Status != failed().Status || got.Error != err.Error() {
+			t.Errorf(
+				"%s saved with status %q, error %q; want %q, %q",
+				ns, got.Status, got.Error, failed().Status, err.Error(),
+			)
+		}
+	}
+
+	if n := countNotifications(deps, "NS Failure: "+host); n != 2 {
+		t.Errorf("sent %d NS Failure notifications, want 2", n)
+	}
+
+	w.UpdateHostnameState(t.Context(), host, watcher.BuildHostnameState(
+		bothAnswer, current, time.Now(),
+	))
+
+	if n := countNotifications(deps, "NS Recovery: "+host); n != 2 {
+		t.Errorf("sent %d NS Recovery notifications, want 2", n)
+	}
+}
+
+// TestFailedNameserversOnlyForThoseOfThePreviousCheck gives
+// FailedNameservers lookup errors after a check in which nsA and nsB
+// answered. Only an error that names both makes them failed.
+func TestFailedNameserversOnlyForThoseOfThePreviousCheck(t *testing.T) {
+	t.Parallel()
+
+	records := map[string][]string{"A": {ip1}}
+	bothAnswered := saved(map[string]*state.NameserverRecordState{
+		nsA: answered(records), nsB: answered(records),
+	})
+	noAnswer := func(nameservers ...string) error {
+		return &resolver.NoAnswerError{Nameservers: nameservers, Err: errNoReply}
+	}
+
+	tests := []struct {
+		name       string
+		err        error
+		prev       *state.HostnameState
+		wantFailed bool
+	}{
+		{"both did not answer", noAnswer(nsA, nsB), bothAnswered, true},
+		{
+			"the servers of a parent zone did not answer",
+			noAnswer(nsC), bothAnswered, false,
+		},
+		{
+			"the zone's delegation lists only nsA now",
+			noAnswer(nsA), bothAnswered, false,
+		},
+		{"no root server answered", errNoReply, bothAnswered, false},
+		{"no previous check", noAnswer(nsA, nsB), nil, false},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			t.Parallel()
+
+			got := watcher.FailedNameservers(tt.err, tt.prev)
+			if (got != nil) != tt.wantFailed {
+				t.Errorf("returned %v, want failed: %v", got, tt.wantFailed)
+			}
+		})
+	}
+}
+
 // TestNameserverThatNeverAnswers asks a nameserver address where
 // nothing answers, 192.0.2.1, and checks what the watcher saves for it.
 // The deadline outlasts the resolver's first two-second try, as in the

internal/watcher/watcher.go

@@ -465,12 +465,15 @@ func (w *Watcher) detectNSAddressChanges(
 }
 
 // checkHostname checks hostname's records and reports whether it saved
-// them; it does not when their lookup fails.
+// them; it does not when their lookup fails, unless the nameservers that
+// did not answer are those of the previous check (see failedNameservers).
 func (w *Watcher) checkHostname(
 	ctx context.Context,
 	hostname string,
 ) bool {
 	results, err := w.resolver.LookupAllRecords(ctx, hostname)
+	prev, _ := w.state.GetHostnameState(hostname)
+
 	if err != nil {
 		w.logFailedLookup(
 			ctx,
@@ -479,11 +482,12 @@ func (w *Watcher) checkHostname(
 			"error", err,
 		)
 
-		return false
+		results = failedNameservers(err, prev)
+		if results == nil {
+			return false
+		}
 	}
 
-	prev, _ := w.state.GetHostnameState(hostname)
-
 	w.updateHostnameState(
 		ctx, hostname, buildHostnameState(results, prev, time.Now().UTC()),
 	)
@@ -491,6 +495,40 @@ func (w *Watcher) checkHostname(
 	return true
 }
 
+// failedNameservers returns a failed response for each nameserver in
+// prev, a name's state from the previous check, with err as the reason,
+// when err says that the walk to the name's nameservers reached their
+// zone and none of them answered. For any other error, as when the walk
+// got no answer before it reached their zone, or when prev is nil, it
+// returns nil.
+func failedNameservers(
+	err error,
+	prev *state.HostnameState,
+) map[string]*resolver.NameserverResponse {
+	var noAnswer *resolver.NoAnswerError
+	if !errors.As(err, &noAnswer) || prev == nil {
+		return nil
+	}
+
+	saved := slices.Sorted(maps.Keys(prev.RecordsByNameserver))
+	if !slices.Equal(noAnswer.Nameservers, saved) {
+		return nil
+	}
+
+	results := make(map[string]*resolver.NameserverResponse)
+
+	for _, ns := range saved {
+		results[ns] = &resolver.NameserverResponse{
+			Nameserver: ns,
+			Records:    make(map[string][]string),
+			Status:     resolver.StatusError,
+			Error:      err.Error(),
+		}
+	}
+
+	return results
+}
+
 // updateHostnameState finishes a check of hostname from newState, built
 // from its nameservers' answers: it follows the CNAME in them, notifies
 // what changed since the previous check, and saves newState.