watcher: a removed nameserver is an NS Change, not an NS Failure (closes #262)

A nameserver that left a domain's delegation was reported twice: as the
domain's NS Change, and by the check of the domain's own records as an
error-level NS Failure saying it disappeared. The check of a name's
records now sends one NS Change at warning level, starting Hostname: or
Domain:, listing the nameservers it was asked at before and not now.
For a domain it leaves out those of its saved delegation, which the
domain's own NS Change lists, so a domain's new delegation is now
notified and saved only after, and only together with, its records.
Nothing is sent for a nameserver that joins.

A nameserver that was failing before it left is now listed too; the
NS Failure skipped it.

Model: opus-5-5

internal/watcher/message_test.go

@@ -5,7 +5,6 @@ import (
 	"strings"
 	"testing"
 
-	"sneak.berlin/go/dnswatcher/internal/config"
 	"sneak.berlin/go/dnswatcher/internal/state"
 	"sneak.berlin/go/dnswatcher/internal/watcher"
 )
@@ -74,21 +73,19 @@ b.ns.example.net.: 192.0.2.2`,
 func TestDomainRecordNotificationsNameTheDomain(t *testing.T) {
 	t.Parallel()
 
-	notifier := &mockNotifier{}
-	w := watcher.NewForTest(
-		&config.Config{Domains: []string{domain}},
-		nil, nil, nil, nil, notifier,
-	)
+	cfg := defaultTestConfig(t)
+	cfg.Domains = []string{domain}
+
+	deps := newTestDeps(t, cfg)
+	w := watcher.NewForTest(cfg, deps.state, nil, nil, nil, deps.notifier)
 
 	// nsA's address changes, which also makes it differ from nsC; nsB
-	// fails; nsC answers again; nsD is gone.
-	nsD := "d.ns.example.net."
+	// fails; nsC answers again.
 	w.DetectHostnameChanges(t.Context(), domain,
 		saved(map[string]*state.NameserverRecordState{
 			nsA: answered(map[string][]string{"A": {ip1}}),
 			nsB: answered(map[string][]string{"A": {ip1}}),
 			nsC: failed(),
-			nsD: answered(map[string][]string{"A": {ip1}}),
 		}),
 		saved(map[string]*state.NameserverRecordState{
 			nsA: answered(map[string][]string{"A": {ip2}}),
@@ -102,18 +99,17 @@ func TestDomainRecordNotificationsNameTheDomain(t *testing.T) {
 		t.Context(), domain, cnameState(ip1), cnameState(ip2),
 	)
 
-	// NS Failure is sent for nsB failing and for nsD being gone.
 	want := map[string]int{
 		"Record Change":        1,
 		"Inconsistency":        1,
-		"NS Failure":           2,
+		"NS Failure":           1,
 		"NS Recovery":          1,
 		"CNAME Address Change": 1,
 	}
 
 	sent := make(map[string]int)
 
-	for _, n := range notifier.getNotifications() {
+	for _, n := range deps.notifier.getNotifications() {
 		kind, _, _ := strings.Cut(n.Title, ":")
 		sent[kind]++
 
@@ -128,6 +124,89 @@ func TestDomainRecordNotificationsNameTheDomain(t *testing.T) {
 	}
 }
 
+// nsB, which answered, and nsC, which did not, are no longer among the
+// nameservers. One NS Change lists both, for the hostname and for the
+// configured apex domain with no delegation of its own. Nothing is sent
+// for the domain whose delegation on the previous check listed them:
+// the check of its delegation sends its NS Change.
+func TestRemovedNameserversAreAnNSChange(t *testing.T) {
+	t.Parallel()
+
+	tests := []struct {
+		name   string
+		target string
+		// The domain's nameservers saved by the previous check.
+		delegation []string
+		// The NS Change sent, or "" for none.
+		want string
+	}{
+		{
+			name:   "hostname",
+			target: host,
+			want: `Hostname: www.example.net
+Removed: b.ns.example.net., c.ns.example.net.`,
+		},
+		{
+			name:       "domain with no delegation of its own",
+			target:     domain,
+			delegation: []string{},
+			want: `Domain: example.net
+Removed: b.ns.example.net., c.ns.example.net.`,
+		},
+		{
+			name:       "domain whose delegation listed them",
+			target:     domain,
+			delegation: []string{nsA, nsB, nsC},
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			t.Parallel()
+
+			cfg := defaultTestConfig(t)
+			cfg.Domains = []string{domain}
+
+			deps := newTestDeps(t, cfg)
+			w := watcher.NewForTest(
+				cfg, deps.state, nil, nil, nil, deps.notifier,
+			)
+
+			deps.state.SetDomainState(domain, &state.DomainState{
+				Nameservers: tt.delegation,
+			})
+
+			prev := saved(map[string]*state.NameserverRecordState{
+				nsA: answered(map[string][]string{"A": {ip1}}),
+				nsB: answered(map[string][]string{"A": {ip1}}),
+				nsC: failed(),
+			})
+			current := saved(map[string]*state.NameserverRecordState{
+				nsA: answered(map[string][]string{"A": {ip1}}),
+			})
+
+			w.DetectHostnameChanges(t.Context(), tt.target, prev, current)
+
+			got := deps.notifier.getNotifications()
+
+			if tt.want == "" {
+				if len(got) != 0 {
+					t.Errorf("sent %v, want nothing", got)
+				}
+
+				return
+			}
+
+			title := "NS Change: " + tt.target
+			if len(got) != 1 || got[0].Title != title ||
+				got[0].Message != tt.want {
+				t.Errorf("sent %v, want one %q with message:\n%s",
+					got, title, tt.want)
+			}
+		})
+	}
+}
+
 // The startup notification counts the configured domains and hostnames,
 // although the state's hostnames also hold the apex domain's own
 // records. Nothing is looked up: the watcher has no resolver.

internal/watcher/watcher.go

@@ -322,6 +322,20 @@ func (w *Watcher) checkDomain(
 		ctx, nameservers, prevAddresses,
 	)
 
+	// A domain that does not exist has no records of its own: none are
+	// asked for, and those saved by an earlier check are removed.
+	// Otherwise the apex domain's records are also checked and saved as
+	// a hostname's, so that the port and TLS checks find its addresses.
+	// Notifications about them name it as a domain (see nameLine). When
+	// their lookup fails, the check fails as when the NS lookup does, so
+	// the saved nameservers stay those the saved records were asked at,
+	// as detectNSDisappearances needs.
+	if nxdomain {
+		w.state.DeleteHostnameState(domain)
+	} else if !w.checkHostname(ctx, domain) {
+		return
+	}
+
 	if hasPrev && !w.firstRun {
 		w.detectNSChanges(ctx, domain, prev.Nameservers, nameservers)
 		w.detectNSAddressChanges(ctx, domain, prevAddresses, addresses)
@@ -333,19 +347,6 @@ func (w *Watcher) checkDomain(
 		NXDomain:            nxdomain,
 		LastChecked:         now,
 	})
-
-	// A domain that does not exist has no records of its own: none are
-	// asked for, and those saved by an earlier check are removed.
-	if nxdomain {
-		w.state.DeleteHostnameState(domain)
-
-		return
-	}
-
-	// The apex domain's records are also checked and saved as a
-	// hostname's, so that the port and TLS checks find its addresses.
-	// Notifications about them name it as a domain (see nameLine).
-	w.checkHostname(ctx, domain)
 }
 
 func (w *Watcher) detectNSChanges(
@@ -456,10 +457,12 @@ func (w *Watcher) detectNSAddressChanges(
 	}
 }
 
+// checkHostname checks hostname's records and reports whether it saved
+// them; it does not when their lookup fails.
 func (w *Watcher) checkHostname(
 	ctx context.Context,
 	hostname string,
-) {
+) bool {
 	results, err := w.resolver.LookupAllRecords(ctx, hostname)
 	if err != nil {
 		w.logFailedLookup(
@@ -469,7 +472,7 @@ func (w *Watcher) checkHostname(
 			"error", err,
 		)
 
-		return
+		return false
 	}
 
 	prev, _ := w.state.GetHostnameState(hostname)
@@ -477,6 +480,8 @@ func (w *Watcher) checkHostname(
 	w.updateHostnameState(
 		ctx, hostname, buildHostnameState(results, prev, time.Now().UTC()),
 	)
+
+	return true
 }
 
 // updateHostnameState finishes a check of hostname from newState, built
@@ -772,28 +777,53 @@ func (w *Watcher) detectRecordChanges(
 	}
 }
 
+// detectNSDisappearances sends one NS Change listing the nameservers of
+// the previous check that are no longer those of the zone hostname is
+// in. For a configured apex domain it leaves out those of the domain's
+// saved delegation, which detectNSChanges notifies when they are removed
+// from it. checkDomain saves a new delegation only after this runs, and
+// only together with the records, so the saved one is the delegation
+// prev was asked at.
 func (w *Watcher) detectNSDisappearances(
 	ctx context.Context,
 	hostname string,
 	prev, current *state.HostnameState,
 ) {
-	for ns, prevNS := range prev.RecordsByNameserver {
-		if _, ok := current.RecordsByNameserver[ns]; ok || prevNS.Status != statusOK {
-			continue
+	var delegation []string
+
+	if w.isDomain(hostname) {
+		if domainState, ok := w.state.GetDomainState(hostname); ok {
+			delegation = domainState.Nameservers
 		}
-
-		msg := fmt.Sprintf(
-			"%s\nNameserver: %s disappeared",
-			w.nameLine(hostname), ns,
-		)
-
-		w.notify.SendNotification(
-			ctx,
-			"NS Failure: "+hostname,
-			msg,
-			"error",
-		)
 	}
+
+	var removed []string
+
+	for ns := range prev.RecordsByNameserver {
+		_, ok := current.RecordsByNameserver[ns]
+		if !ok && !slices.Contains(delegation, ns) {
+			removed = append(removed, ns)
+		}
+	}
+
+	if len(removed) == 0 {
+		return
+	}
+
+	slices.Sort(removed)
+
+	msg := fmt.Sprintf(
+		"%s\nRemoved: %s",
+		w.nameLine(hostname),
+		strings.Join(removed, ", "),
+	)
+
+	w.notify.SendNotification(
+		ctx,
+		"NS Change: "+hostname,
+		msg,
+		"warning",
+	)
 }
 
 // detectNSFailures notifies when a nameserver that answered on the

internal/watcher/watcher_test.go

@@ -139,6 +139,22 @@ func (m *mockNotifier) getNotifications() []notification {
 	return result
 }
 
+// cancelOnNotify is a mockNotifier that calls cancel at each
+// notification, as if shutdown cut the check short there.
+type cancelOnNotify struct {
+	*mockNotifier
+
+	cancel context.CancelFunc
+}
+
+func (n cancelOnNotify) SendNotification(
+	ctx context.Context,
+	title, message, priority string,
+) {
+	n.mockNotifier.SendNotification(ctx, title, message, priority)
+	n.cancel()
+}
+
 // --- Helpers to build a Watcher and run its checks against live DNS ---
 
 type testDeps struct {
@@ -558,6 +574,11 @@ func TestDomainThatDoesNotExist(t *testing.T) {
 // nameservers and without nxdomain, and its records, asked at the
 // codeberg.page servers, are saved. Those are testSmallDomain's two
 // nameservers; github.io, the zone of the README's example, has eight.
+// The previous check saved a delegation to oldNS1, now withdrawn, and
+// records asked at oldNS1. A first check is cut short at its first
+// notification, as shutdown would, and a second runs in full: across
+// them the NS Change of its delegation removes oldNS1, and no other NS
+// Change lists it again.
 func TestDomainWithNoDelegationOfItsOwn(t *testing.T) {
 	t.Parallel()
 
@@ -569,9 +590,30 @@ func TestDomainWithNoDelegationOfItsOwn(t *testing.T) {
 	var deps *testDeps
 
 	livednstest.Retry(t, "watcher checks", func(ctx context.Context) error {
-		var w *watcher.Watcher
+		deps = newTestDeps(t, cfg)
 
-		w, deps = newTestWatcher(t, cfg)
+		firstCtx, cancel := context.WithCancel(ctx)
+		defer cancel()
+
+		w := watcher.NewForTest(
+			cfg,
+			deps.state,
+			resolver.NewFromLogger(slog.Default()),
+			deps.portChecker,
+			deps.tlsChecker,
+			cancelOnNotify{mockNotifier: deps.notifier, cancel: cancel},
+		)
+
+		deps.state.SetDomainState(domain, &state.DomainState{
+			Nameservers: []string{oldNS1},
+		})
+		deps.state.SetHostnameState(domain, saved(
+			map[string]*state.NameserverRecordState{
+				oldNS1: answered(map[string][]string{"A": {oldIP}}),
+			},
+		))
+
+		w.RunOnce(firstCtx)
 
 		err := checkOnce(ctx, w, deps)
 
@@ -593,6 +635,22 @@ func TestDomainWithNoDelegationOfItsOwn(t *testing.T) {
 	if _, ok := deps.state.GetHostnameState(domain); !ok {
 		t.Errorf("no records saved for %s", domain)
 	}
+
+	title := "NS Change: " + domain
+	want := "Domain: " + domain + "\nAdded: \nRemoved: " + oldNS1
+
+	var changes []notification
+
+	for _, n := range deps.notifier.getNotifications() {
+		if n.Title == title {
+			changes = append(changes, n)
+		}
+	}
+
+	if len(changes) != 1 || changes[0].Message != want {
+		t.Errorf("sent %v, want one %q with message:\n%s",
+			changes, title, want)
+	}
 }
 
 func TestNSAddressChangeDetection(t *testing.T) {
@@ -1049,7 +1107,7 @@ func TestSendTestNotification_Disabled(t *testing.T) {
 	}
 }
 
-func TestNSFailureAndRecovery(t *testing.T) {
+func TestNSChangeAndRecovery(t *testing.T) {
 	t.Parallel()
 
 	cfg := defaultTestConfig(t)
@@ -1059,7 +1117,7 @@ func TestNSFailureAndRecovery(t *testing.T) {
 
 	// The saved state has every nameserver live DNS lists as one that
 	// did not answer, and, as answering, one that live DNS does not
-	// list, which then disappears.
+	// list, which the check then finds removed.
 	_, deps := runChecks(t, cfg, func(deps *testDeps) {
 		byNameserver := map[string]*state.NameserverRecordState{
 			oldNS1: answered(map[string][]string{"A": {oldIP}}),
@@ -1071,7 +1129,7 @@ func TestNSFailureAndRecovery(t *testing.T) {
 		deps.state.SetHostnameState(testHost, saved(byNameserver))
 	})
 
-	assertNotified(t, deps, "NS Failure: "+testHost, "error")
+	assertNotified(t, deps, "NS Change: "+testHost, "warning")
 	assertNotified(t, deps, "NS Recovery: "+testHost, "success")
 
 	// A nameserver that did not answer has no records to compare, so