watcher: send the inconsistency alert once per disagreement (closes #158)

detectInconsistencies alerted for neighbouring pairs of nameservers whose
records differed, on every DNS check, for as long as they differed. It now
also takes the previous hostname state, compares every pair of nameservers,
and alerts for a pair only when both were in that state with equal records.
The state loaded at startup is the previous state for the first check, so a
disagreement saved before a restart is not reported again. The choice of
pairs is tested on record data, and the alert itself through the hostname
change detection with the notifier stand-in and no resolver. The README
describes the new behaviour.

Model: opus-5-5

README.md

@@ -76,10 +76,10 @@ rejected.
     responding again.
   - **Inconsistency detected**: Two nameservers that agreed on the
     previous check now return different record sets for the same
-    hostname. This is sent once, on the check where they start to
+    hostname. Every pair of nameservers is compared. The alert is sent
-    disagree, and not again while they keep disagreeing, including
+    once for each such pair, on the check where they start to disagree,
-    after a restart. If they agree again and later disagree, it is
+    and not again while they keep disagreeing, including after a restart.
-    sent again.
+    If they agree again and later disagree, it is sent again.
 
 ### TCP Port Monitoring
 

TODO.md

@@ -25,7 +25,8 @@ Rationale, Design, TODO, License, Author) if any are still missing.
 
 - 2026-09-28: the inconsistency alert is sent once, on the check where two
   nameservers that agreed start to disagree, instead of on every check while
-  they disagree, and not again after a restart (closes #158).
+  they disagree, and not again after a restart. Every pair of nameservers is
+  compared, not only neighbours in sorted order of name (closes #158).
 - 2026-09-28: DNS names in record values (CNAME, MX, SRV and NS targets) are
   lower-cased, so nameservers that answer in different letter case no longer
   count as inconsistent or as a record change (closes #157).

internal/watcher/export_test.go

@@ -1,6 +1,10 @@
 package watcher
 
-import "sneak.berlin/go/dnswatcher/internal/state"
+import (
+	"context"
+
+	"sneak.berlin/go/dnswatcher/internal/state"
+)
 
 // NewlyDisagreeingPairs exports newlyDisagreeingPairs for testing.
 func NewlyDisagreeingPairs(
@@ -9,3 +13,13 @@ func NewlyDisagreeingPairs(
 ) [][2]string {
 	return newlyDisagreeingPairs(prev, current)
 }
+
+// DetectHostnameChanges exports detectHostnameChanges for testing.
+func (w *Watcher) DetectHostnameChanges(
+	ctx context.Context,
+	hostname string,
+	prev *state.HostnameState,
+	current map[string]map[string][]string,
+) {
+	w.detectHostnameChanges(ctx, hostname, prev, current)
+}

internal/watcher/inconsistency_test.go

@@ -8,6 +8,16 @@ import (
 	"sneak.berlin/go/dnswatcher/internal/watcher"
 )
 
+const (
+	host = "www.example.net"
+	nsA  = "a.ns.example.net."
+	nsB  = "b.ns.example.net."
+	nsC  = "c.ns.example.net."
+	ip1  = "192.0.2.1"
+	ip2  = "192.0.2.2"
+	ip3  = "192.0.2.3"
+)
+
 // hostnameState builds the state a check with these records leaves behind.
 func hostnameState(
 	records map[string]map[string][]string,
@@ -29,24 +39,20 @@ func hostnameState(
 func TestNewlyDisagreeingPairs(t *testing.T) {
 	t.Parallel()
 
-	const (
-		nsA = "a.ns.example.net."
-		nsB = "b.ns.example.net."
-		ip1 = "192.0.2.1"
-		ip2 = "192.0.2.2"
-	)
-
 	onlyA := map[string]map[string][]string{nsA: {"A": {ip1}}}
-	agree := map[string]map[string][]string{
+	agree := map[string]map[string][]string{nsA: {"A": {ip1}}, nsB: {"A": {ip1}}}
-		nsA: {"A": {ip1}},
+	disagree := map[string]map[string][]string{nsA: {"A": {ip1}}, nsB: {"A": {ip2}}}
-		nsB: {"A": {ip1}},
-	}
-	disagree := map[string]map[string][]string{
-		nsA: {"A": {ip1}},
-		nsB: {"A": {ip2}},
-	}
 	alert := [][2]string{{nsA, nsB}}
 
+	// b already disagrees with a and c; then c changes, so a and c,
+	// which agreed, now differ.
+	bDiffers := map[string]map[string][]string{
+		nsA: {"A": {ip1}}, nsB: {"A": {ip2}}, nsC: {"A": {ip1}},
+	}
+	cChanges := map[string]map[string][]string{
+		nsA: {"A": {ip1}}, nsB: {"A": {ip2}}, nsC: {"A": {ip3}},
+	}
+
 	// Each case starts from the state loaded at startup and runs the
 	// checks in order; want[i] is what check i alerts for.
 	tests := []struct {
@@ -85,6 +91,12 @@ func TestNewlyDisagreeingPairs(t *testing.T) {
 			checks: []map[string]map[string][]string{disagree, agree, disagree},
 			want:   [][][2]string{alert, nil, alert},
 		},
+		{
+			name:   "new disagreement while another nameserver differs alerts",
+			loaded: bDiffers,
+			checks: []map[string]map[string][]string{cChanges, cChanges},
+			want:   [][][2]string{{{nsA, nsC}}, nil},
+		},
 	}
 
 	for _, tt := range tests {
@@ -107,3 +119,58 @@ func TestNewlyDisagreeingPairs(t *testing.T) {
 		})
 	}
 }
+
+func TestInconsistencyAlert(t *testing.T) {
+	t.Parallel()
+
+	agree := map[string]map[string][]string{nsA: {"A": {ip1}}, nsB: {"A": {ip1}}}
+	disagree := map[string]map[string][]string{nsA: {"A": {ip1}}, nsB: {"A": {ip2}}}
+
+	// Each case starts from the state loaded at startup and then sees
+	// the nameservers disagree on three checks in a row.
+	tests := []struct {
+		name   string
+		loaded map[string]map[string][]string
+		want   int
+	}{
+		{
+			name:   "disagreement lasting several checks alerts once",
+			loaded: agree,
+			want:   1,
+		},
+		{
+			name:   "disagreement in the loaded state does not alert",
+			loaded: disagree,
+			want:   0,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			t.Parallel()
+
+			// The hostname change detection uses only the notifier.
+			notifier := &mockNotifier{}
+			w := watcher.NewForTest(nil, nil, nil, nil, nil, notifier)
+
+			prev := hostnameState(tt.loaded)
+
+			for range 3 {
+				w.DetectHostnameChanges(t.Context(), host, prev, disagree)
+				prev = hostnameState(disagree)
+			}
+
+			got := 0
+
+			for _, n := range notifier.getNotifications() {
+				if n.Title == "Inconsistency: "+host {
+					got++
+				}
+			}
+
+			if got != tt.want {
+				t.Errorf("sent %d inconsistency alerts, want %d", got, tt.want)
+			}
+		})
+	}
+}

internal/watcher/watcher.go

@@ -470,8 +470,8 @@ func (w *Watcher) detectInconsistencies(
 	}
 }
 
-// newlyDisagreeingPairs returns the pairs of nameservers, neighbours in
+// newlyDisagreeingPairs returns every pair of nameservers whose records
-// sorted order, whose records differ in current but were equal in prev.
+// differ in current but were equal in prev, in sorted order of name.
 // A pair with a nameserver missing from prev is not returned.
 func newlyDisagreeingPairs(
 	prev *state.HostnameState,
@@ -486,22 +486,21 @@ func newlyDisagreeingPairs(
 
 	var pairs [][2]string
 
-	for i := range len(nameservers) - 1 {
+	for i, ns1 := range nameservers {
-		ns1 := nameservers[i]
+		for _, ns2 := range nameservers[i+1:] {
-		ns2 := nameservers[i+1]
+			if recordsEqual(current[ns1], current[ns2]) {
+				continue
+			}
 
-		if recordsEqual(current[ns1], current[ns2]) {
+			prev1, ok1 := prev.RecordsByNameserver[ns1]
-			continue
+			prev2, ok2 := prev.RecordsByNameserver[ns2]
+
+			if !ok1 || !ok2 || !recordsEqual(prev1.Records, prev2.Records) {
+				continue
+			}
+
+			pairs = append(pairs, [2]string{ns1, ns2})
 		}
-
-		prev1, ok1 := prev.RecordsByNameserver[ns1]
-		prev2, ok2 := prev.RecordsByNameserver[ns2]
-
-		if !ok1 || !ok2 || !recordsEqual(prev1.Records, prev2.Records) {
-			continue
-		}
-
-		pairs = append(pairs, [2]string{ns1, ns2})
 	}
 
 	return pairs