watcher: send the inconsistency alert once per disagreement (closes #158)

detectInconsistencies alerted for neighbouring pairs of nameservers whose
records differed, on every DNS check, for as long as they differed. It now
also takes the previous hostname state, compares every pair of nameservers,
and alerts for a pair that differs unless both were in that state and
already differed there, so a nameserver new on a check that answers
differently is reported once. The state loaded at startup is the previous
state for the first check, so a disagreement saved before a restart is not
reported again. The choice of pairs is tested on record data, and the alert
through the hostname change detection with the notifier stand-in and no
resolver. The README describes the new behaviour.

Model: opus-5-5

README.md

@@ -74,12 +74,15 @@ rejected.
     This is distinct from "responded with no records."
   - **NS recovery**: A previously-unreachable nameserver starts
     responding again.
-  - **Inconsistency detected**: Two nameservers that agreed on the
-    previous check now return different record sets for the same
-    hostname. This is sent once, on the check where they start to
-    disagree, and not again while they keep disagreeing, including
-    after a restart. If they agree again and later disagree, it is
-    sent again.
+  - **Inconsistency detected**: Two nameservers return different record
+    sets for the same hostname and did not already differ on the previous
+    check. Every pair of nameservers is compared. The alert is sent once
+    for each such pair, on the check where they start to disagree, and not
+    again while they keep disagreeing, including after a restart. A
+    nameserver that was not in the previous check (newly added, or back
+    after dropping out) and answers differently is reported on the check
+    where it appears. If a pair agrees again and later disagrees, the
+    alert is sent again.
 
 ### TCP Port Monitoring
 

TODO.md

@@ -24,8 +24,10 @@ Rationale, Design, TODO, License, Author) if any are still missing.
 # Completed Steps
 
 - 2026-09-28: the inconsistency alert is sent once, on the check where two
-  nameservers that agreed start to disagree, instead of on every check while
-  they disagree, and not again after a restart (closes #158).
+  nameservers start to disagree or where a nameserver that disagrees first
+  appears, instead of on every check while they disagree, and not again after
+  a restart. Every pair of nameservers is compared, not only neighbours in
+  sorted order of name (closes #158).
 - 2026-09-28: DNS names in record values (CNAME, MX, SRV and NS targets) are
   lower-cased, so nameservers that answer in different letter case no longer
   count as inconsistent or as a record change (closes #157).

internal/watcher/export_test.go

@@ -1,6 +1,10 @@
 package watcher
 
-import "sneak.berlin/go/dnswatcher/internal/state"
+import (
+	"context"
+
+	"sneak.berlin/go/dnswatcher/internal/state"
+)
 
 // NewlyDisagreeingPairs exports newlyDisagreeingPairs for testing.
 func NewlyDisagreeingPairs(
@@ -9,3 +13,13 @@ func NewlyDisagreeingPairs(
 ) [][2]string {
 	return newlyDisagreeingPairs(prev, current)
 }
+
+// DetectHostnameChanges exports detectHostnameChanges for testing.
+func (w *Watcher) DetectHostnameChanges(
+	ctx context.Context,
+	hostname string,
+	prev *state.HostnameState,
+	current map[string]map[string][]string,
+) {
+	w.detectHostnameChanges(ctx, hostname, prev, current)
+}

internal/watcher/inconsistency_test.go

@@ -8,6 +8,16 @@ import (
 	"sneak.berlin/go/dnswatcher/internal/watcher"
 )
 
+const (
+	host = "www.example.net"
+	nsA  = "a.ns.example.net."
+	nsB  = "b.ns.example.net."
+	nsC  = "c.ns.example.net."
+	ip1  = "192.0.2.1"
+	ip2  = "192.0.2.2"
+	ip3  = "192.0.2.3"
+)
+
 // hostnameState builds the state a check with these records leaves behind.
 func hostnameState(
 	records map[string]map[string][]string,
@@ -29,24 +39,20 @@ func hostnameState(
 func TestNewlyDisagreeingPairs(t *testing.T) {
 	t.Parallel()
 
-	const (
-		nsA = "a.ns.example.net."
-		nsB = "b.ns.example.net."
-		ip1 = "192.0.2.1"
-		ip2 = "192.0.2.2"
-	)
-
 	onlyA := map[string]map[string][]string{nsA: {"A": {ip1}}}
-	agree := map[string]map[string][]string{
-		nsA: {"A": {ip1}},
-		nsB: {"A": {ip1}},
-	}
-	disagree := map[string]map[string][]string{
-		nsA: {"A": {ip1}},
-		nsB: {"A": {ip2}},
-	}
+	agree := map[string]map[string][]string{nsA: {"A": {ip1}}, nsB: {"A": {ip1}}}
+	disagree := map[string]map[string][]string{nsA: {"A": {ip1}}, nsB: {"A": {ip2}}}
 	alert := [][2]string{{nsA, nsB}}
 
+	// b already disagrees with a and c; then c changes, so a and c,
+	// which agreed, now differ.
+	bDiffers := map[string]map[string][]string{
+		nsA: {"A": {ip1}}, nsB: {"A": {ip2}}, nsC: {"A": {ip1}},
+	}
+	cChanges := map[string]map[string][]string{
+		nsA: {"A": {ip1}}, nsB: {"A": {ip2}}, nsC: {"A": {ip3}},
+	}
+
 	// Each case starts from the state loaded at startup and runs the
 	// checks in order; want[i] is what check i alerts for.
 	tests := []struct {
@@ -74,10 +80,10 @@ func TestNewlyDisagreeingPairs(t *testing.T) {
 			want:   [][][2]string{nil, nil},
 		},
 		{
-			name:   "nameserver new on this check does not alert",
+			name:   "nameserver new on the first check and disagreeing alerts once",
 			loaded: onlyA,
-			checks: []map[string]map[string][]string{disagree},
-			want:   [][][2]string{nil},
+			checks: []map[string]map[string][]string{disagree, disagree},
+			want:   [][][2]string{alert, nil},
 		},
 		{
 			name:   "disagreement after agreeing again alerts again",
@@ -85,6 +91,12 @@ func TestNewlyDisagreeingPairs(t *testing.T) {
 			checks: []map[string]map[string][]string{disagree, agree, disagree},
 			want:   [][][2]string{alert, nil, alert},
 		},
+		{
+			name:   "new disagreement while another nameserver differs alerts",
+			loaded: bDiffers,
+			checks: []map[string]map[string][]string{cChanges, cChanges},
+			want:   [][][2]string{{{nsA, nsC}}, nil},
+		},
 	}
 
 	for _, tt := range tests {
@@ -107,3 +119,64 @@ func TestNewlyDisagreeingPairs(t *testing.T) {
 		})
 	}
 }
+
+func TestInconsistencyAlert(t *testing.T) {
+	t.Parallel()
+
+	onlyA := map[string]map[string][]string{nsA: {"A": {ip1}}}
+	agree := map[string]map[string][]string{nsA: {"A": {ip1}}, nsB: {"A": {ip1}}}
+	disagree := map[string]map[string][]string{nsA: {"A": {ip1}}, nsB: {"A": {ip2}}}
+
+	// Each case starts from the state loaded at startup and then sees
+	// the nameservers disagree on three checks in a row.
+	tests := []struct {
+		name   string
+		loaded map[string]map[string][]string
+		want   int
+	}{
+		{
+			name:   "disagreement lasting several checks alerts once",
+			loaded: agree,
+			want:   1,
+		},
+		{
+			name:   "disagreement in the loaded state does not alert",
+			loaded: disagree,
+			want:   0,
+		},
+		{
+			name:   "nameserver new on the first check and disagreeing alerts once",
+			loaded: onlyA,
+			want:   1,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			t.Parallel()
+
+			// The hostname change detection uses only the notifier.
+			notifier := &mockNotifier{}
+			w := watcher.NewForTest(nil, nil, nil, nil, nil, notifier)
+
+			prev := hostnameState(tt.loaded)
+
+			for range 3 {
+				w.DetectHostnameChanges(t.Context(), host, prev, disagree)
+				prev = hostnameState(disagree)
+			}
+
+			got := 0
+
+			for _, n := range notifier.getNotifications() {
+				if n.Title == "Inconsistency: "+host {
+					got++
+				}
+			}
+
+			if got != tt.want {
+				t.Errorf("sent %d inconsistency alerts, want %d", got, tt.want)
+			}
+		})
+	}
+}

internal/watcher/watcher.go

@@ -470,9 +470,10 @@ func (w *Watcher) detectInconsistencies(
 	}
 }
 
-// newlyDisagreeingPairs returns the pairs of nameservers, neighbours in
-// sorted order, whose records differ in current but were equal in prev.
-// A pair with a nameserver missing from prev is not returned.
+// newlyDisagreeingPairs returns every pair of nameservers whose records
+// differ in current, in sorted order of name, except pairs where both
+// nameservers were in prev and already differed there. A nameserver
+// missing from prev is paired with every nameserver it differs from.
 func newlyDisagreeingPairs(
 	prev *state.HostnameState,
 	current map[string]map[string][]string,
@@ -486,22 +487,21 @@ func newlyDisagreeingPairs(
 
 	var pairs [][2]string
 
-	for i := range len(nameservers) - 1 {
-		ns1 := nameservers[i]
-		ns2 := nameservers[i+1]
+	for i, ns1 := range nameservers {
+		for _, ns2 := range nameservers[i+1:] {
+			if recordsEqual(current[ns1], current[ns2]) {
+				continue
+			}
 
-		if recordsEqual(current[ns1], current[ns2]) {
-			continue
+			prev1, ok1 := prev.RecordsByNameserver[ns1]
+			prev2, ok2 := prev.RecordsByNameserver[ns2]
+
+			if ok1 && ok2 && !recordsEqual(prev1.Records, prev2.Records) {
+				continue
+			}
+
+			pairs = append(pairs, [2]string{ns1, ns2})
 		}
-
-		prev1, ok1 := prev.RecordsByNameserver[ns1]
-		prev2, ok2 := prev.RecordsByNameserver[ns2]
-
-		if !ok1 || !ok2 || !recordsEqual(prev1.Records, prev2.Records) {
-			continue
-		}
-
-		pairs = append(pairs, [2]string{ns1, ns2})
 	}
 
 	return pairs