watcher: follow a watched name's CNAME for port and TLS checks (closes #203)

When a watched name's nameservers answer with a CNAME and no address,
the DNS check follows every CNAME target they gave with
ResolveIPAddresses and saves the addresses found for all of them in the
hostname state as cnameAddresses, so nameservers that disagree on the
target do not change them from check to check. The port and TLS checks
use them. A change in them is notified as a CNAME address change, also
from or to none. A state file without the field loads them as not known
(nil), so its first check sends nothing for them. When a target cannot
be followed, or none of the name's nameservers answered, the last
check's addresses are kept. The domain check now runs the hostname
check for the apex instead of a copy of it.

Model: opus-5-5

README.md

@@ -126,10 +126,10 @@ notification endpoint set, changes show only on the dashboard; see
       nameservers answer with a CNAME and no address; a name that answers with
       an address has none. A change from or to no addresses is sent too, as when
       a name moves between A records and a CNAME. Nothing is sent when the
-      previous addresses were kept because the chain could not be followed or
-      none of the name's nameservers answered. The first check after loading a
-      state file without `cnameAddresses` sends nothing: it saves the addresses
-      it finds for the next check to compare.
+      previous addresses were kept because a chain could not be followed or none
+      of the name's nameservers answered. The first check after loading a state
+      file without `cnameAddresses` sends nothing: it saves the addresses it
+      finds for the next check to compare.
 
 ### TCP Port Monitoring
 
@@ -137,9 +137,10 @@ notification endpoint set, changes show only on the dashboard; see
   the IPv4 and IPv6 addresses in the A and AAAA records its authoritative
   nameservers returned. When they returned a CNAME and no address, the CNAME
   chain is followed and the addresses at its end are used, and a change in those
-  is notified as a CNAME address change. When the chain cannot be followed, or
-  none of the name's nameservers answered, the addresses the last check found at
-  its end are used.
+  is notified as a CNAME address change. When the nameservers gave different
+  CNAME targets, each is followed and the addresses of all are used. When a
+  chain cannot be followed, or none of the name's nameservers answered, the
+  addresses the last check found at its end are used.
 - Checks TCP connectivity on ports **80** and **443** for each IP address.
 - Every **1 hour** by default, re-checks all ports.
 - Any change in port availability triggers a notification:
@@ -434,10 +435,10 @@ This approach ensures:
   servers.
 
 A watched name's records are stored as its nameservers return them, CNAME
-included. When they return a CNAME and no address, the CNAME chain is followed
-(with a depth limit to prevent loops) to the A and AAAA records at its end, and
-the port and TLS checks use those addresses. Nameservers' addresses are found
-the same way.
+included. When they return a CNAME and no address, the chain of every CNAME
+target they gave is followed (with a depth limit to prevent loops) to the A and
+AAAA records at its end, and the port and TLS checks use those addresses.
+Nameservers' addresses are also found by following CNAME chains.
 
 Sending a notification or a Sentry report is the one use of the system's
 resolver: the HTTP client looks up the webhook's or Sentry's host name with it.
@@ -531,12 +532,13 @@ certificate entry whose TLS connection or handshake failed likewise has status
 resolves to. A state file without it loads, and the next check fills it in
 without a notification.
 
-`cnameAddresses` lists the sorted addresses at the end of a hostname's CNAME
-chain, found when its nameservers answered with a CNAME and no address; it is
-empty when they answered with an address. When the chain cannot be followed, or
-none of the name's nameservers answered, the previous check's list is kept, or
-`null` when no earlier check saved one. A state file without it loads, and the
-first check after that saves it without a notification.
+`cnameAddresses` lists the sorted addresses at the end of the chain of every
+CNAME target a hostname's nameservers gave, found when they answered with a
+CNAME and no address; it is empty when they answered with an address. When a
+chain cannot be followed, or none of the name's nameservers answered, the
+previous check's list is kept, or `null` when no earlier check saved one. A
+state file without it loads, and the first check after that saves it without a
+notification.
 
 A port entry in the older format, with one `hostname` instead of the `hostnames`
 list, loads as a list of that one name.

internal/watcher/cname_test.go

@@ -52,15 +52,12 @@ func TestCNAMEIntoAnotherZonePortAndTLSChecks(t *testing.T) {
 	}
 }
 
-// TestCNAMEThatCannotBeFollowedKeepsPrevious gives a name under
-// .invalid, whose lookup fails, answers with a CNAME and no address.
-// The addresses the previous check saved from following its CNAME are
-// kept.
+// TestCNAMEThatCannotBeFollowedKeepsPrevious gives a name a CNAME to a
+// target under .invalid, whose lookup fails. The addresses the previous
+// check saved from following its CNAME are kept.
 func TestCNAMEThatCannotBeFollowedKeepsPrevious(t *testing.T) {
 	t.Parallel()
 
-	const name = "www.example.invalid"
-
 	w := watcher.NewForTest(
 		nil, nil, resolver.NewFromLogger(slog.Default()), nil, nil, nil,
 	)
@@ -73,7 +70,7 @@ func TestCNAMEThatCannotBeFollowedKeepsPrevious(t *testing.T) {
 	// The result is the same whether or not live DNS answers, so the
 	// lookup is not retried.
 	_ = livednstest.Run(func(ctx context.Context) error {
-		w.ResolveCNAMEAddresses(ctx, name, current, prev)
+		w.ResolveCNAMEAddresses(ctx, host, current, prev)
 
 		return nil
 	})
@@ -86,6 +83,93 @@ func TestCNAMEThatCannotBeFollowedKeepsPrevious(t *testing.T) {
 	}
 }
 
+// followLive follows in live DNS the CNAMEs in a name's records, built
+// from records, and returns the addresses saved for the name. The
+// previous check saved oldIP, which is kept when a target cannot be
+// followed; that is retried.
+func followLive(
+	t *testing.T,
+	records map[string]map[string][]string,
+) []string {
+	t.Helper()
+
+	w := watcher.NewForTest(
+		nil, nil, resolver.NewFromLogger(slog.Default()), nil, nil, nil,
+	)
+	prev := cnameState(oldIP)
+
+	var current *state.HostnameState
+
+	livednstest.Retry(t, "following CNAMEs", func(ctx context.Context) error {
+		current = hostnameState(records)
+
+		w.ResolveCNAMEAddresses(ctx, host, current, prev)
+
+		if slices.Equal(current.CNAMEAddresses, prev.CNAMEAddresses) {
+			return livednstest.ErrNoAnswer
+		}
+
+		return nil
+	})
+
+	return current.CNAMEAddresses
+}
+
+// TestCNAMEAddressesOfEveryTarget gives a name's two nameservers
+// different CNAME targets, as when a secondary still serves an old one.
+// The addresses at the end of both are saved, whichever answer is read
+// first: one.one.one.one has 1.1.1.1, and dns.google has 8.8.8.8.
+func TestCNAMEAddressesOfEveryTarget(t *testing.T) {
+	t.Parallel()
+
+	found := followLive(t, map[string]map[string][]string{
+		nsA: {"CNAME": {"one.one.one.one."}},
+		nsB: {"CNAME": {"dns.google."}},
+	})
+
+	for _, ip := range []string{"1.1.1.1", "8.8.8.8"} {
+		if !slices.Contains(found, ip) {
+			t.Errorf("saved %v, want %s among them", found, ip)
+		}
+	}
+}
+
+// TestCNAMEChainEndingInNoAddressSavesEmptyList follows a CNAME to a
+// name live DNS answers with NXDOMAIN. An empty list is saved, not nil,
+// which would mean the addresses are not known.
+func TestCNAMEChainEndingInNoAddressSavesEmptyList(t *testing.T) {
+	t.Parallel()
+
+	found := followLive(t, map[string]map[string][]string{
+		nsA: {"CNAME": {"this-surely-does-not-exist-xyz.google.com."}},
+	})
+
+	if found == nil || len(found) != 0 {
+		t.Errorf("saved %#v, want an empty list", found)
+	}
+}
+
+// TestCNAMEBesideAnAddressNotFollowed gives one nameserver of a name an
+// address and another a CNAME. The CNAME is not followed: an empty list
+// is saved, not nil, and nothing is looked up, the watcher having no
+// resolver.
+func TestCNAMEBesideAnAddressNotFollowed(t *testing.T) {
+	t.Parallel()
+
+	w := watcher.NewForTest(nil, nil, nil, nil, nil, nil)
+
+	current := hostnameState(map[string]map[string][]string{
+		nsA: {"A": {ip1}},
+		nsB: {"CNAME": {"target.example.org."}},
+	})
+
+	w.ResolveCNAMEAddresses(t.Context(), host, current, nil)
+
+	if current.CNAMEAddresses == nil || len(current.CNAMEAddresses) != 0 {
+		t.Errorf("saved %#v, want an empty list", current.CNAMEAddresses)
+	}
+}
+
 // TestCNAMEWhoseNameserversAllFailedKeepsPrevious checks a name none of
 // whose nameservers answered. The addresses the previous check saved
 // from following its CNAME are kept, and nothing is looked up: the

internal/watcher/watcher.go

@@ -394,11 +394,13 @@ func (w *Watcher) checkHostname(
 
 // resolveCNAMEAddresses saves in current the addresses at the end of
 // hostname's CNAME chain, when the nameservers' answers in current hold
-// a CNAME and no address, and an empty list otherwise.
-// ResolveIPAddresses looks the name up again and follows the chain. The
-// addresses saved in prev, which may be nil, are kept when none of the
-// name's nameservers answered, and when the chain cannot be followed, as
-// when no nameserver of a zone in it answers.
+// a CNAME and no address, and an empty list otherwise. Every CNAME
+// target the nameservers gave is followed with ResolveIPAddresses and
+// the addresses found for all of them are saved, so nameservers that
+// disagree on the target do not change the result from check to check.
+// The addresses saved in prev, which may be nil, are kept when none of
+// the name's nameservers answered, and when a target cannot be
+// followed, as when no nameserver of a zone in its chain answers.
 func (w *Watcher) resolveCNAMEAddresses(
 	ctx context.Context,
 	hostname string,
@@ -413,7 +415,7 @@ func (w *Watcher) resolveCNAMEAddresses(
 	current.CNAMEAddresses = []string{}
 
 	answered := false
-	hasCNAME := false
+	targets := make(map[string]bool)
 
 	for _, nsState := range current.RecordsByNameserver {
 		if nsState.Status != statusOK {
@@ -426,8 +428,8 @@ func (w *Watcher) resolveCNAMEAddresses(
 			return
 		}
 
-		if len(nsState.Records["CNAME"]) > 0 {
-			hasCNAME = true
+		for _, target := range nsState.Records["CNAME"] {
+			targets[target] = true
 		}
 	}
 
@@ -437,26 +439,27 @@ func (w *Watcher) resolveCNAMEAddresses(
 		return
 	}
 
-	if !hasCNAME {
-		return
+	for target := range targets {
+		ips, err := w.resolver.ResolveIPAddresses(ctx, target)
+		if err != nil {
+			w.log.Error(
+				"failed to follow CNAME",
+				"hostname", hostname,
+				"target", target,
+				"error", err,
+			)
+
+			current.CNAMEAddresses = prevAddresses
+
+			return
+		}
+
+		current.CNAMEAddresses = append(current.CNAMEAddresses, ips...)
 	}
 
-	ips, err := w.resolver.ResolveIPAddresses(ctx, hostname)
-	if err != nil {
-		w.log.Error(
-			"failed to follow CNAME",
-			"hostname", hostname,
-			"error", err,
-		)
-
-		current.CNAMEAddresses = prevAddresses
-
-		return
-	}
-
-	// Appended to the empty list, so a chain that ends in no address is
-	// saved as empty, not nil.
-	current.CNAMEAddresses = append(current.CNAMEAddresses, ips...)
+	// Still the empty list when every chain ends in no address.
+	slices.Sort(current.CNAMEAddresses)
+	current.CNAMEAddresses = slices.Compact(current.CNAMEAddresses)
 }
 
 // buildHostnameState saves each nameserver's response. A nameserver