resolver: try servers in a random order on each resolution (closes #138)

Every resolution walked the root servers in a fixed order, so
a.root-servers.net got every first query and its timeouts were paid on
every lookup. Each list of servers the resolver walks, the root servers
and the nameservers of each zone below them, is now walked in a random
order from the standard library's rand.Shuffle, chosen anew each time.
Failover is unchanged: a server that does not reply, or refuses, is
passed over for the next; any other reply, even a SERVFAIL, is used.
The shuffle is passed in, so the tests check the order with a seeded
source; which server a live query reached is not observable, so no
test fails if the walk stops shuffling.

Model: opus-5-5

README.md

@@ -110,7 +110,9 @@ Contributions that introduce mocked, faked, or stubbed DNS will be rejected.
     - Port transitioned from open to closed (or vice versa).
     - New IP appeared (from DNS change) and its port state was recorded.
     - IP disappeared (from DNS change) — noted in the DNS change notification;
-      port state for that IP is removed.
+      port state for that IP is removed. When none of a name's nameservers
+      answered, its addresses are not known, so the port state saved for them is
+      kept.
 
 ### TLS Certificate Monitoring
 
@@ -399,6 +401,11 @@ performs full iterative resolution:
 4. **Authoritative query**: Queries all discovered authoritative nameservers
    directly for the requested records.
 
+In steps 2 and 3 the servers are asked one at a time in a random order, chosen
+anew each time, so no one root server gets every first query. A server that does
+not reply, or refuses the query, is passed over for the next one; the first
+other reply is used, even a SERVFAIL, and no further server is asked.
+
 This approach ensures:
 
 - Independence from any upstream resolver's cache or filtering.

TODO.md

@@ -19,6 +19,10 @@ trial run of the finished image: https://git.eeqj.de/sneak/dnswatcher/issues/149
 
 # Completed Steps
 
+- 2026-10-01: the resolver tries root servers, and every other server list it
+  walks, in a random order each time, not always from the top (closes #138).
+- 2026-10-01: when none of a configured name's nameservers answered, the port
+  state saved for its addresses is kept, not removed (closes #193).
 - 2026-10-01: `ResolveIPAddresses` returns an error, not no addresses, when no
   nameserver of the name's zone answered (closes #190).
 - 2026-10-01: `make fmt` and `make fmt-check` cover Markdown with prettier, run
@@ -120,5 +124,4 @@ trial run of the finished image: https://git.eeqj.de/sneak/dnswatcher/issues/149
 - README accuracy sweep: https://git.eeqj.de/sneak/dnswatcher/issues/108
 - README sections required by policy:
   https://git.eeqj.de/sneak/dnswatcher/issues/173
-- fixed root server order: https://git.eeqj.de/sneak/dnswatcher/issues/138
 - review toward 1.0: https://git.eeqj.de/sneak/dnswatcher/issues/144

internal/resolver/export_test.go

@@ -26,3 +26,16 @@ func (r *Resolver) QueryEachNS(
 ) (map[string]*NameserverResponse, error) {
 	return r.queryEachNS(ctx, nameservers, hostname)
 }
+
+// RootServerList exports rootServerList for testing.
+func RootServerList() []string {
+	return rootServerList()
+}
+
+// Shuffled exports shuffled for testing.
+func Shuffled(
+	servers []string,
+	shuffle func(n int, swap func(i, j int)),
+) []string {
+	return shuffled(servers, shuffle)
+}

internal/resolver/iterative.go

@@ -4,7 +4,9 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"math/rand/v2"
 	"net"
+	"slices"
 	"sort"
 	"strings"
 	"time"
@@ -255,6 +257,24 @@ func (r *Resolver) followDelegation(
 	return nil, ErrNoNameservers
 }
 
+// shuffled returns a copy of servers in the order shuffle puts them
+// in. The resolver passes rand.Shuffle, so each time it walks a list of
+// servers it starts at a random one, and no one server gets every
+// first query.
+func shuffled(
+	servers []string,
+	shuffle func(n int, swap func(i, j int)),
+) []string {
+	order := slices.Clone(servers)
+	shuffle(len(order), func(i, j int) {
+		order[i], order[j] = order[j], order[i]
+	})
+
+	return order
+}
+
+// queryServers asks the servers in a random order and returns the
+// first reply; it fails only when every server has failed.
 func (r *Resolver) queryServers(
 	ctx context.Context,
 	servers []string,
@@ -263,7 +283,7 @@ func (r *Resolver) queryServers(
 ) (*dns.Msg, error) {
 	var lastErr error
 
-	for _, ip := range servers {
+	for _, ip := range shuffled(servers, rand.Shuffle) {
 		if checkCtx(ctx) != nil {
 			return nil, ErrContextCanceled
 		}
@@ -279,13 +299,15 @@ func (r *Resolver) queryServers(
 	return nil, fmt.Errorf("all servers failed: %w", lastErr)
 }
 
+// resolveNSIPs returns the addresses of one of the nameservers, trying
+// their names in a random order until one resolves.
 func (r *Resolver) resolveNSIPs(
 	ctx context.Context,
 	nsNames []string,
 ) []string {
 	var ips []string
 
-	for _, ns := range nsNames {
+	for _, ns := range shuffled(nsNames, rand.Shuffle) {
 		resolved, err := r.resolveARecord(ctx, ns)
 		if err == nil {
 			ips = append(ips, resolved...)

internal/resolver/iterative_test.go

@@ -1,6 +1,8 @@
 package resolver_test
 
 import (
+	"math/rand/v2"
+	"slices"
 	"testing"
 
 	"github.com/miekg/dns"
@@ -92,3 +94,30 @@ func TestExtractRecordValue_LetterCase(t *testing.T) {
 		})
 	}
 }
+
+// TestShuffled shuffles the root servers with many seeds. Every order
+// must hold each root server once, so each is tried before a
+// resolution fails; each root server must come first for some seed, so
+// no one root server gets every first query; and the list passed in
+// must be left as it was.
+func TestShuffled(t *testing.T) {
+	t.Parallel()
+
+	const seeds = 1000
+
+	roots := resolver.RootServerList()
+	before := slices.Clone(roots)
+	first := make(map[string]bool)
+
+	for seed := range uint64(seeds) {
+		rng := rand.New(rand.NewPCG(seed, 0)) //nolint:gosec // seeded on purpose
+		order := resolver.Shuffled(roots, rng.Shuffle)
+
+		assert.ElementsMatch(t, roots, order)
+
+		first[order[0]] = true
+	}
+
+	assert.Len(t, first, len(roots))
+	assert.Equal(t, before, roots)
+}

internal/watcher/export_test.go

@@ -67,6 +67,11 @@ func (w *Watcher) DetectNSAddressChanges(
 	w.detectNSAddressChanges(ctx, domain, prev, current)
 }
 
+// CheckAllPorts exports checkAllPorts for testing.
+func (w *Watcher) CheckAllPorts(ctx context.Context) {
+	w.checkAllPorts(ctx)
+}
+
 // BuildHostnameState exports buildHostnameState for testing.
 func BuildHostnameState(
 	results map[string]*resolver.NameserverResponse,

internal/watcher/nsfailure_test.go

@@ -331,3 +331,118 @@ func TestNameserverThatRefuses(t *testing.T) {
 		)
 	}
 }
+
+// TestPortStateWhenNoNameserverAnswered runs the port checks on
+// hostname state built here, which gives the name no address. The port
+// state saved for its old address is kept only when the name is a
+// configured hostname or domain and none of its nameservers answered.
+func TestPortStateWhenNoNameserverAnswered(t *testing.T) {
+	t.Parallel()
+
+	noneAnswered := saved(map[string]*state.NameserverRecordState{
+		nsA: failed(), nsB: failed(),
+	})
+	oneAnsweredNoAddress := saved(map[string]*state.NameserverRecordState{
+		nsA: answered(map[string][]string{}), nsB: failed(),
+	})
+
+	configured := []string{host}
+
+	tests := []struct {
+		name      string
+		hostname  *state.HostnameState
+		hostnames []string
+		domains   []string
+		wantKept  bool
+	}{
+		{"no nameserver answered", noneAnswered, configured, nil, true},
+		{
+			"no nameserver answered, configured as a domain",
+			noneAnswered, nil, configured, true,
+		},
+		{
+			"one answered with no address",
+			oneAnsweredNoAddress, configured, nil, false,
+		},
+		{"no nameserver answered, not configured", noneAnswered, nil, nil, false},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			t.Parallel()
+
+			cfg := defaultTestConfig(t)
+			cfg.Hostnames = tt.hostnames
+			cfg.Domains = tt.domains
+
+			// The port checks read the saved hostname state and look
+			// nothing up, so the watcher has no resolver.
+			deps := newTestDeps(t, cfg)
+			w := watcher.NewForTest(
+				cfg, deps.state, nil,
+				deps.portChecker, deps.tlsChecker, deps.notifier,
+			)
+
+			key := ip1 + ":443"
+
+			deps.state.SetHostnameState(host, tt.hostname)
+			deps.state.SetPortState(key, &state.PortState{
+				Open: true, Hostnames: []string{host},
+			})
+
+			w.CheckAllPorts(t.Context())
+
+			_, kept := deps.state.GetPortState(key)
+			if kept != tt.wantKept {
+				t.Errorf("port state %s kept: %v, want %v", key, kept, tt.wantKept)
+			}
+		})
+	}
+}
+
+// TestPortStateWhenNoNameserverAnsweredAndOtherNameMovesAway saves the
+// port state of an address two configured hostnames resolve to. While
+// none of the first one's nameservers answer, the port checks run with
+// the other one still at that address, then after it moved away; the
+// port state is kept both times.
+func TestPortStateWhenNoNameserverAnsweredAndOtherNameMovesAway(
+	t *testing.T,
+) {
+	t.Parallel()
+
+	const other = "mail.example.net"
+
+	cfg := defaultTestConfig(t)
+	cfg.Hostnames = []string{host, other}
+
+	// The port checks read the saved hostname state and look nothing
+	// up, so the watcher has no resolver.
+	deps := newTestDeps(t, cfg)
+	w := watcher.NewForTest(
+		cfg, deps.state, nil,
+		deps.portChecker, deps.tlsChecker, deps.notifier,
+	)
+
+	key := ip1 + ":443"
+
+	deps.state.SetPortState(key, &state.PortState{
+		Open: true, Hostnames: []string{host, other},
+	})
+	deps.state.SetHostnameState(host, saved(
+		map[string]*state.NameserverRecordState{nsA: failed(), nsB: failed()},
+	))
+
+	for _, otherIP := range []string{ip1, ip2} {
+		deps.state.SetHostnameState(other, saved(
+			map[string]*state.NameserverRecordState{
+				nsA: answered(map[string][]string{"A": {otherIP}}),
+			},
+		))
+
+		w.CheckAllPorts(t.Context())
+
+		if _, kept := deps.state.GetPortState(key); !kept {
+			t.Fatalf("port state %s removed with %s at %s", key, other, otherIP)
+		}
+	}
+}

internal/watcher/watcher.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"fmt"
 	"log/slog"
+	"slices"
 	"sort"
 	"strings"
 	"sync"
@@ -708,15 +709,46 @@ func parsePortKey(key string) (string, int) {
 }
 
 // cleanupStalePorts removes port state entries that are no
-// longer referenced by any hostname in the current DNS data.
+// longer referenced by any hostname in the current DNS data. An
+// entry saved for a configured name none of whose nameservers
+// answered is kept: that name's addresses are not known, not gone.
 func (w *Watcher) cleanupStalePorts(
 	currentAssociations map[string][]string,
 ) {
 	for _, key := range w.state.GetAllPortKeys() {
-		if _, exists := currentAssociations[key]; !exists {
+		if _, exists := currentAssociations[key]; exists {
+			continue
+		}
+
+		ps, ok := w.state.GetPortState(key)
+		if ok && slices.ContainsFunc(ps.Hostnames, w.noNameserverAnswered) {
+			continue
+		}
+
 		w.state.DeletePortState(key)
 	}
 }
+
+// noNameserverAnswered reports whether name is a configured domain or
+// hostname and none of its nameservers answered on its last check.
+func (w *Watcher) noNameserverAnswered(name string) bool {
+	if !slices.Contains(w.config.Hostnames, name) &&
+		!slices.Contains(w.config.Domains, name) {
+		return false
+	}
+
+	hs, ok := w.state.GetHostnameState(name)
+	if !ok {
+		return false
+	}
+
+	for _, nsState := range hs.RecordsByNameserver {
+		if nsState.Status == statusOK {
+			return false
+		}
+	}
+
+	return true
 }
 
 func (w *Watcher) collectIPs(hostname string) []string {
@@ -795,9 +827,24 @@ func (w *Watcher) checkSinglePort(
 		)
 	}
 
+	// A configured name on the saved list none of whose nameservers
+	// answered stays on it, so the entry is kept when the other names
+	// stop resolving to this address.
+	savedHostnames := slices.Clone(hostnames)
+
+	if hasPrev {
+		for _, name := range prev.Hostnames {
+			if !slices.Contains(hostnames, name) && w.noNameserverAnswered(name) {
+				savedHostnames = append(savedHostnames, name)
+			}
+		}
+
+		sort.Strings(savedHostnames)
+	}
+
 	w.state.SetPortState(key, &state.PortState{
 		Open:        result.Open,
-		Hostnames:   hostnames,
+		Hostnames:   savedHostnames,
 		LastChecked: now,
 	})
 }