resolver: try servers in a random order on each resolution (closes #138)

Every resolution walked the root servers in a fixed order, so
a.root-servers.net got every first query and its timeouts were paid on
every lookup. Each list of servers the resolver walks is now walked in a
random order from rand.Shuffle, chosen anew each time; a server that does
not reply, refuses, or gives an error reply or a referral that leads no
closer is still passed over for the next. When a referral names a zone's
nameservers without their addresses, all of them are now looked up, not
only the first that resolves, so the zone is not given up because the
first nameserver whose address was found gave no usable reply. No test
fails if the walk stops shuffling: which server a live query reached is
not observable.

Model: opus-5-5

README.md

@@ -407,6 +407,13 @@ it watches. Instead, it performs full iterative resolution:
 4. **Authoritative query**: Queries all discovered authoritative nameservers
    directly for the requested records.
 
+In steps 2 and 3 the servers are asked one at a time in a random order, chosen
+anew each time, so no one root server gets every first query. A server that does
+not reply, refuses the query, or gives an error reply such as SERVFAIL or a
+referral that leads no closer to the name is passed over for the next one. When
+a referral names a zone's nameservers without their addresses, the addresses of
+all of them are looked up, so that each can be asked.
+
 This approach ensures:
 
 - Independence from any upstream resolver's cache or filtering.

TODO.md

@@ -19,6 +19,8 @@ trial run of the finished image: https://git.eeqj.de/sneak/dnswatcher/issues/149
 
 # Completed Steps
 
+- 2026-10-02: the resolver tries root servers, and every other server list it
+  walks, in a random order each time, not always from the top (closes #138).
 - 2026-10-02: a name listed more than once in `DNSWATCHER_TARGETS`, in any
   letter case or with a trailing dot, is watched once (closes #207).
 - 2026-10-01: README checked against the code and corrected: metrics, CORS,
@@ -131,5 +133,4 @@ trial run of the finished image: https://git.eeqj.de/sneak/dnswatcher/issues/149
 
 - 1.0 readiness: run it with a real config and read the logs:
   https://git.eeqj.de/sneak/dnswatcher/issues/66
-- fixed root server order: https://git.eeqj.de/sneak/dnswatcher/issues/138
 - review toward 1.0: https://git.eeqj.de/sneak/dnswatcher/issues/144

internal/livednstest/livednstest.go

@@ -9,11 +9,11 @@
 //
 //  1. Bounded concurrency. Tests run in parallel and the build hosts
 //     have many cores, so without a limit every test starts its own
-//     iterative resolution at the same instant and they all hit the
+//     iterative resolution at the same instant and they all send their
-//     first root server within a few milliseconds of each other. Root
+//     first queries to the root servers within a few milliseconds of
-//     servers rate-limit that, which shows up as a different arbitrary
+//     each other. Root servers rate-limit that, which shows up as a
-//     subset of tests failing on each run. Run caps how many live
+//     different arbitrary subset of tests failing on each run. Run caps
-//     operations are in flight at once in one test binary.
+//     how many live operations are in flight at once in one test binary.
 //
 //  2. Retry with exponential backoff. Each live operation gets several
 //     attempts with its own timeout. An attempt is retried when it

internal/resolver/export_test.go

@@ -36,3 +36,24 @@ func (r *Resolver) QueryEachNS(
 ) (map[string]*NameserverResponse, error) {
 	return r.queryEachNS(ctx, nameservers, hostname)
 }
+
+// ResolveNSIPs exports resolveNSIPs for testing.
+func (r *Resolver) ResolveNSIPs(
+	ctx context.Context,
+	nsNames []string,
+) []string {
+	return r.resolveNSIPs(ctx, nsNames)
+}
+
+// RootServerList exports rootServerList for testing.
+func RootServerList() []string {
+	return rootServerList()
+}
+
+// Shuffled exports shuffled for testing.
+func Shuffled(
+	servers []string,
+	shuffle func(n int, swap func(i, j int)),
+) []string {
+	return shuffled(servers, shuffle)
+}

internal/resolver/iterative.go

@@ -4,7 +4,9 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"math/rand/v2"
 	"net"
+	"slices"
 	"sort"
 	"strings"
 	"time"
@@ -259,9 +261,25 @@ func (r *Resolver) followDelegation(
 	return nil, ErrNoNameservers
 }
 
-// queryServers asks servers, the servers of zone, about name until one
+// shuffled returns a copy of servers in the order shuffle puts them
-// gives a usable reply. A server that times out, refuses or gives a
+// in. The resolver passes rand.Shuffle, so each time it walks a list of
-// reply that is not usable is passed over for the next.
+// servers it starts at a random one, and no one server gets every
+// first query.
+func shuffled(
+	servers []string,
+	shuffle func(n int, swap func(i, j int)),
+) []string {
+	order := slices.Clone(servers)
+	shuffle(len(order), func(i, j int) {
+		order[i], order[j] = order[j], order[i]
+	})
+
+	return order
+}
+
+// queryServers asks servers, the servers of zone, about name in a random
+// order until one gives a usable reply. A server that times out, refuses
+// or gives a reply that is not usable is passed over for the next.
 func (r *Resolver) queryServers(
 	ctx context.Context,
 	servers []string,
@@ -271,7 +289,7 @@ func (r *Resolver) queryServers(
 ) (*dns.Msg, error) {
 	var lastErr error
 
-	for _, ip := range servers {
+	for _, ip := range shuffled(servers, rand.Shuffle) {
 		if checkCtx(ctx) != nil {
 			return nil, ErrContextCanceled
 		}
@@ -340,6 +358,10 @@ func nsSetFrom(resp *dns.Msg, domain string) []string {
 	return extractNSSet(resp.Answer)
 }
 
+// resolveNSIPs returns the addresses of every nameserver in nsNames
+// whose name resolves, for a referral that carries none. The walk can
+// then go on to the zone's other nameservers when one gives no usable
+// reply.
 func (r *Resolver) resolveNSIPs(
 	ctx context.Context,
 	nsNames []string,
@@ -351,10 +373,6 @@ func (r *Resolver) resolveNSIPs(
 		if err == nil {
 			ips = append(ips, resolved...)
 		}
-
-		if len(ips) > 0 {
-			break
-		}
 	}
 
 	return ips

internal/resolver/iterative_test.go

@@ -1,6 +1,8 @@
 package resolver_test
 
 import (
+	"math/rand/v2"
+	"slices"
 	"testing"
 
 	"github.com/miekg/dns"
@@ -235,3 +237,30 @@ func TestExtractRecordValue_LetterCase(t *testing.T) {
 		})
 	}
 }
+
+// TestShuffled shuffles the root servers with many seeds. Every order
+// must hold each root server once, so each is tried before a
+// resolution fails; each root server must come first for some seed, so
+// no one root server gets every first query; and the list passed in
+// must be left as it was.
+func TestShuffled(t *testing.T) {
+	t.Parallel()
+
+	const seeds = 1000
+
+	roots := resolver.RootServerList()
+	before := slices.Clone(roots)
+	first := make(map[string]bool)
+
+	for seed := range uint64(seeds) {
+		rng := rand.New(rand.NewPCG(seed, 0)) //nolint:gosec // seeded on purpose
+		order := resolver.Shuffled(roots, rng.Shuffle)
+
+		assert.ElementsMatch(t, roots, order)
+
+		first[order[0]] = true
+	}
+
+	assert.Len(t, first, len(roots))
+	assert.Equal(t, before, roots)
+}

internal/resolver/livedns_test.go

@@ -383,3 +383,37 @@ func liveResolveIPsAllowingEmpty(
 
 	return out
 }
+
+// liveResolveNSIPs looks up the addresses of the nameservers named
+// names, retrying until there are at least atLeast of them: a name
+// whose lookup got no reply is left out of the result, not an error.
+func liveResolveNSIPs(
+	t *testing.T,
+	r *resolver.Resolver,
+	names []string,
+	atLeast int,
+) []string {
+	t.Helper()
+
+	var out []string
+
+	livednstest.Retry(
+		t,
+		"ResolveNSIPs("+strings.Join(names, ", ")+")",
+		func(ctx context.Context) error {
+			ips := r.ResolveNSIPs(ctx, names)
+			if len(ips) < atLeast {
+				return fmt.Errorf(
+					"%w: %d addresses, expected at least %d",
+					livednstest.ErrNoAnswer, len(ips), atLeast,
+				)
+			}
+
+			out = ips
+
+			return nil
+		},
+	)
+
+	return out
+}

internal/resolver/resolver_test.go

@@ -139,6 +139,28 @@ func TestFindAuthoritativeNameservers_CloudflareDomain(
 	}
 }
 
+// TestResolveNSIPs_EveryNameserver looks up the addresses of two of
+// google.com's nameservers together, as the walk does when a referral
+// names a zone's nameservers without their addresses, and compares them
+// with each looked up alone. Together they must give the addresses of
+// both, not only of the first that resolves, so that when one gives no
+// usable reply the walk goes on to the other.
+func TestResolveNSIPs_EveryNameserver(t *testing.T) {
+	t.Parallel()
+
+	r := newTestResolver(t)
+	names := []string{"ns3.google.com.", "ns4.google.com."}
+	want := make([]string, 0, len(names))
+
+	for _, name := range names {
+		want = append(want, liveResolveNSIPs(t, r, []string{name}, 1)...)
+	}
+
+	got := liveResolveNSIPs(t, r, names, len(want))
+
+	assert.ElementsMatch(t, want, got)
+}
+
 // ----------------------------------------------------------------
 // QueryNameserver tests
 // ----------------------------------------------------------------