test: make live DNS tests robust instead of gated (closes #93)

The resolver's live-DNS tests failed nondeterministically, a different
subset each run. Three structural causes, all test-side:

- Burst fan-out. Every test in the package is parallel and the build
  hosts have many cores, so all ~35 iterative resolutions started at
  the same instant and, because queryServers walks rootServerList() in
  fixed order, hit the same root server within milliseconds. Root
  servers rate-limit that.
- No retry anywhere. One dropped UDP packet in a delegation chain
  failed a test outright.
- Unanimity assertions. TestQueryAllNameservers_AllReturnOK and
  _NXDomainFromAllNS required every one of a domain's nameservers to
  answer, with no tolerance for one being slow.

New internal/resolver/livedns_test.go addresses each: a package-wide
gate bounds how many live resolutions are in flight at once, every
live operation gets three attempts with exponential backoff and its
own deadline, and multi-nameserver assertions now need a strict
majority rather than unanimity. The retry predicate is deliberately
transport-level -- "did a nameserver answer at all" -- never the
assertion under test, so a resolver that answers incorrectly still
fails on the first attempt. A nameserver that stays silent is
tolerated; one that answers wrongly is not.

livedns_harness_test.go tests that machinery directly: quorum
arithmetic, status counting, the gate's concurrency bound, per-attempt
deadlines, and recovery from a transient failure. It touches no DNS.

Nothing is mocked, faked, stubbed, recorded, skipped or build-tagged,
and production resolver behaviour is unchanged.

Test caps move to the new org-wide values ruled at prompts issue 41:
60s hard cap, 20s target, 90s -timeout backstop. REPO_POLICIES.md is
re-vendored byte-identical from sneak/prompts rather than hand-edited,
which also picks up the golangci-lint paragraph this copy had drifted
behind on. TESTING.md's stale 30-second target follows to 60.

#93

internal/resolver/livedns_test.go

@@ -0,0 +1,437 @@
+package resolver_test
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"sort"
+	"strings"
+	"testing"
+	"time"
+
+	"sneak.berlin/go/dnswatcher/internal/resolver"
+)
+
+// ----------------------------------------------------------------
+// Live DNS test support
+// ----------------------------------------------------------------
+//
+// Every test in this package resolves against the real, live DNS —
+// see TESTING.md. Nothing here mocks, fakes, stubs, records or
+// replays DNS, and nothing here skips or gates a test: the helpers
+// below only change *how* the live queries are issued, so that a
+// single dropped UDP packet or one slow authoritative server does
+// not turn a correct resolver into a red build.
+//
+// Three mechanisms, all test-side:
+//
+//  1. Bounded concurrency. The package's tests are parallel and the
+//     build hosts have many cores, so without a limit every test
+//     starts its own iterative resolution at the same instant and
+//     they all hit the first root server in rootServerList() within
+//     a few milliseconds of each other. Root servers rate-limit
+//     that, which shows up as a different arbitrary subset of tests
+//     failing on each run. liveGate caps how many resolutions are
+//     in flight at once.
+//
+//  2. Retry with exponential backoff. Each live operation gets
+//     several attempts with its own timeout. The retry predicate is
+//     strictly transport-level — "did a nameserver answer at all" —
+//     never the assertion the test is making. A resolver that
+//     answers incorrectly still fails on the first attempt.
+//
+//  3. Quorum. Where an assertion spans several independent
+//     nameservers, a strict majority answering as expected is
+//     enough; a server that fails to answer is tolerated, while a
+//     server that answers *wrongly* still fails the test.
+
+const (
+	// liveAttempts is how many times a live DNS operation is
+	// attempted before the test fails.
+	liveAttempts = 3
+
+	// liveAttemptTimeout bounds one attempt. Worst case for an
+	// operation is liveAttempts * liveAttemptTimeout plus the
+	// backoff — about 26 seconds, well inside the 90-second
+	// `go test -timeout` backstop even when several operations
+	// exhaust their attempts.
+	liveAttemptTimeout = 8 * time.Second
+
+	// liveBackoffBase is the delay after the first failed
+	// attempt; it is multiplied by liveBackoffFactor each time.
+	liveBackoffBase = 500 * time.Millisecond
+
+	// liveBackoffFactor is the exponential backoff multiplier.
+	liveBackoffFactor = 2
+
+	// liveConcurrency caps how many live resolutions may be in
+	// flight across the whole package at once.
+	liveConcurrency = 6
+
+	// minNameservers is the smallest nameserver count a
+	// well-run zone is expected to publish.
+	minNameservers = 2
+)
+
+// liveGate bounds concurrent live resolutions package-wide. It has
+// to be package scoped: the whole point is that it is shared by
+// every parallel test in the package.
+//
+//nolint:gochecknoglobals // package-wide live query rate limit
+var liveGate = make(chan struct{}, liveConcurrency)
+
+var (
+	// errLiveNoAnswer reports that a live operation produced no
+	// usable answer, which is retried rather than asserted on.
+	errLiveNoAnswer = errors.New("no answer from live DNS")
+
+	// errLiveNoQuorum reports that too few of a domain's
+	// nameservers answered for a quorum assertion to be made.
+	errLiveNoQuorum = errors.New("no nameserver quorum")
+)
+
+// runLive executes one attempt of a live operation, holding a slot
+// in liveGate for its duration and bounding it with its own
+// timeout.
+func runLive(op func(ctx context.Context) error) error {
+	liveGate <- struct{}{}
+	defer func() { <-liveGate }()
+
+	ctx, cancel := context.WithTimeout(
+		context.Background(), liveAttemptTimeout,
+	)
+	defer cancel()
+
+	return op(ctx)
+}
+
+// retryLive runs op until it reports success, retrying transport
+// failures with exponential backoff, and fails the test if every
+// attempt fails. op returns an error only for a failure to obtain
+// an answer — never for an answer the test disagrees with, which
+// belongs in an assertion so that it fails immediately. op stores
+// whatever it obtained where its caller can find it.
+func retryLive(
+	t *testing.T,
+	what string,
+	op func(ctx context.Context) error,
+) {
+	t.Helper()
+
+	var last error
+
+	backoff := liveBackoffBase
+
+	for attempt := range liveAttempts {
+		if attempt > 0 {
+			t.Logf(
+				"%s: attempt %d of %d failed (%v), "+
+					"retrying in %s",
+				what, attempt, liveAttempts, last, backoff,
+			)
+			time.Sleep(backoff)
+
+			backoff *= liveBackoffFactor
+		}
+
+		last = runLive(op)
+		if last == nil {
+			return
+		}
+	}
+
+	t.Fatalf(
+		"%s: no answer after %d live attempts: %v",
+		what, liveAttempts, last,
+	)
+}
+
+// liveQuorum is how many of total nameservers must agree for a
+// multi-nameserver assertion to hold: a strict majority.
+func liveQuorum(total int) int {
+	if total < 1 {
+		return 1
+	}
+
+	return total/2 + 1
+}
+
+// countStatus counts the responses carrying the given status.
+func countStatus(
+	results map[string]*resolver.NameserverResponse,
+	status string,
+) int {
+	n := 0
+
+	for _, resp := range results {
+		if resp.Status == status {
+			n++
+		}
+	}
+
+	return n
+}
+
+// answeredCount counts the nameservers that produced an answer of
+// any kind, as opposed to failing or timing out.
+func answeredCount(
+	results map[string]*resolver.NameserverResponse,
+) int {
+	return len(results) -
+		countStatus(results, resolver.StatusError) -
+		countStatus(results, resolver.StatusTimeout)
+}
+
+// describeStatuses renders per-nameserver statuses for use in
+// assertion failure messages.
+func describeStatuses(
+	results map[string]*resolver.NameserverResponse,
+) string {
+	parts := make([]string, 0, len(results))
+	for ns, resp := range results {
+		parts = append(
+			parts, fmt.Sprintf("%s=%s", ns, resp.Status),
+		)
+	}
+
+	sort.Strings(parts)
+
+	return strings.Join(parts, " ")
+}
+
+// ----------------------------------------------------------------
+// Live operation wrappers
+// ----------------------------------------------------------------
+
+// liveFindAuthoritative resolves a domain's authoritative
+// nameservers, retrying until the delegation chain can be walked.
+func liveFindAuthoritative(
+	t *testing.T,
+	r *resolver.Resolver,
+	domain string,
+) []string {
+	t.Helper()
+
+	var out []string
+
+	retryLive(
+		t,
+		"FindAuthoritativeNameservers("+domain+")",
+		func(ctx context.Context) error {
+			ns, err := r.FindAuthoritativeNameservers(ctx, domain)
+			if err != nil {
+				return err
+			}
+
+			if len(ns) == 0 {
+				return fmt.Errorf(
+					"%w: %s has no nameservers",
+					errLiveNoAnswer, domain,
+				)
+			}
+
+			out = ns
+
+			return nil
+		},
+	)
+
+	return out
+}
+
+// liveLookupNS is liveFindAuthoritative through the LookupNS entry
+// point, so that both entry points stay independently exercised.
+func liveLookupNS(
+	t *testing.T,
+	r *resolver.Resolver,
+	domain string,
+) []string {
+	t.Helper()
+
+	var out []string
+
+	retryLive(
+		t,
+		"LookupNS("+domain+")",
+		func(ctx context.Context) error {
+			ns, err := r.LookupNS(ctx, domain)
+			if err != nil {
+				return err
+			}
+
+			if len(ns) == 0 {
+				return fmt.Errorf(
+					"%w: %s has no nameservers",
+					errLiveNoAnswer, domain,
+				)
+			}
+
+			out = ns
+
+			return nil
+		},
+	)
+
+	return out
+}
+
+// liveQueryNameserver queries one nameserver, retrying while that
+// nameserver fails to answer. NXDOMAIN and NODATA are answers and
+// are returned to the caller to assert on.
+func liveQueryNameserver(
+	t *testing.T,
+	r *resolver.Resolver,
+	nameserver string,
+	hostname string,
+) *resolver.NameserverResponse {
+	t.Helper()
+
+	what := fmt.Sprintf(
+		"QueryNameserver(%s, %s)", nameserver, hostname,
+	)
+
+	var out *resolver.NameserverResponse
+
+	retryLive(
+		t,
+		what,
+		func(ctx context.Context) error {
+			resp, err := r.QueryNameserver(
+				ctx, nameserver, hostname,
+			)
+			if err != nil {
+				return err
+			}
+
+			if resp.Status == resolver.StatusTimeout ||
+				resp.Status == resolver.StatusError {
+				return fmt.Errorf(
+					"%w: %s returned %s: %s",
+					errLiveNoAnswer, nameserver,
+					resp.Status, resp.Error,
+				)
+			}
+
+			out = resp
+
+			return nil
+		},
+	)
+
+	return out
+}
+
+// liveQueryAllNameservers queries every authoritative nameserver
+// for a hostname, retrying until a quorum of them has answered.
+// Individual nameservers that stay silent are left in the result
+// for the caller to account for.
+func liveQueryAllNameservers(
+	t *testing.T,
+	r *resolver.Resolver,
+	hostname string,
+) map[string]*resolver.NameserverResponse {
+	t.Helper()
+
+	var out map[string]*resolver.NameserverResponse
+
+	retryLive(
+		t,
+		"QueryAllNameservers("+hostname+")",
+		func(ctx context.Context) error {
+			results, err := r.QueryAllNameservers(ctx, hostname)
+			if err != nil {
+				return err
+			}
+
+			if len(results) == 0 {
+				return fmt.Errorf(
+					"%w: no nameservers queried for %s",
+					errLiveNoAnswer, hostname,
+				)
+			}
+
+			answered := answeredCount(results)
+			if answered < liveQuorum(len(results)) {
+				return fmt.Errorf(
+					"%w: %d of %d answered: %s",
+					errLiveNoQuorum, answered,
+					len(results), describeStatuses(results),
+				)
+			}
+
+			out = results
+
+			return nil
+		},
+	)
+
+	return out
+}
+
+// liveResolveIPs resolves a hostname that is expected to have
+// addresses, retrying until at least one is returned.
+func liveResolveIPs(
+	t *testing.T,
+	r *resolver.Resolver,
+	hostname string,
+) []string {
+	t.Helper()
+
+	var out []string
+
+	retryLive(
+		t,
+		"ResolveIPAddresses("+hostname+")",
+		func(ctx context.Context) error {
+			ips, err := r.ResolveIPAddresses(ctx, hostname)
+			if err != nil {
+				return err
+			}
+
+			if len(ips) == 0 {
+				return fmt.Errorf(
+					"%w: no addresses for %s",
+					errLiveNoAnswer, hostname,
+				)
+			}
+
+			out = ips
+
+			return nil
+		},
+	)
+
+	return out
+}
+
+// liveResolveIPsAllowingEmpty resolves a hostname that may legitimately
+// have no addresses, so the empty result is returned rather than
+// retried. Used for names that must not exist; the corresponding
+// QueryAllNameservers test is what proves the nameservers actively
+// said NXDOMAIN rather than merely staying silent.
+func liveResolveIPsAllowingEmpty(
+	t *testing.T,
+	r *resolver.Resolver,
+	hostname string,
+) []string {
+	t.Helper()
+
+	var out []string
+
+	retryLive(
+		t,
+		"ResolveIPAddresses("+hostname+")",
+		func(ctx context.Context) error {
+			ips, err := r.ResolveIPAddresses(ctx, hostname)
+			if err != nil {
+				return err
+			}
+
+			out = ips
+
+			return nil
+		},
+	)
+
+	return out
+}