dnswatcher/internal/resolver/livedns_test.go

package resolver_test

import (
	"context"
	"errors"
	"fmt"
	"sort"
	"strings"
	"testing"
	"time"

	"sneak.berlin/go/dnswatcher/internal/resolver"
)

// ----------------------------------------------------------------
// Live DNS test support
// ----------------------------------------------------------------
//
// Every test in this package resolves against the real, live DNS —
// see TESTING.md. Nothing here mocks, fakes, stubs, records or
// replays DNS, and nothing here skips or gates a test: the helpers
// below only change *how* the live queries are issued, so that a
// single dropped UDP packet or one slow authoritative server does
// not turn a correct resolver into a red build.
//
// Three mechanisms, all test-side:
//
//  1. Bounded concurrency. The package's tests are parallel and the
//     build hosts have many cores, so without a limit every test
//     starts its own iterative resolution at the same instant and
//     they all hit the first root server in rootServerList() within
//     a few milliseconds of each other. Root servers rate-limit
//     that, which shows up as a different arbitrary subset of tests
//     failing on each run. liveGate caps how many resolutions are
//     in flight at once.
//
//  2. Retry with exponential backoff. Each live operation gets
//     several attempts with its own timeout. The retry predicate is
//     strictly transport-level — "did a nameserver answer at all" —
//     never the assertion the test is making. A resolver that
//     answers incorrectly still fails on the first attempt.
//
//  3. Quorum. Where an assertion spans several independent
//     nameservers, a strict majority answering as expected is
//     enough; a server that fails to answer is tolerated, while a
//     server that answers *wrongly* still fails the test.

const (
	// liveAttempts is how many times a live DNS operation is
	// attempted before the test fails.
	liveAttempts = 3

	// liveAttemptTimeout bounds one attempt. Worst case for an
	// operation is liveAttempts * liveAttemptTimeout plus the
	// backoff — about 26 seconds, well inside the 90-second
	// `go test -timeout` backstop even when several operations
	// exhaust their attempts.
	liveAttemptTimeout = 8 * time.Second

	// liveBackoffBase is the delay after the first failed
	// attempt; it is multiplied by liveBackoffFactor each time.
	liveBackoffBase = 500 * time.Millisecond

	// liveBackoffFactor is the exponential backoff multiplier.
	liveBackoffFactor = 2

	// liveConcurrency caps how many live resolutions may be in
	// flight across the whole package at once.
	liveConcurrency = 6

	// minNameservers is the smallest nameserver count a
	// well-run zone is expected to publish.
	minNameservers = 2
)

// liveGate bounds concurrent live resolutions package-wide. It has
// to be package scoped: the whole point is that it is shared by
// every parallel test in the package.
//
//nolint:gochecknoglobals // package-wide live query rate limit
var liveGate = make(chan struct{}, liveConcurrency)

var (
	// errLiveNoAnswer reports that a live operation produced no
	// usable answer, which is retried rather than asserted on.
	errLiveNoAnswer = errors.New("no answer from live DNS")

	// errLiveNoQuorum reports that too few of a domain's
	// nameservers answered for a quorum assertion to be made.
	errLiveNoQuorum = errors.New("no nameserver quorum")
)

// runLive executes one attempt of a live operation, holding a slot
// in liveGate for its duration and bounding it with its own
// timeout.
func runLive(op func(ctx context.Context) error) error {
	liveGate <- struct{}{}
	defer func() { <-liveGate }()

	ctx, cancel := context.WithTimeout(
		context.Background(), liveAttemptTimeout,
	)
	defer cancel()

	return op(ctx)
}

// retryLive runs op until it reports success, retrying transport
// failures with exponential backoff, and fails the test if every
// attempt fails. op returns an error only for a failure to obtain
// an answer — never for an answer the test disagrees with, which
// belongs in an assertion so that it fails immediately. op stores
// whatever it obtained where its caller can find it.
func retryLive(
	t *testing.T,
	what string,
	op func(ctx context.Context) error,
) {
	t.Helper()

	var last error

	backoff := liveBackoffBase

	for attempt := range liveAttempts {
		if attempt > 0 {
			t.Logf(
				"%s: attempt %d of %d failed (%v), "+
					"retrying in %s",
				what, attempt, liveAttempts, last, backoff,
			)
			time.Sleep(backoff)

			backoff *= liveBackoffFactor
		}

		last = runLive(op)
		if last == nil {
			return
		}
	}

	t.Fatalf(
		"%s: no answer after %d live attempts: %v",
		what, liveAttempts, last,
	)
}

// liveQuorum is how many of total nameservers must agree for a
// multi-nameserver assertion to hold: a strict majority.
func liveQuorum(total int) int {
	if total < 1 {
		return 1
	}

	return total/2 + 1
}

// countStatus counts the responses carrying the given status.
func countStatus(
	results map[string]*resolver.NameserverResponse,
	status string,
) int {
	n := 0

	for _, resp := range results {
		if resp.Status == status {
			n++
		}
	}

	return n
}

// answeredCount counts the nameservers that produced an answer of
// any kind, as opposed to failing or timing out.
func answeredCount(
	results map[string]*resolver.NameserverResponse,
) int {
	return len(results) -
		countStatus(results, resolver.StatusError) -
		countStatus(results, resolver.StatusTimeout)
}

// describeStatuses renders per-nameserver statuses for use in
// assertion failure messages.
func describeStatuses(
	results map[string]*resolver.NameserverResponse,
) string {
	parts := make([]string, 0, len(results))
	for ns, resp := range results {
		parts = append(
			parts, fmt.Sprintf("%s=%s", ns, resp.Status),
		)
	}

	sort.Strings(parts)

	return strings.Join(parts, " ")
}

// ----------------------------------------------------------------
// Live operation wrappers
// ----------------------------------------------------------------

// liveFindAuthoritative resolves a domain's authoritative
// nameservers, retrying until the delegation chain can be walked.
func liveFindAuthoritative(
	t *testing.T,
	r *resolver.Resolver,
	domain string,
) []string {
	t.Helper()

	var out []string

	retryLive(
		t,
		"FindAuthoritativeNameservers("+domain+")",
		func(ctx context.Context) error {
			ns, err := r.FindAuthoritativeNameservers(ctx, domain)
			if err != nil {
				return err
			}

			if len(ns) == 0 {
				return fmt.Errorf(
					"%w: %s has no nameservers",
					errLiveNoAnswer, domain,
				)
			}

			out = ns

			return nil
		},
	)

	return out
}

// liveLookupNS is liveFindAuthoritative through the LookupNS entry
// point, so that both entry points stay independently exercised.
func liveLookupNS(
	t *testing.T,
	r *resolver.Resolver,
	domain string,
) []string {
	t.Helper()

	var out []string

	retryLive(
		t,
		"LookupNS("+domain+")",
		func(ctx context.Context) error {
			ns, err := r.LookupNS(ctx, domain)
			if err != nil {
				return err
			}

			if len(ns) == 0 {
				return fmt.Errorf(
					"%w: %s has no nameservers",
					errLiveNoAnswer, domain,
				)
			}

			out = ns

			return nil
		},
	)

	return out
}

// liveQueryNameserver queries one nameserver, retrying while that
// nameserver fails to answer. NXDOMAIN and NODATA are answers and
// are returned to the caller to assert on.
func liveQueryNameserver(
	t *testing.T,
	r *resolver.Resolver,
	nameserver string,
	hostname string,
) *resolver.NameserverResponse {
	t.Helper()

	what := fmt.Sprintf(
		"QueryNameserver(%s, %s)", nameserver, hostname,
	)

	var out *resolver.NameserverResponse

	retryLive(
		t,
		what,
		func(ctx context.Context) error {
			resp, err := r.QueryNameserver(
				ctx, nameserver, hostname,
			)
			if err != nil {
				return err
			}

			if resp.Status == resolver.StatusTimeout ||
				resp.Status == resolver.StatusError {
				return fmt.Errorf(
					"%w: %s returned %s: %s",
					errLiveNoAnswer, nameserver,
					resp.Status, resp.Error,
				)
			}

			out = resp

			return nil
		},
	)

	return out
}

// liveQueryAllNameservers queries every authoritative nameserver
// for a hostname, retrying until a quorum of them has answered.
// Individual nameservers that stay silent are left in the result
// for the caller to account for.
func liveQueryAllNameservers(
	t *testing.T,
	r *resolver.Resolver,
	hostname string,
) map[string]*resolver.NameserverResponse {
	t.Helper()

	var out map[string]*resolver.NameserverResponse

	retryLive(
		t,
		"QueryAllNameservers("+hostname+")",
		func(ctx context.Context) error {
			results, err := r.QueryAllNameservers(ctx, hostname)
			if err != nil {
				return err
			}

			if len(results) == 0 {
				return fmt.Errorf(
					"%w: no nameservers queried for %s",
					errLiveNoAnswer, hostname,
				)
			}

			answered := answeredCount(results)
			if answered < liveQuorum(len(results)) {
				return fmt.Errorf(
					"%w: %d of %d answered: %s",
					errLiveNoQuorum, answered,
					len(results), describeStatuses(results),
				)
			}

			out = results

			return nil
		},
	)

	return out
}

// liveResolveIPs resolves a hostname that is expected to have
// addresses, retrying until at least one is returned.
func liveResolveIPs(
	t *testing.T,
	r *resolver.Resolver,
	hostname string,
) []string {
	t.Helper()

	var out []string

	retryLive(
		t,
		"ResolveIPAddresses("+hostname+")",
		func(ctx context.Context) error {
			ips, err := r.ResolveIPAddresses(ctx, hostname)
			if err != nil {
				return err
			}

			if len(ips) == 0 {
				return fmt.Errorf(
					"%w: no addresses for %s",
					errLiveNoAnswer, hostname,
				)
			}

			out = ips

			return nil
		},
	)

	return out
}

// liveResolveIPsAllowingEmpty resolves a hostname that may legitimately
// have no addresses, so the empty result is returned rather than
// retried. Used for names that must not exist; the corresponding
// QueryAllNameservers test is what proves the nameservers actively
// said NXDOMAIN rather than merely staying silent.
func liveResolveIPsAllowingEmpty(
	t *testing.T,
	r *resolver.Resolver,
	hostname string,
) []string {
	t.Helper()

	var out []string

	retryLive(
		t,
		"ResolveIPAddresses("+hostname+")",
		func(ctx context.Context) error {
			ips, err := r.ResolveIPAddresses(ctx, hostname)
			if err != nil {
				return err
			}

			out = ips

			return nil
		},
	)

	return out
}