package resolver_test import ( "context" "errors" "fmt" "sort" "strings" "testing" "time" "sneak.berlin/go/dnswatcher/internal/resolver" ) // ---------------------------------------------------------------- // Live DNS test support // ---------------------------------------------------------------- // // Every test in this package resolves against the real, live DNS — // see TESTING.md. Nothing here mocks, fakes, stubs, records or // replays DNS, and nothing here skips or gates a test: the helpers // below only change *how* the live queries are issued, so that a // single dropped UDP packet or one slow authoritative server does // not turn a correct resolver into a red build. // // Three mechanisms, all test-side: // // 1. Bounded concurrency. The package's tests are parallel and the // build hosts have many cores, so without a limit every test // starts its own iterative resolution at the same instant and // they all hit the first root server in rootServerList() within // a few milliseconds of each other. Root servers rate-limit // that, which shows up as a different arbitrary subset of tests // failing on each run. liveGate caps how many resolutions are // in flight at once. // // 2. Retry with exponential backoff. Each live operation gets // several attempts with its own timeout. The retry predicate is // strictly transport-level — "did a nameserver answer at all" — // never the assertion the test is making. A resolver that // answers incorrectly still fails on the first attempt. // // 3. Quorum. Where an assertion spans several independent // nameservers, a strict majority answering as expected is // enough; a server that fails to answer is tolerated, while a // server that answers *wrongly* still fails the test. const ( // liveAttempts is how many times a live DNS operation is // attempted before the test fails. liveAttempts = 3 // liveAttemptTimeout bounds one attempt. Worst case for an // operation is liveAttempts * liveAttemptTimeout plus the // backoff — about 26 seconds, well inside the 90-second // `go test -timeout` backstop even when several operations // exhaust their attempts. liveAttemptTimeout = 8 * time.Second // liveBackoffBase is the delay after the first failed // attempt; it is multiplied by liveBackoffFactor each time. liveBackoffBase = 500 * time.Millisecond // liveBackoffFactor is the exponential backoff multiplier. liveBackoffFactor = 2 // liveConcurrency caps how many live resolutions may be in // flight across the whole package at once. liveConcurrency = 6 // minNameservers is the smallest nameserver count a // well-run zone is expected to publish. minNameservers = 2 ) // liveGate bounds concurrent live resolutions package-wide. It has // to be package scoped: the whole point is that it is shared by // every parallel test in the package. // //nolint:gochecknoglobals // package-wide live query rate limit var liveGate = make(chan struct{}, liveConcurrency) var ( // errLiveNoAnswer reports that a live operation produced no // usable answer, which is retried rather than asserted on. errLiveNoAnswer = errors.New("no answer from live DNS") // errLiveNoQuorum reports that too few of a domain's // nameservers answered for a quorum assertion to be made. errLiveNoQuorum = errors.New("no nameserver quorum") ) // runLive executes one attempt of a live operation, holding a slot // in liveGate for its duration and bounding it with its own // timeout. func runLive(op func(ctx context.Context) error) error { liveGate <- struct{}{} defer func() { <-liveGate }() ctx, cancel := context.WithTimeout( context.Background(), liveAttemptTimeout, ) defer cancel() return op(ctx) } // retryLive runs op until it reports success, retrying transport // failures with exponential backoff, and fails the test if every // attempt fails. op returns an error only for a failure to obtain // an answer — never for an answer the test disagrees with, which // belongs in an assertion so that it fails immediately. op stores // whatever it obtained where its caller can find it. func retryLive( t *testing.T, what string, op func(ctx context.Context) error, ) { t.Helper() var last error backoff := liveBackoffBase for attempt := range liveAttempts { if attempt > 0 { t.Logf( "%s: attempt %d of %d failed (%v), "+ "retrying in %s", what, attempt, liveAttempts, last, backoff, ) time.Sleep(backoff) backoff *= liveBackoffFactor } last = runLive(op) if last == nil { return } } t.Fatalf( "%s: no answer after %d live attempts: %v", what, liveAttempts, last, ) } // liveQuorum is how many of total nameservers must agree for a // multi-nameserver assertion to hold: a strict majority. func liveQuorum(total int) int { if total < 1 { return 1 } return total/2 + 1 } // countStatus counts the responses carrying the given status. func countStatus( results map[string]*resolver.NameserverResponse, status string, ) int { n := 0 for _, resp := range results { if resp.Status == status { n++ } } return n } // answeredCount counts the nameservers that produced an answer of // any kind, as opposed to failing or timing out. func answeredCount( results map[string]*resolver.NameserverResponse, ) int { return len(results) - countStatus(results, resolver.StatusError) - countStatus(results, resolver.StatusTimeout) } // describeStatuses renders per-nameserver statuses for use in // assertion failure messages. func describeStatuses( results map[string]*resolver.NameserverResponse, ) string { parts := make([]string, 0, len(results)) for ns, resp := range results { parts = append( parts, fmt.Sprintf("%s=%s", ns, resp.Status), ) } sort.Strings(parts) return strings.Join(parts, " ") } // ---------------------------------------------------------------- // Live operation wrappers // ---------------------------------------------------------------- // liveFindAuthoritative resolves a domain's authoritative // nameservers, retrying until the delegation chain can be walked. func liveFindAuthoritative( t *testing.T, r *resolver.Resolver, domain string, ) []string { t.Helper() var out []string retryLive( t, "FindAuthoritativeNameservers("+domain+")", func(ctx context.Context) error { ns, err := r.FindAuthoritativeNameservers(ctx, domain) if err != nil { return err } if len(ns) == 0 { return fmt.Errorf( "%w: %s has no nameservers", errLiveNoAnswer, domain, ) } out = ns return nil }, ) return out } // liveLookupNS is liveFindAuthoritative through the LookupNS entry // point, so that both entry points stay independently exercised. func liveLookupNS( t *testing.T, r *resolver.Resolver, domain string, ) []string { t.Helper() var out []string retryLive( t, "LookupNS("+domain+")", func(ctx context.Context) error { ns, err := r.LookupNS(ctx, domain) if err != nil { return err } if len(ns) == 0 { return fmt.Errorf( "%w: %s has no nameservers", errLiveNoAnswer, domain, ) } out = ns return nil }, ) return out } // liveQueryNameserver queries one nameserver, retrying while that // nameserver fails to answer. NXDOMAIN and NODATA are answers and // are returned to the caller to assert on. func liveQueryNameserver( t *testing.T, r *resolver.Resolver, nameserver string, hostname string, ) *resolver.NameserverResponse { t.Helper() what := fmt.Sprintf( "QueryNameserver(%s, %s)", nameserver, hostname, ) var out *resolver.NameserverResponse retryLive( t, what, func(ctx context.Context) error { resp, err := r.QueryNameserver( ctx, nameserver, hostname, ) if err != nil { return err } if resp.Status == resolver.StatusTimeout || resp.Status == resolver.StatusError { return fmt.Errorf( "%w: %s returned %s: %s", errLiveNoAnswer, nameserver, resp.Status, resp.Error, ) } out = resp return nil }, ) return out } // liveQueryAllNameservers queries every authoritative nameserver // for a hostname, retrying until a quorum of them has answered. // Individual nameservers that stay silent are left in the result // for the caller to account for. func liveQueryAllNameservers( t *testing.T, r *resolver.Resolver, hostname string, ) map[string]*resolver.NameserverResponse { t.Helper() var out map[string]*resolver.NameserverResponse retryLive( t, "QueryAllNameservers("+hostname+")", func(ctx context.Context) error { results, err := r.QueryAllNameservers(ctx, hostname) if err != nil { return err } if len(results) == 0 { return fmt.Errorf( "%w: no nameservers queried for %s", errLiveNoAnswer, hostname, ) } answered := answeredCount(results) if answered < liveQuorum(len(results)) { return fmt.Errorf( "%w: %d of %d answered: %s", errLiveNoQuorum, answered, len(results), describeStatuses(results), ) } out = results return nil }, ) return out } // liveResolveIPs resolves a hostname that is expected to have // addresses, retrying until at least one is returned. func liveResolveIPs( t *testing.T, r *resolver.Resolver, hostname string, ) []string { t.Helper() var out []string retryLive( t, "ResolveIPAddresses("+hostname+")", func(ctx context.Context) error { ips, err := r.ResolveIPAddresses(ctx, hostname) if err != nil { return err } if len(ips) == 0 { return fmt.Errorf( "%w: no addresses for %s", errLiveNoAnswer, hostname, ) } out = ips return nil }, ) return out } // liveResolveIPsAllowingEmpty resolves a hostname that may legitimately // have no addresses, so the empty result is returned rather than // retried. Used for names that must not exist; the corresponding // QueryAllNameservers test is what proves the nameservers actively // said NXDOMAIN rather than merely staying silent. func liveResolveIPsAllowingEmpty( t *testing.T, r *resolver.Resolver, hostname string, ) []string { t.Helper() var out []string retryLive( t, "ResolveIPAddresses("+hostname+")", func(ctx context.Context) error { ips, err := r.ResolveIPAddresses(ctx, hostname) if err != nil { return err } out = ips return nil }, ) return out }