dnswatcher/internal/livednstest/livednstest.go

// Package livednstest runs the live DNS operations of tests. Tests that
// look something up in DNS query live DNS servers, never a stand-in —
// see TESTING.md. Nothing here mocks, fakes, stubs, records or replays
// DNS, and nothing here skips a test: it only changes *how* the live
// queries are issued, so that a single dropped UDP packet or one slow
// authoritative server does not turn correct code into a red build.
//
// Two mechanisms:
//
//  1. Bounded concurrency. Tests run in parallel and the build hosts
//     have many cores, so without a limit every test starts its own
//     iterative resolution at the same instant and they all hit the
//     first root server within a few milliseconds of each other. Root
//     servers rate-limit that, which shows up as a different arbitrary
//     subset of tests failing on each run. Run caps how many live
//     operations are in flight at once in one test binary.
//
//  2. Retry with exponential backoff. Each live operation gets several
//     attempts with its own timeout. An attempt is retried when it
//     obtained nothing to check, never because of what the test
//     asserts about the result, so a wrong result still fails on the
//     first attempt. A fault in the code under test that leaves
//     nothing to check looks the same as live DNS not answering, and
//     fails only after the last attempt.
package livednstest

import (
	"context"
	"errors"
	"testing"
	"time"
)

const (
	// attempts is how many times a live DNS operation is attempted
	// before the test fails.
	attempts = 3

	// AttemptTimeout bounds one attempt. Worst case for an operation
	// is attempts * AttemptTimeout plus the backoff — about 26
	// seconds, well inside the 90-second `go test -timeout` backstop
	// even when several operations exhaust their attempts.
	AttemptTimeout = 8 * time.Second

	// backoffBase is the delay after the first failed attempt; it is
	// multiplied by backoffFactor each time.
	backoffBase = 500 * time.Millisecond

	// backoffFactor is the exponential backoff multiplier.
	backoffFactor = 2

	// Concurrency caps how many live operations may be in flight
	// across one test binary at once.
	Concurrency = 6
)

// gate bounds concurrent live operations. It has to be package scoped:
// the whole point is that it is shared by every parallel test in the
// test binary.
//
//nolint:gochecknoglobals // package-wide live query rate limit
var gate = make(chan struct{}, Concurrency)

// ErrNoAnswer reports that a live operation produced no usable answer,
// which is retried rather than asserted on.
var ErrNoAnswer = errors.New("no answer from live DNS")

// Run executes one attempt of a live operation, holding a slot in gate
// for its duration and bounding it with its own timeout.
func Run(op func(ctx context.Context) error) error {
	gate <- struct{}{}
	defer func() { <-gate }()

	ctx, cancel := context.WithTimeout(
		context.Background(), AttemptTimeout,
	)
	defer cancel()

	return op(ctx)
}

// Retry runs op until it reports success, retrying failures with
// exponential backoff, and fails the test if every attempt fails. op
// returns an error only for a failure to obtain an answer — never for
// an answer the test disagrees with, which belongs in an assertion so
// that it fails immediately. op stores whatever it obtained where its
// caller can find it.
func Retry(
	t *testing.T,
	what string,
	op func(ctx context.Context) error,
) {
	t.Helper()

	var last error

	backoff := backoffBase

	for attempt := range attempts {
		if attempt > 0 {
			t.Logf(
				"%s: attempt %d of %d failed (%v), "+
					"retrying in %s",
				what, attempt, attempts, last, backoff,
			)
			time.Sleep(backoff)

			backoff *= backoffFactor
		}

		last = Run(op)
		if last == nil {
			return
		}
	}

	t.Fatalf(
		"%s: all %d live attempts failed: %v",
		what, attempts, last,
	)
}