dnswatcher/internal/watcher/watcher_test.go

package watcher_test

import (
	"context"
	"errors"
	"fmt"
	"log/slog"
	"os"
	"slices"
	"sort"
	"strings"
	"sync"
	"testing"
	"time"

	"sneak.berlin/go/dnswatcher/internal/config"
	"sneak.berlin/go/dnswatcher/internal/portcheck"
	"sneak.berlin/go/dnswatcher/internal/resolver"
	"sneak.berlin/go/dnswatcher/internal/state"
	"sneak.berlin/go/dnswatcher/internal/tlscheck"
	"sneak.berlin/go/dnswatcher/internal/watcher"
)

// These tests exercise the watcher against live DNS. DNS is never
// mocked in this repository (see TESTING.md): the watcher is wired
// to the real iterative resolver and queries stable, well-known
// names. Only the non-DNS collaborators (port checker, TLS
// checker, notifier) are test doubles so that notification and
// state behavior can be asserted deterministically.
//
// Change detection is exercised by seeding the state store with a
// synthetic previous observation that live DNS cannot match; DNS
// itself stays live in every run.

const (
	// testDomain and testHostname are IANA-operated names
	// reserved for documentation use; they are stable and
	// resolve worldwide.
	testDomain   = "example.com"
	testHostname = "www.example.com"

	// fakeNS uses the RFC 2606 reserved .invalid TLD, so it can
	// never collide with a nameserver name in live DNS results.
	fakeNS = "ns-baseline.invalid."

	// fakeIP is an RFC 5737 documentation address that live DNS
	// can never return for a real hostname.
	fakeIP = "203.0.113.1"

	startupTitle = "✅ dnswatcher startup complete"

	// longLife is a certificate lifetime far outside the expiry
	// warning window; shortLife is inside it.
	longLife  = 90 * 24 * time.Hour
	shortLife = 3 * 24 * time.Hour

	// monitoredTestPorts and tlsCheckPort mirror the ports the
	// watcher checks. They are duplicated here deliberately: a
	// test that imported the production values would agree with
	// the watcher no matter which ports it drifted to.
	tlsCheckPort = 443

	pollInterval = 100 * time.Millisecond
	scanTimeout  = 25 * time.Second
	notifyGrace  = 500 * time.Millisecond
	shutdownWait = 5 * time.Second
)

// errNotFound is returned when a test double has no data.
var errNotFound = errors.New("not found")

// --- Test doubles for non-DNS collaborators ---

// portCall records one CheckPort invocation. The arguments matter,
// not just the count: a watcher that dutifully checks ports on the
// wrong addresses produces exactly the same call count as one that
// resolved correctly, so the tests assert what was checked.
type portCall struct {
	address string
	port    int
}

type mockPortChecker struct {
	mu      sync.Mutex
	openAll bool
	err     error
	calls   int
	seen    []portCall
}

func (m *mockPortChecker) CheckPort(
	_ context.Context,
	address string,
	port int,
) (*portcheck.PortResult, error) {
	m.mu.Lock()
	defer m.mu.Unlock()

	m.calls++
	m.seen = append(m.seen, portCall{address: address, port: port})

	if m.err != nil {
		return nil, m.err
	}

	return &portcheck.PortResult{Open: m.openAll}, nil
}

func (m *mockPortChecker) setOpenAll(open bool) {
	m.mu.Lock()
	defer m.mu.Unlock()

	m.openAll = open
}

func (m *mockPortChecker) callCount() int {
	m.mu.Lock()
	defer m.mu.Unlock()

	return m.calls
}

// checkedKeys returns the distinct "address:port" pairs the checker
// was asked about, sorted, in the same form as the state store's
// port keys so the two can be compared directly.
func (m *mockPortChecker) checkedKeys() []string {
	m.mu.Lock()
	defer m.mu.Unlock()

	keys := make([]string, 0, len(m.seen))
	for _, c := range m.seen {
		keys = append(
			keys, fmt.Sprintf("%s:%d", c.address, c.port),
		)
	}

	return sortedSet(keys)
}

// tlsCall records one CheckCertificate invocation, for the same
// reason portCall records CheckPort's arguments.
type tlsCall struct {
	address  string
	hostname string
}

type mockTLSChecker struct {
	mu    sync.Mutex
	cert  *tlscheck.CertificateInfo
	err   error
	calls int
	seen  []tlsCall
}

func (m *mockTLSChecker) CheckCertificate(
	_ context.Context,
	address string,
	hostname string,
) (*tlscheck.CertificateInfo, error) {
	m.mu.Lock()
	defer m.mu.Unlock()

	m.calls++
	m.seen = append(
		m.seen, tlsCall{address: address, hostname: hostname},
	)

	if m.err != nil {
		return nil, m.err
	}

	if m.cert == nil {
		return nil, fmt.Errorf(
			"%w: cert for %s", errNotFound, hostname,
		)
	}

	return m.cert, nil
}

func (m *mockTLSChecker) setCert(cert *tlscheck.CertificateInfo) {
	m.mu.Lock()
	defer m.mu.Unlock()

	m.cert = cert
}

func (m *mockTLSChecker) callCount() int {
	m.mu.Lock()
	defer m.mu.Unlock()

	return m.calls
}

// checkedKeys returns the distinct certificate keys the checker was
// asked about, sorted, in the state store's "address:port:hostname"
// form so the two can be compared directly.
func (m *mockTLSChecker) checkedKeys() []string {
	m.mu.Lock()
	defer m.mu.Unlock()

	keys := make([]string, 0, len(m.seen))
	for _, c := range m.seen {
		keys = append(keys, fmt.Sprintf(
			"%s:%d:%s", c.address, tlsCheckPort, c.hostname,
		))
	}

	return sortedSet(keys)
}

type notification struct {
	Title    string
	Message  string
	Priority string
}

type mockNotifier struct {
	mu            sync.Mutex
	notifications []notification
}

func (m *mockNotifier) SendNotification(
	_ context.Context,
	title, message, priority string,
) {
	m.mu.Lock()
	defer m.mu.Unlock()

	m.notifications = append(m.notifications, notification{
		Title:    title,
		Message:  message,
		Priority: priority,
	})
}

func (m *mockNotifier) getNotifications() []notification {
	m.mu.Lock()
	defer m.mu.Unlock()

	result := make([]notification, len(m.notifications))
	copy(result, m.notifications)

	return result
}

// --- Helpers to build a Watcher for testing ---

type testDeps struct {
	portChecker *mockPortChecker
	tlsChecker  *mockTLSChecker
	notifier    *mockNotifier
	state       *state.State
	config      *config.Config
}

// liveWatcherConcurrency caps how many of this package's tests may
// be driving live DNS at once. Every test here runs a full iterative
// resolution, and the package is entirely parallel, so without a
// bound all of them burst against the same root servers in the same
// instant and get rate-limited — the fan-out pathology that
// internal/resolver/livedns_test.go exists to prevent. That gate is
// package-scoped and so cannot reach across into this package's test
// binary; this is its counterpart here.
const liveWatcherConcurrency = 3

// liveWatcherGate bounds concurrent live-DNS watcher tests. It has to
// be package scoped: the point is that every parallel test shares it.
//
//nolint:gochecknoglobals // package-wide live query rate limit
var liveWatcherGate = make(chan struct{}, liveWatcherConcurrency)

// acquireLiveSlot blocks until this test may run live DNS, releasing
// the slot when the test ends.
func acquireLiveSlot(t *testing.T) {
	t.Helper()

	liveWatcherGate <- struct{}{}

	t.Cleanup(func() { <-liveWatcherGate })
}

func newTestWatcher(
	t *testing.T,
	cfg *config.Config,
) (*watcher.Watcher, *testDeps) {
	t.Helper()

	acquireLiveSlot(t)

	deps := &testDeps{
		portChecker: &mockPortChecker{},
		tlsChecker:  &mockTLSChecker{},
		notifier:    &mockNotifier{},
		config:      cfg,
	}

	deps.state = state.NewForTestWithDataDir(cfg.DataDir)

	log := slog.New(slog.NewTextHandler(
		os.Stderr,
		&slog.HandlerOptions{Level: slog.LevelDebug},
	))

	w := watcher.NewForTest(
		deps.config,
		deps.state,
		resolver.NewFromLogger(log),
		deps.portChecker,
		deps.tlsChecker,
		deps.notifier,
	)

	return w, deps
}

func defaultTestConfig(t *testing.T) *config.Config {
	t.Helper()

	return &config.Config{
		DNSInterval:      time.Hour,
		TLSInterval:      12 * time.Hour,
		TLSExpiryWarning: 7,
		DataDir:          t.TempDir(),
	}
}

func testCert(
	hostname string,
	lifetime time.Duration,
) *tlscheck.CertificateInfo {
	return &tlscheck.CertificateInfo{
		CommonName:              hostname,
		Issuer:                  "Test CA",
		NotAfter:                time.Now().Add(lifetime),
		SubjectAlternativeNames: []string{hostname},
	}
}

// setupHealthyEndpoints makes every port report open and every TLS
// check return a long-lived certificate, regardless of which IPs
// live DNS resolves.
func setupHealthyEndpoints(deps *testDeps, hostname string) {
	deps.portChecker.setOpenAll(true)
	deps.tlsChecker.setCert(testCert(hostname, longLife))
}

// countTitled counts notifications whose title starts with prefix.
func countTitled(deps *testDeps, prefix string) int {
	count := 0

	for _, n := range deps.notifier.getNotifications() {
		if strings.HasPrefix(n.Title, prefix) {
			count++
		}
	}

	return count
}

// waitFor polls cond until it returns true or timeout elapses.
func waitFor(timeout time.Duration, cond func() bool) bool {
	deadline := time.Now().Add(timeout)

	for time.Now().Before(deadline) {
		if cond() {
			return true
		}

		time.Sleep(pollInterval)
	}

	return cond()
}

// startWatcher runs w.Run in a goroutine and returns a stop
// function that cancels it and waits for a clean shutdown.
func startWatcher(
	t *testing.T,
	w *watcher.Watcher,
) func() {
	t.Helper()

	ctx, cancel := context.WithCancel(t.Context())
	done := make(chan struct{})

	go func() {
		w.Run(ctx)
		close(done)
	}()

	return func() {
		cancel()

		select {
		case <-done:
			// Shut down cleanly.
		case <-time.After(shutdownWait):
			t.Error("watcher did not shut down within timeout")
		}
	}
}

// liveIPs extracts all A/AAAA addresses recorded for hostname in
// the snapshot, i.e. what live DNS actually resolved.
func liveIPs(snap state.Snapshot, hostname string) []string {
	hs, ok := snap.Hostnames[hostname]
	if !ok {
		return nil
	}

	seen := make(map[string]bool)

	var ips []string

	for _, nsState := range hs.RecordsByNameserver {
		for _, recType := range []string{"A", "AAAA"} {
			for _, ip := range nsState.Records[recType] {
				if !seen[ip] {
					seen[ip] = true

					ips = append(ips, ip)
				}
			}
		}
	}

	sort.Strings(ips)

	return ips
}

// monitoredTestPorts is the set of ports the watcher is expected to
// check on every resolved address.
func monitoredTestPorts() []int {
	return []int{80, tlsCheckPort}
}

// sortedSet deduplicates and sorts, so that call records and state
// keys can be compared as sets.
func sortedSet(values []string) []string {
	seen := make(map[string]bool, len(values))

	out := make([]string, 0, len(values))

	for _, v := range values {
		if seen[v] {
			continue
		}

		seen[v] = true

		out = append(out, v)
	}

	sort.Strings(out)

	return out
}

// watchedNames is every name the config puts under observation.
// Domains get hostname state too, for their port and TLS checks.
func watchedNames(cfg *config.Config) []string {
	names := make([]string, 0, len(cfg.Hostnames)+len(cfg.Domains))
	names = append(names, cfg.Hostnames...)
	names = append(names, cfg.Domains...)

	return sortedSet(names)
}

// stateHostnames is the set of names the run actually recorded.
func stateHostnames(snap state.Snapshot) []string {
	names := make([]string, 0, len(snap.Hostnames))
	for name := range snap.Hostnames {
		names = append(names, name)
	}

	return sortedSet(names)
}

// expectedPortKeys is every "address:port" the watcher should have
// touched: the monitored ports on every address live DNS resolved
// for every watched name, and nothing else.
func expectedPortKeys(
	snap state.Snapshot,
	names []string,
) []string {
	var keys []string

	for _, name := range names {
		for _, ip := range liveIPs(snap, name) {
			for _, port := range monitoredTestPorts() {
				keys = append(
					keys, fmt.Sprintf("%s:%d", ip, port),
				)
			}
		}
	}

	return sortedSet(keys)
}

// expectedCertKeys is every "address:port:hostname" the watcher
// should have a certificate for, given that every port reports open.
func expectedCertKeys(
	snap state.Snapshot,
	names []string,
) []string {
	var keys []string

	for _, name := range names {
		for _, ip := range liveIPs(snap, name) {
			keys = append(keys, fmt.Sprintf(
				"%s:%d:%s", ip, tlsCheckPort, name,
			))
		}
	}

	return sortedSet(keys)
}

// stateKeys sorts a state map's keys for set comparison.
func stateKeys[V any](m map[string]V) []string {
	keys := make([]string, 0, len(m))
	for k := range m {
		keys = append(keys, k)
	}

	return sortedSet(keys)
}

// assertSameSet fails with both sides named when they differ.
func assertSameSet(t *testing.T, what string, want, got []string) {
	t.Helper()

	if !slices.Equal(want, got) {
		t.Errorf(
			"%s: expected %v, got %v", what, want, got,
		)
	}
}

// --- Tests ---

func TestFirstRunBaseline(t *testing.T) {
	t.Parallel()

	cfg := defaultTestConfig(t)
	cfg.Domains = []string{testDomain}
	cfg.Hostnames = []string{testHostname}

	w, deps := newTestWatcher(t, cfg)
	setupHealthyEndpoints(deps, testHostname)

	w.RunOnce(t.Context())

	assertNoNotifications(t, deps)
	assertStatePopulated(t, deps)
}

func assertNoNotifications(
	t *testing.T,
	deps *testDeps,
) {
	t.Helper()

	notifications := deps.notifier.getNotifications()
	if len(notifications) != 0 {
		t.Errorf(
			"expected 0 notifications on first run, got %d: %v",
			len(notifications), notifications,
		)
	}
}

func assertStatePopulated(
	t *testing.T,
	deps *testDeps,
) {
	t.Helper()

	snap := deps.state.GetSnapshot()

	if len(snap.Domains) != 1 {
		t.Errorf(
			"expected 1 domain in state, got %d",
			len(snap.Domains),
		)
	}

	// Hostname state covers both explicit hostnames and domains
	// (domains also get hostname state for port/TLS checks), and
	// covers exactly those: a name in state that nothing asked
	// for, or a configured name missing from it, is a bug.
	names := watchedNames(deps.config)
	assertSameSet(
		t, "hostnames in state", names, stateHostnames(snap),
	)

	// Every watched name must have resolved to something, or the
	// key comparisons below would pass vacuously on empty sets.
	for _, name := range names {
		if len(liveIPs(snap, name)) == 0 {
			t.Errorf("no addresses resolved for %s", name)
		}
	}

	// The addresses the port checker was aimed at must be the
	// addresses live DNS returned. Counting calls would not
	// notice the watcher checking the wrong hosts.
	assertSameSet(
		t, "ports checked",
		expectedPortKeys(snap, names),
		deps.portChecker.checkedKeys(),
	)
}

func TestDomainPortAndTLSChecks(t *testing.T) {
	t.Parallel()

	cfg := defaultTestConfig(t)
	cfg.Domains = []string{testDomain}

	w, deps := newTestWatcher(t, cfg)
	setupHealthyEndpoints(deps, testDomain)

	w.RunOnce(t.Context())

	snap := deps.state.GetSnapshot()

	names := watchedNames(cfg)

	ips := liveIPs(snap, testDomain)
	if len(ips) == 0 {
		t.Fatal("live DNS resolved no addresses for " + testDomain)
	}

	// Port and certificate state must be keyed by the addresses
	// live DNS actually returned — every one of them, and no
	// others. Asserting only that the maps are non-empty would
	// hold just as well if the watcher had resolved the wrong
	// name or dropped all but one of its addresses.
	wantPorts := expectedPortKeys(snap, names)
	assertSameSet(t, "port state", wantPorts, stateKeys(snap.Ports))
	assertSameSet(
		t, "ports checked", wantPorts,
		deps.portChecker.checkedKeys(),
	)

	wantCerts := expectedCertKeys(snap, names)
	assertSameSet(
		t, "certificate state", wantCerts,
		stateKeys(snap.Certificates),
	)
	assertSameSet(
		t, "certificates checked", wantCerts,
		deps.tlsChecker.checkedKeys(),
	)
}

func TestNSChangeDetection(t *testing.T) {
	t.Parallel()

	cfg := defaultTestConfig(t)
	cfg.Domains = []string{testDomain}

	w, deps := newTestWatcher(t, cfg)

	ctx := t.Context()
	w.RunOnce(ctx)

	// Replace the recorded NS baseline with a synthetic value
	// that live DNS cannot return; the next live run must
	// detect the difference.
	deps.state.SetDomainState(testDomain, &state.DomainState{
		Nameservers: []string{fakeNS},
		LastChecked: time.Now().UTC(),
	})

	w.RunOnce(ctx)

	if countTitled(deps, "NS Change: ") == 0 {
		t.Error("expected notification for NS change")
	}

	found := false

	for _, n := range deps.notifier.getNotifications() {
		if n.Priority == "warning" {
			found = true
		}
	}

	if !found {
		t.Error("expected warning-priority NS change notification")
	}
}

func TestRecordChangeDetection(t *testing.T) {
	t.Parallel()

	cfg := defaultTestConfig(t)
	cfg.Hostnames = []string{testHostname}

	w, deps := newTestWatcher(t, cfg)

	ctx := t.Context()
	w.RunOnce(ctx)

	hs, ok := deps.state.GetHostnameState(testHostname)
	if !ok || len(hs.RecordsByNameserver) == 0 {
		t.Fatal("expected hostname state after first live run")
	}

	// Rewrite every nameserver's recorded answer to a
	// documentation address; the next live run must observe
	// different records and notify.
	tampered := &state.HostnameState{
		RecordsByNameserver: make(
			map[string]*state.NameserverRecordState,
		),
		LastChecked: hs.LastChecked,
	}

	for ns := range hs.RecordsByNameserver {
		tampered.RecordsByNameserver[ns] = &state.NameserverRecordState{
			Records:     map[string][]string{"A": {fakeIP}},
			Status:      "ok",
			LastChecked: time.Now().UTC(),
		}
	}

	deps.state.SetHostnameState(testHostname, tampered)

	w.RunOnce(ctx)

	if countTitled(deps, "Record Change: ") == 0 {
		t.Error("expected notification for record change")
	}
}

func TestPortStateChange(t *testing.T) {
	t.Parallel()

	cfg := defaultTestConfig(t)
	cfg.Hostnames = []string{testHostname}

	w, deps := newTestWatcher(t, cfg)
	setupHealthyEndpoints(deps, testHostname)

	ctx := t.Context()
	w.RunOnce(ctx)

	// All live-resolved addresses flip from open to closed.
	deps.portChecker.setOpenAll(false)

	w.RunOnce(ctx)

	if countTitled(deps, "Port Change: ") == 0 {
		t.Error("expected notification for port state change")
	}
}

func TestTLSExpiryWarning(t *testing.T) {
	t.Parallel()

	cfg := defaultTestConfig(t)
	cfg.Hostnames = []string{testHostname}

	w, deps := newTestWatcher(t, cfg)
	deps.portChecker.setOpenAll(true)
	deps.tlsChecker.setCert(testCert(testHostname, shortLife))

	w.RunOnce(t.Context())

	found := false

	for _, n := range deps.notifier.getNotifications() {
		if strings.HasPrefix(n.Title, "TLS Expiry Warning: ") &&
			n.Priority == "warning" {
			found = true
		}
	}

	if !found {
		t.Errorf(
			"expected expiry warning, got: %v",
			deps.notifier.getNotifications(),
		)
	}
}

func TestTLSExpiryWarningDedup(t *testing.T) {
	t.Parallel()

	cfg := defaultTestConfig(t)
	cfg.Hostnames = []string{testHostname}
	cfg.TLSInterval = 24 * time.Hour

	w, deps := newTestWatcher(t, cfg)
	deps.portChecker.setOpenAll(true)
	deps.tlsChecker.setCert(testCert(testHostname, shortLife))

	ctx := t.Context()

	// First run fires one expiry warning per live-resolved IP.
	w.RunOnce(ctx)

	afterFirst := countTitled(deps, "TLS Expiry Warning: ")
	if afterFirst == 0 {
		t.Fatal("expected at least one expiry warning")
	}

	// A second run within the TLS interval must not re-notify.
	w.RunOnce(ctx)

	afterSecond := countTitled(deps, "TLS Expiry Warning: ")
	if afterSecond != afterFirst {
		t.Errorf(
			"expected expiry warnings deduplicated at %d, got %d",
			afterFirst, afterSecond,
		)
	}
}

func TestGracefulShutdown(t *testing.T) {
	t.Parallel()

	cfg := defaultTestConfig(t)
	cfg.Domains = []string{testDomain}
	cfg.DNSInterval = 100 * time.Millisecond
	cfg.TLSInterval = 100 * time.Millisecond

	w, _ := newTestWatcher(t, cfg)

	stop := startWatcher(t, w)

	// Cancel while live DNS work is likely still in flight; the
	// watcher must still shut down promptly.
	time.Sleep(250 * time.Millisecond)
	stop()
}

func TestDNSRunsBeforePortAndTLSChecks(t *testing.T) {
	t.Parallel()

	cfg := defaultTestConfig(t)
	cfg.Hostnames = []string{testHostname}

	w, deps := newTestWatcher(t, cfg)
	setupHealthyEndpoints(deps, testHostname)

	// Seed stale state pointing at an address live DNS cannot
	// return. A correct cycle refreshes DNS before the port and
	// TLS phases, so the stale address must be dropped, not
	// probed.
	deps.state.SetHostnameState(testHostname, &state.HostnameState{
		RecordsByNameserver: map[string]*state.NameserverRecordState{
			fakeNS: {
				Records:     map[string][]string{"A": {fakeIP}},
				Status:      "ok",
				LastChecked: time.Now().UTC(),
			},
		},
		LastChecked: time.Now().UTC(),
	})
	deps.state.SetPortState(fakeIP+":80", &state.PortState{
		Open:        true,
		Hostnames:   []string{testHostname},
		LastChecked: time.Now().UTC(),
	})

	w.RunOnce(t.Context())

	snap := deps.state.GetSnapshot()

	ips := liveIPs(snap, testHostname)
	if len(ips) == 0 {
		t.Fatal("expected live-resolved IPs in hostname state")
	}

	for _, ip := range ips {
		if _, ok := snap.Ports[ip+":80"]; !ok {
			t.Errorf(
				"port check missed fresh DNS address %s", ip,
			)
		}
	}

	if _, ok := snap.Ports[fakeIP+":80"]; ok {
		t.Error("port check used stale DNS: " + fakeIP + ":80")
	}

	certKey := ips[0] + ":443:" + testHostname
	if _, ok := snap.Certificates[certKey]; !ok {
		t.Error("TLS check used stale DNS: missing " + certKey)
	}
}

func TestSendTestNotification_Enabled(t *testing.T) {
	t.Parallel()

	cfg := defaultTestConfig(t)
	cfg.Hostnames = []string{testHostname}
	cfg.SendTestNotification = true

	w, deps := newTestWatcher(t, cfg)
	setupHealthyEndpoints(deps, testHostname)

	w.RunOnce(t.Context())

	// RunOnce does not send the test notification — it is
	// sent by Run after RunOnce completes.
	notifications := deps.notifier.getNotifications()
	if len(notifications) != 0 {
		t.Errorf(
			"RunOnce should not send test notification, got %d",
			len(notifications),
		)
	}
}

func TestSendTestNotification_ViaRun(t *testing.T) {
	t.Parallel()

	cfg := defaultTestConfig(t)
	cfg.Hostnames = []string{testHostname}
	cfg.SendTestNotification = true
	cfg.DNSInterval = 24 * time.Hour
	cfg.TLSInterval = 24 * time.Hour

	w, deps := newTestWatcher(t, cfg)
	setupHealthyEndpoints(deps, testHostname)

	stop := startWatcher(t, w)

	// The startup notification is sent after the initial live
	// scan completes; wait for it rather than assuming a scan
	// duration.
	found := waitFor(scanTimeout, func() bool {
		return countTitled(deps, startupTitle) > 0
	})

	stop()

	if !found {
		t.Errorf(
			"expected startup test notification, got: %v",
			deps.notifier.getNotifications(),
		)
	}

	for _, n := range deps.notifier.getNotifications() {
		if n.Title == startupTitle && n.Priority != "success" {
			t.Errorf(
				"expected success priority, got %q", n.Priority,
			)
		}
	}
}

func TestSendTestNotification_Disabled(t *testing.T) {
	t.Parallel()

	cfg := defaultTestConfig(t)
	cfg.Hostnames = []string{testHostname}
	cfg.SendTestNotification = false
	cfg.DNSInterval = 24 * time.Hour
	cfg.TLSInterval = 24 * time.Hour

	w, deps := newTestWatcher(t, cfg)
	setupHealthyEndpoints(deps, testHostname)

	stop := startWatcher(t, w)

	// Wait until the initial scan has completed (RunOnce saves
	// state as its final step), then allow a grace period for
	// any notification that would follow it.
	scanned := waitFor(scanTimeout, func() bool {
		return !deps.state.GetSnapshot().LastUpdated.IsZero()
	})

	time.Sleep(notifyGrace)
	stop()

	if !scanned {
		t.Fatal("initial live scan did not complete in time")
	}

	if countTitled(deps, startupTitle) != 0 {
		t.Error(
			"test notification should not be sent when disabled",
		)
	}
}

func TestNSFailureAndRecovery(t *testing.T) {
	t.Parallel()

	cfg := defaultTestConfig(t)
	cfg.Hostnames = []string{testHostname}

	w, deps := newTestWatcher(t, cfg)

	ctx := t.Context()
	w.RunOnce(ctx)

	tamperNSBaseline(t, deps)

	w.RunOnce(ctx)

	// fakeNS was recorded as healthy but is absent from live
	// results: a disappearance.
	if countTitled(deps, "NS Failure: ") == 0 {
		t.Error("expected notification for NS disappearance")
	}

	// A real nameserver was recorded as failed but answers in
	// live results: a recovery.
	if countTitled(deps, "NS Recovery: ") == 0 {
		t.Error("expected notification for NS recovery")
	}
}

// tamperNSBaseline rewrites the recorded hostname state so that
// one real (live-observed) nameserver is marked as failed and a
// synthetic nameserver is recorded as healthy. The next live run
// must report the synthetic one as disappeared and the real one
// as recovered.
func tamperNSBaseline(t *testing.T, deps *testDeps) {
	t.Helper()

	hs, ok := deps.state.GetHostnameState(testHostname)
	if !ok || len(hs.RecordsByNameserver) == 0 {
		t.Fatal("expected hostname state after first live run")
	}

	tampered := &state.HostnameState{
		RecordsByNameserver: make(
			map[string]*state.NameserverRecordState,
		),
		LastChecked: hs.LastChecked,
	}

	realNS := make([]string, 0, len(hs.RecordsByNameserver))

	for ns, rec := range hs.RecordsByNameserver {
		tampered.RecordsByNameserver[ns] = rec
		realNS = append(realNS, ns)
	}

	sort.Strings(realNS)

	failed := realNS[0]
	tampered.RecordsByNameserver[failed] = &state.NameserverRecordState{
		Records:     hs.RecordsByNameserver[failed].Records,
		Status:      "error",
		Error:       "synthetic baseline failure",
		LastChecked: time.Now().UTC(),
	}

	tampered.RecordsByNameserver[fakeNS] = &state.NameserverRecordState{
		Records:     map[string][]string{"A": {fakeIP}},
		Status:      "ok",
		LastChecked: time.Now().UTC(),
	}

	deps.state.SetHostnameState(testHostname, tampered)
}