dnswatcher/internal/watcher/nsfailure_test.go

package watcher_test

import (
	"context"
	"errors"
	"fmt"
	"log/slog"
	"strings"
	"testing"
	"time"

	"sneak.berlin/go/dnswatcher/internal/livednstest"
	"sneak.berlin/go/dnswatcher/internal/resolver"
	"sneak.berlin/go/dnswatcher/internal/state"
	"sneak.berlin/go/dnswatcher/internal/watcher"
)

// errNoReply is why a walk that got no reply from a zone's servers
// failed.
var errNoReply = errors.New("all servers failed: i/o timeout")

// answered is what a check saves for a nameserver that answered with
// these records.
func answered(records map[string][]string) *state.NameserverRecordState {
	return &state.NameserverRecordState{Records: records, Status: "ok"}
}

// failed is what a check saves for a nameserver that did not answer.
func failed() *state.NameserverRecordState {
	return &state.NameserverRecordState{
		Records: map[string][]string{},
		Status:  "error",
		Error:   "all queries timed out",
	}
}

// saved builds the hostname state a check saves.
func saved(
	byNameserver map[string]*state.NameserverRecordState,
) *state.HostnameState {
	return &state.HostnameState{RecordsByNameserver: byNameserver}
}

// alertCounts counts the hostname alerts sent, by kind.
type alertCounts struct {
	failures, recoveries, recordChanges, inconsistencies int
}

// countAlerts runs the hostname change detection from the state loaded
// at startup through each check in turn, and counts the alerts sent.
func countAlerts(
	t *testing.T,
	loaded *state.HostnameState,
	checks []*state.HostnameState,
) alertCounts {
	t.Helper()

	// The hostname change detection uses only the notifier.
	notifier := &mockNotifier{}
	w := watcher.NewForTest(nil, nil, nil, nil, nil, notifier)

	prev := loaded

	for _, current := range checks {
		w.DetectHostnameChanges(t.Context(), host, prev, current)
		prev = current
	}

	var got alertCounts

	for _, n := range notifier.getNotifications() {
		kind, _, _ := strings.Cut(n.Title, ":")

		switch kind {
		case "NS Failure":
			got.failures++
		case "NS Recovery":
			got.recoveries++
		case "Record Change":
			got.recordChanges++
		case "Inconsistency":
			got.inconsistencies++
		}
	}

	return got
}

func TestNSFailureAndRecoveryAlerts(t *testing.T) {
	t.Parallel()

	records := map[string][]string{"A": {ip1}}

	bothAnswer := saved(map[string]*state.NameserverRecordState{
		nsA: answered(records), nsB: answered(records),
	})
	bFails := saved(map[string]*state.NameserverRecordState{
		nsA: answered(records), nsB: failed(),
	})
	onlyA := saved(map[string]*state.NameserverRecordState{
		nsA: answered(records),
	})
	bAnswersNoRecords := saved(map[string]*state.NameserverRecordState{
		nsA: answered(records), nsB: answered(map[string][]string{}),
	})
	bAnswersDifferently := saved(map[string]*state.NameserverRecordState{
		nsA: answered(records), nsB: answered(map[string][]string{"A": {ip2}}),
	})

	// Each case starts from the state loaded at startup and runs the
	// checks in order.
	tests := []struct {
		name   string
		loaded *state.HostnameState
		checks []*state.HostnameState
		want   alertCounts
	}{
		{
			"failure lasting several checks alerts once",
			bothAnswer, []*state.HostnameState{bFails, bFails, bFails},
			alertCounts{failures: 1},
		},
		{
			"recovery alerts once",
			bFails, []*state.HostnameState{bothAnswer, bothAnswer},
			alertCounts{recoveries: 1},
		},
		{
			"failing again after recovering alerts again",
			bothAnswer, []*state.HostnameState{bFails, bothAnswer, bFails},
			alertCounts{failures: 2, recoveries: 1},
		},
		{
			"nameserver failing when first seen does not alert",
			onlyA, []*state.HostnameState{bFails, bFails},
			alertCounts{},
		},
		{
			"answer with no records is a record change, not a failure",
			bothAnswer, []*state.HostnameState{bAnswersNoRecords},
			alertCounts{recordChanges: 1, inconsistencies: 1},
		},
		{
			"recovered nameserver that answers differently disagrees",
			bFails, []*state.HostnameState{bAnswersDifferently},
			alertCounts{recoveries: 1, inconsistencies: 1},
		},
	}

	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
			t.Parallel()

			got := countAlerts(t, tt.loaded, tt.checks)
			if got != tt.want {
				t.Errorf("sent %+v, want %+v", got, tt.want)
			}
		})
	}
}

func TestNSFailureAlertNamesHostnameNameserverAndReason(t *testing.T) {
	t.Parallel()

	records := map[string][]string{"A": {ip1}}

	notifier := &mockNotifier{}
	w := watcher.NewForTest(nil, nil, nil, nil, nil, notifier)

	w.DetectHostnameChanges(
		t.Context(), host,
		saved(map[string]*state.NameserverRecordState{nsA: answered(records)}),
		saved(map[string]*state.NameserverRecordState{nsA: failed()}),
	)

	notifications := notifier.getNotifications()
	if len(notifications) != 1 {
		t.Fatalf("sent %v, want one NS Failure", notifications)
	}

	msg := notifications[0].Message
	if !strings.Contains(msg, host) || !strings.Contains(msg, nsA) ||
		!strings.Contains(msg, failed().Error) {
		t.Errorf(
			"message %q does not name %s, %s and the reason",
			msg, host, nsA,
		)
	}
}

// TestZoneNameserversAllStopAnswering runs three checks of host from
// the point where its lookup has returned. On the first, nsA and nsB,
// the nameservers of its zone, answer. On the second, the lookup fails
// with the error the walk to them returns when neither answers (see
// the resolver's TestFollowDelegation_NoAnswerAfterReferral), given
// here because no live zone has nameservers that never answer. For a
// hostname it is the lookup of its records that fails, and for a domain
// with no delegation of its own the lookup of its delegation. Both are
// saved as failed, with the reason, and each gets an NS Failure. On the
// third, both answer, and each gets an NS Recovery. A domain's records
// are checked as a hostname's, so its first and third checks are too.
func TestZoneNameserversAllStopAnswering(t *testing.T) {
	t.Parallel()

	records := map[string][]string{"A": {ip1}}
	bothAnswer := map[string]*resolver.NameserverResponse{
		nsA: response(records), nsB: response(records),
	}
	noAnswer := &resolver.NoAnswerError{
		Nameservers: []string{nsA, nsB}, Err: errNoReply,
	}

	tests := []struct {
		name     string
		isDomain bool
	}{
		{"hostname", false},
		{"domain with no delegation of its own", true},
	}

	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
			t.Parallel()

			cfg := defaultTestConfig(t)
			if tt.isDomain {
				cfg.Domains = []string{host}
			} else {
				cfg.Hostnames = []string{host}
			}

			// The watcher is given each lookup's result, so it has no
			// resolver.
			deps := newTestDeps(t, cfg)
			w := watcher.NewForTest(
				cfg, deps.state, nil,
				deps.portChecker, deps.tlsChecker, deps.notifier,
			)
			w.SetFirstRun(false)

			w.CheckHostnameResult(t.Context(), host, bothAnswer, nil)

			if tt.isDomain {
				w.CheckDomainResult(t.Context(), host, nil, noAnswer)
			} else {
				w.CheckHostnameResult(t.Context(), host, nil, noAnswer)
			}

			current, _ := deps.state.GetHostnameState(host)

			for _, ns := range []string{nsA, nsB} {
				got := current.RecordsByNameserver[ns]
				if got.Status != failed().Status || got.Error != errNoReply.Error() {
					t.Errorf(
						"%s saved with status %q, error %q; want %q, %q",
						ns, got.Status, got.Error, failed().Status, errNoReply,
					)
				}
			}

			if n := countNotifications(deps, "NS Failure: "+host); n != 2 {
				t.Errorf("sent %d NS Failure notifications, want 2", n)
			}

			w.CheckHostnameResult(t.Context(), host, bothAnswer, nil)

			if n := countNotifications(deps, "NS Recovery: "+host); n != 2 {
				t.Errorf("sent %d NS Recovery notifications, want 2", n)
			}
		})
	}
}

// TestFailedNameserversOnlyForThoseOfThePreviousCheck gives
// FailedNameservers lookup errors after a check in which nsA and nsB
// answered. Only an error that names both makes them failed.
func TestFailedNameserversOnlyForThoseOfThePreviousCheck(t *testing.T) {
	t.Parallel()

	records := map[string][]string{"A": {ip1}}
	bothAnswered := saved(map[string]*state.NameserverRecordState{
		nsA: answered(records), nsB: answered(records),
	})
	noAnswer := func(nameservers ...string) error {
		return &resolver.NoAnswerError{Nameservers: nameservers, Err: errNoReply}
	}

	tests := []struct {
		name       string
		err        error
		prev       *state.HostnameState
		wantFailed bool
	}{
		{"both did not answer", noAnswer(nsA, nsB), bothAnswered, true},
		{
			"the servers of a parent zone did not answer",
			noAnswer(nsC), bothAnswered, false,
		},
		{
			"the zone's delegation lists only nsA now",
			noAnswer(nsA), bothAnswered, false,
		},
		{"no root server answered", errNoReply, bothAnswered, false},
		{"no previous check", noAnswer(nsA, nsB), nil, false},
	}

	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
			t.Parallel()

			got := watcher.FailedNameservers(tt.err, tt.prev)
			if (got != nil) != tt.wantFailed {
				t.Errorf("returned %v, want failed: %v", got, tt.wantFailed)
			}
		})
	}
}

// TestNameserverThatNeverAnswers asks a nameserver address where
// nothing answers, 192.0.2.1, and checks what the watcher saves for it.
// The deadline outlasts the resolver's first two-second try, as in the
// resolver's timeout test.
func TestNameserverThatNeverAnswers(t *testing.T) {
	t.Parallel()

	ctx, cancel := context.WithTimeout(t.Context(), 3*time.Second)
	t.Cleanup(cancel)

	res := resolver.NewFromLogger(slog.Default())

	resp, err := res.QueryNameserverIP(ctx, nsA, "192.0.2.1", host)
	if err != nil {
		t.Fatal(err)
	}

	hs := watcher.BuildHostnameState(
		map[string]*resolver.NameserverResponse{nsA: resp}, nil, time.Now(),
	)

	got := hs.RecordsByNameserver[nsA]
	if got.Status != "error" || got.Error == "" {
		t.Errorf(
			"saved status %q, error %q; want status error with a reason",
			got.Status, got.Error,
		)
	}
}

// TestNameserverThatAnswersNXDOMAIN asks a real nameserver about a name
// that does not exist and checks what the watcher saves for it: NXDOMAIN
// is an answer, so the nameserver is saved as ok with no error.
func TestNameserverThatAnswersNXDOMAIN(t *testing.T) {
	t.Parallel()

	res := resolver.NewFromLogger(slog.Default())
	name := "this-surely-does-not-exist-xyz." + testDomain

	var (
		ns   string
		resp *resolver.NameserverResponse
	)

	livednstest.Retry(t, "QueryNameserver("+name+")", func(ctx context.Context) error {
		nameservers, err := res.LookupNS(ctx, testDomain)
		if err != nil {
			return err
		}

		ns = nameservers[0]

		resp, err = res.QueryNameserver(ctx, ns, name)
		if err != nil {
			return err
		}

		// A timeout or a failure is no answer to check.
		if resp.Status == resolver.StatusTimeout ||
			resp.Status == resolver.StatusError {
			return fmt.Errorf(
				"%w: %s: %s", livednstest.ErrNoAnswer, ns, resp.Error,
			)
		}

		return nil
	})

	if resp.Status != resolver.StatusNXDomain {
		t.Fatalf("%s answered %q for %s, want NXDOMAIN", ns, resp.Status, name)
	}

	hs := watcher.BuildHostnameState(
		map[string]*resolver.NameserverResponse{ns: resp}, nil, time.Now(),
	)

	got := hs.RecordsByNameserver[ns]
	if got.Status != "ok" || got.Error != "" {
		t.Errorf(
			"saved status %q, error %q; want status ok with no error",
			got.Status, got.Error,
		)
	}
}

// TestNameserverThatRefuses asks a google.com nameserver about
// cloudflare.com, a zone it does not serve, which it refuses, and checks
// what the watcher saves for it: REFUSED is no answer, so the nameserver
// is saved as error with the reason.
func TestNameserverThatRefuses(t *testing.T) {
	t.Parallel()

	const reason = "server returned REFUSED"

	res := resolver.NewFromLogger(slog.Default())

	var (
		ns   string
		resp *resolver.NameserverResponse
	)

	livednstest.Retry(
		t,
		"QueryNameserver(cloudflare.com)",
		func(ctx context.Context) error {
			nameservers, err := res.LookupNS(ctx, testDomain)
			if err != nil {
				return err
			}

			ns = nameservers[0]

			resp, err = res.QueryNameserver(ctx, ns, "cloudflare.com")
			if err != nil {
				return err
			}

			// A timeout or a network error is no reply at all.
			if resp.Status == resolver.StatusTimeout ||
				strings.HasPrefix(resp.Error, "network error") {
				return fmt.Errorf(
					"%w: %s: %s", livednstest.ErrNoAnswer, ns, resp.Error,
				)
			}

			return nil
		},
	)

	if resp.Error != reason {
		t.Fatalf(
			"%s answered %q (%s) for cloudflare.com, want REFUSED",
			ns, resp.Status, resp.Error,
		)
	}

	hs := watcher.BuildHostnameState(
		map[string]*resolver.NameserverResponse{ns: resp}, nil, time.Now(),
	)

	got := hs.RecordsByNameserver[ns]
	if got.Status != failed().Status || got.Error != reason {
		t.Errorf(
			"saved status %q, error %q; want status %q, error %q",
			got.Status, got.Error, failed().Status, reason,
		)
	}
}

// TestPortStateWhenNoNameserverAnswered runs the port checks on
// hostname state built here, which gives the name no address. The port
// state saved for its old address is kept only when the name is a
// configured hostname or domain and none of its nameservers answered.
func TestPortStateWhenNoNameserverAnswered(t *testing.T) {
	t.Parallel()

	noneAnswered := saved(map[string]*state.NameserverRecordState{
		nsA: failed(), nsB: failed(),
	})
	oneAnsweredNoAddress := saved(map[string]*state.NameserverRecordState{
		nsA: answered(map[string][]string{}), nsB: failed(),
	})

	configured := []string{host}

	tests := []struct {
		name      string
		hostname  *state.HostnameState
		hostnames []string
		domains   []string
		wantKept  bool
	}{
		{"no nameserver answered", noneAnswered, configured, nil, true},
		{
			"no nameserver answered, configured as a domain",
			noneAnswered, nil, configured, true,
		},
		{
			"one answered with no address",
			oneAnsweredNoAddress, configured, nil, false,
		},
		{"no nameserver answered, not configured", noneAnswered, nil, nil, false},
	}

	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
			t.Parallel()

			cfg := defaultTestConfig(t)
			cfg.Hostnames = tt.hostnames
			cfg.Domains = tt.domains

			// The port checks read the saved hostname state and look
			// nothing up, so the watcher has no resolver.
			deps := newTestDeps(t, cfg)
			w := watcher.NewForTest(
				cfg, deps.state, nil,
				deps.portChecker, deps.tlsChecker, deps.notifier,
			)

			key := ip1 + ":443"

			deps.state.SetHostnameState(host, tt.hostname)
			deps.state.SetPortState(key, &state.PortState{
				Open: true, Hostnames: []string{host},
			})

			w.CheckAllPorts(t.Context())

			_, kept := deps.state.GetPortState(key)
			if kept != tt.wantKept {
				t.Errorf("port state %s kept: %v, want %v", key, kept, tt.wantKept)
			}
		})
	}
}

// TestPortStateWhenNoNameserverAnsweredAndOtherNameMovesAway saves the
// port state of an address two configured hostnames resolve to. While
// none of the first one's nameservers answer, the port checks run with
// the other one still at that address, then after it moved away; the
// port state is kept both times.
func TestPortStateWhenNoNameserverAnsweredAndOtherNameMovesAway(
	t *testing.T,
) {
	t.Parallel()

	const other = "mail.example.net"

	cfg := defaultTestConfig(t)
	cfg.Hostnames = []string{host, other}

	// The port checks read the saved hostname state and look nothing
	// up, so the watcher has no resolver.
	deps := newTestDeps(t, cfg)
	w := watcher.NewForTest(
		cfg, deps.state, nil,
		deps.portChecker, deps.tlsChecker, deps.notifier,
	)

	key := ip1 + ":443"

	deps.state.SetPortState(key, &state.PortState{
		Open: true, Hostnames: []string{host, other},
	})
	deps.state.SetHostnameState(host, saved(
		map[string]*state.NameserverRecordState{nsA: failed(), nsB: failed()},
	))

	for _, otherIP := range []string{ip1, ip2} {
		deps.state.SetHostnameState(other, saved(
			map[string]*state.NameserverRecordState{
				nsA: answered(map[string][]string{"A": {otherIP}}),
			},
		))

		w.CheckAllPorts(t.Context())

		if _, kept := deps.state.GetPortState(key); !kept {
			t.Fatalf("port state %s removed with %s at %s", key, other, otherIP)
		}
	}
}