watcher: notify NS query failure and recovery (closes #104)
check / check (push) Successful in 1m37s
check / check (push) Successful in 1m37s
LookupAllRecords now returns each nameserver's response, so the watcher saves its status: ok when it answered, NXDOMAIN and no records included, and error with the reason when it timed out, answered SERVFAIL or REFUSED, or could not be reached. A nameserver that starts failing sends NS Failure and one that answers again sends NS Recovery. A failing nameserver is left out of the record change and inconsistency comparisons. The resolver used to report REFUSED and network errors as an answer with no records; they are now errors. A lookup cut short by its context now returns an error instead of a failure of the nameserver it was querying. Model: opus-5-5
This commit is contained in:
@@ -0,0 +1,333 @@
|
||||
package watcher_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"sneak.berlin/go/dnswatcher/internal/livednstest"
|
||||
"sneak.berlin/go/dnswatcher/internal/resolver"
|
||||
"sneak.berlin/go/dnswatcher/internal/state"
|
||||
"sneak.berlin/go/dnswatcher/internal/watcher"
|
||||
)
|
||||
|
||||
// answered is what a check saves for a nameserver that answered with
|
||||
// these records.
|
||||
func answered(records map[string][]string) *state.NameserverRecordState {
|
||||
return &state.NameserverRecordState{Records: records, Status: "ok"}
|
||||
}
|
||||
|
||||
// failed is what a check saves for a nameserver that did not answer.
|
||||
func failed() *state.NameserverRecordState {
|
||||
return &state.NameserverRecordState{
|
||||
Records: map[string][]string{},
|
||||
Status: "error",
|
||||
Error: "all queries timed out",
|
||||
}
|
||||
}
|
||||
|
||||
// saved builds the hostname state a check saves.
|
||||
func saved(
|
||||
byNameserver map[string]*state.NameserverRecordState,
|
||||
) *state.HostnameState {
|
||||
return &state.HostnameState{RecordsByNameserver: byNameserver}
|
||||
}
|
||||
|
||||
// alertCounts counts the hostname alerts sent, by kind.
|
||||
type alertCounts struct {
|
||||
failures, recoveries, recordChanges, inconsistencies int
|
||||
}
|
||||
|
||||
// countAlerts runs the hostname change detection from the state loaded
|
||||
// at startup through each check in turn, and counts the alerts sent.
|
||||
func countAlerts(
|
||||
t *testing.T,
|
||||
loaded *state.HostnameState,
|
||||
checks []*state.HostnameState,
|
||||
) alertCounts {
|
||||
t.Helper()
|
||||
|
||||
// The hostname change detection uses only the notifier.
|
||||
notifier := &mockNotifier{}
|
||||
w := watcher.NewForTest(nil, nil, nil, nil, nil, notifier)
|
||||
|
||||
prev := loaded
|
||||
|
||||
for _, current := range checks {
|
||||
w.DetectHostnameChanges(t.Context(), host, prev, current)
|
||||
prev = current
|
||||
}
|
||||
|
||||
var got alertCounts
|
||||
|
||||
for _, n := range notifier.getNotifications() {
|
||||
kind, _, _ := strings.Cut(n.Title, ":")
|
||||
|
||||
switch kind {
|
||||
case "NS Failure":
|
||||
got.failures++
|
||||
case "NS Recovery":
|
||||
got.recoveries++
|
||||
case "Record Change":
|
||||
got.recordChanges++
|
||||
case "Inconsistency":
|
||||
got.inconsistencies++
|
||||
}
|
||||
}
|
||||
|
||||
return got
|
||||
}
|
||||
|
||||
func TestNSFailureAndRecoveryAlerts(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
records := map[string][]string{"A": {ip1}}
|
||||
|
||||
bothAnswer := saved(map[string]*state.NameserverRecordState{
|
||||
nsA: answered(records), nsB: answered(records),
|
||||
})
|
||||
bFails := saved(map[string]*state.NameserverRecordState{
|
||||
nsA: answered(records), nsB: failed(),
|
||||
})
|
||||
onlyA := saved(map[string]*state.NameserverRecordState{
|
||||
nsA: answered(records),
|
||||
})
|
||||
bAnswersNoRecords := saved(map[string]*state.NameserverRecordState{
|
||||
nsA: answered(records), nsB: answered(map[string][]string{}),
|
||||
})
|
||||
bAnswersDifferently := saved(map[string]*state.NameserverRecordState{
|
||||
nsA: answered(records), nsB: answered(map[string][]string{"A": {ip2}}),
|
||||
})
|
||||
|
||||
// Each case starts from the state loaded at startup and runs the
|
||||
// checks in order.
|
||||
tests := []struct {
|
||||
name string
|
||||
loaded *state.HostnameState
|
||||
checks []*state.HostnameState
|
||||
want alertCounts
|
||||
}{
|
||||
{
|
||||
"failure lasting several checks alerts once",
|
||||
bothAnswer, []*state.HostnameState{bFails, bFails, bFails},
|
||||
alertCounts{failures: 1},
|
||||
},
|
||||
{
|
||||
"recovery alerts once",
|
||||
bFails, []*state.HostnameState{bothAnswer, bothAnswer},
|
||||
alertCounts{recoveries: 1},
|
||||
},
|
||||
{
|
||||
"failing again after recovering alerts again",
|
||||
bothAnswer, []*state.HostnameState{bFails, bothAnswer, bFails},
|
||||
alertCounts{failures: 2, recoveries: 1},
|
||||
},
|
||||
{
|
||||
"nameserver failing when first seen does not alert",
|
||||
onlyA, []*state.HostnameState{bFails, bFails},
|
||||
alertCounts{},
|
||||
},
|
||||
{
|
||||
"answer with no records is a record change, not a failure",
|
||||
bothAnswer, []*state.HostnameState{bAnswersNoRecords},
|
||||
alertCounts{recordChanges: 1, inconsistencies: 1},
|
||||
},
|
||||
{
|
||||
"recovered nameserver that answers differently disagrees",
|
||||
bFails, []*state.HostnameState{bAnswersDifferently},
|
||||
alertCounts{recoveries: 1, inconsistencies: 1},
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
got := countAlerts(t, tt.loaded, tt.checks)
|
||||
if got != tt.want {
|
||||
t.Errorf("sent %+v, want %+v", got, tt.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestNSFailureAlertNamesHostnameNameserverAndReason(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
records := map[string][]string{"A": {ip1}}
|
||||
|
||||
notifier := &mockNotifier{}
|
||||
w := watcher.NewForTest(nil, nil, nil, nil, nil, notifier)
|
||||
|
||||
w.DetectHostnameChanges(
|
||||
t.Context(), host,
|
||||
saved(map[string]*state.NameserverRecordState{nsA: answered(records)}),
|
||||
saved(map[string]*state.NameserverRecordState{nsA: failed()}),
|
||||
)
|
||||
|
||||
notifications := notifier.getNotifications()
|
||||
if len(notifications) != 1 {
|
||||
t.Fatalf("sent %v, want one NS Failure", notifications)
|
||||
}
|
||||
|
||||
msg := notifications[0].Message
|
||||
if !strings.Contains(msg, host) || !strings.Contains(msg, nsA) ||
|
||||
!strings.Contains(msg, failed().Error) {
|
||||
t.Errorf(
|
||||
"message %q does not name %s, %s and the reason",
|
||||
msg, host, nsA,
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
// TestNameserverThatNeverAnswers asks a nameserver address where
|
||||
// nothing answers, 192.0.2.1, and checks what the watcher saves for it.
|
||||
// The deadline outlasts the resolver's first two-second try, as in the
|
||||
// resolver's timeout test.
|
||||
func TestNameserverThatNeverAnswers(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
ctx, cancel := context.WithTimeout(t.Context(), 3*time.Second)
|
||||
t.Cleanup(cancel)
|
||||
|
||||
res := resolver.NewFromLogger(slog.Default())
|
||||
|
||||
resp, err := res.QueryNameserverIP(ctx, nsA, "192.0.2.1", host)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
hs := watcher.BuildHostnameState(
|
||||
map[string]*resolver.NameserverResponse{nsA: resp}, time.Now(),
|
||||
)
|
||||
|
||||
got := hs.RecordsByNameserver[nsA]
|
||||
if got.Status != "error" || got.Error == "" {
|
||||
t.Errorf(
|
||||
"saved status %q, error %q; want status error with a reason",
|
||||
got.Status, got.Error,
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
// TestNameserverThatAnswersNXDOMAIN asks a real nameserver about a name
|
||||
// that does not exist and checks what the watcher saves for it: NXDOMAIN
|
||||
// is an answer, so the nameserver is saved as ok with no error.
|
||||
func TestNameserverThatAnswersNXDOMAIN(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
res := resolver.NewFromLogger(slog.Default())
|
||||
name := "this-surely-does-not-exist-xyz." + testDomain
|
||||
|
||||
var (
|
||||
ns string
|
||||
resp *resolver.NameserverResponse
|
||||
)
|
||||
|
||||
livednstest.Retry(t, "QueryNameserver("+name+")", func(ctx context.Context) error {
|
||||
nameservers, err := res.LookupNS(ctx, testDomain)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
ns = nameservers[0]
|
||||
|
||||
resp, err = res.QueryNameserver(ctx, ns, name)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// A timeout or a failure is no answer to check.
|
||||
if resp.Status == resolver.StatusTimeout ||
|
||||
resp.Status == resolver.StatusError {
|
||||
return fmt.Errorf(
|
||||
"%w: %s: %s", livednstest.ErrNoAnswer, ns, resp.Error,
|
||||
)
|
||||
}
|
||||
|
||||
return nil
|
||||
})
|
||||
|
||||
if resp.Status != resolver.StatusNXDomain {
|
||||
t.Fatalf("%s answered %q for %s, want NXDOMAIN", ns, resp.Status, name)
|
||||
}
|
||||
|
||||
hs := watcher.BuildHostnameState(
|
||||
map[string]*resolver.NameserverResponse{ns: resp}, time.Now(),
|
||||
)
|
||||
|
||||
got := hs.RecordsByNameserver[ns]
|
||||
if got.Status != "ok" || got.Error != "" {
|
||||
t.Errorf(
|
||||
"saved status %q, error %q; want status ok with no error",
|
||||
got.Status, got.Error,
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
// TestNameserverThatRefuses asks a google.com nameserver about
|
||||
// cloudflare.com, a zone it does not serve, which it refuses, and checks
|
||||
// what the watcher saves for it: REFUSED is no answer, so the nameserver
|
||||
// is saved as error with the reason.
|
||||
func TestNameserverThatRefuses(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
const reason = "server returned REFUSED"
|
||||
|
||||
res := resolver.NewFromLogger(slog.Default())
|
||||
|
||||
var (
|
||||
ns string
|
||||
resp *resolver.NameserverResponse
|
||||
)
|
||||
|
||||
livednstest.Retry(
|
||||
t,
|
||||
"QueryNameserver(cloudflare.com)",
|
||||
func(ctx context.Context) error {
|
||||
nameservers, err := res.LookupNS(ctx, testDomain)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
ns = nameservers[0]
|
||||
|
||||
resp, err = res.QueryNameserver(ctx, ns, "cloudflare.com")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// A timeout or a network error is no reply at all.
|
||||
if resp.Status == resolver.StatusTimeout ||
|
||||
strings.HasPrefix(resp.Error, "network error") {
|
||||
return fmt.Errorf(
|
||||
"%w: %s: %s", livednstest.ErrNoAnswer, ns, resp.Error,
|
||||
)
|
||||
}
|
||||
|
||||
return nil
|
||||
},
|
||||
)
|
||||
|
||||
if resp.Error != reason {
|
||||
t.Fatalf(
|
||||
"%s answered %q (%s) for cloudflare.com, want REFUSED",
|
||||
ns, resp.Status, resp.Error,
|
||||
)
|
||||
}
|
||||
|
||||
hs := watcher.BuildHostnameState(
|
||||
map[string]*resolver.NameserverResponse{ns: resp}, time.Now(),
|
||||
)
|
||||
|
||||
got := hs.RecordsByNameserver[ns]
|
||||
if got.Status != failed().Status || got.Error != reason {
|
||||
t.Errorf(
|
||||
"saved status %q, error %q; want status %q, error %q",
|
||||
got.Status, got.Error, failed().Status, reason,
|
||||
)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user