resolver, watcher: a record type whose query fails keeps its previous records (closes #231)

The resolver lists in FailedTypes each record type whose query to a
nameserver got no usable reply (no reply, a code other than NOERROR or
NXDOMAIN, a referral, or a truncated reply whose TCP retry failed) and
logs it unless shutdown cut it short. A nameserver that answered no
type has failed.
The watcher saves such a type in failedTypes with the previous check's
records, leaves it out of the comparison with other nameservers on that
check, and compares it with the next answer. When the previous check
did not know its records either, it is also in unknownTypes and not
compared until it answers. A nameserver whose A, AAAA or CNAME query
failed is no answer when following a CNAME or resolving addresses.

Model: opus-5-5

internal/resolver/errors.go

@@ -22,6 +22,12 @@ var (
 		"reply is an error or a referral that leads no closer",
 	)
 
+	// ErrTruncated is the reason given for a reply too large for UDP
+	// whose retry over TCP failed.
+	ErrTruncated = errors.New(
+		"reply truncated and its retry over TCP failed",
+	)
+
 	// ErrIntercepted is returned when every root server refused a
 	// query. Root servers refuse no query, so the refusals came from
 	// something on the network answering in their place.

internal/resolver/export_test.go

@@ -2,10 +2,21 @@ package resolver
 
 import (
 	"context"
+	"log/slog"
+	"time"
 
 	"github.com/miekg/dns"
 )
 
+// NewWithFailingTCP returns a Resolver whose TCP client gives up before
+// it can connect, so the retry over TCP of every truncated reply fails.
+func NewWithFailingTCP(log *slog.Logger) *Resolver {
+	r := NewFromLogger(log)
+	r.tcp = &tcpClient{timeout: time.Nanosecond}
+
+	return r
+}
+
 // ExtractRecordValue exports extractRecordValue for testing.
 func ExtractRecordValue(rr dns.RR) string {
 	return extractRecordValue(rr)

internal/resolver/iterative.go

@@ -8,6 +8,7 @@ import (
 	"net"
 	"slices"
 	"sort"
+	"strconv"
 	"strings"
 	"time"
 
@@ -99,6 +100,9 @@ func (r *Resolver) tryExchange(
 	return resp, err
 }
 
+// retryTCP returns the reply to msg over TCP when resp, its reply over
+// UDP, is truncated. When that fails it returns resp, still truncated,
+// which holds only the records that fit.
 func (r *Resolver) retryTCP(
 	ctx context.Context,
 	msg *dns.Msg,
@@ -320,13 +324,20 @@ func (r *Resolver) queryServers(
 	return nil, fmt.Errorf("all servers failed: %w", lastErr)
 }
 
+// isErrorReply reports whether msg is an error reply: one with any code
+// but NOERROR and NXDOMAIN, such as SERVFAIL, NOTIMP or FORMERR. An error
+// reply says nothing about the name's records.
+func isErrorReply(msg *dns.Msg) bool {
+	return msg.Rcode != dns.RcodeSuccess && msg.Rcode != dns.RcodeNameError
+}
+
 // usableReply reports whether resp, a reply from one of the servers of
 // zone to a query about name, is usable. An error reply such as SERVFAIL
 // is not. Nor is a referral, unless it refers the query to a zone below
 // zone that name is in: a server that refers it back to zone, up or
 // sideways does not serve zone as it should.
 func usableReply(resp *dns.Msg, zone string, name string) bool {
-	if resp.Rcode != dns.RcodeSuccess && resp.Rcode != dns.RcodeNameError {
+	if isErrorReply(resp) {
 		return false
 	}
 
@@ -714,15 +725,21 @@ func (r *Resolver) queryTypes(
 }
 
 type queryState struct {
-	gotNXDomain bool
-	gotSERVFAIL bool
-	gotRefused  bool
-	gotTimeout  bool
-	gotReferral bool
-	netErr      error
-	hasRecords  bool
+	gotNXDomain   bool
+	gotErrorReply bool
+	errorReply    string // its code, such as SERVFAIL, or number if unnamed
+	gotRefused    bool
+	gotTimeout    bool
+	gotReferral   bool
+	netErr        error
+	hasRecords    bool
+	answered      bool
 }
 
+// queryEachType asks the nameserver at nsIP about hostname once for each
+// record type in qtypes, and lists in resp.FailedTypes the types whose
+// query got no usable reply, logging each with the reason unless ctx was
+// cancelled: shutdown cancels it, and a query it cut short did not fail.
 func (r *Resolver) queryEachType(
 	ctx context.Context,
 	nsIP string,
@@ -737,7 +754,34 @@ func (r *Resolver) queryEachType(
 			break
 		}
 
-		r.querySingleType(ctx, nsIP, hostname, qtype, resp, &state)
+		err := r.querySingleType(ctx, nsIP, hostname, qtype, resp, &state)
+		if err == nil {
+			state.answered = true
+
+			continue
+		}
+
+		rtype := dns.TypeToString[qtype]
+		resp.FailedTypes = append(resp.FailedTypes, rtype)
+
+		if errors.Is(ctx.Err(), context.Canceled) {
+			continue
+		}
+
+		r.log.Warn(
+			"record type query failed",
+			"hostname", hostname,
+			"nameserver", resp.Nameserver,
+			"type", rtype,
+			"error", err,
+		)
+	}
+
+	// The reply about another type can carry the name's CNAME. When the
+	// query for CNAME itself failed, that is left out too, so Records
+	// holds nothing for a failed type.
+	for _, rtype := range resp.FailedTypes {
+		delete(resp.Records, rtype)
 	}
 
 	for k := range resp.Records {
@@ -747,6 +791,9 @@ func (r *Resolver) queryEachType(
 	return state
 }
 
+// querySingleType asks the nameserver at nsIP about hostname's records
+// of type qtype. It returns nil when the nameserver answered: with
+// records, with none, or with NXDOMAIN; otherwise it returns why not.
 func (r *Resolver) querySingleType(
 	ctx context.Context,
 	nsIP string,
@@ -754,7 +801,7 @@ func (r *Resolver) querySingleType(
 	qtype uint16,
 	resp *NameserverResponse,
 	state *queryState,
-) {
+) error {
 	msg, err := r.queryDNS(ctx, nsIP, hostname, qtype)
 	if err != nil {
 		switch {
@@ -766,19 +813,40 @@ func (r *Resolver) querySingleType(
 			state.netErr = err
 		}
 
-		return
+		return err
 	}
 
+	return readReply(msg, resp, state)
+}
+
+// readReply adds to resp the records in msg, a nameserver's reply to a
+// query about one record type. It returns nil when the nameserver
+// answered: with records, with none, or with NXDOMAIN; otherwise it
+// returns why not.
+func readReply(
+	msg *dns.Msg,
+	resp *NameserverResponse,
+	state *queryState,
+) error {
 	if msg.Rcode == dns.RcodeNameError {
 		state.gotNXDomain = true
 
-		return
+		return nil
 	}
 
-	if msg.Rcode == dns.RcodeServerFailure {
-		state.gotSERVFAIL = true
+	if isErrorReply(msg) {
+		state.gotErrorReply = true
 
-		return
+		code, named := dns.RcodeToString[msg.Rcode]
+		if !named {
+			code = strconv.Itoa(msg.Rcode)
+		}
+
+		state.errorReply = code
+
+		return fmt.Errorf(
+			"server returned %s: %w", state.errorReply, ErrUnusableReply,
+		)
 	}
 
 	// A reply with no answer that lists other nameservers, from a server
@@ -791,10 +859,20 @@ func (r *Resolver) querySingleType(
 		len(extractNSSet(msg.Ns)) > 0 {
 		state.gotReferral = true
 
-		return
+		return fmt.Errorf("server returned a referral: %w", ErrUnusableReply)
+	}
+
+	// A reply still truncated is one whose TCP retry failed, and holds
+	// only the records that fit.
+	if msg.Truncated {
+		state.netErr = ErrTruncated
+
+		return ErrTruncated
 	}
 
 	collectAnswerRecords(msg, resp, state)
+
+	return nil
 }
 
 // collectAnswerRecords adds the records in msg's answer to resp, each
@@ -833,23 +911,26 @@ func isTimeout(err error) bool {
 	return false
 }
 
+// classifyResponse sets the nameserver's status. One that answered no
+// record type has failed, and Error says why; one that answered some has
+// the status of those answers.
 func classifyResponse(resp *NameserverResponse, state queryState) {
 	switch {
 	case state.gotNXDomain && !state.hasRecords:
 		resp.Status = StatusNXDomain
-	case state.gotTimeout && !state.hasRecords:
+	case state.gotTimeout && !state.answered:
 		resp.Status = StatusTimeout
 		resp.Error = "all queries timed out"
-	case state.gotSERVFAIL && !state.hasRecords:
+	case state.gotErrorReply && !state.answered:
 		resp.Status = StatusError
-		resp.Error = "server returned SERVFAIL"
-	case state.gotRefused && !state.hasRecords:
+		resp.Error = "server returned " + state.errorReply
+	case state.gotRefused && !state.answered:
 		resp.Status = StatusError
 		resp.Error = "server returned REFUSED"
-	case state.netErr != nil && !state.hasRecords:
+	case state.netErr != nil && !state.answered:
 		resp.Status = StatusError
 		resp.Error = "network error: " + state.netErr.Error()
-	case state.gotReferral && !state.hasRecords:
+	case state.gotReferral && !state.answered:
 		resp.Status = StatusError
 		resp.Error = "server returned a referral"
 	case !state.hasRecords && !state.gotNXDomain:
@@ -1010,9 +1091,11 @@ func (r *Resolver) resolveIPWithCNAME(
 }
 
 // collectIPs returns the addresses in the nameservers' answers and the
-// first CNAME target among them. It returns ErrNoNameserverAnswered when
-// every nameserver timed out, failed or returned a referral: that is not
-// a name with no addresses.
+// first CNAME target among them. A nameserver whose query for one of the
+// types failed gave only part of the addresses, and is left out. It
+// returns ErrNoNameserverAnswered when every nameserver timed out,
+// failed, returned a referral or was left out: that is not a name with
+// no addresses.
 func collectIPs(
 	results map[string]*NameserverResponse,
 ) ([]string, string, error) {
@@ -1025,7 +1108,8 @@ func collectIPs(
 	answered := false
 
 	for _, resp := range results {
-		if resp.Status == StatusTimeout || resp.Status == StatusError {
+		if resp.Status == StatusTimeout || resp.Status == StatusError ||
+			len(resp.FailedTypes) > 0 {
 			continue
 		}
 

internal/resolver/iterative_internal_test.go

@@ -0,0 +1,131 @@
+package resolver
+
+import (
+	"strconv"
+	"syscall"
+	"testing"
+
+	"github.com/miekg/dns"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// TestClassifyResponse sets a nameserver's status from the results of
+// its queries, built here. One that answered some record types, even
+// with no records, has not failed when its query for another type got
+// no usable reply, whatever the reason; one whose every query got none
+// has.
+func TestClassifyResponse(t *testing.T) {
+	t.Parallel()
+
+	tests := []struct {
+		name       string
+		results    queryState
+		wantStatus string
+		wantError  string
+	}{
+		{
+			"some types answered with no records, another timed out",
+			queryState{answered: true, gotTimeout: true},
+			StatusNoData, "",
+		},
+		{
+			"some types answered with no records, another got SERVFAIL",
+			queryState{
+				answered: true, gotErrorReply: true, errorReply: "SERVFAIL",
+			},
+			StatusNoData, "",
+		},
+		{
+			"some types answered with no records, another was refused",
+			queryState{answered: true, gotRefused: true},
+			StatusNoData, "",
+		},
+		{
+			"some types answered with no records, another got a network error",
+			queryState{answered: true, netErr: syscall.ECONNREFUSED},
+			StatusNoData, "",
+		},
+		{
+			"some types answered with no records, another's reply was " +
+				"truncated and its retry over TCP failed",
+			queryState{answered: true, netErr: ErrTruncated},
+			StatusNoData, "",
+		},
+		{
+			"some types answered with no records, another got a referral",
+			queryState{answered: true, gotReferral: true},
+			StatusNoData, "",
+		},
+		{
+			"every query timed out",
+			queryState{gotTimeout: true},
+			StatusTimeout, "all queries timed out",
+		},
+		{
+			"every query got NOTIMP",
+			queryState{gotErrorReply: true, errorReply: "NOTIMP"},
+			StatusError, "server returned NOTIMP",
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			t.Parallel()
+
+			resp := &NameserverResponse{Status: StatusOK}
+			classifyResponse(resp, tt.results)
+
+			assert.Equal(t, tt.wantStatus, resp.Status)
+			assert.Equal(t, tt.wantError, resp.Error)
+		})
+	}
+}
+
+// TestReadReply checks which replies to a query about one record type,
+// built here, are an answer: one with the code NOERROR or NXDOMAIN. A
+// reply with any other code is not, and the type's query has failed; a
+// nameserver whose only reply it is has failed, and Error gives the
+// code, or its number when the code has no name.
+func TestReadReply(t *testing.T) {
+	t.Parallel()
+
+	tests := []struct {
+		rcode      int
+		wantStatus string
+		wantError  string
+	}{
+		{dns.RcodeSuccess, StatusNoData, ""},
+		{dns.RcodeNameError, StatusNXDomain, ""},
+		{dns.RcodeServerFailure, StatusError, "server returned SERVFAIL"},
+		{dns.RcodeNotImplemented, StatusError, "server returned NOTIMP"},
+		{dns.RcodeFormatError, StatusError, "server returned FORMERR"},
+		{12, StatusError, "server returned 12"}, // unassigned, no name
+	}
+
+	for _, tt := range tests {
+		t.Run(strconv.Itoa(tt.rcode), func(t *testing.T) {
+			t.Parallel()
+
+			msg := new(dns.Msg)
+			msg.Authoritative = true
+			msg.Rcode = tt.rcode
+
+			resp := &NameserverResponse{Records: map[string][]string{}}
+
+			var state queryState
+
+			err := readReply(msg, resp, &state)
+			classifyResponse(resp, state)
+
+			if tt.wantStatus == StatusError {
+				require.ErrorIs(t, err, ErrUnusableReply)
+			} else {
+				require.NoError(t, err)
+			}
+
+			assert.Equal(t, tt.wantStatus, resp.Status)
+			assert.Equal(t, tt.wantError, resp.Error)
+		})
+	}
+}

internal/resolver/iterative_test.go

@@ -43,6 +43,25 @@ func TestCollectIPs_FailedIsNoAnswer(t *testing.T) {
 	assert.Empty(t, ips)
 }
 
+// TestCollectIPs_FailedTypeIsNoAnswer checks that a nameserver whose
+// query for one of the types failed is no answer: its addresses are
+// only part of them.
+func TestCollectIPs_FailedTypeIsNoAnswer(t *testing.T) {
+	t.Parallel()
+
+	ips, _, err := resolver.CollectIPs(
+		map[string]*resolver.NameserverResponse{
+			nsExample1: {
+				Records:     map[string][]string{"A": {"192.0.2.1"}},
+				FailedTypes: []string{"AAAA"},
+				Status:      resolver.StatusOK,
+			},
+		},
+	)
+	require.ErrorIs(t, err, resolver.ErrNoNameserverAnswered)
+	assert.Empty(t, ips)
+}
+
 const (
 	// exampleCom is the zone most cases of TestUsableReply and
 	// TestNSSetFrom are about, and wwwExampleCom a name in it.

internal/resolver/resolver.go

@@ -31,11 +31,15 @@ type Params struct {
 }
 
 // NameserverResponse holds one nameserver's response for a query.
+// FailedTypes lists the record types whose query got no usable reply,
+// and Records holds nothing for them: their records are not known. When
+// no record type got one, Status and Error say the nameserver failed.
 type NameserverResponse struct {
-	Nameserver string
-	Records    map[string][]string
-	Status     string
-	Error      string
+	Nameserver  string
+	Records     map[string][]string
+	FailedTypes []string
+	Status      string
+	Error       string
 }
 
 // Resolver performs iterative DNS resolution from root servers.

internal/resolver/resolver_test.go

@@ -1,6 +1,7 @@
 package resolver_test
 
 import (
+	"bytes"
 	"context"
 	"errors"
 	"fmt"
@@ -366,6 +367,30 @@ func TestQueryNameserver_TXT(t *testing.T) {
 	)
 }
 
+// TestQueryNameserver_TruncatedReplyWhoseTCPRetryFails asks a google.com
+// nameserver about google.com with a resolver whose retries over TCP
+// fail. google.com's TXT records do not fit in a reply over UDP, so TXT
+// is reported as failed, holding none of the records that fit, and
+// logged with the reason, while the nameserver, which answered the other
+// types, is ok.
+func TestQueryNameserver_TruncatedReplyWhoseTCPRetryFails(t *testing.T) {
+	t.Parallel()
+
+	ns := findOneNSForDomain(t, newTestResolver(t), "google.com")
+
+	var logs bytes.Buffer
+
+	r := resolver.NewWithFailingTCP(slog.New(slog.NewTextHandler(&logs, nil)))
+	resp := liveQueryNameserver(t, r, ns, "google.com")
+
+	assert.Equal(t, resolver.StatusOK, resp.Status)
+	assert.Contains(t, resp.FailedTypes, "TXT")
+	assert.NotContains(t, resp.Records, "TXT")
+	assert.Contains(t, logs.String(),
+		"hostname=google.com. nameserver="+ns+" type=TXT error=",
+	)
+}
+
 func TestQueryNameserver_NXDomain(t *testing.T) {
 	t.Parallel()
 
@@ -1001,6 +1026,29 @@ func TestQueryNameserverIP_Timeout(t *testing.T) {
 	assert.NotEmpty(t, resp.Error)
 }
 
+// TestQueryNameserverIP_CancelledLogsNothing cancels the context while
+// a query to 192.0.2.1, where nothing answers, is waiting for a reply,
+// as shutdown does. The query was cut short, not failed, so nothing is
+// logged.
+func TestQueryNameserverIP_CancelledLogsNothing(t *testing.T) {
+	t.Parallel()
+
+	var logs bytes.Buffer
+
+	r := resolver.NewFromLogger(slog.New(slog.NewTextHandler(&logs, nil)))
+
+	ctx, cancel := context.WithCancel(context.Background())
+	t.Cleanup(cancel)
+	time.AfterFunc(100*time.Millisecond, cancel)
+
+	_, err := r.QueryNameserverIP(
+		ctx, "unreachable.test.", "192.0.2.1", "example.com",
+	)
+	require.NoError(t, err)
+
+	assert.Empty(t, logs.String())
+}
+
 // TestCollectIPs_NoNameserverAnswered takes the response of a
 // nameserver at 192.0.2.1, where nothing answers, as
 // TestQueryNameserverIP_Timeout does. Addresses collected from