Retry a WHOIS lookup that failed instead of storing it as complete (closes #105)

A lookup whose RIR query failed, or whose RIR answer carried none of the
WHOIS fields (a refusal of service, for one), returned IANA's answer as
a success. The fetcher stored it as complete with blank fields, did not
look the AS up again for 30 days, and shortened its interval. Such a
lookup is now an error: the fetcher backs off and leaves the row as it
was. The fields are read from the RIR's answer alone. An AS IANA refers
to no RIR is stored with no WHOIS data, so it is not looked up again
until it is stale, counting as neither a success nor a failure. Tests
reach fake WHOIS servers through the client's DialContext.

Model: opus-5-5

internal/routewatch/asnfetcher.go

@@ -264,7 +264,10 @@ func (f *ASNFetcher) fetchNextStale(ctx context.Context) {
 }
 
 // fetchAndUpdate performs a WHOIS lookup and updates the database.
-// Returns true if successful.
+// Returns true if successful. A failed lookup leaves the AS's row as it was,
+// so the AS stays stale. An AS IANA refers to no RIR is stored with no
+// WHOIS data, so it is not looked up again until it is stale; that counts
+// as neither a success nor a failure.
 func (f *ASNFetcher) fetchAndUpdate(ctx context.Context, asn int) bool {
 	ctx, cancel := context.WithTimeout(ctx, time.Minute)
 	defer cancel()
@@ -272,6 +275,19 @@ func (f *ASNFetcher) fetchAndUpdate(ctx context.Context, asn int) bool {
 	f.logger.Info("Fetching WHOIS data", "asn", asn)
 
 	info, err := f.whoisClient.LookupASN(ctx, asn)
+	if errors.Is(err, whois.ErrNoReferral) {
+		f.logger.Info("IANA refers the AS to no RIR", "asn", asn)
+
+		err = f.db.UpdateASNWHOIS(ctx, &database.ASNWHOISUpdate{ASN: asn})
+		if err != nil {
+			f.logger.Error("Failed to update ASN WHOIS data",
+				"asn", asn, "error", err)
+			f.recordFailure()
+		}
+
+		return false
+	}
+
 	if err != nil {
 		f.logger.Error("WHOIS lookup failed", "asn", asn, "error", err)
 		f.recordFailure()

internal/routewatch/asnfetcher_internal_test.go

@@ -0,0 +1,157 @@
+package routewatch
+
+import (
+	"bufio"
+	"context"
+	"errors"
+	"io"
+	"net"
+	"sync"
+	"syscall"
+	"testing"
+	"time"
+
+	"git.eeqj.de/sneak/routewatch/internal/database"
+	"git.eeqj.de/sneak/routewatch/internal/logger"
+)
+
+// The WHOIS servers a lookup asks: IANA first, then the RIR IANA refers it
+// to.
+const (
+	ianaWHOISServer = "whois.iana.org:43"
+	ripeWHOISServer = "whois.ripe.net:43"
+)
+
+// ianaRefersToRIPE is IANA's answer for an AS in a block it gave to RIPE.
+const ianaRefersToRIPE = `refer:        whois.ripe.net
+
+as-block:     64496-64511
+organisation: Assigned by RIPE NCC
+changed:      2002-09-01
+source:       IANA
+`
+
+// ripeAccessDenied is RIPE's answer to a host it no longer serves.
+const ripeAccessDenied = `% This is the RIPE Database query service.
+%ERROR:201: access denied for 192.0.2.1
+`
+
+// ianaPrivateUseBlock is IANA's answer for a private-use AS: its block and
+// no referral.
+const ianaPrivateUseBlock = `as-block:     64512-65534
+organisation: Reserved for Private Use
+source:       IANA
+`
+
+// newFetcherWithWHOISAnswers returns an ASN fetcher on a new database that
+// holds testASNA, never looked up. The fetcher's WHOIS client connects to a
+// fake server for each address in answers, which reads the query and sends
+// answers[address]; connecting to any other address is refused.
+func newFetcherWithWHOISAnswers(
+	t *testing.T, answers map[string]string,
+) (*ASNFetcher, *database.Database) {
+	t.Helper()
+
+	db := newTestDatabase(t)
+
+	_, err := db.GetOrCreateASN(testASNA, time.Now())
+	if err != nil {
+		t.Fatalf("GetOrCreateASN: %v", err)
+	}
+
+	var servers sync.WaitGroup
+
+	t.Cleanup(servers.Wait)
+
+	f := NewASNFetcher(db, logger.New().Logger)
+	f.whoisClient.DialContext = func(
+		_ context.Context, _, address string,
+	) (net.Conn, error) {
+		answer, ok := answers[address]
+		if !ok {
+			return nil, syscall.ECONNREFUSED
+		}
+
+		client, server := net.Pipe()
+
+		servers.Go(func() {
+			defer func() { _ = server.Close() }()
+
+			_, _ = bufio.NewReader(server).ReadString('\n')
+			_, _ = io.WriteString(server, answer)
+		})
+
+		return client, nil
+	}
+
+	return f, db
+}
+
+// TestFetchAndUpdateKeepsTheRowWhenTheLookupFails looks up an AS IANA refers
+// to RIPE when RIPE's server refuses the connection, and when it answers only
+// with an error. The AS must stay never looked up, and the fetcher must back
+// off.
+func TestFetchAndUpdateKeepsTheRowWhenTheLookupFails(t *testing.T) {
+	t.Parallel()
+
+	for name, answers := range map[string]map[string]string{
+		"connection refused": {ianaWHOISServer: ianaRefersToRIPE},
+		"only an error": {
+			ianaWHOISServer: ianaRefersToRIPE,
+			ripeWHOISServer: ripeAccessDenied,
+		},
+	} {
+		t.Run(name, func(t *testing.T) {
+			t.Parallel()
+
+			f, db := newFetcherWithWHOISAnswers(t, answers)
+
+			if f.fetchAndUpdate(t.Context(), testASNA) {
+				t.Error("fetchAndUpdate reported success")
+			}
+
+			stats, err := db.GetWHOISStats(t.Context(), whoisStaleThreshold)
+			if err != nil {
+				t.Fatalf("GetWHOISStats: %v", err)
+			}
+
+			if stats.NeverFetched != 1 {
+				t.Errorf("never looked up = %d, want 1: the failed lookup "+
+					"was stored", stats.NeverFetched)
+			}
+
+			got := f.GetStats()
+			if got.CurrentInterval != baseInterval*backoffMultiplier ||
+				got.ConsecutiveFails != 1 {
+				t.Errorf("interval %v after %d failures, want %v after 1",
+					got.CurrentInterval, got.ConsecutiveFails,
+					baseInterval*backoffMultiplier)
+			}
+		})
+	}
+}
+
+// TestFetchAndUpdateMarksAnASWithoutReferral looks up a private-use AS, which
+// IANA refers to no RIR. The AS must not be looked up again until it is
+// stale, and the lookup counts as neither a success nor a failure.
+func TestFetchAndUpdateMarksAnASWithoutReferral(t *testing.T) {
+	t.Parallel()
+
+	f, db := newFetcherWithWHOISAnswers(t,
+		map[string]string{ianaWHOISServer: ianaPrivateUseBlock})
+
+	f.fetchAndUpdate(t.Context(), testASNA)
+
+	asn, err := db.GetNextStaleASN(t.Context(), whoisStaleThreshold)
+	if !errors.Is(err, database.ErrNoStaleASN) {
+		t.Errorf("GetNextStaleASN = %d, %v; want ErrNoStaleASN", asn, err)
+	}
+
+	got := f.GetStats()
+	if got.CurrentInterval != baseInterval || got.SuccessesLastHour != 0 ||
+		got.ErrorsLastHour != 0 {
+		t.Errorf("interval %v, %d successes, %d errors; want %v, 0, 0",
+			got.CurrentInterval, got.SuccessesLastHour, got.ErrorsLastHour,
+			baseInterval)
+	}
+}

internal/whois/whois.go

@@ -4,6 +4,7 @@ package whois
 import (
 	"bufio"
 	"context"
+	"errors"
 	"fmt"
 	"net"
 	"regexp"
@@ -43,6 +44,17 @@ const (
 	RIRAFRNIC = "AFRINIC"
 )
 
+// Errors LookupASN returns when a WHOIS answer has nothing to store.
+var (
+	// ErrNoReferral is returned when IANA refers the AS to no RIR, as it
+	// does for a reserved, private-use or unallocated AS number.
+	ErrNoReferral = errors.New("IANA refers the AS to no RIR")
+
+	errNoASBlock = errors.New("IANA answered with neither a referral " +
+		"nor an AS block")
+	errNoFields = errors.New("RIR answered with no WHOIS fields")
+)
+
 // ASNInfo contains parsed WHOIS information for an ASN.
 type ASNInfo struct {
 	ASN         int
@@ -63,20 +75,24 @@ type ASNInfo struct {
 
 // Client performs WHOIS lookups for ASNs.
 type Client struct {
-	// Dialer for creating connections (can be overridden for testing)
-	dialer *net.Dialer
+	// DialContext opens the connection to a WHOIS server, given as
+	// host:port. NewClient sets a net.Dialer's; tests replace it to reach
+	// local servers instead.
+	DialContext func(ctx context.Context, network, address string) (net.Conn, error)
 }
 
 // NewClient creates a new WHOIS client.
 func NewClient() *Client {
-	return &Client{
-		dialer: &net.Dialer{
-			Timeout: dialTimeout,
-		},
-	}
+	dialer := &net.Dialer{Timeout: dialTimeout}
+
+	return &Client{DialContext: dialer.DialContext}
 }
 
-// LookupASN queries WHOIS for the given ASN and returns parsed information.
+// LookupASN asks IANA which RIR holds the given ASN, asks that RIR, and
+// returns what the RIR's answer says. It returns ErrNoReferral when IANA
+// refers the AS to no RIR, and another error when a query fails, when IANA's
+// answer has neither a referral nor the AS's block, or when the RIR's answer
+// carries none of ASNInfo's fields, as a refusal of service does.
 func (c *Client) LookupASN(ctx context.Context, asn int) (*ASNInfo, error) {
 	// Query IANA first to find the authoritative RIR
 	query := fmt.Sprintf("AS%d", asn)
@@ -89,32 +105,46 @@ func (c *Client) LookupASN(ctx context.Context, asn int) (*ASNInfo, error) {
 	// Determine RIR from IANA response
 	rir, whoisServer := c.parseIANAReferral(ianaResp)
 	if whoisServer == "" {
-		// No referral, try to parse what we have
-		return c.parseResponse(asn, rir, ianaResp), nil
+		// IANA names the block of an AS it refers to no RIR. An answer
+		// without that, such as an empty one, says nothing about the AS.
+		if !strings.Contains(ianaResp, "as-block:") {
+			return nil, fmt.Errorf("%s: %w", query, errNoASBlock)
+		}
+
+		return nil, fmt.Errorf("%s: %w", query, ErrNoReferral)
 	}
 
 	// Query the authoritative RIR
 	rirResp, err := c.query(ctx, whoisServer, query)
 	if err != nil {
-		// Return partial data from IANA if RIR query fails
-		info := c.parseResponse(asn, rir, ianaResp)
-		info.RawResponse = fmt.Sprintf("%s\n--- RIR query failed: %v ---\n",
-			ianaResp, err)
-
-		return info, nil
+		return nil, fmt.Errorf("RIR query failed: %w", err)
 	}
 
-	// Combine responses and parse
-	fullResponse := ianaResp + "\n" + rirResp
-	info := c.parseResponse(asn, rir, fullResponse)
-	info.RawResponse = fullResponse
+	// The fields come from the RIR's answer alone: IANA's describes the
+	// block the AS is in.
+	info := c.parseResponse(asn, rir, rirResp)
+	if info.isEmpty() {
+		return nil, fmt.Errorf("%s from %s: %w", query, whoisServer, errNoFields)
+	}
+
+	info.RawResponse = ianaResp + "\n" + rirResp
 
 	return info, nil
 }
 
+// isEmpty reports whether info holds none of the fields a WHOIS answer
+// fills, as when the answer is only comment and error lines.
+func (info *ASNInfo) isEmpty() bool {
+	return info.ASName == "" && info.OrgName == "" && info.OrgID == "" &&
+		info.Address == "" && info.CountryCode == "" &&
+		info.AbuseEmail == "" && info.AbusePhone == "" &&
+		info.TechEmail == "" && info.TechPhone == "" &&
+		info.RegDate == nil && info.LastMod == nil
+}
+
 // query performs a raw WHOIS query to the specified server.
 func (c *Client) query(ctx context.Context, server, query string) (string, error) {
-	conn, err := c.dialer.DialContext(ctx, "tcp", server)
+	conn, err := c.DialContext(ctx, "tcp", server)
 	if err != nil {
 		return "", fmt.Errorf("dial %s: %w", server, err)
 	}
@@ -155,9 +185,8 @@ func (c *Client) query(ctx context.Context, server, query string) (string, error
 }
 
 // parseIANAReferral extracts the RIR and WHOIS server from an IANA response.
+// The server is "" when the response has no referral.
 func (c *Client) parseIANAReferral(response string) (string, string) {
-	var rir string
-
 	for line := range strings.SplitSeq(response, "\n") {
 		line = strings.TrimSpace(line)
 
@@ -186,36 +215,9 @@ func (c *Client) parseIANAReferral(response string) (string, string) {
 				return "", server
 			}
 		}
-
-		// Also check organisation line for RIR hints
-		if strings.HasPrefix(strings.ToLower(line), "organisation:") {
-			orgRIR := rirInOrganisationLine(strings.ToLower(line))
-			if orgRIR != "" {
-				rir = orgRIR
-			}
-		}
 	}
 
-	return rir, ""
-}
-
-// rirInOrganisationLine returns the RIR a lowercased IANA "organisation:"
-// line names, or "" when it names none.
-func rirInOrganisationLine(org string) string {
-	switch {
-	case strings.Contains(org, "arin"):
-		return RIRARIN
-	case strings.Contains(org, "ripe"):
-		return RIRRIPE
-	case strings.Contains(org, "apnic"):
-		return RIRAPNIC
-	case strings.Contains(org, "lacnic"):
-		return RIRLACNIC
-	case strings.Contains(org, "afrinic"):
-		return RIRAFRNIC
-	default:
-		return ""
-	}
+	return "", ""
 }
 
 // parseResponse extracts ASN information from a WHOIS response.

internal/whois/whois_test.go

@@ -0,0 +1,189 @@
+package whois_test
+
+import (
+	"bufio"
+	"context"
+	"errors"
+	"io"
+	"net"
+	"sync"
+	"syscall"
+	"testing"
+	"time"
+
+	"git.eeqj.de/sneak/routewatch/internal/whois"
+)
+
+// The servers LookupASN asks: IANA first, then the RIR IANA refers it to.
+const (
+	ianaServer = "whois.iana.org:43"
+	ripeServer = "whois.ripe.net:43"
+)
+
+// testASN is the AS the tests look up.
+const testASN = 3333
+
+// ianaReferral is IANA's answer for an AS in a block it gave to RIPE.
+const ianaReferral = `% IANA WHOIS server
+
+refer:        whois.ripe.net
+
+as-block:     3154-3353
+organisation: Assigned by RIPE NCC
+whois:        whois.ripe.net
+changed:      2002-09-01
+source:       IANA
+`
+
+// ripeRecord is RIPE's answer for the AS.
+const ripeRecord = `% This is the RIPE Database query service.
+
+aut-num:        AS3333
+as-name:        RIPE-NCC-AS
+org:            ORG-RIEN1-RIPE
+created:        2002-09-17T13:08:32Z
+last-modified:  2024-01-09T08:55:41Z
+source:         RIPE
+`
+
+// ripeRefusal is RIPE's answer to a host it no longer serves.
+const ripeRefusal = `% This is the RIPE Database query service.
+% The objects are in RPSL format.
+%
+%ERROR:201: access denied for 192.0.2.1
+%
+% Sorry, access from your host has been permanently
+% denied because of a repeated excessive querying.
+`
+
+// ianaPrivateUse is IANA's answer for a private-use AS: its block and no
+// referral.
+const ianaPrivateUse = `% IANA WHOIS server
+
+as-block:     64512-65534
+organisation: Reserved for Private Use
+descr:        Reserved for Private Use
+source:       IANA
+`
+
+// ianaNoObject is IANA's answer when it has nothing for the query.
+const ianaNoObject = `% IANA WHOIS server
+% This query returned 0 objects.
+`
+
+// fakeServers returns a DialContext for a whois.Client that connects to a
+// fake server for each address in answers, which reads the query and sends
+// answers[address]. Connecting to any other address is refused.
+func fakeServers(
+	t *testing.T, answers map[string]string,
+) func(context.Context, string, string) (net.Conn, error) {
+	t.Helper()
+
+	var servers sync.WaitGroup
+
+	t.Cleanup(servers.Wait)
+
+	return func(_ context.Context, _, address string) (net.Conn, error) {
+		answer, ok := answers[address]
+		if !ok {
+			return nil, syscall.ECONNREFUSED
+		}
+
+		client, server := net.Pipe()
+
+		servers.Go(func() {
+			defer func() { _ = server.Close() }()
+
+			_, _ = bufio.NewReader(server).ReadString('\n')
+			_, _ = io.WriteString(server, answer)
+		})
+
+		return client, nil
+	}
+}
+
+// lookUp looks testASN up with fake servers sending answers.
+func lookUp(
+	t *testing.T, answers map[string]string,
+) (*whois.ASNInfo, error) {
+	t.Helper()
+
+	client := whois.NewClient()
+	client.DialContext = fakeServers(t, answers)
+
+	return client.LookupASN(t.Context(), testASN)
+}
+
+// TestLookupASNReadsTheRIRAnswer looks up an AS IANA refers to RIPE. The
+// fields come from RIPE's answer, the last-modified date included: IANA's
+// "changed:" line dates its block, not the AS.
+func TestLookupASNReadsTheRIRAnswer(t *testing.T) {
+	t.Parallel()
+
+	info, err := lookUp(t, map[string]string{
+		ianaServer: ianaReferral,
+		ripeServer: ripeRecord,
+	})
+	if err != nil {
+		t.Fatalf("LookupASN: %v", err)
+	}
+
+	if info.ASName != "RIPE-NCC-AS" || info.OrgID != "ORG-RIEN1-RIPE" ||
+		info.RIR != whois.RIRRIPE {
+		t.Errorf("LookupASN = AS name %q, org %q, RIR %q; "+
+			"want RIPE-NCC-AS, ORG-RIEN1-RIPE, RIPE",
+			info.ASName, info.OrgID, info.RIR)
+	}
+
+	wantLastMod := time.Date(2024, 1, 9, 8, 55, 41, 0, time.UTC)
+	if info.LastMod == nil || !info.LastMod.Equal(wantLastMod) {
+		t.Errorf("LookupASN last modified = %v, want %v",
+			info.LastMod, wantLastMod)
+	}
+}
+
+// TestLookupASNFailsWhenTheRIRGivesNoAnswer looks up an AS IANA refers to
+// RIPE when RIPE's server refuses the connection, and when it answers only
+// with an error. Neither is a lookup that found the AS.
+func TestLookupASNFailsWhenTheRIRGivesNoAnswer(t *testing.T) {
+	t.Parallel()
+
+	for name, answers := range map[string]map[string]string{
+		"connection refused": {ianaServer: ianaReferral},
+		"only an error":      {ianaServer: ianaReferral, ripeServer: ripeRefusal},
+	} {
+		t.Run(name, func(t *testing.T) {
+			t.Parallel()
+
+			_, err := lookUp(t, answers)
+			if err == nil || errors.Is(err, whois.ErrNoReferral) {
+				t.Errorf("LookupASN error = %v, want one that is not "+
+					"ErrNoReferral", err)
+			}
+		})
+	}
+}
+
+// TestLookupASNReportsNoReferral looks up a private-use AS, which IANA
+// refers to no RIR.
+func TestLookupASNReportsNoReferral(t *testing.T) {
+	t.Parallel()
+
+	_, err := lookUp(t, map[string]string{ianaServer: ianaPrivateUse})
+	if !errors.Is(err, whois.ErrNoReferral) {
+		t.Errorf("LookupASN error = %v, want ErrNoReferral", err)
+	}
+}
+
+// TestLookupASNFailsWhenIANAAnswersNoBlock looks up an AS when IANA answers
+// with neither a referral nor the AS's block. That is not IANA saying no
+// RIR holds the AS.
+func TestLookupASNFailsWhenIANAAnswersNoBlock(t *testing.T) {
+	t.Parallel()
+
+	_, err := lookUp(t, map[string]string{ianaServer: ianaNoObject})
+	if err == nil || errors.Is(err, whois.ErrNoReferral) {
+		t.Errorf("LookupASN error = %v, want one that is not ErrNoReferral",
+			err)
+	}
+}