internal/routewatch/asnfetcher.go

@@ -264,7 +264,10 @@ func (f *ASNFetcher) fetchNextStale(ctx context.Context) {
 }
 
 // fetchAndUpdate performs a WHOIS lookup and updates the database.
-// Returns true if successful.
+// Returns true if successful. A failed lookup leaves the AS's row as it was,
+// so the AS stays stale. An AS IANA refers to no RIR is stored with no
+// WHOIS data, so it is not looked up again until it is stale; that counts
+// as neither a success nor a failure.
 func (f *ASNFetcher) fetchAndUpdate(ctx context.Context, asn int) bool {
 	ctx, cancel := context.WithTimeout(ctx, time.Minute)
 	defer cancel()
@@ -272,6 +275,19 @@ func (f *ASNFetcher) fetchAndUpdate(ctx context.Context, asn int) bool {
 	f.logger.Info("Fetching WHOIS data", "asn", asn)
 
 	info, err := f.whoisClient.LookupASN(ctx, asn)
+	if errors.Is(err, whois.ErrNoReferral) {
+		f.logger.Info("IANA refers the AS to no RIR", "asn", asn)
+
+		err = f.db.UpdateASNWHOIS(ctx, &database.ASNWHOISUpdate{ASN: asn})
+		if err != nil {
+			f.logger.Error("Failed to update ASN WHOIS data",
+				"asn", asn, "error", err)
+			f.recordFailure()
+		}
+
+		return false
+	}
+
 	if err != nil {
 		f.logger.Error("WHOIS lookup failed", "asn", asn, "error", err)
 		f.recordFailure()

internal/routewatch/asnfetcher_internal_test.go

@@ -0,0 +1,157 @@
+package routewatch
+
+import (
+	"bufio"
+	"context"
+	"errors"
+	"io"
+	"net"
+	"sync"
+	"syscall"
+	"testing"
+	"time"
+
+	"git.eeqj.de/sneak/routewatch/internal/database"
+	"git.eeqj.de/sneak/routewatch/internal/logger"
+)
+
+// The WHOIS servers a lookup asks: IANA first, then the RIR IANA refers it
+// to.
+const (
+	ianaWHOISServer = "whois.iana.org:43"
+	ripeWHOISServer = "whois.ripe.net:43"
+)
+
+// ianaRefersToRIPE is IANA's answer for an AS in a block it gave to RIPE.
+const ianaRefersToRIPE = `refer:        whois.ripe.net
+
+as-block:     64496-64511
+organisation: Assigned by RIPE NCC
+changed:      2002-09-01
+source:       IANA
+`
+
+// ripeAccessDenied is RIPE's answer to a host it no longer serves.
+const ripeAccessDenied = `% This is the RIPE Database query service.
+%ERROR:201: access denied for 192.0.2.1
+`
+
+// ianaPrivateUseBlock is IANA's answer for a private-use AS: its block and
+// no referral.
+const ianaPrivateUseBlock = `as-block:     64512-65534
+organisation: Reserved for Private Use
+source:       IANA
+`
+
+// newFetcherWithWHOISAnswers returns an ASN fetcher on a new database that
+// holds testASNA, never looked up. The fetcher's WHOIS client connects to a
+// fake server for each address in answers, which reads the query and sends
+// answers[address]; connecting to any other address is refused.
+func newFetcherWithWHOISAnswers(
+	t *testing.T, answers map[string]string,
+) (*ASNFetcher, *database.Database) {
+	t.Helper()
+
+	db := newTestDatabase(t)
+
+	_, err := db.GetOrCreateASN(testASNA, time.Now())
+	if err != nil {
+		t.Fatalf("GetOrCreateASN: %v", err)
+	}
+
+	var servers sync.WaitGroup
+
+	t.Cleanup(servers.Wait)
+
+	f := NewASNFetcher(db, logger.New().Logger)
+	f.whoisClient.DialContext = func(
+		_ context.Context, _, address string,
+	) (net.Conn, error) {
+		answer, ok := answers[address]
+		if !ok {
+			return nil, syscall.ECONNREFUSED
+		}
+
+		client, server := net.Pipe()
+
+		servers.Go(func() {
+			defer func() { _ = server.Close() }()
+
+			_, _ = bufio.NewReader(server).ReadString('\n')
+			_, _ = io.WriteString(server, answer)
+		})
+
+		return client, nil
+	}
+
+	return f, db
+}
+
+// TestFetchAndUpdateKeepsTheRowWhenTheLookupFails looks up an AS IANA refers
+// to RIPE when RIPE's server refuses the connection, and when it answers only
+// with an error. The AS must stay never looked up, and the fetcher must back
+// off.
+func TestFetchAndUpdateKeepsTheRowWhenTheLookupFails(t *testing.T) {
+	t.Parallel()
+
+	for name, answers := range map[string]map[string]string{
+		"connection refused": {ianaWHOISServer: ianaRefersToRIPE},
+		"only an error": {
+			ianaWHOISServer: ianaRefersToRIPE,
+			ripeWHOISServer: ripeAccessDenied,
+		},
+	} {
+		t.Run(name, func(t *testing.T) {
+			t.Parallel()
+
+			f, db := newFetcherWithWHOISAnswers(t, answers)
+
+			if f.fetchAndUpdate(t.Context(), testASNA) {
+				t.Error("fetchAndUpdate reported success")
+			}
+
+			stats, err := db.GetWHOISStats(t.Context(), whoisStaleThreshold)
+			if err != nil {
+				t.Fatalf("GetWHOISStats: %v", err)
+			}
+
+			if stats.NeverFetched != 1 {
+				t.Errorf("never looked up = %d, want 1: the failed lookup "+
+					"was stored", stats.NeverFetched)
+			}
+
+			got := f.GetStats()
+			if got.CurrentInterval != baseInterval*backoffMultiplier ||
+				got.ConsecutiveFails != 1 {
+				t.Errorf("interval %v after %d failures, want %v after 1",
+					got.CurrentInterval, got.ConsecutiveFails,
+					baseInterval*backoffMultiplier)
+			}
+		})
+	}
+}
+
+// TestFetchAndUpdateMarksAnASWithoutReferral looks up a private-use AS, which
+// IANA refers to no RIR. The AS must not be looked up again until it is
+// stale, and the lookup counts as neither a success nor a failure.
+func TestFetchAndUpdateMarksAnASWithoutReferral(t *testing.T) {
+	t.Parallel()
+
+	f, db := newFetcherWithWHOISAnswers(t,
+		map[string]string{ianaWHOISServer: ianaPrivateUseBlock})
+
+	f.fetchAndUpdate(t.Context(), testASNA)
+
+	asn, err := db.GetNextStaleASN(t.Context(), whoisStaleThreshold)
+	if !errors.Is(err, database.ErrNoStaleASN) {
+		t.Errorf("GetNextStaleASN = %d, %v; want ErrNoStaleASN", asn, err)
+	}
+
+	got := f.GetStats()
+	if got.CurrentInterval != baseInterval || got.SuccessesLastHour != 0 ||
+		got.ErrorsLastHour != 0 {
+		t.Errorf("interval %v, %d successes, %d errors; want %v, 0, 0",
+			got.CurrentInterval, got.SuccessesLastHour, got.ErrorsLastHour,
+			baseInterval)
+	}
+}

internal/whois/whois.go

@@ -4,6 +4,7 @@ package whois
 import (
 	"bufio"
 	"context"
+	"errors"
 	"fmt"
 	"net"
 	"regexp"
@@ -43,6 +44,17 @@ const (
 	RIRAFRNIC = "AFRINIC"
 )
 
+// Errors LookupASN returns when a WHOIS answer has nothing to store.
+var (
+	// ErrNoReferral is returned when IANA refers the AS to no RIR, as it
+	// does for a reserved, private-use or unallocated AS number.
+	ErrNoReferral = errors.New("IANA refers the AS to no RIR")
+
+	errNoASBlock = errors.New("IANA answered with neither a referral " +
+		"nor an AS block")
+	errNoFields = errors.New("RIR answered with no WHOIS fields")
+)
+
 // ASNInfo contains parsed WHOIS information for an ASN.
 type ASNInfo struct {
 	ASN         int
@@ -63,20 +75,24 @@ type ASNInfo struct {
 
 // Client performs WHOIS lookups for ASNs.
 type Client struct {
-	// Dialer for creating connections (can be overridden for testing)
+	// DialContext opens the connection to a WHOIS server, given as
-	dialer *net.Dialer
+	// host:port. NewClient sets a net.Dialer's; tests replace it to reach
+	// local servers instead.
+	DialContext func(ctx context.Context, network, address string) (net.Conn, error)
 }
 
 // NewClient creates a new WHOIS client.
 func NewClient() *Client {
-	return &Client{
+	dialer := &net.Dialer{Timeout: dialTimeout}
-		dialer: &net.Dialer{
+
-			Timeout: dialTimeout,
+	return &Client{DialContext: dialer.DialContext}
-		},
-	}
 }
 
-// LookupASN queries WHOIS for the given ASN and returns parsed information.
+// LookupASN asks IANA which RIR holds the given ASN, asks that RIR, and
+// returns what the RIR's answer says. It returns ErrNoReferral when IANA
+// refers the AS to no RIR, and another error when a query fails, when IANA's
+// answer has neither a referral nor the AS's block, or when the RIR's answer
+// carries none of ASNInfo's fields, as a refusal of service does.
 func (c *Client) LookupASN(ctx context.Context, asn int) (*ASNInfo, error) {
 	// Query IANA first to find the authoritative RIR
 	query := fmt.Sprintf("AS%d", asn)
@@ -89,32 +105,46 @@ func (c *Client) LookupASN(ctx context.Context, asn int) (*ASNInfo, error) {
 	// Determine RIR from IANA response
 	rir, whoisServer := c.parseIANAReferral(ianaResp)
 	if whoisServer == "" {
-		// No referral, try to parse what we have
+		// IANA names the block of an AS it refers to no RIR. An answer
-		return c.parseResponse(asn, rir, ianaResp), nil
+		// without that, such as an empty one, says nothing about the AS.
+		if !strings.Contains(ianaResp, "as-block:") {
+			return nil, fmt.Errorf("%s: %w", query, errNoASBlock)
+		}
+
+		return nil, fmt.Errorf("%s: %w", query, ErrNoReferral)
 	}
 
 	// Query the authoritative RIR
 	rirResp, err := c.query(ctx, whoisServer, query)
 	if err != nil {
-		// Return partial data from IANA if RIR query fails
+		return nil, fmt.Errorf("RIR query failed: %w", err)
-		info := c.parseResponse(asn, rir, ianaResp)
-		info.RawResponse = fmt.Sprintf("%s\n--- RIR query failed: %v ---\n",
-			ianaResp, err)
-
-		return info, nil
 	}
 
-	// Combine responses and parse
+	// The fields come from the RIR's answer alone: IANA's describes the
-	fullResponse := ianaResp + "\n" + rirResp
+	// block the AS is in.
-	info := c.parseResponse(asn, rir, fullResponse)
+	info := c.parseResponse(asn, rir, rirResp)
-	info.RawResponse = fullResponse
+	if info.isEmpty() {
+		return nil, fmt.Errorf("%s from %s: %w", query, whoisServer, errNoFields)
+	}
+
+	info.RawResponse = ianaResp + "\n" + rirResp
 
 	return info, nil
 }
 
+// isEmpty reports whether info holds none of the fields a WHOIS answer
+// fills, as when the answer is only comment and error lines.
+func (info *ASNInfo) isEmpty() bool {
+	return info.ASName == "" && info.OrgName == "" && info.OrgID == "" &&
+		info.Address == "" && info.CountryCode == "" &&
+		info.AbuseEmail == "" && info.AbusePhone == "" &&
+		info.TechEmail == "" && info.TechPhone == "" &&
+		info.RegDate == nil && info.LastMod == nil
+}
+
 // query performs a raw WHOIS query to the specified server.
 func (c *Client) query(ctx context.Context, server, query string) (string, error) {
-	conn, err := c.dialer.DialContext(ctx, "tcp", server)
+	conn, err := c.DialContext(ctx, "tcp", server)
 	if err != nil {
 		return "", fmt.Errorf("dial %s: %w", server, err)
 	}
@@ -155,9 +185,8 @@ func (c *Client) query(ctx context.Context, server, query string) (string, error
 }
 
 // parseIANAReferral extracts the RIR and WHOIS server from an IANA response.
+// The server is "" when the response has no referral.
 func (c *Client) parseIANAReferral(response string) (string, string) {
-	var rir string
-
 	for line := range strings.SplitSeq(response, "\n") {
 		line = strings.TrimSpace(line)
 
@@ -186,36 +215,9 @@ func (c *Client) parseIANAReferral(response string) (string, string) {
 				return "", server
 			}
 		}
-
-		// Also check organisation line for RIR hints
-		if strings.HasPrefix(strings.ToLower(line), "organisation:") {
-			orgRIR := rirInOrganisationLine(strings.ToLower(line))
-			if orgRIR != "" {
-				rir = orgRIR
-			}
-		}
 	}
 
-	return rir, ""
+	return "", ""
-}
-
-// rirInOrganisationLine returns the RIR a lowercased IANA "organisation:"
-// line names, or "" when it names none.
-func rirInOrganisationLine(org string) string {
-	switch {
-	case strings.Contains(org, "arin"):
-		return RIRARIN
-	case strings.Contains(org, "ripe"):
-		return RIRRIPE
-	case strings.Contains(org, "apnic"):
-		return RIRAPNIC
-	case strings.Contains(org, "lacnic"):
-		return RIRLACNIC
-	case strings.Contains(org, "afrinic"):
-		return RIRAFRNIC
-	default:
-		return ""
-	}
 }
 
 // parseResponse extracts ASN information from a WHOIS response.

internal/whois/whois_test.go

@@ -0,0 +1,189 @@
+package whois_test
+
+import (
+	"bufio"
+	"context"
+	"errors"
+	"io"
+	"net"
+	"sync"
+	"syscall"
+	"testing"
+	"time"
+
+	"git.eeqj.de/sneak/routewatch/internal/whois"
+)
+
+// The servers LookupASN asks: IANA first, then the RIR IANA refers it to.
+const (
+	ianaServer = "whois.iana.org:43"
+	ripeServer = "whois.ripe.net:43"
+)
+
+// testASN is the AS the tests look up.
+const testASN = 3333
+
+// ianaReferral is IANA's answer for an AS in a block it gave to RIPE.
+const ianaReferral = `% IANA WHOIS server
+
+refer:        whois.ripe.net
+
+as-block:     3154-3353
+organisation: Assigned by RIPE NCC
+whois:        whois.ripe.net
+changed:      2002-09-01
+source:       IANA
+`
+
+// ripeRecord is RIPE's answer for the AS.
+const ripeRecord = `% This is the RIPE Database query service.
+
+aut-num:        AS3333
+as-name:        RIPE-NCC-AS
+org:            ORG-RIEN1-RIPE
+created:        2002-09-17T13:08:32Z
+last-modified:  2024-01-09T08:55:41Z
+source:         RIPE
+`
+
+// ripeRefusal is RIPE's answer to a host it no longer serves.
+const ripeRefusal = `% This is the RIPE Database query service.
+% The objects are in RPSL format.
+%
+%ERROR:201: access denied for 192.0.2.1
+%
+% Sorry, access from your host has been permanently
+% denied because of a repeated excessive querying.
+`
+
+// ianaPrivateUse is IANA's answer for a private-use AS: its block and no
+// referral.
+const ianaPrivateUse = `% IANA WHOIS server
+
+as-block:     64512-65534
+organisation: Reserved for Private Use
+descr:        Reserved for Private Use
+source:       IANA
+`
+
+// ianaNoObject is IANA's answer when it has nothing for the query.
+const ianaNoObject = `% IANA WHOIS server
+% This query returned 0 objects.
+`
+
+// fakeServers returns a DialContext for a whois.Client that connects to a
+// fake server for each address in answers, which reads the query and sends
+// answers[address]. Connecting to any other address is refused.
+func fakeServers(
+	t *testing.T, answers map[string]string,
+) func(context.Context, string, string) (net.Conn, error) {
+	t.Helper()
+
+	var servers sync.WaitGroup
+
+	t.Cleanup(servers.Wait)
+
+	return func(_ context.Context, _, address string) (net.Conn, error) {
+		answer, ok := answers[address]
+		if !ok {
+			return nil, syscall.ECONNREFUSED
+		}
+
+		client, server := net.Pipe()
+
+		servers.Go(func() {
+			defer func() { _ = server.Close() }()
+
+			_, _ = bufio.NewReader(server).ReadString('\n')
+			_, _ = io.WriteString(server, answer)
+		})
+
+		return client, nil
+	}
+}
+
+// lookUp looks testASN up with fake servers sending answers.
+func lookUp(
+	t *testing.T, answers map[string]string,
+) (*whois.ASNInfo, error) {
+	t.Helper()
+
+	client := whois.NewClient()
+	client.DialContext = fakeServers(t, answers)
+
+	return client.LookupASN(t.Context(), testASN)
+}
+
+// TestLookupASNReadsTheRIRAnswer looks up an AS IANA refers to RIPE. The
+// fields come from RIPE's answer, the last-modified date included: IANA's
+// "changed:" line dates its block, not the AS.
+func TestLookupASNReadsTheRIRAnswer(t *testing.T) {
+	t.Parallel()
+
+	info, err := lookUp(t, map[string]string{
+		ianaServer: ianaReferral,
+		ripeServer: ripeRecord,
+	})
+	if err != nil {
+		t.Fatalf("LookupASN: %v", err)
+	}
+
+	if info.ASName != "RIPE-NCC-AS" || info.OrgID != "ORG-RIEN1-RIPE" ||
+		info.RIR != whois.RIRRIPE {
+		t.Errorf("LookupASN = AS name %q, org %q, RIR %q; "+
+			"want RIPE-NCC-AS, ORG-RIEN1-RIPE, RIPE",
+			info.ASName, info.OrgID, info.RIR)
+	}
+
+	wantLastMod := time.Date(2024, 1, 9, 8, 55, 41, 0, time.UTC)
+	if info.LastMod == nil || !info.LastMod.Equal(wantLastMod) {
+		t.Errorf("LookupASN last modified = %v, want %v",
+			info.LastMod, wantLastMod)
+	}
+}
+
+// TestLookupASNFailsWhenTheRIRGivesNoAnswer looks up an AS IANA refers to
+// RIPE when RIPE's server refuses the connection, and when it answers only
+// with an error. Neither is a lookup that found the AS.
+func TestLookupASNFailsWhenTheRIRGivesNoAnswer(t *testing.T) {
+	t.Parallel()
+
+	for name, answers := range map[string]map[string]string{
+		"connection refused": {ianaServer: ianaReferral},
+		"only an error":      {ianaServer: ianaReferral, ripeServer: ripeRefusal},
+	} {
+		t.Run(name, func(t *testing.T) {
+			t.Parallel()
+
+			_, err := lookUp(t, answers)
+			if err == nil || errors.Is(err, whois.ErrNoReferral) {
+				t.Errorf("LookupASN error = %v, want one that is not "+
+					"ErrNoReferral", err)
+			}
+		})
+	}
+}
+
+// TestLookupASNReportsNoReferral looks up a private-use AS, which IANA
+// refers to no RIR.
+func TestLookupASNReportsNoReferral(t *testing.T) {
+	t.Parallel()
+
+	_, err := lookUp(t, map[string]string{ianaServer: ianaPrivateUse})
+	if !errors.Is(err, whois.ErrNoReferral) {
+		t.Errorf("LookupASN error = %v, want ErrNoReferral", err)
+	}
+}
+
+// TestLookupASNFailsWhenIANAAnswersNoBlock looks up an AS when IANA answers
+// with neither a referral nor the AS's block. That is not IANA saying no
+// RIR holds the AS.
+func TestLookupASNFailsWhenIANAAnswersNoBlock(t *testing.T) {
+	t.Parallel()
+
+	_, err := lookUp(t, map[string]string{ianaServer: ianaNoObject})
+	if err == nil || errors.Is(err, whois.ErrNoReferral) {
+		t.Errorf("LookupASN error = %v, want one that is not ErrNoReferral",
+			err)
+	}
+}