Back off WHOIS lookups more slowly and stop asking for personal data (closes #159 )

A failed lookup now makes the fetcher's wait 1.5 times as long, up to one minute instead of five. A success still halves it. RIPE counts the person and role objects an answer holds against a daily limit per IP address, and refuses an address for good that passes it too often. RIPE, APNIC and AFRINIC are now asked with --no-personal, which leaves those objects out but keeps the organisation object. An answer with RIPE's access-denied or limit-reached error line is now reported as a refusal holding that line, not as an answer with no fields. Model: opus-5-5
2026-10-26 22:26:29 +00:00
5 changed files with 300 additions and 42 deletions
@@ -33,6 +33,11 @@ tested.

 # Completed Steps

+- 2026-10-26: the WHOIS fetcher backs off more slowly. A failure makes its wait
+  1.5 times as long, up to one minute instead of five. Its queries to RIPE,
+  APNIC and AFRINIC leave out person and role objects, which RIPE counts against
+  a daily limit per address. A refused query is logged with the RIR's error line
+  (closes #159)
 - 2026-10-26: the home page's AS name search lists the ASes whose handle or
  description holds the text, as links to their pages, from the new
  `/api/v1/search/as`; the search ignores case. When the database has no record
@@ -21,10 +21,16 @@ const (
 	minInterval = 1 * time.Second

 	// maxInterval is the maximum interval after failures (backoff cap).
-	maxInterval = 5 * time.Minute
+	maxInterval = time.Minute

-	// backoffMultiplier is how much to multiply interval on failure.
-	backoffMultiplier = 2
+	// A failure multiplies the interval by backoffNumerator divided by
+	// backoffDenominator, 1.5, up to maxInterval.
+	backoffNumerator   = 3
+	backoffDenominator = 2
+
+	// A success divides the interval by successDivisor, down to
+	// minInterval.
+	successDivisor = 2

 	// whoisStaleThreshold is how old WHOIS data can be before refresh.
 	whoisStaleThreshold = 30 * 24 * time.Hour // 30 days
@@ -160,7 +166,7 @@ func (f *ASNFetcher) recordSuccess() {
 	f.consecutiveFails = 0

 	// Decrease interval by half, but not below minimum
-	newInterval := max(f.currentInterval/backoffMultiplier, minInterval)
+	newInterval := max(f.currentInterval/successDivisor, minInterval)

 	if newInterval != f.currentInterval {
 		f.logger.Debug("Decreased fetch interval",
@@ -184,8 +190,9 @@ func (f *ASNFetcher) recordFailure() {
 	f.intervalMu.Lock()
 	f.consecutiveFails++

-	// Exponential backoff: multiply by 2, capped at max
-	newInterval := min(f.currentInterval*backoffMultiplier, maxInterval)
+	// Exponential backoff: multiply by 1.5, capped at max
+	newInterval := min(
+		f.currentInterval*backoffNumerator/backoffDenominator, maxInterval)

 	if newInterval != f.currentInterval {
 		f.logger.Debug("Increased fetch interval due to failure",
@@ -120,17 +120,58 @@ func TestFetchAndUpdateKeepsTheRowWhenTheLookupFails(t *testing.T) {
 					"was stored", stats.NeverFetched)
 			}

+			want := baseInterval * backoffNumerator / backoffDenominator
+
 			got := f.GetStats()
-			if got.CurrentInterval != baseInterval*backoffMultiplier ||
-				got.ConsecutiveFails != 1 {
+			if got.CurrentInterval != want || got.ConsecutiveFails != 1 {
 				t.Errorf("interval %v after %d failures, want %v after 1",
-					got.CurrentInterval, got.ConsecutiveFails,
-					baseInterval*backoffMultiplier)
+					got.CurrentInterval, got.ConsecutiveFails, want)
 			}
 		})
 	}
 }

+// TestRecordStepsTheInterval has the fetcher fail from its base interval of
+// 15 seconds, then succeed. Each failure makes the interval 1.5 times as
+// long, up to one minute, and each success halves it, down to one second.
+func TestRecordStepsTheInterval(t *testing.T) {
+	t.Parallel()
+
+	f := NewASNFetcher(newMockStore(), logger.New().Logger)
+
+	for i, want := range []time.Duration{
+		22500 * time.Millisecond,
+		33750 * time.Millisecond,
+		50625 * time.Millisecond,
+		time.Minute,
+		time.Minute,
+	} {
+		f.recordFailure()
+
+		if got := f.getInterval(); got != want {
+			t.Errorf("interval after %d failures = %v, want %v",
+				i+1, got, want)
+		}
+	}
+
+	for i, want := range []time.Duration{
+		30 * time.Second,
+		15 * time.Second,
+		7500 * time.Millisecond,
+		3750 * time.Millisecond,
+		1875 * time.Millisecond,
+		time.Second,
+		time.Second,
+	} {
+		f.recordSuccess()
+
+		if got := f.getInterval(); got != want {
+			t.Errorf("interval after %d successes = %v, want %v",
+				i+1, got, want)
+		}
+	}
+}
+
 // TestFetchAndUpdateMarksAnASWithoutReferral looks up a private-use AS, which
 // IANA refers to no RIR. The AS must not be looked up again until it is
 // stale, and the lookup counts as neither a success nor a failure.
@@ -49,6 +49,31 @@ const (
 	RIRAFRNIC = "AFRINIC"
 )

+// noPersonalFlag keeps person and role objects out of an answer. The
+// organisation object stays in it. RIPE counts the person and role objects
+// it returns against a daily limit per IP address. Role objects for abuse
+// contacts do not count. RIPE refuses an address for good that passes the
+// limit too often:
+// https://docs.db.ripe.net/Types-of-Queries/Access-Control-for-Queries/
+//
+// RIPE documents the flag here:
+// https://docs.db.ripe.net/Types-of-Queries/Referenced-Objects-in-Query-Response/
+// APNIC and AFRINIC list it in their answers to the query "help". AFRINIC
+// sends readers to that answer for its full list of flags:
+// https://afrinic.net/whois-getting-started
+const noPersonalFlag = "--no-personal"
+
+// The error lines with which RIPE's database software refuses a query:
+// access denied to the client's IP address, and that address's limit
+// reached. APNIC and AFRINIC run the same software. The lines are listed
+// under "Access Errors" on this page, whose URL is split over two lines:
+// https://docs.db.ripe.net/Appendices/
+// Appendix-C--RIPE-Database-Query-Server-Response-Codes-and-Messages/
+const (
+	accessDeniedError = "%ERROR:201:"
+	accessLimitError  = "%ERROR:202:"
+)
+
 // Errors LookupASN returns when a WHOIS answer has nothing to store.
 var (
 	// ErrNoReferral is returned when IANA refers the AS to no RIR, as it
@@ -58,6 +83,7 @@ var (
 	errNoASBlock = errors.New("IANA answered with neither a referral " +
 		"nor an AS block")
 	errNoFields       = errors.New("RIR answered with no WHOIS fields")
+	errRefused        = errors.New("RIR refused the query")
 	errUnknownServer  = errors.New("IANA refers the AS to a server no RIR runs")
 	errAnswerTooLarge = errors.New("WHOIS answer reached the size cap")
 )
@@ -97,10 +123,10 @@ func NewClient() *Client {

 // LookupASN asks IANA which RIR holds the given ASN, asks that RIR, and
 // returns what the RIR's answer says. It returns ErrNoReferral when IANA
-// refers the AS to no RIR, and another error when a query fails, when IANA's
-// answer has neither a referral nor the AS's block, when IANA refers the AS
-// to a server none of the five RIRs run, or when the RIR's answer carries
-// none of ASNInfo's fields, as a refusal of service does.
+// refers the AS to no RIR, and another error when a query fails, when the
+// RIR refuses it, when IANA's answer has neither a referral nor the AS's
+// block, when IANA refers the AS to a server none of the five RIRs run, or
+// when the RIR's answer carries none of ASNInfo's fields.
 func (c *Client) LookupASN(ctx context.Context, asn int) (*ASNInfo, error) {
 	// Query IANA first to find the authoritative RIR
 	query := fmt.Sprintf("AS%d", asn)
@@ -127,7 +153,7 @@ func (c *Client) LookupASN(ctx context.Context, asn int) (*ASNInfo, error) {
 	}

 	// Query the authoritative RIR
-	rirResp, err := c.query(ctx, whoisServer, query)
+	rirResp, err := c.queryRIR(ctx, whoisServer, rirQuery(rir, query))
 	if err != nil {
 		return nil, fmt.Errorf("RIR query failed: %w", err)
 	}
@@ -154,6 +180,39 @@ func (info *ASNInfo) isEmpty() bool {
 		info.RegDate == nil && info.LastMod == nil
 }

+// rirQuery returns the query sent to rir's server for asQuery, an AS
+// number such as "AS3333". RIPE, APNIC and AFRINIC are asked to keep person
+// and role objects out of the answer.
+func rirQuery(rir, asQuery string) string {
+	switch rir {
+	case RIRRIPE, RIRAPNIC, RIRAFRNIC:
+		return noPersonalFlag + " " + asQuery
+	default:
+		return asQuery
+	}
+}
+
+// queryRIR sends query to an RIR's server and returns the answer. An answer
+// holding an error line with which the server refuses the query is an
+// error, and the error holds that line.
+func (c *Client) queryRIR(
+	ctx context.Context, server, query string,
+) (string, error) {
+	answer, err := c.query(ctx, server, query)
+	if err != nil {
+		return "", err
+	}
+
+	for line := range strings.SplitSeq(answer, "\n") {
+		if strings.HasPrefix(line, accessDeniedError) ||
+			strings.HasPrefix(line, accessLimitError) {
+			return "", fmt.Errorf("%w: %s", errRefused, line)
+		}
+	}
+
+	return answer, nil
+}
+
 // query performs a raw WHOIS query to the specified server. An answer that
 // reaches maxAnswerSize is an error.
 func (c *Client) query(ctx context.Context, server, query string) (string, error) {
@@ -6,6 +6,7 @@ import (
 	"errors"
 	"io"
 	"net"
+	"slices"
 	"strings"
 	"sync"
 	"syscall"
@@ -21,8 +22,20 @@ const (
 	ripeServer = "whois.ripe.net:43"
 )

-// testASN is the AS the tests look up.
-const testASN = 3333
+// testASN is the AS the tests look up, and asQuery the query for it.
+const (
+	testASN = 3333
+	asQuery = "AS3333"
+)
+
+// The fake servers' answers are keyed by a server's address, a space and a
+// query. These are the keys of the queries LookupASN sends for testASN:
+// IANA's, then RIPE's, which asks RIPE to leave out person and role
+// objects.
+const (
+	ianaQuery = ianaServer + " " + asQuery
+	ripeQuery = ripeServer + " --no-personal " + asQuery
+)

 // ianaReferral is IANA's answer for an AS in a block it gave to RIPE.
 const ianaReferral = `% IANA WHOIS server
@@ -36,27 +49,75 @@ changed:      2002-09-01
 source:       IANA
 `

-// ripeRecord is RIPE's answer for the AS.
+// ripeRecord is RIPE's answer for the AS when asked to leave out person and
+// role objects: the abuse address as a comment, the AS's object and the
+// organisation object it refers to.
 const ripeRecord = `% This is the RIPE Database query service.
+% The objects are in RPSL format.
+
+% Note: --no-personal means ALL personal data has been filtered from this output.
+
+% Information related to 'AS3333'
+
+% Abuse contact for 'AS3333' is 'abuse@ripe.net'

 aut-num:        AS3333
 as-name:        RIPE-NCC-AS
 org:            ORG-RIEN1-RIPE
+admin-c:        BRD-RIPE
+tech-c:         OPS4-RIPE
+abuse-c:        OPS4-RIPE
+mnt-by:         RIPE-NCC-MNT
 created:        2002-09-17T13:08:32Z
 last-modified:  2024-01-09T08:55:41Z
 source:         RIPE
+
+organisation:   ORG-RIEN1-RIPE
+org-name:       Reseaux IP Europeens Network Coordination Centre (RIPE NCC)
+country:        NL
+org-type:       RIR
+address:        P.O. Box 10096
+address:        1001EB Amsterdam
+abuse-c:        OPS4-RIPE
+created:        2012-03-09T13:08:29Z
+last-modified:  2024-03-26T10:55:38Z
+source:         RIPE
 `

+// The error lines with which RIPE refuses a query: access denied to the
+// host, and the host's limit on returned objects reached.
+const (
+	ripeAccessDenied = "%ERROR:201: access denied for 192.0.2.1"
+	ripeLimitReached = "%ERROR:202: access control limit reached"
+)
+
 // ripeRefusal is RIPE's answer to a host it no longer serves.
 const ripeRefusal = `% This is the RIPE Database query service.
 % The objects are in RPSL format.
 %
-%ERROR:201: access denied for 192.0.2.1
+` + ripeAccessDenied + `
 %
 % Sorry, access from your host has been permanently
 % denied because of a repeated excessive querying.
 `

+// apnicAfrinicRecord is the answer of APNIC or AFRINIC for the AS when asked
+// to leave out person and role objects. Like RIPE's, it holds the abuse
+// address as a comment, the AS's object and the organisation object.
+const apnicAfrinicRecord = `% Abuse contact for 'AS3333' is 'abuse@example.net'
+
+aut-num:        AS3333
+as-name:        EXAMPLE-AS
+org:            ORG-EXAMPLE1
+admin-c:        EX1-TEST
+last-modified:  2026-08-26T00:40:00Z
+
+organisation:   ORG-EXAMPLE1
+org-name:       Example Networks
+country:        ZA
+address:        1 Example Road
+`
+
 // ianaPrivateUse is IANA's answer for a private-use AS: its block and no
 // referral.
 const ianaPrivateUse = `% IANA WHOIS server
@@ -80,20 +141,29 @@ const fillerLine = "remarks:        and more\n"
 const endlessAnswerSize = 4 << 20

 // fakeServers returns a DialContext for a whois.Client that connects to a
-// fake server for each address in answers, which reads the query and sends
-// answers[address]. Connecting to any other address is refused.
+// fake server at each address answers names. Each key of answers is a
+// server's address, a space and a query, and its value is that server's
+// answer to that query. A server reads one query and sends its answer, or
+// nothing for a query it has no answer for. Connecting to any other address
+// is refused.
 func fakeServers(
 	t *testing.T, answers map[string]string,
 ) func(context.Context, string, string) (net.Conn, error) {
 	t.Helper()

+	addresses := make(map[string]bool)
+
+	for key := range answers {
+		address, _, _ := strings.Cut(key, " ")
+		addresses[address] = true
+	}
+
 	var servers sync.WaitGroup

 	t.Cleanup(servers.Wait)

 	return func(_ context.Context, _, address string) (net.Conn, error) {
-		answer, ok := answers[address]
-		if !ok {
+		if !addresses[address] {
 			return nil, syscall.ECONNREFUSED
 		}

@@ -102,7 +172,8 @@ func fakeServers(
 		servers.Go(func() {
 			defer func() { _ = server.Close() }()

-			_, _ = bufio.NewReader(server).ReadString('\n')
+			query, _ := bufio.NewReader(server).ReadString('\n')
+			answer := answers[address+" "+strings.TrimSpace(query)]
 			_, _ = io.WriteString(server, answer)
 		})

@@ -122,25 +193,41 @@ func lookUp(
 	return client.LookupASN(t.Context(), testASN)
 }

-// TestLookupASNReadsTheRIRAnswer looks up an AS IANA refers to RIPE. The
-// fields come from RIPE's answer, the last-modified date included: IANA's
-// "changed:" line dates its block, not the AS.
+// TestLookupASNReadsTheRIRAnswer looks up an AS IANA refers to RIPE. RIPE
+// answers only a query that leaves out person and role objects, and its
+// answer to that still fills the AS's fields: the abuse address from the
+// comment line, and the organisation's name and country from its object.
+// The dates are the AS's: IANA's "changed:" line dates its block.
 func TestLookupASNReadsTheRIRAnswer(t *testing.T) {
 	t.Parallel()

 	info, err := lookUp(t, map[string]string{
-		ianaServer: ianaReferral,
-		ripeServer: ripeRecord,
+		ianaQuery: ianaReferral,
+		ripeQuery: ripeRecord,
 	})
 	if err != nil {
 		t.Fatalf("LookupASN: %v", err)
 	}

-	if info.ASName != "RIPE-NCC-AS" || info.OrgID != "ORG-RIEN1-RIPE" ||
-		info.RIR != whois.RIRRIPE {
-		t.Errorf("LookupASN = AS name %q, org %q, RIR %q; "+
-			"want RIPE-NCC-AS, ORG-RIEN1-RIPE, RIPE",
-			info.ASName, info.OrgID, info.RIR)
+	got := []string{
+		info.ASName, info.OrgID, info.OrgName, info.CountryCode,
+		info.AbuseEmail, info.RIR,
+	}
+	want := []string{
+		"RIPE-NCC-AS", "ORG-RIEN1-RIPE",
+		"Reseaux IP Europeens Network Coordination Centre (RIPE NCC)", "NL",
+		"abuse@ripe.net", whois.RIRRIPE,
+	}
+
+	if !slices.Equal(got, want) {
+		t.Errorf("LookupASN = AS name, org ID, org name, country, abuse "+
+			"address, RIR %q; want %q", got, want)
+	}
+
+	wantRegDate := time.Date(2002, 9, 17, 13, 8, 32, 0, time.UTC)
+	if info.RegDate == nil || !info.RegDate.Equal(wantRegDate) {
+		t.Errorf("LookupASN registered = %v, want %v",
+			info.RegDate, wantRegDate)
 	}

 	wantLastMod := time.Date(2024, 1, 9, 8, 55, 41, 0, time.UTC)
@@ -150,6 +237,65 @@ func TestLookupASNReadsTheRIRAnswer(t *testing.T) {
 	}
 }

+// TestLookupASNAsksAPNICAndAFRINICToLeaveOutPersonalData looks up an AS at
+// APNIC and at AFRINIC. Each answers only a query that leaves out person
+// and role objects, as RIPE does. That one answer still holds the
+// organisation object, so the organisation's name and country are filled.
+func TestLookupASNAsksAPNICAndAFRINICToLeaveOutPersonalData(t *testing.T) {
+	t.Parallel()
+
+	for _, server := range []string{"whois.apnic.net", "whois.afrinic.net"} {
+		t.Run(server, func(t *testing.T) {
+			t.Parallel()
+
+			address := server + ":43"
+
+			info, err := lookUp(t, map[string]string{
+				ianaQuery:                             "refer:        " + server + "\n",
+				address + " --no-personal " + asQuery: apnicAfrinicRecord,
+			})
+			if err != nil {
+				t.Fatalf("LookupASN: %v", err)
+			}
+
+			got := []string{
+				info.ASName, info.OrgName, info.CountryCode, info.AbuseEmail,
+			}
+			want := []string{
+				"EXAMPLE-AS", "Example Networks", "ZA", "abuse@example.net",
+			}
+
+			if !slices.Equal(got, want) {
+				t.Errorf("LookupASN = AS name, org name, country, abuse "+
+					"address %q; want %q", got, want)
+			}
+		})
+	}
+}
+
+// TestLookupASNReportsARefusal looks up an AS IANA refers to RIPE when RIPE
+// refuses the query, with each of the error lines RIPE refuses with. The
+// error holds RIPE's line, so the log says why the lookup failed.
+func TestLookupASNReportsARefusal(t *testing.T) {
+	t.Parallel()
+
+	for _, line := range []string{ripeAccessDenied, ripeLimitReached} {
+		t.Run(line, func(t *testing.T) {
+			t.Parallel()
+
+			_, err := lookUp(t, map[string]string{
+				ianaQuery: ianaReferral,
+				ripeQuery: "% This is the RIPE Database query service.\n" +
+					line + "\n",
+			})
+			if err == nil || !strings.Contains(err.Error(), line) {
+				t.Errorf("LookupASN error = %v, want one holding %q",
+					err, line)
+			}
+		})
+	}
+}
+
 // TestLookupASNFailsWhenTheRIRGivesNoAnswer looks up an AS IANA refers to
 // RIPE when RIPE's server refuses the connection, and when it answers only
 // with an error. Neither is a lookup that found the AS.
@@ -157,8 +303,8 @@ func TestLookupASNFailsWhenTheRIRGivesNoAnswer(t *testing.T) {
 	t.Parallel()

 	for name, answers := range map[string]map[string]string{
-		"connection refused": {ianaServer: ianaReferral},
-		"only an error":      {ianaServer: ianaReferral, ripeServer: ripeRefusal},
+		"connection refused": {ianaQuery: ianaReferral},
+		"only an error":      {ianaQuery: ianaReferral, ripeQuery: ripeRefusal},
 	} {
 		t.Run(name, func(t *testing.T) {
 			t.Parallel()
@@ -177,7 +323,7 @@ func TestLookupASNFailsWhenTheRIRGivesNoAnswer(t *testing.T) {
 func TestLookupASNReportsNoReferral(t *testing.T) {
 	t.Parallel()

-	_, err := lookUp(t, map[string]string{ianaServer: ianaPrivateUse})
+	_, err := lookUp(t, map[string]string{ianaQuery: ianaPrivateUse})
 	if !errors.Is(err, whois.ErrNoReferral) {
 		t.Errorf("LookupASN error = %v, want ErrNoReferral", err)
 	}
@@ -189,7 +335,7 @@ func TestLookupASNReportsNoReferral(t *testing.T) {
 func TestLookupASNFailsWhenIANAAnswersNoBlock(t *testing.T) {
 	t.Parallel()

-	_, err := lookUp(t, map[string]string{ianaServer: ianaNoObject})
+	_, err := lookUp(t, map[string]string{ianaQuery: ianaNoObject})
 	if err == nil || errors.Is(err, whois.ErrNoReferral) {
 		t.Errorf("LookupASN error = %v, want one that is not ErrNoReferral",
 			err)
@@ -206,8 +352,8 @@ func TestLookupASNFailsWhenTheRIRAnswerDoesNotEnd(t *testing.T) {
 		strings.Repeat(fillerLine, endlessAnswerSize/len(fillerLine))

 	_, err := lookUp(t, map[string]string{
-		ianaServer: ianaReferral,
-		ripeServer: endless,
+		ianaQuery: ianaReferral,
+		ripeQuery: endless,
 	})
 	if err == nil || errors.Is(err, whois.ErrNoReferral) {
 		t.Errorf("LookupASN error = %v, want one that is not ErrNoReferral",
@@ -233,9 +379,9 @@ func TestLookupASNFailsOnAReferralToAnotherServer(t *testing.T) {
 			t.Parallel()

 			_, err := lookUp(t, map[string]string{
-				ianaServer: "refer:        " + refer + "\n\n" +
+				ianaQuery: "refer:        " + refer + "\n\n" +
 					"as-block:     3154-3353\n",
-				address: ripeRecord,
+				address + " " + asQuery: ripeRecord,
 			})
 			if err == nil || errors.Is(err, whois.ErrNoReferral) {
 				t.Errorf("LookupASN error = %v, want one that is not "+