Pause a WHOIS server for an hour after it refuses a query (closes #218)

The fetcher kept one interval for every registry, so a registry that
refused a query was asked again within seconds. RIPE blocked the test
host's address that way.

The WHOIS client now asks a server nothing for an hour after the server
refuses a query. A refusal is a refused or reset connection, or an
%ERROR:201: or %ERROR:202: line. The other servers are still asked. A
lookup that needs a paused server leaves the AS stale, and counts as
neither a success nor a failure. NIC.br's hour-long skip after its
over-limit answer is now the same pause.

Model: opus-5-5

internal/whois/whois.go

@@ -12,6 +12,7 @@ import (
 	"regexp"
 	"strings"
 	"sync"
+	"syscall"
 	"time"
 	"unicode/utf8"
 
@@ -70,12 +71,11 @@ const nicbrHeader = "% Copyright (c) Nic.br"
 // queries to stop the data being taken in bulk.
 const nicbrLimitReached = "Query rate limit exceeded"
 
-// NICbrSkipTime is how long the client stops asking NIC.br after NIC.br
-// answers that the client is over its query limit. NIC.br publishes
-// neither the limit nor how long it lasts. An hour outlasts a limit
-// counted per minute or per hour. The ASes looked up meanwhile get
-// NIC.br's fields when they are next refreshed.
-const NICbrSkipTime = time.Hour
+// PauseTime is how long the client asks a WHOIS server nothing after the
+// server refuses a query, as ask says. RIPE refuses an address for good
+// that keeps passing its limit, as noPersonalFlag says. An hour outlasts a
+// limit counted per minute or per hour.
+const PauseTime = time.Hour
 
 // RIR identifiers.
 const (
@@ -125,6 +125,10 @@ var (
 	errAnswerTooLarge = errors.New("WHOIS answer reached the size cap")
 )
 
+// ErrPaused is returned when a lookup needs a server that refused a query
+// less than PauseTime ago. The client sends that server nothing meanwhile.
+var ErrPaused = errors.New("WHOIS server paused after refusing a query")
+
 // ASNInfo contains parsed WHOIS information for an ASN.
 type ASNInfo struct {
 	ASN         int
@@ -156,12 +160,12 @@ type Client struct {
 
 	logger *slog.Logger
 
-	// mu guards nicbrSkipUntil.
+	// mu guards pausedUntil.
 	mu sync.Mutex
 
-	// nicbrSkipUntil is when the client next asks NIC.br, after NIC.br
-	// answered that the client is over its query limit.
-	nicbrSkipUntil time.Time
+	// pausedUntil holds, for each server that refused a query, given as
+	// host:port, when the client next asks it.
+	pausedUntil map[string]time.Time
 }
 
 // NewClient creates a new WHOIS client that logs to logger.
@@ -172,24 +176,26 @@ func NewClient(logger *slog.Logger) *Client {
 		DialContext: dialer.DialContext,
 		Now:         time.Now,
 		logger:      logger,
+		pausedUntil: make(map[string]time.Time),
 	}
 }
 
 // LookupASN asks IANA which RIR holds the given ASN, asks that RIR, and
 // returns what the RIR's answer says. When LACNIC's answer is NIC.br's, it
 // returns what NIC.br's own server says instead, unless that query fails or
-// is skipped, as askNICbr says.
+// NIC.br is paused, as askNICbr says.
 //
-// It returns ErrNoReferral when IANA refers the AS to no RIR, and another
-// error when the query to IANA or to the RIR fails, when the RIR refuses
-// it, when IANA's answer has neither a referral nor the AS's block, when
-// IANA refers the AS to a server none of the five RIRs run, or when the
-// answer carries none of ASNInfo's fields.
+// It returns ErrNoReferral when IANA refers the AS to no RIR. Its error
+// wraps ErrPaused when IANA or the RIR is paused. It returns another error
+// when the query to IANA or to the RIR fails, when the RIR refuses it, when
+// IANA's answer has neither a referral nor the AS's block, when IANA refers
+// the AS to a server none of the five RIRs run, or when the answer carries
+// none of ASNInfo's fields.
 func (c *Client) LookupASN(ctx context.Context, asn int) (*ASNInfo, error) {
 	// Query IANA first to find the authoritative RIR
 	query := fmt.Sprintf("AS%d", asn)
 
-	ianaResp, err := c.query(ctx, whoisServerIANA, query)
+	ianaResp, err := c.ask(ctx, whoisServerIANA, query)
 	if err != nil {
 		return nil, fmt.Errorf("IANA query failed: %w", err)
 	}
@@ -211,7 +217,7 @@ func (c *Client) LookupASN(ctx context.Context, asn int) (*ASNInfo, error) {
 	}
 
 	// Query the authoritative RIR
-	rirResp, err := c.queryRIR(ctx, whoisServer, rirQuery(rir, query))
+	rirResp, err := c.ask(ctx, whoisServer, rirQuery(rir, query))
 	if err != nil {
 		return nil, fmt.Errorf("RIR query failed: %w", err)
 	}
@@ -241,22 +247,19 @@ func (c *Client) LookupASN(ctx context.Context, asn int) (*ASNInfo, error) {
 // askNICbr returns lacnicInfo, what LACNIC's answer says, and logs a
 // warning when the query fails, when NIC.br sends the reduced answer for a
 // client over its limit, or when NIC.br's answer carries none of ASNInfo's
-// fields. After the reduced answer, it returns lacnicInfo without asking
-// NIC.br until NICbrSkipTime has passed.
+// fields. The reduced answer pauses NIC.br, as a refusal does. While NIC.br
+// is paused, askNICbr returns lacnicInfo without a warning. The ASes looked
+// up meanwhile get NIC.br's fields when they are next refreshed.
 func (c *Client) askNICbr(
 	ctx context.Context, asQuery string, lacnicInfo *ASNInfo,
 ) *ASNInfo {
 	asn := lacnicInfo.ASN
 
-	c.mu.Lock()
-	skipping := c.Now().Before(c.nicbrSkipUntil)
-	c.mu.Unlock()
-
-	if skipping {
+	answer, err := c.ask(ctx, whoisServerNICbr, asQuery)
+	if errors.Is(err, ErrPaused) {
 		return lacnicInfo
 	}
 
-	answer, err := c.query(ctx, whoisServerNICbr, asQuery)
 	if err != nil {
 		c.logger.Warn("NIC.br query failed, keeping LACNIC's fields",
 			"asn", asn, "error", err)
@@ -265,13 +268,11 @@ func (c *Client) askNICbr(
 	}
 
 	if strings.Contains(answer, nicbrLimitReached) {
-		c.mu.Lock()
-		c.nicbrSkipUntil = c.Now().Add(NICbrSkipTime)
-		c.mu.Unlock()
+		c.pause(whoisServerNICbr)
 
 		c.logger.Warn("NIC.br answered that the client is over its query "+
-			"limit, keeping LACNIC's fields and skipping NIC.br",
-			"asn", asn, "skip_for", NICbrSkipTime)
+			"limit, keeping LACNIC's fields and pausing NIC.br",
+			"asn", asn, "pause_for", PauseTime)
 
 		return lacnicInfo
 	}
@@ -309,25 +310,61 @@ func rirQuery(rir, asQuery string) string {
 	}
 }
 
-// queryRIR sends query to an RIR's server and returns the answer. An answer
-// holding an error line with which the server refuses the query is an
-// error, and the error holds that line.
-func (c *Client) queryRIR(
-	ctx context.Context, server, query string,
-) (string, error) {
+// ask sends query to server and returns the answer. A server that refuses
+// the query is paused: ask then sends it nothing until PauseTime has
+// passed, and returns an error that wraps ErrPaused instead. Each of these
+// is a refusal, and an error:
+//
+//   - a refused connection
+//   - a reset connection
+//   - an answer with one of the error lines that RIPE's database software
+//     refuses a query with. The error holds that line.
+func (c *Client) ask(ctx context.Context, server, query string) (string, error) {
+	c.mu.Lock()
+	pausedUntil := c.pausedUntil[server]
+	c.mu.Unlock()
+
+	if c.Now().Before(pausedUntil) {
+		return "", fmt.Errorf("%w: %s until %s", ErrPaused, server,
+			pausedUntil.UTC().Format(time.RFC3339))
+	}
+
 	answer, err := c.query(ctx, server, query)
+	if err == nil {
+		err = checkRefusal(answer)
+	}
+
+	if errors.Is(err, errRefused) || errors.Is(err, syscall.ECONNREFUSED) ||
+		errors.Is(err, syscall.ECONNRESET) {
+		c.pause(server)
+	}
+
 	if err != nil {
 		return "", err
 	}
 
+	return answer, nil
+}
+
+// checkRefusal returns an error holding the first line of answer with which
+// RIPE's database software refuses a query, or nil when there is none.
+func checkRefusal(answer string) error {
 	for line := range strings.SplitSeq(answer, "\n") {
 		if strings.HasPrefix(line, accessDeniedError) ||
 			strings.HasPrefix(line, accessLimitError) {
-			return "", fmt.Errorf("%w: %s", errRefused, line)
+			return fmt.Errorf("%w: %s", errRefused, line)
 		}
 	}
 
-	return answer, nil
+	return nil
+}
+
+// pause makes the client ask server nothing until PauseTime has passed.
+func (c *Client) pause(server string) {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+
+	c.pausedUntil[server] = c.Now().Add(PauseTime)
 }
 
 // query performs a raw WHOIS query to the specified server and returns the