diff --git a/TODO.md b/TODO.md index 6c89695..fa26f64 100644 --- a/TODO.md +++ b/TODO.md @@ -33,6 +33,11 @@ tested. # Completed Steps +- 2026-10-26: the WHOIS fetcher backs off more slowly. A failure makes its wait + 1.5 times as long, up to one minute instead of five. Its queries to RIPE, + APNIC and AFRINIC leave out person and role objects, which RIPE counts against + a daily limit per address. A refused query is logged with the RIR's error line + (closes #159) - 2026-10-26: the home page's AS name search lists the ASes whose handle or description holds the text, as links to their pages, from the new `/api/v1/search/as`; the search ignores case. When the database has no record diff --git a/internal/routewatch/asnfetcher.go b/internal/routewatch/asnfetcher.go index 6d51b24..655c5d5 100644 --- a/internal/routewatch/asnfetcher.go +++ b/internal/routewatch/asnfetcher.go @@ -21,10 +21,16 @@ const ( minInterval = 1 * time.Second // maxInterval is the maximum interval after failures (backoff cap). - maxInterval = 5 * time.Minute + maxInterval = time.Minute - // backoffMultiplier is how much to multiply interval on failure. - backoffMultiplier = 2 + // A failure multiplies the interval by backoffNumerator divided by + // backoffDenominator, 1.5, up to maxInterval. + backoffNumerator = 3 + backoffDenominator = 2 + + // A success divides the interval by successDivisor, down to + // minInterval. + successDivisor = 2 // whoisStaleThreshold is how old WHOIS data can be before refresh. whoisStaleThreshold = 30 * 24 * time.Hour // 30 days @@ -160,7 +166,7 @@ func (f *ASNFetcher) recordSuccess() { f.consecutiveFails = 0 // Decrease interval by half, but not below minimum - newInterval := max(f.currentInterval/backoffMultiplier, minInterval) + newInterval := max(f.currentInterval/successDivisor, minInterval) if newInterval != f.currentInterval { f.logger.Debug("Decreased fetch interval", @@ -184,8 +190,9 @@ func (f *ASNFetcher) recordFailure() { f.intervalMu.Lock() f.consecutiveFails++ - // Exponential backoff: multiply by 2, capped at max - newInterval := min(f.currentInterval*backoffMultiplier, maxInterval) + // Exponential backoff: multiply by 1.5, capped at max + newInterval := min( + f.currentInterval*backoffNumerator/backoffDenominator, maxInterval) if newInterval != f.currentInterval { f.logger.Debug("Increased fetch interval due to failure", diff --git a/internal/routewatch/asnfetcher_internal_test.go b/internal/routewatch/asnfetcher_internal_test.go index 26c73f8..6779d3d 100644 --- a/internal/routewatch/asnfetcher_internal_test.go +++ b/internal/routewatch/asnfetcher_internal_test.go @@ -120,17 +120,58 @@ func TestFetchAndUpdateKeepsTheRowWhenTheLookupFails(t *testing.T) { "was stored", stats.NeverFetched) } + want := baseInterval * backoffNumerator / backoffDenominator + got := f.GetStats() - if got.CurrentInterval != baseInterval*backoffMultiplier || - got.ConsecutiveFails != 1 { + if got.CurrentInterval != want || got.ConsecutiveFails != 1 { t.Errorf("interval %v after %d failures, want %v after 1", - got.CurrentInterval, got.ConsecutiveFails, - baseInterval*backoffMultiplier) + got.CurrentInterval, got.ConsecutiveFails, want) } }) } } +// TestRecordStepsTheInterval has the fetcher fail from its base interval of +// 15 seconds, then succeed. Each failure makes the interval 1.5 times as +// long, up to one minute, and each success halves it, down to one second. +func TestRecordStepsTheInterval(t *testing.T) { + t.Parallel() + + f := NewASNFetcher(newMockStore(), logger.New().Logger) + + for i, want := range []time.Duration{ + 22500 * time.Millisecond, + 33750 * time.Millisecond, + 50625 * time.Millisecond, + time.Minute, + time.Minute, + } { + f.recordFailure() + + if got := f.getInterval(); got != want { + t.Errorf("interval after %d failures = %v, want %v", + i+1, got, want) + } + } + + for i, want := range []time.Duration{ + 30 * time.Second, + 15 * time.Second, + 7500 * time.Millisecond, + 3750 * time.Millisecond, + 1875 * time.Millisecond, + time.Second, + time.Second, + } { + f.recordSuccess() + + if got := f.getInterval(); got != want { + t.Errorf("interval after %d successes = %v, want %v", + i+1, got, want) + } + } +} + // TestFetchAndUpdateMarksAnASWithoutReferral looks up a private-use AS, which // IANA refers to no RIR. The AS must not be looked up again until it is // stale, and the lookup counts as neither a success nor a failure. diff --git a/internal/whois/whois.go b/internal/whois/whois.go index 92df9fa..2427260 100644 --- a/internal/whois/whois.go +++ b/internal/whois/whois.go @@ -49,6 +49,31 @@ const ( RIRAFRNIC = "AFRINIC" ) +// noPersonalFlag keeps person and role objects out of an answer. The +// organisation object stays in it. RIPE counts the person and role objects +// it returns against a daily limit per IP address. Role objects for abuse +// contacts do not count. RIPE refuses an address for good that passes the +// limit too often: +// https://docs.db.ripe.net/Types-of-Queries/Access-Control-for-Queries/ +// +// RIPE documents the flag here: +// https://docs.db.ripe.net/Types-of-Queries/Referenced-Objects-in-Query-Response/ +// APNIC and AFRINIC list it in their answers to the query "help". AFRINIC +// sends readers to that answer for its full list of flags: +// https://afrinic.net/whois-getting-started +const noPersonalFlag = "--no-personal" + +// The error lines with which RIPE's database software refuses a query: +// access denied to the client's IP address, and that address's limit +// reached. APNIC and AFRINIC run the same software. The lines are listed +// under "Access Errors" on this page, whose URL is split over two lines: +// https://docs.db.ripe.net/Appendices/ +// Appendix-C--RIPE-Database-Query-Server-Response-Codes-and-Messages/ +const ( + accessDeniedError = "%ERROR:201:" + accessLimitError = "%ERROR:202:" +) + // Errors LookupASN returns when a WHOIS answer has nothing to store. var ( // ErrNoReferral is returned when IANA refers the AS to no RIR, as it @@ -58,6 +83,7 @@ var ( errNoASBlock = errors.New("IANA answered with neither a referral " + "nor an AS block") errNoFields = errors.New("RIR answered with no WHOIS fields") + errRefused = errors.New("RIR refused the query") errUnknownServer = errors.New("IANA refers the AS to a server no RIR runs") errAnswerTooLarge = errors.New("WHOIS answer reached the size cap") ) @@ -97,10 +123,10 @@ func NewClient() *Client { // LookupASN asks IANA which RIR holds the given ASN, asks that RIR, and // returns what the RIR's answer says. It returns ErrNoReferral when IANA -// refers the AS to no RIR, and another error when a query fails, when IANA's -// answer has neither a referral nor the AS's block, when IANA refers the AS -// to a server none of the five RIRs run, or when the RIR's answer carries -// none of ASNInfo's fields, as a refusal of service does. +// refers the AS to no RIR, and another error when a query fails, when the +// RIR refuses it, when IANA's answer has neither a referral nor the AS's +// block, when IANA refers the AS to a server none of the five RIRs run, or +// when the RIR's answer carries none of ASNInfo's fields. func (c *Client) LookupASN(ctx context.Context, asn int) (*ASNInfo, error) { // Query IANA first to find the authoritative RIR query := fmt.Sprintf("AS%d", asn) @@ -127,7 +153,7 @@ func (c *Client) LookupASN(ctx context.Context, asn int) (*ASNInfo, error) { } // Query the authoritative RIR - rirResp, err := c.query(ctx, whoisServer, query) + rirResp, err := c.queryRIR(ctx, whoisServer, rirQuery(rir, query)) if err != nil { return nil, fmt.Errorf("RIR query failed: %w", err) } @@ -154,6 +180,39 @@ func (info *ASNInfo) isEmpty() bool { info.RegDate == nil && info.LastMod == nil } +// rirQuery returns the query sent to rir's server for asQuery, an AS +// number such as "AS3333". RIPE, APNIC and AFRINIC are asked to keep person +// and role objects out of the answer. +func rirQuery(rir, asQuery string) string { + switch rir { + case RIRRIPE, RIRAPNIC, RIRAFRNIC: + return noPersonalFlag + " " + asQuery + default: + return asQuery + } +} + +// queryRIR sends query to an RIR's server and returns the answer. An answer +// holding an error line with which the server refuses the query is an +// error, and the error holds that line. +func (c *Client) queryRIR( + ctx context.Context, server, query string, +) (string, error) { + answer, err := c.query(ctx, server, query) + if err != nil { + return "", err + } + + for line := range strings.SplitSeq(answer, "\n") { + if strings.HasPrefix(line, accessDeniedError) || + strings.HasPrefix(line, accessLimitError) { + return "", fmt.Errorf("%w: %s", errRefused, line) + } + } + + return answer, nil +} + // query performs a raw WHOIS query to the specified server. An answer that // reaches maxAnswerSize is an error. func (c *Client) query(ctx context.Context, server, query string) (string, error) { diff --git a/internal/whois/whois_test.go b/internal/whois/whois_test.go index bcc8ef9..0fee412 100644 --- a/internal/whois/whois_test.go +++ b/internal/whois/whois_test.go @@ -6,6 +6,7 @@ import ( "errors" "io" "net" + "slices" "strings" "sync" "syscall" @@ -21,8 +22,20 @@ const ( ripeServer = "whois.ripe.net:43" ) -// testASN is the AS the tests look up. -const testASN = 3333 +// testASN is the AS the tests look up, and asQuery the query for it. +const ( + testASN = 3333 + asQuery = "AS3333" +) + +// The fake servers' answers are keyed by a server's address, a space and a +// query. These are the keys of the queries LookupASN sends for testASN: +// IANA's, then RIPE's, which asks RIPE to leave out person and role +// objects. +const ( + ianaQuery = ianaServer + " " + asQuery + ripeQuery = ripeServer + " --no-personal " + asQuery +) // ianaReferral is IANA's answer for an AS in a block it gave to RIPE. const ianaReferral = `% IANA WHOIS server @@ -36,27 +49,75 @@ changed: 2002-09-01 source: IANA ` -// ripeRecord is RIPE's answer for the AS. +// ripeRecord is RIPE's answer for the AS when asked to leave out person and +// role objects: the abuse address as a comment, the AS's object and the +// organisation object it refers to. const ripeRecord = `% This is the RIPE Database query service. +% The objects are in RPSL format. + +% Note: --no-personal means ALL personal data has been filtered from this output. + +% Information related to 'AS3333' + +% Abuse contact for 'AS3333' is 'abuse@ripe.net' aut-num: AS3333 as-name: RIPE-NCC-AS org: ORG-RIEN1-RIPE +admin-c: BRD-RIPE +tech-c: OPS4-RIPE +abuse-c: OPS4-RIPE +mnt-by: RIPE-NCC-MNT created: 2002-09-17T13:08:32Z last-modified: 2024-01-09T08:55:41Z source: RIPE + +organisation: ORG-RIEN1-RIPE +org-name: Reseaux IP Europeens Network Coordination Centre (RIPE NCC) +country: NL +org-type: RIR +address: P.O. Box 10096 +address: 1001EB Amsterdam +abuse-c: OPS4-RIPE +created: 2012-03-09T13:08:29Z +last-modified: 2024-03-26T10:55:38Z +source: RIPE ` +// The error lines with which RIPE refuses a query: access denied to the +// host, and the host's limit on returned objects reached. +const ( + ripeAccessDenied = "%ERROR:201: access denied for 192.0.2.1" + ripeLimitReached = "%ERROR:202: access control limit reached" +) + // ripeRefusal is RIPE's answer to a host it no longer serves. const ripeRefusal = `% This is the RIPE Database query service. % The objects are in RPSL format. % -%ERROR:201: access denied for 192.0.2.1 +` + ripeAccessDenied + ` % % Sorry, access from your host has been permanently % denied because of a repeated excessive querying. ` +// apnicAfrinicRecord is the answer of APNIC or AFRINIC for the AS when asked +// to leave out person and role objects. Like RIPE's, it holds the abuse +// address as a comment, the AS's object and the organisation object. +const apnicAfrinicRecord = `% Abuse contact for 'AS3333' is 'abuse@example.net' + +aut-num: AS3333 +as-name: EXAMPLE-AS +org: ORG-EXAMPLE1 +admin-c: EX1-TEST +last-modified: 2026-08-26T00:40:00Z + +organisation: ORG-EXAMPLE1 +org-name: Example Networks +country: ZA +address: 1 Example Road +` + // ianaPrivateUse is IANA's answer for a private-use AS: its block and no // referral. const ianaPrivateUse = `% IANA WHOIS server @@ -80,20 +141,29 @@ const fillerLine = "remarks: and more\n" const endlessAnswerSize = 4 << 20 // fakeServers returns a DialContext for a whois.Client that connects to a -// fake server for each address in answers, which reads the query and sends -// answers[address]. Connecting to any other address is refused. +// fake server at each address answers names. Each key of answers is a +// server's address, a space and a query, and its value is that server's +// answer to that query. A server reads one query and sends its answer, or +// nothing for a query it has no answer for. Connecting to any other address +// is refused. func fakeServers( t *testing.T, answers map[string]string, ) func(context.Context, string, string) (net.Conn, error) { t.Helper() + addresses := make(map[string]bool) + + for key := range answers { + address, _, _ := strings.Cut(key, " ") + addresses[address] = true + } + var servers sync.WaitGroup t.Cleanup(servers.Wait) return func(_ context.Context, _, address string) (net.Conn, error) { - answer, ok := answers[address] - if !ok { + if !addresses[address] { return nil, syscall.ECONNREFUSED } @@ -102,7 +172,8 @@ func fakeServers( servers.Go(func() { defer func() { _ = server.Close() }() - _, _ = bufio.NewReader(server).ReadString('\n') + query, _ := bufio.NewReader(server).ReadString('\n') + answer := answers[address+" "+strings.TrimSpace(query)] _, _ = io.WriteString(server, answer) }) @@ -122,25 +193,41 @@ func lookUp( return client.LookupASN(t.Context(), testASN) } -// TestLookupASNReadsTheRIRAnswer looks up an AS IANA refers to RIPE. The -// fields come from RIPE's answer, the last-modified date included: IANA's -// "changed:" line dates its block, not the AS. +// TestLookupASNReadsTheRIRAnswer looks up an AS IANA refers to RIPE. RIPE +// answers only a query that leaves out person and role objects, and its +// answer to that still fills the AS's fields: the abuse address from the +// comment line, and the organisation's name and country from its object. +// The dates are the AS's: IANA's "changed:" line dates its block. func TestLookupASNReadsTheRIRAnswer(t *testing.T) { t.Parallel() info, err := lookUp(t, map[string]string{ - ianaServer: ianaReferral, - ripeServer: ripeRecord, + ianaQuery: ianaReferral, + ripeQuery: ripeRecord, }) if err != nil { t.Fatalf("LookupASN: %v", err) } - if info.ASName != "RIPE-NCC-AS" || info.OrgID != "ORG-RIEN1-RIPE" || - info.RIR != whois.RIRRIPE { - t.Errorf("LookupASN = AS name %q, org %q, RIR %q; "+ - "want RIPE-NCC-AS, ORG-RIEN1-RIPE, RIPE", - info.ASName, info.OrgID, info.RIR) + got := []string{ + info.ASName, info.OrgID, info.OrgName, info.CountryCode, + info.AbuseEmail, info.RIR, + } + want := []string{ + "RIPE-NCC-AS", "ORG-RIEN1-RIPE", + "Reseaux IP Europeens Network Coordination Centre (RIPE NCC)", "NL", + "abuse@ripe.net", whois.RIRRIPE, + } + + if !slices.Equal(got, want) { + t.Errorf("LookupASN = AS name, org ID, org name, country, abuse "+ + "address, RIR %q; want %q", got, want) + } + + wantRegDate := time.Date(2002, 9, 17, 13, 8, 32, 0, time.UTC) + if info.RegDate == nil || !info.RegDate.Equal(wantRegDate) { + t.Errorf("LookupASN registered = %v, want %v", + info.RegDate, wantRegDate) } wantLastMod := time.Date(2024, 1, 9, 8, 55, 41, 0, time.UTC) @@ -150,6 +237,65 @@ func TestLookupASNReadsTheRIRAnswer(t *testing.T) { } } +// TestLookupASNAsksAPNICAndAFRINICToLeaveOutPersonalData looks up an AS at +// APNIC and at AFRINIC. Each answers only a query that leaves out person +// and role objects, as RIPE does. That one answer still holds the +// organisation object, so the organisation's name and country are filled. +func TestLookupASNAsksAPNICAndAFRINICToLeaveOutPersonalData(t *testing.T) { + t.Parallel() + + for _, server := range []string{"whois.apnic.net", "whois.afrinic.net"} { + t.Run(server, func(t *testing.T) { + t.Parallel() + + address := server + ":43" + + info, err := lookUp(t, map[string]string{ + ianaQuery: "refer: " + server + "\n", + address + " --no-personal " + asQuery: apnicAfrinicRecord, + }) + if err != nil { + t.Fatalf("LookupASN: %v", err) + } + + got := []string{ + info.ASName, info.OrgName, info.CountryCode, info.AbuseEmail, + } + want := []string{ + "EXAMPLE-AS", "Example Networks", "ZA", "abuse@example.net", + } + + if !slices.Equal(got, want) { + t.Errorf("LookupASN = AS name, org name, country, abuse "+ + "address %q; want %q", got, want) + } + }) + } +} + +// TestLookupASNReportsARefusal looks up an AS IANA refers to RIPE when RIPE +// refuses the query, with each of the error lines RIPE refuses with. The +// error holds RIPE's line, so the log says why the lookup failed. +func TestLookupASNReportsARefusal(t *testing.T) { + t.Parallel() + + for _, line := range []string{ripeAccessDenied, ripeLimitReached} { + t.Run(line, func(t *testing.T) { + t.Parallel() + + _, err := lookUp(t, map[string]string{ + ianaQuery: ianaReferral, + ripeQuery: "% This is the RIPE Database query service.\n" + + line + "\n", + }) + if err == nil || !strings.Contains(err.Error(), line) { + t.Errorf("LookupASN error = %v, want one holding %q", + err, line) + } + }) + } +} + // TestLookupASNFailsWhenTheRIRGivesNoAnswer looks up an AS IANA refers to // RIPE when RIPE's server refuses the connection, and when it answers only // with an error. Neither is a lookup that found the AS. @@ -157,8 +303,8 @@ func TestLookupASNFailsWhenTheRIRGivesNoAnswer(t *testing.T) { t.Parallel() for name, answers := range map[string]map[string]string{ - "connection refused": {ianaServer: ianaReferral}, - "only an error": {ianaServer: ianaReferral, ripeServer: ripeRefusal}, + "connection refused": {ianaQuery: ianaReferral}, + "only an error": {ianaQuery: ianaReferral, ripeQuery: ripeRefusal}, } { t.Run(name, func(t *testing.T) { t.Parallel() @@ -177,7 +323,7 @@ func TestLookupASNFailsWhenTheRIRGivesNoAnswer(t *testing.T) { func TestLookupASNReportsNoReferral(t *testing.T) { t.Parallel() - _, err := lookUp(t, map[string]string{ianaServer: ianaPrivateUse}) + _, err := lookUp(t, map[string]string{ianaQuery: ianaPrivateUse}) if !errors.Is(err, whois.ErrNoReferral) { t.Errorf("LookupASN error = %v, want ErrNoReferral", err) } @@ -189,7 +335,7 @@ func TestLookupASNReportsNoReferral(t *testing.T) { func TestLookupASNFailsWhenIANAAnswersNoBlock(t *testing.T) { t.Parallel() - _, err := lookUp(t, map[string]string{ianaServer: ianaNoObject}) + _, err := lookUp(t, map[string]string{ianaQuery: ianaNoObject}) if err == nil || errors.Is(err, whois.ErrNoReferral) { t.Errorf("LookupASN error = %v, want one that is not ErrNoReferral", err) @@ -206,8 +352,8 @@ func TestLookupASNFailsWhenTheRIRAnswerDoesNotEnd(t *testing.T) { strings.Repeat(fillerLine, endlessAnswerSize/len(fillerLine)) _, err := lookUp(t, map[string]string{ - ianaServer: ianaReferral, - ripeServer: endless, + ianaQuery: ianaReferral, + ripeQuery: endless, }) if err == nil || errors.Is(err, whois.ErrNoReferral) { t.Errorf("LookupASN error = %v, want one that is not ErrNoReferral", @@ -233,9 +379,9 @@ func TestLookupASNFailsOnAReferralToAnotherServer(t *testing.T) { t.Parallel() _, err := lookUp(t, map[string]string{ - ianaServer: "refer: " + refer + "\n\n" + + ianaQuery: "refer: " + refer + "\n\n" + "as-block: 3154-3353\n", - address: ripeRecord, + address + " " + asQuery: ripeRecord, }) if err == nil || errors.Is(err, whois.ErrNoReferral) { t.Errorf("LookupASN error = %v, want one that is not "+