diff --git a/internal/routewatch/asnfetcher.go b/internal/routewatch/asnfetcher.go index 7320276..5dbf1eb 100644 --- a/internal/routewatch/asnfetcher.go +++ b/internal/routewatch/asnfetcher.go @@ -264,7 +264,10 @@ func (f *ASNFetcher) fetchNextStale(ctx context.Context) { } // fetchAndUpdate performs a WHOIS lookup and updates the database. -// Returns true if successful. +// Returns true if successful. A failed lookup leaves the AS's row as it was, +// so the AS stays stale. An AS IANA refers to no RIR is stored with no +// WHOIS data, so it is not looked up again until it is stale; that counts +// as neither a success nor a failure. func (f *ASNFetcher) fetchAndUpdate(ctx context.Context, asn int) bool { ctx, cancel := context.WithTimeout(ctx, time.Minute) defer cancel() @@ -272,6 +275,19 @@ func (f *ASNFetcher) fetchAndUpdate(ctx context.Context, asn int) bool { f.logger.Info("Fetching WHOIS data", "asn", asn) info, err := f.whoisClient.LookupASN(ctx, asn) + if errors.Is(err, whois.ErrNoReferral) { + f.logger.Info("IANA refers the AS to no RIR", "asn", asn) + + err = f.db.UpdateASNWHOIS(ctx, &database.ASNWHOISUpdate{ASN: asn}) + if err != nil { + f.logger.Error("Failed to update ASN WHOIS data", + "asn", asn, "error", err) + f.recordFailure() + } + + return false + } + if err != nil { f.logger.Error("WHOIS lookup failed", "asn", asn, "error", err) f.recordFailure() diff --git a/internal/routewatch/asnfetcher_internal_test.go b/internal/routewatch/asnfetcher_internal_test.go new file mode 100644 index 0000000..d929068 --- /dev/null +++ b/internal/routewatch/asnfetcher_internal_test.go @@ -0,0 +1,157 @@ +package routewatch + +import ( + "bufio" + "context" + "errors" + "io" + "net" + "sync" + "syscall" + "testing" + "time" + + "git.eeqj.de/sneak/routewatch/internal/database" + "git.eeqj.de/sneak/routewatch/internal/logger" +) + +// The WHOIS servers a lookup asks: IANA first, then the RIR IANA refers it +// to. +const ( + ianaWHOISServer = "whois.iana.org:43" + ripeWHOISServer = "whois.ripe.net:43" +) + +// ianaRefersToRIPE is IANA's answer for an AS in a block it gave to RIPE. +const ianaRefersToRIPE = `refer: whois.ripe.net + +as-block: 64496-64511 +organisation: Assigned by RIPE NCC +changed: 2002-09-01 +source: IANA +` + +// ripeAccessDenied is RIPE's answer to a host it no longer serves. +const ripeAccessDenied = `% This is the RIPE Database query service. +%ERROR:201: access denied for 192.0.2.1 +` + +// ianaPrivateUseBlock is IANA's answer for a private-use AS: its block and +// no referral. +const ianaPrivateUseBlock = `as-block: 64512-65534 +organisation: Reserved for Private Use +source: IANA +` + +// newFetcherWithWHOISAnswers returns an ASN fetcher on a new database that +// holds testASNA, never looked up. The fetcher's WHOIS client connects to a +// fake server for each address in answers, which reads the query and sends +// answers[address]; connecting to any other address is refused. +func newFetcherWithWHOISAnswers( + t *testing.T, answers map[string]string, +) (*ASNFetcher, *database.Database) { + t.Helper() + + db := newTestDatabase(t) + + _, err := db.GetOrCreateASN(testASNA, time.Now()) + if err != nil { + t.Fatalf("GetOrCreateASN: %v", err) + } + + var servers sync.WaitGroup + + t.Cleanup(servers.Wait) + + f := NewASNFetcher(db, logger.New().Logger) + f.whoisClient.DialContext = func( + _ context.Context, _, address string, + ) (net.Conn, error) { + answer, ok := answers[address] + if !ok { + return nil, syscall.ECONNREFUSED + } + + client, server := net.Pipe() + + servers.Go(func() { + defer func() { _ = server.Close() }() + + _, _ = bufio.NewReader(server).ReadString('\n') + _, _ = io.WriteString(server, answer) + }) + + return client, nil + } + + return f, db +} + +// TestFetchAndUpdateKeepsTheRowWhenTheLookupFails looks up an AS IANA refers +// to RIPE when RIPE's server refuses the connection, and when it answers only +// with an error. The AS must stay never looked up, and the fetcher must back +// off. +func TestFetchAndUpdateKeepsTheRowWhenTheLookupFails(t *testing.T) { + t.Parallel() + + for name, answers := range map[string]map[string]string{ + "connection refused": {ianaWHOISServer: ianaRefersToRIPE}, + "only an error": { + ianaWHOISServer: ianaRefersToRIPE, + ripeWHOISServer: ripeAccessDenied, + }, + } { + t.Run(name, func(t *testing.T) { + t.Parallel() + + f, db := newFetcherWithWHOISAnswers(t, answers) + + if f.fetchAndUpdate(t.Context(), testASNA) { + t.Error("fetchAndUpdate reported success") + } + + stats, err := db.GetWHOISStats(t.Context(), whoisStaleThreshold) + if err != nil { + t.Fatalf("GetWHOISStats: %v", err) + } + + if stats.NeverFetched != 1 { + t.Errorf("never looked up = %d, want 1: the failed lookup "+ + "was stored", stats.NeverFetched) + } + + got := f.GetStats() + if got.CurrentInterval != baseInterval*backoffMultiplier || + got.ConsecutiveFails != 1 { + t.Errorf("interval %v after %d failures, want %v after 1", + got.CurrentInterval, got.ConsecutiveFails, + baseInterval*backoffMultiplier) + } + }) + } +} + +// TestFetchAndUpdateMarksAnASWithoutReferral looks up a private-use AS, which +// IANA refers to no RIR. The AS must not be looked up again until it is +// stale, and the lookup counts as neither a success nor a failure. +func TestFetchAndUpdateMarksAnASWithoutReferral(t *testing.T) { + t.Parallel() + + f, db := newFetcherWithWHOISAnswers(t, + map[string]string{ianaWHOISServer: ianaPrivateUseBlock}) + + f.fetchAndUpdate(t.Context(), testASNA) + + asn, err := db.GetNextStaleASN(t.Context(), whoisStaleThreshold) + if !errors.Is(err, database.ErrNoStaleASN) { + t.Errorf("GetNextStaleASN = %d, %v; want ErrNoStaleASN", asn, err) + } + + got := f.GetStats() + if got.CurrentInterval != baseInterval || got.SuccessesLastHour != 0 || + got.ErrorsLastHour != 0 { + t.Errorf("interval %v, %d successes, %d errors; want %v, 0, 0", + got.CurrentInterval, got.SuccessesLastHour, got.ErrorsLastHour, + baseInterval) + } +} diff --git a/internal/whois/whois.go b/internal/whois/whois.go index 3713c9f..9e94a92 100644 --- a/internal/whois/whois.go +++ b/internal/whois/whois.go @@ -4,6 +4,7 @@ package whois import ( "bufio" "context" + "errors" "fmt" "net" "regexp" @@ -43,6 +44,17 @@ const ( RIRAFRNIC = "AFRINIC" ) +// Errors LookupASN returns when a WHOIS answer has nothing to store. +var ( + // ErrNoReferral is returned when IANA refers the AS to no RIR, as it + // does for a reserved, private-use or unallocated AS number. + ErrNoReferral = errors.New("IANA refers the AS to no RIR") + + errNoASBlock = errors.New("IANA answered with neither a referral " + + "nor an AS block") + errNoFields = errors.New("RIR answered with no WHOIS fields") +) + // ASNInfo contains parsed WHOIS information for an ASN. type ASNInfo struct { ASN int @@ -63,20 +75,24 @@ type ASNInfo struct { // Client performs WHOIS lookups for ASNs. type Client struct { - // Dialer for creating connections (can be overridden for testing) - dialer *net.Dialer + // DialContext opens the connection to a WHOIS server, given as + // host:port. NewClient sets a net.Dialer's; tests replace it to reach + // local servers instead. + DialContext func(ctx context.Context, network, address string) (net.Conn, error) } // NewClient creates a new WHOIS client. func NewClient() *Client { - return &Client{ - dialer: &net.Dialer{ - Timeout: dialTimeout, - }, - } + dialer := &net.Dialer{Timeout: dialTimeout} + + return &Client{DialContext: dialer.DialContext} } -// LookupASN queries WHOIS for the given ASN and returns parsed information. +// LookupASN asks IANA which RIR holds the given ASN, asks that RIR, and +// returns what the RIR's answer says. It returns ErrNoReferral when IANA +// refers the AS to no RIR, and another error when a query fails, when IANA's +// answer has neither a referral nor the AS's block, or when the RIR's answer +// carries none of ASNInfo's fields, as a refusal of service does. func (c *Client) LookupASN(ctx context.Context, asn int) (*ASNInfo, error) { // Query IANA first to find the authoritative RIR query := fmt.Sprintf("AS%d", asn) @@ -89,32 +105,46 @@ func (c *Client) LookupASN(ctx context.Context, asn int) (*ASNInfo, error) { // Determine RIR from IANA response rir, whoisServer := c.parseIANAReferral(ianaResp) if whoisServer == "" { - // No referral, try to parse what we have - return c.parseResponse(asn, rir, ianaResp), nil + // IANA names the block of an AS it refers to no RIR. An answer + // without that, such as an empty one, says nothing about the AS. + if !strings.Contains(ianaResp, "as-block:") { + return nil, fmt.Errorf("%s: %w", query, errNoASBlock) + } + + return nil, fmt.Errorf("%s: %w", query, ErrNoReferral) } // Query the authoritative RIR rirResp, err := c.query(ctx, whoisServer, query) if err != nil { - // Return partial data from IANA if RIR query fails - info := c.parseResponse(asn, rir, ianaResp) - info.RawResponse = fmt.Sprintf("%s\n--- RIR query failed: %v ---\n", - ianaResp, err) - - return info, nil + return nil, fmt.Errorf("RIR query failed: %w", err) } - // Combine responses and parse - fullResponse := ianaResp + "\n" + rirResp - info := c.parseResponse(asn, rir, fullResponse) - info.RawResponse = fullResponse + // The fields come from the RIR's answer alone: IANA's describes the + // block the AS is in. + info := c.parseResponse(asn, rir, rirResp) + if info.isEmpty() { + return nil, fmt.Errorf("%s from %s: %w", query, whoisServer, errNoFields) + } + + info.RawResponse = ianaResp + "\n" + rirResp return info, nil } +// isEmpty reports whether info holds none of the fields a WHOIS answer +// fills, as when the answer is only comment and error lines. +func (info *ASNInfo) isEmpty() bool { + return info.ASName == "" && info.OrgName == "" && info.OrgID == "" && + info.Address == "" && info.CountryCode == "" && + info.AbuseEmail == "" && info.AbusePhone == "" && + info.TechEmail == "" && info.TechPhone == "" && + info.RegDate == nil && info.LastMod == nil +} + // query performs a raw WHOIS query to the specified server. func (c *Client) query(ctx context.Context, server, query string) (string, error) { - conn, err := c.dialer.DialContext(ctx, "tcp", server) + conn, err := c.DialContext(ctx, "tcp", server) if err != nil { return "", fmt.Errorf("dial %s: %w", server, err) } @@ -155,9 +185,8 @@ func (c *Client) query(ctx context.Context, server, query string) (string, error } // parseIANAReferral extracts the RIR and WHOIS server from an IANA response. +// The server is "" when the response has no referral. func (c *Client) parseIANAReferral(response string) (string, string) { - var rir string - for line := range strings.SplitSeq(response, "\n") { line = strings.TrimSpace(line) @@ -186,36 +215,9 @@ func (c *Client) parseIANAReferral(response string) (string, string) { return "", server } } - - // Also check organisation line for RIR hints - if strings.HasPrefix(strings.ToLower(line), "organisation:") { - orgRIR := rirInOrganisationLine(strings.ToLower(line)) - if orgRIR != "" { - rir = orgRIR - } - } } - return rir, "" -} - -// rirInOrganisationLine returns the RIR a lowercased IANA "organisation:" -// line names, or "" when it names none. -func rirInOrganisationLine(org string) string { - switch { - case strings.Contains(org, "arin"): - return RIRARIN - case strings.Contains(org, "ripe"): - return RIRRIPE - case strings.Contains(org, "apnic"): - return RIRAPNIC - case strings.Contains(org, "lacnic"): - return RIRLACNIC - case strings.Contains(org, "afrinic"): - return RIRAFRNIC - default: - return "" - } + return "", "" } // parseResponse extracts ASN information from a WHOIS response. diff --git a/internal/whois/whois_test.go b/internal/whois/whois_test.go new file mode 100644 index 0000000..8d3a150 --- /dev/null +++ b/internal/whois/whois_test.go @@ -0,0 +1,189 @@ +package whois_test + +import ( + "bufio" + "context" + "errors" + "io" + "net" + "sync" + "syscall" + "testing" + "time" + + "git.eeqj.de/sneak/routewatch/internal/whois" +) + +// The servers LookupASN asks: IANA first, then the RIR IANA refers it to. +const ( + ianaServer = "whois.iana.org:43" + ripeServer = "whois.ripe.net:43" +) + +// testASN is the AS the tests look up. +const testASN = 3333 + +// ianaReferral is IANA's answer for an AS in a block it gave to RIPE. +const ianaReferral = `% IANA WHOIS server + +refer: whois.ripe.net + +as-block: 3154-3353 +organisation: Assigned by RIPE NCC +whois: whois.ripe.net +changed: 2002-09-01 +source: IANA +` + +// ripeRecord is RIPE's answer for the AS. +const ripeRecord = `% This is the RIPE Database query service. + +aut-num: AS3333 +as-name: RIPE-NCC-AS +org: ORG-RIEN1-RIPE +created: 2002-09-17T13:08:32Z +last-modified: 2024-01-09T08:55:41Z +source: RIPE +` + +// ripeRefusal is RIPE's answer to a host it no longer serves. +const ripeRefusal = `% This is the RIPE Database query service. +% The objects are in RPSL format. +% +%ERROR:201: access denied for 192.0.2.1 +% +% Sorry, access from your host has been permanently +% denied because of a repeated excessive querying. +` + +// ianaPrivateUse is IANA's answer for a private-use AS: its block and no +// referral. +const ianaPrivateUse = `% IANA WHOIS server + +as-block: 64512-65534 +organisation: Reserved for Private Use +descr: Reserved for Private Use +source: IANA +` + +// ianaNoObject is IANA's answer when it has nothing for the query. +const ianaNoObject = `% IANA WHOIS server +% This query returned 0 objects. +` + +// fakeServers returns a DialContext for a whois.Client that connects to a +// fake server for each address in answers, which reads the query and sends +// answers[address]. Connecting to any other address is refused. +func fakeServers( + t *testing.T, answers map[string]string, +) func(context.Context, string, string) (net.Conn, error) { + t.Helper() + + var servers sync.WaitGroup + + t.Cleanup(servers.Wait) + + return func(_ context.Context, _, address string) (net.Conn, error) { + answer, ok := answers[address] + if !ok { + return nil, syscall.ECONNREFUSED + } + + client, server := net.Pipe() + + servers.Go(func() { + defer func() { _ = server.Close() }() + + _, _ = bufio.NewReader(server).ReadString('\n') + _, _ = io.WriteString(server, answer) + }) + + return client, nil + } +} + +// lookUp looks testASN up with fake servers sending answers. +func lookUp( + t *testing.T, answers map[string]string, +) (*whois.ASNInfo, error) { + t.Helper() + + client := whois.NewClient() + client.DialContext = fakeServers(t, answers) + + return client.LookupASN(t.Context(), testASN) +} + +// TestLookupASNReadsTheRIRAnswer looks up an AS IANA refers to RIPE. The +// fields come from RIPE's answer, the last-modified date included: IANA's +// "changed:" line dates its block, not the AS. +func TestLookupASNReadsTheRIRAnswer(t *testing.T) { + t.Parallel() + + info, err := lookUp(t, map[string]string{ + ianaServer: ianaReferral, + ripeServer: ripeRecord, + }) + if err != nil { + t.Fatalf("LookupASN: %v", err) + } + + if info.ASName != "RIPE-NCC-AS" || info.OrgID != "ORG-RIEN1-RIPE" || + info.RIR != whois.RIRRIPE { + t.Errorf("LookupASN = AS name %q, org %q, RIR %q; "+ + "want RIPE-NCC-AS, ORG-RIEN1-RIPE, RIPE", + info.ASName, info.OrgID, info.RIR) + } + + wantLastMod := time.Date(2024, 1, 9, 8, 55, 41, 0, time.UTC) + if info.LastMod == nil || !info.LastMod.Equal(wantLastMod) { + t.Errorf("LookupASN last modified = %v, want %v", + info.LastMod, wantLastMod) + } +} + +// TestLookupASNFailsWhenTheRIRGivesNoAnswer looks up an AS IANA refers to +// RIPE when RIPE's server refuses the connection, and when it answers only +// with an error. Neither is a lookup that found the AS. +func TestLookupASNFailsWhenTheRIRGivesNoAnswer(t *testing.T) { + t.Parallel() + + for name, answers := range map[string]map[string]string{ + "connection refused": {ianaServer: ianaReferral}, + "only an error": {ianaServer: ianaReferral, ripeServer: ripeRefusal}, + } { + t.Run(name, func(t *testing.T) { + t.Parallel() + + _, err := lookUp(t, answers) + if err == nil || errors.Is(err, whois.ErrNoReferral) { + t.Errorf("LookupASN error = %v, want one that is not "+ + "ErrNoReferral", err) + } + }) + } +} + +// TestLookupASNReportsNoReferral looks up a private-use AS, which IANA +// refers to no RIR. +func TestLookupASNReportsNoReferral(t *testing.T) { + t.Parallel() + + _, err := lookUp(t, map[string]string{ianaServer: ianaPrivateUse}) + if !errors.Is(err, whois.ErrNoReferral) { + t.Errorf("LookupASN error = %v, want ErrNoReferral", err) + } +} + +// TestLookupASNFailsWhenIANAAnswersNoBlock looks up an AS when IANA answers +// with neither a referral nor the AS's block. That is not IANA saying no +// RIR holds the AS. +func TestLookupASNFailsWhenIANAAnswersNoBlock(t *testing.T) { + t.Parallel() + + _, err := lookUp(t, map[string]string{ianaServer: ianaNoObject}) + if err == nil || errors.Is(err, whois.ErrNoReferral) { + t.Errorf("LookupASN error = %v, want one that is not ErrNoReferral", + err) + } +}