diff --git a/TODO.md b/TODO.md index 57d039b..eb42c18 100644 --- a/TODO.md +++ b/TODO.md @@ -33,6 +33,12 @@ tested. # Completed Steps +- 2026-10-27: the WHOIS details of a Brazilian AS keep their accented letters. + LACNIC's answer for such an AS comes from NIC.br, Brazil's registry, with each + accented letter replaced. The details now come from NIC.br's own server, + `whois.registro.br`. When that query fails, the details from LACNIC's answer + are kept and a warning is logged. After NIC.br answers that RouteWatch is over + its query limit, RouteWatch stops asking NIC.br for an hour (closes #174) - 2026-10-27: `make asupdate` stops with an error when the download of `as.csv` breaks off. The generator used to retry the failed read forever. A record that does not parse is still logged and skipped (closes #177) diff --git a/internal/routewatch/asnfetcher.go b/internal/routewatch/asnfetcher.go index 655c5d5..4f5eb7d 100644 --- a/internal/routewatch/asnfetcher.go +++ b/internal/routewatch/asnfetcher.go @@ -68,10 +68,12 @@ type ASNFetcher struct { // NewASNFetcher creates a new ASN fetcher. func NewASNFetcher(db database.Store, logger *slog.Logger) *ASNFetcher { + logger = logger.With("component", "asn_fetcher") + return &ASNFetcher{ db: db, - whoisClient: whois.NewClient(), - logger: logger.With("component", "asn_fetcher"), + whoisClient: whois.NewClient(logger), + logger: logger, immediateQueue: make(chan int, immediateQueueSize), stopCh: make(chan struct{}), done: make(chan struct{}), diff --git a/internal/whois/whois.go b/internal/whois/whois.go index b13e9bf..6c41f00 100644 --- a/internal/whois/whois.go +++ b/internal/whois/whois.go @@ -7,9 +7,11 @@ import ( "errors" "fmt" "io" + "log/slog" "net" "regexp" "strings" + "sync" "time" "unicode/utf8" @@ -49,8 +51,32 @@ const ( whoisServerAPNIC = "whois.apnic.net:43" whoisServerLACNIC = "whois.lacnic.net:43" whoisServerAFRINIC = "whois.afrinic.net:43" + + // NIC.br registers Brazil's ASes under LACNIC. + whoisServerNICbr = "whois.registro.br:43" ) +// nicbrHeader starts the copyright notice at the top of NIC.br's WHOIS +// answers. LACNIC's answer for an AS that NIC.br registers is NIC.br's, and +// carries the same notice: +// +// % Copyright (c) Nic.br - Use of this data is governed by the Use and +// % Privacy Policy at https://registro.br/upp . Distribution, +const nicbrHeader = "% Copyright (c) Nic.br" + +// nicbrLimitReached is in the line that marks NIC.br's reduced answer to a +// client over its query limit. NIC.br publishes no limit. Its Use and +// Privacy Policy, https://registro.br/upp, says only that it restricts +// queries to stop the data being taken in bulk. +const nicbrLimitReached = "Query rate limit exceeded" + +// NICbrSkipTime is how long the client stops asking NIC.br after NIC.br +// answers that the client is over its query limit. NIC.br publishes +// neither the limit nor how long it lasts. An hour outlasts a limit +// counted per minute or per hour. The ASes looked up meanwhile get +// NIC.br's fields when they are next refreshed. +const NICbrSkipTime = time.Hour + // RIR identifiers. const ( RIRARIN = "ARIN" @@ -123,21 +149,42 @@ type Client struct { // host:port. NewClient sets a net.Dialer's; tests replace it to reach // local servers instead. DialContext func(ctx context.Context, network, address string) (net.Conn, error) + + // Now returns the current time. NewClient sets time.Now; tests replace + // it to move the clock. + Now func() time.Time + + logger *slog.Logger + + // mu guards nicbrSkipUntil. + mu sync.Mutex + + // nicbrSkipUntil is when the client next asks NIC.br, after NIC.br + // answered that the client is over its query limit. + nicbrSkipUntil time.Time } -// NewClient creates a new WHOIS client. -func NewClient() *Client { +// NewClient creates a new WHOIS client that logs to logger. +func NewClient(logger *slog.Logger) *Client { dialer := &net.Dialer{Timeout: dialTimeout} - return &Client{DialContext: dialer.DialContext} + return &Client{ + DialContext: dialer.DialContext, + Now: time.Now, + logger: logger, + } } // LookupASN asks IANA which RIR holds the given ASN, asks that RIR, and -// returns what the RIR's answer says. It returns ErrNoReferral when IANA -// refers the AS to no RIR, and another error when a query fails, when the -// RIR refuses it, when IANA's answer has neither a referral nor the AS's -// block, when IANA refers the AS to a server none of the five RIRs run, or -// when the RIR's answer carries none of ASNInfo's fields. +// returns what the RIR's answer says. When LACNIC's answer is NIC.br's, it +// returns what NIC.br's own server says instead, unless that query fails or +// is skipped, as askNICbr says. +// +// It returns ErrNoReferral when IANA refers the AS to no RIR, and another +// error when the query to IANA or to the RIR fails, when the RIR refuses +// it, when IANA's answer has neither a referral nor the AS's block, when +// IANA refers the AS to a server none of the five RIRs run, or when the +// answer carries none of ASNInfo's fields. func (c *Client) LookupASN(ctx context.Context, asn int) (*ASNInfo, error) { // Query IANA first to find the authoritative RIR query := fmt.Sprintf("AS%d", asn) @@ -169,18 +216,77 @@ func (c *Client) LookupASN(ctx context.Context, asn int) (*ASNInfo, error) { return nil, fmt.Errorf("RIR query failed: %w", err) } - // The fields come from the RIR's answer alone: IANA's describes the - // block the AS is in. + // The fields never come from IANA's answer, which describes the block + // the AS is in. info := c.parseResponse(asn, rir, rirResp) + + if rir == RIRLACNIC && strings.Contains(rirResp, nicbrHeader) { + info = c.askNICbr(ctx, query, info) + } + if info.isEmpty() { return nil, fmt.Errorf("%s from %s: %w", query, whoisServer, errNoFields) } - info.RawResponse = ianaResp + "\n" + rirResp + info.RawResponse = ianaResp + "\n" + info.RawResponse return info, nil } +// askNICbr asks NIC.br's server about an AS whose answer from LACNIC is +// NIC.br's, and returns what NIC.br's answer says. LACNIC replaces each +// accented letter in such an answer with U+FFFD. NIC.br's own server keeps +// them, in Latin-1. +// +// askNICbr returns lacnicInfo, what LACNIC's answer says, and logs a +// warning when the query fails, when NIC.br sends the reduced answer for a +// client over its limit, or when NIC.br's answer carries none of ASNInfo's +// fields. After the reduced answer, it returns lacnicInfo without asking +// NIC.br until NICbrSkipTime has passed. +func (c *Client) askNICbr( + ctx context.Context, asQuery string, lacnicInfo *ASNInfo, +) *ASNInfo { + asn := lacnicInfo.ASN + + c.mu.Lock() + skipping := c.Now().Before(c.nicbrSkipUntil) + c.mu.Unlock() + + if skipping { + return lacnicInfo + } + + answer, err := c.query(ctx, whoisServerNICbr, asQuery) + if err != nil { + c.logger.Warn("NIC.br query failed, keeping LACNIC's fields", + "asn", asn, "error", err) + + return lacnicInfo + } + + if strings.Contains(answer, nicbrLimitReached) { + c.mu.Lock() + c.nicbrSkipUntil = c.Now().Add(NICbrSkipTime) + c.mu.Unlock() + + c.logger.Warn("NIC.br answered that the client is over its query "+ + "limit, keeping LACNIC's fields and skipping NIC.br", + "asn", asn, "skip_for", NICbrSkipTime) + + return lacnicInfo + } + + info := c.parseResponse(asn, RIRLACNIC, answer) + if info.isEmpty() { + c.logger.Warn("NIC.br answered with no WHOIS fields, "+ + "keeping LACNIC's fields", "asn", asn) + + return lacnicInfo + } + + return info +} + // isEmpty reports whether info holds none of the fields a WHOIS answer // fills, as when the answer is only comment and error lines. func (info *ASNInfo) isEmpty() bool { @@ -276,9 +382,9 @@ func (c *Client) query(ctx context.Context, server, query string) (string, error } // toUTF8 returns a WHOIS answer in UTF-8. An answer that is valid UTF-8 is -// returned as it is. Any other answer is decoded as Latin-1, which RIPE and -// APNIC send. LACNIC, ARIN and AFRINIC send UTF-8. RIPE's database working -// group lists the five here: +// returned as it is. Any other answer is decoded as Latin-1, which RIPE, +// APNIC and NIC.br send. LACNIC, ARIN and AFRINIC send UTF-8. RIPE's +// database working group lists the five RIRs here: // https://www.ripe.net/ripe/mail/archives/db-wg/2024-May/008075.html func toUTF8(answer string) (string, error) { if utf8.ValidString(answer) { diff --git a/internal/whois/whois_test.go b/internal/whois/whois_test.go index 8284cec..bc16f52 100644 --- a/internal/whois/whois_test.go +++ b/internal/whois/whois_test.go @@ -2,9 +2,13 @@ package whois_test import ( "bufio" + "bytes" "context" + "encoding/json" "errors" "io" + "log/slog" + "maps" "net" "slices" "strings" @@ -50,6 +54,10 @@ changed: 2002-09-01 source: IANA ` +// lacnicReferral is IANA's answer for an AS in a block it gave to LACNIC, +// shortened to the referral. +const lacnicReferral = "refer: whois.lacnic.net\n" + // ripeRecord is RIPE's answer for the AS when asked to leave out person and // role objects. It starts with the as-block object of the block the AS is // in. Then come the abuse address as a comment, the AS's object and the @@ -272,7 +280,7 @@ func lookUp( ) (*whois.ASNInfo, error) { t.Helper() - client := whois.NewClient() + client := whois.NewClient(slog.New(slog.DiscardHandler)) client.DialContext = fakeServers(t, answers) return client.LookupASN(t.Context(), testASN) @@ -330,7 +338,7 @@ func TestLookupASNReadsTheRIRAnswer(t *testing.T) { func TestLookupASNReadsOnlyTheASAndItsOrganisation(t *testing.T) { t.Parallel() - client := whois.NewClient() + client := whois.NewClient(slog.New(slog.DiscardHandler)) client.DialContext = fakeServers(t, map[string]string{ apnicIANAQuery: "refer: whois.apnic.net\n", apnicQuery: apnicRecord, @@ -421,7 +429,7 @@ func TestLookupASNStoresAnswersInUTF8(t *testing.T) { "org-name: " + latin1OrgName + "\n", }, "LACNIC in UTF-8": { - ianaQuery: "refer: whois.lacnic.net\n", + ianaQuery: lacnicReferral, "whois.lacnic.net:43 " + asQuery: "aut-num: AS3333\n" + "owner: " + utf8OrgName + "\n", }, @@ -448,6 +456,312 @@ func TestLookupASNStoresAnswersInUTF8(t *testing.T) { } } +// lacnicServer is LACNIC's WHOIS server and nicbrServer NIC.br's, which +// registers Brazil's ASes under LACNIC. brazilASN is one of those ASes. +const ( + lacnicServer = "whois.lacnic.net:43" + nicbrServer = "whois.registro.br:43" + brazilASN = 52751 +) + +// The keys of the queries LookupASN sends for brazilASN: IANA's, LACNIC's, +// then NIC.br's. +const ( + brazilIANAQuery = ianaServer + " AS52751" + brazilLACNICQuery = lacnicServer + " AS52751" + brazilNICbrQuery = nicbrServer + " AS52751" +) + +// The name of brazilASN's organisation in three forms: as LACNIC sends it, +// with each accented letter replaced by U+FFFD; as NIC.br sends it, in +// Latin-1; and in UTF-8. +const ( + lacnicBrazilOrgName = "Brasrede Telecomunica\U0000FFFD\U0000FFFDes LTDA" + nicbrBrazilOrgName = "Brasrede Telecomunica\xe7\xf5es LTDA" + brazilOrgName = "Brasrede Telecomunicações LTDA" +) + +// nicbrCopyright is the copyright notice at the top of NIC.br's answers. +const nicbrCopyright = "% Copyright (c) Nic.br - " + + "Use of this data is governed by the Use and\n" + + "% Privacy Policy at https://registro.br/upp . Distribution,\n" + + "% commercialization, reproduction, and use for advertising or similar\n" + + "% purposes are expressly prohibited.\n" + +// lacnicBrazilRecord is LACNIC's answer for brazilASN, recorded on +// 2026-10-26. It is NIC.br's answer, which LACNIC passes on after a line +// of its own and a space. It is shortened. The client's address is a +// documentation address. The contact's name, the contact's object and the +// footer are left out. +const lacnicBrazilRecord = "% IP Client: 192.0.2.1\n" + + " " + nicbrCopyright + `% 2026-10-26T23:12:22-03:00 - 192.0.2.1 + +aut-num: AS52751 +owner: ` + lacnicBrazilOrgName + ` +owner-c: AVL217 +routing-c: AVL217 +abuse-c: AVL217 +created: 20120829 +changed: 20130306 +inetnum: 177.38.152.0/21 +inetnum: 2804:b34::/32 +inetnum: 191.243.4.0/22 +inetnum: 131.72.248.0/22 +inetnum: 138.94.228.0/22 +inetnum: 170.233.8.0/22 +inetnum: 170.246.4.0/22 +` + +// nicbrRecord is NIC.br's answer for brazilASN, in Latin-1, recorded on +// 2026-10-26 and shortened as lacnicBrazilRecord is. +const nicbrRecord = nicbrCopyright + `% 2026-10-26T23:14:41-03:00 - 192.0.2.1 + +aut-num: AS52751 +owner: ` + nicbrBrazilOrgName + ` +owner-c: AVL217 +routing-c: AVL217 +abuse-c: AVL217 +created: 20120829 +changed: 20130306 +inetnum: 177.38.152.0/21 +inetnum: 2804:b34::/32 +inetnum: 191.243.4.0/22 +inetnum: 131.72.248.0/22 +inetnum: 138.94.228.0/22 +inetnum: 170.233.8.0/22 +inetnum: 170.246.4.0/22 +` + +// nicbrOverLimit stands for NIC.br's answer for brazilASN to a client over +// its query limit, and is not recorded. Such an answer leaves information +// out, here the owner, and says so in its first line. +const nicbrOverLimit = `% Query rate limit exceeded. Reduced information. + +aut-num: AS52751 +created: 20120829 +changed: 20130306 +` + +// newNICbrClient returns a client whose fake servers send answers and +// whose logger writes JSON lines to logs. It adds one to *nicbrAsks each +// time it connects to NIC.br's server. +func newNICbrClient( + t *testing.T, answers map[string]string, logs io.Writer, nicbrAsks *int, +) *whois.Client { + t.Helper() + + dial := fakeServers(t, answers) + + client := whois.NewClient(slog.New(slog.NewJSONHandler(logs, nil))) + client.DialContext = func( + ctx context.Context, network, address string, + ) (net.Conn, error) { + if address == nicbrServer { + *nicbrAsks++ + } + + return dial(ctx, network, address) + } + + return client +} + +// lookUpBrazilianAS looks brazilASN up with fake servers sending answers, +// and reports whether the client connected to NIC.br's server. +func lookUpBrazilianAS( + t *testing.T, answers map[string]string, +) (*whois.ASNInfo, bool, error) { + t.Helper() + + nicbrAsks := 0 + client := newNICbrClient(t, answers, io.Discard, &nicbrAsks) + + info, err := client.LookupASN(t.Context(), brazilASN) + + return info, nicbrAsks > 0, err +} + +// TestLookupASNTakesABrazilianASFromNICbr looks up an AS that NIC.br +// registers. LACNIC's answer is NIC.br's with each accented letter replaced +// by U+FFFD, so the client also asks NIC.br's server, which answers in +// Latin-1. The organisation's name has its accented letters, in the parsed +// field and in the stored answer. +func TestLookupASNTakesABrazilianASFromNICbr(t *testing.T) { + t.Parallel() + + info, _, err := lookUpBrazilianAS(t, map[string]string{ + brazilIANAQuery: lacnicReferral, + brazilLACNICQuery: lacnicBrazilRecord, + brazilNICbrQuery: nicbrRecord, + }) + if err != nil { + t.Fatalf("LookupASN: %v", err) + } + + if info.OrgName != brazilOrgName || info.RIR != whois.RIRLACNIC { + t.Errorf("LookupASN org name, RIR = %q, %q; want %q, %q", + info.OrgName, info.RIR, brazilOrgName, whois.RIRLACNIC) + } + + if !strings.Contains(info.RawResponse, brazilOrgName) { + t.Errorf("LookupASN answer = %q, want one holding %q", + info.RawResponse, brazilOrgName) + } + + wantRegDate := time.Date(2012, 8, 29, 0, 0, 0, 0, time.UTC) + if info.RegDate == nil || !info.RegDate.Equal(wantRegDate) { + t.Errorf("LookupASN registered = %v, want %v", + info.RegDate, wantRegDate) + } +} + +// TestLookupASNAsksNICbrOnlyForItsASes looks up an AS at LACNIC whose +// answer does not carry NIC.br's copyright notice, as for an AS that LACNIC +// registers itself. The client does not connect to NIC.br's server, which +// would answer with a record, and keeps LACNIC's fields. +func TestLookupASNAsksNICbrOnlyForItsASes(t *testing.T) { + t.Parallel() + + info, askedNICbr, err := lookUpBrazilianAS(t, map[string]string{ + brazilIANAQuery: lacnicReferral, + brazilLACNICQuery: "aut-num: AS52751\n" + + "owner: " + utf8OrgName + "\n", + brazilNICbrQuery: nicbrRecord, + }) + if err != nil { + t.Fatalf("LookupASN: %v", err) + } + + if askedNICbr { + t.Error("LookupASN asked NIC.br's server about an AS whose " + + "answer from LACNIC is not NIC.br's") + } + + if info.OrgName != utf8OrgName { + t.Errorf("LookupASN org name = %q, want LACNIC's %q", + info.OrgName, utf8OrgName) + } +} + +// TestLookupASNKeepsLACNICsAnswerWhenNICbrFails looks up an AS that NIC.br +// registers when NIC.br's server refuses the connection, sends nothing, or +// sends the reduced answer for a client over its query limit. The client +// asks NIC.br, and the lookup still succeeds with LACNIC's fields. The +// client logs one warning, which names the AS and the reason. +func TestLookupASNKeepsLACNICsAnswerWhenNICbrFails(t *testing.T) { + t.Parallel() + + for name, test := range map[string]struct { + nicbrAnswer map[string]string + reason string + }{ + "connection refused": { + nicbrAnswer: map[string]string{}, + reason: syscall.ECONNREFUSED.Error(), + }, + "no answer": { + nicbrAnswer: map[string]string{brazilNICbrQuery: ""}, + reason: "no WHOIS fields", + }, + "over the query limit": { + nicbrAnswer: map[string]string{brazilNICbrQuery: nicbrOverLimit}, + reason: "over its query limit", + }, + } { + t.Run(name, func(t *testing.T) { + t.Parallel() + + answers := map[string]string{ + brazilIANAQuery: lacnicReferral, + brazilLACNICQuery: lacnicBrazilRecord, + } + maps.Copy(answers, test.nicbrAnswer) + + var logs bytes.Buffer + + nicbrAsks := 0 + client := newNICbrClient(t, answers, &logs, &nicbrAsks) + + info, err := client.LookupASN(t.Context(), brazilASN) + if err != nil { + t.Fatalf("LookupASN: %v", err) + } + + if nicbrAsks == 0 { + t.Error("LookupASN did not ask NIC.br's server") + } + + if info.OrgName != lacnicBrazilOrgName { + t.Errorf("LookupASN org name = %q, want LACNIC's %q", + info.OrgName, lacnicBrazilOrgName) + } + + var logged struct { + Level string `json:"level"` + ASN int `json:"asn"` + } + + err = json.Unmarshal(logs.Bytes(), &logged) + if err != nil { + t.Fatalf("want one log line, got %q: %v", logs.String(), err) + } + + if logged.Level != slog.LevelWarn.String() || + logged.ASN != brazilASN || + !strings.Contains(logs.String(), test.reason) { + t.Errorf("logged %q, want a warning for AS%d holding %q", + logs.String(), brazilASN, test.reason) + } + }) + } +} + +// TestLookupASNSkipsNICbrAfterItsLimit looks up an AS that NIC.br +// registers three times with one client, while NIC.br answers that the +// client is over its query limit. The first lookup asks NIC.br. The +// second, a second before NICbrSkipTime has passed, does not. The third, +// once NICbrSkipTime has passed, asks again. Each keeps LACNIC's fields. +func TestLookupASNSkipsNICbrAfterItsLimit(t *testing.T) { + t.Parallel() + + nicbrAsks := 0 + client := newNICbrClient(t, map[string]string{ + brazilIANAQuery: lacnicReferral, + brazilLACNICQuery: lacnicBrazilRecord, + brazilNICbrQuery: nicbrOverLimit, + }, io.Discard, &nicbrAsks) + + start := time.Date(2026, 10, 27, 0, 0, 0, 0, time.UTC) + now := start + client.Now = func() time.Time { return now } + + for _, step := range []struct { + after time.Duration + wantAsks int + }{ + {after: 0, wantAsks: 1}, + {after: whois.NICbrSkipTime - time.Second, wantAsks: 1}, + {after: whois.NICbrSkipTime, wantAsks: 2}, + } { + now = start.Add(step.after) + + info, err := client.LookupASN(t.Context(), brazilASN) + if err != nil { + t.Fatalf("LookupASN after %v: %v", step.after, err) + } + + if nicbrAsks != step.wantAsks { + t.Errorf("after %v NIC.br's server was asked %d times, want %d", + step.after, nicbrAsks, step.wantAsks) + } + + if info.OrgName != lacnicBrazilOrgName { + t.Errorf("LookupASN after %v org name = %q, want LACNIC's %q", + step.after, info.OrgName, lacnicBrazilOrgName) + } + } +} + // TestLookupASNReportsARefusal looks up an AS IANA refers to RIPE when RIPE // refuses the query, with each of the error lines RIPE refuses with. The // error holds RIPE's line, so the log says why the lookup failed.