2026-10-27 03:59:12 +01:00
4 changed files with 447 additions and 19 deletions
@@ -33,6 +33,12 @@ tested.

 # Completed Steps

+- 2026-10-27: the WHOIS details of a Brazilian AS keep their accented letters.
+  LACNIC's answer for such an AS comes from NIC.br, Brazil's registry, with each
+  accented letter replaced. The details now come from NIC.br's own server,
+  `whois.registro.br`. When that query fails, the details from LACNIC's answer
+  are kept and a warning is logged. After NIC.br answers that RouteWatch is over
+  its query limit, RouteWatch stops asking NIC.br for an hour (closes #174)
 - 2026-10-27: `make asupdate` stops with an error when the download of `as.csv`
  breaks off. The generator used to retry the failed read forever. A record that
  does not parse is still logged and skipped (closes #177)
@@ -68,10 +68,12 @@ type ASNFetcher struct {

 // NewASNFetcher creates a new ASN fetcher.
 func NewASNFetcher(db database.Store, logger *slog.Logger) *ASNFetcher {
+	logger = logger.With("component", "asn_fetcher")
+
 	return &ASNFetcher{
 		db:              db,
-		whoisClient:     whois.NewClient(),
-		logger:          logger.With("component", "asn_fetcher"),
+		whoisClient:     whois.NewClient(logger),
+		logger:          logger,
 		immediateQueue:  make(chan int, immediateQueueSize),
 		stopCh:          make(chan struct{}),
 		done:            make(chan struct{}),
@@ -7,9 +7,11 @@ import (
 	"errors"
 	"fmt"
 	"io"
+	"log/slog"
 	"net"
 	"regexp"
 	"strings"
+	"sync"
 	"time"
 	"unicode/utf8"

@@ -49,8 +51,32 @@ const (
 	whoisServerAPNIC   = "whois.apnic.net:43"
 	whoisServerLACNIC  = "whois.lacnic.net:43"
 	whoisServerAFRINIC = "whois.afrinic.net:43"
+
+	// NIC.br registers Brazil's ASes under LACNIC.
+	whoisServerNICbr = "whois.registro.br:43"
 )

+// nicbrHeader starts the copyright notice at the top of NIC.br's WHOIS
+// answers. LACNIC's answer for an AS that NIC.br registers is NIC.br's, and
+// carries the same notice:
+//
+//	% Copyright (c) Nic.br - Use of this data is governed by the Use and
+//	% Privacy Policy at https://registro.br/upp . Distribution,
+const nicbrHeader = "% Copyright (c) Nic.br"
+
+// nicbrLimitReached is in the line that marks NIC.br's reduced answer to a
+// client over its query limit. NIC.br publishes no limit. Its Use and
+// Privacy Policy, https://registro.br/upp, says only that it restricts
+// queries to stop the data being taken in bulk.
+const nicbrLimitReached = "Query rate limit exceeded"
+
+// NICbrSkipTime is how long the client stops asking NIC.br after NIC.br
+// answers that the client is over its query limit. NIC.br publishes
+// neither the limit nor how long it lasts. An hour outlasts a limit
+// counted per minute or per hour. The ASes looked up meanwhile get
+// NIC.br's fields when they are next refreshed.
+const NICbrSkipTime = time.Hour
+
 // RIR identifiers.
 const (
 	RIRARIN   = "ARIN"
@@ -123,21 +149,42 @@ type Client struct {
 	// host:port. NewClient sets a net.Dialer's; tests replace it to reach
 	// local servers instead.
 	DialContext func(ctx context.Context, network, address string) (net.Conn, error)
+
+	// Now returns the current time. NewClient sets time.Now; tests replace
+	// it to move the clock.
+	Now func() time.Time
+
+	logger *slog.Logger
+
+	// mu guards nicbrSkipUntil.
+	mu sync.Mutex
+
+	// nicbrSkipUntil is when the client next asks NIC.br, after NIC.br
+	// answered that the client is over its query limit.
+	nicbrSkipUntil time.Time
 }

-// NewClient creates a new WHOIS client.
-func NewClient() *Client {
+// NewClient creates a new WHOIS client that logs to logger.
+func NewClient(logger *slog.Logger) *Client {
 	dialer := &net.Dialer{Timeout: dialTimeout}

-	return &Client{DialContext: dialer.DialContext}
+	return &Client{
+		DialContext: dialer.DialContext,
+		Now:         time.Now,
+		logger:      logger,
+	}
 }

 // LookupASN asks IANA which RIR holds the given ASN, asks that RIR, and
-// returns what the RIR's answer says. It returns ErrNoReferral when IANA
-// refers the AS to no RIR, and another error when a query fails, when the
-// RIR refuses it, when IANA's answer has neither a referral nor the AS's
-// block, when IANA refers the AS to a server none of the five RIRs run, or
-// when the RIR's answer carries none of ASNInfo's fields.
+// returns what the RIR's answer says. When LACNIC's answer is NIC.br's, it
+// returns what NIC.br's own server says instead, unless that query fails or
+// is skipped, as askNICbr says.
+//
+// It returns ErrNoReferral when IANA refers the AS to no RIR, and another
+// error when the query to IANA or to the RIR fails, when the RIR refuses
+// it, when IANA's answer has neither a referral nor the AS's block, when
+// IANA refers the AS to a server none of the five RIRs run, or when the
+// answer carries none of ASNInfo's fields.
 func (c *Client) LookupASN(ctx context.Context, asn int) (*ASNInfo, error) {
 	// Query IANA first to find the authoritative RIR
 	query := fmt.Sprintf("AS%d", asn)
@@ -169,18 +216,77 @@ func (c *Client) LookupASN(ctx context.Context, asn int) (*ASNInfo, error) {
 		return nil, fmt.Errorf("RIR query failed: %w", err)
 	}

-	// The fields come from the RIR's answer alone: IANA's describes the
-	// block the AS is in.
+	// The fields never come from IANA's answer, which describes the block
+	// the AS is in.
 	info := c.parseResponse(asn, rir, rirResp)
+
+	if rir == RIRLACNIC && strings.Contains(rirResp, nicbrHeader) {
+		info = c.askNICbr(ctx, query, info)
+	}
+
 	if info.isEmpty() {
 		return nil, fmt.Errorf("%s from %s: %w", query, whoisServer, errNoFields)
 	}

-	info.RawResponse = ianaResp + "\n" + rirResp
+	info.RawResponse = ianaResp + "\n" + info.RawResponse

 	return info, nil
 }

+// askNICbr asks NIC.br's server about an AS whose answer from LACNIC is
+// NIC.br's, and returns what NIC.br's answer says. LACNIC replaces each
+// accented letter in such an answer with U+FFFD. NIC.br's own server keeps
+// them, in Latin-1.
+//
+// askNICbr returns lacnicInfo, what LACNIC's answer says, and logs a
+// warning when the query fails, when NIC.br sends the reduced answer for a
+// client over its limit, or when NIC.br's answer carries none of ASNInfo's
+// fields. After the reduced answer, it returns lacnicInfo without asking
+// NIC.br until NICbrSkipTime has passed.
+func (c *Client) askNICbr(
+	ctx context.Context, asQuery string, lacnicInfo *ASNInfo,
+) *ASNInfo {
+	asn := lacnicInfo.ASN
+
+	c.mu.Lock()
+	skipping := c.Now().Before(c.nicbrSkipUntil)
+	c.mu.Unlock()
+
+	if skipping {
+		return lacnicInfo
+	}
+
+	answer, err := c.query(ctx, whoisServerNICbr, asQuery)
+	if err != nil {
+		c.logger.Warn("NIC.br query failed, keeping LACNIC's fields",
+			"asn", asn, "error", err)
+
+		return lacnicInfo
+	}
+
+	if strings.Contains(answer, nicbrLimitReached) {
+		c.mu.Lock()
+		c.nicbrSkipUntil = c.Now().Add(NICbrSkipTime)
+		c.mu.Unlock()
+
+		c.logger.Warn("NIC.br answered that the client is over its query "+
+			"limit, keeping LACNIC's fields and skipping NIC.br",
+			"asn", asn, "skip_for", NICbrSkipTime)
+
+		return lacnicInfo
+	}
+
+	info := c.parseResponse(asn, RIRLACNIC, answer)
+	if info.isEmpty() {
+		c.logger.Warn("NIC.br answered with no WHOIS fields, "+
+			"keeping LACNIC's fields", "asn", asn)
+
+		return lacnicInfo
+	}
+
+	return info
+}
+
 // isEmpty reports whether info holds none of the fields a WHOIS answer
 // fills, as when the answer is only comment and error lines.
 func (info *ASNInfo) isEmpty() bool {
@@ -276,9 +382,9 @@ func (c *Client) query(ctx context.Context, server, query string) (string, error
 }

 // toUTF8 returns a WHOIS answer in UTF-8. An answer that is valid UTF-8 is
-// returned as it is. Any other answer is decoded as Latin-1, which RIPE and
-// APNIC send. LACNIC, ARIN and AFRINIC send UTF-8. RIPE's database working
-// group lists the five here:
+// returned as it is. Any other answer is decoded as Latin-1, which RIPE,
+// APNIC and NIC.br send. LACNIC, ARIN and AFRINIC send UTF-8. RIPE's
+// database working group lists the five RIRs here:
 // https://www.ripe.net/ripe/mail/archives/db-wg/2024-May/008075.html
 func toUTF8(answer string) (string, error) {
 	if utf8.ValidString(answer) {
@@ -2,9 +2,13 @@ package whois_test

 import (
 	"bufio"
+	"bytes"
 	"context"
+	"encoding/json"
 	"errors"
 	"io"
+	"log/slog"
+	"maps"
 	"net"
 	"slices"
 	"strings"
@@ -50,6 +54,10 @@ changed:      2002-09-01
 source:       IANA
 `

+// lacnicReferral is IANA's answer for an AS in a block it gave to LACNIC,
+// shortened to the referral.
+const lacnicReferral = "refer:        whois.lacnic.net\n"
+
 // ripeRecord is RIPE's answer for the AS when asked to leave out person and
 // role objects. It starts with the as-block object of the block the AS is
 // in. Then come the abuse address as a comment, the AS's object and the
@@ -272,7 +280,7 @@ func lookUp(
 ) (*whois.ASNInfo, error) {
 	t.Helper()

-	client := whois.NewClient()
+	client := whois.NewClient(slog.New(slog.DiscardHandler))
 	client.DialContext = fakeServers(t, answers)

 	return client.LookupASN(t.Context(), testASN)
@@ -330,7 +338,7 @@ func TestLookupASNReadsTheRIRAnswer(t *testing.T) {
 func TestLookupASNReadsOnlyTheASAndItsOrganisation(t *testing.T) {
 	t.Parallel()

-	client := whois.NewClient()
+	client := whois.NewClient(slog.New(slog.DiscardHandler))
 	client.DialContext = fakeServers(t, map[string]string{
 		apnicIANAQuery: "refer:        whois.apnic.net\n",
 		apnicQuery:     apnicRecord,
@@ -421,7 +429,7 @@ func TestLookupASNStoresAnswersInUTF8(t *testing.T) {
 				"org-name:       " + latin1OrgName + "\n",
 		},
 		"LACNIC in UTF-8": {
-			ianaQuery: "refer:        whois.lacnic.net\n",
+			ianaQuery: lacnicReferral,
 			"whois.lacnic.net:43 " + asQuery: "aut-num:     AS3333\n" +
 				"owner:       " + utf8OrgName + "\n",
 		},
@@ -448,6 +456,312 @@ func TestLookupASNStoresAnswersInUTF8(t *testing.T) {
 	}
 }

+// lacnicServer is LACNIC's WHOIS server and nicbrServer NIC.br's, which
+// registers Brazil's ASes under LACNIC. brazilASN is one of those ASes.
+const (
+	lacnicServer = "whois.lacnic.net:43"
+	nicbrServer  = "whois.registro.br:43"
+	brazilASN    = 52751
+)
+
+// The keys of the queries LookupASN sends for brazilASN: IANA's, LACNIC's,
+// then NIC.br's.
+const (
+	brazilIANAQuery   = ianaServer + " AS52751"
+	brazilLACNICQuery = lacnicServer + " AS52751"
+	brazilNICbrQuery  = nicbrServer + " AS52751"
+)
+
+// The name of brazilASN's organisation in three forms: as LACNIC sends it,
+// with each accented letter replaced by U+FFFD; as NIC.br sends it, in
+// Latin-1; and in UTF-8.
+const (
+	lacnicBrazilOrgName = "Brasrede Telecomunica\U0000FFFD\U0000FFFDes LTDA"
+	nicbrBrazilOrgName  = "Brasrede Telecomunica\xe7\xf5es LTDA"
+	brazilOrgName       = "Brasrede Telecomunicações LTDA"
+)
+
+// nicbrCopyright is the copyright notice at the top of NIC.br's answers.
+const nicbrCopyright = "% Copyright (c) Nic.br - " +
+	"Use of this data is governed by the Use and\n" +
+	"% Privacy Policy at https://registro.br/upp . Distribution,\n" +
+	"% commercialization, reproduction, and use for advertising or similar\n" +
+	"% purposes are expressly prohibited.\n"
+
+// lacnicBrazilRecord is LACNIC's answer for brazilASN, recorded on
+// 2026-10-26. It is NIC.br's answer, which LACNIC passes on after a line
+// of its own and a space. It is shortened. The client's address is a
+// documentation address. The contact's name, the contact's object and the
+// footer are left out.
+const lacnicBrazilRecord = "% IP Client: 192.0.2.1\n" +
+	" " + nicbrCopyright + `% 2026-10-26T23:12:22-03:00 - 192.0.2.1
+
+aut-num:     AS52751
+owner:       ` + lacnicBrazilOrgName + `
+owner-c:     AVL217
+routing-c:   AVL217
+abuse-c:     AVL217
+created:     20120829
+changed:     20130306
+inetnum:     177.38.152.0/21
+inetnum:     2804:b34::/32
+inetnum:     191.243.4.0/22
+inetnum:     131.72.248.0/22
+inetnum:     138.94.228.0/22
+inetnum:     170.233.8.0/22
+inetnum:     170.246.4.0/22
+`
+
+// nicbrRecord is NIC.br's answer for brazilASN, in Latin-1, recorded on
+// 2026-10-26 and shortened as lacnicBrazilRecord is.
+const nicbrRecord = nicbrCopyright + `% 2026-10-26T23:14:41-03:00 - 192.0.2.1
+
+aut-num:     AS52751
+owner:       ` + nicbrBrazilOrgName + `
+owner-c:     AVL217
+routing-c:   AVL217
+abuse-c:     AVL217
+created:     20120829
+changed:     20130306
+inetnum:     177.38.152.0/21
+inetnum:     2804:b34::/32
+inetnum:     191.243.4.0/22
+inetnum:     131.72.248.0/22
+inetnum:     138.94.228.0/22
+inetnum:     170.233.8.0/22
+inetnum:     170.246.4.0/22
+`
+
+// nicbrOverLimit stands for NIC.br's answer for brazilASN to a client over
+// its query limit, and is not recorded. Such an answer leaves information
+// out, here the owner, and says so in its first line.
+const nicbrOverLimit = `% Query rate limit exceeded. Reduced information.
+
+aut-num:     AS52751
+created:     20120829
+changed:     20130306
+`
+
+// newNICbrClient returns a client whose fake servers send answers and
+// whose logger writes JSON lines to logs. It adds one to *nicbrAsks each
+// time it connects to NIC.br's server.
+func newNICbrClient(
+	t *testing.T, answers map[string]string, logs io.Writer, nicbrAsks *int,
+) *whois.Client {
+	t.Helper()
+
+	dial := fakeServers(t, answers)
+
+	client := whois.NewClient(slog.New(slog.NewJSONHandler(logs, nil)))
+	client.DialContext = func(
+		ctx context.Context, network, address string,
+	) (net.Conn, error) {
+		if address == nicbrServer {
+			*nicbrAsks++
+		}
+
+		return dial(ctx, network, address)
+	}
+
+	return client
+}
+
+// lookUpBrazilianAS looks brazilASN up with fake servers sending answers,
+// and reports whether the client connected to NIC.br's server.
+func lookUpBrazilianAS(
+	t *testing.T, answers map[string]string,
+) (*whois.ASNInfo, bool, error) {
+	t.Helper()
+
+	nicbrAsks := 0
+	client := newNICbrClient(t, answers, io.Discard, &nicbrAsks)
+
+	info, err := client.LookupASN(t.Context(), brazilASN)
+
+	return info, nicbrAsks > 0, err
+}
+
+// TestLookupASNTakesABrazilianASFromNICbr looks up an AS that NIC.br
+// registers. LACNIC's answer is NIC.br's with each accented letter replaced
+// by U+FFFD, so the client also asks NIC.br's server, which answers in
+// Latin-1. The organisation's name has its accented letters, in the parsed
+// field and in the stored answer.
+func TestLookupASNTakesABrazilianASFromNICbr(t *testing.T) {
+	t.Parallel()
+
+	info, _, err := lookUpBrazilianAS(t, map[string]string{
+		brazilIANAQuery:   lacnicReferral,
+		brazilLACNICQuery: lacnicBrazilRecord,
+		brazilNICbrQuery:  nicbrRecord,
+	})
+	if err != nil {
+		t.Fatalf("LookupASN: %v", err)
+	}
+
+	if info.OrgName != brazilOrgName || info.RIR != whois.RIRLACNIC {
+		t.Errorf("LookupASN org name, RIR = %q, %q; want %q, %q",
+			info.OrgName, info.RIR, brazilOrgName, whois.RIRLACNIC)
+	}
+
+	if !strings.Contains(info.RawResponse, brazilOrgName) {
+		t.Errorf("LookupASN answer = %q, want one holding %q",
+			info.RawResponse, brazilOrgName)
+	}
+
+	wantRegDate := time.Date(2012, 8, 29, 0, 0, 0, 0, time.UTC)
+	if info.RegDate == nil || !info.RegDate.Equal(wantRegDate) {
+		t.Errorf("LookupASN registered = %v, want %v",
+			info.RegDate, wantRegDate)
+	}
+}
+
+// TestLookupASNAsksNICbrOnlyForItsASes looks up an AS at LACNIC whose
+// answer does not carry NIC.br's copyright notice, as for an AS that LACNIC
+// registers itself. The client does not connect to NIC.br's server, which
+// would answer with a record, and keeps LACNIC's fields.
+func TestLookupASNAsksNICbrOnlyForItsASes(t *testing.T) {
+	t.Parallel()
+
+	info, askedNICbr, err := lookUpBrazilianAS(t, map[string]string{
+		brazilIANAQuery: lacnicReferral,
+		brazilLACNICQuery: "aut-num:     AS52751\n" +
+			"owner:       " + utf8OrgName + "\n",
+		brazilNICbrQuery: nicbrRecord,
+	})
+	if err != nil {
+		t.Fatalf("LookupASN: %v", err)
+	}
+
+	if askedNICbr {
+		t.Error("LookupASN asked NIC.br's server about an AS whose " +
+			"answer from LACNIC is not NIC.br's")
+	}
+
+	if info.OrgName != utf8OrgName {
+		t.Errorf("LookupASN org name = %q, want LACNIC's %q",
+			info.OrgName, utf8OrgName)
+	}
+}
+
+// TestLookupASNKeepsLACNICsAnswerWhenNICbrFails looks up an AS that NIC.br
+// registers when NIC.br's server refuses the connection, sends nothing, or
+// sends the reduced answer for a client over its query limit. The client
+// asks NIC.br, and the lookup still succeeds with LACNIC's fields. The
+// client logs one warning, which names the AS and the reason.
+func TestLookupASNKeepsLACNICsAnswerWhenNICbrFails(t *testing.T) {
+	t.Parallel()
+
+	for name, test := range map[string]struct {
+		nicbrAnswer map[string]string
+		reason      string
+	}{
+		"connection refused": {
+			nicbrAnswer: map[string]string{},
+			reason:      syscall.ECONNREFUSED.Error(),
+		},
+		"no answer": {
+			nicbrAnswer: map[string]string{brazilNICbrQuery: ""},
+			reason:      "no WHOIS fields",
+		},
+		"over the query limit": {
+			nicbrAnswer: map[string]string{brazilNICbrQuery: nicbrOverLimit},
+			reason:      "over its query limit",
+		},
+	} {
+		t.Run(name, func(t *testing.T) {
+			t.Parallel()
+
+			answers := map[string]string{
+				brazilIANAQuery:   lacnicReferral,
+				brazilLACNICQuery: lacnicBrazilRecord,
+			}
+			maps.Copy(answers, test.nicbrAnswer)
+
+			var logs bytes.Buffer
+
+			nicbrAsks := 0
+			client := newNICbrClient(t, answers, &logs, &nicbrAsks)
+
+			info, err := client.LookupASN(t.Context(), brazilASN)
+			if err != nil {
+				t.Fatalf("LookupASN: %v", err)
+			}
+
+			if nicbrAsks == 0 {
+				t.Error("LookupASN did not ask NIC.br's server")
+			}
+
+			if info.OrgName != lacnicBrazilOrgName {
+				t.Errorf("LookupASN org name = %q, want LACNIC's %q",
+					info.OrgName, lacnicBrazilOrgName)
+			}
+
+			var logged struct {
+				Level string `json:"level"`
+				ASN   int    `json:"asn"`
+			}
+
+			err = json.Unmarshal(logs.Bytes(), &logged)
+			if err != nil {
+				t.Fatalf("want one log line, got %q: %v", logs.String(), err)
+			}
+
+			if logged.Level != slog.LevelWarn.String() ||
+				logged.ASN != brazilASN ||
+				!strings.Contains(logs.String(), test.reason) {
+				t.Errorf("logged %q, want a warning for AS%d holding %q",
+					logs.String(), brazilASN, test.reason)
+			}
+		})
+	}
+}
+
+// TestLookupASNSkipsNICbrAfterItsLimit looks up an AS that NIC.br
+// registers three times with one client, while NIC.br answers that the
+// client is over its query limit. The first lookup asks NIC.br. The
+// second, a second before NICbrSkipTime has passed, does not. The third,
+// once NICbrSkipTime has passed, asks again. Each keeps LACNIC's fields.
+func TestLookupASNSkipsNICbrAfterItsLimit(t *testing.T) {
+	t.Parallel()
+
+	nicbrAsks := 0
+	client := newNICbrClient(t, map[string]string{
+		brazilIANAQuery:   lacnicReferral,
+		brazilLACNICQuery: lacnicBrazilRecord,
+		brazilNICbrQuery:  nicbrOverLimit,
+	}, io.Discard, &nicbrAsks)
+
+	start := time.Date(2026, 10, 27, 0, 0, 0, 0, time.UTC)
+	now := start
+	client.Now = func() time.Time { return now }
+
+	for _, step := range []struct {
+		after    time.Duration
+		wantAsks int
+	}{
+		{after: 0, wantAsks: 1},
+		{after: whois.NICbrSkipTime - time.Second, wantAsks: 1},
+		{after: whois.NICbrSkipTime, wantAsks: 2},
+	} {
+		now = start.Add(step.after)
+
+		info, err := client.LookupASN(t.Context(), brazilASN)
+		if err != nil {
+			t.Fatalf("LookupASN after %v: %v", step.after, err)
+		}
+
+		if nicbrAsks != step.wantAsks {
+			t.Errorf("after %v NIC.br's server was asked %d times, want %d",
+				step.after, nicbrAsks, step.wantAsks)
+		}
+
+		if info.OrgName != lacnicBrazilOrgName {
+			t.Errorf("LookupASN after %v org name = %q, want LACNIC's %q",
+				step.after, info.OrgName, lacnicBrazilOrgName)
+		}
+	}
+}
+
 // TestLookupASNReportsARefusal looks up an AS IANA refers to RIPE when RIPE
 // refuses the query, with each of the error lines RIPE refuses with. The
 // error holds RIPE's line, so the log says why the lookup failed.