Keep accented letters in AS names (closes #165)

The built-in AS list had no letter outside ASCII, and make asupdate
rejected today's as.csv, which added a country code column.

The generator now checks only the first three header columns and reads
past the rest; the parsing moved into parseCSV. The list is regenerated.
The new source spells some names differently and no longer lists AS 0
or the documentation AS numbers, so the tests that relied on those
entries changed.

A WHOIS answer that is not valid UTF-8 is now decoded as Latin-1, which
RIPE and APNIC send, before it is parsed and stored. LACNIC answers in
UTF-8.

Model: opus-5-5

internal/whois/whois.go

@@ -11,6 +11,9 @@ import (
 	"regexp"
 	"strings"
 	"time"
+	"unicode/utf8"
+
+	"golang.org/x/text/encoding/charmap"
 )
 
 // Timeout constants for WHOIS queries.
@@ -213,8 +216,8 @@ func (c *Client) queryRIR(
 	return answer, nil
 }
 
-// query performs a raw WHOIS query to the specified server. An answer that
-// reaches maxAnswerSize is an error.
+// query performs a raw WHOIS query to the specified server and returns the
+// answer in UTF-8. An answer that reaches maxAnswerSize is an error.
 func (c *Client) query(ctx context.Context, server, query string) (string, error) {
 	conn, err := c.DialContext(ctx, "tcp", server)
 	if err != nil {
@@ -261,7 +264,25 @@ func (c *Client) query(ctx context.Context, server, query string) (string, error
 		return "", fmt.Errorf("read response: %w", errAnswerTooLarge)
 	}
 
-	return sb.String(), nil
+	return toUTF8(sb.String())
+}
+
+// toUTF8 returns a WHOIS answer in UTF-8. An answer that is valid UTF-8 is
+// returned as it is. Any other answer is decoded as Latin-1, which RIPE and
+// APNIC send. LACNIC, ARIN and AFRINIC send UTF-8. RIPE's database working
+// group lists the five here:
+// https://www.ripe.net/ripe/mail/archives/db-wg/2024-May/008075.html
+func toUTF8(answer string) (string, error) {
+	if utf8.ValidString(answer) {
+		return answer, nil
+	}
+
+	decoded, err := charmap.ISO8859_1.NewDecoder().String(answer)
+	if err != nil {
+		return "", fmt.Errorf("decode Latin-1 answer: %w", err)
+	}
+
+	return decoded, nil
 }
 
 // parseIANAReferral extracts the RIR and WHOIS server from an IANA response.

internal/whois/whois_test.go

@@ -12,6 +12,7 @@ import (
 	"syscall"
 	"testing"
 	"time"
+	"unicode/utf8"
 
 	"git.eeqj.de/sneak/routewatch/internal/whois"
 )
@@ -273,6 +274,55 @@ func TestLookupASNAsksAPNICAndAFRINICToLeaveOutPersonalData(t *testing.T) {
 	}
 }
 
+// The name of an organisation with accented letters, in Latin-1, one byte for
+// each letter, and in UTF-8.
+const (
+	latin1OrgName = "Fundaci\xf3n Ra\xedces de Espa\xf1a"
+	utf8OrgName   = "Fundación Raíces de España"
+)
+
+// TestLookupASNStoresAnswersInUTF8 looks up an AS at RIPE, which answers in
+// Latin-1, and at LACNIC, which answers in UTF-8. Both answers give the
+// organisation's name with its accented letters in UTF-8, in the parsed field
+// and in the stored answer. Before the fix the Latin-1 answer was kept as
+// received, which is not valid UTF-8.
+func TestLookupASNStoresAnswersInUTF8(t *testing.T) {
+	t.Parallel()
+
+	for name, answers := range map[string]map[string]string{
+		"RIPE in Latin-1": {
+			ianaQuery: ianaReferral,
+			ripeQuery: "aut-num:        AS3333\n" +
+				"org-name:       " + latin1OrgName + "\n",
+		},
+		"LACNIC in UTF-8": {
+			ianaQuery: "refer:        whois.lacnic.net\n",
+			"whois.lacnic.net:43 " + asQuery: "aut-num:     AS3333\n" +
+				"owner:       " + utf8OrgName + "\n",
+		},
+	} {
+		t.Run(name, func(t *testing.T) {
+			t.Parallel()
+
+			info, err := lookUp(t, answers)
+			if err != nil {
+				t.Fatalf("LookupASN: %v", err)
+			}
+
+			if info.OrgName != utf8OrgName {
+				t.Errorf("LookupASN org name = %q, want %q",
+					info.OrgName, utf8OrgName)
+			}
+
+			if !utf8.ValidString(info.RawResponse) ||
+				!strings.Contains(info.RawResponse, utf8OrgName) {
+				t.Errorf("LookupASN answer = %q, want UTF-8 holding %q",
+					info.RawResponse, utf8OrgName)
+			}
+		})
+	}
+}
+
 // TestLookupASNReportsARefusal looks up an AS IANA refers to RIPE when RIPE
 // refuses the query, with each of the error lines RIPE refuses with. The
 // error holds RIPE's line, so the log says why the lookup failed.