Keep accented letters in AS names (closes #165)

The built-in AS list had no letter outside ASCII, and make asupdate
rejected today's as.csv, which added a country code column.

The generator now checks only the first three header columns and reads
past the rest; the parsing moved into parseCSV. The list is regenerated.
The new source spells some names differently and no longer lists AS 0
or the documentation AS numbers, so the tests that relied on those
entries changed.

A WHOIS answer that is not valid UTF-8 is now decoded as Latin-1, which
RIPE and APNIC send, before it is parsed and stored. LACNIC answers in
UTF-8.

Model: opus-5-5

internal/whois/whois_test.go

@@ -12,6 +12,7 @@ import (
 	"syscall"
 	"testing"
 	"time"
+	"unicode/utf8"
 
 	"git.eeqj.de/sneak/routewatch/internal/whois"
 )
@@ -273,6 +274,55 @@ func TestLookupASNAsksAPNICAndAFRINICToLeaveOutPersonalData(t *testing.T) {
 	}
 }
 
+// The name of an organisation with accented letters, in Latin-1, one byte for
+// each letter, and in UTF-8.
+const (
+	latin1OrgName = "Fundaci\xf3n Ra\xedces de Espa\xf1a"
+	utf8OrgName   = "Fundación Raíces de España"
+)
+
+// TestLookupASNStoresAnswersInUTF8 looks up an AS at RIPE, which answers in
+// Latin-1, and at LACNIC, which answers in UTF-8. Both answers give the
+// organisation's name with its accented letters in UTF-8, in the parsed field
+// and in the stored answer. Before the fix the Latin-1 answer was kept as
+// received, which is not valid UTF-8.
+func TestLookupASNStoresAnswersInUTF8(t *testing.T) {
+	t.Parallel()
+
+	for name, answers := range map[string]map[string]string{
+		"RIPE in Latin-1": {
+			ianaQuery: ianaReferral,
+			ripeQuery: "aut-num:        AS3333\n" +
+				"org-name:       " + latin1OrgName + "\n",
+		},
+		"LACNIC in UTF-8": {
+			ianaQuery: "refer:        whois.lacnic.net\n",
+			"whois.lacnic.net:43 " + asQuery: "aut-num:     AS3333\n" +
+				"owner:       " + utf8OrgName + "\n",
+		},
+	} {
+		t.Run(name, func(t *testing.T) {
+			t.Parallel()
+
+			info, err := lookUp(t, answers)
+			if err != nil {
+				t.Fatalf("LookupASN: %v", err)
+			}
+
+			if info.OrgName != utf8OrgName {
+				t.Errorf("LookupASN org name = %q, want %q",
+					info.OrgName, utf8OrgName)
+			}
+
+			if !utf8.ValidString(info.RawResponse) ||
+				!strings.Contains(info.RawResponse, utf8OrgName) {
+				t.Errorf("LookupASN answer = %q, want UTF-8 holding %q",
+					info.RawResponse, utf8OrgName)
+			}
+		})
+	}
+}
+
 // TestLookupASNReportsARefusal looks up an AS IANA refers to RIPE when RIPE
 // refuses the query, with each of the error lines RIPE refuses with. The
 // error holds RIPE's line, so the log says why the lookup failed.