Keep accented letters in AS names (closes #165)

The built-in AS list had no letter outside ASCII, and make asupdate
rejected today's as.csv, which added a country code column.

The generator now checks only the first three header columns and reads
past the rest; the parsing moved into parseCSV. The list is regenerated.
The new source spells some names differently and no longer lists AS 0
or the documentation AS numbers, so the tests that relied on those
entries changed.

A WHOIS answer that is not valid UTF-8 is now decoded as Latin-1, which
RIPE and APNIC send, before it is parsed and stored. LACNIC answers in
UTF-8.

Model: opus-5-5

cmd/asinfo-gen/main.go

@@ -6,6 +6,7 @@ import (
 	"encoding/csv"
 	"encoding/json"
 	"errors"
+	"fmt"
 	"io"
 	"log"
 	"net/http"
@@ -16,6 +17,17 @@ import (
 
 const asCsvURL = "https://github.com/ipverse/asn-info/raw/refs/heads/master/as.csv"
 
+// The first columns of as.csv, the ones the generator reads. It reads past
+// the columns after them, such as the country code.
+const (
+	asnColumn         = "asn"
+	handleColumn      = "handle"
+	descriptionColumn = "description"
+	columnsRead       = 3
+)
+
+var errUnexpectedHeader = errors.New("unexpected CSV header")
+
 // ASInfo represents information about an Autonomous System
 type ASInfo struct {
 	ASN         int    `json:"asn"`
@@ -45,24 +57,50 @@ func main() {
 		log.Fatalf("Failed to fetch CSV: HTTP %d", resp.StatusCode)
 	}
 
-	// Parse CSV
-	reader := csv.NewReader(resp.Body)
+	asInfos, err := parseCSV(resp.Body)
+
+	// Closed here rather than deferred: log.Fatalf exits without running
+	// deferred calls.
+	_ = resp.Body.Close()
+
+	if err != nil {
+		log.Fatalf("Failed to parse CSV: %v", err)
+	}
+
+	log.Printf("Parsed %d AS records", len(asInfos))
+
+	// Convert to JSON and output to stdout
+	encoder := json.NewEncoder(os.Stdout)
+	encoder.SetIndent("", "\t")
+
+	err = encoder.Encode(asInfos)
+	if err != nil {
+		log.Fatalf("Failed to encode JSON: %v", err)
+	}
+}
+
+// parseCSV reads as.csv from r and returns its ASes, with the handle and the
+// description as the file spells them. A header whose first columns are not
+// the AS number, the handle and the description is an error. A record that
+// does not parse is logged and skipped.
+func parseCSV(r io.Reader) ([]ASInfo, error) {
+	reader := csv.NewReader(r)
 
-	// Read header
 	header, err := reader.Read()
 	if err != nil {
-		log.Fatalf("Failed to read CSV header: %v", err)
+		return nil, fmt.Errorf("read CSV header: %w", err)
 	}
 
-	if len(header) != 3 || header[0] != "asn" || header[1] != "handle" ||
-		header[2] != "description" {
-		log.Fatalf("Unexpected CSV header: %v", header)
+	if len(header) < columnsRead || header[0] != asnColumn ||
+		header[1] != handleColumn || header[2] != descriptionColumn {
+		return nil, fmt.Errorf("%w: %v", errUnexpectedHeader, header)
 	}
 
-	// Read all records
 	var asInfos []ASInfo
 
 	for {
+		// The reader returns an error for a record with a different number
+		// of fields than the header.
 		record, err := reader.Read()
 		if errors.Is(err, io.EOF) {
 			break
@@ -74,13 +112,6 @@ func main() {
 			continue
 		}
 
-		const expectedFields = 3
-		if len(record) != expectedFields {
-			log.Printf("Skipping invalid record: %v", record)
-
-			continue
-		}
-
 		asn, err := strconv.Atoi(record[0])
 		if err != nil {
 			log.Printf("Invalid ASN %q: %v", record[0], err)
@@ -95,18 +126,5 @@ func main() {
 		})
 	}
 
-	// Closed here rather than deferred: log.Fatalf exits without running
-	// deferred calls.
-	_ = resp.Body.Close()
-
-	log.Printf("Parsed %d AS records", len(asInfos))
-
-	// Convert to JSON and output to stdout
-	encoder := json.NewEncoder(os.Stdout)
-	encoder.SetIndent("", "\t")
-
-	err = encoder.Encode(asInfos)
-	if err != nil {
-		log.Fatalf("Failed to encode JSON: %v", err)
-	}
+	return asInfos, nil
 }

cmd/asinfo-gen/main_test.go

@@ -0,0 +1,55 @@
+package main
+
+import (
+	"slices"
+	"strings"
+	"testing"
+)
+
+// TestParseCSVReadsTodaysLayout parses an as.csv with the country code column
+// that the source added after the description. It expects each AS with its
+// handle and description spelled exactly as in the file, accents included.
+func TestParseCSVReadsTodaysLayout(t *testing.T) {
+	t.Parallel()
+
+	const asCSV = `asn,handle,description,country-code
+1,LVLT-1,Level 3 Parent LLC,US
+1251,FUNDACAO-AMPARO-PESQUISA,FUNDAÇÃO DE AMPARO À PESQUISA DO ESTADO SÃO PAULO,BR
+`
+
+	got, err := parseCSV(strings.NewReader(asCSV))
+	if err != nil {
+		t.Fatalf("parseCSV: %v", err)
+	}
+
+	want := []ASInfo{
+		{ASN: 1, Handle: "LVLT-1", Description: "Level 3 Parent LLC"},
+		{
+			ASN:    1251,
+			Handle: "FUNDACAO-AMPARO-PESQUISA",
+			Description: "FUNDAÇÃO DE AMPARO À PESQUISA DO ESTADO " +
+				"SÃO PAULO",
+		},
+	}
+
+	if !slices.Equal(got, want) {
+		t.Errorf("parseCSV = %+v, want %+v", got, want)
+	}
+}
+
+// TestParseCSVRejectsAnotherHeader parses files whose first columns are not
+// the AS number, the handle and the description. It expects an error rather
+// than a list read from the wrong columns.
+func TestParseCSVRejectsAnotherHeader(t *testing.T) {
+	t.Parallel()
+
+	for _, asCSV := range []string{
+		"asn,handle\n1,LVLT-1\n",
+		"asn,description,handle\n1,Level 3 Parent LLC,LVLT-1\n",
+	} {
+		_, err := parseCSV(strings.NewReader(asCSV))
+		if err == nil {
+			t.Errorf("parseCSV(%q): no error", asCSV)
+		}
+	}
+}