The built-in AS list had no letter outside ASCII, and make asupdate rejected today's as.csv, which added a country code column. The generator now checks only the first three header columns and reads past the rest; the parsing moved into parseCSV. The list is regenerated. The new source spells some names differently and no longer lists AS 0 or the documentation AS numbers, so the tests that relied on those entries changed. A WHOIS answer that is not valid UTF-8 is now decoded as Latin-1, which RIPE and APNIC send, before it is parsed and stored. LACNIC answers in UTF-8. Model: opus-5-5
This commit was merged in pull request #172.
This commit is contained in:
@@ -33,6 +33,11 @@ tested.
|
|||||||
|
|
||||||
# Completed Steps
|
# Completed Steps
|
||||||
|
|
||||||
|
- 2026-10-27: AS names keep their accented letters. The AS list built into
|
||||||
|
RouteWatch is rebuilt from today's `as.csv`, which added a country code
|
||||||
|
column. That file no longer lists AS 0 or the documentation AS numbers. A
|
||||||
|
WHOIS answer that is not valid UTF-8 is decoded as Latin-1, which RIPE and
|
||||||
|
APNIC send, before it is parsed and stored (closes #165)
|
||||||
- 2026-10-27: the Age column on the AS and prefix pages reads in words, such as
|
- 2026-10-27: the Age column on the AS and prefix pages reads in words, such as
|
||||||
"5 minutes ago" or "2 weeks ago", instead of Go durations such as "360h0m0s
|
"5 minutes ago" or "2 weeks ago", instead of Go durations such as "360h0m0s
|
||||||
ago". An age under a minute reads in seconds, and one over a month in months
|
ago". An age under a minute reads in seconds, and one over a month in months
|
||||||
|
|||||||
+47
-29
@@ -6,6 +6,7 @@ import (
|
|||||||
"encoding/csv"
|
"encoding/csv"
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
"errors"
|
"errors"
|
||||||
|
"fmt"
|
||||||
"io"
|
"io"
|
||||||
"log"
|
"log"
|
||||||
"net/http"
|
"net/http"
|
||||||
@@ -16,6 +17,17 @@ import (
|
|||||||
|
|
||||||
const asCsvURL = "https://github.com/ipverse/asn-info/raw/refs/heads/master/as.csv"
|
const asCsvURL = "https://github.com/ipverse/asn-info/raw/refs/heads/master/as.csv"
|
||||||
|
|
||||||
|
// The first columns of as.csv, the ones the generator reads. It reads past
|
||||||
|
// the columns after them, such as the country code.
|
||||||
|
const (
|
||||||
|
asnColumn = "asn"
|
||||||
|
handleColumn = "handle"
|
||||||
|
descriptionColumn = "description"
|
||||||
|
columnsRead = 3
|
||||||
|
)
|
||||||
|
|
||||||
|
var errUnexpectedHeader = errors.New("unexpected CSV header")
|
||||||
|
|
||||||
// ASInfo represents information about an Autonomous System
|
// ASInfo represents information about an Autonomous System
|
||||||
type ASInfo struct {
|
type ASInfo struct {
|
||||||
ASN int `json:"asn"`
|
ASN int `json:"asn"`
|
||||||
@@ -45,24 +57,50 @@ func main() {
|
|||||||
log.Fatalf("Failed to fetch CSV: HTTP %d", resp.StatusCode)
|
log.Fatalf("Failed to fetch CSV: HTTP %d", resp.StatusCode)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Parse CSV
|
asInfos, err := parseCSV(resp.Body)
|
||||||
reader := csv.NewReader(resp.Body)
|
|
||||||
|
// Closed here rather than deferred: log.Fatalf exits without running
|
||||||
|
// deferred calls.
|
||||||
|
_ = resp.Body.Close()
|
||||||
|
|
||||||
|
if err != nil {
|
||||||
|
log.Fatalf("Failed to parse CSV: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
log.Printf("Parsed %d AS records", len(asInfos))
|
||||||
|
|
||||||
|
// Convert to JSON and output to stdout
|
||||||
|
encoder := json.NewEncoder(os.Stdout)
|
||||||
|
encoder.SetIndent("", "\t")
|
||||||
|
|
||||||
|
err = encoder.Encode(asInfos)
|
||||||
|
if err != nil {
|
||||||
|
log.Fatalf("Failed to encode JSON: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// parseCSV reads as.csv from r and returns its ASes, with the handle and the
|
||||||
|
// description as the file spells them. A header whose first columns are not
|
||||||
|
// the AS number, the handle and the description is an error. A record that
|
||||||
|
// does not parse is logged and skipped.
|
||||||
|
func parseCSV(r io.Reader) ([]ASInfo, error) {
|
||||||
|
reader := csv.NewReader(r)
|
||||||
|
|
||||||
// Read header
|
|
||||||
header, err := reader.Read()
|
header, err := reader.Read()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Fatalf("Failed to read CSV header: %v", err)
|
return nil, fmt.Errorf("read CSV header: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
if len(header) != 3 || header[0] != "asn" || header[1] != "handle" ||
|
if len(header) < columnsRead || header[0] != asnColumn ||
|
||||||
header[2] != "description" {
|
header[1] != handleColumn || header[2] != descriptionColumn {
|
||||||
log.Fatalf("Unexpected CSV header: %v", header)
|
return nil, fmt.Errorf("%w: %v", errUnexpectedHeader, header)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Read all records
|
|
||||||
var asInfos []ASInfo
|
var asInfos []ASInfo
|
||||||
|
|
||||||
for {
|
for {
|
||||||
|
// The reader returns an error for a record with a different number
|
||||||
|
// of fields than the header.
|
||||||
record, err := reader.Read()
|
record, err := reader.Read()
|
||||||
if errors.Is(err, io.EOF) {
|
if errors.Is(err, io.EOF) {
|
||||||
break
|
break
|
||||||
@@ -74,13 +112,6 @@ func main() {
|
|||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
|
||||||
const expectedFields = 3
|
|
||||||
if len(record) != expectedFields {
|
|
||||||
log.Printf("Skipping invalid record: %v", record)
|
|
||||||
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
|
|
||||||
asn, err := strconv.Atoi(record[0])
|
asn, err := strconv.Atoi(record[0])
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Printf("Invalid ASN %q: %v", record[0], err)
|
log.Printf("Invalid ASN %q: %v", record[0], err)
|
||||||
@@ -95,18 +126,5 @@ func main() {
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
// Closed here rather than deferred: log.Fatalf exits without running
|
return asInfos, nil
|
||||||
// deferred calls.
|
|
||||||
_ = resp.Body.Close()
|
|
||||||
|
|
||||||
log.Printf("Parsed %d AS records", len(asInfos))
|
|
||||||
|
|
||||||
// Convert to JSON and output to stdout
|
|
||||||
encoder := json.NewEncoder(os.Stdout)
|
|
||||||
encoder.SetIndent("", "\t")
|
|
||||||
|
|
||||||
err = encoder.Encode(asInfos)
|
|
||||||
if err != nil {
|
|
||||||
log.Fatalf("Failed to encode JSON: %v", err)
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,55 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"slices"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
// TestParseCSVReadsTodaysLayout parses an as.csv with the country code column
|
||||||
|
// that the source added after the description. It expects each AS with its
|
||||||
|
// handle and description spelled exactly as in the file, accents included.
|
||||||
|
func TestParseCSVReadsTodaysLayout(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
const asCSV = `asn,handle,description,country-code
|
||||||
|
1,LVLT-1,Level 3 Parent LLC,US
|
||||||
|
1251,FUNDACAO-AMPARO-PESQUISA,FUNDAÇÃO DE AMPARO À PESQUISA DO ESTADO SÃO PAULO,BR
|
||||||
|
`
|
||||||
|
|
||||||
|
got, err := parseCSV(strings.NewReader(asCSV))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("parseCSV: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
want := []ASInfo{
|
||||||
|
{ASN: 1, Handle: "LVLT-1", Description: "Level 3 Parent LLC"},
|
||||||
|
{
|
||||||
|
ASN: 1251,
|
||||||
|
Handle: "FUNDACAO-AMPARO-PESQUISA",
|
||||||
|
Description: "FUNDAÇÃO DE AMPARO À PESQUISA DO ESTADO " +
|
||||||
|
"SÃO PAULO",
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
if !slices.Equal(got, want) {
|
||||||
|
t.Errorf("parseCSV = %+v, want %+v", got, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestParseCSVRejectsAnotherHeader parses files whose first columns are not
|
||||||
|
// the AS number, the handle and the description. It expects an error rather
|
||||||
|
// than a list read from the wrong columns.
|
||||||
|
func TestParseCSVRejectsAnotherHeader(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
for _, asCSV := range []string{
|
||||||
|
"asn,handle\n1,LVLT-1\n",
|
||||||
|
"asn,description,handle\n1,Level 3 Parent LLC,LVLT-1\n",
|
||||||
|
} {
|
||||||
|
_, err := parseCSV(strings.NewReader(asCSV))
|
||||||
|
if err == nil {
|
||||||
|
t.Errorf("parseCSV(%q): no error", asCSV)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -10,6 +10,7 @@ require (
|
|||||||
github.com/rcrowley/go-metrics v0.0.0-20250401214520-65e299d6c5c9
|
github.com/rcrowley/go-metrics v0.0.0-20250401214520-65e299d6c5c9
|
||||||
go.uber.org/fx v1.24.0
|
go.uber.org/fx v1.24.0
|
||||||
golang.org/x/term v0.33.0
|
golang.org/x/term v0.33.0
|
||||||
|
golang.org/x/text v0.43.0
|
||||||
)
|
)
|
||||||
|
|
||||||
require (
|
require (
|
||||||
|
|||||||
@@ -28,5 +28,7 @@ golang.org/x/sys v0.34.0 h1:H5Y5sJ2L2JRdyv7ROF1he/lPdvFsd0mJHFw2ThKHxLA=
|
|||||||
golang.org/x/sys v0.34.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k=
|
golang.org/x/sys v0.34.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k=
|
||||||
golang.org/x/term v0.33.0 h1:NuFncQrRcaRvVmgRkvM3j/F00gWIAlcmlB8ACEKmGIg=
|
golang.org/x/term v0.33.0 h1:NuFncQrRcaRvVmgRkvM3j/F00gWIAlcmlB8ACEKmGIg=
|
||||||
golang.org/x/term v0.33.0/go.mod h1:s18+ql9tYWp1IfpV9DmCtQDDSRBUjKaw9M1eAv5UeF0=
|
golang.org/x/term v0.33.0/go.mod h1:s18+ql9tYWp1IfpV9DmCtQDDSRBUjKaw9M1eAv5UeF0=
|
||||||
|
golang.org/x/text v0.43.0 h1:1QivrlhwAsnMOcqgOdl9my7s9OyZl8O5/UGUTJVXoWc=
|
||||||
|
golang.org/x/text v0.43.0/go.mod h1:hbSIYA/amXcRXTFSZqh8tWHEoqRvvbKPJ2uz/KU+a+s=
|
||||||
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
|
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
|
||||||
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||||
|
|||||||
@@ -458,8 +458,8 @@ func TestPrefixDetailShowsOriginNames(t *testing.T) {
|
|||||||
// TestPagesSayOriginZeroIsNoKnownOrigin stores a route with origin 0, as the
|
// TestPagesSayOriginZeroIsNoKnownOrigin stores a route with origin 0, as the
|
||||||
// prefix handler stores a route whose AS path ends in an AS set of several
|
// prefix handler stores a route whose AS path ends in an AS set of several
|
||||||
// members, and checks that the prefix page and the prefix length page say it
|
// members, and checks that the prefix page and the prefix length page say it
|
||||||
// has no known origin, without a link to AS0 or the description of the
|
// has no known origin, without a link to AS0
|
||||||
// reserved AS 0 (https://git.eeqj.de/sneak/routewatch/issues/75).
|
// (https://git.eeqj.de/sneak/routewatch/issues/75).
|
||||||
func TestPagesSayOriginZeroIsNoKnownOrigin(t *testing.T) {
|
func TestPagesSayOriginZeroIsNoKnownOrigin(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
@@ -479,11 +479,6 @@ func TestPagesSayOriginZeroIsNoKnownOrigin(t *testing.T) {
|
|||||||
t.Fatalf("UpsertLiveRouteBatch: %v", err)
|
t.Fatalf("UpsertLiveRouteBatch: %v", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
reserved, ok := asinfo.Get(0)
|
|
||||||
if !ok {
|
|
||||||
t.Fatal("AS0 is not in the AS info data")
|
|
||||||
}
|
|
||||||
|
|
||||||
s := New(db, streamer.New(logger.New(), metrics.New()), logger.New(), &config.Config{})
|
s := New(db, streamer.New(logger.New(), metrics.New()), logger.New(), &config.Config{})
|
||||||
|
|
||||||
for _, path := range []string{"/prefix/" + testPrefixV4, prefixLengthPagePath} {
|
for _, path := range []string{"/prefix/" + testPrefixV4, prefixLengthPagePath} {
|
||||||
@@ -505,10 +500,6 @@ func TestPagesSayOriginZeroIsNoKnownOrigin(t *testing.T) {
|
|||||||
if strings.Contains(page, `href="/as/0"`) {
|
if strings.Contains(page, `href="/as/0"`) {
|
||||||
t.Errorf("%s links to AS0", path)
|
t.Errorf("%s links to AS0", path)
|
||||||
}
|
}
|
||||||
|
|
||||||
if strings.Contains(page, html.EscapeString(reserved.Description)) {
|
|
||||||
t.Errorf("%s shows AS0's description %q", path, reserved.Description)
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1327,7 +1318,7 @@ func TestASSearchAnswersMatches(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
cloudflare := asinfo.Info{
|
cloudflare := asinfo.Info{
|
||||||
ASN: 13335, Handle: "CLOUDFLARENET", Description: "Cloudflare, Inc.",
|
ASN: 13335, Handle: "CLOUDFLARENET", Description: "Cloudflare Inc.",
|
||||||
}
|
}
|
||||||
if !slices.Contains(data.Results, cloudflare) {
|
if !slices.Contains(data.Results, cloudflare) {
|
||||||
t.Errorf("results %v do not hold %v", data.Results, cloudflare)
|
t.Errorf("results %v do not hold %v", data.Results, cloudflare)
|
||||||
@@ -1400,8 +1391,9 @@ func TestASSearchShowsFirstMatchesAndTotal(t *testing.T) {
|
|||||||
// The AS numbers the AS detail tests ask for. storedASN originates the route
|
// The AS numbers the AS detail tests ask for. storedASN originates the route
|
||||||
// for testPrefixV4 and is in the database. routeOnlyASN originates the route
|
// for testPrefixV4 and is in the database. routeOnlyASN originates the route
|
||||||
// for testPrefixV6, but the database has no record of the AS itself, as when
|
// for testPrefixV6, but the database has no record of the AS itself, as when
|
||||||
// a route is stored before its AS. builtInOnlyASN is in the AS list built into
|
// a route is stored before its AS. Both are documentation AS numbers, which
|
||||||
// asinfo but not in the database. unknownASN is in neither.
|
// the AS list built into asinfo does not hold. builtInOnlyASN is in that list
|
||||||
|
// but not in the database. unknownASN is in neither.
|
||||||
const (
|
const (
|
||||||
storedASN = 64500
|
storedASN = 64500
|
||||||
routeOnlyASN = 64501
|
routeOnlyASN = 64501
|
||||||
@@ -1417,14 +1409,12 @@ const firstSeenCard = `<div class="info-label">First Seen</div>`
|
|||||||
|
|
||||||
// newServerWithStoredAS returns newServerWithTestRoutes's server with storedASN
|
// newServerWithStoredAS returns newServerWithTestRoutes's server with storedASN
|
||||||
// added to the database. It fails the test when the AS list built into asinfo
|
// added to the database. It fails the test when the AS list built into asinfo
|
||||||
// lacks routeOnlyASN or builtInOnlyASN, or holds unknownASN.
|
// lacks builtInOnlyASN or holds unknownASN.
|
||||||
func newServerWithStoredAS(t *testing.T) *Server {
|
func newServerWithStoredAS(t *testing.T) *Server {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
|
|
||||||
for _, asn := range []int{routeOnlyASN, builtInOnlyASN} {
|
if _, ok := asinfo.Get(builtInOnlyASN); !ok {
|
||||||
if _, ok := asinfo.Get(asn); !ok {
|
t.Fatalf("AS%d is not in the AS info data", builtInOnlyASN)
|
||||||
t.Fatalf("AS%d is not in the AS info data", asn)
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
if _, ok := asinfo.Get(unknownASN); ok {
|
if _, ok := asinfo.Get(unknownASN); ok {
|
||||||
@@ -1442,10 +1432,10 @@ func newServerWithStoredAS(t *testing.T) *Server {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// TestASDetailJSONAnswersBuiltInAS asks /api/v1/as/ for each of the ASes above.
|
// TestASDetailJSONAnswersBuiltInAS asks /api/v1/as/ for each of the ASes above.
|
||||||
// All but unknownASN answer 200 with the handle and description from the
|
// All but unknownASN answer 200 with the handle and description the built-in
|
||||||
// built-in AS list and their prefixes, as lists even when empty. Only storedASN
|
// AS list has for them, if any, and their prefixes, as lists even when empty.
|
||||||
// has a first_seen time. unknownASN answers 404. Before the fix builtInOnlyASN
|
// Only storedASN has a first_seen time. unknownASN answers 404. Before the fix
|
||||||
// and routeOnlyASN answered 404 too
|
// builtInOnlyASN and routeOnlyASN answered 404 too
|
||||||
// (https://git.eeqj.de/sneak/routewatch/issues/151).
|
// (https://git.eeqj.de/sneak/routewatch/issues/151).
|
||||||
func TestASDetailJSONAnswersBuiltInAS(t *testing.T) {
|
func TestASDetailJSONAnswersBuiltInAS(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
@@ -1499,13 +1489,13 @@ func TestASDetailJSONAnswersBuiltInAS(t *testing.T) {
|
|||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
|
||||||
info, _ := asinfo.Get(tt.asn)
|
handle, description := asinfo.GetHandle(tt.asn), asinfo.GetDescription(tt.asn)
|
||||||
got := body.Data.ASN
|
got := body.Data.ASN
|
||||||
|
|
||||||
if got.ASN != tt.asn || got.Handle != info.Handle ||
|
if got.ASN != tt.asn || got.Handle != handle ||
|
||||||
got.Description != info.Description {
|
got.Description != description {
|
||||||
t.Errorf("%s: AS %d %q %q, want %d %q %q", path, got.ASN,
|
t.Errorf("%s: AS %d %q %q, want %d %q %q", path, got.ASN,
|
||||||
got.Handle, got.Description, tt.asn, info.Handle, info.Description)
|
got.Handle, got.Description, tt.asn, handle, description)
|
||||||
}
|
}
|
||||||
|
|
||||||
if body.Data.IPv4Prefixes == nil || body.Data.IPv6Prefixes == nil {
|
if body.Data.IPv4Prefixes == nil || body.Data.IPv6Prefixes == nil {
|
||||||
@@ -1532,11 +1522,11 @@ func TestASDetailJSONAnswersBuiltInAS(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// TestASDetailPageShowsBuiltInAS asks /as/ for each of the ASes above. All but
|
// TestASDetailPageShowsBuiltInAS asks /as/ for each of the ASes above. All but
|
||||||
// unknownASN answer 200 with the handle and description from the built-in AS
|
// unknownASN answer 200 with the handle and description the built-in AS list
|
||||||
// list. storedASN and routeOnlyASN list their prefixes, and builtInOnlyASN says
|
// has for them, if any. storedASN and routeOnlyASN list their prefixes, and
|
||||||
// the AS announces none. Only storedASN shows a first-seen date, and no page
|
// builtInOnlyASN says the AS announces none. Only storedASN shows a first-seen
|
||||||
// shows a date that was never set. unknownASN answers 404. Before the fix
|
// date, and no page shows a date that was never set. unknownASN answers 404.
|
||||||
// builtInOnlyASN and routeOnlyASN answered 404 too
|
// Before the fix builtInOnlyASN and routeOnlyASN answered 404 too
|
||||||
// (https://git.eeqj.de/sneak/routewatch/issues/151).
|
// (https://git.eeqj.de/sneak/routewatch/issues/151).
|
||||||
func TestASDetailPageShowsBuiltInAS(t *testing.T) {
|
func TestASDetailPageShowsBuiltInAS(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
@@ -1572,12 +1562,11 @@ func TestASDetailPageShowsBuiltInAS(t *testing.T) {
|
|||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
|
||||||
info, _ := asinfo.Get(tt.asn)
|
|
||||||
page := rec.Body.String()
|
page := rec.Body.String()
|
||||||
|
|
||||||
for _, want := range []string{
|
for _, want := range []string{
|
||||||
html.EscapeString(info.Handle),
|
html.EscapeString(asinfo.GetHandle(tt.asn)),
|
||||||
html.EscapeString(info.Description),
|
html.EscapeString(asinfo.GetDescription(tt.asn)),
|
||||||
tt.wantText,
|
tt.wantText,
|
||||||
} {
|
} {
|
||||||
if !strings.Contains(page, want) {
|
if !strings.Contains(page, want) {
|
||||||
@@ -1599,17 +1588,11 @@ func TestASDetailPageShowsBuiltInAS(t *testing.T) {
|
|||||||
|
|
||||||
// TestNoOriginHasNoASAnswers stores a route with no known origin, which has
|
// TestNoOriginHasNoASAnswers stores a route with no known origin, which has
|
||||||
// origin AS 0 (https://git.eeqj.de/sneak/routewatch/issues/75). It expects
|
// origin AS 0 (https://git.eeqj.de/sneak/routewatch/issues/75). It expects
|
||||||
// /as/0 and /api/v1/as/0 to answer 404. It expects a search for "iana" to leave
|
// /as/0 and /api/v1/as/0 to answer 404.
|
||||||
// AS 0 out of its results and its total, although the AS list built into
|
|
||||||
// asinfo holds AS 0. That search has more matches than an answer holds, so it
|
|
||||||
// still lists maxASSearchResults of them.
|
|
||||||
func TestNoOriginHasNoASAnswers(t *testing.T) {
|
func TestNoOriginHasNoASAnswers(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
const (
|
const noOriginPrefix = "203.0.113.0/24"
|
||||||
noOriginPrefix = "203.0.113.0/24"
|
|
||||||
query = "iana"
|
|
||||||
)
|
|
||||||
|
|
||||||
s := newServerWithTestRoutes(t)
|
s := newServerWithTestRoutes(t)
|
||||||
|
|
||||||
@@ -1635,30 +1618,6 @@ func TestNoOriginHasNoASAnswers(t *testing.T) {
|
|||||||
path, rec.Code, http.StatusNotFound, rec.Body)
|
path, rec.Code, http.StatusNotFound, rec.Body)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
first, total := asinfo.Search(query, 1)
|
|
||||||
if len(first) != 1 || first[0].ASN != ristypes.NoOrigin {
|
|
||||||
t.Fatalf("AS%d is not the first match for %q in the AS info data",
|
|
||||||
ristypes.NoOrigin, query)
|
|
||||||
}
|
|
||||||
|
|
||||||
data := searchAS(t, s, "q="+query)
|
|
||||||
|
|
||||||
for _, info := range data.Results {
|
|
||||||
if info.ASN == ristypes.NoOrigin {
|
|
||||||
t.Errorf("the search for %q lists AS%d", query, info.ASN)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if data.Total != total-1 {
|
|
||||||
t.Errorf("the search for %q has a total of %d, want %d",
|
|
||||||
query, data.Total, total-1)
|
|
||||||
}
|
|
||||||
|
|
||||||
if len(data.Results) != maxASSearchResults {
|
|
||||||
t.Errorf("the search for %q lists %d matches, want %d",
|
|
||||||
query, len(data.Results), maxASSearchResults)
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// settledGoroutineCount lets transient goroutines finish, then reports the
|
// settledGoroutineCount lets transient goroutines finish, then reports the
|
||||||
|
|||||||
+24
-3
@@ -11,6 +11,9 @@ import (
|
|||||||
"regexp"
|
"regexp"
|
||||||
"strings"
|
"strings"
|
||||||
"time"
|
"time"
|
||||||
|
"unicode/utf8"
|
||||||
|
|
||||||
|
"golang.org/x/text/encoding/charmap"
|
||||||
)
|
)
|
||||||
|
|
||||||
// Timeout constants for WHOIS queries.
|
// Timeout constants for WHOIS queries.
|
||||||
@@ -213,8 +216,8 @@ func (c *Client) queryRIR(
|
|||||||
return answer, nil
|
return answer, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// query performs a raw WHOIS query to the specified server. An answer that
|
// query performs a raw WHOIS query to the specified server and returns the
|
||||||
// reaches maxAnswerSize is an error.
|
// answer in UTF-8. An answer that reaches maxAnswerSize is an error.
|
||||||
func (c *Client) query(ctx context.Context, server, query string) (string, error) {
|
func (c *Client) query(ctx context.Context, server, query string) (string, error) {
|
||||||
conn, err := c.DialContext(ctx, "tcp", server)
|
conn, err := c.DialContext(ctx, "tcp", server)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -261,7 +264,25 @@ func (c *Client) query(ctx context.Context, server, query string) (string, error
|
|||||||
return "", fmt.Errorf("read response: %w", errAnswerTooLarge)
|
return "", fmt.Errorf("read response: %w", errAnswerTooLarge)
|
||||||
}
|
}
|
||||||
|
|
||||||
return sb.String(), nil
|
return toUTF8(sb.String())
|
||||||
|
}
|
||||||
|
|
||||||
|
// toUTF8 returns a WHOIS answer in UTF-8. An answer that is valid UTF-8 is
|
||||||
|
// returned as it is. Any other answer is decoded as Latin-1, which RIPE and
|
||||||
|
// APNIC send. LACNIC, ARIN and AFRINIC send UTF-8. RIPE's database working
|
||||||
|
// group lists the five here:
|
||||||
|
// https://www.ripe.net/ripe/mail/archives/db-wg/2024-May/008075.html
|
||||||
|
func toUTF8(answer string) (string, error) {
|
||||||
|
if utf8.ValidString(answer) {
|
||||||
|
return answer, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
decoded, err := charmap.ISO8859_1.NewDecoder().String(answer)
|
||||||
|
if err != nil {
|
||||||
|
return "", fmt.Errorf("decode Latin-1 answer: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
return decoded, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// parseIANAReferral extracts the RIR and WHOIS server from an IANA response.
|
// parseIANAReferral extracts the RIR and WHOIS server from an IANA response.
|
||||||
|
|||||||
@@ -12,6 +12,7 @@ import (
|
|||||||
"syscall"
|
"syscall"
|
||||||
"testing"
|
"testing"
|
||||||
"time"
|
"time"
|
||||||
|
"unicode/utf8"
|
||||||
|
|
||||||
"git.eeqj.de/sneak/routewatch/internal/whois"
|
"git.eeqj.de/sneak/routewatch/internal/whois"
|
||||||
)
|
)
|
||||||
@@ -273,6 +274,55 @@ func TestLookupASNAsksAPNICAndAFRINICToLeaveOutPersonalData(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// The name of an organisation with accented letters, in Latin-1, one byte for
|
||||||
|
// each letter, and in UTF-8.
|
||||||
|
const (
|
||||||
|
latin1OrgName = "Fundaci\xf3n Ra\xedces de Espa\xf1a"
|
||||||
|
utf8OrgName = "Fundación Raíces de España"
|
||||||
|
)
|
||||||
|
|
||||||
|
// TestLookupASNStoresAnswersInUTF8 looks up an AS at RIPE, which answers in
|
||||||
|
// Latin-1, and at LACNIC, which answers in UTF-8. Both answers give the
|
||||||
|
// organisation's name with its accented letters in UTF-8, in the parsed field
|
||||||
|
// and in the stored answer. Before the fix the Latin-1 answer was kept as
|
||||||
|
// received, which is not valid UTF-8.
|
||||||
|
func TestLookupASNStoresAnswersInUTF8(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
for name, answers := range map[string]map[string]string{
|
||||||
|
"RIPE in Latin-1": {
|
||||||
|
ianaQuery: ianaReferral,
|
||||||
|
ripeQuery: "aut-num: AS3333\n" +
|
||||||
|
"org-name: " + latin1OrgName + "\n",
|
||||||
|
},
|
||||||
|
"LACNIC in UTF-8": {
|
||||||
|
ianaQuery: "refer: whois.lacnic.net\n",
|
||||||
|
"whois.lacnic.net:43 " + asQuery: "aut-num: AS3333\n" +
|
||||||
|
"owner: " + utf8OrgName + "\n",
|
||||||
|
},
|
||||||
|
} {
|
||||||
|
t.Run(name, func(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
info, err := lookUp(t, answers)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("LookupASN: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if info.OrgName != utf8OrgName {
|
||||||
|
t.Errorf("LookupASN org name = %q, want %q",
|
||||||
|
info.OrgName, utf8OrgName)
|
||||||
|
}
|
||||||
|
|
||||||
|
if !utf8.ValidString(info.RawResponse) ||
|
||||||
|
!strings.Contains(info.RawResponse, utf8OrgName) {
|
||||||
|
t.Errorf("LookupASN answer = %q, want UTF-8 holding %q",
|
||||||
|
info.RawResponse, utf8OrgName)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// TestLookupASNReportsARefusal looks up an AS IANA refers to RIPE when RIPE
|
// TestLookupASNReportsARefusal looks up an AS IANA refers to RIPE when RIPE
|
||||||
// refuses the query, with each of the error lines RIPE refuses with. The
|
// refuses the query, with each of the error lines RIPE refuses with. The
|
||||||
// error holds RIPE's line, so the log says why the lookup failed.
|
// error holds RIPE's line, so the log says why the lookup failed.
|
||||||
|
|||||||
Binary file not shown.
@@ -22,7 +22,7 @@ func TestGet(t *testing.T) {
|
|||||||
name: "AS1 Level 3",
|
name: "AS1 Level 3",
|
||||||
asn: 1,
|
asn: 1,
|
||||||
wantOK: true,
|
wantOK: true,
|
||||||
wantDesc: "Level 3 Parent, LLC",
|
wantDesc: "Level 3 Parent LLC",
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
name: "AS15169 Google",
|
name: "AS15169 Google",
|
||||||
@@ -36,10 +36,10 @@ func TestGet(t *testing.T) {
|
|||||||
wantOK: false,
|
wantOK: false,
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
name: "AS0 IANA Reserved",
|
name: "AS1251 with accented letters",
|
||||||
asn: 0,
|
asn: 1251,
|
||||||
wantOK: true,
|
wantOK: true,
|
||||||
wantDesc: "Internet Assigned Numbers Authority",
|
wantDesc: "FUNDAÇÃO DE AMPARO À PESQUISA DO ESTADO SÃO PAULO",
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -69,7 +69,7 @@ func TestGetDescription(t *testing.T) {
|
|||||||
asn int
|
asn int
|
||||||
want string
|
want string
|
||||||
}{
|
}{
|
||||||
{1, "Level 3 Parent, LLC"},
|
{1, "Level 3 Parent LLC"},
|
||||||
{999999999, ""},
|
{999999999, ""},
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -92,8 +92,7 @@ func TestGetHandle(t *testing.T) {
|
|||||||
asn int
|
asn int
|
||||||
want string
|
want string
|
||||||
}{
|
}{
|
||||||
{1, "LVLT"},
|
{1, "LVLT-1"},
|
||||||
{0, "IANA-RSVD"},
|
|
||||||
{999999999, ""},
|
{999999999, ""},
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -142,6 +141,11 @@ func TestSearch(t *testing.T) {
|
|||||||
query: "University",
|
query: "University",
|
||||||
wantMin: 10,
|
wantMin: 10,
|
||||||
},
|
},
|
||||||
|
{
|
||||||
|
name: "Search for São Paulo",
|
||||||
|
query: "São Paulo",
|
||||||
|
wantMin: 1,
|
||||||
|
},
|
||||||
{
|
{
|
||||||
name: "Non-existent org",
|
name: "Non-existent org",
|
||||||
query: "XYZABC123456",
|
query: "XYZABC123456",
|
||||||
@@ -189,13 +193,13 @@ func TestSearchEmptyQueryMatchesNothing(t *testing.T) {
|
|||||||
func TestSearchIgnoresCase(t *testing.T) {
|
func TestSearchIgnoresCase(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
const cloudflareASN = 13335 // handle CLOUDFLARENET, "Cloudflare, Inc."
|
const cloudflareASN = 13335 // handle CLOUDFLARENET, "Cloudflare Inc."
|
||||||
|
|
||||||
isCloudflare := func(info asinfo.Info) bool {
|
isCloudflare := func(info asinfo.Info) bool {
|
||||||
return info.ASN == cloudflareASN
|
return info.ASN == cloudflareASN
|
||||||
}
|
}
|
||||||
|
|
||||||
for _, query := range []string{"cloudflarenet", "CLOUDFLARE, INC."} {
|
for _, query := range []string{"cloudflarenet", "CLOUDFLARE INC."} {
|
||||||
results, _ := asinfo.Search(query, asinfo.Total())
|
results, _ := asinfo.Search(query, asinfo.Total())
|
||||||
if !slices.ContainsFunc(results, isCloudflare) {
|
if !slices.ContainsFunc(results, isCloudflare) {
|
||||||
t.Errorf("Search(%q) did not find AS%d", query, cloudflareASN)
|
t.Errorf("Search(%q) did not find AS%d", query, cloudflareASN)
|
||||||
|
|||||||
@@ -17,7 +17,7 @@ func ExampleGet() {
|
|||||||
func ExampleGetDescription() {
|
func ExampleGetDescription() {
|
||||||
desc := asinfo.GetDescription(13335)
|
desc := asinfo.GetDescription(13335)
|
||||||
fmt.Println(desc)
|
fmt.Println(desc)
|
||||||
// Output: Cloudflare, Inc.
|
// Output: Cloudflare Inc.
|
||||||
}
|
}
|
||||||
|
|
||||||
func ExampleSearch() {
|
func ExampleSearch() {
|
||||||
|
|||||||
Reference in New Issue
Block a user