clawbot
cd1b9b94a4
check / check (push) Canceled after 0s
parseCSV skipped a record on any read error and asked for the next one. When the download of as.csv broke off, the response body returned the same error on every read, so the generator logged it forever and make asupdate never finished. parseCSV now skips only a record that does not parse, a csv.ParseError. Any other read error ends it with that error, so the generator exits non-zero and script/asupdate keeps the old list. Model: opus-5-5
145 lines
3.4 KiB
Go
145 lines
3.4 KiB
Go
// Package main generates the embedded AS info data for the asinfo package
|
|
package main
|
|
|
|
import (
|
|
"context"
|
|
"encoding/csv"
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"io"
|
|
"log"
|
|
"net/http"
|
|
"os"
|
|
"strconv"
|
|
"strings"
|
|
)
|
|
|
|
const asCsvURL = "https://github.com/ipverse/as-metadata/raw/refs/heads/master/as.csv"
|
|
|
|
// The first columns of as.csv, the ones the generator reads. It reads past
|
|
// the columns after them, such as the country code.
|
|
const (
|
|
asnColumn = "asn"
|
|
handleColumn = "handle"
|
|
descriptionColumn = "description"
|
|
columnsRead = 3
|
|
)
|
|
|
|
var (
|
|
errUnexpectedHeader = errors.New("unexpected CSV header")
|
|
errNoASes = errors.New("no AS in the CSV")
|
|
)
|
|
|
|
// ASInfo represents information about an Autonomous System
|
|
type ASInfo struct {
|
|
ASN int `json:"asn"`
|
|
Handle string `json:"handle"`
|
|
Description string `json:"description"`
|
|
}
|
|
|
|
func main() {
|
|
// Configure logger to write to stderr
|
|
log.SetOutput(os.Stderr)
|
|
|
|
// Fetch the CSV data
|
|
log.Println("Fetching AS CSV data...")
|
|
|
|
req, err := http.NewRequestWithContext(
|
|
context.Background(), http.MethodGet, asCsvURL, nil)
|
|
if err != nil {
|
|
log.Fatalf("Failed to fetch CSV: %v", err)
|
|
}
|
|
|
|
resp, err := http.DefaultClient.Do(req)
|
|
if err != nil {
|
|
log.Fatalf("Failed to fetch CSV: %v", err)
|
|
}
|
|
|
|
if resp.StatusCode != http.StatusOK {
|
|
log.Fatalf("Failed to fetch CSV: HTTP %d", resp.StatusCode)
|
|
}
|
|
|
|
asInfos, err := parseCSV(resp.Body)
|
|
|
|
// Closed here rather than deferred: log.Fatalf exits without running
|
|
// deferred calls.
|
|
_ = resp.Body.Close()
|
|
|
|
if err != nil {
|
|
log.Fatalf("Failed to parse CSV: %v", err)
|
|
}
|
|
|
|
log.Printf("Parsed %d AS records", len(asInfos))
|
|
|
|
// Convert to JSON and output to stdout
|
|
encoder := json.NewEncoder(os.Stdout)
|
|
encoder.SetIndent("", "\t")
|
|
|
|
err = encoder.Encode(asInfos)
|
|
if err != nil {
|
|
log.Fatalf("Failed to encode JSON: %v", err)
|
|
}
|
|
}
|
|
|
|
// parseCSV reads as.csv from r and returns its ASes, with the handle and the
|
|
// description as the file spells them. A header whose first columns are not
|
|
// the AS number, the handle and the description is an error. A record that
|
|
// does not parse is logged and skipped. A read that fails, such as on a broken
|
|
// download, is an error. A file left with no AS is an error.
|
|
func parseCSV(r io.Reader) ([]ASInfo, error) {
|
|
reader := csv.NewReader(r)
|
|
|
|
header, err := reader.Read()
|
|
if err != nil {
|
|
return nil, fmt.Errorf("read CSV header: %w", err)
|
|
}
|
|
|
|
if len(header) < columnsRead || header[0] != asnColumn ||
|
|
header[1] != handleColumn || header[2] != descriptionColumn {
|
|
return nil, fmt.Errorf("%w: %v", errUnexpectedHeader, header)
|
|
}
|
|
|
|
var asInfos []ASInfo
|
|
|
|
for {
|
|
// The reader returns an error for a record with a different number
|
|
// of fields than the header.
|
|
record, err := reader.Read()
|
|
if errors.Is(err, io.EOF) {
|
|
break
|
|
}
|
|
|
|
if _, ok := errors.AsType[*csv.ParseError](err); ok {
|
|
log.Printf("Error reading CSV record: %v", err)
|
|
|
|
continue
|
|
}
|
|
|
|
// Any other error comes from the download, such as a broken
|
|
// connection. Every later read returns it again.
|
|
if err != nil {
|
|
return nil, fmt.Errorf("read CSV record: %w", err)
|
|
}
|
|
|
|
asn, err := strconv.Atoi(record[0])
|
|
if err != nil {
|
|
log.Printf("Invalid ASN %q: %v", record[0], err)
|
|
|
|
continue
|
|
}
|
|
|
|
asInfos = append(asInfos, ASInfo{
|
|
ASN: asn,
|
|
Handle: strings.TrimSpace(record[1]),
|
|
Description: strings.TrimSpace(record[2]),
|
|
})
|
|
}
|
|
|
|
if len(asInfos) == 0 {
|
|
return nil, errNoASes
|
|
}
|
|
|
|
return asInfos, nil
|
|
}
|