3 Commits
Author SHA1 Message Date
sneak 50c8e9d276 WIP routewatch lint fixes
check / check (push) Canceled after 0s
Model: opus-5-5
2026-10-14 13:03:14 +00:00
sneak c5ae72fca7 WIP db-tests lint fixes
Model: opus-5-5
2026-10-14 13:03:14 +00:00
sneak c9fb637ecd WIP: lint fixes in the smaller packages
Model: opus-5-5
2026-10-14 13:03:14 +00:00
31 changed files with 1300 additions and 911 deletions

No files matched your search

+22 -7
View File
@@ -2,8 +2,10 @@
package main package main
import ( import (
"context"
"encoding/csv" "encoding/csv"
"encoding/json" "encoding/json"
"errors"
"io" "io"
"log" "log"
"net/http" "net/http"
@@ -27,13 +29,17 @@ func main() {
// Fetch the CSV data // Fetch the CSV data
log.Println("Fetching AS CSV data...") log.Println("Fetching AS CSV data...")
resp, err := http.Get(asCsvURL)
req, err := http.NewRequestWithContext(
context.Background(), http.MethodGet, asCsvURL, nil)
if err != nil {
log.Fatalf("Failed to fetch CSV: %v", err)
}
resp, err := http.DefaultClient.Do(req)
if err != nil { if err != nil {
log.Fatalf("Failed to fetch CSV: %v", err) log.Fatalf("Failed to fetch CSV: %v", err)
} }
defer func() {
_ = resp.Body.Close()
}()
if resp.StatusCode != http.StatusOK { if resp.StatusCode != http.StatusOK {
log.Fatalf("Failed to fetch CSV: HTTP %d", resp.StatusCode) log.Fatalf("Failed to fetch CSV: HTTP %d", resp.StatusCode)
@@ -48,17 +54,20 @@ func main() {
log.Fatalf("Failed to read CSV header: %v", err) log.Fatalf("Failed to read CSV header: %v", err)
} }
if len(header) != 3 || header[0] != "asn" || header[1] != "handle" || header[2] != "description" { if len(header) != 3 || header[0] != "asn" || header[1] != "handle" ||
header[2] != "description" {
log.Fatalf("Unexpected CSV header: %v", header) log.Fatalf("Unexpected CSV header: %v", header)
} }
// Read all records // Read all records
var asInfos []ASInfo var asInfos []ASInfo
for { for {
record, err := reader.Read() record, err := reader.Read()
if err == io.EOF { if errors.Is(err, io.EOF) {
break break
} }
if err != nil { if err != nil {
log.Printf("Error reading CSV record: %v", err) log.Printf("Error reading CSV record: %v", err)
@@ -86,12 +95,18 @@ func main() {
}) })
} }
// Closed here rather than deferred: log.Fatalf exits without running
// deferred calls.
_ = resp.Body.Close()
log.Printf("Parsed %d AS records", len(asInfos)) log.Printf("Parsed %d AS records", len(asInfos))
// Convert to JSON and output to stdout // Convert to JSON and output to stdout
encoder := json.NewEncoder(os.Stdout) encoder := json.NewEncoder(os.Stdout)
encoder.SetIndent("", "\t") encoder.SetIndent("", "\t")
if err := encoder.Encode(asInfos); err != nil {
err = encoder.Encode(asInfos)
if err != nil {
log.Fatalf("Failed to encode JSON: %v", err) log.Fatalf("Failed to encode JSON: %v", err)
} }
} }
+10 -7
View File
@@ -26,9 +26,17 @@ func main() {
// Register raw message handler that prints to stdout // Register raw message handler that prints to stdout
s.RegisterRawHandler(func(line string) { s.RegisterRawHandler(func(line string) {
fmt.Println(line) _, _ = fmt.Fprintln(os.Stdout, line)
}) })
// Start streaming before any defer: log.Fatal exits without running
// deferred calls.
err := s.Start()
if err != nil {
log.Fatal("Failed to start streamer:", err)
}
defer s.Stop()
// Set up context with cancellation // Set up context with cancellation
ctx, cancel := context.WithCancel(context.Background()) ctx, cancel := context.WithCancel(context.Background())
defer cancel() defer cancel()
@@ -36,18 +44,13 @@ func main() {
// Handle signals // Handle signals
sigChan := make(chan os.Signal, 1) sigChan := make(chan os.Signal, 1)
signal.Notify(sigChan, os.Interrupt, syscall.SIGTERM) signal.Notify(sigChan, os.Interrupt, syscall.SIGTERM)
go func() { go func() {
<-sigChan <-sigChan
log.Println("Received shutdown signal") log.Println("Received shutdown signal")
cancel() cancel()
}() }()
// Start streaming
if err := s.Start(); err != nil {
log.Fatal("Failed to start streamer:", err)
}
defer s.Stop()
// Wait for context cancellation // Wait for context cancellation
<-ctx.Done() <-ctx.Done()
} }
+27 -12
View File
@@ -2,6 +2,7 @@
package config package config
import ( import (
"errors"
"fmt" "fmt"
"os" "os"
"path/filepath" "path/filepath"
@@ -27,6 +28,12 @@ const (
maxPort = 65535 maxPort = 65535
) )
var (
errRelativeXDGDataHome = errors.New("XDG_DATA_HOME must be an absolute path")
errUnsupportedOS = errors.New("unsupported operating system")
errInvalidPort = errors.New("PORT must be a whole number")
)
// Config holds configuration for the entire application // Config holds configuration for the entire application
type Config struct { type Config struct {
// StateDir is the directory for all application state (database, snapshots) // StateDir is the directory for all application state (database, snapshots)
@@ -38,11 +45,14 @@ type Config struct {
// MaxRuntime is the maximum runtime (0 = run forever) // MaxRuntime is the maximum runtime (0 = run forever)
MaxRuntime time.Duration MaxRuntime time.Duration
// EnableBatchedDatabaseWrites enables batched database operations for better performance // EnableBatchedDatabaseWrites enables batched database operations for
// better performance
EnableBatchedDatabaseWrites bool EnableBatchedDatabaseWrites bool
// RouteExpirationTimeout is how long a route can go without being refreshed before expiring // RouteExpirationTimeout is how long a route can go without being
// Default is 2 hours which is conservative for BGP (typical BGP hold time is 90-180 seconds) // refreshed before expiring
// Default is 2 hours which is conservative for BGP (typical BGP hold
// time is 90-180 seconds)
RouteExpirationTimeout time.Duration RouteExpirationTimeout time.Duration
} }
@@ -59,11 +69,14 @@ func New() (*Config, error) {
} }
return &Config{ return &Config{
StateDir: stateDir, StateDir: stateDir,
Port: port, Port: port,
MaxRuntime: 0, // Run forever by default // Run forever by default
EnableBatchedDatabaseWrites: true, // Enable batching by default MaxRuntime: 0,
RouteExpirationTimeout: defaultRouteExpirationMinutes * time.Minute, // For active route monitoring // Enable batching by default
EnableBatchedDatabaseWrites: true,
// For active route monitoring
RouteExpirationTimeout: defaultRouteExpirationMinutes * time.Minute,
}, nil }, nil
} }
@@ -89,7 +102,7 @@ func getStateDirectory() (string, error) {
// the database somewhere unexpected. // the database somewhere unexpected.
xdgData := os.Getenv("XDG_DATA_HOME") xdgData := os.Getenv("XDG_DATA_HOME")
if xdgData != "" && !filepath.IsAbs(xdgData) { if xdgData != "" && !filepath.IsAbs(xdgData) {
return "", fmt.Errorf("XDG_DATA_HOME must be an absolute path, got %q", xdgData) return "", fmt.Errorf("%w, got %q", errRelativeXDGDataHome, xdgData)
} }
// Unix-like: /var/lib/berlin.sneak.app.routewatch if root, else XDG_DATA_HOME // Unix-like: /var/lib/berlin.sneak.app.routewatch if root, else XDG_DATA_HOME
@@ -111,7 +124,7 @@ func getStateDirectory() (string, error) {
return filepath.Join(home, ".local", "share", AppIdentifier), nil return filepath.Join(home, ".local", "share", AppIdentifier), nil
default: default:
return "", fmt.Errorf("unsupported operating system: %s", runtime.GOOS) return "", fmt.Errorf("%w: %s", errUnsupportedOS, runtime.GOOS)
} }
} }
@@ -125,7 +138,8 @@ func getPort() (int, error) {
// ParseUint, unlike Atoi, refuses a sign: the health check URL cannot use "+9090" // ParseUint, unlike Atoi, refuses a sign: the health check URL cannot use "+9090"
port, err := strconv.ParseUint(value, 10, 0) port, err := strconv.ParseUint(value, 10, 0)
if err != nil || port < 1 || port > maxPort { if err != nil || port < 1 || port > maxPort {
return 0, fmt.Errorf("PORT must be a whole number from 1 to %d, got %q", maxPort, value) return 0, fmt.Errorf("%w from 1 to %d, got %q",
errInvalidPort, maxPort, value)
} }
return int(port), nil return int(port), nil
@@ -134,7 +148,8 @@ func getPort() (int, error) {
// EnsureDirectories creates all necessary directories if they don't exist // EnsureDirectories creates all necessary directories if they don't exist
func (c *Config) EnsureDirectories() error { func (c *Config) EnsureDirectories() error {
// Ensure state directory exists // Ensure state directory exists
if err := os.MkdirAll(c.StateDir, dirPermissions); err != nil { err := os.MkdirAll(c.StateDir, dirPermissions)
if err != nil {
return fmt.Errorf("failed to create state directory: %w", err) return fmt.Errorf("failed to create state directory: %w", err)
} }
@@ -36,7 +36,8 @@ func TestNewRefusesInvalidPort(t *testing.T) {
t.Setenv("PORT", value) t.Setenv("PORT", value)
t.Setenv("XDG_DATA_HOME", "") t.Setenv("XDG_DATA_HOME", "")
if _, err := New(); err == nil { _, err := New()
if err == nil {
t.Errorf("New() with PORT=%q returned no error", value) t.Errorf("New() with PORT=%q returned no error", value)
} }
}) })
@@ -51,12 +52,16 @@ func TestNewRefusesRelativeXDGDataHome(t *testing.T) {
t.Setenv("PORT", "") t.Setenv("PORT", "")
t.Setenv("XDG_DATA_HOME", "relative/path") t.Setenv("XDG_DATA_HOME", "relative/path")
if _, err := New(); err == nil {
_, err := New()
if err == nil {
t.Error("New() with a relative XDG_DATA_HOME returned no error") t.Error("New() with a relative XDG_DATA_HOME returned no error")
} }
t.Setenv("XDG_DATA_HOME", "/var/lib") t.Setenv("XDG_DATA_HOME", "/var/lib")
if _, err := New(); err != nil {
_, err = New()
if err != nil {
t.Errorf("New() with XDG_DATA_HOME=/var/lib: %v", err) t.Errorf("New() with XDG_DATA_HOME=/var/lib: %v", err)
} }
} }
@@ -12,6 +12,12 @@ import (
"github.com/google/uuid" "github.com/google/uuid"
) )
// testV4PeerIP is the peer the IPv4 routes in these tests are learned from.
const testV4PeerIP = "192.0.2.1"
// testV6Prefix is the prefix of every IPv6 route in these tests.
const testV6Prefix = "2001:db8::/32"
// mkV4Route builds an IPv4 live route with its mask length taken from the // mkV4Route builds an IPv4 live route with its mask length taken from the
// prefix. // prefix.
func mkV4Route(t *testing.T, prefix string, asn int, ts time.Time) *LiveRoute { func mkV4Route(t *testing.T, prefix string, asn int, ts time.Time) *LiveRoute {
@@ -28,18 +34,18 @@ func mkV4Route(t *testing.T, prefix string, asn int, ts time.Time) *LiveRoute {
MaskLength: maskLength, MaskLength: maskLength,
IPVersion: ipVersionV4, IPVersion: ipVersionV4,
OriginASN: asn, OriginASN: asn,
PeerIP: "192.0.2.1", PeerIP: testV4PeerIP,
ASPath: []int{asn}, ASPath: []int{asn},
NextHop: "192.0.2.254", NextHop: "192.0.2.254",
LastUpdated: ts, LastUpdated: ts,
} }
} }
// mkV6Route builds an IPv6 live route. // mkV6Route builds an IPv6 live route for testV6Prefix.
func mkV6Route(prefix string, asn int, ts time.Time) *LiveRoute { func mkV6Route(asn int, ts time.Time) *LiveRoute {
return &LiveRoute{ return &LiveRoute{
ID: uuid.New(), ID: uuid.New(),
Prefix: prefix, Prefix: testV6Prefix,
MaskLength: 32, MaskLength: 32,
IPVersion: ipVersionV6, IPVersion: ipVersionV6,
OriginASN: asn, OriginASN: asn,
@@ -56,6 +62,8 @@ func mkV6Route(prefix string, asn int, ts time.Time) *LiveRoute {
// deleted. These counts are what /api/v1/stats reports; before this change the // deleted. These counts are what /api/v1/stats reports; before this change the
// endpoint recomputed them with a COUNT(*) over each table on every request. // endpoint recomputed them with a COUNT(*) over each table on every request.
func TestLiveCountsTrackWritesInRealtime(t *testing.T) { func TestLiveCountsTrackWritesInRealtime(t *testing.T) {
t.Parallel()
cfg := &config.Config{StateDir: t.TempDir()} cfg := &config.Config{StateDir: t.TempDir()}
db, err := New(cfg, logger.New()) db, err := New(cfg, logger.New())
@@ -70,6 +78,7 @@ func TestLiveCountsTrackWritesInRealtime(t *testing.T) {
if err != nil { if err != nil {
t.Fatalf("GetStatsContext on empty database: %v", err) t.Fatalf("GetStatsContext on empty database: %v", err)
} }
if empty.ASNs != 0 || empty.Prefixes != 0 || empty.Peerings != 0 || if empty.ASNs != 0 || empty.Prefixes != 0 || empty.Peerings != 0 ||
empty.Peers != 0 || empty.LiveRoutes != 0 { empty.Peers != 0 || empty.LiveRoutes != 0 {
t.Fatalf("empty database counts nonzero: %+v", empty) t.Fatalf("empty database counts nonzero: %+v", empty)
@@ -77,30 +86,16 @@ func TestLiveCountsTrackWritesInRealtime(t *testing.T) {
ts := time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC) ts := time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC)
if err := db.GetOrCreateASNBatch(map[int]time.Time{64500: ts, 64501: ts}); err != nil { writeASNsPrefixesAndPeers(t, db, ts)
t.Fatalf("GetOrCreateASNBatch: %v", err)
}
if err := db.UpdatePrefixesBatch(map[string]time.Time{
"198.51.100.0/24": ts,
"2001:db8::/32": ts,
}); err != nil {
t.Fatalf("UpdatePrefixesBatch: %v", err)
}
if err := db.UpdatePeerBatch(map[string]PeerUpdate{
"192.0.2.1": {PeerIP: "192.0.2.1", PeerASN: 64500, MessageType: "UPDATE", Timestamp: ts},
}); err != nil {
t.Fatalf("UpdatePeerBatch: %v", err)
}
if err := db.RecordPeering(64500, 64501, ts); err != nil {
t.Fatalf("RecordPeering: %v", err)
}
routes := []*LiveRoute{ routes := []*LiveRoute{
mkV4Route(t, "198.51.100.0/24", 64500, ts), mkV4Route(t, "198.51.100.0/24", 64500, ts),
mkV4Route(t, "203.0.113.0/24", 64501, ts.Add(time.Minute)), mkV4Route(t, "203.0.113.0/24", 64501, ts.Add(time.Minute)),
mkV6Route("2001:db8::/32", 64502, ts.Add(2*time.Minute)), mkV6Route(64502, ts.Add(2*time.Minute)),
} }
if err := db.UpsertLiveRouteBatch(routes); err != nil {
err = db.UpsertLiveRouteBatch(routes)
if err != nil {
t.Fatalf("UpsertLiveRouteBatch: %v", err) t.Fatalf("UpsertLiveRouteBatch: %v", err)
} }
@@ -108,44 +103,88 @@ func TestLiveCountsTrackWritesInRealtime(t *testing.T) {
if err != nil { if err != nil {
t.Fatalf("GetStatsContext: %v", err) t.Fatalf("GetStatsContext: %v", err)
} }
assertCounts(t, "after inserts", stats, wantCounts{ assertCounts(t, "after inserts", stats, wantCounts{
asns: 2, prefixes: 2, peerings: 1, peers: 1, asns: 2, prefixes: 2, peerings: 1, peers: 1,
ipv4Routes: 2, ipv6Routes: 1, liveRoutes: 3, ipv4Routes: 2, ipv6Routes: 1, liveRoutes: 3,
}) })
// Re-announcing the same routes is an update, not an insert: counts hold. // Re-announcing the same routes is an update, not an insert: counts hold.
if err := db.UpsertLiveRouteBatch(routes); err != nil { err = db.UpsertLiveRouteBatch(routes)
if err != nil {
t.Fatalf("UpsertLiveRouteBatch (re-announce): %v", err) t.Fatalf("UpsertLiveRouteBatch (re-announce): %v", err)
} }
stats, err = db.GetStatsContext(ctx) stats, err = db.GetStatsContext(ctx)
if err != nil { if err != nil {
t.Fatalf("GetStatsContext: %v", err) t.Fatalf("GetStatsContext: %v", err)
} }
assertCounts(t, "after re-announce", stats, wantCounts{ assertCounts(t, "after re-announce", stats, wantCounts{
asns: 2, prefixes: 2, peerings: 1, peers: 1, asns: 2, prefixes: 2, peerings: 1, peers: 1,
ipv4Routes: 2, ipv6Routes: 1, liveRoutes: 3, ipv4Routes: 2, ipv6Routes: 1, liveRoutes: 3,
}) })
// A withdrawal removes one route. // A withdrawal removes one route.
if err := db.DeleteLiveRouteBatch([]LiveRouteDeletion{ err = db.DeleteLiveRouteBatch([]LiveRouteDeletion{{
{Prefix: "203.0.113.0/24", OriginASN: 64501, PeerIP: "192.0.2.1", IPVersion: ipVersionV4}, Prefix: "203.0.113.0/24", OriginASN: 64501,
}); err != nil { PeerIP: testV4PeerIP, IPVersion: ipVersionV4,
}})
if err != nil {
t.Fatalf("DeleteLiveRouteBatch: %v", err) t.Fatalf("DeleteLiveRouteBatch: %v", err)
} }
stats, err = db.GetStatsContext(ctx) stats, err = db.GetStatsContext(ctx)
if err != nil { if err != nil {
t.Fatalf("GetStatsContext: %v", err) t.Fatalf("GetStatsContext: %v", err)
} }
assertCounts(t, "after delete", stats, wantCounts{ assertCounts(t, "after delete", stats, wantCounts{
asns: 2, prefixes: 2, peerings: 1, peers: 1, asns: 2, prefixes: 2, peerings: 1, peers: 1,
ipv4Routes: 1, ipv6Routes: 1, liveRoutes: 2, ipv4Routes: 1, ipv6Routes: 1, liveRoutes: 2,
}) })
} }
// writeASNsPrefixesAndPeers writes two ASNs, two prefixes, one peer and one
// peering: the rows besides live routes that the stats count.
func writeASNsPrefixesAndPeers(t *testing.T, db *Database, ts time.Time) {
t.Helper()
err := db.GetOrCreateASNBatch(map[int]time.Time{64500: ts, 64501: ts})
if err != nil {
t.Fatalf("GetOrCreateASNBatch: %v", err)
}
err = db.UpdatePrefixesBatch(map[string]time.Time{
"198.51.100.0/24": ts,
testV6Prefix: ts,
})
if err != nil {
t.Fatalf("UpdatePrefixesBatch: %v", err)
}
err = db.UpdatePeerBatch(map[string]PeerUpdate{
testV4PeerIP: {
PeerIP: testV4PeerIP, PeerASN: 64500,
MessageType: "UPDATE", Timestamp: ts,
},
})
if err != nil {
t.Fatalf("UpdatePeerBatch: %v", err)
}
err = db.RecordPeering(64500, 64501, ts)
if err != nil {
t.Fatalf("RecordPeering: %v", err)
}
}
// TestLiveCountsSeededFromDatabaseAtStartup writes rows, reopens the same // TestLiveCountsSeededFromDatabaseAtStartup writes rows, reopens the same
// database file, and checks the counts and the prefix distribution come back // database file, and checks the counts and the prefix distribution come back
// from the seed scan rather than starting at zero. // from the seed scan rather than starting at zero.
func TestLiveCountsSeededFromDatabaseAtStartup(t *testing.T) { func TestLiveCountsSeededFromDatabaseAtStartup(t *testing.T) {
t.Parallel()
cfg := &config.Config{StateDir: t.TempDir()} cfg := &config.Config{StateDir: t.TempDir()}
db, err := New(cfg, logger.New()) db, err := New(cfg, logger.New())
@@ -154,16 +193,22 @@ func TestLiveCountsSeededFromDatabaseAtStartup(t *testing.T) {
} }
ts := time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC) ts := time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC)
if err := db.GetOrCreateASNBatch(map[int]time.Time{64500: ts, 64501: ts, 64502: ts}); err != nil {
err = db.GetOrCreateASNBatch(map[int]time.Time{64500: ts, 64501: ts, 64502: ts})
if err != nil {
t.Fatalf("GetOrCreateASNBatch: %v", err) t.Fatalf("GetOrCreateASNBatch: %v", err)
} }
if err := db.UpsertLiveRouteBatch([]*LiveRoute{
err = db.UpsertLiveRouteBatch([]*LiveRoute{
mkV4Route(t, "198.51.100.0/24", 64500, ts), mkV4Route(t, "198.51.100.0/24", 64500, ts),
mkV6Route("2001:db8::/32", 64502, ts), mkV6Route(64502, ts),
}); err != nil { })
if err != nil {
t.Fatalf("UpsertLiveRouteBatch: %v", err) t.Fatalf("UpsertLiveRouteBatch: %v", err)
} }
if err := db.Close(); err != nil {
err = db.Close()
if err != nil {
t.Fatalf("Close: %v", err) t.Fatalf("Close: %v", err)
} }
@@ -177,13 +222,16 @@ func TestLiveCountsSeededFromDatabaseAtStartup(t *testing.T) {
if err != nil { if err != nil {
t.Fatalf("GetStatsContext after reopen: %v", err) t.Fatalf("GetStatsContext after reopen: %v", err)
} }
if stats.ASNs != 3 { if stats.ASNs != 3 {
t.Errorf("seeded ASNs = %d, want 3", stats.ASNs) t.Errorf("seeded ASNs = %d, want 3", stats.ASNs)
} }
if stats.IPv4Routes != 1 || stats.IPv6Routes != 1 || stats.LiveRoutes != 2 { if stats.IPv4Routes != 1 || stats.IPv6Routes != 1 || stats.LiveRoutes != 2 {
t.Errorf("seeded routes = (v4 %d, v6 %d, total %d), want (1, 1, 2)", t.Errorf("seeded routes = (v4 %d, v6 %d, total %d), want (1, 1, 2)",
stats.IPv4Routes, stats.IPv6Routes, stats.LiveRoutes) stats.IPv4Routes, stats.IPv6Routes, stats.LiveRoutes)
} }
assertDistribution(t, "seeded IPv4 distribution", stats.IPv4PrefixDistribution, assertDistribution(t, "seeded IPv4 distribution", stats.IPv4PrefixDistribution,
[]PrefixDistribution{{MaskLength: 24, Count: 1}}) []PrefixDistribution{{MaskLength: 24, Count: 1}})
assertDistribution(t, "seeded IPv6 distribution", stats.IPv6PrefixDistribution, assertDistribution(t, "seeded IPv6 distribution", stats.IPv6PrefixDistribution,
@@ -196,26 +244,91 @@ func TestLiveCountsSeededFromDatabaseAtStartup(t *testing.T) {
// tables. The steps run once through the batch methods the prefix handler uses // tables. The steps run once through the batch methods the prefix handler uses
// and once through the single-route methods. // and once through the single-route methods.
func TestPrefixDistributionTracksWrites(t *testing.T) { func TestPrefixDistributionTracksWrites(t *testing.T) {
t.Parallel()
steps := prefixDistributionSteps(t)
for _, batch := range []bool{true, false} {
name := "single-route writes"
if batch {
name = "batch writes"
}
t.Run(name, func(t *testing.T) {
t.Parallel()
db, err := New(&config.Config{StateDir: t.TempDir()}, logger.New())
if err != nil {
t.Fatalf("failed to create database: %v", err)
}
defer func() { _ = db.Close() }()
ctx := context.Background()
for _, step := range steps {
err := announceRoutes(db, batch, step.announce)
if err != nil {
t.Fatalf("%s: announce: %v", step.name, err)
}
err = withdrawRoutes(db, batch, step.withdraw)
if err != nil {
t.Fatalf("%s: withdraw: %v", step.name, err)
}
stats, err := db.GetStatsContext(ctx)
if err != nil {
t.Fatalf("%s: GetStatsContext: %v", step.name, err)
}
queryV4, queryV6, err := db.GetPrefixDistributionContext(ctx)
if err != nil {
t.Fatalf("%s: GetPrefixDistributionContext: %v", step.name, err)
}
assertDistribution(t, step.name+": IPv4 distribution",
stats.IPv4PrefixDistribution, step.wantV4)
assertDistribution(t, step.name+": IPv6 distribution",
stats.IPv6PrefixDistribution, step.wantV6)
assertDistribution(t, step.name+": IPv4 distribution query",
queryV4, step.wantV4)
assertDistribution(t, step.name+": IPv6 distribution query",
queryV6, step.wantV6)
}
})
}
}
// prefixDistributionStep is one step of TestPrefixDistributionTracksWrites:
// the routes it announces, then the routes it withdraws, then the
// distributions both the stats and the distribution query must report.
type prefixDistributionStep struct {
name string
announce []*LiveRoute
withdraw []*LiveRoute
wantV4 []PrefixDistribution
wantV6 []PrefixDistribution
}
// prefixDistributionSteps returns the steps of
// TestPrefixDistributionTracksWrites, in the order they run.
func prefixDistributionSteps(t *testing.T) []prefixDistributionStep {
t.Helper()
ts := time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC) ts := time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC)
shared := mkV4Route(t, "198.51.100.0/24", 64500, ts) shared := mkV4Route(t, "198.51.100.0/24", 64500, ts)
sharedSecondPeer := mkV4Route(t, "198.51.100.0/24", 64500, ts) sharedSecondPeer := mkV4Route(t, "198.51.100.0/24", 64500, ts)
sharedSecondPeer.PeerIP = "192.0.2.2" sharedSecondPeer.PeerIP = "192.0.2.2"
other := mkV4Route(t, "203.0.113.0/24", 64501, ts) other := mkV4Route(t, "203.0.113.0/24", 64501, ts)
wide := mkV4Route(t, "172.16.0.0/16", 64502, ts) wide := mkV4Route(t, "172.16.0.0/16", 64502, ts)
v6 := mkV6Route("2001:db8::/32", 64503, ts) v6 := mkV6Route(64503, ts)
v6SecondPeer := mkV6Route("2001:db8::/32", 64503, ts) v6SecondPeer := mkV6Route(64503, ts)
v6SecondPeer.PeerIP = "2001:db8::2" v6SecondPeer.PeerIP = "2001:db8::2"
all := []PrefixDistribution{{MaskLength: 16, Count: 1}, {MaskLength: 24, Count: 2}} all := []PrefixDistribution{{MaskLength: 16, Count: 1}, {MaskLength: 24, Count: 2}}
v6Only := []PrefixDistribution{{MaskLength: 32, Count: 1}} v6Only := []PrefixDistribution{{MaskLength: 32, Count: 1}}
steps := []struct { return []prefixDistributionStep{
name string
announce []*LiveRoute
withdraw []*LiveRoute
wantV4 []PrefixDistribution
wantV6 []PrefixDistribution
}{
{ {
name: "new routes", name: "new routes",
announce: []*LiveRoute{shared, other, wide, v6}, announce: []*LiveRoute{shared, other, wide, v6},
@@ -239,7 +352,9 @@ func TestPrefixDistributionTracksWrites(t *testing.T) {
{ {
name: "withdrawal of the last route for a prefix", name: "withdrawal of the last route for a prefix",
withdraw: []*LiveRoute{sharedSecondPeer, v6SecondPeer}, withdraw: []*LiveRoute{sharedSecondPeer, v6SecondPeer},
wantV4: []PrefixDistribution{{MaskLength: 16, Count: 1}, {MaskLength: 24, Count: 1}}, wantV4: []PrefixDistribution{
{MaskLength: 16, Count: 1}, {MaskLength: 24, Count: 1},
},
}, },
{ {
name: "withdrawal of every remaining route", name: "withdrawal of every remaining route",
@@ -270,45 +385,6 @@ func TestPrefixDistributionTracksWrites(t *testing.T) {
wantV6: v6Only, wantV6: v6Only,
}, },
} }
for _, batch := range []bool{true, false} {
name := "single-route writes"
if batch {
name = "batch writes"
}
t.Run(name, func(t *testing.T) {
db, err := New(&config.Config{StateDir: t.TempDir()}, logger.New())
if err != nil {
t.Fatalf("failed to create database: %v", err)
}
defer func() { _ = db.Close() }()
ctx := context.Background()
for _, step := range steps {
if err := announceRoutes(db, batch, step.announce); err != nil {
t.Fatalf("%s: announce: %v", step.name, err)
}
if err := withdrawRoutes(db, batch, step.withdraw); err != nil {
t.Fatalf("%s: withdraw: %v", step.name, err)
}
stats, err := db.GetStatsContext(ctx)
if err != nil {
t.Fatalf("%s: GetStatsContext: %v", step.name, err)
}
queryV4, queryV6, err := db.GetPrefixDistributionContext(ctx)
if err != nil {
t.Fatalf("%s: GetPrefixDistributionContext: %v", step.name, err)
}
assertDistribution(t, step.name+": IPv4 distribution", stats.IPv4PrefixDistribution, step.wantV4)
assertDistribution(t, step.name+": IPv6 distribution", stats.IPv6PrefixDistribution, step.wantV6)
assertDistribution(t, step.name+": IPv4 distribution query", queryV4, step.wantV4)
assertDistribution(t, step.name+": IPv6 distribution query", queryV6, step.wantV6)
}
})
}
} }
// announceRoutes writes routes in one UpsertLiveRouteBatch, or with one // announceRoutes writes routes in one UpsertLiveRouteBatch, or with one
@@ -317,8 +393,10 @@ func announceRoutes(db *Database, batch bool, routes []*LiveRoute) error {
if batch { if batch {
return db.UpsertLiveRouteBatch(routes) return db.UpsertLiveRouteBatch(routes)
} }
for _, route := range routes { for _, route := range routes {
if err := db.UpsertLiveRoute(route); err != nil { err := db.UpsertLiveRoute(route)
if err != nil {
return err return err
} }
} }
@@ -333,7 +411,8 @@ func announceRoutes(db *Database, batch bool, routes []*LiveRoute) error {
func withdrawRoutes(db *Database, batch bool, routes []*LiveRoute) error { func withdrawRoutes(db *Database, batch bool, routes []*LiveRoute) error {
if !batch { if !batch {
for _, route := range routes { for _, route := range routes {
if err := db.DeleteLiveRoute(route.Prefix, 0, route.PeerIP); err != nil { err := db.DeleteLiveRoute(route.Prefix, 0, route.PeerIP)
if err != nil {
return err return err
} }
} }
@@ -356,6 +435,8 @@ func withdrawRoutes(db *Database, batch bool, routes []*LiveRoute) error {
// MIN/MAX union query read its result into *time.Time, which the driver could // MIN/MAX union query read its result into *time.Time, which the driver could
// not parse, so it logged a warning every call and left both timestamps nil. // not parse, so it logged a warning every call and left both timestamps nil.
func TestStatsRouteTimestamps(t *testing.T) { func TestStatsRouteTimestamps(t *testing.T) {
t.Parallel()
cfg := &config.Config{StateDir: t.TempDir()} cfg := &config.Config{StateDir: t.TempDir()}
db, err := New(cfg, logger.New()) db, err := New(cfg, logger.New())
@@ -370,6 +451,7 @@ func TestStatsRouteTimestamps(t *testing.T) {
if err != nil { if err != nil {
t.Fatalf("GetStatsContext on empty database: %v", err) t.Fatalf("GetStatsContext on empty database: %v", err)
} }
if empty.OldestRoute != nil || empty.NewestRoute != nil { if empty.OldestRoute != nil || empty.NewestRoute != nil {
t.Fatalf("empty database timestamps = (%v, %v), want (nil, nil)", t.Fatalf("empty database timestamps = (%v, %v), want (nil, nil)",
empty.OldestRoute, empty.NewestRoute) empty.OldestRoute, empty.NewestRoute)
@@ -378,11 +460,13 @@ func TestStatsRouteTimestamps(t *testing.T) {
base := time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC) base := time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC)
oldest := base oldest := base
newest := base.Add(2 * time.Minute) newest := base.Add(2 * time.Minute)
if err := db.UpsertLiveRouteBatch([]*LiveRoute{
err = db.UpsertLiveRouteBatch([]*LiveRoute{
mkV4Route(t, "198.51.100.0/24", 64500, base.Add(time.Minute)), mkV4Route(t, "198.51.100.0/24", 64500, base.Add(time.Minute)),
mkV4Route(t, "203.0.113.0/24", 64501, oldest), mkV4Route(t, "203.0.113.0/24", 64501, oldest),
mkV6Route("2001:db8::/32", 64502, newest), mkV6Route(64502, newest),
}); err != nil { })
if err != nil {
t.Fatalf("UpsertLiveRouteBatch: %v", err) t.Fatalf("UpsertLiveRouteBatch: %v", err)
} }
@@ -390,9 +474,11 @@ func TestStatsRouteTimestamps(t *testing.T) {
if err != nil { if err != nil {
t.Fatalf("GetStatsContext: %v", err) t.Fatalf("GetStatsContext: %v", err)
} }
if stats.OldestRoute == nil || !stats.OldestRoute.Equal(oldest) { if stats.OldestRoute == nil || !stats.OldestRoute.Equal(oldest) {
t.Errorf("OldestRoute = %v, want %v", stats.OldestRoute, oldest) t.Errorf("OldestRoute = %v, want %v", stats.OldestRoute, oldest)
} }
if stats.NewestRoute == nil || !stats.NewestRoute.Equal(newest) { if stats.NewestRoute == nil || !stats.NewestRoute.Equal(newest) {
t.Errorf("NewestRoute = %v, want %v", stats.NewestRoute, newest) t.Errorf("NewestRoute = %v, want %v", stats.NewestRoute, newest)
} }
@@ -401,6 +487,8 @@ func TestStatsRouteTimestamps(t *testing.T) {
// TestLiveCountsConcurrentReadWrite runs writers and stats readers at once so // TestLiveCountsConcurrentReadWrite runs writers and stats readers at once so
// the race detector proves the counters are safe under concurrent use. // the race detector proves the counters are safe under concurrent use.
func TestLiveCountsConcurrentReadWrite(t *testing.T) { func TestLiveCountsConcurrentReadWrite(t *testing.T) {
t.Parallel()
cfg := &config.Config{StateDir: t.TempDir()} cfg := &config.Config{StateDir: t.TempDir()}
db, err := New(cfg, logger.New()) db, err := New(cfg, logger.New())
@@ -412,17 +500,21 @@ func TestLiveCountsConcurrentReadWrite(t *testing.T) {
ts := time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC) ts := time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC)
const writers = 4 const writers = 4
var wg sync.WaitGroup
var wg sync.WaitGroup
wg.Add(writers) wg.Add(writers)
for w := range writers { for w := range writers {
go func(base int) { go func(base int) {
defer wg.Done() defer wg.Done()
for i := range 25 { for i := range 25 {
asn := 65000 + base*100 + i asn := 65000 + base*100 + i
route := mkV6Route("2001:db8::/32", asn, ts) route := mkV6Route(asn, ts)
route.PeerIP = "2001:db8::" + uuid.NewString() route.PeerIP = "2001:db8::" + uuid.NewString()
if err := db.UpsertLiveRoute(route); err != nil {
err := db.UpsertLiveRoute(route)
if err != nil {
t.Errorf("UpsertLiveRoute: %v", err) t.Errorf("UpsertLiveRoute: %v", err)
return return
@@ -433,15 +525,19 @@ func TestLiveCountsConcurrentReadWrite(t *testing.T) {
var readerWG sync.WaitGroup var readerWG sync.WaitGroup
readerWG.Add(1) readerWG.Add(1)
stop := make(chan struct{}) stop := make(chan struct{})
go func() { go func() {
defer readerWG.Done() defer readerWG.Done()
for { for {
select { select {
case <-stop: case <-stop:
return return
default: default:
if _, err := db.GetStatsContext(context.Background()); err != nil { _, err := db.GetStatsContext(context.Background())
if err != nil {
t.Errorf("GetStatsContext: %v", err) t.Errorf("GetStatsContext: %v", err)
return return
@@ -458,6 +554,7 @@ func TestLiveCountsConcurrentReadWrite(t *testing.T) {
if err != nil { if err != nil {
t.Fatalf("GetStatsContext: %v", err) t.Fatalf("GetStatsContext: %v", err)
} }
if want := writers * 25; stats.IPv6Routes != want { if want := writers * 25; stats.IPv6Routes != want {
t.Errorf("IPv6Routes = %d, want %d", stats.IPv6Routes, want) t.Errorf("IPv6Routes = %d, want %d", stats.IPv6Routes, want)
} }
@@ -482,21 +579,27 @@ func assertCounts(t *testing.T, when string, got Stats, want wantCounts) {
if got.ASNs != want.asns { if got.ASNs != want.asns {
t.Errorf("%s: ASNs = %d, want %d", when, got.ASNs, want.asns) t.Errorf("%s: ASNs = %d, want %d", when, got.ASNs, want.asns)
} }
if got.Prefixes != want.prefixes { if got.Prefixes != want.prefixes {
t.Errorf("%s: Prefixes = %d, want %d", when, got.Prefixes, want.prefixes) t.Errorf("%s: Prefixes = %d, want %d", when, got.Prefixes, want.prefixes)
} }
if got.Peerings != want.peerings { if got.Peerings != want.peerings {
t.Errorf("%s: Peerings = %d, want %d", when, got.Peerings, want.peerings) t.Errorf("%s: Peerings = %d, want %d", when, got.Peerings, want.peerings)
} }
if got.Peers != want.peers { if got.Peers != want.peers {
t.Errorf("%s: Peers = %d, want %d", when, got.Peers, want.peers) t.Errorf("%s: Peers = %d, want %d", when, got.Peers, want.peers)
} }
if got.IPv4Routes != want.ipv4Routes { if got.IPv4Routes != want.ipv4Routes {
t.Errorf("%s: IPv4Routes = %d, want %d", when, got.IPv4Routes, want.ipv4Routes) t.Errorf("%s: IPv4Routes = %d, want %d", when, got.IPv4Routes, want.ipv4Routes)
} }
if got.IPv6Routes != want.ipv6Routes { if got.IPv6Routes != want.ipv6Routes {
t.Errorf("%s: IPv6Routes = %d, want %d", when, got.IPv6Routes, want.ipv6Routes) t.Errorf("%s: IPv6Routes = %d, want %d", when, got.IPv6Routes, want.ipv6Routes)
} }
if got.LiveRoutes != want.liveRoutes { if got.LiveRoutes != want.liveRoutes {
t.Errorf("%s: LiveRoutes = %d, want %d", when, got.LiveRoutes, want.liveRoutes) t.Errorf("%s: LiveRoutes = %d, want %d", when, got.LiveRoutes, want.liveRoutes)
} }
@@ -50,6 +50,8 @@ const (
// address, ErrNoRoute when none covers it, and the next less specific prefix // address, ErrNoRoute when none covers it, and the next less specific prefix
// once the only route of the most specific one is withdrawn. // once the only route of the most specific one is withdrawn.
func TestGetIPInfoFindsMostSpecificLiveRoute(t *testing.T) { func TestGetIPInfoFindsMostSpecificLiveRoute(t *testing.T) {
t.Parallel()
cfg := &config.Config{StateDir: t.TempDir()} cfg := &config.Config{StateDir: t.TempDir()}
db, err := New(cfg, logger.New()) db, err := New(cfg, logger.New())
@@ -67,24 +69,7 @@ func TestGetIPInfoFindsMostSpecificLiveRoute(t *testing.T) {
"2001:db8:1::/48": 64501, "2001:db8:1::/48": 64501,
"2001:db8:1:2::/64": 64502, "2001:db8:1:2::/64": 64502,
} }
ts := time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC) writeOriginRoutes(t, db, origins)
routes := make([]*LiveRoute, 0, len(origins))
for prefix, asn := range origins {
routes = append(routes, &LiveRoute{
ID: uuid.New(),
Prefix: prefix,
MaskLength: netip.MustParsePrefix(prefix).Bits(),
IPVersion: detectIPVersion(prefix),
OriginASN: asn,
PeerIP: testPeerIP,
ASPath: []int{asn},
NextHop: testPeerIP,
LastUpdated: ts,
})
}
if err := db.UpsertLiveRouteBatch(routes); err != nil {
t.Fatalf("UpsertLiveRouteBatch: %v", err)
}
// lookup checks that ip resolves to the live prefix want, or to ErrNoRoute // lookup checks that ip resolves to the live prefix want, or to ErrNoRoute
// when want is empty. // when want is empty.
@@ -99,11 +84,13 @@ func TestGetIPInfoFindsMostSpecificLiveRoute(t *testing.T) {
return return
} }
if err != nil { if err != nil {
t.Errorf("GetIPInfo(%s): %v", ip, err) t.Errorf("GetIPInfo(%s): %v", ip, err)
return return
} }
if info.Netblock != want || info.MaskLength != netip.MustParsePrefix(want).Bits() || if info.Netblock != want || info.MaskLength != netip.MustParsePrefix(want).Bits() ||
info.ASN != origins[want] { info.ASN != origins[want] {
t.Errorf("GetIPInfo(%s) = %s (mask %d) AS%d, want %s AS%d", t.Errorf("GetIPInfo(%s) = %s (mask %d) AS%d, want %s AS%d",
@@ -122,8 +109,14 @@ func TestGetIPInfoFindsMostSpecificLiveRoute(t *testing.T) {
lookup("2001:db9::1", "") lookup("2001:db9::1", "")
err = db.DeleteLiveRouteBatch([]LiveRouteDeletion{ err = db.DeleteLiveRouteBatch([]LiveRouteDeletion{
{Prefix: "10.1.2.0/24", OriginASN: 64502, PeerIP: testPeerIP, IPVersion: ipVersionV4}, {
{Prefix: "2001:db8:1:2::/64", OriginASN: 64502, PeerIP: testPeerIP, IPVersion: ipVersionV6}, Prefix: "10.1.2.0/24", OriginASN: 64502,
PeerIP: testPeerIP, IPVersion: ipVersionV4,
},
{
Prefix: "2001:db8:1:2::/64", OriginASN: 64502,
PeerIP: testPeerIP, IPVersion: ipVersionV6,
},
}) })
if err != nil { if err != nil {
t.Fatalf("DeleteLiveRouteBatch: %v", err) t.Fatalf("DeleteLiveRouteBatch: %v", err)
@@ -133,11 +126,41 @@ func TestGetIPInfoFindsMostSpecificLiveRoute(t *testing.T) {
lookup("2001:db8:1:2::3", "2001:db8:1::/48") lookup("2001:db8:1:2::3", "2001:db8:1::/48")
} }
// writeOriginRoutes writes one live route from testPeerIP for each prefix in
// origins, originated by the AS it maps to.
func writeOriginRoutes(t *testing.T, db *Database, origins map[string]int) {
t.Helper()
ts := time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC)
routes := make([]*LiveRoute, 0, len(origins))
for prefix, asn := range origins {
routes = append(routes, &LiveRoute{
ID: uuid.New(),
Prefix: prefix,
MaskLength: netip.MustParsePrefix(prefix).Bits(),
IPVersion: detectIPVersion(prefix),
OriginASN: asn,
PeerIP: testPeerIP,
ASPath: []int{asn},
NextHop: testPeerIP,
LastUpdated: ts,
})
}
err := db.UpsertLiveRouteBatch(routes)
if err != nil {
t.Fatalf("UpsertLiveRouteBatch: %v", err)
}
}
// TestConnectionPoolPragmas holds several pooled connections open at once and // TestConnectionPoolPragmas holds several pooled connections open at once and
// checks each one carries the per-connection settings from the DSN, plus the // checks each one carries the per-connection settings from the DSN, plus the
// process-wide hard heap limit, and sees the new file with auto_vacuum // process-wide hard heap limit, and sees the new file with auto_vacuum
// incremental. // incremental.
func TestConnectionPoolPragmas(t *testing.T) { func TestConnectionPoolPragmas(t *testing.T) {
t.Parallel()
cfg := &config.Config{StateDir: t.TempDir()} cfg := &config.Config{StateDir: t.TempDir()}
db, err := New(cfg, logger.New()) db, err := New(cfg, logger.New())
@@ -157,57 +180,64 @@ func TestConnectionPoolPragmas(t *testing.T) {
} }
}() }()
for i := 0; i < heldConnections; i++ { for i := range heldConnections {
c, err := db.db.Conn(ctx) c, err := db.db.Conn(ctx)
if err != nil { if err != nil {
t.Fatalf("failed to open connection %d: %v", i, err) t.Fatalf("failed to open connection %d: %v", i, err)
} }
conns = append(conns, c) conns = append(conns, c)
} }
for i, c := range conns { for i, c := range conns {
var cacheSize int cacheSize := readPragma(ctx, t, c, i, "cache_size")
if err := c.QueryRowContext(ctx, "PRAGMA cache_size").Scan(&cacheSize); err != nil {
t.Fatalf("conn %d: failed to read cache_size: %v", i, err)
}
if cacheSize != sqliteCacheSizeKiB { if cacheSize != sqliteCacheSizeKiB {
t.Errorf("conn %d: cache_size = %d, want %d", i, cacheSize, sqliteCacheSizeKiB) t.Errorf("conn %d: cache_size = %d, want %d",
i, cacheSize, sqliteCacheSizeKiB)
} }
var busyTimeout int busyTimeout := readPragma(ctx, t, c, i, "busy_timeout")
if err := c.QueryRowContext(ctx, "PRAGMA busy_timeout").Scan(&busyTimeout); err != nil {
t.Fatalf("conn %d: failed to read busy_timeout: %v", i, err)
}
if busyTimeout != sqliteBusyTimeoutMs { if busyTimeout != sqliteBusyTimeoutMs {
t.Errorf("conn %d: busy_timeout = %d, want %d", i, busyTimeout, sqliteBusyTimeoutMs) t.Errorf("conn %d: busy_timeout = %d, want %d",
i, busyTimeout, sqliteBusyTimeoutMs)
} }
var tempStore int tempStore := readPragma(ctx, t, c, i, "temp_store")
if err := c.QueryRowContext(ctx, "PRAGMA temp_store").Scan(&tempStore); err != nil {
t.Fatalf("conn %d: failed to read temp_store: %v", i, err)
}
if tempStore == tempStoreMemory { if tempStore == tempStoreMemory {
t.Errorf("conn %d: temp_store = %d, want anything but %d (MEMORY)", i, tempStore, tempStoreMemory) t.Errorf("conn %d: temp_store = %d, want anything but %d (MEMORY)",
i, tempStore, tempStoreMemory)
} }
var hardHeapLimit int64 hardHeapLimit := readPragma(ctx, t, c, i, "hard_heap_limit")
if err := c.QueryRowContext(ctx, "PRAGMA hard_heap_limit").Scan(&hardHeapLimit); err != nil {
t.Fatalf("conn %d: failed to read hard_heap_limit: %v", i, err)
}
if hardHeapLimit != sqliteHardHeapLimitBytes { if hardHeapLimit != sqliteHardHeapLimitBytes {
t.Errorf("conn %d: hard_heap_limit = %d, want %d", i, hardHeapLimit, sqliteHardHeapLimitBytes) t.Errorf("conn %d: hard_heap_limit = %d, want %d",
i, hardHeapLimit, sqliteHardHeapLimitBytes)
} }
var autoVacuum int autoVacuum := readPragma(ctx, t, c, i, "auto_vacuum")
if err := c.QueryRowContext(ctx, "PRAGMA auto_vacuum").Scan(&autoVacuum); err != nil {
t.Fatalf("conn %d: failed to read auto_vacuum: %v", i, err)
}
if autoVacuum != autoVacuumIncremental { if autoVacuum != autoVacuumIncremental {
t.Errorf("conn %d: auto_vacuum = %d, want %d (incremental)", i, autoVacuum, autoVacuumIncremental) t.Errorf("conn %d: auto_vacuum = %d, want %d (incremental)",
i, autoVacuum, autoVacuumIncremental)
} }
} }
} }
// readPragma reads the integer value of PRAGMA name on connection i, c.
func readPragma(
ctx context.Context, t *testing.T, c *sql.Conn, i int, name string,
) int64 {
t.Helper()
var value int64
err := c.QueryRowContext(ctx, "PRAGMA "+name).Scan(&value)
if err != nil {
t.Fatalf("conn %d: failed to read %s: %v", i, name, err)
}
return value
}
// TestVacuumReturnsFreePages checks that after routes are deleted from a new // TestVacuumReturnsFreePages checks that after routes are deleted from a new
// database, one Vacuum call returns every page they used. The deletes leave // database, one Vacuum call returns every page they used. The deletes leave
// fewer free pages than the 1000 Vacuum frees per call, so none may remain. // fewer free pages than the 1000 Vacuum frees per call, so none may remain.
@@ -215,6 +245,8 @@ func TestConnectionPoolPragmas(t *testing.T) {
// the free pages stayed in the file, and with the PRAGMA run by ExecContext // the free pages stayed in the file, and with the PRAGMA run by ExecContext
// Vacuum freed only one page per call. // Vacuum freed only one page per call.
func TestVacuumReturnsFreePages(t *testing.T) { func TestVacuumReturnsFreePages(t *testing.T) {
t.Parallel()
cfg := &config.Config{StateDir: t.TempDir()} cfg := &config.Config{StateDir: t.TempDir()}
db, err := New(cfg, logger.New()) db, err := New(cfg, logger.New())
@@ -225,10 +257,12 @@ func TestVacuumReturnsFreePages(t *testing.T) {
ctx := context.Background() ctx := context.Background()
ts := time.Now().UTC() ts := time.Now().UTC()
const asn = 64500 const asn = 64500
routes := make([]*LiveRoute, 0, vacuumTestRoutes) routes := make([]*LiveRoute, 0, vacuumTestRoutes)
deletions := make([]LiveRouteDeletion, 0, vacuumTestRoutes) deletions := make([]LiveRouteDeletion, 0, vacuumTestRoutes)
for i := range vacuumTestRoutes { for i := range vacuumTestRoutes {
route := mkV4Route(t, fmt.Sprintf("10.%d.%d.0/24", i/256, i%256), asn, ts) route := mkV4Route(t, fmt.Sprintf("10.%d.%d.0/24", i/256, i%256), asn, ts)
routes = append(routes, route) routes = append(routes, route)
@@ -240,29 +274,39 @@ func TestVacuumReturnsFreePages(t *testing.T) {
}) })
} }
if err := db.UpsertLiveRouteBatch(routes); err != nil { err = db.UpsertLiveRouteBatch(routes)
if err != nil {
t.Fatalf("failed to write routes: %v", err) t.Fatalf("failed to write routes: %v", err)
} }
if err := db.DeleteLiveRouteBatch(deletions); err != nil {
err = db.DeleteLiveRouteBatch(deletions)
if err != nil {
t.Fatalf("failed to delete routes: %v", err) t.Fatalf("failed to delete routes: %v", err)
} }
var before int var before int
if err := db.db.QueryRowContext(ctx, "PRAGMA freelist_count").Scan(&before); err != nil {
err = db.db.QueryRowContext(ctx, "PRAGMA freelist_count").Scan(&before)
if err != nil {
t.Fatalf("failed to read freelist_count: %v", err) t.Fatalf("failed to read freelist_count: %v", err)
} }
if before == 0 { if before == 0 {
t.Fatalf("no free pages after deleting %d routes", vacuumTestRoutes) t.Fatalf("no free pages after deleting %d routes", vacuumTestRoutes)
} }
if err := db.Vacuum(ctx); err != nil { err = db.Vacuum(ctx)
if err != nil {
t.Fatalf("Vacuum failed: %v", err) t.Fatalf("Vacuum failed: %v", err)
} }
var after int var after int
if err := db.db.QueryRowContext(ctx, "PRAGMA freelist_count").Scan(&after); err != nil {
err = db.db.QueryRowContext(ctx, "PRAGMA freelist_count").Scan(&after)
if err != nil {
t.Fatalf("failed to read freelist_count: %v", err) t.Fatalf("failed to read freelist_count: %v", err)
} }
if after != 0 { if after != 0 {
t.Errorf("free pages after Vacuum = %d of %d, want 0", after, before) t.Errorf("free pages after Vacuum = %d of %d, want 0", after, before)
} }
@@ -277,6 +321,8 @@ func TestVacuumReturnsFreePages(t *testing.T) {
// no batch is dropped. The checkpoint runs without the Database mutex, exactly // no batch is dropped. The checkpoint runs without the Database mutex, exactly
// as the background maintainer does in production. // as the background maintainer does in production.
func TestBatchWriteDuringCheckpoint(t *testing.T) { func TestBatchWriteDuringCheckpoint(t *testing.T) {
t.Parallel()
cfg := &config.Config{StateDir: t.TempDir()} cfg := &config.Config{StateDir: t.TempDir()}
db, err := New(cfg, logger.New()) db, err := New(cfg, logger.New())
@@ -289,8 +335,10 @@ func TestBatchWriteDuringCheckpoint(t *testing.T) {
var wg sync.WaitGroup var wg sync.WaitGroup
wg.Add(1) wg.Add(1)
go func() { go func() {
defer wg.Done() defer wg.Done()
for { for {
select { select {
case <-ctx.Done(): case <-ctx.Done():
@@ -302,12 +350,14 @@ func TestBatchWriteDuringCheckpoint(t *testing.T) {
}() }()
ts := time.Now().UTC() ts := time.Now().UTC()
for i := 0; i < contentionIterations; i++ { for i := range contentionIterations {
asns := map[int]time.Time{ asns := map[int]time.Time{
i % contendedASNCount: ts, i % contendedASNCount: ts,
(i % contendedASNCount) + asnSecondBand: ts, (i % contendedASNCount) + asnSecondBand: ts,
} }
if err := db.GetOrCreateASNBatch(asns); err != nil {
err := db.GetOrCreateASNBatch(asns)
if err != nil {
cancel() cancel()
wg.Wait() wg.Wait()
t.Fatalf("batch write failed under checkpoint contention: %v", err) t.Fatalf("batch write failed under checkpoint contention: %v", err)
+26 -19
View File
@@ -1,4 +1,3 @@
// Package database provides SQLite storage for BGP routing data.
package database package database
import ( import (
@@ -9,6 +8,8 @@ import (
// ASN represents an Autonomous System Number with its metadata including // ASN represents an Autonomous System Number with its metadata including
// handle, description, WHOIS data, and first/last seen timestamps. // handle, description, WHOIS data, and first/last seen timestamps.
//
//nolint:tagliatelle // field names are the existing JSON API
type ASN struct { type ASN struct {
ASN int `json:"asn"` ASN int `json:"asn"`
Handle string `json:"handle"` Handle string `json:"handle"`
@@ -36,37 +37,39 @@ type ASN struct {
// Prefix represents an IP prefix (CIDR block) with its IP version (4 or 6) // Prefix represents an IP prefix (CIDR block) with its IP version (4 or 6)
// and first/last seen timestamps. // and first/last seen timestamps.
type Prefix struct { type Prefix struct {
ID uuid.UUID `json:"id"` ID uuid.UUID
Prefix string `json:"prefix"` Prefix string
IPVersion int `json:"ip_version"` // 4 or 6 IPVersion int // 4 or 6
FirstSeen time.Time `json:"first_seen"` FirstSeen time.Time
LastSeen time.Time `json:"last_seen"` LastSeen time.Time
} }
// Announcement represents a BGP announcement or withdrawal event, // Announcement represents a BGP announcement or withdrawal event,
// containing the prefix, AS path, origin ASN, peer ASN, next hop, and timestamp. // containing the prefix, AS path, origin ASN, peer ASN, next hop, and timestamp.
type Announcement struct { type Announcement struct {
ID uuid.UUID `json:"id"` ID uuid.UUID
PrefixID uuid.UUID `json:"prefix_id"` PrefixID uuid.UUID
PeerASN int `json:"peer_asn"` PeerASN int
OriginASN int `json:"origin_asn"` OriginASN int
Path string `json:"path"` // JSON-encoded AS path Path string // JSON-encoded AS path
NextHop string `json:"next_hop"` NextHop string
Timestamp time.Time `json:"timestamp"` Timestamp time.Time
IsWithdrawal bool `json:"is_withdrawal"` IsWithdrawal bool
} }
// ASNPeering represents a peering relationship between two ASNs, // ASNPeering represents a peering relationship between two ASNs,
// stored with the lower ASN as ASA and the higher as ASB. // stored with the lower ASN as ASA and the higher as ASB.
type ASNPeering struct { type ASNPeering struct {
ID uuid.UUID `json:"id"` ID uuid.UUID
ASA int `json:"as_a"` ASA int
ASB int `json:"as_b"` ASB int
FirstSeen time.Time `json:"first_seen"` FirstSeen time.Time
LastSeen time.Time `json:"last_seen"` LastSeen time.Time
} }
// LiveRoute represents a route in the live routing table // LiveRoute represents a route in the live routing table
//
//nolint:tagliatelle // field names are the existing JSON API
type LiveRoute struct { type LiveRoute struct {
ID uuid.UUID `json:"id"` ID uuid.UUID `json:"id"`
Prefix string `json:"prefix"` Prefix string `json:"prefix"`
@@ -80,12 +83,16 @@ type LiveRoute struct {
} }
// PrefixDistribution represents the distribution of prefixes by mask length // PrefixDistribution represents the distribution of prefixes by mask length
//
//nolint:tagliatelle // field names are the existing JSON API
type PrefixDistribution struct { type PrefixDistribution struct {
MaskLength int `json:"mask_length"` MaskLength int `json:"mask_length"`
Count int `json:"count"` Count int `json:"count"`
} }
// IPInfo represents comprehensive IP information for the /ip endpoint // IPInfo represents comprehensive IP information for the /ip endpoint
//
//nolint:tagliatelle // field names are the existing JSON API
type IPInfo struct { type IPInfo struct {
IP string `json:"ip"` IP string `json:"ip"`
PTR []string `json:"ptr,omitempty"` PTR []string `json:"ptr,omitempty"`
+1
View File
@@ -74,6 +74,7 @@ func getSourceAttrs() []slog.Attr {
// Get the function name // Get the function name
fn := runtime.FuncForPC(pc) fn := runtime.FuncForPC(pc)
var funcName string var funcName string
if fn != nil { if fn != nil {
funcName = filepath.Base(fn.Name()) funcName = filepath.Base(fn.Name())
+9 -4
View File
@@ -115,9 +115,11 @@ func (t *Tracker) GetStreamMetrics() StreamMetrics {
if msgCount >= 0 { if msgCount >= 0 {
totalMessages = uint64(msgCount) totalMessages = uint64(msgCount)
} }
if byteCount >= 0 { if byteCount >= 0 {
totalBytes = uint64(byteCount) totalBytes = uint64(byteCount)
} }
if wireByteCount >= 0 { if wireByteCount >= 0 {
totalWireBytes = uint64(wireByteCount) totalWireBytes = uint64(wireByteCount)
} }
@@ -161,7 +163,7 @@ func (t *Tracker) RecordWithdrawal() {
func (t *Tracker) SetBGPPeerCount(count int) { func (t *Tracker) SetBGPPeerCount(count int) {
// BGP peer count is always small (< 1000), so int32 is safe // BGP peer count is always small (< 1000), so int32 is safe
if count > 0 && count < 1<<31 { if count > 0 && count < 1<<31 {
t.bgpPeerCount.Store(int32(count)) //nolint:gosec // count is validated t.bgpPeerCount.Store(int32(count))
} }
} }
@@ -209,7 +211,8 @@ type StreamMetrics struct {
TotalMessages uint64 TotalMessages uint64
// TotalBytes is the total number of decompressed bytes received since startup // TotalBytes is the total number of decompressed bytes received since startup
TotalBytes uint64 TotalBytes uint64
// TotalWireBytes is the total number of bytes received on the wire (before decompression) // TotalWireBytes is the total number of bytes received on the wire (before
// decompression)
TotalWireBytes uint64 TotalWireBytes uint64
// ConnectedSince is the time when the current connection was established // ConnectedSince is the time when the current connection was established
ConnectedSince time.Time ConnectedSince time.Time
@@ -217,9 +220,11 @@ type StreamMetrics struct {
Connected bool Connected bool
// MessagesPerSec is the rate of messages received per second (1-minute average) // MessagesPerSec is the rate of messages received per second (1-minute average)
MessagesPerSec float64 MessagesPerSec float64
// BitsPerSec is the rate of decompressed bits received per second (1-minute average) // BitsPerSec is the rate of decompressed bits received per second (1-minute
// average)
BitsPerSec float64 BitsPerSec float64
// WireBitsPerSec is the rate of bits received on the wire per second (1-minute average) // WireBitsPerSec is the rate of bits received on the wire per second
// (1-minute average)
WireBitsPerSec float64 WireBitsPerSec float64
// ReconnectCount is the number of reconnections since startup // ReconnectCount is the number of reconnections since startup
ReconnectCount uint64 ReconnectCount uint64
+15 -5
View File
@@ -1,4 +1,5 @@
// Package ristypes defines the data structures for RIS Live BGP messages and announcements. // Package ristypes defines the data structures for RIS Live BGP messages and
// announcements.
package ristypes package ristypes
import ( import (
@@ -17,25 +18,30 @@ type ASPath []int
func (p *ASPath) UnmarshalJSON(data []byte) error { func (p *ASPath) UnmarshalJSON(data []byte) error {
// First try to unmarshal as a simple array of integers // First try to unmarshal as a simple array of integers
var simple []int var simple []int
if err := json.Unmarshal(data, &simple); err == nil {
err := json.Unmarshal(data, &simple)
if err == nil {
*p = ASPath(simple) *p = ASPath(simple)
return nil return nil
} }
// If that fails, unmarshal as array of interfaces and flatten // If that fails, unmarshal as array of interfaces and flatten
var raw []interface{} var raw []any
if err := json.Unmarshal(data, &raw); err != nil {
err = json.Unmarshal(data, &raw)
if err != nil {
return err return err
} }
// Flatten the array // Flatten the array
result := make([]int, 0) result := make([]int, 0)
for _, item := range raw { for _, item := range raw {
switch v := item.(type) { switch v := item.(type) {
case float64: case float64:
result = append(result, int(v)) result = append(result, int(v))
case []interface{}: case []any:
// Nested array - flatten it // Nested array - flatten it
for _, nested := range v { for _, nested := range v {
if num, ok := nested.(float64); ok { if num, ok := nested.(float64); ok {
@@ -62,6 +68,8 @@ type RISLiveMessage struct {
// It contains metadata about the BGP session (peer, ASN, host) along with // It contains metadata about the BGP session (peer, ASN, host) along with
// the actual BGP update data including AS path, communities, announcements, // the actual BGP update data including AS path, communities, announcements,
// and withdrawals. // and withdrawals.
//
//nolint:tagliatelle // field names are the RIS Live feed's
type RISMessage struct { type RISMessage struct {
Type string `json:"type"` Type string `json:"type"`
Timestamp float64 `json:"timestamp"` Timestamp float64 `json:"timestamp"`
@@ -90,6 +98,8 @@ type RISMessage struct {
// RISAnnouncement represents a BGP route announcement within a RIS message. // RISAnnouncement represents a BGP route announcement within a RIS message.
// It contains the next hop IP address and the list of prefixes being announced // It contains the next hop IP address and the list of prefixes being announced
// via that next hop. // via that next hop.
//
//nolint:tagliatelle // field names are the RIS Live feed's
type RISAnnouncement struct { type RISAnnouncement struct {
NextHop string `json:"next_hop"` NextHop string `json:"next_hop"`
Prefixes []string `json:"prefixes"` Prefixes []string `json:"prefixes"`
+24 -7
View File
@@ -1,10 +1,12 @@
package ristypes package ristypes_test
import ( import (
"bufio" "bufio"
"encoding/json" "encoding/json"
"os" "os"
"testing" "testing"
"git.eeqj.de/sneak/routewatch/internal/ristypes"
) )
// messageExamplesPath is the captured RIS Live feed used as decode fixtures, // messageExamplesPath is the captured RIS Live feed used as decode fixtures,
@@ -15,32 +17,43 @@ const messageExamplesPath = "../../docs/message-examples.json"
// streamer does and checks that the fields no handler reads (Community, Raw) // streamer does and checks that the fields no handler reads (Community, Raw)
// stay empty while the fields handlers use (Path, Announcements) still decode. // stay empty while the fields handlers use (Path, Announcements) still decode.
func TestDecodeDropsCommunityAndRaw(t *testing.T) { func TestDecodeDropsCommunityAndRaw(t *testing.T) {
t.Parallel()
f, err := os.Open(messageExamplesPath) f, err := os.Open(messageExamplesPath)
if err != nil { if err != nil {
t.Fatalf("open fixtures: %v", err) t.Fatalf("open fixtures: %v", err)
} }
defer f.Close()
defer func() {
_ = f.Close()
}()
var messages, withPath, withAnnouncements int var messages, withPath, withAnnouncements int
scanner := bufio.NewScanner(f) scanner := bufio.NewScanner(f)
scanner.Buffer(make([]byte, 0, 64*1024), 1024*1024) scanner.Buffer(make([]byte, 0, 64*1024), 1024*1024)
for scanner.Scan() { for scanner.Scan() {
line := scanner.Bytes() line := scanner.Bytes()
if len(line) == 0 { if len(line) == 0 {
continue continue
} }
var wrapper RISLiveMessage var wrapper ristypes.RISLiveMessage
if err := json.Unmarshal(line, &wrapper); err != nil {
err := json.Unmarshal(line, &wrapper)
if err != nil {
t.Fatalf("unmarshal message %d: %v", messages+1, err) t.Fatalf("unmarshal message %d: %v", messages+1, err)
} }
messages++ messages++
msg := wrapper.Data msg := wrapper.Data
if msg.Community != nil { if msg.Community != nil {
t.Errorf("message %d: Community decoded, want empty: %v", messages, msg.Community) t.Errorf("message %d: Community decoded, want empty: %v",
messages, msg.Community)
} }
if msg.Raw != "" { if msg.Raw != "" {
t.Errorf("message %d: Raw decoded, want empty", messages) t.Errorf("message %d: Raw decoded, want empty", messages)
} }
@@ -48,11 +61,14 @@ func TestDecodeDropsCommunityAndRaw(t *testing.T) {
if len(msg.Path) > 0 { if len(msg.Path) > 0 {
withPath++ withPath++
} }
if len(msg.Announcements) > 0 { if len(msg.Announcements) > 0 {
withAnnouncements++ withAnnouncements++
} }
} }
if err := scanner.Err(); err != nil {
err = scanner.Err()
if err != nil {
t.Fatalf("scan fixtures: %v", err) t.Fatalf("scan fixtures: %v", err)
} }
@@ -62,6 +78,7 @@ func TestDecodeDropsCommunityAndRaw(t *testing.T) {
// The fixtures include announcement messages; a used field must still decode, // The fixtures include announcement messages; a used field must still decode,
// otherwise an empty Community/Raw would prove nothing. // otherwise an empty Community/Raw would prove nothing.
if withPath == 0 || withAnnouncements == 0 { if withPath == 0 || withAnnouncements == 0 {
t.Fatalf("used fields did not decode: withPath=%d withAnnouncements=%d", withPath, withAnnouncements) t.Fatalf("used fields did not decode: withPath=%d withAnnouncements=%d",
withPath, withAnnouncements)
} }
} }
+32 -16
View File
@@ -1,10 +1,11 @@
// Package routewatch contains the primary RouteWatch type that represents a running instance // Package routewatch contains the primary RouteWatch type that represents a
// of the application and contains pointers to its core dependencies, and is responsible for initialization. // running instance of the application and contains pointers to its core
// dependencies, and is responsible for initialization.
package routewatch package routewatch
import ( import (
"context" "context"
"fmt" "errors"
"sync" "sync"
"time" "time"
@@ -18,6 +19,10 @@ import (
"go.uber.org/fx" "go.uber.org/fx"
) )
// errNonBatchedNotImplemented is returned by Run when batched database writes
// are disabled, since only the batched handlers exist.
var errNonBatchedNotImplemented = errors.New("non-batched handlers not implemented")
// Dependencies contains all dependencies for RouteWatch // Dependencies contains all dependencies for RouteWatch
type Dependencies struct { type Dependencies struct {
fx.In fx.In
@@ -68,8 +73,10 @@ func (rw *RouteWatch) Run(ctx context.Context) error {
// Apply runtime limit if specified // Apply runtime limit if specified
if rw.maxRuntime > 0 { if rw.maxRuntime > 0 {
var cancel context.CancelFunc var cancel context.CancelFunc
ctx, cancel = context.WithTimeout(ctx, rw.maxRuntime) ctx, cancel = context.WithTimeout(ctx, rw.maxRuntime)
defer cancel() defer cancel()
rw.logger.Info("Running with time limit", "max_runtime", rw.maxRuntime) rw.logger.Info("Running with time limit", "max_runtime", rw.maxRuntime)
} }
@@ -96,29 +103,31 @@ func (rw *RouteWatch) Run(ctx context.Context) error {
// Non-batched handlers not implemented yet // Non-batched handlers not implemented yet
rw.logger.Error("Non-batched handlers not implemented") rw.logger.Error("Non-batched handlers not implemented")
return fmt.Errorf("non-batched handlers not implemented") return errNonBatchedNotImplemented
} }
// No longer need routing table handler - PrefixHandler maintains live_routes table // No longer need routing table handler - PrefixHandler maintains live_routes table
// Start streaming // Start streaming
if err := rw.streamer.Start(); err != nil { err := rw.streamer.Start()
if err != nil {
return err return err
} }
// Start HTTP server // Start HTTP server
if err := rw.server.Start(); err != nil { err = rw.server.Start()
if err != nil {
return err return err
} }
// Start ASN WHOIS fetcher for background updates // Start ASN WHOIS fetcher for background updates
rw.asnFetcher = NewASNFetcher(rw.db, rw.logger.Logger) rw.asnFetcher = NewASNFetcher(rw.db, rw.logger.Logger)
rw.asnFetcher.Start() rw.asnFetcher.Start(ctx)
rw.server.SetASNFetcher(rw.asnFetcher) rw.server.SetASNFetcher(rw.asnFetcher)
// Start database maintenance goroutine // Start database maintenance goroutine
rw.dbMaintainer = NewDBMaintainer(rw.db, rw.logger.Logger) rw.dbMaintainer = NewDBMaintainer(rw.db, rw.logger.Logger)
rw.dbMaintainer.Start() rw.dbMaintainer.Start(ctx)
// Wait for context cancellation // Wait for context cancellation
<-ctx.Done() <-ctx.Done()
@@ -127,13 +136,14 @@ func (rw *RouteWatch) Run(ctx context.Context) error {
} }
// Shutdown performs graceful shutdown of all services // Shutdown performs graceful shutdown of all services
func (rw *RouteWatch) Shutdown() { func (rw *RouteWatch) Shutdown(ctx context.Context) {
rw.mu.Lock() rw.mu.Lock()
if rw.shutdown { if rw.shutdown {
rw.mu.Unlock() rw.mu.Unlock()
return return
} }
rw.shutdown = true rw.shutdown = true
rw.mu.Unlock() rw.mu.Unlock()
@@ -142,14 +152,17 @@ func (rw *RouteWatch) Shutdown() {
rw.logger.Info("Flushing database handler") rw.logger.Info("Flushing database handler")
rw.dbHandler.Stop() rw.dbHandler.Stop()
} }
if rw.peerHandler != nil { if rw.peerHandler != nil {
rw.logger.Info("Flushing peer handler") rw.logger.Info("Flushing peer handler")
rw.peerHandler.Stop() rw.peerHandler.Stop()
} }
if rw.prefixHandler != nil { if rw.prefixHandler != nil {
rw.logger.Info("Flushing prefix handler") rw.logger.Info("Flushing prefix handler")
rw.prefixHandler.Stop() rw.prefixHandler.Stop()
} }
if rw.peeringHandler != nil { if rw.peeringHandler != nil {
rw.logger.Info("Flushing peering handler") rw.logger.Info("Flushing peering handler")
rw.peeringHandler.Stop() rw.peeringHandler.Stop()
@@ -168,11 +181,15 @@ func (rw *RouteWatch) Shutdown() {
// Stop services // Stop services
rw.streamer.Stop() rw.streamer.Stop()
// Stop HTTP server with a timeout // Stop HTTP server with a timeout of its own, however much of ctx is left
const serverStopTimeout = 5 * time.Second const serverStopTimeout = 5 * time.Second
stopCtx, cancel := context.WithTimeout(context.Background(), serverStopTimeout)
stopCtx, cancel := context.WithTimeout(
context.WithoutCancel(ctx), serverStopTimeout)
defer cancel() defer cancel()
if err := rw.server.Stop(stopCtx); err != nil {
err := rw.server.Stop(stopCtx)
if err != nil {
rw.logger.Error("Failed to stop HTTP server gracefully", "error", err) rw.logger.Error("Failed to stop HTTP server gracefully", "error", err)
} }
@@ -185,12 +202,11 @@ func (rw *RouteWatch) Shutdown() {
"bits_per_sec", metrics.BitsPerSec, "bits_per_sec", metrics.BitsPerSec,
"duration", time.Since(metrics.ConnectedSince), "duration", time.Since(metrics.ConnectedSince),
) )
} }
// getModule provides all fx dependencies // getModule provides all fx dependencies
func getModule() fx.Option { func getModule() []fx.Option {
return fx.Options( return []fx.Option{
fx.Provide( fx.Provide(
logger.New, logger.New,
config.New, config.New,
@@ -203,5 +219,5 @@ func getModule() fx.Option {
server.New, server.New,
New, New,
), ),
) }
} }
@@ -50,7 +50,9 @@ func newMockStore() *mockStore {
} }
// GetOrCreateASN mock implementation // GetOrCreateASN mock implementation
func (m *mockStore) GetOrCreateASN(number int, timestamp time.Time) (*database.ASN, error) { func (m *mockStore) GetOrCreateASN(
number int, timestamp time.Time,
) (*database.ASN, error) {
m.mu.Lock() m.mu.Lock()
defer m.mu.Unlock() defer m.mu.Unlock()
@@ -99,11 +101,14 @@ func (m *mockStore) UpdatePrefixesBatch(prefixes map[string]time.Time) error {
} }
} }
} }
return nil return nil
} }
// GetOrCreatePrefix mock implementation // GetOrCreatePrefix mock implementation
func (m *mockStore) GetOrCreatePrefix(prefix string, timestamp time.Time) (*database.Prefix, error) { func (m *mockStore) GetOrCreatePrefix(
prefix string, timestamp time.Time,
) (*database.Prefix, error) {
m.mu.Lock() m.mu.Lock()
defer m.mu.Unlock() defer m.mu.Unlock()
@@ -168,7 +173,7 @@ func (m *mockStore) RecordPeering(asA, asB int, _ time.Time) error {
} }
// UpdatePeer mock implementation // UpdatePeer mock implementation
func (m *mockStore) UpdatePeer(peerIP string, peerASN int, messageType string, timestamp time.Time) error { func (m *mockStore) UpdatePeer(_ string, _ int, _ string, _ time.Time) error {
// Simple mock - just return nil // Simple mock - just return nil
return nil return nil
} }
@@ -194,41 +199,45 @@ func (m *mockStore) GetStats() (database.Stats, error) {
} }
// GetStatsContext returns statistics about the mock store with context support // GetStatsContext returns statistics about the mock store with context support
func (m *mockStore) GetStatsContext(ctx context.Context) (database.Stats, error) { func (m *mockStore) GetStatsContext(_ context.Context) (database.Stats, error) {
return m.GetStats() return m.GetStats()
} }
// UpsertLiveRoute mock implementation // UpsertLiveRoute mock implementation
func (m *mockStore) UpsertLiveRoute(route *database.LiveRoute) error { func (m *mockStore) UpsertLiveRoute(_ *database.LiveRoute) error {
// Simple mock - just return nil // Simple mock - just return nil
return nil return nil
} }
// DeleteLiveRoute mock implementation // DeleteLiveRoute mock implementation
func (m *mockStore) DeleteLiveRoute(prefix string, originASN int, peerIP string) error { func (m *mockStore) DeleteLiveRoute(_ string, _ int, _ string) error {
// Simple mock - just return nil // Simple mock - just return nil
return nil return nil
} }
// GetPrefixDistribution mock implementation // GetPrefixDistribution mock implementation
func (m *mockStore) GetPrefixDistribution() (ipv4 []database.PrefixDistribution, ipv6 []database.PrefixDistribution, err error) { func (m *mockStore) GetPrefixDistribution() (
[]database.PrefixDistribution, []database.PrefixDistribution, error,
) {
// Return empty distributions for now // Return empty distributions for now
return nil, nil, nil return nil, nil, nil
} }
// GetPrefixDistributionContext mock implementation with context support // GetPrefixDistributionContext mock implementation with context support
func (m *mockStore) GetPrefixDistributionContext(ctx context.Context) (ipv4 []database.PrefixDistribution, ipv6 []database.PrefixDistribution, err error) { func (m *mockStore) GetPrefixDistributionContext(_ context.Context) (
[]database.PrefixDistribution, []database.PrefixDistribution, error,
) {
return m.GetPrefixDistribution() return m.GetPrefixDistribution()
} }
// GetLiveRouteCounts mock implementation // GetLiveRouteCounts mock implementation
func (m *mockStore) GetLiveRouteCounts() (ipv4Count, ipv6Count int, err error) { func (m *mockStore) GetLiveRouteCounts() (int, int, error) {
// Return mock counts // Return mock counts
return m.RouteCount / 2, m.RouteCount / 2, nil return m.RouteCount / 2, m.RouteCount / 2, nil
} }
// GetLiveRouteCountsContext mock implementation with context support // GetLiveRouteCountsContext mock implementation with context support
func (m *mockStore) GetLiveRouteCountsContext(ctx context.Context) (ipv4Count, ipv6Count int, err error) { func (m *mockStore) GetLiveRouteCountsContext(_ context.Context) (int, int, error) {
return m.GetLiveRouteCounts() return m.GetLiveRouteCounts()
} }
@@ -247,39 +256,49 @@ func (m *mockStore) GetASDetails(asn int) (*database.ASN, []database.LiveRoute,
} }
// GetASDetailsContext mock implementation with context support // GetASDetailsContext mock implementation with context support
func (m *mockStore) GetASDetailsContext(ctx context.Context, asn int) (*database.ASN, []database.LiveRoute, error) { func (m *mockStore) GetASDetailsContext(
_ context.Context, asn int,
) (*database.ASN, []database.LiveRoute, error) {
return m.GetASDetails(asn) return m.GetASDetails(asn)
} }
// GetPrefixDetails mock implementation // GetPrefixDetails mock implementation
func (m *mockStore) GetPrefixDetails(prefix string) ([]database.LiveRoute, error) { func (m *mockStore) GetPrefixDetails(_ string) ([]database.LiveRoute, error) {
// Return empty routes for now // Return empty routes for now
return []database.LiveRoute{}, nil return []database.LiveRoute{}, nil
} }
// GetPrefixDetailsContext mock implementation with context support // GetPrefixDetailsContext mock implementation with context support
func (m *mockStore) GetPrefixDetailsContext(ctx context.Context, prefix string) ([]database.LiveRoute, error) { func (m *mockStore) GetPrefixDetailsContext(
_ context.Context, prefix string,
) ([]database.LiveRoute, error) {
return m.GetPrefixDetails(prefix) return m.GetPrefixDetails(prefix)
} }
func (m *mockStore) GetRandomPrefixesByLength(maskLength, ipVersion, limit int) ([]database.LiveRoute, error) { func (m *mockStore) GetRandomPrefixesByLength(
_, _, _ int,
) ([]database.LiveRoute, error) {
// Return empty routes for now // Return empty routes for now
return []database.LiveRoute{}, nil return []database.LiveRoute{}, nil
} }
// GetRandomPrefixesByLengthContext mock implementation with context support // GetRandomPrefixesByLengthContext mock implementation with context support
func (m *mockStore) GetRandomPrefixesByLengthContext(ctx context.Context, maskLength, ipVersion, limit int) ([]database.LiveRoute, error) { func (m *mockStore) GetRandomPrefixesByLengthContext(
_ context.Context, maskLength, ipVersion, limit int,
) ([]database.LiveRoute, error) {
return m.GetRandomPrefixesByLength(maskLength, ipVersion, limit) return m.GetRandomPrefixesByLength(maskLength, ipVersion, limit)
} }
// GetASPeers mock implementation // GetASPeers mock implementation
func (m *mockStore) GetASPeers(asn int) ([]database.ASPeer, error) { func (m *mockStore) GetASPeers(_ int) ([]database.ASPeer, error) {
// Return empty peers for now // Return empty peers for now
return []database.ASPeer{}, nil return []database.ASPeer{}, nil
} }
// GetASPeersContext mock implementation with context support // GetASPeersContext mock implementation with context support
func (m *mockStore) GetASPeersContext(ctx context.Context, asn int) ([]database.ASPeer, error) { func (m *mockStore) GetASPeersContext(
_ context.Context, asn int,
) ([]database.ASPeer, error) {
return m.GetASPeers(asn) return m.GetASPeers(asn)
} }
@@ -289,8 +308,11 @@ func (m *mockStore) GetIPInfo(ip string) (*database.IPInfo, error) {
} }
// GetIPInfoContext mock implementation with context support // GetIPInfoContext mock implementation with context support
func (m *mockStore) GetIPInfoContext(ctx context.Context, ip string) (*database.IPInfo, error) { func (m *mockStore) GetIPInfoContext(
_ context.Context, ip string,
) (*database.IPInfo, error) {
now := time.Now() now := time.Now()
return &database.IPInfo{ return &database.IPInfo{
IP: ip, IP: ip,
Netblock: "8.8.8.0/24", Netblock: "8.8.8.0/24",
@@ -307,17 +329,21 @@ func (m *mockStore) GetIPInfoContext(ctx context.Context, ip string) (*database.
} }
// GetNextStaleASN mock implementation // GetNextStaleASN mock implementation
func (m *mockStore) GetNextStaleASN(ctx context.Context, staleThreshold time.Duration) (int, error) { func (m *mockStore) GetNextStaleASN(_ context.Context, _ time.Duration) (int, error) {
return 0, database.ErrNoStaleASN return 0, database.ErrNoStaleASN
} }
// UpdateASNWHOIS mock implementation // UpdateASNWHOIS mock implementation
func (m *mockStore) UpdateASNWHOIS(ctx context.Context, update *database.ASNWHOISUpdate) error { func (m *mockStore) UpdateASNWHOIS(
_ context.Context, _ *database.ASNWHOISUpdate,
) error {
return nil return nil
} }
// GetWHOISStats mock implementation // GetWHOISStats mock implementation
func (m *mockStore) GetWHOISStats(ctx context.Context, staleThreshold time.Duration) (*database.WHOISStats, error) { func (m *mockStore) GetWHOISStats(
_ context.Context, _ time.Duration,
) (*database.WHOISStats, error) {
m.mu.Lock() m.mu.Lock()
defer m.mu.Unlock() defer m.mu.Unlock()
@@ -345,19 +371,22 @@ func (m *mockStore) UpsertLiveRouteBatch(routes []*database.LiveRoute) error {
LastSeen: route.LastUpdated, LastSeen: route.LastUpdated,
} }
m.PrefixCount++ m.PrefixCount++
if route.IPVersion == 4 { if route.IPVersion == 4 {
m.IPv4Prefixes++ m.IPv4Prefixes++
} else { } else {
m.IPv6Prefixes++ m.IPv6Prefixes++
} }
} }
m.RouteCount++ m.RouteCount++
} }
return nil return nil
} }
// DeleteLiveRouteBatch mock implementation // DeleteLiveRouteBatch mock implementation
func (m *mockStore) DeleteLiveRouteBatch(deletions []database.LiveRouteDeletion) error { func (m *mockStore) DeleteLiveRouteBatch(_ []database.LiveRouteDeletion) error {
// Simple mock - just return nil // Simple mock - just return nil
return nil return nil
} }
@@ -377,43 +406,53 @@ func (m *mockStore) GetOrCreateASNBatch(asns map[int]time.Time) error {
m.ASNCount++ m.ASNCount++
} }
} }
return nil return nil
} }
// UpdatePeerBatch mock implementation // UpdatePeerBatch mock implementation
func (m *mockStore) UpdatePeerBatch(peers map[string]database.PeerUpdate) error { func (m *mockStore) UpdatePeerBatch(_ map[string]database.PeerUpdate) error {
// Simple mock - just return nil // Simple mock - just return nil
return nil return nil
} }
// Vacuum mock implementation // Vacuum mock implementation
func (m *mockStore) Vacuum(ctx context.Context) error { func (m *mockStore) Vacuum(_ context.Context) error {
return nil return nil
} }
// Analyze mock implementation // Analyze mock implementation
func (m *mockStore) Analyze(ctx context.Context) error { func (m *mockStore) Analyze(_ context.Context) error {
return nil return nil
} }
// Checkpoint mock implementation // Checkpoint mock implementation
func (m *mockStore) Checkpoint(ctx context.Context) error { func (m *mockStore) Checkpoint(_ context.Context) error {
return nil return nil
} }
// Ping mock implementation // Ping mock implementation
func (m *mockStore) Ping(ctx context.Context) error { func (m *mockStore) Ping(_ context.Context) error {
return nil return nil
} }
func TestRouteWatchLiveFeed(t *testing.T) { func TestRouteWatchLiveFeed(t *testing.T) {
t.Parallel()
if testing.Short() { if testing.Short() {
t.Skip("skipping live RIPE RIS network feed test in short mode; run without -short to include it") t.Skip("skipping live RIPE RIS network feed test in short mode; " +
"run without -short to include it")
} }
// Create mock database // Create mock database
mockDB := newMockStore() mockDB := newMockStore()
defer mockDB.Close()
defer func() {
err := mockDB.Close()
if err != nil {
t.Errorf("failed to close mock database: %v", err)
}
}()
logger := logger.New() logger := logger.New()
@@ -466,16 +505,19 @@ func TestRouteWatchLiveFeed(t *testing.T) {
if stats.ASNs == 0 { if stats.ASNs == 0 {
t.Error("Expected to receive some ASNs from live feed") t.Error("Expected to receive some ASNs from live feed")
} }
t.Logf("Received %d unique ASNs in 5 seconds", stats.ASNs) t.Logf("Received %d unique ASNs in 5 seconds", stats.ASNs)
if stats.Prefixes == 0 { if stats.Prefixes == 0 {
t.Error("Expected to receive some prefixes from live feed") t.Error("Expected to receive some prefixes from live feed")
} }
t.Logf("Received %d unique prefixes (%d IPv4, %d IPv6) in 5 seconds", stats.Prefixes, stats.IPv4Prefixes, stats.IPv6Prefixes)
t.Logf("Received %d unique prefixes (%d IPv4, %d IPv6) in 5 seconds",
stats.Prefixes, stats.IPv4Prefixes, stats.IPv6Prefixes)
if stats.Peerings == 0 { if stats.Peerings == 0 {
t.Error("Expected to receive some peerings from live feed") t.Error("Expected to receive some peerings from live feed")
} }
t.Logf("Recorded %d AS peering relationships in 5 seconds", stats.Peerings)
t.Logf("Recorded %d AS peering relationships in 5 seconds", stats.Peerings)
} }
+1 -1
View File
@@ -1,3 +1,3 @@
package routewatch package routewatch_test
// Tests for routewatch package are in app_integration_test.go // Tests for routewatch package are in app_integration_test.go
+18 -14
View File
@@ -71,7 +71,7 @@ func NewASHandler(db database.Store, logger *logger.Logger) *ASHandler {
// the AS path information needed to track autonomous systems. // the AS path information needed to track autonomous systems.
func (h *ASHandler) WantsMessage(messageType string) bool { func (h *ASHandler) WantsMessage(messageType string) bool {
// We only care about UPDATE messages for the database // We only care about UPDATE messages for the database
return messageType == "UPDATE" return messageType == updateMessageType
} }
// QueueCapacity returns the recommended message queue size for this handler. // QueueCapacity returns the recommended message queue size for this handler.
@@ -124,9 +124,20 @@ func (h *ASHandler) HandleMessage(msg *ristypes.RISMessage) {
} }
} }
// Stop gracefully shuts down the ASHandler by signaling the background flush
// goroutine to terminate and waiting for it to complete. Any pending ASN
// operations in the current batch are flushed to the database before Stop
// returns. This method should be called during application shutdown to ensure
// no data is lost.
func (h *ASHandler) Stop() {
close(h.stopCh)
h.wg.Wait()
}
// flushLoop runs in a goroutine and periodically flushes batches // flushLoop runs in a goroutine and periodically flushes batches
func (h *ASHandler) flushLoop() { func (h *ASHandler) flushLoop() {
defer h.wg.Done() defer h.wg.Done()
ticker := time.NewTicker(asnBatchTimeout) ticker := time.NewTicker(asnBatchTimeout)
defer ticker.Stop() defer ticker.Stop()
@@ -149,7 +160,8 @@ func (h *ASHandler) flushLoop() {
} }
} }
// flushBatchLocked flushes the ASN batch to the database (must be called with mutex held) // flushBatchLocked flushes the ASN batch to the database (must be called with
// mutex held)
func (h *ASHandler) flushBatchLocked() { func (h *ASHandler) flushBatchLocked() {
if len(h.batch) == 0 { if len(h.batch) == 0 {
return return
@@ -164,21 +176,13 @@ func (h *ASHandler) flushBatchLocked() {
} }
// Process all ASNs in a single batch transaction // Process all ASNs in a single batch transaction
if err := h.db.GetOrCreateASNBatch(asnMap); err != nil { err := h.db.GetOrCreateASNBatch(asnMap)
h.logger.Error("Failed to process ASN batch", "error", err, "count", len(asnMap)) if err != nil {
h.logger.Error("Failed to process ASN batch",
"error", err, "count", len(asnMap))
} }
// Clear batch // Clear batch
h.batch = h.batch[:0] h.batch = h.batch[:0]
h.lastFlush = time.Now() h.lastFlush = time.Now()
} }
// Stop gracefully shuts down the ASHandler by signaling the background flush
// goroutine to terminate and waiting for it to complete. Any pending ASN
// operations in the current batch are flushed to the database before Stop
// returns. This method should be called during application shutdown to ensure
// no data is lost.
func (h *ASHandler) Stop() {
close(h.stopCh)
h.wg.Wait()
}
+21 -24
View File
@@ -1,8 +1,8 @@
// Package routewatch contains the ASN WHOIS fetcher for background updates.
package routewatch package routewatch
import ( import (
"context" "context"
"errors"
"log/slog" "log/slog"
"sync" "sync"
"time" "time"
@@ -73,10 +73,13 @@ func NewASNFetcher(db database.Store, logger *slog.Logger) *ASNFetcher {
} }
} }
// Start begins the background ASN fetcher goroutine. // Start begins the background ASN fetcher goroutine. It runs until Stop:
func (f *ASNFetcher) Start() { // the end of ctx does not stop it.
func (f *ASNFetcher) Start(ctx context.Context) {
f.wg.Add(1) f.wg.Add(1)
go f.run()
go f.run(context.WithoutCancel(ctx))
f.logger.Info("ASN fetcher started", f.logger.Info("ASN fetcher started",
"base_interval", baseInterval, "base_interval", baseInterval,
"min_interval", minInterval, "min_interval", minInterval,
@@ -151,10 +154,7 @@ func (f *ASNFetcher) recordSuccess() {
f.consecutiveFails = 0 f.consecutiveFails = 0
// Decrease interval by half, but not below minimum // Decrease interval by half, but not below minimum
newInterval := f.currentInterval / backoffMultiplier newInterval := max(f.currentInterval/backoffMultiplier, minInterval)
if newInterval < minInterval {
newInterval = minInterval
}
if newInterval != f.currentInterval { if newInterval != f.currentInterval {
f.logger.Debug("Decreased fetch interval", f.logger.Debug("Decreased fetch interval",
@@ -177,10 +177,7 @@ func (f *ASNFetcher) recordFailure() {
f.consecutiveFails++ f.consecutiveFails++
// Exponential backoff: multiply by 2, capped at max // Exponential backoff: multiply by 2, capped at max
newInterval := f.currentInterval * backoffMultiplier newInterval := min(f.currentInterval*backoffMultiplier, maxInterval)
if newInterval > maxInterval {
newInterval = maxInterval
}
if newInterval != f.currentInterval { if newInterval != f.currentInterval {
f.logger.Debug("Increased fetch interval due to failure", f.logger.Debug("Increased fetch interval due to failure",
@@ -199,7 +196,7 @@ func (f *ASNFetcher) recordFailure() {
} }
// run is the main background loop. // run is the main background loop.
func (f *ASNFetcher) run() { func (f *ASNFetcher) run(ctx context.Context) {
defer f.wg.Done() defer f.wg.Done()
timer := time.NewTimer(f.getInterval()) timer := time.NewTimer(f.getInterval())
@@ -212,13 +209,13 @@ func (f *ASNFetcher) run() {
case asn := <-f.immediateQueue: case asn := <-f.immediateQueue:
// Process immediate request (respects lock) // Process immediate request (respects lock)
f.tryFetch(asn) f.tryFetch(ctx, asn)
// Reset timer after immediate fetch // Reset timer after immediate fetch
timer.Reset(f.getInterval()) timer.Reset(f.getInterval())
case <-timer.C: case <-timer.C:
// Background fetch of stale/missing ASN // Background fetch of stale/missing ASN
f.fetchNextStale() f.fetchNextStale(ctx)
// Reset timer with potentially updated interval // Reset timer with potentially updated interval
timer.Reset(f.getInterval()) timer.Reset(f.getInterval())
} }
@@ -227,7 +224,7 @@ func (f *ASNFetcher) run() {
// tryFetch attempts to fetch and update an ASN, respecting the fetch lock. // tryFetch attempts to fetch and update an ASN, respecting the fetch lock.
// Returns true if fetch was successful. // Returns true if fetch was successful.
func (f *ASNFetcher) tryFetch(asn int) bool { func (f *ASNFetcher) tryFetch(ctx context.Context, asn int) bool {
// Try to acquire lock, skip if another fetch is running // Try to acquire lock, skip if another fetch is running
if !f.fetchMu.TryLock() { if !f.fetchMu.TryLock() {
f.logger.Debug("Skipping fetch, another fetch in progress", "asn", asn) f.logger.Debug("Skipping fetch, another fetch in progress", "asn", asn)
@@ -236,11 +233,11 @@ func (f *ASNFetcher) tryFetch(asn int) bool {
} }
defer f.fetchMu.Unlock() defer f.fetchMu.Unlock()
return f.fetchAndUpdate(asn) return f.fetchAndUpdate(ctx, asn)
} }
// fetchNextStale finds and fetches the next ASN needing WHOIS data. // fetchNextStale finds and fetches the next ASN needing WHOIS data.
func (f *ASNFetcher) fetchNextStale() { func (f *ASNFetcher) fetchNextStale(ctx context.Context) {
// Try to acquire lock, skip if another fetch is running // Try to acquire lock, skip if another fetch is running
if !f.fetchMu.TryLock() { if !f.fetchMu.TryLock() {
f.logger.Debug("Skipping stale fetch, another fetch in progress") f.logger.Debug("Skipping stale fetch, another fetch in progress")
@@ -249,12 +246,12 @@ func (f *ASNFetcher) fetchNextStale() {
} }
defer f.fetchMu.Unlock() defer f.fetchMu.Unlock()
ctx, cancel := context.WithTimeout(context.Background(), time.Minute) lookupCtx, cancel := context.WithTimeout(ctx, time.Minute)
defer cancel() defer cancel()
asn, err := f.db.GetNextStaleASN(ctx, whoisStaleThreshold) asn, err := f.db.GetNextStaleASN(lookupCtx, whoisStaleThreshold)
if err != nil { if err != nil {
if err != database.ErrNoStaleASN { if !errors.Is(err, database.ErrNoStaleASN) {
f.logger.Error("Failed to get stale ASN", "error", err) f.logger.Error("Failed to get stale ASN", "error", err)
f.recordFailure() f.recordFailure()
} }
@@ -263,13 +260,13 @@ func (f *ASNFetcher) fetchNextStale() {
return return
} }
f.fetchAndUpdate(asn) f.fetchAndUpdate(ctx, asn)
} }
// fetchAndUpdate performs a WHOIS lookup and updates the database. // fetchAndUpdate performs a WHOIS lookup and updates the database.
// Returns true if successful. // Returns true if successful.
func (f *ASNFetcher) fetchAndUpdate(asn int) bool { func (f *ASNFetcher) fetchAndUpdate(ctx context.Context, asn int) bool {
ctx, cancel := context.WithTimeout(context.Background(), time.Minute) ctx, cancel := context.WithTimeout(ctx, time.Minute)
defer cancel() defer cancel()
f.logger.Info("Fetching WHOIS data", "asn", asn) f.logger.Info("Fetching WHOIS data", "asn", asn)
+17 -9
View File
@@ -55,14 +55,19 @@ func logDebugStats(logger *logger.Logger) {
// CLIEntry is the main entry point for the routewatch command-line interface. // CLIEntry is the main entry point for the routewatch command-line interface.
// It initializes the application using the fx dependency injection framework, // It initializes the application using the fx dependency injection framework,
// sets up signal handling for graceful shutdown, and starts the RouteWatch service. // sets up signal handling for graceful shutdown, and starts the RouteWatch
// This function blocks until the application receives a shutdown signal or encounters // service. This function blocks until the application receives a shutdown
// a fatal error. // signal or encounters a fatal error.
func CLIEntry() { func CLIEntry() {
app := fx.New( app := fx.New(
getModule(), fx.Options(getModule()...),
fx.StopTimeout(shutdownTimeout), // Allow 60 seconds for graceful shutdown fx.StopTimeout(shutdownTimeout), // Allow 60 seconds for graceful shutdown
fx.Invoke(func(lc fx.Lifecycle, rw *RouteWatch, logger *logger.Logger, shutdowner fx.Shutdowner) { fx.Invoke(func(
lc fx.Lifecycle,
rw *RouteWatch,
logger *logger.Logger,
shutdowner fx.Shutdowner,
) {
lc.Append(fx.Hook{ lc.Append(fx.Hook{
OnStart: func(ctx context.Context) error { OnStart: func(ctx context.Context) error {
// Start debug stats logging // Start debug stats logging
@@ -75,22 +80,25 @@ func CLIEntry() {
go func() { go func() {
<-sigCh <-sigCh
logger.Info("Received shutdown signal") logger.Info("Received shutdown signal")
if err := shutdowner.Shutdown(); err != nil {
err := shutdowner.Shutdown()
if err != nil {
logger.Error("Failed to shutdown gracefully", "error", err) logger.Error("Failed to shutdown gracefully", "error", err)
} }
}() }()
go func() { go func() {
if err := rw.Run(ctx); err != nil { err := rw.Run(ctx)
if err != nil {
logger.Error("RouteWatch error", "error", err) logger.Error("RouteWatch error", "error", err)
} }
}() }()
return nil return nil
}, },
OnStop: func(_ context.Context) error { OnStop: func(ctx context.Context) error {
logger.Info("Shutting down RouteWatch") logger.Info("Shutting down RouteWatch")
rw.Shutdown() rw.Shutdown(ctx)
return nil return nil
}, },
+61 -59
View File
@@ -1,4 +1,3 @@
// Package routewatch contains the database maintainer for background maintenance tasks.
package routewatch package routewatch
import ( import (
@@ -64,10 +63,13 @@ func NewDBMaintainer(db database.Store, logger *slog.Logger) *DBMaintainer {
} }
} }
// Start begins the background maintenance goroutine. // Start begins the background maintenance goroutine. It runs until Stop:
func (m *DBMaintainer) Start() { // the end of ctx does not stop it.
func (m *DBMaintainer) Start(ctx context.Context) {
m.wg.Add(1) m.wg.Add(1)
go m.run()
go m.run(context.WithoutCancel(ctx))
m.logger.Info("Database maintainer started", m.logger.Info("Database maintainer started",
"checkpoint_interval", checkpointInterval, "checkpoint_interval", checkpointInterval,
"vacuum_interval", vacuumInterval, "vacuum_interval", vacuumInterval,
@@ -83,15 +85,17 @@ func (m *DBMaintainer) Stop() {
} }
// run is the main background loop. // run is the main background loop.
func (m *DBMaintainer) run() { func (m *DBMaintainer) run(ctx context.Context) {
defer m.wg.Done() defer m.wg.Done()
// Use different timers for each task // Use different timers for each task
checkpointTimer := time.NewTimer(checkpointInterval) checkpointTimer := time.NewTimer(checkpointInterval)
vacuumTimer := time.NewTimer(vacuumInterval)
analyzeTimer := time.NewTimer(analyzeInterval)
defer checkpointTimer.Stop() defer checkpointTimer.Stop()
vacuumTimer := time.NewTimer(vacuumInterval)
defer vacuumTimer.Stop() defer vacuumTimer.Stop()
analyzeTimer := time.NewTimer(analyzeInterval)
defer analyzeTimer.Stop() defer analyzeTimer.Stop()
for { for {
@@ -100,90 +104,88 @@ func (m *DBMaintainer) run() {
return return
case <-checkpointTimer.C: case <-checkpointTimer.C:
m.runCheckpoint() m.runCheckpoint(ctx)
checkpointTimer.Reset(checkpointInterval) checkpointTimer.Reset(checkpointInterval)
case <-vacuumTimer.C: case <-vacuumTimer.C:
m.runVacuum() m.runVacuum(ctx)
vacuumTimer.Reset(vacuumInterval) vacuumTimer.Reset(vacuumInterval)
case <-analyzeTimer.C: case <-analyzeTimer.C:
m.runAnalyze() m.runAnalyze(ctx)
analyzeTimer.Reset(analyzeInterval) analyzeTimer.Reset(analyzeInterval)
} }
} }
} }
// runCheckpoint performs a WAL checkpoint to keep the WAL file small. // runCheckpoint performs a WAL checkpoint to keep the WAL file small.
func (m *DBMaintainer) runCheckpoint() { func (m *DBMaintainer) runCheckpoint(ctx context.Context) {
ctx, cancel := context.WithTimeout(context.Background(), checkpointTimeout) duration, err := runWithTimeout(ctx, checkpointTimeout, m.db.Checkpoint)
defer cancel() m.recordResult(&m.lastCheckpoint, &m.lastCheckpointError, &m.checkpointCount, err)
startTime := time.Now()
err := m.db.Checkpoint(ctx)
m.statsMu.Lock()
m.lastCheckpoint = time.Now()
m.lastCheckpointError = err
if err == nil {
m.checkpointCount++
}
m.statsMu.Unlock()
if err != nil { if err != nil {
m.logger.Error("WAL checkpoint failed", "error", err, "duration", time.Since(startTime)) m.logger.Error("WAL checkpoint failed", "error", err, "duration", duration)
} else { } else {
m.logger.Debug("WAL checkpoint completed", "duration", time.Since(startTime)) m.logger.Debug("WAL checkpoint completed", "duration", duration)
} }
} }
// runVacuum performs an incremental vacuum operation on the database. // runVacuum performs an incremental vacuum operation on the database.
func (m *DBMaintainer) runVacuum() { func (m *DBMaintainer) runVacuum(ctx context.Context) {
ctx, cancel := context.WithTimeout(context.Background(), vacuumTimeout)
defer cancel()
m.logger.Debug("Running incremental vacuum") m.logger.Debug("Running incremental vacuum")
startTime := time.Now()
err := m.db.Vacuum(ctx) duration, err := runWithTimeout(ctx, vacuumTimeout, m.db.Vacuum)
m.recordResult(&m.lastVacuum, &m.lastVacuumError, &m.vacuumCount, err)
m.statsMu.Lock()
m.lastVacuum = time.Now()
m.lastVacuumError = err
if err == nil {
m.vacuumCount++
}
m.statsMu.Unlock()
if err != nil { if err != nil {
m.logger.Error("Incremental vacuum failed", "error", err, "duration", time.Since(startTime)) m.logger.Error("Incremental vacuum failed", "error", err, "duration", duration)
} else { } else {
m.logger.Debug("Incremental vacuum completed", "duration", time.Since(startTime)) m.logger.Debug("Incremental vacuum completed", "duration", duration)
} }
} }
// runAnalyze performs an ANALYZE operation on the database. // runAnalyze performs an ANALYZE operation on the database.
func (m *DBMaintainer) runAnalyze() { func (m *DBMaintainer) runAnalyze(ctx context.Context) {
ctx, cancel := context.WithTimeout(context.Background(), analyzeTimeout)
defer cancel()
m.logger.Info("Starting database ANALYZE") m.logger.Info("Starting database ANALYZE")
startTime := time.Now()
err := m.db.Analyze(ctx) duration, err := runWithTimeout(ctx, analyzeTimeout, m.db.Analyze)
m.recordResult(&m.lastAnalyze, &m.lastAnalyzeError, &m.analyzeCount, err)
m.statsMu.Lock()
m.lastAnalyze = time.Now()
m.lastAnalyzeError = err
if err == nil {
m.analyzeCount++
}
m.statsMu.Unlock()
if err != nil { if err != nil {
m.logger.Error("ANALYZE failed", "error", err, "duration", time.Since(startTime)) m.logger.Error("ANALYZE failed", "error", err, "duration", duration)
} else { } else {
m.logger.Info("ANALYZE completed", "duration", time.Since(startTime)) m.logger.Info("ANALYZE completed", "duration", duration)
}
}
// runWithTimeout runs operation with a context that ends after timeout and
// returns how long it took and its error.
func runWithTimeout(
ctx context.Context,
timeout time.Duration,
operation func(context.Context) error,
) (time.Duration, error) {
ctx, cancel := context.WithTimeout(ctx, timeout)
defer cancel()
startTime := time.Now()
err := operation(ctx)
return time.Since(startTime), err
}
// recordResult stores when a maintenance task last ran and its error, and
// counts it when it succeeded.
func (m *DBMaintainer) recordResult(
lastRun *time.Time, lastErr *error, successCount *int, err error,
) {
m.statsMu.Lock()
defer m.statsMu.Unlock()
*lastRun = time.Now()
*lastErr = err
if err == nil {
*successCount++
} }
} }
+16 -14
View File
@@ -1,10 +1,16 @@
package routewatch package routewatch
import ( import (
"slices"
"git.eeqj.de/sneak/routewatch/internal/logger" "git.eeqj.de/sneak/routewatch/internal/logger"
"git.eeqj.de/sneak/routewatch/internal/ristypes" "git.eeqj.de/sneak/routewatch/internal/ristypes"
) )
// updateMessageType is the RIS message type of BGP UPDATE messages, the only
// type the database handlers process.
const updateMessageType = "UPDATE"
// SimpleHandler is a basic implementation of streamer.MessageHandler that // SimpleHandler is a basic implementation of streamer.MessageHandler that
// filters messages by type and delegates processing to a callback function. // filters messages by type and delegates processing to a callback function.
// It provides a simple way to handle specific RIS message types without // It provides a simple way to handle specific RIS message types without
@@ -15,10 +21,11 @@ type SimpleHandler struct {
callback func(*ristypes.RISMessage) callback func(*ristypes.RISMessage)
} }
// NewSimpleHandler creates a new SimpleHandler that accepts specific message types. // NewSimpleHandler creates a new SimpleHandler that accepts specific message
// The messageTypes parameter specifies which RIS message types this handler will process. // types. The messageTypes parameter specifies which RIS message types this
// If messageTypes is empty, the handler will accept all message types. // handler will process. If messageTypes is empty, the handler will accept all
// The callback function is invoked for each message that passes the type filter. // message types. The callback function is invoked for each message that
// passes the type filter.
func NewSimpleHandler( func NewSimpleHandler(
logger *logger.Logger, logger *logger.Logger,
messageTypes []string, messageTypes []string,
@@ -31,22 +38,17 @@ func NewSimpleHandler(
} }
} }
// WantsMessage returns true if this handler wants to process messages of the given type. // WantsMessage returns true if this handler wants to process messages of the
// It checks whether messageType is in the handler's configured list of accepted types. // given type. It checks whether messageType is in the handler's configured
// If no specific types were configured (empty messageTypes slice), it returns true for all types. // list of accepted types. If no specific types were configured (empty
// messageTypes slice), it returns true for all types.
func (h *SimpleHandler) WantsMessage(messageType string) bool { func (h *SimpleHandler) WantsMessage(messageType string) bool {
// If no specific types are set, accept all messages // If no specific types are set, accept all messages
if len(h.messageTypes) == 0 { if len(h.messageTypes) == 0 {
return true return true
} }
for _, t := range h.messageTypes { return slices.Contains(h.messageTypes, messageType)
if t == messageType {
return true
}
}
return false
} }
// HandleMessage processes a RIS message by invoking the configured callback function. // HandleMessage processes a RIS message by invoking the configured callback function.
+21 -13
View File
@@ -1,7 +1,8 @@
package routewatch package routewatch
// peerhandler.go provides batched peer tracking functionality for BGP route monitoring. // peerhandler.go provides batched peer tracking functionality for BGP route
// It tracks BGP peers from all incoming RIS messages and maintains peer state in the database. // monitoring. It tracks BGP peers from all incoming RIS messages and
// maintains peer state in the database.
import ( import (
"strconv" "strconv"
@@ -92,8 +93,10 @@ func (h *PeerHandler) QueueCapacity() int {
func (h *PeerHandler) HandleMessage(msg *ristypes.RISMessage) { func (h *PeerHandler) HandleMessage(msg *ristypes.RISMessage) {
// Parse peer ASN from string // Parse peer ASN from string
peerASN := 0 peerASN := 0
if msg.PeerASN != "" { if msg.PeerASN != "" {
if asn, err := strconv.Atoi(msg.PeerASN); err == nil { asn, err := strconv.Atoi(msg.PeerASN)
if err == nil {
peerASN = asn peerASN = asn
} }
} }
@@ -115,9 +118,16 @@ func (h *PeerHandler) HandleMessage(msg *ristypes.RISMessage) {
} }
} }
// Stop gracefully stops the handler and flushes remaining batches
func (h *PeerHandler) Stop() {
close(h.stopCh)
h.wg.Wait()
}
// flushLoop runs in a goroutine and periodically flushes batches // flushLoop runs in a goroutine and periodically flushes batches
func (h *PeerHandler) flushLoop() { func (h *PeerHandler) flushLoop() {
defer h.wg.Done() defer h.wg.Done()
ticker := time.NewTicker(peerBatchTimeout) ticker := time.NewTicker(peerBatchTimeout)
defer ticker.Stop() defer ticker.Stop()
@@ -140,7 +150,8 @@ func (h *PeerHandler) flushLoop() {
} }
} }
// flushBatchLocked flushes the peer batch to the database (must be called with mutex held) // flushBatchLocked flushes the peer batch to the database (must be called
// with mutex held)
func (h *PeerHandler) flushBatchLocked() { func (h *PeerHandler) flushBatchLocked() {
if len(h.peerBatch) == 0 { if len(h.peerBatch) == 0 {
return return
@@ -149,7 +160,8 @@ func (h *PeerHandler) flushBatchLocked() {
// Deduplicate by peer IP, keeping the latest update for each peer // Deduplicate by peer IP, keeping the latest update for each peer
peerMap := make(map[string]peerUpdate) peerMap := make(map[string]peerUpdate)
for _, update := range h.peerBatch { for _, update := range h.peerBatch {
if existing, ok := peerMap[update.peerIP]; !ok || update.timestamp.After(existing.timestamp) { existing, ok := peerMap[update.peerIP]
if !ok || update.timestamp.After(existing.timestamp) {
peerMap[update.peerIP] = update peerMap[update.peerIP] = update
} }
} }
@@ -166,17 +178,13 @@ func (h *PeerHandler) flushBatchLocked() {
} }
// Process all peers in a single batch transaction // Process all peers in a single batch transaction
if err := h.db.UpdatePeerBatch(dbPeerMap); err != nil { err := h.db.UpdatePeerBatch(dbPeerMap)
h.logger.Error("Failed to process peer batch", "error", err, "count", len(dbPeerMap)) if err != nil {
h.logger.Error("Failed to process peer batch",
"error", err, "count", len(dbPeerMap))
} }
// Clear batch // Clear batch
h.peerBatch = h.peerBatch[:0] h.peerBatch = h.peerBatch[:0]
h.lastFlush = time.Now() h.lastFlush = time.Now()
} }
// Stop gracefully stops the handler and flushes remaining batches
func (h *PeerHandler) Stop() {
close(h.stopCh)
h.wg.Wait()
}
+24 -19
View File
@@ -72,7 +72,7 @@ func NewPeeringHandler(db database.Store, logger *logger.Logger) *PeeringHandler
// the AS path information needed to extract peering relationships. // the AS path information needed to extract peering relationships.
func (h *PeeringHandler) WantsMessage(messageType string) bool { func (h *PeeringHandler) WantsMessage(messageType string) bool {
// We only care about UPDATE messages that have AS paths // We only care about UPDATE messages that have AS paths
return messageType == "UPDATE" return messageType == updateMessageType
} }
// QueueCapacity returns the buffer size for the handler's message queue. // QueueCapacity returns the buffer size for the handler's message queue.
@@ -101,6 +101,7 @@ func (h *PeeringHandler) HandleMessage(msg *ristypes.RISMessage) {
return return
} }
key := string(pathJSON) key := string(pathJSON)
h.mu.Lock() h.mu.Lock()
@@ -116,6 +117,23 @@ func (h *PeeringHandler) HandleMessage(msg *ristypes.RISMessage) {
h.mu.Unlock() h.mu.Unlock()
} }
// ProcessPeeringsNow triggers immediate processing of all accumulated AS
// paths into peering records. This bypasses the normal periodic processing
// schedule and is primarily intended for testing purposes.
func (h *PeeringHandler) ProcessPeeringsNow() {
h.processPeerings()
}
// Stop gracefully shuts down the handler by signaling the background
// goroutine to stop and performing a final synchronous processing of
// any remaining AS paths. This ensures no peering data is lost during
// shutdown.
func (h *PeeringHandler) Stop() {
close(h.stopCh)
// Process any remaining peerings synchronously
h.processPeerings()
}
// processLoop runs periodically to process AS paths into peerings // processLoop runs periodically to process AS paths into peerings
func (h *PeeringHandler) processLoop() { func (h *PeeringHandler) processLoop() {
ticker := time.NewTicker(peeringProcessInterval) ticker := time.NewTicker(peeringProcessInterval)
@@ -134,13 +152,6 @@ func (h *PeeringHandler) processLoop() {
} }
} }
// ProcessPeeringsNow triggers immediate processing of all accumulated AS
// paths into peering records. This bypasses the normal periodic processing
// schedule and is primarily intended for testing purposes.
func (h *PeeringHandler) ProcessPeeringsNow() {
h.processPeerings()
}
// processPeerings extracts peerings from AS paths and writes to database // processPeerings extracts peerings from AS paths and writes to database
func (h *PeeringHandler) processPeerings() { func (h *PeeringHandler) processPeerings() {
// Take the accumulated paths and replace the map with a fresh empty one // Take the accumulated paths and replace the map with a fresh empty one
@@ -160,11 +171,14 @@ func (h *PeeringHandler) processPeerings() {
type peeringKey struct { type peeringKey struct {
low, high int low, high int
} }
peerings := make(map[peeringKey]time.Time) peerings := make(map[peeringKey]time.Time)
for pathJSON, timestamp := range paths { for pathJSON, timestamp := range paths {
var path []int var path []int
if err := json.Unmarshal([]byte(pathJSON), &path); err != nil {
err := json.Unmarshal([]byte(pathJSON), &path)
if err != nil {
h.logger.Error("Failed to decode AS path", "error", err) h.logger.Error("Failed to decode AS path", "error", err)
continue continue
@@ -197,6 +211,7 @@ func (h *PeeringHandler) processPeerings() {
// Record peerings in database // Record peerings in database
start := time.Now() start := time.Now()
successCount := 0 successCount := 0
for key, ts := range peerings { for key, ts := range peerings {
err := h.db.RecordPeering(key.low, key.high, ts) err := h.db.RecordPeering(key.low, key.high, ts)
if err != nil { if err != nil {
@@ -218,13 +233,3 @@ func (h *PeeringHandler) processPeerings() {
"duration", time.Since(start), "duration", time.Since(start),
) )
} }
// Stop gracefully shuts down the handler by signaling the background
// goroutine to stop and performing a final synchronous processing of
// any remaining AS paths. This ensures no peering data is lost during
// shutdown.
func (h *PeeringHandler) Stop() {
close(h.stopCh)
// Process any remaining peerings synchronously
h.processPeerings()
}
@@ -53,7 +53,8 @@ func newTestHandler(db database.Store) *PeeringHandler {
} }
} }
func snapshot(h *PeeringHandler) (tracked, dropped int) { // snapshot returns how many paths the handler tracks and how many it dropped.
func snapshot(h *PeeringHandler) (int, int) {
h.mu.Lock() h.mu.Lock()
defer h.mu.Unlock() defer h.mu.Unlock()
@@ -82,6 +83,8 @@ func handle(h *PeeringHandler, ts time.Time, asns ...int) {
// empties the path map (the swap) and that a path seen again after a run is // empties the path map (the swap) and that a path seen again after a run is
// recorded in the next run too. // recorded in the next run too.
func TestPeeringHandlerProcessesEachRunAndEmpties(t *testing.T) { func TestPeeringHandlerProcessesEachRunAndEmpties(t *testing.T) {
t.Parallel()
store := &recordingStore{mockStore: newMockStore()} store := &recordingStore{mockStore: newMockStore()}
h := newTestHandler(store) h := newTestHandler(store)
@@ -94,6 +97,7 @@ func TestPeeringHandlerProcessesEachRunAndEmpties(t *testing.T) {
if tracked, _ := snapshot(h); tracked != 0 { if tracked, _ := snapshot(h); tracked != 0 {
t.Fatalf("map not empty after first run: %d paths remain", tracked) t.Fatalf("map not empty after first run: %d paths remain", tracked)
} }
if got := store.callCount(); got != 1 { if got := store.callCount(); got != 1 {
t.Fatalf("want 1 RecordPeering call after first run, got %d", got) t.Fatalf("want 1 RecordPeering call after first run, got %d", got)
} }
@@ -105,6 +109,7 @@ func TestPeeringHandlerProcessesEachRunAndEmpties(t *testing.T) {
if tracked, _ := snapshot(h); tracked != 0 { if tracked, _ := snapshot(h); tracked != 0 {
t.Fatalf("map not empty after second run: %d paths remain", tracked) t.Fatalf("map not empty after second run: %d paths remain", tracked)
} }
if got := store.callCount(); got != 2 { if got := store.callCount(); got != 2 {
t.Fatalf("want 2 RecordPeering calls after second run, got %d", got) t.Fatalf("want 2 RecordPeering calls after second run, got %d", got)
} }
@@ -114,6 +119,8 @@ func TestPeeringHandlerProcessesEachRunAndEmpties(t *testing.T) {
// and counts them, while a path already tracked is refreshed rather than // and counts them, while a path already tracked is refreshed rather than
// dropped. // dropped.
func TestPeeringHandlerCapDropsAndCounts(t *testing.T) { func TestPeeringHandlerCapDropsAndCounts(t *testing.T) {
t.Parallel()
store := &recordingStore{mockStore: newMockStore()} store := &recordingStore{mockStore: newMockStore()}
h := newTestHandler(store) h := newTestHandler(store)
@@ -126,6 +133,7 @@ func TestPeeringHandlerCapDropsAndCounts(t *testing.T) {
h.mu.Lock() h.mu.Lock()
h.asPaths[existing] = now h.asPaths[existing] = now
for i := 0; len(h.asPaths) < maxTrackedPaths; i++ { for i := 0; len(h.asPaths) < maxTrackedPaths; i++ {
h.asPaths[strconv.Itoa(i)] = now h.asPaths[strconv.Itoa(i)] = now
} }
@@ -138,6 +146,7 @@ func TestPeeringHandlerCapDropsAndCounts(t *testing.T) {
if tracked != maxTrackedPaths { if tracked != maxTrackedPaths {
t.Fatalf("want map size %d after drop, got %d", maxTrackedPaths, tracked) t.Fatalf("want map size %d after drop, got %d", maxTrackedPaths, tracked)
} }
if dropped != 1 { if dropped != 1 {
t.Fatalf("want dropped count 1, got %d", dropped) t.Fatalf("want dropped count 1, got %d", dropped)
} }
@@ -150,6 +159,7 @@ func TestPeeringHandlerCapDropsAndCounts(t *testing.T) {
if tracked != maxTrackedPaths { if tracked != maxTrackedPaths {
t.Fatalf("want map size %d after refresh, got %d", maxTrackedPaths, tracked) t.Fatalf("want map size %d after refresh, got %d", maxTrackedPaths, tracked)
} }
if dropped != 1 { if dropped != 1 {
t.Fatalf("want dropped count still 1 after refresh, got %d", dropped) t.Fatalf("want dropped count still 1 after refresh, got %d", dropped)
} }
@@ -157,7 +167,9 @@ func TestPeeringHandlerCapDropsAndCounts(t *testing.T) {
h.mu.Lock() h.mu.Lock()
gotTS := h.asPaths[existing] gotTS := h.asPaths[existing]
h.mu.Unlock() h.mu.Unlock()
if !gotTS.Equal(refreshed) { if !gotTS.Equal(refreshed) {
t.Fatalf("existing path timestamp not refreshed: want %v, got %v", refreshed, gotTS) t.Fatalf("existing path timestamp not refreshed: want %v, got %v",
refreshed, gotTS)
} }
} }
+70 -181
View File
@@ -33,8 +33,8 @@ const (
ipv6Version = 6 ipv6Version = 6
) )
// PrefixHandler tracks BGP prefixes and maintains a live routing table in the database. // PrefixHandler tracks BGP prefixes and maintains a live routing table in the
// Routes are added on announcement and deleted on withdrawal. // database. Routes are added on announcement and deleted on withdrawal.
type PrefixHandler struct { type PrefixHandler struct {
db database.Store db database.Store
logger *logger.Logger logger *logger.Logger
@@ -79,10 +79,11 @@ func (h *PrefixHandler) SetMetricsTracker(metrics *metrics.Tracker) {
h.metrics = metrics h.metrics = metrics
} }
// WantsMessage returns true if this handler wants to process messages of the given type // WantsMessage returns true if this handler wants to process messages of the
// given type
func (h *PrefixHandler) WantsMessage(messageType string) bool { func (h *PrefixHandler) WantsMessage(messageType string) bool {
// We only care about UPDATE messages for the routing table // We only care about UPDATE messages for the routing table
return messageType == "UPDATE" return messageType == updateMessageType
} }
// QueueCapacity returns the desired queue capacity for this handler // QueueCapacity returns the desired queue capacity for this handler
@@ -145,9 +146,16 @@ func (h *PrefixHandler) HandleMessage(msg *ristypes.RISMessage) {
} }
} }
// Stop gracefully stops the handler and flushes remaining batches
func (h *PrefixHandler) Stop() {
close(h.stopCh)
h.wg.Wait()
}
// flushLoop runs in a goroutine and periodically flushes batches // flushLoop runs in a goroutine and periodically flushes batches
func (h *PrefixHandler) flushLoop() { func (h *PrefixHandler) flushLoop() {
defer h.wg.Done() defer h.wg.Done()
ticker := time.NewTicker(prefixBatchTimeout) ticker := time.NewTicker(prefixBatchTimeout)
defer ticker.Stop() defer ticker.Stop()
@@ -170,7 +178,8 @@ func (h *PrefixHandler) flushLoop() {
} }
} }
// flushBatchLocked flushes the prefix batch to the database (must be called with mutex held) // flushBatchLocked flushes the prefix batch to the database (must be called
// with mutex held)
func (h *PrefixHandler) flushBatchLocked() { func (h *PrefixHandler) flushBatchLocked() {
if len(h.batch) == 0 { if len(h.batch) == 0 {
return return
@@ -182,23 +191,29 @@ func (h *PrefixHandler) flushBatchLocked() {
// Group updates by prefix to deduplicate // Group updates by prefix to deduplicate
// For each prefix, keep the latest update // For each prefix, keep the latest update
prefixMap := make(map[string]prefixUpdate) prefixMap := make(map[string]prefixUpdate)
for _, update := range h.batch { for _, update := range h.batch {
key := update.prefix key := update.prefix
if existing, ok := prefixMap[key]; !ok || update.timestamp.After(existing.timestamp) {
existing, ok := prefixMap[key]
if !ok || update.timestamp.After(existing.timestamp) {
prefixMap[key] = update prefixMap[key] = update
} }
} }
// Collect routes to upsert and delete // Collect routes to upsert and delete
var routesToUpsert []*database.LiveRoute var (
var routesToDelete []database.LiveRouteDeletion routesToUpsert []*database.LiveRoute
routesToDelete []database.LiveRouteDeletion
)
// Collect unique prefixes to update // Collect unique prefixes to update
prefixesToUpdate := make(map[string]time.Time) prefixesToUpdate := make(map[string]time.Time)
for _, update := range prefixMap { for _, update := range prefixMap {
// Track prefix for both announcements and withdrawals // Track prefix for both announcements and withdrawals
if _, exists := prefixesToUpdate[update.prefix]; !exists || update.timestamp.After(prefixesToUpdate[update.prefix]) { seen, exists := prefixesToUpdate[update.prefix]
if !exists || update.timestamp.After(seen) {
prefixesToUpdate[update.prefix] = update.timestamp prefixesToUpdate[update.prefix] = update.timestamp
} }
@@ -212,7 +227,8 @@ func (h *PrefixHandler) flushBatchLocked() {
// Parse CIDR to get IP version // Parse CIDR to get IP version
_, ipVersion, err := parseCIDR(update.prefix) _, ipVersion, err := parseCIDR(update.prefix)
if err != nil { if err != nil {
h.logger.Error("Failed to parse CIDR for withdrawal", "prefix", update.prefix, "error", err) h.logger.Error("Failed to parse CIDR for withdrawal",
"prefix", update.prefix, "error", err)
continue continue
} }
@@ -227,30 +243,7 @@ func (h *PrefixHandler) flushBatchLocked() {
} }
} }
// Process batch operations successCount := h.writeBatch(routesToUpsert, routesToDelete, prefixesToUpdate)
successCount := 0
if len(routesToUpsert) > 0 {
if err := h.db.UpsertLiveRouteBatch(routesToUpsert); err != nil {
h.logger.Error("Failed to upsert route batch", "error", err, "count", len(routesToUpsert))
} else {
successCount += len(routesToUpsert)
}
}
if len(routesToDelete) > 0 {
if err := h.db.DeleteLiveRouteBatch(routesToDelete); err != nil {
h.logger.Error("Failed to delete route batch", "error", err, "count", len(routesToDelete))
} else {
successCount += len(routesToDelete)
}
}
// Update prefix tables
if len(prefixesToUpdate) > 0 {
if err := h.db.UpdatePrefixesBatch(prefixesToUpdate); err != nil {
h.logger.Error("Failed to update prefix batch", "error", err, "count", len(prefixesToUpdate))
}
}
elapsed := time.Since(startTime) elapsed := time.Since(startTime)
h.logger.Debug("Flushed prefix batch", h.logger.Debug("Flushed prefix batch",
@@ -265,6 +258,47 @@ func (h *PrefixHandler) flushBatchLocked() {
h.lastFlush = time.Now() h.lastFlush = time.Now()
} }
// writeBatch writes the collected route upserts, route deletions and prefix
// updates to the database and returns how many routes it wrote.
func (h *PrefixHandler) writeBatch(
routesToUpsert []*database.LiveRoute,
routesToDelete []database.LiveRouteDeletion,
prefixesToUpdate map[string]time.Time,
) int {
successCount := 0
if len(routesToUpsert) > 0 {
err := h.db.UpsertLiveRouteBatch(routesToUpsert)
if err != nil {
h.logger.Error("Failed to upsert route batch",
"error", err, "count", len(routesToUpsert))
} else {
successCount += len(routesToUpsert)
}
}
if len(routesToDelete) > 0 {
err := h.db.DeleteLiveRouteBatch(routesToDelete)
if err != nil {
h.logger.Error("Failed to delete route batch",
"error", err, "count", len(routesToDelete))
} else {
successCount += len(routesToDelete)
}
}
// Update prefix tables
if len(prefixesToUpdate) > 0 {
err := h.db.UpdatePrefixesBatch(prefixesToUpdate)
if err != nil {
h.logger.Error("Failed to update prefix batch",
"error", err, "count", len(prefixesToUpdate))
}
}
return successCount
}
// canonicalPrefix returns prefix in the text form net/netip prints, the form // canonicalPrefix returns prefix in the text form net/netip prints, the form
// the IP lookup builds when it looks a prefix up. The feed sends IPv6 // the IP lookup builds when it looks a prefix up. The feed sends IPv6
// announcements compressed ("2001:db8::/32") but IPv6 withdrawals uncompressed // announcements compressed ("2001:db8::/32") but IPv6 withdrawals uncompressed
@@ -280,8 +314,8 @@ func canonicalPrefix(prefix string) string {
return p.Masked().String() return p.Masked().String()
} }
// parseCIDR extracts the mask length and IP version from a prefix string // parseCIDR returns the mask length and the IP version of a prefix string
func parseCIDR(prefix string) (maskLength int, ipVersion int, err error) { func parseCIDR(prefix string) (int, int, error) {
_, ipNet, err := net.ParseCIDR(prefix) _, ipNet, err := net.ParseCIDR(prefix)
if err != nil { if err != nil {
return 0, 0, err return 0, 0, err
@@ -295,50 +329,6 @@ func parseCIDR(prefix string) (maskLength int, ipVersion int, err error) {
return ones, ipv4Version, nil return ones, ipv4Version, nil
} }
// processAnnouncement handles storing an announcement in the database
// nolint:unused // kept for potential future use
func (h *PrefixHandler) processAnnouncement(_ *database.Prefix, update prefixUpdate) {
// Parse CIDR to get mask length
maskLength, ipVersion, err := parseCIDR(update.prefix)
if err != nil {
h.logger.Error("Failed to parse CIDR",
"prefix", update.prefix,
"error", err,
)
return
}
// Track route update metrics
if h.metrics != nil {
if ipVersion == ipv4Version {
h.metrics.RecordIPv4Update()
} else {
h.metrics.RecordIPv6Update()
}
}
// Create live route record
liveRoute := &database.LiveRoute{
ID: uuid.New(),
Prefix: update.prefix,
MaskLength: maskLength,
IPVersion: ipVersion,
OriginASN: update.originASN,
PeerIP: update.peer,
ASPath: update.path,
NextHop: update.peer, // Using peer as next hop
LastUpdated: update.timestamp,
}
if err := h.db.UpsertLiveRoute(liveRoute); err != nil {
h.logger.Error("Failed to upsert live route",
"prefix", update.prefix,
"error", err,
)
}
}
// createLiveRoute creates a LiveRoute from a prefix update // createLiveRoute creates a LiveRoute from a prefix update
func (h *PrefixHandler) createLiveRoute(update prefixUpdate) *database.LiveRoute { func (h *PrefixHandler) createLiveRoute(update prefixUpdate) *database.LiveRoute {
// Parse CIDR to get mask length // Parse CIDR to get mask length
@@ -376,104 +366,3 @@ func (h *PrefixHandler) createLiveRoute(update prefixUpdate) *database.LiveRoute
return liveRoute return liveRoute
} }
// processAnnouncementDirect handles storing an announcement directly without prefix table
// nolint:unused // kept for potential future use
func (h *PrefixHandler) processAnnouncementDirect(update prefixUpdate) {
// Parse CIDR to get mask length
maskLength, ipVersion, err := parseCIDR(update.prefix)
if err != nil {
h.logger.Error("Failed to parse CIDR",
"prefix", update.prefix,
"error", err,
)
return
}
// Track route update metrics
if h.metrics != nil {
if ipVersion == ipv4Version {
h.metrics.RecordIPv4Update()
} else {
h.metrics.RecordIPv6Update()
}
}
// Create live route record
liveRoute := &database.LiveRoute{
ID: uuid.New(),
Prefix: update.prefix,
MaskLength: maskLength,
IPVersion: ipVersion,
OriginASN: update.originASN,
PeerIP: update.peer,
ASPath: update.path,
NextHop: update.peer, // Using peer as next hop
LastUpdated: update.timestamp,
}
if err := h.db.UpsertLiveRoute(liveRoute); err != nil {
h.logger.Error("Failed to upsert live route",
"prefix", update.prefix,
"error", err,
)
}
}
// processWithdrawalDirect handles removing a route directly without prefix table
// nolint:unused // kept for potential future use
func (h *PrefixHandler) processWithdrawalDirect(update prefixUpdate) {
// For withdrawals, we need to delete the route from live_routes
if update.originASN > 0 {
if err := h.db.DeleteLiveRoute(update.prefix, update.originASN, update.peer); err != nil {
h.logger.Error("Failed to delete live route",
"prefix", update.prefix,
"origin_asn", update.originASN,
"peer", update.peer,
"error", err,
)
}
} else {
// If no origin ASN, just delete all routes for this prefix from this peer
if err := h.db.DeleteLiveRoute(update.prefix, 0, update.peer); err != nil {
h.logger.Error("Failed to delete live route (no origin ASN)",
"prefix", update.prefix,
"peer", update.peer,
"error", err,
)
}
}
}
// processWithdrawal handles removing a route from the live routing table
// nolint:unused // kept for potential future use
func (h *PrefixHandler) processWithdrawal(_ *database.Prefix, update prefixUpdate) {
// For withdrawals, we need to delete the route from live_routes
// Since we have the origin ASN from the update, we can delete the specific route
if update.originASN > 0 {
if err := h.db.DeleteLiveRoute(update.prefix, update.originASN, update.peer); err != nil {
h.logger.Error("Failed to delete live route",
"prefix", update.prefix,
"origin_asn", update.originASN,
"peer", update.peer,
"error", err,
)
}
} else {
// If no origin ASN, just delete all routes for this prefix from this peer
if err := h.db.DeleteLiveRoute(update.prefix, 0, update.peer); err != nil {
h.logger.Error("Failed to delete live route (no origin ASN)",
"prefix", update.prefix,
"peer", update.peer,
"error", err,
)
}
}
}
// Stop gracefully stops the handler and flushes remaining batches
func (h *PrefixHandler) Stop() {
close(h.stopCh)
h.wg.Wait()
}
@@ -11,7 +11,10 @@ import (
"git.eeqj.de/sneak/routewatch/internal/ristypes" "git.eeqj.de/sneak/routewatch/internal/ristypes"
) )
const testPeerIP = "2001:db8:ffff::1" const (
testPeerIP = "2001:db8:ffff::1"
testV4Prefix = "192.0.2.0/24"
)
// TestPrefixHandlerStoresPrefixesTheIPLookupFinds runs announcements and // TestPrefixHandlerStoresPrefixesTheIPLookupFinds runs announcements and
// withdrawals through the prefix handler into a real database and looks the // withdrawals through the prefix handler into a real database and looks the
@@ -20,6 +23,8 @@ const testPeerIP = "2001:db8:ffff::1"
// remove the route the announcement stored, and the IP lookup must find the // remove the route the announcement stored, and the IP lookup must find the
// stored prefix. // stored prefix.
func TestPrefixHandlerStoresPrefixesTheIPLookupFinds(t *testing.T) { func TestPrefixHandlerStoresPrefixesTheIPLookupFinds(t *testing.T) {
t.Parallel()
db, err := database.New(&config.Config{StateDir: t.TempDir()}, logger.New()) db, err := database.New(&config.Config{StateDir: t.TempDir()}, logger.New())
if err != nil { if err != nil {
t.Fatalf("failed to create database: %v", err) t.Fatalf("failed to create database: %v", err)
@@ -31,6 +36,7 @@ func TestPrefixHandlerStoresPrefixesTheIPLookupFinds(t *testing.T) {
flush := func() { flush := func() {
h.mu.Lock() h.mu.Lock()
defer h.mu.Unlock() defer h.mu.Unlock()
h.flushBatchLocked() h.flushBatchLocked()
} }
@@ -41,34 +47,38 @@ func TestPrefixHandlerStoresPrefixesTheIPLookupFinds(t *testing.T) {
ParsedTimestamp: ts, ParsedTimestamp: ts,
Announcements: []ristypes.RISAnnouncement{{ Announcements: []ristypes.RISAnnouncement{{
NextHop: testPeerIP, NextHop: testPeerIP,
Prefixes: []string{"2001:db8:1::/48", "192.0.2.0/24"}, Prefixes: []string{"2001:db8:1::/48", testV4Prefix},
}}, }},
}) })
flush() flush()
for ip, want := range map[string]string{ for ip, want := range map[string]string{
"2001:db8:1::1": "2001:db8:1::/48", "2001:db8:1::1": "2001:db8:1::/48",
"192.0.2.1": "192.0.2.0/24", "192.0.2.1": testV4Prefix,
} { } {
info, err := db.GetIPInfo(ip) info, err := db.GetIPInfo(ip)
if err != nil { if err != nil {
t.Fatalf("GetIPInfo(%s) after announcement: %v", ip, err) t.Fatalf("GetIPInfo(%s) after announcement: %v", ip, err)
} }
if info.Netblock != want || info.ASN != testASNB { if info.Netblock != want || info.ASN != testASNB {
t.Errorf("GetIPInfo(%s) = %s AS%d, want %s AS%d", ip, info.Netblock, info.ASN, want, testASNB) t.Errorf("GetIPInfo(%s) = %s AS%d, want %s AS%d",
ip, info.Netblock, info.ASN, want, testASNB)
} }
} }
h.HandleMessage(&ristypes.RISMessage{ h.HandleMessage(&ristypes.RISMessage{
Peer: testPeerIP, Peer: testPeerIP,
ParsedTimestamp: ts.Add(time.Minute), ParsedTimestamp: ts.Add(time.Minute),
Withdrawals: []string{"2001:db8:1:0:0:0:0:0/48", "192.0.2.0/24"}, Withdrawals: []string{"2001:db8:1:0:0:0:0:0/48", testV4Prefix},
}) })
flush() flush()
for _, ip := range []string{"2001:db8:1::1", "192.0.2.1"} { for _, ip := range []string{"2001:db8:1::1", "192.0.2.1"} {
if info, err := db.GetIPInfo(ip); !errors.Is(err, database.ErrNoRoute) { info, err := db.GetIPInfo(ip)
t.Errorf("GetIPInfo(%s) after withdrawal = %+v, %v; want ErrNoRoute", ip, info, err) if !errors.Is(err, database.ErrNoRoute) {
t.Errorf("GetIPInfo(%s) after withdrawal = %+v, %v; want ErrNoRoute",
ip, info, err)
} }
} }
} }
+269 -201
View File
@@ -1,5 +1,6 @@
// Package streamer implements an HTTP client that connects to the RIPE RIS Live streaming API, // Package streamer implements an HTTP client that connects to the RIPE RIS Live
// parses BGP UPDATE messages from the JSON stream, and dispatches them to registered handlers. // streaming API, parses BGP UPDATE messages from the JSON stream, and
// dispatches them to registered handlers.
package streamer package streamer
import ( import (
@@ -7,6 +8,7 @@ import (
"compress/gzip" "compress/gzip"
"context" "context"
"encoding/json" "encoding/json"
"errors"
"fmt" "fmt"
"io" "io"
"math" "math"
@@ -24,20 +26,20 @@ import (
// countingReader wraps an io.Reader and counts bytes read // countingReader wraps an io.Reader and counts bytes read
type countingReader struct { type countingReader struct {
reader io.Reader reader io.Reader
count int64 count atomic.Int64
} }
// Read implements io.Reader and counts bytes // Read implements io.Reader and counts bytes
func (c *countingReader) Read(p []byte) (int, error) { func (c *countingReader) Read(p []byte) (int, error) {
n, err := c.reader.Read(p) n, err := c.reader.Read(p)
atomic.AddInt64(&c.count, int64(n)) c.count.Add(int64(n))
return n, err return n, err
} }
// Count returns the total bytes read // Count returns the total bytes read
func (c *countingReader) Count() int64 { func (c *countingReader) Count() int64 {
return atomic.LoadInt64(&c.count) return c.count.Load()
} }
// Configuration constants for the RIS Live streamer. // Configuration constants for the RIS Live streamer.
@@ -58,11 +60,17 @@ const (
backpressureSlope = 2.0 // Slope for linear drop probability increase backpressureSlope = 2.0 // Slope for linear drop probability increase
) )
var (
errAlreadyRunning = errors.New("streamer already running")
errUnexpectedStatus = errors.New("unexpected status code")
)
// MessageHandler defines the interface for processing RIS messages. // MessageHandler defines the interface for processing RIS messages.
// Implementations must specify which message types they want to receive, // Implementations must specify which message types they want to receive,
// how to process messages, and their desired queue capacity. // how to process messages, and their desired queue capacity.
type MessageHandler interface { type MessageHandler interface {
// WantsMessage returns true if this handler wants to process messages of the given type. // WantsMessage returns true if this handler wants to process messages of
// the given type.
WantsMessage(messageType string) bool WantsMessage(messageType string) bool
// HandleMessage processes a RIS message. This method is called from a dedicated // HandleMessage processes a RIS message. This method is called from a dedicated
@@ -113,7 +121,7 @@ type Streamer struct {
cancel context.CancelFunc cancel context.CancelFunc
running bool running bool
metrics *metrics.Tracker metrics *metrics.Tracker
totalDropped uint64 // Total dropped messages across all handlers totalDropped atomic.Uint64 // Total dropped messages across all handlers
random *rand.Rand // Random number generator for backpressure drops random *rand.Rand // Random number generator for backpressure drops
bgpPeers map[string]bool // Track active BGP peers by peer IP bgpPeers map[string]bool // Track active BGP peers by peer IP
bgpPeersMu sync.RWMutex // Protects bgpPeers map bgpPeersMu sync.RWMutex // Protects bgpPeers map
@@ -121,7 +129,8 @@ type Streamer struct {
// New creates a new Streamer instance configured to connect to the RIS Live API. // New creates a new Streamer instance configured to connect to the RIS Live API.
// The logger is used for structured logging of connection events and errors. // The logger is used for structured logging of connection events and errors.
// The metrics tracker is used to record message counts, bytes received, and connection status. // The metrics tracker is used to record message counts, bytes received, and
// connection status.
func New(logger *logger.Logger, metrics *metrics.Tracker) *Streamer { func New(logger *logger.Logger, metrics *metrics.Tracker) *Streamer {
return &Streamer{ return &Streamer{
logger: logger, logger: logger,
@@ -153,7 +162,8 @@ func (s *Streamer) RegisterHandler(handler MessageHandler) {
handler: handler, handler: handler,
queue: make(chan *ristypes.RISMessage, handler.QueueCapacity()), queue: make(chan *ristypes.RISMessage, handler.QueueCapacity()),
metrics: handlerMetrics{ metrics: handlerMetrics{
minTime: time.Duration(math.MaxInt64), // Initialize to max so first value sets the floor // Initialize to max so first value sets the floor
minTime: time.Duration(math.MaxInt64),
}, },
} }
@@ -171,6 +181,7 @@ func (s *Streamer) RegisterHandler(handler MessageHandler) {
func (s *Streamer) RegisterRawHandler(handler RawMessageHandler) { func (s *Streamer) RegisterRawHandler(handler RawMessageHandler) {
s.mu.Lock() s.mu.Lock()
defer s.mu.Unlock() defer s.mu.Unlock()
s.rawHandler = handler s.rawHandler = handler
} }
@@ -183,7 +194,7 @@ func (s *Streamer) Start() error {
defer s.mu.Unlock() defer s.mu.Unlock()
if s.running { if s.running {
return fmt.Errorf("streamer already running") return errAlreadyRunning
} }
ctx, cancel := context.WithCancel(context.Background()) ctx, cancel := context.WithCancel(context.Background())
@@ -216,44 +227,21 @@ func (s *Streamer) Stop() {
return return
} }
s.cancel() s.cancel()
s.cancel = nil s.cancel = nil
// Close all handler queues to signal workers to stop // Close all handler queues to signal workers to stop
for _, info := range s.handlers { for _, info := range s.handlers {
close(info.queue) close(info.queue)
} }
s.running = false s.running = false
s.mu.Unlock() s.mu.Unlock()
s.metrics.SetConnected(false) s.metrics.SetConnected(false)
} }
// runHandlerWorker processes messages for a specific handler // IsRunning reports whether the streamer is currently connected and processing
func (s *Streamer) runHandlerWorker(info *handlerInfo) { // messages. This is safe to call concurrently from multiple goroutines.
for msg := range info.queue {
start := time.Now()
info.handler.HandleMessage(msg)
elapsed := time.Since(start)
// Update metrics
info.metrics.mu.Lock()
info.metrics.processedCount++
info.metrics.totalTime += elapsed
// Update min time
if elapsed < info.metrics.minTime {
info.metrics.minTime = elapsed
}
// Update max time
if elapsed > info.metrics.maxTime {
info.metrics.maxTime = elapsed
}
info.metrics.mu.Unlock()
}
}
// IsRunning reports whether the streamer is currently connected and processing messages.
// This is safe to call concurrently from multiple goroutines.
func (s *Streamer) IsRunning() bool { func (s *Streamer) IsRunning() bool {
s.mu.RLock() s.mu.RLock()
defer s.mu.RUnlock() defer s.mu.RUnlock()
@@ -314,12 +302,15 @@ func (s *Streamer) GetHandlerStats() []HandlerStats {
// Calculate average time // Calculate average time
if info.metrics.processedCount > 0 { if info.metrics.processedCount > 0 {
processedCount := info.metrics.processedCount processedCount := info.metrics.processedCount
const maxInt64 = 1<<63 - 1 const maxInt64 = 1<<63 - 1
if processedCount > maxInt64 { if processedCount > maxInt64 {
processedCount = maxInt64 processedCount = maxInt64
} }
//nolint:gosec // processedCount is explicitly bounded above
hs.AvgProcessTime = info.metrics.totalTime / time.Duration(processedCount) hs.AvgProcessTime = info.metrics.totalTime /
time.Duration(processedCount)
} }
info.metrics.mu.Unlock() info.metrics.mu.Unlock()
@@ -330,11 +321,38 @@ func (s *Streamer) GetHandlerStats() []HandlerStats {
return stats return stats
} }
// GetDroppedMessages returns the total number of messages dropped across all handlers // GetDroppedMessages returns the total number of messages dropped across all
// due to queue overflow or backpressure. This counter is monotonically increasing // handlers due to queue overflow or backpressure. This counter is monotonically
// and is safe to call concurrently. // increasing and is safe to call concurrently.
func (s *Streamer) GetDroppedMessages() uint64 { func (s *Streamer) GetDroppedMessages() uint64 {
return atomic.LoadUint64(&s.totalDropped) return s.totalDropped.Load()
}
// runHandlerWorker processes messages for a specific handler
func (s *Streamer) runHandlerWorker(info *handlerInfo) {
for msg := range info.queue {
start := time.Now()
info.handler.HandleMessage(msg)
elapsed := time.Since(start)
// Update metrics
info.metrics.mu.Lock()
info.metrics.processedCount++
info.metrics.totalTime += elapsed
// Update min time
if elapsed < info.metrics.minTime {
info.metrics.minTime = elapsed
}
// Update max time
if elapsed > info.metrics.maxTime {
info.metrics.maxTime = elapsed
}
info.metrics.mu.Unlock()
}
} }
// logMetrics logs the current streaming statistics // logMetrics logs the current streaming statistics
@@ -343,7 +361,8 @@ func (s *Streamer) logMetrics() {
uptime := time.Since(metrics.ConnectedSince) uptime := time.Since(metrics.ConnectedSince)
const bitsPerMegabit = 1000000 const bitsPerMegabit = 1000000
totalDropped := atomic.LoadUint64(&s.totalDropped)
totalDropped := s.totalDropped.Load()
s.logger.Info( s.logger.Info(
"Stream statistics", "Stream statistics",
@@ -369,16 +388,19 @@ func (s *Streamer) logMetrics() {
// Log per-handler statistics // Log per-handler statistics
s.mu.RLock() s.mu.RLock()
for i, info := range s.handlers { for i, info := range s.handlers {
info.metrics.mu.Lock() info.metrics.mu.Lock()
if info.metrics.processedCount > 0 { if info.metrics.processedCount > 0 {
// Safe conversion: processedCount is bounded by maxInt64 // Safe conversion: processedCount is bounded by maxInt64
processedCount := info.metrics.processedCount processedCount := info.metrics.processedCount
const maxInt64 = 1<<63 - 1 const maxInt64 = 1<<63 - 1
if processedCount > maxInt64 { if processedCount > maxInt64 {
processedCount = maxInt64 processedCount = maxInt64
} }
//nolint:gosec // processedCount is explicitly bounded above
avgTime := info.metrics.totalTime / time.Duration(processedCount) avgTime := info.metrics.totalTime / time.Duration(processedCount)
s.logger.Info( s.logger.Info(
"Handler statistics", "Handler statistics",
@@ -395,6 +417,7 @@ func (s *Streamer) logMetrics() {
} }
info.metrics.mu.Unlock() info.metrics.mu.Unlock()
} }
s.mu.RUnlock() s.mu.RUnlock()
} }
@@ -403,7 +426,8 @@ func (s *Streamer) updateMetrics(messageBytes int) {
s.metrics.RecordMessage(int64(messageBytes)) s.metrics.RecordMessage(int64(messageBytes))
} }
// streamWithReconnect handles streaming with automatic reconnection and exponential backoff // streamWithReconnect handles streaming with automatic reconnection and
// exponential backoff
func (s *Streamer) streamWithReconnect(ctx context.Context) { func (s *Streamer) streamWithReconnect(ctx context.Context) {
backoffDelay := minBackoffDelay backoffDelay := minBackoffDelay
consecutiveFailures := 0 consecutiveFailures := 0
@@ -444,6 +468,7 @@ func (s *Streamer) streamWithReconnect(ctx context.Context) {
if streamDuration > 30*time.Second { if streamDuration > 30*time.Second {
s.logger.Info("Resetting backoff delay due to successful connection", s.logger.Info("Resetting backoff delay due to successful connection",
"stream_duration", streamDuration) "stream_duration", streamDuration)
backoffDelay = minBackoffDelay backoffDelay = minBackoffDelay
consecutiveFailures = 0 consecutiveFailures = 0
} else { } else {
@@ -477,7 +502,7 @@ func (s *Streamer) stream(ctx context.Context) error {
connCtx, connCancel := context.WithCancel(ctx) connCtx, connCancel := context.WithCancel(ctx)
defer connCancel() defer connCancel()
req, err := http.NewRequestWithContext(ctx, "GET", s.url, nil) req, err := http.NewRequestWithContext(ctx, http.MethodGet, s.url, nil)
if err != nil { if err != nil {
return fmt.Errorf("failed to create request: %w", err) return fmt.Errorf("failed to create request: %w", err)
} }
@@ -490,13 +515,14 @@ func (s *Streamer) stream(ctx context.Context) error {
return fmt.Errorf("failed to connect to RIS Live: %w", err) return fmt.Errorf("failed to connect to RIS Live: %w", err)
} }
defer func() { defer func() {
if err := resp.Body.Close(); err != nil { err := resp.Body.Close()
if err != nil {
s.logger.Error("Failed to close response body", "error", err) s.logger.Error("Failed to close response body", "error", err)
} }
}() }()
if resp.StatusCode != http.StatusOK { if resp.StatusCode != http.StatusOK {
return fmt.Errorf("unexpected status code: %d", resp.StatusCode) return fmt.Errorf("%w: %d", errUnexpectedStatus, resp.StatusCode)
} }
// Wrap body with counting reader to track actual wire bytes // Wrap body with counting reader to track actual wire bytes
@@ -510,7 +536,9 @@ func (s *Streamer) stream(ctx context.Context) error {
return fmt.Errorf("failed to create gzip reader: %w", err) return fmt.Errorf("failed to create gzip reader: %w", err)
} }
defer func() { _ = gzReader.Close() }() defer func() { _ = gzReader.Close() }()
reader = gzReader reader = gzReader
s.logger.Info("Connected to RIS Live stream", "compression", "gzip") s.logger.Info("Connected to RIS Live stream", "compression", "gzip")
} else { } else {
s.logger.Info("Connected to RIS Live stream", "compression", "none") s.logger.Info("Connected to RIS Live stream", "compression", "none")
@@ -518,9 +546,6 @@ func (s *Streamer) stream(ctx context.Context) error {
s.metrics.SetConnected(true) s.metrics.SetConnected(true)
// Track wire bytes for metrics updates
var lastWireBytes int64
// Start metrics logging goroutine // Start metrics logging goroutine
metricsTicker := time.NewTicker(metricsLogInterval) metricsTicker := time.NewTicker(metricsLogInterval)
defer metricsTicker.Stop() defer metricsTicker.Stop()
@@ -537,25 +562,41 @@ func (s *Streamer) stream(ctx context.Context) error {
}() }()
// Wire byte update ticker - update metrics with actual wire bytes periodically // Wire byte update ticker - update metrics with actual wire bytes periodically
go s.recordWireBytes(connCtx, wireCounter)
return s.readStream(ctx, reader)
}
// recordWireBytes adds the bytes wireCounter has counted to the metrics once a
// second until ctx is cancelled.
func (s *Streamer) recordWireBytes(
ctx context.Context, wireCounter *countingReader,
) {
wireUpdateTicker := time.NewTicker(time.Second) wireUpdateTicker := time.NewTicker(time.Second)
defer wireUpdateTicker.Stop() defer wireUpdateTicker.Stop()
go func() { var lastWireBytes int64
for {
select {
case <-wireUpdateTicker.C:
currentBytes := wireCounter.Count()
delta := currentBytes - lastWireBytes
if delta > 0 {
s.metrics.RecordWireBytes(delta)
lastWireBytes = currentBytes
}
case <-connCtx.Done():
return
}
}
}()
for {
select {
case <-wireUpdateTicker.C:
currentBytes := wireCounter.Count()
delta := currentBytes - lastWireBytes
if delta > 0 {
s.metrics.RecordWireBytes(delta)
lastWireBytes = currentBytes
}
case <-ctx.Done():
return
}
}
}
// readStream hands each line of the stream to processLine until the stream
// ends, a line fails, or ctx is cancelled.
func (s *Streamer) readStream(ctx context.Context, reader io.Reader) error {
scanner := bufio.NewScanner(reader) scanner := bufio.NewScanner(reader)
for scanner.Scan() { for scanner.Scan() {
@@ -572,149 +613,176 @@ func (s *Streamer) stream(ctx context.Context) error {
continue continue
} }
// Update metrics with decompressed message size err := s.processLine(ctx, line)
s.updateMetrics(len(line)) if err != nil {
return err
// Call raw handler if registered
s.mu.RLock()
rawHandler := s.rawHandler
s.mu.RUnlock()
if rawHandler != nil {
// Call raw handler synchronously to preserve order
rawHandler(string(line))
} }
// Parse the message first
var wrapper ristypes.RISLiveMessage
if err := json.Unmarshal(line, &wrapper); err != nil {
// Log the error and return to trigger reconnection
s.logger.Error("Failed to parse JSON",
"error", err,
"line", string(line),
"line_length", len(line))
return fmt.Errorf("JSON parse error: %w", err)
}
// Check if it's a ris_message wrapper
if wrapper.Type != "ris_message" {
s.logger.Error("Unexpected wrapper type",
"type", wrapper.Type,
"line", string(line),
)
continue
}
// Get the actual message
msg := wrapper.Data
// Parse the timestamp
msg.ParsedTimestamp = time.Unix(int64(msg.Timestamp), 0).UTC()
// Process based on message type
switch msg.Type {
case "UPDATE":
// Process BGP UPDATE messages
// Will be dispatched to handlers
case "RIS_PEER_STATE":
// RIS peer state messages - silently ignore
continue
case "KEEPALIVE":
// BGP keepalive messages - silently process
continue
case "OPEN":
// BGP open messages - track peer as active
s.bgpPeersMu.Lock()
s.bgpPeers[msg.Peer] = true
peerCount := len(s.bgpPeers)
s.bgpPeersMu.Unlock()
s.metrics.SetBGPPeerCount(peerCount)
s.logger.Info("BGP session opened",
"peer", msg.Peer,
"peer_asn", msg.PeerASN,
"total_peers", peerCount,
)
continue
case "NOTIFICATION":
// BGP notification messages (session closed)
s.bgpPeersMu.Lock()
delete(s.bgpPeers, msg.Peer)
peerCount := len(s.bgpPeers)
s.bgpPeersMu.Unlock()
s.metrics.SetBGPPeerCount(peerCount)
s.logger.Warn("BGP notification",
"peer", msg.Peer,
"peer_asn", msg.PeerASN,
"total_peers", peerCount,
)
continue
case "STATE":
// Peer state changes - silently ignore
continue
default:
s.logger.Warn("Unknown message type, skipping",
"type", msg.Type,
)
continue
}
// Dispatch to interested handlers. Stop cancels ctx and closes the
// queues under the write lock, so if ctx is cancelled here, under the
// read lock, the queues are closed and must not be sent to.
s.mu.RLock()
if ctx.Err() != nil {
s.mu.RUnlock()
return ctx.Err()
}
for _, info := range s.handlers {
if !info.handler.WantsMessage(msg.Type) {
continue
}
// Check if we should drop due to backpressure
if s.shouldDropForBackpressure(info) {
atomic.AddUint64(&info.metrics.droppedCount, 1)
atomic.AddUint64(&s.totalDropped, 1)
continue
}
// Try to queue the message
select {
case info.queue <- &msg:
// Message queued successfully
// Update high water mark if needed
queueLen := len(info.queue)
info.metrics.mu.Lock()
if queueLen > info.metrics.queueHighWaterMark {
info.metrics.queueHighWaterMark = queueLen
}
info.metrics.mu.Unlock()
default:
// Queue is full, drop the message
atomic.AddUint64(&info.metrics.droppedCount, 1)
atomic.AddUint64(&s.totalDropped, 1)
}
}
s.mu.RUnlock()
} }
if err := scanner.Err(); err != nil { err := scanner.Err()
if err != nil {
return fmt.Errorf("scanner error: %w", err) return fmt.Errorf("scanner error: %w", err)
} }
return nil return nil
} }
// shouldDropForBackpressure determines if a message should be dropped based on queue utilization // processLine passes one line of the stream to the raw handler, parses it,
// and dispatches an UPDATE message to the handlers. An error ends the
// connection.
func (s *Streamer) processLine(ctx context.Context, line []byte) error {
// Update metrics with decompressed message size
s.updateMetrics(len(line))
// Call raw handler if registered
s.mu.RLock()
rawHandler := s.rawHandler
s.mu.RUnlock()
if rawHandler != nil {
// Call raw handler synchronously to preserve order
rawHandler(string(line))
}
// Parse the message first
var wrapper ristypes.RISLiveMessage
err := json.Unmarshal(line, &wrapper)
if err != nil {
// Log the error and return to trigger reconnection
s.logger.Error("Failed to parse JSON",
"error", err,
"line", string(line),
"line_length", len(line))
return fmt.Errorf("JSON parse error: %w", err)
}
// Check if it's a ris_message wrapper
if wrapper.Type != "ris_message" {
s.logger.Error("Unexpected wrapper type",
"type", wrapper.Type,
"line", string(line),
)
return nil
}
// Get the actual message
msg := wrapper.Data
// Parse the timestamp
msg.ParsedTimestamp = time.Unix(int64(msg.Timestamp), 0).UTC()
// Process based on message type
switch msg.Type {
case "UPDATE":
// Process BGP UPDATE messages
// Will be dispatched to handlers
case "RIS_PEER_STATE":
// RIS peer state messages - silently ignore
return nil
case "KEEPALIVE":
// BGP keepalive messages - silently process
return nil
case "OPEN":
// BGP open messages - track peer as active
s.bgpPeersMu.Lock()
s.bgpPeers[msg.Peer] = true
peerCount := len(s.bgpPeers)
s.bgpPeersMu.Unlock()
s.metrics.SetBGPPeerCount(peerCount)
s.logger.Info("BGP session opened",
"peer", msg.Peer,
"peer_asn", msg.PeerASN,
"total_peers", peerCount,
)
return nil
case "NOTIFICATION":
// BGP notification messages (session closed)
s.bgpPeersMu.Lock()
delete(s.bgpPeers, msg.Peer)
peerCount := len(s.bgpPeers)
s.bgpPeersMu.Unlock()
s.metrics.SetBGPPeerCount(peerCount)
s.logger.Warn("BGP notification",
"peer", msg.Peer,
"peer_asn", msg.PeerASN,
"total_peers", peerCount,
)
return nil
case "STATE":
// Peer state changes - silently ignore
return nil
default:
s.logger.Warn("Unknown message type, skipping",
"type", msg.Type,
)
return nil
}
return s.dispatchMessage(ctx, &msg)
}
// dispatchMessage queues msg for each handler that wants it, dropping it for
// a handler whose queue is full or under backpressure.
func (s *Streamer) dispatchMessage(
ctx context.Context, msg *ristypes.RISMessage,
) error {
// Stop cancels ctx and closes the queues under the write lock, so if ctx
// is cancelled here, under the read lock, the queues are closed and must
// not be sent to.
s.mu.RLock()
if ctx.Err() != nil {
s.mu.RUnlock()
return ctx.Err()
}
for _, info := range s.handlers {
if !info.handler.WantsMessage(msg.Type) {
continue
}
// Check if we should drop due to backpressure
if s.shouldDropForBackpressure(info) {
atomic.AddUint64(&info.metrics.droppedCount, 1)
s.totalDropped.Add(1)
continue
}
// Try to queue the message
select {
case info.queue <- msg:
// Message queued successfully
// Update high water mark if needed
queueLen := len(info.queue)
info.metrics.mu.Lock()
if queueLen > info.metrics.queueHighWaterMark {
info.metrics.queueHighWaterMark = queueLen
}
info.metrics.mu.Unlock()
default:
// Queue is full, drop the message
atomic.AddUint64(&info.metrics.droppedCount, 1)
s.totalDropped.Add(1)
}
}
s.mu.RUnlock()
return nil
}
// shouldDropForBackpressure determines if a message should be dropped based on
// queue utilization
func (s *Streamer) shouldDropForBackpressure(info *handlerInfo) bool { func (s *Streamer) shouldDropForBackpressure(info *handlerInfo) bool {
// Calculate queue utilization // Calculate queue utilization
queueLen := len(info.queue) queueLen := len(info.queue)
@@ -16,6 +16,8 @@ import (
) )
func TestNewStreamer(t *testing.T) { func TestNewStreamer(t *testing.T) {
t.Parallel()
logger := logger.New() logger := logger.New()
metricsTracker := metrics.New() metricsTracker := metrics.New()
s := New(logger, metricsTracker) s := New(logger, metricsTracker)
@@ -46,10 +48,13 @@ func TestNewStreamer(t *testing.T) {
// goroutine count returns to its starting value. Each connection starts two // goroutine count returns to its starting value. Each connection starts two
// ticker goroutines; before the fix they lived until the streamer's lifetime // ticker goroutines; before the fix they lived until the streamer's lifetime
// context was cancelled, so every reconnect leaked two. // context was cancelled, so every reconnect leaked two.
//
//nolint:paralleltest // counts goroutines, so no other test may run alongside
func TestStreamDoesNotLeakTickersAcrossReconnects(t *testing.T) { func TestStreamDoesNotLeakTickersAcrossReconnects(t *testing.T) {
// The handler returns immediately, so the response body is empty and each // The handler returns immediately, so the response body is empty and each
// stream call ends at once, standing in for a dropped connection. // stream call ends at once, standing in for a dropped connection.
srv := httptest.NewServer(http.HandlerFunc(func(_ http.ResponseWriter, _ *http.Request) {})) srv := httptest.NewServer(http.HandlerFunc(
func(_ http.ResponseWriter, _ *http.Request) {}))
defer srv.Close() defer srv.Close()
s := New(logger.New(), metrics.New()) s := New(logger.New(), metrics.New())
@@ -57,19 +62,23 @@ func TestStreamDoesNotLeakTickersAcrossReconnects(t *testing.T) {
// One warm-up connection so any persistent HTTP transport goroutine exists // One warm-up connection so any persistent HTTP transport goroutine exists
// before we take the baseline. // before we take the baseline.
if err := s.stream(context.Background()); err != nil { err := s.stream(context.Background())
if err != nil {
t.Fatalf("warm-up stream returned error: %v", err) t.Fatalf("warm-up stream returned error: %v", err)
} }
s.client.CloseIdleConnections() s.client.CloseIdleConnections()
baseline := settledGoroutineCount() baseline := settledGoroutineCount()
const reconnects = 20 const reconnects = 20
for range reconnects { for range reconnects {
if err := s.stream(context.Background()); err != nil { err := s.stream(context.Background())
if err != nil {
t.Fatalf("stream returned error: %v", err) t.Fatalf("stream returned error: %v", err)
} }
} }
s.client.CloseIdleConnections() s.client.CloseIdleConnections()
if !waitForGoroutines(baseline) { if !waitForGoroutines(baseline) {
@@ -81,9 +90,13 @@ func TestStreamDoesNotLeakTickersAcrossReconnects(t *testing.T) {
// updateHandler wants UPDATE messages and does nothing with them. // updateHandler wants UPDATE messages and does nothing with them.
type updateHandler struct{} type updateHandler struct{}
func (updateHandler) WantsMessage(messageType string) bool { return messageType == "UPDATE" } func (updateHandler) WantsMessage(messageType string) bool {
func (updateHandler) HandleMessage(*ristypes.RISMessage) {} return messageType == "UPDATE"
func (updateHandler) QueueCapacity() int { return 10 } }
func (updateHandler) HandleMessage(*ristypes.RISMessage) {}
func (updateHandler) QueueCapacity() int { return 10 }
// TestStopBeforeMessageReachesQueues stops the streamer after the read loop // TestStopBeforeMessageReachesQueues stops the streamer after the read loop
// has checked for cancellation but before it hands the message to the handler // has checked for cancellation but before it hands the message to the handler
@@ -91,11 +104,15 @@ func (updateHandler) QueueCapacity() int { return 10 }
// used to end in "send on closed channel". The raw handler runs in that gap on // used to end in "send on closed channel". The raw handler runs in that gap on
// the read loop itself, so calling Stop from it hits the gap every time. // the read loop itself, so calling Stop from it hits the gap every time.
func TestStopBeforeMessageReachesQueues(t *testing.T) { func TestStopBeforeMessageReachesQueues(t *testing.T) {
const line = `{"type":"ris_message","data":{"type":"UPDATE","peer":"192.0.2.1",` + t.Parallel()
`"peer_asn":"64496","timestamp":1700000000}}` + "\n"
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { const line = `{"type":"ris_message","data":{"type":"UPDATE",` +
_, _ = io.WriteString(w, line) `"peer":"192.0.2.1","peer_asn":"64496","timestamp":1700000000}}` + "\n"
}))
srv := httptest.NewServer(http.HandlerFunc(
func(w http.ResponseWriter, _ *http.Request) {
_, _ = io.WriteString(w, line)
}))
defer srv.Close() defer srv.Close()
s := New(logger.New(), metrics.New()) s := New(logger.New(), metrics.New())
@@ -109,7 +126,8 @@ func TestStopBeforeMessageReachesQueues(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background()) ctx, cancel := context.WithCancel(context.Background())
s.cancel = cancel s.cancel = cancel
if err := s.stream(ctx); !errors.Is(err, context.Canceled) { err := s.stream(ctx)
if !errors.Is(err, context.Canceled) {
t.Fatalf("stream returned %v, want %v", err, context.Canceled) t.Fatalf("stream returned %v, want %v", err, context.Canceled)
} }
@@ -121,12 +139,15 @@ func TestStopBeforeMessageReachesQueues(t *testing.T) {
// current count. // current count.
func settledGoroutineCount() int { func settledGoroutineCount() int {
prev := runtime.NumGoroutine() prev := runtime.NumGoroutine()
for range 20 { for range 20 {
time.Sleep(10 * time.Millisecond) time.Sleep(10 * time.Millisecond)
cur := runtime.NumGoroutine() cur := runtime.NumGoroutine()
if cur == prev { if cur == prev {
return cur return cur
} }
prev = cur prev = cur
} }
@@ -139,6 +160,7 @@ func waitForGoroutines(target int) bool {
if runtime.NumGoroutine() <= target { if runtime.NumGoroutine() <= target {
return true return true
} }
time.Sleep(10 * time.Millisecond) time.Sleep(10 * time.Millisecond)
} }
+11 -4
View File
@@ -60,6 +60,7 @@ func timeSince(t time.Time) string {
if duration < time.Minute { if duration < time.Minute {
return "just now" return "just now"
} }
if duration < time.Hour { if duration < time.Hour {
minutes := int(duration.Minutes()) minutes := int(duration.Minutes())
if minutes == 1 { if minutes == 1 {
@@ -68,6 +69,7 @@ func timeSince(t time.Time) string {
return duration.Truncate(time.Minute).String() + " ago" return duration.Truncate(time.Minute).String() + " ago"
} }
if duration < hoursPerDay*time.Hour { if duration < hoursPerDay*time.Hour {
hours := int(duration.Hours()) hours := int(duration.Hours())
if hours == 1 { if hours == 1 {
@@ -76,10 +78,12 @@ func timeSince(t time.Time) string {
return duration.Truncate(time.Hour).String() + " ago" return duration.Truncate(time.Hour).String() + " ago"
} }
days := int(duration.Hours() / hoursPerDay) days := int(duration.Hours() / hoursPerDay)
if days == 1 { if days == 1 {
return "1 day ago" return "1 day ago"
} }
if days < daysPerMonth { if days < daysPerMonth {
return duration.Truncate(hoursPerDay*time.Hour).String() + " ago" return duration.Truncate(hoursPerDay*time.Hour).String() + " ago"
} }
@@ -118,7 +122,7 @@ func initTemplates() {
"appLicense": func() string { return version.License }, "appLicense": func() string { return version.License },
"appRepoURL": func() string { return version.RepoURL }, "appRepoURL": func() string { return version.RepoURL },
"appGitRevision": func() string { return version.GitRevisionShort }, "appGitRevision": func() string { return version.GitRevisionShort },
"appGitCommitURL": func() string { return version.CommitURL() }, "appGitCommitURL": version.CommitURL,
} }
// Parse index template // Parse index template
@@ -134,19 +138,22 @@ func initTemplates() {
} }
// Parse AS detail template // Parse AS detail template
defaultTemplates.ASDetail, err = template.New("asDetail").Funcs(funcs).Parse(asDetailHTML) defaultTemplates.ASDetail, err = template.New("asDetail").
Funcs(funcs).Parse(asDetailHTML)
if err != nil { if err != nil {
panic("failed to parse AS detail template: " + err.Error()) panic("failed to parse AS detail template: " + err.Error())
} }
// Parse prefix detail template // Parse prefix detail template
defaultTemplates.PrefixDetail, err = template.New("prefixDetail").Funcs(funcs).Parse(prefixDetailHTML) defaultTemplates.PrefixDetail, err = template.New("prefixDetail").
Funcs(funcs).Parse(prefixDetailHTML)
if err != nil { if err != nil {
panic("failed to parse prefix detail template: " + err.Error()) panic("failed to parse prefix detail template: " + err.Error())
} }
// Parse prefix length template // Parse prefix length template
defaultTemplates.PrefixLength, err = template.New("prefixLength").Funcs(funcs).Parse(prefixLengthHTML) defaultTemplates.PrefixLength, err = template.New("prefixLength").
Funcs(funcs).Parse(prefixLengthHTML)
if err != nil { if err != nil {
panic("failed to parse prefix length template: " + err.Error()) panic("failed to parse prefix length template: " + err.Error())
} }
+1 -1
View File
@@ -3,7 +3,7 @@ package version
// Build-time variables set via ldflags // Build-time variables set via ldflags
// //
//nolint:gochecknoglobals // These must be variables to allow ldflags injection at build time //nolint:gochecknoglobals // variables so that ldflags can set them at build time
var ( var (
// GitRevision is the git commit hash // GitRevision is the git commit hash
GitRevision = "unknown" GitRevision = "unknown"
+106 -78
View File
@@ -98,7 +98,8 @@ func (c *Client) LookupASN(ctx context.Context, asn int) (*ASNInfo, error) {
if err != nil { if err != nil {
// Return partial data from IANA if RIR query fails // Return partial data from IANA if RIR query fails
info := c.parseResponse(asn, rir, ianaResp) info := c.parseResponse(asn, rir, ianaResp)
info.RawResponse = ianaResp + "\n--- RIR query failed: " + err.Error() + " ---\n" info.RawResponse = fmt.Sprintf("%s\n--- RIR query failed: %v ---\n",
ianaResp, err)
return info, nil return info, nil
} }
@@ -120,28 +121,33 @@ func (c *Client) query(ctx context.Context, server, query string) (string, error
defer func() { _ = conn.Close() }() defer func() { _ = conn.Close() }()
// Set deadlines // Set deadlines
if err := conn.SetWriteDeadline(time.Now().Add(writeTimeout)); err != nil { err = conn.SetWriteDeadline(time.Now().Add(writeTimeout))
if err != nil {
return "", fmt.Errorf("set write deadline: %w", err) return "", fmt.Errorf("set write deadline: %w", err)
} }
// Send query // Send query
if _, err := fmt.Fprintf(conn, "%s\r\n", query); err != nil { _, err = fmt.Fprintf(conn, "%s\r\n", query)
if err != nil {
return "", fmt.Errorf("write query: %w", err) return "", fmt.Errorf("write query: %w", err)
} }
// Read response // Read response
if err := conn.SetReadDeadline(time.Now().Add(readTimeout)); err != nil { err = conn.SetReadDeadline(time.Now().Add(readTimeout))
if err != nil {
return "", fmt.Errorf("set read deadline: %w", err) return "", fmt.Errorf("set read deadline: %w", err)
} }
var sb strings.Builder var sb strings.Builder
scanner := bufio.NewScanner(conn) scanner := bufio.NewScanner(conn)
for scanner.Scan() { for scanner.Scan() {
sb.WriteString(scanner.Text()) sb.WriteString(scanner.Text())
sb.WriteString("\n") sb.WriteString("\n")
} }
if err := scanner.Err(); err != nil { err = scanner.Err()
if err != nil {
return sb.String(), fmt.Errorf("read response: %w", err) return sb.String(), fmt.Errorf("read response: %w", err)
} }
@@ -149,10 +155,10 @@ func (c *Client) query(ctx context.Context, server, query string) (string, error
} }
// parseIANAReferral extracts the RIR and WHOIS server from an IANA response. // parseIANAReferral extracts the RIR and WHOIS server from an IANA response.
func (c *Client) parseIANAReferral(response string) (rir, whoisServer string) { func (c *Client) parseIANAReferral(response string) (string, string) {
lines := strings.Split(response, "\n") var rir string
for _, line := range lines { for line := range strings.SplitSeq(response, "\n") {
line = strings.TrimSpace(line) line = strings.TrimSpace(line)
// Look for "refer:" line // Look for "refer:" line
@@ -183,18 +189,9 @@ func (c *Client) parseIANAReferral(response string) (rir, whoisServer string) {
// Also check organisation line for RIR hints // Also check organisation line for RIR hints
if strings.HasPrefix(strings.ToLower(line), "organisation:") { if strings.HasPrefix(strings.ToLower(line), "organisation:") {
org := strings.ToLower(line) orgRIR := rirInOrganisationLine(strings.ToLower(line))
switch { if orgRIR != "" {
case strings.Contains(org, "arin"): rir = orgRIR
rir = RIRARIN
case strings.Contains(org, "ripe"):
rir = RIRRIPE
case strings.Contains(org, "apnic"):
rir = RIRAPNIC
case strings.Contains(org, "lacnic"):
rir = RIRLACNIC
case strings.Contains(org, "afrinic"):
rir = RIRAFRNIC
} }
} }
} }
@@ -202,6 +199,25 @@ func (c *Client) parseIANAReferral(response string) (rir, whoisServer string) {
return rir, "" return rir, ""
} }
// rirInOrganisationLine returns the RIR a lowercased IANA "organisation:"
// line names, or "" when it names none.
func rirInOrganisationLine(org string) string {
switch {
case strings.Contains(org, "arin"):
return RIRARIN
case strings.Contains(org, "ripe"):
return RIRRIPE
case strings.Contains(org, "apnic"):
return RIRAPNIC
case strings.Contains(org, "lacnic"):
return RIRLACNIC
case strings.Contains(org, "afrinic"):
return RIRAFRNIC
default:
return ""
}
}
// parseResponse extracts ASN information from a WHOIS response. // parseResponse extracts ASN information from a WHOIS response.
func (c *Client) parseResponse(asn int, rir, response string) *ASNInfo { func (c *Client) parseResponse(asn int, rir, response string) *ASNInfo {
info := &ASNInfo{ info := &ASNInfo{
@@ -211,11 +227,13 @@ func (c *Client) parseResponse(asn int, rir, response string) *ASNInfo {
} }
lines := strings.Split(response, "\n") lines := strings.Split(response, "\n")
var addressLines []string var addressLines []string
for _, line := range lines { for _, line := range lines {
line = strings.TrimSpace(line) line = strings.TrimSpace(line)
if line == "" || strings.HasPrefix(line, "%") || strings.HasPrefix(line, "#") { if line == "" || strings.HasPrefix(line, "%") ||
strings.HasPrefix(line, "#") {
continue continue
} }
@@ -232,63 +250,14 @@ func (c *Client) parseResponse(asn int, rir, response string) *ASNInfo {
continue continue
} }
switch key {
// AS Name (varies by RIR)
case "asname", "as-name":
if info.ASName == "" {
info.ASName = value
}
// Organization
case "orgname", "org-name", "owner":
if info.OrgName == "" {
info.OrgName = value
}
case "orgid", "org-id", "org":
if info.OrgID == "" {
info.OrgID = value
}
// Address (collect multiple lines) // Address (collect multiple lines)
case "address": if key == "address" {
addressLines = append(addressLines, value) addressLines = append(addressLines, value)
// Country continue
case "country":
if info.CountryCode == "" && len(value) == 2 {
info.CountryCode = strings.ToUpper(value)
}
// Abuse contact
case "orgabuseemail", "abuse-mailbox":
if info.AbuseEmail == "" {
info.AbuseEmail = value
}
case "orgabusephone":
if info.AbusePhone == "" {
info.AbusePhone = value
}
// Tech contact
case "orgtechemail":
if info.TechEmail == "" {
info.TechEmail = value
}
case "orgtechphone":
if info.TechPhone == "" {
info.TechPhone = value
}
// Registration dates
case "regdate", "created":
if info.RegDate == nil {
info.RegDate = c.parseDate(value)
}
case "updated", "last-modified", "changed":
if info.LastMod == nil {
info.LastMod = c.parseDate(value)
}
} }
c.setResponseField(info, key, value)
} }
// Combine address lines // Combine address lines
@@ -304,6 +273,65 @@ func (c *Client) parseResponse(asn int, rir, response string) *ASNInfo {
return info return info
} }
// setResponseField stores the value of one "key: value" line of a WHOIS
// response in info, keeping the first value seen for each field.
func (c *Client) setResponseField(info *ASNInfo, key, value string) {
switch key {
// Country
case "country":
if info.CountryCode == "" && len(value) == 2 {
info.CountryCode = strings.ToUpper(value)
}
// Registration dates
case "regdate", "created":
if info.RegDate == nil {
info.RegDate = c.parseDate(value)
}
case "updated", "last-modified", "changed":
if info.LastMod == nil {
info.LastMod = c.parseDate(value)
}
default:
field := textFieldForKey(info, key)
if field != nil && *field == "" {
*field = value
}
}
}
// textFieldForKey returns the text field of info that a WHOIS response key
// fills, or nil when the key fills none.
func textFieldForKey(info *ASNInfo, key string) *string {
switch key {
// AS Name (varies by RIR)
case "asname", "as-name":
return &info.ASName
// Organization
case "orgname", "org-name", "owner":
return &info.OrgName
case "orgid", "org-id", "org":
return &info.OrgID
// Abuse contact
case "orgabuseemail", "abuse-mailbox":
return &info.AbuseEmail
case "orgabusephone":
return &info.AbusePhone
// Tech contact
case "orgtechemail":
return &info.TechEmail
case "orgtechphone":
return &info.TechPhone
default:
return nil
}
}
// parseDate attempts to parse various date formats used in WHOIS responses. // parseDate attempts to parse various date formats used in WHOIS responses.
func (c *Client) parseDate(value string) *time.Time { func (c *Client) parseDate(value string) *time.Time {
// Common formats // Common formats
@@ -319,15 +347,15 @@ func (c *Client) parseDate(value string) *time.Time {
value = strings.TrimSpace(value) value = strings.TrimSpace(value)
// Handle "YYYYMMDD" format from LACNIC // Handle "YYYYMMDD" format from LACNIC
if len(value) == lacnicDateFormatLen { if len(value) == lacnicDateFormatLen {
if _, err := time.Parse("20060102", value); err == nil { t, err := time.Parse("20060102", value)
t, _ := time.Parse("20060102", value) if err == nil {
return &t return &t
} }
} }
for _, format := range formats { for _, format := range formats {
if t, err := time.Parse(format, value); err == nil { t, err := time.Parse(format, value)
if err == nil {
return &t return &t
} }
} }
+4 -2
View File
@@ -1,4 +1,3 @@
// Package asinfo provides information about Autonomous Systems (AS)
package asinfo package asinfo
import ( import (
@@ -54,7 +53,9 @@ func initRegistry() {
} }
var asInfos []Info var asInfos []Info
if err := json.Unmarshal(jsonData, &asInfos); err != nil {
err = json.Unmarshal(jsonData, &asInfos)
if err != nil {
panic("failed to unmarshal embedded AS data: " + err.Error()) panic("failed to unmarshal embedded AS data: " + err.Error())
} }
@@ -137,6 +138,7 @@ func Search(query string) []Info {
defer defaultRegistry.mu.RUnlock() defer defaultRegistry.mu.RUnlock()
var results []Info var results []Info
for _, info := range defaultRegistry.byASN { for _, info := range defaultRegistry.byASN {
if contains(info.Handle, query) || contains(info.Description, query) { if contains(info.Handle, query) || contains(info.Description, query) {
results = append(results, *info) results = append(results, *info)
+52 -16
View File
@@ -1,10 +1,15 @@
package asinfo package asinfo_test
import ( import (
"strings"
"testing" "testing"
"git.eeqj.de/sneak/routewatch/pkg/asinfo"
) )
func TestGet(t *testing.T) { func TestGet(t *testing.T) {
t.Parallel()
tests := []struct { tests := []struct {
name string name string
asn int asn int
@@ -38,20 +43,26 @@ func TestGet(t *testing.T) {
for _, tt := range tests { for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) { t.Run(tt.name, func(t *testing.T) {
info, ok := Get(tt.asn) t.Parallel()
info, ok := asinfo.Get(tt.asn)
if ok != tt.wantOK { if ok != tt.wantOK {
t.Errorf("Get(%d) ok = %v, want %v", tt.asn, ok, tt.wantOK) t.Errorf("Get(%d) ok = %v, want %v", tt.asn, ok, tt.wantOK)
return return
} }
if ok && info.Description != tt.wantDesc { if ok && info.Description != tt.wantDesc {
t.Errorf("Get(%d) description = %q, want %q", tt.asn, info.Description, tt.wantDesc) t.Errorf("Get(%d) description = %q, want %q",
tt.asn, info.Description, tt.wantDesc)
} }
}) })
} }
} }
func TestGetDescription(t *testing.T) { func TestGetDescription(t *testing.T) {
t.Parallel()
tests := []struct { tests := []struct {
asn int asn int
want string want string
@@ -62,7 +73,10 @@ func TestGetDescription(t *testing.T) {
for _, tt := range tests { for _, tt := range tests {
t.Run("", func(t *testing.T) { t.Run("", func(t *testing.T) {
if got := GetDescription(tt.asn); got != tt.want { t.Parallel()
got := asinfo.GetDescription(tt.asn)
if got != tt.want {
t.Errorf("GetDescription(%d) = %q, want %q", tt.asn, got, tt.want) t.Errorf("GetDescription(%d) = %q, want %q", tt.asn, got, tt.want)
} }
}) })
@@ -70,6 +84,8 @@ func TestGetDescription(t *testing.T) {
} }
func TestGetHandle(t *testing.T) { func TestGetHandle(t *testing.T) {
t.Parallel()
tests := []struct { tests := []struct {
asn int asn int
want string want string
@@ -81,7 +97,10 @@ func TestGetHandle(t *testing.T) {
for _, tt := range tests { for _, tt := range tests {
t.Run("", func(t *testing.T) { t.Run("", func(t *testing.T) {
if got := GetHandle(tt.asn); got != tt.want { t.Parallel()
got := asinfo.GetHandle(tt.asn)
if got != tt.want {
t.Errorf("GetHandle(%d) = %q, want %q", tt.asn, got, tt.want) t.Errorf("GetHandle(%d) = %q, want %q", tt.asn, got, tt.want)
} }
}) })
@@ -89,18 +108,23 @@ func TestGetHandle(t *testing.T) {
} }
func TestTotal(t *testing.T) { func TestTotal(t *testing.T) {
total := Total() t.Parallel()
total := asinfo.Total()
if total < 100000 { if total < 100000 {
t.Errorf("Total() = %d, expected > 100000", total) t.Errorf("Total() = %d, expected > 100000", total)
} }
// Verify it's consistent // Verify it's consistent
if total2 := Total(); total2 != total { total2 := asinfo.Total()
if total2 != total {
t.Errorf("Total() returned different values: %d vs %d", total, total2) t.Errorf("Total() returned different values: %d vs %d", total, total2)
} }
} }
func TestSearch(t *testing.T) { func TestSearch(t *testing.T) {
t.Parallel()
tests := []struct { tests := []struct {
name string name string
query string query string
@@ -130,16 +154,21 @@ func TestSearch(t *testing.T) {
for _, tt := range tests { for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) { t.Run(tt.name, func(t *testing.T) {
results := Search(tt.query) t.Parallel()
results := asinfo.Search(tt.query)
if len(results) < tt.wantMin { if len(results) < tt.wantMin {
t.Errorf("Search(%q) returned %d results, want at least %d", tt.query, len(results), tt.wantMin) t.Errorf("Search(%q) returned %d results, want at least %d",
tt.query, len(results), tt.wantMin)
} }
// Verify all results contain the query // Verify all results contain the query
if tt.query != "" { if tt.query != "" {
for _, r := range results { for _, r := range results {
if !contains(r.Handle, tt.query) && !contains(r.Description, tt.query) { if !strings.Contains(r.Handle, tt.query) &&
t.Errorf("Search result ASN %d doesn't contain query %q", r.ASN, tt.query) !strings.Contains(r.Description, tt.query) {
t.Errorf("Search result ASN %d doesn't contain query %q",
r.ASN, tt.query)
} }
} }
} }
@@ -148,13 +177,17 @@ func TestSearch(t *testing.T) {
} }
func TestDataIntegrity(t *testing.T) { func TestDataIntegrity(t *testing.T) {
t.Parallel()
// Verify no duplicate ASNs // Verify no duplicate ASNs
all := All() all := asinfo.All()
seen := make(map[int]bool) seen := make(map[int]bool)
for _, info := range all { for _, info := range all {
if seen[info.ASN] { if seen[info.ASN] {
t.Errorf("Duplicate ASN found: %d", info.ASN) t.Errorf("Duplicate ASN found: %d", info.ASN)
} }
seen[info.ASN] = true seen[info.ASN] = true
} }
@@ -163,6 +196,7 @@ func TestDataIntegrity(t *testing.T) {
if info.Handle == "" && info.ASN != 0 { if info.Handle == "" && info.ASN != 0 {
t.Errorf("ASN %d has empty handle", info.ASN) t.Errorf("ASN %d has empty handle", info.ASN)
} }
if info.Description == "" { if info.Description == "" {
t.Errorf("ASN %d has empty description", info.ASN) t.Errorf("ASN %d has empty description", info.ASN)
} }
@@ -174,8 +208,9 @@ func BenchmarkGet(b *testing.B) {
asns := []int{1, 15169, 13335, 32934, 8075, 16509} asns := []int{1, 15169, 13335, 32934, 8075, 16509}
b.ResetTimer() b.ResetTimer()
for i := 0; i < b.N; i++ {
Get(asns[i%len(asns)]) for i := range b.N {
asinfo.Get(asns[i%len(asns)])
} }
} }
@@ -183,7 +218,8 @@ func BenchmarkSearch(b *testing.B) {
queries := []string{"Google", "Amazon", "Microsoft", "University"} queries := []string{"Google", "Amazon", "Microsoft", "University"}
b.ResetTimer() b.ResetTimer()
for i := 0; i < b.N; i++ {
Search(queries[i%len(queries)]) for i := range b.N {
asinfo.Search(queries[i%len(queries)])
} }
} }