sneak/routewatch
1
0
Fork 0
Code Issues 5 Pull Requests 2 Actions 35 Packages Projects Releases Wiki Activity

Remove the per-peer route tables (closes #122)
check / check (push) Canceled after 0s
Details

Browse Source 
The routing table in memory now holds every route, so the two per-peer route
tables go, with every write, delete, count and read that served them. The
prefix handler applies each update to the table, writes the table's changes
every second, and stores each new prefix with its first-seen time. The peer
delete, its retry on the next flush and the updates held for it go too. The
types and errors of the server's answers move to internal/server.

The image's memory settings fit a full table of about 1 GB: GOMEMLIMIT of
2.5 GiB, a 512 MiB page cache for the writing connection, and SQLite heap
limits of 768 MiB and 1 GiB. The README describes the new table, its
database copy and the memory budget.

Model: opus-5-5
This commit was merged in pull request #195.
This commit is contained in:
clawbot
2026-10-27 17:22:32 +01:00
parent 506e289a63
commit 1c65656b58
23 changed files with 701 additions and 5273 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/database/counts.go
+9 -100
View File
@@ -6,11 +6,10 @@ import (
"sync"
)
// liveCounts holds the running row counts and the prefix distribution that the
// stats endpoints report. They are seeded once at startup from the tables and
// then adjusted on every write, so a stats read serves them from memory instead
// of running a query over the tables. The COUNT(*) scans (issue 27) and then
// the prefix distribution query (issue 30) each grew with the database until
// liveCounts holds the running row counts that the stats endpoints report.
// They are seeded once at startup from the tables and then adjusted on every
// write, so a stats read serves them from memory instead of running a query
// over the tables. The COUNT(*) scans (issue 27) grew with the database until
// they took the whole request timeout and made /api/v1/stats return 500.
//
// A single mutex guards all fields so the stats reader takes a consistent
@@ -23,20 +22,11 @@ type liveCounts struct {
prefixesV6 int
peerings int
peers int
routesV4 int
routesV6 int
// The prefix distribution: for each mask length, the number of distinct
// prefixes that have at least one live route.
distributionV4 [ipv4Bits + 1]int
distributionV6 [ipv6Bits + 1]int
}
// seed sets every count to the value read from the tables at startup. It runs
// before any writer, so it needs no coordination with the adjust methods.
func (c *liveCounts) seed(
asns, prefixesV4, prefixesV6, peerings, peers, routesV4, routesV6 int,
distributionV4, distributionV6 []PrefixDistribution,
) {
func (c *liveCounts) seed(asns, prefixesV4, prefixesV6, peerings, peers int) {
c.mu.Lock()
defer c.mu.Unlock()
@@ -45,16 +35,6 @@ func (c *liveCounts) seed(
c.prefixesV6 = prefixesV6
c.peerings = peerings
c.peers = peers
c.routesV4 = routesV4
c.routesV6 = routesV6
for _, entry := range distributionV4 {
addAtMaskLength(c.distributionV4[:], entry.MaskLength, entry.Count)
}
for _, entry := range distributionV6 {
addAtMaskLength(c.distributionV6[:], entry.MaskLength, entry.Count)
}
}
// addASNs adds n to the ASN count.
@@ -86,41 +66,7 @@ func (c *liveCounts) addPeers(n int) {
c.mu.Unlock()
}
// addRoutes adds to the IPv4 and IPv6 live-route counts. Deletions pass
// negative values.
func (c *liveCounts) addRoutes(v4, v6 int) {
c.mu.Lock()
c.routesV4 += v4
c.routesV6 += v6
c.mu.Unlock()
}
// addToDistribution adds n to the IPv4 and IPv6 prefix distributions once for
// each listed mask length. A write lists the mask lengths of the prefixes it
// gave their first live route with n = 1, and of the prefixes it left with no
// live route with n = -1.
func (c *liveCounts) addToDistribution(maskLengthsV4, maskLengthsV6 []int, n int) {
c.mu.Lock()
defer c.mu.Unlock()
for _, maskLength := range maskLengthsV4 {
addAtMaskLength(c.distributionV4[:], maskLength, n)
}
for _, maskLength := range maskLengthsV6 {
addAtMaskLength(c.distributionV6[:], maskLength, n)
}
}
// addAtMaskLength adds n to counts[maskLength]. A mask length the array has no
// entry for is ignored, so a malformed route cannot crash the daemon.
func addAtMaskLength(counts []int, maskLength, n int) {
if maskLength >= 0 && maskLength < len(counts) {
counts[maskLength] += n
}
}
// fill copies the counts into a Stats, including the derived totals, under a
// fill copies the counts into a Stats, including the derived total, under a
// single read lock so the reader sees one consistent snapshot.
func (c *liveCounts) fill(s *Stats) {
c.mu.RLock()
@@ -132,26 +78,6 @@ func (c *liveCounts) fill(s *Stats) {
s.Prefixes = c.prefixesV4 + c.prefixesV6
s.Peerings = c.peerings
s.Peers = c.peers
s.IPv4Routes = c.routesV4
s.IPv6Routes = c.routesV6
s.LiveRoutes = c.routesV4 + c.routesV6
s.IPv4PrefixDistribution = distributionList(c.distributionV4[:])
s.IPv6PrefixDistribution = distributionList(c.distributionV6[:])
}
// distributionList lists the mask lengths that have at least one prefix, in
// ascending order, the way the distribution query returns them.
func distributionList(counts []int) []PrefixDistribution {
var list []PrefixDistribution
for maskLength, count := range counts {
if count > 0 {
list = append(list,
PrefixDistribution{MaskLength: maskLength, Count: count})
}
}
return list
}
// countRows returns the number of rows in the named table. It is used only at
@@ -167,9 +93,8 @@ func (d *Database) countRows(ctx context.Context, table string) (int, error) {
return n, nil
}
// seedCounts reads the current row counts and prefix distribution from the
// tables into the in-memory counters. It runs once at startup, before the
// streamer begins writing.
// seedCounts reads the current row counts from the tables into the in-memory
// counters. It runs once at startup, before the streamer begins writing.
func (d *Database) seedCounts(ctx context.Context) error {
asns, err := d.countRows(ctx, "asns")
if err != nil {
@@ -196,23 +121,7 @@ func (d *Database) seedCounts(ctx context.Context) error {
return err
}
routesV4, err := d.countRows(ctx, tableLiveRoutesV4)
if err != nil {
return err
}
routesV6, err := d.countRows(ctx, tableLiveRoutesV6)
if err != nil {
return err
}
distributionV4, distributionV6, err := d.GetPrefixDistributionContext(ctx)
if err != nil {
return err
}
d.counts.seed(asns, prefixesV4, prefixesV6, peerings, peers, routesV4, routesV6,
distributionV4, distributionV6)
d.counts.seed(asns, prefixesV4, prefixesV6, peerings, peers)
return nil
}
internal/database/counts_internal_test.go
+25 -577
View File
@@ -3,62 +3,23 @@ package database
import (
"context"
"fmt"
"slices"
"sync"
"testing"
"time"
"git.eeqj.de/sneak/routewatch/internal/config"
"git.eeqj.de/sneak/routewatch/internal/logger"
"github.com/google/uuid"
)
// testV4PeerIP is the peer the IPv4 routes in these tests are learned from.
// testV4PeerIP is the peer the IPv4 rows in these tests are learned from.
const testV4PeerIP = "192.0.2.1"
// testV6Prefix is the prefix of the IPv6 routes in these tests.
// testV6Prefix is the IPv6 prefix of these tests.
const testV6Prefix = "2001:db8::/32"
// mkV4Route builds an IPv4 live route with its mask length taken from the
// prefix.
func mkV4Route(t *testing.T, prefix string, asn int, ts time.Time) *LiveRoute {
t.Helper()
maskLength, err := prefixMaskLength(prefix)
if err != nil {
t.Fatalf("prefixMaskLength(%s): %v", prefix, err)
}
return &LiveRoute{
Prefix: prefix,
MaskLength: maskLength,
IPVersion: ipVersionV4,
OriginASN: asn,
PeerIP: testV4PeerIP,
ASPath: []int{asn},
NextHop: "192.0.2.254",
LastUpdated: ts,
}
}
// mkV6Route builds an IPv6 live route.
func mkV6Route(prefix string, asn int, ts time.Time) *LiveRoute {
return &LiveRoute{
Prefix: prefix,
MaskLength: 32,
IPVersion: ipVersionV6,
OriginASN: asn,
PeerIP: "2001:db8::1",
ASPath: []int{asn},
NextHop: "2001:db8::ffff",
LastUpdated: ts,
}
}
// TestLiveCountsTrackWritesInRealtime checks that the stats counts start at
// zero, reflect each write the moment it commits (no recompute, no timer), do
// not move when a route is merely re-announced, and drop when a route is
// deleted. These counts are what /api/v1/stats reports; before this change the
// zero and reflect each write the moment it commits (no recompute, no timer).
// These counts are what /api/v1/stats reports; before this change the
// endpoint recomputed them with a COUNT(*) over each table on every request.
func TestLiveCountsTrackWritesInRealtime(t *testing.T) {
t.Parallel()
@@ -79,7 +40,7 @@ func TestLiveCountsTrackWritesInRealtime(t *testing.T) {
}
if empty.ASNs != 0 || empty.Prefixes != 0 || empty.Peerings != 0 ||
empty.Peers != 0 || empty.LiveRoutes != 0 {
empty.Peers != 0 {
t.Fatalf("empty database counts nonzero: %+v", empty)
}
@@ -87,17 +48,6 @@ func TestLiveCountsTrackWritesInRealtime(t *testing.T) {
writeASNsPrefixesAndPeers(t, db, ts)
routes := []*LiveRoute{
mkV4Route(t, "198.51.100.0/24", 64500, ts),
mkV4Route(t, "203.0.113.0/24", 64501, ts.Add(time.Minute)),
mkV6Route(testV6Prefix, 64502, ts.Add(2*time.Minute)),
}
err = db.UpsertLiveRouteBatch(routes)
if err != nil {
t.Fatalf("UpsertLiveRouteBatch: %v", err)
}
stats, err := db.GetStatsContext(ctx)
if err != nil {
t.Fatalf("GetStatsContext: %v", err)
@@ -105,46 +55,11 @@ func TestLiveCountsTrackWritesInRealtime(t *testing.T) {
assertCounts(t, "after inserts", stats, wantCounts{
asns: 2, prefixes: 2, peerings: 1, peers: 1,
ipv4Routes: 2, ipv6Routes: 1, liveRoutes: 3,
})
// Re-announcing the same routes is an update, not an insert: counts hold.
err = db.UpsertLiveRouteBatch(routes)
if err != nil {
t.Fatalf("UpsertLiveRouteBatch (re-announce): %v", err)
}
stats, err = db.GetStatsContext(ctx)
if err != nil {
t.Fatalf("GetStatsContext: %v", err)
}
assertCounts(t, "after re-announce", stats, wantCounts{
asns: 2, prefixes: 2, peerings: 1, peers: 1,
ipv4Routes: 2, ipv6Routes: 1, liveRoutes: 3,
})
// A withdrawal removes one route.
err = db.DeleteLiveRouteBatch([]LiveRouteDeletion{
{Prefix: "203.0.113.0/24", PeerIP: testV4PeerIP, IPVersion: ipVersionV4},
})
if err != nil {
t.Fatalf("DeleteLiveRouteBatch: %v", err)
}
stats, err = db.GetStatsContext(ctx)
if err != nil {
t.Fatalf("GetStatsContext: %v", err)
}
assertCounts(t, "after delete", stats, wantCounts{
asns: 2, prefixes: 2, peerings: 1, peers: 1,
ipv4Routes: 1, ipv6Routes: 1, liveRoutes: 2,
})
}
// writeASNsPrefixesAndPeers writes two ASNs, two prefixes, one peer and one
// peering: the rows besides live routes that the stats count.
// peering: the rows the stats count.
func writeASNsPrefixesAndPeers(t *testing.T, db *Database, ts time.Time) {
t.Helper()
@@ -180,8 +95,8 @@ func writeASNsPrefixesAndPeers(t *testing.T, db *Database, ts time.Time) {
}
// TestLiveCountsSeededFromDatabaseAtStartup writes rows, reopens the same
// database file, and checks the counts and the prefix distribution come back
// from the seed scan rather than starting at zero.
// database file, and checks the counts come back from the seed scan rather
// than starting at zero.
func TestLiveCountsSeededFromDatabaseAtStartup(t *testing.T) {
t.Parallel()
@@ -192,23 +107,7 @@ func TestLiveCountsSeededFromDatabaseAtStartup(t *testing.T) {
t.Fatalf("failed to create database: %v", err)
}
ts := time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC)
seen := SeenTimes{First: ts, Last: ts}
err = db.GetOrCreateASNBatch(
map[int]SeenTimes{64500: seen, 64501: seen, 64502: seen})
if err != nil {
t.Fatalf("GetOrCreateASNBatch: %v", err)
}
err = db.UpsertLiveRouteBatch([]*LiveRoute{
mkV4Route(t, "198.51.100.0/24", 64500, ts),
mkV6Route(testV6Prefix, 64502, ts),
})
if err != nil {
t.Fatalf("UpsertLiveRouteBatch: %v", err)
}
writeASNsPrefixesAndPeers(t, db, time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC))
err = db.Close()
if err != nil {
@@ -226,435 +125,9 @@ func TestLiveCountsSeededFromDatabaseAtStartup(t *testing.T) {
t.Fatalf("GetStatsContext after reopen: %v", err)
}
if stats.ASNs != 3 {
t.Errorf("seeded ASNs = %d, want 3", stats.ASNs)
}
if stats.IPv4Routes != 1 || stats.IPv6Routes != 1 || stats.LiveRoutes != 2 {
t.Errorf("seeded routes = (v4 %d, v6 %d, total %d), want (1, 1, 2)",
stats.IPv4Routes, stats.IPv6Routes, stats.LiveRoutes)
}
assertDistribution(t, "seeded IPv4 distribution", stats.IPv4PrefixDistribution,
[]PrefixDistribution{{MaskLength: 24, Count: 1}})
assertDistribution(t, "seeded IPv6 distribution", stats.IPv6PrefixDistribution,
[]PrefixDistribution{{MaskLength: 32, Count: 1}})
}
// TestPrefixDistributionTracksWrites checks that the prefix distribution the
// stats read reports stays exact across each kind of live-route write, and that
// after every step it equals what the distribution query reads from the route
// tables. The steps run once through the batch methods the prefix handler uses
// and once through the single-route methods.
func TestPrefixDistributionTracksWrites(t *testing.T) {
t.Parallel()
steps := prefixDistributionSteps(t)
for _, batch := range []bool{true, false} {
name := "single-route writes"
if batch {
name = "batch writes"
}
t.Run(name, func(t *testing.T) {
t.Parallel()
db, err := New(&config.Config{StateDir: t.TempDir()}, logger.New())
if err != nil {
t.Fatalf("failed to create database: %v", err)
}
defer func() { _ = db.Close() }()
ctx := context.Background()
for _, step := range steps {
err := announceRoutes(db, batch, step.announce)
if err != nil {
t.Fatalf("%s: announce: %v", step.name, err)
}
err = withdrawRoutes(db, batch, step.withdraw)
if err != nil {
t.Fatalf("%s: withdraw: %v", step.name, err)
}
stats, err := db.GetStatsContext(ctx)
if err != nil {
t.Fatalf("%s: GetStatsContext: %v", step.name, err)
}
queryV4, queryV6, err := db.GetPrefixDistributionContext(ctx)
if err != nil {
t.Fatalf("%s: GetPrefixDistributionContext: %v", step.name, err)
}
assertDistribution(t, step.name+": IPv4 distribution",
stats.IPv4PrefixDistribution, step.wantV4)
assertDistribution(t, step.name+": IPv6 distribution",
stats.IPv6PrefixDistribution, step.wantV6)
assertDistribution(t, step.name+": IPv4 distribution query",
queryV4, step.wantV4)
assertDistribution(t, step.name+": IPv6 distribution query",
queryV6, step.wantV6)
}
})
}
}
// prefixDistributionStep is one step of TestPrefixDistributionTracksWrites:
// the routes it announces, then the routes it withdraws, then the
// distributions both the stats and the distribution query must report.
type prefixDistributionStep struct {
name string
announce []*LiveRoute
withdraw []*LiveRoute
wantV4 []PrefixDistribution
wantV6 []PrefixDistribution
}
// prefixDistributionSteps returns the steps of
// TestPrefixDistributionTracksWrites, in the order they run.
func prefixDistributionSteps(t *testing.T) []prefixDistributionStep {
t.Helper()
ts := time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC)
shared := mkV4Route(t, "198.51.100.0/24", 64500, ts)
sharedSecondPeer := mkV4Route(t, "198.51.100.0/24", 64500, ts)
sharedSecondPeer.PeerIP = "192.0.2.2"
other := mkV4Route(t, "203.0.113.0/24", 64501, ts)
wide := mkV4Route(t, "172.16.0.0/16", 64502, ts)
v6 := mkV6Route(testV6Prefix, 64503, ts)
v6SecondPeer := mkV6Route(testV6Prefix, 64503, ts)
v6SecondPeer.PeerIP = "2001:db8::2"
all := []PrefixDistribution{{MaskLength: 16, Count: 1}, {MaskLength: 24, Count: 2}}
v6Only := []PrefixDistribution{{MaskLength: 32, Count: 1}}
return []prefixDistributionStep{
{
name: "new routes",
announce: []*LiveRoute{shared, other, wide, v6},
wantV4: all, wantV6: v6Only,
},
{
name: "re-announcement",
announce: []*LiveRoute{shared, other, wide, v6},
wantV4: all, wantV6: v6Only,
},
{
name: "second peer announces a prefix that has a live route",
announce: []*LiveRoute{sharedSecondPeer, v6SecondPeer},
wantV4: all, wantV6: v6Only,
},
{
name: "withdrawal of a route that is not the last for its prefix",
withdraw: []*LiveRoute{shared, v6},
wantV4: all, wantV6: v6Only,
},
{
name: "withdrawal of the last route for a prefix",
withdraw: []*LiveRoute{sharedSecondPeer, v6SecondPeer},
wantV4: []PrefixDistribution{
{MaskLength: 16, Count: 1}, {MaskLength: 24, Count: 1},
},
},
{
name: "withdrawal of every remaining route",
withdraw: []*LiveRoute{other, wide},
},
{
name: "two peers announce a new prefix together",
announce: []*LiveRoute{shared, sharedSecondPeer, v6, v6SecondPeer},
wantV4: []PrefixDistribution{{MaskLength: 24, Count: 1}},
wantV6: v6Only,
},
{
name: "both routes for a prefix withdrawn together",
withdraw: []*LiveRoute{shared, sharedSecondPeer, v6, v6SecondPeer},
},
// The feed often withdraws a route that is not live. That must not take
// the prefix out of the distribution. A count wrongly taken below zero is
// left out of the answer, so the next step shows it: its announcement
// at the same mask length would then not be counted.
{
name: "withdrawal of routes that are not live",
withdraw: []*LiveRoute{other, v6},
},
{
name: "announcement after a withdrawal of routes that are not live",
announce: []*LiveRoute{other, v6},
wantV4: []PrefixDistribution{{MaskLength: 24, Count: 1}},
wantV6: v6Only,
},
}
}
// announceRoutes writes routes in one UpsertLiveRouteBatch, or with one
// UpsertLiveRoute each.
func announceRoutes(db *Database, batch bool, routes []*LiveRoute) error {
if batch {
return db.UpsertLiveRouteBatch(routes)
}
for _, route := range routes {
err := db.UpsertLiveRoute(route)
if err != nil {
return err
}
}
return nil
}
// withdrawRoutes removes routes in one DeleteLiveRouteBatch, or with one
// DeleteLiveRoute each.
func withdrawRoutes(db *Database, batch bool, routes []*LiveRoute) error {
if !batch {
for _, route := range routes {
err := db.DeleteLiveRoute(route.Prefix, route.PeerIP)
if err != nil {
return err
}
}
return nil
}
deletions := make([]LiveRouteDeletion, 0, len(routes))
for _, route := range routes {
deletions = append(deletions, LiveRouteDeletion{
Prefix: route.Prefix, PeerIP: route.PeerIP, IPVersion: route.IPVersion,
})
}
return db.DeleteLiveRouteBatch(deletions)
}
// TestDeleteLiveRoutesOfPeer gives two peers IPv4 and IPv6 live routes, some
// for the same prefixes, and deletes all of one peer's routes. The other peer's
// routes stay, and the route counts and prefix distribution held in memory
// equal what is read from the route tables.
func TestDeleteLiveRoutesOfPeer(t *testing.T) {
t.Parallel()
const (
peerIP = "192.0.2.1"
otherPeerIP = "192.0.2.2"
sharedV4 = "198.51.100.0/24"
sharedV6 = "2001:db8::/32"
)
db, err := New(&config.Config{StateDir: t.TempDir()}, logger.New())
if err != nil {
t.Fatalf("failed to create database: %v", err)
}
defer func() { _ = db.Close() }()
ts := time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC)
ofPeer := func(peer string, route *LiveRoute) *LiveRoute {
route.PeerIP = peer
return route
}
onlyV6 := mkV6Route("2001:db8:1::/48", 64503, ts)
onlyV6.MaskLength = 48
err = db.UpsertLiveRouteBatch([]*LiveRoute{
ofPeer(peerIP, mkV4Route(t, sharedV4, 64500, ts)),
ofPeer(otherPeerIP, mkV4Route(t, sharedV4, 64500, ts)),
ofPeer(peerIP, mkV4Route(t, "203.0.113.0/24", 64501, ts)),
ofPeer(peerIP, mkV6Route(sharedV6, 64502, ts)),
ofPeer(otherPeerIP, mkV6Route(sharedV6, 64502, ts)),
ofPeer(peerIP, onlyV6),
assertCounts(t, "after reopen", stats, wantCounts{
asns: 2, prefixes: 2, peerings: 1, peers: 1,
})
if err != nil {
t.Fatalf("UpsertLiveRouteBatch: %v", err)
}
// peerIP has two IPv4 and two IPv6 routes; a second delete finds none.
for _, want := range []int{4, 0} {
deleted, err := db.DeleteLiveRoutesOfPeer(peerIP)
if err != nil {
t.Fatalf("DeleteLiveRoutesOfPeer: %v", err)
}
if deleted != want {
t.Errorf("DeleteLiveRoutesOfPeer deleted %d routes, want %d",
deleted, want)
}
}
for _, prefix := range []string{sharedV4, sharedV6} {
routes, _, err := db.GetPrefixDetailsContext(context.Background(), prefix)
if err != nil {
t.Fatalf("GetPrefixDetailsContext(%s): %v", prefix, err)
}
if len(routes) != 1 || routes[0].PeerIP != otherPeerIP {
t.Errorf("live routes for %s = %+v, want only the one from %s",
prefix, routes, otherPeerIP)
}
}
checkOneLiveRoutePerIPVersion(t, db)
}
// TestDeleteLiveRoutesOfPeerDeletesInChunks gives a peer one IPv4 route more
// than a chunk holds, two of them for prefixes another peer also has. One chunk
// deletes peerRouteChunkSize of the peer's routes and commits, leaving the last
// one. The routes are written again, and DeleteLiveRoutesOfPeer deletes all of
// them, a chunk and then the last one. After each step the route count and
// prefix distribution held in memory and those read from the route table are
// the ones wanted.
func TestDeleteLiveRoutesOfPeerDeletesInChunks(t *testing.T) {
t.Parallel()
const (
peerIP = "192.0.2.1"
otherPeerIP = "192.0.2.2"
// How many of the peer's prefixes the other peer also has, the first
// ones in prefix order.
sharedPrefixes = 2
)
db, err := New(&config.Config{StateDir: t.TempDir()}, logger.New())
if err != nil {
t.Fatalf("failed to create database: %v", err)
}
defer func() { _ = db.Close() }()
ts := time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC)
peerRoutes := make([]*LiveRoute, 0, peerRouteChunkSize+1)
otherPeerRoutes := make([]*LiveRoute, 0, sharedPrefixes)
for i := range peerRouteChunkSize + 1 {
prefix := fmt.Sprintf("10.%d.%d.0/24", i/256, i%256)
route := mkV4Route(t, prefix, 64500, ts)
route.PeerIP = peerIP
peerRoutes = append(peerRoutes, route)
if i < sharedPrefixes {
route := mkV4Route(t, prefix, 64500, ts)
route.PeerIP = otherPeerIP
otherPeerRoutes = append(otherPeerRoutes, route)
}
}
err = db.UpsertLiveRouteBatch(slices.Concat(peerRoutes, otherPeerRoutes))
if err != nil {
t.Fatalf("UpsertLiveRouteBatch: %v", err)
}
deletedV4, deletedV6, err := db.deletePeerRouteChunk(peerIP)
if err != nil {
t.Fatalf("deletePeerRouteChunk: %v", err)
}
if deletedV4 != peerRouteChunkSize || deletedV6 != 0 {
t.Errorf("deletePeerRouteChunk deleted (%d, %d) routes, want (%d, 0)",
deletedV4, deletedV6, peerRouteChunkSize)
}
// The shared prefixes went with the chunk, so each route left has a
// prefix of its own.
checkV4RoutesOfOwnPrefixes(t, db, "after one chunk", 1+sharedPrefixes)
err = db.UpsertLiveRouteBatch(peerRoutes)
if err != nil {
t.Fatalf("UpsertLiveRouteBatch: %v", err)
}
deleted, err := db.DeleteLiveRoutesOfPeer(peerIP)
if err != nil {
t.Fatalf("DeleteLiveRoutesOfPeer: %v", err)
}
if deleted != len(peerRoutes) {
t.Errorf("DeleteLiveRoutesOfPeer deleted %d routes, want %d",
deleted, len(peerRoutes))
}
checkV4RoutesOfOwnPrefixes(t, db, "after DeleteLiveRoutesOfPeer",
sharedPrefixes)
}
// checkV4RoutesOfOwnPrefixes checks that the database has wantRoutes IPv4 live
// routes, each for a /24 no other live route has, both as counted in memory and
// as read from the route table. when says which step of the test checks.
func checkV4RoutesOfOwnPrefixes(
t *testing.T, db *Database, when string, wantRoutes int,
) {
t.Helper()
ctx := context.Background()
stats, err := db.GetStatsContext(ctx)
if err != nil {
t.Fatalf("GetStatsContext: %v", err)
}
tableRoutes, _, err := db.GetLiveRouteCountsContext(ctx)
if err != nil {
t.Fatalf("GetLiveRouteCountsContext: %v", err)
}
if stats.IPv4Routes != wantRoutes || tableRoutes != wantRoutes {
t.Errorf("%s: IPv4 live routes: %d in memory, %d in the table, want %d",
when, stats.IPv4Routes, tableRoutes, wantRoutes)
}
queryDistribution, _, err := db.GetPrefixDistributionContext(ctx)
if err != nil {
t.Fatalf("GetPrefixDistributionContext: %v", err)
}
want := []PrefixDistribution{{MaskLength: 24, Count: wantRoutes}}
assertDistribution(t, when+": IPv4 distribution",
stats.IPv4PrefixDistribution, want)
assertDistribution(t, when+": IPv4 distribution query",
queryDistribution, want)
}
// checkOneLiveRoutePerIPVersion checks that the database has one IPv4 live
// route, for a /24, and one IPv6 live route, for a /32, both as counted in
// memory and as read from the route tables.
func checkOneLiveRoutePerIPVersion(t *testing.T, db *Database) {
t.Helper()
ctx := context.Background()
stats, err := db.GetStatsContext(ctx)
if err != nil {
t.Fatalf("GetStatsContext: %v", err)
}
tableV4, tableV6, err := db.GetLiveRouteCountsContext(ctx)
if err != nil {
t.Fatalf("GetLiveRouteCountsContext: %v", err)
}
if stats.IPv4Routes != 1 || tableV4 != 1 || stats.IPv6Routes != 1 ||
tableV6 != 1 {
t.Errorf("live routes (IPv4, IPv6): (%d, %d) in memory, (%d, %d) in "+
"the tables, want (1, 1)",
stats.IPv4Routes, stats.IPv6Routes, tableV4, tableV6)
}
queryV4, queryV6, err := db.GetPrefixDistributionContext(ctx)
if err != nil {
t.Fatalf("GetPrefixDistributionContext: %v", err)
}
wantV4 := []PrefixDistribution{{MaskLength: 24, Count: 1}}
wantV6 := []PrefixDistribution{{MaskLength: 32, Count: 1}}
assertDistribution(t, "IPv4 distribution", stats.IPv4PrefixDistribution, wantV4)
assertDistribution(t, "IPv6 distribution", stats.IPv6PrefixDistribution, wantV6)
assertDistribution(t, "IPv4 distribution query", queryV4, wantV4)
assertDistribution(t, "IPv6 distribution query", queryV6, wantV6)
}
// TestLiveCountsConcurrentReadWrite runs writers and stats readers at once so
@@ -672,7 +145,10 @@ func TestLiveCountsConcurrentReadWrite(t *testing.T) {
ts := time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC)
const writers = 4
const (
writers = 4
prefixesPerWrite = 25
)
var wg sync.WaitGroup
wg.Add(writers)
@@ -681,14 +157,12 @@ func TestLiveCountsConcurrentReadWrite(t *testing.T) {
go func(base int) {
defer wg.Done()
for i := range 25 {
asn := 65000 + base*100 + i
route := mkV6Route(testV6Prefix, asn, ts)
route.PeerIP = "2001:db8::" + uuid.NewString()
for i := range prefixesPerWrite {
prefix := fmt.Sprintf("10.%d.%d.0/24", base, i)
err := db.UpsertLiveRoute(route)
err := db.UpdatePrefixesBatch(map[string]time.Time{prefix: ts})
if err != nil {
t.Errorf("UpsertLiveRoute: %v", err)
t.Errorf("UpdatePrefixesBatch: %v", err)
return
}
@@ -728,22 +202,16 @@ func TestLiveCountsConcurrentReadWrite(t *testing.T) {
t.Fatalf("GetStatsContext: %v", err)
}
if want := writers * 25; stats.IPv6Routes != want {
t.Errorf("IPv6Routes = %d, want %d", stats.IPv6Routes, want)
if want := writers * prefixesPerWrite; stats.IPv4Prefixes != want {
t.Errorf("IPv4Prefixes = %d, want %d", stats.IPv4Prefixes, want)
}
// Every writer announced the same prefix, so it counts once.
assertDistribution(t, "IPv6 distribution", stats.IPv6PrefixDistribution,
[]PrefixDistribution{{MaskLength: 32, Count: 1}})
}
type wantCounts struct {
asns int
prefixes int
peerings int
peers int
ipv4Routes int
ipv6Routes int
liveRoutes int
asns int
prefixes int
peerings int
peers int
}
func assertCounts(t *testing.T, when string, got Stats, want wantCounts) {
@@ -764,24 +232,4 @@ func assertCounts(t *testing.T, when string, got Stats, want wantCounts) {
if got.Peers != want.peers {
t.Errorf("%s: Peers = %d, want %d", when, got.Peers, want.peers)
}
if got.IPv4Routes != want.ipv4Routes {
t.Errorf("%s: IPv4Routes = %d, want %d", when, got.IPv4Routes, want.ipv4Routes)
}
if got.IPv6Routes != want.ipv6Routes {
t.Errorf("%s: IPv6Routes = %d, want %d", when, got.IPv6Routes, want.ipv6Routes)
}
if got.LiveRoutes != want.liveRoutes {
t.Errorf("%s: LiveRoutes = %d, want %d", when, got.LiveRoutes, want.liveRoutes)
}
}
func assertDistribution(t *testing.T, what string, got, want []PrefixDistribution) {
t.Helper()
if !slices.Equal(got, want) {
t.Errorf("%s = %v, want %v", what, got, want)
}
}
internal/database/database.go
+15 -1454
View File
File diff suppressed because it is too large Load Diff
internal/database/database_internal_test.go
+55 -1270
View File
File diff suppressed because it is too large Load Diff
internal/database/interface.go
+14 -47
View File
@@ -1,7 +1,6 @@
// Package database provides SQLite storage for BGP routing data including ASNs,
// prefixes, announcements, peerings, and live route tables. Its routes table
// holds a copy of the whole routing table that internal/routetable keeps in
// memory.
// Package database provides SQLite storage for BGP routing data: ASNs,
// prefixes, announcements, peerings and BGP peers. Its routes table holds a
// copy of the whole routing table that internal/routetable keeps in memory.
package database
import (
@@ -12,26 +11,21 @@ import (
"git.eeqj.de/sneak/routewatch/internal/routetable"
)
// Stats contains database statistics including counts for ASNs, prefixes,
// peerings, peers, and live routes, as well as file size and prefix distribution data.
// Stats contains database statistics: counts of ASNs, prefixes, peerings and
// peers, and the file size.
type Stats struct {
ASNs int
Prefixes int
IPv4Prefixes int
IPv6Prefixes int
Peerings int
Peers int
FileSizeBytes int64
LiveRoutes int
IPv4Routes int
IPv6Routes int
IPv4PrefixDistribution []PrefixDistribution
IPv6PrefixDistribution []PrefixDistribution
ASNs int
Prefixes int
IPv4Prefixes int
IPv6Prefixes int
Peerings int
Peers int
FileSizeBytes int64
}
// Store defines the interface for database operations. It provides methods for
// managing ASNs, prefixes, announcements, peerings, BGP peers, and live routes.
// Implementations must be safe for concurrent use.
// managing ASNs, prefixes, announcements, peerings, BGP peers, and the copy of
// the routing table. Implementations must be safe for concurrent use.
//
//nolint:interfacebloat // the one seam handlers and tests depend on
type Store interface {
@@ -57,27 +51,10 @@ type Store interface {
UpdatePeer(peerIP string, peerASN int, messageType string, timestamp time.Time) error
UpdatePeerBatch(peers map[string]PeerUpdate) error
// Live route operations
UpsertLiveRoute(route *LiveRoute) error
UpsertLiveRouteBatch(routes []*LiveRoute) error
DeleteLiveRoute(prefix, peerIP string) error
DeleteLiveRouteBatch(deletions []LiveRouteDeletion) error
DeleteLiveRoutesOfPeer(peerIP string) (deleted int, err error)
GetPrefixDistribution() (
ipv4 []PrefixDistribution, ipv6 []PrefixDistribution, err error)
GetPrefixDistributionContext(ctx context.Context) (
ipv4 []PrefixDistribution, ipv6 []PrefixDistribution, err error)
GetLiveRouteCounts() (ipv4Count, ipv6Count int, err error)
GetLiveRouteCountsContext(ctx context.Context) (ipv4Count, ipv6Count int, err error)
// Routing table operations: the database's copy of the table in memory
WriteRouteChanges(changes []routetable.Change, newPeers map[string]uint) error
ReadRouteTable(ctx context.Context) (*routetable.Table, error)
// IP lookup operations
GetIPInfo(ip string) (*IPInfo, error)
GetIPInfoContext(ctx context.Context, ip string) (*IPInfo, error)
// ASN WHOIS operations
GetNextStaleASN(ctx context.Context, staleThreshold time.Duration) (int, error)
UpdateASNWHOIS(ctx context.Context, update *ASNWHOISUpdate) error
@@ -87,18 +64,8 @@ type Store interface {
GetAS(ctx context.Context, asn int) (*ASN, bool, error)
GetASWHOIS(ctx context.Context, asn int) (*ASN, error)
GetPrefixFirstSeen(ctx context.Context, prefix netip.Prefix) (time.Time, error)
GetASDetails(asn int) (*ASN, []LiveRoute, error)
GetASDetailsContext(ctx context.Context, asn int) (*ASN, []LiveRoute, error)
GetASPeers(asn int) ([]ASPeer, error)
GetASPeersContext(ctx context.Context, asn int) ([]ASPeer, error)
GetPrefixDetails(prefix string) ([]LiveRoute, map[int]ASN, error)
GetPrefixDetailsContext(
ctx context.Context, prefix string,
) ([]LiveRoute, map[int]ASN, error)
GetRandomPrefixesByLength(maskLength, ipVersion, limit int) ([]LiveRoute, error)
GetRandomPrefixesByLengthContext(
ctx context.Context, maskLength, ipVersion, limit int,
) ([]LiveRoute, error)
// Lifecycle
Close() error
internal/database/models.go
-59
View File
@@ -66,65 +66,6 @@ type ASNPeering struct {
LastSeen time.Time
}
// LiveRoute represents a route in the live routing table. JSON leaves out an
// empty AS path, next hop or time: the routing table in memory keeps them only
// for the two newest routes of each prefix and origin AS.
//
//nolint:tagliatelle // field names are the existing JSON API
type LiveRoute struct {
Prefix string `json:"prefix"`
MaskLength int `json:"mask_length"`
IPVersion int `json:"ip_version"`
OriginASN int `json:"origin_asn"`
PeerIP string `json:"peer_ip"`
ASPath []int `json:"as_path,omitempty"`
NextHop string `json:"next_hop,omitempty"`
LastUpdated time.Time `json:"last_updated,omitzero"`
}
// PrefixDistribution represents the distribution of prefixes by mask length
//
//nolint:tagliatelle // field names are the existing JSON API
type PrefixDistribution struct {
MaskLength int `json:"mask_length"`
Count int `json:"count"`
}
// IPInfo represents comprehensive IP information for the /ip endpoint
//
//nolint:tagliatelle // field names are the existing JSON API
type IPInfo struct {
IP string `json:"ip"`
Netblock string `json:"netblock"`
MaskLength int `json:"mask_length"`
IPVersion int `json:"ip_version"`
NumPeers int `json:"num_peers"`
// AS information
ASN int `json:"asn"`
ASName string `json:"as_name,omitempty"`
Handle string `json:"handle,omitempty"`
Description string `json:"description,omitempty"`
OrgName string `json:"org_name,omitempty"`
OrgID string `json:"org_id,omitempty"`
Address string `json:"address,omitempty"`
CountryCode string `json:"country_code,omitempty"`
AbuseEmail string `json:"abuse_email,omitempty"`
RIR string `json:"rir,omitempty"`
// Timestamps. LastSeen is zero, and left out of JSON, when the prefix
// keeps no route.
FirstSeen time.Time `json:"first_seen"`
LastSeen time.Time `json:"last_seen,omitzero"`
// Indicates if WHOIS data needs refresh (not serialized)
NeedsWHOISRefresh bool `json:"-"`
}
// LiveRouteDeletion represents parameters for deleting a live route
type LiveRouteDeletion struct {
Prefix string
PeerIP string
IPVersion int
}
// SeenTimes is the earliest and the latest time one batch of updates saw an AS,
// a peer or a peering. A row the batch adds is stored first seen at First and
// last seen at Last. A row already stored takes First as its first_seen only
internal/database/schema.sql
-59
View File
@@ -89,65 +89,6 @@ CREATE INDEX IF NOT EXISTS idx_bgp_peers_asn ON bgp_peers(peer_asn);
CREATE INDEX IF NOT EXISTS idx_bgp_peers_last_seen ON bgp_peers(last_seen);
CREATE INDEX IF NOT EXISTS idx_bgp_peers_ip ON bgp_peers(peer_ip);
-- IPv4 routing table maintained by PrefixHandler. A route write changes the
-- table and its entry in every index on a column the write sets, so the table
-- has only the indexes that the queries named with them use.
CREATE TABLE IF NOT EXISTS live_routes_v4 (
-- The row id, which the prefix-length pages' sample draws through the
-- mask_length index. Declared as a column, it stays the same for as long
-- as the route is live, also through a VACUUM.
id INTEGER PRIMARY KEY,
prefix TEXT NOT NULL,
mask_length INTEGER NOT NULL, -- CIDR mask length (0-32)
origin_asn INTEGER NOT NULL,
peer_ip TEXT NOT NULL,
as_path TEXT NOT NULL, -- JSON array
next_hop TEXT NOT NULL,
last_updated DATETIME NOT NULL,
-- A peer has at most one route for a prefix. This key finds the route a
-- write updates or deletes and, peer_ip coming first, all of a peer's
-- routes when its BGP session goes down.
UNIQUE(peer_ip, prefix)
);
-- IPv6 routing table maintained by PrefixHandler, as live_routes_v4.
CREATE TABLE IF NOT EXISTS live_routes_v6 (
id INTEGER PRIMARY KEY,
prefix TEXT NOT NULL,
mask_length INTEGER NOT NULL, -- CIDR mask length (0-128)
origin_asn INTEGER NOT NULL,
peer_ip TEXT NOT NULL,
as_path TEXT NOT NULL, -- JSON array
next_hop TEXT NOT NULL,
last_updated DATETIME NOT NULL,
UNIQUE(peer_ip, prefix)
);
-- Indexes for both live route tables, and the queries that use each.
--
-- prefix, last_updated: a prefix's routes, for the prefix page and its API
-- answer, the IP lookup and its peer count, and the check whether a prefix
-- still has a live route; and a prefix's newest route for the prefix-length
-- pages and the AS page, in one search whatever the number of peers.
CREATE INDEX IF NOT EXISTS idx_live_routes_v4_prefix_last_updated ON live_routes_v4(prefix, last_updated);
CREATE INDEX IF NOT EXISTS idx_live_routes_v6_prefix_last_updated ON live_routes_v6(prefix, last_updated);
-- mask_length: the prefix-length pages' random sample, the row id range of a
-- mask length and the first route of it at or after a row id.
CREATE INDEX IF NOT EXISTS idx_live_routes_v4_mask_length ON live_routes_v4(mask_length);
CREATE INDEX IF NOT EXISTS idx_live_routes_v6_mask_length ON live_routes_v6(mask_length);
-- mask_length, prefix: the prefix-length pages' list of a mask length's
-- prefixes in text order, and the prefix distribution read at startup.
CREATE INDEX IF NOT EXISTS idx_live_routes_v4_mask_prefix ON live_routes_v4(mask_length, prefix);
CREATE INDEX IF NOT EXISTS idx_live_routes_v6_mask_prefix ON live_routes_v6(mask_length, prefix);
-- origin_asn, prefix: the prefixes an AS originates, each in one search
-- whatever the number of peers, for the AS page and its API answer.
CREATE INDEX IF NOT EXISTS idx_live_routes_v4_origin_prefix ON live_routes_v4(origin_asn, prefix);
CREATE INDEX IF NOT EXISTS idx_live_routes_v6_origin_prefix ON live_routes_v6(origin_asn, prefix);
-- last_updated: the oldest and newest route times in /api/v1/stats and
-- /status.json.
CREATE INDEX IF NOT EXISTS idx_live_routes_v4_last_updated ON live_routes_v4(last_updated);
CREATE INDEX IF NOT EXISTS idx_live_routes_v6_last_updated ON live_routes_v6(last_updated);
-- The whole routing table, as internal/routetable keeps it in memory: one row
-- for each prefix and origin AS that some peer carries. Startup reads every
-- row, and the rows of the pairs that change are written back.
internal/database/utils.go
-14
View File
@@ -1,8 +1,6 @@
package database
import (
"fmt"
"net"
"strings"
"github.com/google/uuid"
@@ -20,15 +18,3 @@ func detectIPVersion(prefix string) int {
return ipVersionV4
}
// prefixMaskLength returns the mask length of a prefix such as 192.0.2.0/24.
func prefixMaskLength(prefix string) (int, error) {
_, network, err := net.ParseCIDR(prefix)
if err != nil {
return 0, fmt.Errorf("invalid prefix %s: %w", prefix, err)
}
maskLength, _ := network.Mask.Size()
return maskLength, nil
}
internal/routewatch/app.go
+3 -4
View File
@@ -109,8 +109,9 @@ func (rw *RouteWatch) Run(ctx context.Context) error {
rw.peerHandler = NewPeerHandler(rw.db, rw.logger)
rw.streamer.RegisterHandler(rw.peerHandler)
// PrefixHandler maintains the prefixes and live_routes tables, and
// the routing table in memory and its copy in the routes table
// PrefixHandler keeps the routing table in memory current, and writes
// its changes to the routes table and new prefixes to the prefix
// tables
rw.prefixHandler = NewPrefixHandler(rw.db, rw.logger, rw.table)
rw.prefixHandler.SetMetricsTracker(rw.streamer.GetMetricsTracker())
rw.streamer.RegisterHandler(rw.prefixHandler)
@@ -125,8 +126,6 @@ func (rw *RouteWatch) Run(ctx context.Context) error {
return errNonBatchedNotImplemented
}
// No longer need routing table handler - PrefixHandler maintains live_routes table
// Start streaming
err := rw.streamer.Start(ctx)
if err != nil {
internal/routewatch/app_integration_internal_test.go
+7 -157
View File
@@ -33,14 +33,12 @@ type mockStore struct {
ASNCount int
PrefixCount int
PeeringCount int
RouteCount int
WithdrawalCount int
// Track unique items
ASNs map[int]*database.ASN
Prefixes map[string]*database.Prefix
Peerings map[string]bool // key is "from_to"
Routes map[string]bool // key is "prefix_origin_peer"
// TablePrefixes holds the prefixes that the written changes of the
// routing table leave some peer carrying.
TablePrefixes map[string]bool
@@ -56,7 +54,6 @@ func newMockStore() *mockStore {
ASNs: make(map[int]*database.ASN),
Prefixes: make(map[string]*database.Prefix),
Peerings: make(map[string]bool),
Routes: make(map[string]bool),
TablePrefixes: make(map[string]bool),
}
}
@@ -107,6 +104,13 @@ func (m *mockStore) UpdatePrefixesBatch(prefixes map[string]time.Time) error {
IPVersion: ipVersion,
FirstSeen: timestamp,
}
m.PrefixCount++
if ipVersion == ipVersionV4 {
m.IPv4Prefixes++
} else {
m.IPv6Prefixes++
}
}
}
@@ -207,65 +211,6 @@ func (m *mockStore) GetStatsContext(_ context.Context) (database.Stats, error) {
return m.GetStats()
}
// UpsertLiveRoute mock implementation
func (m *mockStore) UpsertLiveRoute(_ *database.LiveRoute) error {
// Simple mock - just return nil
return nil
}
// DeleteLiveRoute mock implementation
func (m *mockStore) DeleteLiveRoute(_, _ string) error {
// Simple mock - just return nil
return nil
}
// GetPrefixDistribution mock implementation
func (m *mockStore) GetPrefixDistribution() (
[]database.PrefixDistribution, []database.PrefixDistribution, error,
) {
// Return empty distributions for now
return nil, nil, nil
}
// GetPrefixDistributionContext mock implementation with context support
func (m *mockStore) GetPrefixDistributionContext(_ context.Context) (
[]database.PrefixDistribution, []database.PrefixDistribution, error,
) {
return m.GetPrefixDistribution()
}
// GetLiveRouteCounts mock implementation
func (m *mockStore) GetLiveRouteCounts() (int, int, error) {
// Return mock counts
return m.RouteCount / 2, m.RouteCount / 2, nil
}
// GetLiveRouteCountsContext mock implementation with context support
func (m *mockStore) GetLiveRouteCountsContext(_ context.Context) (int, int, error) {
return m.GetLiveRouteCounts()
}
// GetASDetails mock implementation
func (m *mockStore) GetASDetails(asn int) (*database.ASN, []database.LiveRoute, error) {
m.mu.Lock()
defer m.mu.Unlock()
// Check if ASN exists
if asnInfo, exists := m.ASNs[asn]; exists {
// Return empty prefixes for now
return asnInfo, []database.LiveRoute{}, nil
}
return nil, nil, database.ErrNoRoute
}
// GetASDetailsContext mock implementation with context support
func (m *mockStore) GetASDetailsContext(
_ context.Context, asn int,
) (*database.ASN, []database.LiveRoute, error) {
return m.GetASDetails(asn)
}
// GetAS returns the stored AS, or only its number when none is stored.
func (m *mockStore) GetAS(_ context.Context, asn int) (*database.ASN, bool, error) {
m.mu.Lock()
@@ -292,35 +237,6 @@ func (m *mockStore) GetPrefixFirstSeen(
return time.Time{}, nil
}
// GetPrefixDetails mock implementation
func (m *mockStore) GetPrefixDetails(
_ string,
) ([]database.LiveRoute, map[int]database.ASN, error) {
// Return empty routes for now
return []database.LiveRoute{}, map[int]database.ASN{}, nil
}
// GetPrefixDetailsContext mock implementation with context support
func (m *mockStore) GetPrefixDetailsContext(
_ context.Context, prefix string,
) ([]database.LiveRoute, map[int]database.ASN, error) {
return m.GetPrefixDetails(prefix)
}
func (m *mockStore) GetRandomPrefixesByLength(
_, _, _ int,
) ([]database.LiveRoute, error) {
// Return empty routes for now
return []database.LiveRoute{}, nil
}
// GetRandomPrefixesByLengthContext mock implementation with context support
func (m *mockStore) GetRandomPrefixesByLengthContext(
_ context.Context, maskLength, ipVersion, limit int,
) ([]database.LiveRoute, error) {
return m.GetRandomPrefixesByLength(maskLength, ipVersion, limit)
}
// GetASPeers mock implementation
func (m *mockStore) GetASPeers(_ int) ([]database.ASPeer, error) {
// Return empty peers for now
@@ -334,32 +250,6 @@ func (m *mockStore) GetASPeersContext(
return m.GetASPeers(asn)
}
// GetIPInfo mock implementation
func (m *mockStore) GetIPInfo(ip string) (*database.IPInfo, error) {
return m.GetIPInfoContext(context.Background(), ip)
}
// GetIPInfoContext mock implementation with context support
func (m *mockStore) GetIPInfoContext(
_ context.Context, ip string,
) (*database.IPInfo, error) {
now := time.Now()
return &database.IPInfo{
IP: ip,
Netblock: "8.8.8.0/24",
MaskLength: 24,
IPVersion: 4,
NumPeers: 3,
ASN: 15169,
Handle: "GOOGLE",
Description: "Google LLC",
CountryCode: "US",
FirstSeen: now.Add(-24 * time.Hour),
LastSeen: now,
}, nil
}
// GetNextStaleASN mock implementation
func (m *mockStore) GetNextStaleASN(_ context.Context, _ time.Duration) (int, error) {
return 0, database.ErrNoStaleASN
@@ -387,46 +277,6 @@ func (m *mockStore) GetWHOISStats(
}, nil
}
// UpsertLiveRouteBatch mock implementation
func (m *mockStore) UpsertLiveRouteBatch(routes []*database.LiveRoute) error {
m.mu.Lock()
defer m.mu.Unlock()
for _, route := range routes {
// Track prefix
if _, exists := m.Prefixes[route.Prefix]; !exists {
m.Prefixes[route.Prefix] = &database.Prefix{
Prefix: route.Prefix,
IPVersion: route.IPVersion,
FirstSeen: route.LastUpdated,
}
m.PrefixCount++
if route.IPVersion == 4 {
m.IPv4Prefixes++
} else {
m.IPv6Prefixes++
}
}
m.RouteCount++
}
return nil
}
// DeleteLiveRouteBatch mock implementation
func (m *mockStore) DeleteLiveRouteBatch(_ []database.LiveRouteDeletion) error {
// Simple mock - just return nil
return nil
}
// DeleteLiveRoutesOfPeer mock implementation
func (m *mockStore) DeleteLiveRoutesOfPeer(_ string) (int, error) {
// Simple mock - deletes nothing
return 0, nil
}
// WriteRouteChanges records the prefixes the changes leave some peer carrying,
// whatever their origin AS.
func (m *mockStore) WriteRouteChanges(
internal/routewatch/ashandler_internal_test.go
+2 -2
View File
@@ -29,9 +29,9 @@ func TestASHandlerStoresTheEarliestAndLatestTimesOfANewAS(t *testing.T) {
h.HandleMessage(second)
h.flushBatch()
asn, _, err := db.GetASDetails(testASNA)
asn, _, err := db.GetAS(t.Context(), testASNA)
if err != nil {
t.Fatalf("GetASDetails(%d): %v", testASNA, err)
t.Fatalf("GetAS(%d): %v", testASNA, err)
}
if !asn.FirstSeen.Equal(first.ParsedTimestamp) {
internal/routewatch/batchwrite_internal_test.go
+15 -25
View File
@@ -81,9 +81,9 @@ func TestFlushLoopWritesOnEveryTick(t *testing.T) {
}
// blockingStore is a mockStore whose batch writes for the AS, peer and prefix
// handlers, delete of a peer's routes and peering write wait until the test
// calls releaseWrites. writeStarted is closed when the first of them starts. It
// records what each batch write was given.
// handlers and peering write wait until the test calls releaseWrites.
// writeStarted is closed when the first of them starts. It records what each
// batch write was given.
type blockingStore struct {
*mockStore
@@ -109,13 +109,13 @@ func newBlockingStore() *blockingStore {
}
}
func (s *blockingStore) UpsertLiveRouteBatch(routes []*database.LiveRoute) error {
func (s *blockingStore) UpdatePrefixesBatch(prefixes map[string]time.Time) error {
s.waitForRelease()
s.mu.Lock()
defer s.mu.Unlock()
for _, route := range routes {
s.prefixes[route.Prefix] = true
for prefix := range prefixes {
s.prefixes[prefix] = true
}
return nil
@@ -145,12 +145,6 @@ func (s *blockingStore) UpdatePeerBatch(peers map[string]database.PeerUpdate) er
return nil
}
func (s *blockingStore) DeleteLiveRoutesOfPeer(peerIP string) (int, error) {
s.waitForRelease()
return s.mockStore.DeleteLiveRoutesOfPeer(peerIP)
}
func (s *blockingStore) RecordPeering(
asA, asB int, seen database.SeenTimes,
) error {
@@ -176,7 +170,6 @@ func (s *blockingStore) releaseWrites() {
type stopLog struct {
Msg string `json:"msg"`
Updates int `json:"updates"`
PeersDown int `json:"peers_down"`
TableChanges int `json:"table_changes"`
Paths int `json:"paths"`
}
@@ -374,10 +367,10 @@ func stopWhileAWriteIsBlocked(
}
// TestPrefixHandlerSkipsItsLastWriteAfterTheStopDeadline also checks the
// routing table's changes: the flush whose delete is held wrote those of the
// fill, so Stop must log the one of the extra prefix as not written. The flush
// timer must not fire. A flush it started before the fill would be the one
// held, and it would not have written the fill's change.
// routing table's changes: the flush whose prefix write is held wrote those of
// the fill, so Stop must log the one of the extra prefix as not written. The
// flush timer must not fire. A flush it started before the fill would be the
// one held, and it would not have written the fill's change.
func TestPrefixHandlerSkipsItsLastWriteAfterTheStopDeadline(t *testing.T) {
t.Parallel()
@@ -389,10 +382,8 @@ func TestPrefixHandlerSkipsItsLastWriteAfterTheStopDeadline(t *testing.T) {
h := newPrefixHandler(store, newErrorLogger(&logs),
routetable.New(routetable.Params{}), longerThanAnyTest)
fill := slices.Repeat([]string{testV4Prefix}, prefixBatchSize)
// The peer's session goes down first, so the write held is the delete of
// its routes, and Stop must log that peer down too.
stopWhileAWriteIsBlocked(t, store, h, h.done,
[]*ristypes.RISMessage{sessionDown(), prefixAnnouncement(fill)},
[]*ristypes.RISMessage{prefixAnnouncement(fill)},
prefixAnnouncement([]string{extraPrefix}))
store.mu.Lock()
@@ -403,11 +394,10 @@ func TestPrefixHandlerSkipsItsLastWriteAfterTheStopDeadline(t *testing.T) {
}
logged := lastLogged(t, &logs)
if logged.Updates != prefixBatchSize+1 || logged.PeersDown != 1 ||
logged.TableChanges != 1 {
t.Errorf("Stop logged %d updates, %d peer downs and %d table changes "+
"not written, want %d, 1 and 1", logged.Updates, logged.PeersDown,
logged.TableChanges, prefixBatchSize+1)
if logged.Updates != prefixBatchSize+1 || logged.TableChanges != 1 {
t.Errorf("Stop logged %d updates and %d table changes not written, "+
"want %d and 1", logged.Updates, logged.TableChanges,
prefixBatchSize+1)
}
}
internal/routewatch/prefixhandler.go
+78 -352
View File
@@ -2,10 +2,8 @@ package routewatch
import (
"context"
"net"
"net/netip"
"slices"
"strings"
"sync"
"time"
@@ -28,11 +26,9 @@ const (
prefixBatchSize = 25000
// prefixBatchMaxSize is the most prefix updates the batch holds while the
// flush goroutine writes the previous one: ten batches, about 60 MiB
// flush goroutine writes the previous one: ten batches, about 15 MiB
// (README "Memory"). At this size HandleMessage waits until the flush
// goroutine takes the batch. On top of these, the handler holds at most
// prefixBatchSize updates, about 6 MiB, that a flush kept for the peers
// whose route delete failed (held).
// goroutine takes the batch.
prefixBatchMaxSize = 250000
// prefixBatchTimeout is the maximum time to wait before flushing a batch
@@ -48,47 +44,31 @@ const (
// most one pair. So the million pairs that a full-table peer's session
// going down changes are written in about a minute.
maxTableChangesPerWrite = 20000
// IP version constants
ipv4Version = 4
ipv6Version = 6
)
// PrefixHandler tracks BGP prefixes and maintains a live routing table in the
// database. Routes are added on announcement and deleted on withdrawal or on an
// announcement whose AS path is empty or holds AS 0 anywhere, and all routes of
// a peer are deleted when its BGP session goes down.
// PrefixHandler keeps the routing table in memory current from the feed. It
// applies each update to the table as it handles it. An announcement gives
// the peer a route to the prefix. A withdrawal removes the peer's route, as
// does an announcement whose AS path is empty or holds AS 0 anywhere. A peer
// whose BGP session goes down leaves every prefix.
//
// It applies the same updates to the routing table in memory as it handles
// them, and writes the table's changes to the database with each flush.
// Each flush writes the table's changes to the database. It also stores each
// prefix the updates name with the time it was first seen.
type PrefixHandler struct {
db database.Store
logger *logger.Logger
metrics *metrics.Tracker
table *routetable.Table
// Batching. mu guards batch, held, peersDown, writing, deleting and
// writingTableChanges only: the flush goroutine takes them under mu and
// writes them after releasing mu, so HandleMessage waits for a database
// write only while batch is at prefixBatchMaxSize.
// Batching. mu guards batch, writing and writingTableChanges only: the
// flush goroutine takes them under mu and writes them after releasing
// mu, so HandleMessage waits for a database write only while batch is at
// prefixBatchMaxSize.
mu sync.Mutex
batch []prefixUpdate
// held holds the updates that the peers whose route delete failed in the
// last flush sent after their session went down, for the next flush to
// write after it retries the delete. They are not in batch, so they do
// not count toward a full batch: a delete that keeps failing is retried
// at the pace of the flush timer.
held []prefixUpdate
// peersDown holds, each once, the peers whose BGP session went down
// since the batch was taken, and those whose route delete failed in the
// last flush.
peersDown []string
batch []seenPrefix
// writing is how many updates the flush goroutine took and has not
// finished writing.
writing int
// deleting is how many peers from peersDown the flush goroutine took and
// has not yet deleted the routes of.
deleting int
// writingTableChanges is how many changes the flush goroutine took from
// the routing table and has not finished writing.
writingTableChanges int
@@ -102,6 +82,15 @@ type PrefixHandler struct {
done chan struct{} // closed when the flush goroutine returns
}
// seenPrefix is what the batch keeps of an update: the prefix it names and its
// time. A flush needs only these to store each new prefix's first-seen time.
type seenPrefix struct {
prefix string
timestamp time.Time
}
// prefixUpdate is one prefix of an update message, as applyToTable applies it
// to the routing table.
type prefixUpdate struct {
prefix string
originASN int
@@ -144,7 +133,7 @@ func newPrefixHandler(
db: db,
logger: logger,
table: table,
batch: make([]prefixUpdate, 0, prefixBatchSize),
batch: make([]seenPrefix, 0, prefixBatchSize),
batchFullCh: make(chan struct{}, 1),
stopCh: make(chan context.Context, 1),
done: make(chan struct{}),
@@ -181,7 +170,7 @@ func (h *PrefixHandler) QueueCapacity() int {
func (h *PrefixHandler) HandleMessage(msg *ristypes.RISMessage) {
if msg.Type == ristypes.MessageTypeState {
if msg.State == ristypes.StateDown {
h.peerSessionDown(msg.Peer)
h.table.PeerDown(msg.Peer)
}
return
@@ -206,12 +195,11 @@ func (h *PrefixHandler) HandleMessage(msg *ristypes.RISMessage) {
h.batchTaken.Wait()
}
// Withdrawals are queued, and applied to the routing table, before
// announcements, because the batch keeps each peer's last update for a
// prefix: a message that lists a prefix as both withdrawn and announced
// keeps the route (RFC 4271 section 4.3). A withdrawal names only the
// prefix and peer: the message's AS path, and so its origin AS, belongs
// to the announced prefixes.
// Withdrawals are applied to the routing table before announcements, so
// a message that lists a prefix as both withdrawn and announced keeps the
// route (RFC 4271 section 4.3). A withdrawal names only the prefix and
// peer: the message's AS path, and so its origin AS, belongs to the
// announced prefixes.
for _, prefix := range msg.Withdrawals {
update := prefixUpdate{
prefix: canonicalPrefix(prefix),
@@ -219,7 +207,7 @@ func (h *PrefixHandler) HandleMessage(msg *ristypes.RISMessage) {
messageType: "withdrawal",
timestamp: timestamp,
}
h.batch = append(h.batch, update)
h.batch = append(h.batch, seenPrefix{update.prefix, update.timestamp})
h.applyToTable(update)
// Record withdrawal in metrics
@@ -239,7 +227,7 @@ func (h *PrefixHandler) HandleMessage(msg *ristypes.RISMessage) {
path: path,
endsInSet: len(msg.Path.Set) > 0,
}
h.batch = append(h.batch, update)
h.batch = append(h.batch, seenPrefix{update.prefix, update.timestamp})
h.applyToTable(update)
// Record announcement in metrics
@@ -261,14 +249,12 @@ func (h *PrefixHandler) HandleMessage(msg *ristypes.RISMessage) {
}
// Stop stops the flush goroutine, which makes one last flush before it returns,
// and waits for it until ctx ends. That flush writes the updates and peer downs
// the handler holds, and at most maxTableChangesPerWrite of the routing table's
// changes. Once ctx has ended the flush goroutine skips that last flush, and a
// write still running is lost when the process exits. A peer whose route delete
// fails in the last flush is not retried either, nor is a failed write of the
// table's changes. Stop logs how many updates are not written, how many peer
// downs have not had their routes deleted, and how many of the table's changes
// are not written.
// and waits for it until ctx ends. That flush writes at most
// maxTableChangesPerWrite of the routing table's changes, and the prefixes of
// the updates the handler holds. Once ctx has ended the flush goroutine skips
// that last flush, and a write still running is lost when the process exits. A
// failed write of the table's changes is not retried either. Stop logs how many
// updates and how many of the table's changes are not written.
func (h *PrefixHandler) Stop(ctx context.Context) {
h.stopCh <- ctx
@@ -278,176 +264,96 @@ func (h *PrefixHandler) Stop(ctx context.Context) {
}
h.mu.Lock()
unwritten := h.writing + len(h.batch) + len(h.held)
peersDown := h.deleting + len(h.peersDown)
unwritten := h.writing + len(h.batch)
tableChanges := h.writingTableChanges + h.table.ChangeCount()
h.mu.Unlock()
if unwritten == 0 && peersDown == 0 && tableChanges == 0 {
if unwritten == 0 && tableChanges == 0 {
return
}
if ctx.Err() != nil {
h.logger.Error("Stop deadline passed before the prefix handler wrote "+
"its updates", "updates", unwritten, "peers_down", peersDown,
"table_changes", tableChanges)
"its updates", "updates", unwritten, "table_changes", tableChanges)
return
}
h.logger.Error("The prefix handler stopped before it wrote everything",
"updates", unwritten, "peers_down", peersDown,
"table_changes", tableChanges)
"updates", unwritten, "table_changes", tableChanges)
}
// peerSessionDown notes that the peer's BGP session went down, so that the next
// flush deletes all of the peer's live routes before it writes the batch. The
// peer's updates still in the batch or held came before the down, which
// replaces them, so they are dropped; updates the peer sends later stay in the
// batch and are written after the delete. A peer already in peersDown is not
// added again: one delete removes all of its routes. In the routing table in
// memory the down takes effect at once.
func (h *PrefixHandler) peerSessionDown(peer string) {
h.mu.Lock()
defer h.mu.Unlock()
h.table.PeerDown(peer)
isPeers := func(update prefixUpdate) bool { return update.peer == peer }
h.batch = slices.DeleteFunc(h.batch, isPeers)
h.held = slices.DeleteFunc(h.held, isPeers)
if !slices.Contains(h.peersDown, peer) {
h.peersDown = append(h.peersDown, peer)
}
}
// takeBatch returns the pending updates, the held ones first, and the peers in
// peersDown, and leaves an empty batch, no held updates and no peers in their
// takeBatch returns the pending updates and leaves an empty batch in their
// place for HandleMessage to fill. The updates count in writing until
// flushBatch has written them, and the peers in deleting until their routes
// are deleted.
func (h *PrefixHandler) takeBatch() ([]prefixUpdate, []string) {
// flushBatch has written them.
func (h *PrefixHandler) takeBatch() []seenPrefix {
h.mu.Lock()
defer h.mu.Unlock()
if len(h.held) == 0 && len(h.batch) == 0 && len(h.peersDown) == 0 {
return nil, nil
}
batch, peersDown := h.batch, h.peersDown
if len(h.held) > 0 {
batch = slices.Concat(h.held, h.batch)
if len(h.batch) == 0 {
return nil
}
batch := h.batch
h.writing = len(batch)
h.deleting = len(peersDown)
h.held = nil
h.batch = make([]prefixUpdate, 0, prefixBatchSize)
h.peersDown = nil
h.batch = make([]seenPrefix, 0, prefixBatchSize)
h.batchTaken.Broadcast()
return batch, peersDown
return batch
}
// flushBatch writes the routing table's changes, then takes the pending batch
// and writes it to the database, without holding the mutex during the writes.
// and stores the prefixes it names, without holding the mutex during the
// writes.
func (h *PrefixHandler) flushBatch() {
h.writeTableChanges()
batch, peersDown := h.takeBatch()
if len(batch) == 0 && len(peersDown) == 0 {
batch := h.takeBatch()
if len(batch) == 0 {
return
}
startTime := time.Now()
batchSize := len(batch)
// The routes of a peer whose session went down are deleted first: the
// batch holds only the updates the peer sent after the down. A peer whose
// delete failed is kept, with those updates, for the next flush to retry.
failedPeers := h.deleteLiveRoutesOfPeers(peersDown)
if len(failedPeers) > 0 {
batch = h.keepForNextFlush(batch, failedPeers)
}
// For each prefix and peer, keep only the last update the feed delivered.
// Updates from different peers for the same prefix are all kept. Arrival
// order decides, not the timestamp: timestamps are whole seconds, so
// updates within one second carry the same one.
type prefixAndPeer struct{ prefix, peer string }
lastUpdates := make(map[prefixAndPeer]prefixUpdate)
// Each prefix the batch names, announced or withdrawn, with the earliest
// timestamp among all of its updates, not only each peer's last one: the
// prefix tables store only first_seen, written when the prefix is new and
// never changed after.
// timestamp among its updates: the prefix tables store only first_seen,
// written when the prefix is new and never changed after.
prefixesToUpdate := make(map[string]time.Time)
for _, update := range batch {
lastUpdates[prefixAndPeer{update.prefix, update.peer}] = update
seen, exists := prefixesToUpdate[update.prefix]
if !exists || update.timestamp.Before(seen) {
prefixesToUpdate[update.prefix] = update.timestamp
}
}
// Collect routes to upsert and delete
var (
routesToUpsert []*database.LiveRoute
routesToDelete []database.LiveRouteDeletion
)
for _, update := range lastUpdates {
if update.announcesRoute() {
// Create live route for batch upsert
route := h.createLiveRoute(update)
if route != nil {
routesToUpsert = append(routesToUpsert, route)
}
} else {
// Parse CIDR to get IP version
_, ipVersion, err := parseCIDR(update.prefix)
if err != nil {
h.logger.Error("Failed to parse CIDR for route deletion",
"prefix", update.prefix, "error", err)
continue
}
// Create deletion record for batch delete
routesToDelete = append(routesToDelete, database.LiveRouteDeletion{
Prefix: update.prefix,
PeerIP: update.peer,
IPVersion: ipVersion,
})
}
err := h.db.UpdatePrefixesBatch(prefixesToUpdate)
if err != nil {
h.logger.Error("Failed to update prefix batch",
"error", err, "count", len(prefixesToUpdate))
}
successCount := h.writeBatch(routesToUpsert, routesToDelete, prefixesToUpdate)
h.mu.Lock()
h.writing = 0
h.mu.Unlock()
elapsed := time.Since(startTime)
h.logger.Debug("Flushed prefix batch",
"batch_size", batchSize,
"peers_down", len(peersDown),
"batch_size", len(batch),
"unique_prefixes", len(prefixesToUpdate),
"success", successCount,
"duration_ms", elapsed.Milliseconds(),
"duration_ms", time.Since(startTime).Milliseconds(),
)
}
// applyToTable applies update to the routing table in memory: as an
// announcement when it gives the peer a route, otherwise as a withdrawal. A
// prefix that does not parse is left out; flushBatch logs it.
// prefix that does not parse is logged and left out. Each announcement counts
// in the IPv4 or IPv6 route update rate.
func (h *PrefixHandler) applyToTable(update prefixUpdate) {
prefix, err := netip.ParsePrefix(update.prefix)
if err != nil {
h.logger.Error("Failed to parse prefix",
"prefix", update.prefix, "error", err)
return
}
@@ -457,6 +363,14 @@ func (h *PrefixHandler) applyToTable(update prefixUpdate) {
return
}
if h.metrics != nil {
if prefix.Addr().Is4() {
h.metrics.RecordIPv4Update()
} else {
h.metrics.RecordIPv6Update()
}
}
h.table.Announce(prefix, update.originASN, routetable.Route{
Peer: update.peer,
ASPath: update.path,
@@ -495,147 +409,11 @@ func (h *PrefixHandler) writeTableChanges() {
}
}
// deleteLiveRoutesOfPeers deletes all live routes of each of peers, the peers
// whose BGP session went down, and returns those whose delete failed. Most
// downs come from peers that have no routes, so only a delete that removed
// routes is logged. A peer whose delete succeeded no longer counts in deleting;
// a failed one counts until keepForNextFlush puts it back in peersDown.
func (h *PrefixHandler) deleteLiveRoutesOfPeers(peers []string) []string {
var failed []string
for _, peer := range peers {
deleted, err := h.db.DeleteLiveRoutesOfPeer(peer)
if err != nil {
h.logger.Error(
"Failed to delete the live routes of a peer whose session "+
"went down; the next flush retries",
"peer", peer, "routes_deleted", deleted, "error", err)
failed = append(failed, peer)
continue
}
h.mu.Lock()
h.deleting--
h.mu.Unlock()
if deleted > 0 {
h.logger.Info(
"Deleted the live routes of a peer whose session went down",
"peer", peer, "routes", deleted)
}
}
return failed
}
// keepForNextFlush puts the peers whose route delete failed back in peersDown,
// and their updates from batch in held, so that the next flush retries each
// delete before it writes the peer's updates; written now, they would be
// deleted by the retry. The updates of a peer whose session went down again
// since the batch was taken came before that down, so they are dropped, as
// peerSessionDown drops them, and the peer is already in peersDown. It returns
// the updates of the other peers, for this flush to write.
//
// While a delete keeps failing, the held updates would grow with every flush,
// so only the last prefixBatchSize of them are kept and the older ones are
// dropped. This never leaves in place a route the peer withdrew: a prefix with
// a kept update keeps its last one, and the retry deletes all of the peer's
// routes, so a prefix whose updates were all dropped has no route from the peer
// until the peer announces it again.
func (h *PrefixHandler) keepForNextFlush(
batch []prefixUpdate, failedPeers []string,
) []prefixUpdate {
var kept, toWrite []prefixUpdate
for _, update := range batch {
if slices.Contains(failedPeers, update.peer) {
kept = append(kept, update)
} else {
toWrite = append(toWrite, update)
}
}
h.mu.Lock()
defer h.mu.Unlock()
kept = slices.DeleteFunc(kept, func(update prefixUpdate) bool {
return slices.Contains(h.peersDown, update.peer)
})
if len(kept) > prefixBatchSize {
h.logger.Error(
"Dropped the oldest updates of peers whose route delete failed",
"peers", failedPeers, "dropped", len(kept)-prefixBatchSize)
// A copy, so that the dropped updates are freed.
kept = slices.Clone(kept[len(kept)-prefixBatchSize:])
}
// takeBatch left held empty, and only this flush fills it. The kept
// updates now count in held, not in writing, and the failed peers in
// peersDown, not in deleting.
h.held = kept
h.writing = len(toWrite)
h.deleting = 0
for _, peer := range failedPeers {
if !slices.Contains(h.peersDown, peer) {
h.peersDown = append(h.peersDown, peer)
}
}
return toWrite
}
// writeBatch writes the collected route upserts, route deletions and prefix
// updates to the database and returns how many routes it wrote.
func (h *PrefixHandler) writeBatch(
routesToUpsert []*database.LiveRoute,
routesToDelete []database.LiveRouteDeletion,
prefixesToUpdate map[string]time.Time,
) int {
successCount := 0
if len(routesToUpsert) > 0 {
err := h.db.UpsertLiveRouteBatch(routesToUpsert)
if err != nil {
h.logger.Error("Failed to upsert route batch",
"error", err, "count", len(routesToUpsert))
} else {
successCount += len(routesToUpsert)
}
}
if len(routesToDelete) > 0 {
err := h.db.DeleteLiveRouteBatch(routesToDelete)
if err != nil {
h.logger.Error("Failed to delete route batch",
"error", err, "count", len(routesToDelete))
} else {
successCount += len(routesToDelete)
}
}
// Update prefix tables
if len(prefixesToUpdate) > 0 {
err := h.db.UpdatePrefixesBatch(prefixesToUpdate)
if err != nil {
h.logger.Error("Failed to update prefix batch",
"error", err, "count", len(prefixesToUpdate))
}
}
return successCount
}
// canonicalPrefix returns prefix in the text form net/netip prints, the form
// the IP lookup builds when it looks a prefix up. The feed sends IPv6
// the prefix tables store and the IP lookup reads them in. The feed sends IPv6
// announcements compressed ("2001:db8::/32") but IPv6 withdrawals uncompressed
// ("2001:db8:0:0:0:0:0:0/32"); stored as received, a withdrawal would not match
// the route its announcement stored. A prefix that does not parse is returned
// unchanged, and the batch flush reports it.
// ("2001:db8:0:0:0:0:0:0/32"). A prefix that does not parse is returned
// unchanged, and applyToTable logs it.
func canonicalPrefix(prefix string) string {
p, err := netip.ParsePrefix(prefix)
if err != nil {
@@ -644,55 +422,3 @@ func canonicalPrefix(prefix string) string {
return p.Masked().String()
}
// parseCIDR returns the mask length and the IP version of a prefix string
func parseCIDR(prefix string) (int, int, error) {
_, ipNet, err := net.ParseCIDR(prefix)
if err != nil {
return 0, 0, err
}
ones, _ := ipNet.Mask.Size()
if strings.Contains(prefix, ":") {
return ones, ipv6Version, nil
}
return ones, ipv4Version, nil
}
// createLiveRoute creates a LiveRoute from a prefix update
func (h *PrefixHandler) createLiveRoute(update prefixUpdate) *database.LiveRoute {
// Parse CIDR to get mask length
maskLength, ipVersion, err := parseCIDR(update.prefix)
if err != nil {
h.logger.Error("Failed to parse CIDR",
"prefix", update.prefix,
"error", err,
)
return nil
}
// Track route update metrics
if h.metrics != nil {
if ipVersion == ipv4Version {
h.metrics.RecordIPv4Update()
} else {
h.metrics.RecordIPv6Update()
}
}
// Create live route record
liveRoute := &database.LiveRoute{
Prefix: update.prefix,
MaskLength: maskLength,
IPVersion: ipVersion,
OriginASN: update.originASN,
PeerIP: update.peer,
ASPath: update.path,
NextHop: update.peer, // Using peer as next hop
LastUpdated: update.timestamp,
}
return liveRoute
}
internal/routewatch/prefixhandler_internal_test.go
+45 -915
View File
File diff suppressed because it is too large Load Diff
internal/routewatch/prefixhandler_table_internal_test.go
+82 -5
View File
@@ -8,6 +8,7 @@ import (
"net/netip"
"reflect"
"slices"
"strings"
"testing"
"time"
@@ -84,13 +85,25 @@ func TestPrefixHandlerAppliesUpdatesToTheRoutingTable(t *testing.T) {
testV6Prefix: {},
},
},
{
name: "an announcement after the peer's session went down",
messages: []*ristypes.RISMessage{
peerAnnouncement(testPeerIP, testV4Prefix),
sessionDown(),
peerAnnouncement(testPeerIP, otherV4Prefix),
},
want: map[string]map[int][]string{
testV4Prefix: {},
otherV4Prefix: {testASNB: {testPeerIP}},
},
},
})
}
// TestPrefixHandlerAppliesAPathWithoutAnOriginToTheRoutingTable hands the
// prefix handler announcements whose AS path gives no origin. One whose path
// holds AS 0 removes the peer's route, as a withdrawal does; one whose path ends
// in an AS set of several members is kept with origin 0.
// is empty or holds AS 0 removes the peer's route, as a withdrawal does; one
// whose path ends in an AS set of several members is kept with origin 0.
func TestPrefixHandlerAppliesAPathWithoutAnOriginToTheRoutingTable(t *testing.T) {
t.Parallel()
@@ -110,6 +123,14 @@ func TestPrefixHandlerAppliesAPathWithoutAnOriginToTheRoutingTable(t *testing.T)
},
want: map[string]map[int][]string{testV4Prefix: {}},
},
{
name: "an empty path",
messages: []*ristypes.RISMessage{
peerAnnouncement(testPeerIP, testV4Prefix),
announceWithPath(ristypes.ASPath{}),
},
want: map[string]map[int][]string{testV4Prefix: {}},
},
{
name: "a path with AS 0 before its end",
messages: []*ristypes.RISMessage{
@@ -128,6 +149,55 @@ func TestPrefixHandlerAppliesAPathWithoutAnOriginToTheRoutingTable(t *testing.T)
})
}
// TestPrefixHandlerKeepsThePathOfAPathEndingInASet announces a prefix with AS
// paths that end in an AS set. Any member of a set of several may be the
// origin, so the routing table keeps the route with origin 0 (none); a set of
// one member counts as that AS. The kept AS path lists the set's members after
// the rest of the path.
func TestPrefixHandlerKeepsThePathOfAPathEndingInASet(t *testing.T) {
t.Parallel()
tests := []struct {
name string
path []any
wantOrigin int
wantPath []int
}{
{
name: "a set of several members",
path: []any{testASNA, testASNB, []int{testASNC, testASND}},
wantPath: []int{testASNA, testASNB, testASNC, testASND},
},
{
name: "a set of one member",
path: []any{testASNA, testASNB, []int{testASNC}},
wantOrigin: testASNC,
wantPath: []int{testASNA, testASNB, testASNC},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
t.Parallel()
msg := peerAnnouncement(testPeerIP, testV4Prefix)
msg.Path = decodePath(t, tt.path...)
h := prefixHandlerWithoutTimer(newMockStore())
h.HandleMessage(msg)
origins := h.table.Origins(netip.MustParsePrefix(testV4Prefix))
if len(origins) != 1 || origins[0].ASN != tt.wantOrigin ||
len(origins[0].Routes) != 1 ||
!slices.Equal(origins[0].Routes[0].ASPath, tt.wantPath) {
t.Errorf("Origins(%s) = %+v, want one route with origin AS%d "+
"and AS path %v", testV4Prefix, origins, tt.wantOrigin,
tt.wantPath)
}
})
}
}
// checkTableCases hands the messages of each case to a new prefix handler and
// checks its routing table right after, with no flush.
func checkTableCases(t *testing.T, tests []tableCase) {
@@ -343,8 +413,9 @@ func TestPrefixHandlerWritesManyTableChangesOverSeveralFlushes(t *testing.T) {
// TestPrefixHandlerStopLogsTheTableChangesLeftUnwritten stops a prefix handler,
// with no deadline, whose routing table holds more changed pairs than one write
// takes. Its last flush writes one write's worth, so Stop must log the pairs
// that no write took. The flush timer must not fire, or its writes could leave
// no pairs for Stop to log.
// that no write took, and must not say that the stop deadline passed. The
// flush timer must not fire, or its writes could leave no pairs for Stop to
// log.
func TestPrefixHandlerStopLogsTheTableChangesLeftUnwritten(t *testing.T) {
t.Parallel()
@@ -364,7 +435,13 @@ func TestPrefixHandlerStopLogsTheTableChangesLeftUnwritten(t *testing.T) {
written += n
}
if logged := lastLogged(t, &logs); logged.TableChanges != pairs-written {
logged := lastLogged(t, &logs)
if strings.Contains(logged.Msg, "deadline") {
t.Errorf("Stop logged %q although its deadline had not passed",
logged.Msg)
}
if logged.TableChanges != pairs-written {
t.Errorf("Stop logged %d table changes not written, want %d",
logged.TableChanges, pairs-written)
}
internal/server/handlers.go
+94 -94
View File
@@ -83,7 +83,7 @@ const (
maxASSearchResults = 50
)
// IP versions as database.LiveRoute.IPVersion records them.
// IP versions as LiveRoute.IPVersion records them.
const (
ipVersionV4 = 4
ipVersionV6 = 6
@@ -233,34 +233,34 @@ type WHOISStatsInfo struct {
//
//nolint:tagliatelle // field names are the existing JSON API
type Stats struct {
Uptime string `json:"uptime"`
TotalMessages uint64 `json:"total_messages"`
TotalBytes uint64 `json:"total_bytes"`
TotalWireBytes uint64 `json:"total_wire_bytes"`
MessagesPerSec float64 `json:"messages_per_sec"`
MbitsPerSec float64 `json:"mbits_per_sec"`
WireMbitsPerSec float64 `json:"wire_mbits_per_sec"`
Connected bool `json:"connected"`
GoVersion string `json:"go_version"`
Goroutines int `json:"goroutines"`
MemoryUsage string `json:"memory_usage"`
ASNs int `json:"asns"`
Prefixes int `json:"prefixes"`
IPv4Prefixes int `json:"ipv4_prefixes"`
IPv6Prefixes int `json:"ipv6_prefixes"`
Peerings int `json:"peerings"`
Peers int `json:"peers"`
DatabaseSizeBytes int64 `json:"database_size_bytes"`
LiveRoutes int `json:"live_routes"`
IPv4Routes int `json:"ipv4_routes"`
IPv6Routes int `json:"ipv6_routes"`
OldestRoute *time.Time `json:"oldest_route,omitempty"`
NewestRoute *time.Time `json:"newest_route,omitempty"`
IPv4UpdatesPerSec float64 `json:"ipv4_updates_per_sec"`
IPv6UpdatesPerSec float64 `json:"ipv6_updates_per_sec"`
IPv4PrefixDistribution []database.PrefixDistribution `json:"ipv4_prefix_distribution"`
IPv6PrefixDistribution []database.PrefixDistribution `json:"ipv6_prefix_distribution"`
WHOISStats *WHOISStatsInfo `json:"whois_stats,omitempty"`
Uptime string `json:"uptime"`
TotalMessages uint64 `json:"total_messages"`
TotalBytes uint64 `json:"total_bytes"`
TotalWireBytes uint64 `json:"total_wire_bytes"`
MessagesPerSec float64 `json:"messages_per_sec"`
MbitsPerSec float64 `json:"mbits_per_sec"`
WireMbitsPerSec float64 `json:"wire_mbits_per_sec"`
Connected bool `json:"connected"`
GoVersion string `json:"go_version"`
Goroutines int `json:"goroutines"`
MemoryUsage string `json:"memory_usage"`
ASNs int `json:"asns"`
Prefixes int `json:"prefixes"`
IPv4Prefixes int `json:"ipv4_prefixes"`
IPv6Prefixes int `json:"ipv6_prefixes"`
Peerings int `json:"peerings"`
Peers int `json:"peers"`
DatabaseSizeBytes int64 `json:"database_size_bytes"`
LiveRoutes int `json:"live_routes"`
IPv4Routes int `json:"ipv4_routes"`
IPv6Routes int `json:"ipv6_routes"`
OldestRoute *time.Time `json:"oldest_route,omitempty"`
NewestRoute *time.Time `json:"newest_route,omitempty"`
IPv4UpdatesPerSec float64 `json:"ipv4_updates_per_sec"`
IPv6UpdatesPerSec float64 `json:"ipv6_updates_per_sec"`
IPv4PrefixDistribution []PrefixDistribution `json:"ipv4_prefix_distribution"`
IPv6PrefixDistribution []PrefixDistribution `json:"ipv6_prefix_distribution"`
WHOISStats *WHOISStatsInfo `json:"whois_stats,omitempty"`
// OriginsWithoutKeptRoute counts the prefix and origin AS pairs that peers
// carry but whose kept routes were all withdrawn.
@@ -394,12 +394,12 @@ func (s *Server) statusStats(
// prefixDistribution lists the mask lengths that have at least one prefix, in
// ascending order, with their prefix counts. countsByLength[n] is the number
// of prefixes of mask length n.
func prefixDistribution(countsByLength []int) []database.PrefixDistribution {
var distribution []database.PrefixDistribution
func prefixDistribution(countsByLength []int) []PrefixDistribution {
var distribution []PrefixDistribution
for maskLength, count := range countsByLength {
if count > 0 {
distribution = append(distribution, database.PrefixDistribution{
distribution = append(distribution, PrefixDistribution{
MaskLength: maskLength, Count: count,
})
}
@@ -508,39 +508,39 @@ type StreamStats struct {
//
//nolint:tagliatelle // field names are the existing JSON API
type StatsResponse struct {
Uptime string `json:"uptime"`
TotalMessages uint64 `json:"total_messages"`
TotalBytes uint64 `json:"total_bytes"`
TotalWireBytes uint64 `json:"total_wire_bytes"`
MessagesPerSec float64 `json:"messages_per_sec"`
MbitsPerSec float64 `json:"mbits_per_sec"`
WireMbitsPerSec float64 `json:"wire_mbits_per_sec"`
Connected bool `json:"connected"`
ConnectionDuration string `json:"connection_duration"`
ReconnectCount uint64 `json:"reconnect_count"`
GoVersion string `json:"go_version"`
Goroutines int `json:"goroutines"`
MemoryUsage string `json:"memory_usage"`
GC GCStats `json:"gc"`
Stream StreamStats `json:"stream"`
ASNs int `json:"asns"`
Prefixes int `json:"prefixes"`
IPv4Prefixes int `json:"ipv4_prefixes"`
IPv6Prefixes int `json:"ipv6_prefixes"`
Peerings int `json:"peerings"`
Peers int `json:"peers"`
DatabaseSizeBytes int64 `json:"database_size_bytes"`
LiveRoutes int `json:"live_routes"`
IPv4Routes int `json:"ipv4_routes"`
IPv6Routes int `json:"ipv6_routes"`
OldestRoute *time.Time `json:"oldest_route,omitempty"`
NewestRoute *time.Time `json:"newest_route,omitempty"`
IPv4UpdatesPerSec float64 `json:"ipv4_updates_per_sec"`
IPv6UpdatesPerSec float64 `json:"ipv6_updates_per_sec"`
HandlerStats []HandlerStatsInfo `json:"handler_stats"`
IPv4PrefixDistribution []database.PrefixDistribution `json:"ipv4_prefix_distribution"`
IPv6PrefixDistribution []database.PrefixDistribution `json:"ipv6_prefix_distribution"`
WHOISStats *WHOISStatsInfo `json:"whois_stats,omitempty"`
Uptime string `json:"uptime"`
TotalMessages uint64 `json:"total_messages"`
TotalBytes uint64 `json:"total_bytes"`
TotalWireBytes uint64 `json:"total_wire_bytes"`
MessagesPerSec float64 `json:"messages_per_sec"`
MbitsPerSec float64 `json:"mbits_per_sec"`
WireMbitsPerSec float64 `json:"wire_mbits_per_sec"`
Connected bool `json:"connected"`
ConnectionDuration string `json:"connection_duration"`
ReconnectCount uint64 `json:"reconnect_count"`
GoVersion string `json:"go_version"`
Goroutines int `json:"goroutines"`
MemoryUsage string `json:"memory_usage"`
GC GCStats `json:"gc"`
Stream StreamStats `json:"stream"`
ASNs int `json:"asns"`
Prefixes int `json:"prefixes"`
IPv4Prefixes int `json:"ipv4_prefixes"`
IPv6Prefixes int `json:"ipv6_prefixes"`
Peerings int `json:"peerings"`
Peers int `json:"peers"`
DatabaseSizeBytes int64 `json:"database_size_bytes"`
LiveRoutes int `json:"live_routes"`
IPv4Routes int `json:"ipv4_routes"`
IPv6Routes int `json:"ipv6_routes"`
OldestRoute *time.Time `json:"oldest_route,omitempty"`
NewestRoute *time.Time `json:"newest_route,omitempty"`
IPv4UpdatesPerSec float64 `json:"ipv4_updates_per_sec"`
IPv6UpdatesPerSec float64 `json:"ipv6_updates_per_sec"`
HandlerStats []HandlerStatsInfo `json:"handler_stats"`
IPv4PrefixDistribution []PrefixDistribution `json:"ipv4_prefix_distribution"`
IPv6PrefixDistribution []PrefixDistribution `json:"ipv6_prefix_distribution"`
WHOISStats *WHOISStatsInfo `json:"whois_stats,omitempty"`
OriginsWithoutKeptRoute int `json:"origins_without_kept_route"`
}
@@ -753,7 +753,7 @@ type IPLookupResponse struct {
// fields. IP hides the IPInfo field of the same name, so that an address with
// no route has it too.
type IPLookupResult struct {
*database.IPInfo
*IPInfo
IP string `json:"ip"`
PTR []string `json:"ptr,omitempty"`
@@ -813,10 +813,10 @@ func (s *Server) handleIPInfo() http.HandlerFunc {
}
switch {
case errors.Is(err, database.ErrNoRoute):
case errors.Is(err, ErrNoRoute):
// The address still gets a result, without route fields.
ipInfo = nil
case errors.Is(err, database.ErrInvalidIP):
case errors.Is(err, ErrInvalidIP):
response.Errors = append(response.Errors, ip+": "+err.Error())
continue
@@ -860,15 +860,15 @@ func (s *Server) handleIPInfo() http.HandlerFunc {
// route. The database gives the prefix's first-seen time and the AS's WHOIS
// details. It returns ErrNoRoute when no prefix holds the address, and
// ErrInvalidIP when ip does not parse.
func (s *Server) ipInfo(ctx context.Context, ip string) (*database.IPInfo, error) {
func (s *Server) ipInfo(ctx context.Context, ip string) (*IPInfo, error) {
addr, err := netip.ParseAddr(ip)
if err != nil {
return nil, fmt.Errorf("%w: %s", database.ErrInvalidIP, ip)
return nil, fmt.Errorf("%w: %s", ErrInvalidIP, ip)
}
summary, found := s.table.Lookup(addr)
if !found {
return nil, fmt.Errorf("%w for IP %s", database.ErrNoRoute, ip)
return nil, fmt.Errorf("%w for IP %s", ErrNoRoute, ip)
}
firstSeen, err := s.db.GetPrefixFirstSeen(ctx, summary.Prefix)
@@ -881,7 +881,7 @@ func (s *Server) ipInfo(ctx context.Context, ip string) (*database.IPInfo, error
return nil, err
}
info := &database.IPInfo{
info := &IPInfo{
IP: ip,
Netblock: summary.Prefix.String(),
MaskLength: summary.Prefix.Bits(),
@@ -1165,7 +1165,7 @@ func (s *Server) handleASDetailJSON() http.HandlerFunc {
}
if err != nil {
if errors.Is(err, database.ErrNoRoute) {
if errors.Is(err, ErrNoRoute) {
s.writeJSONError(w, http.StatusNotFound, err.Error())
} else {
s.logger.Error("Failed to get AS details", "asn", asn, "error", err)
@@ -1201,9 +1201,9 @@ func (s *Server) handleASDetailJSON() http.HandlerFunc {
// routing table all lack.
func (s *Server) asDetails(
ctx context.Context, asn int,
) (*database.ASN, []database.LiveRoute, error) {
) (*database.ASN, []LiveRoute, error) {
if asn == ristypes.NoOrigin {
return nil, nil, fmt.Errorf("%w: AS%d", database.ErrNoRoute, asn)
return nil, nil, fmt.Errorf("%w: AS%d", ErrNoRoute, asn)
}
asInfo, known, err := s.db.GetAS(ctx, asn)
@@ -1213,13 +1213,13 @@ func (s *Server) asDetails(
originated := s.table.OriginatedPrefixes(asn)
if !known && len(originated) == 0 {
return nil, nil, fmt.Errorf("%w: AS%d", database.ErrNoRoute, asn)
return nil, nil, fmt.Errorf("%w: AS%d", ErrNoRoute, asn)
}
prefixes := make([]database.LiveRoute, 0, len(originated))
prefixes := make([]LiveRoute, 0, len(originated))
for _, p := range originated {
route := database.LiveRoute{
route := LiveRoute{
Prefix: p.Prefix.String(),
MaskLength: p.Prefix.Bits(),
IPVersion: ipVersionOf(p.Prefix),
@@ -1239,9 +1239,9 @@ func (s *Server) asDetails(
// splitRoutesByIPVersion groups routes into the IPv4 and the IPv6 ones. Neither
// group is nil, so that a JSON answer holds [] for an empty one and not null.
func splitRoutesByIPVersion(
routes []database.LiveRoute,
) ([]database.LiveRoute, []database.LiveRoute) {
ipv4Routes, ipv6Routes := []database.LiveRoute{}, []database.LiveRoute{}
routes []LiveRoute,
) ([]LiveRoute, []LiveRoute) {
ipv4Routes, ipv6Routes := []LiveRoute{}, []LiveRoute{}
for _, route := range routes {
if route.IPVersion == ipVersionV4 {
@@ -1256,8 +1256,8 @@ func splitRoutesByIPVersion(
// sortRoutesByMaskLength sorts routes by mask length, shortest first, and
// routes with the same mask length by network address.
func sortRoutesByMaskLength(routes []database.LiveRoute) {
slices.SortFunc(routes, func(a, b database.LiveRoute) int {
func sortRoutesByMaskLength(routes []LiveRoute) {
slices.SortFunc(routes, func(a, b LiveRoute) int {
prefixA, _ := netip.ParsePrefix(a.Prefix)
prefixB, _ := netip.ParsePrefix(b.Prefix)
@@ -1294,7 +1294,7 @@ func (s *Server) handlePrefixDetailJSON() http.HandlerFunc {
origins := s.table.Origins(parsed)
if len(origins) == 0 {
s.writeJSONError(w, http.StatusNotFound,
database.ErrNoRoute.Error()+": "+prefix)
ErrNoRoute.Error()+": "+prefix)
return
}
@@ -1302,7 +1302,7 @@ func (s *Server) handlePrefixDetailJSON() http.HandlerFunc {
routes := prefixRoutes(parsed, origins)
// Group by origin AS
originMap := make(map[int][]database.LiveRoute)
originMap := make(map[int][]LiveRoute)
for _, route := range routes {
originMap[route.OriginASN] = append(originMap[route.OriginASN], route)
}
@@ -1392,7 +1392,7 @@ func (s *Server) handleASDetail() http.HandlerFunc {
}
if err != nil {
if errors.Is(err, database.ErrNoRoute) {
if errors.Is(err, ErrNoRoute) {
http.Error(w, "AS not found", http.StatusNotFound)
} else {
s.logger.Error("Failed to get AS details", "error", err)
@@ -1425,8 +1425,8 @@ func (s *Server) handleASDetail() http.HandlerFunc {
// Prepare template data
data := struct {
ASN *database.ASN
IPv4Prefixes []database.LiveRoute
IPv6Prefixes []database.LiveRoute
IPv4Prefixes []LiveRoute
IPv6Prefixes []LiveRoute
TotalCount int
IPv4Count int
IPv6Count int
@@ -1585,11 +1585,11 @@ func (s *Server) originASNs(
// them, since the routing table keeps only their addresses.
func prefixRoutes(
prefix netip.Prefix, origins []routetable.Origin,
) []database.LiveRoute {
var routes []database.LiveRoute
) []LiveRoute {
var routes []LiveRoute
for _, origin := range origins {
base := database.LiveRoute{
base := LiveRoute{
Prefix: prefix.String(),
MaskLength: prefix.Bits(),
IPVersion: ipVersionOf(prefix),
@@ -1630,14 +1630,14 @@ type ASPathEntry struct {
// EnhancedRoute is a live route with the handle of each AS in its path.
type EnhancedRoute struct {
database.LiveRoute
LiveRoute
ASPathWithHandle []ASPathEntry
}
// routesWithASPathHandles looks up the handle of each AS in the path of each
// route.
func routesWithASPathHandles(routes []database.LiveRoute) []EnhancedRoute {
func routesWithASPathHandles(routes []LiveRoute) []EnhancedRoute {
enhancedRoutes := make([]EnhancedRoute, len(routes))
for i, route := range routes {
@@ -1736,7 +1736,7 @@ func (s *Server) prefixLengthHandler(ipVersion, maxMaskLength int) http.HandlerF
// EnhancedPrefix is a prefix on the prefix length page, with the origin AS
// that most peers carry it with, that AS's description, and its age.
type EnhancedPrefix struct {
database.LiveRoute
LiveRoute
OriginASDescription string
Age string
internal/server/handlers_internal_test.go
+12 -30
View File
@@ -158,8 +158,8 @@ type tableStats struct {
OldestRoute time.Time `json:"oldest_route"`
NewestRoute time.Time `json:"newest_route"`
IPv4Distribution []database.PrefixDistribution `json:"ipv4_prefix_distribution"`
IPv6Distribution []database.PrefixDistribution `json:"ipv6_prefix_distribution"`
IPv4Distribution []PrefixDistribution `json:"ipv4_prefix_distribution"`
IPv6Distribution []PrefixDistribution `json:"ipv6_prefix_distribution"`
}
// TestStatsHandlersAnswerFromMemory checks that both stats handlers answer 200
@@ -211,8 +211,8 @@ func TestStatsHandlersAnswerFromMemory(t *testing.T) {
OriginsWithoutKeptRoute: 1,
OldestRoute: testRouteTime(0),
NewestRoute: testRouteTime(4),
IPv4Distribution: []database.PrefixDistribution{{MaskLength: 24, Count: 2}},
IPv6Distribution: []database.PrefixDistribution{{MaskLength: 32, Count: 1}},
IPv4Distribution: []PrefixDistribution{{MaskLength: 24, Count: 2}},
IPv6Distribution: []PrefixDistribution{{MaskLength: 32, Count: 1}},
}
for name, handler := range handlers {
@@ -422,27 +422,9 @@ func TestIPLookupsReadPercentEncodedAddresses(t *testing.T) {
}
}
// errASDetailsLookupFailed is what failingASDetailsDB answers every AS details
// lookup with.
var errASDetailsLookupFailed = errors.New("AS details lookup failed")
// failingASDetailsDB wraps a real store and fails every AS details lookup.
type failingASDetailsDB struct {
database.Store
}
func (failingASDetailsDB) GetASDetailsContext(
_ context.Context, _ int,
) (*database.ASN, []database.LiveRoute, error) {
return nil, nil, errASDetailsLookupFailed
}
// TestPrefixDetailShowsOriginNames checks that the prefix page shows each
// origin AS's handle and description without looking up the AS's details,
// which also read every route the AS originates. Before the fix the page did
// that lookup once per origin, so with it failing the names were missing
// (https://git.eeqj.de/sneak/routewatch/issues/67). The AS path column shows
// handles too, so the test looks for the origin list's "(handle)" form.
// origin AS's handle and description. The AS path column shows handles too, so
// the test looks for the origin list's "(handle)" form.
func TestPrefixDetailShowsOriginNames(t *testing.T) {
t.Parallel()
@@ -465,7 +447,7 @@ func TestPrefixDetailShowsOriginNames(t *testing.T) {
announce(table, testPrefixV4, asn, testPeer(i+1), testRouteTime(0))
}
s := New(failingASDetailsDB{db}, table,
s := New(db, table,
streamer.New(logger.New(), metrics.New()), logger.New(), &config.Config{})
rec := httptest.NewRecorder()
s.router.ServeHTTP(rec, httptest.NewRequestWithContext(
@@ -1716,9 +1698,9 @@ func TestASDetailJSONAnswersBuiltInAS(t *testing.T) {
//nolint:tagliatelle // field names are the existing JSON API
var body struct {
Data struct {
ASN database.ASN `json:"asn"`
IPv4Prefixes []database.LiveRoute `json:"ipv4_prefixes"`
IPv6Prefixes []database.LiveRoute `json:"ipv6_prefixes"`
ASN database.ASN `json:"asn"`
IPv4Prefixes []LiveRoute `json:"ipv4_prefixes"`
IPv6Prefixes []LiveRoute `json:"ipv6_prefixes"`
} `json:"data"`
}
@@ -1863,9 +1845,9 @@ func TestSortRoutesByMaskLength(t *testing.T) {
}
for _, tt := range tests {
routes := make([]database.LiveRoute, 0, len(tt.prefixes))
routes := make([]LiveRoute, 0, len(tt.prefixes))
for _, prefix := range tt.prefixes {
routes = append(routes, database.LiveRoute{Prefix: prefix})
routes = append(routes, LiveRoute{Prefix: prefix})
}
sortRoutesByMaskLength(routes)
internal/server/handlers_table_internal_test.go
+29
View File
@@ -11,6 +11,7 @@ import (
"time"
"git.eeqj.de/sneak/routewatch/internal/database"
"git.eeqj.de/sneak/routewatch/internal/ristypes"
"git.eeqj.de/sneak/routewatch/internal/routetable"
)
@@ -134,6 +135,34 @@ func TestIPLookupGivesTheOriginMostPeersCarry(t *testing.T) {
}
}
// TestIPLookupAsksForNoWHOISLookupOfOriginZero checks that the IP lookup asks
// for no WHOIS lookup of AS 0, which means no known origin. It asks for one of
// an AS whose WHOIS details were never fetched or are stale, and not of one
// whose details are fresh.
func TestIPLookupAsksForNoWHOISLookupOfOriginZero(t *testing.T) {
t.Parallel()
fresh := time.Now().Add(-time.Hour)
stale := time.Now().Add(-2 * whoisStaleThreshold)
tests := []struct {
name string
asInfo database.ASN
want bool
}{
{"no known origin", database.ASN{ASN: ristypes.NoOrigin}, false},
{"details never fetched", database.ASN{ASN: mostPeersASN}, true},
{"stale details", database.ASN{ASN: mostPeersASN, WHOISUpdatedAt: &stale}, true},
{"fresh details", database.ASN{ASN: mostPeersASN, WHOISUpdatedAt: &fresh}, false},
}
for _, tt := range tests {
if got := needsWHOISRefresh(&tt.asInfo); got != tt.want {
t.Errorf("%s: needsWHOISRefresh = %t, want %t", tt.name, got, tt.want)
}
}
}
// prefixRoute is one route of the prefix API's answer. A field the answer
// leaves out stays nil.
//
internal/server/models.go
+67
View File
@@ -0,0 +1,67 @@
package server
import (
"errors"
"time"
)
var (
// ErrInvalidIP says that an IP address is malformed. The IP lookup returns
// it.
ErrInvalidIP = errors.New("invalid IP address")
// ErrNoRoute says that the routing table has no route for an address, a
// prefix or an AS. The lookups return it.
ErrNoRoute = errors.New("no route found")
)
// LiveRoute represents a route in the live routing table. JSON leaves out an
// empty AS path, next hop or time: the routing table in memory keeps them only
// for the two newest routes of each prefix and origin AS.
//
//nolint:tagliatelle // field names are the existing JSON API
type LiveRoute struct {
Prefix string `json:"prefix"`
MaskLength int `json:"mask_length"`
IPVersion int `json:"ip_version"`
OriginASN int `json:"origin_asn"`
PeerIP string `json:"peer_ip"`
ASPath []int `json:"as_path,omitempty"`
NextHop string `json:"next_hop,omitempty"`
LastUpdated time.Time `json:"last_updated,omitzero"`
}
// PrefixDistribution represents the distribution of prefixes by mask length
//
//nolint:tagliatelle // field names are the existing JSON API
type PrefixDistribution struct {
MaskLength int `json:"mask_length"`
Count int `json:"count"`
}
// IPInfo represents comprehensive IP information for the /ip endpoint
//
//nolint:tagliatelle // field names are the existing JSON API
type IPInfo struct {
IP string `json:"ip"`
Netblock string `json:"netblock"`
MaskLength int `json:"mask_length"`
IPVersion int `json:"ip_version"`
NumPeers int `json:"num_peers"`
// AS information
ASN int `json:"asn"`
ASName string `json:"as_name,omitempty"`
Handle string `json:"handle,omitempty"`
Description string `json:"description,omitempty"`
OrgName string `json:"org_name,omitempty"`
OrgID string `json:"org_id,omitempty"`
Address string `json:"address,omitempty"`
CountryCode string `json:"country_code,omitempty"`
AbuseEmail string `json:"abuse_email,omitempty"`
RIR string `json:"rir,omitempty"`
// Timestamps. LastSeen is zero, and left out of JSON, when the prefix
// keeps no route.
FirstSeen time.Time `json:"first_seen"`
LastSeen time.Time `json:"last_seen,omitzero"`
// Indicates if WHOIS data needs refresh (not serialized)
NeedsWHOISRefresh bool `json:"-"`
}
internal/templates/templates_test.go
+5 -4
View File
@@ -7,6 +7,7 @@ import (
"testing"
"git.eeqj.de/sneak/routewatch/internal/database"
"git.eeqj.de/sneak/routewatch/internal/server"
"git.eeqj.de/sneak/routewatch/internal/templates"
)
@@ -114,14 +115,14 @@ func TestASDetailPrefixListsFold(t *testing.T) {
prefixV6 = "2001:db8::/32"
)
ipv4 := []database.LiveRoute{{Prefix: prefixV4}}
ipv6 := []database.LiveRoute{{Prefix: prefixV6}}
ipv4 := []server.LiveRoute{{Prefix: prefixV4}}
ipv6 := []server.LiveRoute{{Prefix: prefixV6}}
// The fields the AS page handler gives the template.
data := struct {
ASN *database.ASN
IPv4Prefixes []database.LiveRoute
IPv6Prefixes []database.LiveRoute
IPv4Prefixes []server.LiveRoute
IPv6Prefixes []server.LiveRoute
TotalCount int
IPv4Count int
IPv6Count int