From d45c805384a0130847131aa85de27add8574d549 Mon Sep 17 00:00:00 2001 From: sneak Date: Wed, 14 Oct 2026 13:15:21 +0000 Subject: [PATCH] Sample the prefix-length pages with bounded lookups, each prefix once (closes #66) /prefixlength/{length} and /prefixlength6/{length} grouped every live route of that length by prefix, shuffled all the prefixes and joined the routes back, so a request grew with the number of routes and listed a prefix once for each peer that shared its newest time. Now a length with at most the page's 500 prefixes has all of them listed. A longer one is sampled: each lookup draws a random row id and takes the next route of that length from the mask length index, at most four lookups per prefix asked for. Each picked prefix is shown once, with its newest route. Model: opus-5-5 --- TODO.md | 5 + internal/database/database.go | 178 ++++++++++++++++++++++++----- internal/database/database_test.go | 93 +++++++++++++++ 3 files changed, 247 insertions(+), 29 deletions(-) diff --git a/TODO.md b/TODO.md index d0b7920..688aa05 100644 --- a/TODO.md +++ b/TODO.md @@ -27,6 +27,11 @@ real 5 GiB limit (https://git.eeqj.de/sneak/routewatch/issues/3). # Completed Steps +- 2026-10-14: `/prefixlength/{length}` and `/prefixlength6/{length}` no + longer read every route of that length: a length with at most 500 + prefixes has all of them listed, a longer one is sampled with random + lookups on the mask length index, and each prefix is listed once, with + its newest route (closes #66) - 2026-10-03: `/api/v1/stats` serves the prefix distribution from memory, seeded at startup and adjusted on every live-route write, so a request no longer reads every live route (closes #30) diff --git a/internal/database/database.go b/internal/database/database.go index 5393f74..5701af0 100644 --- a/internal/database/database.go +++ b/internal/database/database.go @@ -8,6 +8,7 @@ import ( "encoding/json" "errors" "fmt" + "math/rand/v2" "net" "net/netip" "os" @@ -53,6 +54,13 @@ const ( sqliteHardHeapLimitBytes = 1610612736 ) +// sampleLookupsPerPrefix bounds the random lookups GetRandomPrefixesByLength +// makes when a mask length has more prefixes than it lists: at most this many +// for each prefix asked for. A lookup that lands on a prefix already picked is +// spent, so a sample can come back a little short when there are only a few +// more prefixes than asked for, or their routes are spread unevenly. +const sampleLookupsPerPrefix = 4 + // Common errors var ( // ErrInvalidIP is returned when an IP address is malformed @@ -1696,45 +1704,158 @@ func (d *Database) GetRandomPrefixesByLength(maskLength, ipVersion, limit int) ( return d.GetRandomPrefixesByLengthContext(context.Background(), maskLength, ipVersion, limit) } -// GetRandomPrefixesByLengthContext returns a random sample of prefixes with context support +// GetRandomPrefixesByLengthContext returns up to limit distinct prefixes with +// the given mask length, each with its most recently updated live route. Its +// work grows with limit, not with the number of routes: a mask length with at +// most limit prefixes has all of them listed, any other is sampled. func (d *Database) GetRandomPrefixesByLengthContext( ctx context.Context, maskLength, ipVersion, limit int) ([]LiveRoute, error) { - // Select unique prefixes with their most recent route information tableName := "live_routes_v4" if ipVersion == ipVersionV6 { tableName = "live_routes_v6" } + // Reading one prefix more than limit tells whether there are more prefixes + // to choose from than the page lists. + prefixes, err := d.firstPrefixesByLength(ctx, tableName, maskLength, limit+1) + if err != nil { + return nil, err + } + if len(prefixes) > limit { + prefixes, err = d.samplePrefixesByLength(ctx, tableName, maskLength, limit) + if err != nil { + return nil, err + } + } + + return d.newestRouteOfEachPrefix(ctx, tableName, ipVersion, prefixes) +} + +// firstPrefixesByLength returns the first n distinct prefixes, in text order, +// with the given mask length. It reads only their entries in the mask length +// and prefix index. +func (d *Database) firstPrefixesByLength( + ctx context.Context, tableName string, maskLength, n int) ([]string, error) { //nolint:gosec // Table name is hardcoded based on IP version query := fmt.Sprintf(` - WITH unique_prefixes AS ( - SELECT prefix, MAX(last_updated) as max_updated - FROM %s - WHERE mask_length = ? - GROUP BY prefix - ORDER BY RANDOM() - LIMIT ? - ) - SELECT lr.prefix, lr.mask_length, lr.origin_asn, lr.as_path, - lr.peer_ip, lr.last_updated - FROM %s lr - INNER JOIN unique_prefixes up ON lr.prefix = up.prefix AND lr.last_updated = up.max_updated - WHERE lr.mask_length = ? - `, tableName, tableName) + SELECT DISTINCT prefix FROM %s + WHERE mask_length = ? + ORDER BY prefix + LIMIT ? + `, tableName) - rows, err := d.db.QueryContext(ctx, query, maskLength, limit, maskLength) + rows, err := d.db.QueryContext(ctx, query, maskLength, n) if err != nil { - return nil, fmt.Errorf("failed to query random prefixes: %w", err) + return nil, fmt.Errorf("failed to list prefixes: %w", err) } - defer func() { - _ = rows.Close() - }() + defer func() { _ = rows.Close() }() - var routes []LiveRoute + var prefixes []string for rows.Next() { - var route LiveRoute + var prefix string + if err := rows.Scan(&prefix); err != nil { + return nil, fmt.Errorf("failed to scan prefix: %w", err) + } + prefixes = append(prefixes, prefix) + } + if err := rows.Err(); err != nil { + return nil, fmt.Errorf("failed to list prefixes: %w", err) + } + + return prefixes, nil +} + +// samplePrefixesByLength picks up to limit distinct prefixes with the given +// mask length at random, in at most sampleLookupsPerPrefix lookups for each +// prefix asked for. ORDER BY RANDOM() would read every route with that mask +// length. Instead each lookup draws a row id between those of the first and +// last route with that mask length and takes the prefix of the first such +// route at or after it. SQLite keeps the row id at the end of every index +// entry, so the mask length index is ordered by mask length, then row id, and +// the lookup is one search of it. A prefix with more routes, or with more rows +// of other mask lengths just before its own, comes up more often: the page +// needs a varied sample, not an even one. +func (d *Database) samplePrefixesByLength( + ctx context.Context, tableName string, maskLength, limit int) ([]string, error) { + //nolint:gosec // Table name is hardcoded based on IP version + rangeQuery := fmt.Sprintf(` + SELECT + (SELECT MIN(rowid) FROM %[1]s WHERE mask_length = ?), + (SELECT MAX(rowid) FROM %[1]s WHERE mask_length = ?) + `, tableName) + + var first, last sql.NullInt64 + if err := d.db.QueryRowContext(ctx, rangeQuery, maskLength, maskLength).Scan(&first, &last); err != nil { + return nil, fmt.Errorf("failed to read row id range: %w", err) + } + if !first.Valid { + // Every route with this mask length was withdrawn since they were listed. + return nil, nil + } + + //nolint:gosec // Table name is hardcoded based on IP version + lookupQuery := fmt.Sprintf(` + SELECT prefix FROM %s + WHERE mask_length = ? AND rowid >= ? + ORDER BY rowid + LIMIT 1 + `, tableName) + + stmt, err := d.db.PrepareContext(ctx, lookupQuery) + if err != nil { + return nil, fmt.Errorf("failed to prepare prefix lookup: %w", err) + } + defer func() { _ = stmt.Close() }() + + picked := make(map[string]bool, limit) + prefixes := make([]string, 0, limit) + for lookups := 0; lookups < limit*sampleLookupsPerPrefix && len(prefixes) < limit; lookups++ { + //nolint:gosec // The sample needs variety, not unpredictability. + rowID := first.Int64 + rand.Int64N(last.Int64-first.Int64+1) + + var prefix string + err := stmt.QueryRowContext(ctx, maskLength, rowID).Scan(&prefix) + if errors.Is(err, sql.ErrNoRows) { + // The routes from rowID on were withdrawn since the range was read. + continue + } + if err != nil { + return nil, fmt.Errorf("failed to look up prefix: %w", err) + } + if !picked[prefix] { + picked[prefix] = true + prefixes = append(prefixes, prefix) + } + } + + return prefixes, nil +} + +// newestRouteOfEachPrefix returns the most recently updated live route of each +// prefix, one lookup on the prefix index per prefix. A prefix whose routes were +// all withdrawn since it was picked is left out. +func (d *Database) newestRouteOfEachPrefix( + ctx context.Context, tableName string, ipVersion int, prefixes []string) ([]LiveRoute, error) { + //nolint:gosec // Table name is hardcoded based on IP version + query := fmt.Sprintf(` + SELECT prefix, mask_length, origin_asn, as_path, peer_ip, last_updated + FROM %s + WHERE prefix = ? + ORDER BY last_updated DESC + LIMIT 1 + `, tableName) + + stmt, err := d.db.PrepareContext(ctx, query) + if err != nil { + return nil, fmt.Errorf("failed to prepare route query: %w", err) + } + defer func() { _ = stmt.Close() }() + + routes := make([]LiveRoute, 0, len(prefixes)) + for _, prefix := range prefixes { + route := LiveRoute{IPVersion: ipVersion} var pathJSON string - err := rows.Scan( + err := stmt.QueryRowContext(ctx, prefix).Scan( &route.Prefix, &route.MaskLength, &route.OriginASN, @@ -1742,17 +1863,16 @@ func (d *Database) GetRandomPrefixesByLengthContext( &route.PeerIP, &route.LastUpdated, ) - // Set IP version based on which table we queried - route.IPVersion = ipVersion - if err != nil { + if errors.Is(err, sql.ErrNoRows) { continue } + if err != nil { + return nil, fmt.Errorf("failed to query route of %s: %w", prefix, err) + } - // Decode AS path if err := json.Unmarshal([]byte(pathJSON), &route.ASPath); err != nil { route.ASPath = []int{} } - routes = append(routes, route) } diff --git a/internal/database/database_test.go b/internal/database/database_test.go index 82753a9..af408c9 100644 --- a/internal/database/database_test.go +++ b/internal/database/database_test.go @@ -45,6 +45,19 @@ const ( asnSecondBand = 100 ) +// Parameters for the prefix-length sample tests. +const ( + // sampleLimit is how many prefixes the tests ask the sample for. + sampleLimit = 10 + // sampledPrefixCount is how many prefixes the sampling test stores, more + // than sampleLimit so the sample has to choose among them. + sampledPrefixCount = 30 + // secondV4PeerIP and secondV6PeerIP are the second peer each prefix is + // learned from, besides the one mkV4Route and mkV6Route set. + secondV4PeerIP = "192.0.2.2" + secondV6PeerIP = "2001:db8::2" +) + // TestGetIPInfoFindsMostSpecificLiveRoute stores nested live prefixes for both // families and checks that a lookup returns the most specific one covering the // address, ErrNoRoute when none covers it, and the next less specific prefix @@ -317,3 +330,83 @@ func TestBatchWriteDuringCheckpoint(t *testing.T) { cancel() wg.Wait() } + +// TestGetRandomPrefixesByLengthListsPrefixOnce stores one prefix of each +// family learned from two peers in the same second and checks that the +// prefix-length sample lists each of them once. +func TestGetRandomPrefixesByLengthListsPrefixOnce(t *testing.T) { + cfg := &config.Config{StateDir: t.TempDir()} + + db, err := New(cfg, logger.New()) + if err != nil { + t.Fatalf("failed to create database: %v", err) + } + defer func() { _ = db.Close() }() + + ts := time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC) + v4First := mkV4Route(t, "198.51.100.0/24", 64500, ts) + v4Second := mkV4Route(t, "198.51.100.0/24", 64500, ts) + v4Second.PeerIP = secondV4PeerIP + v6First := mkV6Route("2001:db8::/32", 64500, ts) + v6Second := mkV6Route("2001:db8::/32", 64500, ts) + v6Second.PeerIP = secondV6PeerIP + if err := db.UpsertLiveRouteBatch([]*LiveRoute{v4First, v4Second, v6First, v6Second}); err != nil { + t.Fatalf("UpsertLiveRouteBatch: %v", err) + } + + for _, want := range []*LiveRoute{v4First, v6First} { + routes, err := db.GetRandomPrefixesByLength(want.MaskLength, want.IPVersion, sampleLimit) + if err != nil { + t.Fatalf("GetRandomPrefixesByLength(%d, %d): %v", want.MaskLength, want.IPVersion, err) + } + if len(routes) != 1 || routes[0].Prefix != want.Prefix { + t.Errorf("GetRandomPrefixesByLength(%d, %d) = %+v, want %s listed once", + want.MaskLength, want.IPVersion, routes, want.Prefix) + } + } +} + +// TestGetRandomPrefixesByLengthSamplesManyPrefixes stores more /24 prefixes +// than the sample asks for, each learned from two peers in the same second, +// plus a /16, and checks that the sample lists as many prefixes as it asks +// for, each once and each a /24. +func TestGetRandomPrefixesByLengthSamplesManyPrefixes(t *testing.T) { + cfg := &config.Config{StateDir: t.TempDir()} + + db, err := New(cfg, logger.New()) + if err != nil { + t.Fatalf("failed to create database: %v", err) + } + defer func() { _ = db.Close() }() + + ts := time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC) + var routes []*LiveRoute + for i := range sampledPrefixCount { + prefix := fmt.Sprintf("10.0.%d.0/24", i) + first := mkV4Route(t, prefix, 64500, ts) + second := mkV4Route(t, prefix, 64500, ts) + second.PeerIP = secondV4PeerIP + routes = append(routes, first, second) + } + routes = append(routes, mkV4Route(t, "10.1.0.0/16", 64501, ts)) + if err := db.UpsertLiveRouteBatch(routes); err != nil { + t.Fatalf("UpsertLiveRouteBatch: %v", err) + } + + maskLength := routes[0].MaskLength + sample, err := db.GetRandomPrefixesByLength(maskLength, ipVersionV4, sampleLimit) + if err != nil { + t.Fatalf("GetRandomPrefixesByLength: %v", err) + } + if len(sample) != sampleLimit { + t.Errorf("sample listed %d routes, want %d", len(sample), sampleLimit) + } + listed := make(map[string]bool) + for _, route := range sample { + if listed[route.Prefix] || route.MaskLength != maskLength { + t.Errorf("sample lists %s (mask %d) twice or at the wrong mask length", + route.Prefix, route.MaskLength) + } + listed[route.Prefix] = true + } +}