Sample the prefix-length pages with bounded lookups, each prefix once (closes #66)

/prefixlength/{length} and /prefixlength6/{length} grouped every live route
of that length by prefix, shuffled all the prefixes and joined the routes
back, so a request grew with the number of routes and listed a prefix once
for each peer that shared its newest time.

Now a length with at most the page's 500 prefixes has all of them listed. A
longer one is sampled: each lookup draws a random row id and takes the next
route of that length from the mask length index, at most four lookups per
prefix asked for. Each picked prefix is shown once, with its newest route.

Model: opus-5-5

TODO.md

@@ -27,6 +27,11 @@ real 5 GiB limit (https://git.eeqj.de/sneak/routewatch/issues/3).
 
 # Completed Steps
 
+- 2026-10-14: `/prefixlength/{length}` and `/prefixlength6/{length}` no
+  longer read every route of that length: a length with at most 500
+  prefixes has all of them listed, a longer one is sampled with random
+  lookups on the mask length index, and each prefix is listed once, with
+  its newest route (closes #66)
 - 2026-10-03: `/api/v1/stats` serves the prefix distribution from memory,
   seeded at startup and adjusted on every live-route write, so a request no
   longer reads every live route (closes #30)

internal/database/database.go

@@ -54,16 +54,12 @@ const (
 	sqliteHardHeapLimitBytes = 1610612736
 )
 
-// Bounds on the work GetRandomPrefixesByLength does to pick its sample.
-const (
-	// maxPrefixesToShuffle is the most prefixes of one mask length that are
-	// read in full and shuffled to pick the sample. A mask length with more is
-	// sampled with random lookups, which then seldom land on a prefix twice.
-	maxPrefixesToShuffle = 5000
-	// sampleLookupsPerPrefix is the most random lookups made for each prefix
-	// asked for.
-	sampleLookupsPerPrefix = 4
-)
+// sampleLookupsPerPrefix bounds the random lookups GetRandomPrefixesByLength
+// makes when a mask length has more prefixes than it lists: at most this many
+// for each prefix asked for. A lookup that lands on a prefix already picked is
+// spent, so a sample can come back a little short when there are only a few
+// more prefixes than asked for, or their routes are spread unevenly.
+const sampleLookupsPerPrefix = 4
 
 // Common errors
 var (
@@ -1708,11 +1704,10 @@ func (d *Database) GetRandomPrefixesByLength(maskLength, ipVersion, limit int) (
 	return d.GetRandomPrefixesByLengthContext(context.Background(), maskLength, ipVersion, limit)
 }
 
-// GetRandomPrefixesByLengthContext returns limit prefixes with the given mask
-// length picked at random, or all of them when there are fewer, each once with
-// its most recently updated live route. However many routes there are, it
-// reads at most maxPrefixesToShuffle+1 prefixes and makes at most
-// sampleLookupsPerPrefix lookups for each prefix asked for.
+// GetRandomPrefixesByLengthContext returns up to limit distinct prefixes with
+// the given mask length, each with its most recently updated live route. Its
+// work grows with limit, not with the number of routes: a mask length with at
+// most limit prefixes has all of them listed, any other is sampled.
 func (d *Database) GetRandomPrefixesByLengthContext(
 	ctx context.Context, maskLength, ipVersion, limit int) ([]LiveRoute, error) {
 	tableName := "live_routes_v4"
@@ -1720,22 +1715,18 @@ func (d *Database) GetRandomPrefixesByLengthContext(
 		tableName = "live_routes_v6"
 	}
 
-	// Reading one prefix more than maxPrefixesToShuffle tells whether these
-	// are all the prefixes with this mask length.
-	prefixes, err := d.firstPrefixesByLength(ctx, tableName, maskLength, maxPrefixesToShuffle+1)
+	// Reading one prefix more than limit tells whether there are more prefixes
+	// to choose from than the page lists.
+	prefixes, err := d.firstPrefixesByLength(ctx, tableName, maskLength, limit+1)
 	if err != nil {
 		return nil, err
 	}
-	rand.Shuffle(len(prefixes), func(i, j int) {
-		prefixes[i], prefixes[j] = prefixes[j], prefixes[i]
-	})
-	if len(prefixes) > maxPrefixesToShuffle {
-		prefixes, err = d.samplePrefixesByLength(ctx, tableName, maskLength, limit, prefixes)
+	if len(prefixes) > limit {
+		prefixes, err = d.samplePrefixesByLength(ctx, tableName, maskLength, limit)
 		if err != nil {
 			return nil, err
 		}
 	}
-	prefixes = prefixes[:min(limit, len(prefixes))]
 
 	return d.newestRouteOfEachPrefix(ctx, tableName, ipVersion, prefixes)
 }
@@ -1774,20 +1765,18 @@ func (d *Database) firstPrefixesByLength(
 	return prefixes, nil
 }
 
-// samplePrefixesByLength picks limit distinct prefixes with the given mask
-// length at random, in at most sampleLookupsPerPrefix lookups for each prefix
-// asked for. ORDER BY RANDOM() would read every route with that mask length.
-// Instead each lookup draws a row id between those of the first and last route
-// with that mask length and takes the prefix of the first such route at or
-// after it. SQLite keeps the row id at the end of every index entry, so the
-// mask length index is ordered by mask length, then row id, and the lookup is
-// one search of it. A prefix with more routes, or with more rows of other mask
-// lengths just before its own, comes up more often: the page needs a varied
-// sample, not an even one. Should the lookups keep landing on prefixes already
-// picked, the rest are taken from shuffled, prefixes of that length in random
-// order.
+// samplePrefixesByLength picks up to limit distinct prefixes with the given
+// mask length at random, in at most sampleLookupsPerPrefix lookups for each
+// prefix asked for. ORDER BY RANDOM() would read every route with that mask
+// length. Instead each lookup draws a row id between those of the first and
+// last route with that mask length and takes the prefix of the first such
+// route at or after it. SQLite keeps the row id at the end of every index
+// entry, so the mask length index is ordered by mask length, then row id, and
+// the lookup is one search of it. A prefix with more routes, or with more rows
+// of other mask lengths just before its own, comes up more often: the page
+// needs a varied sample, not an even one.
 func (d *Database) samplePrefixesByLength(
-	ctx context.Context, tableName string, maskLength, limit int, shuffled []string) ([]string, error) {
+	ctx context.Context, tableName string, maskLength, limit int) ([]string, error) {
 	//nolint:gosec // Table name is hardcoded based on IP version
 	rangeQuery := fmt.Sprintf(`
 		SELECT
@@ -1820,13 +1809,6 @@ func (d *Database) samplePrefixesByLength(
 
 	picked := make(map[string]bool, limit)
 	prefixes := make([]string, 0, limit)
-	pick := func(prefix string) {
-		if !picked[prefix] {
-			picked[prefix] = true
-			prefixes = append(prefixes, prefix)
-		}
-	}
-
 	for lookups := 0; lookups < limit*sampleLookupsPerPrefix && len(prefixes) < limit; lookups++ {
 		//nolint:gosec // The sample needs variety, not unpredictability.
 		rowID := first.Int64 + rand.Int64N(last.Int64-first.Int64+1)
@@ -1840,14 +1822,10 @@ func (d *Database) samplePrefixesByLength(
 		if err != nil {
 			return nil, fmt.Errorf("failed to look up prefix: %w", err)
 		}
-		pick(prefix)
-	}
-
-	for _, prefix := range shuffled {
-		if len(prefixes) == limit {
-			break
+		if !picked[prefix] {
+			picked[prefix] = true
+			prefixes = append(prefixes, prefix)
 		}
-		pick(prefix)
 	}
 
 	return prefixes, nil

internal/database/database_test.go

@@ -6,7 +6,6 @@ import (
 	"errors"
 	"fmt"
 	"net/netip"
-	"slices"
 	"sync"
 	"testing"
 	"time"
@@ -367,42 +366,10 @@ func TestGetRandomPrefixesByLengthListsPrefixOnce(t *testing.T) {
 	}
 }
 
-// TestGetRandomPrefixesByLengthListsNewestRoute stores a prefix learned from
-// two peers at different times, the older route first, and checks that the
-// prefix-length sample lists the newer route.
-func TestGetRandomPrefixesByLengthListsNewestRoute(t *testing.T) {
-	cfg := &config.Config{StateDir: t.TempDir()}
-
-	db, err := New(cfg, logger.New())
-	if err != nil {
-		t.Fatalf("failed to create database: %v", err)
-	}
-	defer func() { _ = db.Close() }()
-
-	older := time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC)
-	olderRoute := mkV4Route(t, "198.51.100.0/24", 64500, older)
-	newerRoute := mkV4Route(t, "198.51.100.0/24", 64500, older.Add(time.Minute))
-	newerRoute.PeerIP = secondV4PeerIP
-	if err := db.UpsertLiveRouteBatch([]*LiveRoute{olderRoute, newerRoute}); err != nil {
-		t.Fatalf("UpsertLiveRouteBatch: %v", err)
-	}
-
-	routes, err := db.GetRandomPrefixesByLength(newerRoute.MaskLength, ipVersionV4, sampleLimit)
-	if err != nil {
-		t.Fatalf("GetRandomPrefixesByLength: %v", err)
-	}
-	if len(routes) != 1 || routes[0].PeerIP != newerRoute.PeerIP {
-		t.Errorf("GetRandomPrefixesByLength = %+v, want only the newer route, from %s",
-			routes, newerRoute.PeerIP)
-	}
-}
-
 // TestGetRandomPrefixesByLengthSamplesManyPrefixes stores more /24 prefixes
-// than the sample asks for, each followed by a /16 so that routes of another
-// mask length sit between the /24 rows, and each prefix learned from two peers
-// in the same second. It checks both the sample the pages get, picked from the
-// shuffled prefixes, and the random lookups used for a mask length with more
-// than maxPrefixesToShuffle prefixes.
+// than the sample asks for, each learned from two peers in the same second,
+// plus a /16, and checks that the sample lists as many prefixes as it asks
+// for, each once and each a /24.
 func TestGetRandomPrefixesByLengthSamplesManyPrefixes(t *testing.T) {
 	cfg := &config.Config{StateDir: t.TempDir()}
 
@@ -414,58 +381,32 @@ func TestGetRandomPrefixesByLengthSamplesManyPrefixes(t *testing.T) {
 
 	ts := time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC)
 	var routes []*LiveRoute
-	var stored []string
 	for i := range sampledPrefixCount {
 		prefix := fmt.Sprintf("10.0.%d.0/24", i)
-		for _, p := range []string{prefix, fmt.Sprintf("172.%d.0.0/16", i)} {
-			first := mkV4Route(t, p, 64500, ts)
-			second := mkV4Route(t, p, 64500, ts)
-			second.PeerIP = secondV4PeerIP
-			routes = append(routes, first, second)
-		}
-		stored = append(stored, prefix)
+		first := mkV4Route(t, prefix, 64500, ts)
+		second := mkV4Route(t, prefix, 64500, ts)
+		second.PeerIP = secondV4PeerIP
+		routes = append(routes, first, second)
 	}
+	routes = append(routes, mkV4Route(t, "10.1.0.0/16", 64501, ts))
 	if err := db.UpsertLiveRouteBatch(routes); err != nil {
 		t.Fatalf("UpsertLiveRouteBatch: %v", err)
 	}
-	slices.Sort(stored)
 
 	maskLength := routes[0].MaskLength
 	sample, err := db.GetRandomPrefixesByLength(maskLength, ipVersionV4, sampleLimit)
 	if err != nil {
 		t.Fatalf("GetRandomPrefixesByLength: %v", err)
 	}
-	var listed []string
-	for _, route := range sample {
-		listed = append(listed, route.Prefix)
-	}
-	checkPrefixSample(t, "GetRandomPrefixesByLength", listed, stored)
-
-	looked, err := db.samplePrefixesByLength(
-		context.Background(), "live_routes_v4", maskLength, sampleLimit, nil)
-	if err != nil {
-		t.Fatalf("samplePrefixesByLength: %v", err)
-	}
-	checkPrefixSample(t, "samplePrefixesByLength", looked, stored)
-}
-
-// checkPrefixSample checks that sample lists sampleLimit prefixes, each once
-// and each one of stored, and that it is not just the first sampleLimit of
-// stored, which is in text order.
-func checkPrefixSample(t *testing.T, name string, sample, stored []string) {
-	t.Helper()
-
 	if len(sample) != sampleLimit {
-		t.Errorf("%s listed %d prefixes, want %d", name, len(sample), sampleLimit)
+		t.Errorf("sample listed %d routes, want %d", len(sample), sampleLimit)
 	}
 	listed := make(map[string]bool)
-	for _, prefix := range sample {
-		if listed[prefix] || !slices.Contains(stored, prefix) {
-			t.Errorf("%s lists %s twice or it is not one of the /24s", name, prefix)
+	for _, route := range sample {
+		if listed[route.Prefix] || route.MaskLength != maskLength {
+			t.Errorf("sample lists %s (mask %d) twice or at the wrong mask length",
+				route.Prefix, route.MaskLength)
 		}
-		listed[prefix] = true
-	}
-	if slices.Equal(slices.Sorted(slices.Values(sample)), stored[:sampleLimit]) {
-		t.Errorf("%s lists just the first %d prefixes in text order", name, sampleLimit)
+		listed[route.Prefix] = true
 	}
 }

internal/metrics/metrics.go

@@ -60,22 +60,17 @@ func New() *Tracker {
 	}
 }
 
-// SetConnected updates the connection status. Every connection after the
-// first counts one reconnect.
+// SetConnected updates the connection status
 func (t *Tracker) SetConnected(connected bool) {
-	t.isConnected.Store(connected)
-	if !connected {
-		return
-	}
-
-	t.mu.Lock()
-	// connectedSince is zero until the first connection.
-	connectedBefore := !t.connectedSince.IsZero()
-	t.connectedSince = time.Now()
-	t.mu.Unlock()
-
-	if connectedBefore {
-		t.reconnectCount.Add(1)
+	wasConnected := t.isConnected.Swap(connected)
+	if connected {
+		t.mu.Lock()
+		t.connectedSince = time.Now()
+		t.mu.Unlock()
+		// Increment reconnect count (but not for the initial connection)
+		if wasConnected || t.reconnectCount.Load() > 0 {
+			t.reconnectCount.Add(1)
+		}
 	}
 }
 

internal/metrics/metrics_test.go

@@ -1,21 +0,0 @@
-package metrics
-
-import "testing"
-
-// TestReconnectCount connects three times with a disconnect between each, the
-// way the streamer does, and expects every connection after the first to count
-// one reconnect.
-func TestReconnectCount(t *testing.T) {
-	tracker := New()
-
-	tracker.SetConnected(true)
-	tracker.SetConnected(false)
-	tracker.SetConnected(true)
-	tracker.SetConnected(false)
-	tracker.SetConnected(true)
-
-	const wantReconnects = 2
-	if got := tracker.GetReconnectCount(); got != wantReconnects {
-		t.Fatalf("reconnect count = %d, want %d", got, wantReconnects)
-	}
-}