Sample the prefix-length pages with bounded lookups, each prefix once (closes #66)

/prefixlength/{length} and /prefixlength6/{length} grouped every live route
of that length by prefix, shuffled all the prefixes and joined the routes
back, so a request grew with the number of routes and listed a prefix once
for each peer that shared its newest time.

Now a length with at most the page's 500 prefixes has all of them listed. A
longer one is sampled: each lookup draws a random row id and takes the next
route of that length from the mask length index, at most four lookups per
prefix asked for. Each picked prefix is shown once, with its newest route.

Model: opus-5-5

TODO.md

@@ -27,6 +27,11 @@ real 5 GiB limit (https://git.eeqj.de/sneak/routewatch/issues/3).
 
 # Completed Steps
 
+- 2026-10-14: `/prefixlength/{length}` and `/prefixlength6/{length}` no
+  longer read every route of that length: a length with at most 500
+  prefixes has all of them listed, a longer one is sampled with random
+  lookups on the mask length index, and each prefix is listed once, with
+  its newest route (closes #66)
 - 2026-10-03: `/api/v1/stats` serves the prefix distribution from memory,
   seeded at startup and adjusted on every live-route write, so a request no
   longer reads every live route (closes #30)

internal/database/database.go

@@ -54,16 +54,12 @@ const (
 	sqliteHardHeapLimitBytes = 1610612736
 )
 
-// Bounds on the work GetRandomPrefixesByLength does to pick its sample.
+// sampleLookupsPerPrefix bounds the random lookups GetRandomPrefixesByLength
-const (
+// makes when a mask length has more prefixes than it lists: at most this many
-	// maxPrefixesToShuffle is the most prefixes of one mask length that are
+// for each prefix asked for. A lookup that lands on a prefix already picked is
-	// read in full and shuffled to pick the sample. A mask length with more is
+// spent, so a sample can come back a little short when there are only a few
-	// sampled with random lookups, which then seldom land on a prefix twice.
+// more prefixes than asked for, or their routes are spread unevenly.
-	maxPrefixesToShuffle = 5000
+const sampleLookupsPerPrefix = 4
-	// sampleLookupsPerPrefix is the most random lookups made for each prefix
-	// asked for.
-	sampleLookupsPerPrefix = 4
-)
 
 // Common errors
 var (
@@ -1708,11 +1704,10 @@ func (d *Database) GetRandomPrefixesByLength(maskLength, ipVersion, limit int) (
 	return d.GetRandomPrefixesByLengthContext(context.Background(), maskLength, ipVersion, limit)
 }
 
-// GetRandomPrefixesByLengthContext returns limit prefixes with the given mask
+// GetRandomPrefixesByLengthContext returns up to limit distinct prefixes with
-// length picked at random, or all of them when there are fewer, each once with
+// the given mask length, each with its most recently updated live route. Its
-// its most recently updated live route. However many routes there are, it
+// work grows with limit, not with the number of routes: a mask length with at
-// reads at most maxPrefixesToShuffle+1 prefixes and makes at most
+// most limit prefixes has all of them listed, any other is sampled.
-// sampleLookupsPerPrefix lookups for each prefix asked for.
 func (d *Database) GetRandomPrefixesByLengthContext(
 	ctx context.Context, maskLength, ipVersion, limit int) ([]LiveRoute, error) {
 	tableName := "live_routes_v4"
@@ -1720,22 +1715,18 @@ func (d *Database) GetRandomPrefixesByLengthContext(
 		tableName = "live_routes_v6"
 	}
 
-	// Reading one prefix more than maxPrefixesToShuffle tells whether these
+	// Reading one prefix more than limit tells whether there are more prefixes
-	// are all the prefixes with this mask length.
+	// to choose from than the page lists.
-	prefixes, err := d.firstPrefixesByLength(ctx, tableName, maskLength, maxPrefixesToShuffle+1)
+	prefixes, err := d.firstPrefixesByLength(ctx, tableName, maskLength, limit+1)
 	if err != nil {
 		return nil, err
 	}
-	rand.Shuffle(len(prefixes), func(i, j int) {
+	if len(prefixes) > limit {
-		prefixes[i], prefixes[j] = prefixes[j], prefixes[i]
+		prefixes, err = d.samplePrefixesByLength(ctx, tableName, maskLength, limit)
-	})
-	if len(prefixes) > maxPrefixesToShuffle {
-		prefixes, err = d.samplePrefixesByLength(ctx, tableName, maskLength, limit, prefixes)
 		if err != nil {
 			return nil, err
 		}
 	}
-	prefixes = prefixes[:min(limit, len(prefixes))]
 
 	return d.newestRouteOfEachPrefix(ctx, tableName, ipVersion, prefixes)
 }
@@ -1774,20 +1765,18 @@ func (d *Database) firstPrefixesByLength(
 	return prefixes, nil
 }
 
-// samplePrefixesByLength picks limit distinct prefixes with the given mask
+// samplePrefixesByLength picks up to limit distinct prefixes with the given
-// length at random, in at most sampleLookupsPerPrefix lookups for each prefix
+// mask length at random, in at most sampleLookupsPerPrefix lookups for each
-// asked for. ORDER BY RANDOM() would read every route with that mask length.
+// prefix asked for. ORDER BY RANDOM() would read every route with that mask
-// Instead each lookup draws a row id between those of the first and last route
+// length. Instead each lookup draws a row id between those of the first and
-// with that mask length and takes the prefix of the first such route at or
+// last route with that mask length and takes the prefix of the first such
-// after it. SQLite keeps the row id at the end of every index entry, so the
+// route at or after it. SQLite keeps the row id at the end of every index
-// mask length index is ordered by mask length, then row id, and the lookup is
+// entry, so the mask length index is ordered by mask length, then row id, and
-// one search of it. A prefix with more routes, or with more rows of other mask
+// the lookup is one search of it. A prefix with more routes, or with more rows
-// lengths just before its own, comes up more often: the page needs a varied
+// of other mask lengths just before its own, comes up more often: the page
-// sample, not an even one. Should the lookups keep landing on prefixes already
+// needs a varied sample, not an even one.
-// picked, the rest are taken from shuffled, prefixes of that length in random
-// order.
 func (d *Database) samplePrefixesByLength(
-	ctx context.Context, tableName string, maskLength, limit int, shuffled []string) ([]string, error) {
+	ctx context.Context, tableName string, maskLength, limit int) ([]string, error) {
 	//nolint:gosec // Table name is hardcoded based on IP version
 	rangeQuery := fmt.Sprintf(`
 		SELECT
@@ -1820,13 +1809,6 @@ func (d *Database) samplePrefixesByLength(
 
 	picked := make(map[string]bool, limit)
 	prefixes := make([]string, 0, limit)
-	pick := func(prefix string) {
-		if !picked[prefix] {
-			picked[prefix] = true
-			prefixes = append(prefixes, prefix)
-		}
-	}
-
 	for lookups := 0; lookups < limit*sampleLookupsPerPrefix && len(prefixes) < limit; lookups++ {
 		//nolint:gosec // The sample needs variety, not unpredictability.
 		rowID := first.Int64 + rand.Int64N(last.Int64-first.Int64+1)
@@ -1840,14 +1822,10 @@ func (d *Database) samplePrefixesByLength(
 		if err != nil {
 			return nil, fmt.Errorf("failed to look up prefix: %w", err)
 		}
-		pick(prefix)
+		if !picked[prefix] {
+			picked[prefix] = true
+			prefixes = append(prefixes, prefix)
 		}
-
-	for _, prefix := range shuffled {
-		if len(prefixes) == limit {
-			break
-		}
-		pick(prefix)
 	}
 
 	return prefixes, nil

internal/database/database_test.go

@@ -6,7 +6,6 @@ import (
 	"errors"
 	"fmt"
 	"net/netip"
-	"slices"
 	"sync"
 	"testing"
 	"time"
@@ -367,42 +366,10 @@ func TestGetRandomPrefixesByLengthListsPrefixOnce(t *testing.T) {
 	}
 }
 
-// TestGetRandomPrefixesByLengthListsNewestRoute stores a prefix learned from
-// two peers at different times, the older route first, and checks that the
-// prefix-length sample lists the newer route.
-func TestGetRandomPrefixesByLengthListsNewestRoute(t *testing.T) {
-	cfg := &config.Config{StateDir: t.TempDir()}
-
-	db, err := New(cfg, logger.New())
-	if err != nil {
-		t.Fatalf("failed to create database: %v", err)
-	}
-	defer func() { _ = db.Close() }()
-
-	older := time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC)
-	olderRoute := mkV4Route(t, "198.51.100.0/24", 64500, older)
-	newerRoute := mkV4Route(t, "198.51.100.0/24", 64500, older.Add(time.Minute))
-	newerRoute.PeerIP = secondV4PeerIP
-	if err := db.UpsertLiveRouteBatch([]*LiveRoute{olderRoute, newerRoute}); err != nil {
-		t.Fatalf("UpsertLiveRouteBatch: %v", err)
-	}
-
-	routes, err := db.GetRandomPrefixesByLength(newerRoute.MaskLength, ipVersionV4, sampleLimit)
-	if err != nil {
-		t.Fatalf("GetRandomPrefixesByLength: %v", err)
-	}
-	if len(routes) != 1 || routes[0].PeerIP != newerRoute.PeerIP {
-		t.Errorf("GetRandomPrefixesByLength = %+v, want only the newer route, from %s",
-			routes, newerRoute.PeerIP)
-	}
-}
-
 // TestGetRandomPrefixesByLengthSamplesManyPrefixes stores more /24 prefixes
-// than the sample asks for, each followed by a /16 so that routes of another
+// than the sample asks for, each learned from two peers in the same second,
-// mask length sit between the /24 rows, and each prefix learned from two peers
+// plus a /16, and checks that the sample lists as many prefixes as it asks
-// in the same second. It checks both the sample the pages get, picked from the
+// for, each once and each a /24.
-// shuffled prefixes, and the random lookups used for a mask length with more
-// than maxPrefixesToShuffle prefixes.
 func TestGetRandomPrefixesByLengthSamplesManyPrefixes(t *testing.T) {
 	cfg := &config.Config{StateDir: t.TempDir()}
 
@@ -414,58 +381,32 @@ func TestGetRandomPrefixesByLengthSamplesManyPrefixes(t *testing.T) {
 
 	ts := time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC)
 	var routes []*LiveRoute
-	var stored []string
 	for i := range sampledPrefixCount {
 		prefix := fmt.Sprintf("10.0.%d.0/24", i)
-		for _, p := range []string{prefix, fmt.Sprintf("172.%d.0.0/16", i)} {
+		first := mkV4Route(t, prefix, 64500, ts)
-			first := mkV4Route(t, p, 64500, ts)
+		second := mkV4Route(t, prefix, 64500, ts)
-			second := mkV4Route(t, p, 64500, ts)
 		second.PeerIP = secondV4PeerIP
 		routes = append(routes, first, second)
 	}
-		stored = append(stored, prefix)
+	routes = append(routes, mkV4Route(t, "10.1.0.0/16", 64501, ts))
-	}
 	if err := db.UpsertLiveRouteBatch(routes); err != nil {
 		t.Fatalf("UpsertLiveRouteBatch: %v", err)
 	}
-	slices.Sort(stored)
 
 	maskLength := routes[0].MaskLength
 	sample, err := db.GetRandomPrefixesByLength(maskLength, ipVersionV4, sampleLimit)
 	if err != nil {
 		t.Fatalf("GetRandomPrefixesByLength: %v", err)
 	}
-	var listed []string
-	for _, route := range sample {
-		listed = append(listed, route.Prefix)
-	}
-	checkPrefixSample(t, "GetRandomPrefixesByLength", listed, stored)
-
-	looked, err := db.samplePrefixesByLength(
-		context.Background(), "live_routes_v4", maskLength, sampleLimit, nil)
-	if err != nil {
-		t.Fatalf("samplePrefixesByLength: %v", err)
-	}
-	checkPrefixSample(t, "samplePrefixesByLength", looked, stored)
-}
-
-// checkPrefixSample checks that sample lists sampleLimit prefixes, each once
-// and each one of stored, and that it is not just the first sampleLimit of
-// stored, which is in text order.
-func checkPrefixSample(t *testing.T, name string, sample, stored []string) {
-	t.Helper()
-
 	if len(sample) != sampleLimit {
-		t.Errorf("%s listed %d prefixes, want %d", name, len(sample), sampleLimit)
+		t.Errorf("sample listed %d routes, want %d", len(sample), sampleLimit)
 	}
 	listed := make(map[string]bool)
-	for _, prefix := range sample {
+	for _, route := range sample {
-		if listed[prefix] || !slices.Contains(stored, prefix) {
+		if listed[route.Prefix] || route.MaskLength != maskLength {
-			t.Errorf("%s lists %s twice or it is not one of the /24s", name, prefix)
+			t.Errorf("sample lists %s (mask %d) twice or at the wrong mask length",
+				route.Prefix, route.MaskLength)
 		}
-		listed[prefix] = true
+		listed[route.Prefix] = true
-	}
-	if slices.Equal(slices.Sorted(slices.Values(sample)), stored[:sampleLimit]) {
-		t.Errorf("%s lists just the first %d prefixes in text order", name, sampleLimit)
 	}
 }

internal/metrics/metrics.go

@@ -60,24 +60,19 @@ func New() *Tracker {
 	}
 }
 
-// SetConnected updates the connection status. Every connection after the
+// SetConnected updates the connection status
-// first counts one reconnect.
 func (t *Tracker) SetConnected(connected bool) {
-	t.isConnected.Store(connected)
+	wasConnected := t.isConnected.Swap(connected)
-	if !connected {
+	if connected {
-		return
-	}
-
 		t.mu.Lock()
-	// connectedSince is zero until the first connection.
-	connectedBefore := !t.connectedSince.IsZero()
 		t.connectedSince = time.Now()
 		t.mu.Unlock()
-
+		// Increment reconnect count (but not for the initial connection)
-	if connectedBefore {
+		if wasConnected || t.reconnectCount.Load() > 0 {
 			t.reconnectCount.Add(1)
 		}
 	}
+}
 
 // GetReconnectCount returns the number of reconnections since startup
 func (t *Tracker) GetReconnectCount() uint64 {

internal/metrics/metrics_test.go

@@ -1,21 +0,0 @@
-package metrics
-
-import "testing"
-
-// TestReconnectCount connects three times with a disconnect between each, the
-// way the streamer does, and expects every connection after the first to count
-// one reconnect.
-func TestReconnectCount(t *testing.T) {
-	tracker := New()
-
-	tracker.SetConnected(true)
-	tracker.SetConnected(false)
-	tracker.SetConnected(true)
-	tracker.SetConnected(false)
-	tracker.SetConnected(true)
-
-	const wantReconnects = 2
-	if got := tracker.GetReconnectCount(); got != wantReconnects {
-		t.Fatalf("reconnect count = %d, want %d", got, wantReconnects)
-	}
-}