Sample the prefix-length pages with bounded lookups, each prefix once (closes #66)

/prefixlength/{length} and /prefixlength6/{length} grouped every live route
of that length by prefix, shuffled all the prefixes and joined the routes
back, so a request grew with the number of routes and listed a prefix once
for each peer that shared its newest time.

Now a length with at most 5000 prefixes has them all read and shuffled, and
the page lists 500 of them. A longer one is sampled with random row-id
lookups on the mask length index, at most four per prefix asked for, filled
up from the shuffled prefixes if the lookups keep repeating. Each prefix is
shown once, with its newest route.

Model: opus-5-5

README.md

@@ -141,7 +141,10 @@ routewatch/
 - **Batched Writes**: All database operations are batched for performance
 - **Backpressure**: Probabilistic message dropping when queues exceed 50% capacity
 - **Graceful Shutdown**: 60-second timeout, flushes all pending batches
-- **Reconnection**: Exponential backoff (5s-320s) with reset after 30s of stable connection
+- **Reconnection**: Any end of the stream other than shutdown, including RIS Live
+  ending the response normally, marks the feed disconnected and reconnects with
+  exponential backoff (5s-320s). The backoff resets after a stream that lasted
+  more than 30s. A stream with no line for 60s is dropped and reconnected.
 - **IP Lookup**: the most specific live route is found by looking up the
   address's prefix at each mask length, longest first, in the prefix index
   (at most 25 lookups for IPv4, 113 for IPv6); prefixes are stored in the

internal/database/database.go

@@ -8,6 +8,7 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
+	"math/rand/v2"
 	"net"
 	"net/netip"
 	"os"
@@ -61,6 +62,17 @@ const (
 	sqliteHardHeapLimitBytes = 1610612736
 )
 
+// Bounds on the work GetRandomPrefixesByLength does to pick its sample.
+const (
+	// maxPrefixesToShuffle is the most prefixes of one mask length that are
+	// read in full and shuffled to pick the sample. A mask length with more is
+	// sampled with random lookups.
+	maxPrefixesToShuffle = 5000
+	// sampleLookupsPerPrefix is the most random lookups made for each prefix
+	// asked for.
+	sampleLookupsPerPrefix = 4
+)
+
 // Common errors
 var (
 	// ErrInvalidIP is returned when an IP address is malformed
@@ -1711,45 +1723,183 @@ func (d *Database) GetRandomPrefixesByLength(maskLength, ipVersion, limit int) (
 	return d.GetRandomPrefixesByLengthContext(context.Background(), maskLength, ipVersion, limit)
 }
 
-// GetRandomPrefixesByLengthContext returns a random sample of prefixes with context support
+// GetRandomPrefixesByLengthContext returns limit prefixes with the given mask
+// length picked at random, or all of them when there are fewer, each once with
+// its most recently updated live route, for a limit up to
+// maxPrefixesToShuffle. However many routes there are, it reads at most
+// maxPrefixesToShuffle+1 prefixes and makes at most sampleLookupsPerPrefix
+// lookups for each prefix asked for.
 func (d *Database) GetRandomPrefixesByLengthContext(
 	ctx context.Context, maskLength, ipVersion, limit int) ([]LiveRoute, error) {
-	// Select unique prefixes with their most recent route information
 	tableName := "live_routes_v4"
 	if ipVersion == ipVersionV6 {
 		tableName = "live_routes_v6"
 	}
 
+	// Reading one prefix more than maxPrefixesToShuffle tells whether these
+	// are all the prefixes with this mask length.
+	prefixes, err := d.firstPrefixesByLength(ctx, tableName, maskLength, maxPrefixesToShuffle+1)
+	if err != nil {
+		return nil, err
+	}
+	rand.Shuffle(len(prefixes), func(i, j int) {
+		prefixes[i], prefixes[j] = prefixes[j], prefixes[i]
+	})
+	if len(prefixes) > maxPrefixesToShuffle {
+		prefixes, err = d.samplePrefixesByLength(ctx, tableName, maskLength, limit, prefixes)
+		if err != nil {
+			return nil, err
+		}
+	}
+	prefixes = prefixes[:min(limit, len(prefixes))]
+
+	return d.newestRouteOfEachPrefix(ctx, tableName, ipVersion, prefixes)
+}
+
+// firstPrefixesByLength returns the first n distinct prefixes, in text order,
+// with the given mask length. It reads only their entries in the mask length
+// and prefix index.
+func (d *Database) firstPrefixesByLength(
+	ctx context.Context, tableName string, maskLength, n int) ([]string, error) {
 	//nolint:gosec // Table name is hardcoded based on IP version
 	query := fmt.Sprintf(`
-		WITH unique_prefixes AS (
+		SELECT DISTINCT prefix FROM %s
-			SELECT prefix, MAX(last_updated) as max_updated
+		WHERE mask_length = ?
-			FROM %s
+		ORDER BY prefix
-			WHERE mask_length = ?
+		LIMIT ?
-			GROUP BY prefix
+	`, tableName)
-			ORDER BY RANDOM()
-			LIMIT ?
-		)
-		SELECT lr.prefix, lr.mask_length, lr.origin_asn, lr.as_path, 
-			lr.peer_ip, lr.last_updated
-		FROM %s lr
-		INNER JOIN unique_prefixes up ON lr.prefix = up.prefix AND lr.last_updated = up.max_updated
-		WHERE lr.mask_length = ?
-	`, tableName, tableName)
 
-	rows, err := d.db.QueryContext(ctx, query, maskLength, limit, maskLength)
+	rows, err := d.db.QueryContext(ctx, query, maskLength, n)
 	if err != nil {
-		return nil, fmt.Errorf("failed to query random prefixes: %w", err)
+		return nil, fmt.Errorf("failed to list prefixes: %w", err)
 	}
-	defer func() {
+	defer func() { _ = rows.Close() }()
-		_ = rows.Close()
-	}()
 
-	var routes []LiveRoute
+	var prefixes []string
 	for rows.Next() {
-		var route LiveRoute
+		var prefix string
+		if err := rows.Scan(&prefix); err != nil {
+			return nil, fmt.Errorf("failed to scan prefix: %w", err)
+		}
+		prefixes = append(prefixes, prefix)
+	}
+	if err := rows.Err(); err != nil {
+		return nil, fmt.Errorf("failed to list prefixes: %w", err)
+	}
+
+	return prefixes, nil
+}
+
+// samplePrefixesByLength picks limit distinct prefixes with the given mask
+// length at random, in at most sampleLookupsPerPrefix lookups for each prefix
+// asked for. ORDER BY RANDOM() would read every route with that mask length.
+// Instead each lookup draws a row id between those of the first and last route
+// with that mask length and takes the prefix of the first such route at or
+// after it. SQLite keeps the row id at the end of every index entry, so the
+// mask length index is ordered by mask length, then row id, and the lookup is
+// one search of it.
+//
+// The sample is uneven, as the page needs a varied sample, not an even one: a
+// route is picked in proportion to the row ids between it and the previous
+// route with that mask length, including the row ids of routes with other mask
+// lengths and of withdrawn routes. A route that follows a long run of
+// withdrawn routes is picked far more often than one that directly follows
+// another route with that mask length.
+//
+// Should the lookups keep landing on prefixes already picked, the rest are
+// taken from shuffled, prefixes of that length in random order.
+func (d *Database) samplePrefixesByLength(
+	ctx context.Context, tableName string, maskLength, limit int, shuffled []string) ([]string, error) {
+	//nolint:gosec // Table name is hardcoded based on IP version
+	rangeQuery := fmt.Sprintf(`
+		SELECT
+			(SELECT MIN(rowid) FROM %[1]s WHERE mask_length = ?),
+			(SELECT MAX(rowid) FROM %[1]s WHERE mask_length = ?)
+	`, tableName)
+
+	var first, last sql.NullInt64
+	if err := d.db.QueryRowContext(ctx, rangeQuery, maskLength, maskLength).Scan(&first, &last); err != nil {
+		return nil, fmt.Errorf("failed to read row id range: %w", err)
+	}
+	if !first.Valid {
+		// Every route with this mask length was withdrawn since they were listed.
+		return nil, nil
+	}
+
+	//nolint:gosec // Table name is hardcoded based on IP version
+	lookupQuery := fmt.Sprintf(`
+		SELECT prefix FROM %s
+		WHERE mask_length = ? AND rowid >= ?
+		ORDER BY rowid
+		LIMIT 1
+	`, tableName)
+
+	stmt, err := d.db.PrepareContext(ctx, lookupQuery)
+	if err != nil {
+		return nil, fmt.Errorf("failed to prepare prefix lookup: %w", err)
+	}
+	defer func() { _ = stmt.Close() }()
+
+	picked := make(map[string]bool, limit)
+	prefixes := make([]string, 0, limit)
+	pick := func(prefix string) {
+		if !picked[prefix] {
+			picked[prefix] = true
+			prefixes = append(prefixes, prefix)
+		}
+	}
+
+	for lookups := 0; lookups < limit*sampleLookupsPerPrefix && len(prefixes) < limit; lookups++ {
+		//nolint:gosec // The sample needs variety, not unpredictability.
+		rowID := first.Int64 + rand.Int64N(last.Int64-first.Int64+1)
+
+		var prefix string
+		err := stmt.QueryRowContext(ctx, maskLength, rowID).Scan(&prefix)
+		if errors.Is(err, sql.ErrNoRows) {
+			// The routes from rowID on were withdrawn since the range was read.
+			continue
+		}
+		if err != nil {
+			return nil, fmt.Errorf("failed to look up prefix: %w", err)
+		}
+		pick(prefix)
+	}
+
+	for _, prefix := range shuffled {
+		if len(prefixes) == limit {
+			break
+		}
+		pick(prefix)
+	}
+
+	return prefixes, nil
+}
+
+// newestRouteOfEachPrefix returns the most recently updated live route of each
+// prefix, one lookup on the prefix index per prefix. A prefix whose routes were
+// all withdrawn since it was picked is left out.
+func (d *Database) newestRouteOfEachPrefix(
+	ctx context.Context, tableName string, ipVersion int, prefixes []string) ([]LiveRoute, error) {
+	//nolint:gosec // Table name is hardcoded based on IP version
+	query := fmt.Sprintf(`
+		SELECT prefix, mask_length, origin_asn, as_path, peer_ip, last_updated
+		FROM %s
+		WHERE prefix = ?
+		ORDER BY last_updated DESC
+		LIMIT 1
+	`, tableName)
+
+	stmt, err := d.db.PrepareContext(ctx, query)
+	if err != nil {
+		return nil, fmt.Errorf("failed to prepare route query: %w", err)
+	}
+	defer func() { _ = stmt.Close() }()
+
+	routes := make([]LiveRoute, 0, len(prefixes))
+	for _, prefix := range prefixes {
+		route := LiveRoute{IPVersion: ipVersion}
 		var pathJSON string
-		err := rows.Scan(
+		err := stmt.QueryRowContext(ctx, prefix).Scan(
 			&route.Prefix,
 			&route.MaskLength,
 			&route.OriginASN,
@@ -1757,17 +1907,16 @@ func (d *Database) GetRandomPrefixesByLengthContext(
 			&route.PeerIP,
 			&route.LastUpdated,
 		)
-		// Set IP version based on which table we queried
+		if errors.Is(err, sql.ErrNoRows) {
-		route.IPVersion = ipVersion
-		if err != nil {
 			continue
 		}
+		if err != nil {
+			return nil, fmt.Errorf("failed to query route of %s: %w", prefix, err)
+		}
 
-		// Decode AS path
 		if err := json.Unmarshal([]byte(pathJSON), &route.ASPath); err != nil {
 			route.ASPath = []int{}
 		}
-
 		routes = append(routes, route)
 	}
 

internal/database/database_test.go

@@ -6,6 +6,7 @@ import (
 	"errors"
 	"fmt"
 	"net/netip"
+	"slices"
 	"sync"
 	"testing"
 	"time"
@@ -45,6 +46,32 @@ const (
 	asnSecondBand = 100
 )
 
+// Parameters for the prefix-length sample tests.
+const (
+	// sampleLimit is how many prefixes the tests ask the sample for.
+	sampleLimit = 10
+	// sampledPrefixCount is how many prefixes the sampling test stores, more
+	// than sampleLimit so the sample has to choose among them.
+	sampledPrefixCount = 30
+	// secondV4PeerIP and secondV6PeerIP are the second peer each prefix is
+	// learned from, besides the one mkV4Route and mkV6Route set.
+	secondV4PeerIP = "192.0.2.2"
+	secondV6PeerIP = "2001:db8::2"
+	// largeSamplePrefixCount is how many prefixes the test of a mask length
+	// with more than maxPrefixesToShuffle prefixes stores: twice that, so about
+	// half of them come after the first maxPrefixesToShuffle+1 in text order.
+	largeSamplePrefixCount = 2 * maxPrefixesToShuffle
+	// largeSampleLimit is how many prefixes that test asks for. Each random
+	// lookup lands after the first maxPrefixesToShuffle+1 in text order about
+	// half the time, so none of them doing so is about as likely as
+	// largeSampleLimit coin tosses all coming up heads.
+	largeSampleLimit = 30
+	// liveRoutesV4Table and liveRoutesV6Table are the live route tables the
+	// tests pass to the sampling steps.
+	liveRoutesV4Table = "live_routes_v4"
+	liveRoutesV6Table = "live_routes_v6"
+)
+
 // TestGetIPInfoFindsMostSpecificLiveRoute stores nested live prefixes for both
 // families and checks that a lookup returns the most specific one covering the
 // address, ErrNoRoute when none covers it, and the next less specific prefix
@@ -389,3 +416,281 @@ func TestBatchWriteDuringCheckpoint(t *testing.T) {
 	cancel()
 	wg.Wait()
 }
+
+// TestGetRandomPrefixesByLengthListsPrefixOnce stores one prefix of each
+// family learned from two peers in the same second and checks that the
+// prefix-length sample lists each of them once.
+func TestGetRandomPrefixesByLengthListsPrefixOnce(t *testing.T) {
+	cfg := &config.Config{StateDir: t.TempDir()}
+
+	db, err := New(cfg, logger.New())
+	if err != nil {
+		t.Fatalf("failed to create database: %v", err)
+	}
+	defer func() { _ = db.Close() }()
+
+	ts := time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC)
+	v4First := mkV4Route(t, "198.51.100.0/24", 64500, ts)
+	v4Second := mkV4Route(t, "198.51.100.0/24", 64500, ts)
+	v4Second.PeerIP = secondV4PeerIP
+	v6First := mkV6Route("2001:db8::/32", 64500, ts)
+	v6Second := mkV6Route("2001:db8::/32", 64500, ts)
+	v6Second.PeerIP = secondV6PeerIP
+	if err := db.UpsertLiveRouteBatch([]*LiveRoute{v4First, v4Second, v6First, v6Second}); err != nil {
+		t.Fatalf("UpsertLiveRouteBatch: %v", err)
+	}
+
+	for _, want := range []*LiveRoute{v4First, v6First} {
+		routes, err := db.GetRandomPrefixesByLength(want.MaskLength, want.IPVersion, sampleLimit)
+		if err != nil {
+			t.Fatalf("GetRandomPrefixesByLength(%d, %d): %v", want.MaskLength, want.IPVersion, err)
+		}
+		if len(routes) != 1 || routes[0].Prefix != want.Prefix {
+			t.Errorf("GetRandomPrefixesByLength(%d, %d) = %+v, want %s listed once",
+				want.MaskLength, want.IPVersion, routes, want.Prefix)
+		}
+	}
+}
+
+// TestGetRandomPrefixesByLengthListsNewestRoute stores a prefix learned from
+// two peers at different times, the older route first, and checks that the
+// prefix-length sample lists the newer route.
+func TestGetRandomPrefixesByLengthListsNewestRoute(t *testing.T) {
+	cfg := &config.Config{StateDir: t.TempDir()}
+
+	db, err := New(cfg, logger.New())
+	if err != nil {
+		t.Fatalf("failed to create database: %v", err)
+	}
+	defer func() { _ = db.Close() }()
+
+	older := time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC)
+	olderRoute := mkV4Route(t, "198.51.100.0/24", 64500, older)
+	newerRoute := mkV4Route(t, "198.51.100.0/24", 64500, older.Add(time.Minute))
+	newerRoute.PeerIP = secondV4PeerIP
+	if err := db.UpsertLiveRouteBatch([]*LiveRoute{olderRoute, newerRoute}); err != nil {
+		t.Fatalf("UpsertLiveRouteBatch: %v", err)
+	}
+
+	routes, err := db.GetRandomPrefixesByLength(newerRoute.MaskLength, ipVersionV4, sampleLimit)
+	if err != nil {
+		t.Fatalf("GetRandomPrefixesByLength: %v", err)
+	}
+	if len(routes) != 1 || routes[0].PeerIP != newerRoute.PeerIP {
+		t.Errorf("GetRandomPrefixesByLength = %+v, want only the newer route, from %s",
+			routes, newerRoute.PeerIP)
+	}
+}
+
+// TestGetRandomPrefixesByLengthSamplesManyPrefixes stores more /24 prefixes
+// than the sample asks for, each followed by a /16 so that routes of another
+// mask length sit between the /24 rows, and each prefix learned from two peers
+// in the same second. It checks both the sample the pages get, picked from the
+// shuffled prefixes, and the random lookups used for a mask length with more
+// than maxPrefixesToShuffle prefixes.
+func TestGetRandomPrefixesByLengthSamplesManyPrefixes(t *testing.T) {
+	cfg := &config.Config{StateDir: t.TempDir()}
+
+	db, err := New(cfg, logger.New())
+	if err != nil {
+		t.Fatalf("failed to create database: %v", err)
+	}
+	defer func() { _ = db.Close() }()
+
+	ts := time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC)
+	var routes []*LiveRoute
+	var stored []string
+	for i := range sampledPrefixCount {
+		prefix := fmt.Sprintf("10.0.%d.0/24", i)
+		for _, p := range []string{prefix, fmt.Sprintf("172.%d.0.0/16", i)} {
+			first := mkV4Route(t, p, 64500, ts)
+			second := mkV4Route(t, p, 64500, ts)
+			second.PeerIP = secondV4PeerIP
+			routes = append(routes, first, second)
+		}
+		stored = append(stored, prefix)
+	}
+	if err := db.UpsertLiveRouteBatch(routes); err != nil {
+		t.Fatalf("UpsertLiveRouteBatch: %v", err)
+	}
+	slices.Sort(stored)
+
+	maskLength := routes[0].MaskLength
+	sample, err := db.GetRandomPrefixesByLength(maskLength, ipVersionV4, sampleLimit)
+	if err != nil {
+		t.Fatalf("GetRandomPrefixesByLength: %v", err)
+	}
+	var listed []string
+	for _, route := range sample {
+		listed = append(listed, route.Prefix)
+	}
+	checkPrefixSample(t, "GetRandomPrefixesByLength", listed, stored)
+
+	looked, err := db.samplePrefixesByLength(
+		context.Background(), liveRoutesV4Table, maskLength, sampleLimit, nil)
+	if err != nil {
+		t.Fatalf("samplePrefixesByLength: %v", err)
+	}
+	checkPrefixSample(t, "samplePrefixesByLength", looked, stored)
+}
+
+// checkPrefixSample checks that sample lists sampleLimit prefixes, each once
+// and each one of stored, and that it is not just the first sampleLimit of
+// stored, which is in text order.
+func checkPrefixSample(t *testing.T, name string, sample, stored []string) {
+	t.Helper()
+
+	if len(sample) != sampleLimit {
+		t.Errorf("%s listed %d prefixes, want %d", name, len(sample), sampleLimit)
+	}
+	listed := make(map[string]bool)
+	for _, prefix := range sample {
+		if listed[prefix] || !slices.Contains(stored, prefix) {
+			t.Errorf("%s lists %s twice or it is not one of the /24s", name, prefix)
+		}
+		listed[prefix] = true
+	}
+	if slices.Equal(slices.Sorted(slices.Values(sample)), stored[:sampleLimit]) {
+		t.Errorf("%s lists just the first %d prefixes in text order", name, sampleLimit)
+	}
+}
+
+// TestGetRandomPrefixesByLengthSamplesPastFirstPrefixes stores more prefixes
+// of one mask length than maxPrefixesToShuffle. It checks that
+// firstPrefixesByLength reads just the first maxPrefixesToShuffle+1 in text
+// order, and that the sample lists the prefixes asked for, each once, some of
+// them after those first ones, which only the random lookups reach.
+func TestGetRandomPrefixesByLengthSamplesPastFirstPrefixes(t *testing.T) {
+	cfg := &config.Config{StateDir: t.TempDir()}
+
+	db, err := New(cfg, logger.New())
+	if err != nil {
+		t.Fatalf("failed to create database: %v", err)
+	}
+	defer func() { _ = db.Close() }()
+
+	ts := time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC)
+	var routes []*LiveRoute
+	var stored []string
+	for i := range largeSamplePrefixCount {
+		route := mkV6Route(fmt.Sprintf("2001:%x::/32", i), 64500, ts)
+		routes = append(routes, route)
+		stored = append(stored, route.Prefix)
+	}
+	if err := db.UpsertLiveRouteBatch(routes); err != nil {
+		t.Fatalf("UpsertLiveRouteBatch: %v", err)
+	}
+	slices.Sort(stored)
+	firstPrefixes := stored[:maxPrefixesToShuffle+1]
+
+	maskLength := routes[0].MaskLength
+	read, err := db.firstPrefixesByLength(
+		context.Background(), liveRoutesV6Table, maskLength, maxPrefixesToShuffle+1)
+	if err != nil {
+		t.Fatalf("firstPrefixesByLength: %v", err)
+	}
+	if !slices.Equal(read, firstPrefixes) {
+		t.Errorf("firstPrefixesByLength read %d prefixes, want the first %d in text order",
+			len(read), len(firstPrefixes))
+	}
+
+	sample, err := db.GetRandomPrefixesByLength(maskLength, ipVersionV6, largeSampleLimit)
+	if err != nil {
+		t.Fatalf("GetRandomPrefixesByLength: %v", err)
+	}
+	if len(sample) != largeSampleLimit {
+		t.Errorf("GetRandomPrefixesByLength listed %d prefixes, want %d", len(sample), largeSampleLimit)
+	}
+	listed := make(map[string]bool)
+	pastFirstPrefixes := 0
+	for _, route := range sample {
+		if listed[route.Prefix] {
+			t.Errorf("GetRandomPrefixesByLength lists %s twice", route.Prefix)
+		}
+		listed[route.Prefix] = true
+		if !slices.Contains(firstPrefixes, route.Prefix) {
+			pastFirstPrefixes++
+		}
+	}
+	if pastFirstPrefixes == 0 {
+		t.Errorf("GetRandomPrefixesByLength lists only prefixes among the first %d in text order",
+			len(firstPrefixes))
+	}
+}
+
+// TestSamplePrefixesByLengthFillsUpFromShuffled stores a single prefix, so
+// every random lookup lands on it, and checks that the sample still lists
+// sampleLimit prefixes: the stored one, then the shuffled ones in order. The
+// stored prefix is the second shuffled one, so the fill-up must skip it.
+func TestSamplePrefixesByLengthFillsUpFromShuffled(t *testing.T) {
+	cfg := &config.Config{StateDir: t.TempDir()}
+
+	db, err := New(cfg, logger.New())
+	if err != nil {
+		t.Fatalf("failed to create database: %v", err)
+	}
+	defer func() { _ = db.Close() }()
+
+	var shuffled []string
+	for i := range sampledPrefixCount {
+		shuffled = append(shuffled, fmt.Sprintf("10.0.%d.0/24", i))
+	}
+	route := mkV4Route(t, shuffled[1], 64500, time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC))
+	if err := db.UpsertLiveRouteBatch([]*LiveRoute{route}); err != nil {
+		t.Fatalf("UpsertLiveRouteBatch: %v", err)
+	}
+
+	sample, err := db.samplePrefixesByLength(
+		context.Background(), liveRoutesV4Table, route.MaskLength, sampleLimit, shuffled)
+	if err != nil {
+		t.Fatalf("samplePrefixesByLength: %v", err)
+	}
+	want := append([]string{shuffled[1], shuffled[0]}, shuffled[2:sampleLimit]...)
+	if !slices.Equal(sample, want) {
+		t.Errorf("samplePrefixesByLength = %v, want %v", sample, want)
+	}
+}
+
+// TestGetRandomPrefixesByLengthSkipsWithdrawnRoutes withdraws a route after
+// storing it, as can happen between the steps of GetRandomPrefixesByLength. It
+// checks that the random lookups pick nothing for a mask length with no route
+// left, and that only the prefixes that still have a route get their newest
+// route listed.
+func TestGetRandomPrefixesByLengthSkipsWithdrawnRoutes(t *testing.T) {
+	cfg := &config.Config{StateDir: t.TempDir()}
+
+	db, err := New(cfg, logger.New())
+	if err != nil {
+		t.Fatalf("failed to create database: %v", err)
+	}
+	defer func() { _ = db.Close() }()
+
+	ts := time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC)
+	kept := mkV4Route(t, "198.51.100.0/24", 64500, ts)
+	withdrawn := mkV4Route(t, "172.16.0.0/16", 64500, ts)
+	if err := db.UpsertLiveRouteBatch([]*LiveRoute{kept, withdrawn}); err != nil {
+		t.Fatalf("UpsertLiveRouteBatch: %v", err)
+	}
+	if err := db.DeleteLiveRoute(withdrawn.Prefix, withdrawn.OriginASN, withdrawn.PeerIP); err != nil {
+		t.Fatalf("DeleteLiveRoute: %v", err)
+	}
+
+	ctx := context.Background()
+	sample, err := db.samplePrefixesByLength(
+		ctx, liveRoutesV4Table, withdrawn.MaskLength, sampleLimit, []string{withdrawn.Prefix})
+	if err != nil {
+		t.Fatalf("samplePrefixesByLength: %v", err)
+	}
+	if len(sample) != 0 {
+		t.Errorf("samplePrefixesByLength = %v, want none for a mask length with no route left", sample)
+	}
+
+	routes, err := db.newestRouteOfEachPrefix(
+		ctx, liveRoutesV4Table, ipVersionV4, []string{withdrawn.Prefix, kept.Prefix})
+	if err != nil {
+		t.Fatalf("newestRouteOfEachPrefix: %v", err)
+	}
+	if len(routes) != 1 || routes[0].Prefix != kept.Prefix {
+		t.Errorf("newestRouteOfEachPrefix = %+v, want only the route of %s", routes, kept.Prefix)
+	}
+}

internal/streamer/streamer.go

@@ -7,6 +7,7 @@ import (
 	"compress/gzip"
 	"context"
 	"encoding/json"
+	"errors"
 	"fmt"
 	"io"
 	"math"
@@ -48,6 +49,7 @@ const (
 	metricsUpdateRate     = time.Second
 	minBackoffDelay       = 5 * time.Second
 	maxBackoffDelay       = 320 * time.Second
+	streamIdleTimeout     = 60 * time.Second // Reconnect when no line arrives for this long
 	metricsLogInterval    = 10 * time.Second
 	bytesPerKB            = 1024
 	bytesPerMB            = 1024 * 1024
@@ -58,6 +60,9 @@ const (
 	backpressureSlope     = 2.0 // Slope for linear drop probability increase
 )
 
+// errStreamEnded is returned by stream when RIS Live ends the response.
+var errStreamEnded = errors.New("RIS Live ended the stream")
+
 // MessageHandler defines the interface for processing RIS messages.
 // Implementations must specify which message types they want to receive,
 // how to process messages, and their desired queue capacity.
@@ -107,6 +112,8 @@ type Streamer struct {
 	logger       *logger.Logger
 	client       *http.Client
 	url          string
+	idleTimeout  time.Duration // Drop the connection when no line arrives for this long
+	minBackoff   time.Duration // First wait before reconnecting; doubles up to maxBackoffDelay
 	handlers     []*handlerInfo
 	rawHandler   RawMessageHandler
 	mu           sync.RWMutex
@@ -124,8 +131,10 @@ type Streamer struct {
 // The metrics tracker is used to record message counts, bytes received, and connection status.
 func New(logger *logger.Logger, metrics *metrics.Tracker) *Streamer {
 	return &Streamer{
-		logger: logger,
+		logger:      logger,
-		url:    risLiveURL,
+		url:         risLiveURL,
+		idleTimeout: streamIdleTimeout,
+		minBackoff:  minBackoffDelay,
 		client: &http.Client{
 			Timeout: 0, // No timeout for streaming
 			Transport: &http.Transport{
@@ -405,7 +414,7 @@ func (s *Streamer) updateMetrics(messageBytes int) {
 
 // streamWithReconnect handles streaming with automatic reconnection and exponential backoff
 func (s *Streamer) streamWithReconnect(ctx context.Context) {
-	backoffDelay := minBackoffDelay
+	backoffDelay := s.minBackoff
 	consecutiveFailures := 0
 
 	for {
@@ -422,8 +431,9 @@ func (s *Streamer) streamWithReconnect(ctx context.Context) {
 		err := s.stream(ctx)
 		streamDuration := time.Since(startTime)
 
-		if err == nil {
+		// Only Stop ends the streamer. A stream that ends any other way,
-			// Clean exit (context cancelled)
+		// with or without an error, is a disconnect and is reconnected.
+		if ctx.Err() != nil {
 			return
 		}
 
@@ -434,17 +444,13 @@ func (s *Streamer) streamWithReconnect(ctx context.Context) {
 			"stream_duration", streamDuration)
 		s.metrics.SetConnected(false)
 
-		// Check if context is cancelled
+		// A stream that lasted more than 30 seconds resets the backoff. That
-		if ctx.Err() != nil {
+		// includes a stream dropped after streamIdleTimeout with no line, so
-			return
+		// a silent feed is retried after the first backoff delay every time.
-		}
-
-		// If we streamed for more than 30 seconds, reset the backoff
-		// This indicates we had a successful connection that received data
 		if streamDuration > 30*time.Second {
 			s.logger.Info("Resetting backoff delay due to successful connection",
 				"stream_duration", streamDuration)
-			backoffDelay = minBackoffDelay
+			backoffDelay = s.minBackoff
 			consecutiveFailures = 0
 		} else {
 			// Increment consecutive failures
@@ -477,7 +483,13 @@ func (s *Streamer) stream(ctx context.Context) error {
 	connCtx, connCancel := context.WithCancel(ctx)
 	defer connCancel()
 
-	req, err := http.NewRequestWithContext(ctx, "GET", s.url, nil)
+	// The idle timer drops the connection, by cancelling connCtx, when no
+	// line arrives for s.idleTimeout. Each line read starts it over. It
+	// also covers the wait for the server to answer.
+	idleTimer := time.AfterFunc(s.idleTimeout, connCancel)
+	defer idleTimer.Stop()
+
+	req, err := http.NewRequestWithContext(connCtx, "GET", s.url, nil)
 	if err != nil {
 		return fmt.Errorf("failed to create request: %w", err)
 	}
@@ -559,6 +571,8 @@ func (s *Streamer) stream(ctx context.Context) error {
 	scanner := bufio.NewScanner(reader)
 
 	for scanner.Scan() {
+		idleTimer.Reset(s.idleTimeout)
+
 		select {
 		case <-ctx.Done():
 			s.logger.Info("Stream stopped by context")
@@ -707,11 +721,19 @@ func (s *Streamer) stream(ctx context.Context) error {
 		s.mu.RUnlock()
 	}
 
+	if ctx.Err() != nil {
+		return ctx.Err()
+	}
+
+	if connCtx.Err() != nil {
+		return fmt.Errorf("no line received for %s", s.idleTimeout)
+	}
+
 	if err := scanner.Err(); err != nil {
 		return fmt.Errorf("scanner error: %w", err)
 	}
 
-	return nil
+	return errStreamEnded
 }
 
 // shouldDropForBackpressure determines if a message should be dropped based on queue utilization

internal/streamer/streamer_test.go

@@ -7,6 +7,7 @@ import (
 	"net/http"
 	"net/http/httptest"
 	"runtime"
+	"sync/atomic"
 	"testing"
 	"time"
 
@@ -39,6 +40,14 @@ func TestNewStreamer(t *testing.T) {
 	if s.metrics != metricsTracker {
 		t.Error("metrics tracker not set correctly")
 	}
+
+	if s.idleTimeout != streamIdleTimeout {
+		t.Errorf("idleTimeout = %s, want %s", s.idleTimeout, streamIdleTimeout)
+	}
+
+	if s.minBackoff != minBackoffDelay {
+		t.Errorf("minBackoff = %s, want %s", s.minBackoff, minBackoffDelay)
+	}
 }
 
 // TestStreamDoesNotLeakTickersAcrossReconnects drives many short-lived
@@ -57,8 +66,8 @@ func TestStreamDoesNotLeakTickersAcrossReconnects(t *testing.T) {
 
 	// One warm-up connection so any persistent HTTP transport goroutine exists
 	// before we take the baseline.
-	if err := s.stream(context.Background()); err != nil {
+	if err := s.stream(context.Background()); !errors.Is(err, errStreamEnded) {
-		t.Fatalf("warm-up stream returned error: %v", err)
+		t.Fatalf("warm-up stream returned %v, want %v", err, errStreamEnded)
 	}
 	s.client.CloseIdleConnections()
 
@@ -66,8 +75,8 @@ func TestStreamDoesNotLeakTickersAcrossReconnects(t *testing.T) {
 
 	const reconnects = 20
 	for range reconnects {
-		if err := s.stream(context.Background()); err != nil {
+		if err := s.stream(context.Background()); !errors.Is(err, errStreamEnded) {
-			t.Fatalf("stream returned error: %v", err)
+			t.Fatalf("stream returned %v, want %v", err, errStreamEnded)
 		}
 	}
 	s.client.CloseIdleConnections()
@@ -78,23 +87,35 @@ func TestStreamDoesNotLeakTickersAcrossReconnects(t *testing.T) {
 	}
 }
 
-// updateHandler wants UPDATE messages and does nothing with them.
+// updateLine is one UPDATE message as RIS Live sends it.
-type updateHandler struct{}
+const updateLine = `{"type":"ris_message","data":{"type":"UPDATE","peer":"192.0.2.1",` +
+	`"peer_asn":"64496","timestamp":1700000000}}` + "\n"
+
+// updateHandler wants UPDATE messages. For each one it handles it sends on
+// handled if there is room, and otherwise does nothing; a nil handled is
+// never sent on.
+type updateHandler struct {
+	handled chan struct{}
+}
 
 func (updateHandler) WantsMessage(messageType string) bool { return messageType == "UPDATE" }
-func (updateHandler) HandleMessage(*ristypes.RISMessage)   {}
 func (updateHandler) QueueCapacity() int                   { return 10 }
 
+func (h updateHandler) HandleMessage(*ristypes.RISMessage) {
+	select {
+	case h.handled <- struct{}{}:
+	default:
+	}
+}
+
 // TestStopBeforeMessageReachesQueues stops the streamer after the read loop
 // has checked for cancellation but before it hands the message to the handler
 // queues. That is the gap a stop from another goroutine can land in, and it
 // used to end in "send on closed channel". The raw handler runs in that gap on
 // the read loop itself, so calling Stop from it hits the gap every time.
 func TestStopBeforeMessageReachesQueues(t *testing.T) {
-	const line = `{"type":"ris_message","data":{"type":"UPDATE","peer":"192.0.2.1",` +
-		`"peer_asn":"64496","timestamp":1700000000}}` + "\n"
 	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
-		_, _ = io.WriteString(w, line)
+		_, _ = io.WriteString(w, updateLine)
 	}))
 	defer srv.Close()
 
@@ -117,6 +138,117 @@ func TestStopBeforeMessageReachesQueues(t *testing.T) {
 	s.Stop()
 }
 
+// TestReconnectWhenStreamEnds has the server end the first response
+// normally, as when RIS Live restarts. The streamer must treat that as a
+// disconnect and connect again.
+func TestReconnectWhenStreamEnds(t *testing.T) {
+	endResponse := func(http.ResponseWriter, *http.Request) {}
+	checkReconnect(t, endResponse, streamIdleTimeout)
+}
+
+// TestReconnectWhenStreamGoesIdle has the server hold the first response
+// open without sending anything. The streamer must drop it once the idle
+// timeout passes and connect again.
+func TestReconnectWhenStreamGoesIdle(t *testing.T) {
+	const idleTimeout = 50 * time.Millisecond
+	holdResponse := func(w http.ResponseWriter, r *http.Request) {
+		w.(http.Flusher).Flush()
+		<-r.Context().Done()
+	}
+	checkReconnect(t, holdResponse, idleTimeout)
+}
+
+// TestReconnectWhenServerNeverAnswers has the server accept the first
+// request and never answer it, not even with headers. The streamer must give
+// up on it once the idle timeout passes and connect again.
+func TestReconnectWhenServerNeverAnswers(t *testing.T) {
+	const idleTimeout = 50 * time.Millisecond
+	neverAnswer := func(_ http.ResponseWriter, r *http.Request) {
+		<-r.Context().Done()
+	}
+	checkReconnect(t, neverAnswer, idleTimeout)
+}
+
+// TestStreamNotDroppedWhileLinesArrive has the server send lines well within
+// the idle timeout for several times the timeout, then end the response. Each
+// line must start the idle timeout over, so the stream lasts until the server
+// ends it instead of being dropped as idle.
+func TestStreamNotDroppedWhileLinesArrive(t *testing.T) {
+	const (
+		idleTimeout  = 100 * time.Millisecond
+		lineInterval = idleTimeout / 10
+		sendFor      = 5 * idleTimeout
+	)
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
+		deadline := time.Now().Add(sendFor)
+		for time.Now().Before(deadline) {
+			_, _ = io.WriteString(w, updateLine)
+			w.(http.Flusher).Flush()
+			time.Sleep(lineInterval)
+		}
+	}))
+	defer srv.Close()
+
+	s := New(logger.New(), metrics.New())
+	s.url = srv.URL
+	s.idleTimeout = idleTimeout
+
+	if err := s.stream(context.Background()); !errors.Is(err, errStreamEnded) {
+		t.Fatalf("stream returned %v, want %v", err, errStreamEnded)
+	}
+}
+
+// checkReconnect runs a streamer against a fake RIS Live server. The first
+// response is served by firstResponse; every later one carries one UPDATE
+// line and then stays open. The streamer must come back for a second
+// response, must not be marked connected when it does, and must handle the
+// UPDATE in it.
+func checkReconnect(t *testing.T, firstResponse http.HandlerFunc, idleTimeout time.Duration) {
+	t.Helper()
+
+	const maxWait = 5 * time.Second
+
+	tracker := metrics.New()
+	connectedAtReconnect := make(chan bool, 1)
+	var requests atomic.Int32
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if requests.Add(1) == 1 {
+			firstResponse(w, r)
+
+			return
+		}
+		select {
+		case connectedAtReconnect <- tracker.IsConnected():
+		default:
+		}
+		_, _ = io.WriteString(w, updateLine)
+		w.(http.Flusher).Flush()
+		<-r.Context().Done()
+	}))
+	defer srv.Close()
+
+	s := New(logger.New(), tracker)
+	s.url = srv.URL
+	s.idleTimeout = idleTimeout
+	s.minBackoff = time.Millisecond
+	handler := updateHandler{handled: make(chan struct{}, 1)}
+	s.RegisterHandler(handler)
+	if err := s.Start(); err != nil {
+		t.Fatalf("Start returned error: %v", err)
+	}
+	defer s.Stop()
+
+	select {
+	case <-handler.handled:
+	case <-time.After(maxWait):
+		t.Fatalf("no message from a second response handled within %s", maxWait)
+	}
+
+	if <-connectedAtReconnect {
+		t.Error("streamer was still marked connected when it came back for a second response")
+	}
+}
+
 // settledGoroutineCount lets transient goroutines finish, then reports the
 // current count.
 func settledGoroutineCount() int {