Make live route writes cheaper so the prefix handler keeps up (closes #96)

The live route tables lose their random UUID key; the row id the
prefix-length sample draws is declared as the integer key. The prefix
tables are keyed by prefix and keep only first_seen; a batch inserts only
prefixes not stored yet. A route update sets only the columns that
change. Each kept index names the queries using it. Every write goes
through one connection kept for writing, with a 1 GiB page cache; reading
connections get 16 MiB, heap limits rise to 1.5 and 2 GiB (README
"Memory"). Batch writes go in key order. A benchmark run by hand writes
250,000 updates on 5 million routes faster than the feed's 10,000 a
second; a live run follows.

Model: opus-5-5

internal/database/database_internal_test.go

@@ -6,15 +6,16 @@ import (
 	"errors"
 	"fmt"
 	"log/slog"
+	"math/rand/v2"
 	"net/netip"
 	"slices"
+	"strconv"
 	"sync"
 	"testing"
 	"time"
 
 	"git.eeqj.de/sneak/routewatch/internal/config"
 	"git.eeqj.de/sneak/routewatch/internal/logger"
-	"github.com/google/uuid"
 	"github.com/mattn/go-sqlite3"
 )
 
@@ -33,6 +34,9 @@ const vacuumTestRoutes = 2000
 // once so each is a distinct SQLite connection that parsed the DSN.
 const heldConnections = 5
 
+// bytesPerKiB turns a cache_size in KiB into bytes.
+const bytesPerKiB = 1024
+
 // testPeerIP is the peer every route in the IP lookup test is learned from.
 const testPeerIP = "192.0.2.254"
 
@@ -293,7 +297,7 @@ func TestLiveRouteTablesHoldOneRoutePerPrefixAndPeer(t *testing.T) {
 			t.Fatalf("UpsertLiveRoute(%s): %v", tt.route.Prefix, err)
 		}
 
-		_, err = db.db.ExecContext(t.Context(), tt.insertSQL, uuid.NewString(),
+		_, err = db.db.ExecContext(t.Context(), tt.insertSQL,
 			tt.route.Prefix, tt.route.MaskLength, otherOriginASN,
 			tt.route.PeerIP, "[]", tt.route.NextHop, ts)
 
@@ -317,7 +321,6 @@ func writeOriginRoutes(t *testing.T, db *Database, origins map[string]int) {
 
 	for prefix, asn := range origins {
 		routes = append(routes, &LiveRoute{
-			ID:          uuid.New(),
 			Prefix:      prefix,
 			MaskLength:  netip.MustParsePrefix(prefix).Bits(),
 			IPVersion:   detectIPVersion(prefix),
@@ -337,7 +340,7 @@ func writeOriginRoutes(t *testing.T, db *Database, origins map[string]int) {
 
 // TestConnectionPoolPragmas holds several pooled connections open at once and
 // checks each one carries the per-connection settings from the DSN, plus the
-// process-wide hard heap limit, and sees the new file with auto_vacuum
+// process-wide heap limits, and sees the new file with auto_vacuum
 // incremental.
 func TestConnectionPoolPragmas(t *testing.T) {
 	t.Parallel()
@@ -372,9 +375,9 @@ func TestConnectionPoolPragmas(t *testing.T) {
 
 	for i, c := range conns {
 		cacheSize := readPragma(ctx, t, c, i, "cache_size")
-		if cacheSize != sqliteCacheSizeKiB {
+		if cacheSize != sqliteReadCacheSizeKiB {
 			t.Errorf("conn %d: cache_size = %d, want %d",
-				i, cacheSize, sqliteCacheSizeKiB)
+				i, cacheSize, sqliteReadCacheSizeKiB)
 		}
 
 		busyTimeout := readPragma(ctx, t, c, i, "busy_timeout")
@@ -389,6 +392,12 @@ func TestConnectionPoolPragmas(t *testing.T) {
 				i, tempStore, tempStoreMemory)
 		}
 
+		softHeapLimit := readPragma(ctx, t, c, i, "soft_heap_limit")
+		if softHeapLimit != sqliteSoftHeapLimitBytes {
+			t.Errorf("conn %d: soft_heap_limit = %d, want %d",
+				i, softHeapLimit, sqliteSoftHeapLimitBytes)
+		}
+
 		hardHeapLimit := readPragma(ctx, t, c, i, "hard_heap_limit")
 		if hardHeapLimit != sqliteHardHeapLimitBytes {
 			t.Errorf("conn %d: hard_heap_limit = %d, want %d",
@@ -403,6 +412,51 @@ func TestConnectionPoolPragmas(t *testing.T) {
 	}
 }
 
+// TestWritingConnectionCache checks that the connection kept for writing has
+// the large cache, and that the soft heap limit is above the caches of all the
+// connections together: SQLite shrinks the caches to stay under it.
+func TestWritingConnectionCache(t *testing.T) {
+	t.Parallel()
+
+	db, err := New(&config.Config{StateDir: t.TempDir()}, logger.New())
+	if err != nil {
+		t.Fatalf("failed to create database: %v", err)
+	}
+	defer func() { _ = db.Close() }()
+
+	ctx := t.Context()
+
+	var writeCacheSize, readCacheSize, softHeapLimit int64
+
+	err = db.writer.QueryRowContext(ctx, "PRAGMA cache_size").
+		Scan(&writeCacheSize)
+	if err != nil {
+		t.Fatalf("failed to read the writing connection's cache_size: %v", err)
+	}
+
+	if writeCacheSize != sqliteWriteCacheSizeKiB {
+		t.Errorf("writing connection: cache_size = %d, want %d",
+			writeCacheSize, sqliteWriteCacheSizeKiB)
+	}
+
+	err = db.db.QueryRowContext(ctx, "PRAGMA cache_size").Scan(&readCacheSize)
+	if err != nil {
+		t.Fatalf("failed to read a reading connection's cache_size: %v", err)
+	}
+
+	err = db.db.QueryRowContext(ctx, "PRAGMA soft_heap_limit").
+		Scan(&softHeapLimit)
+	if err != nil {
+		t.Fatalf("failed to read soft_heap_limit: %v", err)
+	}
+
+	cachesBytes := -bytesPerKiB * (writeCacheSize + (maxConns-1)*readCacheSize)
+	if softHeapLimit <= cachesBytes {
+		t.Errorf("soft_heap_limit = %d, not above the %d bytes of the caches",
+			softHeapLimit, cachesBytes)
+	}
+}
+
 // readPragma reads the integer value of PRAGMA name on connection i, c.
 func readPragma(
 	ctx context.Context, t *testing.T, c *sql.Conn, i int, name string,
@@ -1090,3 +1144,380 @@ func TestGetRandomPrefixesByLengthSkipsWithdrawnRoutes(t *testing.T) {
 			routes, kept.Prefix)
 	}
 }
+
+// TestLiveRouteUpdateSetsOriginWhenItChanges announces a peer's route for a
+// prefix, then again with the same origin AS and a new path, then with a new
+// origin AS. Each time the route must carry the announced origin AS, path and
+// time, and the AS details must list the prefix under that origin AS only. The
+// update sets the origin AS with a statement of its own, which changes nothing
+// while the origin AS stays the same.
+func TestLiveRouteUpdateSetsOriginWhenItChanges(t *testing.T) {
+	t.Parallel()
+
+	db, err := New(&config.Config{StateDir: t.TempDir()}, logger.New())
+	if err != nil {
+		t.Fatalf("failed to create database: %v", err)
+	}
+	defer func() { _ = db.Close() }()
+
+	const firstOrigin, secondOrigin = 64500, 64501
+
+	ts := time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC)
+
+	for _, asn := range []int{firstOrigin, secondOrigin} {
+		_, err := db.GetOrCreateASN(asn, ts)
+		if err != nil {
+			t.Fatalf("GetOrCreateASN(%d): %v", asn, err)
+		}
+	}
+
+	route := mkV4Route(t, "198.51.100.0/24", firstOrigin, ts)
+
+	for _, announced := range []struct {
+		origin int
+		path   []int
+	}{
+		{firstOrigin, []int{firstOrigin}},
+		{firstOrigin, []int{secondOrigin, firstOrigin}},
+		{secondOrigin, []int{firstOrigin, secondOrigin}},
+	} {
+		ts = ts.Add(time.Minute)
+		route.OriginASN, route.ASPath, route.LastUpdated =
+			announced.origin, announced.path, ts
+
+		err := db.UpsertLiveRouteBatch([]*LiveRoute{route})
+		if err != nil {
+			t.Fatalf("UpsertLiveRouteBatch: %v", err)
+		}
+
+		routes, _, err := db.GetPrefixDetails(route.Prefix)
+		if err != nil {
+			t.Fatalf("GetPrefixDetails: %v", err)
+		}
+
+		if len(routes) != 1 || routes[0].OriginASN != announced.origin ||
+			!slices.Equal(routes[0].ASPath, announced.path) ||
+			!routes[0].LastUpdated.Equal(ts) {
+			t.Errorf("after announcing origin AS%d, path %v at %s: routes %+v",
+				announced.origin, announced.path, ts, routes)
+		}
+
+		for _, asn := range []int{firstOrigin, secondOrigin} {
+			_, prefixes, err := db.GetASDetails(asn)
+			if err != nil {
+				t.Fatalf("GetASDetails(%d): %v", asn, err)
+			}
+
+			listed := len(prefixes) == 1 && prefixes[0].Prefix == route.Prefix
+			if listed != (asn == announced.origin) {
+				t.Errorf("after announcing origin AS%d, AS%d lists %+v",
+					announced.origin, asn, prefixes)
+			}
+		}
+	}
+}
+
+// TestUpdatePrefixesBatchKeepsFirstSeen writes a prefix, then a batch with it
+// and a new prefix. Only the new prefix is inserted and counted, and the known
+// one keeps its first_seen time.
+func TestUpdatePrefixesBatchKeepsFirstSeen(t *testing.T) {
+	t.Parallel()
+
+	db, err := New(&config.Config{StateDir: t.TempDir()}, logger.New())
+	if err != nil {
+		t.Fatalf("failed to create database: %v", err)
+	}
+	defer func() { _ = db.Close() }()
+
+	const known = "198.51.100.0/24"
+
+	first := time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC)
+	later := first.Add(time.Hour)
+
+	err = db.UpdatePrefixesBatch(map[string]time.Time{known: first})
+	if err != nil {
+		t.Fatalf("UpdatePrefixesBatch: %v", err)
+	}
+
+	err = db.UpdatePrefixesBatch(map[string]time.Time{
+		known: later, testV6Prefix: later,
+	})
+	if err != nil {
+		t.Fatalf("UpdatePrefixesBatch: %v", err)
+	}
+
+	stats, err := db.GetStats()
+	if err != nil {
+		t.Fatalf("GetStats: %v", err)
+	}
+
+	if stats.IPv4Prefixes != 1 || stats.IPv6Prefixes != 1 {
+		t.Errorf("prefix counts = (v4 %d, v6 %d), want (1, 1)",
+			stats.IPv4Prefixes, stats.IPv6Prefixes)
+	}
+
+	for _, tt := range []struct {
+		query, prefix string
+		firstSeen     time.Time
+	}{
+		{
+			query:  "SELECT first_seen FROM prefixes_v4 WHERE prefix = ?",
+			prefix: known, firstSeen: first,
+		},
+		{
+			query:  "SELECT first_seen FROM prefixes_v6 WHERE prefix = ?",
+			prefix: testV6Prefix, firstSeen: later,
+		},
+	} {
+		var firstSeen time.Time
+
+		err := db.db.QueryRowContext(t.Context(), tt.query, tt.prefix).
+			Scan(&firstSeen)
+		if err != nil {
+			t.Fatalf("reading %s: %v", tt.prefix, err)
+		}
+
+		if !firstSeen.Equal(tt.firstSeen) {
+			t.Errorf("%s: first seen %s, want %s", tt.prefix, firstSeen,
+				tt.firstSeen)
+		}
+	}
+}
+
+// TestUpsertLiveRouteBatchWritesInKeyOrder writes new routes of two peers out
+// of key order, the last one a second announcement of the first, and checks
+// that the rows were inserted in key order, peer then prefix (each new row's id
+// is one above the last), and that the route announced twice carries the later
+// announcement.
+func TestUpsertLiveRouteBatchWritesInKeyOrder(t *testing.T) {
+	t.Parallel()
+
+	db, err := New(&config.Config{StateDir: t.TempDir()}, logger.New())
+	if err != nil {
+		t.Fatalf("failed to create database: %v", err)
+	}
+	defer func() { _ = db.Close() }()
+
+	const (
+		firstOrigin, laterOrigin = 64500, 64501
+		// Three prefixes, in text order.
+		firstPrefix  = "192.0.2.0/24"
+		secondPrefix = "198.51.100.0/24"
+		thirdPrefix  = "203.0.113.0/24"
+	)
+
+	type row struct {
+		peer, prefix string
+		origin       int
+	}
+
+	ts := time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC)
+
+	announcements := []row{
+		{secondV4PeerIP, secondPrefix, firstOrigin},
+		{testV4PeerIP, thirdPrefix, firstOrigin},
+		{secondV4PeerIP, firstPrefix, firstOrigin},
+		{testV4PeerIP, secondPrefix, firstOrigin},
+		{secondV4PeerIP, secondPrefix, laterOrigin},
+	}
+
+	routes := make([]*LiveRoute, 0, len(announcements))
+
+	for _, announced := range announcements {
+		route := mkV4Route(t, announced.prefix, announced.origin, ts)
+		route.PeerIP = announced.peer
+		routes = append(routes, route)
+	}
+
+	err = db.UpsertLiveRouteBatch(routes)
+	if err != nil {
+		t.Fatalf("UpsertLiveRouteBatch: %v", err)
+	}
+
+	rows, err := db.db.QueryContext(t.Context(),
+		"SELECT peer_ip, prefix, origin_asn FROM live_routes_v4 ORDER BY id")
+	if err != nil {
+		t.Fatalf("reading the routes: %v", err)
+	}
+	defer func() { _ = rows.Close() }()
+
+	var stored []row
+
+	for rows.Next() {
+		var r row
+
+		err := rows.Scan(&r.peer, &r.prefix, &r.origin)
+		if err != nil {
+			t.Fatalf("reading a route: %v", err)
+		}
+
+		stored = append(stored, r)
+	}
+
+	err = rows.Err()
+	if err != nil {
+		t.Fatalf("reading the routes: %v", err)
+	}
+
+	want := []row{
+		{testV4PeerIP, secondPrefix, firstOrigin},
+		{testV4PeerIP, thirdPrefix, firstOrigin},
+		{secondV4PeerIP, firstPrefix, firstOrigin},
+		{secondV4PeerIP, secondPrefix, laterOrigin},
+	}
+
+	if !slices.Equal(stored, want) {
+		t.Errorf("routes in insertion order = %+v, want %+v", stored, want)
+	}
+}
+
+// Parameters for BenchmarkPrefixBatchOnFiveMillionRoutes.
+const (
+	// benchPeers and benchPrefixes size the table the batches are written to:
+	// every peer has a route for every prefix, 5 million routes in all.
+	benchPeers    = 10
+	benchPrefixes = 500000
+	// benchBatchSize is how many updates each batch holds, as many as a full
+	// prefix handler batch (prefixBatchSize in internal/routewatch).
+	benchBatchSize = 25000
+	// benchMaxBatchSize is the largest batch the prefix handler writes, the
+	// one it holds while a slow write runs (prefixBatchMaxSize in
+	// internal/routewatch).
+	benchMaxBatchSize = 250000
+	// benchWithdrawalEvery makes every fifth update of a timed batch a
+	// withdrawal; the others are announcements.
+	benchWithdrawalEvery = 5
+	// benchOrigins is how many origin ASes the prefixes are spread over.
+	benchOrigins = 50000
+	// benchFirstASN is the lowest AS number in the benchmark's AS paths.
+	benchFirstASN = 64500
+	// benchMaskLength is the mask length of every prefix in the benchmark.
+	benchMaskLength = 24
+)
+
+// BenchmarkPrefixBatchOnFiveMillionRoutes times the writes the prefix handler
+// makes for one batch of announcements and withdrawals of routes picked at
+// random, to a database that already holds 5 million live routes: 500,000
+// IPv4 prefixes, each from 10 peers. It times a full batch of 25,000 updates,
+// then the largest batch, 250,000. It builds that database first, one batch at
+// a time, which takes some minutes and a few GB of disk under TMPDIR. It is
+// run by hand, not by make test, for example in the test image:
+//
+//	go test -run '^$' -bench FiveMillion -benchtime 10x ./internal/database/
+func BenchmarkPrefixBatchOnFiveMillionRoutes(b *testing.B) {
+	db, err := New(&config.Config{StateDir: b.TempDir()}, logger.New())
+	if err != nil {
+		b.Fatalf("failed to create database: %v", err)
+	}
+	defer func() { _ = db.Close() }()
+
+	//nolint:gosec // The benchmark needs the same batches every run.
+	rng := rand.New(rand.NewPCG(1, 2))
+	ts := time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC)
+
+	// Each peer sends its whole table, as when its session comes up.
+	for peer := range benchPeers {
+		for first := 0; first < benchPrefixes; first += benchBatchSize {
+			routes := make([]*LiveRoute, 0, benchBatchSize)
+			for prefix := first; prefix < first+benchBatchSize; prefix++ {
+				routes = append(routes, benchRoute(rng, prefix, peer, ts))
+			}
+
+			writeBenchBatch(b, db, routes, nil, ts)
+		}
+	}
+
+	for _, batchSize := range []int{benchBatchSize, benchMaxBatchSize} {
+		b.Run(strconv.Itoa(batchSize), func(b *testing.B) {
+			for b.Loop() {
+				ts = ts.Add(time.Second)
+
+				var (
+					routes      []*LiveRoute
+					withdrawals []LiveRouteDeletion
+				)
+
+				for i := range batchSize {
+					route := benchRoute(rng, rng.IntN(benchPrefixes),
+						rng.IntN(benchPeers), ts)
+					if i%benchWithdrawalEvery == 0 {
+						withdrawals = append(withdrawals, LiveRouteDeletion{
+							Prefix: route.Prefix, PeerIP: route.PeerIP,
+							IPVersion: ipVersionV4,
+						})
+					} else {
+						routes = append(routes, route)
+					}
+				}
+
+				writeBenchBatch(b, db, routes, withdrawals, ts)
+			}
+		})
+	}
+}
+
+// benchRoute returns the route peer has for the prefix numbered prefix, a /24
+// from 1.0.0.0/24 on, with a new AS path of three to six ASes from the peer's
+// AS to the prefix's origin AS.
+func benchRoute(rng *rand.Rand, prefix, peer int, ts time.Time) *LiveRoute {
+	const minPathLength, extraPathLength = 3, 4
+
+	origin := benchFirstASN + prefix%benchOrigins
+	pathLength := minPathLength + rng.IntN(extraPathLength)
+
+	path := make([]int, 0, pathLength)
+	path = append(path, benchFirstASN+peer)
+
+	for len(path) < pathLength-1 {
+		path = append(path, benchFirstASN+rng.IntN(benchOrigins))
+	}
+
+	path = append(path, origin)
+	peerIP := fmt.Sprintf("198.51.100.%d", 1+peer)
+
+	return &LiveRoute{
+		Prefix: fmt.Sprintf("%d.%d.%d.0/%d",
+			1+prefix>>16, (prefix>>8)%256, prefix%256, benchMaskLength),
+		MaskLength:  benchMaskLength,
+		IPVersion:   ipVersionV4,
+		OriginASN:   origin,
+		PeerIP:      peerIP,
+		ASPath:      path,
+		NextHop:     peerIP,
+		LastUpdated: ts,
+	}
+}
+
+// writeBenchBatch writes what the prefix handler writes for a batch: the
+// announced routes, the withdrawals, and every prefix the batch names, of
+// which only the ones not stored yet are inserted.
+func writeBenchBatch(
+	b *testing.B, db *Database, routes []*LiveRoute,
+	withdrawals []LiveRouteDeletion, ts time.Time,
+) {
+	b.Helper()
+
+	prefixes := make(map[string]time.Time, len(routes)+len(withdrawals))
+	for _, route := range routes {
+		prefixes[route.Prefix] = ts
+	}
+
+	for _, withdrawal := range withdrawals {
+		prefixes[withdrawal.Prefix] = ts
+	}
+
+	err := db.UpsertLiveRouteBatch(routes)
+	if err != nil {
+		b.Fatalf("UpsertLiveRouteBatch: %v", err)
+	}
+
+	err = db.DeleteLiveRouteBatch(withdrawals)
+	if err != nil {
+		b.Fatalf("DeleteLiveRouteBatch: %v", err)
+	}
+
+	err = db.UpdatePrefixesBatch(prefixes)
+	if err != nil {
+		b.Fatalf("UpdatePrefixesBatch: %v", err)
+	}
+}