Make live route writes cheaper so the prefix handler keeps up (closes #96)

The live route tables lose their random UUID key; the row id the
prefix-length sample draws is declared as the integer key. The prefix
tables are keyed by prefix and keep only first_seen; a batch inserts only
prefixes not stored yet. A route update sets only the columns that
change. Each kept index names the queries using it. Every write goes
through one connection kept for writing, with a 1 GiB page cache; reading
connections get 16 MiB, heap limits rise to 1.5 and 2 GiB (README
"Memory"). Batch writes go in key order. A benchmark run by hand writes
250,000 updates on 5 million routes faster than the feed's 10,000 a
second; a live run follows.

Model: opus-5-5

internal/database/database.go

@@ -1,25 +1,27 @@
 package database
 
 import (
+	"cmp"
 	"context"
 	"database/sql"
 	_ "embed"
 	"encoding/json"
 	"errors"
 	"fmt"
+	"maps"
 	"math/rand/v2"
 	"net"
 	"net/netip"
 	"os"
 	"path/filepath"
 	"runtime"
+	"slices"
 	"sync"
 	"time"
 
 	"git.eeqj.de/sneak/routewatch/internal/config"
 	"git.eeqj.de/sneak/routewatch/internal/logger"
 	"git.eeqj.de/sneak/routewatch/pkg/asinfo"
-	"github.com/google/uuid"
 	_ "github.com/mattn/go-sqlite3" // CGO SQLite driver
 )
 
@@ -52,20 +54,31 @@ const (
 	minLookupMaskLengthV6 = 16
 )
 
-// SQLite memory tuning. cache_size and busy_timeout go in the DSN so every
-// pooled connection gets them; the heap limits are process-wide and set once.
+// SQLite memory tuning (README "Memory"). cache_size and busy_timeout go in the
+// DSN so every pooled connection gets them; the connection kept for writing
+// then sets its own cache size. The heap limits are process-wide and set once.
 const (
-	// sqliteCacheSizeKiB is the per-connection page cache; negative means KiB.
-	// -65536 = 64 MiB, so at most 640 MiB across the 10-connection pool.
-	sqliteCacheSizeKiB = -65536
+	// maxConns is the size of the connection pool: the connection kept for
+	// writing and 9 reading connections.
+	maxConns = 10
+	// sqliteReadCacheSizeKiB is the page cache of each reading connection;
+	// negative means KiB. -16384 = 16 MiB, at most 144 MiB for the 9. A reading
+	// connection's cache is emptied whenever another connection has written
+	// since its last read, many times a minute, so a large one is wasted.
+	sqliteReadCacheSizeKiB = -16384
+	// sqliteWriteCacheSizeKiB is the page cache of the connection kept for
+	// writing, -1048576 = 1 GiB. It keeps its pages from one write to the next,
+	// so the index pages the route writes touch stay in memory.
+	sqliteWriteCacheSizeKiB = -1048576
 	// sqliteBusyTimeoutMs is how long a connection waits on a locked database.
 	sqliteBusyTimeoutMs = 5000
-	// sqliteSoftHeapLimitBytes (1 GiB) makes SQLite recycle its cache rather
-	// than allocate once its C heap passes this size.
-	sqliteSoftHeapLimitBytes = 1073741824
-	// sqliteHardHeapLimitBytes (1.5 GiB) fails a statement with SQLITE_NOMEM
+	// sqliteSoftHeapLimitBytes (1.5 GiB) makes SQLite recycle its cache rather
+	// than allocate once its C heap passes this size. It is above the caches
+	// together, so it does not shrink them.
+	sqliteSoftHeapLimitBytes = 1610612736
+	// sqliteHardHeapLimitBytes (2 GiB) fails a statement with SQLITE_NOMEM
 	// instead of growing the C heap without bound.
-	sqliteHardHeapLimitBytes = 1610612736
+	sqliteHardHeapLimitBytes = 2147483648
 )
 
 // Settings of the ANALYZE that Analyze runs.
@@ -106,7 +119,10 @@ var (
 
 // Database manages the SQLite database connection and operations.
 type Database struct {
-	db       *sql.DB
+	db *sql.DB
+	// writer is the connection kept for writing, taken from db when the
+	// database opens. Every write uses it while holding mu.
+	writer   *sql.Conn
 	logger   *logger.Logger
 	path     string
 	mu       sync.Mutex
@@ -144,7 +160,7 @@ func New(cfg *config.Config, logger *logger.Logger) (*Database, error) {
 		"file:%s?_cache_size=%d&_synchronous=OFF&_busy_timeout=%d"+
 			"&_auto_vacuum=incremental&_journal_mode=WAL&_txlock=immediate",
 		dbPath,
-		sqliteCacheSizeKiB,
+		sqliteReadCacheSizeKiB,
 		sqliteBusyTimeoutMs,
 	)
 
@@ -153,20 +169,34 @@ func New(cfg *config.Config, logger *logger.Logger) (*Database, error) {
 		return nil, fmt.Errorf("failed to open database: %w", err)
 	}
 
-	err = db.PingContext(context.Background())
+	ctx := context.Background()
+
+	err = db.PingContext(ctx)
 	if err != nil {
 		return nil, fmt.Errorf("failed to ping database: %w", err)
 	}
 
 	// Set connection pool parameters
 	// Multiple connections allow concurrent reads while writes are serialized
-	const maxConns = 10
 	db.SetMaxOpenConns(maxConns)
 	db.SetMaxIdleConns(maxConns)
 	db.SetConnMaxLifetime(0)
 
+	writer, err := db.Conn(ctx)
+	if err != nil {
+		return nil, fmt.Errorf("failed to open the writing connection: %w", err)
+	}
+
+	_, err = writer.ExecContext(ctx,
+		fmt.Sprintf("PRAGMA cache_size=%d", sqliteWriteCacheSizeKiB))
+	if err != nil {
+		return nil, fmt.Errorf("failed to set the writing connection's cache: %w",
+			err)
+	}
+
 	database := &Database{
-		db: db, logger: logger, path: dbPath, counts: &liveCounts{},
+		db: db, writer: writer, logger: logger, path: dbPath,
+		counts: &liveCounts{},
 	}
 
 	err = database.Initialize()
@@ -188,10 +218,11 @@ func New(cfg *config.Config, logger *logger.Logger) (*Database, error) {
 // Initialize creates the database schema if it doesn't exist.
 func (d *Database) Initialize() error {
 	// Set SQLite pragmas for performance. Per-connection settings (cache_size,
-	// synchronous, busy_timeout, journal_mode) live in the DSN; temp_store is
-	// left at its default so DISTINCT temp B-trees spill to disk instead of C
-	// heap. The heap limits below are process-wide, so setting them once here is
-	// enough for the whole pool.
+	// synchronous, busy_timeout, journal_mode) live in the DSN, and New sets the
+	// writing connection's cache_size; temp_store is left at its default so
+	// DISTINCT temp B-trees spill to disk instead of C heap. The heap limits
+	// below are process-wide, so setting them once here is enough for the whole
+	// pool.
 	pragmas := []string{
 		"PRAGMA journal_mode=WAL", // Write-Ahead Logging
 		fmt.Sprintf("PRAGMA soft_heap_limit=%d", sqliteSoftHeapLimitBytes),
@@ -208,7 +239,7 @@ func (d *Database) Initialize() error {
 	// Run WAL checkpoint on startup to consolidate any existing WAL
 	var walPages, checkpointedPages, movedPages int
 
-	err := d.db.QueryRowContext(context.Background(),
+	err := d.writer.QueryRowContext(context.Background(),
 		"PRAGMA wal_checkpoint(TRUNCATE)",
 	).Scan(&walPages, &checkpointedPages, &movedPages)
 	if err != nil {
@@ -229,35 +260,47 @@ func (d *Database) Initialize() error {
 	return nil
 }
 
-// Close closes the database connection.
+// Close closes the writing connection and then the connection pool.
 func (d *Database) Close() error {
-	return d.db.Close()
+	return errors.Join(d.writer.Close(), d.db.Close())
 }
 
-// A live-route upsert is an UPDATE followed, only when no row matched, by an
-// INSERT. A live route is keyed by prefix and peer, so an announcement replaces
-// the peer's route for the prefix whatever its origin AS. The UPDATE's
-// rows-affected count (1 for an existing key, 0 for a new one) is what lets the
-// in-memory route counters stay exact without a COUNT(*). Callers hold the
-// database write lock, so no other writer can insert the same key between the
-// two statements. The id column is set only on INSERT, so an updated route
-// keeps its original id, exactly as the previous ON CONFLICT upsert did.
+// A live-route upsert sets the origin AS of the peer's route for the prefix
+// when the announcement changes it, then updates the rest of the route, and
+// inserts the route only when that update matched no row. A live route is keyed
+// by prefix and peer, so an announcement replaces the peer's route for the
+// prefix whatever its origin AS. The update's rows-affected count (1 for an
+// existing key, 0 for a new one) is what lets the in-memory route counters stay
+// exact without a COUNT(*). Callers hold the database write lock, so no other
+// writer can insert the same key between the statements.
+//
+// SQLite rewrites a row's entry in every index on a column an UPDATE sets, even
+// when the value stays the same, so the update sets only the columns that
+// change: mask_length follows from the prefix, and origin_asn has a statement
+// of its own that changes nothing when the origin AS stays the same, as it
+// mostly does.
 const (
+	updateLiveRouteOriginV4SQL = `UPDATE live_routes_v4 SET origin_asn = ?
+		WHERE prefix = ? AND peer_ip = ? AND origin_asn != ?`
+
 	updateLiveRouteV4SQL = `UPDATE live_routes_v4
-		SET mask_length = ?, origin_asn = ?, as_path = ?, next_hop = ?,
-		last_updated = ? WHERE prefix = ? AND peer_ip = ?`
+		SET as_path = ?, next_hop = ?, last_updated = ?
+		WHERE prefix = ? AND peer_ip = ?`
 
 	insertLiveRouteV4SQL = `INSERT INTO live_routes_v4
-		(id, prefix, mask_length, origin_asn, peer_ip, as_path, next_hop,
-		last_updated) VALUES (?, ?, ?, ?, ?, ?, ?, ?)`
+		(prefix, mask_length, origin_asn, peer_ip, as_path, next_hop,
+		last_updated) VALUES (?, ?, ?, ?, ?, ?, ?)`
+
+	updateLiveRouteOriginV6SQL = `UPDATE live_routes_v6 SET origin_asn = ?
+		WHERE prefix = ? AND peer_ip = ? AND origin_asn != ?`
 
 	updateLiveRouteV6SQL = `UPDATE live_routes_v6
-		SET mask_length = ?, origin_asn = ?, as_path = ?, next_hop = ?,
-		last_updated = ? WHERE prefix = ? AND peer_ip = ?`
+		SET as_path = ?, next_hop = ?, last_updated = ?
+		WHERE prefix = ? AND peer_ip = ?`
 
 	insertLiveRouteV6SQL = `INSERT INTO live_routes_v6
-		(id, prefix, mask_length, origin_asn, peer_ip, as_path, next_hop,
-		last_updated) VALUES (?, ?, ?, ?, ?, ?, ?, ?)`
+		(prefix, mask_length, origin_asn, peer_ip, as_path, next_hop,
+		last_updated) VALUES (?, ?, ?, ?, ?, ?, ?)`
 )
 
 // Before a new route is inserted and after a route is deleted, the write looks
@@ -278,15 +321,22 @@ const (
 		WHERE prefix = ? AND peer_ip = ?`
 )
 
-// upsertRouteRow updates a live route with the update statement of its IP
-// version's table, inserting it when no row matched. It reports whether a new
-// row was inserted and whether that row is the first live route for its prefix.
+// upsertRouteRow updates a live route with the origin and update statements of
+// its IP version's table, inserting it when no row matched. It reports whether
+// a new row was inserted and whether that row is the first live route for its
+// prefix.
 func upsertRouteRow(
-	ctx context.Context, upd, ins, has *sql.Stmt, route *LiveRoute,
+	ctx context.Context, setOrigin, upd, ins, has *sql.Stmt, route *LiveRoute,
 	pathJSON string,
 ) (bool, bool, error) {
-	res, err := upd.ExecContext(ctx, route.MaskLength, route.OriginASN, pathJSON,
-		route.NextHop, route.LastUpdated, route.Prefix, route.PeerIP)
+	_, err := setOrigin.ExecContext(ctx, route.OriginASN, route.Prefix,
+		route.PeerIP, route.OriginASN)
+	if err != nil {
+		return false, false, err
+	}
+
+	res, err := upd.ExecContext(ctx, pathJSON, route.NextHop, route.LastUpdated,
+		route.Prefix, route.PeerIP)
 	if err != nil {
 		return false, false, err
 	}
@@ -307,9 +357,9 @@ func upsertRouteRow(
 		return false, false, err
 	}
 
-	_, err = ins.ExecContext(ctx, route.ID.String(), route.Prefix,
-		route.MaskLength, route.OriginASN, route.PeerIP, pathJSON,
-		route.NextHop, route.LastUpdated)
+	_, err = ins.ExecContext(ctx, route.Prefix, route.MaskLength,
+		route.OriginASN, route.PeerIP, pathJSON, route.NextHop,
+		route.LastUpdated)
 	if err != nil {
 		return false, false, err
 	}
@@ -327,13 +377,23 @@ func upsertRoutesInTable(
 		return 0, nil, nil
 	}
 
-	family, updateSQL, insertSQL, lookupSQL := ipv4Family, updateLiveRouteV4SQL,
-		insertLiveRouteV4SQL, prefixHasLiveRouteV4SQL
+	family, originSQL, updateSQL := ipv4Family, updateLiveRouteOriginV4SQL,
+		updateLiveRouteV4SQL
+	insertSQL, lookupSQL := insertLiveRouteV4SQL, prefixHasLiveRouteV4SQL
+
 	if ipVersion == ipVersionV6 {
-		family, updateSQL, insertSQL, lookupSQL = ipv6Family,
-			updateLiveRouteV6SQL, insertLiveRouteV6SQL, prefixHasLiveRouteV6SQL
+		family, originSQL, updateSQL = ipv6Family, updateLiveRouteOriginV6SQL,
+			updateLiveRouteV6SQL
+		insertSQL, lookupSQL = insertLiveRouteV6SQL, prefixHasLiveRouteV6SQL
 	}
 
+	setOrigin, err := tx.PrepareContext(ctx, originSQL)
+	if err != nil {
+		return 0, nil, fmt.Errorf("failed to prepare %s origin statement: %w",
+			family, err)
+	}
+	defer func() { _ = setOrigin.Close() }()
+
 	upd, err := tx.PrepareContext(ctx, updateSQL)
 	if err != nil {
 		return 0, nil, fmt.Errorf("failed to prepare %s update statement: %w",
@@ -365,8 +425,8 @@ func upsertRoutesInTable(
 			return 0, nil, fmt.Errorf("failed to encode AS path: %w", err)
 		}
 
-		inserted, newPrefix, err := upsertRouteRow(ctx, upd, ins, has, route,
-			string(pathJSON))
+		inserted, newPrefix, err := upsertRouteRow(ctx, setOrigin, upd, ins, has,
+			route, string(pathJSON))
 		if err != nil {
 			return 0, nil, fmt.Errorf("failed to upsert route %s: %w",
 				route.Prefix, err)
@@ -400,9 +460,18 @@ func (d *Database) UpsertLiveRouteBatch(routes []*LiveRoute) error {
 	}
 	defer d.rollback(tx)
 
+	// The rows are written in key order, peer then prefix, so consecutive writes
+	// land on the same index pages. The sort is stable, so of two updates of
+	// one route the later one is still written last.
+	sorted := slices.SortedStableFunc(slices.Values(routes),
+		func(a, b *LiveRoute) int {
+			return cmp.Or(cmp.Compare(a.PeerIP, b.PeerIP),
+				cmp.Compare(a.Prefix, b.Prefix))
+		})
+
 	var routesV4, routesV6 []*LiveRoute
 
-	for _, route := range routes {
+	for _, route := range sorted {
 		if route.IPVersion == ipVersionV4 {
 			routesV4 = append(routesV4, route)
 		} else {
@@ -546,9 +615,17 @@ func (d *Database) DeleteLiveRouteBatch(deletions []LiveRouteDeletion) error {
 	}
 	defer d.rollback(tx)
 
+	// The rows are deleted in key order, peer then prefix, as
+	// UpsertLiveRouteBatch writes them.
+	sorted := slices.SortedFunc(slices.Values(deletions),
+		func(a, b LiveRouteDeletion) int {
+			return cmp.Or(cmp.Compare(a.PeerIP, b.PeerIP),
+				cmp.Compare(a.Prefix, b.Prefix))
+		})
+
 	var deletionsV4, deletionsV6 []LiveRouteDeletion
 
-	for _, del := range deletions {
+	for _, del := range sorted {
 		if del.IPVersion == ipVersionV4 {
 			deletionsV4 = append(deletionsV4, del)
 		} else {
@@ -683,46 +760,11 @@ func (d *Database) DeleteLiveRoutesOfPeer(peerIP string) (int, error) {
 	return int(deletedV4 + deletedV6), nil
 }
 
-// upsertPrefix updates the last_seen time of prefix with the statements of its
-// IP version's table, inserting the prefix when it is new. It reports whether
-// it inserted it.
-func upsertPrefix(
-	ctx context.Context, selectStmt, updateStmt, insertStmt *sql.Stmt,
-	prefix string, timestamp time.Time,
-) (bool, error) {
-	var id string
-
-	err := selectStmt.QueryRowContext(ctx, prefix).Scan(&id)
-
-	switch {
-	case err == nil:
-		// Prefix exists, update last_seen
-		_, err = updateStmt.ExecContext(ctx, timestamp, prefix)
-		if err != nil {
-			return false, fmt.Errorf("failed to update prefix %s: %w",
-				prefix, err)
-		}
-
-		return false, nil
-	case errors.Is(err, sql.ErrNoRows):
-		// Prefix doesn't exist, create it
-		_, err = insertStmt.ExecContext(ctx, generateUUID().String(), prefix,
-			timestamp, timestamp)
-		if err != nil {
-			return false, fmt.Errorf("failed to insert prefix %s: %w",
-				prefix, err)
-		}
-
-		return true, nil
-	default:
-		return false, fmt.Errorf("failed to query prefix %s: %w", prefix, err)
-	}
-}
-
-// upsertPrefixesInTable updates the last_seen time of prefixes, all of one IP
-// version, in the prefix table of that version, inserting the ones that are
-// new. It returns how many it inserted.
-func upsertPrefixesInTable(
+// insertPrefixesInTable inserts the prefixes, all of one IP version, that the
+// prefix table of that version does not hold yet, each first seen at its time
+// in prefixes. A prefix already stored is left as it is. It returns how many it
+// inserted.
+func insertPrefixesInTable(
 	ctx context.Context, tx *loggingTx, prefixes map[string]time.Time,
 	ipVersion int,
 ) (int, error) {
@@ -731,33 +773,15 @@ func upsertPrefixesInTable(
 	}
 
 	family := ipv4Family
-	selectSQL := "SELECT id FROM prefixes_v4 WHERE prefix = ?"
-	updateSQL := "UPDATE prefixes_v4 SET last_seen = ? WHERE prefix = ?"
-	insertSQL := "INSERT INTO prefixes_v4 (id, prefix, first_seen, last_seen) " +
-		"VALUES (?, ?, ?, ?)"
+	insertSQL := "INSERT INTO prefixes_v4 (prefix, first_seen) VALUES (?, ?) " +
+		"ON CONFLICT (prefix) DO NOTHING"
 
 	if ipVersion == ipVersionV6 {
 		family = ipv6Family
-		selectSQL = "SELECT id FROM prefixes_v6 WHERE prefix = ?"
-		updateSQL = "UPDATE prefixes_v6 SET last_seen = ? WHERE prefix = ?"
-		insertSQL = "INSERT INTO prefixes_v6 (id, prefix, first_seen, last_seen) " +
-			"VALUES (?, ?, ?, ?)"
+		insertSQL = "INSERT INTO prefixes_v6 (prefix, first_seen) VALUES (?, ?) " +
+			"ON CONFLICT (prefix) DO NOTHING"
 	}
 
-	selectStmt, err := tx.PrepareContext(ctx, selectSQL)
-	if err != nil {
-		return 0, fmt.Errorf("failed to prepare %s select statement: %w",
-			family, err)
-	}
-	defer func() { _ = selectStmt.Close() }()
-
-	updateStmt, err := tx.PrepareContext(ctx, updateSQL)
-	if err != nil {
-		return 0, fmt.Errorf("failed to prepare %s update statement: %w",
-			family, err)
-	}
-	defer func() { _ = updateStmt.Close() }()
-
 	insertStmt, err := tx.PrepareContext(ctx, insertSQL)
 	if err != nil {
 		return 0, fmt.Errorf("failed to prepare %s insert statement: %w",
@@ -767,14 +791,21 @@ func upsertPrefixesInTable(
 
 	var newPrefixes int
 
-	for prefix, timestamp := range prefixes {
-		inserted, err := upsertPrefix(ctx, selectStmt, updateStmt, insertStmt,
-			prefix, timestamp)
+	// The rows are written in key order, as UpsertLiveRouteBatch writes routes.
+	for _, prefix := range slices.Sorted(maps.Keys(prefixes)) {
+		res, err := insertStmt.ExecContext(ctx, prefix, prefixes[prefix])
 		if err != nil {
-			return 0, err
+			return 0, fmt.Errorf("failed to insert prefix %s: %w", prefix, err)
 		}
 
-		if inserted {
+		// One row for a new prefix, none for one already stored.
+		inserted, err := res.RowsAffected()
+		if err != nil {
+			return 0, fmt.Errorf("failed to count inserted prefix %s: %w",
+				prefix, err)
+		}
+
+		if inserted > 0 {
 			newPrefixes++
 		}
 	}
@@ -782,8 +813,9 @@ func upsertPrefixesInTable(
 	return newPrefixes, nil
 }
 
-// UpdatePrefixesBatch updates the last_seen time for multiple prefixes in a
-// single transaction, the IPv4 prefixes first and then the IPv6 prefixes.
+// UpdatePrefixesBatch stores, in a single transaction, the prefixes that are
+// not stored yet, each first seen at its time in prefixes: the IPv4 prefixes
+// first and then the IPv6 prefixes.
 func (d *Database) UpdatePrefixesBatch(prefixes map[string]time.Time) error {
 	if len(prefixes) == 0 {
 		return nil
@@ -811,12 +843,12 @@ func (d *Database) UpdatePrefixesBatch(prefixes map[string]time.Time) error {
 
 	ctx := context.Background()
 
-	newV4, err := upsertPrefixesInTable(ctx, tx, prefixesV4, ipVersionV4)
+	newV4, err := insertPrefixesInTable(ctx, tx, prefixesV4, ipVersionV4)
 	if err != nil {
 		return err
 	}
 
-	newV6, err := upsertPrefixesInTable(ctx, tx, prefixesV6, ipVersionV6)
+	newV6, err := insertPrefixesInTable(ctx, tx, prefixesV6, ipVersionV6)
 	if err != nil {
 		return err
 	}
@@ -1047,35 +1079,15 @@ func (d *Database) GetOrCreatePrefix(
 		tableName = "prefixes_v6"
 	}
 
-	var (
-		p     Prefix
-		idStr string
-	)
+	var p Prefix
 
-	query := fmt.Sprintf("SELECT id, prefix, first_seen, last_seen "+
-		"FROM %s WHERE prefix = ?", tableName)
+	query := fmt.Sprintf("SELECT prefix, first_seen FROM %s WHERE prefix = ?",
+		tableName)
 
-	err = tx.QueryRow(query, prefix).Scan(&idStr, &p.Prefix, &p.FirstSeen, &p.LastSeen)
+	err = tx.QueryRow(query, prefix).Scan(&p.Prefix, &p.FirstSeen)
 	if err == nil {
-		// Prefix exists, update last_seen
-		p.ID, _ = uuid.Parse(idStr)
+		// Prefix exists; the deferred rollback ends the read.
 		p.IPVersion = ipVersion
-		updateQuery := fmt.Sprintf("UPDATE %s SET last_seen = ? WHERE id = ?", tableName)
-
-		_, err = tx.Exec(updateQuery, timestamp, p.ID.String())
-		if err != nil {
-			return nil, err
-		}
-
-		p.LastSeen = timestamp
-
-		err = tx.Commit()
-		if err != nil {
-			d.logger.Error("Failed to commit transaction for prefix update",
-				"prefix", prefix, "error", err)
-
-			return nil, err
-		}
 
 		return &p, nil
 	}
@@ -1085,12 +1097,11 @@ func (d *Database) GetOrCreatePrefix(
 	}
 
 	// Prefix doesn't exist, create it
-	p = Prefix{ID: generateUUID(), Prefix: prefix, IPVersion: ipVersion,
-		FirstSeen: timestamp, LastSeen: timestamp}
+	p = Prefix{Prefix: prefix, IPVersion: ipVersion, FirstSeen: timestamp}
 	insertQuery := fmt.Sprintf("INSERT INTO %s "+
-		"(id, prefix, first_seen, last_seen) VALUES (?, ?, ?, ?)", tableName)
+		"(prefix, first_seen) VALUES (?, ?)", tableName)
 
-	_, err = tx.Exec(insertQuery, p.ID.String(), p.Prefix, p.FirstSeen, p.LastSeen)
+	_, err = tx.Exec(insertQuery, p.Prefix, p.FirstSeen)
 	if err != nil {
 		return nil, err
 	}
@@ -1389,37 +1400,44 @@ func (d *Database) UpsertLiveRoute(route *LiveRoute) error {
 		return fmt.Errorf("failed to encode AS path: %w", err)
 	}
 
-	updateSQL, insertSQL, lookupSQL := updateLiveRouteV4SQL,
-		insertLiveRouteV4SQL, prefixHasLiveRouteV4SQL
+	originSQL, updateSQL := updateLiveRouteOriginV4SQL, updateLiveRouteV4SQL
+	insertSQL, lookupSQL := insertLiveRouteV4SQL, prefixHasLiveRouteV4SQL
+
 	if route.IPVersion == ipVersionV6 {
-		updateSQL, insertSQL, lookupSQL = updateLiveRouteV6SQL,
-			insertLiveRouteV6SQL, prefixHasLiveRouteV6SQL
+		originSQL, updateSQL = updateLiveRouteOriginV6SQL, updateLiveRouteV6SQL
+		insertSQL, lookupSQL = insertLiveRouteV6SQL, prefixHasLiveRouteV6SQL
 	}
 
 	ctx := context.Background()
 
 	// The write lock is held, so no other writer can insert this key between the
 	// update and the insert even though they are separate autocommit statements.
-	upd, err := d.db.PrepareContext(ctx, updateSQL)
+	setOrigin, err := d.writer.PrepareContext(ctx, originSQL)
+	if err != nil {
+		return fmt.Errorf("failed to prepare origin statement: %w", err)
+	}
+	defer func() { _ = setOrigin.Close() }()
+
+	upd, err := d.writer.PrepareContext(ctx, updateSQL)
 	if err != nil {
 		return fmt.Errorf("failed to prepare update statement: %w", err)
 	}
 	defer func() { _ = upd.Close() }()
 
-	ins, err := d.db.PrepareContext(ctx, insertSQL)
+	ins, err := d.writer.PrepareContext(ctx, insertSQL)
 	if err != nil {
 		return fmt.Errorf("failed to prepare insert statement: %w", err)
 	}
 	defer func() { _ = ins.Close() }()
 
-	has, err := d.db.PrepareContext(ctx, lookupSQL)
+	has, err := d.writer.PrepareContext(ctx, lookupSQL)
 	if err != nil {
 		return fmt.Errorf("failed to prepare prefix lookup statement: %w", err)
 	}
 	defer func() { _ = has.Close() }()
 
-	inserted, newPrefix, err := upsertRouteRow(ctx, upd, ins, has, route,
-		string(pathJSON))
+	inserted, newPrefix, err := upsertRouteRow(ctx, setOrigin, upd, ins, has,
+		route, string(pathJSON))
 	if err != nil {
 		return fmt.Errorf("failed to upsert route %s: %w", route.Prefix, err)
 	}
@@ -2008,7 +2026,7 @@ func (d *Database) UpdateASNWHOIS(
 		WHERE asn = ?
 	`
 
-	_, err := d.db.ExecContext(ctx, query,
+	_, err := d.writer.ExecContext(ctx, query,
 		update.ASName,
 		update.OrgName,
 		update.OrgID,
@@ -2165,7 +2183,8 @@ func (d *Database) GetIPInfoContext(
 // Vacuum runs incremental vacuum to return up to 1000 unused pages to the
 // filesystem. Like Analyze and Checkpoint, it holds the database lock, so it
 // and the batch writes wait for each other instead of failing on SQLite's
-// write lock.
+// write lock, and returns ctx's error without running if ctx ended while it
+// waited: the writing connection it uses does not check ctx itself.
 func (d *Database) Vacuum(ctx context.Context) error {
 	// Free up to 1000 pages per call (~4MB with default 4KB page size)
 	// This keeps each vacuum operation quick
@@ -2174,7 +2193,12 @@ func (d *Database) Vacuum(ctx context.Context) error {
 	d.lock("Vacuum")
 	defer d.unlock()
 
-	rows, err := d.db.QueryContext(ctx,
+	err := ctx.Err()
+	if err != nil {
+		return fmt.Errorf("failed to run incremental vacuum: %w", err)
+	}
+
+	rows, err := d.writer.QueryContext(ctx,
 		fmt.Sprintf("PRAGMA incremental_vacuum(%d)", pagesToFree))
 	if err != nil {
 		return fmt.Errorf("failed to run incremental vacuum: %w", err)
@@ -2204,9 +2228,14 @@ func (d *Database) Analyze(ctx context.Context) error {
 	d.lock("Analyze")
 	defer d.unlock()
 
+	err := ctx.Err()
+	if err != nil {
+		return fmt.Errorf("failed to analyze database: %w", err)
+	}
+
 	// analysis_limit is a setting of the connection, so it is set in the same
 	// call as PRAGMA optimize, which runs both statements on one connection.
-	_, err := d.db.ExecContext(ctx, fmt.Sprintf(
+	_, err = d.writer.ExecContext(ctx, fmt.Sprintf(
 		"PRAGMA analysis_limit=%d; PRAGMA optimize=%#x",
 		analysisLimit, optimizeAllTables))
 	if err != nil {
@@ -2223,7 +2252,12 @@ func (d *Database) Checkpoint(ctx context.Context) error {
 	d.lock("Checkpoint")
 	defer d.unlock()
 
-	_, err := d.db.ExecContext(ctx, "PRAGMA wal_checkpoint(TRUNCATE)")
+	err := ctx.Err()
+	if err != nil {
+		return fmt.Errorf("failed to checkpoint WAL: %w", err)
+	}
+
+	_, err = d.writer.ExecContext(ctx, "PRAGMA wal_checkpoint(TRUNCATE)")
 	if err != nil {
 		return fmt.Errorf("failed to checkpoint WAL: %w", err)
 	}
@@ -2279,9 +2313,9 @@ func (d *Database) unlock() {
 		"held_by", lockedBy, "duration_ms", holdDuration.Milliseconds())
 }
 
-// beginTx starts a new transaction with logging
+// beginTx starts a new transaction on the writing connection, with logging
 func (d *Database) beginTx() (*loggingTx, error) {
-	tx, err := d.db.BeginTx(context.Background(), nil)
+	tx, err := d.writer.BeginTx(context.Background(), nil)
 	if err != nil {
 		return nil, err
 	}