Make live route writes cheaper so the prefix handler keeps up (closes #96)

The live route tables lose their random UUID key; the row id the
prefix-length sample draws is declared as the integer key. The prefix
tables are keyed by prefix and keep only first_seen; a batch inserts only
prefixes not stored yet. A route update sets only the columns that
change. Each kept index names the queries using it. Every write goes
through one connection kept for writing, with a 1 GiB page cache; reading
connections get 16 MiB, heap limits rise to 1.5 and 2 GiB (README
"Memory"). Batch writes go in key order. A benchmark run by hand writes
250,000 updates on 5 million routes faster than the feed's 10,000 a
second; a live run follows.

Model: opus-5-5

internal/database/schema.sql

@@ -27,21 +27,18 @@ CREATE TABLE IF NOT EXISTS asns (
 	whois_updated_at DATETIME    -- when we last fetched WHOIS data
 );
 
--- IPv4 prefixes table
+-- IPv4 prefixes table. Every read and write finds a row by its prefix, so the
+-- rows are stored by prefix (WITHOUT ROWID) and need no index of their own.
 CREATE TABLE IF NOT EXISTS prefixes_v4 (
-	id TEXT PRIMARY KEY,
-	prefix TEXT UNIQUE NOT NULL,
-	first_seen DATETIME NOT NULL,
-	last_seen DATETIME NOT NULL
-);
+	prefix TEXT PRIMARY KEY,
+	first_seen DATETIME NOT NULL
+) WITHOUT ROWID;
 
--- IPv6 prefixes table
+-- IPv6 prefixes table, stored as prefixes_v4 is.
 CREATE TABLE IF NOT EXISTS prefixes_v6 (
-	id TEXT PRIMARY KEY,
-	prefix TEXT UNIQUE NOT NULL,
-	first_seen DATETIME NOT NULL,
-	last_seen DATETIME NOT NULL
-);
+	prefix TEXT PRIMARY KEY,
+	first_seen DATETIME NOT NULL
+) WITHOUT ROWID;
 
 CREATE TABLE IF NOT EXISTS announcements (
 	id TEXT PRIMARY KEY,
@@ -75,11 +72,6 @@ CREATE TABLE IF NOT EXISTS bgp_peers (
 	last_message_type TEXT
 );
 
--- Indexes for prefixes_v4 table
-CREATE INDEX IF NOT EXISTS idx_prefixes_v4_prefix ON prefixes_v4(prefix);
-
--- Indexes for prefixes_v6 table
-CREATE INDEX IF NOT EXISTS idx_prefixes_v6_prefix ON prefixes_v6(prefix);
 CREATE INDEX IF NOT EXISTS idx_announcements_timestamp ON announcements(timestamp);
 CREATE INDEX IF NOT EXISTS idx_announcements_prefix_id ON announcements(prefix_id);
 CREATE INDEX IF NOT EXISTS idx_announcements_peer_asn ON announcements(peer_asn);
@@ -97,9 +89,14 @@ CREATE INDEX IF NOT EXISTS idx_bgp_peers_asn ON bgp_peers(peer_asn);
 CREATE INDEX IF NOT EXISTS idx_bgp_peers_last_seen ON bgp_peers(last_seen);
 CREATE INDEX IF NOT EXISTS idx_bgp_peers_ip ON bgp_peers(peer_ip);
 
--- IPv4 routing table maintained by PrefixHandler
+-- IPv4 routing table maintained by PrefixHandler. A route write changes the
+-- table and its entry in every index on a column the write sets, so the table
+-- has only the indexes that the queries named with them use.
 CREATE TABLE IF NOT EXISTS live_routes_v4 (
-	id TEXT PRIMARY KEY,
+	-- The row id, which the prefix-length pages' sample draws through the
+	-- mask_length index. Declared as a column, it stays the same for as long
+	-- as the route is live, also through a VACUUM.
+	id INTEGER PRIMARY KEY,
 	prefix TEXT NOT NULL,
 	mask_length INTEGER NOT NULL, -- CIDR mask length (0-32)
 	origin_asn INTEGER NOT NULL,
@@ -107,15 +104,15 @@ CREATE TABLE IF NOT EXISTS live_routes_v4 (
 	as_path TEXT NOT NULL, -- JSON array
 	next_hop TEXT NOT NULL,
 	last_updated DATETIME NOT NULL,
-	-- A peer has at most one route for a prefix. peer_ip comes first so that
-	-- this key also finds all of a peer's routes, when its BGP session goes
-	-- down; lookups by prefix use the prefix index below.
+	-- A peer has at most one route for a prefix. This key finds the route a
+	-- write updates or deletes and, peer_ip coming first, all of a peer's
+	-- routes when its BGP session goes down.
 	UNIQUE(peer_ip, prefix)
 );
 
--- IPv6 routing table maintained by PrefixHandler
+-- IPv6 routing table maintained by PrefixHandler, as live_routes_v4.
 CREATE TABLE IF NOT EXISTS live_routes_v6 (
-	id TEXT PRIMARY KEY,
+	id INTEGER PRIMARY KEY,
 	prefix TEXT NOT NULL,
 	mask_length INTEGER NOT NULL, -- CIDR mask length (0-128)
 	origin_asn INTEGER NOT NULL,
@@ -123,24 +120,29 @@ CREATE TABLE IF NOT EXISTS live_routes_v6 (
 	as_path TEXT NOT NULL, -- JSON array
 	next_hop TEXT NOT NULL,
 	last_updated DATETIME NOT NULL,
-	-- As in live_routes_v4: peer_ip first, to find all of a peer's routes.
 	UNIQUE(peer_ip, prefix)
 );
 
--- Indexes for live_routes_v4 table
--- Finds a prefix's newest route in one search, whatever the number of peers.
+-- Indexes for both live route tables, and the queries that use each.
+--
+-- prefix, last_updated: a prefix's routes, for the prefix page and its API
+-- answer, the IP lookup and its peer count, and the check whether a prefix
+-- still has a live route; and a prefix's newest route for the prefix-length
+-- pages, in one search whatever the number of peers.
 CREATE INDEX IF NOT EXISTS idx_live_routes_v4_prefix_last_updated ON live_routes_v4(prefix, last_updated);
-CREATE INDEX IF NOT EXISTS idx_live_routes_v4_mask_length ON live_routes_v4(mask_length);
-CREATE INDEX IF NOT EXISTS idx_live_routes_v4_origin_asn ON live_routes_v4(origin_asn);
-CREATE INDEX IF NOT EXISTS idx_live_routes_v4_last_updated ON live_routes_v4(last_updated);
--- Index to optimize prefix distribution queries
-CREATE INDEX IF NOT EXISTS idx_live_routes_v4_mask_prefix ON live_routes_v4(mask_length, prefix);
-
--- Indexes for live_routes_v6 table
--- Finds a prefix's newest route in one search, whatever the number of peers.
 CREATE INDEX IF NOT EXISTS idx_live_routes_v6_prefix_last_updated ON live_routes_v6(prefix, last_updated);
+-- mask_length: the prefix-length pages' random sample, the row id range of a
+-- mask length and the first route of it at or after a row id.
+CREATE INDEX IF NOT EXISTS idx_live_routes_v4_mask_length ON live_routes_v4(mask_length);
 CREATE INDEX IF NOT EXISTS idx_live_routes_v6_mask_length ON live_routes_v6(mask_length);
+-- mask_length, prefix: the prefix-length pages' list of a mask length's
+-- prefixes in text order, and the prefix distribution read at startup.
+CREATE INDEX IF NOT EXISTS idx_live_routes_v4_mask_prefix ON live_routes_v4(mask_length, prefix);
+CREATE INDEX IF NOT EXISTS idx_live_routes_v6_mask_prefix ON live_routes_v6(mask_length, prefix);
+-- origin_asn: the prefixes an AS originates, for the AS page and its API answer.
+CREATE INDEX IF NOT EXISTS idx_live_routes_v4_origin_asn ON live_routes_v4(origin_asn);
 CREATE INDEX IF NOT EXISTS idx_live_routes_v6_origin_asn ON live_routes_v6(origin_asn);
-CREATE INDEX IF NOT EXISTS idx_live_routes_v6_last_updated ON live_routes_v6(last_updated);
--- Index to optimize prefix distribution queries
-CREATE INDEX IF NOT EXISTS idx_live_routes_v6_mask_prefix ON live_routes_v6(mask_length, prefix);
+-- last_updated: the oldest and newest route times in /api/v1/stats and
+-- /status.json.
+CREATE INDEX IF NOT EXISTS idx_live_routes_v4_last_updated ON live_routes_v4(last_updated);
+CREATE INDEX IF NOT EXISTS idx_live_routes_v6_last_updated ON live_routes_v6(last_updated);