Make live route writes cheaper so the prefix handler keeps up (closes #96)

The live route tables lose their random UUID key: the row id, which the
prefix-length sample draws, is now declared as the integer key. The prefix
tables are keyed by the prefix, without the UUID or the duplicate index.
A route update sets only the columns that change, so SQLite no longer
rewrites the mask length indexes on every update, nor the origin index
while the origin AS stays the same. Each kept index names the queries
that use it. BenchmarkPrefixBatchOnFiveMillionRoutes, run by hand, times
one 25,000-update batch on 5 million routes.

Model: opus-5-5

internal/database/schema.sql

@@ -27,21 +27,20 @@ CREATE TABLE IF NOT EXISTS asns (
 	whois_updated_at DATETIME    -- when we last fetched WHOIS data
 );
 
--- IPv4 prefixes table
+-- IPv4 prefixes table. Every read and write finds a row by its prefix, so the
+-- rows are stored by prefix (WITHOUT ROWID) and need no index of their own.
 CREATE TABLE IF NOT EXISTS prefixes_v4 (
-	id TEXT PRIMARY KEY,
-	prefix TEXT UNIQUE NOT NULL,
+	prefix TEXT PRIMARY KEY,
 	first_seen DATETIME NOT NULL,
 	last_seen DATETIME NOT NULL
-);
+) WITHOUT ROWID;
 
--- IPv6 prefixes table
+-- IPv6 prefixes table, stored as prefixes_v4 is.
 CREATE TABLE IF NOT EXISTS prefixes_v6 (
-	id TEXT PRIMARY KEY,
-	prefix TEXT UNIQUE NOT NULL,
+	prefix TEXT PRIMARY KEY,
 	first_seen DATETIME NOT NULL,
 	last_seen DATETIME NOT NULL
-);
+) WITHOUT ROWID;
 
 CREATE TABLE IF NOT EXISTS announcements (
 	id TEXT PRIMARY KEY,
@@ -75,11 +74,6 @@ CREATE TABLE IF NOT EXISTS bgp_peers (
 	last_message_type TEXT
 );
 
--- Indexes for prefixes_v4 table
-CREATE INDEX IF NOT EXISTS idx_prefixes_v4_prefix ON prefixes_v4(prefix);
-
--- Indexes for prefixes_v6 table
-CREATE INDEX IF NOT EXISTS idx_prefixes_v6_prefix ON prefixes_v6(prefix);
 CREATE INDEX IF NOT EXISTS idx_announcements_timestamp ON announcements(timestamp);
 CREATE INDEX IF NOT EXISTS idx_announcements_prefix_id ON announcements(prefix_id);
 CREATE INDEX IF NOT EXISTS idx_announcements_peer_asn ON announcements(peer_asn);
@@ -97,9 +91,14 @@ CREATE INDEX IF NOT EXISTS idx_bgp_peers_asn ON bgp_peers(peer_asn);
 CREATE INDEX IF NOT EXISTS idx_bgp_peers_last_seen ON bgp_peers(last_seen);
 CREATE INDEX IF NOT EXISTS idx_bgp_peers_ip ON bgp_peers(peer_ip);
 
--- IPv4 routing table maintained by PrefixHandler
+-- IPv4 routing table maintained by PrefixHandler. A route write changes the
+-- table and its entry in every index on a column the write sets, so the table
+-- has only the indexes that the queries named with them use.
 CREATE TABLE IF NOT EXISTS live_routes_v4 (
-	id TEXT PRIMARY KEY,
+	-- The row id, which the prefix-length pages' sample draws through the
+	-- mask_length index. Declared as a column, it stays the same for as long
+	-- as the route is live, also through a VACUUM.
+	id INTEGER PRIMARY KEY,
 	prefix TEXT NOT NULL,
 	mask_length INTEGER NOT NULL, -- CIDR mask length (0-32)
 	origin_asn INTEGER NOT NULL,
@@ -107,15 +106,15 @@ CREATE TABLE IF NOT EXISTS live_routes_v4 (
 	as_path TEXT NOT NULL, -- JSON array
 	next_hop TEXT NOT NULL,
 	last_updated DATETIME NOT NULL,
-	-- A peer has at most one route for a prefix. peer_ip comes first so that
-	-- this key also finds all of a peer's routes, when its BGP session goes
-	-- down; lookups by prefix use the prefix index below.
+	-- A peer has at most one route for a prefix. This key finds the route a
+	-- write updates or deletes and, peer_ip coming first, all of a peer's
+	-- routes when its BGP session goes down.
 	UNIQUE(peer_ip, prefix)
 );
 
--- IPv6 routing table maintained by PrefixHandler
+-- IPv6 routing table maintained by PrefixHandler, as live_routes_v4.
 CREATE TABLE IF NOT EXISTS live_routes_v6 (
-	id TEXT PRIMARY KEY,
+	id INTEGER PRIMARY KEY,
 	prefix TEXT NOT NULL,
 	mask_length INTEGER NOT NULL, -- CIDR mask length (0-128)
 	origin_asn INTEGER NOT NULL,
@@ -123,24 +122,29 @@ CREATE TABLE IF NOT EXISTS live_routes_v6 (
 	as_path TEXT NOT NULL, -- JSON array
 	next_hop TEXT NOT NULL,
 	last_updated DATETIME NOT NULL,
-	-- As in live_routes_v4: peer_ip first, to find all of a peer's routes.
 	UNIQUE(peer_ip, prefix)
 );
 
--- Indexes for live_routes_v4 table
--- Finds a prefix's newest route in one search, whatever the number of peers.
+-- Indexes for both live route tables, and the queries that use each.
+--
+-- prefix, last_updated: a prefix's routes, for the prefix page and its API
+-- answer, the IP lookup and its peer count, and the check whether a prefix
+-- still has a live route; and a prefix's newest route for the prefix-length
+-- pages, in one search whatever the number of peers.
 CREATE INDEX IF NOT EXISTS idx_live_routes_v4_prefix_last_updated ON live_routes_v4(prefix, last_updated);
-CREATE INDEX IF NOT EXISTS idx_live_routes_v4_mask_length ON live_routes_v4(mask_length);
-CREATE INDEX IF NOT EXISTS idx_live_routes_v4_origin_asn ON live_routes_v4(origin_asn);
-CREATE INDEX IF NOT EXISTS idx_live_routes_v4_last_updated ON live_routes_v4(last_updated);
--- Index to optimize prefix distribution queries
-CREATE INDEX IF NOT EXISTS idx_live_routes_v4_mask_prefix ON live_routes_v4(mask_length, prefix);
-
--- Indexes for live_routes_v6 table
--- Finds a prefix's newest route in one search, whatever the number of peers.
 CREATE INDEX IF NOT EXISTS idx_live_routes_v6_prefix_last_updated ON live_routes_v6(prefix, last_updated);
+-- mask_length: the prefix-length pages' random sample, the row id range of a
+-- mask length and the first route of it at or after a row id.
+CREATE INDEX IF NOT EXISTS idx_live_routes_v4_mask_length ON live_routes_v4(mask_length);
 CREATE INDEX IF NOT EXISTS idx_live_routes_v6_mask_length ON live_routes_v6(mask_length);
+-- mask_length, prefix: the prefix-length pages' list of a mask length's
+-- prefixes in text order, and the prefix distribution read at startup.
+CREATE INDEX IF NOT EXISTS idx_live_routes_v4_mask_prefix ON live_routes_v4(mask_length, prefix);
+CREATE INDEX IF NOT EXISTS idx_live_routes_v6_mask_prefix ON live_routes_v6(mask_length, prefix);
+-- origin_asn: the prefixes an AS originates, for the AS page and its API answer.
+CREATE INDEX IF NOT EXISTS idx_live_routes_v4_origin_asn ON live_routes_v4(origin_asn);
 CREATE INDEX IF NOT EXISTS idx_live_routes_v6_origin_asn ON live_routes_v6(origin_asn);
-CREATE INDEX IF NOT EXISTS idx_live_routes_v6_last_updated ON live_routes_v6(last_updated);
--- Index to optimize prefix distribution queries
-CREATE INDEX IF NOT EXISTS idx_live_routes_v6_mask_prefix ON live_routes_v6(mask_length, prefix);
+-- last_updated: the oldest and newest route times in /api/v1/stats and
+-- /status.json.
+CREATE INDEX IF NOT EXISTS idx_live_routes_v4_last_updated ON live_routes_v4(last_updated);
+CREATE INDEX IF NOT EXISTS idx_live_routes_v6_last_updated ON live_routes_v6(last_updated);