clawbot
1f612642e3
check / check (push) Canceled after 0s
The live route tables lose their random UUID key: the row id, which the prefix-length sample draws, is now declared as the integer key. The prefix tables are keyed by the prefix, without the UUID or the duplicate index. A route update sets only the columns that change, so SQLite no longer rewrites the mask length indexes on every update, nor the origin index while the origin AS stays the same. Each kept index names the queries that use it. BenchmarkPrefixBatchOnFiveMillionRoutes, run by hand, times one 25,000-update batch on 5 million routes. Model: opus-5-5
150 lines
6.1 KiB
SQL
150 lines
6.1 KiB
SQL
-- IMPORTANT: This is the ONLY place where schema changes should be made.
|
|
-- We do NOT support migrations. All schema changes MUST be in this file.
|
|
-- DO NOT make schema changes anywhere else in the codebase.
|
|
|
|
CREATE TABLE IF NOT EXISTS asns (
|
|
asn INTEGER PRIMARY KEY,
|
|
handle TEXT,
|
|
description TEXT,
|
|
-- WHOIS parsed fields
|
|
as_name TEXT,
|
|
org_name TEXT,
|
|
org_id TEXT,
|
|
address TEXT, -- full address (may be multi-line)
|
|
country_code TEXT,
|
|
abuse_email TEXT,
|
|
abuse_phone TEXT,
|
|
tech_email TEXT,
|
|
tech_phone TEXT,
|
|
rir TEXT, -- ARIN, RIPE, APNIC, LACNIC, AFRINIC
|
|
rir_registration_date DATETIME,
|
|
rir_last_modified DATETIME,
|
|
-- Raw WHOIS response
|
|
whois_raw TEXT, -- complete WHOIS response text
|
|
-- Timestamps
|
|
first_seen DATETIME NOT NULL,
|
|
last_seen DATETIME NOT NULL,
|
|
whois_updated_at DATETIME -- when we last fetched WHOIS data
|
|
);
|
|
|
|
-- IPv4 prefixes table. Every read and write finds a row by its prefix, so the
|
|
-- rows are stored by prefix (WITHOUT ROWID) and need no index of their own.
|
|
CREATE TABLE IF NOT EXISTS prefixes_v4 (
|
|
prefix TEXT PRIMARY KEY,
|
|
first_seen DATETIME NOT NULL,
|
|
last_seen DATETIME NOT NULL
|
|
) WITHOUT ROWID;
|
|
|
|
-- IPv6 prefixes table, stored as prefixes_v4 is.
|
|
CREATE TABLE IF NOT EXISTS prefixes_v6 (
|
|
prefix TEXT PRIMARY KEY,
|
|
first_seen DATETIME NOT NULL,
|
|
last_seen DATETIME NOT NULL
|
|
) WITHOUT ROWID;
|
|
|
|
CREATE TABLE IF NOT EXISTS announcements (
|
|
id TEXT PRIMARY KEY,
|
|
prefix_id TEXT NOT NULL,
|
|
peer_asn INTEGER NOT NULL,
|
|
origin_asn INTEGER NOT NULL,
|
|
path TEXT NOT NULL,
|
|
next_hop TEXT,
|
|
timestamp DATETIME NOT NULL,
|
|
is_withdrawal BOOLEAN NOT NULL DEFAULT 0,
|
|
FOREIGN KEY (peer_asn) REFERENCES asns(asn),
|
|
FOREIGN KEY (origin_asn) REFERENCES asns(asn)
|
|
);
|
|
|
|
CREATE TABLE IF NOT EXISTS peerings (
|
|
id TEXT PRIMARY KEY,
|
|
as_a INTEGER NOT NULL,
|
|
as_b INTEGER NOT NULL,
|
|
first_seen DATETIME NOT NULL,
|
|
last_seen DATETIME NOT NULL,
|
|
UNIQUE(as_a, as_b)
|
|
);
|
|
|
|
-- BGP peers that send us messages
|
|
CREATE TABLE IF NOT EXISTS bgp_peers (
|
|
id TEXT PRIMARY KEY,
|
|
peer_ip TEXT UNIQUE NOT NULL,
|
|
peer_asn INTEGER NOT NULL,
|
|
first_seen DATETIME NOT NULL,
|
|
last_seen DATETIME NOT NULL,
|
|
last_message_type TEXT
|
|
);
|
|
|
|
CREATE INDEX IF NOT EXISTS idx_announcements_timestamp ON announcements(timestamp);
|
|
CREATE INDEX IF NOT EXISTS idx_announcements_prefix_id ON announcements(prefix_id);
|
|
CREATE INDEX IF NOT EXISTS idx_announcements_peer_asn ON announcements(peer_asn);
|
|
CREATE INDEX IF NOT EXISTS idx_announcements_origin_asn ON announcements(origin_asn);
|
|
CREATE INDEX IF NOT EXISTS idx_peerings_as_a ON peerings(as_a);
|
|
CREATE INDEX IF NOT EXISTS idx_peerings_as_b ON peerings(as_b);
|
|
CREATE INDEX IF NOT EXISTS idx_peerings_lookup ON peerings(as_a, as_b);
|
|
|
|
-- Indexes for asns table
|
|
CREATE INDEX IF NOT EXISTS idx_asns_asn ON asns(asn);
|
|
CREATE INDEX IF NOT EXISTS idx_asns_whois_updated_at ON asns(whois_updated_at);
|
|
|
|
-- Indexes for bgp_peers table
|
|
CREATE INDEX IF NOT EXISTS idx_bgp_peers_asn ON bgp_peers(peer_asn);
|
|
CREATE INDEX IF NOT EXISTS idx_bgp_peers_last_seen ON bgp_peers(last_seen);
|
|
CREATE INDEX IF NOT EXISTS idx_bgp_peers_ip ON bgp_peers(peer_ip);
|
|
|
|
-- IPv4 routing table maintained by PrefixHandler. A route write changes the
|
|
-- table and its entry in every index on a column the write sets, so the table
|
|
-- has only the indexes that the queries named with them use.
|
|
CREATE TABLE IF NOT EXISTS live_routes_v4 (
|
|
-- The row id, which the prefix-length pages' sample draws through the
|
|
-- mask_length index. Declared as a column, it stays the same for as long
|
|
-- as the route is live, also through a VACUUM.
|
|
id INTEGER PRIMARY KEY,
|
|
prefix TEXT NOT NULL,
|
|
mask_length INTEGER NOT NULL, -- CIDR mask length (0-32)
|
|
origin_asn INTEGER NOT NULL,
|
|
peer_ip TEXT NOT NULL,
|
|
as_path TEXT NOT NULL, -- JSON array
|
|
next_hop TEXT NOT NULL,
|
|
last_updated DATETIME NOT NULL,
|
|
-- A peer has at most one route for a prefix. This key finds the route a
|
|
-- write updates or deletes and, peer_ip coming first, all of a peer's
|
|
-- routes when its BGP session goes down.
|
|
UNIQUE(peer_ip, prefix)
|
|
);
|
|
|
|
-- IPv6 routing table maintained by PrefixHandler, as live_routes_v4.
|
|
CREATE TABLE IF NOT EXISTS live_routes_v6 (
|
|
id INTEGER PRIMARY KEY,
|
|
prefix TEXT NOT NULL,
|
|
mask_length INTEGER NOT NULL, -- CIDR mask length (0-128)
|
|
origin_asn INTEGER NOT NULL,
|
|
peer_ip TEXT NOT NULL,
|
|
as_path TEXT NOT NULL, -- JSON array
|
|
next_hop TEXT NOT NULL,
|
|
last_updated DATETIME NOT NULL,
|
|
UNIQUE(peer_ip, prefix)
|
|
);
|
|
|
|
-- Indexes for both live route tables, and the queries that use each.
|
|
--
|
|
-- prefix, last_updated: a prefix's routes, for the prefix page and its API
|
|
-- answer, the IP lookup and its peer count, and the check whether a prefix
|
|
-- still has a live route; and a prefix's newest route for the prefix-length
|
|
-- pages, in one search whatever the number of peers.
|
|
CREATE INDEX IF NOT EXISTS idx_live_routes_v4_prefix_last_updated ON live_routes_v4(prefix, last_updated);
|
|
CREATE INDEX IF NOT EXISTS idx_live_routes_v6_prefix_last_updated ON live_routes_v6(prefix, last_updated);
|
|
-- mask_length: the prefix-length pages' random sample, the row id range of a
|
|
-- mask length and the first route of it at or after a row id.
|
|
CREATE INDEX IF NOT EXISTS idx_live_routes_v4_mask_length ON live_routes_v4(mask_length);
|
|
CREATE INDEX IF NOT EXISTS idx_live_routes_v6_mask_length ON live_routes_v6(mask_length);
|
|
-- mask_length, prefix: the prefix-length pages' list of a mask length's
|
|
-- prefixes in text order, and the prefix distribution read at startup.
|
|
CREATE INDEX IF NOT EXISTS idx_live_routes_v4_mask_prefix ON live_routes_v4(mask_length, prefix);
|
|
CREATE INDEX IF NOT EXISTS idx_live_routes_v6_mask_prefix ON live_routes_v6(mask_length, prefix);
|
|
-- origin_asn: the prefixes an AS originates, for the AS page and its API answer.
|
|
CREATE INDEX IF NOT EXISTS idx_live_routes_v4_origin_asn ON live_routes_v4(origin_asn);
|
|
CREATE INDEX IF NOT EXISTS idx_live_routes_v6_origin_asn ON live_routes_v6(origin_asn);
|
|
-- last_updated: the oldest and newest route times in /api/v1/stats and
|
|
-- /status.json.
|
|
CREATE INDEX IF NOT EXISTS idx_live_routes_v4_last_updated ON live_routes_v4(last_updated);
|
|
CREATE INDEX IF NOT EXISTS idx_live_routes_v6_last_updated ON live_routes_v6(last_updated); |