clawbot
d84dd56460
check / check (push) Canceled after 0s
The live route tables lose their random UUID key; the row id the prefix-length sample draws is declared as the integer key. The prefix tables are keyed by prefix and keep only first_seen; a batch inserts only prefixes not stored yet. A route update sets only the columns that change. Each kept index names the queries using it. Every write goes through one connection kept for writing, with a 1 GiB page cache; reading connections get 16 MiB, heap limits rise to 1.5 and 2 GiB (README "Memory"). Batch writes go in key order. A benchmark run by hand writes 250,000 updates on 5 million routes faster than the feed's 10,000 a second; a live run follows. Model: opus-5-5
148 lines
6.0 KiB
SQL
148 lines
6.0 KiB
SQL
-- IMPORTANT: This is the ONLY place where schema changes should be made.
|
|
-- We do NOT support migrations. All schema changes MUST be in this file.
|
|
-- DO NOT make schema changes anywhere else in the codebase.
|
|
|
|
CREATE TABLE IF NOT EXISTS asns (
|
|
asn INTEGER PRIMARY KEY,
|
|
handle TEXT,
|
|
description TEXT,
|
|
-- WHOIS parsed fields
|
|
as_name TEXT,
|
|
org_name TEXT,
|
|
org_id TEXT,
|
|
address TEXT, -- full address (may be multi-line)
|
|
country_code TEXT,
|
|
abuse_email TEXT,
|
|
abuse_phone TEXT,
|
|
tech_email TEXT,
|
|
tech_phone TEXT,
|
|
rir TEXT, -- ARIN, RIPE, APNIC, LACNIC, AFRINIC
|
|
rir_registration_date DATETIME,
|
|
rir_last_modified DATETIME,
|
|
-- Raw WHOIS response
|
|
whois_raw TEXT, -- complete WHOIS response text
|
|
-- Timestamps
|
|
first_seen DATETIME NOT NULL,
|
|
last_seen DATETIME NOT NULL,
|
|
whois_updated_at DATETIME -- when we last fetched WHOIS data
|
|
);
|
|
|
|
-- IPv4 prefixes table. Every read and write finds a row by its prefix, so the
|
|
-- rows are stored by prefix (WITHOUT ROWID) and need no index of their own.
|
|
CREATE TABLE IF NOT EXISTS prefixes_v4 (
|
|
prefix TEXT PRIMARY KEY,
|
|
first_seen DATETIME NOT NULL
|
|
) WITHOUT ROWID;
|
|
|
|
-- IPv6 prefixes table, stored as prefixes_v4 is.
|
|
CREATE TABLE IF NOT EXISTS prefixes_v6 (
|
|
prefix TEXT PRIMARY KEY,
|
|
first_seen DATETIME NOT NULL
|
|
) WITHOUT ROWID;
|
|
|
|
CREATE TABLE IF NOT EXISTS announcements (
|
|
id TEXT PRIMARY KEY,
|
|
prefix_id TEXT NOT NULL,
|
|
peer_asn INTEGER NOT NULL,
|
|
origin_asn INTEGER NOT NULL,
|
|
path TEXT NOT NULL,
|
|
next_hop TEXT,
|
|
timestamp DATETIME NOT NULL,
|
|
is_withdrawal BOOLEAN NOT NULL DEFAULT 0,
|
|
FOREIGN KEY (peer_asn) REFERENCES asns(asn),
|
|
FOREIGN KEY (origin_asn) REFERENCES asns(asn)
|
|
);
|
|
|
|
CREATE TABLE IF NOT EXISTS peerings (
|
|
id TEXT PRIMARY KEY,
|
|
as_a INTEGER NOT NULL,
|
|
as_b INTEGER NOT NULL,
|
|
first_seen DATETIME NOT NULL,
|
|
last_seen DATETIME NOT NULL,
|
|
UNIQUE(as_a, as_b)
|
|
);
|
|
|
|
-- BGP peers that send us messages
|
|
CREATE TABLE IF NOT EXISTS bgp_peers (
|
|
id TEXT PRIMARY KEY,
|
|
peer_ip TEXT UNIQUE NOT NULL,
|
|
peer_asn INTEGER NOT NULL,
|
|
first_seen DATETIME NOT NULL,
|
|
last_seen DATETIME NOT NULL,
|
|
last_message_type TEXT
|
|
);
|
|
|
|
CREATE INDEX IF NOT EXISTS idx_announcements_timestamp ON announcements(timestamp);
|
|
CREATE INDEX IF NOT EXISTS idx_announcements_prefix_id ON announcements(prefix_id);
|
|
CREATE INDEX IF NOT EXISTS idx_announcements_peer_asn ON announcements(peer_asn);
|
|
CREATE INDEX IF NOT EXISTS idx_announcements_origin_asn ON announcements(origin_asn);
|
|
CREATE INDEX IF NOT EXISTS idx_peerings_as_a ON peerings(as_a);
|
|
CREATE INDEX IF NOT EXISTS idx_peerings_as_b ON peerings(as_b);
|
|
CREATE INDEX IF NOT EXISTS idx_peerings_lookup ON peerings(as_a, as_b);
|
|
|
|
-- Indexes for asns table
|
|
CREATE INDEX IF NOT EXISTS idx_asns_asn ON asns(asn);
|
|
CREATE INDEX IF NOT EXISTS idx_asns_whois_updated_at ON asns(whois_updated_at);
|
|
|
|
-- Indexes for bgp_peers table
|
|
CREATE INDEX IF NOT EXISTS idx_bgp_peers_asn ON bgp_peers(peer_asn);
|
|
CREATE INDEX IF NOT EXISTS idx_bgp_peers_last_seen ON bgp_peers(last_seen);
|
|
CREATE INDEX IF NOT EXISTS idx_bgp_peers_ip ON bgp_peers(peer_ip);
|
|
|
|
-- IPv4 routing table maintained by PrefixHandler. A route write changes the
|
|
-- table and its entry in every index on a column the write sets, so the table
|
|
-- has only the indexes that the queries named with them use.
|
|
CREATE TABLE IF NOT EXISTS live_routes_v4 (
|
|
-- The row id, which the prefix-length pages' sample draws through the
|
|
-- mask_length index. Declared as a column, it stays the same for as long
|
|
-- as the route is live, also through a VACUUM.
|
|
id INTEGER PRIMARY KEY,
|
|
prefix TEXT NOT NULL,
|
|
mask_length INTEGER NOT NULL, -- CIDR mask length (0-32)
|
|
origin_asn INTEGER NOT NULL,
|
|
peer_ip TEXT NOT NULL,
|
|
as_path TEXT NOT NULL, -- JSON array
|
|
next_hop TEXT NOT NULL,
|
|
last_updated DATETIME NOT NULL,
|
|
-- A peer has at most one route for a prefix. This key finds the route a
|
|
-- write updates or deletes and, peer_ip coming first, all of a peer's
|
|
-- routes when its BGP session goes down.
|
|
UNIQUE(peer_ip, prefix)
|
|
);
|
|
|
|
-- IPv6 routing table maintained by PrefixHandler, as live_routes_v4.
|
|
CREATE TABLE IF NOT EXISTS live_routes_v6 (
|
|
id INTEGER PRIMARY KEY,
|
|
prefix TEXT NOT NULL,
|
|
mask_length INTEGER NOT NULL, -- CIDR mask length (0-128)
|
|
origin_asn INTEGER NOT NULL,
|
|
peer_ip TEXT NOT NULL,
|
|
as_path TEXT NOT NULL, -- JSON array
|
|
next_hop TEXT NOT NULL,
|
|
last_updated DATETIME NOT NULL,
|
|
UNIQUE(peer_ip, prefix)
|
|
);
|
|
|
|
-- Indexes for both live route tables, and the queries that use each.
|
|
--
|
|
-- prefix, last_updated: a prefix's routes, for the prefix page and its API
|
|
-- answer, the IP lookup and its peer count, and the check whether a prefix
|
|
-- still has a live route; and a prefix's newest route for the prefix-length
|
|
-- pages, in one search whatever the number of peers.
|
|
CREATE INDEX IF NOT EXISTS idx_live_routes_v4_prefix_last_updated ON live_routes_v4(prefix, last_updated);
|
|
CREATE INDEX IF NOT EXISTS idx_live_routes_v6_prefix_last_updated ON live_routes_v6(prefix, last_updated);
|
|
-- mask_length: the prefix-length pages' random sample, the row id range of a
|
|
-- mask length and the first route of it at or after a row id.
|
|
CREATE INDEX IF NOT EXISTS idx_live_routes_v4_mask_length ON live_routes_v4(mask_length);
|
|
CREATE INDEX IF NOT EXISTS idx_live_routes_v6_mask_length ON live_routes_v6(mask_length);
|
|
-- mask_length, prefix: the prefix-length pages' list of a mask length's
|
|
-- prefixes in text order, and the prefix distribution read at startup.
|
|
CREATE INDEX IF NOT EXISTS idx_live_routes_v4_mask_prefix ON live_routes_v4(mask_length, prefix);
|
|
CREATE INDEX IF NOT EXISTS idx_live_routes_v6_mask_prefix ON live_routes_v6(mask_length, prefix);
|
|
-- origin_asn: the prefixes an AS originates, for the AS page and its API answer.
|
|
CREATE INDEX IF NOT EXISTS idx_live_routes_v4_origin_asn ON live_routes_v4(origin_asn);
|
|
CREATE INDEX IF NOT EXISTS idx_live_routes_v6_origin_asn ON live_routes_v6(origin_asn);
|
|
-- last_updated: the oldest and newest route times in /api/v1/stats and
|
|
-- /status.json.
|
|
CREATE INDEX IF NOT EXISTS idx_live_routes_v4_last_updated ON live_routes_v4(last_updated);
|
|
CREATE INDEX IF NOT EXISTS idx_live_routes_v6_last_updated ON live_routes_v6(last_updated); |