diff --git a/Dockerfile b/Dockerfile
index f2f9d83..e54904d 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -121,10 +121,11 @@ RUN chown -R routewatch:routewatch /app
ENV XDG_DATA_HOME=/var/lib
-# Cap the Go heap at 1.5 GiB so the runtime collects harder before the
-# container's memory limit is reached. setpriv in the entrypoint preserves this
-# the way it does XDG_DATA_HOME above.
-ENV GOMEMLIMIT=1536MiB
+# Cap the Go heap at 2.5 GiB so the runtime collects harder before the
+# container's memory limit is reached. A full routing table takes about 1 GB
+# of it (README "Memory"). setpriv in the entrypoint preserves this the way it
+# does XDG_DATA_HOME above.
+ENV GOMEMLIMIT=2560MiB
# Cap glibc's malloc arenas. The SQLite C library allocates and frees millions
# of small page-cache chunks from many threads; glibc otherwise creates up to
@@ -139,8 +140,13 @@ EXPOSE 8080
COPY ./entrypoint.sh /entrypoint.sh
-# Health check using the health endpoint, on the port PORT names
-HEALTHCHECK --interval=30s --timeout=5s --start-period=10s --retries=3 \
+# Health check using the health endpoint, on the port PORT names. During the
+# start period, Docker 25 and later checks every 5 seconds, and the first pass
+# makes the container healthy. After it, Docker checks every 30 seconds. The
+# start period covers the 60 seconds after which upaas reads the health
+# (README "Running under upaas"). Startup reads the routing table first,
+# which takes about 25 seconds when the table is full.
+HEALTHCHECK --interval=30s --timeout=5s --start-period=60s --retries=3 \
CMD curl -sf "http://localhost:${PORT:-8080}/.well-known/healthcheck.json" || exit 1
ENTRYPOINT ["/bin/bash", "/entrypoint.sh" ]
diff --git a/Makefile b/Makefile
index 4c39949..bf3f73e 100644
--- a/Makefile
+++ b/Makefile
@@ -52,7 +52,6 @@ clean:
run: build
DEBUG=routewatch ./bin/routewatch 2>&1 | tee log.txt
+# Rebuild the AS list built into RouteWatch from as.csv (needs the network).
asupdate:
- @echo "Updating AS info data..."
- @go run cmd/asinfo-gen/main.go | gzip > pkg/asinfo/asdata.json.gz.tmp && \
- mv pkg/asinfo/asdata.json.gz.tmp pkg/asinfo/asdata.json.gz
+ @script/asupdate
diff --git a/README.md b/README.md
index 7c3629f..29fcb56 100644
--- a/README.md
+++ b/README.md
@@ -1,25 +1,27 @@
# RouteWatch
RouteWatch is an MIT-licensed Go daemon by @sneak that monitors the BGP routing
-table in real time: it streams BGP UPDATE messages from the RIPE RIS Live
-service, maintains a live routing table in SQLite, and provides HTTP APIs for
-querying routing information.
+table in real time. It works like this:
+
+- It streams BGP UPDATE messages from the RIPE RIS Live service.
+- It keeps the whole routing table in memory, with a copy in SQLite.
+- It serves web pages and a JSON API for looking routes up.
## Rationale
RIPE RIS Live streams the BGP updates that RIPE's route collectors receive, as
they happen, from the moment a client connects. It reports changes, not the
routing table they add up to, so it cannot say which AS an IP address is routed
-to now or which routes a prefix has. RouteWatch applies the feed's updates to
-live routing tables in its own SQLite database and answers IP-to-AS lookups and
-prefix pages from them.
+to now or which routes a prefix has. RouteWatch applies the feed's updates to a
+routing table that it keeps in memory, with a copy in its own SQLite database.
+It answers IP-to-AS lookups and prefix pages from that table.
-The feed sends no table when a client connects, so these tables hold only the
-routes the feed has announced since RouteWatch first started: an IP-to-AS lookup
-finds nothing for an address whose routes have not been announced since then.
-The tables also miss the updates RouteWatch drops when it falls behind under
-load and those sent while it is stopped, so a lookup or a prefix page can show a
-route the feed has withdrawn.
+The feed sends no table when a client connects. So the routing table holds only
+the routes announced since RouteWatch first started. An IP-to-AS lookup finds
+nothing for an address whose routes have not been announced since then. The
+routing table also misses updates RouteWatch drops when it falls behind, and
+updates sent while it is stopped. So a lookup or a prefix page can show a route
+the feed has withdrawn.
## Features
@@ -27,6 +29,8 @@ route the feed has withdrawn.
- Maintains live IPv4 and IPv6 routing tables
- Tracks AS peering relationships
- HTTP API for IP-to-AS lookups, prefix details, and AS information
+- IP lookups report the public blocklists an address is on, from copies kept in
+ memory and on disk
- Automatic reconnection with exponential backoff
- Batched database writes for high performance
- Backpressure handling to prevent memory exhaustion
@@ -54,22 +58,37 @@ DEBUG=routewatch ./routewatch
Every route answers `HEAD` as it answers `GET`, without the body.
+The pages and the API read routes from the routing table that RouteWatch keeps
+in memory. For each prefix and origin AS, the table keeps the two newest routes
+in full, with their AS paths and times. These are the kept routes. The table
+keeps only the address of every other peer that carries the prefix. When both
+kept routes of an origin AS are withdrawn, that origin has no kept route until
+one of its peers sends an update.
+
### Web Interface
-- `GET /` - Home page (HTML): headline figures, and lookups by AS number and by
- IP address
+- `GET /` - Home page (HTML): headline figures, lookups by AS number and by IP
+ address, and a search of AS names. A card shows the `curl` command that looks
+ up the visitor's own address at `/ip/{addr}`, and that command's live answer.
- `GET /status` - HTML status dashboard, which reads `/api/v1/stats`
-- `GET /status.json` - JSON statistics: feed, memory, database and route figures
+- `GET /status.json` - JSON statistics: feed, memory, database and route
+ figures. The route figures come from the routing table. `oldest_route` and
+ `newest_route` are the times of the oldest and the newest kept routes.
+ `origins_without_kept_route` counts, across all prefixes, the origin ASes that
+ have no kept route.
- `GET /.well-known/healthcheck.json` - Health check (JSON): 200 when the
database answers and the RIS Live feed is connected, otherwise 503;
`data.status` is `ok` or `error` and `data.checks` says which check failed
- `GET /as/{asn}` - AS detail page (HTML): the prefixes the AS originates and
- its peers
+ its peers. When the database has no record of the AS, its handle and
+ description come from the AS list built into RouteWatch. The page then shows
+ no first-seen date.
- `GET /prefix/{prefix}/{length}` - Prefix detail page (HTML), such as
- `/prefix/192.0.2.0/24`: every live route for the prefix, one per peer, and its
- origin ASes
+ `/prefix/192.0.2.0/24`: its origin ASes and every peer that carries it. Only
+ the kept routes show an AS path, a next hop and a time.
- `GET /prefixlength/{length}` - Up to 500 IPv4 prefixes with that mask length,
- picked at random, each with its newest route (HTML)
+ picked at random (HTML). Each shows the origin AS that most peers carry it
+ with, and the age of its newest kept route.
- `GET /prefixlength6/{length}` - The same for IPv6
- `GET /ip/{addr}` - The same JSON answer as `/api/v1/ip/{ip}`
- `GET /ip/` - The same, for the caller's own address: the address the request
@@ -79,23 +98,51 @@ Every route answers `HEAD` as it answers `GET`, without the body.
### API v1
- `GET /api/v1/stats` - The `/status.json` figures plus connection, garbage
- collector, stream and handler queue statistics
-- `GET /api/v1/ip/{ip}` - Looks up each address: its most specific live route
- (`netblock`), that route's origin AS with its WHOIS details, and the address's
- PTR names. `{ip}` is an IP address, a host name, or a comma-separated list of
- at most 10 of these. Of the addresses a host name resolves to, only public
- ones are looked up: a name with none, such as one that resolves to a private
- address, is reported as `host name has no public address`, without its
- addresses. `results` has one entry per address with a live route and `errors`
- one per address or name that failed. The answer is 400 when `{ip}` lists more
- than 10, when it comes to more than 10 addresses once its names are resolved,
- or when nothing in it is an IP address or a host name with a public address;
- and 404 when none of the addresses has a live route. A PTR lookup gets 2
- seconds; an address whose lookup takes longer is answered without PTR names.
+ collector, stream and handler queue statistics. `stream.route_churn_per_min`
+ is the number of prefix announcements and withdrawals, from all peers, in the
+ last full clock minute.
+- `GET /api/v1/ip/{ip}` - Looks up each address. `{ip}` is an IP address, a host
+ name, or a comma-separated list of at most 10 of these. `results` has one
+ entry for each address, with:
+ - `ip` and `ptr`: the address and its PTR names
+ - `netblock` and the fields beside it: the most specific prefix that holds
+ the address, and its origin AS with that AS's WHOIS details. For a prefix
+ with several origin ASes, `asn` is the one that most peers carry.
+ `last_seen` is the time of the prefix's newest kept route.
+ - `blocklists`: the public blocklists the address is on (see Blocklists)
+
+ An address with no live route has a result without the route and AS fields.
+ Of the addresses a host name resolves to, only public ones are looked up. A
+ name with none, such as one that resolves to a private address, is reported
+ as `host name has no public address`, without its addresses. `errors` has
+ one entry for each host name that failed. The answer is 400 when:
+ - `{ip}` lists more than 10
+ - `{ip}` comes to more than 10 addresses once its names are resolved
+ - nothing in `{ip}` is an IP address or a host name with a public address
+
+ A PTR lookup gets 2 seconds. An address whose lookup takes longer is
+ answered without PTR names.
+
- `GET /api/v1/as/{asn}` - The AS and the prefixes it originates, IPv4 and IPv6
- apart, each with when its newest route was updated
-- `GET /api/v1/prefix/{prefix}/{length}` - Every live route for the prefix, one
- per peer, and the same routes grouped by origin AS in `origins`
+ apart, each with the time of its newest kept route. A prefix that keeps no
+ route with the AS has no `last_updated`. When the database has no record of
+ the AS, its handle and description come from the AS list built into
+ RouteWatch. The answer then has no `first_seen` or `last_seen`.
+- `GET /api/v1/prefix/{prefix}/{length}` - One route for each peer that carries
+ the prefix, and the same routes grouped by origin AS in `origins`. Only the
+ kept routes have `as_path`, `next_hop` and `last_updated`.
+- `GET /api/v1/search/as?q={text}` - Finds the ASes whose handle or description
+ contains `{text}`, ignoring case. `data` holds:
+ - `query`: `{text}` without its leading and trailing spaces
+ - `total`: the number of ASes that match
+ - `results`: the first 50 of them in AS number order, each with `asn`,
+ `handle` and `description`
+
+ The search reads the AS list built into RouteWatch, not the database. The
+ list comes from
+ [ipverse/as-metadata](https://github.com/ipverse/as-metadata). A match can
+ therefore be an AS that the database has no record of. A `q` that is
+ missing, empty or only spaces answers 400.
A JSON answer is `{"status": "ok", "data": ...}`, or for an error
`{"status": "error", "error": {"msg": ..., "code": ...}}`, and carries an
@@ -106,13 +153,16 @@ took. The health check is the exception: its 503 answer is also
A request that runs past its time limit answers 504: the limit is 30 seconds,
and 4 seconds for the database reads behind `/status.json` and `/api/v1/stats`.
An AS number, prefix or mask length that does not parse answers 400, as does a
-mask length out of range for its IP version. An AS the database does not hold,
-or a prefix with no live route, answers 404. The HTML pages give these errors as
-plain text. Under `/api/v1` and `/ip/`, a path no route matches answers 404 with
-the message `Not found`, and `POST`, `PUT`, `PATCH`, `DELETE`, `OPTIONS`,
-`TRACE` or `CONNECT` answers 405 with `Method not allowed`. A 500 answer says
-only `Internal server error`, and a host name that does not resolve only
-`could not resolve host name`; the detail goes to the log.
+mask length out of range for its IP version. An AS answers 404 when it
+originates no prefix and has no record in the database or in the AS list built
+into RouteWatch. AS 0 always answers 404, because it means no known origin. The
+AS name search never lists it. A prefix with no live route answers 404 too. The
+HTML pages give these errors as plain text. Under `/api/v1` and `/ip/`, a path
+no route matches answers 404 with the message `Not found`, and `POST`, `PUT`,
+`PATCH`, `DELETE`, `OPTIONS`, `TRACE` or `CONNECT` answers 405 with
+`Method not allowed`. A 500 answer says only `Internal server error`, and a host
+name that does not resolve only `could not resolve host name`; the detail goes
+to the log.
Every answer carries `Strict-Transport-Security`, `Content-Security-Policy`,
`X-Frame-Options`, `X-Content-Type-Options`, `Referrer-Policy` and
@@ -128,6 +178,66 @@ in a prefix's `origins`) means the route has no known origin: its AS path ends
in an AS set of several members, any of which may have originated it. The pages
show such a route as having no known origin.
+## Blocklists
+
+The IP lookup reports which public blocklists list each address. Each result has
+a `blocklists` list with one entry for each list the address is on:
+
+- `name`: the list's name, from the lists below
+- `description`: what being on the list means
+- `updated`: when RouteWatch last downloaded the list, or last heard from its
+ publisher that its copy was current
+
+An address on no list has an empty `blocklists`. An address with no live route
+still has a result with its `blocklists`.
+
+The lists:
+
+- `dshield`: the 20 networks that DShield saw scan the most targets in the last
+ three days. From [DShield](https://isc.sans.edu/feeds_doc.html), downloaded
+ every hour.
+- `feodo_tracker`: botnet command-and-control servers. From abuse.ch
+ [Feodo Tracker](https://feodotracker.abuse.ch/blocklist/), downloaded every 15
+ minutes.
+- `firehol_abusers_30d`: addresses reported for comment, form or forum spam in
+ the last 30 days. From
+ [FireHOL](https://iplists.firehol.org/?ipset=firehol_abusers_30d), downloaded
+ every 5 hours.
+- `firehol_level3`: addresses reported for attacks, spyware or viruses in the
+ last 30 days. From
+ [FireHOL](https://iplists.firehol.org/?ipset=firehol_level3), downloaded every
+ 3 hours.
+- `spamhaus_drop` and `spamhaus_dropv6`: IPv4 and IPv6 networks that Spamhaus
+ says spammers or cyber criminals have hijacked or lease. From
+ [Spamhaus DROP](https://www.spamhaus.org/blocklists/do-not-route-or-peer/),
+ downloaded once a day. Spamhaus merged its EDROP list into DROP.
+- `tor_exits`: Tor exit nodes, where traffic from the Tor network leaves. From
+ the [Tor Project](https://check.torproject.org/torbulkexitlist), downloaded
+ every hour.
+
+No list is downloaded more often than its publisher updates it or allows. Each
+can be downloaded whole without an account or a key.
+
+How RouteWatch keeps the lists:
+
+- It holds each list in memory, so a lookup makes no outside request.
+- It keeps a copy of each list in the `blocklists` directory of the state
+ directory (see Configuration). Each copy is a file named after its list.
+- Beside each copy, a file ending in `.validators` keeps the `Last-Modified` and
+ `ETag` the publisher sent with that copy.
+- At startup it loads each list from its copy. It downloads a list at once when
+ the copy is older than the list's interval, or when there is no copy.
+- A download sends the copy's `Last-Modified` as `If-Modified-Since` and its
+ `ETag` as `If-None-Match`. The publisher then sends the list only when it has
+ changed. Without either, the download asks for the whole list.
+- A download fails when it takes over a minute or its file is over 16 MiB.
+- When a download fails, or its file does not parse, RouteWatch keeps the
+ previous copy and logs the failure. It tries again one interval later.
+
+A successful IP lookup answer carries `Cache-Control` with a `max-age` of 15
+minutes, the shortest list interval. The answer for the caller's own address
+(`/ip/`) is `private`, and the others are `public`.
+
## Code Structure
```
@@ -137,9 +247,11 @@ routewatch/
│ ├── asinfo-gen/ # Utility to generate AS info data
│ └── streamdumper/ # Debug utility for raw stream output
├── internal/
+│ ├── blocklist/ # Public IP blocklists: downloads and lookups
│ ├── routewatch/ # Core application logic
│ ├── server/ # HTTP server and handlers
│ ├── database/ # SQLite storage layer
+│ ├── routetable/ # The whole routing table in memory
│ ├── streamer/ # RIPE RIS Live client
│ ├── ristypes/ # BGP message data structures
│ ├── logger/ # Structured logging wrapper
@@ -156,31 +268,36 @@ routewatch/
### Component Relationships
+`internal/routewatch/app.go` builds and starts these parts:
+
```
-┌─────────────────────────────────────────────────────────────────┐
-│ RouteWatch │
-│ (internal/routewatch/app.go - main orchestrator) │
-├─────────────────────────────────────────────────────────────────┤
-│ │
-│ ┌──────────────┐ ┌──────────────┐ ┌──────────────┐ │
-│ │ Streamer │───▶│ Handlers │───▶│ Database │ │
-│ │ │ │ │ │ │ │
-│ │ RIS Live │ │ - ASHandler │ │ SQLite with │ │
-│ │ HTTP stream │ │ - PeerHandler│ │ WAL mode │ │
-│ │ client │ │ - PrefixHdlr │ │ │ │
-│ │ │ │ - PeeringHdlr│ │ Tables: │ │
-│ └──────────────┘ └──────────────┘ │ - asns │ │
-│ │ - prefixes │ │
-│ ┌──────────────┐ ┌──────────────┐ │ - live_routes│ │
-│ │ Server │───▶│ Handlers │───▶│ - peerings │ │
-│ │ │ │ │ │ - bgp_peers │ │
-│ │ Chi router │ │ Status, API │ └──────────────┘ │
-│ │ port 8080 │ │ AS, Prefix │ │
-│ └──────────────┘ └──────────────┘ │
-│ │
-└─────────────────────────────────────────────────────────────────┘
+┌──────────────┐ ┌──────────────┐ ┌──────────────┐
+│ Streamer │───▶│ Handlers │───▶│ Database │
+│ │ │ │ │ │
+│ RIS Live │ │ - ASHandler │ │ SQLite with │
+│ HTTP stream │ │ - PeerHandler│ │ WAL mode │
+│ client │ │ - PrefixHdlr │ │ │
+│ │ │ - PeeringHdlr│ │ Tables: │
+└──────────────┘ └──────┬───────┘ │ - asns │
+ │ │ - prefixes │
+ ▼ │ - routes │
+ ┌──────────────┐ │ - route_peers│
+ │ Routing table│───▶│ - peerings │
+ │ in memory │ │ - bgp_peers │
+ └──────────────┘ └──────────────┘
+ ▲ ▲
+┌──────────────┐ ┌──────┴───────┐ │
+│ Server │───▶│ Handlers │───────────┘
+│ │ │ │
+│ Chi router │ │ Status, API │
+│ port 8080 │ │ AS, Prefix │
+└──────────────┘ └──────────────┘
```
+The prefix handler keeps the routing table in memory current. It writes the
+table's changes to the database every second. The web handlers read routes from
+the routing table, and everything else from the database.
+
### Execution Flow
1. **Startup** (`cmd/routewatch/main.go` → `internal/routewatch/cli.go`)
@@ -189,13 +306,17 @@ routewatch/
2. **Initialization** (`internal/routewatch/app.go`)
- Database created with SQLite schema (WAL mode, page caches as in "Memory")
+ - The routing table read from the `routes` table into memory, before the
+ streamer connects. A full table takes about 25 seconds to read.
- Message handlers registered with the streamer
- HTTP server started on configured port
3. **Message Processing Pipeline**
```
- RIS Live Stream → JSON Parser → Message Dispatcher → Handler Queues → Batch Writers → SQLite
+ RIS Live Stream → JSON Parser → Message Dispatcher → Handler Queues
+ ├→ Batch Writers → SQLite
+ └→ PrefixHandler → Routing Table in Memory → SQLite, every second
```
- Streamer connects to `ris-live.ripe.net` via HTTP
@@ -210,21 +331,41 @@ routewatch/
4. **Handler Details**
- **ASHandler**: Tracks all ASNs seen in AS paths
- **PeerHandler**: Records BGP peer information
- - **PrefixHandler**: Maintains live routing table (upserts on announcement,
- deletes on withdrawal or on an announcement whose AS path is empty or
- holds AS 0 anywhere, deletes all of a peer's routes when its BGP session
- goes down). Routes are kept per peer IP, so when one peer IP has sessions
- on two collectors, a session going down on one also removes the routes the
- other still carries, until they are announced again.
+ - **PrefixHandler**: Keeps the routing table in memory current. It applies
+ each update to the table as it handles it:
+ - An announcement gives the peer a route to the prefix.
+ - A withdrawal removes the peer's route. So does an announcement whose
+ AS path is empty or holds AS 0 anywhere.
+ - A peer whose BGP session goes down leaves every prefix.
+
+ Every second it writes the changed prefix and origin AS pairs to the
+ `routes` table. A write takes at most 20,000 pairs, and the rest follow
+ in the next writes. It also stores each new prefix with the time it was
+ first seen. Peers are known by their IP address. A peer IP can have
+ sessions on two collectors. A session going down on one then also
+ removes the routes the other still carries, until they are announced
+ again.
+
- **PeeringHandler**: Extracts AS peering relationships from AS paths
5. **HTTP Request Flow**
```
- Request → Chi Router → Middleware (timeout, logging) → Handler → Database Query → Response
+ Request → Chi Router → Middleware (timeout, logging) → Handler → Routing Table or Database → Response
```
### Key Design Patterns
+- **Routing Table in Memory**: `internal/routetable` holds the whole routing
+ table. For each prefix and origin AS it keeps:
+ - every peer whose current route to the prefix has that origin AS, one bit
+ per peer;
+ - the newest two of those routes in full, with AS path and time.
+
+ Of the other routes, only the peer is kept. The prefix handler writes the
+ changed pairs to the `routes` table every second, one row per prefix and
+ origin AS. Startup reads that table back into memory before the streamer
+ connects. A crash loses only the changes not yet written.
+
- **Batched Writes**: The handlers write the feed's updates to the database in
batches for performance
- **Backpressure**: Probabilistic message dropping when queues exceed 50%
@@ -239,11 +380,10 @@ routewatch/
with exponential backoff (5s-320s). The backoff resets after a stream that
lasted more than 30s. A stream with no line for 60s is dropped and
reconnected.
-- **IP Lookup**: the most specific live route is found by looking up the
- address's prefix at each mask length, longest first, in the prefix index (at
- most 25 lookups for IPv4, 113 for IPv6); prefixes are stored in the text form
- Go's `net/netip` prints. Routes shorter than /8 for IPv4 or /16 for IPv6, such
- as a default route, are not used.
+- **IP Lookup**: the routing table in memory finds the most specific prefix that
+ holds the address. It looks up the address's prefix at each mask length,
+ longest first: at most 25 lookups for IPv4 and 113 for IPv6. Prefixes shorter
+ than /8 for IPv4 or /16 for IPv6, such as a default route, are not used.
### Database Schema
@@ -257,13 +397,14 @@ asns(asn PRIMARY KEY, handle, description,
prefixes_v4(prefix PRIMARY KEY, first_seen) WITHOUT ROWID
prefixes_v6(prefix PRIMARY KEY, first_seen) WITHOUT ROWID
--- Live routing tables (one per IP version), one route per peer and prefix
-live_routes_v4(id INTEGER PRIMARY KEY, prefix, mask_length, origin_asn,
- peer_ip, as_path, next_hop, last_updated,
- UNIQUE(peer_ip, prefix))
-live_routes_v6(id INTEGER PRIMARY KEY, prefix, mask_length, origin_asn,
- peer_ip, as_path, next_hop, last_updated,
- UNIQUE(peer_ip, prefix))
+-- The routing table, one row per prefix and origin AS that some peer
+-- carries: the peer set, and the newest two routes in full
+routes(prefix, origin_asn, peers,
+ newest_peer, newest_as_path, newest_time,
+ older_peer, older_as_path, older_time,
+ PRIMARY KEY (prefix, origin_asn)) WITHOUT ROWID
+-- The peers of the routes table, each with its number in the peer sets
+route_peers(number PRIMARY KEY, address UNIQUE)
-- Relationship tracking
peerings(id PRIMARY KEY, as_a, as_b, first_seen, last_seen,
@@ -287,7 +428,7 @@ Configuration is handled via environment variables and OS-specific paths:
| `PORT` | `8080` | HTTP server port, a whole number from 1 to 65535 |
| `DEBUG` | (empty) | Set to `routewatch` for debug logging |
| `XDG_DATA_HOME` | `/var/lib` (in the Docker image) | Base of the state directory; must be an absolute path |
-| `GOMEMLIMIT` | `1536MiB` (in the Docker image) | Go soft memory limit; see Memory |
+| `GOMEMLIMIT` | `2560MiB` (in the Docker image) | Go soft memory limit; see Memory |
| `MALLOC_ARENA_MAX` | `2` (in the Docker image) | glibc malloc arena cap, a positive whole number; see Memory |
| `TRUSTED_PROXIES` | (empty) | Addresses and CIDR ranges of trusted proxies; see below |
@@ -303,7 +444,7 @@ client sent. For a request from any other address they use the address the
request came from, whatever forwarding headers it carries. `X-Real-IP` and
`True-Client-IP` are not read.
-State directory (database location):
+State directory (the database and the blocklist copies):
- macOS: `~/Library/Application Support/berlin.sneak.app.routewatch/`
- Linux: `/var/lib/berlin.sneak.app.routewatch/` when running as root, otherwise
@@ -313,28 +454,37 @@ State directory (database location):
## Memory
-The daemon holds a live routing table, so its memory grows with the size of the
-data it tracks. The image sets ceilings that keep it inside a 5 GiB container.
+The daemon keeps the whole routing table in memory. The image sets ceilings that
+keep it inside a 5 GiB container.
Budget:
-- Go heap: a 1.5 GiB soft limit (`GOMEMLIMIT=1536MiB`, set in the image).
-- SQLite: 1 GiB of page cache for the one connection every write uses, which
- keeps the pages the route writes touch from one batch to the next, and 16 MiB
- for each of the 9 reading connections, whose cache SQLite empties whenever
- another connection has written: about 1.15 GiB in all. A 1.5 GiB soft heap
- limit sits above the caches, so it does not shrink them, and a 2 GiB hard heap
- limit caps the C library.
-- glibc allocator: the SQLite C library runs on glibc `malloc`, which frees
- page-cache chunks back to per-arena free lists rather than to the kernel, so
- process RSS tracks the high-water mark of those arenas, not SQLite's live
- heap. glibc creates up to eight arenas per core, so on a many-core host the
- retained memory — and thus RSS — grows with the core count. The image sets
- `MALLOC_ARENA_MAX=2` to bound it; the two-arena cap costs nothing here because
- database writes are already serialized.
+- Go heap: a 2.5 GiB soft limit (`GOMEMLIMIT=2560MiB`, set in the image).
+ - The routing table takes about 700 bytes per prefix and origin AS. A full
+ table has about 1.4 million of these pairs, so it takes about 1 GB.
+ - The rest of the heap takes up to about 0.5 GiB. Most of it is the
+ handlers' batches and queues, described below.
+ - The remaining 1 GiB is room for the garbage collector. With much less, it
+ would run almost all the time.
+- SQLite: 656 MiB of page cache.
+ - The connection that does every write has 512 MiB. It keeps the pages the
+ writes touch from one write to the next. With a full routing table, the
+ `routes` table takes about 450 MB, so its pages fit.
+ - Each of the 9 reading connections has 16 MiB. SQLite empties a reading
+ connection's cache whenever another connection has written.
+ - A 768 MiB soft heap limit sits above the caches, so it does not shrink
+ them. A 1 GiB hard heap limit caps the C library.
+- glibc allocator: the SQLite C library allocates memory with glibc `malloc`.
+ - glibc keeps freed page-cache chunks in per-arena free lists instead of
+ returning them to the kernel. So the process's RSS follows the high-water
+ mark of those arenas, not SQLite's live heap.
+ - glibc creates up to eight arenas per core, so on a many-core host RSS
+ grows with the core count.
+ - The image sets `MALLOC_ARENA_MAX=2` to bound it. Two arenas cost nothing
+ here, because database writes already run one at a time.
- About 0.2 GiB for everything else in the runtime.
-The ceilings add up to 3.7 GiB: the Go heap's 1.5 GiB, SQLite's 2 GiB and the
+The ceilings add up to 3.7 GiB: the Go heap's 2.5 GiB, SQLite's 1 GiB and the
0.2 GiB for the rest.
Run the container with a memory limit of 5 GiB and swap disabled:
@@ -346,38 +496,36 @@ docker run --memory=5g --memory-swap=5g ...
or the equivalent in your deployment tool. This leaves headroom above the
ceilings for spikes and the kernel page cache.
-Override the Go soft limit by setting `GOMEMLIMIT` in the environment (for
-example `-e GOMEMLIMIT=1GiB`); this replaces the image default.
-`MALLOC_ARENA_MAX` can be overridden the same way, but raising it lets RSS climb
-again on a many-core host.
+To replace the image's Go soft limit, set `GOMEMLIMIT` in the environment, for
+example `-e GOMEMLIMIT=3GiB`. A limit close to the routing table's size makes
+the garbage collector run almost all the time. `MALLOC_ARENA_MAX` can be
+overridden the same way, but raising it lets RSS climb again on a many-core
+host.
What happens at each limit:
- Go soft limit: as the heap approaches `GOMEMLIMIT`, the runtime runs garbage
collection more aggressively rather than growing further.
-- SQLite: at the 1.5 GiB soft heap limit it recycles its page cache instead of
- allocating more; at the 2 GiB hard heap limit a statement fails with an
- out-of-memory error, and the handler logs it and drops that batch. The process
- keeps running.
+- SQLite: at the 768 MiB soft heap limit it recycles its page cache instead of
+ allocating more. At the 1 GiB hard heap limit a statement fails with an
+ out-of-memory error, and the handler logs it. The prefix handler writes the
+ routing table's changes again on a later write. The other writes drop their
+ batch. The process keeps running.
- Handler queues: each of the four handler queues holds at most 20,000 messages.
When a queue fills, the streamer drops messages instead of blocking, except a
`STATE` message, which it holds, in feed order, until there is room: a few
hundred bytes each, about ten a second.
- Handler batches: while the AS, peer or prefix handler writes a batch to the
- database, the updates that arrive collect in its next batch, which stops at
- ten times the batch size: 300,000 updates for the AS handler
- (`asnBatchMaxSize`, about 10 MiB), 100,000 for the peer handler
- (`peerBatchMaxSize`, about 10 MiB) and 250,000 for the prefix handler
- (`prefixBatchMaxSize`, about 60 MiB). At that size the handler takes nothing
- more off its queue until the write ends, so the queue fills and the streamer
- drops messages as above. When the delete of a peer's live routes after its
- session went down fails, the prefix handler keeps the updates the peer sent
- after the down for the next write, which retries the delete first; on top of
- its 250,000 it keeps at most the last 25,000 of them (`prefixBatchSize`, about
- 6 MiB) and drops and logs the older ones, so while the delete keeps failing
- the other peers' updates still come off the queue. With the batches being
- written, the three handlers' batches take at most about 0.25 GiB of the Go
- heap's 1.5 GiB.
+ database, the updates that arrive collect in its next batch. That batch stops
+ at ten times the batch size:
+ - 300,000 updates for the AS handler (`asnBatchMaxSize`, about 10 MiB);
+ - 100,000 for the peer handler (`peerBatchMaxSize`, about 10 MiB);
+ - 250,000 for the prefix handler (`prefixBatchMaxSize`, about 15 MiB).
+
+ At that size the handler takes nothing more off its queue until the write
+ ends. The queue then fills, and the streamer drops messages as above. With
+ the batches being written, the three handlers' batches take at most about
+ 0.1 GiB of the Go heap.
With `DEBUG=routewatch` the daemon logs a `System stats` line every 60 seconds
with the goroutine count and Go memory figures.
@@ -399,25 +547,25 @@ What the [upaas](https://git.eeqj.de/sneak/upaas) app needs:
- Health check: the image's `HEALTHCHECK` requests
`/.well-known/healthcheck.json` on the container port. upaas reads the
container's health 60 seconds after a deploy and fails the deploy unless it is
- `healthy`.
+ `healthy`. For the first 60 seconds, Docker 25 and later runs the check every
+ 5 seconds, and the first pass makes the container healthy. Startup first reads
+ the routing table, which takes about 25 seconds when the table is full.
- Stop grace: the container needs 9 seconds between SIGTERM and SIGKILL. upaas
gives 10 (`stopTimeoutSeconds` in its `internal/docker/client.go`), as
`docker stop` does by default. The daemon allows 8 seconds for the handlers to
- write what they hold and for the requests in progress to finish, then stops
+ write what they hold and for the requests in progress to finish. Then it stops
the HTTP server, closes the database and exits with status 0. A longer grace
changes nothing. What is not written after 8 seconds is lost, and the streamer
- and each handler log how much. When the prefix handler has fallen behind the
- feed, a stop loses far more: its batch is full at 250,000 updates and the
- streamer's worker for it waits for room, so the streamer's stop takes the
- whole 8 seconds and every handler then skips its last write. Lost then are the
- messages left on the four handlers' queues, what the AS, peer and peering
- handlers hold, the prefix handler's batch, and the write it has in progress,
- up to another 250,000 updates, which take about 20 seconds and are cut off by
- the exit; withdrawals and peer downs are among them. A route withdrawn in them
- stays in the live route table until its peer announces or withdraws the prefix
- again or its session goes down. When the 8 seconds run out the database is not
- closed, and SQLite rolls back a write the exit cut off when the database next
- opens.
+ and each handler log how much.
+ - The prefix handler's last write takes at most 20,000 of the routing
+ table's changed pairs. A full-table peer's session going down changes
+ about a million pairs, which take about a minute to write. A stop in that
+ minute leaves the rest unwritten.
+ - A lost withdrawal or peer down leaves its routes in the `routes` table,
+ and the next start reads them back. They stay until the peer sends another
+ update for the prefix, or its session goes down.
+ - When the 8 seconds run out the database is not closed. When the database
+ next opens, SQLite rolls back any write the exit cut off.
## Development
@@ -465,6 +613,11 @@ provide:
or `go.sum`) and `go fmt` (writes), then `script/check`
- `script/install-precommit` — install the git pre-commit hook that runs
`script/precommit`
+- `script/asupdate` — rebuild the AS list built into RouteWatch,
+ `pkg/asinfo/asdata.json.gz`, from the `as.csv` file of
+ [ipverse/as-metadata](https://github.com/ipverse/as-metadata). It needs the
+ network. It runs `cmd/asinfo-gen`, which fails when it finds no AS. When
+ `cmd/asinfo-gen` fails, the old list stays and the script exits non-zero.
Tests and the linter run only in Docker, as the `test` and `lint` phases of the
`Dockerfile`; the image cannot be built unless both pass. Every `docker build`
diff --git a/TODO.md b/TODO.md
index a4f5e9a..c095961 100644
--- a/TODO.md
+++ b/TODO.md
@@ -25,14 +25,121 @@ are measured by a benchmark, not yet on the live feed.
# Next Step
-Run `next` on the live feed, as README "Running under upaas" describes, for the
-final check https://git.eeqj.de/sneak/routewatch/issues/96 names: whether the
-prefix handler still drops updates after the first hour. If it does, a new issue
+Run `next` on the live feed, as README "Running under upaas" describes, until
+the routing table stops growing. Post its size and memory figures on
+https://git.eeqj.de/sneak/routewatch/issues/122. The same run makes the final
+check https://git.eeqj.de/sneak/routewatch/issues/96 names: whether the prefix
+handler still drops updates after the first hour. If it does, a new issue
follows with the numbers. Then an audit of the code at the commit that run
tested.
# Completed Steps
+- 2026-10-27: the per-peer route tables are gone, with the code that wrote,
+ counted and read them. The prefix handler applies each update to the routing
+ table in memory and writes the table's changes every second. It also stores
+ each new prefix with its first-seen time. The image's memory limits are set
+ for a full routing table of about 1 GB: `GOMEMLIMIT` is 2.5 GiB, the writing
+ connection's page cache 512 MiB, and SQLite's heap limits 768 MiB and 1 GiB
+ (closes #122)
+- 2026-10-27: the pages, the API and the stats read every route from the routing
+ table in memory. For a prefix with several origin ASes, the IP lookup gives
+ the one most peers carry. Its `last_seen` is the newest kept route's time. The
+ prefix page and API list every peer that carries the prefix, with an AS path,
+ next hop and time only for the kept routes. The stats count the origins whose
+ kept routes were all withdrawn, in `origins_without_kept_route` (part of #122)
+- 2026-10-27: the prefix, AS and peer handlers make a timed write on every tick
+ of their timers, every 1, 2 and 2 seconds. They used to skip every other tick.
+ A full batch's write restarts the timer, so the next timed write comes one
+ interval after that write began (closes #192)
+- 2026-10-27: once the stop deadline has passed, the AS, peer and peering
+ handlers start no write either. A full batch or a tick of the timer that
+ waited while a write ran past the deadline could still start one. The flush
+ goroutines of the prefix, AS and peer handlers now run the same code (closes
+ #189)
+- 2026-10-27: once the stop deadline has passed, the prefix handler starts no
+ flush. A batch that filled while a write ran past the deadline could still be
+ written after it (closes #188)
+- 2026-10-27: the prefix handler applies each update to the routing table in
+ memory, and writes the table's changes to the database every second. A write
+ takes at most 20,000 changed pairs, and the rest follow in the next writes. A
+ failed write is written again later. Startup reads the table before the
+ streamer connects. The pages still read the per-peer route tables (part of
+ #122)
+- 2026-10-27: the database keeps a copy of the whole routing table in a new
+ `routes` table, one row per prefix and origin AS. The peers' numbers are in a
+ new `route_peers` table. One function writes the pairs that changed in memory,
+ and another reads the whole table back into memory at startup. Nothing uses
+ them yet (part of #122)
+- 2026-10-27: a new package, `internal/routetable`, holds the whole routing
+ table in memory. For each prefix and origin AS it keeps the peers that carry
+ it, one bit per peer, and the two newest of their routes. Nothing uses it yet
+ (part of #122)
+- 2026-10-27: each result of the IP lookup lists the public blocklists the
+ address is on, in `blocklists`. The lists are DShield, Feodo Tracker, two
+ FireHOL lists, Spamhaus DROP and the Tor exit list. RouteWatch downloads each
+ list on its own schedule and answers from copies in memory. The copies are
+ also kept in the state directory's `blocklists` directory. IP lookup answers
+ carry `Cache-Control`. An address with no live route now has a result too,
+ without route fields, so the IP lookup no longer answers 404 (closes #158)
+- 2026-10-27: the home page has an IP information API card. It shows the `curl`
+ command that looks up the visitor's own address at `/ip/{addr}`, and that
+ command's live answer (closes #156)
+- 2026-10-27: on the AS page, a click on the heading of the IPv4 or the IPv6
+ prefix list folds or unfolds the list. Both lists start open (closes #155)
+- 2026-10-27: the AS page lists its IPv4 and its IPv6 prefixes by mask length,
+ shortest first, and prefixes of one mask length by address. They used to be
+ listed by address (closes #153)
+- 2026-10-27: on `/` and `/status`, a spinner beside the status turns one frame
+ each time the stats show more messages than the time before. A page that has
+ not seen more messages for three refreshes shows "stale" and the time since
+ then instead, in red (closes #152)
+- 2026-10-27: the WHOIS details of a Brazilian AS keep their accented letters.
+ LACNIC's answer for such an AS comes from NIC.br, Brazil's registry, with each
+ accented letter replaced. The details now come from NIC.br's own server,
+ `whois.registro.br`. When that query fails, the details from LACNIC's answer
+ are kept and a warning is logged. After NIC.br answers that RouteWatch is over
+ its query limit, RouteWatch stops asking NIC.br for an hour (closes #174)
+- 2026-10-27: `make asupdate` stops with an error when the download of `as.csv`
+ breaks off. The generator used to retry the failed read forever. A record that
+ does not parse is still logged and skipped (closes #177)
+- 2026-10-27: `make asupdate` runs the new `script/asupdate`. It replaces the AS
+ list built into RouteWatch only when the generator succeeds and finds at least
+ one AS. A failed download used to replace the list with an empty file. The
+ generator reads `as.csv` from `ipverse/as-metadata`, the source's new name
+ (closes #173)
+- 2026-10-27: the WHOIS details of an AS at RIPE, APNIC or AFRINIC come only
+ from its `aut-num` object and the organisation object it names. RIPE and APNIC
+ start their answer with the `as-block` object of the AS's block. Its dates
+ were stored as the AS's (closes #168)
+- 2026-10-27: AS names keep their accented letters. The AS list built into
+ RouteWatch is rebuilt from today's `as.csv`, which added a country code
+ column. That file no longer lists AS 0 or the documentation AS numbers. A
+ WHOIS answer that is not valid UTF-8 is decoded as Latin-1, which RIPE and
+ APNIC send, before it is parsed and stored (closes #165)
+- 2026-10-27: the Age column on the AS and prefix pages reads in words, such as
+ "5 minutes ago" or "2 weeks ago", instead of Go durations such as "360h0m0s
+ ago". An age under a minute reads in seconds, and one over a month in months
+ or years instead of a date (closes #163)
+- 2026-10-26: a write to a stored AS, peering or peer keeps the earlier of the
+ two first-seen times and the later of the two last-seen times. An old update
+ from a collector catching up on a backlog no longer moves last seen back
+ (closes #162)
+- 2026-10-26: the status page shows "Route Churn/min", the number of prefix
+ announcements and withdrawals, from all peers, in the last full clock minute.
+ `/api/v1/stats` reports it as `route_churn_per_min` in `stream`, which
+ replaces the moving average `route_churn_per_sec`. The messages and updates
+ per second on `/` and `/status` are whole numbers (closes #160)
+- 2026-10-26: the WHOIS fetcher backs off more slowly. A failure makes its wait
+ 1.5 times as long, up to one minute instead of five. Its queries to RIPE,
+ APNIC and AFRINIC leave out person and role objects, which RIPE counts against
+ a daily limit per address. A refused query is logged with the RIR's error line
+ (closes #159)
+- 2026-10-26: the home page's AS name search lists the ASes whose handle or
+ description holds the text, as links to their pages, from the new
+ `/api/v1/search/as`; the search ignores case. When the database has no record
+ of an AS, its page takes the handle and description from the AS list built
+ into RouteWatch and still lists the AS's prefixes (closes #151)
- 2026-10-19: README "HTTP Endpoints" lists every route with what it answers,
"Database Schema" matches `schema.sql`, and the rest of the README matches the
code; this file brought up to date (closes #72)
@@ -179,4 +286,4 @@ tested.
# Future Steps
-None open.
+None.
diff --git a/cmd/asinfo-gen/main.go b/cmd/asinfo-gen/main.go
index fbc0a50..346c74d 100644
--- a/cmd/asinfo-gen/main.go
+++ b/cmd/asinfo-gen/main.go
@@ -6,6 +6,7 @@ import (
"encoding/csv"
"encoding/json"
"errors"
+ "fmt"
"io"
"log"
"net/http"
@@ -14,7 +15,21 @@ import (
"strings"
)
-const asCsvURL = "https://github.com/ipverse/asn-info/raw/refs/heads/master/as.csv"
+const asCsvURL = "https://github.com/ipverse/as-metadata/raw/refs/heads/master/as.csv"
+
+// The first columns of as.csv, the ones the generator reads. It reads past
+// the columns after them, such as the country code.
+const (
+ asnColumn = "asn"
+ handleColumn = "handle"
+ descriptionColumn = "description"
+ columnsRead = 3
+)
+
+var (
+ errUnexpectedHeader = errors.New("unexpected CSV header")
+ errNoASes = errors.New("no AS in the CSV")
+)
// ASInfo represents information about an Autonomous System
type ASInfo struct {
@@ -45,40 +60,66 @@ func main() {
log.Fatalf("Failed to fetch CSV: HTTP %d", resp.StatusCode)
}
- // Parse CSV
- reader := csv.NewReader(resp.Body)
+ asInfos, err := parseCSV(resp.Body)
+
+ // Closed here rather than deferred: log.Fatalf exits without running
+ // deferred calls.
+ _ = resp.Body.Close()
+
+ if err != nil {
+ log.Fatalf("Failed to parse CSV: %v", err)
+ }
+
+ log.Printf("Parsed %d AS records", len(asInfos))
+
+ // Convert to JSON and output to stdout
+ encoder := json.NewEncoder(os.Stdout)
+ encoder.SetIndent("", "\t")
+
+ err = encoder.Encode(asInfos)
+ if err != nil {
+ log.Fatalf("Failed to encode JSON: %v", err)
+ }
+}
+
+// parseCSV reads as.csv from r and returns its ASes, with the handle and the
+// description as the file spells them. A header whose first columns are not
+// the AS number, the handle and the description is an error. A record that
+// does not parse is logged and skipped. A read that fails, such as on a broken
+// download, is an error. A file left with no AS is an error.
+func parseCSV(r io.Reader) ([]ASInfo, error) {
+ reader := csv.NewReader(r)
- // Read header
header, err := reader.Read()
if err != nil {
- log.Fatalf("Failed to read CSV header: %v", err)
+ return nil, fmt.Errorf("read CSV header: %w", err)
}
- if len(header) != 3 || header[0] != "asn" || header[1] != "handle" ||
- header[2] != "description" {
- log.Fatalf("Unexpected CSV header: %v", header)
+ if len(header) < columnsRead || header[0] != asnColumn ||
+ header[1] != handleColumn || header[2] != descriptionColumn {
+ return nil, fmt.Errorf("%w: %v", errUnexpectedHeader, header)
}
- // Read all records
var asInfos []ASInfo
for {
+ // The reader returns an error for a record with a different number
+ // of fields than the header.
record, err := reader.Read()
if errors.Is(err, io.EOF) {
break
}
- if err != nil {
+ if _, ok := errors.AsType[*csv.ParseError](err); ok {
log.Printf("Error reading CSV record: %v", err)
continue
}
- const expectedFields = 3
- if len(record) != expectedFields {
- log.Printf("Skipping invalid record: %v", record)
-
- continue
+ // Any other error comes from the download, such as a broken
+ // connection. Every later read returns it again.
+ if err != nil {
+ return nil, fmt.Errorf("read CSV record: %w", err)
}
asn, err := strconv.Atoi(record[0])
@@ -95,18 +136,9 @@ func main() {
})
}
- // Closed here rather than deferred: log.Fatalf exits without running
- // deferred calls.
- _ = resp.Body.Close()
-
- log.Printf("Parsed %d AS records", len(asInfos))
-
- // Convert to JSON and output to stdout
- encoder := json.NewEncoder(os.Stdout)
- encoder.SetIndent("", "\t")
-
- err = encoder.Encode(asInfos)
- if err != nil {
- log.Fatalf("Failed to encode JSON: %v", err)
+ if len(asInfos) == 0 {
+ return nil, errNoASes
}
+
+ return asInfos, nil
}
diff --git a/cmd/asinfo-gen/main_test.go b/cmd/asinfo-gen/main_test.go
new file mode 100644
index 0000000..f629989
--- /dev/null
+++ b/cmd/asinfo-gen/main_test.go
@@ -0,0 +1,142 @@
+package main
+
+import (
+ "errors"
+ "io"
+ "slices"
+ "strings"
+ "testing"
+ "testing/iotest"
+ "time"
+)
+
+// parseTimeLimit is how long a test waits for parseCSV to return before it
+// fails.
+const parseTimeLimit = 2 * time.Second
+
+// TestParseCSVReadsTodaysLayout parses an as.csv with the country code column
+// that the source added after the description. It expects each AS with its
+// handle and description spelled exactly as in the file, accents included.
+func TestParseCSVReadsTodaysLayout(t *testing.T) {
+ t.Parallel()
+
+ const asCSV = `asn,handle,description,country-code
+1,LVLT-1,Level 3 Parent LLC,US
+1251,FUNDACAO-AMPARO-PESQUISA,FUNDAÇÃO DE AMPARO À PESQUISA DO ESTADO SÃO PAULO,BR
+`
+
+ got, err := parseCSV(strings.NewReader(asCSV))
+ if err != nil {
+ t.Fatalf("parseCSV: %v", err)
+ }
+
+ want := []ASInfo{
+ {ASN: 1, Handle: "LVLT-1", Description: "Level 3 Parent LLC"},
+ {
+ ASN: 1251,
+ Handle: "FUNDACAO-AMPARO-PESQUISA",
+ Description: "FUNDAÇÃO DE AMPARO À PESQUISA DO ESTADO " +
+ "SÃO PAULO",
+ },
+ }
+
+ if !slices.Equal(got, want) {
+ t.Errorf("parseCSV = %+v, want %+v", got, want)
+ }
+}
+
+// TestParseCSVRejectsAnotherHeader parses files whose first columns are not
+// the AS number, the handle and the description. It expects an error rather
+// than a list read from the wrong columns.
+func TestParseCSVRejectsAnotherHeader(t *testing.T) {
+ t.Parallel()
+
+ for _, asCSV := range []string{
+ "asn,handle\n1,LVLT-1\n",
+ "asn,description,handle\n1,Level 3 Parent LLC,LVLT-1\n",
+ } {
+ _, err := parseCSV(strings.NewReader(asCSV))
+ if err == nil {
+ t.Errorf("parseCSV(%q): no error", asCSV)
+ }
+ }
+}
+
+// TestParseCSVRejectsAFileWithNoASes parses files that hold no AS, one with
+// only the header and one whose only record has no valid AS number. It
+// expects errNoASes, so that make asupdate keeps the old list.
+func TestParseCSVRejectsAFileWithNoASes(t *testing.T) {
+ t.Parallel()
+
+ for _, asCSV := range []string{
+ "asn,handle,description,country-code\n",
+ "asn,handle,description,country-code\nAS1,LVLT-1,Level 3,US\n",
+ } {
+ _, err := parseCSV(strings.NewReader(asCSV))
+ if !errors.Is(err, errNoASes) {
+ t.Errorf("parseCSV(%q) = %v, want %v", asCSV, err, errNoASes)
+ }
+ }
+}
+
+// TestParseCSVSkipsARecordThatDoesNotParse parses an as.csv with a record
+// that has too few fields between two valid records. It expects both valid
+// ASes and no error, so that one bad record does not stop make asupdate.
+func TestParseCSVSkipsARecordThatDoesNotParse(t *testing.T) {
+ t.Parallel()
+
+ const asCSV = "asn,handle,description,country-code\n" +
+ "1,LVLT-1,Level 3 Parent LLC,US\n" +
+ "2,UDEL-DCN\n" +
+ "3,MIT-GATEWAYS,Massachusetts Institute of Technology,US\n"
+
+ got, err := parseCSV(strings.NewReader(asCSV))
+ if err != nil {
+ t.Fatalf("parseCSV: %v", err)
+ }
+
+ want := []ASInfo{
+ {ASN: 1, Handle: "LVLT-1", Description: "Level 3 Parent LLC"},
+ {
+ ASN: 3,
+ Handle: "MIT-GATEWAYS",
+ Description: "Massachusetts Institute of Technology",
+ },
+ }
+
+ if !slices.Equal(got, want) {
+ t.Errorf("parseCSV = %+v, want %+v", got, want)
+ }
+}
+
+// TestParseCSVStopsWhenTheDownloadBreaksOff parses an as.csv whose download
+// breaks off after the first record. From then on every read fails, as on a
+// broken connection. It expects parseCSV to return that error, so that make
+// asupdate keeps the old list.
+func TestParseCSVStopsWhenTheDownloadBreaksOff(t *testing.T) {
+ t.Parallel()
+
+ const asCSV = "asn,handle,description,country-code\n" +
+ "1,LVLT-1,Level 3 Parent LLC,US\n"
+
+ download := io.MultiReader(
+ strings.NewReader(asCSV), iotest.ErrReader(io.ErrUnexpectedEOF))
+
+ // parseCSV runs on its own goroutine, so that a parseCSV that never
+ // returns fails the test instead of hanging it.
+ parsed := make(chan error, 1)
+
+ go func() {
+ _, err := parseCSV(download)
+ parsed <- err
+ }()
+
+ select {
+ case err := <-parsed:
+ if !errors.Is(err, io.ErrUnexpectedEOF) {
+ t.Errorf("parseCSV = %v, want %v", err, io.ErrUnexpectedEOF)
+ }
+ case <-time.After(parseTimeLimit):
+ t.Fatalf("parseCSV did not return within %v", parseTimeLimit)
+ }
+}
diff --git a/go.mod b/go.mod
index 10d57f5..b10f623 100644
--- a/go.mod
+++ b/go.mod
@@ -3,13 +3,17 @@ module git.eeqj.de/sneak/routewatch
go 1.27.1
require (
+ github.com/bits-and-blooms/bitset v1.26.0
github.com/dustin/go-humanize v1.0.1
github.com/go-chi/chi/v5 v5.2.2
+ github.com/google/btree v1.1.3
github.com/google/uuid v1.6.0
github.com/mattn/go-sqlite3 v1.14.29
github.com/rcrowley/go-metrics v0.0.0-20250401214520-65e299d6c5c9
go.uber.org/fx v1.24.0
+ go4.org/netipx v0.0.0-20260823151212-3075585bcbeb
golang.org/x/term v0.33.0
+ golang.org/x/text v0.43.0
)
require (
diff --git a/go.sum b/go.sum
index 4e183bc..244a36b 100644
--- a/go.sum
+++ b/go.sum
@@ -1,9 +1,13 @@
+github.com/bits-and-blooms/bitset v1.26.0 h1:WPvcPZbP1yuKofmjs2SYIFO1EjzIGhCP9ducElLsh5Q=
+github.com/bits-and-blooms/bitset v1.26.0/go.mod h1:7hO7Gc7Pp1vODcmWvKMRA9BNmbv6a/7QIWpPxHddWR8=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
github.com/go-chi/chi/v5 v5.2.2 h1:CMwsvRVTbXVytCk1Wd72Zy1LAsAh9GxMmSNWLHCG618=
github.com/go-chi/chi/v5 v5.2.2/go.mod h1:L2yAIGWB3H+phAw1NxKwWM+7eUH/lU8pOMm5hHcoops=
+github.com/google/btree v1.1.3 h1:CVpQJjYgC4VbzxeGVHfvZrv1ctoYCAI8vbl07Fcxlyg=
+github.com/google/btree v1.1.3/go.mod h1:qOPhT0dTNdNzV6Z/lhRX0YXUafgPLFUh+gZMl761Gm4=
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/mattn/go-sqlite3 v1.14.29 h1:1O6nRLJKvsi1H2Sj0Hzdfojwt8GiGKm+LOfLaBFaouQ=
@@ -24,9 +28,13 @@ go.uber.org/multierr v1.10.0 h1:S0h4aNzvfcFsC3dRF1jLoaov7oRaKqRGC/pUEJ2yvPQ=
go.uber.org/multierr v1.10.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y=
go.uber.org/zap v1.26.0 h1:sI7k6L95XOKS281NhVKOFCUNIvv9e0w4BF8N3u+tCRo=
go.uber.org/zap v1.26.0/go.mod h1:dtElttAiwGvoJ/vj4IwHBS/gXsEu/pZ50mUIRWuG0so=
+go4.org/netipx v0.0.0-20260823151212-3075585bcbeb h1:XBM4hvfwGAttkkiTIFfeigdfcL1xIfdKXqFdgiHGtDs=
+go4.org/netipx v0.0.0-20260823151212-3075585bcbeb/go.mod h1:PLyyIXexvUFg3Owu6p/WfdlivPbZJsZdgWZlrGope/Y=
golang.org/x/sys v0.34.0 h1:H5Y5sJ2L2JRdyv7ROF1he/lPdvFsd0mJHFw2ThKHxLA=
golang.org/x/sys v0.34.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k=
golang.org/x/term v0.33.0 h1:NuFncQrRcaRvVmgRkvM3j/F00gWIAlcmlB8ACEKmGIg=
golang.org/x/term v0.33.0/go.mod h1:s18+ql9tYWp1IfpV9DmCtQDDSRBUjKaw9M1eAv5UeF0=
+golang.org/x/text v0.43.0 h1:1QivrlhwAsnMOcqgOdl9my7s9OyZl8O5/UGUTJVXoWc=
+golang.org/x/text v0.43.0/go.mod h1:hbSIYA/amXcRXTFSZqh8tWHEoqRvvbKPJ2uz/KU+a+s=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
diff --git a/internal/blocklist/blocklist.go b/internal/blocklist/blocklist.go
new file mode 100644
index 0000000..9e16898
--- /dev/null
+++ b/internal/blocklist/blocklist.go
@@ -0,0 +1,184 @@
+// Package blocklist downloads public IP blocklists on a schedule, keeps the
+// last good copy of each on disk and in memory, and answers which lists
+// contain an address.
+package blocklist
+
+import (
+ "context"
+ "log/slog"
+ "net/netip"
+ "sync"
+ "time"
+
+ "go4.org/netipx"
+)
+
+// Format is how a list's file is written.
+type Format int
+
+const (
+ // FormatLines has one address or network per line, such as 192.0.2.1 or
+ // 192.0.2.0/24. Lines that start with # are comments.
+ FormatLines Format = iota
+
+ // FormatSpamhausJSON has one JSON object per line, with a network in its
+ // cidr field. The last line holds the file's metadata and no network.
+ FormatSpamhausJSON
+
+ // FormatDShield has one network per line, as tab-separated fields that
+ // start with the network's first and last address. Lines that start
+ // with # are comments.
+ FormatDShield
+)
+
+// List is one public blocklist.
+type List struct {
+ // Name names the list in lookup answers, and names its copy on disk.
+ Name string
+
+ // Description says in one line what being on the list means.
+ Description string
+
+ // URL is where the publisher serves the whole list.
+ URL string
+
+ // Interval is how long a copy is used before the list is downloaded
+ // again.
+ Interval time.Duration
+
+ // Format is how the list's file is written.
+ Format Format
+}
+
+// Listing says that a list contains an address.
+type Listing struct {
+ Name string `json:"name"`
+ Description string `json:"description"`
+
+ // Updated is when routewatch last downloaded the list, or last heard from
+ // its publisher that its copy was still current.
+ Updated time.Time `json:"updated"`
+}
+
+// listCopy is a list's last good copy in memory.
+type listCopy struct {
+ addresses *netipx.IPSet
+ updated time.Time
+
+ // validators identify the version of the list this copy holds.
+ validators validators
+}
+
+// Params is what New needs.
+type Params struct {
+ // Dir is the directory that holds the copies on disk.
+ Dir string
+
+ // Lists are the lists to download and look addresses up in.
+ Lists []List
+
+ Logger *slog.Logger
+}
+
+// Blocklists keeps a copy of each list and answers which lists contain an
+// address. Start loads the copies from disk and starts downloading the lists.
+type Blocklists struct {
+ dir string
+ lists []List
+ logger *slog.Logger
+
+ // copies holds each list's last good copy, by list name. A list with no
+ // good copy yet has none.
+ mu sync.RWMutex
+ copies map[string]listCopy
+
+ // stop ends the goroutine Start runs, and done is closed when it returns.
+ stop context.CancelFunc
+ done chan struct{}
+}
+
+// New returns a Blocklists with no copies. Start loads them.
+func New(params Params) *Blocklists {
+ return &Blocklists{
+ dir: params.Dir,
+ lists: params.Lists,
+ logger: params.Logger.With("component", "blocklists"),
+ copies: make(map[string]listCopy),
+ done: make(chan struct{}),
+ }
+}
+
+// Lookup returns a Listing for each list whose copy contains addr, in the
+// order of the lists given to New. It makes no outside request. When no list
+// contains addr, it returns an empty list, not nil.
+func (b *Blocklists) Lookup(addr netip.Addr) []Listing {
+ b.mu.RLock()
+ defer b.mu.RUnlock()
+
+ listings := []Listing{}
+
+ for _, list := range b.lists {
+ c, ok := b.copies[list.Name]
+ if ok && c.addresses.Contains(addr.Unmap()) {
+ listings = append(listings, Listing{
+ Name: list.Name,
+ Description: list.Description,
+ Updated: c.updated,
+ })
+ }
+ }
+
+ return listings
+}
+
+// ShortestInterval returns the shortest Interval of the lists. That is the
+// soonest a copy can change, and so how long a lookup answer stays current.
+func (b *Blocklists) ShortestInterval() time.Duration {
+ var shortest time.Duration
+
+ for i, list := range b.lists {
+ if i == 0 || list.Interval < shortest {
+ shortest = list.Interval
+ }
+ }
+
+ return shortest
+}
+
+// updated returns when the named list's copy was last downloaded or found
+// current, or the zero time when the list has no copy.
+func (b *Blocklists) updated(name string) time.Time {
+ b.mu.RLock()
+ defer b.mu.RUnlock()
+
+ return b.copies[name].updated
+}
+
+// copyValidators returns the validators of the named list's copy. They are
+// empty when the list has no copy.
+func (b *Blocklists) copyValidators(name string) validators {
+ b.mu.RLock()
+ defer b.mu.RUnlock()
+
+ return b.copies[name].validators
+}
+
+// setCopy makes c the named list's copy.
+func (b *Blocklists) setCopy(name string, c listCopy) {
+ b.mu.Lock()
+ defer b.mu.Unlock()
+
+ b.copies[name] = c
+}
+
+// setUpdated records that the named list's copy was found current at t.
+func (b *Blocklists) setUpdated(name string, t time.Time) {
+ b.mu.Lock()
+ defer b.mu.Unlock()
+
+ c, ok := b.copies[name]
+ if ok {
+ c.updated = t
+ b.copies[name] = c
+ }
+}
diff --git a/internal/blocklist/blocklist_internal_test.go b/internal/blocklist/blocklist_internal_test.go
new file mode 100644
index 0000000..0ce9770
--- /dev/null
+++ b/internal/blocklist/blocklist_internal_test.go
@@ -0,0 +1,494 @@
+package blocklist
+
+import (
+ "context"
+ "errors"
+ "io"
+ "io/fs"
+ "log/slog"
+ "net/http"
+ "net/http/httptest"
+ "net/netip"
+ "os"
+ "path/filepath"
+ "strings"
+ "sync"
+ "testing"
+ "time"
+)
+
+// The test list's name and description, the address its good version lists,
+// and the address a newer version lists instead.
+const (
+ testListName = "test_list"
+ testListDescription = "On the test list"
+ listedAddr = "192.0.2.1"
+ goodListBody = "# A test list\n" + listedAddr + "\n"
+ newListedAddr = "192.0.2.2"
+ newListBody = "# A test list\n" + newListedAddr + "\n"
+)
+
+// testETag is the ETag a publisher sends with a version of the test list.
+const testETag = `"v1"`
+
+// maxWait is how long a test waits for the download goroutine.
+const maxWait = 10 * time.Second
+
+// publisher stands in for a list's publisher. It answers every request with
+// its status and body, and keeps the headers of each request. A 200 answer
+// carries lastModified and etag, when set, as Last-Modified and ETag. As
+// from a real publisher, a request whose If-None-Match or If-Modified-Since
+// matches them is answered "not modified".
+type publisher struct {
+ mu sync.Mutex
+ status int
+ body string
+ lastModified time.Time
+ etag string
+ headers []http.Header
+}
+
+// newPublisher starts a publisher that answers 200 with body, and returns it
+// and its URL.
+func newPublisher(t *testing.T, body string) (*publisher, string) {
+ t.Helper()
+
+ p := &publisher{status: http.StatusOK, body: body}
+ srv := httptest.NewServer(p)
+ t.Cleanup(srv.Close)
+
+ return p, srv.URL
+}
+
+func (p *publisher) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+ p.mu.Lock()
+ defer p.mu.Unlock()
+
+ p.headers = append(p.headers, r.Header.Clone())
+
+ if p.status != http.StatusOK {
+ w.WriteHeader(p.status)
+ _, _ = io.WriteString(w, p.body)
+
+ return
+ }
+
+ if p.etag != "" {
+ w.Header().Set(etagHeader, p.etag)
+ }
+
+ // ServeContent sends Last-Modified unless lastModified is zero, and
+ // answers "not modified" when the request's conditions match.
+ http.ServeContent(w, r, testListName, p.lastModified,
+ strings.NewReader(p.body))
+}
+
+// answer makes the publisher answer later requests with status and body.
+func (p *publisher) answer(status int, body string) {
+ p.mu.Lock()
+ defer p.mu.Unlock()
+
+ p.status, p.body = status, body
+}
+
+// publish makes the publisher answer later requests with a version of the
+// list: body, with lastModified and etag. A zero lastModified or an empty
+// etag is not sent.
+func (p *publisher) publish(body string, lastModified time.Time, etag string) {
+ p.mu.Lock()
+ defer p.mu.Unlock()
+
+ p.status, p.body = http.StatusOK, body
+ p.lastModified, p.etag = lastModified, etag
+}
+
+// requests returns the headers of the requests the publisher has had.
+func (p *publisher) requests() []http.Header {
+ p.mu.Lock()
+ defer p.mu.Unlock()
+
+ return append([]http.Header(nil), p.headers...)
+}
+
+// testList returns a list served at url, written in format.
+func testList(url string, format Format) List {
+ return List{
+ Name: testListName,
+ Description: testListDescription,
+ URL: url,
+ Interval: time.Hour,
+ Format: format,
+ }
+}
+
+func newTestBlocklists(dir string, list List) *Blocklists {
+ return New(Params{
+ Dir: dir,
+ Lists: []List{list},
+ Logger: slog.New(slog.DiscardHandler),
+ })
+}
+
+// lookup looks addr up in b.
+func lookup(t *testing.T, b *Blocklists, addr string) []Listing {
+ t.Helper()
+
+ return b.Lookup(netip.MustParseAddr(addr))
+}
+
+// TestLookupFindsListedAddresses downloads a list in each format and looks
+// up addresses on it, inside its networks, and off it, in IPv4 and IPv6.
+// A listed address has one listing that names the list; any other has an
+// empty list of listings.
+func TestLookupFindsListedAddresses(t *testing.T) {
+ t.Parallel()
+
+ tests := []struct {
+ name string
+ format Format
+ body string
+ listed []string
+ unlisted []string
+ }{
+ {
+ name: "single addresses",
+ format: FormatLines,
+ body: "# comment\n192.0.2.1\n\n2001:db8::1\n",
+ listed: []string{"192.0.2.1", "::ffff:192.0.2.1", "2001:db8::1"},
+ unlisted: []string{"192.0.2.2", "2001:db8::2"},
+ },
+ {
+ name: "networks",
+ format: FormatLines,
+ body: "# comment\n198.51.100.0/24\n2001:db8:1::/48\n",
+ listed: []string{"198.51.100.0", "198.51.100.255", "2001:db8:1::5"},
+ unlisted: []string{"198.51.101.0", "2001:db8:2::1"},
+ },
+ {
+ name: "Spamhaus networks",
+ format: FormatSpamhausJSON,
+ body: `{"cidr":"203.0.113.0/25","sblid":"SBL1","rir":"arin"}` + "\n" +
+ `{"cidr":"2001:db8:3::/48","sblid":"SBL2","rir":"ripencc"}` + "\n" +
+ `{"type":"metadata","timestamp":1793074442,"records":2}` + "\n",
+ listed: []string{"203.0.113.1", "2001:db8:3::1"},
+ unlisted: []string{"203.0.113.200", "2001:db8:4::1"},
+ },
+ {
+ name: "DShield networks",
+ format: FormatDShield,
+ body: "# DShield.org Recommended Block List\n" +
+ "192.0.2.0\t192.0.2.255\t24\t327\tEXAMPLE\tUS\tNone\n",
+ listed: []string{"192.0.2.0", "192.0.2.77"},
+ unlisted: []string{"192.0.3.1", "2001:db8::1"},
+ },
+ }
+
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ t.Parallel()
+
+ _, url := newPublisher(t, tt.body)
+ b := newTestBlocklists(t.TempDir(), testList(url, tt.format))
+
+ err := b.download(t.Context(), b.lists[0])
+ if err != nil {
+ t.Fatalf("download: %v", err)
+ }
+
+ for _, addr := range tt.listed {
+ listings := lookup(t, b, addr)
+ if len(listings) != 1 || listings[0].Name != testListName ||
+ listings[0].Description != testListDescription ||
+ listings[0].Updated.IsZero() {
+ t.Errorf("%s: listings %+v, want one for %s", addr,
+ listings, testListName)
+ }
+ }
+
+ for _, addr := range tt.unlisted {
+ listings := lookup(t, b, addr)
+ if listings == nil || len(listings) != 0 {
+ t.Errorf("%s: listings %#v, want an empty list", addr, listings)
+ }
+ }
+ })
+ }
+}
+
+// TestFailedDownloadKeepsThePreviousCopy downloads a good version of a list,
+// then one that fails. The good copy stays in memory and on disk, and no
+// part of the failed download is left beside it.
+func TestFailedDownloadKeepsThePreviousCopy(t *testing.T) {
+ t.Parallel()
+
+ tests := []struct {
+ name string
+ status int
+ body string
+ }{
+ {"server error", http.StatusInternalServerError, "unavailable"},
+ {"not a list", http.StatusOK, "\n\n"},
+ {
+ "over the size limit", http.StatusOK,
+ strings.Repeat(listedAddr+"\n", maxDownloadSize/len(listedAddr)),
+ },
+ }
+
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ t.Parallel()
+
+ dir := t.TempDir()
+ p, url := newPublisher(t, goodListBody)
+ b := newTestBlocklists(dir, testList(url, FormatLines))
+
+ err := b.download(t.Context(), b.lists[0])
+ if err != nil {
+ t.Fatalf("first download: %v", err)
+ }
+
+ before := lookup(t, b, listedAddr)
+
+ p.answer(tt.status, tt.body)
+
+ err = b.download(t.Context(), b.lists[0])
+ if err == nil {
+ t.Fatal("second download returned no error")
+ }
+
+ after := lookup(t, b, listedAddr)
+ if len(after) != 1 || len(before) != 1 ||
+ !after[0].Updated.Equal(before[0].Updated) {
+ t.Errorf("listings %+v after the failure, want %+v", after, before)
+ }
+
+ copies := os.DirFS(dir)
+
+ onDisk, err := fs.ReadFile(copies, testListName)
+ if err != nil || string(onDisk) != goodListBody {
+ t.Errorf("copy on disk %q (%v), want %q", onDisk, err, goodListBody)
+ }
+
+ _, err = fs.Stat(copies, testListName+downloadSuffix)
+ if !errors.Is(err, fs.ErrNotExist) {
+ t.Errorf("the failed download's file is still there (%v)", err)
+ }
+ })
+ }
+}
+
+// TestRestartLoadsCopiesFromDisk downloads a list, then starts a new
+// Blocklists on the same directory, as a restart does. The new one answers
+// from the copy on disk at once, and does not download the list again while
+// the copy is younger than the list's interval.
+func TestRestartLoadsCopiesFromDisk(t *testing.T) {
+ t.Parallel()
+
+ dir := t.TempDir()
+ p, url := newPublisher(t, goodListBody)
+ list := testList(url, FormatLines)
+
+ first := newTestBlocklists(dir, list)
+
+ err := first.download(t.Context(), list)
+ if err != nil {
+ t.Fatalf("download: %v", err)
+ }
+
+ downloaded := lookup(t, first, listedAddr)[0].Updated
+
+ second := newTestBlocklists(dir, list)
+ second.Start(t.Context())
+ second.Stop(t.Context())
+
+ listings := lookup(t, second, listedAddr)
+ if len(listings) != 1 ||
+ listings[0].Updated.Sub(downloaded).Abs() > time.Second {
+ t.Errorf("after the restart, listings %+v, want one updated at %s",
+ listings, downloaded)
+ }
+
+ if got := len(p.requests()); got != 1 {
+ t.Errorf("the publisher had %d requests, want 1", got)
+ }
+}
+
+// TestStartRefreshesAnOldCopy starts on a copy older than its list's
+// interval. The list is downloaded at once, with routewatch's User-Agent and
+// the Last-Modified and ETag the publisher sent with the copy, kept on disk.
+// The publisher answers that the copy is still current, so the copy is kept
+// and marked as updated now.
+func TestStartRefreshesAnOldCopy(t *testing.T) {
+ t.Parallel()
+
+ dir := t.TempDir()
+ p, url := newPublisher(t, goodListBody)
+ list := testList(url, FormatLines)
+
+ lastModified := time.Now().Add(-3 * list.Interval).UTC().Truncate(time.Second)
+ p.publish(goodListBody, lastModified, testETag)
+
+ err := newTestBlocklists(dir, list).download(t.Context(), list)
+ if err != nil {
+ t.Fatalf("download: %v", err)
+ }
+
+ old := time.Now().Add(-2 * list.Interval).UTC().Truncate(time.Second)
+
+ err = os.Chtimes(filepath.Join(dir, testListName), old, old)
+ if err != nil {
+ t.Fatalf("Chtimes: %v", err)
+ }
+
+ b := newTestBlocklists(dir, list)
+ b.Start(t.Context())
+
+ deadline := time.Now().Add(maxWait)
+ for !lookup(t, b, listedAddr)[0].Updated.After(old) {
+ if time.Now().After(deadline) {
+ t.Fatalf("the copy was not marked updated within %s", maxWait)
+ }
+
+ time.Sleep(time.Millisecond)
+ }
+
+ b.Stop(t.Context())
+
+ requests := p.requests()
+ last := requests[len(requests)-1]
+
+ want := lastModified.Format(http.TimeFormat)
+ if got := last.Get(ifModifiedSinceHeader); got != want {
+ t.Errorf("If-Modified-Since %q, want %q", got, want)
+ }
+
+ if got := last.Get(ifNoneMatchHeader); got != testETag {
+ t.Errorf("If-None-Match %q, want %q", got, testETag)
+ }
+
+ if got := last.Get(userAgentHeader); got != userAgent {
+ t.Errorf("User-Agent %q, want %q", got, userAgent)
+ }
+}
+
+// TestNewVersionOlderThanTheLastDownloadReplacesTheCopy downloads a list.
+// The publisher then puts out a new version whose Last-Modified is earlier
+// than that download, as when a cache in front of the publisher passes the
+// version on late. The next download sends the Last-Modified of the copy,
+// not the time of the download, so it gets the new version.
+func TestNewVersionOlderThanTheLastDownloadReplacesTheCopy(t *testing.T) {
+ t.Parallel()
+
+ p, url := newPublisher(t, goodListBody)
+ list := testList(url, FormatLines)
+ b := newTestBlocklists(t.TempDir(), list)
+
+ firstVersion := time.Now().Add(-2 * list.Interval).UTC().Truncate(time.Second)
+ p.publish(goodListBody, firstVersion, "")
+
+ err := b.download(t.Context(), list)
+ if err != nil {
+ t.Fatalf("first download: %v", err)
+ }
+
+ p.publish(newListBody, firstVersion.Add(list.Interval), "")
+
+ err = b.download(t.Context(), list)
+ if err != nil {
+ t.Fatalf("second download: %v", err)
+ }
+
+ if listings := lookup(t, b, newListedAddr); len(listings) != 1 {
+ t.Errorf("%s: listings %+v, want one from the new version",
+ newListedAddr, listings)
+ }
+
+ want := firstVersion.Format(http.TimeFormat)
+ if got := p.requests()[1].Get(ifModifiedSinceHeader); got != want {
+ t.Errorf("If-Modified-Since %q, want %q", got, want)
+ }
+}
+
+// TestDownloadWithoutLastModifiedOrETagSendsNoCondition downloads a list
+// twice from a publisher that sends neither Last-Modified nor ETag. Neither
+// request carries If-Modified-Since or If-None-Match, so the second download
+// gets the whole list again.
+func TestDownloadWithoutLastModifiedOrETagSendsNoCondition(t *testing.T) {
+ t.Parallel()
+
+ p, url := newPublisher(t, goodListBody)
+ list := testList(url, FormatLines)
+ b := newTestBlocklists(t.TempDir(), list)
+
+ err := b.download(t.Context(), list)
+ if err != nil {
+ t.Fatalf("first download: %v", err)
+ }
+
+ err = b.download(t.Context(), list)
+ if err != nil {
+ t.Fatalf("second download: %v", err)
+ }
+
+ for i, request := range p.requests() {
+ if request.Get(ifModifiedSinceHeader) != "" ||
+ request.Get(ifNoneMatchHeader) != "" {
+ t.Errorf("request %d has a condition: %v", i, request)
+ }
+ }
+}
+
+// TestStopCancelsADownloadInProgress stops while the publisher has not
+// answered. Stop cancels the download and returns once the download
+// goroutine has stopped.
+func TestStopCancelsADownloadInProgress(t *testing.T) {
+ t.Parallel()
+
+ arrived := make(chan struct{})
+
+ var once sync.Once
+
+ srv := httptest.NewServer(http.HandlerFunc(
+ func(_ http.ResponseWriter, r *http.Request) {
+ once.Do(func() { close(arrived) })
+ <-r.Context().Done()
+ }))
+ t.Cleanup(srv.Close)
+
+ b := newTestBlocklists(t.TempDir(), testList(srv.URL, FormatLines))
+ b.Start(t.Context())
+
+ select {
+ case <-arrived:
+ case <-time.After(maxWait):
+ t.Fatalf("no download began within %s", maxWait)
+ }
+
+ ctx, cancel := context.WithTimeout(t.Context(), maxWait)
+ defer cancel()
+
+ b.Stop(ctx)
+
+ select {
+ case <-b.done:
+ default:
+ t.Error("Stop returned before the download goroutine stopped")
+ }
+}
+
+// TestPublicListsAreComplete checks that every public list has what a
+// download and a lookup answer need, and a unique name.
+func TestPublicListsAreComplete(t *testing.T) {
+ t.Parallel()
+
+ names := make(map[string]bool)
+
+ for _, list := range Lists() {
+ if list.Name == "" || list.Description == "" || list.URL == "" ||
+ list.Interval <= 0 || names[list.Name] {
+ t.Errorf("list %+v is incomplete or has a name used before", list)
+ }
+
+ names[list.Name] = true
+ }
+}
diff --git a/internal/blocklist/download.go b/internal/blocklist/download.go
new file mode 100644
index 0000000..dc6ac07
--- /dev/null
+++ b/internal/blocklist/download.go
@@ -0,0 +1,379 @@
+package blocklist
+
+import (
+ "context"
+ "encoding/json"
+ "errors"
+ "fmt"
+ "io"
+ "io/fs"
+ "net/http"
+ "os"
+ "slices"
+ "time"
+
+ "go4.org/netipx"
+)
+
+// Limits of one download.
+const (
+ // downloadTimeout is how long one download may take.
+ downloadTimeout = time.Minute
+
+ // maxDownloadSize is the size of the largest list file accepted, in
+ // bytes. The largest list, FireHOL abusers, is about 2.5 MiB.
+ maxDownloadSize = 16 << 20
+)
+
+// The request headers a download sets.
+const (
+ userAgentHeader = "User-Agent"
+ ifModifiedSinceHeader = "If-Modified-Since"
+ ifNoneMatchHeader = "If-None-Match"
+
+ // userAgent names routewatch to the publishers.
+ userAgent = "routewatch (+https://git.eeqj.de/sneak/routewatch)"
+)
+
+// The response headers that identify the version of a list a publisher sent.
+const (
+ lastModifiedHeader = "Last-Modified"
+ etagHeader = "ETag"
+)
+
+const (
+ // downloadSuffix ends the name of the file a download is written to,
+ // beside the list's copy, until it has parsed.
+ downloadSuffix = ".download"
+
+ // validatorsSuffix ends the name of the file beside a list's copy that
+ // keeps the copy's validators.
+ validatorsSuffix = ".validators"
+
+ // dirPermissions are those of the directory that holds the copies.
+ dirPermissions = 0o750
+
+ // validatorsPermissions are those of the file that keeps a copy's
+ // validators.
+ validatorsPermissions = 0o600
+)
+
+// validators are the Last-Modified and ETag headers a publisher sent with a
+// version of a list. A download sends them back, as If-Modified-Since and
+// If-None-Match, so that the publisher sends the list only when it has
+// changed. They are kept as the publisher wrote them.
+//
+//nolint:tagliatelle // snake_case keys, as the Go style guide asks
+type validators struct {
+ LastModified string `json:"last_modified,omitempty"`
+ ETag string `json:"etag,omitempty"`
+}
+
+var (
+ errUnexpectedStatus = errors.New("unexpected HTTP status")
+ errTooLarge = errors.New("list is over the size limit")
+)
+
+// Start loads each list's copy from disk. It then starts the goroutine that
+// downloads each list once its copy is older than its Interval, and a list
+// with no copy at once. The goroutine runs until Stop: the end of ctx does
+// not stop it.
+func (b *Blocklists) Start(ctx context.Context) {
+ err := os.MkdirAll(b.dir, dirPermissions)
+ if err != nil {
+ b.logger.Error("Failed to create the blocklists directory",
+ "dir", b.dir, "error", err)
+ }
+
+ nextDownload := make([]time.Time, len(b.lists))
+
+ for i, list := range b.lists {
+ loadErr := b.loadCopy(list)
+ if loadErr != nil && !errors.Is(loadErr, fs.ErrNotExist) {
+ b.logger.Error("Failed to load a blocklist copy",
+ "list", list.Name, "error", loadErr)
+ }
+
+ // A list with no copy has the zero time, so it is due at once.
+ nextDownload[i] = b.updated(list.Name).Add(list.Interval)
+ }
+
+ ctx, b.stop = context.WithCancel(context.WithoutCancel(ctx))
+
+ go func() {
+ b.run(ctx, nextDownload)
+ }()
+
+ b.logger.Info("Blocklist downloads started", "dir", b.dir)
+}
+
+// Stop stops the downloads, cancelling one in progress, and waits until the
+// goroutine has returned or ctx ends. The copies in memory stay, so Lookup
+// still answers.
+func (b *Blocklists) Stop(ctx context.Context) {
+ b.stop()
+
+ select {
+ case <-b.done:
+ b.logger.Info("Blocklist downloads stopped")
+ case <-ctx.Done():
+ b.logger.Warn("Stop deadline passed before the blocklist downloads " +
+ "stopped")
+ }
+}
+
+// run downloads each list when nextDownload says, until ctx ends. After each
+// try, whether it worked or not, the list's next download is an Interval
+// later.
+func (b *Blocklists) run(ctx context.Context, nextDownload []time.Time) {
+ defer close(b.done)
+
+ if len(nextDownload) == 0 {
+ return
+ }
+
+ for {
+ soonest := slices.MinFunc(nextDownload, time.Time.Compare)
+ timer := time.NewTimer(time.Until(soonest))
+
+ select {
+ case <-ctx.Done():
+ timer.Stop()
+
+ return
+ case <-timer.C:
+ }
+
+ for i, list := range b.lists {
+ if time.Now().Before(nextDownload[i]) {
+ continue
+ }
+
+ err := b.download(ctx, list)
+ if ctx.Err() != nil {
+ return
+ }
+
+ if err != nil {
+ b.logger.Error("Blocklist download failed; keeping the "+
+ "previous copy", "list", list.Name, "error", err)
+ }
+
+ nextDownload[i] = time.Now().Add(list.Interval)
+ }
+ }
+}
+
+// download asks the publisher for list, sending the validators of the list's
+// copy when it has them. A new version that parses becomes the list's copy,
+// on disk and in memory. An answer that the copy is still current marks the
+// copy as updated now. Anything else leaves the copy as it is and returns an
+// error.
+func (b *Blocklists) download(ctx context.Context, list List) error {
+ ctx, cancel := context.WithTimeout(ctx, downloadTimeout)
+ defer cancel()
+
+ req, err := http.NewRequestWithContext(ctx, http.MethodGet, list.URL, nil)
+ if err != nil {
+ return err
+ }
+
+ req.Header.Set(userAgentHeader, userAgent)
+
+ held := b.copyValidators(list.Name)
+ if held.LastModified != "" {
+ req.Header.Set(ifModifiedSinceHeader, held.LastModified)
+ }
+
+ if held.ETag != "" {
+ req.Header.Set(ifNoneMatchHeader, held.ETag)
+ }
+
+ resp, err := http.DefaultClient.Do(req)
+ if err != nil {
+ return err
+ }
+
+ defer func() { _ = resp.Body.Close() }()
+
+ switch resp.StatusCode {
+ case http.StatusOK:
+ return b.replaceCopy(list, resp.Body, validators{
+ LastModified: resp.Header.Get(lastModifiedHeader),
+ ETag: resp.Header.Get(etagHeader),
+ })
+ case http.StatusNotModified:
+ return b.markCurrent(list)
+ default:
+ return fmt.Errorf("%w: %s", errUnexpectedStatus, resp.Status)
+ }
+}
+
+// replaceCopy writes a new version of list, read from body, beside the list's
+// copy. Only when it parses does it replace the copy, on disk and in memory.
+// sent are the validators the publisher sent with the new version.
+func (b *Blocklists) replaceCopy(list List, body io.Reader, sent validators) error {
+ root, err := os.OpenRoot(b.dir)
+ if err != nil {
+ return err
+ }
+
+ defer func() { _ = root.Close() }()
+
+ downloadName := list.Name + downloadSuffix
+
+ addresses, err := writeAndParse(root, downloadName, body, list.Format)
+ if err != nil {
+ _ = root.Remove(downloadName)
+
+ return err
+ }
+
+ // The old copy's validators are removed before the copy is replaced, so
+ // that they are never kept with the new copy.
+ err = root.Remove(list.Name + validatorsSuffix)
+ if err != nil && !errors.Is(err, fs.ErrNotExist) {
+ return err
+ }
+
+ err = root.Rename(downloadName, list.Name)
+ if err != nil {
+ return err
+ }
+
+ b.setCopy(list.Name, listCopy{
+ addresses: addresses, updated: time.Now().UTC(), validators: sent,
+ })
+ b.logger.Info("Downloaded a new version of a blocklist", "list", list.Name)
+
+ // Without this file, the first download after a restart asks for the
+ // whole list.
+ err = writeValidators(root, list.Name, sent)
+ if err != nil {
+ b.logger.Error("Failed to keep the validators of a blocklist copy",
+ "list", list.Name, "error", err)
+ }
+
+ return nil
+}
+
+// writeAndParse writes body to the file name in root, up to maxDownloadSize
+// bytes, and parses the file as format.
+func writeAndParse(
+ root *os.Root, name string, body io.Reader, format Format,
+) (*netipx.IPSet, error) {
+ f, err := root.Create(name)
+ if err != nil {
+ return nil, err
+ }
+
+ defer func() { _ = f.Close() }()
+
+ written, err := io.Copy(f, io.LimitReader(body, maxDownloadSize+1))
+ if err != nil {
+ return nil, err
+ }
+
+ if written > maxDownloadSize {
+ return nil, errTooLarge
+ }
+
+ _, err = f.Seek(0, io.SeekStart)
+ if err != nil {
+ return nil, err
+ }
+
+ return parse(f, format)
+}
+
+// markCurrent records that the publisher says the list's copy is still
+// current: the copy, on disk and in memory, is marked as updated now.
+func (b *Blocklists) markCurrent(list List) error {
+ root, err := os.OpenRoot(b.dir)
+ if err != nil {
+ return err
+ }
+
+ defer func() { _ = root.Close() }()
+
+ now := time.Now().UTC()
+
+ err = root.Chtimes(list.Name, now, now)
+ if err != nil {
+ return err
+ }
+
+ b.setUpdated(list.Name, now)
+ b.logger.Debug("Blocklist copy is still current", "list", list.Name)
+
+ return nil
+}
+
+// loadCopy makes the list's copy on disk, with the validators kept beside it,
+// its copy in memory. The file's modification time is when the copy was
+// updated.
+func (b *Blocklists) loadCopy(list List) error {
+ root, err := os.OpenRoot(b.dir)
+ if err != nil {
+ return err
+ }
+
+ defer func() { _ = root.Close() }()
+
+ f, err := root.Open(list.Name)
+ if err != nil {
+ return err
+ }
+
+ defer func() { _ = f.Close() }()
+
+ info, err := f.Stat()
+ if err != nil {
+ return err
+ }
+
+ addresses, err := parse(f, list.Format)
+ if err != nil {
+ return err
+ }
+
+ held, err := readValidators(root, list.Name)
+ if err != nil {
+ return err
+ }
+
+ b.setCopy(list.Name, listCopy{
+ addresses: addresses, updated: info.ModTime().UTC(), validators: held,
+ })
+
+ return nil
+}
+
+// writeValidators keeps held in the file beside the named list's copy.
+func writeValidators(root *os.Root, name string, held validators) error {
+ data, err := json.Marshal(held)
+ if err != nil {
+ return err
+ }
+
+ return root.WriteFile(name+validatorsSuffix, data, validatorsPermissions)
+}
+
+// readValidators returns the validators kept beside the named list's copy.
+// They are empty when there is no such file.
+func readValidators(root *os.Root, name string) (validators, error) {
+ var held validators
+
+ data, err := root.ReadFile(name + validatorsSuffix)
+ if errors.Is(err, fs.ErrNotExist) {
+ return held, nil
+ }
+
+ if err != nil {
+ return held, err
+ }
+
+ err = json.Unmarshal(data, &held)
+
+ return held, err
+}
diff --git a/internal/blocklist/lists.go b/internal/blocklist/lists.go
new file mode 100644
index 0000000..2aa3507
--- /dev/null
+++ b/internal/blocklist/lists.go
@@ -0,0 +1,93 @@
+package blocklist
+
+import "time"
+
+// How often each list is downloaded. No list is downloaded more often than
+// its publisher updates it or allows.
+const (
+ // DShield asks that its feeds be downloaded at most once an hour:
+ // https://isc.sans.edu/feeds_doc.html
+ dshieldInterval = time.Hour
+
+ // abuse.ch makes the list every 5 minutes and recommends a download at
+ // least every 15: https://feodotracker.abuse.ch/blocklist/
+ feodoTrackerInterval = 15 * time.Minute
+
+ // FireHOL's page for each list gives how often the list changes on
+ // average: about every 5 hours for abusers and every 3 hours for level
+ // 3. FireHOL sets no download limit.
+ // https://iplists.firehol.org/?ipset=firehol_abusers_30d
+ // https://iplists.firehol.org/?ipset=firehol_level3
+ fireHOLAbusersInterval = 5 * time.Hour
+ fireHOLLevel3Interval = 3 * time.Hour
+
+ // Spamhaus asks that DROP be downloaded at most once a day:
+ // https://www.spamhaus.org/blocklists/do-not-route-or-peer/
+ spamhausDROPInterval = 24 * time.Hour
+
+ // Tor publishes a new exit list every hour and sets no download limit:
+ // https://collector.torproject.org/recent/exit-lists/
+ torExitsInterval = time.Hour
+)
+
+// Lists returns the public blocklists routewatch downloads. Each can be
+// downloaded whole without an account or a key. Spamhaus merged its EDROP
+// list into DROP, so DROP covers both.
+func Lists() []List {
+ return []List{
+ {
+ Name: "dshield",
+ Description: "In one of the 20 networks that DShield saw scan " +
+ "the most targets in the last three days",
+ URL: "https://feeds.dshield.org/block.txt",
+ Interval: dshieldInterval,
+ Format: FormatDShield,
+ },
+ {
+ Name: "feodo_tracker",
+ Description: "A botnet command-and-control server",
+ URL: "https://feodotracker.abuse.ch/downloads/ipblocklist.txt",
+ Interval: feodoTrackerInterval,
+ Format: FormatLines,
+ },
+ {
+ Name: "firehol_abusers_30d",
+ Description: "Reported for comment, form or forum spam in the " +
+ "last 30 days",
+ URL: "https://iplists.firehol.org/files/firehol_abusers_30d.netset",
+ Interval: fireHOLAbusersInterval,
+ Format: FormatLines,
+ },
+ {
+ Name: "firehol_level3",
+ Description: "Reported for attacks, spyware or viruses in the " +
+ "last 30 days",
+ URL: "https://iplists.firehol.org/files/firehol_level3.netset",
+ Interval: fireHOLLevel3Interval,
+ Format: FormatLines,
+ },
+ {
+ Name: "spamhaus_drop",
+ Description: "In an IPv4 network that Spamhaus says spammers " +
+ "or cyber criminals have hijacked or lease",
+ URL: "https://www.spamhaus.org/drop/drop_v4.json",
+ Interval: spamhausDROPInterval,
+ Format: FormatSpamhausJSON,
+ },
+ {
+ Name: "spamhaus_dropv6",
+ Description: "In an IPv6 network that Spamhaus says spammers " +
+ "or cyber criminals have hijacked or lease",
+ URL: "https://www.spamhaus.org/drop/drop_v6.json",
+ Interval: spamhausDROPInterval,
+ Format: FormatSpamhausJSON,
+ },
+ {
+ Name: "tor_exits",
+ Description: "A Tor exit node, where traffic from the Tor network leaves",
+ URL: "https://check.torproject.org/torbulkexitlist",
+ Interval: torExitsInterval,
+ Format: FormatLines,
+ },
+ }
+}
diff --git a/internal/blocklist/parse.go b/internal/blocklist/parse.go
new file mode 100644
index 0000000..7602cf0
--- /dev/null
+++ b/internal/blocklist/parse.go
@@ -0,0 +1,158 @@
+package blocklist
+
+import (
+ "bufio"
+ "encoding/json"
+ "errors"
+ "fmt"
+ "io"
+ "net/netip"
+ "strings"
+
+ "go4.org/netipx"
+)
+
+const (
+ // commentPrefix starts a comment line in a list file.
+ commentPrefix = "#"
+
+ // networkSeparator separates the address and the mask length of a
+ // network, as in 192.0.2.0/24.
+ networkSeparator = "/"
+
+ // dshieldFieldSeparator separates the fields of a DShield line.
+ dshieldFieldSeparator = "\t"
+
+ // spamhausMetadataType is the type of the line that ends a Spamhaus
+ // file. That line holds the file's metadata and no network.
+ spamhausMetadataType = "metadata"
+)
+
+var (
+ errUnknownFormat = errors.New("unknown list format")
+ errNoNetwork = errors.New("no network on the line")
+ errBadRange = errors.New("first and last address make no range")
+)
+
+// parse reads a list file written in format and returns the addresses it
+// holds. A line it cannot read fails the whole file, so a file that is not
+// the list, such as an error page, never replaces a good copy.
+func parse(r io.Reader, format Format) (*netipx.IPSet, error) {
+ var builder netipx.IPSetBuilder
+
+ scanner := bufio.NewScanner(r)
+
+ for lineNumber := 1; scanner.Scan(); lineNumber++ {
+ line := strings.TrimSpace(scanner.Text())
+ if line == "" || strings.HasPrefix(line, commentPrefix) {
+ continue
+ }
+
+ err := addLine(&builder, line, format)
+ if err != nil {
+ return nil, fmt.Errorf("line %d: %w", lineNumber, err)
+ }
+ }
+
+ err := scanner.Err()
+ if err != nil {
+ return nil, err
+ }
+
+ return builder.IPSet()
+}
+
+// addLine adds the address or network on one line of a list file to builder.
+func addLine(builder *netipx.IPSetBuilder, line string, format Format) error {
+ switch format {
+ case FormatLines:
+ return addAddressOrNetwork(builder, line)
+ case FormatSpamhausJSON:
+ return addSpamhausNetwork(builder, line)
+ case FormatDShield:
+ return addDShieldNetwork(builder, line)
+ default:
+ return errUnknownFormat
+ }
+}
+
+// addAddressOrNetwork adds an address, such as 192.0.2.1, or a network, such
+// as 192.0.2.0/24, to builder.
+func addAddressOrNetwork(builder *netipx.IPSetBuilder, text string) error {
+ if strings.Contains(text, networkSeparator) {
+ network, err := netip.ParsePrefix(text)
+ if err != nil {
+ return err
+ }
+
+ builder.AddPrefix(network)
+
+ return nil
+ }
+
+ addr, err := netip.ParseAddr(text)
+ if err != nil {
+ return err
+ }
+
+ builder.Add(addr)
+
+ return nil
+}
+
+// addSpamhausNetwork adds the network on a line of a Spamhaus file to
+// builder. The metadata line adds nothing.
+func addSpamhausNetwork(builder *netipx.IPSetBuilder, line string) error {
+ var entry struct {
+ CIDR string `json:"cidr"`
+ Type string `json:"type"`
+ }
+
+ err := json.Unmarshal([]byte(line), &entry)
+ if err != nil {
+ return err
+ }
+
+ if entry.Type == spamhausMetadataType {
+ return nil
+ }
+
+ network, err := netip.ParsePrefix(entry.CIDR)
+ if err != nil {
+ return err
+ }
+
+ builder.AddPrefix(network)
+
+ return nil
+}
+
+// addDShieldNetwork adds the network on a line of a DShield file to builder.
+// The line's first two fields are the network's first and last address.
+func addDShieldNetwork(builder *netipx.IPSetBuilder, line string) error {
+ firstField, rest, found := strings.Cut(line, dshieldFieldSeparator)
+ if !found {
+ return errNoNetwork
+ }
+
+ lastField, _, _ := strings.Cut(rest, dshieldFieldSeparator)
+
+ first, err := netip.ParseAddr(firstField)
+ if err != nil {
+ return err
+ }
+
+ last, err := netip.ParseAddr(lastField)
+ if err != nil {
+ return err
+ }
+
+ network := netipx.IPRangeFrom(first, last)
+ if !network.IsValid() {
+ return errBadRange
+ }
+
+ builder.AddRange(network)
+
+ return nil
+}
diff --git a/internal/database/counts.go b/internal/database/counts.go
index 5c8d4a5..033a8f1 100644
--- a/internal/database/counts.go
+++ b/internal/database/counts.go
@@ -6,11 +6,10 @@ import (
"sync"
)
-// liveCounts holds the running row counts and the prefix distribution that the
-// stats endpoints report. They are seeded once at startup from the tables and
-// then adjusted on every write, so a stats read serves them from memory instead
-// of running a query over the tables. The COUNT(*) scans (issue 27) and then
-// the prefix distribution query (issue 30) each grew with the database until
+// liveCounts holds the running row counts that the stats endpoints report.
+// They are seeded once at startup from the tables and then adjusted on every
+// write, so a stats read serves them from memory instead of running a query
+// over the tables. The COUNT(*) scans (issue 27) grew with the database until
// they took the whole request timeout and made /api/v1/stats return 500.
//
// A single mutex guards all fields so the stats reader takes a consistent
@@ -23,20 +22,11 @@ type liveCounts struct {
prefixesV6 int
peerings int
peers int
- routesV4 int
- routesV6 int
- // The prefix distribution: for each mask length, the number of distinct
- // prefixes that have at least one live route.
- distributionV4 [ipv4Bits + 1]int
- distributionV6 [ipv6Bits + 1]int
}
// seed sets every count to the value read from the tables at startup. It runs
// before any writer, so it needs no coordination with the adjust methods.
-func (c *liveCounts) seed(
- asns, prefixesV4, prefixesV6, peerings, peers, routesV4, routesV6 int,
- distributionV4, distributionV6 []PrefixDistribution,
-) {
+func (c *liveCounts) seed(asns, prefixesV4, prefixesV6, peerings, peers int) {
c.mu.Lock()
defer c.mu.Unlock()
@@ -45,16 +35,6 @@ func (c *liveCounts) seed(
c.prefixesV6 = prefixesV6
c.peerings = peerings
c.peers = peers
- c.routesV4 = routesV4
- c.routesV6 = routesV6
-
- for _, entry := range distributionV4 {
- addAtMaskLength(c.distributionV4[:], entry.MaskLength, entry.Count)
- }
-
- for _, entry := range distributionV6 {
- addAtMaskLength(c.distributionV6[:], entry.MaskLength, entry.Count)
- }
}
// addASNs adds n to the ASN count.
@@ -86,41 +66,7 @@ func (c *liveCounts) addPeers(n int) {
c.mu.Unlock()
}
-// addRoutes adds to the IPv4 and IPv6 live-route counts. Deletions pass
-// negative values.
-func (c *liveCounts) addRoutes(v4, v6 int) {
- c.mu.Lock()
- c.routesV4 += v4
- c.routesV6 += v6
- c.mu.Unlock()
-}
-
-// addToDistribution adds n to the IPv4 and IPv6 prefix distributions once for
-// each listed mask length. A write lists the mask lengths of the prefixes it
-// gave their first live route with n = 1, and of the prefixes it left with no
-// live route with n = -1.
-func (c *liveCounts) addToDistribution(maskLengthsV4, maskLengthsV6 []int, n int) {
- c.mu.Lock()
- defer c.mu.Unlock()
-
- for _, maskLength := range maskLengthsV4 {
- addAtMaskLength(c.distributionV4[:], maskLength, n)
- }
-
- for _, maskLength := range maskLengthsV6 {
- addAtMaskLength(c.distributionV6[:], maskLength, n)
- }
-}
-
-// addAtMaskLength adds n to counts[maskLength]. A mask length the array has no
-// entry for is ignored, so a malformed route cannot crash the daemon.
-func addAtMaskLength(counts []int, maskLength, n int) {
- if maskLength >= 0 && maskLength < len(counts) {
- counts[maskLength] += n
- }
-}
-
-// fill copies the counts into a Stats, including the derived totals, under a
+// fill copies the counts into a Stats, including the derived total, under a
// single read lock so the reader sees one consistent snapshot.
func (c *liveCounts) fill(s *Stats) {
c.mu.RLock()
@@ -132,26 +78,6 @@ func (c *liveCounts) fill(s *Stats) {
s.Prefixes = c.prefixesV4 + c.prefixesV6
s.Peerings = c.peerings
s.Peers = c.peers
- s.IPv4Routes = c.routesV4
- s.IPv6Routes = c.routesV6
- s.LiveRoutes = c.routesV4 + c.routesV6
- s.IPv4PrefixDistribution = distributionList(c.distributionV4[:])
- s.IPv6PrefixDistribution = distributionList(c.distributionV6[:])
-}
-
-// distributionList lists the mask lengths that have at least one prefix, in
-// ascending order, the way the distribution query returns them.
-func distributionList(counts []int) []PrefixDistribution {
- var list []PrefixDistribution
-
- for maskLength, count := range counts {
- if count > 0 {
- list = append(list,
- PrefixDistribution{MaskLength: maskLength, Count: count})
- }
- }
-
- return list
}
// countRows returns the number of rows in the named table. It is used only at
@@ -167,9 +93,8 @@ func (d *Database) countRows(ctx context.Context, table string) (int, error) {
return n, nil
}
-// seedCounts reads the current row counts and prefix distribution from the
-// tables into the in-memory counters. It runs once at startup, before the
-// streamer begins writing.
+// seedCounts reads the current row counts from the tables into the in-memory
+// counters. It runs once at startup, before the streamer begins writing.
func (d *Database) seedCounts(ctx context.Context) error {
asns, err := d.countRows(ctx, "asns")
if err != nil {
@@ -196,23 +121,7 @@ func (d *Database) seedCounts(ctx context.Context) error {
return err
}
- routesV4, err := d.countRows(ctx, tableLiveRoutesV4)
- if err != nil {
- return err
- }
-
- routesV6, err := d.countRows(ctx, tableLiveRoutesV6)
- if err != nil {
- return err
- }
-
- distributionV4, distributionV6, err := d.GetPrefixDistributionContext(ctx)
- if err != nil {
- return err
- }
-
- d.counts.seed(asns, prefixesV4, prefixesV6, peerings, peers, routesV4, routesV6,
- distributionV4, distributionV6)
+ d.counts.seed(asns, prefixesV4, prefixesV6, peerings, peers)
return nil
}
diff --git a/internal/database/counts_internal_test.go b/internal/database/counts_internal_test.go
index 3fcd7b5..d2e1d1f 100644
--- a/internal/database/counts_internal_test.go
+++ b/internal/database/counts_internal_test.go
@@ -3,62 +3,23 @@ package database
import (
"context"
"fmt"
- "slices"
"sync"
"testing"
"time"
"git.eeqj.de/sneak/routewatch/internal/config"
"git.eeqj.de/sneak/routewatch/internal/logger"
- "github.com/google/uuid"
)
-// testV4PeerIP is the peer the IPv4 routes in these tests are learned from.
+// testV4PeerIP is the peer the IPv4 rows in these tests are learned from.
const testV4PeerIP = "192.0.2.1"
-// testV6Prefix is the prefix of the IPv6 routes in these tests.
+// testV6Prefix is the IPv6 prefix of these tests.
const testV6Prefix = "2001:db8::/32"
-// mkV4Route builds an IPv4 live route with its mask length taken from the
-// prefix.
-func mkV4Route(t *testing.T, prefix string, asn int, ts time.Time) *LiveRoute {
- t.Helper()
-
- maskLength, err := prefixMaskLength(prefix)
- if err != nil {
- t.Fatalf("prefixMaskLength(%s): %v", prefix, err)
- }
-
- return &LiveRoute{
- Prefix: prefix,
- MaskLength: maskLength,
- IPVersion: ipVersionV4,
- OriginASN: asn,
- PeerIP: testV4PeerIP,
- ASPath: []int{asn},
- NextHop: "192.0.2.254",
- LastUpdated: ts,
- }
-}
-
-// mkV6Route builds an IPv6 live route.
-func mkV6Route(prefix string, asn int, ts time.Time) *LiveRoute {
- return &LiveRoute{
- Prefix: prefix,
- MaskLength: 32,
- IPVersion: ipVersionV6,
- OriginASN: asn,
- PeerIP: "2001:db8::1",
- ASPath: []int{asn},
- NextHop: "2001:db8::ffff",
- LastUpdated: ts,
- }
-}
-
// TestLiveCountsTrackWritesInRealtime checks that the stats counts start at
-// zero, reflect each write the moment it commits (no recompute, no timer), do
-// not move when a route is merely re-announced, and drop when a route is
-// deleted. These counts are what /api/v1/stats reports; before this change the
+// zero and reflect each write the moment it commits (no recompute, no timer).
+// These counts are what /api/v1/stats reports; before this change the
// endpoint recomputed them with a COUNT(*) over each table on every request.
func TestLiveCountsTrackWritesInRealtime(t *testing.T) {
t.Parallel()
@@ -79,7 +40,7 @@ func TestLiveCountsTrackWritesInRealtime(t *testing.T) {
}
if empty.ASNs != 0 || empty.Prefixes != 0 || empty.Peerings != 0 ||
- empty.Peers != 0 || empty.LiveRoutes != 0 {
+ empty.Peers != 0 {
t.Fatalf("empty database counts nonzero: %+v", empty)
}
@@ -87,17 +48,6 @@ func TestLiveCountsTrackWritesInRealtime(t *testing.T) {
writeASNsPrefixesAndPeers(t, db, ts)
- routes := []*LiveRoute{
- mkV4Route(t, "198.51.100.0/24", 64500, ts),
- mkV4Route(t, "203.0.113.0/24", 64501, ts.Add(time.Minute)),
- mkV6Route(testV6Prefix, 64502, ts.Add(2*time.Minute)),
- }
-
- err = db.UpsertLiveRouteBatch(routes)
- if err != nil {
- t.Fatalf("UpsertLiveRouteBatch: %v", err)
- }
-
stats, err := db.GetStatsContext(ctx)
if err != nil {
t.Fatalf("GetStatsContext: %v", err)
@@ -105,46 +55,11 @@ func TestLiveCountsTrackWritesInRealtime(t *testing.T) {
assertCounts(t, "after inserts", stats, wantCounts{
asns: 2, prefixes: 2, peerings: 1, peers: 1,
- ipv4Routes: 2, ipv6Routes: 1, liveRoutes: 3,
- })
-
- // Re-announcing the same routes is an update, not an insert: counts hold.
- err = db.UpsertLiveRouteBatch(routes)
- if err != nil {
- t.Fatalf("UpsertLiveRouteBatch (re-announce): %v", err)
- }
-
- stats, err = db.GetStatsContext(ctx)
- if err != nil {
- t.Fatalf("GetStatsContext: %v", err)
- }
-
- assertCounts(t, "after re-announce", stats, wantCounts{
- asns: 2, prefixes: 2, peerings: 1, peers: 1,
- ipv4Routes: 2, ipv6Routes: 1, liveRoutes: 3,
- })
-
- // A withdrawal removes one route.
- err = db.DeleteLiveRouteBatch([]LiveRouteDeletion{
- {Prefix: "203.0.113.0/24", PeerIP: testV4PeerIP, IPVersion: ipVersionV4},
- })
- if err != nil {
- t.Fatalf("DeleteLiveRouteBatch: %v", err)
- }
-
- stats, err = db.GetStatsContext(ctx)
- if err != nil {
- t.Fatalf("GetStatsContext: %v", err)
- }
-
- assertCounts(t, "after delete", stats, wantCounts{
- asns: 2, prefixes: 2, peerings: 1, peers: 1,
- ipv4Routes: 1, ipv6Routes: 1, liveRoutes: 2,
})
}
// writeASNsPrefixesAndPeers writes two ASNs, two prefixes, one peer and one
-// peering: the rows besides live routes that the stats count.
+// peering: the rows the stats count.
func writeASNsPrefixesAndPeers(t *testing.T, db *Database, ts time.Time) {
t.Helper()
@@ -180,8 +95,8 @@ func writeASNsPrefixesAndPeers(t *testing.T, db *Database, ts time.Time) {
}
// TestLiveCountsSeededFromDatabaseAtStartup writes rows, reopens the same
-// database file, and checks the counts and the prefix distribution come back
-// from the seed scan rather than starting at zero.
+// database file, and checks the counts come back from the seed scan rather
+// than starting at zero.
func TestLiveCountsSeededFromDatabaseAtStartup(t *testing.T) {
t.Parallel()
@@ -192,23 +107,7 @@ func TestLiveCountsSeededFromDatabaseAtStartup(t *testing.T) {
t.Fatalf("failed to create database: %v", err)
}
- ts := time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC)
-
- seen := SeenTimes{First: ts, Last: ts}
-
- err = db.GetOrCreateASNBatch(
- map[int]SeenTimes{64500: seen, 64501: seen, 64502: seen})
- if err != nil {
- t.Fatalf("GetOrCreateASNBatch: %v", err)
- }
-
- err = db.UpsertLiveRouteBatch([]*LiveRoute{
- mkV4Route(t, "198.51.100.0/24", 64500, ts),
- mkV6Route(testV6Prefix, 64502, ts),
- })
- if err != nil {
- t.Fatalf("UpsertLiveRouteBatch: %v", err)
- }
+ writeASNsPrefixesAndPeers(t, db, time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC))
err = db.Close()
if err != nil {
@@ -226,489 +125,9 @@ func TestLiveCountsSeededFromDatabaseAtStartup(t *testing.T) {
t.Fatalf("GetStatsContext after reopen: %v", err)
}
- if stats.ASNs != 3 {
- t.Errorf("seeded ASNs = %d, want 3", stats.ASNs)
- }
-
- if stats.IPv4Routes != 1 || stats.IPv6Routes != 1 || stats.LiveRoutes != 2 {
- t.Errorf("seeded routes = (v4 %d, v6 %d, total %d), want (1, 1, 2)",
- stats.IPv4Routes, stats.IPv6Routes, stats.LiveRoutes)
- }
-
- assertDistribution(t, "seeded IPv4 distribution", stats.IPv4PrefixDistribution,
- []PrefixDistribution{{MaskLength: 24, Count: 1}})
- assertDistribution(t, "seeded IPv6 distribution", stats.IPv6PrefixDistribution,
- []PrefixDistribution{{MaskLength: 32, Count: 1}})
-}
-
-// TestPrefixDistributionTracksWrites checks that the prefix distribution the
-// stats read reports stays exact across each kind of live-route write, and that
-// after every step it equals what the distribution query reads from the route
-// tables. The steps run once through the batch methods the prefix handler uses
-// and once through the single-route methods.
-func TestPrefixDistributionTracksWrites(t *testing.T) {
- t.Parallel()
-
- steps := prefixDistributionSteps(t)
-
- for _, batch := range []bool{true, false} {
- name := "single-route writes"
- if batch {
- name = "batch writes"
- }
-
- t.Run(name, func(t *testing.T) {
- t.Parallel()
-
- db, err := New(&config.Config{StateDir: t.TempDir()}, logger.New())
- if err != nil {
- t.Fatalf("failed to create database: %v", err)
- }
- defer func() { _ = db.Close() }()
-
- ctx := context.Background()
-
- for _, step := range steps {
- err := announceRoutes(db, batch, step.announce)
- if err != nil {
- t.Fatalf("%s: announce: %v", step.name, err)
- }
-
- err = withdrawRoutes(db, batch, step.withdraw)
- if err != nil {
- t.Fatalf("%s: withdraw: %v", step.name, err)
- }
-
- stats, err := db.GetStatsContext(ctx)
- if err != nil {
- t.Fatalf("%s: GetStatsContext: %v", step.name, err)
- }
-
- queryV4, queryV6, err := db.GetPrefixDistributionContext(ctx)
- if err != nil {
- t.Fatalf("%s: GetPrefixDistributionContext: %v", step.name, err)
- }
-
- assertDistribution(t, step.name+": IPv4 distribution",
- stats.IPv4PrefixDistribution, step.wantV4)
- assertDistribution(t, step.name+": IPv6 distribution",
- stats.IPv6PrefixDistribution, step.wantV6)
- assertDistribution(t, step.name+": IPv4 distribution query",
- queryV4, step.wantV4)
- assertDistribution(t, step.name+": IPv6 distribution query",
- queryV6, step.wantV6)
- }
- })
- }
-}
-
-// prefixDistributionStep is one step of TestPrefixDistributionTracksWrites:
-// the routes it announces, then the routes it withdraws, then the
-// distributions both the stats and the distribution query must report.
-type prefixDistributionStep struct {
- name string
- announce []*LiveRoute
- withdraw []*LiveRoute
- wantV4 []PrefixDistribution
- wantV6 []PrefixDistribution
-}
-
-// prefixDistributionSteps returns the steps of
-// TestPrefixDistributionTracksWrites, in the order they run.
-func prefixDistributionSteps(t *testing.T) []prefixDistributionStep {
- t.Helper()
-
- ts := time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC)
- shared := mkV4Route(t, "198.51.100.0/24", 64500, ts)
- sharedSecondPeer := mkV4Route(t, "198.51.100.0/24", 64500, ts)
- sharedSecondPeer.PeerIP = "192.0.2.2"
- other := mkV4Route(t, "203.0.113.0/24", 64501, ts)
- wide := mkV4Route(t, "172.16.0.0/16", 64502, ts)
- v6 := mkV6Route(testV6Prefix, 64503, ts)
- v6SecondPeer := mkV6Route(testV6Prefix, 64503, ts)
- v6SecondPeer.PeerIP = "2001:db8::2"
-
- all := []PrefixDistribution{{MaskLength: 16, Count: 1}, {MaskLength: 24, Count: 2}}
- v6Only := []PrefixDistribution{{MaskLength: 32, Count: 1}}
-
- return []prefixDistributionStep{
- {
- name: "new routes",
- announce: []*LiveRoute{shared, other, wide, v6},
- wantV4: all, wantV6: v6Only,
- },
- {
- name: "re-announcement",
- announce: []*LiveRoute{shared, other, wide, v6},
- wantV4: all, wantV6: v6Only,
- },
- {
- name: "second peer announces a prefix that has a live route",
- announce: []*LiveRoute{sharedSecondPeer, v6SecondPeer},
- wantV4: all, wantV6: v6Only,
- },
- {
- name: "withdrawal of a route that is not the last for its prefix",
- withdraw: []*LiveRoute{shared, v6},
- wantV4: all, wantV6: v6Only,
- },
- {
- name: "withdrawal of the last route for a prefix",
- withdraw: []*LiveRoute{sharedSecondPeer, v6SecondPeer},
- wantV4: []PrefixDistribution{
- {MaskLength: 16, Count: 1}, {MaskLength: 24, Count: 1},
- },
- },
- {
- name: "withdrawal of every remaining route",
- withdraw: []*LiveRoute{other, wide},
- },
- {
- name: "two peers announce a new prefix together",
- announce: []*LiveRoute{shared, sharedSecondPeer, v6, v6SecondPeer},
- wantV4: []PrefixDistribution{{MaskLength: 24, Count: 1}},
- wantV6: v6Only,
- },
- {
- name: "both routes for a prefix withdrawn together",
- withdraw: []*LiveRoute{shared, sharedSecondPeer, v6, v6SecondPeer},
- },
- // The feed often withdraws a route that is not live. That must not take
- // the prefix out of the distribution. A count wrongly taken below zero is
- // left out of the answer, so the next step shows it: its announcement
- // at the same mask length would then not be counted.
- {
- name: "withdrawal of routes that are not live",
- withdraw: []*LiveRoute{other, v6},
- },
- {
- name: "announcement after a withdrawal of routes that are not live",
- announce: []*LiveRoute{other, v6},
- wantV4: []PrefixDistribution{{MaskLength: 24, Count: 1}},
- wantV6: v6Only,
- },
- }
-}
-
-// announceRoutes writes routes in one UpsertLiveRouteBatch, or with one
-// UpsertLiveRoute each.
-func announceRoutes(db *Database, batch bool, routes []*LiveRoute) error {
- if batch {
- return db.UpsertLiveRouteBatch(routes)
- }
-
- for _, route := range routes {
- err := db.UpsertLiveRoute(route)
- if err != nil {
- return err
- }
- }
-
- return nil
-}
-
-// withdrawRoutes removes routes in one DeleteLiveRouteBatch, or with one
-// DeleteLiveRoute each.
-func withdrawRoutes(db *Database, batch bool, routes []*LiveRoute) error {
- if !batch {
- for _, route := range routes {
- err := db.DeleteLiveRoute(route.Prefix, route.PeerIP)
- if err != nil {
- return err
- }
- }
-
- return nil
- }
-
- deletions := make([]LiveRouteDeletion, 0, len(routes))
- for _, route := range routes {
- deletions = append(deletions, LiveRouteDeletion{
- Prefix: route.Prefix, PeerIP: route.PeerIP, IPVersion: route.IPVersion,
- })
- }
-
- return db.DeleteLiveRouteBatch(deletions)
-}
-
-// TestDeleteLiveRoutesOfPeer gives two peers IPv4 and IPv6 live routes, some
-// for the same prefixes, and deletes all of one peer's routes. The other peer's
-// routes stay, and the route counts and prefix distribution held in memory
-// equal what is read from the route tables.
-func TestDeleteLiveRoutesOfPeer(t *testing.T) {
- t.Parallel()
-
- const (
- peerIP = "192.0.2.1"
- otherPeerIP = "192.0.2.2"
- sharedV4 = "198.51.100.0/24"
- sharedV6 = "2001:db8::/32"
- )
-
- db, err := New(&config.Config{StateDir: t.TempDir()}, logger.New())
- if err != nil {
- t.Fatalf("failed to create database: %v", err)
- }
- defer func() { _ = db.Close() }()
-
- ts := time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC)
- ofPeer := func(peer string, route *LiveRoute) *LiveRoute {
- route.PeerIP = peer
-
- return route
- }
- onlyV6 := mkV6Route("2001:db8:1::/48", 64503, ts)
- onlyV6.MaskLength = 48
-
- err = db.UpsertLiveRouteBatch([]*LiveRoute{
- ofPeer(peerIP, mkV4Route(t, sharedV4, 64500, ts)),
- ofPeer(otherPeerIP, mkV4Route(t, sharedV4, 64500, ts)),
- ofPeer(peerIP, mkV4Route(t, "203.0.113.0/24", 64501, ts)),
- ofPeer(peerIP, mkV6Route(sharedV6, 64502, ts)),
- ofPeer(otherPeerIP, mkV6Route(sharedV6, 64502, ts)),
- ofPeer(peerIP, onlyV6),
+ assertCounts(t, "after reopen", stats, wantCounts{
+ asns: 2, prefixes: 2, peerings: 1, peers: 1,
})
- if err != nil {
- t.Fatalf("UpsertLiveRouteBatch: %v", err)
- }
-
- // peerIP has two IPv4 and two IPv6 routes; a second delete finds none.
- for _, want := range []int{4, 0} {
- deleted, err := db.DeleteLiveRoutesOfPeer(peerIP)
- if err != nil {
- t.Fatalf("DeleteLiveRoutesOfPeer: %v", err)
- }
-
- if deleted != want {
- t.Errorf("DeleteLiveRoutesOfPeer deleted %d routes, want %d",
- deleted, want)
- }
- }
-
- for _, prefix := range []string{sharedV4, sharedV6} {
- routes, _, err := db.GetPrefixDetailsContext(context.Background(), prefix)
- if err != nil {
- t.Fatalf("GetPrefixDetailsContext(%s): %v", prefix, err)
- }
-
- if len(routes) != 1 || routes[0].PeerIP != otherPeerIP {
- t.Errorf("live routes for %s = %+v, want only the one from %s",
- prefix, routes, otherPeerIP)
- }
- }
-
- checkOneLiveRoutePerIPVersion(t, db)
-}
-
-// TestDeleteLiveRoutesOfPeerDeletesInChunks gives a peer one IPv4 route more
-// than a chunk holds, two of them for prefixes another peer also has. One chunk
-// deletes peerRouteChunkSize of the peer's routes and commits, leaving the last
-// one. The routes are written again, and DeleteLiveRoutesOfPeer deletes all of
-// them, a chunk and then the last one. After each step the route count and
-// prefix distribution held in memory and those read from the route table are
-// the ones wanted.
-func TestDeleteLiveRoutesOfPeerDeletesInChunks(t *testing.T) {
- t.Parallel()
-
- const (
- peerIP = "192.0.2.1"
- otherPeerIP = "192.0.2.2"
- // How many of the peer's prefixes the other peer also has, the first
- // ones in prefix order.
- sharedPrefixes = 2
- )
-
- db, err := New(&config.Config{StateDir: t.TempDir()}, logger.New())
- if err != nil {
- t.Fatalf("failed to create database: %v", err)
- }
- defer func() { _ = db.Close() }()
-
- ts := time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC)
-
- peerRoutes := make([]*LiveRoute, 0, peerRouteChunkSize+1)
- otherPeerRoutes := make([]*LiveRoute, 0, sharedPrefixes)
-
- for i := range peerRouteChunkSize + 1 {
- prefix := fmt.Sprintf("10.%d.%d.0/24", i/256, i%256)
-
- route := mkV4Route(t, prefix, 64500, ts)
- route.PeerIP = peerIP
- peerRoutes = append(peerRoutes, route)
-
- if i < sharedPrefixes {
- route := mkV4Route(t, prefix, 64500, ts)
- route.PeerIP = otherPeerIP
- otherPeerRoutes = append(otherPeerRoutes, route)
- }
- }
-
- err = db.UpsertLiveRouteBatch(slices.Concat(peerRoutes, otherPeerRoutes))
- if err != nil {
- t.Fatalf("UpsertLiveRouteBatch: %v", err)
- }
-
- deletedV4, deletedV6, err := db.deletePeerRouteChunk(peerIP)
- if err != nil {
- t.Fatalf("deletePeerRouteChunk: %v", err)
- }
-
- if deletedV4 != peerRouteChunkSize || deletedV6 != 0 {
- t.Errorf("deletePeerRouteChunk deleted (%d, %d) routes, want (%d, 0)",
- deletedV4, deletedV6, peerRouteChunkSize)
- }
-
- // The shared prefixes went with the chunk, so each route left has a
- // prefix of its own.
- checkV4RoutesOfOwnPrefixes(t, db, "after one chunk", 1+sharedPrefixes)
-
- err = db.UpsertLiveRouteBatch(peerRoutes)
- if err != nil {
- t.Fatalf("UpsertLiveRouteBatch: %v", err)
- }
-
- deleted, err := db.DeleteLiveRoutesOfPeer(peerIP)
- if err != nil {
- t.Fatalf("DeleteLiveRoutesOfPeer: %v", err)
- }
-
- if deleted != len(peerRoutes) {
- t.Errorf("DeleteLiveRoutesOfPeer deleted %d routes, want %d",
- deleted, len(peerRoutes))
- }
-
- checkV4RoutesOfOwnPrefixes(t, db, "after DeleteLiveRoutesOfPeer",
- sharedPrefixes)
-}
-
-// checkV4RoutesOfOwnPrefixes checks that the database has wantRoutes IPv4 live
-// routes, each for a /24 no other live route has, both as counted in memory and
-// as read from the route table. when says which step of the test checks.
-func checkV4RoutesOfOwnPrefixes(
- t *testing.T, db *Database, when string, wantRoutes int,
-) {
- t.Helper()
-
- ctx := context.Background()
-
- stats, err := db.GetStatsContext(ctx)
- if err != nil {
- t.Fatalf("GetStatsContext: %v", err)
- }
-
- tableRoutes, _, err := db.GetLiveRouteCountsContext(ctx)
- if err != nil {
- t.Fatalf("GetLiveRouteCountsContext: %v", err)
- }
-
- if stats.IPv4Routes != wantRoutes || tableRoutes != wantRoutes {
- t.Errorf("%s: IPv4 live routes: %d in memory, %d in the table, want %d",
- when, stats.IPv4Routes, tableRoutes, wantRoutes)
- }
-
- queryDistribution, _, err := db.GetPrefixDistributionContext(ctx)
- if err != nil {
- t.Fatalf("GetPrefixDistributionContext: %v", err)
- }
-
- want := []PrefixDistribution{{MaskLength: 24, Count: wantRoutes}}
- assertDistribution(t, when+": IPv4 distribution",
- stats.IPv4PrefixDistribution, want)
- assertDistribution(t, when+": IPv4 distribution query",
- queryDistribution, want)
-}
-
-// checkOneLiveRoutePerIPVersion checks that the database has one IPv4 live
-// route, for a /24, and one IPv6 live route, for a /32, both as counted in
-// memory and as read from the route tables.
-func checkOneLiveRoutePerIPVersion(t *testing.T, db *Database) {
- t.Helper()
-
- ctx := context.Background()
-
- stats, err := db.GetStatsContext(ctx)
- if err != nil {
- t.Fatalf("GetStatsContext: %v", err)
- }
-
- tableV4, tableV6, err := db.GetLiveRouteCountsContext(ctx)
- if err != nil {
- t.Fatalf("GetLiveRouteCountsContext: %v", err)
- }
-
- if stats.IPv4Routes != 1 || tableV4 != 1 || stats.IPv6Routes != 1 ||
- tableV6 != 1 {
- t.Errorf("live routes (IPv4, IPv6): (%d, %d) in memory, (%d, %d) in "+
- "the tables, want (1, 1)",
- stats.IPv4Routes, stats.IPv6Routes, tableV4, tableV6)
- }
-
- queryV4, queryV6, err := db.GetPrefixDistributionContext(ctx)
- if err != nil {
- t.Fatalf("GetPrefixDistributionContext: %v", err)
- }
-
- wantV4 := []PrefixDistribution{{MaskLength: 24, Count: 1}}
- wantV6 := []PrefixDistribution{{MaskLength: 32, Count: 1}}
-
- assertDistribution(t, "IPv4 distribution", stats.IPv4PrefixDistribution, wantV4)
- assertDistribution(t, "IPv6 distribution", stats.IPv6PrefixDistribution, wantV6)
- assertDistribution(t, "IPv4 distribution query", queryV4, wantV4)
- assertDistribution(t, "IPv6 distribution query", queryV6, wantV6)
-}
-
-// TestStatsRouteTimestamps checks the oldest/newest route timestamps are read
-// from the right rows across both tables and parse into time.Time. The old
-// MIN/MAX union query read its result into *time.Time, which the driver could
-// not parse, so it logged a warning every call and left both timestamps nil.
-func TestStatsRouteTimestamps(t *testing.T) {
- t.Parallel()
-
- cfg := &config.Config{StateDir: t.TempDir()}
-
- db, err := New(cfg, logger.New())
- if err != nil {
- t.Fatalf("failed to create database: %v", err)
- }
- defer func() { _ = db.Close() }()
-
- ctx := context.Background()
-
- empty, err := db.GetStatsContext(ctx)
- if err != nil {
- t.Fatalf("GetStatsContext on empty database: %v", err)
- }
-
- if empty.OldestRoute != nil || empty.NewestRoute != nil {
- t.Fatalf("empty database timestamps = (%v, %v), want (nil, nil)",
- empty.OldestRoute, empty.NewestRoute)
- }
-
- base := time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC)
- oldest := base
- newest := base.Add(2 * time.Minute)
-
- err = db.UpsertLiveRouteBatch([]*LiveRoute{
- mkV4Route(t, "198.51.100.0/24", 64500, base.Add(time.Minute)),
- mkV4Route(t, "203.0.113.0/24", 64501, oldest),
- mkV6Route(testV6Prefix, 64502, newest),
- })
- if err != nil {
- t.Fatalf("UpsertLiveRouteBatch: %v", err)
- }
-
- stats, err := db.GetStatsContext(ctx)
- if err != nil {
- t.Fatalf("GetStatsContext: %v", err)
- }
-
- if stats.OldestRoute == nil || !stats.OldestRoute.Equal(oldest) {
- t.Errorf("OldestRoute = %v, want %v", stats.OldestRoute, oldest)
- }
-
- if stats.NewestRoute == nil || !stats.NewestRoute.Equal(newest) {
- t.Errorf("NewestRoute = %v, want %v", stats.NewestRoute, newest)
- }
}
// TestLiveCountsConcurrentReadWrite runs writers and stats readers at once so
@@ -726,7 +145,10 @@ func TestLiveCountsConcurrentReadWrite(t *testing.T) {
ts := time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC)
- const writers = 4
+ const (
+ writers = 4
+ prefixesPerWrite = 25
+ )
var wg sync.WaitGroup
wg.Add(writers)
@@ -735,14 +157,12 @@ func TestLiveCountsConcurrentReadWrite(t *testing.T) {
go func(base int) {
defer wg.Done()
- for i := range 25 {
- asn := 65000 + base*100 + i
- route := mkV6Route(testV6Prefix, asn, ts)
- route.PeerIP = "2001:db8::" + uuid.NewString()
+ for i := range prefixesPerWrite {
+ prefix := fmt.Sprintf("10.%d.%d.0/24", base, i)
- err := db.UpsertLiveRoute(route)
+ err := db.UpdatePrefixesBatch(map[string]time.Time{prefix: ts})
if err != nil {
- t.Errorf("UpsertLiveRoute: %v", err)
+ t.Errorf("UpdatePrefixesBatch: %v", err)
return
}
@@ -782,22 +202,16 @@ func TestLiveCountsConcurrentReadWrite(t *testing.T) {
t.Fatalf("GetStatsContext: %v", err)
}
- if want := writers * 25; stats.IPv6Routes != want {
- t.Errorf("IPv6Routes = %d, want %d", stats.IPv6Routes, want)
+ if want := writers * prefixesPerWrite; stats.IPv4Prefixes != want {
+ t.Errorf("IPv4Prefixes = %d, want %d", stats.IPv4Prefixes, want)
}
- // Every writer announced the same prefix, so it counts once.
- assertDistribution(t, "IPv6 distribution", stats.IPv6PrefixDistribution,
- []PrefixDistribution{{MaskLength: 32, Count: 1}})
}
type wantCounts struct {
- asns int
- prefixes int
- peerings int
- peers int
- ipv4Routes int
- ipv6Routes int
- liveRoutes int
+ asns int
+ prefixes int
+ peerings int
+ peers int
}
func assertCounts(t *testing.T, when string, got Stats, want wantCounts) {
@@ -818,24 +232,4 @@ func assertCounts(t *testing.T, when string, got Stats, want wantCounts) {
if got.Peers != want.peers {
t.Errorf("%s: Peers = %d, want %d", when, got.Peers, want.peers)
}
-
- if got.IPv4Routes != want.ipv4Routes {
- t.Errorf("%s: IPv4Routes = %d, want %d", when, got.IPv4Routes, want.ipv4Routes)
- }
-
- if got.IPv6Routes != want.ipv6Routes {
- t.Errorf("%s: IPv6Routes = %d, want %d", when, got.IPv6Routes, want.ipv6Routes)
- }
-
- if got.LiveRoutes != want.liveRoutes {
- t.Errorf("%s: LiveRoutes = %d, want %d", when, got.LiveRoutes, want.liveRoutes)
- }
-}
-
-func assertDistribution(t *testing.T, what string, got, want []PrefixDistribution) {
- t.Helper()
-
- if !slices.Equal(got, want) {
- t.Errorf("%s = %v, want %v", what, got, want)
- }
}
diff --git a/internal/database/database.go b/internal/database/database.go
index caa7f69..4988e69 100644
--- a/internal/database/database.go
+++ b/internal/database/database.go
@@ -1,16 +1,12 @@
package database
import (
- "cmp"
"context"
"database/sql"
_ "embed"
- "encoding/json"
"errors"
"fmt"
"maps"
- "math/rand/v2"
- "net"
"net/netip"
"os"
"path/filepath"
@@ -35,25 +31,12 @@ const (
dirPermissions = 0750 // rwxr-x---
ipVersionV4 = 4
ipVersionV6 = 6
- ipv4Bits = 32
- ipv6Bits = 128
-
- tableLiveRoutesV4 = "live_routes_v4"
- tableLiveRoutesV6 = "live_routes_v6"
// Names of the IP versions in error and log messages.
ipv4Family = "IPv4"
ipv6Family = "IPv6"
)
-// The IP lookup does not use routes shorter than these mask lengths. A default
-// route or a route this short only says where a peer sends traffic it has no
-// better route for; it does not say who holds the address.
-const (
- minLookupMaskLengthV4 = 8
- minLookupMaskLengthV6 = 16
-)
-
// SQLite memory tuning (README "Memory"). cache_size and busy_timeout go in the
// DSN so every pooled connection gets them; the connection kept for writing
// then sets its own cache size. The heap limits are process-wide and set once.
@@ -67,18 +50,19 @@ const (
// since its last read, many times a minute, so a large one is wasted.
sqliteReadCacheSizeKiB = -16384
// sqliteWriteCacheSizeKiB is the page cache of the connection kept for
- // writing, -1048576 = 1 GiB. It keeps its pages from one write to the next,
- // so the index pages the route writes touch stay in memory.
- sqliteWriteCacheSizeKiB = -1048576
+ // writing, -524288 = 512 MiB. It keeps its pages from one write to the
+ // next. The routes table of a full routing table takes about 450 MB, so
+ // the pages the writes of the table's changes touch stay in memory.
+ sqliteWriteCacheSizeKiB = -524288
// sqliteBusyTimeoutMs is how long a connection waits on a locked database.
sqliteBusyTimeoutMs = 5000
- // sqliteSoftHeapLimitBytes (1.5 GiB) makes SQLite recycle its cache rather
+ // sqliteSoftHeapLimitBytes (768 MiB) makes SQLite recycle its cache rather
// than allocate once its C heap passes this size. It is above the caches
- // together, so it does not shrink them.
- sqliteSoftHeapLimitBytes = 1610612736
- // sqliteHardHeapLimitBytes (2 GiB) fails a statement with SQLITE_NOMEM
+ // together, 656 MiB, so it does not shrink them.
+ sqliteSoftHeapLimitBytes = 805306368
+ // sqliteHardHeapLimitBytes (1 GiB) fails a statement with SQLITE_NOMEM
// instead of growing the C heap without bound.
- sqliteHardHeapLimitBytes = 2147483648
+ sqliteHardHeapLimitBytes = 1073741824
)
// Settings that bound the WAL file's size once no read is open (see
@@ -110,23 +94,8 @@ const (
optimizeAllTables = 0x10002
)
-// Bounds on the work GetRandomPrefixesByLength does to pick its sample.
-const (
- // maxPrefixesToShuffle is the most prefixes of one mask length that are
- // read in full and shuffled to pick the sample. A mask length with more is
- // sampled with random lookups.
- maxPrefixesToShuffle = 5000
- // sampleLookupsPerPrefix is the most random lookups made for each prefix
- // asked for.
- sampleLookupsPerPrefix = 4
-)
-
// Common errors
var (
- // ErrInvalidIP is returned when an IP address is malformed
- ErrInvalidIP = errors.New("invalid IP address")
- // ErrNoRoute is returned when no route is found for an IP
- ErrNoRoute = errors.New("no route found")
// ErrNoStaleASN is returned when no ASN needs WHOIS refresh
ErrNoStaleASN = errors.New("no stale ASN found")
// ErrCheckpointIncomplete is returned by Checkpoint when it left part of
@@ -331,511 +300,6 @@ func (d *Database) Close() error {
return errors.Join(d.writer.Close(), d.db.Close())
}
-// A live-route upsert sets the origin AS of the peer's route for the prefix
-// when the announcement changes it, then updates the rest of the route, and
-// inserts the route only when that update matched no row. A live route is keyed
-// by prefix and peer, so an announcement replaces the peer's route for the
-// prefix whatever its origin AS. The update's rows-affected count (1 for an
-// existing key, 0 for a new one) is what lets the in-memory route counters stay
-// exact without a COUNT(*). Callers hold the database write lock, so no other
-// writer can insert the same key between the statements.
-//
-// SQLite rewrites a row's entry in every index on a column an UPDATE sets, even
-// when the value stays the same, so the update sets only the columns that
-// change: mask_length follows from the prefix, and origin_asn has a statement
-// of its own that changes nothing when the origin AS stays the same, as it
-// mostly does.
-const (
- updateLiveRouteOriginV4SQL = `UPDATE live_routes_v4 SET origin_asn = ?
- WHERE prefix = ? AND peer_ip = ? AND origin_asn != ?`
-
- updateLiveRouteV4SQL = `UPDATE live_routes_v4
- SET as_path = ?, next_hop = ?, last_updated = ?
- WHERE prefix = ? AND peer_ip = ?`
-
- insertLiveRouteV4SQL = `INSERT INTO live_routes_v4
- (prefix, mask_length, origin_asn, peer_ip, as_path, next_hop,
- last_updated) VALUES (?, ?, ?, ?, ?, ?, ?)`
-
- updateLiveRouteOriginV6SQL = `UPDATE live_routes_v6 SET origin_asn = ?
- WHERE prefix = ? AND peer_ip = ? AND origin_asn != ?`
-
- updateLiveRouteV6SQL = `UPDATE live_routes_v6
- SET as_path = ?, next_hop = ?, last_updated = ?
- WHERE prefix = ? AND peer_ip = ?`
-
- insertLiveRouteV6SQL = `INSERT INTO live_routes_v6
- (prefix, mask_length, origin_asn, peer_ip, as_path, next_hop,
- last_updated) VALUES (?, ?, ?, ?, ?, ?, ?)`
-)
-
-// Before a new route is inserted and after a route is deleted, the write looks
-// up whether any live route has that prefix, to keep the in-memory prefix
-// distribution exact. The lookup reads one entry of the prefix index.
-const (
- prefixHasLiveRouteV4SQL = `SELECT EXISTS
- (SELECT 1 FROM live_routes_v4 WHERE prefix = ?)`
- prefixHasLiveRouteV6SQL = `SELECT EXISTS
- (SELECT 1 FROM live_routes_v6 WHERE prefix = ?)`
-)
-
-// A deletion removes the live route the peer has for the prefix, if any.
-const (
- deleteLiveRouteV4SQL = `DELETE FROM live_routes_v4
- WHERE prefix = ? AND peer_ip = ?`
- deleteLiveRouteV6SQL = `DELETE FROM live_routes_v6
- WHERE prefix = ? AND peer_ip = ?`
-)
-
-// upsertRouteRow updates a live route with the origin and update statements of
-// its IP version's table, inserting it when no row matched. It reports whether
-// a new row was inserted and whether that row is the first live route for its
-// prefix.
-func upsertRouteRow(
- ctx context.Context, setOrigin, upd, ins, has *sql.Stmt, route *LiveRoute,
- pathJSON string,
-) (bool, bool, error) {
- _, err := setOrigin.ExecContext(ctx, route.OriginASN, route.Prefix,
- route.PeerIP, route.OriginASN)
- if err != nil {
- return false, false, err
- }
-
- res, err := upd.ExecContext(ctx, pathJSON, route.NextHop, route.LastUpdated,
- route.Prefix, route.PeerIP)
- if err != nil {
- return false, false, err
- }
-
- affected, err := res.RowsAffected()
- if err != nil {
- return false, false, err
- }
-
- if affected > 0 {
- return false, false, nil
- }
-
- var prefixHadRoute bool
-
- err = has.QueryRowContext(ctx, route.Prefix).Scan(&prefixHadRoute)
- if err != nil {
- return false, false, err
- }
-
- _, err = ins.ExecContext(ctx, route.Prefix, route.MaskLength,
- route.OriginASN, route.PeerIP, pathJSON, route.NextHop,
- route.LastUpdated)
- if err != nil {
- return false, false, err
- }
-
- return true, !prefixHadRoute, nil
-}
-
-// upsertRoutesInTable upserts routes, all of one IP version, in the live route
-// table of that version. It returns how many new rows it inserted and the mask
-// lengths of the prefixes that got their first live route.
-func upsertRoutesInTable(
- ctx context.Context, tx *loggingTx, routes []*LiveRoute, ipVersion int,
-) (int, []int, error) {
- if len(routes) == 0 {
- return 0, nil, nil
- }
-
- family, originSQL, updateSQL := ipv4Family, updateLiveRouteOriginV4SQL,
- updateLiveRouteV4SQL
- insertSQL, lookupSQL := insertLiveRouteV4SQL, prefixHasLiveRouteV4SQL
-
- if ipVersion == ipVersionV6 {
- family, originSQL, updateSQL = ipv6Family, updateLiveRouteOriginV6SQL,
- updateLiveRouteV6SQL
- insertSQL, lookupSQL = insertLiveRouteV6SQL, prefixHasLiveRouteV6SQL
- }
-
- setOrigin, err := tx.PrepareContext(ctx, originSQL)
- if err != nil {
- return 0, nil, fmt.Errorf("failed to prepare %s origin statement: %w",
- family, err)
- }
- defer func() { _ = setOrigin.Close() }()
-
- upd, err := tx.PrepareContext(ctx, updateSQL)
- if err != nil {
- return 0, nil, fmt.Errorf("failed to prepare %s update statement: %w",
- family, err)
- }
- defer func() { _ = upd.Close() }()
-
- ins, err := tx.PrepareContext(ctx, insertSQL)
- if err != nil {
- return 0, nil, fmt.Errorf("failed to prepare %s insert statement: %w",
- family, err)
- }
- defer func() { _ = ins.Close() }()
-
- has, err := tx.PrepareContext(ctx, lookupSQL)
- if err != nil {
- return 0, nil, fmt.Errorf(
- "failed to prepare %s prefix lookup statement: %w", family, err)
- }
- defer func() { _ = has.Close() }()
-
- var newRows int
- // Mask lengths of the prefixes that get their first live route.
- var newPrefixMaskLengths []int
-
- for _, route := range routes {
- pathJSON, err := json.Marshal(route.ASPath)
- if err != nil {
- return 0, nil, fmt.Errorf("failed to encode AS path: %w", err)
- }
-
- inserted, newPrefix, err := upsertRouteRow(ctx, setOrigin, upd, ins, has,
- route, string(pathJSON))
- if err != nil {
- return 0, nil, fmt.Errorf("failed to upsert route %s: %w",
- route.Prefix, err)
- }
-
- if inserted {
- newRows++
- }
-
- if newPrefix {
- newPrefixMaskLengths = append(newPrefixMaskLengths, route.MaskLength)
- }
- }
-
- return newRows, newPrefixMaskLengths, nil
-}
-
-// UpsertLiveRouteBatch inserts or updates multiple live routes in a single
-// transaction, the IPv4 routes first and then the IPv6 routes.
-func (d *Database) UpsertLiveRouteBatch(routes []*LiveRoute) error {
- if len(routes) == 0 {
- return nil
- }
-
- d.lock("UpsertLiveRouteBatch")
- defer d.unlock()
-
- tx, err := d.beginTx()
- if err != nil {
- return fmt.Errorf("failed to begin transaction: %w", err)
- }
- defer d.rollback(tx)
-
- // The rows are written in key order, peer then prefix, so consecutive writes
- // land on the same index pages. The sort is stable, so of two updates of
- // one route the later one is still written last.
- sorted := slices.SortedStableFunc(slices.Values(routes),
- func(a, b *LiveRoute) int {
- return cmp.Or(cmp.Compare(a.PeerIP, b.PeerIP),
- cmp.Compare(a.Prefix, b.Prefix))
- })
-
- var routesV4, routesV6 []*LiveRoute
-
- for _, route := range sorted {
- if route.IPVersion == ipVersionV4 {
- routesV4 = append(routesV4, route)
- } else {
- routesV6 = append(routesV6, route)
- }
- }
-
- ctx := context.Background()
-
- newV4, newPrefixMaskLengthsV4, err := upsertRoutesInTable(ctx, tx,
- routesV4, ipVersionV4)
- if err != nil {
- return err
- }
-
- newV6, newPrefixMaskLengthsV6, err := upsertRoutesInTable(ctx, tx,
- routesV6, ipVersionV6)
- if err != nil {
- return err
- }
-
- err = tx.Commit()
- if err != nil {
- return fmt.Errorf("failed to commit transaction: %w", err)
- }
-
- d.counts.addRoutes(newV4, newV6)
- d.counts.addToDistribution(newPrefixMaskLengthsV4, newPrefixMaskLengthsV6, 1)
-
- return nil
-}
-
-// deleteRouteRow runs one deletion with the delete statement of its IP
-// version's table. It returns how many rows it removed and whether that left
-// the prefix with no live route.
-func deleteRouteRow(
- ctx context.Context, deleteStmt, has *sql.Stmt, del LiveRouteDeletion,
-) (int64, bool, error) {
- res, err := deleteStmt.ExecContext(ctx, del.Prefix, del.PeerIP)
- if err != nil {
- return 0, false, fmt.Errorf("failed to delete route %s: %w",
- del.Prefix, err)
- }
-
- // The feed often withdraws a route that is not live, which deletes
- // nothing, so the in-memory route counters use the rows-affected count.
- affected, err := res.RowsAffected()
- if err != nil {
- return 0, false, fmt.Errorf("failed to count deleted route %s: %w",
- del.Prefix, err)
- }
-
- if affected == 0 {
- return 0, false, nil
- }
-
- // The prefix leaves the distribution when no live route has it any more.
- var prefixHasRoute bool
-
- err = has.QueryRowContext(ctx, del.Prefix).Scan(&prefixHasRoute)
- if err != nil {
- return 0, false, fmt.Errorf("failed to look up prefix %s: %w",
- del.Prefix, err)
- }
-
- return affected, !prefixHasRoute, nil
-}
-
-// deleteRoutesInTable runs deletions, all of one IP version, on the live route
-// table of that version. It returns how many rows it removed and the mask
-// lengths of the prefixes it left with no live route.
-func deleteRoutesInTable(
- ctx context.Context, tx *loggingTx, deletions []LiveRouteDeletion,
- ipVersion int,
-) (int64, []int, error) {
- if len(deletions) == 0 {
- return 0, nil, nil
- }
-
- family, deleteSQL, lookupSQL := ipv4Family, deleteLiveRouteV4SQL,
- prefixHasLiveRouteV4SQL
- if ipVersion == ipVersionV6 {
- family, deleteSQL, lookupSQL = ipv6Family, deleteLiveRouteV6SQL,
- prefixHasLiveRouteV6SQL
- }
-
- deleteStmt, err := tx.PrepareContext(ctx, deleteSQL)
- if err != nil {
- return 0, nil, fmt.Errorf("failed to prepare %s delete statement: %w",
- family, err)
- }
- defer func() { _ = deleteStmt.Close() }()
-
- has, err := tx.PrepareContext(ctx, lookupSQL)
- if err != nil {
- return 0, nil, fmt.Errorf(
- "failed to prepare %s prefix lookup statement: %w", family, err)
- }
- defer func() { _ = has.Close() }()
-
- var deleted int64
- // Mask lengths of the prefixes left with no live route.
- var gonePrefixMaskLengths []int
-
- for _, del := range deletions {
- affected, prefixGone, err := deleteRouteRow(ctx, deleteStmt, has, del)
- if err != nil {
- return 0, nil, err
- }
-
- deleted += affected
-
- if !prefixGone {
- continue
- }
-
- maskLength, err := prefixMaskLength(del.Prefix)
- if err != nil {
- return 0, nil, err
- }
-
- gonePrefixMaskLengths = append(gonePrefixMaskLengths, maskLength)
- }
-
- return deleted, gonePrefixMaskLengths, nil
-}
-
-// DeleteLiveRouteBatch deletes multiple live routes in a single transaction,
-// the IPv4 routes first and then the IPv6 routes.
-func (d *Database) DeleteLiveRouteBatch(deletions []LiveRouteDeletion) error {
- if len(deletions) == 0 {
- return nil
- }
-
- d.lock("DeleteLiveRouteBatch")
- defer d.unlock()
-
- tx, err := d.beginTx()
- if err != nil {
- return fmt.Errorf("failed to begin transaction: %w", err)
- }
- defer d.rollback(tx)
-
- // The rows are deleted in key order, peer then prefix, as
- // UpsertLiveRouteBatch writes them.
- sorted := slices.SortedFunc(slices.Values(deletions),
- func(a, b LiveRouteDeletion) int {
- return cmp.Or(cmp.Compare(a.PeerIP, b.PeerIP),
- cmp.Compare(a.Prefix, b.Prefix))
- })
-
- var deletionsV4, deletionsV6 []LiveRouteDeletion
-
- for _, del := range sorted {
- if del.IPVersion == ipVersionV4 {
- deletionsV4 = append(deletionsV4, del)
- } else {
- deletionsV6 = append(deletionsV6, del)
- }
- }
-
- ctx := context.Background()
-
- deletedV4, gonePrefixMaskLengthsV4, err := deleteRoutesInTable(ctx, tx,
- deletionsV4, ipVersionV4)
- if err != nil {
- return err
- }
-
- deletedV6, gonePrefixMaskLengthsV6, err := deleteRoutesInTable(ctx, tx,
- deletionsV6, ipVersionV6)
- if err != nil {
- return err
- }
-
- err = tx.Commit()
- if err != nil {
- return fmt.Errorf("failed to commit transaction: %w", err)
- }
-
- d.counts.addRoutes(-int(deletedV4), -int(deletedV6))
- d.counts.addToDistribution(gonePrefixMaskLengthsV4, gonePrefixMaskLengthsV6, -1)
-
- return nil
-}
-
-// peerRouteChunkSize is the most routes of a peer that one transaction of
-// DeleteLiveRoutesOfPeer deletes from each route table. A peer can have a
-// million routes; deleted in one transaction, they would hold the database lock
-// for most of a minute.
-const peerRouteChunkSize = 10000
-
-// Deleting a chunk of a peer's live routes. The first query of each table
-// selects the peer's first routes in prefix order, through the
-// UNIQUE(peer_ip, prefix) key, each with its mask length and whether another
-// peer has a live route for its prefix: the delete takes the prefixes no other
-// peer has out of the prefix distribution. The second query deletes the peer's
-// routes up to the last prefix the first one selected.
-const (
- peerRouteChunkV4SQL = `SELECT r.prefix, r.mask_length,
- EXISTS (SELECT 1 FROM live_routes_v4 AS o
- WHERE o.prefix = r.prefix AND o.peer_ip != r.peer_ip)
- FROM live_routes_v4 AS r WHERE r.peer_ip = ?
- ORDER BY r.prefix LIMIT ?`
- deletePeerRouteChunkV4SQL = `DELETE FROM live_routes_v4
- WHERE peer_ip = ? AND prefix <= ?`
-
- peerRouteChunkV6SQL = `SELECT r.prefix, r.mask_length,
- EXISTS (SELECT 1 FROM live_routes_v6 AS o
- WHERE o.prefix = r.prefix AND o.peer_ip != r.peer_ip)
- FROM live_routes_v6 AS r WHERE r.peer_ip = ?
- ORDER BY r.prefix LIMIT ?`
- deletePeerRouteChunkV6SQL = `DELETE FROM live_routes_v6
- WHERE peer_ip = ? AND prefix <= ?`
-)
-
-// deletePeerRoutes deletes the first peerRouteChunkSize live routes of peerIP,
-// or as many as it has, from one table, given that table's two statements
-// above. It returns how many routes it deleted and the mask length of each
-// prefix it left with no live route.
-func deletePeerRoutes(
- ctx context.Context, tx *loggingTx, chunkSQL, deleteSQL, peerIP string,
-) (int64, []int, error) {
- rows, err := tx.QueryContext(ctx, chunkSQL, peerIP, peerRouteChunkSize)
- if err != nil {
- return 0, nil, err
- }
- defer func() { _ = rows.Close() }()
-
- var (
- selected int
- lastPrefix string
- goneMaskLengths []int
- )
-
- for rows.Next() {
- var (
- maskLength int
- otherPeerHasARoute bool
- )
-
- err := rows.Scan(&lastPrefix, &maskLength, &otherPeerHasARoute)
- if err != nil {
- return 0, nil, err
- }
-
- selected++
-
- if !otherPeerHasARoute {
- goneMaskLengths = append(goneMaskLengths, maskLength)
- }
- }
-
- err = rows.Err()
- if err != nil {
- return 0, nil, err
- }
-
- if selected == 0 {
- return 0, nil, nil
- }
-
- res, err := tx.ExecContext(ctx, deleteSQL, peerIP, lastPrefix)
- if err != nil {
- return 0, nil, err
- }
-
- deleted, err := res.RowsAffected()
- if err != nil {
- return 0, nil, err
- }
-
- return deleted, goneMaskLengths, nil
-}
-
-// DeleteLiveRoutesOfPeer deletes every live route of the peer, IPv4 and IPv6,
-// as when the peer's BGP session goes down. It deletes them a chunk at a time
-// with deletePeerRouteChunk, each chunk in a transaction of its own, and
-// releases the database lock between chunks so other writes go on. It returns
-// how many routes it deleted; after an error, the chunks it deleted before stay
-// deleted.
-func (d *Database) DeleteLiveRoutesOfPeer(peerIP string) (int, error) {
- total := 0
-
- for {
- deletedV4, deletedV6, err := d.deletePeerRouteChunk(peerIP)
- if err != nil {
- return total, err
- }
-
- total += deletedV4 + deletedV6
-
- if deletedV4 < peerRouteChunkSize && deletedV6 < peerRouteChunkSize {
- return total, nil
- }
- }
-}
-
// insertPrefixesInTable inserts the prefixes, all of one IP version, that the
// prefix table of that version does not hold yet, each first seen at its time
// in prefixes. A prefix already stored is left as it is. It returns how many it
@@ -867,7 +331,8 @@ func insertPrefixesInTable(
var newPrefixes int
- // The rows are written in key order, as UpsertLiveRouteBatch writes routes.
+ // The rows are written in key order, so that consecutive writes land on
+ // the same index pages.
for _, prefix := range slices.Sorted(maps.Keys(prefixes)) {
res, err := insertStmt.ExecContext(ctx, prefix, prefixes[prefix])
if err != nil {
@@ -944,6 +409,26 @@ func (d *Database) UpdatePrefixesBatch(prefixes map[string]time.Time) error {
return nil
}
+// These statements update a stored AS, peering or peer. Each keeps the earlier
+// of the stored and the new first_seen, and the later of the stored and the new
+// last_seen. Some collectors send updates days old while they catch up on a
+// backlog. MIN and MAX compare the times as text, which is time order here: the
+// driver writes every time in one format, and every time is UTC.
+const (
+ updateASNSQL = `UPDATE asns
+ SET first_seen = MIN(first_seen, ?), last_seen = MAX(last_seen, ?)
+ WHERE asn = ?`
+
+ updatePeeringSQL = `UPDATE peerings
+ SET first_seen = MIN(first_seen, ?), last_seen = MAX(last_seen, ?)
+ WHERE as_a = ? AND as_b = ?`
+
+ updatePeerSQL = `UPDATE bgp_peers
+ SET peer_asn = ?, last_message_type = ?,
+ first_seen = MIN(first_seen, ?), last_seen = MAX(last_seen, ?)
+ WHERE peer_ip = ?`
+)
+
// newASN returns a new ASN record first seen at seen.First and last seen at
// seen.Last, with the handle and description from the AS info lookup when it
// knows the number.
@@ -963,9 +448,8 @@ func newASN(number int, seen SeenTimes) ASN {
return asn
}
-// upsertASN sets the last_seen time of ASN number to seen.Last with the given
-// statements, inserting the ASN when it is new. It reports whether it inserted
-// it.
+// upsertASN stores the seen times of ASN number with the given statements,
+// inserting the ASN when it is new. It reports whether it inserted it.
func upsertASN(
ctx context.Context, selectStmt, updateStmt, insertStmt *sql.Stmt,
number int, seen SeenTimes,
@@ -978,8 +462,8 @@ func upsertASN(
err := selectStmt.QueryRowContext(ctx, number).Scan(&asn.ASN, &handle,
&description, &asn.FirstSeen, &asn.LastSeen)
if err == nil {
- // ASN exists, update last_seen
- _, err = updateStmt.ExecContext(ctx, seen.Last, number)
+ // ASN exists, update its seen times
+ _, err = updateStmt.ExecContext(ctx, seen.First, seen.Last, number)
if err != nil {
return false, fmt.Errorf("failed to update ASN %d: %w", number, err)
}
@@ -1029,8 +513,7 @@ func (d *Database) GetOrCreateASNBatch(asns map[int]SeenTimes) error {
}
defer func() { _ = selectStmt.Close() }()
- updateStmt, err := tx.PrepareContext(ctx,
- "UPDATE asns SET last_seen = ? WHERE asn = ?")
+ updateStmt, err := tx.PrepareContext(ctx, updateASNSQL)
if err != nil {
return fmt.Errorf("failed to prepare update statement: %w", err)
}
@@ -1089,17 +572,19 @@ func (d *Database) GetOrCreateASN(number int, timestamp time.Time) (*ASN, error)
"FROM asns WHERE asn = ?", number).
Scan(&asn.ASN, &handle, &description, &asn.FirstSeen, &asn.LastSeen)
if err == nil {
- // ASN exists, update last_seen
+ // ASN exists, update its seen times
asn.Handle = handle.String
asn.Description = description.String
- _, err = tx.Exec("UPDATE asns SET last_seen = ? WHERE asn = ?",
- timestamp, number)
+ _, err = tx.Exec(updateASNSQL, timestamp, timestamp, number)
if err != nil {
return nil, err
}
- asn.LastSeen = timestamp
+ // Return the seen times the update stored.
+ seen := SeenTimes{First: asn.FirstSeen, Last: asn.LastSeen}
+ seen.Add(timestamp)
+ asn.FirstSeen, asn.LastSeen = seen.First, seen.Last
err = tx.Commit()
if err != nil {
@@ -1258,9 +743,7 @@ func (d *Database) RecordPeering(asA, asB int, seen SeenTimes) error {
}
if exists {
- _, err = tx.Exec(
- "UPDATE peerings SET last_seen = ? WHERE as_a = ? AND as_b = ?",
- seen.Last, asA, asB)
+ _, err = tx.Exec(updatePeeringSQL, seen.First, seen.Last, asA, asB)
} else {
_, err = tx.Exec(`
INSERT INTO peerings (id, as_a, as_b, first_seen, last_seen)
@@ -1315,9 +798,7 @@ func (d *Database) UpdatePeerBatch(peers map[string]PeerUpdate) error {
}
defer func() { _ = checkStmt.Close() }()
- updateStmt, err := tx.PrepareContext(ctx,
- "UPDATE bgp_peers SET peer_asn = ?, last_seen = ?, "+
- "last_message_type = ? WHERE peer_ip = ?")
+ updateStmt, err := tx.PrepareContext(ctx, updatePeerSQL)
if err != nil {
return fmt.Errorf("failed to prepare update statement: %w", err)
}
@@ -1343,7 +824,8 @@ func (d *Database) UpdatePeerBatch(peers map[string]PeerUpdate) error {
if exists {
_, err = updateStmt.ExecContext(ctx, update.PeerASN,
- update.Seen.Last, update.MessageType, update.PeerIP)
+ update.MessageType, update.Seen.First, update.Seen.Last,
+ update.PeerIP)
} else {
_, err = insertStmt.ExecContext(ctx,
generateUUID().String(), update.PeerIP, update.PeerASN,
@@ -1392,11 +874,8 @@ func (d *Database) UpdatePeer(
}
if exists {
- _, err = tx.Exec(
- "UPDATE bgp_peers SET peer_asn = ?, last_seen = ?, "+
- "last_message_type = ? WHERE peer_ip = ?",
- peerASN, timestamp, messageType, peerIP,
- )
+ _, err = tx.Exec(updatePeerSQL,
+ peerASN, messageType, timestamp, timestamp, peerIP)
} else {
_, err = tx.Exec(
"INSERT INTO bgp_peers (id, peer_ip, peer_asn, first_seen, "+
@@ -1435,16 +914,13 @@ func (d *Database) GetStats() (Stats, error) {
// GetStatsContext returns database statistics with context support.
//
-// The row counts (ASNs, prefixes, peerings, peers, live routes) and the prefix
-// distribution come from the in-memory counters, seeded at startup and kept
-// current on every write. The oldest/newest route timestamps are read from the
-// ends of the last_updated index, and the file size from a stat(). No part of
-// the read scans a table or a whole index.
-func (d *Database) GetStatsContext(ctx context.Context) (Stats, error) {
+// The row counts (ASNs, prefixes, peerings, peers) come from the in-memory
+// counters, seeded at startup and kept current on every write, and the file
+// size from a stat(). The read touches no table.
+func (d *Database) GetStatsContext(_ context.Context) (Stats, error) {
var stats Stats
- // Row counts and prefix distribution from memory, as a single consistent
- // snapshot.
+ // Row counts from memory, as a single consistent snapshot.
d.counts.fill(&stats)
// Database file size is a cheap stat() on the file.
@@ -1455,303 +931,14 @@ func (d *Database) GetStatsContext(ctx context.Context) (Stats, error) {
stats.FileSizeBytes = fileInfo.Size()
}
- // Oldest and newest route timestamps read one row from each end of the
- // last_updated index (a log-time lookup, not a scan). Selecting the column
- // directly lets the driver parse the DATETIME into time.Time; the old
- // MIN/MAX union scan returned an untyped string that failed to scan and
- // logged a warning on every call.
- oldest, newest, err := d.routeTimestampRange(ctx)
- if err != nil {
- // Display-only fields; log but keep the rest of the stats.
- d.logger.Warn("Failed to get route timestamps", "error", err)
- } else {
- stats.OldestRoute = oldest
- stats.NewestRoute = newest
- }
-
return stats, nil
}
-// UpsertLiveRoute inserts or updates a live route
-func (d *Database) UpsertLiveRoute(route *LiveRoute) error {
- d.lock("UpsertLiveRoute")
- defer d.unlock()
-
- pathJSON, err := json.Marshal(route.ASPath)
- if err != nil {
- return fmt.Errorf("failed to encode AS path: %w", err)
- }
-
- originSQL, updateSQL := updateLiveRouteOriginV4SQL, updateLiveRouteV4SQL
- insertSQL, lookupSQL := insertLiveRouteV4SQL, prefixHasLiveRouteV4SQL
-
- if route.IPVersion == ipVersionV6 {
- originSQL, updateSQL = updateLiveRouteOriginV6SQL, updateLiveRouteV6SQL
- insertSQL, lookupSQL = insertLiveRouteV6SQL, prefixHasLiveRouteV6SQL
- }
-
- ctx := context.Background()
-
- // The write lock is held, so no other writer can insert this key between the
- // update and the insert even though they are separate autocommit statements.
- setOrigin, err := d.writer.PrepareContext(ctx, originSQL)
- if err != nil {
- return fmt.Errorf("failed to prepare origin statement: %w", err)
- }
- defer func() { _ = setOrigin.Close() }()
-
- upd, err := d.writer.PrepareContext(ctx, updateSQL)
- if err != nil {
- return fmt.Errorf("failed to prepare update statement: %w", err)
- }
- defer func() { _ = upd.Close() }()
-
- ins, err := d.writer.PrepareContext(ctx, insertSQL)
- if err != nil {
- return fmt.Errorf("failed to prepare insert statement: %w", err)
- }
- defer func() { _ = ins.Close() }()
-
- has, err := d.writer.PrepareContext(ctx, lookupSQL)
- if err != nil {
- return fmt.Errorf("failed to prepare prefix lookup statement: %w", err)
- }
- defer func() { _ = has.Close() }()
-
- inserted, newPrefix, err := upsertRouteRow(ctx, setOrigin, upd, ins, has,
- route, string(pathJSON))
- if err != nil {
- return fmt.Errorf("failed to upsert route %s: %w", route.Prefix, err)
- }
-
- if inserted {
- if route.IPVersion == ipVersionV4 {
- d.counts.addRoutes(1, 0)
- } else {
- d.counts.addRoutes(0, 1)
- }
- }
-
- if newPrefix {
- if route.IPVersion == ipVersionV4 {
- d.counts.addToDistribution([]int{route.MaskLength}, nil, 1)
- } else {
- d.counts.addToDistribution(nil, []int{route.MaskLength}, 1)
- }
- }
-
- return nil
-}
-
-// DeleteLiveRoute deletes the live route the peer has for the prefix
-func (d *Database) DeleteLiveRoute(prefix, peerIP string) error {
- d.lock("DeleteLiveRoute")
- defer d.unlock()
-
- // Determine table based on prefix IP version
- _, ipnet, err := net.ParseCIDR(prefix)
- if err != nil {
- return fmt.Errorf("invalid prefix format: %w", err)
- }
-
- isV4 := ipnet.IP.To4() != nil
-
- // The delete and the prefix lookup after it share one transaction, so the
- // in-memory counts change only once both have succeeded.
- tx, err := d.beginTx()
- if err != nil {
- return fmt.Errorf("failed to begin transaction: %w", err)
- }
- defer d.rollback(tx)
-
- // Literal per-table queries (rather than one formatted with the table name)
- // so the delete carries no dynamically built SQL.
- deleteSQL := deleteLiveRouteV6SQL
- if isV4 {
- deleteSQL = deleteLiveRouteV4SQL
- }
-
- res, err := tx.Exec(deleteSQL, prefix, peerIP)
- if err != nil {
- return err
- }
-
- affected, err := res.RowsAffected()
- if err != nil {
- return err
- }
-
- if affected == 0 {
- return nil
- }
-
- // The prefix leaves the distribution when no live route has it any more.
- lookupSQL := prefixHasLiveRouteV6SQL
- if isV4 {
- lookupSQL = prefixHasLiveRouteV4SQL
- }
-
- var prefixHasRoute bool
-
- err = tx.QueryRow(lookupSQL, prefix).Scan(&prefixHasRoute)
- if err != nil {
- return err
- }
-
- err = tx.Commit()
- if err != nil {
- return fmt.Errorf("failed to commit transaction: %w", err)
- }
-
- if isV4 {
- d.counts.addRoutes(-int(affected), 0)
- } else {
- d.counts.addRoutes(0, -int(affected))
- }
-
- if !prefixHasRoute {
- maskLength, _ := ipnet.Mask.Size()
- if isV4 {
- d.counts.addToDistribution([]int{maskLength}, nil, -1)
- } else {
- d.counts.addToDistribution(nil, []int{maskLength}, -1)
- }
- }
-
- return nil
-}
-
-// GetPrefixDistribution returns the distribution of unique prefixes by mask
-// length
-func (d *Database) GetPrefixDistribution() (
- []PrefixDistribution, []PrefixDistribution, error,
-) {
- return d.GetPrefixDistributionContext(context.Background())
-}
-
-// GetPrefixDistributionContext returns the distribution of unique prefixes by
-// mask length with context support. It reads every live route, so the stats
-// read does not call it; it seeds the in-memory distribution once at startup.
-func (d *Database) GetPrefixDistributionContext(ctx context.Context) (
- []PrefixDistribution, []PrefixDistribution, error,
-) {
- var ipv4, ipv6 []PrefixDistribution
-
- // IPv4 distribution - count unique prefixes from v4 table
- query := `
- SELECT mask_length, COUNT(DISTINCT prefix) as count
- FROM live_routes_v4
- GROUP BY mask_length
- ORDER BY mask_length
- `
-
- rows4, err := d.db.QueryContext(ctx, query)
- if err != nil {
- return nil, nil, fmt.Errorf("failed to query IPv4 distribution: %w", err)
- }
- defer func() {
- if rows4 != nil {
- _ = rows4.Close()
- }
- }()
-
- for rows4.Next() {
- var dist PrefixDistribution
-
- err := rows4.Scan(&dist.MaskLength, &dist.Count)
- if err != nil {
- return nil, nil, fmt.Errorf("failed to scan IPv4 distribution: %w",
- err)
- }
-
- ipv4 = append(ipv4, dist)
- }
-
- // A read that stops partway ends the loop without an error of its own.
- err = rows4.Err()
- if err != nil {
- return nil, nil, fmt.Errorf("failed to read IPv4 distribution: %w", err)
- }
-
- // IPv6 distribution - count unique prefixes from v6 table
- query = `
- SELECT mask_length, COUNT(DISTINCT prefix) as count
- FROM live_routes_v6
- GROUP BY mask_length
- ORDER BY mask_length
- `
-
- rows6, err := d.db.QueryContext(ctx, query)
- if err != nil {
- return nil, nil, fmt.Errorf("failed to query IPv6 distribution: %w", err)
- }
- defer func() {
- if rows6 != nil {
- _ = rows6.Close()
- }
- }()
-
- for rows6.Next() {
- var dist PrefixDistribution
-
- err := rows6.Scan(&dist.MaskLength, &dist.Count)
- if err != nil {
- return nil, nil, fmt.Errorf("failed to scan IPv6 distribution: %w",
- err)
- }
-
- ipv6 = append(ipv6, dist)
- }
-
- err = rows6.Err()
- if err != nil {
- return nil, nil, fmt.Errorf("failed to read IPv6 distribution: %w", err)
- }
-
- return ipv4, ipv6, nil
-}
-
-// GetLiveRouteCounts returns the count of IPv4 and IPv6 routes
-func (d *Database) GetLiveRouteCounts() (int, int, error) {
- return d.GetLiveRouteCountsContext(context.Background())
-}
-
-// GetLiveRouteCountsContext returns the count of IPv4 and IPv6 routes with
-// context support
-func (d *Database) GetLiveRouteCountsContext(
- ctx context.Context,
-) (int, int, error) {
- var ipv4Count, ipv6Count int
-
- // Get IPv4 count from dedicated table
- err := d.db.QueryRowContext(ctx, "SELECT COUNT(*) FROM live_routes_v4").
- Scan(&ipv4Count)
- if err != nil {
- return 0, 0, fmt.Errorf("failed to count IPv4 routes: %w", err)
- }
-
- // Get IPv6 count from dedicated table
- err = d.db.QueryRowContext(ctx, "SELECT COUNT(*) FROM live_routes_v6").
- Scan(&ipv6Count)
- if err != nil {
- return 0, 0, fmt.Errorf("failed to count IPv6 routes: %w", err)
- }
-
- return ipv4Count, ipv6Count, nil
-}
-
-// GetASDetails returns detailed information about an AS including prefixes
-func (d *Database) GetASDetails(asn int) (*ASN, []LiveRoute, error) {
- return d.GetASDetailsContext(context.Background(), asn)
-}
-
-// GetASDetailsContext returns the AS and the prefixes it originates, the IPv4
-// ones first, each once with the time of its newest route. It makes two index
-// searches per prefix, however many peers carry the prefix.
-func (d *Database) GetASDetailsContext(
- ctx context.Context, asn int,
-) (*ASN, []LiveRoute, error) {
- // Get AS information
+// GetAS returns the AS's handle, description and first and last seen times
+// from the asns table. When the table has no record of the AS, its handle and
+// description come from the AS info lookup, and it has no seen times. The bool
+// says whether the asns table or the AS info lookup holds the AS.
+func (d *Database) GetAS(ctx context.Context, asn int) (*ASN, bool, error) {
var (
asnInfo ASN
handle, description sql.NullString
@@ -1763,28 +950,107 @@ func (d *Database) GetASDetailsContext(
asn,
).Scan(&asnInfo.ASN, &handle, &description, &asnInfo.FirstSeen,
&asnInfo.LastSeen)
- if err != nil {
- if errors.Is(err, sql.ErrNoRows) {
- return nil, nil, fmt.Errorf("%w: AS%d", ErrNoRoute, asn)
+
+ switch {
+ case errors.Is(err, sql.ErrNoRows):
+ asnInfo.ASN = asn
+
+ info, ok := asinfo.Get(asn)
+ if ok {
+ asnInfo.Handle = info.Handle
+ asnInfo.Description = info.Description
}
- return nil, nil, fmt.Errorf("failed to query AS: %w", err)
+ return &asnInfo, ok, nil
+ case err != nil:
+ return nil, false, fmt.Errorf("failed to query AS: %w", err)
}
asnInfo.Handle = handle.String
asnInfo.Description = description.String
- allPrefixes, err := d.originPrefixes(ctx, asn, ipVersionV4)
- if err != nil {
- return &asnInfo, nil, err
+ return &asnInfo, true, nil
+}
+
+// asWHOISQuery reads the handle, description and WHOIS details of one AS, and
+// when the WHOIS details were fetched.
+const asWHOISQuery = `
+ SELECT handle, description, as_name, org_name, org_id, address,
+ country_code, abuse_email, rir, whois_updated_at
+ FROM asns WHERE asn = ?`
+
+// GetASWHOIS returns the handle, description and WHOIS details that the asns
+// table holds for the AS, and when the WHOIS details were fetched. When the
+// table has no record of the AS, it returns an ASN with only its number.
+func (d *Database) GetASWHOIS(ctx context.Context, asn int) (*ASN, error) {
+ var (
+ handle, description, asName, orgName, orgID sql.NullString
+ address, countryCode, abuseEmail, rir sql.NullString
+ whoisUpdatedAt sql.NullTime
+ )
+
+ err := d.db.QueryRowContext(ctx, asWHOISQuery, asn).Scan(
+ &handle, &description, &asName, &orgName, &orgID, &address,
+ &countryCode, &abuseEmail, &rir, &whoisUpdatedAt,
+ )
+ if errors.Is(err, sql.ErrNoRows) {
+ return &ASN{ASN: asn}, nil
}
- prefixesV6, err := d.originPrefixes(ctx, asn, ipVersionV6)
if err != nil {
- return &asnInfo, allPrefixes, err
+ return nil, fmt.Errorf("failed to query AS: %w", err)
}
- return &asnInfo, append(allPrefixes, prefixesV6...), nil
+ info := &ASN{
+ ASN: asn,
+ Handle: handle.String,
+ Description: description.String,
+ ASName: asName.String,
+ OrgName: orgName.String,
+ OrgID: orgID.String,
+ Address: address.String,
+ CountryCode: countryCode.String,
+ AbuseEmail: abuseEmail.String,
+ RIR: rir.String,
+ }
+
+ if whoisUpdatedAt.Valid {
+ info.WHOISUpdatedAt = &whoisUpdatedAt.Time
+ }
+
+ return info, nil
+}
+
+// The queries GetPrefixFirstSeen reads an IPv4 and an IPv6 prefix's first-seen
+// time with.
+const (
+ prefixFirstSeenV4SQL = `SELECT first_seen FROM prefixes_v4 WHERE prefix = ?`
+ prefixFirstSeenV6SQL = `SELECT first_seen FROM prefixes_v6 WHERE prefix = ?`
+)
+
+// GetPrefixFirstSeen returns when the prefix tables first saw prefix, or the
+// zero time when they have no record of it.
+func (d *Database) GetPrefixFirstSeen(
+ ctx context.Context, prefix netip.Prefix,
+) (time.Time, error) {
+ query := prefixFirstSeenV6SQL
+ if prefix.Addr().Is4() {
+ query = prefixFirstSeenV4SQL
+ }
+
+ var firstSeen time.Time
+
+ err := d.db.QueryRowContext(ctx, query, prefix.Masked().String()).
+ Scan(&firstSeen)
+ if errors.Is(err, sql.ErrNoRows) {
+ return time.Time{}, nil
+ }
+
+ if err != nil {
+ return time.Time{}, fmt.Errorf("failed to query prefix: %w", err)
+ }
+
+ return firstSeen, nil
}
// ASPeer represents a peering relationship with another AS including handle,
@@ -1858,144 +1124,6 @@ func (d *Database) GetASPeersContext(
return peers, nil
}
-// GetPrefixDetails returns detailed information about a prefix
-func (d *Database) GetPrefixDetails(
- prefix string,
-) ([]LiveRoute, map[int]ASN, error) {
- return d.GetPrefixDetailsContext(context.Background(), prefix)
-}
-
-// GetPrefixDetailsContext returns the live routes for a prefix and, keyed by
-// AS number, the handle and description of each AS that originates it.
-func (d *Database) GetPrefixDetailsContext(
- ctx context.Context, prefix string,
-) ([]LiveRoute, map[int]ASN, error) {
- // Determine if it's IPv4 or IPv6 by parsing the prefix
- _, ipnet, err := net.ParseCIDR(prefix)
- if err != nil {
- return nil, nil, fmt.Errorf("invalid prefix format: %w", err)
- }
-
- tableName := tableLiveRoutesV4
- ipVersion := ipVersionV4
-
- if ipnet.IP.To4() == nil {
- tableName = tableLiveRoutesV6
- ipVersion = ipVersionV6
- }
-
- query := fmt.Sprintf(`
- SELECT lr.origin_asn, lr.peer_ip, lr.as_path, lr.next_hop,
- lr.last_updated, a.handle, a.description
- FROM %s lr
- LEFT JOIN asns a ON a.asn = lr.origin_asn
- WHERE lr.prefix = ?
- ORDER BY lr.origin_asn, lr.peer_ip
- `, tableName)
-
- rows, err := d.db.QueryContext(ctx, query, prefix)
- if err != nil {
- return nil, nil, fmt.Errorf("failed to query prefix details: %w", err)
- }
- defer func() { _ = rows.Close() }()
-
- var routes []LiveRoute
-
- origins := make(map[int]ASN)
-
- for rows.Next() {
- var (
- route LiveRoute
- pathJSON string
- handle, description sql.NullString
- )
-
- err := rows.Scan(
- &route.OriginASN, &route.PeerIP, &pathJSON, &route.NextHop,
- &route.LastUpdated, &handle, &description,
- )
- if err != nil {
- continue
- }
-
- // Decode AS path
- err = json.Unmarshal([]byte(pathJSON), &route.ASPath)
- if err != nil {
- route.ASPath = []int{}
- }
-
- route.Prefix = prefix
- route.IPVersion = ipVersion
- route.MaskLength, _ = ipnet.Mask.Size()
- routes = append(routes, route)
- origins[route.OriginASN] = ASN{
- ASN: route.OriginASN,
- Handle: handle.String,
- Description: description.String,
- }
- }
-
- err = rows.Err()
- if err != nil {
- return nil, nil, fmt.Errorf("failed to read prefix details: %w", err)
- }
-
- if len(routes) == 0 {
- return nil, nil, fmt.Errorf("%w: %s", ErrNoRoute, prefix)
- }
-
- return routes, origins, nil
-}
-
-// GetRandomPrefixesByLength returns a random sample of prefixes with the
-// specified mask length
-func (d *Database) GetRandomPrefixesByLength(
- maskLength, ipVersion, limit int,
-) ([]LiveRoute, error) {
- return d.GetRandomPrefixesByLengthContext(context.Background(),
- maskLength, ipVersion, limit)
-}
-
-// GetRandomPrefixesByLengthContext returns limit prefixes with the given mask
-// length picked at random, or all of them when there are fewer, each once with
-// its most recently updated live route, for a limit up to
-// maxPrefixesToShuffle. However many routes there are, it makes at most
-// maxPrefixesToShuffle+1 index searches to list prefixes, at most
-// sampleLookupsPerPrefix random lookups for each prefix asked for, and one
-// index search for each listed prefix's newest route.
-func (d *Database) GetRandomPrefixesByLengthContext(
- ctx context.Context, maskLength, ipVersion, limit int) ([]LiveRoute, error) {
- tableName := tableLiveRoutesV4
- if ipVersion == ipVersionV6 {
- tableName = tableLiveRoutesV6
- }
-
- // Reading one prefix more than maxPrefixesToShuffle tells whether these
- // are all the prefixes with this mask length.
- prefixes, err := d.firstPrefixesByLength(ctx, tableName, maskLength,
- maxPrefixesToShuffle+1)
- if err != nil {
- return nil, err
- }
-
- //nolint:gosec // The sample needs variety, not unpredictability.
- rand.Shuffle(len(prefixes), func(i, j int) {
- prefixes[i], prefixes[j] = prefixes[j], prefixes[i]
- })
-
- if len(prefixes) > maxPrefixesToShuffle {
- prefixes, err = d.samplePrefixesByLength(ctx, tableName, maskLength,
- limit, prefixes)
- if err != nil {
- return nil, err
- }
- }
-
- prefixes = prefixes[:min(limit, len(prefixes))]
-
- return d.newestRouteOfEachPrefix(ctx, tableName, ipVersion, prefixes)
-}
-
// GetNextStaleASN returns a random ASN that needs WHOIS data refresh.
func (d *Database) GetNextStaleASN(
ctx context.Context, staleThreshold time.Duration,
@@ -2118,136 +1246,6 @@ func (d *Database) UpdateASNWHOIS(
return nil
}
-// GetIPInfo returns comprehensive IP information for the /ip endpoint.
-func (d *Database) GetIPInfo(ip string) (*IPInfo, error) {
- return d.GetIPInfoContext(context.Background(), ip)
-}
-
-// ipInfoQuery reads the live route for one prefix with its peer count, the
-// prefix's first_seen and the origin AS's details. %[1]s is the live route
-// table and %[2]s the prefix table of the address's IP version.
-const ipInfoQuery = `
- SELECT
- lr.prefix,
- lr.mask_length,
- lr.origin_asn,
- lr.last_updated,
- (SELECT COUNT(DISTINCT peer_ip) FROM %[1]s
- WHERE prefix = lr.prefix) as num_peers,
- p.first_seen,
- a.handle,
- a.description,
- a.as_name,
- a.org_name,
- a.org_id,
- a.address,
- a.country_code,
- a.abuse_email,
- a.rir,
- a.whois_updated_at
- FROM %[1]s lr
- LEFT JOIN %[2]s p ON p.prefix = lr.prefix
- LEFT JOIN asns a ON a.asn = lr.origin_asn
- WHERE lr.prefix = ?
- LIMIT 1
- `
-
-// GetIPInfoContext returns comprehensive IP information with context support.
-//
-// It finds the most specific live route for the address with point lookups on
-// the prefix index: for each mask length from the longest (32 or 128) down to
-// minLookupMaskLengthV4 or minLookupMaskLengthV6 it looks up the address's
-// network prefix at that length, and the first length with a live route wins.
-// That is at most 25 or 113 indexed lookups, however many routes are live. The
-// prefix is looked up in the text form net/netip prints, which is the form the
-// prefix handler stores.
-func (d *Database) GetIPInfoContext(
- ctx context.Context, ip string,
-) (*IPInfo, error) {
- addr, err := netip.ParseAddr(ip)
- if err != nil {
- return nil, fmt.Errorf("%w: %s", ErrInvalidIP, ip)
- }
- // An IPv4 address written in IPv6 form (::ffff:192.0.2.1) is an IPv4 lookup.
- addr = addr.Unmap()
-
- info := &IPInfo{IP: ip, IPVersion: ipVersionV6}
- minMaskLength := minLookupMaskLengthV6
-
- routesTable, prefixesTable := tableLiveRoutesV6, "prefixes_v6"
- if addr.Is4() {
- routesTable, prefixesTable = tableLiveRoutesV4, "prefixes_v4"
- minMaskLength = minLookupMaskLengthV4
- info.IPVersion = ipVersionV4
- }
-
- // Route info with peer count and prefix first_seen
- query := fmt.Sprintf(ipInfoQuery, routesTable, prefixesTable)
-
- stmt, err := d.db.PrepareContext(ctx, query)
- if err != nil {
- return nil, fmt.Errorf("failed to prepare route query: %w", err)
- }
- defer func() { _ = stmt.Close() }()
-
- var (
- handle, description, asName, orgName, orgID sql.NullString
- address, countryCode, abuseEmail, rir sql.NullString
- prefixFirstSeen, whoisUpdatedAt sql.NullTime
- )
-
- found := false
-
- for bits := addr.BitLen(); bits >= minMaskLength; bits-- {
- prefix := netip.PrefixFrom(addr, bits).Masked().String()
-
- err := stmt.QueryRowContext(ctx, prefix).Scan(
- &info.Netblock, &info.MaskLength, &info.ASN, &info.LastSeen,
- &info.NumPeers, &prefixFirstSeen, &handle, &description,
- &asName, &orgName, &orgID, &address, &countryCode, &abuseEmail,
- &rir, &whoisUpdatedAt,
- )
- if errors.Is(err, sql.ErrNoRows) {
- continue
- }
-
- if err != nil {
- return nil, fmt.Errorf("failed to query routes: %w", err)
- }
-
- found = true
-
- break
- }
-
- if !found {
- return nil, fmt.Errorf("%w for IP %s", ErrNoRoute, ip)
- }
-
- info.Handle = handle.String
- info.Description = description.String
- info.ASName = asName.String
- info.OrgName = orgName.String
- info.OrgID = orgID.String
- info.Address = address.String
- info.CountryCode = countryCode.String
- info.AbuseEmail = abuseEmail.String
- info.RIR = rir.String
-
- if prefixFirstSeen.Valid {
- info.FirstSeen = prefixFirstSeen.Time
- }
-
- // Check if WHOIS data needs refresh (never fetched or older than 30 days).
- // A route with origin 0 has no known origin, so no AS to look up.
- const staleThreshold = 30 * 24 * time.Hour
-
- info.NeedsWHOISRefresh = info.ASN != 0 && (!whoisUpdatedAt.Valid ||
- time.Since(whoisUpdatedAt.Time) > staleThreshold)
-
- return info, nil
-}
-
// Vacuum runs incremental vacuum to return up to 1000 unused pages to the
// filesystem. Like Analyze and Checkpoint, it holds the database lock, so it
// and the batch writes wait for each other instead of failing on SQLite's
@@ -2479,391 +1477,3 @@ func (d *Database) rollback(tx *loggingTx) {
d.logger.Error("Failed to rollback transaction", "error", err)
}
}
-
-// deletePeerRouteChunk deletes, in one transaction, the first
-// peerRouteChunkSize IPv4 and the first peerRouteChunkSize IPv6 live routes of
-// the peer, or as many as it has. It returns how many of each it deleted.
-func (d *Database) deletePeerRouteChunk(peerIP string) (int, int, error) {
- d.lock("DeleteLiveRoutesOfPeer")
- defer d.unlock()
-
- tx, err := d.beginTx()
- if err != nil {
- return 0, 0, fmt.Errorf("failed to begin transaction: %w", err)
- }
- defer d.rollback(tx)
-
- ctx := context.Background()
-
- deletedV4, goneMaskLengthsV4, err := deletePeerRoutes(ctx, tx,
- peerRouteChunkV4SQL, deletePeerRouteChunkV4SQL, peerIP)
- if err != nil {
- return 0, 0, fmt.Errorf("failed to delete the IPv4 routes of peer %s: %w",
- peerIP, err)
- }
-
- deletedV6, goneMaskLengthsV6, err := deletePeerRoutes(ctx, tx,
- peerRouteChunkV6SQL, deletePeerRouteChunkV6SQL, peerIP)
- if err != nil {
- return 0, 0, fmt.Errorf("failed to delete the IPv6 routes of peer %s: %w",
- peerIP, err)
- }
-
- err = tx.Commit()
- if err != nil {
- return 0, 0, fmt.Errorf("failed to commit transaction: %w", err)
- }
-
- d.counts.addRoutes(-int(deletedV4), -int(deletedV6))
- d.counts.addToDistribution(goneMaskLengthsV4, goneMaskLengthsV6, -1)
-
- return int(deletedV4), int(deletedV6), nil
-}
-
-// routeTimestampRange returns the earliest and latest last_updated across both
-// live route tables, or nil values when both tables are empty. Each query reads
-// one row from an end of the last_updated index rather than scanning the tables.
-func (d *Database) routeTimestampRange(
- ctx context.Context,
-) (*time.Time, *time.Time, error) {
- var oldest, newest *time.Time
-
- oldestV4, ok, err := d.scanRouteTimestamp(ctx, "SELECT last_updated "+
- "FROM live_routes_v4 ORDER BY last_updated ASC LIMIT 1")
- if err != nil {
- return nil, nil, err
- }
-
- if ok {
- oldest = &oldestV4
- }
-
- oldestV6, ok, err := d.scanRouteTimestamp(ctx, "SELECT last_updated "+
- "FROM live_routes_v6 ORDER BY last_updated ASC LIMIT 1")
- if err != nil {
- return nil, nil, err
- }
-
- if ok && (oldest == nil || oldestV6.Before(*oldest)) {
- oldest = &oldestV6
- }
-
- newestV4, ok, err := d.scanRouteTimestamp(ctx, "SELECT last_updated "+
- "FROM live_routes_v4 ORDER BY last_updated DESC LIMIT 1")
- if err != nil {
- return nil, nil, err
- }
-
- if ok {
- newest = &newestV4
- }
-
- newestV6, ok, err := d.scanRouteTimestamp(ctx, "SELECT last_updated "+
- "FROM live_routes_v6 ORDER BY last_updated DESC LIMIT 1")
- if err != nil {
- return nil, nil, err
- }
-
- if ok && (newest == nil || newestV6.After(*newest)) {
- newest = &newestV6
- }
-
- return oldest, newest, nil
-}
-
-// scanRouteTimestamp runs a single-row timestamp query. The bool is false when
-// the table is empty. The query selects the last_updated column directly so the
-// driver parses the DATETIME value into a time.Time.
-func (d *Database) scanRouteTimestamp(
- ctx context.Context, query string,
-) (time.Time, bool, error) {
- var ts time.Time
-
- err := d.db.QueryRowContext(ctx, query).Scan(&ts)
- switch {
- case errors.Is(err, sql.ErrNoRows):
- return time.Time{}, false, nil
- case err != nil:
- return time.Time{}, false, err
- default:
- return ts, true, nil
- }
-}
-
-// The AS page lists the prefixes an AS originates one at a time, each in two
-// index searches that read no table row: one of the origin_asn and prefix
-// index for the AS's first prefix after the previous one, and one of the
-// prefix and last_updated index for the time of its newest route. Neither
-// reads more for a prefix that more peers carry.
-const (
- nextOriginPrefixV4SQL = `SELECT prefix FROM live_routes_v4
- WHERE origin_asn = ? AND prefix > ? ORDER BY prefix LIMIT 1`
- newestRouteTimeV4SQL = `SELECT last_updated FROM live_routes_v4
- WHERE prefix = ? ORDER BY last_updated DESC LIMIT 1`
-
- nextOriginPrefixV6SQL = `SELECT prefix FROM live_routes_v6
- WHERE origin_asn = ? AND prefix > ? ORDER BY prefix LIMIT 1`
- newestRouteTimeV6SQL = `SELECT last_updated FROM live_routes_v6
- WHERE prefix = ? ORDER BY last_updated DESC LIMIT 1`
-)
-
-// originPrefixes returns every prefix asn originates in the live route table
-// of ipVersion, in text order, each with its mask length and the time of the
-// prefix's newest route, whichever AS that route names as origin. A prefix
-// whose routes were all withdrawn between the two searches is left out.
-func (d *Database) originPrefixes(
- ctx context.Context, asn, ipVersion int,
-) ([]LiveRoute, error) {
- family, nextSQL, newestSQL := ipv4Family, nextOriginPrefixV4SQL,
- newestRouteTimeV4SQL
- if ipVersion == ipVersionV6 {
- family, nextSQL, newestSQL = ipv6Family, nextOriginPrefixV6SQL,
- newestRouteTimeV6SQL
- }
-
- nextStmt, err := d.db.PrepareContext(ctx, nextSQL)
- if err != nil {
- return nil, fmt.Errorf("failed to prepare %s prefix query: %w",
- family, err)
- }
- defer func() { _ = nextStmt.Close() }()
-
- newestStmt, err := d.db.PrepareContext(ctx, newestSQL)
- if err != nil {
- return nil, fmt.Errorf("failed to prepare %s route time query: %w",
- family, err)
- }
- defer func() { _ = newestStmt.Close() }()
-
- var routes []LiveRoute
-
- previous := "" // every prefix sorts after the empty string
-
- for {
- var prefix string
-
- err := nextStmt.QueryRowContext(ctx, asn, previous).Scan(&prefix)
- if errors.Is(err, sql.ErrNoRows) {
- return routes, nil
- }
-
- if err != nil {
- return nil, fmt.Errorf("failed to list %s prefixes: %w", family, err)
- }
-
- previous = prefix
- route := LiveRoute{Prefix: prefix, IPVersion: ipVersion, OriginASN: asn}
-
- err = newestStmt.QueryRowContext(ctx, prefix).Scan(&route.LastUpdated)
- if errors.Is(err, sql.ErrNoRows) {
- continue
- }
-
- if err != nil {
- return nil, fmt.Errorf("failed to query route time of %s: %w",
- prefix, err)
- }
-
- route.MaskLength, err = prefixMaskLength(prefix)
- if err != nil {
- return nil, err
- }
-
- routes = append(routes, route)
- }
-}
-
-// firstPrefixesByLength returns the first n prefixes, in text order, with the
-// given mask length. Each prefix is one search of the mask length and prefix
-// index for the prefix after the previous one, so the read does not grow with
-// the number of peers that carry a prefix, as a SELECT DISTINCT would.
-func (d *Database) firstPrefixesByLength(
- ctx context.Context, tableName string, maskLength, n int) ([]string, error) {
- //nolint:gosec // Table name is hardcoded based on IP version
- query := fmt.Sprintf(`
- SELECT prefix FROM %s
- WHERE mask_length = ? AND prefix > ?
- ORDER BY prefix
- LIMIT 1
- `, tableName)
-
- stmt, err := d.db.PrepareContext(ctx, query)
- if err != nil {
- return nil, fmt.Errorf("failed to prepare prefix query: %w", err)
- }
- defer func() { _ = stmt.Close() }()
-
- var prefixes []string
-
- previous := "" // every prefix sorts after the empty string
-
- for len(prefixes) < n {
- var prefix string
-
- err := stmt.QueryRowContext(ctx, maskLength, previous).Scan(&prefix)
- if errors.Is(err, sql.ErrNoRows) {
- break
- }
-
- if err != nil {
- return nil, fmt.Errorf("failed to list prefixes: %w", err)
- }
-
- prefixes = append(prefixes, prefix)
- previous = prefix
- }
-
- return prefixes, nil
-}
-
-// samplePrefixesByLength picks limit distinct prefixes with the given mask
-// length at random, in at most sampleLookupsPerPrefix lookups for each prefix
-// asked for. ORDER BY RANDOM() would read every route with that mask length.
-// Instead each lookup draws a row id between those of the first and last route
-// with that mask length and takes the prefix of the first such route at or
-// after it. SQLite keeps the row id at the end of every index entry, so the
-// mask length index is ordered by mask length, then row id, and the lookup is
-// one search of it.
-//
-// The sample is uneven, as the page needs a varied sample, not an even one: a
-// route is picked in proportion to the row ids between it and the previous
-// route with that mask length, including the row ids of routes with other mask
-// lengths and of withdrawn routes. A route that follows a long run of
-// withdrawn routes is picked far more often than one that directly follows
-// another route with that mask length.
-//
-// Should the lookups keep landing on prefixes already picked, the rest are
-// taken from shuffled, prefixes of that length in random order.
-func (d *Database) samplePrefixesByLength(
- ctx context.Context, tableName string, maskLength, limit int,
- shuffled []string,
-) ([]string, error) {
- //nolint:gosec // Table name is hardcoded based on IP version
- rangeQuery := fmt.Sprintf(`
- SELECT
- (SELECT MIN(rowid) FROM %[1]s WHERE mask_length = ?),
- (SELECT MAX(rowid) FROM %[1]s WHERE mask_length = ?)
- `, tableName)
-
- var first, last sql.NullInt64
-
- err := d.db.QueryRowContext(ctx, rangeQuery, maskLength, maskLength).
- Scan(&first, &last)
- if err != nil {
- return nil, fmt.Errorf("failed to read row id range: %w", err)
- }
-
- if !first.Valid {
- // Every route with this mask length was withdrawn since they were listed.
- return nil, nil
- }
-
- //nolint:gosec // Table name is hardcoded based on IP version
- lookupQuery := fmt.Sprintf(`
- SELECT prefix FROM %s
- WHERE mask_length = ? AND rowid >= ?
- ORDER BY rowid
- LIMIT 1
- `, tableName)
-
- stmt, err := d.db.PrepareContext(ctx, lookupQuery)
- if err != nil {
- return nil, fmt.Errorf("failed to prepare prefix lookup: %w", err)
- }
- defer func() { _ = stmt.Close() }()
-
- picked := make(map[string]bool, limit)
- prefixes := make([]string, 0, limit)
- pick := func(prefix string) {
- if !picked[prefix] {
- picked[prefix] = true
- prefixes = append(prefixes, prefix)
- }
- }
-
- maxLookups := limit * sampleLookupsPerPrefix
- for lookups := 0; lookups < maxLookups && len(prefixes) < limit; lookups++ {
- //nolint:gosec // The sample needs variety, not unpredictability.
- rowID := first.Int64 + rand.Int64N(last.Int64-first.Int64+1)
-
- var prefix string
-
- err := stmt.QueryRowContext(ctx, maskLength, rowID).Scan(&prefix)
- if errors.Is(err, sql.ErrNoRows) {
- // The routes from rowID on were withdrawn since the range was read.
- continue
- }
-
- if err != nil {
- return nil, fmt.Errorf("failed to look up prefix: %w", err)
- }
-
- pick(prefix)
- }
-
- for _, prefix := range shuffled {
- if len(prefixes) == limit {
- break
- }
-
- pick(prefix)
- }
-
- return prefixes, nil
-}
-
-// newestRouteOfEachPrefix returns the most recently updated live route of each
-// prefix, one search of the prefix and last_updated index per prefix however
-// many peers carry it. A prefix whose routes were all withdrawn since it was
-// picked is left out.
-func (d *Database) newestRouteOfEachPrefix(
- ctx context.Context, tableName string, ipVersion int, prefixes []string,
-) ([]LiveRoute, error) {
- //nolint:gosec // Table name is hardcoded based on IP version
- query := fmt.Sprintf(`
- SELECT prefix, mask_length, origin_asn, as_path, peer_ip, last_updated
- FROM %s
- WHERE prefix = ?
- ORDER BY last_updated DESC
- LIMIT 1
- `, tableName)
-
- stmt, err := d.db.PrepareContext(ctx, query)
- if err != nil {
- return nil, fmt.Errorf("failed to prepare route query: %w", err)
- }
- defer func() { _ = stmt.Close() }()
-
- routes := make([]LiveRoute, 0, len(prefixes))
-
- for _, prefix := range prefixes {
- route := LiveRoute{IPVersion: ipVersion}
-
- var pathJSON string
-
- err := stmt.QueryRowContext(ctx, prefix).Scan(
- &route.Prefix,
- &route.MaskLength,
- &route.OriginASN,
- &pathJSON,
- &route.PeerIP,
- &route.LastUpdated,
- )
- if errors.Is(err, sql.ErrNoRows) {
- continue
- }
-
- if err != nil {
- return nil, fmt.Errorf("failed to query route of %s: %w", prefix, err)
- }
-
- err = json.Unmarshal([]byte(pathJSON), &route.ASPath)
- if err != nil {
- route.ASPath = []int{}
- }
-
- routes = append(routes, route)
- }
-
- return routes, nil
-}
diff --git a/internal/database/database_internal_test.go b/internal/database/database_internal_test.go
index 2f8f242..3195ead 100644
--- a/internal/database/database_internal_test.go
+++ b/internal/database/database_internal_test.go
@@ -8,19 +8,17 @@ import (
"errors"
"fmt"
"log/slog"
- "math/rand/v2"
"net/netip"
"os"
- "slices"
- "strconv"
- "strings"
"sync"
"testing"
"time"
"git.eeqj.de/sneak/routewatch/internal/config"
"git.eeqj.de/sneak/routewatch/internal/logger"
- "github.com/mattn/go-sqlite3"
+ "git.eeqj.de/sneak/routewatch/internal/ristypes"
+ "git.eeqj.de/sneak/routewatch/internal/routetable"
+ "git.eeqj.de/sneak/routewatch/pkg/asinfo"
)
// tempStoreMemory is the PRAGMA temp_store value meaning "hold temp B-trees in
@@ -30,8 +28,8 @@ const tempStoreMemory = 2
// autoVacuumIncremental is the PRAGMA auto_vacuum value meaning "incremental".
const autoVacuumIncremental = 2
-// vacuumTestRoutes is how many routes the vacuum test writes and then deletes,
-// enough to leave many free pages in the file.
+// vacuumTestRoutes is how many rows of the routes table the vacuum test writes
+// and then deletes, enough to leave many free pages in the file.
const vacuumTestRoutes = 2000
// heldConnections is how many pooled connections the pragma test holds open at
@@ -41,7 +39,7 @@ const heldConnections = 5
// bytesPerKiB turns a cache_size in KiB into bytes.
const bytesPerKiB = 1024
-// testPeerIP is the peer every route in the IP lookup test is learned from.
+// testPeerIP is the address of the BGP peer the peer write tests store.
const testPeerIP = "192.0.2.254"
// Parameters for the checkpoint-contention regression test.
@@ -71,6 +69,9 @@ const (
lockWaitTimeout = time.Millisecond
// lockTestASN is the AS the batch write in the lock tests stores.
lockTestASN = 64500
+ // lockTestPrefix is the prefix the batch writes in the checkpoint tests
+ // store.
+ lockTestPrefix = "192.0.2.0/24"
// readerCheckpointLimit is how long a checkpoint, and a batch write
// waiting for it, may take while a reader is open: well under the busy
// timeout, which a checkpoint that waits for the reader takes.
@@ -82,275 +83,6 @@ const (
time.Second
)
-// Parameters for the prefix-length sample tests.
-const (
- // sampleLimit is how many prefixes the tests ask the sample for.
- sampleLimit = 10
- // sampledPrefixCount is how many prefixes the sampling test stores, more
- // than sampleLimit so the sample has to choose among them.
- sampledPrefixCount = 30
- // secondV4PeerIP and secondV6PeerIP are the second peer each prefix is
- // learned from, besides the one mkV4Route and mkV6Route set.
- secondV4PeerIP = "192.0.2.2"
- secondV6PeerIP = "2001:db8::2"
- // largeSamplePrefixCount is how many prefixes the test of a mask length
- // with more than maxPrefixesToShuffle prefixes stores: twice that, so about
- // half of them come after the first maxPrefixesToShuffle+1 in text order.
- largeSamplePrefixCount = 2 * maxPrefixesToShuffle
- // largeSampleLimit is how many prefixes that test asks for. Each random
- // lookup lands after the first maxPrefixesToShuffle+1 in text order about
- // half the time, so none of them doing so is about as likely as
- // largeSampleLimit coin tosses all coming up heads.
- largeSampleLimit = 30
- // liveRoutesV4Table and liveRoutesV6Table are the live route tables the
- // tests pass to the sampling steps.
- liveRoutesV4Table = "live_routes_v4"
- liveRoutesV6Table = "live_routes_v6"
-)
-
-// TestGetIPInfoFindsMostSpecificLiveRoute stores nested live prefixes for both
-// families and checks that a lookup returns the most specific one covering the
-// address, ErrNoRoute when none covers it, and the next less specific prefix
-// once the only route of the most specific one is withdrawn.
-func TestGetIPInfoFindsMostSpecificLiveRoute(t *testing.T) {
- t.Parallel()
-
- cfg := &config.Config{StateDir: t.TempDir()}
-
- db, err := New(cfg, logger.New())
- if err != nil {
- t.Fatalf("failed to create database: %v", err)
- }
- defer func() { _ = db.Close() }()
-
- // Nested live prefixes, each originated by its own AS.
- origins := map[string]int{
- "10.0.0.0/8": 64500,
- "10.1.0.0/16": 64501,
- "10.1.2.0/24": 64502,
- "2001:db8::/32": 64500,
- "2001:db8:1::/48": 64501,
- "2001:db8:1:2::/64": 64502,
- }
- writeOriginRoutes(t, db, origins)
-
- // lookup checks that ip resolves to the live prefix want, or to ErrNoRoute
- // when want is empty.
- lookup := func(ip, want string) {
- t.Helper()
-
- info, err := db.GetIPInfo(ip)
- if want == "" {
- if !errors.Is(err, ErrNoRoute) {
- t.Errorf("GetIPInfo(%s) = %+v, %v; want ErrNoRoute", ip, info, err)
- }
-
- return
- }
-
- if err != nil {
- t.Errorf("GetIPInfo(%s): %v", ip, err)
-
- return
- }
-
- if info.Netblock != want || info.MaskLength != netip.MustParsePrefix(want).Bits() ||
- info.ASN != origins[want] {
- t.Errorf("GetIPInfo(%s) = %s (mask %d) AS%d, want %s AS%d",
- ip, info.Netblock, info.MaskLength, info.ASN, want, origins[want])
- }
- }
-
- lookup("10.1.2.3", "10.1.2.0/24")
- lookup("::ffff:10.1.2.3", "10.1.2.0/24")
- lookup("10.1.3.4", "10.1.0.0/16")
- lookup("10.2.0.1", "10.0.0.0/8")
- lookup("192.0.2.1", "")
- lookup("2001:db8:1:2::3", "2001:db8:1:2::/64")
- lookup("2001:db8:1:3::4", "2001:db8:1::/48")
- lookup("2001:db8:2::1", "2001:db8::/32")
- lookup("2001:db9::1", "")
-
- err = db.DeleteLiveRouteBatch([]LiveRouteDeletion{
- {Prefix: "10.1.2.0/24", PeerIP: testPeerIP, IPVersion: ipVersionV4},
- {Prefix: "2001:db8:1:2::/64", PeerIP: testPeerIP, IPVersion: ipVersionV6},
- })
- if err != nil {
- t.Fatalf("DeleteLiveRouteBatch: %v", err)
- }
-
- lookup("10.1.2.3", "10.1.0.0/16")
- lookup("2001:db8:1:2::3", "2001:db8:1::/48")
-}
-
-// TestGetIPInfoIgnoresShortRoutes checks that the IP lookup does not report a
-// default route, or a route shorter than /8 for IPv4 or /16 for IPv6, as the
-// route for an address, and that a route of exactly /8 or /16, or longer, still
-// is. The prefixes are written out so that a wrong limit fails the test.
-func TestGetIPInfoIgnoresShortRoutes(t *testing.T) {
- t.Parallel()
-
- cfg := &config.Config{StateDir: t.TempDir()}
-
- db, err := New(cfg, logger.New())
- if err != nil {
- t.Fatalf("failed to create database: %v", err)
- }
- defer func() { _ = db.Close() }()
-
- // announce writes one live route for prefix, originated by asn.
- announce := func(prefix string, asn int) {
- t.Helper()
-
- writeOriginRoutes(t, db, map[string]int{prefix: asn})
- }
-
- // lookup checks that ip resolves to the live prefix netblock from asn.
- lookup := func(ip, netblock string, asn int) {
- t.Helper()
-
- info, err := db.GetIPInfo(ip)
- if err != nil {
- t.Errorf("GetIPInfo(%s): %v", ip, err)
-
- return
- }
-
- if info.Netblock != netblock || info.ASN != asn {
- t.Errorf("GetIPInfo(%s) = %s AS%d, want %s AS%d",
- ip, info.Netblock, info.ASN, netblock, asn)
- }
- }
-
- // Default and very short routes, and routes one bit shorter than each limit.
- announce("2000::/3", 64500)
- announce("0.0.0.0/0", 64500)
- announce("2000::/15", 64500)
- announce("192.0.0.0/7", 64500)
-
- for _, ip := range []string{"2001:db8::1", "192.0.2.1"} {
- info, err := db.GetIPInfo(ip)
- if !errors.Is(err, ErrNoRoute) {
- t.Errorf("GetIPInfo(%s) with only short routes = %+v, %v; "+
- "want ErrNoRoute", ip, info, err)
- }
- }
-
- // Routes of exactly the minimum length are used.
- announce("2001::/16", 64501)
- announce("192.0.0.0/8", 64501)
-
- lookup("2001:db8::1", "2001::/16", 64501)
- lookup("192.0.2.1", "192.0.0.0/8", 64501)
-
- announce("2001:db8::/32", 64502)
-
- lookup("2001:db8::1", "2001:db8::/32", 64502)
-}
-
-// TestGetIPInfoNeedsNoWHOISRefreshForOriginZero checks that the IP lookup asks
-// for no WHOIS lookup when the address's route has origin 0, no known origin,
-// and still asks for one when its origin AS has no WHOIS data.
-func TestGetIPInfoNeedsNoWHOISRefreshForOriginZero(t *testing.T) {
- t.Parallel()
-
- db, err := New(&config.Config{StateDir: t.TempDir()}, logger.New())
- if err != nil {
- t.Fatalf("failed to create database: %v", err)
- }
- defer func() { _ = db.Close() }()
-
- writeOriginRoutes(t, db, map[string]int{"10.0.0.0/8": 0, "11.0.0.0/8": 64500})
-
- for ip, want := range map[string]bool{"10.0.0.1": false, "11.0.0.1": true} {
- info, err := db.GetIPInfo(ip)
- if err != nil {
- t.Fatalf("GetIPInfo(%s): %v", ip, err)
- }
-
- if info.NeedsWHOISRefresh != want {
- t.Errorf("GetIPInfo(%s) asks for a WHOIS lookup of AS%d: %t, want %t",
- ip, info.ASN, info.NeedsWHOISRefresh, want)
- }
- }
-}
-
-// TestLiveRouteTablesHoldOneRoutePerPrefixAndPeer checks that each live route
-// table refuses a second route from a peer for a prefix, even with another
-// origin AS. An upsert updates the peer's route before it would insert one, so
-// only an insert straight into the table shows what the table allows.
-func TestLiveRouteTablesHoldOneRoutePerPrefixAndPeer(t *testing.T) {
- t.Parallel()
-
- db, err := New(&config.Config{StateDir: t.TempDir()}, logger.New())
- if err != nil {
- t.Fatalf("failed to create database: %v", err)
- }
- defer func() { _ = db.Close() }()
-
- const otherOriginASN = 64501
-
- ts := time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC)
-
- for _, tt := range []struct {
- route *LiveRoute
- insertSQL string
- }{
- {
- route: mkV4Route(t, "198.51.100.0/24", 64500, ts),
- insertSQL: insertLiveRouteV4SQL,
- },
- {
- route: mkV6Route(testV6Prefix, 64500, ts),
- insertSQL: insertLiveRouteV6SQL,
- },
- } {
- err := db.UpsertLiveRoute(tt.route)
- if err != nil {
- t.Fatalf("UpsertLiveRoute(%s): %v", tt.route.Prefix, err)
- }
-
- _, err = db.db.ExecContext(t.Context(), tt.insertSQL,
- tt.route.Prefix, tt.route.MaskLength, otherOriginASN,
- tt.route.PeerIP, "[]", tt.route.NextHop, ts)
-
- var sqliteErr sqlite3.Error
- if !errors.As(err, &sqliteErr) ||
- sqliteErr.ExtendedCode != sqlite3.ErrConstraintUnique {
- t.Errorf("second route from peer %s for %s: error %v, "+
- "want a UNIQUE constraint error",
- tt.route.PeerIP, tt.route.Prefix, err)
- }
- }
-}
-
-// writeOriginRoutes writes one live route from testPeerIP for each prefix in
-// origins, originated by the AS it maps to.
-func writeOriginRoutes(t *testing.T, db *Database, origins map[string]int) {
- t.Helper()
-
- ts := time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC)
- routes := make([]*LiveRoute, 0, len(origins))
-
- for prefix, asn := range origins {
- routes = append(routes, &LiveRoute{
- Prefix: prefix,
- MaskLength: netip.MustParsePrefix(prefix).Bits(),
- IPVersion: detectIPVersion(prefix),
- OriginASN: asn,
- PeerIP: testPeerIP,
- ASPath: []int{asn},
- NextHop: testPeerIP,
- LastUpdated: ts,
- })
- }
-
- err := db.UpsertLiveRouteBatch(routes)
- if err != nil {
- t.Fatalf("UpsertLiveRouteBatch: %v", err)
- }
-}
-
// TestConnectionPoolPragmas holds several pooled connections open at once and
// checks each one carries the per-connection settings from the DSN, plus the
// process-wide heap limits, and sees the new file with auto_vacuum
@@ -513,12 +245,13 @@ func readPragma(
return value
}
-// TestVacuumReturnsFreePages checks that after routes are deleted from a new
-// database, one Vacuum call returns every page they used. The deletes leave
-// fewer free pages than the 1000 Vacuum frees per call, so none may remain.
-// With auto_vacuum off (issue https://git.eeqj.de/sneak/routewatch/issues/43)
-// the free pages stayed in the file, and with the PRAGMA run by ExecContext
-// Vacuum freed only one page per call.
+// TestVacuumReturnsFreePages checks that after the rows of the routes table
+// are deleted from a new database, one Vacuum call returns every page they
+// used. The deletes leave fewer free pages than the 1000 Vacuum frees per
+// call, so none may remain. With auto_vacuum off (issue
+// https://git.eeqj.de/sneak/routewatch/issues/43) the free pages stayed in the
+// file, and with the PRAGMA run by ExecContext Vacuum freed only one page per
+// call.
func TestVacuumReturnsFreePages(t *testing.T) {
t.Parallel()
@@ -531,32 +264,17 @@ func TestVacuumReturnsFreePages(t *testing.T) {
defer func() { _ = db.Close() }()
ctx := context.Background()
- ts := time.Now().UTC()
-
- const asn = 64500
-
- routes := make([]*LiveRoute, 0, vacuumTestRoutes)
- deletions := make([]LiveRouteDeletion, 0, vacuumTestRoutes)
+ table := routetable.New(routetable.Params{})
for i := range vacuumTestRoutes {
- route := mkV4Route(t, fmt.Sprintf("10.%d.%d.0/24", i/256, i%256), asn, ts)
- routes = append(routes, route)
- deletions = append(deletions, LiveRouteDeletion{
- Prefix: route.Prefix,
- PeerIP: route.PeerIP,
- IPVersion: ipVersionV4,
- })
+ announceTo(table, fmt.Sprintf("10.%d.%d.0/24", i/256, i%256), asA, peerA, 0)
}
- err = db.UpsertLiveRouteBatch(routes)
- if err != nil {
- t.Fatalf("failed to write routes: %v", err)
- }
+ writeTableChanges(t, db, table)
- err = db.DeleteLiveRouteBatch(deletions)
- if err != nil {
- t.Fatalf("failed to delete routes: %v", err)
- }
+ // The peer going down removes every prefix, and so every row.
+ table.PeerDown(peerA)
+ writeTableChanges(t, db, table)
var before int
@@ -566,7 +284,7 @@ func TestVacuumReturnsFreePages(t *testing.T) {
}
if before == 0 {
- t.Fatalf("no free pages after deleting %d routes", vacuumTestRoutes)
+ t.Fatalf("no free pages after deleting %d rows", vacuumTestRoutes)
}
err = db.Vacuum(ctx)
@@ -842,12 +560,12 @@ func TestCheckpointLockWaitEndsWithContext(t *testing.T) {
db.mu.Unlock()
}
-// TestCheckpointDoesNotWaitForReader keeps a read transaction on
-// live_routes_v4 open while a route is committed, then starts Checkpoint and a
-// batch write together. It checks that both end within readerCheckpointLimit,
-// whichever of them takes the database lock first, and that Checkpoint reports
-// the part of the WAL the reader still needs as left uncopied. A TRUNCATE
-// checkpoint waited the busy timeout for the reader while holding the lock
+// TestCheckpointDoesNotWaitForReader keeps a read transaction on prefixes_v4
+// open while a prefix is committed, then starts Checkpoint and a batch write
+// together. It checks that both end within readerCheckpointLimit, whichever of
+// them takes the database lock first, and that Checkpoint reports the part of
+// the WAL the reader still needs as left uncopied. A TRUNCATE checkpoint
+// waited the busy timeout for the reader while holding the lock
// (https://git.eeqj.de/sneak/routewatch/issues/102).
func TestCheckpointDoesNotWaitForReader(t *testing.T) {
t.Parallel()
@@ -861,10 +579,9 @@ func TestCheckpointDoesNotWaitForReader(t *testing.T) {
ctx := context.Background()
ts := time.Now().UTC()
- err = db.UpsertLiveRouteBatch(
- []*LiveRoute{mkV4Route(t, "198.51.100.0/24", lockTestASN, ts)})
+ err = db.UpdatePrefixesBatch(map[string]time.Time{"198.51.100.0/24": ts})
if err != nil {
- t.Fatalf("failed to store the route the reader reads: %v", err)
+ t.Fatalf("failed to store the prefix the reader reads: %v", err)
}
reader, err := db.db.Conn(ctx)
@@ -882,18 +599,17 @@ func TestCheckpointDoesNotWaitForReader(t *testing.T) {
}
defer func() { _, _ = reader.ExecContext(ctx, "ROLLBACK") }()
- var routes int
+ var prefixes int
err = reader.QueryRowContext(ctx,
- "SELECT count(*) FROM live_routes_v4").Scan(&routes)
+ "SELECT count(*) FROM prefixes_v4").Scan(&prefixes)
if err != nil {
- t.Fatalf("failed to read the live routes: %v", err)
+ t.Fatalf("failed to read the prefixes: %v", err)
}
- err = db.UpsertLiveRouteBatch(
- []*LiveRoute{mkV4Route(t, "203.0.113.0/24", lockTestASN, ts)})
+ err = db.UpdatePrefixesBatch(map[string]time.Time{"203.0.113.0/24": ts})
if err != nil {
- t.Fatalf("failed to store a route after the read: %v", err)
+ t.Fatalf("failed to store a prefix after the read: %v", err)
}
started := time.Now()
@@ -901,12 +617,13 @@ func TestCheckpointDoesNotWaitForReader(t *testing.T) {
go func() { checkpointed <- db.Checkpoint(ctx) }()
- // A route write, not an AS write: the first new AS loads the embedded AS
+ // A prefix write, not an AS write: the first new AS loads the embedded AS
// names, which takes seconds under -race.
- route := mkV4Route(t, "192.0.2.0/24", lockTestASN, ts)
written := make(chan error, 1)
- go func() { written <- db.UpsertLiveRouteBatch([]*LiveRoute{route}) }()
+ go func() {
+ written <- db.UpdatePrefixesBatch(map[string]time.Time{lockTestPrefix: ts})
+ }()
err = <-checkpointed
if !errors.Is(err, ErrCheckpointIncomplete) {
@@ -964,12 +681,12 @@ func TestCheckpointEmptiesWALOnceNoReadIsOpen(t *testing.T) {
t.Fatalf("failed to begin the read transaction: %v", err)
}
- var routes int
+ var prefixes int
err = reader.QueryRowContext(ctx,
- "SELECT count(*) FROM live_routes_v4").Scan(&routes)
+ "SELECT count(*) FROM prefixes_v4").Scan(&prefixes)
if err != nil {
- t.Fatalf("failed to read the live routes: %v", err)
+ t.Fatalf("failed to read the prefixes: %v", err)
}
fillWALPastLimit(ctx, t, db)
@@ -988,9 +705,8 @@ func TestCheckpointEmptiesWALOnceNoReadIsOpen(t *testing.T) {
took, truncateCheckpointLimit)
}
- err = db.UpsertLiveRouteBatch([]*LiveRoute{
- mkV4Route(t, "192.0.2.0/24", lockTestASN, time.Now().UTC()),
- })
+ err = db.UpdatePrefixesBatch(
+ map[string]time.Time{lockTestPrefix: time.Now().UTC()})
if err != nil {
t.Errorf("batch write after Checkpoint gave up: %v", err)
}
@@ -1054,7 +770,7 @@ func walFileSize(t *testing.T, db *Database) int64 {
return wal.Size()
}
-// TestStartupCheckpointLogsItsFigures opens a new database, writes a route,
+// TestStartupCheckpointLogsItsFigures opens a new database, writes a prefix,
// and opens the same file again while the first stays open, so that its WAL
// file stays. The first open finds no WAL file: it logs a size of 0 and no
// warning. The second logs the size of the WAL file the first left, and the
@@ -1081,16 +797,15 @@ func TestStartupCheckpointLogsItsFigures(t *testing.T) {
"want 0", line["wal_bytes"])
}
- err := db.UpsertLiveRouteBatch([]*LiveRoute{
- mkV4Route(t, "192.0.2.0/24", lockTestASN, time.Now().UTC()),
- })
+ err := db.UpdatePrefixesBatch(
+ map[string]time.Time{lockTestPrefix: time.Now().UTC()})
if err != nil {
- t.Fatalf("failed to store a route: %v", err)
+ t.Fatalf("failed to store a prefix: %v", err)
}
walBytes := walFileSize(t, db)
if walBytes == 0 {
- t.Fatal("WAL file empty after the route write, want frames in it")
+ t.Fatal("WAL file empty after the prefix write, want frames in it")
}
_, lines = openLogged(t, cfg)
@@ -1109,7 +824,7 @@ func TestStartupCheckpointLogsItsFigures(t *testing.T) {
}
// TestStartupCheckpointWarnsWhenIncomplete keeps a read open on a database
-// while a route is written after the read began, then opens the same file
+// while a prefix is written after the read began, then opens the same file
// again. The second open's start-up checkpoint cannot copy the frames written
// after the read began, and gives up once the busy timeout has passed. It
// checks that the checkpoint is logged as a warning with busy 1 and fewer
@@ -1136,19 +851,18 @@ func TestStartupCheckpointWarnsWhenIncomplete(t *testing.T) {
}
defer func() { _, _ = reader.ExecContext(ctx, "ROLLBACK") }()
- var routes int
+ var prefixes int
err = reader.QueryRowContext(ctx,
- "SELECT count(*) FROM live_routes_v4").Scan(&routes)
+ "SELECT count(*) FROM prefixes_v4").Scan(&prefixes)
if err != nil {
- t.Fatalf("failed to read the live routes: %v", err)
+ t.Fatalf("failed to read the prefixes: %v", err)
}
- err = db.UpsertLiveRouteBatch([]*LiveRoute{
- mkV4Route(t, "192.0.2.0/24", lockTestASN, time.Now().UTC()),
- })
+ err = db.UpdatePrefixesBatch(
+ map[string]time.Time{lockTestPrefix: time.Now().UTC()})
if err != nil {
- t.Fatalf("failed to store a route after the read: %v", err)
+ t.Fatalf("failed to store a prefix after the read: %v", err)
}
_, lines := openLogged(t, cfg)
@@ -1224,356 +938,11 @@ func findLine(
return nil
}
-// TestGetRandomPrefixesByLengthListsPrefixOnce stores one prefix of each
-// family learned from two peers in the same second and checks that the
-// prefix-length sample lists each of them once.
-func TestGetRandomPrefixesByLengthListsPrefixOnce(t *testing.T) {
- t.Parallel()
-
- cfg := &config.Config{StateDir: t.TempDir()}
-
- db, err := New(cfg, logger.New())
- if err != nil {
- t.Fatalf("failed to create database: %v", err)
- }
- defer func() { _ = db.Close() }()
-
- ts := time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC)
- v4First := mkV4Route(t, "198.51.100.0/24", 64500, ts)
- v4Second := mkV4Route(t, "198.51.100.0/24", 64500, ts)
- v4Second.PeerIP = secondV4PeerIP
- v6First := mkV6Route(testV6Prefix, 64500, ts)
- v6Second := mkV6Route(testV6Prefix, 64500, ts)
- v6Second.PeerIP = secondV6PeerIP
-
- err = db.UpsertLiveRouteBatch(
- []*LiveRoute{v4First, v4Second, v6First, v6Second})
- if err != nil {
- t.Fatalf("UpsertLiveRouteBatch: %v", err)
- }
-
- for _, want := range []*LiveRoute{v4First, v6First} {
- routes, err := db.GetRandomPrefixesByLength(want.MaskLength,
- want.IPVersion, sampleLimit)
- if err != nil {
- t.Fatalf("GetRandomPrefixesByLength(%d, %d): %v",
- want.MaskLength, want.IPVersion, err)
- }
-
- if len(routes) != 1 || routes[0].Prefix != want.Prefix {
- t.Errorf("GetRandomPrefixesByLength(%d, %d) = %+v, want %s listed once",
- want.MaskLength, want.IPVersion, routes, want.Prefix)
- }
- }
-}
-
-// TestGetRandomPrefixesByLengthListsNewestRoute stores a prefix learned from
-// two peers at different times, the older route first, and checks that the
-// prefix-length sample lists the newer route.
-func TestGetRandomPrefixesByLengthListsNewestRoute(t *testing.T) {
- t.Parallel()
-
- cfg := &config.Config{StateDir: t.TempDir()}
-
- db, err := New(cfg, logger.New())
- if err != nil {
- t.Fatalf("failed to create database: %v", err)
- }
- defer func() { _ = db.Close() }()
-
- older := time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC)
- olderRoute := mkV4Route(t, "198.51.100.0/24", 64500, older)
- newerRoute := mkV4Route(t, "198.51.100.0/24", 64500, older.Add(time.Minute))
- newerRoute.PeerIP = secondV4PeerIP
-
- err = db.UpsertLiveRouteBatch([]*LiveRoute{olderRoute, newerRoute})
- if err != nil {
- t.Fatalf("UpsertLiveRouteBatch: %v", err)
- }
-
- routes, err := db.GetRandomPrefixesByLength(newerRoute.MaskLength,
- ipVersionV4, sampleLimit)
- if err != nil {
- t.Fatalf("GetRandomPrefixesByLength: %v", err)
- }
-
- if len(routes) != 1 || routes[0].PeerIP != newerRoute.PeerIP {
- t.Errorf("GetRandomPrefixesByLength = %+v, want only the newer route, "+
- "from %s", routes, newerRoute.PeerIP)
- }
-}
-
-// TestGetRandomPrefixesByLengthSamplesManyPrefixes stores more /24 prefixes
-// than the sample asks for, each followed by a /16 so that routes of another
-// mask length sit between the /24 rows, and each prefix learned from two peers
-// in the same second. It checks both the sample the pages get, picked from the
-// shuffled prefixes, and the random lookups used for a mask length with more
-// than maxPrefixesToShuffle prefixes.
-func TestGetRandomPrefixesByLengthSamplesManyPrefixes(t *testing.T) {
- t.Parallel()
-
- cfg := &config.Config{StateDir: t.TempDir()}
-
- db, err := New(cfg, logger.New())
- if err != nil {
- t.Fatalf("failed to create database: %v", err)
- }
- defer func() { _ = db.Close() }()
-
- ts := time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC)
-
- var routes []*LiveRoute
-
- stored := make([]string, 0, sampledPrefixCount)
-
- for i := range sampledPrefixCount {
- prefix := fmt.Sprintf("10.0.%d.0/24", i)
- for _, p := range []string{prefix, fmt.Sprintf("172.%d.0.0/16", i)} {
- first := mkV4Route(t, p, 64500, ts)
- second := mkV4Route(t, p, 64500, ts)
- second.PeerIP = secondV4PeerIP
- routes = append(routes, first, second)
- }
-
- stored = append(stored, prefix)
- }
-
- err = db.UpsertLiveRouteBatch(routes)
- if err != nil {
- t.Fatalf("UpsertLiveRouteBatch: %v", err)
- }
-
- slices.Sort(stored)
-
- maskLength := routes[0].MaskLength
-
- sample, err := db.GetRandomPrefixesByLength(maskLength, ipVersionV4, sampleLimit)
- if err != nil {
- t.Fatalf("GetRandomPrefixesByLength: %v", err)
- }
-
- listed := make([]string, 0, len(sample))
- for _, route := range sample {
- listed = append(listed, route.Prefix)
- }
-
- checkPrefixSample(t, "GetRandomPrefixesByLength", listed, stored)
-
- looked, err := db.samplePrefixesByLength(
- context.Background(), liveRoutesV4Table, maskLength, sampleLimit, nil)
- if err != nil {
- t.Fatalf("samplePrefixesByLength: %v", err)
- }
-
- checkPrefixSample(t, "samplePrefixesByLength", looked, stored)
-}
-
-// checkPrefixSample checks that sample lists sampleLimit prefixes, each once
-// and each one of stored, and that it is not just the first sampleLimit of
-// stored, which is in text order.
-func checkPrefixSample(t *testing.T, name string, sample, stored []string) {
- t.Helper()
-
- if len(sample) != sampleLimit {
- t.Errorf("%s listed %d prefixes, want %d", name, len(sample), sampleLimit)
- }
-
- listed := make(map[string]bool)
-
- for _, prefix := range sample {
- if listed[prefix] || !slices.Contains(stored, prefix) {
- t.Errorf("%s lists %s twice or it is not one of the /24s", name, prefix)
- }
-
- listed[prefix] = true
- }
-
- if slices.Equal(slices.Sorted(slices.Values(sample)), stored[:sampleLimit]) {
- t.Errorf("%s lists just the first %d prefixes in text order",
- name, sampleLimit)
- }
-}
-
-// TestGetRandomPrefixesByLengthSamplesPastFirstPrefixes stores more prefixes
-// of one mask length than maxPrefixesToShuffle. It checks that
-// firstPrefixesByLength reads just the first maxPrefixesToShuffle+1 in text
-// order, and that the sample lists the prefixes asked for, each once, some of
-// them after those first ones, which only the random lookups reach.
-func TestGetRandomPrefixesByLengthSamplesPastFirstPrefixes(t *testing.T) {
- t.Parallel()
-
- cfg := &config.Config{StateDir: t.TempDir()}
-
- db, err := New(cfg, logger.New())
- if err != nil {
- t.Fatalf("failed to create database: %v", err)
- }
- defer func() { _ = db.Close() }()
-
- ts := time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC)
- routes := make([]*LiveRoute, 0, largeSamplePrefixCount)
- stored := make([]string, 0, largeSamplePrefixCount)
-
- for i := range largeSamplePrefixCount {
- route := mkV6Route(fmt.Sprintf("2001:%x::/32", i), 64500, ts)
- routes = append(routes, route)
- stored = append(stored, route.Prefix)
- }
-
- err = db.UpsertLiveRouteBatch(routes)
- if err != nil {
- t.Fatalf("UpsertLiveRouteBatch: %v", err)
- }
-
- slices.Sort(stored)
- firstPrefixes := stored[:maxPrefixesToShuffle+1]
- maskLength := routes[0].MaskLength
-
- read, err := db.firstPrefixesByLength(context.Background(),
- liveRoutesV6Table, maskLength, maxPrefixesToShuffle+1)
- if err != nil {
- t.Fatalf("firstPrefixesByLength: %v", err)
- }
-
- if !slices.Equal(read, firstPrefixes) {
- t.Errorf("firstPrefixesByLength read %d prefixes, want the first %d "+
- "in text order", len(read), len(firstPrefixes))
- }
-
- sample, err := db.GetRandomPrefixesByLength(maskLength, ipVersionV6,
- largeSampleLimit)
- if err != nil {
- t.Fatalf("GetRandomPrefixesByLength: %v", err)
- }
-
- if len(sample) != largeSampleLimit {
- t.Errorf("GetRandomPrefixesByLength listed %d prefixes, want %d",
- len(sample), largeSampleLimit)
- }
-
- listed := make(map[string]bool)
- pastFirstPrefixes := 0
-
- for _, route := range sample {
- if listed[route.Prefix] {
- t.Errorf("GetRandomPrefixesByLength lists %s twice", route.Prefix)
- }
-
- listed[route.Prefix] = true
-
- if !slices.Contains(firstPrefixes, route.Prefix) {
- pastFirstPrefixes++
- }
- }
-
- if pastFirstPrefixes == 0 {
- t.Errorf("GetRandomPrefixesByLength lists only prefixes among the first "+
- "%d in text order", len(firstPrefixes))
- }
-}
-
-// TestSamplePrefixesByLengthFillsUpFromShuffled stores a single prefix, so
-// every random lookup lands on it, and checks that the sample still lists
-// sampleLimit prefixes: the stored one, then the shuffled ones in order. The
-// stored prefix is the second shuffled one, so the fill-up must skip it.
-func TestSamplePrefixesByLengthFillsUpFromShuffled(t *testing.T) {
- t.Parallel()
-
- cfg := &config.Config{StateDir: t.TempDir()}
-
- db, err := New(cfg, logger.New())
- if err != nil {
- t.Fatalf("failed to create database: %v", err)
- }
- defer func() { _ = db.Close() }()
-
- shuffled := make([]string, 0, sampledPrefixCount)
- for i := range sampledPrefixCount {
- shuffled = append(shuffled, fmt.Sprintf("10.0.%d.0/24", i))
- }
-
- ts := time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC)
- route := mkV4Route(t, shuffled[1], 64500, ts)
-
- err = db.UpsertLiveRouteBatch([]*LiveRoute{route})
- if err != nil {
- t.Fatalf("UpsertLiveRouteBatch: %v", err)
- }
-
- sample, err := db.samplePrefixesByLength(context.Background(),
- liveRoutesV4Table, route.MaskLength, sampleLimit, shuffled)
- if err != nil {
- t.Fatalf("samplePrefixesByLength: %v", err)
- }
-
- want := append([]string{shuffled[1], shuffled[0]}, shuffled[2:sampleLimit]...)
- if !slices.Equal(sample, want) {
- t.Errorf("samplePrefixesByLength = %v, want %v", sample, want)
- }
-}
-
-// TestGetRandomPrefixesByLengthSkipsWithdrawnRoutes withdraws a route after
-// storing it, as can happen between the steps of GetRandomPrefixesByLength. It
-// checks that the random lookups pick nothing for a mask length with no route
-// left, and that only the prefixes that still have a route get their newest
-// route listed.
-func TestGetRandomPrefixesByLengthSkipsWithdrawnRoutes(t *testing.T) {
- t.Parallel()
-
- cfg := &config.Config{StateDir: t.TempDir()}
-
- db, err := New(cfg, logger.New())
- if err != nil {
- t.Fatalf("failed to create database: %v", err)
- }
- defer func() { _ = db.Close() }()
-
- ts := time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC)
- kept := mkV4Route(t, "198.51.100.0/24", 64500, ts)
- withdrawn := mkV4Route(t, "172.16.0.0/16", 64500, ts)
-
- err = db.UpsertLiveRouteBatch([]*LiveRoute{kept, withdrawn})
- if err != nil {
- t.Fatalf("UpsertLiveRouteBatch: %v", err)
- }
-
- err = db.DeleteLiveRoute(withdrawn.Prefix, withdrawn.PeerIP)
- if err != nil {
- t.Fatalf("DeleteLiveRoute: %v", err)
- }
-
- ctx := context.Background()
-
- sample, err := db.samplePrefixesByLength(ctx, liveRoutesV4Table,
- withdrawn.MaskLength, sampleLimit, []string{withdrawn.Prefix})
- if err != nil {
- t.Fatalf("samplePrefixesByLength: %v", err)
- }
-
- if len(sample) != 0 {
- t.Errorf("samplePrefixesByLength = %v, want none for a mask length "+
- "with no route left", sample)
- }
-
- routes, err := db.newestRouteOfEachPrefix(ctx, liveRoutesV4Table,
- ipVersionV4, []string{withdrawn.Prefix, kept.Prefix})
- if err != nil {
- t.Fatalf("newestRouteOfEachPrefix: %v", err)
- }
-
- if len(routes) != 1 || routes[0].Prefix != kept.Prefix {
- t.Errorf("newestRouteOfEachPrefix = %+v, want only the route of %s",
- routes, kept.Prefix)
- }
-}
-
-// TestLiveRouteUpdateSetsOriginWhenItChanges announces a peer's route for a
-// prefix, then again with the same origin AS and a new path, then with a new
-// origin AS. Each time the route must carry the announced origin AS, path and
-// time, and the AS details must list the prefix under that origin AS only. The
-// update sets the origin AS with a statement of its own, which changes nothing
-// while the origin AS stays the same.
-func TestLiveRouteUpdateSetsOriginWhenItChanges(t *testing.T) {
+// TestGetASReadsTheRecordOrTheBuiltInList asks GetAS for an AS the asns table
+// holds, one only the AS info lookup holds, and one neither holds. The first
+// gets its record's seen times, the second its handle and description from the
+// lookup, and only the third is not known.
+func TestGetASReadsTheRecordOrTheBuiltInList(t *testing.T) {
t.Parallel()
db, err := New(&config.Config{StateDir: t.TempDir()}, logger.New())
@@ -1582,130 +951,63 @@ func TestLiveRouteUpdateSetsOriginWhenItChanges(t *testing.T) {
}
defer func() { _ = db.Close() }()
- const firstOrigin, secondOrigin = 64500, 64501
+ const (
+ storedASN = 64500 // a documentation AS, not in the lookup
+ listedASN = 15169 // in the lookup, not stored
+ unknownASN = 4200000001 // in neither
+ )
ts := time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC)
- for _, asn := range []int{firstOrigin, secondOrigin} {
- _, err := db.GetOrCreateASN(asn, ts)
- if err != nil {
- t.Fatalf("GetOrCreateASN(%d): %v", asn, err)
- }
- }
-
- route := mkV4Route(t, "198.51.100.0/24", firstOrigin, ts)
-
- for _, announced := range []struct {
- origin int
- path []int
- }{
- {firstOrigin, []int{firstOrigin}},
- {firstOrigin, []int{secondOrigin, firstOrigin}},
- {secondOrigin, []int{firstOrigin, secondOrigin}},
- } {
- ts = ts.Add(time.Minute)
- route.OriginASN, route.ASPath, route.LastUpdated =
- announced.origin, announced.path, ts
-
- err := db.UpsertLiveRouteBatch([]*LiveRoute{route})
- if err != nil {
- t.Fatalf("UpsertLiveRouteBatch: %v", err)
- }
-
- routes, _, err := db.GetPrefixDetails(route.Prefix)
- if err != nil {
- t.Fatalf("GetPrefixDetails: %v", err)
- }
-
- if len(routes) != 1 || routes[0].OriginASN != announced.origin ||
- !slices.Equal(routes[0].ASPath, announced.path) ||
- !routes[0].LastUpdated.Equal(ts) {
- t.Errorf("after announcing origin AS%d, path %v at %s: routes %+v",
- announced.origin, announced.path, ts, routes)
- }
-
- for _, asn := range []int{firstOrigin, secondOrigin} {
- _, prefixes, err := db.GetASDetails(asn)
- if err != nil {
- t.Fatalf("GetASDetails(%d): %v", asn, err)
- }
-
- listed := len(prefixes) == 1 && prefixes[0].Prefix == route.Prefix
- if listed != (asn == announced.origin) {
- t.Errorf("after announcing origin AS%d, AS%d lists %+v",
- announced.origin, asn, prefixes)
- }
- }
- }
-}
-
-// TestGetASDetailsListsEachPrefixOnce stores two IPv4 prefixes and an IPv6
-// prefix an AS originates, two of them from two peers at different times, and
-// a prefix another AS originates. The AS details list each of the AS's
-// prefixes once, IPv4 first and in text order, with its mask length and the
-// time of its newest route.
-func TestGetASDetailsListsEachPrefixOnce(t *testing.T) {
- t.Parallel()
-
- db, err := New(&config.Config{StateDir: t.TempDir()}, logger.New())
- if err != nil {
- t.Fatalf("failed to create database: %v", err)
- }
- defer func() { _ = db.Close() }()
-
- const asn, otherASN = 64500, 64501
-
- ts := time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC)
-
- _, err = db.GetOrCreateASN(asn, ts)
+ _, err = db.GetOrCreateASN(storedASN, ts)
if err != nil {
t.Fatalf("GetOrCreateASN: %v", err)
}
- v4Older := mkV4Route(t, "192.0.2.0/24", asn, ts)
- v4Newer := mkV4Route(t, "192.0.2.0/24", asn, ts.Add(time.Minute))
- v4Newer.PeerIP = secondV4PeerIP
- v4Single := mkV4Route(t, "198.51.100.128/25", asn, ts)
- v6Newer := mkV6Route(testV6Prefix, asn, ts.Add(time.Minute))
- v6Older := mkV6Route(testV6Prefix, asn, ts)
- v6Older.PeerIP = secondV6PeerIP
- other := mkV4Route(t, "203.0.113.0/24", otherASN, ts)
-
- err = db.UpsertLiveRouteBatch([]*LiveRoute{
- v4Older, v4Newer, v4Single, v6Newer, v6Older, other,
- })
- if err != nil {
- t.Fatalf("UpsertLiveRouteBatch: %v", err)
+ listed, ok := asinfo.Get(listedASN)
+ if !ok {
+ t.Fatalf("AS%d is not in the AS info data", listedASN)
}
- _, prefixes, err := db.GetASDetails(asn)
- if err != nil {
- t.Fatalf("GetASDetails: %v", err)
+ tests := []struct {
+ asn int
+ want ASN
+ wantKnown bool
+ }{
+ {storedASN, ASN{ASN: storedASN, FirstSeen: ts, LastSeen: ts}, true},
+ {listedASN, ASN{
+ ASN: listedASN, Handle: listed.Handle, Description: listed.Description,
+ }, true},
+ {unknownASN, ASN{ASN: unknownASN}, false},
}
- want := []*LiveRoute{v4Newer, v4Single, v6Newer}
+ for _, tt := range tests {
+ got, known, err := db.GetAS(t.Context(), tt.asn)
+ if err != nil {
+ t.Errorf("GetAS(%d): %v", tt.asn, err)
- listedOnce := slices.EqualFunc(prefixes, want,
- func(got LiveRoute, want *LiveRoute) bool {
- return got.Prefix == want.Prefix &&
- got.MaskLength == want.MaskLength &&
- got.IPVersion == want.IPVersion &&
- got.OriginASN == asn &&
- got.LastUpdated.Equal(want.LastUpdated)
- })
- if !listedOnce {
- t.Errorf("GetASDetails listed %+v, want the prefixes and times of %+v",
- prefixes, want)
+ continue
+ }
+
+ if !got.FirstSeen.Equal(tt.want.FirstSeen) ||
+ !got.LastSeen.Equal(tt.want.LastSeen) {
+ t.Errorf("GetAS(%d) seen %v to %v, want %v to %v", tt.asn,
+ got.FirstSeen, got.LastSeen, tt.want.FirstSeen, tt.want.LastSeen)
+ }
+
+ got.FirstSeen, got.LastSeen = tt.want.FirstSeen, tt.want.LastSeen
+ if *got != tt.want || known != tt.wantKnown {
+ t.Errorf("GetAS(%d) = %+v, known %t; want %+v, known %t",
+ tt.asn, *got, known, tt.want, tt.wantKnown)
+ }
}
}
-// TestASPrefixQueriesUseCoveringIndexes checks the plan of each query that
-// lists the prefixes an AS originates: a search of a covering index and no
-// temporary B-tree, so the list reads one index entry per prefix, not one
-// table row per route. Grouping the AS's routes by prefix read every route's
-// row and sorted them in a temporary B-tree
-// (https://git.eeqj.de/sneak/routewatch/issues/104).
-func TestASPrefixQueriesUseCoveringIndexes(t *testing.T) {
+// TestGetASWHOISReadsTheStoredDetails stores an AS with WHOIS details and asks
+// GetASWHOIS for it and for an AS the asns table does not hold. The first gets
+// its stored handle, description and details, and the time they were fetched.
+// The second gets only its number, although the AS info lookup holds it.
+func TestGetASWHOISReadsTheStoredDetails(t *testing.T) {
t.Parallel()
db, err := New(&config.Config{StateDir: t.TempDir()}, logger.New())
@@ -1714,61 +1016,85 @@ func TestASPrefixQueriesUseCoveringIndexes(t *testing.T) {
}
defer func() { _ = db.Close() }()
- tests := []struct {
- query string
- args []any
- index string
- }{
- {nextOriginPrefixV4SQL, []any{0, ""}, "idx_live_routes_v4_origin_prefix"},
- {newestRouteTimeV4SQL, []any{""}, "idx_live_routes_v4_prefix_last_updated"},
- {nextOriginPrefixV6SQL, []any{0, ""}, "idx_live_routes_v6_origin_prefix"},
- {newestRouteTimeV6SQL, []any{""}, "idx_live_routes_v6_prefix_last_updated"},
+ const storedASN, unstoredASN = 15169, 13335
+
+ _, err = db.GetOrCreateASN(storedASN, time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC))
+ if err != nil {
+ t.Fatalf("GetOrCreateASN: %v", err)
}
- for _, tt := range tests {
- plan := queryPlan(t, db, tt.query, tt.args...)
+ update := &ASNWHOISUpdate{
+ ASN: storedASN, ASName: "TEST-AS", OrgName: "Test Org", OrgID: "ORG-1",
+ Address: "1 Test Street", CountryCode: "DE", AbuseEmail: "abuse@test",
+ RIR: "RIPE",
+ }
- if !strings.Contains(plan, "USING COVERING INDEX "+tt.index+" ") ||
- strings.Contains(plan, "TEMP B-TREE") {
- t.Errorf("plan of %s:\n%s\nwant a search of covering index %s "+
- "and no temporary B-tree", tt.query, plan, tt.index)
- }
+ err = db.UpdateASNWHOIS(t.Context(), update)
+ if err != nil {
+ t.Fatalf("UpdateASNWHOIS: %v", err)
+ }
+
+ got, err := db.GetASWHOIS(t.Context(), storedASN)
+ if err != nil {
+ t.Fatalf("GetASWHOIS(%d): %v", storedASN, err)
+ }
+
+ want := ASN{
+ ASN: storedASN, Handle: asinfo.GetHandle(storedASN),
+ Description: asinfo.GetDescription(storedASN), ASName: update.ASName,
+ OrgName: update.OrgName, OrgID: update.OrgID, Address: update.Address,
+ CountryCode: update.CountryCode, AbuseEmail: update.AbuseEmail,
+ RIR: update.RIR,
+ }
+
+ if got.WHOISUpdatedAt == nil {
+ t.Errorf("GetASWHOIS(%d) has no time the details were fetched", storedASN)
+ }
+
+ got.WHOISUpdatedAt = nil
+ if *got != want {
+ t.Errorf("GetASWHOIS(%d) = %+v, want %+v", storedASN, *got, want)
+ }
+
+ got, err = db.GetASWHOIS(t.Context(), unstoredASN)
+ if err != nil || *got != (ASN{ASN: unstoredASN}) {
+ t.Errorf("GetASWHOIS(%d) = %+v, %v; want only the number",
+ unstoredASN, got, err)
}
}
-// queryPlan returns the steps of EXPLAIN QUERY PLAN for query, one per line.
-func queryPlan(t *testing.T, db *Database, query string, args ...any) string {
- t.Helper()
+// TestGetPrefixFirstSeenReadsThePrefixTables stores an IPv4 and an IPv6
+// prefix and expects GetPrefixFirstSeen to give each its first-seen time, and
+// the zero time for a prefix that is not stored.
+func TestGetPrefixFirstSeenReadsThePrefixTables(t *testing.T) {
+ t.Parallel()
- rows, err := db.db.QueryContext(t.Context(), "EXPLAIN QUERY PLAN "+query,
- args...)
+ db, err := New(&config.Config{StateDir: t.TempDir()}, logger.New())
if err != nil {
- t.Fatalf("EXPLAIN QUERY PLAN %s: %v", query, err)
+ t.Fatalf("failed to create database: %v", err)
}
- defer func() { _ = rows.Close() }()
+ defer func() { _ = db.Close() }()
- var steps []string
+ ts := time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC)
+ stored := map[string]time.Time{
+ "192.0.2.0/24": ts,
+ testV6Prefix: ts.Add(time.Hour),
+ }
- for rows.Next() {
- var (
- id, parent, unused int
- detail string
- )
+ err = db.UpdatePrefixesBatch(stored)
+ if err != nil {
+ t.Fatalf("UpdatePrefixesBatch: %v", err)
+ }
- err := rows.Scan(&id, &parent, &unused, &detail)
- if err != nil {
- t.Fatalf("reading the plan of %s: %v", query, err)
+ stored["198.51.100.0/24"] = time.Time{}
+
+ for prefix, want := range stored {
+ got, err := db.GetPrefixFirstSeen(t.Context(), netip.MustParsePrefix(prefix))
+ if err != nil || !got.Equal(want) {
+ t.Errorf("GetPrefixFirstSeen(%s) = %v, %v; want %v",
+ prefix, got, err, want)
}
-
- steps = append(steps, detail)
}
-
- err = rows.Err()
- if err != nil {
- t.Fatalf("reading the plan of %s: %v", query, err)
- }
-
- return strings.Join(steps, "\n")
}
// TestUpdatePrefixesBatchKeepsFirstSeen writes a prefix, then a batch with it
@@ -1880,12 +1206,129 @@ func TestUpdatePeerBatchStoresTheEarliestAndLatestTimesOfANewPeer(
}
}
-// TestUpsertLiveRouteBatchWritesInKeyOrder writes new routes of two peers out
-// of key order, the last one a second announcement of the first, and checks
-// that the rows were inserted in key order, peer then prefix (each new row's id
-// is one above the last), and that the route announced twice carries the later
-// announcement.
-func TestUpsertLiveRouteBatchWritesInKeyOrder(t *testing.T) {
+// backlogAge is how much older one write is than the other in the tests of
+// first_seen and last_seen, as when a collector catches up on a backlog.
+const backlogAge = 24 * time.Hour
+
+// batchSpan is how much earlier a batch's First is than its Last in the tests
+// of first_seen and last_seen.
+const batchSpan = time.Hour
+
+// TestWritesKeepEarliestFirstSeenAndLatestLastSeen writes a row of each of
+// asns, peerings and bgp_peers through each function that writes one, at a
+// newer time and at an older one. Each row takes the older write's First as
+// first_seen and the newer write's Last as last_seen. The test writes in both
+// orders, each in a database of its own. Newer then older is a collector
+// catching up on a backlog; older then newer is a feed that arrives in order.
+// Each function writes a row of its own. A batch write's First is batchSpan
+// before its Last, so the test catches the two swapped. A single-row write
+// takes one time.
+func TestWritesKeepEarliestFirstSeenAndLatestLastSeen(t *testing.T) {
+ t.Parallel()
+
+ const (
+ batchASN, singleASN = 64500, 64501
+ peerASN = 64502
+ asnQuery = "SELECT first_seen, last_seen FROM asns WHERE asn = ?"
+ peeringQuery = "SELECT first_seen, last_seen FROM peerings " +
+ "WHERE as_a = ? AND as_b = ?"
+ peerQuery = "SELECT first_seen, last_seen FROM bgp_peers " +
+ "WHERE peer_ip = ?"
+ )
+
+ for name, olderFirst := range map[string]bool{
+ "newer then older": false,
+ "older then newer": true,
+ } {
+ t.Run(name, func(t *testing.T) {
+ t.Parallel()
+
+ db, err := New(&config.Config{StateDir: t.TempDir()}, logger.New())
+ if err != nil {
+ t.Fatalf("failed to create database: %v", err)
+ }
+ defer func() { _ = db.Close() }()
+
+ checkSeenTimes(t, db, olderFirst, func(seen SeenTimes) error {
+ return db.GetOrCreateASNBatch(map[int]SeenTimes{batchASN: seen})
+ }, batchSpan, asnQuery, batchASN)
+
+ checkSeenTimes(t, db, olderFirst, func(seen SeenTimes) error {
+ _, err := db.GetOrCreateASN(singleASN, seen.Last)
+
+ return err
+ }, 0, asnQuery, singleASN)
+
+ checkSeenTimes(t, db, olderFirst, func(seen SeenTimes) error {
+ return db.RecordPeering(batchASN, singleASN, seen)
+ }, batchSpan, peeringQuery, batchASN, singleASN)
+
+ checkSeenTimes(t, db, olderFirst, func(seen SeenTimes) error {
+ return db.UpdatePeerBatch(map[string]PeerUpdate{
+ testV4PeerIP: {
+ PeerIP: testV4PeerIP, PeerASN: peerASN,
+ MessageType: ristypes.MessageTypeUpdate,
+ Seen: seen,
+ },
+ })
+ }, batchSpan, peerQuery, testV4PeerIP)
+
+ checkSeenTimes(t, db, olderFirst, func(seen SeenTimes) error {
+ return db.UpdatePeer(testPeerIP, peerASN,
+ ristypes.MessageTypeUpdate, seen.Last)
+ }, 0, peerQuery, testPeerIP)
+ })
+ }
+}
+
+// checkSeenTimes calls write twice, each time with First span before Last: at
+// newer times and at times backlogAge older. It writes the older times first
+// when olderFirst is true. It checks that query, given args, reads the older
+// write's First as first_seen and the newer write's Last as last_seen.
+func checkSeenTimes(
+ t *testing.T, db *Database, olderFirst bool,
+ write func(seen SeenTimes) error,
+ span time.Duration, query string, args ...any,
+) {
+ t.Helper()
+
+ newerLast := time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC)
+ newer := SeenTimes{First: newerLast.Add(-span), Last: newerLast}
+ older := SeenTimes{
+ First: newer.First.Add(-backlogAge),
+ Last: newer.Last.Add(-backlogAge),
+ }
+
+ writes := []SeenTimes{newer, older}
+ if olderFirst {
+ writes = []SeenTimes{older, newer}
+ }
+
+ for _, seen := range writes {
+ err := write(seen)
+ if err != nil {
+ t.Fatalf("writing first %s, last %s: %v", seen.First, seen.Last, err)
+ }
+ }
+
+ var firstSeen, lastSeen time.Time
+
+ err := db.db.QueryRowContext(t.Context(), query, args...).
+ Scan(&firstSeen, &lastSeen)
+ if err != nil {
+ t.Fatalf("reading the row: %v", err)
+ }
+
+ if !firstSeen.Equal(older.First) || !lastSeen.Equal(newer.Last) {
+ t.Errorf("first seen %s, last seen %s; want %s, %s",
+ firstSeen, lastSeen, older.First, newer.Last)
+ }
+}
+
+// TestGetOrCreateASNReturnsStoredSeenTimes stores an AS at a newer time, then
+// at an older one. The second call returns the times the row now holds: the
+// older first_seen and the newer last_seen.
+func TestGetOrCreateASNReturnsStoredSeenTimes(t *testing.T) {
t.Parallel()
db, err := New(&config.Config{StateDir: t.TempDir()}, logger.New())
@@ -1894,289 +1337,23 @@ func TestUpsertLiveRouteBatchWritesInKeyOrder(t *testing.T) {
}
defer func() { _ = db.Close() }()
- const (
- firstOrigin, laterOrigin = 64500, 64501
- // Three prefixes, in text order.
- firstPrefix = "192.0.2.0/24"
- secondPrefix = "198.51.100.0/24"
- thirdPrefix = "203.0.113.0/24"
- )
+ const asn = 64500
- type row struct {
- peer, prefix string
- origin int
- }
+ newer := time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC)
+ older := newer.Add(-backlogAge)
- ts := time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC)
-
- announcements := []row{
- {secondV4PeerIP, secondPrefix, firstOrigin},
- {testV4PeerIP, thirdPrefix, firstOrigin},
- {secondV4PeerIP, firstPrefix, firstOrigin},
- {testV4PeerIP, secondPrefix, firstOrigin},
- {secondV4PeerIP, secondPrefix, laterOrigin},
- }
-
- routes := make([]*LiveRoute, 0, len(announcements))
-
- for _, announced := range announcements {
- route := mkV4Route(t, announced.prefix, announced.origin, ts)
- route.PeerIP = announced.peer
- routes = append(routes, route)
- }
-
- err = db.UpsertLiveRouteBatch(routes)
+ _, err = db.GetOrCreateASN(asn, newer)
if err != nil {
- t.Fatalf("UpsertLiveRouteBatch: %v", err)
+ t.Fatalf("GetOrCreateASN at %s: %v", newer, err)
}
- rows, err := db.db.QueryContext(t.Context(),
- "SELECT peer_ip, prefix, origin_asn FROM live_routes_v4 ORDER BY id")
+ got, err := db.GetOrCreateASN(asn, older)
if err != nil {
- t.Fatalf("reading the routes: %v", err)
- }
- defer func() { _ = rows.Close() }()
-
- var stored []row
-
- for rows.Next() {
- var r row
-
- err := rows.Scan(&r.peer, &r.prefix, &r.origin)
- if err != nil {
- t.Fatalf("reading a route: %v", err)
- }
-
- stored = append(stored, r)
+ t.Fatalf("GetOrCreateASN at %s: %v", older, err)
}
- err = rows.Err()
- if err != nil {
- t.Fatalf("reading the routes: %v", err)
- }
-
- want := []row{
- {testV4PeerIP, secondPrefix, firstOrigin},
- {testV4PeerIP, thirdPrefix, firstOrigin},
- {secondV4PeerIP, firstPrefix, firstOrigin},
- {secondV4PeerIP, secondPrefix, laterOrigin},
- }
-
- if !slices.Equal(stored, want) {
- t.Errorf("routes in insertion order = %+v, want %+v", stored, want)
- }
-}
-
-// Parameters for BenchmarkPrefixBatchOnFiveMillionRoutes.
-const (
- // benchPeers and benchPrefixes size the table the batches are written to:
- // every peer has a route for every prefix, 5 million routes in all.
- benchPeers = 10
- benchPrefixes = 500000
- // benchBatchSize is how many updates each batch holds, as many as a full
- // prefix handler batch (prefixBatchSize in internal/routewatch).
- benchBatchSize = 25000
- // benchMaxBatchSize is the largest batch the prefix handler writes, the
- // one it holds while a slow write runs (prefixBatchMaxSize in
- // internal/routewatch).
- benchMaxBatchSize = 250000
- // benchWithdrawalEvery makes every fifth update of a timed batch a
- // withdrawal; the others are announcements.
- benchWithdrawalEvery = 5
- // benchOrigins is how many origin ASes the prefixes are spread over.
- benchOrigins = 50000
- // benchFirstASN is the lowest AS number in the benchmark's AS paths.
- benchFirstASN = 64500
- // benchMaskLength is the mask length of every prefix in the benchmark.
- benchMaskLength = 24
-)
-
-// BenchmarkPrefixBatchOnFiveMillionRoutes times the writes the prefix handler
-// makes for one batch of announcements and withdrawals of routes picked at
-// random, to a database that already holds 5 million live routes: 500,000
-// IPv4 prefixes, each from 10 peers. It times a full batch of 25,000 updates,
-// then the largest batch, 250,000. It builds that database first, one batch at
-// a time, which takes some minutes and a few GB of disk under TMPDIR. It is
-// run by hand, not by make test, for example in the test image:
-//
-// go test -run '^$' -bench FiveMillion -benchtime 10x ./internal/database/
-func BenchmarkPrefixBatchOnFiveMillionRoutes(b *testing.B) {
- db, err := New(&config.Config{StateDir: b.TempDir()}, logger.New())
- if err != nil {
- b.Fatalf("failed to create database: %v", err)
- }
- defer func() { _ = db.Close() }()
-
- //nolint:gosec // The benchmark needs the same batches every run.
- rng := rand.New(rand.NewPCG(1, 2))
- ts := time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC)
-
- // Each peer sends its whole table, as when its session comes up.
- for peer := range benchPeers {
- for first := 0; first < benchPrefixes; first += benchBatchSize {
- routes := make([]*LiveRoute, 0, benchBatchSize)
- for prefix := first; prefix < first+benchBatchSize; prefix++ {
- routes = append(routes, benchRoute(rng, prefix, peer, ts))
- }
-
- writeBenchBatch(b, db, routes, nil, ts)
- }
- }
-
- for _, batchSize := range []int{benchBatchSize, benchMaxBatchSize} {
- b.Run(strconv.Itoa(batchSize), func(b *testing.B) {
- for b.Loop() {
- ts = ts.Add(time.Second)
-
- var (
- routes []*LiveRoute
- withdrawals []LiveRouteDeletion
- )
-
- for i := range batchSize {
- route := benchRoute(rng, rng.IntN(benchPrefixes),
- rng.IntN(benchPeers), ts)
- if i%benchWithdrawalEvery == 0 {
- withdrawals = append(withdrawals, LiveRouteDeletion{
- Prefix: route.Prefix, PeerIP: route.PeerIP,
- IPVersion: ipVersionV4,
- })
- } else {
- routes = append(routes, route)
- }
- }
-
- writeBenchBatch(b, db, routes, withdrawals, ts)
- }
- })
- }
-}
-
-// benchRoute returns the route peer has for the prefix numbered prefix, a /24
-// from 1.0.0.0/24 on, with a new AS path of three to six ASes from the peer's
-// AS to the prefix's origin AS.
-func benchRoute(rng *rand.Rand, prefix, peer int, ts time.Time) *LiveRoute {
- const minPathLength, extraPathLength = 3, 4
-
- origin := benchFirstASN + prefix%benchOrigins
- pathLength := minPathLength + rng.IntN(extraPathLength)
-
- path := make([]int, 0, pathLength)
- path = append(path, benchFirstASN+peer)
-
- for len(path) < pathLength-1 {
- path = append(path, benchFirstASN+rng.IntN(benchOrigins))
- }
-
- path = append(path, origin)
- peerIP := fmt.Sprintf("198.51.100.%d", 1+peer)
-
- return &LiveRoute{
- Prefix: fmt.Sprintf("%d.%d.%d.0/%d",
- 1+prefix>>16, (prefix>>8)%256, prefix%256, benchMaskLength),
- MaskLength: benchMaskLength,
- IPVersion: ipVersionV4,
- OriginASN: origin,
- PeerIP: peerIP,
- ASPath: path,
- NextHop: peerIP,
- LastUpdated: ts,
- }
-}
-
-// writeBenchBatch writes what the prefix handler writes for a batch: the
-// announced routes, the withdrawals, and every prefix the batch names, of
-// which only the ones not stored yet are inserted.
-func writeBenchBatch(
- b *testing.B, db *Database, routes []*LiveRoute,
- withdrawals []LiveRouteDeletion, ts time.Time,
-) {
- b.Helper()
-
- prefixes := make(map[string]time.Time, len(routes)+len(withdrawals))
- for _, route := range routes {
- prefixes[route.Prefix] = ts
- }
-
- for _, withdrawal := range withdrawals {
- prefixes[withdrawal.Prefix] = ts
- }
-
- err := db.UpsertLiveRouteBatch(routes)
- if err != nil {
- b.Fatalf("UpsertLiveRouteBatch: %v", err)
- }
-
- err = db.DeleteLiveRouteBatch(withdrawals)
- if err != nil {
- b.Fatalf("DeleteLiveRouteBatch: %v", err)
- }
-
- err = db.UpdatePrefixesBatch(prefixes)
- if err != nil {
- b.Fatalf("UpdatePrefixesBatch: %v", err)
- }
-}
-
-// Parameters for BenchmarkGetASDetailsWithFiveHundredPeers: the AS originates
-// benchASPrefixes prefixes and every one of benchASPeers peers has a route for
-// each, 2.5 million routes in all.
-const (
- benchASPrefixes = 5000
- benchASPeers = 500
-)
-
-// BenchmarkGetASDetailsWithFiveHundredPeers times the read behind the AS page
-// for an AS that originates 5,000 IPv4 prefixes, each from 500 peers. It
-// builds that database first, one peer at a time, which takes some minutes and
-// about a GB of disk under TMPDIR. It is run by hand, not by make test, for
-// example in the test image:
-//
-// go test -run '^$' -bench GetASDetails -benchtime 10x ./internal/database/
-func BenchmarkGetASDetailsWithFiveHundredPeers(b *testing.B) {
- db, err := New(&config.Config{StateDir: b.TempDir()}, logger.New())
- if err != nil {
- b.Fatalf("failed to create database: %v", err)
- }
- defer func() { _ = db.Close() }()
-
- ts := time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC)
-
- _, err = db.GetOrCreateASN(benchFirstASN, ts)
- if err != nil {
- b.Fatalf("GetOrCreateASN: %v", err)
- }
-
- for peer := range benchASPeers {
- peerIP := fmt.Sprintf("198.18.%d.%d", peer/256, peer%256)
-
- routes := make([]*LiveRoute, 0, benchASPrefixes)
- for prefix := range benchASPrefixes {
- routes = append(routes, &LiveRoute{
- Prefix: fmt.Sprintf("%d.%d.%d.0/%d",
- 1+prefix>>16, (prefix>>8)%256, prefix%256, benchMaskLength),
- MaskLength: benchMaskLength,
- IPVersion: ipVersionV4,
- OriginASN: benchFirstASN,
- PeerIP: peerIP,
- ASPath: []int{benchFirstASN + 1 + peer, benchFirstASN},
- NextHop: peerIP,
- LastUpdated: ts,
- })
- }
-
- writeBenchBatch(b, db, routes, nil, ts)
- }
-
- for b.Loop() {
- _, prefixes, err := db.GetASDetails(benchFirstASN)
- if err != nil {
- b.Fatalf("GetASDetails: %v", err)
- }
-
- if len(prefixes) != benchASPrefixes {
- b.Fatalf("GetASDetails listed %d prefixes, want %d",
- len(prefixes), benchASPrefixes)
- }
+ if !got.FirstSeen.Equal(older) || !got.LastSeen.Equal(newer) {
+ t.Errorf("first seen %s, last seen %s; want %s, %s",
+ got.FirstSeen, got.LastSeen, older, newer)
}
}
diff --git a/internal/database/interface.go b/internal/database/interface.go
index 1c49b10..895297f 100644
--- a/internal/database/interface.go
+++ b/internal/database/interface.go
@@ -1,34 +1,31 @@
-// Package database provides SQLite storage for BGP routing data including ASNs,
-// prefixes, announcements, peerings, and live route tables.
+// Package database provides SQLite storage for BGP routing data: ASNs,
+// prefixes, announcements, peerings and BGP peers. Its routes table holds a
+// copy of the whole routing table that internal/routetable keeps in memory.
package database
import (
"context"
+ "net/netip"
"time"
+
+ "git.eeqj.de/sneak/routewatch/internal/routetable"
)
-// Stats contains database statistics including counts for ASNs, prefixes,
-// peerings, peers, and live routes, as well as file size and prefix distribution data.
+// Stats contains database statistics: counts of ASNs, prefixes, peerings and
+// peers, and the file size.
type Stats struct {
- ASNs int
- Prefixes int
- IPv4Prefixes int
- IPv6Prefixes int
- Peerings int
- Peers int
- FileSizeBytes int64
- LiveRoutes int
- IPv4Routes int
- IPv6Routes int
- OldestRoute *time.Time
- NewestRoute *time.Time
- IPv4PrefixDistribution []PrefixDistribution
- IPv6PrefixDistribution []PrefixDistribution
+ ASNs int
+ Prefixes int
+ IPv4Prefixes int
+ IPv6Prefixes int
+ Peerings int
+ Peers int
+ FileSizeBytes int64
}
// Store defines the interface for database operations. It provides methods for
-// managing ASNs, prefixes, announcements, peerings, BGP peers, and live routes.
-// Implementations must be safe for concurrent use.
+// managing ASNs, prefixes, announcements, peerings, BGP peers, and the copy of
+// the routing table. Implementations must be safe for concurrent use.
//
//nolint:interfacebloat // the one seam handlers and tests depend on
type Store interface {
@@ -54,22 +51,9 @@ type Store interface {
UpdatePeer(peerIP string, peerASN int, messageType string, timestamp time.Time) error
UpdatePeerBatch(peers map[string]PeerUpdate) error
- // Live route operations
- UpsertLiveRoute(route *LiveRoute) error
- UpsertLiveRouteBatch(routes []*LiveRoute) error
- DeleteLiveRoute(prefix, peerIP string) error
- DeleteLiveRouteBatch(deletions []LiveRouteDeletion) error
- DeleteLiveRoutesOfPeer(peerIP string) (deleted int, err error)
- GetPrefixDistribution() (
- ipv4 []PrefixDistribution, ipv6 []PrefixDistribution, err error)
- GetPrefixDistributionContext(ctx context.Context) (
- ipv4 []PrefixDistribution, ipv6 []PrefixDistribution, err error)
- GetLiveRouteCounts() (ipv4Count, ipv6Count int, err error)
- GetLiveRouteCountsContext(ctx context.Context) (ipv4Count, ipv6Count int, err error)
-
- // IP lookup operations
- GetIPInfo(ip string) (*IPInfo, error)
- GetIPInfoContext(ctx context.Context, ip string) (*IPInfo, error)
+ // Routing table operations: the database's copy of the table in memory
+ WriteRouteChanges(changes []routetable.Change, newPeers map[string]uint) error
+ ReadRouteTable(ctx context.Context) (*routetable.Table, error)
// ASN WHOIS operations
GetNextStaleASN(ctx context.Context, staleThreshold time.Duration) (int, error)
@@ -77,18 +61,11 @@ type Store interface {
GetWHOISStats(ctx context.Context, staleThreshold time.Duration) (*WHOISStats, error)
// AS and prefix detail operations
- GetASDetails(asn int) (*ASN, []LiveRoute, error)
- GetASDetailsContext(ctx context.Context, asn int) (*ASN, []LiveRoute, error)
+ GetAS(ctx context.Context, asn int) (*ASN, bool, error)
+ GetASWHOIS(ctx context.Context, asn int) (*ASN, error)
+ GetPrefixFirstSeen(ctx context.Context, prefix netip.Prefix) (time.Time, error)
GetASPeers(asn int) ([]ASPeer, error)
GetASPeersContext(ctx context.Context, asn int) ([]ASPeer, error)
- GetPrefixDetails(prefix string) ([]LiveRoute, map[int]ASN, error)
- GetPrefixDetailsContext(
- ctx context.Context, prefix string,
- ) ([]LiveRoute, map[int]ASN, error)
- GetRandomPrefixesByLength(maskLength, ipVersion, limit int) ([]LiveRoute, error)
- GetRandomPrefixesByLengthContext(
- ctx context.Context, maskLength, ipVersion, limit int,
- ) ([]LiveRoute, error)
// Lifecycle
Close() error
diff --git a/internal/database/models.go b/internal/database/models.go
index 8797ae7..92c8f67 100644
--- a/internal/database/models.go
+++ b/internal/database/models.go
@@ -28,9 +28,10 @@ type ASN struct {
RIRRegDate *time.Time `json:"rir_registration_date,omitempty"`
RIRLastMod *time.Time `json:"rir_last_modified,omitempty"`
WHOISRaw string `json:"whois_raw,omitempty"`
- // Timestamps
- FirstSeen time.Time `json:"first_seen"`
- LastSeen time.Time `json:"last_seen"`
+ // Timestamps. They are zero, and left out of JSON, for an AS that has no
+ // record in the asns table.
+ FirstSeen time.Time `json:"first_seen,omitzero"`
+ LastSeen time.Time `json:"last_seen,omitzero"`
WHOISUpdatedAt *time.Time `json:"whois_updated_at,omitempty"`
}
@@ -65,67 +66,10 @@ type ASNPeering struct {
LastSeen time.Time
}
-// LiveRoute represents a route in the live routing table
-//
-//nolint:tagliatelle // field names are the existing JSON API
-type LiveRoute struct {
- Prefix string `json:"prefix"`
- MaskLength int `json:"mask_length"`
- IPVersion int `json:"ip_version"`
- OriginASN int `json:"origin_asn"`
- PeerIP string `json:"peer_ip"`
- ASPath []int `json:"as_path"`
- NextHop string `json:"next_hop"`
- LastUpdated time.Time `json:"last_updated"`
-}
-
-// PrefixDistribution represents the distribution of prefixes by mask length
-//
-//nolint:tagliatelle // field names are the existing JSON API
-type PrefixDistribution struct {
- MaskLength int `json:"mask_length"`
- Count int `json:"count"`
-}
-
-// IPInfo represents comprehensive IP information for the /ip endpoint
-//
-//nolint:tagliatelle // field names are the existing JSON API
-type IPInfo struct {
- IP string `json:"ip"`
- PTR []string `json:"ptr,omitempty"`
- Netblock string `json:"netblock"`
- MaskLength int `json:"mask_length"`
- IPVersion int `json:"ip_version"`
- NumPeers int `json:"num_peers"`
- // AS information
- ASN int `json:"asn"`
- ASName string `json:"as_name,omitempty"`
- Handle string `json:"handle,omitempty"`
- Description string `json:"description,omitempty"`
- OrgName string `json:"org_name,omitempty"`
- OrgID string `json:"org_id,omitempty"`
- Address string `json:"address,omitempty"`
- CountryCode string `json:"country_code,omitempty"`
- AbuseEmail string `json:"abuse_email,omitempty"`
- RIR string `json:"rir,omitempty"`
- // Timestamps
- FirstSeen time.Time `json:"first_seen"`
- LastSeen time.Time `json:"last_seen"`
- // Indicates if WHOIS data needs refresh (not serialized)
- NeedsWHOISRefresh bool `json:"-"`
-}
-
-// LiveRouteDeletion represents parameters for deleting a live route
-type LiveRouteDeletion struct {
- Prefix string
- PeerIP string
- IPVersion int
-}
-
// SeenTimes is the earliest and the latest time one batch of updates saw an AS,
// a peer or a peering. A row the batch adds is stored first seen at First and
-// last seen at Last; a row already stored keeps its first_seen and has its
-// last_seen set to Last.
+// last seen at Last. A row already stored takes First as its first_seen only
+// when First is earlier, and Last as its last_seen only when Last is later.
type SeenTimes struct {
First time.Time
Last time.Time
diff --git a/internal/database/routes.go b/internal/database/routes.go
new file mode 100644
index 0000000..2e5897b
--- /dev/null
+++ b/internal/database/routes.go
@@ -0,0 +1,339 @@
+package database
+
+import (
+ "context"
+ "database/sql"
+ "encoding/json"
+ "errors"
+ "fmt"
+ "net/netip"
+ "time"
+
+ "github.com/bits-and-blooms/bitset"
+
+ "git.eeqj.de/sneak/routewatch/internal/routetable"
+)
+
+// keptRoutesPerRow is how many kept routes a row of the routes table has
+// columns for: the newest and the older one.
+const keptRoutesPerRow = 2
+
+// errUnknownRoutePeer is returned for a peer that route_peers lacks: on a
+// write, the peer of a kept route; on a read, a peer number in a row.
+var errUnknownRoutePeer = errors.New("peer not in route_peers")
+
+// The statements of the routes and route_peers tables (schema.sql).
+const (
+ insertRoutePeerSQL = `INSERT INTO route_peers (number, address)
+ VALUES (?, ?)`
+ routePeerNumberSQL = `SELECT number FROM route_peers WHERE address = ?`
+ routePeersSQL = `SELECT number, address FROM route_peers`
+
+ upsertRouteSQL = `INSERT INTO routes (prefix, origin_asn, peers,
+ newest_peer, newest_as_path, newest_time,
+ older_peer, older_as_path, older_time)
+ VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
+ ON CONFLICT (prefix, origin_asn) DO UPDATE SET peers = excluded.peers,
+ newest_peer = excluded.newest_peer,
+ newest_as_path = excluded.newest_as_path,
+ newest_time = excluded.newest_time,
+ older_peer = excluded.older_peer,
+ older_as_path = excluded.older_as_path,
+ older_time = excluded.older_time`
+ deleteRouteSQL = `DELETE FROM routes WHERE prefix = ? AND origin_asn = ?`
+ routesSQL = `SELECT prefix, origin_asn, peers,
+ newest_peer, newest_as_path, newest_time,
+ older_peer, older_as_path, older_time FROM routes`
+)
+
+// WriteRouteChanges writes, in one transaction, what one call of the routing
+// table's TakeChanges returned. It stores each new peer with its number, writes
+// the row of each changed prefix and origin AS pair, and deletes the row of
+// each removed one. Route times are stored to the second, as the feed gives
+// them.
+//
+// When it fails, it writes nothing. The caller then writes these changes and
+// peers again, because later changes can name the peers it did not store.
+func (d *Database) WriteRouteChanges(
+ changes []routetable.Change, newPeers map[string]uint,
+) error {
+ if len(changes) == 0 && len(newPeers) == 0 {
+ return nil
+ }
+
+ d.lock("WriteRouteChanges")
+ defer d.unlock()
+
+ tx, err := d.beginTx()
+ if err != nil {
+ return fmt.Errorf("failed to begin transaction: %w", err)
+ }
+ defer d.rollback(tx)
+
+ ctx := context.Background()
+
+ // The new peers first, so that the rows can look up their numbers.
+ for address, number := range newPeers {
+ _, err := tx.ExecContext(ctx, insertRoutePeerSQL, number, address)
+ if err != nil {
+ return fmt.Errorf("failed to store peer %s: %w", address, err)
+ }
+ }
+
+ err = writeRouteRows(ctx, tx, changes)
+ if err != nil {
+ return err
+ }
+
+ err = tx.Commit()
+ if err != nil {
+ return fmt.Errorf("failed to commit transaction: %w", err)
+ }
+
+ return nil
+}
+
+// writeRouteRows upserts the row of each change in tx, or deletes it for a
+// removed pair.
+func writeRouteRows(
+ ctx context.Context, tx *loggingTx, changes []routetable.Change,
+) error {
+ upsert, err := tx.PrepareContext(ctx, upsertRouteSQL)
+ if err != nil {
+ return fmt.Errorf("failed to prepare route upsert: %w", err)
+ }
+ defer func() { _ = upsert.Close() }()
+
+ del, err := tx.PrepareContext(ctx, deleteRouteSQL)
+ if err != nil {
+ return fmt.Errorf("failed to prepare route delete: %w", err)
+ }
+ defer func() { _ = del.Close() }()
+
+ peerNumber, err := tx.PrepareContext(ctx, routePeerNumberSQL)
+ if err != nil {
+ return fmt.Errorf("failed to prepare peer number lookup: %w", err)
+ }
+ defer func() { _ = peerNumber.Close() }()
+
+ for _, change := range changes {
+ prefix := change.Prefix.String()
+
+ if change.Removed {
+ _, err := del.ExecContext(ctx, prefix, change.OriginASN)
+ if err != nil {
+ return fmt.Errorf("failed to delete route %s: %w", prefix, err)
+ }
+
+ continue
+ }
+
+ row, err := routeRow(ctx, peerNumber, change)
+ if err != nil {
+ return fmt.Errorf("failed to write route %s: %w", prefix, err)
+ }
+
+ _, err = upsert.ExecContext(ctx, row...)
+ if err != nil {
+ return fmt.Errorf("failed to write route %s: %w", prefix, err)
+ }
+ }
+
+ return nil
+}
+
+// routeRow returns the values of the routes row of change, in the order of
+// upsertRouteSQL. peerNumber looks up a peer's number by its address. The
+// table keeps at most keptRoutesPerRow routes of a pair.
+func routeRow(
+ ctx context.Context, peerNumber *sql.Stmt, change routetable.Change,
+) ([]any, error) {
+ peers, err := change.Peers.MarshalBinary()
+ if err != nil {
+ return nil, err
+ }
+
+ row := []any{change.Prefix.String(), change.OriginASN, peers}
+
+ for i := range keptRoutesPerRow {
+ if i >= len(change.Routes) {
+ row = append(row, nil, nil, nil)
+
+ continue
+ }
+
+ r := change.Routes[i]
+
+ var number uint
+
+ err := peerNumber.QueryRowContext(ctx, r.Peer).Scan(&number)
+ if errors.Is(err, sql.ErrNoRows) {
+ return nil, fmt.Errorf("%w: %s", errUnknownRoutePeer, r.Peer)
+ }
+
+ if err != nil {
+ return nil, err
+ }
+
+ path, err := json.Marshal(r.ASPath)
+ if err != nil {
+ return nil, err
+ }
+
+ row = append(row, number, string(path), r.Time.Unix())
+ }
+
+ return row, nil
+}
+
+// ReadRouteTable returns the routing table that the routes and route_peers
+// tables hold, as startup reads it. It reads the peers first, so that the
+// table keeps their numbers, then loads each row into the table as it reads
+// it. The two reads are separate, so nothing may write the tables meanwhile.
+// It fails on a row that names a peer number route_peers lacks.
+func (d *Database) ReadRouteTable(ctx context.Context) (*routetable.Table, error) {
+ peerNumbers, err := d.readRoutePeers(ctx)
+ if err != nil {
+ return nil, err
+ }
+
+ addresses := make(map[uint]string, len(peerNumbers))
+ numbers := new(bitset.BitSet)
+
+ for address, number := range peerNumbers {
+ addresses[number] = address
+ numbers.Set(number)
+ }
+
+ table := routetable.New(routetable.Params{PeerNumbers: peerNumbers})
+
+ rows, err := d.db.QueryContext(ctx, routesSQL)
+ if err != nil {
+ return nil, fmt.Errorf("failed to read routes: %w", err)
+ }
+ defer func() { _ = rows.Close() }()
+
+ for rows.Next() {
+ change, err := scanRouteRow(rows, addresses, numbers)
+ if err != nil {
+ return nil, err
+ }
+
+ table.Load(change)
+ }
+
+ err = rows.Err()
+ if err != nil {
+ return nil, fmt.Errorf("failed to read routes: %w", err)
+ }
+
+ return table, nil
+}
+
+// readRoutePeers returns the number of each peer in route_peers, by address.
+func (d *Database) readRoutePeers(ctx context.Context) (map[string]uint, error) {
+ rows, err := d.db.QueryContext(ctx, routePeersSQL)
+ if err != nil {
+ return nil, fmt.Errorf("failed to read route peers: %w", err)
+ }
+ defer func() { _ = rows.Close() }()
+
+ peerNumbers := make(map[string]uint)
+
+ for rows.Next() {
+ var (
+ number uint
+ address string
+ )
+
+ err := rows.Scan(&number, &address)
+ if err != nil {
+ return nil, fmt.Errorf("failed to read a route peer: %w", err)
+ }
+
+ peerNumbers[address] = number
+ }
+
+ err = rows.Err()
+ if err != nil {
+ return nil, fmt.Errorf("failed to read route peers: %w", err)
+ }
+
+ return peerNumbers, nil
+}
+
+// keptRouteColumns are the three columns of one kept route in a routes row.
+type keptRouteColumns struct {
+ peer sql.Null[uint]
+ asPath sql.NullString
+ time sql.NullInt64
+}
+
+// scanRouteRow returns the content of the routes row rows is at, with each kept
+// route's peer named by its address in addresses. It fails when the row names a
+// peer number that addresses lacks. numbers holds the numbers of addresses as a
+// set, against which a row's whole peer set is checked at once.
+func scanRouteRow(
+ rows *sql.Rows, addresses map[uint]string, numbers *bitset.BitSet,
+) (routetable.Change, error) {
+ var (
+ change routetable.Change
+ prefix string
+ peers []byte
+ kept [keptRoutesPerRow]keptRouteColumns
+ )
+
+ err := rows.Scan(&prefix, &change.OriginASN, &peers,
+ &kept[0].peer, &kept[0].asPath, &kept[0].time,
+ &kept[1].peer, &kept[1].asPath, &kept[1].time)
+ if err != nil {
+ return change, fmt.Errorf("failed to read a route: %w", err)
+ }
+
+ change.Prefix, err = netip.ParsePrefix(prefix)
+ if err != nil {
+ return change, fmt.Errorf("failed to read route %s: %w", prefix, err)
+ }
+
+ change.Peers = new(bitset.BitSet)
+
+ err = change.Peers.UnmarshalBinary(peers)
+ if err != nil {
+ return change, fmt.Errorf("failed to read the peers of route %s: %w",
+ prefix, err)
+ }
+
+ if !numbers.IsSuperSet(change.Peers) {
+ unknown, _ := change.Peers.Difference(numbers).NextSet(0)
+
+ return change, fmt.Errorf("%w: number %d in the peers of route %s",
+ errUnknownRoutePeer, unknown, prefix)
+ }
+
+ for _, columns := range kept {
+ if !columns.peer.Valid {
+ continue
+ }
+
+ address, known := addresses[columns.peer.V]
+ if !known {
+ return change, fmt.Errorf("%w: number %d in route %s",
+ errUnknownRoutePeer, columns.peer.V, prefix)
+ }
+
+ r := routetable.Route{
+ Peer: address,
+ Time: time.Unix(columns.time.Int64, 0).UTC(),
+ }
+
+ err := json.Unmarshal([]byte(columns.asPath.String), &r.ASPath)
+ if err != nil {
+ return change, fmt.Errorf("failed to read an AS path of route %s: %w",
+ prefix, err)
+ }
+
+ change.Routes = append(change.Routes, r)
+ }
+
+ return change, nil
+}
diff --git a/internal/database/routes_internal_test.go b/internal/database/routes_internal_test.go
new file mode 100644
index 0000000..79bc133
--- /dev/null
+++ b/internal/database/routes_internal_test.go
@@ -0,0 +1,527 @@
+package database
+
+import (
+ "errors"
+ "fmt"
+ "math"
+ "math/rand/v2"
+ "net/netip"
+ "os"
+ "reflect"
+ "slices"
+ "testing"
+ "time"
+
+ "github.com/bits-and-blooms/bitset"
+
+ "git.eeqj.de/sneak/routewatch/internal/config"
+ "git.eeqj.de/sneak/routewatch/internal/logger"
+ "git.eeqj.de/sneak/routewatch/internal/routetable"
+)
+
+// Peers, origin ASes and prefixes of the routes table tests.
+const (
+ peerA = "192.0.2.1"
+ peerB = "192.0.2.2"
+ peerC = "2001:db8::3"
+ peerD = "2001:db8::4"
+ peerE = "2001:db8::5"
+
+ asA = 64500
+ asB = 64501
+ // transitAS is the first AS of every test path, the peer's own.
+ transitAS = 64496
+
+ tablePrefixV4 = "10.1.2.0/24"
+ tablePrefixShortV4 = "10.9.0.0/16"
+ tablePrefixV6 = "2001:db8:1::/48"
+)
+
+// tableTime returns the time second seconds after the start of every routes
+// table test.
+func tableTime(second int) time.Time {
+ return time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC).
+ Add(time.Duration(second) * time.Second)
+}
+
+// announceTo has peer announce prefix with origin AS asn second seconds after
+// the start.
+func announceTo(
+ table *routetable.Table, prefix string, asn int, peer string, second int,
+) {
+ table.Announce(netip.MustParsePrefix(prefix), asn, routetable.Route{
+ Peer: peer, ASPath: []int{transitAS, asn}, Time: tableTime(second),
+ })
+}
+
+// newRoutesTestDatabase returns a new database in a temporary directory.
+func newRoutesTestDatabase(t *testing.T) *Database {
+ t.Helper()
+
+ db, err := New(&config.Config{StateDir: t.TempDir()}, logger.New())
+ if err != nil {
+ t.Fatalf("failed to create database: %v", err)
+ }
+
+ t.Cleanup(func() { _ = db.Close() })
+
+ return db
+}
+
+// writeTableChanges writes what table changed since the last call.
+func writeTableChanges(t *testing.T, db *Database, table *routetable.Table) {
+ t.Helper()
+
+ changes, newPeers := table.TakeChanges(math.MaxInt)
+
+ err := db.WriteRouteChanges(changes, newPeers)
+ if err != nil {
+ t.Fatalf("WriteRouteChanges: %v", err)
+ }
+}
+
+func TestReadRouteTableGivesBackWhatWasWritten(t *testing.T) {
+ t.Parallel()
+
+ db := newRoutesTestDatabase(t)
+ source := routetable.New(routetable.Params{})
+
+ // Two origins of one prefix: the routes of peerC and peerB are kept for
+ // asA, peerA keeps only its bit.
+ announceTo(source, tablePrefixV4, asA, peerA, 10)
+ announceTo(source, tablePrefixV4, asA, peerB, 20)
+ announceTo(source, tablePrefixV4, asA, peerC, 30)
+ announceTo(source, tablePrefixV4, asB, peerD, 40)
+ // Two routes with the same time.
+ announceTo(source, tablePrefixV6, asA, peerA, 50)
+ announceTo(source, tablePrefixV6, asA, peerC, 50)
+ // An origin whose kept routes were withdrawn while peerA carries it.
+ announceTo(source, tablePrefixShortV4, asB, peerA, 10)
+ announceTo(source, tablePrefixShortV4, asB, peerB, 20)
+ announceTo(source, tablePrefixShortV4, asB, peerC, 30)
+ source.Withdraw(netip.MustParsePrefix(tablePrefixShortV4), peerB)
+ source.Withdraw(netip.MustParsePrefix(tablePrefixShortV4), peerC)
+
+ writeTableChanges(t, db, source)
+ checkReadTable(t, db, source)
+
+ // A peer moves to another origin, one goes down and a new one comes.
+ announceTo(source, tablePrefixV4, asB, peerA, 60)
+ source.PeerDown(peerC)
+ announceTo(source, tablePrefixV6, asB, peerE, 70)
+
+ writeTableChanges(t, db, source)
+ checkReadTable(t, db, source)
+}
+
+func TestReadRouteTableGivesATableThatTakesLaterUpdates(t *testing.T) {
+ t.Parallel()
+
+ db := newRoutesTestDatabase(t)
+ source := routetable.New(routetable.Params{})
+
+ // peerA and peerB carry tablePrefixV4, peerB and peerC tablePrefixV6.
+ announceTo(source, tablePrefixV4, asA, peerA, 10)
+ announceTo(source, tablePrefixV4, asA, peerB, 20)
+ announceTo(source, tablePrefixV6, asA, peerB, 30)
+ announceTo(source, tablePrefixV6, asA, peerC, 40)
+
+ writeTableChanges(t, db, source)
+ read := checkReadTable(t, db, source)
+
+ // The same updates to both tables: a loaded peer withdraws a prefix,
+ // another one goes down, and a new peer comes.
+ for _, table := range []*routetable.Table{source, read} {
+ table.Withdraw(netip.MustParsePrefix(tablePrefixV4), peerA)
+ table.PeerDown(peerB)
+ announceTo(table, tablePrefixShortV4, asB, peerD, 50)
+ }
+
+ checkSameReads(t, read, source)
+}
+
+func TestReadRouteTableFailsOnAPeerWithoutANumber(t *testing.T) {
+ t.Parallel()
+
+ db := newRoutesTestDatabase(t)
+ table := routetable.New(routetable.Params{})
+
+ // Only the routes of peerB and peerC are kept, so only the peer set names
+ // peerA.
+ announceTo(table, tablePrefixV4, asA, peerA, 10)
+ announceTo(table, tablePrefixV4, asA, peerB, 20)
+ announceTo(table, tablePrefixV4, asA, peerC, 30)
+ writeTableChanges(t, db, table)
+
+ _, err := db.db.ExecContext(t.Context(),
+ "DELETE FROM route_peers WHERE address = ?", peerA)
+ if err != nil {
+ t.Fatalf("deleting a route peer: %v", err)
+ }
+
+ _, err = db.ReadRouteTable(t.Context())
+ if !errors.Is(err, errUnknownRoutePeer) {
+ t.Fatalf("ReadRouteTable = %v, want %v", err, errUnknownRoutePeer)
+ }
+}
+
+// checkReadTable checks that the table ReadRouteTable reads answers every read
+// as want does, and that reading it recorded no change. It returns that table.
+func checkReadTable(
+ t *testing.T, db *Database, want *routetable.Table,
+) *routetable.Table {
+ t.Helper()
+
+ got, err := db.ReadRouteTable(t.Context())
+ if err != nil {
+ t.Fatalf("ReadRouteTable: %v", err)
+ }
+
+ checkSameReads(t, got, want)
+
+ changes, newPeers := got.TakeChanges(math.MaxInt)
+ if len(changes) != 0 || len(newPeers) != 0 {
+ t.Errorf("the read recorded changes %+v and new peers %v, want none",
+ changes, newPeers)
+ }
+
+ return got
+}
+
+// checkSameReads checks that got answers every read as want does, for the
+// prefixes and the origin ASes the tests use.
+func checkSameReads(t *testing.T, got, want *routetable.Table) {
+ t.Helper()
+
+ if !reflect.DeepEqual(got.Counts(), want.Counts()) {
+ t.Errorf("Counts() = %+v, want %+v", got.Counts(), want.Counts())
+ }
+
+ for _, text := range []string{
+ tablePrefixV4, tablePrefixShortV4, tablePrefixV6,
+ } {
+ prefix := netip.MustParsePrefix(text)
+
+ if !reflect.DeepEqual(got.Origins(prefix), want.Origins(prefix)) {
+ t.Errorf("Origins(%s) = %+v, want %+v",
+ prefix, got.Origins(prefix), want.Origins(prefix))
+ }
+
+ gotSummary, gotFound := got.Lookup(prefix.Addr())
+ wantSummary, wantFound := want.Lookup(prefix.Addr())
+
+ if gotFound != wantFound || !reflect.DeepEqual(gotSummary, wantSummary) {
+ t.Errorf("Lookup(%s) = %+v, %v; want %+v, %v", prefix.Addr(),
+ gotSummary, gotFound, wantSummary, wantFound)
+ }
+
+ ipVersion := ipVersionV6
+ if prefix.Addr().Is4() {
+ ipVersion = ipVersionV4
+ }
+
+ gotSample := sortedSample(got, ipVersion, prefix.Bits())
+ wantSample := sortedSample(want, ipVersion, prefix.Bits())
+
+ if !reflect.DeepEqual(gotSample, wantSample) {
+ t.Errorf("SamplePrefixes(%d, %d) = %+v, want %+v",
+ ipVersion, prefix.Bits(), gotSample, wantSample)
+ }
+ }
+
+ for _, asn := range []int{asA, asB} {
+ if !reflect.DeepEqual(got.OriginatedPrefixes(asn),
+ want.OriginatedPrefixes(asn)) {
+ t.Errorf("OriginatedPrefixes(%d) = %+v, want %+v", asn,
+ got.OriginatedPrefixes(asn), want.OriginatedPrefixes(asn))
+ }
+ }
+}
+
+// sortedSample returns every prefix of the IP version and mask length in
+// table, in prefix order.
+func sortedSample(
+ table *routetable.Table, ipVersion, maskLength int,
+) []routetable.PrefixSummary {
+ // More than the tests have of any mask length.
+ const all = 100
+
+ sample := table.SamplePrefixes(ipVersion, maskLength, all)
+ slices.SortFunc(sample, func(a, b routetable.PrefixSummary) int {
+ return a.Prefix.Compare(b.Prefix)
+ })
+
+ return sample
+}
+
+func TestWriteRouteChangesDeletesTheRowOfARemovedPair(t *testing.T) {
+ t.Parallel()
+
+ db := newRoutesTestDatabase(t)
+ table := routetable.New(routetable.Params{})
+
+ announceTo(table, tablePrefixV4, asA, peerA, 10)
+ announceTo(table, tablePrefixV4, asB, peerB, 20)
+ writeTableChanges(t, db, table)
+
+ // The prefix's only peer with asA withdraws it.
+ table.Withdraw(netip.MustParsePrefix(tablePrefixV4), peerA)
+ writeTableChanges(t, db, table)
+
+ var origins []int
+
+ rows, err := db.db.QueryContext(t.Context(),
+ "SELECT origin_asn FROM routes WHERE prefix = ?", tablePrefixV4)
+ if err != nil {
+ t.Fatalf("reading the routes: %v", err)
+ }
+ defer func() { _ = rows.Close() }()
+
+ for rows.Next() {
+ var origin int
+
+ err := rows.Scan(&origin)
+ if err != nil {
+ t.Fatalf("reading a route: %v", err)
+ }
+
+ origins = append(origins, origin)
+ }
+
+ err = rows.Err()
+ if err != nil {
+ t.Fatalf("reading the routes: %v", err)
+ }
+
+ if !slices.Equal(origins, []int{asB}) {
+ t.Errorf("rows of %s have origins %v, want only %d",
+ tablePrefixV4, origins, asB)
+ }
+}
+
+// routePeer is one row of route_peers.
+type routePeer struct {
+ number uint
+ address string
+}
+
+// readRoutePeerRows returns the rows of route_peers in number order.
+func readRoutePeerRows(t *testing.T, db *Database) []routePeer {
+ t.Helper()
+
+ rows, err := db.db.QueryContext(t.Context(),
+ "SELECT number, address FROM route_peers ORDER BY number")
+ if err != nil {
+ t.Fatalf("reading the route peers: %v", err)
+ }
+ defer func() { _ = rows.Close() }()
+
+ var peers []routePeer
+
+ for rows.Next() {
+ var peer routePeer
+
+ err := rows.Scan(&peer.number, &peer.address)
+ if err != nil {
+ t.Fatalf("reading a route peer: %v", err)
+ }
+
+ peers = append(peers, peer)
+ }
+
+ err = rows.Err()
+ if err != nil {
+ t.Fatalf("reading the route peers: %v", err)
+ }
+
+ return peers
+}
+
+func TestWriteRouteChangesStoresNewPeers(t *testing.T) {
+ t.Parallel()
+
+ db := newRoutesTestDatabase(t)
+ table := routetable.New(routetable.Params{})
+
+ announceTo(table, tablePrefixV4, asA, peerA, 10)
+ announceTo(table, tablePrefixV6, asA, peerC, 20)
+ writeTableChanges(t, db, table)
+
+ // peerA is stored already; peerB is new.
+ announceTo(table, tablePrefixV4, asA, peerA, 30)
+ announceTo(table, tablePrefixV4, asA, peerB, 40)
+ writeTableChanges(t, db, table)
+
+ want := []routePeer{{0, peerA}, {1, peerC}, {2, peerB}}
+
+ got := readRoutePeerRows(t, db)
+ if !slices.Equal(got, want) {
+ t.Errorf("route_peers = %+v, want %+v", got, want)
+ }
+}
+
+func TestWriteRouteChangesWritesNothingWhenItFails(t *testing.T) {
+ t.Parallel()
+
+ db := newRoutesTestDatabase(t)
+ table := routetable.New(routetable.Params{})
+
+ announceTo(table, tablePrefixV4, asA, peerA, 10)
+ announceTo(table, tablePrefixV6, asA, peerB, 20)
+
+ // peerB's number is missing, so its kept route cannot be written.
+ changes, newPeers := table.TakeChanges(math.MaxInt)
+ delete(newPeers, peerB)
+
+ err := db.WriteRouteChanges(changes, newPeers)
+ if !errors.Is(err, errUnknownRoutePeer) {
+ t.Fatalf("WriteRouteChanges = %v, want %v", err, errUnknownRoutePeer)
+ }
+
+ peers := readRoutePeerRows(t, db)
+ if len(peers) != 0 {
+ t.Errorf("route_peers = %+v after the failed write, want none", peers)
+ }
+
+ var routes int
+
+ err = db.db.QueryRowContext(t.Context(),
+ "SELECT COUNT(*) FROM routes").Scan(&routes)
+ if err != nil {
+ t.Fatalf("counting the routes: %v", err)
+ }
+
+ if routes != 0 {
+ t.Errorf("routes holds %d rows after the failed write, want none", routes)
+ }
+}
+
+// Parameters for BenchmarkReadRouteTable.
+const (
+ // benchTablePairs is how many prefix and origin AS pairs the table holds:
+ // one origin AS for each prefix. Every benchTableV6Every-th prefix is an
+ // IPv6 /48, the others IPv4 /24s.
+ benchTablePairs = 200000
+ benchTableV6Every = 5
+ // benchTablePeers is how many peers there are, about as many as RIS has.
+ // benchTablePeersPerPair is how many of them, picked at random, carry each
+ // pair: RIS's 460 million routes over 1.4 million pairs.
+ benchTablePeers = 1400
+ benchTablePeersPerPair = 330
+ // benchTableBatch is how many changes each write holds.
+ benchTableBatch = 10000
+)
+
+// BenchmarkReadRouteTable times the read of a routing table of 200,000 prefix
+// and origin AS pairs, each carried by 330 of 1,400 peers and keeping two
+// routes. It writes that table first. It reports the bytes of the database
+// file per pair. It is run by hand, not by make test, for example in the test
+// image:
+//
+// go test -run '^$' -bench ReadRouteTable -benchtime 5x ./internal/database/
+func BenchmarkReadRouteTable(b *testing.B) {
+ db, err := New(&config.Config{StateDir: b.TempDir()}, logger.New())
+ if err != nil {
+ b.Fatalf("failed to create database: %v", err)
+ }
+ defer func() { _ = db.Close() }()
+
+ //nolint:gosec // The benchmark needs the same table every run.
+ rng := rand.New(rand.NewPCG(1, 2))
+
+ newPeers := make(map[string]uint, benchTablePeers)
+ for number := range uint(benchTablePeers) {
+ newPeers[benchPeerAddress(number)] = number
+ }
+
+ for first := 0; first < benchTablePairs; first += benchTableBatch {
+ changes := make([]routetable.Change, 0, benchTableBatch)
+ for pair := first; pair < first+benchTableBatch; pair++ {
+ changes = append(changes, benchTableChange(rng, pair))
+ }
+
+ err := db.WriteRouteChanges(changes, newPeers)
+ if err != nil {
+ b.Fatalf("WriteRouteChanges: %v", err)
+ }
+
+ newPeers = nil
+ }
+
+ // The WAL is copied into the database file, whose size is reported.
+ err = db.Checkpoint(b.Context())
+ if err != nil {
+ b.Fatalf("Checkpoint: %v", err)
+ }
+
+ file, err := os.Stat(db.path)
+ if err != nil {
+ b.Fatalf("reading the database file's size: %v", err)
+ }
+
+ for b.Loop() {
+ table, err := db.ReadRouteTable(b.Context())
+ if err != nil {
+ b.Fatalf("ReadRouteTable: %v", err)
+ }
+
+ counts := table.Counts()
+ if counts.IPv4Prefixes+counts.IPv6Prefixes != benchTablePairs {
+ b.Fatalf("read %d prefixes, want %d",
+ counts.IPv4Prefixes+counts.IPv6Prefixes, benchTablePairs)
+ }
+ }
+
+ // After the loop: b.Loop drops the metrics reported before it starts.
+ b.ReportMetric(float64(file.Size())/benchTablePairs, "file-bytes/pair")
+}
+
+// benchPeerAddress returns the address of the peer numbered number.
+func benchPeerAddress(number uint) string {
+ const addressesPerBlock = 256
+
+ return fmt.Sprintf("198.18.%d.%d",
+ number/addressesPerBlock, number%addressesPerBlock)
+}
+
+// benchTableChange returns the content of the pair numbered pair: its prefix,
+// its origin AS, benchTablePeersPerPair peers picked at random, and the routes
+// of two of them, each with an AS path of four ASes.
+func benchTableChange(rng *rand.Rand, pair int) routetable.Change {
+ const (
+ firstOrigin = 64500
+ transitASes = 50000
+ olderSeconds = 10
+ )
+
+ text := fmt.Sprintf("%d.%d.%d.0/24", 1+pair>>16, (pair>>8)%256, pair%256)
+ if pair%benchTableV6Every == 0 {
+ text = fmt.Sprintf("2001:%x:%x::/48", pair>>16, pair%65536)
+ }
+
+ prefix := netip.MustParsePrefix(text)
+ origin := firstOrigin + pair%transitASes
+ peers := bitset.New(benchTablePeers)
+
+ for peers.Count() < benchTablePeersPerPair {
+ peers.Set(rng.UintN(benchTablePeers))
+ }
+
+ change := routetable.Change{Prefix: prefix, OriginASN: origin, Peers: peers}
+
+ // The routes of the first two peers, newest first.
+ for i, number := range slices.Collect(peers.EachSet())[:keptRoutesPerRow] {
+ change.Routes = append(change.Routes, routetable.Route{
+ Peer: benchPeerAddress(number),
+ ASPath: []int{
+ firstOrigin + rng.IntN(transitASes),
+ firstOrigin + rng.IntN(transitASes),
+ firstOrigin + rng.IntN(transitASes),
+ origin,
+ },
+ Time: tableTime(pair - i*olderSeconds),
+ })
+ }
+
+ return change
+}
diff --git a/internal/database/schema.sql b/internal/database/schema.sql
index 5c0ca22..180c347 100644
--- a/internal/database/schema.sql
+++ b/internal/database/schema.sql
@@ -89,61 +89,31 @@ CREATE INDEX IF NOT EXISTS idx_bgp_peers_asn ON bgp_peers(peer_asn);
CREATE INDEX IF NOT EXISTS idx_bgp_peers_last_seen ON bgp_peers(last_seen);
CREATE INDEX IF NOT EXISTS idx_bgp_peers_ip ON bgp_peers(peer_ip);
--- IPv4 routing table maintained by PrefixHandler. A route write changes the
--- table and its entry in every index on a column the write sets, so the table
--- has only the indexes that the queries named with them use.
-CREATE TABLE IF NOT EXISTS live_routes_v4 (
- -- The row id, which the prefix-length pages' sample draws through the
- -- mask_length index. Declared as a column, it stays the same for as long
- -- as the route is live, also through a VACUUM.
- id INTEGER PRIMARY KEY,
- prefix TEXT NOT NULL,
- mask_length INTEGER NOT NULL, -- CIDR mask length (0-32)
+-- The whole routing table, as internal/routetable keeps it in memory: one row
+-- for each prefix and origin AS that some peer carries. Startup reads every
+-- row, and the rows of the pairs that change are written back.
+CREATE TABLE IF NOT EXISTS routes (
+ prefix TEXT NOT NULL, -- in the text form Go's net/netip prints
origin_asn INTEGER NOT NULL,
- peer_ip TEXT NOT NULL,
- as_path TEXT NOT NULL, -- JSON array
- next_hop TEXT NOT NULL,
- last_updated DATETIME NOT NULL,
- -- A peer has at most one route for a prefix. This key finds the route a
- -- write updates or deletes and, peer_ip coming first, all of a peer's
- -- routes when its BGP session goes down.
- UNIQUE(peer_ip, prefix)
-);
+ -- The peers whose current route to the prefix has this origin AS: a set
+ -- of their numbers in route_peers, as the bitset package's MarshalBinary
+ -- writes it.
+ peers BLOB NOT NULL,
+ -- The newest of those routes and the one before it, the two kept in full.
+ -- Each has its peer's number in route_peers, its AS path as a JSON array,
+ -- and its time in seconds since 1970, UTC. A route that is not kept has
+ -- NULL in its three columns.
+ newest_peer INTEGER,
+ newest_as_path TEXT,
+ newest_time INTEGER,
+ older_peer INTEGER,
+ older_as_path TEXT,
+ older_time INTEGER,
+ PRIMARY KEY (prefix, origin_asn)
+) WITHOUT ROWID;
--- IPv6 routing table maintained by PrefixHandler, as live_routes_v4.
-CREATE TABLE IF NOT EXISTS live_routes_v6 (
- id INTEGER PRIMARY KEY,
- prefix TEXT NOT NULL,
- mask_length INTEGER NOT NULL, -- CIDR mask length (0-128)
- origin_asn INTEGER NOT NULL,
- peer_ip TEXT NOT NULL,
- as_path TEXT NOT NULL, -- JSON array
- next_hop TEXT NOT NULL,
- last_updated DATETIME NOT NULL,
- UNIQUE(peer_ip, prefix)
-);
-
--- Indexes for both live route tables, and the queries that use each.
---
--- prefix, last_updated: a prefix's routes, for the prefix page and its API
--- answer, the IP lookup and its peer count, and the check whether a prefix
--- still has a live route; and a prefix's newest route for the prefix-length
--- pages and the AS page, in one search whatever the number of peers.
-CREATE INDEX IF NOT EXISTS idx_live_routes_v4_prefix_last_updated ON live_routes_v4(prefix, last_updated);
-CREATE INDEX IF NOT EXISTS idx_live_routes_v6_prefix_last_updated ON live_routes_v6(prefix, last_updated);
--- mask_length: the prefix-length pages' random sample, the row id range of a
--- mask length and the first route of it at or after a row id.
-CREATE INDEX IF NOT EXISTS idx_live_routes_v4_mask_length ON live_routes_v4(mask_length);
-CREATE INDEX IF NOT EXISTS idx_live_routes_v6_mask_length ON live_routes_v6(mask_length);
--- mask_length, prefix: the prefix-length pages' list of a mask length's
--- prefixes in text order, and the prefix distribution read at startup.
-CREATE INDEX IF NOT EXISTS idx_live_routes_v4_mask_prefix ON live_routes_v4(mask_length, prefix);
-CREATE INDEX IF NOT EXISTS idx_live_routes_v6_mask_prefix ON live_routes_v6(mask_length, prefix);
--- origin_asn, prefix: the prefixes an AS originates, each in one search
--- whatever the number of peers, for the AS page and its API answer.
-CREATE INDEX IF NOT EXISTS idx_live_routes_v4_origin_prefix ON live_routes_v4(origin_asn, prefix);
-CREATE INDEX IF NOT EXISTS idx_live_routes_v6_origin_prefix ON live_routes_v6(origin_asn, prefix);
--- last_updated: the oldest and newest route times in /api/v1/stats and
--- /status.json.
-CREATE INDEX IF NOT EXISTS idx_live_routes_v4_last_updated ON live_routes_v4(last_updated);
-CREATE INDEX IF NOT EXISTS idx_live_routes_v6_last_updated ON live_routes_v6(last_updated);
\ No newline at end of file
+-- The peers of the routes table, each with the number it has there.
+CREATE TABLE IF NOT EXISTS route_peers (
+ number INTEGER PRIMARY KEY,
+ address TEXT NOT NULL UNIQUE
+);
\ No newline at end of file
diff --git a/internal/database/utils.go b/internal/database/utils.go
index 4ba0b0d..cf1ff34 100644
--- a/internal/database/utils.go
+++ b/internal/database/utils.go
@@ -1,8 +1,6 @@
package database
import (
- "fmt"
- "net"
"strings"
"github.com/google/uuid"
@@ -20,15 +18,3 @@ func detectIPVersion(prefix string) int {
return ipVersionV4
}
-
-// prefixMaskLength returns the mask length of a prefix such as 192.0.2.0/24.
-func prefixMaskLength(prefix string) (int, error) {
- _, network, err := net.ParseCIDR(prefix)
- if err != nil {
- return 0, fmt.Errorf("invalid prefix %s: %w", prefix, err)
- }
-
- maskLength, _ := network.Mask.Size()
-
- return maskLength, nil
-}
diff --git a/internal/metrics/metrics.go b/internal/metrics/metrics.go
index 657d042..e188bcf 100644
--- a/internal/metrics/metrics.go
+++ b/internal/metrics/metrics.go
@@ -34,7 +34,7 @@ type Tracker struct {
// Announcement/withdrawal metrics
announcementCounter metrics.Counter
withdrawalCounter metrics.Counter
- churnRate metrics.Meter // combined announcements + withdrawals per second
+ churnPerMinute minuteCounter // prefix announcements and withdrawals
// BGP peer tracking
bgpPeerCount atomic.Int32
@@ -56,7 +56,7 @@ func New() *Tracker {
ipv6UpdateRate: metrics.NewMeter(),
announcementCounter: metrics.NewCounter(),
withdrawalCounter: metrics.NewCounter(),
- churnRate: metrics.NewMeter(),
+ churnPerMinute: minuteCounter{now: time.Now},
}
}
@@ -156,13 +156,13 @@ func (t *Tracker) RecordIPv6Update() {
// RecordAnnouncement records a route announcement
func (t *Tracker) RecordAnnouncement() {
t.announcementCounter.Inc(1)
- t.churnRate.Mark(1)
+ t.churnPerMinute.add()
}
// RecordWithdrawal records a route withdrawal
func (t *Tracker) RecordWithdrawal() {
t.withdrawalCounter.Inc(1)
- t.churnRate.Mark(1)
+ t.churnPerMinute.add()
}
// SetBGPPeerCount updates the current BGP peer count
@@ -198,9 +198,10 @@ func (t *Tracker) GetWithdrawalCount() uint64 {
return uint64(count)
}
-// GetChurnRate returns the route churn rate per second
-func (t *Tracker) GetChurnRate() float64 {
- return t.churnRate.Rate1()
+// GetChurnPerMinute returns the number of prefix announcements and
+// withdrawals, from all peers, in the last full clock minute.
+func (t *Tracker) GetChurnPerMinute() int {
+ return t.churnPerMinute.lastFullMinute()
}
// GetRouteMetrics returns current route update metrics
diff --git a/internal/metrics/minutecounter.go b/internal/metrics/minutecounter.go
new file mode 100644
index 0000000..88aeb7b
--- /dev/null
+++ b/internal/metrics/minutecounter.go
@@ -0,0 +1,59 @@
+package metrics
+
+import (
+ "sync"
+ "time"
+)
+
+// minuteCounter counts events in each clock minute and reports the count of
+// the last full minute. It is safe for concurrent use.
+type minuteCounter struct {
+ mu sync.Mutex
+ now func() time.Time
+
+ // minute is the start of the minute that count belongs to.
+ minute time.Time
+ count int
+ // lastMinuteCount is the count of the minute just before minute.
+ lastMinuteCount int
+}
+
+// add counts one event in the current minute.
+func (c *minuteCounter) add() {
+ c.mu.Lock()
+ defer c.mu.Unlock()
+
+ c.moveToCurrentMinute()
+ c.count++
+}
+
+// lastFullMinute returns the number of events in the minute that ended most
+// recently.
+func (c *minuteCounter) lastFullMinute() int {
+ c.mu.Lock()
+ defer c.mu.Unlock()
+
+ c.moveToCurrentMinute()
+
+ return c.lastMinuteCount
+}
+
+// moveToCurrentMinute starts a new count when the clock has moved into
+// another minute. The count it ends becomes the last full minute's count when
+// its minute is the one just before the current minute. Otherwise the last
+// full minute had no events. The caller holds mu.
+func (c *minuteCounter) moveToCurrentMinute() {
+ minute := c.now().Truncate(time.Minute)
+ if minute.Equal(c.minute) {
+ return
+ }
+
+ if minute.Sub(c.minute) == time.Minute {
+ c.lastMinuteCount = c.count
+ } else {
+ c.lastMinuteCount = 0
+ }
+
+ c.minute = minute
+ c.count = 0
+}
diff --git a/internal/metrics/minutecounter_internal_test.go b/internal/metrics/minutecounter_internal_test.go
new file mode 100644
index 0000000..02f0686
--- /dev/null
+++ b/internal/metrics/minutecounter_internal_test.go
@@ -0,0 +1,86 @@
+package metrics
+
+import (
+ "sync"
+ "testing"
+ "time"
+)
+
+// TestChurnPerMinuteCountsLastFullMinute announces and withdraws prefixes on
+// an injected clock and expects each minute's whole count once that minute
+// has ended, and 0 for a minute with none.
+func TestChurnPerMinuteCountsLastFullMinute(t *testing.T) {
+ t.Parallel()
+
+ now := time.Date(2026, 10, 26, 12, 0, 30, 0, time.UTC)
+ tracker := New()
+ tracker.churnPerMinute.now = func() time.Time { return now }
+
+ expectChurn := func(want int, why string) {
+ t.Helper()
+
+ got := tracker.GetChurnPerMinute()
+ if got != want {
+ t.Fatalf("at %s churn = %d, want %d: %s",
+ now.Format(time.TimeOnly), got, want, why)
+ }
+ }
+
+ const firstMinuteChurn = 2
+
+ tracker.RecordAnnouncement()
+ tracker.RecordWithdrawal()
+ expectChurn(0, "12:00 has not ended")
+
+ now = time.Date(2026, 10, 26, 12, 1, 59, 0, time.UTC)
+
+ tracker.RecordAnnouncement()
+ expectChurn(firstMinuteChurn, "12:00 had one announcement and one withdrawal")
+
+ now = time.Date(2026, 10, 26, 12, 2, 0, 0, time.UTC)
+
+ expectChurn(1, "12:01 had one announcement")
+
+ now = time.Date(2026, 10, 26, 12, 4, 0, 0, time.UTC)
+
+ expectChurn(0, "12:03 had none")
+
+ tracker.RecordWithdrawal()
+
+ now = time.Date(2026, 10, 26, 12, 6, 0, 0, time.UTC)
+
+ expectChurn(0, "12:05 had none, though 12:04 had one withdrawal")
+}
+
+// TestChurnPerMinuteCountsConcurrentRecords announces prefixes from several
+// goroutines at once and expects every one counted.
+func TestChurnPerMinuteCountsConcurrentRecords(t *testing.T) {
+ t.Parallel()
+
+ const (
+ goroutines = 8
+ recordsPerGoroutine = 1000
+ )
+
+ now := time.Date(2026, 10, 26, 12, 0, 0, 0, time.UTC)
+ tracker := New()
+ tracker.churnPerMinute.now = func() time.Time { return now }
+
+ var wg sync.WaitGroup
+ for range goroutines {
+ wg.Go(func() {
+ for range recordsPerGoroutine {
+ tracker.RecordAnnouncement()
+ }
+ })
+ }
+
+ wg.Wait()
+
+ now = now.Add(time.Minute)
+
+ want := goroutines * recordsPerGoroutine
+ if got := tracker.GetChurnPerMinute(); got != want {
+ t.Fatalf("churn = %d, want %d", got, want)
+ }
+}
diff --git a/internal/ristypes/ris.go b/internal/ristypes/ris.go
index a33f06a..9c40881 100644
--- a/internal/ristypes/ris.go
+++ b/internal/ristypes/ris.go
@@ -39,12 +39,16 @@ type ASPath struct {
Set []int
}
+// NoOrigin is the origin AS of a route whose origin is not known. AS 0 is
+// reserved, so no AS has this number.
+const NoOrigin = 0
+
// Origin returns the AS that originated the route: the last AS of the
-// sequence, or 0 (none) when the path ends in a set of several members or is
+// sequence, or NoOrigin when the path ends in a set of several members or is
// empty.
func (p *ASPath) Origin() int {
if len(p.Set) > 0 || len(p.Sequence) == 0 {
- return 0
+ return NoOrigin
}
return p.Sequence[len(p.Sequence)-1]
diff --git a/internal/routetable/benchmark_test.go b/internal/routetable/benchmark_test.go
new file mode 100644
index 0000000..be4f2c4
--- /dev/null
+++ b/internal/routetable/benchmark_test.go
@@ -0,0 +1,155 @@
+package routetable_test
+
+import (
+ "fmt"
+ "math"
+ "math/rand/v2"
+ "net/netip"
+ "runtime"
+ "testing"
+ "time"
+
+ "git.eeqj.de/sneak/routewatch/internal/routetable"
+)
+
+// Parameters for BenchmarkApplyMixedBatch.
+const (
+ // benchPeers is how many peers there are, about as many as RIS has.
+ benchPeers = 1400
+ // benchPrefixes is how many IPv4 /24 prefixes the table holds, and
+ // benchPeersPerPrefix how many peers, picked at random, carry each.
+ benchPrefixes = 100000
+ benchPeersPerPrefix = 20
+ // benchBatchSize is how many updates the timed batch holds, as many as
+ // the prefix handler's largest batch (prefixBatchMaxSize in
+ // internal/routewatch).
+ benchBatchSize = 250000
+ // In the timed batch every benchWithdrawalEvery-th update is a
+ // withdrawal, and every benchNewOriginEvery-th announcement has an
+ // origin AS other than the prefix's.
+ benchWithdrawalEvery = 5
+ benchNewOriginEvery = 7
+ // benchOrigins is how many origin ASes the prefixes are spread over,
+ // from benchFirstASN on: 20 prefixes each, about as many as an origin AS
+ // has on average in the full table.
+ benchOrigins = 5000
+ benchFirstASN = 64500
+ // benchUpdatesPerSecond is how many updates share one time.
+ benchUpdatesPerSecond = 1000
+ benchMaskLength = 24
+)
+
+// benchUpdate is one update of the benchmark.
+type benchUpdate struct {
+ prefix netip.Prefix
+ origin int
+ route routetable.Route
+ withdraw bool
+}
+
+// BenchmarkApplyMixedBatch times one batch of 250,000 updates of routes picked
+// at random: announcements, withdrawals and announcements with a new origin AS.
+// The table it applies them to holds 100,000 IPv4 prefixes from 5,000 origin
+// ASes, each carried by 20 of 1,400 peers. It reports the heap bytes that table
+// takes per prefix and origin AS pair. It is run by hand, not by make test, for
+// example in the test image:
+//
+// go test -run '^$' -bench MixedBatch -benchtime 10x ./internal/routetable/
+func BenchmarkApplyMixedBatch(b *testing.B) {
+ //nolint:gosec // The benchmark needs the same updates every run.
+ rng := rand.New(rand.NewPCG(1, 2))
+ peers := make([]string, benchPeers)
+
+ for i := range peers {
+ peers[i] = fmt.Sprintf("198.18.%d.%d", i/256, i%256)
+ }
+
+ before := heapBytes()
+ table := routetable.New(routetable.Params{})
+ updates := 0
+
+ for prefix := range benchPrefixes {
+ for range benchPeersPerPrefix {
+ peer := peers[rng.IntN(benchPeers)]
+ apply(table, benchAnnouncement(rng, prefix, peer, updates, false))
+ updates++
+ }
+ }
+
+ // The record of changes is handed over every second while routewatch
+ // runs, so it is not part of the table's size.
+ table.TakeChanges(math.MaxInt)
+
+ bytesPerPair := float64(heapBytes()-before) / benchPrefixes
+
+ batch := make([]benchUpdate, 0, benchBatchSize)
+ for i := range benchBatchSize {
+ update := benchAnnouncement(rng, rng.IntN(benchPrefixes),
+ peers[rng.IntN(benchPeers)], updates+i, i%benchNewOriginEvery == 0)
+ update.withdraw = i%benchWithdrawalEvery == 0
+ batch = append(batch, update)
+ }
+
+ for b.Loop() {
+ for _, update := range batch {
+ apply(table, update)
+ }
+ }
+
+ // After the loop: b.Loop drops the metrics reported before it starts.
+ b.ReportMetric(bytesPerPair, "bytes/pair")
+}
+
+// benchAnnouncement returns the announcement of the prefix numbered prefix, a
+// /24 from 1.0.0.0/24 on, by peer. It is the updates-th update of the
+// benchmark, and its AS path holds four ASes from the peer's AS to the origin
+// AS. newOrigin gives it an origin AS other than the prefix's.
+func benchAnnouncement(
+ rng *rand.Rand, prefix int, peer string, updates int, newOrigin bool,
+) benchUpdate {
+ origin := benchFirstASN + prefix%benchOrigins
+ if newOrigin {
+ origin += benchOrigins
+ }
+
+ start := time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC)
+
+ return benchUpdate{
+ prefix: netip.MustParsePrefix(fmt.Sprintf("%d.%d.%d.0/%d",
+ 1+prefix>>16, (prefix>>8)%256, prefix%256, benchMaskLength)),
+ origin: origin,
+ route: routetable.Route{
+ Peer: peer,
+ ASPath: []int{
+ benchFirstASN + rng.IntN(benchOrigins),
+ benchFirstASN + rng.IntN(benchOrigins),
+ benchFirstASN + rng.IntN(benchOrigins),
+ origin,
+ },
+ Time: start.Add(time.Duration(updates/benchUpdatesPerSecond) *
+ time.Second),
+ },
+ }
+}
+
+// apply applies update to table.
+func apply(table *routetable.Table, update benchUpdate) {
+ if update.withdraw {
+ table.Withdraw(update.prefix, update.route.Peer)
+
+ return
+ }
+
+ table.Announce(update.prefix, update.origin, update.route)
+}
+
+// heapBytes returns the bytes of the live heap objects, after a collection.
+func heapBytes() uint64 {
+ runtime.GC()
+
+ var stats runtime.MemStats
+
+ runtime.ReadMemStats(&stats)
+
+ return stats.HeapAlloc
+}
diff --git a/internal/routetable/reads.go b/internal/routetable/reads.go
new file mode 100644
index 0000000..e108984
--- /dev/null
+++ b/internal/routetable/reads.go
@@ -0,0 +1,233 @@
+package routetable
+
+import (
+ "net/netip"
+ "slices"
+ "time"
+
+ "github.com/bits-and-blooms/bitset"
+)
+
+// PrefixSummary is what the table answers about one prefix.
+type PrefixSummary struct {
+ Prefix netip.Prefix
+ // OriginASN is the origin AS that the most peers carry the prefix with.
+ // When several have as many peers, it is the lowest AS number.
+ OriginASN int
+ // Peers is how many peers carry the prefix.
+ Peers int
+ // Newest is the time of the newest kept route of the prefix, or nil when
+ // it keeps none.
+ Newest *time.Time
+}
+
+// OriginatedPrefix is one prefix an AS originates.
+type OriginatedPrefix struct {
+ Prefix netip.Prefix
+ // Newest is the time of the newest kept route of the prefix with that
+ // origin AS, or nil when it keeps none.
+ Newest *time.Time
+}
+
+// Counts are the table's figures for the stats.
+type Counts struct {
+ // Pairs counts the prefix and origin AS pairs that some peer carries.
+ Pairs int
+ // Peers counts the peers that have a number: those given in
+ // Params.PeerNumbers and those numbered since.
+ Peers int
+ IPv4Prefixes int
+ IPv6Prefixes int
+ // IPv4PrefixesByLength[n] and IPv6PrefixesByLength[n] are how many
+ // prefixes of mask length n some peer carries.
+ IPv4PrefixesByLength []int
+ IPv6PrefixesByLength []int
+ // IPv4Routes and IPv6Routes count peer and prefix pairs: each peer that
+ // carries a prefix counts once for it.
+ IPv4Routes int
+ IPv6Routes int
+ // OriginsWithoutKeptRoute counts the prefix and origin AS pairs that peers
+ // carry but whose kept routes were all withdrawn.
+ OriginsWithoutKeptRoute int
+ // OldestKeptRoute and NewestKeptRoute are the times of the oldest and the
+ // newest kept routes, or nil when the table keeps none.
+ OldestKeptRoute *time.Time
+ NewestKeptRoute *time.Time
+}
+
+// Lookup returns the most specific prefix that holds addr and that some peer
+// carries. It ignores prefixes shorter than /8 for IPv4 and /16 for IPv6. It
+// makes one map lookup per mask length, and reports false when no prefix
+// holds addr.
+func (t *Table) Lookup(addr netip.Addr) (PrefixSummary, bool) {
+ t.mu.RLock()
+ defer t.mu.RUnlock()
+
+ // An IPv4 address written in IPv6 form (::ffff:192.0.2.1) is looked up as
+ // IPv4.
+ addr = addr.Unmap()
+ f := t.familyOf(addr)
+
+ for bits := addr.BitLen(); bits >= f.minLookupLength; bits-- {
+ prefix := netip.PrefixFrom(addr, bits).Masked()
+
+ origins, found := f.byLength[bits][prefix]
+ if found {
+ return summarize(prefix, origins), true
+ }
+ }
+
+ return PrefixSummary{}, false
+}
+
+// OriginatedPrefixes returns the prefixes AS asn originates, in prefix order.
+func (t *Table) OriginatedPrefixes(asn int) []OriginatedPrefix {
+ t.mu.RLock()
+ defer t.mu.RUnlock()
+
+ prefixes := make([]OriginatedPrefix, 0, len(t.byOrigin[asn]))
+ for prefix, o := range t.byOrigin[asn] {
+ prefixes = append(prefixes,
+ OriginatedPrefix{Prefix: prefix, Newest: o.newestTime()})
+ }
+
+ slices.SortFunc(prefixes, func(a, b OriginatedPrefix) int {
+ return a.Prefix.Compare(b.Prefix)
+ })
+
+ return prefixes
+}
+
+// Origins returns each origin AS of prefix, in AS number order, with its peers
+// and its kept routes. It returns nil when no peer carries the prefix.
+func (t *Table) Origins(prefix netip.Prefix) []Origin {
+ t.mu.RLock()
+ defer t.mu.RUnlock()
+
+ if !prefix.IsValid() {
+ return nil
+ }
+
+ var origins []Origin
+
+ for _, o := range t.origins(prefix.Masked()) {
+ origins = append(origins, Origin{
+ ASN: o.asn,
+ Peers: t.addresses(&o.peers),
+ Routes: t.exportRoutes(o.routes),
+ })
+ }
+
+ return origins
+}
+
+// SamplePrefixes returns up to limit of the prefixes of IP version ipVersion
+// (4 or 6) and mask length maskLength. It reads only the prefixes it returns.
+// Go starts each walk over a map at a random place, so each call returns
+// another sample.
+func (t *Table) SamplePrefixes(ipVersion, maskLength, limit int) []PrefixSummary {
+ t.mu.RLock()
+ defer t.mu.RUnlock()
+
+ var f *family
+
+ switch ipVersion {
+ case ipVersion4:
+ f = &t.ipv4
+ case ipVersion6:
+ f = &t.ipv6
+ default:
+ return nil
+ }
+
+ if maskLength < 0 || maskLength >= len(f.byLength) || limit <= 0 {
+ return nil
+ }
+
+ prefixes := f.byLength[maskLength]
+ sample := make([]PrefixSummary, 0, min(limit, len(prefixes)))
+
+ for prefix, origins := range prefixes {
+ if len(sample) == limit {
+ break
+ }
+
+ sample = append(sample, summarize(prefix, origins))
+ }
+
+ return sample
+}
+
+// Counts returns the table's figures for the stats. The writes keep them up to
+// date, so this reads no prefix.
+func (t *Table) Counts() Counts {
+ t.mu.RLock()
+ defer t.mu.RUnlock()
+
+ counts := Counts{
+ Peers: len(t.peerNumbers),
+ IPv4PrefixesByLength: t.ipv4.prefixesByLength(),
+ IPv6PrefixesByLength: t.ipv6.prefixesByLength(),
+ IPv4Routes: t.ipv4.routes,
+ IPv6Routes: t.ipv6.routes,
+ OriginsWithoutKeptRoute: t.originsWithoutKeptRoute,
+ }
+
+ for _, n := range counts.IPv4PrefixesByLength {
+ counts.IPv4Prefixes += n
+ }
+
+ for _, n := range counts.IPv6PrefixesByLength {
+ counts.IPv6Prefixes += n
+ }
+
+ for _, prefixes := range t.byOrigin {
+ counts.Pairs += len(prefixes)
+ }
+
+ oldest, found := t.keptTimes.Min()
+ if found {
+ counts.OldestKeptRoute = &oldest.time
+ }
+
+ newest, found := t.keptTimes.Max()
+ if found {
+ counts.NewestKeptRoute = &newest.time
+ }
+
+ return counts
+}
+
+// addresses returns the addresses of the peers in peers, in number order.
+func (t *Table) addresses(peers *bitset.BitSet) []string {
+ addresses := make([]string, 0, peers.Count())
+ for number := range peers.EachSet() {
+ addresses = append(addresses, t.peerAddresses[number])
+ }
+
+ return addresses
+}
+
+// summarize returns the summary of prefix, whose origins are in AS number
+// order.
+func summarize(prefix netip.Prefix, origins []*origin) PrefixSummary {
+ summary := PrefixSummary{Prefix: prefix}
+ mostPeers := 0
+
+ for _, o := range origins {
+ peers := int(o.peers.Count())
+ summary.Peers += peers
+
+ if peers > mostPeers {
+ mostPeers = peers
+ summary.OriginASN = o.asn
+ }
+
+ newest := o.newestTime()
+ if newest != nil && (summary.Newest == nil || newest.After(*summary.Newest)) {
+ summary.Newest = newest
+ }
+ }
+
+ return summary
+}
diff --git a/internal/routetable/reads_test.go b/internal/routetable/reads_test.go
new file mode 100644
index 0000000..c3f8288
--- /dev/null
+++ b/internal/routetable/reads_test.go
@@ -0,0 +1,265 @@
+package routetable_test
+
+import (
+ "net/netip"
+ "reflect"
+ "testing"
+
+ "git.eeqj.de/sneak/routewatch/internal/routetable"
+)
+
+// announceAll has each listed peer announce prefix with origin AS asn, the
+// peer at index i i*10 seconds after start.
+func announceAll(
+ table *routetable.Table, prefix string, asn, start int, peers ...string,
+) {
+ const secondsBetweenPeers = 10
+
+ for i, peer := range peers {
+ table.Announce(netip.MustParsePrefix(prefix), asn,
+ routeOf(peer, asn, start+i*secondsBetweenPeers))
+ }
+}
+
+func TestLookup(t *testing.T) {
+ t.Parallel()
+
+ table := routetable.New(routetable.Params{})
+ announceAll(table, "10.0.0.0/8", asA, 10, peerA)
+ // Three peers with two origins: asB has more of them.
+ announceAll(table, "10.1.0.0/16", asA, 10, peerA)
+ announceAll(table, "10.1.0.0/16", asB, 20, peerB, peerC)
+ // Two origins with one peer each: the lower AS number wins.
+ announceAll(table, prefixV4, asB, 20, peerB)
+ announceAll(table, prefixV4, asA, 10, peerA)
+ // An origin whose kept routes were withdrawn: no newest time.
+ announceAll(table, "10.9.0.0/16", asA, 10, peerA, peerB, peerC)
+ table.Withdraw(netip.MustParsePrefix("10.9.0.0/16"), peerB)
+ table.Withdraw(netip.MustParsePrefix("10.9.0.0/16"), peerC)
+ announceAll(table, "2001:db8::/32", asA, 10, peerA)
+ announceAll(table, prefixV6, asB, 20, peerB)
+ announceAll(table, "3000::/16", asA, 10, peerA)
+ // Shorter than the lookup uses.
+ announceAll(table, "192.0.0.0/7", asA, 10, peerA)
+ announceAll(table, "2000::/15", asA, 10, peerA)
+
+ tests := []struct {
+ addr string
+ // want is the summary's prefix, or empty when none is found.
+ want string
+ originASN int
+ peers int
+ newest int // in seconds after the start; 0 for none
+ }{
+ {"10.1.2.3", prefixV4, asA, 2, 20},
+ {"::ffff:10.1.2.3", prefixV4, asA, 2, 20},
+ {"10.1.3.4", "10.1.0.0/16", asB, 3, 30},
+ {"10.2.0.1", "10.0.0.0/8", asA, 1, 10},
+ {"10.9.0.1", "10.9.0.0/16", asA, 1, 0},
+ {"192.0.2.1", "", 0, 0, 0},
+ {"2001:db8:1::5", prefixV6, asB, 1, 20},
+ {"2001:db8:2::1", "2001:db8::/32", asA, 1, 10},
+ {"3000::1", "3000::/16", asA, 1, 10},
+ {"2001:db9::1", "", 0, 0, 0},
+ }
+
+ for _, tc := range tests {
+ t.Run(tc.addr, func(t *testing.T) {
+ t.Parallel()
+
+ got, found := table.Lookup(netip.MustParseAddr(tc.addr))
+ if tc.want == "" {
+ if found {
+ t.Errorf("Lookup(%s) = %+v, want none", tc.addr, got)
+ }
+
+ return
+ }
+
+ want := routetable.PrefixSummary{
+ Prefix: netip.MustParsePrefix(tc.want),
+ OriginASN: tc.originASN,
+ Peers: tc.peers,
+ }
+ if tc.newest != 0 {
+ want.Newest = timeAt(tc.newest)
+ }
+
+ if !found || !reflect.DeepEqual(got, want) {
+ t.Errorf("Lookup(%s) = %+v, %v; want %+v", tc.addr, got, found, want)
+ }
+ })
+ }
+}
+
+func TestOriginatedPrefixes(t *testing.T) {
+ t.Parallel()
+
+ table := routetable.New(routetable.Params{})
+ announceAll(table, "2001:db8::/32", asA, 30, peerA)
+ announceAll(table, prefixV4, asA, 10, peerA)
+ // asB's newer route for the same prefix does not count for asA.
+ announceAll(table, prefixV4, asB, 40, peerB)
+ announceAll(table, "10.1.0.0/16", asA, 10, peerA, peerB)
+ announceAll(table, "10.9.0.0/16", asA, 10, peerA, peerB, peerC)
+ table.Withdraw(netip.MustParsePrefix("10.9.0.0/16"), peerB)
+ table.Withdraw(netip.MustParsePrefix("10.9.0.0/16"), peerC)
+
+ want := []routetable.OriginatedPrefix{
+ {Prefix: netip.MustParsePrefix("10.1.0.0/16"), Newest: timeAt(20)},
+ {Prefix: netip.MustParsePrefix(prefixV4), Newest: timeAt(10)},
+ {Prefix: netip.MustParsePrefix("10.9.0.0/16")},
+ {Prefix: netip.MustParsePrefix("2001:db8::/32"), Newest: timeAt(30)},
+ }
+
+ got := table.OriginatedPrefixes(asA)
+ if !reflect.DeepEqual(got, want) {
+ t.Errorf("OriginatedPrefixes(%d) = %+v, want %+v", asA, got, want)
+ }
+
+ got = table.OriginatedPrefixes(asC)
+ if len(got) != 0 {
+ t.Errorf("OriginatedPrefixes(%d) = %+v, want none", asC, got)
+ }
+}
+
+func TestSamplePrefixes(t *testing.T) {
+ t.Parallel()
+
+ table := routetable.New(routetable.Params{})
+
+ announced := []string{
+ "10.0.0.0/24", "10.0.1.0/24", "10.0.2.0/24", "10.0.3.0/24",
+ "10.0.4.0/24", "10.0.0.0/16",
+ "2001:db8::/48", "2001:db8:1::/48", "2001:db8:2::/48",
+ }
+ for _, prefix := range announced {
+ announceAll(table, prefix, asA, 10, peerA)
+ }
+
+ tests := []struct {
+ name string
+ ipVersion, maskLength, limit int
+ want int
+ }{
+ {"fewer than there are", 4, 24, 3, 3},
+ {"more than there are", 4, 24, 10, 5},
+ {"IPv6", 6, 48, 10, 3},
+ {"a mask length with none", 4, 20, 10, 0},
+ {"no such IP version", 5, 24, 10, 0},
+ {"a mask length too long", 4, 33, 10, 0},
+ {"a negative mask length", 4, -1, 10, 0},
+ {"a limit of zero", 4, 24, 0, 0},
+ }
+
+ for _, tc := range tests {
+ t.Run(tc.name, func(t *testing.T) {
+ t.Parallel()
+
+ sample := table.SamplePrefixes(tc.ipVersion, tc.maskLength, tc.limit)
+ if len(sample) != tc.want {
+ t.Fatalf("SamplePrefixes(%d, %d, %d) returned %d prefixes, want %d",
+ tc.ipVersion, tc.maskLength, tc.limit, len(sample), tc.want)
+ }
+
+ seen := make(map[netip.Prefix]bool)
+
+ for _, got := range sample {
+ want := routetable.PrefixSummary{
+ Prefix: got.Prefix, OriginASN: asA, Peers: 1, Newest: timeAt(10),
+ }
+ if seen[got.Prefix] || got.Prefix.Bits() != tc.maskLength ||
+ !reflect.DeepEqual(got, want) {
+ t.Errorf("SamplePrefixes(%d, %d, %d) returned %+v",
+ tc.ipVersion, tc.maskLength, tc.limit, got)
+ }
+
+ seen[got.Prefix] = true
+ }
+ })
+ }
+}
+
+func TestCounts(t *testing.T) {
+ t.Parallel()
+
+ table := routetable.New(routetable.Params{})
+ checkCounts(t, table, routetable.Counts{
+ IPv4PrefixesByLength: byLength(ipv4Bits, nil),
+ IPv6PrefixesByLength: byLength(ipv6Bits, nil),
+ })
+
+ announceAll(table, prefixV4, asA, 10, peerA, peerB)
+ announceAll(table, "10.2.0.0/16", asA, 15, peerA)
+ announceAll(table, "2001:db8::/32", asA, 5, peerA)
+ announceAll(table, "2001:db8::/32", asA, 25, peerC)
+ checkCounts(t, table, routetable.Counts{
+ Pairs: 3,
+ Peers: 3,
+ IPv4Prefixes: 2,
+ IPv6Prefixes: 1,
+ IPv4PrefixesByLength: byLength(ipv4Bits, map[int]int{16: 1, 24: 1}),
+ IPv6PrefixesByLength: byLength(ipv6Bits, map[int]int{32: 1}),
+ IPv4Routes: 3,
+ IPv6Routes: 2,
+ OldestKeptRoute: timeAt(5),
+ NewestKeptRoute: timeAt(25),
+ })
+
+ // peerC's route drops peerA's from the kept ones, and peerA leaves the
+ // IPv6 prefix.
+ announceAll(table, prefixV4, asA, 30, peerC)
+ table.Withdraw(netip.MustParsePrefix("2001:db8::/32"), peerA)
+ checkCounts(t, table, routetable.Counts{
+ Pairs: 3,
+ Peers: 3,
+ IPv4Prefixes: 2,
+ IPv6Prefixes: 1,
+ IPv4PrefixesByLength: byLength(ipv4Bits, map[int]int{16: 1, 24: 1}),
+ IPv6PrefixesByLength: byLength(ipv6Bits, map[int]int{32: 1}),
+ IPv4Routes: 4,
+ IPv6Routes: 1,
+ OldestKeptRoute: timeAt(15),
+ NewestKeptRoute: timeAt(30),
+ })
+
+ // peerC was the last peer of the IPv6 prefix. It keeps its number.
+ table.PeerDown(peerC)
+ checkCounts(t, table, routetable.Counts{
+ Pairs: 2,
+ Peers: 3,
+ IPv4Prefixes: 2,
+ IPv4PrefixesByLength: byLength(ipv4Bits, map[int]int{16: 1, 24: 1}),
+ IPv6PrefixesByLength: byLength(ipv6Bits, nil),
+ IPv4Routes: 3,
+ OldestKeptRoute: timeAt(15),
+ NewestKeptRoute: timeAt(20),
+ })
+}
+
+// The address lengths of the two IP versions.
+const (
+ ipv4Bits = 32
+ ipv6Bits = 128
+)
+
+// byLength returns the prefix counts by mask length, for addresses of bits
+// bits, with counts[n] prefixes of mask length n and none of any other.
+func byLength(bits int, counts map[int]int) []int {
+ all := make([]int, bits+1)
+ for n, count := range counts {
+ all[n] = count
+ }
+
+ return all
+}
+
+// checkCounts checks that the table's counts are want.
+func checkCounts(t *testing.T, table *routetable.Table, want routetable.Counts) {
+ t.Helper()
+
+ got := table.Counts()
+ if !reflect.DeepEqual(got, want) {
+ t.Errorf("Counts() = %+v, want %+v", got, want)
+ }
+}
diff --git a/internal/routetable/routetable.go b/internal/routetable/routetable.go
new file mode 100644
index 0000000..2a737c5
--- /dev/null
+++ b/internal/routetable/routetable.go
@@ -0,0 +1,677 @@
+// Package routetable holds the whole routing table in memory. For each prefix
+// and origin AS, it keeps the peers whose current route to the prefix has that
+// origin AS. It also keeps the two newest of those routes in full. It applies
+// announcements, withdrawals and peers' sessions going down. It answers the
+// route reads that the pages and the API need.
+package routetable
+
+import (
+ "cmp"
+ "maps"
+ "net/netip"
+ "slices"
+ "sync"
+ "time"
+
+ "github.com/bits-and-blooms/bitset"
+ "github.com/google/btree"
+)
+
+const (
+ // keptRoutesPerOrigin is how many routes of each prefix and origin AS are
+ // kept in full: the newest ones. Every other peer that carries the prefix
+ // with that origin AS keeps only its bit in the peer set.
+ keptRoutesPerOrigin = 2
+
+ // keptTimesDegree is the degree of the B-tree that holds the times of the
+ // kept routes in order: each of its nodes holds up to 63 times.
+ keptTimesDegree = 32
+
+ ipv4Bits = 32
+ ipv6Bits = 128
+
+ // The address lookup ignores prefixes shorter than these mask lengths. A
+ // default route or a route this short only says where a peer sends traffic
+ // it has no better route for; it does not say who holds the address.
+ minLookupMaskLengthV4 = 8
+ minLookupMaskLengthV6 = 16
+
+ // The IP versions SamplePrefixes takes.
+ ipVersion4 = 4
+ ipVersion6 = 6
+)
+
+// Route is one peer's route to one prefix. The next hop is the peer itself, so
+// it is not stored.
+type Route struct {
+ // Peer is the peer's address.
+ Peer string
+ ASPath []int
+ // Time is when the peer announced the route.
+ Time time.Time
+}
+
+// Origin is one origin AS of a prefix: the peers whose current route to the
+// prefix has that origin AS, and the newest of those routes in full.
+type Origin struct {
+ // ASN is the origin AS. 0 means no known origin: the path ends in an AS
+ // set of several members.
+ ASN int
+ // Peers are the peers' addresses, in the order of their numbers: the
+ // order they were first seen in.
+ Peers []string
+ // Routes are the kept routes, newest first: at most two, and none when
+ // both were withdrawn while other peers still carry the prefix.
+ Routes []Route
+}
+
+// Change is the current content of one prefix and origin AS pair that changed:
+// what a database row for the pair holds.
+type Change struct {
+ Prefix netip.Prefix
+ OriginASN int
+ // Removed says that no peer carries the prefix with this origin AS any
+ // more. Peers and Routes are then nil.
+ Removed bool
+ // Peers holds the number of each peer that carries the prefix with this
+ // origin AS.
+ Peers *bitset.BitSet
+ // Routes are the kept routes, newest first.
+ Routes []Route
+}
+
+// Params configures New.
+type Params struct {
+ // PeerNumbers holds the number each peer was given before, by address. A
+ // peer not in it gets the next number above all of them the first time it
+ // announces a route.
+ PeerNumbers map[string]uint
+}
+
+// Table is the routing table. Its methods are safe for concurrent use.
+type Table struct {
+ mu sync.RWMutex
+
+ ipv4 family
+ ipv6 family
+ // byOrigin holds, for each origin AS, the prefixes it originates.
+ byOrigin map[int]map[netip.Prefix]*origin
+
+ // peerNumbers and peerAddresses give each peer's number by its address
+ // and its address by its number. nextPeerNumber is the number the next
+ // new peer gets.
+ peerNumbers map[string]uint
+ peerAddresses map[uint]string
+ nextPeerNumber uint
+ // peerPrefixes[n] counts the prefixes the peer numbered n carries, for
+ // each n below nextPeerNumber. It is a slice, not a map, because Load
+ // counts once for every peer of every pair it loads: with a map, reading
+ // a full table at startup took several times as long.
+ peerPrefixes []int
+
+ // newPeers and changed are what TakeChanges hands over: the peers
+ // numbered since it last ran, and the prefix and origin AS pairs changed
+ // since it last took them.
+ newPeers map[string]uint
+ changed map[pair]struct{}
+
+ // keptTimes counts the kept routes at each time, in time order, so its
+ // first and last items are the oldest and newest kept routes.
+ keptTimes *btree.BTreeG[keptTime]
+ // originsWithoutKeptRoute counts the origins that peers carry but that
+ // keep no route.
+ originsWithoutKeptRoute int
+}
+
+// New returns an empty table that numbers new peers on from
+// params.PeerNumbers.
+func New(params Params) *Table {
+ t := &Table{
+ ipv4: newFamily(ipv4Bits, minLookupMaskLengthV4),
+ ipv6: newFamily(ipv6Bits, minLookupMaskLengthV6),
+ byOrigin: make(map[int]map[netip.Prefix]*origin),
+ peerNumbers: make(map[string]uint, len(params.PeerNumbers)),
+ peerAddresses: make(map[uint]string, len(params.PeerNumbers)),
+ newPeers: make(map[string]uint),
+ changed: make(map[pair]struct{}),
+ keptTimes: btree.NewG(keptTimesDegree, func(a, b keptTime) bool {
+ return a.time.Before(b.time)
+ }),
+ }
+
+ for address, number := range params.PeerNumbers {
+ t.peerNumbers[address] = number
+ t.peerAddresses[number] = address
+ t.nextPeerNumber = max(t.nextPeerNumber, number+1)
+ }
+
+ t.peerPrefixes = make([]int, t.nextPeerNumber)
+
+ return t
+}
+
+// Announce applies an announcement: the peer's current route to prefix is now
+// r, with origin AS originASN. A peer that carried the prefix with another
+// origin AS leaves that origin's peer set, and its route there is dropped. r
+// is kept in full when it is one of the two newest routes of its prefix and
+// origin AS; otherwise only the peer's bit is.
+//
+// An announcement whose path is empty or holds AS 0 is a withdrawal, and the
+// caller passes it to Withdraw instead.
+func (t *Table) Announce(prefix netip.Prefix, originASN int, r Route) {
+ t.mu.Lock()
+ defer t.mu.Unlock()
+
+ if !prefix.IsValid() {
+ return
+ }
+
+ prefix = prefix.Masked()
+ peer := t.numberPeer(r.Peer)
+
+ current := originWithPeer(t.origins(prefix), peer)
+ if current != nil && current.asn != originASN {
+ t.leave(prefix, current, peer)
+ }
+
+ t.join(prefix, t.findOrAddOrigin(prefix, originASN), route{
+ peer: peer,
+ path: slices.Clone(r.ASPath),
+ time: r.Time,
+ })
+}
+
+// Withdraw applies a withdrawal: the peer no longer carries prefix. It leaves
+// the prefix's peer set, and its kept route, if any, is dropped.
+func (t *Table) Withdraw(prefix netip.Prefix, peer string) {
+ t.mu.Lock()
+ defer t.mu.Unlock()
+
+ number, known := t.peerNumbers[peer]
+ if !known || !prefix.IsValid() {
+ return
+ }
+
+ prefix = prefix.Masked()
+
+ o := originWithPeer(t.origins(prefix), number)
+ if o != nil {
+ t.leave(prefix, o, number)
+ }
+}
+
+// PeerDown applies a peer's session going down: the peer leaves every peer
+// set, and every route of it is dropped. When the peer carries any prefix, it
+// visits every prefix in the table.
+func (t *Table) PeerDown(peer string) {
+ t.mu.Lock()
+ defer t.mu.Unlock()
+
+ number, known := t.peerNumbers[peer]
+ if !known || t.peerPrefixes[number] == 0 {
+ return
+ }
+
+ for _, f := range []*family{&t.ipv4, &t.ipv6} {
+ for _, prefixes := range f.byLength {
+ for prefix, origins := range prefixes {
+ o := originWithPeer(origins, number)
+ if o != nil {
+ t.leave(prefix, o, number)
+ }
+ }
+ }
+ }
+}
+
+// Load puts one prefix and origin AS pair that the database holds into the
+// table, as startup does with each row it reads. The change's peers join the
+// pair's peer set, and its routes are kept as an announcement's would be.
+// Unlike an update, Load records no change, so TakeChanges does not hand the
+// pair back to be written again. Loading a pair twice changes nothing.
+//
+// The caller makes sure that every peer change names, in its peer set or in
+// its routes, has a number in Params.PeerNumbers.
+func (t *Table) Load(change Change) {
+ t.mu.Lock()
+ defer t.mu.Unlock()
+
+ // A removed pair has no peers, and a pair without peers is not in the
+ // table.
+ if change.Peers == nil || change.Peers.None() || !change.Prefix.IsValid() {
+ return
+ }
+
+ prefix := change.Prefix.Masked()
+ o := t.findOrAddOrigin(prefix, change.OriginASN)
+ wasWithoutKeptRoute := o.withoutKeptRoute()
+
+ // The peers not yet in the peer set join it in one union, which takes
+ // less time than setting their bits one at a time.
+ joining := change.Peers.Difference(&o.peers)
+ o.peers.InPlaceUnion(joining)
+ t.familyOf(prefix.Addr()).routes += int(joining.Count())
+
+ for peer := range joining.EachSet() {
+ t.peerPrefixes[peer]++
+ }
+
+ // Oldest first, as they were announced, so that two routes with the same
+ // time are kept in the order they had.
+ for _, r := range slices.Backward(change.Routes) {
+ peer := t.peerNumbers[r.Peer]
+
+ t.dropKeptRoute(o, peer)
+ t.keepIfNewest(o, route{
+ peer: peer,
+ path: slices.Clone(r.ASPath),
+ time: r.Time,
+ })
+ }
+
+ t.countOriginWithoutKeptRoute(o, wasWithoutKeptRoute)
+}
+
+// TakeChanges returns at most maxChanges of the prefix and origin AS pairs that
+// changed since they were last taken, in prefix order, each with its current
+// content. The other changed pairs wait for a later call. It also returns every
+// peer numbered since it last ran, by address, so a pair taken later names only
+// peers already returned. It then forgets what it returned.
+func (t *Table) TakeChanges(maxChanges int) ([]Change, map[string]uint) {
+ t.mu.Lock()
+ defer t.mu.Unlock()
+
+ changes := make([]Change, 0, min(maxChanges, len(t.changed)))
+ for key := range t.changed {
+ if len(changes) == maxChanges {
+ break
+ }
+
+ changes = append(changes, t.currentContent(key))
+ delete(t.changed, key)
+ }
+
+ // A map keeps the memory of its largest size, so an emptied one is
+ // replaced.
+ if len(t.changed) == 0 {
+ t.changed = make(map[pair]struct{})
+ }
+
+ slices.SortFunc(changes, func(a, b Change) int {
+ return cmp.Or(a.Prefix.Compare(b.Prefix),
+ cmp.Compare(a.OriginASN, b.OriginASN))
+ })
+
+ newPeers := t.newPeers
+ t.newPeers = make(map[string]uint)
+
+ return changes, newPeers
+}
+
+// PutBackChanges records again what one TakeChanges call returned, for a
+// caller that could not write it. The next TakeChanges returns those peers
+// again, and those pairs wait to be taken again like any changed pair. Each is
+// then taken with its content at that time, so a later change of a pair
+// replaces the earlier.
+func (t *Table) PutBackChanges(changes []Change, newPeers map[string]uint) {
+ t.mu.Lock()
+ defer t.mu.Unlock()
+
+ for _, change := range changes {
+ t.changed[pair{prefix: change.Prefix, asn: change.OriginASN}] = struct{}{}
+ }
+
+ maps.Copy(t.newPeers, newPeers)
+}
+
+// ChangeCount returns how many changed prefix and origin AS pairs wait for
+// TakeChanges.
+func (t *Table) ChangeCount() int {
+ t.mu.RLock()
+ defer t.mu.RUnlock()
+
+ return len(t.changed)
+}
+
+// numberPeer returns the peer's number, and gives it the next one the first
+// time the table sees it.
+func (t *Table) numberPeer(address string) uint {
+ number, known := t.peerNumbers[address]
+ if known {
+ return number
+ }
+
+ number = t.nextPeerNumber
+ t.nextPeerNumber++
+ t.peerNumbers[address] = number
+ t.peerAddresses[number] = address
+ t.peerPrefixes = append(t.peerPrefixes, 0)
+ t.newPeers[address] = number
+
+ return number
+}
+
+// familyOf returns the part of the table for addr's IP version.
+func (t *Table) familyOf(addr netip.Addr) *family {
+ if addr.Is4() {
+ return &t.ipv4
+ }
+
+ return &t.ipv6
+}
+
+// prefixesOfLength returns the prefixes of the IP version and the mask length
+// of prefix, a valid masked prefix.
+func (t *Table) prefixesOfLength(prefix netip.Prefix) map[netip.Prefix][]*origin {
+ return t.familyOf(prefix.Addr()).byLength[prefix.Bits()]
+}
+
+// origins returns the origins of prefix, a valid masked prefix.
+func (t *Table) origins(prefix netip.Prefix) []*origin {
+ return t.prefixesOfLength(prefix)[prefix]
+}
+
+// findOrAddOrigin returns the origin of prefix with AS asn, added with no
+// peers when the prefix has none with that AS.
+func (t *Table) findOrAddOrigin(prefix netip.Prefix, asn int) *origin {
+ prefixes := t.prefixesOfLength(prefix)
+ origins := prefixes[prefix]
+
+ i, found := slices.BinarySearchFunc(origins, asn,
+ func(o *origin, asn int) int { return cmp.Compare(o.asn, asn) })
+ if found {
+ return origins[i]
+ }
+
+ o := &origin{asn: asn}
+ prefixes[prefix] = slices.Insert(origins, i, o)
+
+ if t.byOrigin[asn] == nil {
+ t.byOrigin[asn] = make(map[netip.Prefix]*origin)
+ }
+
+ t.byOrigin[asn][prefix] = o
+
+ return o
+}
+
+// removeOrigin removes o, which no peer carries any more, from prefix, and the
+// prefix from the table when it has no origin left.
+func (t *Table) removeOrigin(prefix netip.Prefix, o *origin) {
+ prefixes := t.prefixesOfLength(prefix)
+
+ origins := slices.DeleteFunc(prefixes[prefix],
+ func(other *origin) bool { return other == o })
+ if len(origins) == 0 {
+ delete(prefixes, prefix)
+ } else {
+ prefixes[prefix] = origins
+ }
+
+ delete(t.byOrigin[o.asn], prefix)
+
+ if len(t.byOrigin[o.asn]) == 0 {
+ delete(t.byOrigin, o.asn)
+ }
+}
+
+// join puts the peer of r in the peer set of o, an origin of prefix, and keeps
+// r in full when it is one of the two newest routes of o. The peer's earlier
+// route, if o keeps one, is replaced: it is no longer the peer's current route.
+// The pair is recorded as changed only when its peer set or kept routes change.
+func (t *Table) join(prefix netip.Prefix, o *origin, r route) {
+ if slices.ContainsFunc(o.routes, r.equal) {
+ // The peer announced its kept route again.
+ return
+ }
+
+ wasWithoutKeptRoute := o.withoutKeptRoute()
+
+ joined := !o.peers.Test(r.peer)
+ if joined {
+ o.peers.Set(r.peer)
+
+ t.familyOf(prefix.Addr()).routes++
+ t.peerPrefixes[r.peer]++
+ }
+
+ dropped := t.dropKeptRoute(o, r.peer)
+ kept := t.keepIfNewest(o, r)
+
+ if joined || dropped || kept {
+ t.recordChange(prefix, o, wasWithoutKeptRoute)
+ }
+}
+
+// leave takes peer out of the peer set of o, an origin of prefix, and drops
+// its kept route, if any.
+func (t *Table) leave(prefix netip.Prefix, o *origin, peer uint) {
+ wasWithoutKeptRoute := o.withoutKeptRoute()
+
+ o.peers.Clear(peer)
+ t.dropKeptRoute(o, peer)
+
+ t.familyOf(prefix.Addr()).routes--
+ t.peerPrefixes[peer]--
+
+ t.recordChange(prefix, o, wasWithoutKeptRoute)
+}
+
+// keepIfNewest keeps r in o unless both kept routes of o are newer, and says
+// whether it kept r. When o already keeps two routes, the older one is
+// dropped, and its peer keeps only its bit.
+func (t *Table) keepIfNewest(o *origin, r route) bool {
+ // The kept routes stay newest first: r goes before the first one that is
+ // not newer than it.
+ i := 0
+ for i < len(o.routes) && o.routes[i].time.After(r.time) {
+ i++
+ }
+
+ if i == keptRoutesPerOrigin {
+ return false
+ }
+
+ if len(o.routes) == keptRoutesPerOrigin {
+ t.dropKeptRoute(o, o.routes[keptRoutesPerOrigin-1].peer)
+ }
+
+ o.routes = slices.Insert(o.routes, i, r)
+ t.countKeptTime(r.time, 1)
+
+ return true
+}
+
+// dropKeptRoute drops the peer's route from the kept routes of o, if o keeps
+// one, and says whether it did.
+func (t *Table) dropKeptRoute(o *origin, peer uint) bool {
+ i := slices.IndexFunc(o.routes, func(r route) bool { return r.peer == peer })
+ if i < 0 {
+ return false
+ }
+
+ t.countKeptTime(o.routes[i].time, -1)
+ o.routes = slices.Delete(o.routes, i, i+1)
+
+ return true
+}
+
+// recordChange records that the pair of prefix and the AS of o changed, and
+// brings the count of origins without a kept route up to date.
+// wasWithoutKeptRoute says whether o was such an origin before the change. An
+// origin that no peer carries any more is removed.
+func (t *Table) recordChange(
+ prefix netip.Prefix, o *origin, wasWithoutKeptRoute bool,
+) {
+ t.changed[pair{prefix: prefix, asn: o.asn}] = struct{}{}
+
+ t.countOriginWithoutKeptRoute(o, wasWithoutKeptRoute)
+
+ if o.peers.None() {
+ t.removeOrigin(prefix, o)
+ }
+}
+
+// countOriginWithoutKeptRoute brings the count of origins without a kept route
+// up to date after a change to o. wasWithoutKeptRoute says whether o was such
+// an origin before the change.
+func (t *Table) countOriginWithoutKeptRoute(o *origin, wasWithoutKeptRoute bool) {
+ if wasWithoutKeptRoute {
+ t.originsWithoutKeptRoute--
+ }
+
+ if o.withoutKeptRoute() {
+ t.originsWithoutKeptRoute++
+ }
+}
+
+// countKeptTime adds delta to the number of kept routes with time at.
+func (t *Table) countKeptTime(at time.Time, delta int) {
+ count, _ := t.keptTimes.Get(keptTime{time: at})
+ count.time = at
+ count.routes += delta
+
+ if count.routes == 0 {
+ t.keptTimes.Delete(count)
+
+ return
+ }
+
+ t.keptTimes.ReplaceOrInsert(count)
+}
+
+// currentContent returns what the table holds for the pair now.
+func (t *Table) currentContent(key pair) Change {
+ o := t.byOrigin[key.asn][key.prefix]
+ if o == nil {
+ return Change{Prefix: key.prefix, OriginASN: key.asn, Removed: true}
+ }
+
+ return Change{
+ Prefix: key.prefix,
+ OriginASN: key.asn,
+ Peers: o.peers.Clone(),
+ Routes: t.exportRoutes(o.routes),
+ }
+}
+
+// exportRoutes returns copies of routes that name each peer by its address, or
+// nil when there are none.
+func (t *Table) exportRoutes(routes []route) []Route {
+ if len(routes) == 0 {
+ return nil
+ }
+
+ exported := make([]Route, 0, len(routes))
+ for _, r := range routes {
+ exported = append(exported, Route{
+ Peer: t.peerAddresses[r.peer],
+ ASPath: slices.Clone(r.path),
+ Time: r.time,
+ })
+ }
+
+ return exported
+}
+
+// family is the part of the table for one IP version.
+type family struct {
+ // byLength[n] holds the prefixes of mask length n that some peer carries,
+ // each with its origins in AS number order.
+ byLength []map[netip.Prefix][]*origin
+ // minLookupLength is the shortest mask length the address lookup uses.
+ minLookupLength int
+ // routes counts the peer and prefix pairs: each peer that carries a
+ // prefix counts once for it.
+ routes int
+}
+
+// newFamily returns an empty family for addresses of bits bits.
+func newFamily(bits, minLookupLength int) family {
+ f := family{
+ byLength: make([]map[netip.Prefix][]*origin, bits+1),
+ minLookupLength: minLookupLength,
+ }
+
+ for n := range f.byLength {
+ f.byLength[n] = make(map[netip.Prefix][]*origin)
+ }
+
+ return f
+}
+
+// prefixesByLength returns how many prefixes of each mask length f holds.
+func (f *family) prefixesByLength() []int {
+ counts := make([]int, len(f.byLength))
+ for n, prefixes := range f.byLength {
+ counts[n] = len(prefixes)
+ }
+
+ return counts
+}
+
+// origin is what the table holds for one prefix and origin AS.
+type origin struct {
+ asn int
+ // peers holds the number of each peer whose current route to the prefix
+ // has this origin AS.
+ peers bitset.BitSet
+ // routes are the newest of those routes, newest first, at most
+ // keptRoutesPerOrigin.
+ routes []route
+}
+
+// withoutKeptRoute says whether peers carry o but o keeps no route.
+func (o *origin) withoutKeptRoute() bool {
+ return o.peers.Any() && len(o.routes) == 0
+}
+
+// newestTime returns the time of the newest kept route of o, or nil when o
+// keeps none.
+func (o *origin) newestTime() *time.Time {
+ if len(o.routes) == 0 {
+ return nil
+ }
+
+ newest := o.routes[0].time
+
+ return &newest
+}
+
+// originWithPeer returns the origin whose peer set holds peer, or nil when
+// none does. A peer carries a prefix with one origin AS at most.
+func originWithPeer(origins []*origin, peer uint) *origin {
+ for _, o := range origins {
+ if o.peers.Test(peer) {
+ return o
+ }
+ }
+
+ return nil
+}
+
+// route is a kept route. peer is the peer's number.
+type route struct {
+ peer uint
+ path []int
+ time time.Time
+}
+
+// equal says whether r and other are the same peer's route with the same path
+// and time.
+func (r route) equal(other route) bool {
+ return r.peer == other.peer && r.time.Equal(other.time) &&
+ slices.Equal(r.path, other.path)
+}
+
+// pair names one prefix and origin AS.
+type pair struct {
+ prefix netip.Prefix
+ asn int
+}
+
+// keptTime counts the kept routes with one time.
+type keptTime struct {
+ time time.Time
+ routes int
+}
diff --git a/internal/routetable/routetable_test.go b/internal/routetable/routetable_test.go
new file mode 100644
index 0000000..d77f15f
--- /dev/null
+++ b/internal/routetable/routetable_test.go
@@ -0,0 +1,667 @@
+package routetable_test
+
+import (
+ "math"
+ "net/netip"
+ "reflect"
+ "slices"
+ "sync"
+ "testing"
+ "time"
+
+ "github.com/bits-and-blooms/bitset"
+
+ "git.eeqj.de/sneak/routewatch/internal/routetable"
+)
+
+const (
+ peerA = "192.0.2.1"
+ peerB = "192.0.2.2"
+ peerC = "2001:db8::3"
+ peerD = "2001:db8::4"
+
+ asA = 64500
+ asB = 64501
+ asC = 64502
+ // transitAS is the first AS of every test path, the peer's own.
+ transitAS = 64496
+
+ prefixV4 = "10.1.2.0/24"
+ prefixV6 = "2001:db8:1::/48"
+)
+
+// at returns the time second seconds after the start of every test.
+func at(second int) time.Time {
+ return time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC).
+ Add(time.Duration(second) * time.Second)
+}
+
+// timeAt returns at(second) for a field that may be nil.
+func timeAt(second int) *time.Time {
+ t := at(second)
+
+ return &t
+}
+
+// routeOf returns the route peer announces with origin AS asn second seconds
+// after the start.
+func routeOf(peer string, asn, second int) routetable.Route {
+ return routetable.Route{
+ Peer: peer, ASPath: []int{transitAS, asn}, Time: at(second),
+ }
+}
+
+// step is one update a test applies to a table for one prefix.
+type step func(table *routetable.Table, prefix netip.Prefix)
+
+// announce is the step of peer announcing prefix with origin AS asn second
+// seconds after the start.
+func announce(peer string, asn, second int) step {
+ return func(table *routetable.Table, prefix netip.Prefix) {
+ table.Announce(prefix, asn, routeOf(peer, asn, second))
+ }
+}
+
+// withdraw is the step of peer withdrawing prefix.
+func withdraw(peer string) step {
+ return func(table *routetable.Table, prefix netip.Prefix) {
+ table.Withdraw(prefix, peer)
+ }
+}
+
+// peerDown is the step of peer's session going down.
+func peerDown(peer string) step {
+ return func(table *routetable.Table, _ netip.Prefix) {
+ table.PeerDown(peer)
+ }
+}
+
+// rule is one case of the rule that decides what the table holds for a prefix:
+// the steps applied to an empty table, and the prefix's origins after them.
+// The peers are numbered in the order they first announce, which is the order
+// Origin.Peers lists them in.
+type rule struct {
+ name string
+ steps []step
+ want []routetable.Origin
+}
+
+// checkRules runs each case for an IPv4 and an IPv6 prefix.
+func checkRules(t *testing.T, rules []rule) {
+ t.Helper()
+
+ for _, tc := range rules {
+ for _, text := range []string{prefixV4, prefixV6} {
+ t.Run(tc.name+" "+text, func(t *testing.T) {
+ t.Parallel()
+
+ prefix := netip.MustParsePrefix(text)
+ table := routetable.New(routetable.Params{})
+
+ for _, apply := range tc.steps {
+ apply(table, prefix)
+ }
+
+ got := table.Origins(prefix)
+ if !reflect.DeepEqual(got, tc.want) {
+ t.Errorf("Origins(%s) = %+v, want %+v", prefix, got, tc.want)
+ }
+ })
+ }
+ }
+}
+
+func TestAnnouncementsKeepTheTwoNewestRoutes(t *testing.T) {
+ t.Parallel()
+
+ checkRules(t, []rule{
+ {
+ name: "an announcement is kept",
+ steps: []step{announce(peerA, asA, 10)},
+ want: []routetable.Origin{{ASN: asA, Peers: []string{peerA},
+ Routes: []routetable.Route{routeOf(peerA, asA, 10)}}},
+ },
+ {
+ name: "a lagging peer's older route is not kept",
+ steps: []step{announce(peerA, asA, 20), announce(peerB, asA, 30),
+ announce(peerC, asA, 10)},
+ want: []routetable.Origin{{ASN: asA,
+ Peers: []string{peerA, peerB, peerC}, Routes: []routetable.Route{
+ routeOf(peerB, asA, 30), routeOf(peerA, asA, 20),
+ }}},
+ },
+ {
+ name: "a third newer route drops the older kept one, whose peer " +
+ "keeps its bit",
+ steps: []step{announce(peerA, asA, 10), announce(peerB, asA, 20),
+ announce(peerC, asA, 30)},
+ want: []routetable.Origin{{ASN: asA,
+ Peers: []string{peerA, peerB, peerC}, Routes: []routetable.Route{
+ routeOf(peerC, asA, 30), routeOf(peerB, asA, 20),
+ }}},
+ },
+ {
+ name: "a route as old as the older kept one replaces it",
+ steps: []step{announce(peerA, asA, 10), announce(peerB, asA, 20),
+ announce(peerC, asA, 10)},
+ want: []routetable.Origin{{ASN: asA,
+ Peers: []string{peerA, peerB, peerC}, Routes: []routetable.Route{
+ routeOf(peerB, asA, 20), routeOf(peerC, asA, 10),
+ }}},
+ },
+ {
+ name: "a peer's new route replaces its kept one",
+ steps: []step{announce(peerA, asA, 10), announce(peerB, asA, 20),
+ announce(peerA, asA, 30)},
+ want: []routetable.Origin{{ASN: asA, Peers: []string{peerA, peerB},
+ Routes: []routetable.Route{
+ routeOf(peerA, asA, 30), routeOf(peerB, asA, 20),
+ }}},
+ },
+ {
+ name: "a peer's new route replaces its kept one even when older",
+ steps: []step{announce(peerA, asA, 10), announce(peerB, asA, 20),
+ announce(peerB, asA, 5)},
+ want: []routetable.Origin{{ASN: asA, Peers: []string{peerA, peerB},
+ Routes: []routetable.Route{
+ routeOf(peerA, asA, 10), routeOf(peerB, asA, 5),
+ }}},
+ },
+ })
+}
+
+func TestAPeerCarriesAPrefixWithOneOrigin(t *testing.T) {
+ t.Parallel()
+
+ checkRules(t, []rule{
+ {
+ name: "a peer moving to a new origin leaves the old one",
+ steps: []step{announce(peerA, asA, 10), announce(peerB, asA, 20),
+ announce(peerA, asB, 30)},
+ want: []routetable.Origin{
+ {ASN: asA, Peers: []string{peerB},
+ Routes: []routetable.Route{routeOf(peerB, asA, 20)}},
+ {ASN: asB, Peers: []string{peerA},
+ Routes: []routetable.Route{routeOf(peerA, asB, 30)}},
+ },
+ },
+ {
+ name: "a peer whose route is not kept moves to a new origin",
+ steps: []step{announce(peerA, asA, 10), announce(peerB, asA, 20),
+ announce(peerC, asA, 30), announce(peerA, asB, 40)},
+ want: []routetable.Origin{
+ {ASN: asA, Peers: []string{peerB, peerC},
+ Routes: []routetable.Route{
+ routeOf(peerC, asA, 30), routeOf(peerB, asA, 20),
+ }},
+ {ASN: asB, Peers: []string{peerA},
+ Routes: []routetable.Route{routeOf(peerA, asB, 40)}},
+ },
+ },
+ {
+ name: "origins are listed in AS number order",
+ steps: []step{announce(peerA, asC, 10), announce(peerB, asA, 20),
+ announce(peerC, asB, 30)},
+ want: []routetable.Origin{
+ {ASN: asA, Peers: []string{peerB},
+ Routes: []routetable.Route{routeOf(peerB, asA, 20)}},
+ {ASN: asB, Peers: []string{peerC},
+ Routes: []routetable.Route{routeOf(peerC, asB, 30)}},
+ {ASN: asC, Peers: []string{peerA},
+ Routes: []routetable.Route{routeOf(peerA, asC, 10)}},
+ },
+ },
+ })
+}
+
+func TestWithdrawalsAndPeerDowns(t *testing.T) {
+ t.Parallel()
+
+ // threePeers is peerA, peerB and peerC announcing in turn; the routes of
+ // peerC and peerB are kept.
+ threePeers := []step{announce(peerA, asA, 10), announce(peerB, asA, 20),
+ announce(peerC, asA, 30)}
+
+ checkRules(t, []rule{
+ {
+ name: "a withdrawal drops the peer's kept route",
+ steps: append(slices.Clone(threePeers), withdraw(peerC)),
+ want: []routetable.Origin{{ASN: asA, Peers: []string{peerA, peerB},
+ Routes: []routetable.Route{routeOf(peerB, asA, 20)}}},
+ },
+ {
+ name: "a withdrawal of a route that is not kept leaves the kept ones",
+ steps: append(slices.Clone(threePeers), withdraw(peerA)),
+ want: []routetable.Origin{{ASN: asA, Peers: []string{peerB, peerC},
+ Routes: []routetable.Route{
+ routeOf(peerC, asA, 30), routeOf(peerB, asA, 20),
+ }}},
+ },
+ {
+ name: "an origin whose kept routes are withdrawn keeps its other " +
+ "peers",
+ steps: append(slices.Clone(threePeers), withdraw(peerB),
+ withdraw(peerC)),
+ want: []routetable.Origin{{ASN: asA, Peers: []string{peerA}}},
+ },
+ {
+ name: "an origin without kept routes keeps the next update",
+ steps: append(slices.Clone(threePeers), withdraw(peerB),
+ withdraw(peerC), announce(peerA, asA, 5)),
+ want: []routetable.Origin{{ASN: asA, Peers: []string{peerA},
+ Routes: []routetable.Route{routeOf(peerA, asA, 5)}}},
+ },
+ {
+ name: "an origin no peer carries is removed",
+ steps: []step{announce(peerA, asA, 10), announce(peerB, asB, 20),
+ withdraw(peerA)},
+ want: []routetable.Origin{{ASN: asB, Peers: []string{peerB},
+ Routes: []routetable.Route{routeOf(peerB, asB, 20)}}},
+ },
+ {
+ name: "a prefix no peer carries is removed",
+ steps: []step{announce(peerA, asA, 10), withdraw(peerA)},
+ want: nil,
+ },
+ {
+ name: "a withdrawal from a peer that does not carry it changes nothing",
+ steps: []step{announce(peerA, asA, 10), withdraw(peerB)},
+ want: []routetable.Origin{{ASN: asA, Peers: []string{peerA},
+ Routes: []routetable.Route{routeOf(peerA, asA, 10)}}},
+ },
+ {
+ name: "a peer going down leaves the prefix",
+ steps: []step{announce(peerA, asA, 10), announce(peerB, asA, 20),
+ peerDown(peerB)},
+ want: []routetable.Origin{{ASN: asA, Peers: []string{peerA},
+ Routes: []routetable.Route{routeOf(peerA, asA, 10)}}},
+ },
+ {
+ name: "a peer going down after it withdrew changes nothing",
+ steps: []step{announce(peerA, asA, 10), announce(peerB, asA, 20),
+ withdraw(peerB), peerDown(peerB)},
+ want: []routetable.Origin{{ASN: asA, Peers: []string{peerA},
+ Routes: []routetable.Route{routeOf(peerA, asA, 10)}}},
+ },
+ })
+}
+
+func TestPeerDownLeavesEveryPrefix(t *testing.T) {
+ t.Parallel()
+
+ table := routetable.New(routetable.Params{})
+ prefixes := []netip.Prefix{
+ netip.MustParsePrefix(prefixV4),
+ netip.MustParsePrefix("10.2.0.0/16"),
+ netip.MustParsePrefix(prefixV6),
+ }
+
+ // peerA carries every prefix with origin asA, peerB the first one with
+ // origin asB.
+ for _, prefix := range prefixes {
+ table.Announce(prefix, asA, routeOf(peerA, asA, 10))
+ }
+
+ table.Announce(prefixes[0], asB, routeOf(peerB, asB, 20))
+ table.PeerDown(peerA)
+
+ want := []routetable.Origin{{ASN: asB, Peers: []string{peerB},
+ Routes: []routetable.Route{routeOf(peerB, asB, 20)}}}
+
+ got := table.Origins(prefixes[0])
+ if !reflect.DeepEqual(got, want) {
+ t.Errorf("Origins(%s) = %+v, want %+v", prefixes[0], got, want)
+ }
+
+ for _, prefix := range prefixes[1:] {
+ got := table.Origins(prefix)
+ if got != nil {
+ t.Errorf("Origins(%s) = %+v after the only peer went down, want nil",
+ prefix, got)
+ }
+ }
+
+ counts := table.Counts()
+ if counts.IPv4Prefixes != 1 || counts.IPv6Prefixes != 0 ||
+ counts.IPv4Routes != 1 || counts.IPv6Routes != 0 {
+ t.Errorf("Counts() = %+v, want one IPv4 prefix and route", counts)
+ }
+
+ if len(table.OriginatedPrefixes(asA)) != 0 {
+ t.Errorf("OriginatedPrefixes(%d) = %+v, want none",
+ asA, table.OriginatedPrefixes(asA))
+ }
+}
+
+func TestCountsOriginsWithoutKeptRoute(t *testing.T) {
+ t.Parallel()
+
+ prefix := netip.MustParsePrefix(prefixV4)
+ table := routetable.New(routetable.Params{})
+
+ steps := []struct {
+ apply step
+ want int
+ }{
+ {announce(peerA, asA, 10), 0},
+ {announce(peerB, asA, 20), 0},
+ {announce(peerC, asA, 30), 0},
+ {withdraw(peerC), 0},
+ {withdraw(peerB), 1},
+ // An update from the last peer is kept.
+ {announce(peerA, asA, 40), 0},
+ {withdraw(peerA), 0},
+ // Once more without kept routes, then without peers: the origin
+ // is removed.
+ {announce(peerA, asA, 10), 0},
+ {announce(peerB, asA, 20), 0},
+ {announce(peerC, asA, 30), 0},
+ {withdraw(peerB), 0},
+ {withdraw(peerC), 1},
+ {peerDown(peerA), 0},
+ }
+
+ for i, s := range steps {
+ s.apply(table, prefix)
+
+ got := table.Counts().OriginsWithoutKeptRoute
+ if got != s.want {
+ t.Errorf("after step %d, OriginsWithoutKeptRoute = %d, want %d",
+ i, got, s.want)
+ }
+ }
+}
+
+// change is a routetable.Change with its peer set as a list of peer numbers.
+type change struct {
+ Prefix string
+ OriginASN int
+ Removed bool
+ Peers []uint
+ Routes []routetable.Route
+}
+
+// peerNumbers returns the numbers in set, or nil for a nil set.
+func peerNumbers(set *bitset.BitSet) []uint {
+ if set == nil {
+ return nil
+ }
+
+ return slices.Collect(set.EachSet())
+}
+
+func TestTakeChanges(t *testing.T) {
+ t.Parallel()
+
+ // peerA was numbered 5 before; the next new peer is 6.
+ table := routetable.New(routetable.Params{
+ PeerNumbers: map[string]uint{peerA: 5},
+ })
+ v4 := netip.MustParsePrefix(prefixV4)
+ v6 := netip.MustParsePrefix(prefixV6)
+
+ table.Announce(v6, asA, routeOf(peerA, asA, 10))
+ table.Announce(v4, asA, routeOf(peerA, asA, 10))
+ table.Announce(v4, asA, routeOf(peerB, asA, 20))
+
+ checkChanges(t, table, []change{
+ {Prefix: prefixV4, OriginASN: asA, Peers: []uint{5, 6},
+ Routes: []routetable.Route{
+ routeOf(peerB, asA, 20), routeOf(peerA, asA, 10),
+ }},
+ {Prefix: prefixV6, OriginASN: asA, Peers: []uint{5},
+ Routes: []routetable.Route{routeOf(peerA, asA, 10)}},
+ }, map[string]uint{peerB: 6})
+
+ // TakeChanges cleared the record.
+ checkChanges(t, table, []change{}, map[string]uint{})
+
+ // peerB moves to asB; peerA withdraws, which removes the pair of v4 and
+ // asA.
+ table.Announce(v4, asB, routeOf(peerB, asB, 30))
+ table.Withdraw(v4, peerA)
+
+ checkChanges(t, table, []change{
+ {Prefix: prefixV4, OriginASN: asA, Removed: true},
+ {Prefix: prefixV4, OriginASN: asB, Peers: []uint{6},
+ Routes: []routetable.Route{routeOf(peerB, asB, 30)}},
+ }, map[string]uint{})
+}
+
+func TestTakeChangesHoldsOnlyPairsThatChanged(t *testing.T) {
+ t.Parallel()
+
+ table := routetable.New(routetable.Params{})
+ v4 := netip.MustParsePrefix(prefixV4)
+ v6 := netip.MustParsePrefix(prefixV6)
+
+ // peerA, peerB and peerC are numbered 0, 1 and 2. The routes of peerC and
+ // peerB to v4 are kept; peerA keeps only its bit.
+ announceAll(table, prefixV4, asA, 10, peerA, peerB, peerC)
+ table.Announce(v6, asA, routeOf(peerB, asA, 10))
+ table.TakeChanges(math.MaxInt)
+
+ // peerA announces again, with a time older than both kept routes.
+ table.Announce(v4, asA, routeOf(peerA, asA, 15))
+ checkChanges(t, table, []change{}, map[string]uint{})
+
+ // peerC announces its kept route again.
+ table.Announce(v4, asA, routeOf(peerC, asA, 30))
+ checkChanges(t, table, []change{}, map[string]uint{})
+
+ // peerC announces the same path with a newer time, so its kept route
+ // changes.
+ table.Announce(v4, asA, routeOf(peerC, asA, 40))
+ checkChanges(t, table, []change{
+ {Prefix: prefixV4, OriginASN: asA, Peers: []uint{0, 1, 2},
+ Routes: []routetable.Route{
+ routeOf(peerC, asA, 40), routeOf(peerB, asA, 20),
+ }},
+ }, map[string]uint{})
+
+ // peerA withdraws v6, which it does not carry.
+ table.Withdraw(v6, peerA)
+ checkChanges(t, table, []change{}, map[string]uint{})
+
+ // peerA going down changes v4, the one prefix it carries.
+ table.PeerDown(peerA)
+ checkChanges(t, table, []change{
+ {Prefix: prefixV4, OriginASN: asA, Peers: []uint{1, 2},
+ Routes: []routetable.Route{
+ routeOf(peerC, asA, 40), routeOf(peerB, asA, 20),
+ }},
+ }, map[string]uint{})
+
+ // peerA, which carries nothing now, goes down again.
+ table.PeerDown(peerA)
+ checkChanges(t, table, []change{}, map[string]uint{})
+}
+
+// TestTakeChangesTakesAtMostMaxChanges changes three prefix and origin AS
+// pairs, then takes the changes two at a time. The first call must return two
+// pairs and both new peers, and the second the third pair and no peer.
+func TestTakeChangesTakesAtMostMaxChanges(t *testing.T) {
+ t.Parallel()
+
+ const maxChanges = 2
+
+ table := routetable.New(routetable.Params{})
+ v4 := netip.MustParsePrefix(prefixV4)
+ v6 := netip.MustParsePrefix(prefixV6)
+
+ table.Announce(v4, asA, routeOf(peerA, asA, 10))
+ table.Announce(v4, asB, routeOf(peerB, asB, 10))
+ table.Announce(v6, asA, routeOf(peerA, asA, 10))
+
+ first, firstPeers := table.TakeChanges(maxChanges)
+ second, secondPeers := table.TakeChanges(maxChanges)
+
+ if len(first) != maxChanges || len(second) != 1 {
+ t.Fatalf("TakeChanges(%d) returned %d and then %d pairs, want %d and 1",
+ maxChanges, len(first), len(second), maxChanges)
+ }
+
+ wantPeers := map[string]uint{peerA: 0, peerB: 1}
+ if !reflect.DeepEqual(firstPeers, wantPeers) || len(secondPeers) != 0 {
+ t.Errorf("TakeChanges(%d) returned the new peers %v and then %v, "+
+ "want %v and none", maxChanges, firstPeers, secondPeers, wantPeers)
+ }
+
+ for _, c := range first {
+ if c.Prefix == second[0].Prefix && c.OriginASN == second[0].OriginASN {
+ t.Errorf("both calls returned %s with AS%d", c.Prefix, c.OriginASN)
+ }
+ }
+}
+
+// TestPutBackChanges takes a table's changes and puts them back, as a caller
+// does whose write failed. The next TakeChanges must return them again, with
+// each pair's content then, and with the changes since.
+func TestPutBackChanges(t *testing.T) {
+ t.Parallel()
+
+ table := routetable.New(routetable.Params{})
+ v4 := netip.MustParsePrefix(prefixV4)
+ v6 := netip.MustParsePrefix(prefixV6)
+
+ table.Announce(v4, asA, routeOf(peerA, asA, 10))
+ table.Announce(v6, asA, routeOf(peerA, asA, 10))
+ table.PutBackChanges(table.TakeChanges(math.MaxInt))
+
+ if got := table.ChangeCount(); got != 2 {
+ t.Errorf("ChangeCount() = %d after the changes were put back, want 2",
+ got)
+ }
+
+ // peerA withdraws v4, so v4's earlier content must not come back; peerB
+ // announces v6, and is a new peer as well.
+ table.Withdraw(v4, peerA)
+ table.Announce(v6, asA, routeOf(peerB, asA, 20))
+
+ checkChanges(t, table, []change{
+ {Prefix: prefixV4, OriginASN: asA, Removed: true},
+ {Prefix: prefixV6, OriginASN: asA, Peers: []uint{0, 1},
+ Routes: []routetable.Route{
+ routeOf(peerB, asA, 20), routeOf(peerA, asA, 10),
+ }},
+ }, map[string]uint{peerA: 0, peerB: 1})
+
+ if got := table.ChangeCount(); got != 0 {
+ t.Errorf("ChangeCount() = %d after TakeChanges, want 0", got)
+ }
+}
+
+func TestLoad(t *testing.T) {
+ t.Parallel()
+
+ const otherV4 = "10.9.0.0/16"
+
+ v4 := netip.MustParsePrefix(prefixV4)
+ v6 := netip.MustParsePrefix(prefixV6)
+ other := netip.MustParsePrefix(otherV4)
+
+ // The routes of peerC and peerB to v4 are kept; peerA keeps only its bit.
+ // The two routes to v6 have the same time. The routes of other were
+ // withdrawn while peerA still carries it.
+ source := routetable.New(routetable.Params{})
+ announceAll(source, prefixV4, asA, 10, peerA, peerB, peerC)
+ announceAll(source, prefixV4, asB, 40, peerD)
+ source.Announce(v6, asA, routeOf(peerA, asA, 50))
+ source.Announce(v6, asA, routeOf(peerC, asA, 50))
+ announceAll(source, otherV4, asB, 10, peerA, peerB, peerC)
+ source.Withdraw(other, peerB)
+ source.Withdraw(other, peerC)
+
+ changes, peers := source.TakeChanges(math.MaxInt)
+ loaded := routetable.New(routetable.Params{PeerNumbers: peers})
+
+ // Loading a pair twice changes nothing.
+ for range 2 {
+ for _, change := range changes {
+ loaded.Load(change)
+ }
+ }
+
+ for _, prefix := range []netip.Prefix{v4, v6, other} {
+ got, want := loaded.Origins(prefix), source.Origins(prefix)
+ if !reflect.DeepEqual(got, want) {
+ t.Errorf("Origins(%s) = %+v after the load, want %+v", prefix, got, want)
+ }
+ }
+
+ if !reflect.DeepEqual(loaded.Counts(), source.Counts()) {
+ t.Errorf("Counts() = %+v after the load, want %+v",
+ loaded.Counts(), source.Counts())
+ }
+
+ checkChanges(t, loaded, []change{}, map[string]uint{})
+}
+
+// TestConcurrentUse has goroutines update one table while others read it. The
+// race detector, which make test runs, fails it when a method uses the table
+// without its lock.
+func TestConcurrentUse(t *testing.T) {
+ t.Parallel()
+
+ const (
+ // rounds is how many times each goroutine calls its method.
+ rounds = 1000
+ ipVersion = 4
+ sampleSize = 10
+ )
+
+ table := routetable.New(routetable.Params{})
+ prefix := netip.MustParsePrefix(prefixV4)
+
+ uses := []func(round int){
+ func(round int) { table.Announce(prefix, asA, routeOf(peerA, asA, round)) },
+ func(round int) { table.Announce(prefix, asB, routeOf(peerB, asB, round)) },
+ func(int) { table.Withdraw(prefix, peerA) },
+ func(int) { table.PeerDown(peerB) },
+ func(int) { table.PutBackChanges(table.TakeChanges(math.MaxInt)) },
+ func(int) { table.ChangeCount() },
+ func(int) { table.Lookup(prefix.Addr()) },
+ func(int) { table.Origins(prefix) },
+ func(int) { table.OriginatedPrefixes(asA) },
+ func(int) { table.SamplePrefixes(ipVersion, prefix.Bits(), sampleSize) },
+ func(int) { table.Counts() },
+ }
+
+ var wg sync.WaitGroup
+ for _, use := range uses {
+ wg.Go(func() {
+ for round := range rounds {
+ use(round)
+ }
+ })
+ }
+
+ wg.Wait()
+}
+
+// checkChanges checks that TakeChanges, taking every change, returns want and
+// wantPeers.
+func checkChanges(
+ t *testing.T, table *routetable.Table, want []change,
+ wantPeers map[string]uint,
+) {
+ t.Helper()
+
+ changes, newPeers := table.TakeChanges(math.MaxInt)
+
+ got := make([]change, 0, len(changes))
+ for _, c := range changes {
+ got = append(got, change{
+ Prefix: c.Prefix.String(), OriginASN: c.OriginASN, Removed: c.Removed,
+ Peers: peerNumbers(c.Peers), Routes: c.Routes,
+ })
+ }
+
+ if !reflect.DeepEqual(got, want) {
+ t.Errorf("TakeChanges() changes = %+v, want %+v", got, want)
+ }
+
+ if !reflect.DeepEqual(newPeers, wantPeers) {
+ t.Errorf("TakeChanges() new peers = %v, want %v", newPeers, wantPeers)
+ }
+}
diff --git a/internal/routewatch/app.go b/internal/routewatch/app.go
index 1a4e925..91b2c41 100644
--- a/internal/routewatch/app.go
+++ b/internal/routewatch/app.go
@@ -2,19 +2,25 @@
// running instance of the application and contains pointers to its core
// dependencies, and is responsible for initialization. It also contains the
// ASN WHOIS fetcher for background updates and the database maintainer for
-// background maintenance tasks.
+// background maintenance tasks. The package also holds the handlers that
+// write the feed's updates to the database in batches, and the flush loop
+// that the AS, peer and prefix handlers share.
package routewatch
import (
"context"
"errors"
+ "fmt"
+ "path/filepath"
"sync"
"time"
+ "git.eeqj.de/sneak/routewatch/internal/blocklist"
"git.eeqj.de/sneak/routewatch/internal/config"
"git.eeqj.de/sneak/routewatch/internal/database"
"git.eeqj.de/sneak/routewatch/internal/logger"
"git.eeqj.de/sneak/routewatch/internal/metrics"
+ "git.eeqj.de/sneak/routewatch/internal/routetable"
"git.eeqj.de/sneak/routewatch/internal/server"
"git.eeqj.de/sneak/routewatch/internal/streamer"
@@ -25,11 +31,13 @@ import (
// are disabled, since only the batched handlers exist.
var errNonBatchedNotImplemented = errors.New("non-batched handlers not implemented")
-// Dependencies contains all dependencies for RouteWatch
+// Dependencies contains all dependencies for RouteWatch. Table is the routing
+// table read from the database at startup, which the server reads too.
type Dependencies struct {
fx.In
DB database.Store
+ Table *routetable.Table
Streamer *streamer.Streamer
Server *server.Server
Logger *logger.Logger
@@ -39,6 +47,7 @@ type Dependencies struct {
// RouteWatch represents the main application instance
type RouteWatch struct {
db database.Store
+ table *routetable.Table
streamer *streamer.Streamer
server *server.Server
logger *logger.Logger
@@ -52,12 +61,18 @@ type RouteWatch struct {
peeringHandler *PeeringHandler
asnFetcher *ASNFetcher
dbMaintainer *DBMaintainer
+ blocklists *blocklist.Blocklists
}
+// blocklistsDirName is the directory in the state directory that holds the
+// blocklist copies.
+const blocklistsDirName = "blocklists"
+
// New creates a new RouteWatch instance
func New(deps Dependencies) *RouteWatch {
rw := &RouteWatch{
db: deps.DB,
+ table: deps.Table,
streamer: deps.Streamer,
server: deps.Server,
logger: deps.Logger,
@@ -85,6 +100,7 @@ func (rw *RouteWatch) Run(ctx context.Context) error {
// Register database handler to process BGP UPDATE messages
if rw.config.EnableBatchedDatabaseWrites {
rw.logger.Info("Using batched database handlers for improved performance")
+
// ASHandler maintains the asns table
rw.dbHandler = NewASHandler(rw.db, rw.logger)
rw.streamer.RegisterHandler(rw.dbHandler)
@@ -93,8 +109,10 @@ func (rw *RouteWatch) Run(ctx context.Context) error {
rw.peerHandler = NewPeerHandler(rw.db, rw.logger)
rw.streamer.RegisterHandler(rw.peerHandler)
- // PrefixHandler maintains the prefixes and live_routes tables
- rw.prefixHandler = NewPrefixHandler(rw.db, rw.logger)
+ // PrefixHandler keeps the routing table in memory current, and writes
+ // its changes to the routes table and new prefixes to the prefix
+ // tables
+ rw.prefixHandler = NewPrefixHandler(rw.db, rw.logger, rw.table)
rw.prefixHandler.SetMetricsTracker(rw.streamer.GetMetricsTracker())
rw.streamer.RegisterHandler(rw.prefixHandler)
@@ -108,8 +126,6 @@ func (rw *RouteWatch) Run(ctx context.Context) error {
return errNonBatchedNotImplemented
}
- // No longer need routing table handler - PrefixHandler maintains live_routes table
-
// Start streaming
err := rw.streamer.Start(ctx)
if err != nil {
@@ -122,6 +138,18 @@ func (rw *RouteWatch) Run(ctx context.Context) error {
rw.asnFetcher.Start(ctx)
rw.server.SetASNFetcher(rw.asnFetcher)
+ // Load the blocklist copies from disk and start downloading the lists.
+ // The HTTP handlers read the server's blocklists without a lock, so they
+ // are set before the handlers run.
+ blocklists := blocklist.New(blocklist.Params{
+ Dir: filepath.Join(rw.config.StateDir, blocklistsDirName),
+ Lists: blocklist.Lists(),
+ Logger: rw.logger.Logger,
+ })
+ blocklists.Start(ctx)
+ rw.blocklists = blocklists
+ rw.server.SetBlocklists(blocklists)
+
// Start HTTP server
err = rw.server.Start(ctx)
if err != nil {
@@ -139,11 +167,12 @@ func (rw *RouteWatch) Run(ctx context.Context) error {
}
// Shutdown stops all services in order: the streamer, the handlers, which
-// write what they hold, the WHOIS fetcher, the database maintainer and the
-// HTTP server, and last the database, which it checkpoints and closes. Each
-// stage waits at most until ctx ends, so Shutdown returns soon after: a
-// handler logs what it could not write by then, and the HTTP server closes
-// the connections of requests still running.
+// write what they hold, the WHOIS fetcher, the database maintainer, the
+// blocklist downloads and the HTTP server, and last the database, which it
+// checkpoints and closes. Each stage waits at most until ctx ends, so
+// Shutdown returns soon after: a handler logs what it could not write by
+// then, and the HTTP server closes the connections of requests still
+// running.
func (rw *RouteWatch) Shutdown(ctx context.Context) {
rw.mu.Lock()
if rw.shutdown {
@@ -190,6 +219,10 @@ func (rw *RouteWatch) Shutdown(ctx context.Context) {
rw.dbMaintainer.Stop(ctx)
}
+ if rw.blocklists != nil {
+ rw.blocklists.Stop(ctx)
+ }
+
err := rw.server.Stop(ctx)
if err != nil {
rw.logger.Error("Stopped the HTTP server before every request finished",
@@ -230,6 +263,28 @@ func (rw *RouteWatch) closeDatabase(ctx context.Context) {
}
}
+// readRouteTable reads the routing table from the database and logs how many
+// prefix and origin AS pairs and peers it holds and how long the read took.
+// fx calls it once, to build the server and RouteWatch, before the streamer
+// connects, so the prefix handler applies the feed's updates to all of it. No
+// start timeout limits the read: fx builds them before it runs a start hook.
+func readRouteTable(
+ db database.Store, logger *logger.Logger,
+) (*routetable.Table, error) {
+ start := time.Now()
+
+ table, err := db.ReadRouteTable(context.Background())
+ if err != nil {
+ return nil, fmt.Errorf("failed to read the routing table: %w", err)
+ }
+
+ counts := table.Counts()
+ logger.Info("Read the routing table", "pairs", counts.Pairs,
+ "peers", counts.Peers, "duration_ms", time.Since(start).Milliseconds())
+
+ return table, nil
+}
+
// getModule provides all fx dependencies
func getModule() []fx.Option {
return []fx.Option{
@@ -241,6 +296,7 @@ func getModule() []fx.Option {
database.New,
fx.As(new(database.Store)),
),
+ readRouteTable,
streamer.New,
server.New,
New,
diff --git a/internal/routewatch/app_integration_internal_test.go b/internal/routewatch/app_integration_internal_test.go
index f467126..f736dc2 100644
--- a/internal/routewatch/app_integration_internal_test.go
+++ b/internal/routewatch/app_integration_internal_test.go
@@ -7,6 +7,7 @@ import (
"net"
"net/http"
"net/http/httptest"
+ "net/netip"
"slices"
"strconv"
"strings"
@@ -19,6 +20,7 @@ import (
"git.eeqj.de/sneak/routewatch/internal/logger"
"git.eeqj.de/sneak/routewatch/internal/metrics"
"git.eeqj.de/sneak/routewatch/internal/ristypes"
+ "git.eeqj.de/sneak/routewatch/internal/routetable"
"git.eeqj.de/sneak/routewatch/internal/server"
"git.eeqj.de/sneak/routewatch/internal/streamer"
)
@@ -31,14 +33,15 @@ type mockStore struct {
ASNCount int
PrefixCount int
PeeringCount int
- RouteCount int
WithdrawalCount int
// Track unique items
ASNs map[int]*database.ASN
Prefixes map[string]*database.Prefix
Peerings map[string]bool // key is "from_to"
- Routes map[string]bool // key is "prefix_origin_peer"
+ // TablePrefixes holds the prefixes that the written changes of the
+ // routing table leave some peer carrying.
+ TablePrefixes map[string]bool
// Track IP versions
IPv4Prefixes int
@@ -48,10 +51,10 @@ type mockStore struct {
// newMockStore creates a new mock store
func newMockStore() *mockStore {
return &mockStore{
- ASNs: make(map[int]*database.ASN),
- Prefixes: make(map[string]*database.Prefix),
- Peerings: make(map[string]bool),
- Routes: make(map[string]bool),
+ ASNs: make(map[int]*database.ASN),
+ Prefixes: make(map[string]*database.Prefix),
+ Peerings: make(map[string]bool),
+ TablePrefixes: make(map[string]bool),
}
}
@@ -101,6 +104,13 @@ func (m *mockStore) UpdatePrefixesBatch(prefixes map[string]time.Time) error {
IPVersion: ipVersion,
FirstSeen: timestamp,
}
+ m.PrefixCount++
+
+ if ipVersion == ipVersionV4 {
+ m.IPv4Prefixes++
+ } else {
+ m.IPv6Prefixes++
+ }
}
}
@@ -201,92 +211,30 @@ func (m *mockStore) GetStatsContext(_ context.Context) (database.Stats, error) {
return m.GetStats()
}
-// UpsertLiveRoute mock implementation
-func (m *mockStore) UpsertLiveRoute(_ *database.LiveRoute) error {
- // Simple mock - just return nil
- return nil
-}
-
-// DeleteLiveRoute mock implementation
-func (m *mockStore) DeleteLiveRoute(_, _ string) error {
- // Simple mock - just return nil
- return nil
-}
-
-// GetPrefixDistribution mock implementation
-func (m *mockStore) GetPrefixDistribution() (
- []database.PrefixDistribution, []database.PrefixDistribution, error,
-) {
- // Return empty distributions for now
- return nil, nil, nil
-}
-
-// GetPrefixDistributionContext mock implementation with context support
-func (m *mockStore) GetPrefixDistributionContext(_ context.Context) (
- []database.PrefixDistribution, []database.PrefixDistribution, error,
-) {
- return m.GetPrefixDistribution()
-}
-
-// GetLiveRouteCounts mock implementation
-func (m *mockStore) GetLiveRouteCounts() (int, int, error) {
- // Return mock counts
- return m.RouteCount / 2, m.RouteCount / 2, nil
-}
-
-// GetLiveRouteCountsContext mock implementation with context support
-func (m *mockStore) GetLiveRouteCountsContext(_ context.Context) (int, int, error) {
- return m.GetLiveRouteCounts()
-}
-
-// GetASDetails mock implementation
-func (m *mockStore) GetASDetails(asn int) (*database.ASN, []database.LiveRoute, error) {
+// GetAS returns the stored AS, or only its number when none is stored.
+func (m *mockStore) GetAS(_ context.Context, asn int) (*database.ASN, bool, error) {
m.mu.Lock()
defer m.mu.Unlock()
- // Check if ASN exists
if asnInfo, exists := m.ASNs[asn]; exists {
- // Return empty prefixes for now
- return asnInfo, []database.LiveRoute{}, nil
+ return asnInfo, true, nil
}
- return nil, nil, database.ErrNoRoute
+ return &database.ASN{ASN: asn}, false, nil
}
-// GetASDetailsContext mock implementation with context support
-func (m *mockStore) GetASDetailsContext(
- _ context.Context, asn int,
-) (*database.ASN, []database.LiveRoute, error) {
- return m.GetASDetails(asn)
+// GetASWHOIS returns the stored AS, or only its number when none is stored.
+func (m *mockStore) GetASWHOIS(ctx context.Context, asn int) (*database.ASN, error) {
+ asnInfo, _, err := m.GetAS(ctx, asn)
+
+ return asnInfo, err
}
-// GetPrefixDetails mock implementation
-func (m *mockStore) GetPrefixDetails(
- _ string,
-) ([]database.LiveRoute, map[int]database.ASN, error) {
- // Return empty routes for now
- return []database.LiveRoute{}, map[int]database.ASN{}, nil
-}
-
-// GetPrefixDetailsContext mock implementation with context support
-func (m *mockStore) GetPrefixDetailsContext(
- _ context.Context, prefix string,
-) ([]database.LiveRoute, map[int]database.ASN, error) {
- return m.GetPrefixDetails(prefix)
-}
-
-func (m *mockStore) GetRandomPrefixesByLength(
- _, _, _ int,
-) ([]database.LiveRoute, error) {
- // Return empty routes for now
- return []database.LiveRoute{}, nil
-}
-
-// GetRandomPrefixesByLengthContext mock implementation with context support
-func (m *mockStore) GetRandomPrefixesByLengthContext(
- _ context.Context, maskLength, ipVersion, limit int,
-) ([]database.LiveRoute, error) {
- return m.GetRandomPrefixesByLength(maskLength, ipVersion, limit)
+// GetPrefixFirstSeen returns the zero time: the mock keeps no prefix times.
+func (m *mockStore) GetPrefixFirstSeen(
+ _ context.Context, _ netip.Prefix,
+) (time.Time, error) {
+ return time.Time{}, nil
}
// GetASPeers mock implementation
@@ -302,32 +250,6 @@ func (m *mockStore) GetASPeersContext(
return m.GetASPeers(asn)
}
-// GetIPInfo mock implementation
-func (m *mockStore) GetIPInfo(ip string) (*database.IPInfo, error) {
- return m.GetIPInfoContext(context.Background(), ip)
-}
-
-// GetIPInfoContext mock implementation with context support
-func (m *mockStore) GetIPInfoContext(
- _ context.Context, ip string,
-) (*database.IPInfo, error) {
- now := time.Now()
-
- return &database.IPInfo{
- IP: ip,
- Netblock: "8.8.8.0/24",
- MaskLength: 24,
- IPVersion: 4,
- NumPeers: 3,
- ASN: 15169,
- Handle: "GOOGLE",
- Description: "Google LLC",
- CountryCode: "US",
- FirstSeen: now.Add(-24 * time.Hour),
- LastSeen: now,
- }, nil
-}
-
// GetNextStaleASN mock implementation
func (m *mockStore) GetNextStaleASN(_ context.Context, _ time.Duration) (int, error) {
return 0, database.ErrNoStaleASN
@@ -355,44 +277,28 @@ func (m *mockStore) GetWHOISStats(
}, nil
}
-// UpsertLiveRouteBatch mock implementation
-func (m *mockStore) UpsertLiveRouteBatch(routes []*database.LiveRoute) error {
+// WriteRouteChanges records the prefixes the changes leave some peer carrying,
+// whatever their origin AS.
+func (m *mockStore) WriteRouteChanges(
+ changes []routetable.Change, _ map[string]uint,
+) error {
m.mu.Lock()
defer m.mu.Unlock()
- for _, route := range routes {
- // Track prefix
- if _, exists := m.Prefixes[route.Prefix]; !exists {
- m.Prefixes[route.Prefix] = &database.Prefix{
- Prefix: route.Prefix,
- IPVersion: route.IPVersion,
- FirstSeen: route.LastUpdated,
- }
- m.PrefixCount++
-
- if route.IPVersion == 4 {
- m.IPv4Prefixes++
- } else {
- m.IPv6Prefixes++
- }
+ for _, change := range changes {
+ if change.Removed {
+ delete(m.TablePrefixes, change.Prefix.String())
+ } else {
+ m.TablePrefixes[change.Prefix.String()] = true
}
-
- m.RouteCount++
}
return nil
}
-// DeleteLiveRouteBatch mock implementation
-func (m *mockStore) DeleteLiveRouteBatch(_ []database.LiveRouteDeletion) error {
- // Simple mock - just return nil
- return nil
-}
-
-// DeleteLiveRoutesOfPeer mock implementation
-func (m *mockStore) DeleteLiveRoutesOfPeer(_ string) (int, error) {
- // Simple mock - deletes nothing
- return 0, nil
+// ReadRouteTable returns an empty routing table.
+func (m *mockStore) ReadRouteTable(_ context.Context) (*routetable.Table, error) {
+ return routetable.New(routetable.Params{}), nil
}
// GetOrCreateASNBatch mock implementation
@@ -466,19 +372,22 @@ func TestRouteWatchLiveFeed(t *testing.T) {
// Create streamer
s := streamer.New(logger, metricsTracker)
- // Create test config with empty state dir (no snapshot loading)
+ // The state directory holds the blocklist copies Run downloads.
cfg := &config.Config{
- StateDir: "",
+ StateDir: t.TempDir(),
MaxRuntime: 5 * time.Second,
EnableBatchedDatabaseWrites: true,
}
+ table := routetable.New(routetable.Params{})
+
// Create server
- srv := server.New(mockDB, s, logger, cfg)
+ srv := server.New(mockDB, table, s, logger, cfg)
// Create RouteWatch with 5 second limit
deps := Dependencies{
DB: mockDB,
+ Table: table,
Streamer: s,
Server: srv,
Logger: logger,
@@ -566,7 +475,8 @@ func announcementFeed(prefixes []string) string {
// handler's worker waits on the first one until the server sees the streamer
// close the connection, which only Shutdown does, so all three are still in
// the streamer when Shutdown starts. Each prefix must be in the database when
-// Shutdown returns.
+// Shutdown returns, both in the prefix tables and in the routing table's
+// writes.
func TestShutdownWritesTheUpdatesQueuedInTheStreamer(t *testing.T) {
t.Parallel()
@@ -596,11 +506,11 @@ func TestShutdownWritesTheUpdatesQueuedInTheStreamer(t *testing.T) {
rw := New(Dependencies{
DB: store,
Streamer: s,
- Server: server.New(store, s, log, cfg),
+ Server: server.New(store, nil, s, log, cfg),
Logger: log,
Config: cfg,
})
- rw.prefixHandler = NewPrefixHandler(store, log)
+ rw.prefixHandler = NewPrefixHandler(store, log, routetable.New(routetable.Params{}))
s.RegisterHandler(gatedPrefixHandler{
PrefixHandler: rw.prefixHandler, release: release,
})
@@ -640,9 +550,11 @@ func TestShutdownWritesTheUpdatesQueuedInTheStreamer(t *testing.T) {
defer store.mu.Unlock()
for _, prefix := range prefixes {
- if _, ok := store.Prefixes[prefix]; !ok {
- t.Errorf("%s was queued in the streamer when Shutdown started "+
- "but is not in the database", prefix)
+ _, inPrefixTables := store.Prefixes[prefix]
+ if !inPrefixTables || !store.TablePrefixes[prefix] {
+ t.Errorf("%s was queued in the streamer when Shutdown started; "+
+ "written to the prefix tables %t, in the routing table's "+
+ "writes %t", prefix, inPrefixTables, store.TablePrefixes[prefix])
}
}
}
@@ -696,7 +608,7 @@ func TestShutdownReturnsByItsDeadlineWhileAWriteIsBlocked(t *testing.T) {
rw := New(Dependencies{
DB: store,
Streamer: s,
- Server: server.New(store, s, log, cfg),
+ Server: server.New(store, nil, s, log, cfg),
Logger: log,
Config: cfg,
})
@@ -810,7 +722,7 @@ func TestShutdownCheckpointsAndClosesTheDatabaseLast(t *testing.T) {
rw := New(Dependencies{
DB: store,
Streamer: s,
- Server: server.New(store, s, log, cfg),
+ Server: server.New(store, nil, s, log, cfg),
Logger: log,
Config: cfg,
})
diff --git a/internal/routewatch/ashandler.go b/internal/routewatch/ashandler.go
index d82b9f8..5370bb0 100644
--- a/internal/routewatch/ashandler.go
+++ b/internal/routewatch/ashandler.go
@@ -52,7 +52,6 @@ type ASHandler struct {
// &mu.
batchTaken sync.Cond
batchFullCh chan struct{} // wakes the flush goroutine when batch is full
- lastFlush time.Time // used only by the flush goroutine
// stopCh hands Stop's context to the flush goroutine. It holds one, so
// Stop does not wait for a write in progress to hand it over.
stopCh chan context.Context
@@ -70,19 +69,27 @@ type asnOp struct {
// Stop when finished to ensure all pending operations are flushed and the
// background goroutine is terminated.
func NewASHandler(db database.Store, logger *logger.Logger) *ASHandler {
+ return newASHandler(db, logger, asnBatchTimeout)
+}
+
+// newASHandler is NewASHandler with the flush timer's interval, batchTimeout,
+// in place of asnBatchTimeout. A test that the timer must not disturb gives an
+// interval longer than the test runs.
+func newASHandler(
+ db database.Store, logger *logger.Logger, batchTimeout time.Duration,
+) *ASHandler {
h := &ASHandler{
db: db,
logger: logger,
batch: make([]asnOp, 0, asnBatchSize),
batchFullCh: make(chan struct{}, 1),
- lastFlush: time.Now(),
stopCh: make(chan context.Context, 1),
done: make(chan struct{}),
}
h.batchTaken.L = &h.mu
// Start the flush timer goroutine
- go h.flushLoop()
+ go flushLoop(batchTimeout, h.batchFullCh, h.stopCh, h.done, h.flushBatch)
return h
}
@@ -175,33 +182,6 @@ func (h *ASHandler) Stop(ctx context.Context) {
}
}
-// flushLoop runs in a goroutine, the only one that writes the batch: it
-// flushes when the batch is full, periodically, and once more on Stop.
-func (h *ASHandler) flushLoop() {
- defer close(h.done)
-
- ticker := time.NewTicker(asnBatchTimeout)
- defer ticker.Stop()
-
- for {
- select {
- case <-ticker.C:
- if time.Since(h.lastFlush) >= asnBatchTimeout {
- h.flushBatch()
- }
- case <-h.batchFullCh:
- h.flushBatch()
- case ctx := <-h.stopCh:
- // Final flush, unless the stop deadline has passed
- if ctx.Err() == nil {
- h.flushBatch()
- }
-
- return
- }
- }
-}
-
// takeBatch returns the pending operations and leaves an empty batch in their
// place for HandleMessage to fill. They count in writing until flushBatch has
// written them.
@@ -249,6 +229,4 @@ func (h *ASHandler) flushBatch() {
h.mu.Lock()
h.writing = 0
h.mu.Unlock()
-
- h.lastFlush = time.Now()
}
diff --git a/internal/routewatch/ashandler_internal_test.go b/internal/routewatch/ashandler_internal_test.go
index 33094d1..3e028d0 100644
--- a/internal/routewatch/ashandler_internal_test.go
+++ b/internal/routewatch/ashandler_internal_test.go
@@ -29,9 +29,9 @@ func TestASHandlerStoresTheEarliestAndLatestTimesOfANewAS(t *testing.T) {
h.HandleMessage(second)
h.flushBatch()
- asn, _, err := db.GetASDetails(testASNA)
+ asn, _, err := db.GetAS(t.Context(), testASNA)
if err != nil {
- t.Fatalf("GetASDetails(%d): %v", testASNA, err)
+ t.Fatalf("GetAS(%d): %v", testASNA, err)
}
if !asn.FirstSeen.Equal(first.ParsedTimestamp) {
diff --git a/internal/routewatch/asnfetcher.go b/internal/routewatch/asnfetcher.go
index 6d51b24..4f5eb7d 100644
--- a/internal/routewatch/asnfetcher.go
+++ b/internal/routewatch/asnfetcher.go
@@ -21,10 +21,16 @@ const (
minInterval = 1 * time.Second
// maxInterval is the maximum interval after failures (backoff cap).
- maxInterval = 5 * time.Minute
+ maxInterval = time.Minute
- // backoffMultiplier is how much to multiply interval on failure.
- backoffMultiplier = 2
+ // A failure multiplies the interval by backoffNumerator divided by
+ // backoffDenominator, 1.5, up to maxInterval.
+ backoffNumerator = 3
+ backoffDenominator = 2
+
+ // A success divides the interval by successDivisor, down to
+ // minInterval.
+ successDivisor = 2
// whoisStaleThreshold is how old WHOIS data can be before refresh.
whoisStaleThreshold = 30 * 24 * time.Hour // 30 days
@@ -62,10 +68,12 @@ type ASNFetcher struct {
// NewASNFetcher creates a new ASN fetcher.
func NewASNFetcher(db database.Store, logger *slog.Logger) *ASNFetcher {
+ logger = logger.With("component", "asn_fetcher")
+
return &ASNFetcher{
db: db,
- whoisClient: whois.NewClient(),
- logger: logger.With("component", "asn_fetcher"),
+ whoisClient: whois.NewClient(logger),
+ logger: logger,
immediateQueue: make(chan int, immediateQueueSize),
stopCh: make(chan struct{}),
done: make(chan struct{}),
@@ -160,7 +168,7 @@ func (f *ASNFetcher) recordSuccess() {
f.consecutiveFails = 0
// Decrease interval by half, but not below minimum
- newInterval := max(f.currentInterval/backoffMultiplier, minInterval)
+ newInterval := max(f.currentInterval/successDivisor, minInterval)
if newInterval != f.currentInterval {
f.logger.Debug("Decreased fetch interval",
@@ -184,8 +192,9 @@ func (f *ASNFetcher) recordFailure() {
f.intervalMu.Lock()
f.consecutiveFails++
- // Exponential backoff: multiply by 2, capped at max
- newInterval := min(f.currentInterval*backoffMultiplier, maxInterval)
+ // Exponential backoff: multiply by 1.5, capped at max
+ newInterval := min(
+ f.currentInterval*backoffNumerator/backoffDenominator, maxInterval)
if newInterval != f.currentInterval {
f.logger.Debug("Increased fetch interval due to failure",
diff --git a/internal/routewatch/asnfetcher_internal_test.go b/internal/routewatch/asnfetcher_internal_test.go
index 26c73f8..6779d3d 100644
--- a/internal/routewatch/asnfetcher_internal_test.go
+++ b/internal/routewatch/asnfetcher_internal_test.go
@@ -120,17 +120,58 @@ func TestFetchAndUpdateKeepsTheRowWhenTheLookupFails(t *testing.T) {
"was stored", stats.NeverFetched)
}
+ want := baseInterval * backoffNumerator / backoffDenominator
+
got := f.GetStats()
- if got.CurrentInterval != baseInterval*backoffMultiplier ||
- got.ConsecutiveFails != 1 {
+ if got.CurrentInterval != want || got.ConsecutiveFails != 1 {
t.Errorf("interval %v after %d failures, want %v after 1",
- got.CurrentInterval, got.ConsecutiveFails,
- baseInterval*backoffMultiplier)
+ got.CurrentInterval, got.ConsecutiveFails, want)
}
})
}
}
+// TestRecordStepsTheInterval has the fetcher fail from its base interval of
+// 15 seconds, then succeed. Each failure makes the interval 1.5 times as
+// long, up to one minute, and each success halves it, down to one second.
+func TestRecordStepsTheInterval(t *testing.T) {
+ t.Parallel()
+
+ f := NewASNFetcher(newMockStore(), logger.New().Logger)
+
+ for i, want := range []time.Duration{
+ 22500 * time.Millisecond,
+ 33750 * time.Millisecond,
+ 50625 * time.Millisecond,
+ time.Minute,
+ time.Minute,
+ } {
+ f.recordFailure()
+
+ if got := f.getInterval(); got != want {
+ t.Errorf("interval after %d failures = %v, want %v",
+ i+1, got, want)
+ }
+ }
+
+ for i, want := range []time.Duration{
+ 30 * time.Second,
+ 15 * time.Second,
+ 7500 * time.Millisecond,
+ 3750 * time.Millisecond,
+ 1875 * time.Millisecond,
+ time.Second,
+ time.Second,
+ } {
+ f.recordSuccess()
+
+ if got := f.getInterval(); got != want {
+ t.Errorf("interval after %d successes = %v, want %v",
+ i+1, got, want)
+ }
+ }
+}
+
// TestFetchAndUpdateMarksAnASWithoutReferral looks up a private-use AS, which
// IANA refers to no RIR. The AS must not be looked up again until it is
// stale, and the lookup counts as neither a success nor a failure.
diff --git a/internal/routewatch/batchwrite.go b/internal/routewatch/batchwrite.go
new file mode 100644
index 0000000..6388342
--- /dev/null
+++ b/internal/routewatch/batchwrite.go
@@ -0,0 +1,62 @@
+package routewatch
+
+import (
+ "context"
+ "time"
+)
+
+// flushLoop is the flush goroutine of the prefix, AS and peer handlers. It is
+// the only goroutine that writes a handler's batch. flushBatch takes the batch
+// and writes it.
+//
+// flushLoop calls flushBatch on each tick of a timer, every batchTimeout. It
+// also calls it at once when batchFullCh wakes it. A full batch's write
+// restarts the timer, so the next tick comes batchTimeout after that write
+// began. When stopCh hands it Stop's context, it calls flushBatch once more,
+// unless the stop deadline has passed. Then it closes done and returns.
+func flushLoop(
+ batchTimeout time.Duration,
+ batchFullCh <-chan struct{},
+ stopCh <-chan context.Context,
+ done chan<- struct{},
+ flushBatch func(),
+) {
+ defer close(done)
+
+ ticker := time.NewTicker(batchTimeout)
+ defer ticker.Stop()
+
+ for {
+ // Stop goes first. A write can run past the stop deadline while a full
+ // batch's wake-up or a tick waits beside Stop's context. The select
+ // below picks one of them at random, but past the deadline no flush
+ // may start.
+ select {
+ case ctx := <-stopCh:
+ // Final flush, unless the stop deadline has passed
+ if ctx.Err() == nil {
+ flushBatch()
+ }
+
+ return
+ default:
+ }
+
+ select {
+ case <-ticker.C:
+ flushBatch()
+ case <-batchFullCh:
+ // Reset also drops a tick that waits in ticker.C, so no timed
+ // write follows this one at once.
+ ticker.Reset(batchTimeout)
+ flushBatch()
+ case ctx := <-stopCh:
+ // Final flush, unless the stop deadline has passed
+ if ctx.Err() == nil {
+ flushBatch()
+ }
+
+ return
+ }
+ }
+}
diff --git a/internal/routewatch/batchwrite_internal_test.go b/internal/routewatch/batchwrite_internal_test.go
index b9ce48d..e196fb9 100644
--- a/internal/routewatch/batchwrite_internal_test.go
+++ b/internal/routewatch/batchwrite_internal_test.go
@@ -4,15 +4,18 @@ import (
"bytes"
"context"
"encoding/json"
+ "fmt"
"log/slog"
"slices"
"sync"
"testing"
+ "testing/synctest"
"time"
"git.eeqj.de/sneak/routewatch/internal/database"
"git.eeqj.de/sneak/routewatch/internal/logger"
"git.eeqj.de/sneak/routewatch/internal/ristypes"
+ "git.eeqj.de/sneak/routewatch/internal/routetable"
)
const (
@@ -25,10 +28,62 @@ const (
stillWaitingAfter = 100 * time.Millisecond
)
+// TestFlushLoopWritesOnEveryTick runs flushLoop in a synctest bubble, whose
+// clock is fake: time.Sleep there moves the clock on and returns at once. Each
+// write takes a tenth of the interval. Two timed writes must start one interval
+// apart. A full batch's write, which starts as the second timed write ends,
+// must be followed by a timed write one interval after it began.
+func TestFlushLoopWritesOnEveryTick(t *testing.T) {
+ t.Parallel()
+
+ synctest.Test(t, func(t *testing.T) {
+ const (
+ interval = time.Second
+ writeTime = 100 * time.Millisecond
+ )
+
+ writeStarts := make(chan time.Time)
+ batchFullCh := make(chan struct{}, 1)
+ stopCh := make(chan context.Context, 1)
+ done := make(chan struct{})
+
+ go func() {
+ flushLoop(interval, batchFullCh, stopCh, done, func() {
+ writeStarts <- time.Now()
+
+ time.Sleep(writeTime)
+ })
+ }()
+
+ first := <-writeStarts
+ second := <-writeStarts
+
+ if gap := second.Sub(first); gap != interval {
+ t.Errorf("the second timed write started %s after the first, "+
+ "want %s", gap, interval)
+ }
+
+ batchFullCh <- struct{}{}
+
+ full := <-writeStarts
+ timed := <-writeStarts
+
+ if gap := timed.Sub(full); gap != interval {
+ t.Errorf("the timed write after a full batch's started %s after "+
+ "it, want %s", gap, interval)
+ }
+
+ stopCh <- t.Context()
+
+ <-writeStarts // Stop's last write
+ <-done
+ })
+}
+
// blockingStore is a mockStore whose batch writes for the AS, peer and prefix
-// handlers, delete of a peer's routes and peering write wait until the test
-// calls releaseWrites. writeStarted is closed when the first of them starts. It
-// records what each batch write was given.
+// handlers and peering write wait until the test calls releaseWrites.
+// writeStarted is closed when the first of them starts. It records what each
+// batch write was given.
type blockingStore struct {
*mockStore
@@ -54,13 +109,13 @@ func newBlockingStore() *blockingStore {
}
}
-func (s *blockingStore) UpsertLiveRouteBatch(routes []*database.LiveRoute) error {
+func (s *blockingStore) UpdatePrefixesBatch(prefixes map[string]time.Time) error {
s.waitForRelease()
s.mu.Lock()
defer s.mu.Unlock()
- for _, route := range routes {
- s.prefixes[route.Prefix] = true
+ for prefix := range prefixes {
+ s.prefixes[prefix] = true
}
return nil
@@ -90,12 +145,6 @@ func (s *blockingStore) UpdatePeerBatch(peers map[string]database.PeerUpdate) er
return nil
}
-func (s *blockingStore) DeleteLiveRoutesOfPeer(peerIP string) (int, error) {
- s.waitForRelease()
-
- return s.mockStore.DeleteLiveRoutesOfPeer(peerIP)
-}
-
func (s *blockingStore) RecordPeering(
asA, asB int, seen database.SeenTimes,
) error {
@@ -119,10 +168,10 @@ func (s *blockingStore) releaseWrites() {
//
//nolint:tagliatelle // field names are the keys the handlers log
type stopLog struct {
- Msg string `json:"msg"`
- Updates int `json:"updates"`
- PeersDown int `json:"peers_down"`
- Paths int `json:"paths"`
+ Msg string `json:"msg"`
+ Updates int `json:"updates"`
+ TableChanges int `json:"table_changes"`
+ Paths int `json:"paths"`
}
// newErrorLogger returns a logger that writes only errors, as JSON lines, to
@@ -166,9 +215,10 @@ func returnsInTime(f func()) bool {
}
}
-// startBlockedWrite hands h the messages in fill, which fill its batch so that
-// the handler writes it to store, and waits until that write starts. store
-// holds it until the test calls releaseWrites.
+// startBlockedWrite hands h the messages in fill and waits until h starts to
+// write them to store. The prefix, AS and peer handlers start when fill fills
+// their batch, and the peering handler on its timer. store holds the write
+// until the test calls releaseWrites.
func startBlockedWrite(
t *testing.T,
store *blockingStore,
@@ -191,7 +241,7 @@ func startBlockedWrite(
select {
case <-store.writeStarted:
case <-time.After(handleMessageTimeout):
- t.Fatal("the handler did not write its full batch")
+ t.Fatal("the handler did not start to write the messages")
}
}
@@ -271,7 +321,7 @@ func handleAtMaxBatchSize(
}
// stopWhileAWriteIsBlocked starts a blocked write with the messages in fill,
-// hands h the message extra, and stops h with a deadline that passes while
+// hands h the messages in extra, and stops h with a deadline that passes while
// store holds the write. Stop must return all the same. Then it lets the write
// finish and waits until done, closed when h's flush goroutine returns. Past
// the deadline that goroutine skips its last write, so extra is not written,
@@ -285,7 +335,7 @@ func stopWhileAWriteIsBlocked(
},
done <-chan struct{},
fill []*ristypes.RISMessage,
- extra *ristypes.RISMessage,
+ extra ...*ristypes.RISMessage,
) {
t.Helper()
@@ -294,7 +344,10 @@ func stopWhileAWriteIsBlocked(
defer store.releaseWrites()
startBlockedWrite(t, store, h, fill)
- h.HandleMessage(extra)
+
+ for _, msg := range extra {
+ h.HandleMessage(msg)
+ }
ctx, cancel := context.WithTimeout(t.Context(), stopDeadline)
defer cancel()
@@ -313,6 +366,11 @@ func stopWhileAWriteIsBlocked(
}
}
+// TestPrefixHandlerSkipsItsLastWriteAfterTheStopDeadline also checks the
+// routing table's changes: the flush whose prefix write is held wrote those of
+// the fill, so Stop must log the one of the extra prefix as not written. The
+// flush timer must not fire. A flush it started before the fill would be the
+// one held, and it would not have written the fill's change.
func TestPrefixHandlerSkipsItsLastWriteAfterTheStopDeadline(t *testing.T) {
t.Parallel()
@@ -321,12 +379,11 @@ func TestPrefixHandlerSkipsItsLastWriteAfterTheStopDeadline(t *testing.T) {
var logs bytes.Buffer
store := newBlockingStore()
- h := NewPrefixHandler(store, newErrorLogger(&logs))
+ h := newPrefixHandler(store, newErrorLogger(&logs),
+ routetable.New(routetable.Params{}), longerThanAnyTest)
fill := slices.Repeat([]string{testV4Prefix}, prefixBatchSize)
- // The peer's session goes down first, so the write held is the delete of
- // its routes, and Stop must log that peer down too.
stopWhileAWriteIsBlocked(t, store, h, h.done,
- []*ristypes.RISMessage{sessionDown(), prefixAnnouncement(fill)},
+ []*ristypes.RISMessage{prefixAnnouncement(fill)},
prefixAnnouncement([]string{extraPrefix}))
store.mu.Lock()
@@ -337,9 +394,75 @@ func TestPrefixHandlerSkipsItsLastWriteAfterTheStopDeadline(t *testing.T) {
}
logged := lastLogged(t, &logs)
- if logged.Updates != prefixBatchSize+1 || logged.PeersDown != 1 {
- t.Errorf("Stop logged %d updates and %d peer downs not written, "+
- "want %d and 1", logged.Updates, logged.PeersDown, prefixBatchSize+1)
+ if logged.Updates != prefixBatchSize+1 || logged.TableChanges != 1 {
+ t.Errorf("Stop logged %d updates and %d table changes not written, "+
+ "want %d and 1", logged.Updates, logged.TableChanges,
+ prefixBatchSize+1)
+ }
+}
+
+// TestPrefixHandlerSkipsAFullBatchAfterTheStopDeadline hands the prefix handler
+// a full batch while a write is held, so the batch's wake-up of the flush
+// goroutine waits. The stop deadline passes meanwhile. When the write ends, the
+// flush goroutine must stop instead of writing the batch.
+func TestPrefixHandlerSkipsAFullBatchAfterTheStopDeadline(t *testing.T) {
+ t.Parallel()
+
+ const extraPrefix = "198.51.100.0/24"
+
+ store := newBlockingStore()
+ h := NewPrefixHandler(store, logger.New(), routetable.New(routetable.Params{}))
+ fill := slices.Repeat([]string{testV4Prefix}, prefixBatchSize)
+ full := slices.Repeat([]string{extraPrefix}, prefixBatchSize)
+ stopWhileAWriteIsBlocked(t, store, h, h.done,
+ []*ristypes.RISMessage{prefixAnnouncement(fill)},
+ prefixAnnouncement(full))
+
+ store.mu.Lock()
+ defer store.mu.Unlock()
+
+ if store.prefixes[extraPrefix] {
+ t.Errorf("%s was written after the stop deadline", extraPrefix)
+ }
+}
+
+// tableWriteBlockingStore is a blockingStore whose writes of the routing
+// table's changes also wait until the test calls releaseWrites.
+type tableWriteBlockingStore struct {
+ *blockingStore
+}
+
+func (s tableWriteBlockingStore) WriteRouteChanges(
+ changes []routetable.Change, newPeers map[string]uint,
+) error {
+ s.waitForRelease()
+
+ return s.mockStore.WriteRouteChanges(changes, newPeers)
+}
+
+// TestPrefixHandlerStopCountsTheTableChangesOfAWriteInProgress holds the write
+// of the routing table's changes that a full batch starts, then hands the
+// prefix handler one more prefix and stops it with a deadline that passes
+// meanwhile. Stop must log as not written both the change the held write took
+// and the one since.
+func TestPrefixHandlerStopCountsTheTableChangesOfAWriteInProgress(t *testing.T) {
+ t.Parallel()
+
+ const extraPrefix = "198.51.100.0/24"
+
+ var logs bytes.Buffer
+
+ blocking := newBlockingStore()
+ h := NewPrefixHandler(tableWriteBlockingStore{blockingStore: blocking},
+ newErrorLogger(&logs), routetable.New(routetable.Params{}))
+ fill := slices.Repeat([]string{testV4Prefix}, prefixBatchSize)
+ stopWhileAWriteIsBlocked(t, blocking, h, h.done,
+ []*ristypes.RISMessage{prefixAnnouncement(fill)},
+ prefixAnnouncement([]string{extraPrefix}))
+
+ if logged := lastLogged(t, &logs); logged.TableChanges != 2 {
+ t.Errorf("Stop logged %d table changes not written, want 2",
+ logged.TableChanges)
}
}
@@ -353,7 +476,7 @@ func TestASHandlerSkipsItsLastWriteAfterTheStopDeadline(t *testing.T) {
var logs bytes.Buffer
store := newBlockingStore()
- h := NewASHandler(store, newErrorLogger(&logs))
+ h := newASHandler(store, newErrorLogger(&logs), longerThanAnyTest)
fill := ristypes.ASPath{Sequence: slices.Repeat([]int{testASNA}, asnBatchSize)}
stopWhileAWriteIsBlocked(t, store, h, h.done,
[]*ristypes.RISMessage{{Path: fill}},
@@ -372,6 +495,29 @@ func TestASHandlerSkipsItsLastWriteAfterTheStopDeadline(t *testing.T) {
}
}
+// TestASHandlerSkipsAFullBatchAfterTheStopDeadline hands the AS handler a full
+// batch while a write is held, so the batch's wake-up of the flush goroutine
+// waits. The stop deadline passes meanwhile. When the write ends, the flush
+// goroutine must stop instead of writing the batch.
+func TestASHandlerSkipsAFullBatchAfterTheStopDeadline(t *testing.T) {
+ t.Parallel()
+
+ store := newBlockingStore()
+ h := newASHandler(store, logger.New(), longerThanAnyTest)
+ fill := ristypes.ASPath{Sequence: slices.Repeat([]int{testASNA}, asnBatchSize)}
+ full := ristypes.ASPath{Sequence: slices.Repeat([]int{testASNB}, asnBatchSize)}
+ stopWhileAWriteIsBlocked(t, store, h, h.done,
+ []*ristypes.RISMessage{{Path: fill}},
+ &ristypes.RISMessage{Path: full})
+
+ store.mu.Lock()
+ defer store.mu.Unlock()
+
+ if store.asns[testASNB] {
+ t.Errorf("AS%d was written after the stop deadline", testASNB)
+ }
+}
+
func TestPeerHandlerSkipsItsLastWriteAfterTheStopDeadline(t *testing.T) {
t.Parallel()
@@ -380,7 +526,7 @@ func TestPeerHandlerSkipsItsLastWriteAfterTheStopDeadline(t *testing.T) {
var logs bytes.Buffer
store := newBlockingStore()
- h := NewPeerHandler(store, newErrorLogger(&logs))
+ h := newPeerHandler(store, newErrorLogger(&logs), longerThanAnyTest)
stopWhileAWriteIsBlocked(t, store, h, h.done,
slices.Repeat([]*ristypes.RISMessage{{Peer: testPeerIP}}, peerBatchSize),
&ristypes.RISMessage{Peer: otherPeerIP})
@@ -398,6 +544,30 @@ func TestPeerHandlerSkipsItsLastWriteAfterTheStopDeadline(t *testing.T) {
}
}
+// TestPeerHandlerSkipsAFullBatchAfterTheStopDeadline hands the peer handler a
+// full batch while a write is held, so the batch's wake-up of the flush
+// goroutine waits. The stop deadline passes meanwhile. When the write ends, the
+// flush goroutine must stop instead of writing the batch.
+func TestPeerHandlerSkipsAFullBatchAfterTheStopDeadline(t *testing.T) {
+ t.Parallel()
+
+ const otherPeerIP = "2001:db8:ffff::2"
+
+ store := newBlockingStore()
+ h := newPeerHandler(store, logger.New(), longerThanAnyTest)
+ stopWhileAWriteIsBlocked(t, store, h, h.done,
+ slices.Repeat([]*ristypes.RISMessage{{Peer: testPeerIP}}, peerBatchSize),
+ slices.Repeat([]*ristypes.RISMessage{{Peer: otherPeerIP}},
+ peerBatchSize)...)
+
+ store.mu.Lock()
+ defer store.mu.Unlock()
+
+ if store.peers[otherPeerIP] {
+ t.Errorf("peer %s was written after the stop deadline", otherPeerIP)
+ }
+}
+
// TestPeeringHandlerStopCountsTheWriteInProgress hands a running peering
// handler one AS path and stops it with a deadline that passes while store
// holds Stop's last write, of that path's peering. Stop must log that path as
@@ -437,6 +607,35 @@ func TestPeeringHandlerStopCountsTheWriteInProgress(t *testing.T) {
}
}
+// TestPeeringHandlerStartsNoWriteAfterTheStopDeadline has the peering handler's
+// timer start a write that store holds, and hands the handler one more AS path
+// meanwhile. The stop deadline passes, and a tick of the timer waits. When the
+// write ends, the processing goroutine must stop instead of writing the path.
+func TestPeeringHandlerStartsNoWriteAfterTheStopDeadline(t *testing.T) {
+ t.Parallel()
+
+ // The timer fires many times while Stop waits for its deadline.
+ const processInterval = 10 * time.Millisecond
+
+ store := newBlockingStore()
+ h := newPeeringHandler(store, logger.New(), processInterval)
+ stopWhileAWriteIsBlocked(t, store, h, h.done,
+ []*ristypes.RISMessage{
+ {Path: ristypes.ASPath{Sequence: []int{testASNA, testASNB}}},
+ },
+ &ristypes.RISMessage{
+ Path: ristypes.ASPath{Sequence: []int{testASNC, testASND}},
+ })
+
+ store.mockStore.mu.Lock()
+ defer store.mockStore.mu.Unlock()
+
+ if store.Peerings[fmt.Sprintf("%d_%d", testASNC, testASND)] {
+ t.Errorf("the peering of AS%d and AS%d was written after the stop "+
+ "deadline", testASNC, testASND)
+ }
+}
+
// prefixAnnouncement returns a message in which testPeerIP announces prefixes
// with origin AS testASNA.
func prefixAnnouncement(prefixes []string) *ristypes.RISMessage {
@@ -455,7 +654,7 @@ func TestPrefixHandlerHandlesMessagesWhileAWriteIsBlocked(t *testing.T) {
const extraPrefix = "198.51.100.0/24"
store := newBlockingStore()
- h := NewPrefixHandler(store, logger.New())
+ h := NewPrefixHandler(store, logger.New(), routetable.New(routetable.Params{}))
fill := slices.Repeat([]string{testV4Prefix}, prefixBatchSize)
handleWhileAWriteIsBlocked(t, store, h,
[]*ristypes.RISMessage{prefixAnnouncement(fill)},
@@ -474,7 +673,7 @@ func TestASHandlerHandlesMessagesWhileAWriteIsBlocked(t *testing.T) {
t.Parallel()
store := newBlockingStore()
- h := NewASHandler(store, logger.New())
+ h := newASHandler(store, logger.New(), longerThanAnyTest)
fill := ristypes.ASPath{Sequence: slices.Repeat([]int{testASNA}, asnBatchSize)}
handleWhileAWriteIsBlocked(t, store, h,
[]*ristypes.RISMessage{{Path: fill}},
@@ -495,7 +694,7 @@ func TestPeerHandlerHandlesMessagesWhileAWriteIsBlocked(t *testing.T) {
const otherPeerIP = "2001:db8:ffff::2"
store := newBlockingStore()
- h := NewPeerHandler(store, logger.New())
+ h := newPeerHandler(store, logger.New(), longerThanAnyTest)
handleWhileAWriteIsBlocked(t, store, h,
slices.Repeat([]*ristypes.RISMessage{{Peer: testPeerIP}}, peerBatchSize),
&ristypes.RISMessage{Peer: otherPeerIP})
@@ -515,7 +714,7 @@ func TestPrefixHandlerWaitsAtTheMaximumBatchSize(t *testing.T) {
const extraPrefix = "198.51.100.0/24"
store := newBlockingStore()
- h := NewPrefixHandler(store, logger.New())
+ h := NewPrefixHandler(store, logger.New(), routetable.New(routetable.Params{}))
fill := slices.Repeat([]string{testV4Prefix}, prefixBatchSize)
toMax := slices.Repeat([]string{testV4Prefix}, prefixBatchMaxSize)
handleAtMaxBatchSize(t, store, h,
@@ -536,7 +735,7 @@ func TestASHandlerWaitsAtTheMaximumBatchSize(t *testing.T) {
t.Parallel()
store := newBlockingStore()
- h := NewASHandler(store, logger.New())
+ h := newASHandler(store, logger.New(), longerThanAnyTest)
fill := ristypes.ASPath{Sequence: slices.Repeat([]int{testASNA}, asnBatchSize)}
toMax := ristypes.ASPath{Sequence: slices.Repeat([]int{testASNA}, asnBatchMaxSize)}
handleAtMaxBatchSize(t, store, h,
@@ -553,13 +752,17 @@ func TestASHandlerWaitsAtTheMaximumBatchSize(t *testing.T) {
}
}
+// TestPeerHandlerWaitsAtTheMaximumBatchSize must not have its flush timer fire.
+// A timer flush during the fill would take only part of it. The batch would
+// then reach its maximum size before the test's last message, and the test
+// would hang.
func TestPeerHandlerWaitsAtTheMaximumBatchSize(t *testing.T) {
t.Parallel()
const otherPeerIP = "2001:db8:ffff::2"
store := newBlockingStore()
- h := NewPeerHandler(store, logger.New())
+ h := newPeerHandler(store, logger.New(), longerThanAnyTest)
message := []*ristypes.RISMessage{{Peer: testPeerIP}}
handleAtMaxBatchSize(t, store, h,
slices.Repeat(message, peerBatchSize),
@@ -592,7 +795,10 @@ func TestHandlersWakeTheFlushGoroutineWhenTheBatchIsFull(t *testing.T) {
name: "prefix",
batchSize: prefixBatchSize,
handle: func(n int) chan struct{} {
- h := &PrefixHandler{batchFullCh: make(chan struct{}, 1)}
+ h := &PrefixHandler{
+ batchFullCh: make(chan struct{}, 1),
+ table: routetable.New(routetable.Params{}),
+ }
h.HandleMessage(prefixAnnouncement(
slices.Repeat([]string{testV4Prefix}, n)))
diff --git a/internal/routewatch/peerhandler.go b/internal/routewatch/peerhandler.go
index 6fb3278..b15a58e 100644
--- a/internal/routewatch/peerhandler.go
+++ b/internal/routewatch/peerhandler.go
@@ -56,7 +56,6 @@ type PeerHandler struct {
// is &mu.
batchTaken sync.Cond
batchFullCh chan struct{} // wakes the flush goroutine when peerBatch is full
- lastFlush time.Time // used only by the flush goroutine
// stopCh hands Stop's context to the flush goroutine. It holds one, so
// Stop does not wait for a write in progress to hand it over.
stopCh chan context.Context
@@ -75,19 +74,27 @@ type peerUpdate struct {
// periodically flushes accumulated peer updates to the database. The handler
// should be stopped by calling Stop when it is no longer needed.
func NewPeerHandler(db database.Store, logger *logger.Logger) *PeerHandler {
+ return newPeerHandler(db, logger, peerBatchTimeout)
+}
+
+// newPeerHandler is NewPeerHandler with the flush timer's interval,
+// batchTimeout, in place of peerBatchTimeout. A test that the timer must not
+// disturb gives an interval longer than the test runs.
+func newPeerHandler(
+ db database.Store, logger *logger.Logger, batchTimeout time.Duration,
+) *PeerHandler {
h := &PeerHandler{
db: db,
logger: logger,
peerBatch: make([]peerUpdate, 0, peerBatchSize),
batchFullCh: make(chan struct{}, 1),
- lastFlush: time.Now(),
stopCh: make(chan context.Context, 1),
done: make(chan struct{}),
}
h.batchTaken.L = &h.mu
// Start the flush timer goroutine
- go h.flushLoop()
+ go flushLoop(batchTimeout, h.batchFullCh, h.stopCh, h.done, h.flushBatch)
return h
}
@@ -173,33 +180,6 @@ func (h *PeerHandler) Stop(ctx context.Context) {
}
}
-// flushLoop runs in a goroutine, the only one that writes the batch: it
-// flushes when the batch is full, periodically, and once more on Stop.
-func (h *PeerHandler) flushLoop() {
- defer close(h.done)
-
- ticker := time.NewTicker(peerBatchTimeout)
- defer ticker.Stop()
-
- for {
- select {
- case <-ticker.C:
- if time.Since(h.lastFlush) >= peerBatchTimeout {
- h.flushBatch()
- }
- case <-h.batchFullCh:
- h.flushBatch()
- case ctx := <-h.stopCh:
- // Final flush, unless the stop deadline has passed
- if ctx.Err() == nil {
- h.flushBatch()
- }
-
- return
- }
- }
-}
-
// takeBatch returns the pending updates and leaves an empty batch in their
// place for HandleMessage to fill. They count in writing until flushBatch has
// written them.
@@ -254,6 +234,4 @@ func (h *PeerHandler) flushBatch() {
h.mu.Lock()
h.writing = 0
h.mu.Unlock()
-
- h.lastFlush = time.Now()
}
diff --git a/internal/routewatch/peeringhandler.go b/internal/routewatch/peeringhandler.go
index 8e5f801..046a210 100644
--- a/internal/routewatch/peeringhandler.go
+++ b/internal/routewatch/peeringhandler.go
@@ -62,6 +62,15 @@ type PeeringHandler struct {
// periodically processes accumulated AS paths into peering records. The
// handler begins processing immediately upon creation.
func NewPeeringHandler(db database.Store, logger *logger.Logger) *PeeringHandler {
+ return newPeeringHandler(db, logger, peeringProcessInterval)
+}
+
+// newPeeringHandler is NewPeeringHandler with the processing timer's interval,
+// processInterval, in place of peeringProcessInterval. A test gives a short
+// one to have the timer start a write.
+func newPeeringHandler(
+ db database.Store, logger *logger.Logger, processInterval time.Duration,
+) *PeeringHandler {
h := &PeeringHandler{
db: db,
logger: logger,
@@ -71,7 +80,7 @@ func NewPeeringHandler(db database.Store, logger *logger.Logger) *PeeringHandler
}
// Start the periodic processing goroutine
- go h.processLoop()
+ go h.processLoop(processInterval)
return h
}
@@ -163,15 +172,29 @@ func (h *PeeringHandler) Stop(ctx context.Context) {
}
}
-// processLoop runs periodically to process AS paths into peerings, and once
+// processLoop processes AS paths into peerings every processInterval, and once
// more on Stop.
-func (h *PeeringHandler) processLoop() {
+func (h *PeeringHandler) processLoop(processInterval time.Duration) {
defer close(h.done)
- ticker := time.NewTicker(peeringProcessInterval)
+ ticker := time.NewTicker(processInterval)
defer ticker.Stop()
for {
+ // Stop goes first. A write can run past the stop deadline while a
+ // tick waits beside Stop's context. The select below picks one of
+ // them at random, but past the deadline no write may start.
+ select {
+ case ctx := <-h.stopCh:
+ // Final processing, unless the stop deadline has passed
+ if ctx.Err() == nil {
+ h.processPeerings()
+ }
+
+ return
+ default:
+ }
+
select {
case <-ticker.C:
h.processPeerings()
diff --git a/internal/routewatch/prefixhandler.go b/internal/routewatch/prefixhandler.go
index 8a468fd..6472a39 100644
--- a/internal/routewatch/prefixhandler.go
+++ b/internal/routewatch/prefixhandler.go
@@ -2,10 +2,8 @@ package routewatch
import (
"context"
- "net"
"net/netip"
"slices"
- "strings"
"sync"
"time"
@@ -13,6 +11,7 @@ import (
"git.eeqj.de/sneak/routewatch/internal/logger"
"git.eeqj.de/sneak/routewatch/internal/metrics"
"git.eeqj.de/sneak/routewatch/internal/ristypes"
+ "git.eeqj.de/sneak/routewatch/internal/routetable"
)
const (
@@ -27,64 +26,71 @@ const (
prefixBatchSize = 25000
// prefixBatchMaxSize is the most prefix updates the batch holds while the
- // flush goroutine writes the previous one: ten batches, about 60 MiB
+ // flush goroutine writes the previous one: ten batches, about 15 MiB
// (README "Memory"). At this size HandleMessage waits until the flush
- // goroutine takes the batch. On top of these, the handler holds at most
- // prefixBatchSize updates, about 6 MiB, that a flush kept for the peers
- // whose route delete failed (held).
+ // goroutine takes the batch.
prefixBatchMaxSize = 250000
// prefixBatchTimeout is the maximum time to wait before flushing a batch
// DO NOT reduce this timeout - larger batches are more efficient
prefixBatchTimeout = 1 * time.Second
- // IP version constants
- ipv4Version = 4
- ipv6Version = 6
+ // maxTableChangesPerWrite is the most changed prefix and origin AS pairs
+ // of the routing table that one flush writes; the rest wait for the next
+ // flushes. The change of one pair takes about 430 bytes of memory and 15
+ // microseconds to write. So a write holds about 9 MB and takes about a
+ // third of the one-second flush interval, which leaves the rest to the
+ // batch. The feed peaks at about 5,000 updates a second, each changing at
+ // most one pair. So the million pairs that a full-table peer's session
+ // going down changes are written in about a minute.
+ maxTableChangesPerWrite = 20000
)
-// PrefixHandler tracks BGP prefixes and maintains a live routing table in the
-// database. Routes are added on announcement and deleted on withdrawal or on an
-// announcement whose AS path is empty or holds AS 0 anywhere, and all routes of
-// a peer are deleted when its BGP session goes down.
+// PrefixHandler keeps the routing table in memory current from the feed. It
+// applies each update to the table as it handles it. An announcement gives
+// the peer a route to the prefix. A withdrawal removes the peer's route, as
+// does an announcement whose AS path is empty or holds AS 0 anywhere. A peer
+// whose BGP session goes down leaves every prefix.
+//
+// Each flush writes the table's changes to the database. It also stores each
+// prefix the updates name with the time it was first seen.
type PrefixHandler struct {
db database.Store
logger *logger.Logger
metrics *metrics.Tracker
+ table *routetable.Table
- // Batching. mu guards batch, held, peersDown, writing and deleting only:
- // the flush goroutine takes them under mu and writes them after releasing
+ // Batching. mu guards batch, writing and writingTableChanges only: the
+ // flush goroutine takes them under mu and writes them after releasing
// mu, so HandleMessage waits for a database write only while batch is at
// prefixBatchMaxSize.
mu sync.Mutex
- batch []prefixUpdate
- // held holds the updates that the peers whose route delete failed in the
- // last flush sent after their session went down, for the next flush to
- // write after it retries the delete. They are not in batch, so they do
- // not count toward a full batch: a delete that keeps failing is retried
- // at the pace of the flush timer.
- held []prefixUpdate
- // peersDown holds, each once, the peers whose BGP session went down
- // since the batch was taken, and those whose route delete failed in the
- // last flush.
- peersDown []string
+ batch []seenPrefix
// writing is how many updates the flush goroutine took and has not
// finished writing.
writing int
- // deleting is how many peers from peersDown the flush goroutine took and
- // has not yet deleted the routes of.
- deleting int
+ // writingTableChanges is how many changes the flush goroutine took from
+ // the routing table and has not finished writing.
+ writingTableChanges int
// batchTaken is signalled when the flush goroutine takes batch; its L is
// &mu.
batchTaken sync.Cond
batchFullCh chan struct{} // wakes the flush goroutine when batch is full
- lastFlush time.Time // used only by the flush goroutine
// stopCh hands Stop's context to the flush goroutine. It holds one, so
// Stop does not wait for a write in progress to hand it over.
stopCh chan context.Context
done chan struct{} // closed when the flush goroutine returns
}
+// seenPrefix is what the batch keeps of an update: the prefix it names and its
+// time. A flush needs only these to store each new prefix's first-seen time.
+type seenPrefix struct {
+ prefix string
+ timestamp time.Time
+}
+
+// prefixUpdate is one prefix of an update message, as applyToTable applies it
+// to the routing table.
type prefixUpdate struct {
prefix string
originASN int
@@ -97,21 +103,45 @@ type prefixUpdate struct {
endsInSet bool
}
-// NewPrefixHandler creates a new batched prefix tracking handler
-func NewPrefixHandler(db database.Store, logger *logger.Logger) *PrefixHandler {
+// announcesRoute says whether u gives the peer a route to the prefix: an
+// announcement whose path gives an origin, or ends in an AS set of several
+// members, which gives origin 0 (none). Any other update removes the peer's
+// route for the prefix: a withdrawal, or an announcement whose path is empty
+// or holds AS 0 anywhere (the decoder empties such a path). Such an
+// announcement replaces the peer's route, as any announcement does, with none
+// (RFC 7607 treats AS 0 in a path as a withdrawal).
+func (u prefixUpdate) announcesRoute() bool {
+ return u.messageType == "announcement" && (u.originASN > 0 || u.endsInSet)
+}
+
+// NewPrefixHandler creates a new batched prefix tracking handler, which applies
+// the updates to table, the routing table read from db at startup.
+func NewPrefixHandler(
+ db database.Store, logger *logger.Logger, table *routetable.Table,
+) *PrefixHandler {
+ return newPrefixHandler(db, logger, table, prefixBatchTimeout)
+}
+
+// newPrefixHandler is NewPrefixHandler with the flush timer's interval,
+// batchTimeout, in place of prefixBatchTimeout. A test that the timer must not
+// disturb gives an interval longer than the test runs.
+func newPrefixHandler(
+ db database.Store, logger *logger.Logger, table *routetable.Table,
+ batchTimeout time.Duration,
+) *PrefixHandler {
h := &PrefixHandler{
db: db,
logger: logger,
- batch: make([]prefixUpdate, 0, prefixBatchSize),
+ table: table,
+ batch: make([]seenPrefix, 0, prefixBatchSize),
batchFullCh: make(chan struct{}, 1),
- lastFlush: time.Now(),
stopCh: make(chan context.Context, 1),
done: make(chan struct{}),
}
h.batchTaken.L = &h.mu
// Start the flush timer goroutine
- go h.flushLoop()
+ go flushLoop(batchTimeout, h.batchFullCh, h.stopCh, h.done, h.flushBatch)
return h
}
@@ -140,7 +170,7 @@ func (h *PrefixHandler) QueueCapacity() int {
func (h *PrefixHandler) HandleMessage(msg *ristypes.RISMessage) {
if msg.Type == ristypes.MessageTypeState {
if msg.State == ristypes.StateDown {
- h.peerSessionDown(msg.Peer)
+ h.table.PeerDown(msg.Peer)
}
return
@@ -165,18 +195,21 @@ func (h *PrefixHandler) HandleMessage(msg *ristypes.RISMessage) {
h.batchTaken.Wait()
}
- // Withdrawals are queued before announcements, because the batch keeps
- // each peer's last update for a prefix: a message that lists a prefix as
- // both withdrawn and announced keeps the route (RFC 4271 section 4.3).
- // A withdrawal names only the prefix and peer: the message's AS path, and
- // so its origin AS, belongs to the announced prefixes.
+ // Withdrawals are applied to the routing table before announcements, so
+ // a message that lists a prefix as both withdrawn and announced keeps the
+ // route (RFC 4271 section 4.3). A withdrawal names only the prefix and
+ // peer: the message's AS path, and so its origin AS, belongs to the
+ // announced prefixes.
for _, prefix := range msg.Withdrawals {
- h.batch = append(h.batch, prefixUpdate{
+ update := prefixUpdate{
prefix: canonicalPrefix(prefix),
peer: msg.Peer,
messageType: "withdrawal",
timestamp: timestamp,
- })
+ }
+ h.batch = append(h.batch, seenPrefix{update.prefix, update.timestamp})
+ h.applyToTable(update)
+
// Record withdrawal in metrics
if h.metrics != nil {
h.metrics.RecordWithdrawal()
@@ -185,7 +218,7 @@ func (h *PrefixHandler) HandleMessage(msg *ristypes.RISMessage) {
for _, announcement := range msg.Announcements {
for _, prefix := range announcement.Prefixes {
- h.batch = append(h.batch, prefixUpdate{
+ update := prefixUpdate{
prefix: canonicalPrefix(prefix),
originASN: originASN,
peer: msg.Peer,
@@ -193,7 +226,10 @@ func (h *PrefixHandler) HandleMessage(msg *ristypes.RISMessage) {
timestamp: timestamp,
path: path,
endsInSet: len(msg.Path.Set) > 0,
- })
+ }
+ h.batch = append(h.batch, seenPrefix{update.prefix, update.timestamp})
+ h.applyToTable(update)
+
// Record announcement in metrics
if h.metrics != nil {
h.metrics.RecordAnnouncement()
@@ -212,12 +248,13 @@ func (h *PrefixHandler) HandleMessage(msg *ristypes.RISMessage) {
}
}
-// Stop stops the flush goroutine, which writes the updates and peer downs the
-// handler holds before it returns, and waits for it until ctx ends. Once ctx
-// has ended the flush goroutine skips that last write, and a write still
-// running is lost when the process exits. A peer whose route delete fails in
-// the last write is not retried either. Stop logs how many updates are not
-// written and how many peer downs have not had their routes deleted.
+// Stop stops the flush goroutine, which makes one last flush before it returns,
+// and waits for it until ctx ends. That flush writes at most
+// maxTableChangesPerWrite of the routing table's changes, and the prefixes of
+// the updates the handler holds. Once ctx has ended the flush goroutine skips
+// that last flush, and a write still running is lost when the process exits. A
+// failed write of the table's changes is not retried either. Stop logs how many
+// updates and how many of the table's changes are not written.
func (h *PrefixHandler) Stop(ctx context.Context) {
h.stopCh <- ctx
@@ -227,341 +264,156 @@ func (h *PrefixHandler) Stop(ctx context.Context) {
}
h.mu.Lock()
- unwritten := h.writing + len(h.batch) + len(h.held)
- peersDown := h.deleting + len(h.peersDown)
+ unwritten := h.writing + len(h.batch)
+ tableChanges := h.writingTableChanges + h.table.ChangeCount()
h.mu.Unlock()
- if unwritten == 0 && peersDown == 0 {
+ if unwritten == 0 && tableChanges == 0 {
return
}
if ctx.Err() != nil {
h.logger.Error("Stop deadline passed before the prefix handler wrote "+
- "its updates", "updates", unwritten, "peers_down", peersDown)
+ "its updates", "updates", unwritten, "table_changes", tableChanges)
return
}
- h.logger.Error("The prefix handler stopped without deleting the routes "+
- "of peers whose session went down", "updates", unwritten,
- "peers_down", peersDown)
+ h.logger.Error("The prefix handler stopped before it wrote everything",
+ "updates", unwritten, "table_changes", tableChanges)
}
-// peerSessionDown notes that the peer's BGP session went down, so that the next
-// flush deletes all of the peer's live routes before it writes the batch. The
-// peer's updates still in the batch or held came before the down, which
-// replaces them, so they are dropped; updates the peer sends later stay in the
-// batch and are written after the delete. A peer already in peersDown is not
-// added again: one delete removes all of its routes.
-func (h *PrefixHandler) peerSessionDown(peer string) {
- h.mu.Lock()
- defer h.mu.Unlock()
-
- isPeers := func(update prefixUpdate) bool { return update.peer == peer }
- h.batch = slices.DeleteFunc(h.batch, isPeers)
- h.held = slices.DeleteFunc(h.held, isPeers)
-
- if !slices.Contains(h.peersDown, peer) {
- h.peersDown = append(h.peersDown, peer)
- }
-}
-
-// flushLoop runs in a goroutine, the only one that writes the batch: it
-// flushes when the batch is full, periodically, and once more on Stop.
-func (h *PrefixHandler) flushLoop() {
- defer close(h.done)
-
- ticker := time.NewTicker(prefixBatchTimeout)
- defer ticker.Stop()
-
- for {
- select {
- case <-ticker.C:
- if time.Since(h.lastFlush) >= prefixBatchTimeout {
- h.flushBatch()
- }
- case <-h.batchFullCh:
- h.flushBatch()
- case ctx := <-h.stopCh:
- // Final flush, unless the stop deadline has passed
- if ctx.Err() == nil {
- h.flushBatch()
- }
-
- return
- }
- }
-}
-
-// takeBatch returns the pending updates, the held ones first, and the peers in
-// peersDown, and leaves an empty batch, no held updates and no peers in their
+// takeBatch returns the pending updates and leaves an empty batch in their
// place for HandleMessage to fill. The updates count in writing until
-// flushBatch has written them, and the peers in deleting until their routes
-// are deleted.
-func (h *PrefixHandler) takeBatch() ([]prefixUpdate, []string) {
+// flushBatch has written them.
+func (h *PrefixHandler) takeBatch() []seenPrefix {
h.mu.Lock()
defer h.mu.Unlock()
- if len(h.held) == 0 && len(h.batch) == 0 && len(h.peersDown) == 0 {
- return nil, nil
- }
-
- batch, peersDown := h.batch, h.peersDown
- if len(h.held) > 0 {
- batch = slices.Concat(h.held, h.batch)
+ if len(h.batch) == 0 {
+ return nil
}
+ batch := h.batch
h.writing = len(batch)
- h.deleting = len(peersDown)
- h.held = nil
- h.batch = make([]prefixUpdate, 0, prefixBatchSize)
- h.peersDown = nil
+ h.batch = make([]seenPrefix, 0, prefixBatchSize)
h.batchTaken.Broadcast()
- return batch, peersDown
+ return batch
}
-// flushBatch takes the pending batch and writes it to the database, without
-// holding the mutex during the write.
+// flushBatch writes the routing table's changes, then takes the pending batch
+// and stores the prefixes it names, without holding the mutex during the
+// writes.
func (h *PrefixHandler) flushBatch() {
- batch, peersDown := h.takeBatch()
- if len(batch) == 0 && len(peersDown) == 0 {
+ h.writeTableChanges()
+
+ batch := h.takeBatch()
+ if len(batch) == 0 {
return
}
startTime := time.Now()
- batchSize := len(batch)
-
- // The routes of a peer whose session went down are deleted first: the
- // batch holds only the updates the peer sent after the down. A peer whose
- // delete failed is kept, with those updates, for the next flush to retry.
- failedPeers := h.deleteLiveRoutesOfPeers(peersDown)
- if len(failedPeers) > 0 {
- batch = h.keepForNextFlush(batch, failedPeers)
- }
-
- // For each prefix and peer, keep only the last update the feed delivered.
- // Updates from different peers for the same prefix are all kept. Arrival
- // order decides, not the timestamp: timestamps are whole seconds, so
- // updates within one second carry the same one.
- type prefixAndPeer struct{ prefix, peer string }
-
- lastUpdates := make(map[prefixAndPeer]prefixUpdate)
// Each prefix the batch names, announced or withdrawn, with the earliest
- // timestamp among all of its updates, not only each peer's last one: the
- // prefix tables store only first_seen, written when the prefix is new and
- // never changed after.
+ // timestamp among its updates: the prefix tables store only first_seen,
+ // written when the prefix is new and never changed after.
prefixesToUpdate := make(map[string]time.Time)
for _, update := range batch {
- lastUpdates[prefixAndPeer{update.prefix, update.peer}] = update
-
seen, exists := prefixesToUpdate[update.prefix]
if !exists || update.timestamp.Before(seen) {
prefixesToUpdate[update.prefix] = update.timestamp
}
}
- // Collect routes to upsert and delete
- var (
- routesToUpsert []*database.LiveRoute
- routesToDelete []database.LiveRouteDeletion
- )
-
- for _, update := range lastUpdates {
- // An announcement is stored when its path gives an origin, or when it
- // ends in an AS set of several members: that route is stored with
- // origin 0 (none). Any other update deletes the peer's route for the
- // prefix: a withdrawal, or an announcement whose path is empty or
- // holds AS 0 anywhere (the decoder empties such a path). Such an
- // announcement replaces the peer's route, as any announcement does,
- // with none (RFC 7607 treats AS 0 in a path as a withdrawal).
- if update.messageType == "announcement" &&
- (update.originASN > 0 || update.endsInSet) {
- // Create live route for batch upsert
- route := h.createLiveRoute(update)
- if route != nil {
- routesToUpsert = append(routesToUpsert, route)
- }
- } else {
- // Parse CIDR to get IP version
- _, ipVersion, err := parseCIDR(update.prefix)
- if err != nil {
- h.logger.Error("Failed to parse CIDR for route deletion",
- "prefix", update.prefix, "error", err)
-
- continue
- }
-
- // Create deletion record for batch delete
- routesToDelete = append(routesToDelete, database.LiveRouteDeletion{
- Prefix: update.prefix,
- PeerIP: update.peer,
- IPVersion: ipVersion,
- })
- }
+ err := h.db.UpdatePrefixesBatch(prefixesToUpdate)
+ if err != nil {
+ h.logger.Error("Failed to update prefix batch",
+ "error", err, "count", len(prefixesToUpdate))
}
- successCount := h.writeBatch(routesToUpsert, routesToDelete, prefixesToUpdate)
-
h.mu.Lock()
h.writing = 0
h.mu.Unlock()
- elapsed := time.Since(startTime)
h.logger.Debug("Flushed prefix batch",
- "batch_size", batchSize,
- "peers_down", len(peersDown),
+ "batch_size", len(batch),
"unique_prefixes", len(prefixesToUpdate),
- "success", successCount,
- "duration_ms", elapsed.Milliseconds(),
+ "duration_ms", time.Since(startTime).Milliseconds(),
)
-
- h.lastFlush = time.Now()
}
-// deleteLiveRoutesOfPeers deletes all live routes of each of peers, the peers
-// whose BGP session went down, and returns those whose delete failed. Most
-// downs come from peers that have no routes, so only a delete that removed
-// routes is logged. A peer whose delete succeeded no longer counts in deleting;
-// a failed one counts until keepForNextFlush puts it back in peersDown.
-func (h *PrefixHandler) deleteLiveRoutesOfPeers(peers []string) []string {
- var failed []string
+// applyToTable applies update to the routing table in memory: as an
+// announcement when it gives the peer a route, otherwise as a withdrawal. A
+// prefix that does not parse is logged and left out. Each announcement counts
+// in the IPv4 or IPv6 route update rate.
+func (h *PrefixHandler) applyToTable(update prefixUpdate) {
+ prefix, err := netip.ParsePrefix(update.prefix)
+ if err != nil {
+ h.logger.Error("Failed to parse prefix",
+ "prefix", update.prefix, "error", err)
- for _, peer := range peers {
- deleted, err := h.db.DeleteLiveRoutesOfPeer(peer)
- if err != nil {
- h.logger.Error(
- "Failed to delete the live routes of a peer whose session "+
- "went down; the next flush retries",
- "peer", peer, "routes_deleted", deleted, "error", err)
-
- failed = append(failed, peer)
-
- continue
- }
-
- h.mu.Lock()
- h.deleting--
- h.mu.Unlock()
-
- if deleted > 0 {
- h.logger.Info(
- "Deleted the live routes of a peer whose session went down",
- "peer", peer, "routes", deleted)
- }
+ return
}
- return failed
-}
+ if !update.announcesRoute() {
+ h.table.Withdraw(prefix, update.peer)
-// keepForNextFlush puts the peers whose route delete failed back in peersDown,
-// and their updates from batch in held, so that the next flush retries each
-// delete before it writes the peer's updates; written now, they would be
-// deleted by the retry. The updates of a peer whose session went down again
-// since the batch was taken came before that down, so they are dropped, as
-// peerSessionDown drops them, and the peer is already in peersDown. It returns
-// the updates of the other peers, for this flush to write.
-//
-// While a delete keeps failing, the held updates would grow with every flush,
-// so only the last prefixBatchSize of them are kept and the older ones are
-// dropped. This never leaves in place a route the peer withdrew: a prefix with
-// a kept update keeps its last one, and the retry deletes all of the peer's
-// routes, so a prefix whose updates were all dropped has no route from the peer
-// until the peer announces it again.
-func (h *PrefixHandler) keepForNextFlush(
- batch []prefixUpdate, failedPeers []string,
-) []prefixUpdate {
- var kept, toWrite []prefixUpdate
+ return
+ }
- for _, update := range batch {
- if slices.Contains(failedPeers, update.peer) {
- kept = append(kept, update)
+ if h.metrics != nil {
+ if prefix.Addr().Is4() {
+ h.metrics.RecordIPv4Update()
} else {
- toWrite = append(toWrite, update)
+ h.metrics.RecordIPv6Update()
}
}
+ h.table.Announce(prefix, update.originASN, routetable.Route{
+ Peer: update.peer,
+ ASPath: update.path,
+ Time: update.timestamp,
+ })
+}
+
+// writeTableChanges writes at most maxTableChangesPerWrite changes of the
+// routing table in memory, and every peer the table numbered since the last
+// write.
+// When the write fails, they go back to the table, so that a later write writes
+// them again, each pair with its content then.
+func (h *PrefixHandler) writeTableChanges() {
+ h.mu.Lock()
+ changes, newPeers := h.table.TakeChanges(maxTableChangesPerWrite)
+ h.writingTableChanges = len(changes)
+ h.mu.Unlock()
+
+ if len(changes) == 0 && len(newPeers) == 0 {
+ return
+ }
+
+ err := h.db.WriteRouteChanges(changes, newPeers)
+
h.mu.Lock()
defer h.mu.Unlock()
- kept = slices.DeleteFunc(kept, func(update prefixUpdate) bool {
- return slices.Contains(h.peersDown, update.peer)
- })
+ h.writingTableChanges = 0
- if len(kept) > prefixBatchSize {
- h.logger.Error(
- "Dropped the oldest updates of peers whose route delete failed",
- "peers", failedPeers, "dropped", len(kept)-prefixBatchSize)
+ if err != nil {
+ h.logger.Error("Failed to write the routing table's changes; the next "+
+ "flush retries", "changes", len(changes), "new_peers", len(newPeers),
+ "error", err)
- // A copy, so that the dropped updates are freed.
- kept = slices.Clone(kept[len(kept)-prefixBatchSize:])
+ h.table.PutBackChanges(changes, newPeers)
}
-
- // takeBatch left held empty, and only this flush fills it. The kept
- // updates now count in held, not in writing, and the failed peers in
- // peersDown, not in deleting.
- h.held = kept
- h.writing = len(toWrite)
- h.deleting = 0
-
- for _, peer := range failedPeers {
- if !slices.Contains(h.peersDown, peer) {
- h.peersDown = append(h.peersDown, peer)
- }
- }
-
- return toWrite
-}
-
-// writeBatch writes the collected route upserts, route deletions and prefix
-// updates to the database and returns how many routes it wrote.
-func (h *PrefixHandler) writeBatch(
- routesToUpsert []*database.LiveRoute,
- routesToDelete []database.LiveRouteDeletion,
- prefixesToUpdate map[string]time.Time,
-) int {
- successCount := 0
-
- if len(routesToUpsert) > 0 {
- err := h.db.UpsertLiveRouteBatch(routesToUpsert)
- if err != nil {
- h.logger.Error("Failed to upsert route batch",
- "error", err, "count", len(routesToUpsert))
- } else {
- successCount += len(routesToUpsert)
- }
- }
-
- if len(routesToDelete) > 0 {
- err := h.db.DeleteLiveRouteBatch(routesToDelete)
- if err != nil {
- h.logger.Error("Failed to delete route batch",
- "error", err, "count", len(routesToDelete))
- } else {
- successCount += len(routesToDelete)
- }
- }
-
- // Update prefix tables
- if len(prefixesToUpdate) > 0 {
- err := h.db.UpdatePrefixesBatch(prefixesToUpdate)
- if err != nil {
- h.logger.Error("Failed to update prefix batch",
- "error", err, "count", len(prefixesToUpdate))
- }
- }
-
- return successCount
}
// canonicalPrefix returns prefix in the text form net/netip prints, the form
-// the IP lookup builds when it looks a prefix up. The feed sends IPv6
+// the prefix tables store and the IP lookup reads them in. The feed sends IPv6
// announcements compressed ("2001:db8::/32") but IPv6 withdrawals uncompressed
-// ("2001:db8:0:0:0:0:0:0/32"); stored as received, a withdrawal would not match
-// the route its announcement stored. A prefix that does not parse is returned
-// unchanged, and the batch flush reports it.
+// ("2001:db8:0:0:0:0:0:0/32"). A prefix that does not parse is returned
+// unchanged, and applyToTable logs it.
func canonicalPrefix(prefix string) string {
p, err := netip.ParsePrefix(prefix)
if err != nil {
@@ -570,55 +422,3 @@ func canonicalPrefix(prefix string) string {
return p.Masked().String()
}
-
-// parseCIDR returns the mask length and the IP version of a prefix string
-func parseCIDR(prefix string) (int, int, error) {
- _, ipNet, err := net.ParseCIDR(prefix)
- if err != nil {
- return 0, 0, err
- }
-
- ones, _ := ipNet.Mask.Size()
- if strings.Contains(prefix, ":") {
- return ones, ipv6Version, nil
- }
-
- return ones, ipv4Version, nil
-}
-
-// createLiveRoute creates a LiveRoute from a prefix update
-func (h *PrefixHandler) createLiveRoute(update prefixUpdate) *database.LiveRoute {
- // Parse CIDR to get mask length
- maskLength, ipVersion, err := parseCIDR(update.prefix)
- if err != nil {
- h.logger.Error("Failed to parse CIDR",
- "prefix", update.prefix,
- "error", err,
- )
-
- return nil
- }
-
- // Track route update metrics
- if h.metrics != nil {
- if ipVersion == ipv4Version {
- h.metrics.RecordIPv4Update()
- } else {
- h.metrics.RecordIPv6Update()
- }
- }
-
- // Create live route record
- liveRoute := &database.LiveRoute{
- Prefix: update.prefix,
- MaskLength: maskLength,
- IPVersion: ipVersion,
- OriginASN: update.originASN,
- PeerIP: update.peer,
- ASPath: update.path,
- NextHop: update.peer, // Using peer as next hop
- LastUpdated: update.timestamp,
- }
-
- return liveRoute
-}
diff --git a/internal/routewatch/prefixhandler_internal_test.go b/internal/routewatch/prefixhandler_internal_test.go
index f047a65..c7401d6 100644
--- a/internal/routewatch/prefixhandler_internal_test.go
+++ b/internal/routewatch/prefixhandler_internal_test.go
@@ -3,12 +3,7 @@ package routewatch
import (
"bytes"
"context"
- "errors"
- "fmt"
"net/netip"
- "slices"
- "strings"
- "sync"
"testing"
"time"
@@ -16,6 +11,7 @@ import (
"git.eeqj.de/sneak/routewatch/internal/database"
"git.eeqj.de/sneak/routewatch/internal/logger"
"git.eeqj.de/sneak/routewatch/internal/ristypes"
+ "git.eeqj.de/sneak/routewatch/internal/routetable"
)
const testPeerIP = "2001:db8:ffff::1"
@@ -31,18 +27,20 @@ const (
const secondUpdateAfter = 5 * time.Second
// TestPrefixHandlerStoresPrefixesTheIPLookupFinds runs announcements and
-// withdrawals through the prefix handler into a real database and looks the
-// addresses up. The prefixes are written the way the feed sends them: IPv6
-// announcements compressed, IPv6 withdrawals uncompressed. The withdrawal must
-// remove the route the announcement stored, and the IP lookup must find the
-// stored prefix.
+// withdrawals through the prefix handler into a real database. The prefixes
+// are written the way the feed sends them: IPv6 announcements compressed, IPv6
+// withdrawals uncompressed. The IP lookup must find each announced prefix in
+// the routing table, and its first-seen time in the prefix tables. The
+// withdrawal must remove the peer's route and store no second form of the
+// prefix.
func TestPrefixHandlerStoresPrefixesTheIPLookupFinds(t *testing.T) {
t.Parallel()
- db := newTestDatabase(t)
+ // The two prefixes the announcement names.
+ const wantPrefixes = 2
- // Built without NewPrefixHandler's flush timer, so the test flushes itself.
- h := &PrefixHandler{db: db, logger: logger.New()}
+ db := newTestDatabase(t)
+ h := prefixHandlerWithoutTimer(db)
ts := time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC)
h.HandleMessage(&ristypes.RISMessage{
@@ -60,14 +58,18 @@ func TestPrefixHandlerStoresPrefixesTheIPLookupFinds(t *testing.T) {
"2001:db8:1::1": testV6Prefix,
"192.0.2.1": testV4Prefix,
} {
- info, err := db.GetIPInfo(ip)
- if err != nil {
- t.Fatalf("GetIPInfo(%s) after announcement: %v", ip, err)
+ summary, found := h.table.Lookup(netip.MustParseAddr(ip))
+ if !found || summary.Prefix.String() != want ||
+ summary.OriginASN != testASNB {
+ t.Errorf("Lookup(%s) after announcement = %+v, %t; want %s AS%d",
+ ip, summary, found, want, testASNB)
}
- if info.Netblock != want || info.ASN != testASNB {
- t.Errorf("GetIPInfo(%s) = %s AS%d, want %s AS%d",
- ip, info.Netblock, info.ASN, want, testASNB)
+ firstSeen, err := db.GetPrefixFirstSeen(t.Context(),
+ netip.MustParsePrefix(want))
+ if err != nil || !firstSeen.Equal(ts) {
+ t.Errorf("GetPrefixFirstSeen(%s) = %v, %v; want %v",
+ want, firstSeen, err, ts)
}
}
@@ -79,20 +81,28 @@ func TestPrefixHandlerStoresPrefixesTheIPLookupFinds(t *testing.T) {
h.flushBatch()
for _, ip := range []string{"2001:db8:1::1", "192.0.2.1"} {
- info, err := db.GetIPInfo(ip)
- if !errors.Is(err, database.ErrNoRoute) {
- t.Errorf("GetIPInfo(%s) after withdrawal = %+v, %v; want ErrNoRoute",
- ip, info, err)
+ summary, found := h.table.Lookup(netip.MustParseAddr(ip))
+ if found {
+ t.Errorf("Lookup(%s) after withdrawal = %+v, want none", ip, summary)
}
}
+
+ stats, err := db.GetStatsContext(t.Context())
+ if err != nil {
+ t.Fatalf("GetStatsContext: %v", err)
+ }
+
+ if stats.Prefixes != wantPrefixes {
+ t.Errorf("the prefix tables hold %d prefixes after the withdrawal, "+
+ "want %d", stats.Prefixes, wantPrefixes)
+ }
}
// TestPrefixHandlerStoresANewPrefixFirstSeenAtItsEarliestUpdate announces a
// prefix the database does not hold yet twice in one batch, the second time 5
// seconds after the first, through the prefix handler into a real database. The
// prefix is stored first seen at the time of the first announcement, whether
-// the two come from two peers or from one, and the IP lookup answers with that
-// time.
+// the two come from two peers or from one.
func TestPrefixHandlerStoresANewPrefixFirstSeenAtItsEarliestUpdate(
t *testing.T,
) {
@@ -117,336 +127,20 @@ func TestPrefixHandlerStoresANewPrefixFirstSeenAtItsEarliestUpdate(
second.ParsedTimestamp = first.ParsedTimestamp.Add(secondUpdateAfter)
db := handleBatches(t, [][]*ristypes.RISMessage{{first, second}})
- info, err := db.GetIPInfo("192.0.2.1")
+ firstSeen, err := db.GetPrefixFirstSeen(t.Context(),
+ netip.MustParsePrefix(testV4Prefix))
if err != nil {
- t.Fatalf("GetIPInfo(192.0.2.1): %v", err)
+ t.Fatalf("GetPrefixFirstSeen(%s): %v", testV4Prefix, err)
}
- if !info.FirstSeen.Equal(first.ParsedTimestamp) {
+ if !firstSeen.Equal(first.ParsedTimestamp) {
t.Errorf("first_seen of %s = %s, want %s", testV4Prefix,
- info.FirstSeen, first.ParsedTimestamp)
+ firstSeen, first.ParsedTimestamp)
}
})
}
}
-// TestPrefixHandlerAppliesEachPeersLastUpdateInABatch runs updates for one
-// prefix through the prefix handler into a real database. All updates carry the
-// same timestamp, as updates within one second of the feed do. Within a batch,
-// the last update each peer sent for the prefix decides whether that peer has a
-// live route for it, and updates from different peers are all applied. The
-// in-memory route count and prefix distribution must match the route table.
-func TestPrefixHandlerAppliesEachPeersLastUpdateInABatch(t *testing.T) {
- t.Parallel()
-
- const (
- prefix = "198.51.100.0/24"
- prefixMaskLength = 24
- otherPeerIP = "2001:db8:ffff::2"
- )
-
- ts := time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC)
- announce := func(peer string) *ristypes.RISMessage {
- return &ristypes.RISMessage{
- Peer: peer,
- Path: ristypes.ASPath{Sequence: []int{testASNA, testASNB}},
- ParsedTimestamp: ts,
- Announcements: []ristypes.RISAnnouncement{
- {NextHop: peer, Prefixes: []string{prefix}},
- },
- }
- }
- withdraw := func(peer string) *ristypes.RISMessage {
- return &ristypes.RISMessage{Peer: peer, ParsedTimestamp: ts,
- Withdrawals: []string{prefix}}
- }
- announceAndWithdraw := func(peer string) *ristypes.RISMessage {
- msg := announce(peer)
- msg.Withdrawals = []string{prefix}
-
- return msg
- }
-
- tests := []struct {
- name string
- // Each inner list is handled and then flushed as one batch.
- batches [][]*ristypes.RISMessage
- // The peers left with a live route for the prefix, by peer IP.
- wantPeers []string
- }{
- {
- name: "two peers announce the prefix",
- batches: [][]*ristypes.RISMessage{
- {announce(testPeerIP), announce(otherPeerIP)},
- },
- wantPeers: []string{testPeerIP, otherPeerIP},
- },
- {
- name: "two peers withdraw the prefix",
- batches: [][]*ristypes.RISMessage{
- {announce(testPeerIP)},
- {announce(otherPeerIP)},
- {withdraw(testPeerIP), withdraw(otherPeerIP)},
- },
- },
- {
- name: "a peer announces and then withdraws the prefix",
- batches: [][]*ristypes.RISMessage{{announce(testPeerIP), withdraw(testPeerIP)}},
- },
- {
- name: "a peer withdraws and then re-announces the prefix",
- batches: [][]*ristypes.RISMessage{
- {announce(testPeerIP)},
- {withdraw(testPeerIP), announce(testPeerIP)},
- },
- wantPeers: []string{testPeerIP},
- },
- {
- // RFC 4271 section 4.3: the announcement counts, not the withdrawal.
- name: "one message announces and withdraws the prefix",
- batches: [][]*ristypes.RISMessage{
- {announce(testPeerIP)},
- {announceAndWithdraw(testPeerIP)},
- },
- wantPeers: []string{testPeerIP},
- },
- }
-
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- t.Parallel()
-
- db := handleBatches(t, tt.batches)
- checkLivePeers(t, db, prefix, prefixMaskLength, tt.wantPeers)
- })
- }
-}
-
-// TestPrefixHandlerKeepsOneRoutePerPrefixAndPeer runs updates from one peer
-// through the prefix handler into a real database, flushing each message as its
-// own batch, once with IPv4 prefixes and once with IPv6 prefixes. A peer has at
-// most one route for a prefix: a later announcement replaces it whatever its
-// origin AS, and a withdrawal removes it whatever AS the message's AS path ends
-// in. The in-memory route count and prefix distribution must match the route
-// table.
-func TestPrefixHandlerKeepsOneRoutePerPrefixAndPeer(t *testing.T) {
- t.Parallel()
-
- families := []struct {
- ipVersion int
- prefix, otherPrefix string
- // The mask length of both prefixes.
- maskLength int
- }{
- {
- ipVersion: ipv4Version, maskLength: 24,
- prefix: "198.51.100.0/24", otherPrefix: "203.0.113.0/24",
- },
- {
- ipVersion: ipv6Version, maskLength: 48,
- prefix: testV6Prefix, otherPrefix: "2001:db8:2::/48",
- },
- }
-
- ts := time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC)
- announce := func(path ristypes.ASPath, announced string) *ristypes.RISMessage {
- return &ristypes.RISMessage{
- Peer: testPeerIP,
- Path: path,
- ParsedTimestamp: ts,
- Announcements: []ristypes.RISAnnouncement{
- {NextHop: testPeerIP, Prefixes: []string{announced}},
- },
- }
- }
-
- for _, family := range families {
- prefix, otherPrefix := family.prefix, family.otherPrefix
-
- // One message that announces otherPrefix with origin AS testASNB and
- // withdraws prefix.
- announceOtherAndWithdraw := announce(
- ristypes.ASPath{Sequence: []int{testASNA, testASNB}}, otherPrefix)
- announceOtherAndWithdraw.Withdrawals = []string{prefix}
-
- tests := []struct {
- name string
- // Each message is handled and then flushed as its own batch.
- batches [][]*ristypes.RISMessage
- // The origin AS of the peer's live route for each prefix. A prefix
- // left out must have no live route.
- wantOrigins map[string]int
- }{
- {
- name: "the peer announces the prefix with a new origin AS",
- batches: [][]*ristypes.RISMessage{
- {announce(ristypes.ASPath{Sequence: []int{testASNA}}, prefix)},
- {announce(ristypes.ASPath{Sequence: []int{testASNA, testASNB}}, prefix)},
- },
- wantOrigins: map[string]int{prefix: testASNB},
- },
- {
- name: "one message announces a prefix with another origin AS " +
- "and withdraws the prefix",
- batches: [][]*ristypes.RISMessage{
- {announce(ristypes.ASPath{Sequence: []int{testASNA}}, prefix)},
- {announceOtherAndWithdraw},
- },
- wantOrigins: map[string]int{otherPrefix: testASNB},
- },
- }
-
- for _, tt := range tests {
- name := fmt.Sprintf("IPv%d %s", family.ipVersion, tt.name)
- t.Run(name, func(t *testing.T) {
- t.Parallel()
-
- db := handleBatches(t, tt.batches)
- checkLiveOrigins(t, db, tt.wantOrigins, prefix, otherPrefix)
-
- wantDistribution := []database.PrefixDistribution{
- {MaskLength: family.maskLength, Count: len(tt.wantOrigins)},
- }
- assertCounts(t, db, family.ipVersion, len(tt.wantOrigins),
- wantDistribution)
- })
- }
- }
-}
-
-// TestPrefixHandlerStoresNoOriginForAPathEndingInASet announces a prefix with
-// AS paths that end in an AS set through the prefix handler into a real
-// database. Any member of a set of several may be the origin, so the route is
-// stored with origin 0 (none); a set of one member counts as that AS. The
-// stored AS path lists the set's members after the rest of the path.
-func TestPrefixHandlerStoresNoOriginForAPathEndingInASet(t *testing.T) {
- t.Parallel()
-
- tests := []struct {
- name string
- path []any
- wantOrigin int
- wantPath []int
- }{
- {
- name: "a set of several members",
- path: []any{testASNA, testASNB, []int{testASNC, testASND}},
- wantPath: []int{testASNA, testASNB, testASNC, testASND},
- },
- {
- name: "a set of one member",
- path: []any{testASNA, testASNB, []int{testASNC}},
- wantOrigin: testASNC,
- wantPath: []int{testASNA, testASNB, testASNC},
- },
- }
-
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- t.Parallel()
-
- msg := peerAnnouncement(testPeerIP, testV4Prefix)
- msg.Path = decodePath(t, tt.path...)
- db := handleBatches(t, [][]*ristypes.RISMessage{{msg}})
-
- routes, _, err := db.GetPrefixDetailsContext(
- context.Background(), testV4Prefix)
- if err != nil {
- t.Fatalf("GetPrefixDetailsContext(%s): %v", testV4Prefix, err)
- }
-
- if len(routes) != 1 || routes[0].OriginASN != tt.wantOrigin ||
- !slices.Equal(routes[0].ASPath, tt.wantPath) {
- t.Errorf("live routes for %s = %+v, "+
- "want one with origin AS%d and AS path %v",
- testV4Prefix, routes, tt.wantOrigin, tt.wantPath)
- }
- })
- }
-}
-
-// TestPrefixHandlerDoesNotStoreAnAnnouncementWithAnEmptyPath announces a
-// prefix with an empty AS path, which gives no origin and does not end in an
-// AS set, so no live route is stored.
-func TestPrefixHandlerDoesNotStoreAnAnnouncementWithAnEmptyPath(t *testing.T) {
- t.Parallel()
-
- msg := peerAnnouncement(testPeerIP, testV4Prefix)
- msg.Path = ristypes.ASPath{}
- db := handleBatches(t, [][]*ristypes.RISMessage{{msg}})
-
- assertPeers(t, db, map[string][]string{testV4Prefix: nil})
-}
-
-// TestPrefixHandlerRemovesTheRouteOnAnAnnouncementWithoutAnOrigin runs, through
-// the prefix handler into a real database, a peer's announcement of a prefix
-// with origin AS testASNB and then the peer's announcement of the prefix with
-// an AS path that gives no origin: one whose last AS is 0, an empty one, or one
-// that holds AS 0 before its end, which decodes as an empty path. As any later
-// announcement from the peer does, the second replaces the peer's route, here
-// with none, whether it comes in a later batch or follows a withdrawal in one
-// batch. The in-memory route count and prefix distribution must match the
-// route table.
-func TestPrefixHandlerRemovesTheRouteOnAnAnnouncementWithoutAnOrigin(
- t *testing.T,
-) {
- t.Parallel()
-
- const prefixMaskLength = 24
-
- noOrigin := func(path ristypes.ASPath) *ristypes.RISMessage {
- msg := peerAnnouncement(testPeerIP, testV4Prefix)
- msg.Path = path
-
- return msg
- }
- endsInZero := ristypes.ASPath{Sequence: []int{testASNA, 0}}
- withdrawal := &ristypes.RISMessage{
- Type: ristypes.MessageTypeUpdate,
- Peer: testPeerIP,
- ParsedTimestamp: time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC),
- Withdrawals: []string{testV4Prefix},
- }
-
- tests := []struct {
- name string
- // Handled and flushed as one batch after the announcement with origin
- // AS testASNB.
- batch []*ristypes.RISMessage
- }{
- {
- name: "a path whose last AS is 0",
- batch: []*ristypes.RISMessage{noOrigin(endsInZero)},
- },
- {
- name: "an empty path",
- batch: []*ristypes.RISMessage{noOrigin(ristypes.ASPath{})},
- },
- {
- name: "a path with AS 0 before its end",
- batch: []*ristypes.RISMessage{
- noOrigin(decodePath(t, testASNA, 0, testASNB)),
- },
- },
- {
- name: "a withdrawal and then a path whose last AS is 0",
- batch: []*ristypes.RISMessage{withdrawal, noOrigin(endsInZero)},
- },
- }
-
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- t.Parallel()
-
- db := handleBatches(t, [][]*ristypes.RISMessage{
- {peerAnnouncement(testPeerIP, testV4Prefix)},
- tt.batch,
- })
- checkLivePeers(t, db, testV4Prefix, prefixMaskLength, nil)
- })
- }
-}
-
// TestPrefixHandlerDoesNotStoreAPrefixThatDoesNotParse runs an announcement of
// "not-a-prefix" and a withdrawal of "also/bad" through the prefix handler into
// a real database. Neither is stored in the prefix tables: the prefix count in
@@ -462,8 +156,7 @@ func TestPrefixHandlerDoesNotStoreAPrefixThatDoesNotParse(t *testing.T) {
t.Fatalf("failed to create database: %v", err)
}
- // Built without NewPrefixHandler's flush timer, so the test flushes itself.
- h := &PrefixHandler{db: db, logger: logger.New()}
+ h := prefixHandlerWithoutTimer(db)
withdrawal := &ristypes.RISMessage{
Type: ristypes.MessageTypeUpdate,
@@ -524,468 +217,39 @@ func TestStateMessagesGoToThePrefixHandlerNotThePeerHandler(t *testing.T) {
}
}
-// TestPrefixHandlerDeletesAPeersRoutesWhenItsSessionGoesDown runs
-// announcements from two peers through the prefix handler into a real
-// database, then a batch in which one of the peers announces a prefix and its
-// session goes down. The down deletes every live route of that peer, IPv4 and
-// IPv6, the one announced just before it included, and keeps the other peer's
-// routes for the same prefixes. An announcement the peer sends after the down,
-// in the same batch, is kept. The in-memory route counts and prefix
-// distribution must match the route tables.
-func TestPrefixHandlerDeletesAPeersRoutesWhenItsSessionGoesDown(t *testing.T) {
- t.Parallel()
-
- const (
- otherPeerIP = "2001:db8:ffff::2"
- // Both peers announce the shared prefixes, only testPeerIP the others.
- sharedV4 = "198.51.100.0/24"
- sharedV6 = "2001:db8:1::/48"
- onlyV4 = "203.0.113.0/24"
- onlyV6 = "2001:db8:2::/48"
- beforeDown = "192.0.2.0/24"
- afterDown = "2001:db8:3::/48"
- // The mask lengths of the IPv4 and of the IPv6 prefixes.
- maskLengthV4 = 24
- maskLengthV6 = 48
- )
-
- tests := []struct {
- name string
- // Handled and flushed as one batch after both peers' announcements.
- batch []*ristypes.RISMessage
- // Whether testPeerIP is left with a live route for afterDown.
- wantAfterDown bool
- }{
- {
- name: "the down ends the batch",
- batch: []*ristypes.RISMessage{
- peerAnnouncement(testPeerIP, beforeDown),
- sessionDown(),
- },
- },
- {
- name: "the peer announces a prefix after the down",
- batch: []*ristypes.RISMessage{
- peerAnnouncement(testPeerIP, beforeDown),
- sessionDown(),
- peerAnnouncement(testPeerIP, afterDown),
- },
- wantAfterDown: true,
- },
- }
-
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- t.Parallel()
-
- db := handleBatches(t, [][]*ristypes.RISMessage{
- {
- peerAnnouncement(testPeerIP,
- sharedV4, sharedV6, onlyV4, onlyV6),
- peerAnnouncement(otherPeerIP, sharedV4, sharedV6),
- },
- tt.batch,
- })
-
- // The peers left with a live route for each prefix.
- wantPeers := map[string][]string{
- sharedV4: {otherPeerIP},
- sharedV6: {otherPeerIP},
- onlyV4: nil,
- onlyV6: nil,
- beforeDown: nil,
- afterDown: nil,
- }
- wantRoutesV6 := 1
-
- if tt.wantAfterDown {
- wantPeers[afterDown] = []string{testPeerIP}
- wantRoutesV6++
- }
-
- assertPeers(t, db, wantPeers)
- assertCounts(t, db, ipv4Version, 1, []database.PrefixDistribution{
- {MaskLength: maskLengthV4, Count: 1},
- })
- assertCounts(t, db, ipv6Version, wantRoutesV6,
- []database.PrefixDistribution{
- {MaskLength: maskLengthV6, Count: wantRoutesV6},
- })
- })
- }
-}
-
-// TestPrefixHandlerKeepsWhatAPeerAnnouncesAfterTheFlushOfItsDown runs, through
-// the prefix handler into a real database, one flush after another: a peer's
-// announcement, its session going down, a new announcement from the peer, and
-// another peer's announcement. The down deletes the peer's routes on the flush
-// that handles it and on no later flush, so the route the peer announced after
-// it is kept.
-func TestPrefixHandlerKeepsWhatAPeerAnnouncesAfterTheFlushOfItsDown(t *testing.T) {
- t.Parallel()
-
- const (
- otherPeerIP = "2001:db8:ffff::2"
- beforeDown = "192.0.2.0/24"
- afterDown = "198.51.100.0/24"
- otherPeers = "203.0.113.0/24"
- maskLength = 24
- )
-
- // Each message is handled and then flushed as its own batch.
- db := handleBatches(t, [][]*ristypes.RISMessage{
- {peerAnnouncement(testPeerIP, beforeDown)},
- {sessionDown()},
- {peerAnnouncement(testPeerIP, afterDown)},
- {peerAnnouncement(otherPeerIP, otherPeers)},
- })
-
- assertPeers(t, db, map[string][]string{
- beforeDown: nil,
- afterDown: {testPeerIP},
- otherPeers: {otherPeerIP},
- })
- assertCounts(t, db, ipv4Version, 2, []database.PrefixDistribution{
- {MaskLength: maskLength, Count: 2},
- })
-}
-
-// firstWriteHeldStore is a real database whose first live route batch write
-// waits until the test closes release. writeStarted is closed when it starts.
-type firstWriteHeldStore struct {
- *database.Database
-
- writeStarted chan struct{}
- release chan struct{}
- once sync.Once
-}
-
-// UpsertLiveRouteBatch holds the first call until release is closed, then
-// writes routes to the database.
-func (s *firstWriteHeldStore) UpsertLiveRouteBatch(
- routes []*database.LiveRoute,
-) error {
- s.once.Do(func() {
- close(s.writeStarted)
- <-s.release
- })
-
- return s.Database.UpsertLiveRouteBatch(routes)
-}
-
-// TestPrefixHandlerHandlesADownWhileAFlushWrites holds the write of a flush
-// that stores a peer's route and meanwhile hands the prefix handler the peer's
-// session going down, then a new announcement from the peer. The next flush
-// must delete the route the held write stored and keep the one announced after
-// the down.
-func TestPrefixHandlerHandlesADownWhileAFlushWrites(t *testing.T) {
- t.Parallel()
-
- const (
- beforeDown = "192.0.2.0/24"
- afterDown = "198.51.100.0/24"
- maskLength = 24
- )
-
- db := newTestDatabase(t)
- store := &firstWriteHeldStore{
- Database: db,
- writeStarted: make(chan struct{}),
- release: make(chan struct{}),
- }
-
- // Built without NewPrefixHandler's flush timer, so the test flushes itself.
- h := &PrefixHandler{db: store, logger: logger.New()}
- h.HandleMessage(peerAnnouncement(testPeerIP, beforeDown))
-
- flushed := make(chan struct{})
-
- go func() {
- h.flushBatch()
- close(flushed)
- }()
-
- select {
- case <-store.writeStarted:
- case <-time.After(handleMessageTimeout):
- t.Fatal("the flush did not start its write")
- }
-
- if !returnsInTime(func() {
- h.HandleMessage(sessionDown())
- h.HandleMessage(peerAnnouncement(testPeerIP, afterDown))
- }) {
- t.Fatal("HandleMessage did not return while a database write was held")
- }
-
- close(store.release)
-
- select {
- case <-flushed:
- case <-time.After(handleMessageTimeout):
- t.Fatal("the flush did not finish after its write was let go")
- }
-
- h.flushBatch()
-
- assertPeers(t, db, map[string][]string{
- beforeDown: nil,
- afterDown: {testPeerIP},
- })
- assertCounts(t, db, ipv4Version, 1, []database.PrefixDistribution{
- {MaskLength: maskLength, Count: 1},
- })
-}
-
-// errTestPeerDelete is the error a test store's failing delete of a peer's
-// routes returns.
-var errTestPeerDelete = errors.New("test: the delete of a peer's routes failed")
-
-// firstPeerDeleteFailsStore is a real database whose first delete of a peer's
-// live routes fails, deleting nothing. deletes counts the deletes asked for.
-type firstPeerDeleteFailsStore struct {
- *database.Database
-
- // whileFailing, if not nil, runs during the delete that fails.
- whileFailing func()
- deletes int
-}
-
-// DeleteLiveRoutesOfPeer fails the first call and deletes the peer's routes on
-// every later one.
-func (s *firstPeerDeleteFailsStore) DeleteLiveRoutesOfPeer(
- peerIP string,
-) (int, error) {
- s.deletes++
- if s.deletes > 1 {
- return s.Database.DeleteLiveRoutesOfPeer(peerIP)
- }
-
- if s.whileFailing != nil {
- s.whileFailing()
- }
-
- return 0, errTestPeerDelete
-}
-
-// TestPrefixHandlerRetriesAFailedDeleteOfAPeersRoutes runs, through the prefix
-// handler into a database whose first delete of a peer's routes fails, one flush
-// after another: a peer's announcement, then its session going down and a new
-// announcement from the peer, then nothing. The failed delete must be retried
-// on the next flush, and the announcement the peer sent after the down written
-// only after that retry, so it is kept. In the other cases the peer's session
-// goes down again, and the peer announces another prefix, while the delete
-// fails or after the flush whose delete failed: the announcement sent between
-// the two downs must not be written, and the next flush must delete the peer's
-// routes once, not once for the retry and once for the down.
-func TestPrefixHandlerRetriesAFailedDeleteOfAPeersRoutes(t *testing.T) {
- t.Parallel()
-
- const (
- beforeDown = "192.0.2.0/24"
- afterDown = "198.51.100.0/24"
- afterSecondDown = "203.0.113.0/24"
- maskLength = 24
- // The delete that fails and its retry, which is also the delete for
- // a second down.
- wantDeletes = 2
- )
-
- tests := []struct {
- name string
- // Handled while the delete fails, if not nil.
- whileFailing []*ristypes.RISMessage
- // Handled after the flush whose delete fails, if not nil.
- afterFailing []*ristypes.RISMessage
- // The peers left with a live route for each prefix.
- wantPeers map[string][]string
- }{
- {
- name: "the delete fails once",
- wantPeers: map[string][]string{
- beforeDown: nil, afterDown: {testPeerIP}, afterSecondDown: nil,
- },
- },
- {
- name: "the session goes down again while the delete fails",
- whileFailing: []*ristypes.RISMessage{
- sessionDown(), peerAnnouncement(testPeerIP, afterSecondDown),
- },
- wantPeers: map[string][]string{
- beforeDown: nil, afterDown: nil, afterSecondDown: {testPeerIP},
- },
- },
- {
- name: "the session goes down again after the delete failed",
- afterFailing: []*ristypes.RISMessage{
- sessionDown(), peerAnnouncement(testPeerIP, afterSecondDown),
- },
- wantPeers: map[string][]string{
- beforeDown: nil, afterDown: nil, afterSecondDown: {testPeerIP},
- },
- },
- }
-
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- t.Parallel()
-
- db := newTestDatabase(t)
- store := &firstPeerDeleteFailsStore{Database: db}
-
- // Built without NewPrefixHandler's flush timer, so the test
- // flushes itself.
- h := &PrefixHandler{db: store, logger: logger.New()}
- store.whileFailing = func() {
- for _, msg := range tt.whileFailing {
- h.HandleMessage(msg)
- }
- }
-
- h.HandleMessage(peerAnnouncement(testPeerIP, beforeDown))
- h.flushBatch()
- h.HandleMessage(sessionDown())
- h.HandleMessage(peerAnnouncement(testPeerIP, afterDown))
- h.flushBatch() // the delete fails
-
- for _, msg := range tt.afterFailing {
- h.HandleMessage(msg)
- }
-
- h.flushBatch()
-
- if store.deletes != wantDeletes {
- t.Errorf("the store was asked for %d deletes of the peer's "+
- "routes, want %d", store.deletes, wantDeletes)
- }
-
- assertPeers(t, db, tt.wantPeers)
- assertCounts(t, db, ipv4Version, 1, []database.PrefixDistribution{
- {MaskLength: maskLength, Count: 1},
- })
- })
- }
-}
-
-// peerDeleteAlwaysFailsStore is a real database whose every delete of a peer's
-// live routes fails, deleting nothing.
-type peerDeleteAlwaysFailsStore struct {
- *database.Database
-}
-
-// DeleteLiveRoutesOfPeer fails.
-func (peerDeleteAlwaysFailsStore) DeleteLiveRoutesOfPeer(string) (int, error) {
- return 0, errTestPeerDelete
-}
-
-// TestPrefixHandlerKeepsAtMostABatchOfUpdatesWhileADeleteKeepsFailing runs,
-// through the prefix handler into a database whose every delete of a peer's
-// routes fails, a peer's session going down and then three flushes. Before each
-// flush the peer announces prefixBatchSize new prefixes and another peer
-// announces one. After each flush the handler must hold only the peer's last
-// prefixBatchSize updates, not every update since the down, and the other
-// peer's route must be written. After the last flush, one more update must not
-// wake the flush goroutine: held updates do not count toward a full batch.
-func TestPrefixHandlerKeepsAtMostABatchOfUpdatesWhileADeleteKeepsFailing(
- t *testing.T,
-) {
- t.Parallel()
-
- const otherPeerIP = "2001:db8:ffff::2"
-
- // One prefix the other peer announces before each flush.
- otherPeersPrefixes := []string{testV4Prefix, testV6Prefix, "2001:db8:2::/48"}
-
- db := newTestDatabase(t)
-
- // Built without NewPrefixHandler's flush timer, so the test flushes itself.
- h := &PrefixHandler{
- db: peerDeleteAlwaysFailsStore{Database: db}, logger: logger.New(),
- }
- h.HandleMessage(sessionDown())
-
- // The peer announces a /32 for each address from 10.0.0.0 on, each
- // address once.
- addr := netip.MustParseAddr("10.0.0.0")
-
- for flush, otherPeersPrefix := range otherPeersPrefixes {
- prefixes := make([]string, prefixBatchSize)
- for i := range prefixes {
- prefixes[i] = netip.PrefixFrom(addr, addr.BitLen()).String()
- addr = addr.Next()
- }
-
- h.HandleMessage(peerAnnouncement(testPeerIP, prefixes...))
- h.HandleMessage(peerAnnouncement(otherPeerIP, otherPeersPrefix))
- h.flushBatch()
-
- kept := make([]string, 0, len(h.held))
- for _, update := range h.held {
- kept = append(kept, update.prefix)
- }
-
- if !slices.Equal(kept, prefixes) {
- t.Fatalf("after flush %d the handler holds %d updates, not just "+
- "the peer's last %d", flush+1, len(kept), len(prefixes))
- }
-
- assertPeers(t, db, map[string][]string{otherPeersPrefix: {otherPeerIP}})
- }
-
- // A new channel, so that a wake-up in it comes from the update below.
- h.batchFullCh = make(chan struct{}, 1)
- h.HandleMessage(peerAnnouncement(otherPeerIP, testV4Prefix))
-
- if len(h.batchFullCh) > 0 {
- t.Error("one update after a flush that held a batch of updates " +
- "woke the flush goroutine")
- }
-}
-
-// TestPrefixHandlerStopLogsAFailedDeleteWithoutTheDeadline stops a prefix
-// handler, with no deadline, after a peer's session went down and the peer
-// announced a prefix, into a database whose every delete of a peer's routes
-// fails. Stop must log the peer down and the announcement as not written, and
-// must not say that the stop deadline passed.
-func TestPrefixHandlerStopLogsAFailedDeleteWithoutTheDeadline(t *testing.T) {
+// TestPrefixHandlerStopLogsNothingWhenItWroteEverything stops a prefix
+// handler, with no deadline, after a peer announced a prefix and its session
+// went down. Its last flush writes the routing table's changes and the prefix,
+// so Stop must log nothing.
+func TestPrefixHandlerStopLogsNothingWhenItWroteEverything(t *testing.T) {
t.Parallel()
var logs bytes.Buffer
- h := NewPrefixHandler(peerDeleteAlwaysFailsStore{Database: newTestDatabase(t)},
- newErrorLogger(&logs))
- h.HandleMessage(sessionDown())
+ h := NewPrefixHandler(newTestDatabase(t), newErrorLogger(&logs),
+ routetable.New(routetable.Params{}))
h.HandleMessage(peerAnnouncement(testPeerIP, testV4Prefix))
- h.Stop(t.Context())
-
- logged := lastLogged(t, &logs)
- if strings.Contains(logged.Msg, "deadline") {
- t.Errorf("Stop logged %q although its deadline had not passed",
- logged.Msg)
- }
-
- if logged.Updates != 1 || logged.PeersDown != 1 {
- t.Errorf("Stop logged %d updates and %d peer downs not written, "+
- "want 1 and 1", logged.Updates, logged.PeersDown)
- }
-}
-
-// TestPrefixHandlerStopLogsNothingAfterADeleteThatSucceeds stops a prefix
-// handler, with no deadline, after a peer's session went down, into a database
-// whose delete of the peer's routes succeeds. Stop must log nothing: a peer
-// whose routes are deleted no longer counts as a peer down left undone.
-func TestPrefixHandlerStopLogsNothingAfterADeleteThatSucceeds(t *testing.T) {
- t.Parallel()
-
- var logs bytes.Buffer
-
- h := NewPrefixHandler(newTestDatabase(t), newErrorLogger(&logs))
h.HandleMessage(sessionDown())
h.Stop(t.Context())
if logs.Len() != 0 {
- t.Errorf("Stop logged %q after the routes of the peer whose session "+
- "went down were deleted", logs.String())
+ t.Errorf("Stop logged %q after it wrote everything", logs.String())
+ }
+}
+
+// longerThanAnyTest is a flush timer interval that no test lasts. A test whose
+// prefix, AS or peer handler must flush only when its batch is full and on Stop
+// gives it to newPrefixHandler, newASHandler or newPeerHandler.
+const longerThanAnyTest = time.Hour
+
+// prefixHandlerWithoutTimer returns a prefix handler that writes to store and
+// applies the updates to an empty routing table. It is built without
+// NewPrefixHandler's flush goroutine, so the test flushes itself.
+func prefixHandlerWithoutTimer(store database.Store) *PrefixHandler {
+ return &PrefixHandler{
+ db: store,
+ logger: logger.New(),
+ table: routetable.New(routetable.Params{}),
}
}
@@ -1012,9 +276,7 @@ func handleBatches(
t.Helper()
db := newTestDatabase(t)
-
- // Built without NewPrefixHandler's flush timer, so the test flushes itself.
- h := &PrefixHandler{db: db, logger: logger.New()}
+ h := prefixHandlerWithoutTimer(db)
for _, batch := range batches {
for _, msg := range batch {
@@ -1050,130 +312,3 @@ func sessionDown() *ristypes.RISMessage {
State: ristypes.StateDown,
}
}
-
-// checkLivePeers checks that exactly wantPeers have a live route for the IPv4
-// prefix, and that the in-memory route count and prefix distribution match
-// the route table.
-func checkLivePeers(
- t *testing.T, db *database.Database, prefix string, maskLength int,
- wantPeers []string,
-) {
- t.Helper()
-
- assertPeers(t, db, map[string][]string{prefix: wantPeers})
-
- var wantDistribution []database.PrefixDistribution
- if len(wantPeers) > 0 {
- wantDistribution = []database.PrefixDistribution{
- {MaskLength: maskLength, Count: 1},
- }
- }
-
- assertCounts(t, db, ipv4Version, len(wantPeers), wantDistribution)
-}
-
-// assertPeers checks, for each prefix in wantPeers, that the peers with a live
-// route for it are the ones listed, in peer IP order; nil means none.
-func assertPeers(
- t *testing.T, db *database.Database, wantPeers map[string][]string,
-) {
- t.Helper()
-
- for prefix, want := range wantPeers {
- routes, _, err := db.GetPrefixDetailsContext(context.Background(), prefix)
- if err != nil && !errors.Is(err, database.ErrNoRoute) {
- t.Fatalf("GetPrefixDetailsContext(%s): %v", prefix, err)
- }
-
- got := make([]string, 0, len(routes))
- for _, route := range routes {
- got = append(got, route.PeerIP)
- }
-
- if !slices.Equal(got, want) {
- t.Errorf("peers with a live route for %s = %v, want %v",
- prefix, got, want)
- }
- }
-}
-
-// checkLiveOrigins checks that each of prefixes has one live route, with the
-// origin AS wantOrigins gives for it, or none when wantOrigins leaves it out.
-func checkLiveOrigins(
- t *testing.T, db *database.Database, wantOrigins map[string]int,
- prefixes ...string,
-) {
- t.Helper()
-
- for _, prefix := range prefixes {
- routes, _, err := db.GetPrefixDetailsContext(context.Background(), prefix)
- if err != nil && !errors.Is(err, database.ErrNoRoute) {
- t.Fatalf("GetPrefixDetailsContext(%s): %v", prefix, err)
- }
-
- gotOrigins := make([]int, 0, len(routes))
- for _, route := range routes {
- gotOrigins = append(gotOrigins, route.OriginASN)
- }
-
- var want []int
- if origin, ok := wantOrigins[prefix]; ok {
- want = []int{origin}
- }
-
- if !slices.Equal(gotOrigins, want) {
- t.Errorf("origin ASes of the live routes for %s = %v, want %v",
- prefix, gotOrigins, want)
- }
- }
-}
-
-// assertCounts checks that the live route count and prefix distribution of one
-// IP version are wantRoutes and wantDistribution, both as held in memory and as
-// read from the route table.
-func assertCounts(
- t *testing.T, db *database.Database, ipVersion, wantRoutes int,
- wantDistribution []database.PrefixDistribution,
-) {
- t.Helper()
-
- ctx := context.Background()
-
- stats, err := db.GetStatsContext(ctx)
- if err != nil {
- t.Fatalf("GetStatsContext: %v", err)
- }
-
- tableRoutesV4, tableRoutesV6, err := db.GetLiveRouteCountsContext(ctx)
- if err != nil {
- t.Fatalf("GetLiveRouteCountsContext: %v", err)
- }
-
- queryDistributionV4, queryDistributionV6, err :=
- db.GetPrefixDistributionContext(ctx)
- if err != nil {
- t.Fatalf("GetPrefixDistributionContext: %v", err)
- }
-
- memoryRoutes, tableRoutes := stats.IPv4Routes, tableRoutesV4
- memoryDistribution := stats.IPv4PrefixDistribution
- queryDistribution := queryDistributionV4
-
- if ipVersion == ipv6Version {
- memoryRoutes, tableRoutes = stats.IPv6Routes, tableRoutesV6
- memoryDistribution = stats.IPv6PrefixDistribution
- queryDistribution = queryDistributionV6
- }
-
- if memoryRoutes != wantRoutes || tableRoutes != wantRoutes {
- t.Errorf("IPv%d live routes: %d in memory, %d in the table, want %d",
- ipVersion, memoryRoutes, tableRoutes, wantRoutes)
- }
-
- if !slices.Equal(memoryDistribution, wantDistribution) ||
- !slices.Equal(queryDistribution, wantDistribution) {
- t.Errorf("IPv%d prefix distribution: %v in memory, %v from the table, "+
- "want %v", ipVersion, memoryDistribution, queryDistribution,
- wantDistribution)
- }
-}
diff --git a/internal/routewatch/prefixhandler_table_internal_test.go b/internal/routewatch/prefixhandler_table_internal_test.go
new file mode 100644
index 0000000..b4b894f
--- /dev/null
+++ b/internal/routewatch/prefixhandler_table_internal_test.go
@@ -0,0 +1,607 @@
+package routewatch
+
+import (
+ "bytes"
+ "encoding/json"
+ "errors"
+ "log/slog"
+ "net/netip"
+ "reflect"
+ "slices"
+ "strings"
+ "testing"
+ "time"
+
+ "git.eeqj.de/sneak/routewatch/internal/database"
+ "git.eeqj.de/sneak/routewatch/internal/logger"
+ "git.eeqj.de/sneak/routewatch/internal/ristypes"
+ "git.eeqj.de/sneak/routewatch/internal/routetable"
+)
+
+// More peers and prefixes for the tests of the routing table in memory.
+const (
+ secondPeerIP = "2001:db8:ffff::2"
+ thirdPeerIP = "2001:db8:ffff::3"
+ fourthPeerIP = "192.0.2.4"
+ fifthPeerIP = "192.0.2.5"
+
+ otherV4Prefix = "198.51.100.0/24"
+ setV4Prefix = "203.0.113.0/24"
+
+ // firstManyPrefix is the first prefix announceManyPrefixes announces.
+ firstManyPrefix = "10.0.0.0/32"
+)
+
+// tableWriteTimeout is how long a test waits for the flush timer to write the
+// routing table's changes: a few ticks of it.
+const tableWriteTimeout = 5 * prefixBatchTimeout
+
+// tableCase is one test of the routing table in memory: the messages handed to
+// the prefix handler, and for each prefix the peers that carry it with each
+// origin AS after them.
+type tableCase struct {
+ name string
+ messages []*ristypes.RISMessage
+ want map[string]map[int][]string
+}
+
+// TestPrefixHandlerAppliesUpdatesToTheRoutingTable hands the prefix handler
+// announcements, withdrawals and a peer's session going down.
+func TestPrefixHandlerAppliesUpdatesToTheRoutingTable(t *testing.T) {
+ t.Parallel()
+
+ announceAndWithdraw := peerAnnouncement(testPeerIP, testV4Prefix)
+ announceAndWithdraw.Withdrawals = []string{testV4Prefix}
+
+ checkTableCases(t, []tableCase{
+ {
+ name: "an announcement",
+ messages: []*ristypes.RISMessage{peerAnnouncement(testPeerIP, testV4Prefix)},
+ want: map[string]map[int][]string{testV4Prefix: {testASNB: {testPeerIP}}},
+ },
+ {
+ name: "a withdrawal",
+ messages: []*ristypes.RISMessage{
+ peerAnnouncement(testPeerIP, testV4Prefix),
+ peerWithdrawal(testPeerIP, testV4Prefix),
+ },
+ want: map[string]map[int][]string{testV4Prefix: {}},
+ },
+ {
+ // RFC 4271 section 4.3: the announcement counts, not the withdrawal.
+ name: "a message that announces and withdraws the prefix",
+ messages: []*ristypes.RISMessage{announceAndWithdraw},
+ want: map[string]map[int][]string{testV4Prefix: {testASNB: {testPeerIP}}},
+ },
+ {
+ name: "a peer's session going down",
+ messages: []*ristypes.RISMessage{
+ peerAnnouncement(testPeerIP, testV4Prefix, testV6Prefix),
+ peerAnnouncement(secondPeerIP, testV4Prefix),
+ sessionDown(),
+ },
+ want: map[string]map[int][]string{
+ testV4Prefix: {testASNB: {secondPeerIP}},
+ testV6Prefix: {},
+ },
+ },
+ {
+ name: "an announcement after the peer's session went down",
+ messages: []*ristypes.RISMessage{
+ peerAnnouncement(testPeerIP, testV4Prefix),
+ sessionDown(),
+ peerAnnouncement(testPeerIP, otherV4Prefix),
+ },
+ want: map[string]map[int][]string{
+ testV4Prefix: {},
+ otherV4Prefix: {testASNB: {testPeerIP}},
+ },
+ },
+ })
+}
+
+// TestPrefixHandlerAppliesAPathWithoutAnOriginToTheRoutingTable hands the
+// prefix handler announcements whose AS path gives no origin. One whose path
+// is empty or holds AS 0 removes the peer's route, as a withdrawal does; one
+// whose path ends in an AS set of several members is kept with origin 0.
+func TestPrefixHandlerAppliesAPathWithoutAnOriginToTheRoutingTable(t *testing.T) {
+ t.Parallel()
+
+ announceWithPath := func(path ristypes.ASPath) *ristypes.RISMessage {
+ msg := peerAnnouncement(testPeerIP, testV4Prefix)
+ msg.Path = path
+
+ return msg
+ }
+
+ checkTableCases(t, []tableCase{
+ {
+ name: "a path whose last AS is 0",
+ messages: []*ristypes.RISMessage{
+ peerAnnouncement(testPeerIP, testV4Prefix),
+ announceWithPath(ristypes.ASPath{Sequence: []int{testASNA, 0}}),
+ },
+ want: map[string]map[int][]string{testV4Prefix: {}},
+ },
+ {
+ name: "an empty path",
+ messages: []*ristypes.RISMessage{
+ peerAnnouncement(testPeerIP, testV4Prefix),
+ announceWithPath(ristypes.ASPath{}),
+ },
+ want: map[string]map[int][]string{testV4Prefix: {}},
+ },
+ {
+ name: "a path with AS 0 before its end",
+ messages: []*ristypes.RISMessage{
+ peerAnnouncement(testPeerIP, testV4Prefix),
+ announceWithPath(decodePath(t, testASNA, 0, testASNB)),
+ },
+ want: map[string]map[int][]string{testV4Prefix: {}},
+ },
+ {
+ name: "a path that ends in an AS set",
+ messages: []*ristypes.RISMessage{
+ announceWithPath(decodePath(t, testASNA, []int{testASNC, testASND})),
+ },
+ want: map[string]map[int][]string{testV4Prefix: {0: {testPeerIP}}},
+ },
+ })
+}
+
+// TestPrefixHandlerKeepsThePathOfAPathEndingInASet announces a prefix with AS
+// paths that end in an AS set. Any member of a set of several may be the
+// origin, so the routing table keeps the route with origin 0 (none); a set of
+// one member counts as that AS. The kept AS path lists the set's members after
+// the rest of the path.
+func TestPrefixHandlerKeepsThePathOfAPathEndingInASet(t *testing.T) {
+ t.Parallel()
+
+ tests := []struct {
+ name string
+ path []any
+ wantOrigin int
+ wantPath []int
+ }{
+ {
+ name: "a set of several members",
+ path: []any{testASNA, testASNB, []int{testASNC, testASND}},
+ wantPath: []int{testASNA, testASNB, testASNC, testASND},
+ },
+ {
+ name: "a set of one member",
+ path: []any{testASNA, testASNB, []int{testASNC}},
+ wantOrigin: testASNC,
+ wantPath: []int{testASNA, testASNB, testASNC},
+ },
+ }
+
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ t.Parallel()
+
+ msg := peerAnnouncement(testPeerIP, testV4Prefix)
+ msg.Path = decodePath(t, tt.path...)
+
+ h := prefixHandlerWithoutTimer(newMockStore())
+ h.HandleMessage(msg)
+
+ origins := h.table.Origins(netip.MustParsePrefix(testV4Prefix))
+ if len(origins) != 1 || origins[0].ASN != tt.wantOrigin ||
+ len(origins[0].Routes) != 1 ||
+ !slices.Equal(origins[0].Routes[0].ASPath, tt.wantPath) {
+ t.Errorf("Origins(%s) = %+v, want one route with origin AS%d "+
+ "and AS path %v", testV4Prefix, origins, tt.wantOrigin,
+ tt.wantPath)
+ }
+ })
+ }
+}
+
+// checkTableCases hands the messages of each case to a new prefix handler and
+// checks its routing table right after, with no flush.
+func checkTableCases(t *testing.T, tests []tableCase) {
+ t.Helper()
+
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ t.Parallel()
+
+ h := prefixHandlerWithoutTimer(newMockStore())
+ for _, msg := range tt.messages {
+ h.HandleMessage(msg)
+ }
+
+ for prefix, want := range tt.want {
+ got := tableOrigins(h.table, prefix)
+ if !reflect.DeepEqual(got, want) {
+ t.Errorf("peers of %s by origin AS = %v, want %v",
+ prefix, got, want)
+ }
+ }
+ })
+ }
+}
+
+// tableWritesStore is a real database that hands each write of the routing
+// table's changes to written once it is made.
+type tableWritesStore struct {
+ *database.Database
+
+ written chan []routetable.Change
+}
+
+func (s tableWritesStore) WriteRouteChanges(
+ changes []routetable.Change, newPeers map[string]uint,
+) error {
+ err := s.Database.WriteRouteChanges(changes, newPeers)
+ s.written <- changes
+
+ return err
+}
+
+// TestPrefixHandlerWritesTheTableChangesEachSecond hands a running prefix
+// handler an announcement and then its withdrawal, each far short of a full
+// batch. The flush timer must write the change of each to the database.
+func TestPrefixHandlerWritesTheTableChangesEachSecond(t *testing.T) {
+ t.Parallel()
+
+ // The two writes the test waits for, and room for one more.
+ const writesBuffered = 3
+
+ store := tableWritesStore{
+ Database: newTestDatabase(t),
+ written: make(chan []routetable.Change, writesBuffered),
+ }
+ h := NewPrefixHandler(store, logger.New(), routetable.New(routetable.Params{}))
+
+ defer h.Stop(t.Context())
+
+ prefix := netip.MustParsePrefix(testV4Prefix)
+
+ for _, step := range []struct {
+ msg *ristypes.RISMessage
+ wantRemoved bool
+ }{
+ {msg: peerAnnouncement(testPeerIP, testV4Prefix)},
+ {msg: peerWithdrawal(testPeerIP, testV4Prefix), wantRemoved: true},
+ } {
+ h.HandleMessage(step.msg)
+
+ select {
+ case changes := <-store.written:
+ if len(changes) != 1 || changes[0].Prefix != prefix ||
+ changes[0].OriginASN != testASNB ||
+ changes[0].Removed != step.wantRemoved {
+ t.Fatalf("wrote %+v, want the pair of %s and AS%d with "+
+ "removed %t", changes, prefix, testASNB, step.wantRemoved)
+ }
+ case <-time.After(tableWriteTimeout):
+ t.Fatalf("the routing table's change was not written within %s",
+ tableWriteTimeout)
+ }
+ }
+}
+
+// errTestTableWrite is the error a test store's failing write of the routing
+// table's changes returns.
+var errTestTableWrite = errors.New("test: the write of the table's changes failed")
+
+// firstTableWriteFailsStore is a real database whose first write of the
+// routing table's changes fails, writing nothing. writes counts the writes
+// asked for.
+type firstTableWriteFailsStore struct {
+ *database.Database
+
+ writes int
+}
+
+func (s *firstTableWriteFailsStore) WriteRouteChanges(
+ changes []routetable.Change, newPeers map[string]uint,
+) error {
+ s.writes++
+ if s.writes == 1 {
+ return errTestTableWrite
+ }
+
+ return s.Database.WriteRouteChanges(changes, newPeers)
+}
+
+// TestPrefixHandlerWritesAFailedTableWriteAgain runs two flushes into a
+// database whose first write of the routing table's changes fails. Before the
+// first, two new peers announce a prefix each; before the second, one of them
+// announces its prefix again with another path to the same origin AS. The
+// second write must hold the changes and new peers of the first, with the later
+// path, so the database holds the table that is in memory.
+func TestPrefixHandlerWritesAFailedTableWriteAgain(t *testing.T) {
+ t.Parallel()
+
+ db := newTestDatabase(t)
+ store := &firstTableWriteFailsStore{Database: db}
+ h := prefixHandlerWithoutTimer(store)
+
+ first := peerAnnouncement(testPeerIP, testV4Prefix)
+ h.HandleMessage(first)
+ h.HandleMessage(peerAnnouncement(secondPeerIP, otherV4Prefix))
+ h.flushBatch()
+
+ laterPath := []int{testASNC, testASNB}
+ later := peerAnnouncement(testPeerIP, testV4Prefix)
+ later.Path = ristypes.ASPath{Sequence: laterPath}
+ later.ParsedTimestamp = first.ParsedTimestamp.Add(secondUpdateAfter)
+ h.HandleMessage(later)
+ h.flushBatch()
+
+ if store.writes != 2 {
+ t.Fatalf("the store was asked for %d writes of the table's changes, "+
+ "want 2", store.writes)
+ }
+
+ read, err := db.ReadRouteTable(t.Context())
+ if err != nil {
+ t.Fatalf("ReadRouteTable: %v", err)
+ }
+
+ checkSameTable(t, read, h.table, testV4Prefix, otherV4Prefix)
+
+ origins := read.Origins(netip.MustParsePrefix(testV4Prefix))
+ if len(origins) != 1 || len(origins[0].Routes) != 1 ||
+ !slices.Equal(origins[0].Routes[0].ASPath, laterPath) {
+ t.Errorf("Origins(%s) = %+v, want one route with the path %v",
+ testV4Prefix, origins, laterPath)
+ }
+}
+
+// tableWriteSizesStore is a real database that records how many changes and
+// how many new peers each write of the routing table's changes holds.
+type tableWriteSizesStore struct {
+ *database.Database
+
+ changes []int
+ newPeers []int
+}
+
+func (s *tableWriteSizesStore) WriteRouteChanges(
+ changes []routetable.Change, newPeers map[string]uint,
+) error {
+ s.changes = append(s.changes, len(changes))
+ s.newPeers = append(s.newPeers, len(newPeers))
+
+ return s.Database.WriteRouteChanges(changes, newPeers)
+}
+
+// TestPrefixHandlerWritesManyTableChangesOverSeveralFlushes has one new peer
+// change one and a half times as many prefix and origin AS pairs as one write
+// takes, then flushes three times. The first write must hold as many pairs as
+// one write takes, and the peer. The second must hold the other pairs, and the
+// third flush must write nothing. The database must then hold the table in
+// memory.
+func TestPrefixHandlerWritesManyTableChangesOverSeveralFlushes(t *testing.T) {
+ t.Parallel()
+
+ const (
+ pairs = maxTableChangesPerWrite * 3 / 2
+ flushes = 3
+ )
+
+ db := newTestDatabase(t)
+ store := &tableWriteSizesStore{Database: db}
+ h := prefixHandlerWithoutTimer(store)
+ announceManyPrefixes(h.table, pairs)
+
+ for range flushes {
+ h.flushBatch()
+ }
+
+ wantChanges := []int{maxTableChangesPerWrite, pairs - maxTableChangesPerWrite}
+ wantNewPeers := []int{1, 0}
+
+ if !slices.Equal(store.changes, wantChanges) ||
+ !slices.Equal(store.newPeers, wantNewPeers) {
+ t.Errorf("the writes held %v pairs and %v new peers, want %v and %v",
+ store.changes, store.newPeers, wantChanges, wantNewPeers)
+ }
+
+ read, err := db.ReadRouteTable(t.Context())
+ if err != nil {
+ t.Fatalf("ReadRouteTable: %v", err)
+ }
+
+ checkSameTable(t, read, h.table, firstManyPrefix)
+}
+
+// TestPrefixHandlerStopLogsTheTableChangesLeftUnwritten stops a prefix handler,
+// with no deadline, whose routing table holds more changed pairs than one write
+// takes. Its last flush writes one write's worth, so Stop must log the pairs
+// that no write took, and must not say that the stop deadline passed. The
+// flush timer must not fire, or its writes could leave no pairs for Stop to
+// log.
+func TestPrefixHandlerStopLogsTheTableChangesLeftUnwritten(t *testing.T) {
+ t.Parallel()
+
+ const pairs = maxTableChangesPerWrite + 1
+
+ var logs bytes.Buffer
+
+ store := &tableWriteSizesStore{Database: newTestDatabase(t)}
+ table := routetable.New(routetable.Params{})
+ announceManyPrefixes(table, pairs)
+
+ h := newPrefixHandler(store, newErrorLogger(&logs), table, longerThanAnyTest)
+ h.Stop(t.Context())
+
+ written := 0
+ for _, n := range store.changes {
+ written += n
+ }
+
+ logged := lastLogged(t, &logs)
+ if strings.Contains(logged.Msg, "deadline") {
+ t.Errorf("Stop logged %q although its deadline had not passed",
+ logged.Msg)
+ }
+
+ if logged.TableChanges != pairs-written {
+ t.Errorf("Stop logged %d table changes not written, want %d",
+ logged.TableChanges, pairs-written)
+ }
+}
+
+// readLog holds what readRouteTable logs.
+//
+//nolint:tagliatelle // field names are the keys readRouteTable logs
+type readLog struct {
+ Msg string `json:"msg"`
+ Pairs int `json:"pairs"`
+ Peers int `json:"peers"`
+ DurationMs *int64 `json:"duration_ms"`
+}
+
+// TestARestartReadsTheRoutingTableBack runs announcements, a withdrawal and a
+// peer's session going down through a prefix handler into a database, then
+// reads the table back as startup does. The table read must answer as the one
+// in memory, and the log must give its pairs and peers. A new handler on the
+// table read then takes later updates and writes them as it stops: the
+// database must then hold what the first table holds after the same updates.
+func TestARestartReadsTheRoutingTableBack(t *testing.T) {
+ t.Parallel()
+
+ prefixes := []string{testV4Prefix, testV6Prefix, otherV4Prefix, setV4Prefix}
+
+ db := newTestDatabase(t)
+ h := prefixHandlerWithoutTimer(db)
+
+ viaOtherOrigin := peerAnnouncement(thirdPeerIP, testV4Prefix)
+ viaOtherOrigin.Path = ristypes.ASPath{Sequence: []int{testASNA, testASNC}}
+ endsInSet := peerAnnouncement(secondPeerIP, setV4Prefix)
+ endsInSet.Path = decodePath(t, testASNA, []int{testASNC, testASND})
+
+ for _, msg := range []*ristypes.RISMessage{
+ peerAnnouncement(testPeerIP, testV4Prefix, testV6Prefix),
+ peerAnnouncement(secondPeerIP, testV4Prefix),
+ viaOtherOrigin,
+ endsInSet,
+ peerAnnouncement(thirdPeerIP, otherV4Prefix),
+ peerWithdrawal(thirdPeerIP, otherV4Prefix),
+ peerAnnouncement(fourthPeerIP, otherV4Prefix),
+ sessionDownOf(fourthPeerIP),
+ } {
+ h.HandleMessage(msg)
+ }
+
+ h.flushBatch()
+
+ var logs bytes.Buffer
+
+ table, err := readRouteTable(db,
+ &logger.Logger{Logger: slog.New(slog.NewJSONHandler(&logs, nil))})
+ if err != nil {
+ t.Fatalf("readRouteTable: %v", err)
+ }
+
+ checkSameTable(t, table, h.table, prefixes...)
+
+ // testV4Prefix with two origin ASes, testV6Prefix and setV4Prefix; four
+ // peers, the last of which carries nothing.
+ const wantPairs, wantPeers = 4, 4
+
+ var logged readLog
+
+ err = json.Unmarshal(bytes.TrimSpace(logs.Bytes()), &logged)
+ if err != nil || logged.Pairs != wantPairs || logged.Peers != wantPeers ||
+ logged.DurationMs == nil {
+ t.Errorf("readRouteTable logged %q, want %d pairs, %d peers and the "+
+ "duration", logs.String(), wantPairs, wantPeers)
+ }
+
+ restarted := NewPrefixHandler(db, logger.New(), table)
+
+ for _, msg := range []*ristypes.RISMessage{
+ peerWithdrawal(testPeerIP, testV6Prefix),
+ sessionDownOf(secondPeerIP),
+ peerAnnouncement(fourthPeerIP, otherV4Prefix),
+ peerAnnouncement(fifthPeerIP, setV4Prefix),
+ } {
+ // The first handler only applies the updates to its table.
+ h.HandleMessage(msg)
+ restarted.HandleMessage(msg)
+ }
+
+ restarted.Stop(t.Context())
+
+ read, err := db.ReadRouteTable(t.Context())
+ if err != nil {
+ t.Fatalf("ReadRouteTable after the restart: %v", err)
+ }
+
+ checkSameTable(t, read, h.table, prefixes...)
+}
+
+// peerWithdrawal returns an UPDATE message in which peer withdraws prefixes.
+func peerWithdrawal(peer string, prefixes ...string) *ristypes.RISMessage {
+ return &ristypes.RISMessage{
+ Type: ristypes.MessageTypeUpdate,
+ Peer: peer,
+ ParsedTimestamp: time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC),
+ Withdrawals: prefixes,
+ }
+}
+
+// announceManyPrefixes has testPeerIP announce n prefixes in table, with the
+// path of peerAnnouncement: the /32 of each address from firstManyPrefix on.
+func announceManyPrefixes(table *routetable.Table, n int) {
+ addr := netip.MustParsePrefix(firstManyPrefix).Addr()
+ route := routetable.Route{
+ Peer: testPeerIP,
+ ASPath: []int{testASNA, testASNB},
+ Time: time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC),
+ }
+
+ for range n {
+ table.Announce(netip.PrefixFrom(addr, addr.BitLen()), testASNB, route)
+ addr = addr.Next()
+ }
+}
+
+// sessionDownOf returns the STATE message that says the BGP session of peer
+// went down.
+func sessionDownOf(peer string) *ristypes.RISMessage {
+ return &ristypes.RISMessage{
+ Type: ristypes.MessageTypeState,
+ Peer: peer,
+ State: ristypes.StateDown,
+ }
+}
+
+// tableOrigins returns, for each origin AS of prefix in table, the peers that
+// carry the prefix with it.
+func tableOrigins(table *routetable.Table, prefix string) map[int][]string {
+ origins := make(map[int][]string)
+ for _, o := range table.Origins(netip.MustParsePrefix(prefix)) {
+ origins[o.ASN] = o.Peers
+ }
+
+ return origins
+}
+
+// checkSameTable checks that got answers as want does for each of prefixes, and
+// that the two tables have the same counts.
+func checkSameTable(
+ t *testing.T, got, want *routetable.Table, prefixes ...string,
+) {
+ t.Helper()
+
+ for _, text := range prefixes {
+ prefix := netip.MustParsePrefix(text)
+
+ gotOrigins, wantOrigins := got.Origins(prefix), want.Origins(prefix)
+ if !reflect.DeepEqual(gotOrigins, wantOrigins) {
+ t.Errorf("Origins(%s) = %+v, want %+v", text, gotOrigins, wantOrigins)
+ }
+ }
+
+ gotCounts, wantCounts := got.Counts(), want.Counts()
+ if !reflect.DeepEqual(gotCounts, wantCounts) {
+ t.Errorf("Counts() = %+v, want %+v", gotCounts, wantCounts)
+ }
+}
diff --git a/internal/server/handlers.go b/internal/server/handlers.go
index 2132c0c..53906bd 100644
--- a/internal/server/handlers.go
+++ b/internal/server/handlers.go
@@ -1,21 +1,25 @@
package server
import (
- "bytes"
+ "cmp"
"context"
"encoding/json"
"errors"
+ "fmt"
"net"
"net/http"
"net/netip"
"runtime"
- "sort"
+ "slices"
"strconv"
"strings"
"time"
+ "git.eeqj.de/sneak/routewatch/internal/blocklist"
"git.eeqj.de/sneak/routewatch/internal/database"
"git.eeqj.de/sneak/routewatch/internal/metrics"
+ "git.eeqj.de/sneak/routewatch/internal/ristypes"
+ "git.eeqj.de/sneak/routewatch/internal/routetable"
"git.eeqj.de/sneak/routewatch/internal/streamer"
"git.eeqj.de/sneak/routewatch/internal/templates"
asinfo "git.eeqj.de/sneak/routewatch/pkg/asinfo"
@@ -35,7 +39,7 @@ const (
const (
// maxLookupTargets is the most addresses and host names an IP lookup may
// list, and the most addresses it may have once the names are resolved:
- // each address costs up to 113 database reads and a PTR lookup.
+ // each address costs two database reads and a PTR lookup.
maxLookupTargets = 10
// ptrLookupTimeout is each PTR lookup's own time limit, so that ten
@@ -43,6 +47,15 @@ const (
ptrLookupTimeout = 2 * time.Second
)
+// The Cache-Control header of an IP lookup answer, such as
+// "public, max-age=900".
+const (
+ cacheControlHeader = "Cache-Control"
+ cacheControlPublic = "public"
+ cacheControlPrivate = "private"
+ cacheControlMaxAge = ", max-age="
+)
+
// Keys of the JSON envelope the handlers and middleware write, and the status
// value of an error answer.
const (
@@ -56,13 +69,21 @@ const (
// Error messages an answer sends, in place of the error behind it where there
// is one.
const (
- invalidPrefixMessage = "Invalid prefix"
- hostNotResolvedMessage = "could not resolve host name"
- hostNotPublicMessage = "host name has no public address"
- tooManyTargetsMessage = "Too many addresses or host names"
+ invalidPrefixMessage = "Invalid prefix"
+ hostNotResolvedMessage = "could not resolve host name"
+ hostNotPublicMessage = "host name has no public address"
+ tooManyTargetsMessage = "Too many addresses or host names"
+ searchQueryMissingMessage = "Missing search query"
)
-// IP versions as database.LiveRoute.IPVersion records them.
+// The AS name search reads the text it looks for from the URL query parameter
+// asSearchQueryParam, and answers with at most maxASSearchResults matches.
+const (
+ asSearchQueryParam = "q"
+ maxASSearchResults = 50
+)
+
+// IP versions as LiveRoute.IPVersion records them.
const (
ipVersionV4 = 4
ipVersionV6 = 6
@@ -136,7 +157,10 @@ func (s *Server) handleIndex() http.HandlerFunc {
tmpl := templates.IndexTemplate()
- err := tmpl.Execute(w, pageData{CSPNonce: cspNonce(r.Context())})
+ err := tmpl.Execute(w, indexPageData{
+ CSPNonce: cspNonce(r.Context()),
+ ClientIP: extractClientIP(r),
+ })
if err != nil {
s.logger.Error("Failed to render index template", "error", err)
http.Error(w, "Internal Server Error", http.StatusInternalServerError)
@@ -150,6 +174,15 @@ type pageData struct {
CSPNonce string
}
+// indexPageData is what the home page is rendered with.
+type indexPageData struct {
+ // CSPNonce is the nonce the page's script and style blocks carry.
+ CSPNonce string
+ // ClientIP is the visitor's address, which the IP information API card
+ // looks up: the address /ip/ would look up for the same request.
+ ClientIP string
+}
+
// writeJSONError writes a standardized JSON error response with the given
// status code and error message.
func (s *Server) writeJSONError(w http.ResponseWriter, statusCode int, message string) {
@@ -195,38 +228,43 @@ type WHOISStatsInfo struct {
}
// Stats is the /status.json answer: uptime, message counts, database stats,
-// and route information.
+// and the routing table's figures. OldestRoute and NewestRoute are the times
+// of the oldest and the newest kept routes.
//
//nolint:tagliatelle // field names are the existing JSON API
type Stats struct {
- Uptime string `json:"uptime"`
- TotalMessages uint64 `json:"total_messages"`
- TotalBytes uint64 `json:"total_bytes"`
- TotalWireBytes uint64 `json:"total_wire_bytes"`
- MessagesPerSec float64 `json:"messages_per_sec"`
- MbitsPerSec float64 `json:"mbits_per_sec"`
- WireMbitsPerSec float64 `json:"wire_mbits_per_sec"`
- Connected bool `json:"connected"`
- GoVersion string `json:"go_version"`
- Goroutines int `json:"goroutines"`
- MemoryUsage string `json:"memory_usage"`
- ASNs int `json:"asns"`
- Prefixes int `json:"prefixes"`
- IPv4Prefixes int `json:"ipv4_prefixes"`
- IPv6Prefixes int `json:"ipv6_prefixes"`
- Peerings int `json:"peerings"`
- Peers int `json:"peers"`
- DatabaseSizeBytes int64 `json:"database_size_bytes"`
- LiveRoutes int `json:"live_routes"`
- IPv4Routes int `json:"ipv4_routes"`
- IPv6Routes int `json:"ipv6_routes"`
- OldestRoute *time.Time `json:"oldest_route,omitempty"`
- NewestRoute *time.Time `json:"newest_route,omitempty"`
- IPv4UpdatesPerSec float64 `json:"ipv4_updates_per_sec"`
- IPv6UpdatesPerSec float64 `json:"ipv6_updates_per_sec"`
- IPv4PrefixDistribution []database.PrefixDistribution `json:"ipv4_prefix_distribution"`
- IPv6PrefixDistribution []database.PrefixDistribution `json:"ipv6_prefix_distribution"`
- WHOISStats *WHOISStatsInfo `json:"whois_stats,omitempty"`
+ Uptime string `json:"uptime"`
+ TotalMessages uint64 `json:"total_messages"`
+ TotalBytes uint64 `json:"total_bytes"`
+ TotalWireBytes uint64 `json:"total_wire_bytes"`
+ MessagesPerSec float64 `json:"messages_per_sec"`
+ MbitsPerSec float64 `json:"mbits_per_sec"`
+ WireMbitsPerSec float64 `json:"wire_mbits_per_sec"`
+ Connected bool `json:"connected"`
+ GoVersion string `json:"go_version"`
+ Goroutines int `json:"goroutines"`
+ MemoryUsage string `json:"memory_usage"`
+ ASNs int `json:"asns"`
+ Prefixes int `json:"prefixes"`
+ IPv4Prefixes int `json:"ipv4_prefixes"`
+ IPv6Prefixes int `json:"ipv6_prefixes"`
+ Peerings int `json:"peerings"`
+ Peers int `json:"peers"`
+ DatabaseSizeBytes int64 `json:"database_size_bytes"`
+ LiveRoutes int `json:"live_routes"`
+ IPv4Routes int `json:"ipv4_routes"`
+ IPv6Routes int `json:"ipv6_routes"`
+ OldestRoute *time.Time `json:"oldest_route,omitempty"`
+ NewestRoute *time.Time `json:"newest_route,omitempty"`
+ IPv4UpdatesPerSec float64 `json:"ipv4_updates_per_sec"`
+ IPv6UpdatesPerSec float64 `json:"ipv6_updates_per_sec"`
+ IPv4PrefixDistribution []PrefixDistribution `json:"ipv4_prefix_distribution"`
+ IPv6PrefixDistribution []PrefixDistribution `json:"ipv6_prefix_distribution"`
+ WHOISStats *WHOISStatsInfo `json:"whois_stats,omitempty"`
+
+ // OriginsWithoutKeptRoute counts the prefix and origin AS pairs that peers
+ // carry but whose kept routes were all withdrawn.
+ OriginsWithoutKeptRoute int `json:"origins_without_kept_route"`
}
// databaseStats gets the database stats, giving up when ctx is done; it then
@@ -285,15 +323,6 @@ func (s *Server) handleStatusJSON() http.HandlerFunc {
return
}
- const bitsPerMegabit = 1000000.0
-
- // Get route update metrics
- routeMetrics := s.streamer.GetMetricsTracker().GetRouteMetrics()
-
- // Get memory stats
- var memStats runtime.MemStats
- runtime.ReadMemStats(&memStats)
-
// Get WHOIS stats if fetcher is available
var whoisStats *WHOISStatsInfo
if s.asnFetcher != nil {
@@ -305,44 +334,80 @@ func (s *Server) handleStatusJSON() http.HandlerFunc {
}
}
- stats := Stats{
- Uptime: streamUptime(metrics.ConnectedSince),
- TotalMessages: metrics.TotalMessages,
- TotalBytes: metrics.TotalBytes,
- TotalWireBytes: metrics.TotalWireBytes,
- MessagesPerSec: metrics.MessagesPerSec,
- MbitsPerSec: metrics.BitsPerSec / bitsPerMegabit,
- WireMbitsPerSec: metrics.WireBitsPerSec / bitsPerMegabit,
- Connected: metrics.Connected,
- GoVersion: runtime.Version(),
- Goroutines: runtime.NumGoroutine(),
- MemoryUsage: humanize.Bytes(memStats.Alloc),
- ASNs: dbStats.ASNs,
- Prefixes: dbStats.Prefixes,
- IPv4Prefixes: dbStats.IPv4Prefixes,
- IPv6Prefixes: dbStats.IPv6Prefixes,
- Peerings: dbStats.Peerings,
- Peers: dbStats.Peers,
- DatabaseSizeBytes: dbStats.FileSizeBytes,
- LiveRoutes: dbStats.LiveRoutes,
- IPv4Routes: dbStats.IPv4Routes,
- IPv6Routes: dbStats.IPv6Routes,
- OldestRoute: dbStats.OldestRoute,
- NewestRoute: dbStats.NewestRoute,
- IPv4UpdatesPerSec: routeMetrics.IPv4UpdatesPerSec,
- IPv6UpdatesPerSec: routeMetrics.IPv6UpdatesPerSec,
- IPv4PrefixDistribution: dbStats.IPv4PrefixDistribution,
- IPv6PrefixDistribution: dbStats.IPv6PrefixDistribution,
- WHOISStats: whoisStats,
- }
-
- err = writeJSONSuccess(w, stats)
+ err = writeJSONSuccess(w, s.statusStats(metrics, dbStats, whoisStats))
if err != nil {
s.logger.Error("Failed to encode stats", "error", err)
}
}
}
+// statusStats builds the /status.json answer from the stream metrics, the
+// database stats, the routing table's figures and the WHOIS stats, with the
+// route update rates and memory use as they are now.
+func (s *Server) statusStats(
+ streamMetrics metrics.StreamMetrics,
+ dbStats database.Stats,
+ whoisStats *WHOISStatsInfo,
+) Stats {
+ const bitsPerMegabit = 1000000.0
+
+ routeMetrics := s.streamer.GetMetricsTracker().GetRouteMetrics()
+ counts := s.table.Counts()
+
+ var memStats runtime.MemStats
+ runtime.ReadMemStats(&memStats)
+
+ return Stats{
+ Uptime: streamUptime(streamMetrics.ConnectedSince),
+ TotalMessages: streamMetrics.TotalMessages,
+ TotalBytes: streamMetrics.TotalBytes,
+ TotalWireBytes: streamMetrics.TotalWireBytes,
+ MessagesPerSec: streamMetrics.MessagesPerSec,
+ MbitsPerSec: streamMetrics.BitsPerSec / bitsPerMegabit,
+ WireMbitsPerSec: streamMetrics.WireBitsPerSec / bitsPerMegabit,
+ Connected: streamMetrics.Connected,
+ GoVersion: runtime.Version(),
+ Goroutines: runtime.NumGoroutine(),
+ MemoryUsage: humanize.Bytes(memStats.Alloc),
+ ASNs: dbStats.ASNs,
+ Prefixes: counts.IPv4Prefixes + counts.IPv6Prefixes,
+ IPv4Prefixes: counts.IPv4Prefixes,
+ IPv6Prefixes: counts.IPv6Prefixes,
+ Peerings: dbStats.Peerings,
+ Peers: dbStats.Peers,
+ DatabaseSizeBytes: dbStats.FileSizeBytes,
+ LiveRoutes: counts.IPv4Routes + counts.IPv6Routes,
+ IPv4Routes: counts.IPv4Routes,
+ IPv6Routes: counts.IPv6Routes,
+ OldestRoute: counts.OldestKeptRoute,
+ NewestRoute: counts.NewestKeptRoute,
+ IPv4UpdatesPerSec: routeMetrics.IPv4UpdatesPerSec,
+ IPv6UpdatesPerSec: routeMetrics.IPv6UpdatesPerSec,
+ IPv4PrefixDistribution: prefixDistribution(counts.IPv4PrefixesByLength),
+ IPv6PrefixDistribution: prefixDistribution(counts.IPv6PrefixesByLength),
+ WHOISStats: whoisStats,
+
+ OriginsWithoutKeptRoute: counts.OriginsWithoutKeptRoute,
+ }
+}
+
+// prefixDistribution lists the mask lengths that have at least one prefix, in
+// ascending order, with their prefix counts. countsByLength[n] is the number
+// of prefixes of mask length n.
+func prefixDistribution(countsByLength []int) []PrefixDistribution {
+ var distribution []PrefixDistribution
+
+ for maskLength, count := range countsByLength {
+ if count > 0 {
+ distribution = append(distribution, PrefixDistribution{
+ MaskLength: maskLength, Count: count,
+ })
+ }
+ }
+
+ return distribution
+}
+
// getWHOISStats builds WHOIS statistics from database and fetcher.
func (s *Server) getWHOISStats(ctx context.Context) *WHOISStatsInfo {
// Get database WHOIS stats
@@ -432,49 +497,52 @@ type GCStats struct {
//
//nolint:tagliatelle // field names are the existing JSON API
type StreamStats struct {
- Announcements uint64 `json:"announcements"`
- Withdrawals uint64 `json:"withdrawals"`
- RouteChurnPerSec float64 `json:"route_churn_per_sec"`
- BGPPeerCount int `json:"bgp_peer_count"`
+ Announcements uint64 `json:"announcements"`
+ Withdrawals uint64 `json:"withdrawals"`
+ RouteChurnPerMin int `json:"route_churn_per_min"`
+ BGPPeerCount int `json:"bgp_peer_count"`
}
-// StatsResponse represents the API statistics response
+// StatsResponse represents the API statistics response. The fields it shares
+// with Stats mean what they mean there.
//
//nolint:tagliatelle // field names are the existing JSON API
type StatsResponse struct {
- Uptime string `json:"uptime"`
- TotalMessages uint64 `json:"total_messages"`
- TotalBytes uint64 `json:"total_bytes"`
- TotalWireBytes uint64 `json:"total_wire_bytes"`
- MessagesPerSec float64 `json:"messages_per_sec"`
- MbitsPerSec float64 `json:"mbits_per_sec"`
- WireMbitsPerSec float64 `json:"wire_mbits_per_sec"`
- Connected bool `json:"connected"`
- ConnectionDuration string `json:"connection_duration"`
- ReconnectCount uint64 `json:"reconnect_count"`
- GoVersion string `json:"go_version"`
- Goroutines int `json:"goroutines"`
- MemoryUsage string `json:"memory_usage"`
- GC GCStats `json:"gc"`
- Stream StreamStats `json:"stream"`
- ASNs int `json:"asns"`
- Prefixes int `json:"prefixes"`
- IPv4Prefixes int `json:"ipv4_prefixes"`
- IPv6Prefixes int `json:"ipv6_prefixes"`
- Peerings int `json:"peerings"`
- Peers int `json:"peers"`
- DatabaseSizeBytes int64 `json:"database_size_bytes"`
- LiveRoutes int `json:"live_routes"`
- IPv4Routes int `json:"ipv4_routes"`
- IPv6Routes int `json:"ipv6_routes"`
- OldestRoute *time.Time `json:"oldest_route,omitempty"`
- NewestRoute *time.Time `json:"newest_route,omitempty"`
- IPv4UpdatesPerSec float64 `json:"ipv4_updates_per_sec"`
- IPv6UpdatesPerSec float64 `json:"ipv6_updates_per_sec"`
- HandlerStats []HandlerStatsInfo `json:"handler_stats"`
- IPv4PrefixDistribution []database.PrefixDistribution `json:"ipv4_prefix_distribution"`
- IPv6PrefixDistribution []database.PrefixDistribution `json:"ipv6_prefix_distribution"`
- WHOISStats *WHOISStatsInfo `json:"whois_stats,omitempty"`
+ Uptime string `json:"uptime"`
+ TotalMessages uint64 `json:"total_messages"`
+ TotalBytes uint64 `json:"total_bytes"`
+ TotalWireBytes uint64 `json:"total_wire_bytes"`
+ MessagesPerSec float64 `json:"messages_per_sec"`
+ MbitsPerSec float64 `json:"mbits_per_sec"`
+ WireMbitsPerSec float64 `json:"wire_mbits_per_sec"`
+ Connected bool `json:"connected"`
+ ConnectionDuration string `json:"connection_duration"`
+ ReconnectCount uint64 `json:"reconnect_count"`
+ GoVersion string `json:"go_version"`
+ Goroutines int `json:"goroutines"`
+ MemoryUsage string `json:"memory_usage"`
+ GC GCStats `json:"gc"`
+ Stream StreamStats `json:"stream"`
+ ASNs int `json:"asns"`
+ Prefixes int `json:"prefixes"`
+ IPv4Prefixes int `json:"ipv4_prefixes"`
+ IPv6Prefixes int `json:"ipv6_prefixes"`
+ Peerings int `json:"peerings"`
+ Peers int `json:"peers"`
+ DatabaseSizeBytes int64 `json:"database_size_bytes"`
+ LiveRoutes int `json:"live_routes"`
+ IPv4Routes int `json:"ipv4_routes"`
+ IPv6Routes int `json:"ipv6_routes"`
+ OldestRoute *time.Time `json:"oldest_route,omitempty"`
+ NewestRoute *time.Time `json:"newest_route,omitempty"`
+ IPv4UpdatesPerSec float64 `json:"ipv4_updates_per_sec"`
+ IPv6UpdatesPerSec float64 `json:"ipv6_updates_per_sec"`
+ HandlerStats []HandlerStatsInfo `json:"handler_stats"`
+ IPv4PrefixDistribution []PrefixDistribution `json:"ipv4_prefix_distribution"`
+ IPv6PrefixDistribution []PrefixDistribution `json:"ipv6_prefix_distribution"`
+ WHOISStats *WHOISStatsInfo `json:"whois_stats,omitempty"`
+
+ OriginsWithoutKeptRoute int `json:"origins_without_kept_route"`
}
// handlerStatsInfo converts the streamer's handler statistics for the API
@@ -536,7 +604,7 @@ func (s *Server) streamStats() StreamStats {
return StreamStats{
Announcements: metricsTracker.GetAnnouncementCount(),
Withdrawals: metricsTracker.GetWithdrawalCount(),
- RouteChurnPerSec: metricsTracker.GetChurnRate(),
+ RouteChurnPerMin: metricsTracker.GetChurnPerMinute(),
BGPPeerCount: metricsTracker.GetBGPPeerCount(),
}
}
@@ -594,8 +662,8 @@ func (s *Server) handleStats() http.HandlerFunc {
}
// statsResponse builds the /api/v1/stats answer from the stream metrics, the
-// database stats and the WHOIS stats, with the route update rates, memory use
-// and handler queues as they are now.
+// database stats, the routing table's figures and the WHOIS stats, with the
+// route update rates, memory use and handler queues as they are now.
func (s *Server) statsResponse(
streamMetrics metrics.StreamMetrics,
dbStats database.Stats,
@@ -605,6 +673,7 @@ func (s *Server) statsResponse(
// Get route update metrics
routeMetrics := s.streamer.GetMetricsTracker().GetRouteMetrics()
+ counts := s.table.Counts()
// Get memory stats
var memStats runtime.MemStats
@@ -628,23 +697,25 @@ func (s *Server) statsResponse(
GC: gcStats(&memStats),
Stream: s.streamStats(),
ASNs: dbStats.ASNs,
- Prefixes: dbStats.Prefixes,
- IPv4Prefixes: dbStats.IPv4Prefixes,
- IPv6Prefixes: dbStats.IPv6Prefixes,
+ Prefixes: counts.IPv4Prefixes + counts.IPv6Prefixes,
+ IPv4Prefixes: counts.IPv4Prefixes,
+ IPv6Prefixes: counts.IPv6Prefixes,
Peerings: dbStats.Peerings,
Peers: dbStats.Peers,
DatabaseSizeBytes: dbStats.FileSizeBytes,
- LiveRoutes: dbStats.LiveRoutes,
- IPv4Routes: dbStats.IPv4Routes,
- IPv6Routes: dbStats.IPv6Routes,
- OldestRoute: dbStats.OldestRoute,
- NewestRoute: dbStats.NewestRoute,
+ LiveRoutes: counts.IPv4Routes + counts.IPv6Routes,
+ IPv4Routes: counts.IPv4Routes,
+ IPv6Routes: counts.IPv6Routes,
+ OldestRoute: counts.OldestKeptRoute,
+ NewestRoute: counts.NewestKeptRoute,
IPv4UpdatesPerSec: routeMetrics.IPv4UpdatesPerSec,
IPv6UpdatesPerSec: routeMetrics.IPv6UpdatesPerSec,
HandlerStats: handlerStatsInfo(s.streamer.GetHandlerStats()),
- IPv4PrefixDistribution: dbStats.IPv4PrefixDistribution,
- IPv6PrefixDistribution: dbStats.IPv6PrefixDistribution,
+ IPv4PrefixDistribution: prefixDistribution(counts.IPv4PrefixesByLength),
+ IPv6PrefixDistribution: prefixDistribution(counts.IPv6PrefixesByLength),
WHOISStats: whoisStats,
+
+ OriginsWithoutKeptRoute: counts.OriginsWithoutKeptRoute,
}
}
@@ -671,9 +742,24 @@ func (s *Server) handleIPLookup() http.HandlerFunc {
// IPLookupResponse is the standard response for IP/hostname lookups.
type IPLookupResponse struct {
- Query string `json:"query"`
- Results []*database.IPInfo `json:"results"`
- Errors []string `json:"errors,omitempty"`
+ Query string `json:"query"`
+ Results []IPLookupResult `json:"results"`
+ Errors []string `json:"errors,omitempty"`
+}
+
+// IPLookupResult is the answer for one address: its PTR names, its live route
+// and that route's AS, and the blocklists that list it. IPInfo is nil when
+// the address has no live route, and the answer then has no route or AS
+// fields. IP hides the IPInfo field of the same name, so that an address with
+// no route has it too.
+type IPLookupResult struct {
+ *IPInfo
+
+ IP string `json:"ip"`
+ PTR []string `json:"ptr,omitempty"`
+
+ // Blocklists has one entry for each blocklist the address is on.
+ Blocklists []blocklist.Listing `json:"blocklists"`
}
// handleIPInfo returns a handler that provides comprehensive IP information.
@@ -682,7 +768,7 @@ type IPLookupResponse struct {
// Always returns the same response structure with PTR records for each IP.
func (s *Server) handleIPInfo() http.HandlerFunc {
return func(w http.ResponseWriter, r *http.Request) {
- target := lookupTarget(r)
+ target, ownAddress := lookupTarget(r)
if target == "" {
s.writeJSONError(w, http.StatusBadRequest, "Could not determine IP address")
@@ -690,7 +776,7 @@ func (s *Server) handleIPInfo() http.HandlerFunc {
}
ctx := r.Context()
- response := IPLookupResponse{Query: target, Results: make([]*database.IPInfo, 0)}
+ response := IPLookupResponse{Query: target, Results: make([]IPLookupResult, 0)}
// Collect all IPs to look up
ipsToLookup, resolveErrors, err := s.resolveLookupTargets(ctx, target)
@@ -719,7 +805,7 @@ func (s *Server) handleIPInfo() http.HandlerFunc {
// Look up each IP
for _, ip := range ipsToLookup {
- ipInfo, err := s.db.GetIPInfoContext(ctx, ip)
+ ipInfo, err := s.ipInfo(ctx, ip)
if deadlinePassed(ctx) {
s.writeJSONError(w, http.StatusGatewayTimeout, requestTimeoutMessage)
@@ -727,7 +813,10 @@ func (s *Server) handleIPInfo() http.HandlerFunc {
}
switch {
- case errors.Is(err, database.ErrNoRoute), errors.Is(err, database.ErrInvalidIP):
+ case errors.Is(err, ErrNoRoute):
+ // The address still gets a result, without route fields.
+ ipInfo = nil
+ case errors.Is(err, ErrInvalidIP):
response.Errors = append(response.Errors, ip+": "+err.Error())
continue
@@ -746,13 +835,14 @@ func (s *Server) handleIPInfo() http.HandlerFunc {
return
}
- if len(ptrs) > 0 {
- ipInfo.PTR = ptrs
- }
+ response.Results = append(response.Results, IPLookupResult{
+ IP: ip,
+ PTR: ptrs,
+ IPInfo: ipInfo,
+ Blocklists: s.blocklistListings(ip),
+ })
- response.Results = append(response.Results, ipInfo)
-
- if ipInfo.NeedsWHOISRefresh {
+ if ipInfo != nil && ipInfo.NeedsWHOISRefresh {
refreshASNs[ipInfo.ASN] = true
}
}
@@ -760,10 +850,115 @@ func (s *Server) handleIPInfo() http.HandlerFunc {
// Queue WHOIS refresh for stale ASNs (non-blocking)
s.queueWHOISRefresh(refreshASNs)
- s.writeIPLookupAnswer(w, response)
+ s.writeIPLookupAnswer(w, response, ownAddress)
}
}
+// ipInfo returns the IP lookup's answer for ip. The routing table gives the
+// most specific prefix that holds the address, the number of peers that carry
+// it, the origin AS that most of them carry, and the time of its newest kept
+// route. The database gives the prefix's first-seen time and the AS's WHOIS
+// details. It returns ErrNoRoute when no prefix holds the address, and
+// ErrInvalidIP when ip does not parse.
+func (s *Server) ipInfo(ctx context.Context, ip string) (*IPInfo, error) {
+ addr, err := netip.ParseAddr(ip)
+ if err != nil {
+ return nil, fmt.Errorf("%w: %s", ErrInvalidIP, ip)
+ }
+
+ summary, found := s.table.Lookup(addr)
+ if !found {
+ return nil, fmt.Errorf("%w for IP %s", ErrNoRoute, ip)
+ }
+
+ firstSeen, err := s.db.GetPrefixFirstSeen(ctx, summary.Prefix)
+ if err != nil {
+ return nil, err
+ }
+
+ asInfo, err := s.db.GetASWHOIS(ctx, summary.OriginASN)
+ if err != nil {
+ return nil, err
+ }
+
+ info := &IPInfo{
+ IP: ip,
+ Netblock: summary.Prefix.String(),
+ MaskLength: summary.Prefix.Bits(),
+ IPVersion: ipVersionOf(summary.Prefix),
+ NumPeers: summary.Peers,
+ ASN: summary.OriginASN,
+ ASName: asInfo.ASName,
+ Handle: asInfo.Handle,
+ Description: asInfo.Description,
+ OrgName: asInfo.OrgName,
+ OrgID: asInfo.OrgID,
+ Address: asInfo.Address,
+ CountryCode: asInfo.CountryCode,
+ AbuseEmail: asInfo.AbuseEmail,
+ RIR: asInfo.RIR,
+ FirstSeen: firstSeen,
+ NeedsWHOISRefresh: needsWHOISRefresh(asInfo),
+ }
+
+ if summary.Newest != nil {
+ info.LastSeen = *summary.Newest
+ }
+
+ return info, nil
+}
+
+// needsWHOISRefresh says whether the WHOIS details of asInfo were never
+// fetched or are older than whoisStaleThreshold. AS 0, no known origin, has
+// none to fetch.
+func needsWHOISRefresh(asInfo *database.ASN) bool {
+ if asInfo.ASN == ristypes.NoOrigin {
+ return false
+ }
+
+ return asInfo.WHOISUpdatedAt == nil ||
+ time.Since(*asInfo.WHOISUpdatedAt) > whoisStaleThreshold
+}
+
+// ipVersionOf returns 4 or 6, the IP version of prefix.
+func ipVersionOf(prefix netip.Prefix) int {
+ if prefix.Addr().Is4() {
+ return ipVersionV4
+ }
+
+ return ipVersionV6
+}
+
+// blocklistListings returns the blocklists that list ip. It returns an empty
+// list, never nil, so that an answer's blocklists is [] rather than null.
+func (s *Server) blocklistListings(ip string) []blocklist.Listing {
+ addr, err := netip.ParseAddr(ip)
+ if s.blocklists == nil || err != nil {
+ return []blocklist.Listing{}
+ }
+
+ return s.blocklists.Lookup(addr)
+}
+
+// setIPLookupCacheControl lets caches keep an IP lookup answer for the
+// shortest blocklist interval, the soonest a blocklist copy can change. The
+// answer about the caller's own address is private, so that a shared cache
+// does not give it to another caller. Without blocklists it sets nothing.
+func (s *Server) setIPLookupCacheControl(w http.ResponseWriter, ownAddress bool) {
+ if s.blocklists == nil {
+ return
+ }
+
+ visibility := cacheControlPublic
+ if ownAddress {
+ visibility = cacheControlPrivate
+ }
+
+ maxAgeSeconds := int(s.blocklists.ShortestInterval().Seconds())
+ w.Header().Set(cacheControlHeader,
+ visibility+cacheControlMaxAge+strconv.Itoa(maxAgeSeconds))
+}
+
// noTargetsMessage returns the message of the 400 an IP lookup answers when
// no address is left to look up, followed by the first of resolveErrors, which
// says why, when there is one.
@@ -777,16 +972,12 @@ func noTargetsMessage(resolveErrors []string) string {
return message + ": " + resolveErrors[0]
}
-// writeIPLookupAnswer sends response, or 404 with its first error when no
-// address had a route.
+// writeIPLookupAnswer sends response with its Cache-Control header.
+// ownAddress says the request asked about the caller's own address.
func (s *Server) writeIPLookupAnswer(
- w http.ResponseWriter, response IPLookupResponse,
+ w http.ResponseWriter, response IPLookupResponse, ownAddress bool,
) {
- if len(response.Results) == 0 && len(response.Errors) > 0 {
- s.writeJSONError(w, http.StatusNotFound, "No routes found: "+response.Errors[0])
-
- return
- }
+ s.setIPLookupCacheControl(w, ownAddress)
err := writeJSONSuccess(w, response)
if err != nil {
@@ -795,8 +986,9 @@ func (s *Server) writeIPLookupAnswer(
}
// lookupTarget returns what an IP lookup request asks for: the ip or addr URL
-// parameter, or else the client's own address.
-func lookupTarget(r *http.Request) string {
+// parameter, or else the client's own address. The bool is true for the
+// client's own address.
+func lookupTarget(r *http.Request) (string, bool) {
target := chi.URLParam(r, "ip")
if target == "" {
target = chi.URLParam(r, "addr")
@@ -804,10 +996,10 @@ func lookupTarget(r *http.Request) string {
if target == "" {
// Use client IP (RealIP middleware has already processed this)
- target = extractClientIP(r)
+ return extractClientIP(r), true
}
- return target
+ return target, false
}
// queueWHOISRefresh asks the WHOIS fetcher, when there is one, to refresh
@@ -965,7 +1157,7 @@ func (s *Server) handleASDetailJSON() http.HandlerFunc {
return
}
- asInfo, prefixes, err := s.db.GetASDetailsContext(r.Context(), asn)
+ asInfo, prefixes, err := s.asDetails(r.Context(), asn)
if deadlinePassed(r.Context()) {
s.writeJSONError(w, http.StatusGatewayTimeout, requestTimeoutMessage)
@@ -973,7 +1165,7 @@ func (s *Server) handleASDetailJSON() http.HandlerFunc {
}
if err != nil {
- if errors.Is(err, database.ErrNoRoute) {
+ if errors.Is(err, ErrNoRoute) {
s.writeJSONError(w, http.StatusNotFound, err.Error())
} else {
s.logger.Error("Failed to get AS details", "asn", asn, "error", err)
@@ -1001,11 +1193,55 @@ func (s *Server) handleASDetailJSON() http.HandlerFunc {
}
}
-// splitRoutesByIPVersion groups routes into the IPv4 and the IPv6 ones.
+// asDetails returns the AS's handle, description and seen times, from the
+// database, and the prefixes it originates, from the routing table. Each
+// prefix has the time of its newest kept route, if it keeps one. It returns
+// ErrNoRoute for AS 0, which means no known origin. It also returns ErrNoRoute
+// for an AS that the database, the AS list built into RouteWatch and the
+// routing table all lack.
+func (s *Server) asDetails(
+ ctx context.Context, asn int,
+) (*database.ASN, []LiveRoute, error) {
+ if asn == ristypes.NoOrigin {
+ return nil, nil, fmt.Errorf("%w: AS%d", ErrNoRoute, asn)
+ }
+
+ asInfo, known, err := s.db.GetAS(ctx, asn)
+ if err != nil {
+ return nil, nil, err
+ }
+
+ originated := s.table.OriginatedPrefixes(asn)
+ if !known && len(originated) == 0 {
+ return nil, nil, fmt.Errorf("%w: AS%d", ErrNoRoute, asn)
+ }
+
+ prefixes := make([]LiveRoute, 0, len(originated))
+
+ for _, p := range originated {
+ route := LiveRoute{
+ Prefix: p.Prefix.String(),
+ MaskLength: p.Prefix.Bits(),
+ IPVersion: ipVersionOf(p.Prefix),
+ OriginASN: asn,
+ }
+
+ if p.Newest != nil {
+ route.LastUpdated = *p.Newest
+ }
+
+ prefixes = append(prefixes, route)
+ }
+
+ return asInfo, prefixes, nil
+}
+
+// splitRoutesByIPVersion groups routes into the IPv4 and the IPv6 ones. Neither
+// group is nil, so that a JSON answer holds [] for an empty one and not null.
func splitRoutesByIPVersion(
- routes []database.LiveRoute,
-) ([]database.LiveRoute, []database.LiveRoute) {
- var ipv4Routes, ipv6Routes []database.LiveRoute
+ routes []LiveRoute,
+) ([]LiveRoute, []LiveRoute) {
+ ipv4Routes, ipv6Routes := []LiveRoute{}, []LiveRoute{}
for _, route := range routes {
if route.IPVersion == ipVersionV4 {
@@ -1018,24 +1254,15 @@ func splitRoutesByIPVersion(
return ipv4Routes, ipv6Routes
}
-// sortRoutesByNetwork sorts routes by network address, then by mask length.
-func sortRoutesByNetwork(routes []database.LiveRoute) {
- sort.Slice(routes, func(i, j int) bool {
- // Parse the prefixes to compare network addresses
- ipI, netI, _ := net.ParseCIDR(routes[i].Prefix)
- ipJ, netJ, _ := net.ParseCIDR(routes[j].Prefix)
+// sortRoutesByMaskLength sorts routes by mask length, shortest first, and
+// routes with the same mask length by network address.
+func sortRoutesByMaskLength(routes []LiveRoute) {
+ slices.SortFunc(routes, func(a, b LiveRoute) int {
+ prefixA, _ := netip.ParsePrefix(a.Prefix)
+ prefixB, _ := netip.ParsePrefix(b.Prefix)
- // Compare by network address first
- cmp := bytes.Compare(ipI.To16(), ipJ.To16())
- if cmp != 0 {
- return cmp < 0
- }
-
- // If network addresses are equal, compare by mask length
- onesI, _ := netI.Mask.Size()
- onesJ, _ := netJ.Mask.Size()
-
- return onesI < onesJ
+ return cmp.Or(cmp.Compare(prefixA.Bits(), prefixB.Bits()),
+ prefixA.Addr().Compare(prefixB.Addr()))
})
}
@@ -1053,8 +1280,7 @@ func (s *Server) handlePrefixDetailJSON() http.HandlerFunc {
return
}
- // Look the prefix up in the text form net/netip prints, the form the
- // tables store, so that any valid way of writing it finds the route.
+ // Masking the prefix lets any valid way of writing it find its routes.
parsed, err := netip.ParsePrefix(prefixParam + "/" + lenParam)
if err != nil {
s.writeJSONError(w, http.StatusBadRequest, invalidPrefixMessage)
@@ -1062,30 +1288,21 @@ func (s *Server) handlePrefixDetailJSON() http.HandlerFunc {
return
}
- prefix := parsed.Masked().String()
+ parsed = parsed.Masked()
+ prefix := parsed.String()
- routes, _, err := s.db.GetPrefixDetailsContext(r.Context(), prefix)
- if deadlinePassed(r.Context()) {
- s.writeJSONError(w, http.StatusGatewayTimeout, requestTimeoutMessage)
+ origins := s.table.Origins(parsed)
+ if len(origins) == 0 {
+ s.writeJSONError(w, http.StatusNotFound,
+ ErrNoRoute.Error()+": "+prefix)
return
}
- if err != nil {
- if errors.Is(err, database.ErrNoRoute) {
- s.writeJSONError(w, http.StatusNotFound, err.Error())
- } else {
- s.logger.Error("Failed to get prefix details",
- "prefix", prefix, "error", err)
- s.writeJSONError(w, http.StatusInternalServerError,
- internalErrorMessage)
- }
-
- return
- }
+ routes := prefixRoutes(parsed, origins)
// Group by origin AS
- originMap := make(map[int][]database.LiveRoute)
+ originMap := make(map[int][]LiveRoute)
for _, route := range routes {
originMap[route.OriginASN] = append(originMap[route.OriginASN], route)
}
@@ -1105,6 +1322,56 @@ func (s *Server) handlePrefixDetailJSON() http.HandlerFunc {
}
}
+// ASSearchResponse is the answer to an AS name search: the text searched for,
+// the number of ASes that match it, and the first maxASSearchResults of them in
+// AS number order.
+type ASSearchResponse struct {
+ Query string `json:"query"`
+ Total int `json:"total"`
+ Results []asinfo.Info `json:"results"`
+}
+
+// handleASSearch returns a handler that searches the AS list built into
+// asinfo for the text in the q parameter, without its leading and trailing
+// spaces. It answers 400 when no text is left.
+func (s *Server) handleASSearch() http.HandlerFunc {
+ return func(w http.ResponseWriter, r *http.Request) {
+ query := strings.TrimSpace(r.URL.Query().Get(asSearchQueryParam))
+ if query == "" {
+ s.writeJSONError(w, http.StatusBadRequest, searchQueryMissingMessage)
+
+ return
+ }
+
+ // AS 0 means no known origin and has no AS page, so the answer leaves
+ // it out. The matches come in AS number order, so AS 0 can only be the
+ // first. The search asks for one match more than the answer holds, to
+ // take its place.
+ results, total := asinfo.Search(query, maxASSearchResults+1)
+ if len(results) > 0 && results[0].ASN == ristypes.NoOrigin {
+ results = results[1:]
+ total--
+ }
+
+ results = results[:min(len(results), maxASSearchResults)]
+
+ // results is an empty list rather than nil when nothing matches, so
+ // that the answer holds [] and not null.
+ if results == nil {
+ results = []asinfo.Info{}
+ }
+
+ err := writeJSONSuccess(w, ASSearchResponse{
+ Query: query,
+ Total: total,
+ Results: results,
+ })
+ if err != nil {
+ s.logger.Error("Failed to encode AS search results", "error", err)
+ }
+ }
+}
+
// handleASDetail returns a handler that serves the AS detail HTML page
func (s *Server) handleASDetail() http.HandlerFunc {
return func(w http.ResponseWriter, r *http.Request) {
@@ -1117,7 +1384,7 @@ func (s *Server) handleASDetail() http.HandlerFunc {
return
}
- asInfo, prefixes, err := s.db.GetASDetailsContext(r.Context(), asn)
+ asInfo, prefixes, err := s.asDetails(r.Context(), asn)
if deadlinePassed(r.Context()) {
http.Error(w, requestTimeoutMessage, http.StatusGatewayTimeout)
@@ -1125,7 +1392,7 @@ func (s *Server) handleASDetail() http.HandlerFunc {
}
if err != nil {
- if errors.Is(err, database.ErrNoRoute) {
+ if errors.Is(err, ErrNoRoute) {
http.Error(w, "AS not found", http.StatusNotFound)
} else {
s.logger.Error("Failed to get AS details", "error", err)
@@ -1152,15 +1419,14 @@ func (s *Server) handleASDetail() http.HandlerFunc {
// Group prefixes by IP version
ipv4Prefixes, ipv6Prefixes := splitRoutesByIPVersion(prefixes)
- // Sort prefixes by network address
- sortRoutesByNetwork(ipv4Prefixes)
- sortRoutesByNetwork(ipv6Prefixes)
+ sortRoutesByMaskLength(ipv4Prefixes)
+ sortRoutesByMaskLength(ipv6Prefixes)
// Prepare template data
data := struct {
ASN *database.ASN
- IPv4Prefixes []database.LiveRoute
- IPv6Prefixes []database.LiveRoute
+ IPv4Prefixes []LiveRoute
+ IPv6Prefixes []LiveRoute
TotalCount int
IPv4Count int
IPv6Count int
@@ -1208,8 +1474,7 @@ func (s *Server) handlePrefixDetail() http.HandlerFunc {
return
}
- // Look the prefix up in the text form net/netip prints, the form the
- // tables store, so that any valid way of writing it finds the route.
+ // Masking the prefix lets any valid way of writing it find its routes.
parsed, err := netip.ParsePrefix(prefixParam + "/" + lenParam)
if err != nil {
http.Error(w, invalidPrefixMessage, http.StatusBadRequest)
@@ -1217,9 +1482,16 @@ func (s *Server) handlePrefixDetail() http.HandlerFunc {
return
}
- prefix := parsed.Masked().String()
+ parsed = parsed.Masked()
- routes, originNames, err := s.db.GetPrefixDetailsContext(r.Context(), prefix)
+ origins := s.table.Origins(parsed)
+ if len(origins) == 0 {
+ http.Error(w, "Prefix not found", http.StatusNotFound)
+
+ return
+ }
+
+ originInfos, err := s.originASNs(r.Context(), origins)
if deadlinePassed(r.Context()) {
http.Error(w, requestTimeoutMessage, http.StatusGatewayTimeout)
@@ -1227,27 +1499,13 @@ func (s *Server) handlePrefixDetail() http.HandlerFunc {
}
if err != nil {
- if errors.Is(err, database.ErrNoRoute) {
- http.Error(w, "Prefix not found", http.StatusNotFound)
- } else {
- s.logger.Error("Failed to get prefix details", "error", err)
- http.Error(w, internalErrorMessage, http.StatusInternalServerError)
- }
+ s.logger.Error("Failed to get the origin ASes' names", "error", err)
+ http.Error(w, internalErrorMessage, http.StatusInternalServerError)
return
}
- // Group by origin AS and collect unique AS info
- origins := originASNs(routes, originNames)
-
- // Get the first route to extract some common info
- var maskLength, ipVersion int
- if len(routes) > 0 {
- maskLength, ipVersion = prefixMaskLengthAndIPVersion(prefix)
- }
-
- // Create enhanced routes with AS path handles
- enhancedRoutes := routesWithASPathHandles(routes)
+ routes := prefixRoutes(parsed, origins)
// Prepare template data
data := struct {
@@ -1260,13 +1518,13 @@ func (s *Server) handlePrefixDetail() http.HandlerFunc {
OriginCount int
CSPNonce string
}{
- Prefix: prefix,
- MaskLength: maskLength,
- IPVersion: ipVersion,
- Routes: enhancedRoutes,
- Origins: origins,
+ Prefix: parsed.String(),
+ MaskLength: parsed.Bits(),
+ IPVersion: ipVersionOf(parsed),
+ Routes: routesWithASPathHandles(routes),
+ Origins: originInfos,
PeerCount: len(routes),
- OriginCount: len(origins),
+ OriginCount: len(originInfos),
CSPNonce: cspNonce(r.Context()),
}
@@ -1296,34 +1554,72 @@ type ASNInfo struct {
PeerCount int
}
-// originASNs groups routes by origin AS, with each AS's handle and description
-// from names, as the prefix query returned them, and the number of routes it
-// originates. The origin ASes come in no particular order.
-func originASNs(
- routes []database.LiveRoute,
- names map[int]database.ASN,
-) []*ASNInfo {
- originMap := make(map[int]*ASNInfo)
+// originASNs returns each of origins, a prefix's origin ASes, with its handle
+// and description from the database and the number of peers that carry the
+// prefix with it.
+func (s *Server) originASNs(
+ ctx context.Context, origins []routetable.Origin,
+) ([]*ASNInfo, error) {
+ infos := make([]*ASNInfo, 0, len(origins))
- for _, route := range routes {
- if _, exists := originMap[route.OriginASN]; !exists {
- originMap[route.OriginASN] = &ASNInfo{
- ASN: route.OriginASN,
- Handle: names[route.OriginASN].Handle,
- Description: names[route.OriginASN].Description,
- PeerCount: 0,
- }
+ for _, origin := range origins {
+ asInfo, _, err := s.db.GetAS(ctx, origin.ASN)
+ if err != nil {
+ return nil, err
}
- originMap[route.OriginASN].PeerCount++
+ infos = append(infos, &ASNInfo{
+ ASN: origin.ASN,
+ Handle: asInfo.Handle,
+ Description: asInfo.Description,
+ PeerCount: len(origin.Peers),
+ })
}
- origins := make([]*ASNInfo, 0, len(originMap))
- for _, origin := range originMap {
- origins = append(origins, origin)
+ return infos, nil
+}
+
+// prefixRoutes returns one route for each peer that carries prefix, by origin
+// AS in the order of origins. Each origin's kept routes come first, newest
+// first, with their AS path, next hop and time. Its other peers follow without
+// them, since the routing table keeps only their addresses.
+func prefixRoutes(
+ prefix netip.Prefix, origins []routetable.Origin,
+) []LiveRoute {
+ var routes []LiveRoute
+
+ for _, origin := range origins {
+ base := LiveRoute{
+ Prefix: prefix.String(),
+ MaskLength: prefix.Bits(),
+ IPVersion: ipVersionOf(prefix),
+ OriginASN: origin.ASN,
+ }
+
+ kept := make(map[string]bool, len(origin.Routes))
+
+ for _, r := range origin.Routes {
+ kept[r.Peer] = true
+
+ route := base
+ route.PeerIP = r.Peer
+ route.ASPath = r.ASPath
+ // The next hop is the peer, as the prefix handler records it.
+ route.NextHop = r.Peer
+ route.LastUpdated = r.Time
+ routes = append(routes, route)
+ }
+
+ for _, peer := range origin.Peers {
+ if !kept[peer] {
+ route := base
+ route.PeerIP = peer
+ routes = append(routes, route)
+ }
+ }
}
- return origins
+ return routes
}
// ASPathEntry is one AS of a route's AS path, with its handle.
@@ -1334,14 +1630,14 @@ type ASPathEntry struct {
// EnhancedRoute is a live route with the handle of each AS in its path.
type EnhancedRoute struct {
- database.LiveRoute
+ LiveRoute
ASPathWithHandle []ASPathEntry
}
// routesWithASPathHandles looks up the handle of each AS in the path of each
// route.
-func routesWithASPathHandles(routes []database.LiveRoute) []EnhancedRoute {
+func routesWithASPathHandles(routes []LiveRoute) []EnhancedRoute {
enhancedRoutes := make([]EnhancedRoute, len(routes))
for i, route := range routes {
@@ -1365,23 +1661,6 @@ func routesWithASPathHandles(routes []database.LiveRoute) []EnhancedRoute {
return enhancedRoutes
}
-// prefixMaskLengthAndIPVersion returns the mask length and IP version of a
-// prefix in CIDR notation, or zeros if it does not parse.
-func prefixMaskLengthAndIPVersion(prefix string) (int, int) {
- _, ipNet, err := net.ParseCIDR(prefix)
- if err != nil {
- return 0, 0
- }
-
- maskLength, _ := ipNet.Mask.Size()
-
- if ipNet.IP.To4() != nil {
- return maskLength, ipVersionV4
- }
-
- return maskLength, ipVersionV6
-}
-
// handlePrefixLength shows a random sample of IPv4 prefixes with the
// specified mask length
func (s *Server) handlePrefixLength() http.HandlerFunc {
@@ -1420,29 +1699,11 @@ func (s *Server) prefixLengthHandler(ipVersion, maxMaskLength int) http.HandlerF
// Get random sample of prefixes
const maxPrefixes = 500
- prefixes, err := s.db.GetRandomPrefixesByLengthContext(
- r.Context(), maskLength, ipVersion, maxPrefixes)
- if deadlinePassed(r.Context()) {
- http.Error(w, requestTimeoutMessage, http.StatusGatewayTimeout)
-
- return
- }
-
- if err != nil {
- s.logger.Error("Failed to get prefixes by length", "error", err)
- http.Error(w, internalErrorMessage, http.StatusInternalServerError)
-
- return
- }
+ prefixes := s.table.SamplePrefixes(ipVersion, maskLength, maxPrefixes)
// Sort prefixes for display
- sort.Slice(prefixes, func(i, j int) bool {
- // First compare by IP version
- if prefixes[i].IPVersion != prefixes[j].IPVersion {
- return prefixes[i].IPVersion < prefixes[j].IPVersion
- }
- // Then by prefix
- return prefixes[i].Prefix < prefixes[j].Prefix
+ slices.SortFunc(prefixes, func(a, b routetable.PrefixSummary) int {
+ return a.Prefix.Compare(b.Prefix)
})
// Render template
@@ -1472,23 +1733,33 @@ func (s *Server) prefixLengthHandler(ipVersion, maxMaskLength int) http.HandlerF
}
}
-// EnhancedPrefix is a live route on the prefix length page, with the
-// description of its origin AS and its age.
+// EnhancedPrefix is a prefix on the prefix length page, with the origin AS
+// that most peers carry it with, that AS's description, and its age.
type EnhancedPrefix struct {
- database.LiveRoute
+ LiveRoute
OriginASDescription string
Age string
}
// prefixesWithASDescriptions looks up the description of the origin AS of
-// each prefix and gives each one's age.
-func prefixesWithASDescriptions(prefixes []database.LiveRoute) []EnhancedPrefix {
+// each prefix. It gives each prefix the age of its newest kept route, or no
+// age when it keeps none.
+func prefixesWithASDescriptions(
+ prefixes []routetable.PrefixSummary,
+) []EnhancedPrefix {
enhancedPrefixes := make([]EnhancedPrefix, len(prefixes))
for i, prefix := range prefixes {
enhancedPrefixes[i] = EnhancedPrefix{
- LiveRoute: prefix,
- Age: formatAge(prefix.LastUpdated),
+ Prefix: prefix.Prefix.String(),
+ MaskLength: prefix.Prefix.Bits(),
+ IPVersion: ipVersionOf(prefix.Prefix),
+ OriginASN: prefix.OriginASN,
+ }
+
+ if prefix.Newest != nil {
+ enhancedPrefixes[i].LastUpdated = *prefix.Newest
+ enhancedPrefixes[i].Age = formatAge(*prefix.Newest)
}
// Get AS description
diff --git a/internal/server/handlers_internal_test.go b/internal/server/handlers_internal_test.go
index a734953..f4dd8f7 100644
--- a/internal/server/handlers_internal_test.go
+++ b/internal/server/handlers_internal_test.go
@@ -5,8 +5,11 @@ import (
"encoding/json"
"errors"
"html"
+ "net"
"net/http"
"net/http/httptest"
+ "net/netip"
+ "reflect"
"runtime"
"slices"
"strconv"
@@ -15,10 +18,13 @@ import (
"testing"
"time"
+ "git.eeqj.de/sneak/routewatch/internal/blocklist"
"git.eeqj.de/sneak/routewatch/internal/config"
"git.eeqj.de/sneak/routewatch/internal/database"
"git.eeqj.de/sneak/routewatch/internal/logger"
"git.eeqj.de/sneak/routewatch/internal/metrics"
+ "git.eeqj.de/sneak/routewatch/internal/ristypes"
+ "git.eeqj.de/sneak/routewatch/internal/routetable"
"git.eeqj.de/sneak/routewatch/internal/streamer"
"git.eeqj.de/sneak/routewatch/pkg/asinfo"
)
@@ -26,10 +32,43 @@ import (
// The prefix and peer addresses the prefix page tests store routes for.
const (
testPrefixV4 = "192.0.2.0/24"
+ testPrefixV6 = "2001:db8::/32"
testPeerV4 = "198.51.100.1"
testPeerV6 = "2001:db8::1"
)
+// testPeer returns the address of the nth of the IPv4 peers that tests
+// announce routes from: 198.51.100.n.
+func testPeer(n int) string {
+ return "198.51.100." + strconv.Itoa(n)
+}
+
+// testRouteTime is the time the tests' routes are announced at, or a number of
+// minutes after it.
+func testRouteTime(minutes int) time.Time {
+ return time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC).
+ Add(time.Duration(minutes) * time.Minute)
+}
+
+// announce has peer announce prefix in table at time at, with an AS path
+// that holds only the origin AS asn.
+func announce(
+ table *routetable.Table, prefix string, asn int, peer string, at time.Time,
+) {
+ table.Announce(netip.MustParsePrefix(prefix), asn,
+ routetable.Route{Peer: peer, ASPath: []int{asn}, Time: at})
+}
+
+// withdraw has peer withdraw prefix in table.
+func withdraw(table *routetable.Table, prefix, peer string) {
+ table.Withdraw(netip.MustParsePrefix(prefix), peer)
+}
+
+// emptyTable returns a routing table that holds no route.
+func emptyTable() *routetable.Table {
+ return routetable.New(routetable.Params{})
+}
+
// Paths several tests request.
const (
asPagePath = "/as/64500"
@@ -71,7 +110,8 @@ func (d blockingStatsDB) GetStatsContext(_ context.Context) (database.Stats, err
func TestStatsHandlersDoNotLeakOnTimeout(t *testing.T) {
release := make(chan struct{})
db := blockingStatsDB{release: release}
- s := New(db, streamer.New(logger.New(), metrics.New()), logger.New(), &config.Config{})
+ s := New(db, emptyTable(), streamer.New(logger.New(), metrics.New()),
+ logger.New(), &config.Config{})
handlers := map[string]http.HandlerFunc{
statusJSONHandlerName: s.handleStatusJSON(),
@@ -103,51 +143,78 @@ func TestStatsHandlersDoNotLeakOnTimeout(t *testing.T) {
}
}
+// tableStats holds the figures of a stats answer that come from the routing
+// table.
+//
+//nolint:tagliatelle // field names are the existing JSON API
+type tableStats struct {
+ Prefixes int `json:"prefixes"`
+ IPv4Prefixes int `json:"ipv4_prefixes"`
+ IPv6Prefixes int `json:"ipv6_prefixes"`
+ LiveRoutes int `json:"live_routes"`
+ IPv4Routes int `json:"ipv4_routes"`
+ IPv6Routes int `json:"ipv6_routes"`
+ OriginsWithoutKeptRoute int `json:"origins_without_kept_route"`
+ OldestRoute time.Time `json:"oldest_route"`
+ NewestRoute time.Time `json:"newest_route"`
+
+ IPv4Distribution []PrefixDistribution `json:"ipv4_prefix_distribution"`
+ IPv6Distribution []PrefixDistribution `json:"ipv6_prefix_distribution"`
+}
+
// TestStatsHandlersAnswerFromMemory checks that both stats handlers answer 200
-// with the live route counts and the prefix distribution while the database is
-// closed, so that any query would fail: the request path reads them from
-// memory. The prefix distribution query it used to run read every live route
-// and, on a large database, took the whole 4-second deadline, so
+// with the routing table's figures while the database is closed, so that any
+// query would fail. The prefix distribution query they used to run read every
+// live route and, on a large database, took the whole 4-second deadline, so
// /api/v1/stats answered 500 (https://git.eeqj.de/sneak/routewatch/issues/30).
-// The oldest and newest route times still come from one-row lookups at the ends
-// of an index; with the database closed they are left out of the answer.
+//
+// The table holds two IPv4 prefixes and an IPv6 one. Of the three peers that
+// carried 203.0.113.0/24, the two with kept routes withdrew it, so its origin
+// keeps no route. The oldest and newest routes are those of the other two
+// prefixes.
func TestStatsHandlersAnswerFromMemory(t *testing.T) {
t.Parallel()
+ const withdrawnPrefix = "203.0.113.0/24"
+
+ table := emptyTable()
+ announce(table, testPrefixV4, 64502, testPeer(1), testRouteTime(0))
+
+ for n := 1; n <= 3; n++ {
+ announce(table, withdrawnPrefix, 64500, testPeer(n), testRouteTime(n))
+ }
+
+ withdraw(table, withdrawnPrefix, testPeer(2))
+ withdraw(table, withdrawnPrefix, testPeer(3))
+ announce(table, testPrefixV6, 64501, testPeerV6, testRouteTime(4))
+
db, err := database.New(&config.Config{StateDir: t.TempDir()}, logger.New())
if err != nil {
t.Fatalf("database.New: %v", err)
}
- ts := time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC)
-
- err = db.UpsertLiveRouteBatch([]*database.LiveRoute{
- {
- Prefix: "198.51.100.0/24", MaskLength: 24, IPVersion: 4,
- OriginASN: 64500, PeerIP: "192.0.2.1", ASPath: []int{64500},
- NextHop: "192.0.2.1", LastUpdated: ts,
- },
- {
- Prefix: "2001:db8::/32", MaskLength: 32, IPVersion: 6,
- OriginASN: 64501, PeerIP: testPeerV6, ASPath: []int{64501},
- NextHop: testPeerV6, LastUpdated: ts,
- },
- })
- if err != nil {
- t.Fatalf("UpsertLiveRouteBatch: %v", err)
- }
-
err = db.Close()
if err != nil {
t.Fatalf("Close: %v", err)
}
- s := New(db, streamer.New(logger.New(), metrics.New()), logger.New(), &config.Config{})
+ s := New(db, table, streamer.New(logger.New(), metrics.New()), logger.New(),
+ &config.Config{})
handlers := map[string]http.HandlerFunc{
statusJSONHandlerName: s.handleStatusJSON(),
statsHandlerName: s.handleStats(),
}
+ want := tableStats{
+ Prefixes: 3, IPv4Prefixes: 2, IPv6Prefixes: 1,
+ LiveRoutes: 3, IPv4Routes: 2, IPv6Routes: 1,
+ OriginsWithoutKeptRoute: 1,
+ OldestRoute: testRouteTime(0),
+ NewestRoute: testRouteTime(4),
+ IPv4Distribution: []PrefixDistribution{{MaskLength: 24, Count: 2}},
+ IPv6Distribution: []PrefixDistribution{{MaskLength: 32, Count: 1}},
+ }
+
for name, handler := range handlers {
rec := httptest.NewRecorder()
handler(rec, httptest.NewRequestWithContext(t.Context(), http.MethodGet, "/", nil))
@@ -159,14 +226,8 @@ func TestStatsHandlersAnswerFromMemory(t *testing.T) {
continue
}
- //nolint:tagliatelle // field names are the existing JSON API
var body struct {
- Data struct {
- IPv4Routes int `json:"ipv4_routes"`
- IPv6Routes int `json:"ipv6_routes"`
- IPv4Distribution []database.PrefixDistribution `json:"ipv4_prefix_distribution"`
- IPv6Distribution []database.PrefixDistribution `json:"ipv6_prefix_distribution"`
- } `json:"data"`
+ Data tableStats `json:"data"`
}
err := json.Unmarshal(rec.Body.Bytes(), &body)
@@ -174,30 +235,88 @@ func TestStatsHandlersAnswerFromMemory(t *testing.T) {
t.Fatalf("%s: decoding the answer: %v", name, err)
}
- if body.Data.IPv4Routes != 1 || body.Data.IPv6Routes != 1 {
- t.Errorf("%s: routes = (v4 %d, v6 %d), want (1, 1)",
- name, body.Data.IPv4Routes, body.Data.IPv6Routes)
+ got := body.Data
+ if !got.OldestRoute.Equal(want.OldestRoute) ||
+ !got.NewestRoute.Equal(want.NewestRoute) {
+ t.Errorf("%s: routes from %v to %v, want from %v to %v", name,
+ got.OldestRoute, got.NewestRoute, want.OldestRoute, want.NewestRoute)
}
- wantV4 := []database.PrefixDistribution{{MaskLength: 24, Count: 1}}
- if !slices.Equal(body.Data.IPv4Distribution, wantV4) {
- t.Errorf("%s: IPv4 distribution = %v, want %v",
- name, body.Data.IPv4Distribution, wantV4)
- }
-
- wantV6 := []database.PrefixDistribution{{MaskLength: 32, Count: 1}}
- if !slices.Equal(body.Data.IPv6Distribution, wantV6) {
- t.Errorf("%s: IPv6 distribution = %v, want %v",
- name, body.Data.IPv6Distribution, wantV6)
+ got.OldestRoute, got.NewestRoute = want.OldestRoute, want.NewestRoute
+ if !reflect.DeepEqual(got, want) {
+ t.Errorf("%s: figures %+v, want %+v", name, got, want)
}
}
}
-// newServerWithTestRoutes returns a server on a new database that holds a
-// route for testPrefixV4 and one for 2001:db8::/32.
+// TestStatsAnswerCarriesChurnPerMinute checks that the stream object of the
+// /api/v1/stats answer holds route_churn_per_min as a whole number, and no
+// longer route_churn_per_sec, the moving average it replaces
+// (https://git.eeqj.de/sneak/routewatch/issues/160).
+func TestStatsAnswerCarriesChurnPerMinute(t *testing.T) {
+ t.Parallel()
+
+ s := newServerWithTestRoutes(t)
+
+ rec := httptest.NewRecorder()
+ s.handleStats()(rec,
+ httptest.NewRequestWithContext(t.Context(), http.MethodGet, statsPath, nil))
+
+ if rec.Code != http.StatusOK {
+ t.Fatalf("status %d, want %d; body %s", rec.Code, http.StatusOK, rec.Body)
+ }
+
+ var body struct {
+ Data struct {
+ Stream map[string]json.RawMessage `json:"stream"`
+ } `json:"data"`
+ }
+
+ err := json.Unmarshal(rec.Body.Bytes(), &body)
+ if err != nil {
+ t.Fatalf("decoding the answer: %v", err)
+ }
+
+ churn, ok := body.Data.Stream["route_churn_per_min"]
+ if !ok {
+ t.Fatalf("stream has no route_churn_per_min: %s", rec.Body)
+ }
+
+ var count int
+
+ err = json.Unmarshal(churn, &count)
+ if err != nil {
+ t.Errorf("route_churn_per_min = %s, want a whole number: %v", churn, err)
+ }
+
+ if _, ok := body.Data.Stream["route_churn_per_sec"]; ok {
+ t.Errorf("stream still has route_churn_per_sec: %s", rec.Body)
+ }
+}
+
+// newTestTable returns a routing table that holds a route for testPrefixV4,
+// with origin AS 64500, and one for testPrefixV6, with origin AS 64501.
+func newTestTable() *routetable.Table {
+ table := emptyTable()
+ announce(table, testPrefixV4, 64500, testPeerV4, testRouteTime(0))
+ announce(table, testPrefixV6, 64501, testPeerV6, testRouteTime(0))
+
+ return table
+}
+
+// newServerWithTestRoutes returns a server on a new database that reads the
+// routes of newTestTable.
func newServerWithTestRoutes(t *testing.T) *Server {
t.Helper()
+ return newServerWithTable(t, newTestTable())
+}
+
+// newServerWithTable returns a server on a new database that reads routes from
+// table.
+func newServerWithTable(t *testing.T, table *routetable.Table) *Server {
+ t.Helper()
+
db, err := database.New(&config.Config{StateDir: t.TempDir()}, logger.New())
if err != nil {
t.Fatalf("database.New: %v", err)
@@ -205,25 +324,7 @@ func newServerWithTestRoutes(t *testing.T) *Server {
t.Cleanup(func() { _ = db.Close() })
- ts := time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC)
-
- err = db.UpsertLiveRouteBatch([]*database.LiveRoute{
- {
- Prefix: testPrefixV4, MaskLength: 24, IPVersion: 4,
- OriginASN: 64500, PeerIP: testPeerV4, ASPath: []int{64500},
- NextHop: testPeerV4, LastUpdated: ts,
- },
- {
- Prefix: "2001:db8::/32", MaskLength: 32, IPVersion: 6,
- OriginASN: 64501, PeerIP: testPeerV6, ASPath: []int{64501},
- NextHop: testPeerV6, LastUpdated: ts,
- },
- })
- if err != nil {
- t.Fatalf("UpsertLiveRouteBatch: %v", err)
- }
-
- return New(db, streamer.New(logger.New(), metrics.New()), logger.New(),
+ return New(db, table, streamer.New(logger.New(), metrics.New()), logger.New(),
&config.Config{})
}
@@ -321,27 +422,9 @@ func TestIPLookupsReadPercentEncodedAddresses(t *testing.T) {
}
}
-// errASDetailsLookupFailed is what failingASDetailsDB answers every AS details
-// lookup with.
-var errASDetailsLookupFailed = errors.New("AS details lookup failed")
-
-// failingASDetailsDB wraps a real store and fails every AS details lookup.
-type failingASDetailsDB struct {
- database.Store
-}
-
-func (failingASDetailsDB) GetASDetailsContext(
- _ context.Context, _ int,
-) (*database.ASN, []database.LiveRoute, error) {
- return nil, nil, errASDetailsLookupFailed
-}
-
// TestPrefixDetailShowsOriginNames checks that the prefix page shows each
-// origin AS's handle and description without looking up the AS's details,
-// which also read every route the AS originates. Before the fix the page did
-// that lookup once per origin, so with it failing the names were missing
-// (https://git.eeqj.de/sneak/routewatch/issues/67). The AS path column shows
-// handles too, so the test looks for the origin list's "(handle)" form.
+// origin AS's handle and description. The AS path column shows handles too, so
+// the test looks for the origin list's "(handle)" form.
func TestPrefixDetailShowsOriginNames(t *testing.T) {
t.Parallel()
@@ -352,34 +435,20 @@ func TestPrefixDetailShowsOriginNames(t *testing.T) {
t.Cleanup(func() { _ = db.Close() })
- ts := time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC)
origins := []int{15169, 13335}
+ table := emptyTable()
- for _, asn := range origins {
- _, err := db.GetOrCreateASN(asn, ts)
+ for i, asn := range origins {
+ _, err := db.GetOrCreateASN(asn, testRouteTime(0))
if err != nil {
t.Fatalf("GetOrCreateASN(%d): %v", asn, err)
}
+
+ announce(table, testPrefixV4, asn, testPeer(i+1), testRouteTime(0))
}
- err = db.UpsertLiveRouteBatch([]*database.LiveRoute{
- {
- Prefix: testPrefixV4, MaskLength: 24, IPVersion: 4,
- OriginASN: origins[0], PeerIP: testPeerV4,
- ASPath: []int{origins[0]}, NextHop: testPeerV4, LastUpdated: ts,
- },
- {
- Prefix: testPrefixV4, MaskLength: 24, IPVersion: 4,
- OriginASN: origins[1], PeerIP: "198.51.100.2",
- ASPath: []int{origins[1]}, NextHop: "198.51.100.2", LastUpdated: ts,
- },
- })
- if err != nil {
- t.Fatalf("UpsertLiveRouteBatch: %v", err)
- }
-
- s := New(failingASDetailsDB{db}, streamer.New(logger.New(), metrics.New()),
- logger.New(), &config.Config{})
+ s := New(db, table,
+ streamer.New(logger.New(), metrics.New()), logger.New(), &config.Config{})
rec := httptest.NewRecorder()
s.router.ServeHTTP(rec, httptest.NewRequestWithContext(
t.Context(), http.MethodGet, "/prefix/"+testPrefixV4, nil))
@@ -411,33 +480,19 @@ func TestPrefixDetailShowsOriginNames(t *testing.T) {
// TestPagesSayOriginZeroIsNoKnownOrigin stores a route with origin 0, as the
// prefix handler stores a route whose AS path ends in an AS set of several
// members, and checks that the prefix page and the prefix length page say it
-// has no known origin, without a link to AS0 or the description of the
-// reserved AS 0 (https://git.eeqj.de/sneak/routewatch/issues/75).
+// has no known origin, without a link to AS0
+// (https://git.eeqj.de/sneak/routewatch/issues/75).
func TestPagesSayOriginZeroIsNoKnownOrigin(t *testing.T) {
t.Parallel()
- db, err := database.New(&config.Config{StateDir: t.TempDir()}, logger.New())
- if err != nil {
- t.Fatalf("database.New: %v", err)
- }
+ table := emptyTable()
+ table.Announce(netip.MustParsePrefix(testPrefixV4), ristypes.NoOrigin,
+ routetable.Route{
+ Peer: testPeerV4, ASPath: []int{64500, 64501, 64502},
+ Time: testRouteTime(0),
+ })
- t.Cleanup(func() { _ = db.Close() })
-
- err = db.UpsertLiveRouteBatch([]*database.LiveRoute{{
- Prefix: testPrefixV4, MaskLength: 24, IPVersion: 4,
- OriginASN: 0, PeerIP: testPeerV4, ASPath: []int{64500, 64501, 64502},
- NextHop: testPeerV4, LastUpdated: time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC),
- }})
- if err != nil {
- t.Fatalf("UpsertLiveRouteBatch: %v", err)
- }
-
- reserved, ok := asinfo.Get(0)
- if !ok {
- t.Fatal("AS0 is not in the AS info data")
- }
-
- s := New(db, streamer.New(logger.New(), metrics.New()), logger.New(), &config.Config{})
+ s := newServerWithTable(t, table)
for _, path := range []string{"/prefix/" + testPrefixV4, prefixLengthPagePath} {
rec := httptest.NewRecorder()
@@ -458,9 +513,113 @@ func TestPagesSayOriginZeroIsNoKnownOrigin(t *testing.T) {
if strings.Contains(page, `href="/as/0"`) {
t.Errorf("%s links to AS0", path)
}
+ }
+}
- if strings.Contains(page, html.EscapeString(reserved.Description)) {
- t.Errorf("%s shows AS0's description %q", path, reserved.Description)
+// TestIndexShowsTheClientAddress requests the home page and checks that it has
+// the IP information API card, holding the address /ip/ would look up for the
+// same request: the TCP peer, or, when the peer is a trusted proxy, the client
+// the proxy names in X-Forwarded-For
+// (https://git.eeqj.de/sneak/routewatch/issues/156).
+func TestIndexShowsTheClientAddress(t *testing.T) {
+ t.Parallel()
+
+ const cardHeading = "
IP Information API
"
+
+ // Both requests carry an X-Forwarded-For naming forwardedClient; only a
+ // trusted proxy's is believed.
+ tests := []struct {
+ name string
+ trustedProxies []netip.Prefix
+ peer string
+ want string
+ }{
+ {"direct request", nil, untrustedPeer, untrustedPeerAddr},
+ {
+ "request through a trusted proxy",
+ []netip.Prefix{netip.MustParsePrefix(trustedProxyRange)},
+ trustedPeer, forwardedClient,
+ },
+ }
+
+ for _, tt := range tests {
+ // The home page reads no route and nothing from the database, so the
+ // server has neither a routing table nor a database.
+ s := New(nil, nil, streamer.New(logger.New(), metrics.New()), logger.New(),
+ &config.Config{TrustedProxies: tt.trustedProxies})
+
+ req := httptest.NewRequestWithContext(t.Context(), http.MethodGet, "/", nil)
+ req.RemoteAddr = tt.peer
+ req.Header.Set("X-Forwarded-For", forwardedClient)
+
+ rec := httptest.NewRecorder()
+ s.router.ServeHTTP(rec, req)
+
+ if rec.Code != http.StatusOK {
+ t.Errorf("%s: status %d, want %d; body %s",
+ tt.name, rec.Code, http.StatusOK, rec.Body)
+
+ continue
+ }
+
+ page := rec.Body.String()
+
+ if !strings.Contains(page, cardHeading) {
+ t.Errorf("%s: the page has no IP information API card", tt.name)
+ }
+
+ want := `data-client-ip="` + tt.want + `"`
+ if !strings.Contains(page, want) {
+ t.Errorf("%s: the card does not hold %s", tt.name, want)
+ }
+ }
+}
+
+// TestPagesShowAgesInWords stores routes last updated about 5 minutes, 3 hours
+// and 15 days ago, and checks that the AS page and the prefix pages show each
+// age in words. Before the fix the ages read "5m0s ago", "3h0m0s ago" and
+// "360h0m0s ago" (https://git.eeqj.de/sneak/routewatch/issues/163).
+func TestPagesShowAgesInWords(t *testing.T) {
+ t.Parallel()
+
+ // Each age is well past a whole number of its unit, so the time the test
+ // takes to run does not change the words.
+ ages := []struct {
+ prefix string
+ age time.Duration
+ want string
+ }{
+ {testPrefixV4, 5*time.Minute + 30*time.Second, "5 minutes ago"},
+ {"198.51.100.0/24", 3*time.Hour + 30*time.Minute, "3 hours ago"},
+ {"203.0.113.0/24", 15 * 24 * time.Hour, "2 weeks ago"},
+ }
+
+ now := time.Now().UTC()
+ table := emptyTable()
+
+ for _, a := range ages {
+ announce(table, a.prefix, 64500, testPeerV4, now.Add(-a.age))
+ }
+
+ s := newServerWithTable(t, table)
+
+ for _, a := range ages {
+ cell := `
` + a.want + `
`
+
+ for _, path := range []string{asPagePath, "/prefix/" + a.prefix} {
+ rec := httptest.NewRecorder()
+ s.router.ServeHTTP(rec, httptest.NewRequestWithContext(
+ t.Context(), http.MethodGet, path, nil))
+
+ if rec.Code != http.StatusOK {
+ t.Fatalf("%s: status %d, want %d; body %s",
+ path, rec.Code, http.StatusOK, rec.Body)
+ }
+
+ if !strings.Contains(rec.Body.String(), cell) {
+ t.Errorf("%s does not show the age of the route for %s as %q",
+ path, a.prefix, a.want)
+ }
}
}
}
@@ -479,7 +638,7 @@ const shortRequestDeadline = 50 * time.Millisecond
func TestRequestPastTimeLimitAnswers504(t *testing.T) {
t.Parallel()
- s := New(nil, streamer.New(logger.New(), metrics.New()), logger.New(),
+ s := New(nil, nil, streamer.New(logger.New(), metrics.New()), logger.New(),
&config.Config{})
waitForTimeLimit := func(_ http.ResponseWriter, r *http.Request) {
@@ -529,8 +688,8 @@ func TestStatsHandlersAnswer504WhenDatabaseReadRunsOut(t *testing.T) {
defer close(release)
db := blockingStatsDB{release: release}
- s := New(db, streamer.New(logger.New(), metrics.New()), logger.New(),
- &config.Config{})
+ s := New(db, emptyTable(), streamer.New(logger.New(), metrics.New()),
+ logger.New(), &config.Config{})
handlers := map[string]http.HandlerFunc{
statusJSONHandlerName: s.handleStatusJSON(),
@@ -556,10 +715,9 @@ func TestStatsHandlersAnswer504WhenDatabaseReadRunsOut(t *testing.T) {
// slowDB embeds database.Store (left nil) and overrides the reads the routes
// in TestRoutesAnswer504WhenDatabaseReadIsCutOff make. Each read waits until
// its context ends and then answers as the real store does when the time
-// limit cuts it off: the IP read with the context's error, the prefix read
-// that there is no route, and the others with what they had read, here
-// nothing, and no error. Every other Store method is unused and would panic
-// if called.
+// limit cuts it off: the stats read with what it had read, here nothing, and
+// no error, and the others with the context's error. Every other Store method
+// is unused and would panic if called.
type slowDB struct {
database.Store
}
@@ -570,66 +728,44 @@ func (slowDB) GetStatsContext(ctx context.Context) (database.Stats, error) {
return database.Stats{}, nil
}
-func (slowDB) GetIPInfoContext(
- ctx context.Context, _ string,
-) (*database.IPInfo, error) {
+func (slowDB) GetAS(ctx context.Context, _ int) (*database.ASN, bool, error) {
<-ctx.Done()
- return nil, ctx.Err()
+ return nil, false, ctx.Err()
}
-func (slowDB) GetASDetailsContext(
- ctx context.Context, asn int,
-) (*database.ASN, []database.LiveRoute, error) {
+func (slowDB) GetPrefixFirstSeen(
+ ctx context.Context, _ netip.Prefix,
+) (time.Time, error) {
<-ctx.Done()
- return &database.ASN{ASN: asn}, nil, nil
-}
-
-func (slowDB) GetPrefixDetailsContext(
- ctx context.Context, _ string,
-) ([]database.LiveRoute, map[int]database.ASN, error) {
- <-ctx.Done()
-
- return nil, nil, database.ErrNoRoute
-}
-
-func (slowDB) GetRandomPrefixesByLengthContext(
- ctx context.Context, _, _, _ int,
-) ([]database.LiveRoute, error) {
- <-ctx.Done()
-
- return nil, nil
+ return time.Time{}, ctx.Err()
}
// TestRoutesAnswer504WhenDatabaseReadIsCutOff sends a request through the
// server's router for each route that reads the database, with a store whose
// reads are cut off when the request's time limit passes. Each route answers
// 504 with the time limit message. Without its check after the read, a route
-// answers with what the cut-off read returned: 500, 404, 200, or, on a page
-// that stops before rendering once the request has ended, 504 with an empty
-// body. The health check is left out: it answers 503 when its own 2-second
-// ping runs out.
+// answers with what the cut-off read returned: 500 or 200. The health check is
+// left out: it answers 503 when its own 2-second ping runs out. The prefix
+// length pages and the prefix API read only the routing table.
func TestRoutesAnswer504WhenDatabaseReadIsCutOff(t *testing.T) {
t.Parallel()
- s := New(slowDB{}, streamer.New(logger.New(), metrics.New()), logger.New(),
- &config.Config{})
+ s := New(slowDB{}, newTestTable(), streamer.New(logger.New(), metrics.New()),
+ logger.New(), &config.Config{})
tests := []struct {
path string
wantJSONBody bool
}{
{path: asPagePath},
- {path: "/prefix/192.0.2.0/24"},
- {path: prefixLengthPagePath},
- {path: "/prefixlength6/48"},
+ {path: "/prefix/" + testPrefixV4},
{path: statusJSONPath, wantJSONBody: true},
- {path: "/ip/192.0.2.1", wantJSONBody: true},
+ {path: "/ip/" + addrInTestPrefixV4, wantJSONBody: true},
{path: statsPath, wantJSONBody: true},
- {path: "/api/v1/ip/192.0.2.1", wantJSONBody: true},
+ {path: "/api/v1/ip/" + addrInTestPrefixV4, wantJSONBody: true},
{path: "/api/v1/as/64500", wantJSONBody: true},
- {path: "/api/v1/prefix/192.0.2.0/24", wantJSONBody: true},
}
for _, tt := range tests {
@@ -646,10 +782,8 @@ type slowSecondReadDB struct {
database.Store
}
-func (slowSecondReadDB) GetASDetailsContext(
- _ context.Context, asn int,
-) (*database.ASN, []database.LiveRoute, error) {
- return &database.ASN{ASN: asn}, nil, nil
+func (slowSecondReadDB) GetAS(_ context.Context, asn int) (*database.ASN, bool, error) {
+ return &database.ASN{ASN: asn}, true, nil
}
func (slowSecondReadDB) GetASPeersContext(
@@ -691,8 +825,8 @@ func (idleASNFetcher) GetStats() ASNFetcherStats {
func TestRoutesAnswer504WhenSecondDatabaseReadIsCutOff(t *testing.T) {
t.Parallel()
- s := New(slowSecondReadDB{}, streamer.New(logger.New(), metrics.New()),
- logger.New(), &config.Config{})
+ s := New(slowSecondReadDB{}, emptyTable(),
+ streamer.New(logger.New(), metrics.New()), logger.New(), &config.Config{})
s.SetASNFetcher(idleASNFetcher{})
tests := []struct {
@@ -786,34 +920,27 @@ func (failingDB) GetStatsContext(_ context.Context) (database.Stats, error) {
return database.Stats{}, errDiskIO
}
-func (failingDB) GetIPInfoContext(
- _ context.Context, _ string,
-) (*database.IPInfo, error) {
- return nil, errDiskIO
+func (failingDB) GetAS(_ context.Context, _ int) (*database.ASN, bool, error) {
+ return nil, false, errDiskIO
}
-func (failingDB) GetASDetailsContext(
- _ context.Context, _ int,
-) (*database.ASN, []database.LiveRoute, error) {
- return nil, nil, errDiskIO
+func (failingDB) GetPrefixFirstSeen(
+ _ context.Context, _ netip.Prefix,
+) (time.Time, error) {
+ return time.Time{}, errDiskIO
}
-func (failingDB) GetPrefixDetailsContext(
- _ context.Context, _ string,
-) ([]database.LiveRoute, map[int]database.ASN, error) {
- return nil, nil, errDiskIO
-}
-
-// TestJSONRoutesDoNotSendErrorDetails sends a request to each JSON route and
-// the health check with a store whose reads fail with the driver's error text,
-// and a prefix that does not parse, and checks that no answer carries the
-// driver's or the prefix parser's text. Before the fix the answers sent it to
-// the client (https://git.eeqj.de/sneak/routewatch/issues/108).
+// TestJSONRoutesDoNotSendErrorDetails sends a request to each JSON route that
+// reads the database, and to the health check, with a store whose reads fail
+// with the driver's error text. It also sends a prefix that does not parse.
+// No answer may carry the driver's or the prefix parser's text. Before the fix
+// the answers sent it to the client
+// (https://git.eeqj.de/sneak/routewatch/issues/108).
func TestJSONRoutesDoNotSendErrorDetails(t *testing.T) {
t.Parallel()
- s := New(failingDB{}, streamer.New(logger.New(), metrics.New()), logger.New(),
- &config.Config{})
+ s := New(failingDB{}, newTestTable(), streamer.New(logger.New(), metrics.New()),
+ logger.New(), &config.Config{})
tests := []struct {
path string
@@ -823,8 +950,7 @@ func TestJSONRoutesDoNotSendErrorDetails(t *testing.T) {
{statusJSONPath, http.StatusInternalServerError},
{statsPath, http.StatusInternalServerError},
{"/api/v1/as/64500", http.StatusInternalServerError},
- {"/api/v1/prefix/192.0.2.0/24", http.StatusInternalServerError},
- {"/api/v1/ip/192.0.2.1", http.StatusInternalServerError},
+ {"/api/v1/ip/" + addrInTestPrefixV4, http.StatusInternalServerError},
{"/api/v1/prefix/foo/24", http.StatusBadRequest},
}
@@ -847,45 +973,48 @@ func TestJSONRoutesDoNotSendErrorDetails(t *testing.T) {
}
}
-// The addresses failingIPLookupDB fails the read of and finds a route for.
+// The address whose read failingIPLookupDB fails, in testPrefixV4, and one it
+// reads, in testPrefixV6.
const (
- failingLookupIP = "192.0.2.1"
- routedLookupIP = "192.0.2.2"
+ failingLookupIP = addrInTestPrefixV4
+ routedLookupIP = addrInTestPrefixV6
)
-// failingIPLookupDB embeds database.Store (left nil) and answers the IP lookup
-// with errDiskIO for failingLookupIP, a route for routedLookupIP, and no route
-// for any other address. Every other Store method is unused and would panic if
-// called.
+// failingIPLookupDB embeds database.Store (left nil). It fails the read of
+// testPrefixV4's first-seen time with errDiskIO, and answers the IP lookup's
+// other reads with nothing stored. Every other Store method is unused and
+// would panic if called.
type failingIPLookupDB struct {
database.Store
}
-func (failingIPLookupDB) GetIPInfoContext(
- _ context.Context, ip string,
-) (*database.IPInfo, error) {
- switch ip {
- case failingLookupIP:
- return nil, errDiskIO
- case routedLookupIP:
- return &database.IPInfo{IP: ip}, nil
- default:
- return nil, database.ErrNoRoute
+func (failingIPLookupDB) GetPrefixFirstSeen(
+ _ context.Context, prefix netip.Prefix,
+) (time.Time, error) {
+ if prefix == netip.MustParsePrefix(testPrefixV4) {
+ return time.Time{}, errDiskIO
}
+
+ return time.Time{}, nil
+}
+
+func (failingIPLookupDB) GetASWHOIS(_ context.Context, asn int) (*database.ASN, error) {
+ return &database.ASN{ASN: asn}, nil
}
// TestIPLookupAnswers500WhenDatabaseReadFails sends IP lookups through the
-// server's router with a store whose read of one address fails with the
+// server's router with a store whose read for one address fails with the
// driver's error. A lookup that asks for that address answers 500 with the
-// JSON error body, as the AS and prefix JSON routes do; an address with no
-// route still answers 404. Before the fix the failed read was reported as if
-// the address had no route: 404 for that address alone, and 200 beside a
-// routed address (https://git.eeqj.de/sneak/routewatch/issues/109).
+// JSON error body, as the AS JSON route does; an address with no route answers
+// 200. Before the fix the failed read was reported as if the address had no
+// route: 404 for that address alone, and 200 beside a routed address
+// (https://git.eeqj.de/sneak/routewatch/issues/109).
func TestIPLookupAnswers500WhenDatabaseReadFails(t *testing.T) {
t.Parallel()
- s := New(failingIPLookupDB{}, streamer.New(logger.New(), metrics.New()),
- logger.New(), &config.Config{})
+ s := New(failingIPLookupDB{}, newTestTable(),
+ streamer.New(logger.New(), metrics.New()), logger.New(), &config.Config{})
+ s.resolver = &fakeResolver{}
tests := []struct {
path string
@@ -901,7 +1030,7 @@ func TestIPLookupAnswers500WhenDatabaseReadFails(t *testing.T) {
},
{
path: "/api/v1/ip/203.0.113.1",
- wantCode: http.StatusNotFound,
+ wantCode: http.StatusOK,
},
}
@@ -915,7 +1044,9 @@ func TestIPLookupAnswers500WhenDatabaseReadFails(t *testing.T) {
tt.path, rec.Code, tt.wantCode, rec.Body)
}
- checkJSONErrorCode(t, tt.path, rec, tt.wantCode)
+ if tt.wantCode != http.StatusOK {
+ checkJSONErrorCode(t, tt.path, rec, tt.wantCode)
+ }
}
}
@@ -955,30 +1086,37 @@ func (r *fakeResolver) LookupAddr(ctx context.Context, _ string) ([]string, erro
return nil, errNoSuchHost
}
-// countingIPLookupDB embeds database.Store (left nil) and counts its IP
-// lookups, answering each that the address has no route. Every other Store
-// method is unused and would panic if called.
+// countingIPLookupDB embeds database.Store (left nil) and answers the IP
+// lookup's reads with nothing stored. It counts the lookups, as the reads of a
+// prefix's first-seen time. Every other Store method is unused and would panic
+// if called.
type countingIPLookupDB struct {
database.Store
lookups atomic.Int64
}
-func (d *countingIPLookupDB) GetIPInfoContext(
- _ context.Context, _ string,
-) (*database.IPInfo, error) {
+func (d *countingIPLookupDB) GetPrefixFirstSeen(
+ _ context.Context, _ netip.Prefix,
+) (time.Time, error) {
d.lookups.Add(1)
- return nil, database.ErrNoRoute
+ return time.Time{}, nil
+}
+
+func (d *countingIPLookupDB) GetASWHOIS(
+ _ context.Context, asn int,
+) (*database.ASN, error) {
+ return &database.ASN{ASN: asn}, nil
}
// TestIPLookupRefusesTooManyAddresses sends IP lookups of maxLookupTargets
-// addresses and of one more, listed or as a host name that resolves to them,
-// and a lookup listing maxLookupTargets+1 host names. The longer lookups answer
-// 400 without reading the database, and the list of host names without
-// resolving any of them; the others read the database once per address. Before
-// the fix every address was read and every name resolved, however many there
-// were (https://git.eeqj.de/sneak/routewatch/issues/110).
+// addresses in testPrefixV4 and of one more, listed or as a host name that
+// resolves to them, and a lookup listing maxLookupTargets+1 host names. The
+// longer lookups answer 400 without reading the database, and the list of host
+// names without resolving any of them; the others read the database for each
+// address. Before the fix every address was read and every name resolved,
+// however many there were (https://git.eeqj.de/sneak/routewatch/issues/110).
func TestIPLookupRefusesTooManyAddresses(t *testing.T) {
t.Parallel()
@@ -1003,18 +1141,18 @@ func TestIPLookupRefusesTooManyAddresses(t *testing.T) {
}{
{
strings.Join(addrs[:maxLookupTargets], ","),
- http.StatusNotFound, maxLookupTargets, 0,
+ http.StatusOK, maxLookupTargets, 0,
},
{strings.Join(addrs, ","), http.StatusBadRequest, 0, 0},
- {"most.test", http.StatusNotFound, maxLookupTargets, 1},
+ {"most.test", http.StatusOK, maxLookupTargets, 1},
{"too-many.test", http.StatusBadRequest, 0, 1},
{strings.Join(names, ","), http.StatusBadRequest, 0, 0},
}
for _, tt := range tests {
db := &countingIPLookupDB{}
- s := New(db, streamer.New(logger.New(), metrics.New()), logger.New(),
- &config.Config{})
+ s := New(db, newTestTable(), streamer.New(logger.New(), metrics.New()),
+ logger.New(), &config.Config{})
resolver := &fakeResolver{hosts: hosts}
s.resolver = resolver
@@ -1027,7 +1165,9 @@ func TestIPLookupRefusesTooManyAddresses(t *testing.T) {
tt.target, rec.Code, tt.wantCode, rec.Body)
}
- checkJSONErrorCode(t, tt.target, rec, tt.wantCode)
+ if tt.wantCode != http.StatusOK {
+ checkJSONErrorCode(t, tt.target, rec, tt.wantCode)
+ }
got := db.lookups.Load()
if got != tt.wantLookups {
@@ -1084,6 +1224,191 @@ func TestIPLookupKeepsRouteWhenPTRLookupDoesNotAnswer(t *testing.T) {
}
}
+// addrInTestPrefixV6 is an address in testPrefixV6, which
+// newServerWithTestRoutes stores a route for.
+const addrInTestPrefixV6 = "2001:db8::1"
+
+// testBlocklistInterval is fakeBlocklists' shortest interval.
+const testBlocklistInterval = 15 * time.Minute
+
+// ownAddressPath is the IP lookup of the caller's own address.
+const ownAddressPath = "/ip/"
+
+// listedUnroutedAddr is an address that fakeBlocklists lists, outside the
+// routes newServerWithTestRoutes stores.
+const listedUnroutedAddr = "203.0.113.1"
+
+// fakeBlocklists has one list, which lists addrInTestPrefixV4 and
+// listedUnroutedAddr and no other address.
+type fakeBlocklists struct{}
+
+func testListing() blocklist.Listing {
+ return blocklist.Listing{
+ Name: "test_list",
+ Description: "On the test list",
+ Updated: time.Date(2026, 10, 27, 4, 0, 0, 0, time.UTC),
+ }
+}
+
+func (fakeBlocklists) Lookup(addr netip.Addr) []blocklist.Listing {
+ if addr == netip.MustParseAddr(addrInTestPrefixV4) ||
+ addr == netip.MustParseAddr(listedUnroutedAddr) {
+ return []blocklist.Listing{testListing()}
+ }
+
+ return []blocklist.Listing{}
+}
+
+func (fakeBlocklists) ShortestInterval() time.Duration {
+ return testBlocklistInterval
+}
+
+// TestIPLookupReportsBlocklists looks up two routed addresses, one of them on
+// a blocklist. The listed address's result names the list, says what it means
+// and when it was updated. The other's result has an empty list.
+func TestIPLookupReportsBlocklists(t *testing.T) {
+ t.Parallel()
+
+ s := newServerWithTestRoutes(t)
+ s.resolver = &fakeResolver{}
+ s.SetBlocklists(fakeBlocklists{})
+
+ rec := httptest.NewRecorder()
+ s.router.ServeHTTP(rec, httptest.NewRequestWithContext(t.Context(),
+ http.MethodGet,
+ "/api/v1/ip/"+addrInTestPrefixV4+","+addrInTestPrefixV6, nil))
+
+ if rec.Code != http.StatusOK {
+ t.Fatalf("status %d, want %d; body %s", rec.Code, http.StatusOK, rec.Body)
+ }
+
+ var body struct {
+ Data IPLookupResponse `json:"data"`
+ }
+
+ err := json.Unmarshal(rec.Body.Bytes(), &body)
+ if err != nil {
+ t.Fatalf("decoding the answer %q: %v", rec.Body, err)
+ }
+
+ results := body.Data.Results
+ if len(results) != 2 {
+ t.Fatalf("answer %s, want two results", rec.Body)
+ }
+
+ want := testListing()
+
+ listed := results[0].Blocklists
+ if len(listed) != 1 || listed[0].Name != want.Name ||
+ listed[0].Description != want.Description ||
+ !listed[0].Updated.Equal(want.Updated) {
+ t.Errorf("%s: blocklists %+v, want [%+v]", results[0].IP, listed, want)
+ }
+
+ if !strings.Contains(rec.Body.String(), `"blocklists":[]`) {
+ t.Errorf("answer %s, want an empty blocklists list for %s",
+ rec.Body, addrInTestPrefixV6)
+ }
+}
+
+// TestIPLookupReportsBlocklistsOfAnAddressWithNoRoute looks up a listed
+// address that has no live route, alone and beside a routed address. Each
+// answer is 200 and has a result for the address, with its blocklists and no
+// route fields. Before the fix the lookup of that address alone answered
+// 404, and beside a routed address it was only named in errors, so its
+// blocklists were never reported.
+func TestIPLookupReportsBlocklistsOfAnAddressWithNoRoute(t *testing.T) {
+ t.Parallel()
+
+ s := newServerWithTestRoutes(t)
+ s.resolver = &fakeResolver{}
+ s.SetBlocklists(fakeBlocklists{})
+
+ for _, target := range []string{
+ listedUnroutedAddr,
+ addrInTestPrefixV4 + "," + listedUnroutedAddr,
+ } {
+ rec := httptest.NewRecorder()
+ s.router.ServeHTTP(rec, httptest.NewRequestWithContext(t.Context(),
+ http.MethodGet, "/api/v1/ip/"+target, nil))
+
+ var body struct {
+ Data IPLookupResponse `json:"data"`
+ }
+
+ err := json.Unmarshal(rec.Body.Bytes(), &body)
+ if rec.Code != http.StatusOK || err != nil {
+ t.Errorf("%s: status %d (%v), want %d; body %s",
+ target, rec.Code, err, http.StatusOK, rec.Body)
+
+ continue
+ }
+
+ i := slices.IndexFunc(body.Data.Results, func(r IPLookupResult) bool {
+ return r.IP == listedUnroutedAddr
+ })
+ if i < 0 || len(body.Data.Errors) != 0 {
+ t.Errorf("%s: answer %s, want a result for %s and no errors",
+ target, rec.Body, listedUnroutedAddr)
+
+ continue
+ }
+
+ result := body.Data.Results[i]
+ if result.IPInfo != nil {
+ t.Errorf("%s: result for %s has route fields %+v",
+ target, listedUnroutedAddr, result.IPInfo)
+ }
+
+ if len(result.Blocklists) != 1 ||
+ result.Blocklists[0].Name != testListing().Name {
+ t.Errorf("%s: blocklists %+v, want [%+v]",
+ target, result.Blocklists, testListing())
+ }
+ }
+}
+
+// TestIPLookupAnswersMayBeCached checks the Cache-Control header of the IP
+// lookup answers. Caches may keep them for the blocklists' shortest interval,
+// and only the caller's own browser may keep the answer about the caller's
+// own address.
+func TestIPLookupAnswersMayBeCached(t *testing.T) {
+ t.Parallel()
+
+ s := newServerWithTestRoutes(t)
+ s.resolver = &fakeResolver{}
+ s.SetBlocklists(fakeBlocklists{})
+
+ tests := []struct {
+ path string
+ want string
+ }{
+ {"/api/v1/ip/" + addrInTestPrefixV4, "public, max-age=900"},
+ {ownAddressPath + addrInTestPrefixV4, "public, max-age=900"},
+ {ownAddressPath, "private, max-age=900"},
+ }
+
+ for _, tt := range tests {
+ req := httptest.NewRequestWithContext(t.Context(), http.MethodGet,
+ tt.path, nil)
+ req.RemoteAddr = net.JoinHostPort(addrInTestPrefixV4, "1234")
+
+ rec := httptest.NewRecorder()
+ s.router.ServeHTTP(rec, req)
+
+ if rec.Code != http.StatusOK {
+ t.Errorf("%s: status %d, want %d; body %s",
+ tt.path, rec.Code, http.StatusOK, rec.Body)
+
+ continue
+ }
+
+ if got := rec.Header().Get(cacheControlHeader); got != tt.want {
+ t.Errorf("%s: Cache-Control %q, want %q", tt.path, got, tt.want)
+ }
+ }
+}
+
// The host name TestIPLookupHidesPrivateAddresses resolves to a private
// address, and that address.
const (
@@ -1166,6 +1491,407 @@ func TestIPLookupHidesPrivateAddresses(t *testing.T) {
}
}
+// asSearchPath is the path of the AS name search.
+const asSearchPath = "/api/v1/search/as"
+
+// searchAS sends an AS name search with the URL query rawQuery through s's
+// router, expects a 200 answer, and returns its data.
+func searchAS(t *testing.T, s *Server, rawQuery string) ASSearchResponse {
+ t.Helper()
+
+ rec := httptest.NewRecorder()
+ s.router.ServeHTTP(rec, httptest.NewRequestWithContext(
+ t.Context(), http.MethodGet, asSearchPath+"?"+rawQuery, nil))
+
+ if rec.Code != http.StatusOK {
+ t.Fatalf("%s: status %d, want %d; body %s",
+ rawQuery, rec.Code, http.StatusOK, rec.Body)
+ }
+
+ var body struct {
+ Data ASSearchResponse `json:"data"`
+ }
+
+ err := json.Unmarshal(rec.Body.Bytes(), &body)
+ if err != nil {
+ t.Fatalf("%s: decoding the answer %q: %v", rawQuery, rec.Body, err)
+ }
+
+ return body.Data
+}
+
+// TestASSearchAnswersMatches searches for Cloudflare's handle in lower case,
+// with spaces around it, and expects AS13335 among the matches. Before the fix
+// there was no AS name search, and the home page's search card said it was
+// coming soon (https://git.eeqj.de/sneak/routewatch/issues/151).
+func TestASSearchAnswersMatches(t *testing.T) {
+ t.Parallel()
+
+ const query = "cloudflarenet"
+
+ data := searchAS(t, newServerWithTestRoutes(t), "q=%20"+query+"%20")
+
+ if data.Query != query {
+ t.Errorf("query %q, want %q", data.Query, query)
+ }
+
+ if data.Total != len(data.Results) {
+ t.Errorf("total %d, but %d results", data.Total, len(data.Results))
+ }
+
+ cloudflare := asinfo.Info{
+ ASN: 13335, Handle: "CLOUDFLARENET", Description: "Cloudflare Inc.",
+ }
+ if !slices.Contains(data.Results, cloudflare) {
+ t.Errorf("results %v do not hold %v", data.Results, cloudflare)
+ }
+}
+
+// TestASSearchAnswersEmptyListWhenNothingMatches expects a search that matches
+// nothing to answer 200 with a total of 0 and an empty list, not null.
+func TestASSearchAnswersEmptyListWhenNothingMatches(t *testing.T) {
+ t.Parallel()
+
+ data := searchAS(t, newServerWithTestRoutes(t), "q=XYZABC123456")
+
+ if data.Total != 0 || data.Results == nil || len(data.Results) != 0 {
+ t.Errorf("total %d and results %#v, want 0 and an empty list",
+ data.Total, data.Results)
+ }
+}
+
+// TestASSearchAnswers400ForEmptyQuery expects a search with no q, an empty q
+// or a q of only spaces to answer 400 with the JSON error body.
+func TestASSearchAnswers400ForEmptyQuery(t *testing.T) {
+ t.Parallel()
+
+ s := newServerWithTestRoutes(t)
+
+ for _, path := range []string{
+ asSearchPath, asSearchPath + "?q=", asSearchPath + "?q=%20%20",
+ } {
+ rec := httptest.NewRecorder()
+ s.router.ServeHTTP(rec, httptest.NewRequestWithContext(
+ t.Context(), http.MethodGet, path, nil))
+
+ if rec.Code != http.StatusBadRequest {
+ t.Errorf("%s: status %d, want %d; body %s",
+ path, rec.Code, http.StatusBadRequest, rec.Body)
+ }
+
+ checkJSONErrorCode(t, path, rec, http.StatusBadRequest)
+ checkJSONErrorMessage(t, path, rec, searchQueryMissingMessage)
+ }
+}
+
+// TestASSearchShowsFirstMatchesAndTotal searches for a word with more matches
+// than an answer holds, and expects the first maxASSearchResults of them, in
+// AS number order, and the number of all of them.
+func TestASSearchShowsFirstMatchesAndTotal(t *testing.T) {
+ t.Parallel()
+
+ const query = "university"
+
+ all, _ := asinfo.Search(query, asinfo.Total())
+ if len(all) <= maxASSearchResults {
+ t.Fatalf("Search(%q) has %d matches; the test needs more than %d",
+ query, len(all), maxASSearchResults)
+ }
+
+ data := searchAS(t, newServerWithTestRoutes(t), "q="+query)
+
+ if data.Total != len(all) {
+ t.Errorf("total %d, want %d", data.Total, len(all))
+ }
+
+ if !slices.Equal(data.Results, all[:maxASSearchResults]) {
+ t.Errorf("results %v, want the first %d matches %v",
+ data.Results, maxASSearchResults, all[:maxASSearchResults])
+ }
+}
+
+// The AS numbers the AS detail tests ask for. storedASN originates the route
+// for testPrefixV4 and is in the database. routeOnlyASN originates the route
+// for testPrefixV6, but the database has no record of the AS itself, as when
+// a route is stored before its AS. Both are documentation AS numbers, which
+// the AS list built into asinfo does not hold. builtInOnlyASN is in that list
+// but not in the database. unknownASN is in neither.
+const (
+ storedASN = 64500
+ routeOnlyASN = 64501
+ builtInOnlyASN = 15169
+ unknownASN = 4200000000
+)
+
+// zeroDate is how a page writes a time that was never set.
+const zeroDate = "0001-01-01"
+
+// firstSeenCard is the label of the AS page's first-seen date.
+const firstSeenCard = `
First Seen
`
+
+// newServerWithStoredAS returns newServerWithTestRoutes's server with storedASN
+// added to the database. It fails the test when the AS list built into asinfo
+// lacks builtInOnlyASN or holds unknownASN.
+func newServerWithStoredAS(t *testing.T) *Server {
+ t.Helper()
+
+ if _, ok := asinfo.Get(builtInOnlyASN); !ok {
+ t.Fatalf("AS%d is not in the AS info data", builtInOnlyASN)
+ }
+
+ if _, ok := asinfo.Get(unknownASN); ok {
+ t.Fatalf("AS%d is in the AS info data", unknownASN)
+ }
+
+ s := newServerWithTestRoutes(t)
+
+ _, err := s.db.GetOrCreateASN(storedASN, time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC))
+ if err != nil {
+ t.Fatalf("GetOrCreateASN: %v", err)
+ }
+
+ return s
+}
+
+// TestASDetailJSONAnswersBuiltInAS asks /api/v1/as/ for each of the ASes above.
+// All but unknownASN answer 200 with the handle and description the built-in
+// AS list has for them, if any, and their prefixes, as lists even when empty.
+// Only storedASN has a first_seen time. unknownASN answers 404. Before the fix
+// builtInOnlyASN and routeOnlyASN answered 404 too
+// (https://git.eeqj.de/sneak/routewatch/issues/151).
+func TestASDetailJSONAnswersBuiltInAS(t *testing.T) {
+ t.Parallel()
+
+ s := newServerWithStoredAS(t)
+
+ tests := []struct {
+ asn int
+ wantCode int
+ wantPrefixes []string
+ wantFirstSeen bool
+ }{
+ {storedASN, http.StatusOK, []string{testPrefixV4}, true},
+ {routeOnlyASN, http.StatusOK, []string{testPrefixV6}, false},
+ {builtInOnlyASN, http.StatusOK, nil, false},
+ {unknownASN, http.StatusNotFound, nil, false},
+ }
+
+ for _, tt := range tests {
+ path := "/api/v1/as/" + strconv.Itoa(tt.asn)
+ rec := httptest.NewRecorder()
+ s.router.ServeHTTP(rec, httptest.NewRequestWithContext(
+ t.Context(), http.MethodGet, path, nil))
+
+ if rec.Code != tt.wantCode {
+ t.Errorf("%s: status %d, want %d; body %s",
+ path, rec.Code, tt.wantCode, rec.Body)
+
+ continue
+ }
+
+ if tt.wantCode != http.StatusOK {
+ checkJSONErrorCode(t, path, rec, tt.wantCode)
+
+ continue
+ }
+
+ //nolint:tagliatelle // field names are the existing JSON API
+ var body struct {
+ Data struct {
+ ASN database.ASN `json:"asn"`
+ IPv4Prefixes []LiveRoute `json:"ipv4_prefixes"`
+ IPv6Prefixes []LiveRoute `json:"ipv6_prefixes"`
+ } `json:"data"`
+ }
+
+ err := json.Unmarshal(rec.Body.Bytes(), &body)
+ if err != nil {
+ t.Errorf("%s: decoding the answer %q: %v", path, rec.Body, err)
+
+ continue
+ }
+
+ handle, description := asinfo.GetHandle(tt.asn), asinfo.GetDescription(tt.asn)
+ got := body.Data.ASN
+
+ if got.ASN != tt.asn || got.Handle != handle ||
+ got.Description != description {
+ t.Errorf("%s: AS %d %q %q, want %d %q %q", path, got.ASN,
+ got.Handle, got.Description, tt.asn, handle, description)
+ }
+
+ if body.Data.IPv4Prefixes == nil || body.Data.IPv6Prefixes == nil {
+ t.Errorf("%s: a prefix list is null; body %s", path, rec.Body)
+ }
+
+ routes := slices.Concat(body.Data.IPv4Prefixes, body.Data.IPv6Prefixes)
+
+ gotPrefixes := make([]string, 0, len(routes))
+ for _, route := range routes {
+ gotPrefixes = append(gotPrefixes, route.Prefix)
+ }
+
+ if !slices.Equal(gotPrefixes, tt.wantPrefixes) {
+ t.Errorf("%s: prefixes %v, want %v", path, gotPrefixes, tt.wantPrefixes)
+ }
+
+ gotFirstSeen := strings.Contains(rec.Body.String(), `"first_seen"`)
+ if gotFirstSeen != tt.wantFirstSeen {
+ t.Errorf("%s: has first_seen %t, want %t; body %s",
+ path, gotFirstSeen, tt.wantFirstSeen, rec.Body)
+ }
+ }
+}
+
+// TestASDetailPageShowsBuiltInAS asks /as/ for each of the ASes above. All but
+// unknownASN answer 200 with the handle and description the built-in AS list
+// has for them, if any. storedASN and routeOnlyASN list their prefixes, and
+// builtInOnlyASN says the AS announces none. Only storedASN shows a first-seen
+// date, and no page shows a date that was never set. unknownASN answers 404.
+// Before the fix builtInOnlyASN and routeOnlyASN answered 404 too
+// (https://git.eeqj.de/sneak/routewatch/issues/151).
+func TestASDetailPageShowsBuiltInAS(t *testing.T) {
+ t.Parallel()
+
+ s := newServerWithStoredAS(t)
+
+ tests := []struct {
+ asn int
+ wantCode int
+ wantText string
+ wantFirstSeen bool
+ }{
+ {storedASN, http.StatusOK, testPrefixV4, true},
+ {routeOnlyASN, http.StatusOK, testPrefixV6, false},
+ {builtInOnlyASN, http.StatusOK, "No prefixes announced by this AS", false},
+ {unknownASN, http.StatusNotFound, "", false},
+ }
+
+ for _, tt := range tests {
+ path := "/as/" + strconv.Itoa(tt.asn)
+ rec := httptest.NewRecorder()
+ s.router.ServeHTTP(rec, httptest.NewRequestWithContext(
+ t.Context(), http.MethodGet, path, nil))
+
+ if rec.Code != tt.wantCode {
+ t.Errorf("%s: status %d, want %d; body %s",
+ path, rec.Code, tt.wantCode, rec.Body)
+
+ continue
+ }
+
+ if tt.wantCode != http.StatusOK {
+ continue
+ }
+
+ page := rec.Body.String()
+
+ for _, want := range []string{
+ html.EscapeString(asinfo.GetHandle(tt.asn)),
+ html.EscapeString(asinfo.GetDescription(tt.asn)),
+ tt.wantText,
+ } {
+ if !strings.Contains(page, want) {
+ t.Errorf("%s: page does not show %q", path, want)
+ }
+ }
+
+ if strings.Contains(page, zeroDate) {
+ t.Errorf("%s: page shows the date %s", path, zeroDate)
+ }
+
+ gotFirstSeen := strings.Contains(page, firstSeenCard)
+ if gotFirstSeen != tt.wantFirstSeen {
+ t.Errorf("%s: shows a first-seen date %t, want %t",
+ path, gotFirstSeen, tt.wantFirstSeen)
+ }
+ }
+}
+
+// TestSortRoutesByMaskLength expects the AS page's prefix lists in mask length
+// order, shortest first, and in address order within one mask length. Before
+// the fix they were in address order, so 10.1.0.0/16 came before 11.0.0.0/8
+// (https://git.eeqj.de/sneak/routewatch/issues/153).
+func TestSortRoutesByMaskLength(t *testing.T) {
+ t.Parallel()
+
+ tests := []struct {
+ name string
+ prefixes []string
+ want []string
+ }{
+ {
+ name: "IPv4",
+ prefixes: []string{
+ "100.64.1.0/24", "11.0.0.0/8", "10.1.0.0/16",
+ "20.0.1.0/24", "172.16.0.0/12", "9.0.0.0/8",
+ },
+ want: []string{
+ "9.0.0.0/8", "11.0.0.0/8", "172.16.0.0/12",
+ "10.1.0.0/16", "20.0.1.0/24", "100.64.1.0/24",
+ },
+ },
+ {
+ name: "IPv6",
+ prefixes: []string{
+ "2001:db8:1::/48", "2001:db8:8000::/33", "2001:db8:0:1::/64",
+ "2001:db8::/48", "2001:db8::/33",
+ },
+ want: []string{
+ "2001:db8::/33", "2001:db8:8000::/33", "2001:db8::/48",
+ "2001:db8:1::/48", "2001:db8:0:1::/64",
+ },
+ },
+ }
+
+ for _, tt := range tests {
+ routes := make([]LiveRoute, 0, len(tt.prefixes))
+ for _, prefix := range tt.prefixes {
+ routes = append(routes, LiveRoute{Prefix: prefix})
+ }
+
+ sortRoutesByMaskLength(routes)
+
+ got := make([]string, 0, len(routes))
+ for _, route := range routes {
+ got = append(got, route.Prefix)
+ }
+
+ if !slices.Equal(got, tt.want) {
+ t.Errorf("%s: order %v, want %v", tt.name, got, tt.want)
+ }
+ }
+}
+
+// TestNoOriginHasNoASAnswers stores a route with no known origin, which has
+// origin AS 0 (https://git.eeqj.de/sneak/routewatch/issues/75). It expects
+// /as/0 and /api/v1/as/0 to answer 404.
+func TestNoOriginHasNoASAnswers(t *testing.T) {
+ t.Parallel()
+
+ const noOriginPrefix = "203.0.113.0/24"
+
+ s := newServerWithTestRoutes(t)
+ s.table.Announce(netip.MustParsePrefix(noOriginPrefix), ristypes.NoOrigin,
+ routetable.Route{
+ Peer: testPeerV4, ASPath: []int{64500, 64501, 64502},
+ Time: testRouteTime(0),
+ })
+
+ asn := strconv.Itoa(ristypes.NoOrigin)
+
+ for _, path := range []string{"/as/" + asn, "/api/v1/as/" + asn} {
+ rec := httptest.NewRecorder()
+ s.router.ServeHTTP(rec, httptest.NewRequestWithContext(
+ t.Context(), http.MethodGet, path, nil))
+
+ if rec.Code != http.StatusNotFound {
+ t.Errorf("%s: status %d, want %d; body %s",
+ path, rec.Code, http.StatusNotFound, rec.Body)
+ }
+ }
+}
+
// settledGoroutineCount lets transient goroutines finish, then reports the
// current count.
func settledGoroutineCount() int {
diff --git a/internal/server/handlers_table_internal_test.go b/internal/server/handlers_table_internal_test.go
new file mode 100644
index 0000000..5ff1348
--- /dev/null
+++ b/internal/server/handlers_table_internal_test.go
@@ -0,0 +1,397 @@
+package server
+
+import (
+ "encoding/json"
+ "net/http"
+ "net/http/httptest"
+ "slices"
+ "strconv"
+ "strings"
+ "testing"
+ "time"
+
+ "git.eeqj.de/sneak/routewatch/internal/database"
+ "git.eeqj.de/sneak/routewatch/internal/ristypes"
+ "git.eeqj.de/sneak/routewatch/internal/routetable"
+)
+
+// The origin ASes in the routing table that newTwoOriginTable fills.
+const (
+ mostPeersASN = 64500
+ fewerPeersASN = 64501
+ v6OriginASN = 64502
+)
+
+// noKeptRoutePrefix is the prefix of newTwoOriginTable whose origin keeps no
+// route.
+const noKeptRoutePrefix = "198.51.100.0/24"
+
+// routeTimeFormat is how the pages write a route's time.
+const routeTimeFormat = "2006-01-02 15:04:05"
+
+// newTwoOriginTable returns a routing table that holds:
+// - testPrefixV4 with two origin ASes. Peers 1, 2 and 3 carry it with
+// mostPeersASN, which keeps the routes of peers 3 and 2. Peer 4 carries
+// it with fewerPeersASN, in the prefix's newest route.
+// - noKeptRoutePrefix with mostPeersASN. Only peer 7 carries it, and its
+// route is not kept: peers 5 and 6, whose routes were, withdrew it.
+// - testPrefixV6 with v6OriginASN.
+func newTwoOriginTable() *routetable.Table {
+ table := emptyTable()
+
+ for n := 1; n <= 3; n++ {
+ announce(table, testPrefixV4, mostPeersASN, testPeer(n), testRouteTime(n))
+ }
+
+ announce(table, testPrefixV4, fewerPeersASN, testPeer(4), testRouteTime(5))
+
+ announce(table, noKeptRoutePrefix, mostPeersASN, testPeer(5), testRouteTime(4))
+ announce(table, noKeptRoutePrefix, mostPeersASN, testPeer(6), testRouteTime(6))
+ announce(table, noKeptRoutePrefix, mostPeersASN, testPeer(7), testRouteTime(0))
+ withdraw(table, noKeptRoutePrefix, testPeer(5))
+ withdraw(table, noKeptRoutePrefix, testPeer(6))
+
+ announce(table, testPrefixV6, v6OriginASN, testPeerV6, testRouteTime(0))
+
+ return table
+}
+
+// get sends a GET for path through s's router and fails the test unless it
+// answers 200.
+func get(t *testing.T, s *Server, path string) *httptest.ResponseRecorder {
+ t.Helper()
+
+ rec := httptest.NewRecorder()
+ s.router.ServeHTTP(rec, httptest.NewRequestWithContext(
+ t.Context(), http.MethodGet, path, nil))
+
+ if rec.Code != http.StatusOK {
+ t.Fatalf("%s: status %d, want %d; body %s",
+ path, rec.Code, http.StatusOK, rec.Body)
+ }
+
+ return rec
+}
+
+// TestIPLookupGivesTheOriginMostPeersCarry looks up an address in a prefix
+// with two origin ASes, through each IP lookup route. The answer names the
+// origin AS that more peers carry, counts every peer, and gives the time of
+// the prefix's newest kept route. The prefix's first-seen time and the AS's
+// WHOIS details come from the database.
+func TestIPLookupGivesTheOriginMostPeersCarry(t *testing.T) {
+ t.Parallel()
+
+ s := newServerWithTable(t, newTwoOriginTable())
+ s.resolver = &fakeResolver{}
+
+ firstSeen := testRouteTime(-1)
+
+ err := s.db.UpdatePrefixesBatch(map[string]time.Time{testPrefixV4: firstSeen})
+ if err != nil {
+ t.Fatalf("UpdatePrefixesBatch: %v", err)
+ }
+
+ _, err = s.db.GetOrCreateASN(mostPeersASN, firstSeen)
+ if err != nil {
+ t.Fatalf("GetOrCreateASN: %v", err)
+ }
+
+ whois := &database.ASNWHOISUpdate{
+ ASN: mostPeersASN, ASName: "TEST-AS", CountryCode: "DE",
+ }
+
+ err = s.db.UpdateASNWHOIS(t.Context(), whois)
+ if err != nil {
+ t.Fatalf("UpdateASNWHOIS: %v", err)
+ }
+
+ // ownAddressPath looks up the request's own address, which httptest
+ // gives as addrInTestPrefixV4.
+ for _, path := range []string{
+ "/api/v1/ip/" + addrInTestPrefixV4, "/ip/" + addrInTestPrefixV4,
+ ownAddressPath,
+ } {
+ var body struct {
+ Data IPLookupResponse `json:"data"`
+ }
+
+ err := json.Unmarshal(get(t, s, path).Body.Bytes(), &body)
+ if err != nil || len(body.Data.Results) != 1 ||
+ body.Data.Results[0].IPInfo == nil {
+ t.Fatalf("%s: answer %+v (%v), want one result with a route",
+ path, body.Data, err)
+ }
+
+ got := body.Data.Results[0].IPInfo
+ if got.Netblock != testPrefixV4 || got.ASN != mostPeersASN ||
+ got.NumPeers != 4 || !got.LastSeen.Equal(testRouteTime(5)) ||
+ !got.FirstSeen.Equal(firstSeen) || got.ASName != whois.ASName ||
+ got.CountryCode != whois.CountryCode {
+ t.Errorf("%s: route %+v, want %s from AS%d, 4 peers, last seen %v, "+
+ "first seen %v, %s in %s", path, *got, testPrefixV4,
+ mostPeersASN, testRouteTime(5), firstSeen, whois.ASName,
+ whois.CountryCode)
+ }
+ }
+}
+
+// TestIPLookupAsksForNoWHOISLookupOfOriginZero checks that the IP lookup asks
+// for no WHOIS lookup of AS 0, which means no known origin. It asks for one of
+// an AS whose WHOIS details were never fetched or are stale, and not of one
+// whose details are fresh.
+func TestIPLookupAsksForNoWHOISLookupOfOriginZero(t *testing.T) {
+ t.Parallel()
+
+ fresh := time.Now().Add(-time.Hour)
+ stale := time.Now().Add(-2 * whoisStaleThreshold)
+
+ tests := []struct {
+ name string
+ asInfo database.ASN
+ want bool
+ }{
+ {"no known origin", database.ASN{ASN: ristypes.NoOrigin}, false},
+ {"details never fetched", database.ASN{ASN: mostPeersASN}, true},
+ {"stale details", database.ASN{ASN: mostPeersASN, WHOISUpdatedAt: &stale}, true},
+ {"fresh details", database.ASN{ASN: mostPeersASN, WHOISUpdatedAt: &fresh}, false},
+ }
+
+ for _, tt := range tests {
+ if got := needsWHOISRefresh(&tt.asInfo); got != tt.want {
+ t.Errorf("%s: needsWHOISRefresh = %t, want %t", tt.name, got, tt.want)
+ }
+ }
+}
+
+// prefixRoute is one route of the prefix API's answer. A field the answer
+// leaves out stays nil.
+//
+//nolint:tagliatelle // field names are the existing JSON API
+type prefixRoute struct {
+ OriginASN int `json:"origin_asn"`
+ PeerIP string `json:"peer_ip"`
+ ASPath []int `json:"as_path"`
+ NextHop *string `json:"next_hop"`
+ LastUpdated *time.Time `json:"last_updated"`
+}
+
+// TestPrefixAnswersListEveryPeer asks the prefix API and page for a prefix
+// with two origin ASes, and for one whose origin keeps no route. Each answer
+// lists every peer that carries the prefix, by origin AS. Only the kept routes
+// have an AS path, a next hop and a time, newest first. The other peers follow
+// without them.
+func TestPrefixAnswersListEveryPeer(t *testing.T) {
+ t.Parallel()
+
+ s := newServerWithTable(t, newTwoOriginTable())
+
+ // keptAt[peer] is the time of the peer's kept route; a peer not in it is
+ // listed without a route.
+ keptAt := map[string]time.Time{
+ testPeer(3): testRouteTime(3),
+ testPeer(2): testRouteTime(2),
+ testPeer(4): testRouteTime(5),
+ }
+
+ tests := []struct {
+ prefix string
+ wantPeers []string
+ wantOrigins []int
+ }{
+ {
+ testPrefixV4,
+ []string{testPeer(3), testPeer(2), testPeer(1), testPeer(4)},
+ []int{mostPeersASN, mostPeersASN, mostPeersASN, fewerPeersASN},
+ },
+ {noKeptRoutePrefix, []string{testPeer(7)}, []int{mostPeersASN}},
+ }
+
+ for _, tt := range tests {
+ //nolint:tagliatelle // field names are the existing JSON API
+ var body struct {
+ Data struct {
+ Routes []prefixRoute `json:"routes"`
+ PeerCount int `json:"peer_count"`
+ OriginCount int `json:"origin_count"`
+ } `json:"data"`
+ }
+
+ path := "/api/v1/prefix/" + tt.prefix
+
+ err := json.Unmarshal(get(t, s, path).Body.Bytes(), &body)
+ if err != nil {
+ t.Fatalf("%s: decoding the answer: %v", path, err)
+ }
+
+ wantOriginCount := len(slices.Compact(slices.Clone(tt.wantOrigins)))
+ if body.Data.PeerCount != len(tt.wantPeers) ||
+ body.Data.OriginCount != wantOriginCount {
+ t.Errorf("%s: %d peers and %d origins, want %d and %d", path,
+ body.Data.PeerCount, body.Data.OriginCount, len(tt.wantPeers),
+ wantOriginCount)
+ }
+
+ checkPrefixRoutes(t, path, body.Data.Routes, tt.wantPeers, tt.wantOrigins,
+ keptAt)
+ checkPrefixPage(t, s, tt.prefix, tt.wantPeers, keptAt)
+ }
+}
+
+// checkPrefixRoutes checks that routes come from wantPeers, in that order,
+// with wantOrigins as their origin ASes. The route of a peer in keptAt must
+// have an AS path, the peer as next hop, and the time keptAt gives. Any other
+// route must have none of the three.
+func checkPrefixRoutes(
+ t *testing.T, path string, routes []prefixRoute, wantPeers []string,
+ wantOrigins []int, keptAt map[string]time.Time,
+) {
+ t.Helper()
+
+ if len(routes) != len(wantPeers) {
+ t.Fatalf("%s: routes %+v, want one each from %v", path, routes, wantPeers)
+ }
+
+ for i, route := range routes {
+ if route.PeerIP != wantPeers[i] || route.OriginASN != wantOrigins[i] {
+ t.Errorf("%s: route %d from %s with AS%d, want from %s with AS%d",
+ path, i, route.PeerIP, route.OriginASN, wantPeers[i], wantOrigins[i])
+ }
+
+ at, kept := keptAt[route.PeerIP]
+ if !kept {
+ if route.ASPath != nil || route.NextHop != nil || route.LastUpdated != nil {
+ t.Errorf("%s: route from %s is not kept but has %+v",
+ path, route.PeerIP, route)
+ }
+
+ continue
+ }
+
+ if route.ASPath == nil || route.NextHop == nil ||
+ *route.NextHop != route.PeerIP || route.LastUpdated == nil ||
+ !route.LastUpdated.Equal(at) {
+ t.Errorf("%s: kept route from %s is %+v, want its AS path, the peer "+
+ "as next hop and the time %v", path, route.PeerIP, route, at)
+ }
+ }
+}
+
+// checkPrefixPage checks that the page of prefix shows each of wantPeers, the
+// time of each kept route among them, and no time that was never set.
+func checkPrefixPage(
+ t *testing.T, s *Server, prefix string, wantPeers []string,
+ keptAt map[string]time.Time,
+) {
+ t.Helper()
+
+ path := "/prefix/" + prefix
+ page := get(t, s, path).Body.String()
+
+ for _, peer := range wantPeers {
+ want := `
` + peer + `
`
+ if !strings.Contains(page, want) {
+ t.Errorf("%s does not list the peer %s", path, peer)
+ }
+
+ at, kept := keptAt[peer]
+ if kept && !strings.Contains(page, at.Format(routeTimeFormat)) {
+ t.Errorf("%s does not show the time %v of the route from %s",
+ path, at, peer)
+ }
+ }
+
+ if strings.Contains(page, zeroDate) {
+ t.Errorf("%s shows the date %s", path, zeroDate)
+ }
+}
+
+// TestASAnswersGiveEachPrefixItsNewestKeptRoute asks the AS API and page for
+// mostPeersASN. Its prefixes are testPrefixV4, whose newest kept route with
+// that origin AS is peer 3's, and noKeptRoutePrefix, which keeps no route. The
+// first has that route's time. The second is listed without a time.
+func TestASAnswersGiveEachPrefixItsNewestKeptRoute(t *testing.T) {
+ t.Parallel()
+
+ s := newServerWithTable(t, newTwoOriginTable())
+ path := "/api/v1/as/" + strconv.Itoa(mostPeersASN)
+
+ //nolint:tagliatelle // field names are the existing JSON API
+ var body struct {
+ Data struct {
+ IPv4Prefixes []struct {
+ Prefix string `json:"prefix"`
+ LastUpdated *time.Time `json:"last_updated"`
+ } `json:"ipv4_prefixes"`
+ } `json:"data"`
+ }
+
+ err := json.Unmarshal(get(t, s, path).Body.Bytes(), &body)
+ if err != nil {
+ t.Fatalf("%s: decoding the answer: %v", path, err)
+ }
+
+ prefixes := body.Data.IPv4Prefixes
+ if len(prefixes) != 2 || prefixes[0].Prefix != testPrefixV4 ||
+ prefixes[0].LastUpdated == nil ||
+ !prefixes[0].LastUpdated.Equal(testRouteTime(3)) ||
+ prefixes[1].Prefix != noKeptRoutePrefix || prefixes[1].LastUpdated != nil {
+ t.Errorf("%s: prefixes %+v, want %s updated %v and %s without a time",
+ path, prefixes, testPrefixV4, testRouteTime(3), noKeptRoutePrefix)
+ }
+
+ path = "/as/" + strconv.Itoa(mostPeersASN)
+ page := get(t, s, path).Body.String()
+
+ for _, want := range []string{
+ testPrefixV4, noKeptRoutePrefix, testRouteTime(3).Format(routeTimeFormat),
+ } {
+ if !strings.Contains(page, want) {
+ t.Errorf("%s does not show %s", path, want)
+ }
+ }
+
+ if strings.Contains(page, zeroDate) {
+ t.Errorf("%s shows the date %s", path, zeroDate)
+ }
+}
+
+// TestPrefixLengthPagesShowTheOriginMostPeersCarry asks for the IPv4 /24
+// page and the IPv6 /32 page. The first lists both /24 prefixes with
+// mostPeersASN, the origin AS more peers carry testPrefixV4 with, and gives
+// noKeptRoutePrefix no age. The second lists testPrefixV6.
+func TestPrefixLengthPagesShowTheOriginMostPeersCarry(t *testing.T) {
+ t.Parallel()
+
+ s := newServerWithTable(t, newTwoOriginTable())
+
+ page := get(t, s, prefixLengthPagePath).Body.String()
+
+ for _, want := range []string{
+ ">" + testPrefixV4 + "<", ">" + noKeptRoutePrefix + "<",
+ `href="/as/` + strconv.Itoa(mostPeersASN) + `"`,
+ } {
+ if !strings.Contains(page, want) {
+ t.Errorf("%s does not show %s", prefixLengthPagePath, want)
+ }
+ }
+
+ if strings.Contains(page, `href="/as/`+strconv.Itoa(fewerPeersASN)+`"`) {
+ t.Errorf("%s links to AS%d, which fewer peers carry",
+ prefixLengthPagePath, fewerPeersASN)
+ }
+
+ const noAge = `
`
+ if strings.Count(page, noAge) != 1 {
+ t.Errorf("%s gives %d prefixes no age, want 1 (%s)",
+ prefixLengthPagePath, strings.Count(page, noAge), noKeptRoutePrefix)
+ }
+
+ const v6PagePath = "/prefixlength6/32"
+
+ page = get(t, s, v6PagePath).Body.String()
+ if !strings.Contains(page, ">"+testPrefixV6+"<") ||
+ !strings.Contains(page, `href="/as/`+strconv.Itoa(v6OriginASN)+`"`) {
+ t.Errorf("%s does not show %s with AS%d", v6PagePath, testPrefixV6,
+ v6OriginASN)
+ }
+}
diff --git a/internal/server/middleware_internal_test.go b/internal/server/middleware_internal_test.go
index e3803fb..fc06be0 100644
--- a/internal/server/middleware_internal_test.go
+++ b/internal/server/middleware_internal_test.go
@@ -11,11 +11,12 @@ import (
)
// The addresses the forwarding tests use: the client a proxy names in
-// X-Forwarded-For, a peer inside the trusted range of
-// TestRealIPMiddlewareTrustsOnlyConfiguredProxies, and a peer outside it,
-// which lies in testPrefixV4 so that an IP lookup of it finds a route.
+// X-Forwarded-For, the range of trusted proxies, a peer inside it, and a peer
+// outside it, which lies in testPrefixV4 so that an IP lookup of it finds a
+// route.
const (
forwardedClient = "203.0.113.9"
+ trustedProxyRange = "10.0.0.0/8"
trustedPeer = "10.0.0.2:1234"
untrustedPeerAddr = "192.0.2.1"
untrustedPeer = untrustedPeerAddr + ":1234"
@@ -189,7 +190,7 @@ func TestForwardedForFromUntrustedPeerIsIgnored(t *testing.T) {
func TestRealIPMiddlewareTrustsOnlyConfiguredProxies(t *testing.T) {
t.Parallel()
- trusted := []netip.Prefix{netip.MustParsePrefix("10.0.0.0/8")}
+ trusted := []netip.Prefix{netip.MustParsePrefix(trustedProxyRange)}
tests := []struct {
name string
diff --git a/internal/server/models.go b/internal/server/models.go
new file mode 100644
index 0000000..858a770
--- /dev/null
+++ b/internal/server/models.go
@@ -0,0 +1,67 @@
+package server
+
+import (
+ "errors"
+ "time"
+)
+
+var (
+ // ErrInvalidIP says that an IP address is malformed. The IP lookup returns
+ // it.
+ ErrInvalidIP = errors.New("invalid IP address")
+ // ErrNoRoute says that the routing table has no route for an address, a
+ // prefix or an AS. The lookups return it.
+ ErrNoRoute = errors.New("no route found")
+)
+
+// LiveRoute represents a route in the live routing table. JSON leaves out an
+// empty AS path, next hop or time: the routing table in memory keeps them only
+// for the two newest routes of each prefix and origin AS.
+//
+//nolint:tagliatelle // field names are the existing JSON API
+type LiveRoute struct {
+ Prefix string `json:"prefix"`
+ MaskLength int `json:"mask_length"`
+ IPVersion int `json:"ip_version"`
+ OriginASN int `json:"origin_asn"`
+ PeerIP string `json:"peer_ip"`
+ ASPath []int `json:"as_path,omitempty"`
+ NextHop string `json:"next_hop,omitempty"`
+ LastUpdated time.Time `json:"last_updated,omitzero"`
+}
+
+// PrefixDistribution represents the distribution of prefixes by mask length
+//
+//nolint:tagliatelle // field names are the existing JSON API
+type PrefixDistribution struct {
+ MaskLength int `json:"mask_length"`
+ Count int `json:"count"`
+}
+
+// IPInfo represents comprehensive IP information for the /ip endpoint
+//
+//nolint:tagliatelle // field names are the existing JSON API
+type IPInfo struct {
+ IP string `json:"ip"`
+ Netblock string `json:"netblock"`
+ MaskLength int `json:"mask_length"`
+ IPVersion int `json:"ip_version"`
+ NumPeers int `json:"num_peers"`
+ // AS information
+ ASN int `json:"asn"`
+ ASName string `json:"as_name,omitempty"`
+ Handle string `json:"handle,omitempty"`
+ Description string `json:"description,omitempty"`
+ OrgName string `json:"org_name,omitempty"`
+ OrgID string `json:"org_id,omitempty"`
+ Address string `json:"address,omitempty"`
+ CountryCode string `json:"country_code,omitempty"`
+ AbuseEmail string `json:"abuse_email,omitempty"`
+ RIR string `json:"rir,omitempty"`
+ // Timestamps. LastSeen is zero, and left out of JSON, when the prefix
+ // keeps no route.
+ FirstSeen time.Time `json:"first_seen"`
+ LastSeen time.Time `json:"last_seen,omitzero"`
+ // Indicates if WHOIS data needs refresh (not serialized)
+ NeedsWHOISRefresh bool `json:"-"`
+}
diff --git a/internal/server/routes.go b/internal/server/routes.go
index f995c22..096e092 100644
--- a/internal/server/routes.go
+++ b/internal/server/routes.go
@@ -87,6 +87,7 @@ func (s *Server) setupRoutes() {
r.Get("/ip/{ip}", s.handleIPLookup())
r.Get("/as/{asn}", s.handleASDetailJSON())
r.Get("/prefix/{prefix}/{len}", s.handlePrefixDetailJSON())
+ r.Get("/search/as", s.handleASSearch())
})
s.router = r
diff --git a/internal/server/server.go b/internal/server/server.go
index 0d97002..6d4aadc 100644
--- a/internal/server/server.go
+++ b/internal/server/server.go
@@ -1,4 +1,12 @@
-// Package server provides HTTP endpoints for status monitoring and statistics
+// Package server serves RouteWatch's web pages and JSON API:
+// - the home page, the status figures and the health check
+// - the AS, prefix and prefix length pages, and the JSON answers behind them
+// - the IP lookups
+// - the AS name search
+//
+// Every route it shows comes from the routing table in memory. AS names,
+// WHOIS details and the times a prefix and an AS were first seen come from
+// the database.
package server
import (
@@ -11,9 +19,11 @@ import (
"strconv"
"time"
+ "git.eeqj.de/sneak/routewatch/internal/blocklist"
"git.eeqj.de/sneak/routewatch/internal/config"
"git.eeqj.de/sneak/routewatch/internal/database"
"git.eeqj.de/sneak/routewatch/internal/logger"
+ "git.eeqj.de/sneak/routewatch/internal/routetable"
"git.eeqj.de/sneak/routewatch/internal/streamer"
"github.com/go-chi/chi/v5"
)
@@ -32,6 +42,13 @@ type ASNFetcher interface {
GetStats() ASNFetcherStats
}
+// Blocklists tells which public blocklists list an address, from copies it
+// keeps in memory, as *blocklist.Blocklists does.
+type Blocklists interface {
+ Lookup(addr netip.Addr) []blocklist.Listing
+ ShortestInterval() time.Duration
+}
+
// dnsResolver looks up the addresses of host names and the names of addresses
// in DNS, as *net.Resolver does.
type dnsResolver interface {
@@ -43,11 +60,13 @@ type dnsResolver interface {
type Server struct {
router *chi.Mux
db database.Store
+ table *routetable.Table
streamer *streamer.Streamer
logger *logger.Logger
port int
srv *http.Server
asnFetcher ASNFetcher
+ blocklists Blocklists
// trustedProxies are the reverse proxies whose X-Forwarded-For header
// RealIPMiddleware believes
@@ -58,15 +77,18 @@ type Server struct {
resolver dnsResolver
}
-// New creates a new HTTP server
+// New creates a new HTTP server that reads routes from table, the routing
+// table that the prefix handler keeps current.
func New(
db database.Store,
+ table *routetable.Table,
streamer *streamer.Streamer,
logger *logger.Logger,
cfg *config.Config,
) *Server {
s := &Server{
db: db,
+ table: table,
streamer: streamer,
logger: logger,
port: cfg.Port,
@@ -144,3 +166,9 @@ func (s *Server) Stop(ctx context.Context) error {
func (s *Server) SetASNFetcher(fetcher ASNFetcher) {
s.asnFetcher = fetcher
}
+
+// SetBlocklists sets the blocklists the IP lookups report. Call it before
+// Start: the handlers read the blocklists without a lock.
+func (s *Server) SetBlocklists(blocklists Blocklists) {
+ s.blocklists = blocklists
+}
diff --git a/internal/server/server_test.go b/internal/server/server_test.go
index 1f6964b..5bfec36 100644
--- a/internal/server/server_test.go
+++ b/internal/server/server_test.go
@@ -34,7 +34,7 @@ func TestStartFailsWhenPortIsHeld(t *testing.T) {
port := addr.Port
log := logger.New()
- s := server.New(nil, streamer.New(log, metrics.New()), log,
+ s := server.New(nil, nil, streamer.New(log, metrics.New()), log,
&config.Config{Port: port})
defer func() { _ = s.Stop(context.Background()) }()
diff --git a/internal/templates/as_detail.html b/internal/templates/as_detail.html
index 3f16760..984dcc6 100644
--- a/internal/templates/as_detail.html
+++ b/internal/templates/as_detail.html
@@ -126,6 +126,26 @@
margin: 0;
color: #2c3e50;
}
+ /* A click on a prefix list's heading folds or unfolds the list. Not
+ every browser shows its own marker on a flex element, so the page
+ hides it and draws a triangle before the heading. The triangle
+ points down while the list is open. */
+ summary.prefix-header {
+ cursor: pointer;
+ list-style: none;
+ }
+ summary.prefix-header::-webkit-details-marker {
+ display: none;
+ }
+ summary.prefix-header h2::before {
+ content: "\25B8";
+ display: inline-block;
+ margin-right: 10px;
+ transition: transform 0.2s;
+ }
+ details[open] > summary.prefix-header h2::before {
+ transform: rotate(90deg);
+ }
.prefix-count {
background: #e74c3c;
color: white;
@@ -233,18 +253,20 @@
Search for autonomous systems by organization name
+
Search for autonomous systems by organization name or handle
@@ -326,6 +380,13 @@
+
+
+
IP Information API
+
The API answers what RouteWatch knows about an IP address, such as its AS number and country. /ip/ with no address answers for the caller's own address.