Compare commits
1
Commits
next
..
9ca9c3c9fa
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
9ca9c3c9fa |
+2
-12
@@ -73,8 +73,7 @@
|
|||||||
|
|
||||||
# This repository's host-built artifacts: `make build` writes bin/, `go
|
# This repository's host-built artifacts: `make build` writes bin/, `go
|
||||||
# build` in the root the binaries, `make run` log.txt, `make asupdate` its
|
# build` in the root the binaries, `make run` log.txt, `make asupdate` its
|
||||||
# .tmp file; logs, test binaries, coverage profiles, source archives, and
|
# .tmp file; test binaries, coverage profiles and source archives.
|
||||||
# the binaries and libraries a build for another platform writes.
|
|
||||||
/bin
|
/bin
|
||||||
/routewatch
|
/routewatch
|
||||||
/asinfo-gen
|
/asinfo-gen
|
||||||
@@ -83,24 +82,15 @@
|
|||||||
/out
|
/out
|
||||||
/pkg/asinfo/asdata.json
|
/pkg/asinfo/asdata.json
|
||||||
**/*.tar.zst
|
**/*.tar.zst
|
||||||
**/*.log
|
|
||||||
**/*.test
|
**/*.test
|
||||||
**/*.out
|
**/*.out
|
||||||
**/*.tmp
|
**/*.tmp
|
||||||
**/*.exe
|
|
||||||
**/*.dll
|
|
||||||
**/*.so
|
|
||||||
**/*.dylib
|
|
||||||
|
|
||||||
# Local databases. The image carries a source archive of the whole build
|
# Local databases. The image carries a source archive of the whole build
|
||||||
# context, so these would otherwise ship inside it. The daemon's database
|
# context, so these would otherwise ship inside it.
|
||||||
# is db.sqlite, with its -wal and -shm files while it runs.
|
|
||||||
**/*.db
|
**/*.db
|
||||||
**/*.db-journal
|
**/*.db-journal
|
||||||
**/*.db-wal
|
**/*.db-wal
|
||||||
**/*.sqlite
|
|
||||||
**/*.sqlite-wal
|
|
||||||
**/*.sqlite-shm
|
|
||||||
|
|
||||||
# A local Go workspace points at directories outside the build context.
|
# A local Go workspace points at directories outside the build context.
|
||||||
/go.work
|
/go.work
|
||||||
|
|||||||
+1
-5
@@ -67,14 +67,10 @@ node_modules/
|
|||||||
go.work
|
go.work
|
||||||
go.work.sum
|
go.work.sum
|
||||||
|
|
||||||
# Local databases, temporary files and debug output. The daemon's
|
# Local databases, temporary files and debug output.
|
||||||
# database is db.sqlite, with its -wal and -shm files while it runs.
|
|
||||||
*.db
|
*.db
|
||||||
*.db-journal
|
*.db-journal
|
||||||
*.db-wal
|
*.db-wal
|
||||||
*.sqlite
|
|
||||||
*.sqlite-wal
|
|
||||||
*.sqlite-shm
|
|
||||||
*.tmp
|
*.tmp
|
||||||
log.txt
|
log.txt
|
||||||
out
|
out
|
||||||
|
|||||||
+6
-11
@@ -24,8 +24,8 @@ RUN --mount=type=cache,target=/root/.cache/go-build \
|
|||||||
# image ships and the alpine one does not. -count=1 because the mounted
|
# image ships and the alpine one does not. -count=1 because the mounted
|
||||||
# cache also holds test results, which go test would otherwise replay.
|
# cache also holds test results, which go test would otherwise replay.
|
||||||
# -short skips the live RIS Live feed test, so the suite needs no network.
|
# -short skips the live RIS Live feed test, so the suite needs no network.
|
||||||
# golang:1.27.2-bookworm, 2026-10-19
|
# golang:1.24-bookworm, 2026-09-21
|
||||||
FROM golang@sha256:5cf287a799e6b94384bad13d16b14904c531f51ba65792237e122ce42b392f61 AS test
|
FROM golang@sha256:1a6d4452c65dea36aac2e2d606b01b4a029ec90cc1ae53890540ce6173ea77ac AS test
|
||||||
WORKDIR /src
|
WORKDIR /src
|
||||||
COPY go.mod go.sum ./
|
COPY go.mod go.sum ./
|
||||||
RUN go mod download
|
RUN go mod download
|
||||||
@@ -39,8 +39,8 @@ RUN --mount=type=cache,target=/root/.cache/go-build \
|
|||||||
# Build stage. Nothing is wanted from either phase above; the copies
|
# Build stage. Nothing is wanted from either phase above; the copies
|
||||||
# are what make BuildKit build them first, so nothing after them can
|
# are what make BuildKit build them first, so nothing after them can
|
||||||
# run unless lint and test passed.
|
# run unless lint and test passed.
|
||||||
# golang:1.27.2-bookworm, 2026-10-19
|
# golang:1.24-bookworm, 2026-09-21
|
||||||
FROM golang@sha256:5cf287a799e6b94384bad13d16b14904c531f51ba65792237e122ce42b392f61 AS builder
|
FROM golang@sha256:1a6d4452c65dea36aac2e2d606b01b4a029ec90cc1ae53890540ce6173ea77ac AS builder
|
||||||
|
|
||||||
# Install build dependencies (zstd for archive, gcc for CGO/sqlite3)
|
# Install build dependencies (zstd for archive, gcc for CGO/sqlite3)
|
||||||
RUN apt-get update && apt-get install -y --no-install-recommends \
|
RUN apt-get update && apt-get install -y --no-install-recommends \
|
||||||
@@ -139,13 +139,8 @@ EXPOSE 8080
|
|||||||
|
|
||||||
COPY ./entrypoint.sh /entrypoint.sh
|
COPY ./entrypoint.sh /entrypoint.sh
|
||||||
|
|
||||||
# Health check using the health endpoint, on the port PORT names. During the
|
# Health check using the health endpoint, on the port PORT names
|
||||||
# start period, Docker 25 and later checks every 5 seconds, and the first pass
|
HEALTHCHECK --interval=30s --timeout=5s --start-period=10s --retries=3 \
|
||||||
# makes the container healthy. After it, Docker checks every 30 seconds. The
|
|
||||||
# start period covers the 60 seconds after which upaas reads the health
|
|
||||||
# (README "Running under upaas"). Startup reads the routing table first,
|
|
||||||
# which takes about 25 seconds when the table is full.
|
|
||||||
HEALTHCHECK --interval=30s --timeout=5s --start-period=60s --retries=3 \
|
|
||||||
CMD curl -sf "http://localhost:${PORT:-8080}/.well-known/healthcheck.json" || exit 1
|
CMD curl -sf "http://localhost:${PORT:-8080}/.well-known/healthcheck.json" || exit 1
|
||||||
|
|
||||||
ENTRYPOINT ["/bin/bash", "/entrypoint.sh" ]
|
ENTRYPOINT ["/bin/bash", "/entrypoint.sh" ]
|
||||||
|
|||||||
@@ -52,6 +52,7 @@ clean:
|
|||||||
run: build
|
run: build
|
||||||
DEBUG=routewatch ./bin/routewatch 2>&1 | tee log.txt
|
DEBUG=routewatch ./bin/routewatch 2>&1 | tee log.txt
|
||||||
|
|
||||||
# Rebuild the AS list built into RouteWatch from as.csv (needs the network).
|
|
||||||
asupdate:
|
asupdate:
|
||||||
@script/asupdate
|
@echo "Updating AS info data..."
|
||||||
|
@go run cmd/asinfo-gen/main.go | gzip > pkg/asinfo/asdata.json.gz.tmp && \
|
||||||
|
mv pkg/asinfo/asdata.json.gz.tmp pkg/asinfo/asdata.json.gz
|
||||||
|
|||||||
@@ -1,27 +1,9 @@
|
|||||||
# RouteWatch
|
# RouteWatch
|
||||||
|
|
||||||
RouteWatch is an MIT-licensed Go daemon by @sneak that monitors the BGP routing
|
RouteWatch is an MIT-licensed Go daemon by @sneak that monitors the BGP routing
|
||||||
table in real time. It works like this:
|
table in real time: it streams BGP UPDATE messages from the RIPE RIS Live
|
||||||
|
service, maintains a live routing table in SQLite, and provides HTTP APIs for
|
||||||
- It streams BGP UPDATE messages from the RIPE RIS Live service.
|
querying routing information.
|
||||||
- It keeps a live routing table in SQLite.
|
|
||||||
- It serves web pages and a JSON API for looking routes up.
|
|
||||||
|
|
||||||
## Rationale
|
|
||||||
|
|
||||||
RIPE RIS Live streams the BGP updates that RIPE's route collectors receive, as
|
|
||||||
they happen, from the moment a client connects. It reports changes, not the
|
|
||||||
routing table they add up to, so it cannot say which AS an IP address is routed
|
|
||||||
to now or which routes a prefix has. RouteWatch applies the feed's updates to
|
|
||||||
live routing tables in its own SQLite database and answers IP-to-AS lookups and
|
|
||||||
prefix pages from them.
|
|
||||||
|
|
||||||
The feed sends no table when a client connects, so these tables hold only the
|
|
||||||
routes the feed has announced since RouteWatch first started: an IP-to-AS lookup
|
|
||||||
finds nothing for an address whose routes have not been announced since then.
|
|
||||||
The tables also miss the updates RouteWatch drops when it falls behind under
|
|
||||||
load and those sent while it is stopped, so a lookup or a prefix page can show a
|
|
||||||
route the feed has withdrawn.
|
|
||||||
|
|
||||||
## Features
|
## Features
|
||||||
|
|
||||||
@@ -29,8 +11,6 @@ route the feed has withdrawn.
|
|||||||
- Maintains live IPv4 and IPv6 routing tables
|
- Maintains live IPv4 and IPv6 routing tables
|
||||||
- Tracks AS peering relationships
|
- Tracks AS peering relationships
|
||||||
- HTTP API for IP-to-AS lookups, prefix details, and AS information
|
- HTTP API for IP-to-AS lookups, prefix details, and AS information
|
||||||
- IP lookups report the public blocklists an address is on, from copies kept in
|
|
||||||
memory and on disk
|
|
||||||
- Automatic reconnection with exponential backoff
|
- Automatic reconnection with exponential backoff
|
||||||
- Batched database writes for high performance
|
- Batched database writes for high performance
|
||||||
- Backpressure handling to prevent memory exhaustion
|
- Backpressure handling to prevent memory exhaustion
|
||||||
@@ -56,22 +36,17 @@ DEBUG=routewatch ./routewatch
|
|||||||
|
|
||||||
## HTTP Endpoints
|
## HTTP Endpoints
|
||||||
|
|
||||||
Every route answers `HEAD` as it answers `GET`, without the body.
|
|
||||||
|
|
||||||
### Web Interface
|
### Web Interface
|
||||||
|
|
||||||
- `GET /` - Home page (HTML): headline figures, lookups by AS number and by IP
|
- `GET /` - Home page (HTML): headline figures, and lookups by AS number and by
|
||||||
address, and a search of AS names. A card shows the `curl` command that looks
|
IP address
|
||||||
up the visitor's own address at `/ip/{addr}`, and that command's live answer.
|
|
||||||
- `GET /status` - HTML status dashboard, which reads `/api/v1/stats`
|
- `GET /status` - HTML status dashboard, which reads `/api/v1/stats`
|
||||||
- `GET /status.json` - JSON statistics: feed, memory, database and route figures
|
- `GET /status.json` - JSON statistics: feed, memory, database and route figures
|
||||||
- `GET /.well-known/healthcheck.json` - Health check (JSON): 200 when the
|
- `GET /.well-known/healthcheck.json` - Health check (JSON): 200 when the
|
||||||
database answers and the RIS Live feed is connected, otherwise 503;
|
database answers and the RIS Live feed is connected, otherwise 503;
|
||||||
`data.status` is `ok` or `error` and `data.checks` says which check failed
|
`data.status` is `ok` or `error` and `data.checks` says which check failed
|
||||||
- `GET /as/{asn}` - AS detail page (HTML): the prefixes the AS originates and
|
- `GET /as/{asn}` - AS detail page (HTML): the prefixes the AS originates and
|
||||||
its peers. When the database has no record of the AS, its handle and
|
its peers
|
||||||
description come from the AS list built into RouteWatch. The page then shows
|
|
||||||
no first-seen date.
|
|
||||||
- `GET /prefix/{prefix}/{length}` - Prefix detail page (HTML), such as
|
- `GET /prefix/{prefix}/{length}` - Prefix detail page (HTML), such as
|
||||||
`/prefix/192.0.2.0/24`: every live route for the prefix, one per peer, and its
|
`/prefix/192.0.2.0/24`: every live route for the prefix, one per peer, and its
|
||||||
origin ASes
|
origin ASes
|
||||||
@@ -79,149 +54,34 @@ Every route answers `HEAD` as it answers `GET`, without the body.
|
|||||||
picked at random, each with its newest route (HTML)
|
picked at random, each with its newest route (HTML)
|
||||||
- `GET /prefixlength6/{length}` - The same for IPv6
|
- `GET /prefixlength6/{length}` - The same for IPv6
|
||||||
- `GET /ip/{addr}` - The same JSON answer as `/api/v1/ip/{ip}`
|
- `GET /ip/{addr}` - The same JSON answer as `/api/v1/ip/{ip}`
|
||||||
- `GET /ip/` - The same, for the caller's own address: the address the request
|
- `GET /ip/` - The same, for the caller's own address, or the one a proxy names
|
||||||
came from or, when that is a trusted proxy (`TRUSTED_PROXIES`, see
|
in `True-Client-IP`, `X-Real-IP` or `X-Forwarded-For`
|
||||||
Configuration), the client address in its `X-Forwarded-For`
|
|
||||||
|
|
||||||
### API v1
|
### API v1
|
||||||
|
|
||||||
- `GET /api/v1/stats` - The `/status.json` figures plus connection, garbage
|
- `GET /api/v1/stats` - The `/status.json` figures plus connection, garbage
|
||||||
collector, stream and handler queue statistics. `stream.route_churn_per_min`
|
collector, stream and handler queue statistics
|
||||||
is the number of prefix announcements and withdrawals, from all peers, in the
|
- `GET /api/v1/ip/{ip}` - Looks up each address: its most specific live route
|
||||||
last full clock minute.
|
(`netblock`), that route's origin AS with its WHOIS details, and the address's
|
||||||
- `GET /api/v1/ip/{ip}` - Looks up each address. `{ip}` is an IP address, a host
|
PTR names. `{ip}` is an IP address, a host name (each address it resolves to
|
||||||
name, or a comma-separated list of at most 10 of these. `results` has one
|
is looked up), or a comma-separated list of these. `results` has one entry per
|
||||||
entry for each address, with:
|
address found and `errors` one per address or name that failed; the answer is
|
||||||
- `ip` and `ptr`: the address and its PTR names
|
404 when no address was found.
|
||||||
- `netblock` and the fields beside it: the address's most specific live
|
|
||||||
route, and that route's origin AS with its WHOIS details
|
|
||||||
- `blocklists`: the public blocklists the address is on (see Blocklists)
|
|
||||||
|
|
||||||
An address with no live route has a result without the route and AS fields.
|
|
||||||
Of the addresses a host name resolves to, only public ones are looked up. A
|
|
||||||
name with none, such as one that resolves to a private address, is reported
|
|
||||||
as `host name has no public address`, without its addresses. `errors` has
|
|
||||||
one entry for each host name that failed. The answer is 400 when:
|
|
||||||
- `{ip}` lists more than 10
|
|
||||||
- `{ip}` comes to more than 10 addresses once its names are resolved
|
|
||||||
- nothing in `{ip}` is an IP address or a host name with a public address
|
|
||||||
|
|
||||||
A PTR lookup gets 2 seconds. An address whose lookup takes longer is
|
|
||||||
answered without PTR names.
|
|
||||||
|
|
||||||
- `GET /api/v1/as/{asn}` - The AS and the prefixes it originates, IPv4 and IPv6
|
- `GET /api/v1/as/{asn}` - The AS and the prefixes it originates, IPv4 and IPv6
|
||||||
apart, each with when its newest route was updated. When the database has no
|
apart, each with when its newest route was updated
|
||||||
record of the AS, its handle and description come from the AS list built into
|
|
||||||
RouteWatch. The answer then has no `first_seen` or `last_seen`.
|
|
||||||
- `GET /api/v1/prefix/{prefix}/{length}` - Every live route for the prefix, one
|
- `GET /api/v1/prefix/{prefix}/{length}` - Every live route for the prefix, one
|
||||||
per peer, and the same routes grouped by origin AS in `origins`
|
per peer, and the same routes grouped by origin AS in `origins`
|
||||||
- `GET /api/v1/search/as?q={text}` - Finds the ASes whose handle or description
|
|
||||||
contains `{text}`, ignoring case. `data` holds:
|
|
||||||
- `query`: `{text}` without its leading and trailing spaces
|
|
||||||
- `total`: the number of ASes that match
|
|
||||||
- `results`: the first 50 of them in AS number order, each with `asn`,
|
|
||||||
`handle` and `description`
|
|
||||||
|
|
||||||
The search reads the AS list built into RouteWatch, not the database. The
|
|
||||||
list comes from
|
|
||||||
[ipverse/as-metadata](https://github.com/ipverse/as-metadata). A match can
|
|
||||||
therefore be an AS that the database has no record of. A `q` that is
|
|
||||||
missing, empty or only spaces answers 400.
|
|
||||||
|
|
||||||
A JSON answer is `{"status": "ok", "data": ...}`, or for an error
|
A JSON answer is `{"status": "ok", "data": ...}`, or for an error
|
||||||
`{"status": "error", "error": {"msg": ..., "code": ...}}`, and carries an
|
`{"status": "error", "error": {"msg": ..., "code": ...}}`, and carries an
|
||||||
`@meta` object with the time zone, the API version and the time the request
|
`@meta` object with the time zone, the API version and the time the request
|
||||||
took. The health check is the exception: its 503 answer is also
|
took.
|
||||||
`{"status": "ok", "data": ...}`, with `error` in `data.status`.
|
|
||||||
|
|
||||||
A request that runs past its time limit answers 504: the limit is 30 seconds,
|
|
||||||
and 4 seconds for the database reads behind `/status.json` and `/api/v1/stats`.
|
|
||||||
An AS number, prefix or mask length that does not parse answers 400, as does a
|
|
||||||
mask length out of range for its IP version. An AS answers 404 when it is not in
|
|
||||||
the AS list built into RouteWatch and the database has neither a record of it
|
|
||||||
nor a live route it originates. AS 0 always answers 404, because it means no
|
|
||||||
known origin. The AS name search never lists it. A prefix with no live route
|
|
||||||
answers 404 too. The HTML pages give these errors as plain text. Under `/api/v1`
|
|
||||||
and `/ip/`, a path no route matches answers 404 with the message `Not found`,
|
|
||||||
and `POST`, `PUT`, `PATCH`, `DELETE`, `OPTIONS`, `TRACE` or `CONNECT` answers
|
|
||||||
405 with `Method not allowed`. A 500 answer says only `Internal server error`,
|
|
||||||
and a host name that does not resolve only `could not resolve host name`; the
|
|
||||||
detail goes to the log.
|
|
||||||
|
|
||||||
Every answer carries `Strict-Transport-Security`, `Content-Security-Policy`,
|
|
||||||
`X-Frame-Options`, `X-Content-Type-Options`, `Referrer-Policy` and
|
|
||||||
`Permissions-Policy`. The Content-Security-Policy lets a page load content only
|
|
||||||
from the server, and does not allow inline content with `'unsafe-inline'`: the
|
|
||||||
pages keep their CSS and JavaScript in inline `<style>` and `<script>` blocks,
|
|
||||||
so each answer's policy names a random nonce made for that answer, and the
|
|
||||||
browser applies or runs only the blocks that carry it. The pages therefore use
|
|
||||||
no `style` attributes, which a nonce cannot cover.
|
|
||||||
|
|
||||||
An origin AS of 0 (`origin_asn` in a route, `asn` in an IP lookup, the key `0`
|
An origin AS of 0 (`origin_asn` in a route, `asn` in an IP lookup, the key `0`
|
||||||
in a prefix's `origins`) means the route has no known origin: its AS path ends
|
in a prefix's `origins`) means the route has no known origin: its AS path ends
|
||||||
in an AS set of several members, any of which may have originated it. The pages
|
in an AS set of several members, any of which may have originated it. The pages
|
||||||
show such a route as having no known origin.
|
show such a route as having no known origin.
|
||||||
|
|
||||||
## Blocklists
|
|
||||||
|
|
||||||
The IP lookup reports which public blocklists list each address. Each result has
|
|
||||||
a `blocklists` list with one entry for each list the address is on:
|
|
||||||
|
|
||||||
- `name`: the list's name, from the lists below
|
|
||||||
- `description`: what being on the list means
|
|
||||||
- `updated`: when RouteWatch last downloaded the list, or last heard from its
|
|
||||||
publisher that its copy was current
|
|
||||||
|
|
||||||
An address on no list has an empty `blocklists`. An address with no live route
|
|
||||||
still has a result with its `blocklists`.
|
|
||||||
|
|
||||||
The lists:
|
|
||||||
|
|
||||||
- `dshield`: the 20 networks that DShield saw scan the most targets in the last
|
|
||||||
three days. From [DShield](https://isc.sans.edu/feeds_doc.html), downloaded
|
|
||||||
every hour.
|
|
||||||
- `feodo_tracker`: botnet command-and-control servers. From abuse.ch
|
|
||||||
[Feodo Tracker](https://feodotracker.abuse.ch/blocklist/), downloaded every 15
|
|
||||||
minutes.
|
|
||||||
- `firehol_abusers_30d`: addresses reported for comment, form or forum spam in
|
|
||||||
the last 30 days. From
|
|
||||||
[FireHOL](https://iplists.firehol.org/?ipset=firehol_abusers_30d), downloaded
|
|
||||||
every 5 hours.
|
|
||||||
- `firehol_level3`: addresses reported for attacks, spyware or viruses in the
|
|
||||||
last 30 days. From
|
|
||||||
[FireHOL](https://iplists.firehol.org/?ipset=firehol_level3), downloaded every
|
|
||||||
3 hours.
|
|
||||||
- `spamhaus_drop` and `spamhaus_dropv6`: IPv4 and IPv6 networks that Spamhaus
|
|
||||||
says spammers or cyber criminals have hijacked or lease. From
|
|
||||||
[Spamhaus DROP](https://www.spamhaus.org/blocklists/do-not-route-or-peer/),
|
|
||||||
downloaded once a day. Spamhaus merged its EDROP list into DROP.
|
|
||||||
- `tor_exits`: Tor exit nodes, where traffic from the Tor network leaves. From
|
|
||||||
the [Tor Project](https://check.torproject.org/torbulkexitlist), downloaded
|
|
||||||
every hour.
|
|
||||||
|
|
||||||
No list is downloaded more often than its publisher updates it or allows. Each
|
|
||||||
can be downloaded whole without an account or a key.
|
|
||||||
|
|
||||||
How RouteWatch keeps the lists:
|
|
||||||
|
|
||||||
- It holds each list in memory, so a lookup makes no outside request.
|
|
||||||
- It keeps a copy of each list in the `blocklists` directory of the state
|
|
||||||
directory (see Configuration). Each copy is a file named after its list.
|
|
||||||
- Beside each copy, a file ending in `.validators` keeps the `Last-Modified` and
|
|
||||||
`ETag` the publisher sent with that copy.
|
|
||||||
- At startup it loads each list from its copy. It downloads a list at once when
|
|
||||||
the copy is older than the list's interval, or when there is no copy.
|
|
||||||
- A download sends the copy's `Last-Modified` as `If-Modified-Since` and its
|
|
||||||
`ETag` as `If-None-Match`. The publisher then sends the list only when it has
|
|
||||||
changed. Without either, the download asks for the whole list.
|
|
||||||
- A download fails when it takes over a minute or its file is over 16 MiB.
|
|
||||||
- When a download fails, or its file does not parse, RouteWatch keeps the
|
|
||||||
previous copy and logs the failure. It tries again one interval later.
|
|
||||||
|
|
||||||
A successful IP lookup answer carries `Cache-Control` with a `max-age` of 15
|
|
||||||
minutes, the shortest list interval. The answer for the caller's own address
|
|
||||||
(`/ip/`) is `private`, and the others are `public`.
|
|
||||||
|
|
||||||
## Code Structure
|
## Code Structure
|
||||||
|
|
||||||
```
|
```
|
||||||
@@ -231,11 +91,9 @@ routewatch/
|
|||||||
│ ├── asinfo-gen/ # Utility to generate AS info data
|
│ ├── asinfo-gen/ # Utility to generate AS info data
|
||||||
│ └── streamdumper/ # Debug utility for raw stream output
|
│ └── streamdumper/ # Debug utility for raw stream output
|
||||||
├── internal/
|
├── internal/
|
||||||
│ ├── blocklist/ # Public IP blocklists: downloads and lookups
|
|
||||||
│ ├── routewatch/ # Core application logic
|
│ ├── routewatch/ # Core application logic
|
||||||
│ ├── server/ # HTTP server and handlers
|
│ ├── server/ # HTTP server and handlers
|
||||||
│ ├── database/ # SQLite storage layer
|
│ ├── database/ # SQLite storage layer
|
||||||
│ ├── routetable/ # The whole routing table in memory
|
|
||||||
│ ├── streamer/ # RIPE RIS Live client
|
│ ├── streamer/ # RIPE RIS Live client
|
||||||
│ ├── ristypes/ # BGP message data structures
|
│ ├── ristypes/ # BGP message data structures
|
||||||
│ ├── logger/ # Structured logging wrapper
|
│ ├── logger/ # Structured logging wrapper
|
||||||
@@ -285,8 +143,6 @@ routewatch/
|
|||||||
|
|
||||||
2. **Initialization** (`internal/routewatch/app.go`)
|
2. **Initialization** (`internal/routewatch/app.go`)
|
||||||
- Database created with SQLite schema (WAL mode, page caches as in "Memory")
|
- Database created with SQLite schema (WAL mode, page caches as in "Memory")
|
||||||
- The routing table read from the database into memory, before the streamer
|
|
||||||
connects
|
|
||||||
- Message handlers registered with the streamer
|
- Message handlers registered with the streamer
|
||||||
- HTTP server started on configured port
|
- HTTP server started on configured port
|
||||||
|
|
||||||
@@ -309,14 +165,10 @@ routewatch/
|
|||||||
- **ASHandler**: Tracks all ASNs seen in AS paths
|
- **ASHandler**: Tracks all ASNs seen in AS paths
|
||||||
- **PeerHandler**: Records BGP peer information
|
- **PeerHandler**: Records BGP peer information
|
||||||
- **PrefixHandler**: Maintains live routing table (upserts on announcement,
|
- **PrefixHandler**: Maintains live routing table (upserts on announcement,
|
||||||
deletes on withdrawal or on an announcement whose AS path is empty or
|
deletes on withdrawal, deletes all of a peer's routes when its BGP session
|
||||||
holds AS 0 anywhere, deletes all of a peer's routes when its BGP session
|
|
||||||
goes down). Routes are kept per peer IP, so when one peer IP has sessions
|
goes down). Routes are kept per peer IP, so when one peer IP has sessions
|
||||||
on two collectors, a session going down on one also removes the routes the
|
on two collectors, a session going down on one also removes the routes the
|
||||||
other still carries, until they are announced again. It also applies each
|
other still carries, until they are announced again.
|
||||||
update to the routing table in memory, and writes the table's changes to
|
|
||||||
the database every second. A write takes at most 20,000 changed prefix and
|
|
||||||
origin AS pairs, and the rest follow in the next writes.
|
|
||||||
- **PeeringHandler**: Extracts AS peering relationships from AS paths
|
- **PeeringHandler**: Extracts AS peering relationships from AS paths
|
||||||
|
|
||||||
5. **HTTP Request Flow**
|
5. **HTTP Request Flow**
|
||||||
@@ -330,11 +182,7 @@ routewatch/
|
|||||||
batches for performance
|
batches for performance
|
||||||
- **Backpressure**: Probabilistic message dropping when queues exceed 50%
|
- **Backpressure**: Probabilistic message dropping when queues exceed 50%
|
||||||
capacity
|
capacity
|
||||||
- **Graceful Shutdown**: on SIGTERM or SIGINT the streamer stops, the handlers
|
- **Graceful Shutdown**: 60-second timeout, flushes all pending batches
|
||||||
write the batches they hold, the HTTP server stops and the database is
|
|
||||||
checkpointed and closed. The daemon gives this 8 seconds and exits with status
|
|
||||||
0; "Running under upaas" says what a stop loses, logs and leaves undone when
|
|
||||||
the 8 seconds run out.
|
|
||||||
- **Reconnection**: Any end of the stream other than shutdown, including RIS
|
- **Reconnection**: Any end of the stream other than shutdown, including RIS
|
||||||
Live ending the response normally, marks the feed disconnected and reconnects
|
Live ending the response normally, marks the feed disconnected and reconnects
|
||||||
with exponential backoff (5s-320s). The backoff resets after a stream that
|
with exponential backoff (5s-320s). The backoff resets after a stream that
|
||||||
@@ -390,21 +238,11 @@ Configuration is handled via environment variables and OS-specific paths:
|
|||||||
| `XDG_DATA_HOME` | `/var/lib` (in the Docker image) | Base of the state directory; must be an absolute path |
|
| `XDG_DATA_HOME` | `/var/lib` (in the Docker image) | Base of the state directory; must be an absolute path |
|
||||||
| `GOMEMLIMIT` | `1536MiB` (in the Docker image) | Go soft memory limit; see Memory |
|
| `GOMEMLIMIT` | `1536MiB` (in the Docker image) | Go soft memory limit; see Memory |
|
||||||
| `MALLOC_ARENA_MAX` | `2` (in the Docker image) | glibc malloc arena cap, a positive whole number; see Memory |
|
| `MALLOC_ARENA_MAX` | `2` (in the Docker image) | glibc malloc arena cap, a positive whole number; see Memory |
|
||||||
| `TRUSTED_PROXIES` | (empty) | Addresses and CIDR ranges of trusted proxies; see below |
|
|
||||||
|
|
||||||
A variable that is set to an invalid value stops the start with an error and a
|
A variable that is set to an invalid value stops the start with an error and a
|
||||||
non-zero exit. An empty variable counts as unset.
|
non-zero exit. An empty variable counts as unset.
|
||||||
|
|
||||||
`TRUSTED_PROXIES` lists the reverse proxies whose `X-Forwarded-For` the server
|
State directory (database location):
|
||||||
believes, comma-separated, such as `10.0.0.0/8,192.0.2.1`. For a request from
|
|
||||||
one of them, the request log and `/ip/` use the last address in
|
|
||||||
`X-Forwarded-For` that is not a trusted proxy: each proxy adds the address it
|
|
||||||
received the request from to the end, so the entries before it are whatever the
|
|
||||||
client sent. For a request from any other address they use the address the
|
|
||||||
request came from, whatever forwarding headers it carries. `X-Real-IP` and
|
|
||||||
`True-Client-IP` are not read.
|
|
||||||
|
|
||||||
State directory (the database and the blocklist copies):
|
|
||||||
|
|
||||||
- macOS: `~/Library/Application Support/berlin.sneak.app.routewatch/`
|
- macOS: `~/Library/Application Support/berlin.sneak.app.routewatch/`
|
||||||
- Linux: `/var/lib/berlin.sneak.app.routewatch/` when running as root, otherwise
|
- Linux: `/var/lib/berlin.sneak.app.routewatch/` when running as root, otherwise
|
||||||
@@ -471,14 +309,8 @@ What happens at each limit:
|
|||||||
(`peerBatchMaxSize`, about 10 MiB) and 250,000 for the prefix handler
|
(`peerBatchMaxSize`, about 10 MiB) and 250,000 for the prefix handler
|
||||||
(`prefixBatchMaxSize`, about 60 MiB). At that size the handler takes nothing
|
(`prefixBatchMaxSize`, about 60 MiB). At that size the handler takes nothing
|
||||||
more off its queue until the write ends, so the queue fills and the streamer
|
more off its queue until the write ends, so the queue fills and the streamer
|
||||||
drops messages as above. When the delete of a peer's live routes after its
|
drops messages as above. With the batches being written, the three handlers'
|
||||||
session went down fails, the prefix handler keeps the updates the peer sent
|
batches take at most about 0.25 GiB of the Go heap's 1.5 GiB.
|
||||||
after the down for the next write, which retries the delete first; on top of
|
|
||||||
its 250,000 it keeps at most the last 25,000 of them (`prefixBatchSize`, about
|
|
||||||
6 MiB) and drops and logs the older ones, so while the delete keeps failing
|
|
||||||
the other peers' updates still come off the queue. With the batches being
|
|
||||||
written, the three handlers' batches take at most about 0.25 GiB of the Go
|
|
||||||
heap's 1.5 GiB.
|
|
||||||
|
|
||||||
With `DEBUG=routewatch` the daemon logs a `System stats` line every 60 seconds
|
With `DEBUG=routewatch` the daemon logs a `System stats` line every 60 seconds
|
||||||
with the goroutine count and Go memory figures.
|
with the goroutine count and Go memory figures.
|
||||||
@@ -489,38 +321,15 @@ What the [upaas](https://git.eeqj.de/sneak/upaas) app needs:
|
|||||||
|
|
||||||
- Container port: `8080`.
|
- Container port: `8080`.
|
||||||
- Volume: one, at container path `/var/lib/berlin.sneak.app.routewatch`.
|
- Volume: one, at container path `/var/lib/berlin.sneak.app.routewatch`.
|
||||||
- Environment: `TRUSTED_PROXIES`, set to the address the reverse proxy in front
|
- Environment: nothing is required. Leave `XDG_DATA_HOME`, `GOMEMLIMIT` and
|
||||||
of the app connects from, as the container sees it; without it the request log
|
`MALLOC_ARENA_MAX` at the image's values. `DEBUG=routewatch` is optional and
|
||||||
and `/ip/` give the proxy's address for every request. Nothing else is needed:
|
adds the `System stats` memory line to the log.
|
||||||
leave `XDG_DATA_HOME`, `GOMEMLIMIT` and `MALLOC_ARENA_MAX` at the image's
|
|
||||||
values. `DEBUG=routewatch` is optional and adds the `System stats` memory line
|
|
||||||
to the log.
|
|
||||||
- Memory Limit: `5g`, the 5 GiB limit from Memory above. upaas sets no swap
|
- Memory Limit: `5g`, the 5 GiB limit from Memory above. upaas sets no swap
|
||||||
limit, so on a host with swap Docker allows the same amount of swap again.
|
limit, so on a host with swap Docker allows the same amount of swap again.
|
||||||
- Health check: the image's `HEALTHCHECK` requests
|
- Health check: the image's `HEALTHCHECK` requests
|
||||||
`/.well-known/healthcheck.json` on the container port. upaas reads the
|
`/.well-known/healthcheck.json` on the container port. upaas reads the
|
||||||
container's health 60 seconds after a deploy and fails the deploy unless it is
|
container's health 60 seconds after a deploy and fails the deploy unless it is
|
||||||
`healthy`. For the first 60 seconds, Docker 25 and later runs the check every
|
`healthy`.
|
||||||
5 seconds, and the first pass makes the container healthy. Startup first reads
|
|
||||||
the routing table, which takes about 25 seconds when the table is full.
|
|
||||||
- Stop grace: the container needs 9 seconds between SIGTERM and SIGKILL. upaas
|
|
||||||
gives 10 (`stopTimeoutSeconds` in its `internal/docker/client.go`), as
|
|
||||||
`docker stop` does by default. The daemon allows 8 seconds for the handlers to
|
|
||||||
write what they hold and for the requests in progress to finish, then stops
|
|
||||||
the HTTP server, closes the database and exits with status 0. A longer grace
|
|
||||||
changes nothing. What is not written after 8 seconds is lost, and the streamer
|
|
||||||
and each handler log how much. When the prefix handler has fallen behind the
|
|
||||||
feed, a stop loses far more: its batch is full at 250,000 updates and the
|
|
||||||
streamer's worker for it waits for room, so the streamer's stop takes the
|
|
||||||
whole 8 seconds and every handler then skips its last write. Lost then are the
|
|
||||||
messages left on the four handlers' queues, what the AS, peer and peering
|
|
||||||
handlers hold, the prefix handler's batch, and the write it has in progress,
|
|
||||||
up to another 250,000 updates, which take about 20 seconds and are cut off by
|
|
||||||
the exit; withdrawals and peer downs are among them. A route withdrawn in them
|
|
||||||
stays in the live route table until its peer announces or withdraws the prefix
|
|
||||||
again or its session goes down. When the 8 seconds run out the database is not
|
|
||||||
closed, and SQLite rolls back a write the exit cut off when the database next
|
|
||||||
opens.
|
|
||||||
|
|
||||||
## Development
|
## Development
|
||||||
|
|
||||||
@@ -568,11 +377,6 @@ provide:
|
|||||||
or `go.sum`) and `go fmt` (writes), then `script/check`
|
or `go.sum`) and `go fmt` (writes), then `script/check`
|
||||||
- `script/install-precommit` — install the git pre-commit hook that runs
|
- `script/install-precommit` — install the git pre-commit hook that runs
|
||||||
`script/precommit`
|
`script/precommit`
|
||||||
- `script/asupdate` — rebuild the AS list built into RouteWatch,
|
|
||||||
`pkg/asinfo/asdata.json.gz`, from the `as.csv` file of
|
|
||||||
[ipverse/as-metadata](https://github.com/ipverse/as-metadata). It needs the
|
|
||||||
network. It runs `cmd/asinfo-gen`, which fails when it finds no AS. When
|
|
||||||
`cmd/asinfo-gen` fails, the old list stays and the script exits non-zero.
|
|
||||||
|
|
||||||
Tests and the linter run only in Docker, as the `test` and `lint` phases of the
|
Tests and the linter run only in Docker, as the `test` and `lint` phases of the
|
||||||
`Dockerfile`; the image cannot be built unless both pass. Every `docker build`
|
`Dockerfile`; the image cannot be built unless both pass. Every `docker build`
|
||||||
|
|||||||
@@ -25,97 +25,11 @@ are measured by a benchmark, not yet on the live feed.
|
|||||||
|
|
||||||
# Next Step
|
# Next Step
|
||||||
|
|
||||||
Run `next` on the live feed, as README "Running under upaas" describes, for the
|
`next` waits for sneak to merge it to `main` through
|
||||||
final check https://git.eeqj.de/sneak/routewatch/issues/96 names: whether the
|
https://git.eeqj.de/sneak/routewatch/pulls/87.
|
||||||
prefix handler still drops updates after the first hour. If it does, a new issue
|
|
||||||
follows with the numbers. Then an audit of the code at the commit that run
|
|
||||||
tested.
|
|
||||||
|
|
||||||
# Completed Steps
|
# Completed Steps
|
||||||
|
|
||||||
- 2026-10-27: once the stop deadline has passed, the prefix handler starts no
|
|
||||||
flush. A batch that filled while a write ran past the deadline could still be
|
|
||||||
written after it (closes #188)
|
|
||||||
- 2026-10-27: the prefix handler applies each update to the routing table in
|
|
||||||
memory, and writes the table's changes to the database every second. A write
|
|
||||||
takes at most 20,000 changed pairs, and the rest follow in the next writes. A
|
|
||||||
failed write is written again later. Startup reads the table before the
|
|
||||||
streamer connects. The pages still read the per-peer route tables (part of
|
|
||||||
#122)
|
|
||||||
- 2026-10-27: the database keeps a copy of the whole routing table in a new
|
|
||||||
`routes` table, one row per prefix and origin AS. The peers' numbers are in a
|
|
||||||
new `route_peers` table. One function writes the pairs that changed in memory,
|
|
||||||
and another reads the whole table back into memory at startup. Nothing uses
|
|
||||||
them yet (part of #122)
|
|
||||||
- 2026-10-27: a new package, `internal/routetable`, holds the whole routing
|
|
||||||
table in memory. For each prefix and origin AS it keeps the peers that carry
|
|
||||||
it, one bit per peer, and the two newest of their routes. Nothing uses it yet
|
|
||||||
(part of #122)
|
|
||||||
- 2026-10-27: each result of the IP lookup lists the public blocklists the
|
|
||||||
address is on, in `blocklists`. The lists are DShield, Feodo Tracker, two
|
|
||||||
FireHOL lists, Spamhaus DROP and the Tor exit list. RouteWatch downloads each
|
|
||||||
list on its own schedule and answers from copies in memory. The copies are
|
|
||||||
also kept in the state directory's `blocklists` directory. IP lookup answers
|
|
||||||
carry `Cache-Control`. An address with no live route now has a result too,
|
|
||||||
without route fields, so the IP lookup no longer answers 404 (closes #158)
|
|
||||||
- 2026-10-27: the home page has an IP information API card. It shows the `curl`
|
|
||||||
command that looks up the visitor's own address at `/ip/{addr}`, and that
|
|
||||||
command's live answer (closes #156)
|
|
||||||
- 2026-10-27: on the AS page, a click on the heading of the IPv4 or the IPv6
|
|
||||||
prefix list folds or unfolds the list. Both lists start open (closes #155)
|
|
||||||
- 2026-10-27: the AS page lists its IPv4 and its IPv6 prefixes by mask length,
|
|
||||||
shortest first, and prefixes of one mask length by address. They used to be
|
|
||||||
listed by address (closes #153)
|
|
||||||
- 2026-10-27: on `/` and `/status`, a spinner beside the status turns one frame
|
|
||||||
each time the stats show more messages than the time before. A page that has
|
|
||||||
not seen more messages for three refreshes shows "stale" and the time since
|
|
||||||
then instead, in red (closes #152)
|
|
||||||
- 2026-10-27: the WHOIS details of a Brazilian AS keep their accented letters.
|
|
||||||
LACNIC's answer for such an AS comes from NIC.br, Brazil's registry, with each
|
|
||||||
accented letter replaced. The details now come from NIC.br's own server,
|
|
||||||
`whois.registro.br`. When that query fails, the details from LACNIC's answer
|
|
||||||
are kept and a warning is logged. After NIC.br answers that RouteWatch is over
|
|
||||||
its query limit, RouteWatch stops asking NIC.br for an hour (closes #174)
|
|
||||||
- 2026-10-27: `make asupdate` stops with an error when the download of `as.csv`
|
|
||||||
breaks off. The generator used to retry the failed read forever. A record that
|
|
||||||
does not parse is still logged and skipped (closes #177)
|
|
||||||
- 2026-10-27: `make asupdate` runs the new `script/asupdate`. It replaces the AS
|
|
||||||
list built into RouteWatch only when the generator succeeds and finds at least
|
|
||||||
one AS. A failed download used to replace the list with an empty file. The
|
|
||||||
generator reads `as.csv` from `ipverse/as-metadata`, the source's new name
|
|
||||||
(closes #173)
|
|
||||||
- 2026-10-27: the WHOIS details of an AS at RIPE, APNIC or AFRINIC come only
|
|
||||||
from its `aut-num` object and the organisation object it names. RIPE and APNIC
|
|
||||||
start their answer with the `as-block` object of the AS's block. Its dates
|
|
||||||
were stored as the AS's (closes #168)
|
|
||||||
- 2026-10-27: AS names keep their accented letters. The AS list built into
|
|
||||||
RouteWatch is rebuilt from today's `as.csv`, which added a country code
|
|
||||||
column. That file no longer lists AS 0 or the documentation AS numbers. A
|
|
||||||
WHOIS answer that is not valid UTF-8 is decoded as Latin-1, which RIPE and
|
|
||||||
APNIC send, before it is parsed and stored (closes #165)
|
|
||||||
- 2026-10-27: the Age column on the AS and prefix pages reads in words, such as
|
|
||||||
"5 minutes ago" or "2 weeks ago", instead of Go durations such as "360h0m0s
|
|
||||||
ago". An age under a minute reads in seconds, and one over a month in months
|
|
||||||
or years instead of a date (closes #163)
|
|
||||||
- 2026-10-26: a write to a stored AS, peering or peer keeps the earlier of the
|
|
||||||
two first-seen times and the later of the two last-seen times. An old update
|
|
||||||
from a collector catching up on a backlog no longer moves last seen back
|
|
||||||
(closes #162)
|
|
||||||
- 2026-10-26: the status page shows "Route Churn/min", the number of prefix
|
|
||||||
announcements and withdrawals, from all peers, in the last full clock minute.
|
|
||||||
`/api/v1/stats` reports it as `route_churn_per_min` in `stream`, which
|
|
||||||
replaces the moving average `route_churn_per_sec`. The messages and updates
|
|
||||||
per second on `/` and `/status` are whole numbers (closes #160)
|
|
||||||
- 2026-10-26: the WHOIS fetcher backs off more slowly. A failure makes its wait
|
|
||||||
1.5 times as long, up to one minute instead of five. Its queries to RIPE,
|
|
||||||
APNIC and AFRINIC leave out person and role objects, which RIPE counts against
|
|
||||||
a daily limit per address. A refused query is logged with the RIR's error line
|
|
||||||
(closes #159)
|
|
||||||
- 2026-10-26: the home page's AS name search lists the ASes whose handle or
|
|
||||||
description holds the text, as links to their pages, from the new
|
|
||||||
`/api/v1/search/as`; the search ignores case. When the database has no record
|
|
||||||
of an AS, its page takes the handle and description from the AS list built
|
|
||||||
into RouteWatch and still lists the AS's prefixes (closes #151)
|
|
||||||
- 2026-10-19: README "HTTP Endpoints" lists every route with what it answers,
|
- 2026-10-19: README "HTTP Endpoints" lists every route with what it answers,
|
||||||
"Database Schema" matches `schema.sql`, and the rest of the README matches the
|
"Database Schema" matches `schema.sql`, and the rest of the README matches the
|
||||||
code; this file brought up to date (closes #72)
|
code; this file brought up to date (closes #72)
|
||||||
@@ -262,6 +176,4 @@ tested.
|
|||||||
|
|
||||||
# Future Steps
|
# Future Steps
|
||||||
|
|
||||||
- the rest of https://git.eeqj.de/sneak/routewatch/issues/122, one PR each: the
|
None open.
|
||||||
pages and the API reading from the routing table, then removing the per-peer
|
|
||||||
route tables
|
|
||||||
|
|||||||
+28
-60
@@ -6,7 +6,6 @@ import (
|
|||||||
"encoding/csv"
|
"encoding/csv"
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
"errors"
|
"errors"
|
||||||
"fmt"
|
|
||||||
"io"
|
"io"
|
||||||
"log"
|
"log"
|
||||||
"net/http"
|
"net/http"
|
||||||
@@ -15,21 +14,7 @@ import (
|
|||||||
"strings"
|
"strings"
|
||||||
)
|
)
|
||||||
|
|
||||||
const asCsvURL = "https://github.com/ipverse/as-metadata/raw/refs/heads/master/as.csv"
|
const asCsvURL = "https://github.com/ipverse/asn-info/raw/refs/heads/master/as.csv"
|
||||||
|
|
||||||
// The first columns of as.csv, the ones the generator reads. It reads past
|
|
||||||
// the columns after them, such as the country code.
|
|
||||||
const (
|
|
||||||
asnColumn = "asn"
|
|
||||||
handleColumn = "handle"
|
|
||||||
descriptionColumn = "description"
|
|
||||||
columnsRead = 3
|
|
||||||
)
|
|
||||||
|
|
||||||
var (
|
|
||||||
errUnexpectedHeader = errors.New("unexpected CSV header")
|
|
||||||
errNoASes = errors.New("no AS in the CSV")
|
|
||||||
)
|
|
||||||
|
|
||||||
// ASInfo represents information about an Autonomous System
|
// ASInfo represents information about an Autonomous System
|
||||||
type ASInfo struct {
|
type ASInfo struct {
|
||||||
@@ -60,66 +45,40 @@ func main() {
|
|||||||
log.Fatalf("Failed to fetch CSV: HTTP %d", resp.StatusCode)
|
log.Fatalf("Failed to fetch CSV: HTTP %d", resp.StatusCode)
|
||||||
}
|
}
|
||||||
|
|
||||||
asInfos, err := parseCSV(resp.Body)
|
// Parse CSV
|
||||||
|
reader := csv.NewReader(resp.Body)
|
||||||
// Closed here rather than deferred: log.Fatalf exits without running
|
|
||||||
// deferred calls.
|
|
||||||
_ = resp.Body.Close()
|
|
||||||
|
|
||||||
if err != nil {
|
|
||||||
log.Fatalf("Failed to parse CSV: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
log.Printf("Parsed %d AS records", len(asInfos))
|
|
||||||
|
|
||||||
// Convert to JSON and output to stdout
|
|
||||||
encoder := json.NewEncoder(os.Stdout)
|
|
||||||
encoder.SetIndent("", "\t")
|
|
||||||
|
|
||||||
err = encoder.Encode(asInfos)
|
|
||||||
if err != nil {
|
|
||||||
log.Fatalf("Failed to encode JSON: %v", err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// parseCSV reads as.csv from r and returns its ASes, with the handle and the
|
|
||||||
// description as the file spells them. A header whose first columns are not
|
|
||||||
// the AS number, the handle and the description is an error. A record that
|
|
||||||
// does not parse is logged and skipped. A read that fails, such as on a broken
|
|
||||||
// download, is an error. A file left with no AS is an error.
|
|
||||||
func parseCSV(r io.Reader) ([]ASInfo, error) {
|
|
||||||
reader := csv.NewReader(r)
|
|
||||||
|
|
||||||
|
// Read header
|
||||||
header, err := reader.Read()
|
header, err := reader.Read()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("read CSV header: %w", err)
|
log.Fatalf("Failed to read CSV header: %v", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
if len(header) < columnsRead || header[0] != asnColumn ||
|
if len(header) != 3 || header[0] != "asn" || header[1] != "handle" ||
|
||||||
header[1] != handleColumn || header[2] != descriptionColumn {
|
header[2] != "description" {
|
||||||
return nil, fmt.Errorf("%w: %v", errUnexpectedHeader, header)
|
log.Fatalf("Unexpected CSV header: %v", header)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Read all records
|
||||||
var asInfos []ASInfo
|
var asInfos []ASInfo
|
||||||
|
|
||||||
for {
|
for {
|
||||||
// The reader returns an error for a record with a different number
|
|
||||||
// of fields than the header.
|
|
||||||
record, err := reader.Read()
|
record, err := reader.Read()
|
||||||
if errors.Is(err, io.EOF) {
|
if errors.Is(err, io.EOF) {
|
||||||
break
|
break
|
||||||
}
|
}
|
||||||
|
|
||||||
if _, ok := errors.AsType[*csv.ParseError](err); ok {
|
if err != nil {
|
||||||
log.Printf("Error reading CSV record: %v", err)
|
log.Printf("Error reading CSV record: %v", err)
|
||||||
|
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
|
||||||
// Any other error comes from the download, such as a broken
|
const expectedFields = 3
|
||||||
// connection. Every later read returns it again.
|
if len(record) != expectedFields {
|
||||||
if err != nil {
|
log.Printf("Skipping invalid record: %v", record)
|
||||||
return nil, fmt.Errorf("read CSV record: %w", err)
|
|
||||||
|
continue
|
||||||
}
|
}
|
||||||
|
|
||||||
asn, err := strconv.Atoi(record[0])
|
asn, err := strconv.Atoi(record[0])
|
||||||
@@ -136,9 +95,18 @@ func parseCSV(r io.Reader) ([]ASInfo, error) {
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
if len(asInfos) == 0 {
|
// Closed here rather than deferred: log.Fatalf exits without running
|
||||||
return nil, errNoASes
|
// deferred calls.
|
||||||
}
|
_ = resp.Body.Close()
|
||||||
|
|
||||||
return asInfos, nil
|
log.Printf("Parsed %d AS records", len(asInfos))
|
||||||
|
|
||||||
|
// Convert to JSON and output to stdout
|
||||||
|
encoder := json.NewEncoder(os.Stdout)
|
||||||
|
encoder.SetIndent("", "\t")
|
||||||
|
|
||||||
|
err = encoder.Encode(asInfos)
|
||||||
|
if err != nil {
|
||||||
|
log.Fatalf("Failed to encode JSON: %v", err)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,142 +0,0 @@
|
|||||||
package main
|
|
||||||
|
|
||||||
import (
|
|
||||||
"errors"
|
|
||||||
"io"
|
|
||||||
"slices"
|
|
||||||
"strings"
|
|
||||||
"testing"
|
|
||||||
"testing/iotest"
|
|
||||||
"time"
|
|
||||||
)
|
|
||||||
|
|
||||||
// parseTimeLimit is how long a test waits for parseCSV to return before it
|
|
||||||
// fails.
|
|
||||||
const parseTimeLimit = 2 * time.Second
|
|
||||||
|
|
||||||
// TestParseCSVReadsTodaysLayout parses an as.csv with the country code column
|
|
||||||
// that the source added after the description. It expects each AS with its
|
|
||||||
// handle and description spelled exactly as in the file, accents included.
|
|
||||||
func TestParseCSVReadsTodaysLayout(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
const asCSV = `asn,handle,description,country-code
|
|
||||||
1,LVLT-1,Level 3 Parent LLC,US
|
|
||||||
1251,FUNDACAO-AMPARO-PESQUISA,FUNDAÇÃO DE AMPARO À PESQUISA DO ESTADO SÃO PAULO,BR
|
|
||||||
`
|
|
||||||
|
|
||||||
got, err := parseCSV(strings.NewReader(asCSV))
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("parseCSV: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
want := []ASInfo{
|
|
||||||
{ASN: 1, Handle: "LVLT-1", Description: "Level 3 Parent LLC"},
|
|
||||||
{
|
|
||||||
ASN: 1251,
|
|
||||||
Handle: "FUNDACAO-AMPARO-PESQUISA",
|
|
||||||
Description: "FUNDAÇÃO DE AMPARO À PESQUISA DO ESTADO " +
|
|
||||||
"SÃO PAULO",
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
if !slices.Equal(got, want) {
|
|
||||||
t.Errorf("parseCSV = %+v, want %+v", got, want)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestParseCSVRejectsAnotherHeader parses files whose first columns are not
|
|
||||||
// the AS number, the handle and the description. It expects an error rather
|
|
||||||
// than a list read from the wrong columns.
|
|
||||||
func TestParseCSVRejectsAnotherHeader(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
for _, asCSV := range []string{
|
|
||||||
"asn,handle\n1,LVLT-1\n",
|
|
||||||
"asn,description,handle\n1,Level 3 Parent LLC,LVLT-1\n",
|
|
||||||
} {
|
|
||||||
_, err := parseCSV(strings.NewReader(asCSV))
|
|
||||||
if err == nil {
|
|
||||||
t.Errorf("parseCSV(%q): no error", asCSV)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestParseCSVRejectsAFileWithNoASes parses files that hold no AS, one with
|
|
||||||
// only the header and one whose only record has no valid AS number. It
|
|
||||||
// expects errNoASes, so that make asupdate keeps the old list.
|
|
||||||
func TestParseCSVRejectsAFileWithNoASes(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
for _, asCSV := range []string{
|
|
||||||
"asn,handle,description,country-code\n",
|
|
||||||
"asn,handle,description,country-code\nAS1,LVLT-1,Level 3,US\n",
|
|
||||||
} {
|
|
||||||
_, err := parseCSV(strings.NewReader(asCSV))
|
|
||||||
if !errors.Is(err, errNoASes) {
|
|
||||||
t.Errorf("parseCSV(%q) = %v, want %v", asCSV, err, errNoASes)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestParseCSVSkipsARecordThatDoesNotParse parses an as.csv with a record
|
|
||||||
// that has too few fields between two valid records. It expects both valid
|
|
||||||
// ASes and no error, so that one bad record does not stop make asupdate.
|
|
||||||
func TestParseCSVSkipsARecordThatDoesNotParse(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
const asCSV = "asn,handle,description,country-code\n" +
|
|
||||||
"1,LVLT-1,Level 3 Parent LLC,US\n" +
|
|
||||||
"2,UDEL-DCN\n" +
|
|
||||||
"3,MIT-GATEWAYS,Massachusetts Institute of Technology,US\n"
|
|
||||||
|
|
||||||
got, err := parseCSV(strings.NewReader(asCSV))
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("parseCSV: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
want := []ASInfo{
|
|
||||||
{ASN: 1, Handle: "LVLT-1", Description: "Level 3 Parent LLC"},
|
|
||||||
{
|
|
||||||
ASN: 3,
|
|
||||||
Handle: "MIT-GATEWAYS",
|
|
||||||
Description: "Massachusetts Institute of Technology",
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
if !slices.Equal(got, want) {
|
|
||||||
t.Errorf("parseCSV = %+v, want %+v", got, want)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestParseCSVStopsWhenTheDownloadBreaksOff parses an as.csv whose download
|
|
||||||
// breaks off after the first record. From then on every read fails, as on a
|
|
||||||
// broken connection. It expects parseCSV to return that error, so that make
|
|
||||||
// asupdate keeps the old list.
|
|
||||||
func TestParseCSVStopsWhenTheDownloadBreaksOff(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
const asCSV = "asn,handle,description,country-code\n" +
|
|
||||||
"1,LVLT-1,Level 3 Parent LLC,US\n"
|
|
||||||
|
|
||||||
download := io.MultiReader(
|
|
||||||
strings.NewReader(asCSV), iotest.ErrReader(io.ErrUnexpectedEOF))
|
|
||||||
|
|
||||||
// parseCSV runs on its own goroutine, so that a parseCSV that never
|
|
||||||
// returns fails the test instead of hanging it.
|
|
||||||
parsed := make(chan error, 1)
|
|
||||||
|
|
||||||
go func() {
|
|
||||||
_, err := parseCSV(download)
|
|
||||||
parsed <- err
|
|
||||||
}()
|
|
||||||
|
|
||||||
select {
|
|
||||||
case err := <-parsed:
|
|
||||||
if !errors.Is(err, io.ErrUnexpectedEOF) {
|
|
||||||
t.Errorf("parseCSV = %v, want %v", err, io.ErrUnexpectedEOF)
|
|
||||||
}
|
|
||||||
case <-time.After(parseTimeLimit):
|
|
||||||
t.Fatalf("parseCSV did not return within %v", parseTimeLimit)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -35,7 +35,7 @@ func main() {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
log.Fatal("Failed to start streamer:", err)
|
log.Fatal("Failed to start streamer:", err)
|
||||||
}
|
}
|
||||||
defer s.Stop(context.Background())
|
defer s.Stop()
|
||||||
|
|
||||||
// Set up context with cancellation
|
// Set up context with cancellation
|
||||||
ctx, cancel := context.WithCancel(context.Background())
|
ctx, cancel := context.WithCancel(context.Background())
|
||||||
|
|||||||
@@ -1,19 +1,15 @@
|
|||||||
module git.eeqj.de/sneak/routewatch
|
module git.eeqj.de/sneak/routewatch
|
||||||
|
|
||||||
go 1.27.1
|
go 1.24.4
|
||||||
|
|
||||||
require (
|
require (
|
||||||
github.com/bits-and-blooms/bitset v1.26.0
|
|
||||||
github.com/dustin/go-humanize v1.0.1
|
github.com/dustin/go-humanize v1.0.1
|
||||||
github.com/go-chi/chi/v5 v5.2.2
|
github.com/go-chi/chi/v5 v5.2.2
|
||||||
github.com/google/btree v1.1.3
|
|
||||||
github.com/google/uuid v1.6.0
|
github.com/google/uuid v1.6.0
|
||||||
github.com/mattn/go-sqlite3 v1.14.29
|
github.com/mattn/go-sqlite3 v1.14.29
|
||||||
github.com/rcrowley/go-metrics v0.0.0-20250401214520-65e299d6c5c9
|
github.com/rcrowley/go-metrics v0.0.0-20250401214520-65e299d6c5c9
|
||||||
go.uber.org/fx v1.24.0
|
go.uber.org/fx v1.24.0
|
||||||
go4.org/netipx v0.0.0-20260823151212-3075585bcbeb
|
|
||||||
golang.org/x/term v0.33.0
|
golang.org/x/term v0.33.0
|
||||||
golang.org/x/text v0.43.0
|
|
||||||
)
|
)
|
||||||
|
|
||||||
require (
|
require (
|
||||||
|
|||||||
@@ -1,13 +1,9 @@
|
|||||||
github.com/bits-and-blooms/bitset v1.26.0 h1:WPvcPZbP1yuKofmjs2SYIFO1EjzIGhCP9ducElLsh5Q=
|
|
||||||
github.com/bits-and-blooms/bitset v1.26.0/go.mod h1:7hO7Gc7Pp1vODcmWvKMRA9BNmbv6a/7QIWpPxHddWR8=
|
|
||||||
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
|
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
|
||||||
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||||
github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
|
github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
|
||||||
github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
|
github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
|
||||||
github.com/go-chi/chi/v5 v5.2.2 h1:CMwsvRVTbXVytCk1Wd72Zy1LAsAh9GxMmSNWLHCG618=
|
github.com/go-chi/chi/v5 v5.2.2 h1:CMwsvRVTbXVytCk1Wd72Zy1LAsAh9GxMmSNWLHCG618=
|
||||||
github.com/go-chi/chi/v5 v5.2.2/go.mod h1:L2yAIGWB3H+phAw1NxKwWM+7eUH/lU8pOMm5hHcoops=
|
github.com/go-chi/chi/v5 v5.2.2/go.mod h1:L2yAIGWB3H+phAw1NxKwWM+7eUH/lU8pOMm5hHcoops=
|
||||||
github.com/google/btree v1.1.3 h1:CVpQJjYgC4VbzxeGVHfvZrv1ctoYCAI8vbl07Fcxlyg=
|
|
||||||
github.com/google/btree v1.1.3/go.mod h1:qOPhT0dTNdNzV6Z/lhRX0YXUafgPLFUh+gZMl761Gm4=
|
|
||||||
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
|
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
|
||||||
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
|
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
|
||||||
github.com/mattn/go-sqlite3 v1.14.29 h1:1O6nRLJKvsi1H2Sj0Hzdfojwt8GiGKm+LOfLaBFaouQ=
|
github.com/mattn/go-sqlite3 v1.14.29 h1:1O6nRLJKvsi1H2Sj0Hzdfojwt8GiGKm+LOfLaBFaouQ=
|
||||||
@@ -28,13 +24,9 @@ go.uber.org/multierr v1.10.0 h1:S0h4aNzvfcFsC3dRF1jLoaov7oRaKqRGC/pUEJ2yvPQ=
|
|||||||
go.uber.org/multierr v1.10.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y=
|
go.uber.org/multierr v1.10.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y=
|
||||||
go.uber.org/zap v1.26.0 h1:sI7k6L95XOKS281NhVKOFCUNIvv9e0w4BF8N3u+tCRo=
|
go.uber.org/zap v1.26.0 h1:sI7k6L95XOKS281NhVKOFCUNIvv9e0w4BF8N3u+tCRo=
|
||||||
go.uber.org/zap v1.26.0/go.mod h1:dtElttAiwGvoJ/vj4IwHBS/gXsEu/pZ50mUIRWuG0so=
|
go.uber.org/zap v1.26.0/go.mod h1:dtElttAiwGvoJ/vj4IwHBS/gXsEu/pZ50mUIRWuG0so=
|
||||||
go4.org/netipx v0.0.0-20260823151212-3075585bcbeb h1:XBM4hvfwGAttkkiTIFfeigdfcL1xIfdKXqFdgiHGtDs=
|
|
||||||
go4.org/netipx v0.0.0-20260823151212-3075585bcbeb/go.mod h1:PLyyIXexvUFg3Owu6p/WfdlivPbZJsZdgWZlrGope/Y=
|
|
||||||
golang.org/x/sys v0.34.0 h1:H5Y5sJ2L2JRdyv7ROF1he/lPdvFsd0mJHFw2ThKHxLA=
|
golang.org/x/sys v0.34.0 h1:H5Y5sJ2L2JRdyv7ROF1he/lPdvFsd0mJHFw2ThKHxLA=
|
||||||
golang.org/x/sys v0.34.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k=
|
golang.org/x/sys v0.34.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k=
|
||||||
golang.org/x/term v0.33.0 h1:NuFncQrRcaRvVmgRkvM3j/F00gWIAlcmlB8ACEKmGIg=
|
golang.org/x/term v0.33.0 h1:NuFncQrRcaRvVmgRkvM3j/F00gWIAlcmlB8ACEKmGIg=
|
||||||
golang.org/x/term v0.33.0/go.mod h1:s18+ql9tYWp1IfpV9DmCtQDDSRBUjKaw9M1eAv5UeF0=
|
golang.org/x/term v0.33.0/go.mod h1:s18+ql9tYWp1IfpV9DmCtQDDSRBUjKaw9M1eAv5UeF0=
|
||||||
golang.org/x/text v0.43.0 h1:1QivrlhwAsnMOcqgOdl9my7s9OyZl8O5/UGUTJVXoWc=
|
|
||||||
golang.org/x/text v0.43.0/go.mod h1:hbSIYA/amXcRXTFSZqh8tWHEoqRvvbKPJ2uz/KU+a+s=
|
|
||||||
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
|
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
|
||||||
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||||
|
|||||||
@@ -1,184 +0,0 @@
|
|||||||
// Package blocklist downloads public IP blocklists on a schedule, keeps the
|
|
||||||
// last good copy of each on disk and in memory, and answers which lists
|
|
||||||
// contain an address.
|
|
||||||
package blocklist
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"log/slog"
|
|
||||||
"net/netip"
|
|
||||||
"sync"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"go4.org/netipx"
|
|
||||||
)
|
|
||||||
|
|
||||||
// Format is how a list's file is written.
|
|
||||||
type Format int
|
|
||||||
|
|
||||||
const (
|
|
||||||
// FormatLines has one address or network per line, such as 192.0.2.1 or
|
|
||||||
// 192.0.2.0/24. Lines that start with # are comments.
|
|
||||||
FormatLines Format = iota
|
|
||||||
|
|
||||||
// FormatSpamhausJSON has one JSON object per line, with a network in its
|
|
||||||
// cidr field. The last line holds the file's metadata and no network.
|
|
||||||
FormatSpamhausJSON
|
|
||||||
|
|
||||||
// FormatDShield has one network per line, as tab-separated fields that
|
|
||||||
// start with the network's first and last address. Lines that start
|
|
||||||
// with # are comments.
|
|
||||||
FormatDShield
|
|
||||||
)
|
|
||||||
|
|
||||||
// List is one public blocklist.
|
|
||||||
type List struct {
|
|
||||||
// Name names the list in lookup answers, and names its copy on disk.
|
|
||||||
Name string
|
|
||||||
|
|
||||||
// Description says in one line what being on the list means.
|
|
||||||
Description string
|
|
||||||
|
|
||||||
// URL is where the publisher serves the whole list.
|
|
||||||
URL string
|
|
||||||
|
|
||||||
// Interval is how long a copy is used before the list is downloaded
|
|
||||||
// again.
|
|
||||||
Interval time.Duration
|
|
||||||
|
|
||||||
// Format is how the list's file is written.
|
|
||||||
Format Format
|
|
||||||
}
|
|
||||||
|
|
||||||
// Listing says that a list contains an address.
|
|
||||||
type Listing struct {
|
|
||||||
Name string `json:"name"`
|
|
||||||
Description string `json:"description"`
|
|
||||||
|
|
||||||
// Updated is when routewatch last downloaded the list, or last heard from
|
|
||||||
// its publisher that its copy was still current.
|
|
||||||
Updated time.Time `json:"updated"`
|
|
||||||
}
|
|
||||||
|
|
||||||
// listCopy is a list's last good copy in memory.
|
|
||||||
type listCopy struct {
|
|
||||||
addresses *netipx.IPSet
|
|
||||||
updated time.Time
|
|
||||||
|
|
||||||
// validators identify the version of the list this copy holds.
|
|
||||||
validators validators
|
|
||||||
}
|
|
||||||
|
|
||||||
// Params is what New needs.
|
|
||||||
type Params struct {
|
|
||||||
// Dir is the directory that holds the copies on disk.
|
|
||||||
Dir string
|
|
||||||
|
|
||||||
// Lists are the lists to download and look addresses up in.
|
|
||||||
Lists []List
|
|
||||||
|
|
||||||
Logger *slog.Logger
|
|
||||||
}
|
|
||||||
|
|
||||||
// Blocklists keeps a copy of each list and answers which lists contain an
|
|
||||||
// address. Start loads the copies from disk and starts downloading the lists.
|
|
||||||
type Blocklists struct {
|
|
||||||
dir string
|
|
||||||
lists []List
|
|
||||||
logger *slog.Logger
|
|
||||||
|
|
||||||
// copies holds each list's last good copy, by list name. A list with no
|
|
||||||
// good copy yet has none.
|
|
||||||
mu sync.RWMutex
|
|
||||||
copies map[string]listCopy
|
|
||||||
|
|
||||||
// stop ends the goroutine Start runs, and done is closed when it returns.
|
|
||||||
stop context.CancelFunc
|
|
||||||
done chan struct{}
|
|
||||||
}
|
|
||||||
|
|
||||||
// New returns a Blocklists with no copies. Start loads them.
|
|
||||||
func New(params Params) *Blocklists {
|
|
||||||
return &Blocklists{
|
|
||||||
dir: params.Dir,
|
|
||||||
lists: params.Lists,
|
|
||||||
logger: params.Logger.With("component", "blocklists"),
|
|
||||||
copies: make(map[string]listCopy),
|
|
||||||
done: make(chan struct{}),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Lookup returns a Listing for each list whose copy contains addr, in the
|
|
||||||
// order of the lists given to New. It makes no outside request. When no list
|
|
||||||
// contains addr, it returns an empty list, not nil.
|
|
||||||
func (b *Blocklists) Lookup(addr netip.Addr) []Listing {
|
|
||||||
b.mu.RLock()
|
|
||||||
defer b.mu.RUnlock()
|
|
||||||
|
|
||||||
listings := []Listing{}
|
|
||||||
|
|
||||||
for _, list := range b.lists {
|
|
||||||
c, ok := b.copies[list.Name]
|
|
||||||
if ok && c.addresses.Contains(addr.Unmap()) {
|
|
||||||
listings = append(listings, Listing{
|
|
||||||
Name: list.Name,
|
|
||||||
Description: list.Description,
|
|
||||||
Updated: c.updated,
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return listings
|
|
||||||
}
|
|
||||||
|
|
||||||
// ShortestInterval returns the shortest Interval of the lists. That is the
|
|
||||||
// soonest a copy can change, and so how long a lookup answer stays current.
|
|
||||||
func (b *Blocklists) ShortestInterval() time.Duration {
|
|
||||||
var shortest time.Duration
|
|
||||||
|
|
||||||
for i, list := range b.lists {
|
|
||||||
if i == 0 || list.Interval < shortest {
|
|
||||||
shortest = list.Interval
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return shortest
|
|
||||||
}
|
|
||||||
|
|
||||||
// updated returns when the named list's copy was last downloaded or found
|
|
||||||
// current, or the zero time when the list has no copy.
|
|
||||||
func (b *Blocklists) updated(name string) time.Time {
|
|
||||||
b.mu.RLock()
|
|
||||||
defer b.mu.RUnlock()
|
|
||||||
|
|
||||||
return b.copies[name].updated
|
|
||||||
}
|
|
||||||
|
|
||||||
// copyValidators returns the validators of the named list's copy. They are
|
|
||||||
// empty when the list has no copy.
|
|
||||||
func (b *Blocklists) copyValidators(name string) validators {
|
|
||||||
b.mu.RLock()
|
|
||||||
defer b.mu.RUnlock()
|
|
||||||
|
|
||||||
return b.copies[name].validators
|
|
||||||
}
|
|
||||||
|
|
||||||
// setCopy makes c the named list's copy.
|
|
||||||
func (b *Blocklists) setCopy(name string, c listCopy) {
|
|
||||||
b.mu.Lock()
|
|
||||||
defer b.mu.Unlock()
|
|
||||||
|
|
||||||
b.copies[name] = c
|
|
||||||
}
|
|
||||||
|
|
||||||
// setUpdated records that the named list's copy was found current at t.
|
|
||||||
func (b *Blocklists) setUpdated(name string, t time.Time) {
|
|
||||||
b.mu.Lock()
|
|
||||||
defer b.mu.Unlock()
|
|
||||||
|
|
||||||
c, ok := b.copies[name]
|
|
||||||
if ok {
|
|
||||||
c.updated = t
|
|
||||||
b.copies[name] = c
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,494 +0,0 @@
|
|||||||
package blocklist
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"errors"
|
|
||||||
"io"
|
|
||||||
"io/fs"
|
|
||||||
"log/slog"
|
|
||||||
"net/http"
|
|
||||||
"net/http/httptest"
|
|
||||||
"net/netip"
|
|
||||||
"os"
|
|
||||||
"path/filepath"
|
|
||||||
"strings"
|
|
||||||
"sync"
|
|
||||||
"testing"
|
|
||||||
"time"
|
|
||||||
)
|
|
||||||
|
|
||||||
// The test list's name and description, the address its good version lists,
|
|
||||||
// and the address a newer version lists instead.
|
|
||||||
const (
|
|
||||||
testListName = "test_list"
|
|
||||||
testListDescription = "On the test list"
|
|
||||||
listedAddr = "192.0.2.1"
|
|
||||||
goodListBody = "# A test list\n" + listedAddr + "\n"
|
|
||||||
newListedAddr = "192.0.2.2"
|
|
||||||
newListBody = "# A test list\n" + newListedAddr + "\n"
|
|
||||||
)
|
|
||||||
|
|
||||||
// testETag is the ETag a publisher sends with a version of the test list.
|
|
||||||
const testETag = `"v1"`
|
|
||||||
|
|
||||||
// maxWait is how long a test waits for the download goroutine.
|
|
||||||
const maxWait = 10 * time.Second
|
|
||||||
|
|
||||||
// publisher stands in for a list's publisher. It answers every request with
|
|
||||||
// its status and body, and keeps the headers of each request. A 200 answer
|
|
||||||
// carries lastModified and etag, when set, as Last-Modified and ETag. As
|
|
||||||
// from a real publisher, a request whose If-None-Match or If-Modified-Since
|
|
||||||
// matches them is answered "not modified".
|
|
||||||
type publisher struct {
|
|
||||||
mu sync.Mutex
|
|
||||||
status int
|
|
||||||
body string
|
|
||||||
lastModified time.Time
|
|
||||||
etag string
|
|
||||||
headers []http.Header
|
|
||||||
}
|
|
||||||
|
|
||||||
// newPublisher starts a publisher that answers 200 with body, and returns it
|
|
||||||
// and its URL.
|
|
||||||
func newPublisher(t *testing.T, body string) (*publisher, string) {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
p := &publisher{status: http.StatusOK, body: body}
|
|
||||||
srv := httptest.NewServer(p)
|
|
||||||
t.Cleanup(srv.Close)
|
|
||||||
|
|
||||||
return p, srv.URL
|
|
||||||
}
|
|
||||||
|
|
||||||
func (p *publisher) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
|
||||||
p.mu.Lock()
|
|
||||||
defer p.mu.Unlock()
|
|
||||||
|
|
||||||
p.headers = append(p.headers, r.Header.Clone())
|
|
||||||
|
|
||||||
if p.status != http.StatusOK {
|
|
||||||
w.WriteHeader(p.status)
|
|
||||||
_, _ = io.WriteString(w, p.body)
|
|
||||||
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
if p.etag != "" {
|
|
||||||
w.Header().Set(etagHeader, p.etag)
|
|
||||||
}
|
|
||||||
|
|
||||||
// ServeContent sends Last-Modified unless lastModified is zero, and
|
|
||||||
// answers "not modified" when the request's conditions match.
|
|
||||||
http.ServeContent(w, r, testListName, p.lastModified,
|
|
||||||
strings.NewReader(p.body))
|
|
||||||
}
|
|
||||||
|
|
||||||
// answer makes the publisher answer later requests with status and body.
|
|
||||||
func (p *publisher) answer(status int, body string) {
|
|
||||||
p.mu.Lock()
|
|
||||||
defer p.mu.Unlock()
|
|
||||||
|
|
||||||
p.status, p.body = status, body
|
|
||||||
}
|
|
||||||
|
|
||||||
// publish makes the publisher answer later requests with a version of the
|
|
||||||
// list: body, with lastModified and etag. A zero lastModified or an empty
|
|
||||||
// etag is not sent.
|
|
||||||
func (p *publisher) publish(body string, lastModified time.Time, etag string) {
|
|
||||||
p.mu.Lock()
|
|
||||||
defer p.mu.Unlock()
|
|
||||||
|
|
||||||
p.status, p.body = http.StatusOK, body
|
|
||||||
p.lastModified, p.etag = lastModified, etag
|
|
||||||
}
|
|
||||||
|
|
||||||
// requests returns the headers of the requests the publisher has had.
|
|
||||||
func (p *publisher) requests() []http.Header {
|
|
||||||
p.mu.Lock()
|
|
||||||
defer p.mu.Unlock()
|
|
||||||
|
|
||||||
return append([]http.Header(nil), p.headers...)
|
|
||||||
}
|
|
||||||
|
|
||||||
// testList returns a list served at url, written in format.
|
|
||||||
func testList(url string, format Format) List {
|
|
||||||
return List{
|
|
||||||
Name: testListName,
|
|
||||||
Description: testListDescription,
|
|
||||||
URL: url,
|
|
||||||
Interval: time.Hour,
|
|
||||||
Format: format,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func newTestBlocklists(dir string, list List) *Blocklists {
|
|
||||||
return New(Params{
|
|
||||||
Dir: dir,
|
|
||||||
Lists: []List{list},
|
|
||||||
Logger: slog.New(slog.DiscardHandler),
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
// lookup looks addr up in b.
|
|
||||||
func lookup(t *testing.T, b *Blocklists, addr string) []Listing {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
return b.Lookup(netip.MustParseAddr(addr))
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestLookupFindsListedAddresses downloads a list in each format and looks
|
|
||||||
// up addresses on it, inside its networks, and off it, in IPv4 and IPv6.
|
|
||||||
// A listed address has one listing that names the list; any other has an
|
|
||||||
// empty list of listings.
|
|
||||||
func TestLookupFindsListedAddresses(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
tests := []struct {
|
|
||||||
name string
|
|
||||||
format Format
|
|
||||||
body string
|
|
||||||
listed []string
|
|
||||||
unlisted []string
|
|
||||||
}{
|
|
||||||
{
|
|
||||||
name: "single addresses",
|
|
||||||
format: FormatLines,
|
|
||||||
body: "# comment\n192.0.2.1\n\n2001:db8::1\n",
|
|
||||||
listed: []string{"192.0.2.1", "::ffff:192.0.2.1", "2001:db8::1"},
|
|
||||||
unlisted: []string{"192.0.2.2", "2001:db8::2"},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "networks",
|
|
||||||
format: FormatLines,
|
|
||||||
body: "# comment\n198.51.100.0/24\n2001:db8:1::/48\n",
|
|
||||||
listed: []string{"198.51.100.0", "198.51.100.255", "2001:db8:1::5"},
|
|
||||||
unlisted: []string{"198.51.101.0", "2001:db8:2::1"},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "Spamhaus networks",
|
|
||||||
format: FormatSpamhausJSON,
|
|
||||||
body: `{"cidr":"203.0.113.0/25","sblid":"SBL1","rir":"arin"}` + "\n" +
|
|
||||||
`{"cidr":"2001:db8:3::/48","sblid":"SBL2","rir":"ripencc"}` + "\n" +
|
|
||||||
`{"type":"metadata","timestamp":1793074442,"records":2}` + "\n",
|
|
||||||
listed: []string{"203.0.113.1", "2001:db8:3::1"},
|
|
||||||
unlisted: []string{"203.0.113.200", "2001:db8:4::1"},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "DShield networks",
|
|
||||||
format: FormatDShield,
|
|
||||||
body: "# DShield.org Recommended Block List\n" +
|
|
||||||
"192.0.2.0\t192.0.2.255\t24\t327\tEXAMPLE\tUS\tNone\n",
|
|
||||||
listed: []string{"192.0.2.0", "192.0.2.77"},
|
|
||||||
unlisted: []string{"192.0.3.1", "2001:db8::1"},
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, tt := range tests {
|
|
||||||
t.Run(tt.name, func(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
_, url := newPublisher(t, tt.body)
|
|
||||||
b := newTestBlocklists(t.TempDir(), testList(url, tt.format))
|
|
||||||
|
|
||||||
err := b.download(t.Context(), b.lists[0])
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("download: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, addr := range tt.listed {
|
|
||||||
listings := lookup(t, b, addr)
|
|
||||||
if len(listings) != 1 || listings[0].Name != testListName ||
|
|
||||||
listings[0].Description != testListDescription ||
|
|
||||||
listings[0].Updated.IsZero() {
|
|
||||||
t.Errorf("%s: listings %+v, want one for %s", addr,
|
|
||||||
listings, testListName)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, addr := range tt.unlisted {
|
|
||||||
listings := lookup(t, b, addr)
|
|
||||||
if listings == nil || len(listings) != 0 {
|
|
||||||
t.Errorf("%s: listings %#v, want an empty list", addr, listings)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestFailedDownloadKeepsThePreviousCopy downloads a good version of a list,
|
|
||||||
// then one that fails. The good copy stays in memory and on disk, and no
|
|
||||||
// part of the failed download is left beside it.
|
|
||||||
func TestFailedDownloadKeepsThePreviousCopy(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
tests := []struct {
|
|
||||||
name string
|
|
||||||
status int
|
|
||||||
body string
|
|
||||||
}{
|
|
||||||
{"server error", http.StatusInternalServerError, "unavailable"},
|
|
||||||
{"not a list", http.StatusOK, "<!DOCTYPE html>\n<html></html>\n"},
|
|
||||||
{
|
|
||||||
"over the size limit", http.StatusOK,
|
|
||||||
strings.Repeat(listedAddr+"\n", maxDownloadSize/len(listedAddr)),
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, tt := range tests {
|
|
||||||
t.Run(tt.name, func(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
dir := t.TempDir()
|
|
||||||
p, url := newPublisher(t, goodListBody)
|
|
||||||
b := newTestBlocklists(dir, testList(url, FormatLines))
|
|
||||||
|
|
||||||
err := b.download(t.Context(), b.lists[0])
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("first download: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
before := lookup(t, b, listedAddr)
|
|
||||||
|
|
||||||
p.answer(tt.status, tt.body)
|
|
||||||
|
|
||||||
err = b.download(t.Context(), b.lists[0])
|
|
||||||
if err == nil {
|
|
||||||
t.Fatal("second download returned no error")
|
|
||||||
}
|
|
||||||
|
|
||||||
after := lookup(t, b, listedAddr)
|
|
||||||
if len(after) != 1 || len(before) != 1 ||
|
|
||||||
!after[0].Updated.Equal(before[0].Updated) {
|
|
||||||
t.Errorf("listings %+v after the failure, want %+v", after, before)
|
|
||||||
}
|
|
||||||
|
|
||||||
copies := os.DirFS(dir)
|
|
||||||
|
|
||||||
onDisk, err := fs.ReadFile(copies, testListName)
|
|
||||||
if err != nil || string(onDisk) != goodListBody {
|
|
||||||
t.Errorf("copy on disk %q (%v), want %q", onDisk, err, goodListBody)
|
|
||||||
}
|
|
||||||
|
|
||||||
_, err = fs.Stat(copies, testListName+downloadSuffix)
|
|
||||||
if !errors.Is(err, fs.ErrNotExist) {
|
|
||||||
t.Errorf("the failed download's file is still there (%v)", err)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestRestartLoadsCopiesFromDisk downloads a list, then starts a new
|
|
||||||
// Blocklists on the same directory, as a restart does. The new one answers
|
|
||||||
// from the copy on disk at once, and does not download the list again while
|
|
||||||
// the copy is younger than the list's interval.
|
|
||||||
func TestRestartLoadsCopiesFromDisk(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
dir := t.TempDir()
|
|
||||||
p, url := newPublisher(t, goodListBody)
|
|
||||||
list := testList(url, FormatLines)
|
|
||||||
|
|
||||||
first := newTestBlocklists(dir, list)
|
|
||||||
|
|
||||||
err := first.download(t.Context(), list)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("download: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
downloaded := lookup(t, first, listedAddr)[0].Updated
|
|
||||||
|
|
||||||
second := newTestBlocklists(dir, list)
|
|
||||||
second.Start(t.Context())
|
|
||||||
second.Stop(t.Context())
|
|
||||||
|
|
||||||
listings := lookup(t, second, listedAddr)
|
|
||||||
if len(listings) != 1 ||
|
|
||||||
listings[0].Updated.Sub(downloaded).Abs() > time.Second {
|
|
||||||
t.Errorf("after the restart, listings %+v, want one updated at %s",
|
|
||||||
listings, downloaded)
|
|
||||||
}
|
|
||||||
|
|
||||||
if got := len(p.requests()); got != 1 {
|
|
||||||
t.Errorf("the publisher had %d requests, want 1", got)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestStartRefreshesAnOldCopy starts on a copy older than its list's
|
|
||||||
// interval. The list is downloaded at once, with routewatch's User-Agent and
|
|
||||||
// the Last-Modified and ETag the publisher sent with the copy, kept on disk.
|
|
||||||
// The publisher answers that the copy is still current, so the copy is kept
|
|
||||||
// and marked as updated now.
|
|
||||||
func TestStartRefreshesAnOldCopy(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
dir := t.TempDir()
|
|
||||||
p, url := newPublisher(t, goodListBody)
|
|
||||||
list := testList(url, FormatLines)
|
|
||||||
|
|
||||||
lastModified := time.Now().Add(-3 * list.Interval).UTC().Truncate(time.Second)
|
|
||||||
p.publish(goodListBody, lastModified, testETag)
|
|
||||||
|
|
||||||
err := newTestBlocklists(dir, list).download(t.Context(), list)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("download: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
old := time.Now().Add(-2 * list.Interval).UTC().Truncate(time.Second)
|
|
||||||
|
|
||||||
err = os.Chtimes(filepath.Join(dir, testListName), old, old)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("Chtimes: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
b := newTestBlocklists(dir, list)
|
|
||||||
b.Start(t.Context())
|
|
||||||
|
|
||||||
deadline := time.Now().Add(maxWait)
|
|
||||||
for !lookup(t, b, listedAddr)[0].Updated.After(old) {
|
|
||||||
if time.Now().After(deadline) {
|
|
||||||
t.Fatalf("the copy was not marked updated within %s", maxWait)
|
|
||||||
}
|
|
||||||
|
|
||||||
time.Sleep(time.Millisecond)
|
|
||||||
}
|
|
||||||
|
|
||||||
b.Stop(t.Context())
|
|
||||||
|
|
||||||
requests := p.requests()
|
|
||||||
last := requests[len(requests)-1]
|
|
||||||
|
|
||||||
want := lastModified.Format(http.TimeFormat)
|
|
||||||
if got := last.Get(ifModifiedSinceHeader); got != want {
|
|
||||||
t.Errorf("If-Modified-Since %q, want %q", got, want)
|
|
||||||
}
|
|
||||||
|
|
||||||
if got := last.Get(ifNoneMatchHeader); got != testETag {
|
|
||||||
t.Errorf("If-None-Match %q, want %q", got, testETag)
|
|
||||||
}
|
|
||||||
|
|
||||||
if got := last.Get(userAgentHeader); got != userAgent {
|
|
||||||
t.Errorf("User-Agent %q, want %q", got, userAgent)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestNewVersionOlderThanTheLastDownloadReplacesTheCopy downloads a list.
|
|
||||||
// The publisher then puts out a new version whose Last-Modified is earlier
|
|
||||||
// than that download, as when a cache in front of the publisher passes the
|
|
||||||
// version on late. The next download sends the Last-Modified of the copy,
|
|
||||||
// not the time of the download, so it gets the new version.
|
|
||||||
func TestNewVersionOlderThanTheLastDownloadReplacesTheCopy(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
p, url := newPublisher(t, goodListBody)
|
|
||||||
list := testList(url, FormatLines)
|
|
||||||
b := newTestBlocklists(t.TempDir(), list)
|
|
||||||
|
|
||||||
firstVersion := time.Now().Add(-2 * list.Interval).UTC().Truncate(time.Second)
|
|
||||||
p.publish(goodListBody, firstVersion, "")
|
|
||||||
|
|
||||||
err := b.download(t.Context(), list)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("first download: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
p.publish(newListBody, firstVersion.Add(list.Interval), "")
|
|
||||||
|
|
||||||
err = b.download(t.Context(), list)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("second download: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
if listings := lookup(t, b, newListedAddr); len(listings) != 1 {
|
|
||||||
t.Errorf("%s: listings %+v, want one from the new version",
|
|
||||||
newListedAddr, listings)
|
|
||||||
}
|
|
||||||
|
|
||||||
want := firstVersion.Format(http.TimeFormat)
|
|
||||||
if got := p.requests()[1].Get(ifModifiedSinceHeader); got != want {
|
|
||||||
t.Errorf("If-Modified-Since %q, want %q", got, want)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestDownloadWithoutLastModifiedOrETagSendsNoCondition downloads a list
|
|
||||||
// twice from a publisher that sends neither Last-Modified nor ETag. Neither
|
|
||||||
// request carries If-Modified-Since or If-None-Match, so the second download
|
|
||||||
// gets the whole list again.
|
|
||||||
func TestDownloadWithoutLastModifiedOrETagSendsNoCondition(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
p, url := newPublisher(t, goodListBody)
|
|
||||||
list := testList(url, FormatLines)
|
|
||||||
b := newTestBlocklists(t.TempDir(), list)
|
|
||||||
|
|
||||||
err := b.download(t.Context(), list)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("first download: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
err = b.download(t.Context(), list)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("second download: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
for i, request := range p.requests() {
|
|
||||||
if request.Get(ifModifiedSinceHeader) != "" ||
|
|
||||||
request.Get(ifNoneMatchHeader) != "" {
|
|
||||||
t.Errorf("request %d has a condition: %v", i, request)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestStopCancelsADownloadInProgress stops while the publisher has not
|
|
||||||
// answered. Stop cancels the download and returns once the download
|
|
||||||
// goroutine has stopped.
|
|
||||||
func TestStopCancelsADownloadInProgress(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
arrived := make(chan struct{})
|
|
||||||
|
|
||||||
var once sync.Once
|
|
||||||
|
|
||||||
srv := httptest.NewServer(http.HandlerFunc(
|
|
||||||
func(_ http.ResponseWriter, r *http.Request) {
|
|
||||||
once.Do(func() { close(arrived) })
|
|
||||||
<-r.Context().Done()
|
|
||||||
}))
|
|
||||||
t.Cleanup(srv.Close)
|
|
||||||
|
|
||||||
b := newTestBlocklists(t.TempDir(), testList(srv.URL, FormatLines))
|
|
||||||
b.Start(t.Context())
|
|
||||||
|
|
||||||
select {
|
|
||||||
case <-arrived:
|
|
||||||
case <-time.After(maxWait):
|
|
||||||
t.Fatalf("no download began within %s", maxWait)
|
|
||||||
}
|
|
||||||
|
|
||||||
ctx, cancel := context.WithTimeout(t.Context(), maxWait)
|
|
||||||
defer cancel()
|
|
||||||
|
|
||||||
b.Stop(ctx)
|
|
||||||
|
|
||||||
select {
|
|
||||||
case <-b.done:
|
|
||||||
default:
|
|
||||||
t.Error("Stop returned before the download goroutine stopped")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestPublicListsAreComplete checks that every public list has what a
|
|
||||||
// download and a lookup answer need, and a unique name.
|
|
||||||
func TestPublicListsAreComplete(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
names := make(map[string]bool)
|
|
||||||
|
|
||||||
for _, list := range Lists() {
|
|
||||||
if list.Name == "" || list.Description == "" || list.URL == "" ||
|
|
||||||
list.Interval <= 0 || names[list.Name] {
|
|
||||||
t.Errorf("list %+v is incomplete or has a name used before", list)
|
|
||||||
}
|
|
||||||
|
|
||||||
names[list.Name] = true
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,379 +0,0 @@
|
|||||||
package blocklist
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"encoding/json"
|
|
||||||
"errors"
|
|
||||||
"fmt"
|
|
||||||
"io"
|
|
||||||
"io/fs"
|
|
||||||
"net/http"
|
|
||||||
"os"
|
|
||||||
"slices"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"go4.org/netipx"
|
|
||||||
)
|
|
||||||
|
|
||||||
// Limits of one download.
|
|
||||||
const (
|
|
||||||
// downloadTimeout is how long one download may take.
|
|
||||||
downloadTimeout = time.Minute
|
|
||||||
|
|
||||||
// maxDownloadSize is the size of the largest list file accepted, in
|
|
||||||
// bytes. The largest list, FireHOL abusers, is about 2.5 MiB.
|
|
||||||
maxDownloadSize = 16 << 20
|
|
||||||
)
|
|
||||||
|
|
||||||
// The request headers a download sets.
|
|
||||||
const (
|
|
||||||
userAgentHeader = "User-Agent"
|
|
||||||
ifModifiedSinceHeader = "If-Modified-Since"
|
|
||||||
ifNoneMatchHeader = "If-None-Match"
|
|
||||||
|
|
||||||
// userAgent names routewatch to the publishers.
|
|
||||||
userAgent = "routewatch (+https://git.eeqj.de/sneak/routewatch)"
|
|
||||||
)
|
|
||||||
|
|
||||||
// The response headers that identify the version of a list a publisher sent.
|
|
||||||
const (
|
|
||||||
lastModifiedHeader = "Last-Modified"
|
|
||||||
etagHeader = "ETag"
|
|
||||||
)
|
|
||||||
|
|
||||||
const (
|
|
||||||
// downloadSuffix ends the name of the file a download is written to,
|
|
||||||
// beside the list's copy, until it has parsed.
|
|
||||||
downloadSuffix = ".download"
|
|
||||||
|
|
||||||
// validatorsSuffix ends the name of the file beside a list's copy that
|
|
||||||
// keeps the copy's validators.
|
|
||||||
validatorsSuffix = ".validators"
|
|
||||||
|
|
||||||
// dirPermissions are those of the directory that holds the copies.
|
|
||||||
dirPermissions = 0o750
|
|
||||||
|
|
||||||
// validatorsPermissions are those of the file that keeps a copy's
|
|
||||||
// validators.
|
|
||||||
validatorsPermissions = 0o600
|
|
||||||
)
|
|
||||||
|
|
||||||
// validators are the Last-Modified and ETag headers a publisher sent with a
|
|
||||||
// version of a list. A download sends them back, as If-Modified-Since and
|
|
||||||
// If-None-Match, so that the publisher sends the list only when it has
|
|
||||||
// changed. They are kept as the publisher wrote them.
|
|
||||||
//
|
|
||||||
//nolint:tagliatelle // snake_case keys, as the Go style guide asks
|
|
||||||
type validators struct {
|
|
||||||
LastModified string `json:"last_modified,omitempty"`
|
|
||||||
ETag string `json:"etag,omitempty"`
|
|
||||||
}
|
|
||||||
|
|
||||||
var (
|
|
||||||
errUnexpectedStatus = errors.New("unexpected HTTP status")
|
|
||||||
errTooLarge = errors.New("list is over the size limit")
|
|
||||||
)
|
|
||||||
|
|
||||||
// Start loads each list's copy from disk. It then starts the goroutine that
|
|
||||||
// downloads each list once its copy is older than its Interval, and a list
|
|
||||||
// with no copy at once. The goroutine runs until Stop: the end of ctx does
|
|
||||||
// not stop it.
|
|
||||||
func (b *Blocklists) Start(ctx context.Context) {
|
|
||||||
err := os.MkdirAll(b.dir, dirPermissions)
|
|
||||||
if err != nil {
|
|
||||||
b.logger.Error("Failed to create the blocklists directory",
|
|
||||||
"dir", b.dir, "error", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
nextDownload := make([]time.Time, len(b.lists))
|
|
||||||
|
|
||||||
for i, list := range b.lists {
|
|
||||||
loadErr := b.loadCopy(list)
|
|
||||||
if loadErr != nil && !errors.Is(loadErr, fs.ErrNotExist) {
|
|
||||||
b.logger.Error("Failed to load a blocklist copy",
|
|
||||||
"list", list.Name, "error", loadErr)
|
|
||||||
}
|
|
||||||
|
|
||||||
// A list with no copy has the zero time, so it is due at once.
|
|
||||||
nextDownload[i] = b.updated(list.Name).Add(list.Interval)
|
|
||||||
}
|
|
||||||
|
|
||||||
ctx, b.stop = context.WithCancel(context.WithoutCancel(ctx))
|
|
||||||
|
|
||||||
go func() {
|
|
||||||
b.run(ctx, nextDownload)
|
|
||||||
}()
|
|
||||||
|
|
||||||
b.logger.Info("Blocklist downloads started", "dir", b.dir)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Stop stops the downloads, cancelling one in progress, and waits until the
|
|
||||||
// goroutine has returned or ctx ends. The copies in memory stay, so Lookup
|
|
||||||
// still answers.
|
|
||||||
func (b *Blocklists) Stop(ctx context.Context) {
|
|
||||||
b.stop()
|
|
||||||
|
|
||||||
select {
|
|
||||||
case <-b.done:
|
|
||||||
b.logger.Info("Blocklist downloads stopped")
|
|
||||||
case <-ctx.Done():
|
|
||||||
b.logger.Warn("Stop deadline passed before the blocklist downloads " +
|
|
||||||
"stopped")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// run downloads each list when nextDownload says, until ctx ends. After each
|
|
||||||
// try, whether it worked or not, the list's next download is an Interval
|
|
||||||
// later.
|
|
||||||
func (b *Blocklists) run(ctx context.Context, nextDownload []time.Time) {
|
|
||||||
defer close(b.done)
|
|
||||||
|
|
||||||
if len(nextDownload) == 0 {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
for {
|
|
||||||
soonest := slices.MinFunc(nextDownload, time.Time.Compare)
|
|
||||||
timer := time.NewTimer(time.Until(soonest))
|
|
||||||
|
|
||||||
select {
|
|
||||||
case <-ctx.Done():
|
|
||||||
timer.Stop()
|
|
||||||
|
|
||||||
return
|
|
||||||
case <-timer.C:
|
|
||||||
}
|
|
||||||
|
|
||||||
for i, list := range b.lists {
|
|
||||||
if time.Now().Before(nextDownload[i]) {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
|
|
||||||
err := b.download(ctx, list)
|
|
||||||
if ctx.Err() != nil {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
if err != nil {
|
|
||||||
b.logger.Error("Blocklist download failed; keeping the "+
|
|
||||||
"previous copy", "list", list.Name, "error", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
nextDownload[i] = time.Now().Add(list.Interval)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// download asks the publisher for list, sending the validators of the list's
|
|
||||||
// copy when it has them. A new version that parses becomes the list's copy,
|
|
||||||
// on disk and in memory. An answer that the copy is still current marks the
|
|
||||||
// copy as updated now. Anything else leaves the copy as it is and returns an
|
|
||||||
// error.
|
|
||||||
func (b *Blocklists) download(ctx context.Context, list List) error {
|
|
||||||
ctx, cancel := context.WithTimeout(ctx, downloadTimeout)
|
|
||||||
defer cancel()
|
|
||||||
|
|
||||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, list.URL, nil)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
req.Header.Set(userAgentHeader, userAgent)
|
|
||||||
|
|
||||||
held := b.copyValidators(list.Name)
|
|
||||||
if held.LastModified != "" {
|
|
||||||
req.Header.Set(ifModifiedSinceHeader, held.LastModified)
|
|
||||||
}
|
|
||||||
|
|
||||||
if held.ETag != "" {
|
|
||||||
req.Header.Set(ifNoneMatchHeader, held.ETag)
|
|
||||||
}
|
|
||||||
|
|
||||||
resp, err := http.DefaultClient.Do(req)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
defer func() { _ = resp.Body.Close() }()
|
|
||||||
|
|
||||||
switch resp.StatusCode {
|
|
||||||
case http.StatusOK:
|
|
||||||
return b.replaceCopy(list, resp.Body, validators{
|
|
||||||
LastModified: resp.Header.Get(lastModifiedHeader),
|
|
||||||
ETag: resp.Header.Get(etagHeader),
|
|
||||||
})
|
|
||||||
case http.StatusNotModified:
|
|
||||||
return b.markCurrent(list)
|
|
||||||
default:
|
|
||||||
return fmt.Errorf("%w: %s", errUnexpectedStatus, resp.Status)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// replaceCopy writes a new version of list, read from body, beside the list's
|
|
||||||
// copy. Only when it parses does it replace the copy, on disk and in memory.
|
|
||||||
// sent are the validators the publisher sent with the new version.
|
|
||||||
func (b *Blocklists) replaceCopy(list List, body io.Reader, sent validators) error {
|
|
||||||
root, err := os.OpenRoot(b.dir)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
defer func() { _ = root.Close() }()
|
|
||||||
|
|
||||||
downloadName := list.Name + downloadSuffix
|
|
||||||
|
|
||||||
addresses, err := writeAndParse(root, downloadName, body, list.Format)
|
|
||||||
if err != nil {
|
|
||||||
_ = root.Remove(downloadName)
|
|
||||||
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
// The old copy's validators are removed before the copy is replaced, so
|
|
||||||
// that they are never kept with the new copy.
|
|
||||||
err = root.Remove(list.Name + validatorsSuffix)
|
|
||||||
if err != nil && !errors.Is(err, fs.ErrNotExist) {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
err = root.Rename(downloadName, list.Name)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
b.setCopy(list.Name, listCopy{
|
|
||||||
addresses: addresses, updated: time.Now().UTC(), validators: sent,
|
|
||||||
})
|
|
||||||
b.logger.Info("Downloaded a new version of a blocklist", "list", list.Name)
|
|
||||||
|
|
||||||
// Without this file, the first download after a restart asks for the
|
|
||||||
// whole list.
|
|
||||||
err = writeValidators(root, list.Name, sent)
|
|
||||||
if err != nil {
|
|
||||||
b.logger.Error("Failed to keep the validators of a blocklist copy",
|
|
||||||
"list", list.Name, "error", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// writeAndParse writes body to the file name in root, up to maxDownloadSize
|
|
||||||
// bytes, and parses the file as format.
|
|
||||||
func writeAndParse(
|
|
||||||
root *os.Root, name string, body io.Reader, format Format,
|
|
||||||
) (*netipx.IPSet, error) {
|
|
||||||
f, err := root.Create(name)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
|
|
||||||
defer func() { _ = f.Close() }()
|
|
||||||
|
|
||||||
written, err := io.Copy(f, io.LimitReader(body, maxDownloadSize+1))
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
|
|
||||||
if written > maxDownloadSize {
|
|
||||||
return nil, errTooLarge
|
|
||||||
}
|
|
||||||
|
|
||||||
_, err = f.Seek(0, io.SeekStart)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
|
|
||||||
return parse(f, format)
|
|
||||||
}
|
|
||||||
|
|
||||||
// markCurrent records that the publisher says the list's copy is still
|
|
||||||
// current: the copy, on disk and in memory, is marked as updated now.
|
|
||||||
func (b *Blocklists) markCurrent(list List) error {
|
|
||||||
root, err := os.OpenRoot(b.dir)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
defer func() { _ = root.Close() }()
|
|
||||||
|
|
||||||
now := time.Now().UTC()
|
|
||||||
|
|
||||||
err = root.Chtimes(list.Name, now, now)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
b.setUpdated(list.Name, now)
|
|
||||||
b.logger.Debug("Blocklist copy is still current", "list", list.Name)
|
|
||||||
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// loadCopy makes the list's copy on disk, with the validators kept beside it,
|
|
||||||
// its copy in memory. The file's modification time is when the copy was
|
|
||||||
// updated.
|
|
||||||
func (b *Blocklists) loadCopy(list List) error {
|
|
||||||
root, err := os.OpenRoot(b.dir)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
defer func() { _ = root.Close() }()
|
|
||||||
|
|
||||||
f, err := root.Open(list.Name)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
defer func() { _ = f.Close() }()
|
|
||||||
|
|
||||||
info, err := f.Stat()
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
addresses, err := parse(f, list.Format)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
held, err := readValidators(root, list.Name)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
b.setCopy(list.Name, listCopy{
|
|
||||||
addresses: addresses, updated: info.ModTime().UTC(), validators: held,
|
|
||||||
})
|
|
||||||
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// writeValidators keeps held in the file beside the named list's copy.
|
|
||||||
func writeValidators(root *os.Root, name string, held validators) error {
|
|
||||||
data, err := json.Marshal(held)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
return root.WriteFile(name+validatorsSuffix, data, validatorsPermissions)
|
|
||||||
}
|
|
||||||
|
|
||||||
// readValidators returns the validators kept beside the named list's copy.
|
|
||||||
// They are empty when there is no such file.
|
|
||||||
func readValidators(root *os.Root, name string) (validators, error) {
|
|
||||||
var held validators
|
|
||||||
|
|
||||||
data, err := root.ReadFile(name + validatorsSuffix)
|
|
||||||
if errors.Is(err, fs.ErrNotExist) {
|
|
||||||
return held, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
if err != nil {
|
|
||||||
return held, err
|
|
||||||
}
|
|
||||||
|
|
||||||
err = json.Unmarshal(data, &held)
|
|
||||||
|
|
||||||
return held, err
|
|
||||||
}
|
|
||||||
@@ -1,93 +0,0 @@
|
|||||||
package blocklist
|
|
||||||
|
|
||||||
import "time"
|
|
||||||
|
|
||||||
// How often each list is downloaded. No list is downloaded more often than
|
|
||||||
// its publisher updates it or allows.
|
|
||||||
const (
|
|
||||||
// DShield asks that its feeds be downloaded at most once an hour:
|
|
||||||
// https://isc.sans.edu/feeds_doc.html
|
|
||||||
dshieldInterval = time.Hour
|
|
||||||
|
|
||||||
// abuse.ch makes the list every 5 minutes and recommends a download at
|
|
||||||
// least every 15: https://feodotracker.abuse.ch/blocklist/
|
|
||||||
feodoTrackerInterval = 15 * time.Minute
|
|
||||||
|
|
||||||
// FireHOL's page for each list gives how often the list changes on
|
|
||||||
// average: about every 5 hours for abusers and every 3 hours for level
|
|
||||||
// 3. FireHOL sets no download limit.
|
|
||||||
// https://iplists.firehol.org/?ipset=firehol_abusers_30d
|
|
||||||
// https://iplists.firehol.org/?ipset=firehol_level3
|
|
||||||
fireHOLAbusersInterval = 5 * time.Hour
|
|
||||||
fireHOLLevel3Interval = 3 * time.Hour
|
|
||||||
|
|
||||||
// Spamhaus asks that DROP be downloaded at most once a day:
|
|
||||||
// https://www.spamhaus.org/blocklists/do-not-route-or-peer/
|
|
||||||
spamhausDROPInterval = 24 * time.Hour
|
|
||||||
|
|
||||||
// Tor publishes a new exit list every hour and sets no download limit:
|
|
||||||
// https://collector.torproject.org/recent/exit-lists/
|
|
||||||
torExitsInterval = time.Hour
|
|
||||||
)
|
|
||||||
|
|
||||||
// Lists returns the public blocklists routewatch downloads. Each can be
|
|
||||||
// downloaded whole without an account or a key. Spamhaus merged its EDROP
|
|
||||||
// list into DROP, so DROP covers both.
|
|
||||||
func Lists() []List {
|
|
||||||
return []List{
|
|
||||||
{
|
|
||||||
Name: "dshield",
|
|
||||||
Description: "In one of the 20 networks that DShield saw scan " +
|
|
||||||
"the most targets in the last three days",
|
|
||||||
URL: "https://feeds.dshield.org/block.txt",
|
|
||||||
Interval: dshieldInterval,
|
|
||||||
Format: FormatDShield,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
Name: "feodo_tracker",
|
|
||||||
Description: "A botnet command-and-control server",
|
|
||||||
URL: "https://feodotracker.abuse.ch/downloads/ipblocklist.txt",
|
|
||||||
Interval: feodoTrackerInterval,
|
|
||||||
Format: FormatLines,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
Name: "firehol_abusers_30d",
|
|
||||||
Description: "Reported for comment, form or forum spam in the " +
|
|
||||||
"last 30 days",
|
|
||||||
URL: "https://iplists.firehol.org/files/firehol_abusers_30d.netset",
|
|
||||||
Interval: fireHOLAbusersInterval,
|
|
||||||
Format: FormatLines,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
Name: "firehol_level3",
|
|
||||||
Description: "Reported for attacks, spyware or viruses in the " +
|
|
||||||
"last 30 days",
|
|
||||||
URL: "https://iplists.firehol.org/files/firehol_level3.netset",
|
|
||||||
Interval: fireHOLLevel3Interval,
|
|
||||||
Format: FormatLines,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
Name: "spamhaus_drop",
|
|
||||||
Description: "In an IPv4 network that Spamhaus says spammers " +
|
|
||||||
"or cyber criminals have hijacked or lease",
|
|
||||||
URL: "https://www.spamhaus.org/drop/drop_v4.json",
|
|
||||||
Interval: spamhausDROPInterval,
|
|
||||||
Format: FormatSpamhausJSON,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
Name: "spamhaus_dropv6",
|
|
||||||
Description: "In an IPv6 network that Spamhaus says spammers " +
|
|
||||||
"or cyber criminals have hijacked or lease",
|
|
||||||
URL: "https://www.spamhaus.org/drop/drop_v6.json",
|
|
||||||
Interval: spamhausDROPInterval,
|
|
||||||
Format: FormatSpamhausJSON,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
Name: "tor_exits",
|
|
||||||
Description: "A Tor exit node, where traffic from the Tor network leaves",
|
|
||||||
URL: "https://check.torproject.org/torbulkexitlist",
|
|
||||||
Interval: torExitsInterval,
|
|
||||||
Format: FormatLines,
|
|
||||||
},
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,158 +0,0 @@
|
|||||||
package blocklist
|
|
||||||
|
|
||||||
import (
|
|
||||||
"bufio"
|
|
||||||
"encoding/json"
|
|
||||||
"errors"
|
|
||||||
"fmt"
|
|
||||||
"io"
|
|
||||||
"net/netip"
|
|
||||||
"strings"
|
|
||||||
|
|
||||||
"go4.org/netipx"
|
|
||||||
)
|
|
||||||
|
|
||||||
const (
|
|
||||||
// commentPrefix starts a comment line in a list file.
|
|
||||||
commentPrefix = "#"
|
|
||||||
|
|
||||||
// networkSeparator separates the address and the mask length of a
|
|
||||||
// network, as in 192.0.2.0/24.
|
|
||||||
networkSeparator = "/"
|
|
||||||
|
|
||||||
// dshieldFieldSeparator separates the fields of a DShield line.
|
|
||||||
dshieldFieldSeparator = "\t"
|
|
||||||
|
|
||||||
// spamhausMetadataType is the type of the line that ends a Spamhaus
|
|
||||||
// file. That line holds the file's metadata and no network.
|
|
||||||
spamhausMetadataType = "metadata"
|
|
||||||
)
|
|
||||||
|
|
||||||
var (
|
|
||||||
errUnknownFormat = errors.New("unknown list format")
|
|
||||||
errNoNetwork = errors.New("no network on the line")
|
|
||||||
errBadRange = errors.New("first and last address make no range")
|
|
||||||
)
|
|
||||||
|
|
||||||
// parse reads a list file written in format and returns the addresses it
|
|
||||||
// holds. A line it cannot read fails the whole file, so a file that is not
|
|
||||||
// the list, such as an error page, never replaces a good copy.
|
|
||||||
func parse(r io.Reader, format Format) (*netipx.IPSet, error) {
|
|
||||||
var builder netipx.IPSetBuilder
|
|
||||||
|
|
||||||
scanner := bufio.NewScanner(r)
|
|
||||||
|
|
||||||
for lineNumber := 1; scanner.Scan(); lineNumber++ {
|
|
||||||
line := strings.TrimSpace(scanner.Text())
|
|
||||||
if line == "" || strings.HasPrefix(line, commentPrefix) {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
|
|
||||||
err := addLine(&builder, line, format)
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("line %d: %w", lineNumber, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
err := scanner.Err()
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
|
|
||||||
return builder.IPSet()
|
|
||||||
}
|
|
||||||
|
|
||||||
// addLine adds the address or network on one line of a list file to builder.
|
|
||||||
func addLine(builder *netipx.IPSetBuilder, line string, format Format) error {
|
|
||||||
switch format {
|
|
||||||
case FormatLines:
|
|
||||||
return addAddressOrNetwork(builder, line)
|
|
||||||
case FormatSpamhausJSON:
|
|
||||||
return addSpamhausNetwork(builder, line)
|
|
||||||
case FormatDShield:
|
|
||||||
return addDShieldNetwork(builder, line)
|
|
||||||
default:
|
|
||||||
return errUnknownFormat
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// addAddressOrNetwork adds an address, such as 192.0.2.1, or a network, such
|
|
||||||
// as 192.0.2.0/24, to builder.
|
|
||||||
func addAddressOrNetwork(builder *netipx.IPSetBuilder, text string) error {
|
|
||||||
if strings.Contains(text, networkSeparator) {
|
|
||||||
network, err := netip.ParsePrefix(text)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
builder.AddPrefix(network)
|
|
||||||
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
addr, err := netip.ParseAddr(text)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
builder.Add(addr)
|
|
||||||
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// addSpamhausNetwork adds the network on a line of a Spamhaus file to
|
|
||||||
// builder. The metadata line adds nothing.
|
|
||||||
func addSpamhausNetwork(builder *netipx.IPSetBuilder, line string) error {
|
|
||||||
var entry struct {
|
|
||||||
CIDR string `json:"cidr"`
|
|
||||||
Type string `json:"type"`
|
|
||||||
}
|
|
||||||
|
|
||||||
err := json.Unmarshal([]byte(line), &entry)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
if entry.Type == spamhausMetadataType {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
network, err := netip.ParsePrefix(entry.CIDR)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
builder.AddPrefix(network)
|
|
||||||
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// addDShieldNetwork adds the network on a line of a DShield file to builder.
|
|
||||||
// The line's first two fields are the network's first and last address.
|
|
||||||
func addDShieldNetwork(builder *netipx.IPSetBuilder, line string) error {
|
|
||||||
firstField, rest, found := strings.Cut(line, dshieldFieldSeparator)
|
|
||||||
if !found {
|
|
||||||
return errNoNetwork
|
|
||||||
}
|
|
||||||
|
|
||||||
lastField, _, _ := strings.Cut(rest, dshieldFieldSeparator)
|
|
||||||
|
|
||||||
first, err := netip.ParseAddr(firstField)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
last, err := netip.ParseAddr(lastField)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
network := netipx.IPRangeFrom(first, last)
|
|
||||||
if !network.IsValid() {
|
|
||||||
return errBadRange
|
|
||||||
}
|
|
||||||
|
|
||||||
builder.AddRange(network)
|
|
||||||
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
@@ -4,12 +4,10 @@ package config
|
|||||||
import (
|
import (
|
||||||
"errors"
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
"net/netip"
|
|
||||||
"os"
|
"os"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
"runtime"
|
"runtime"
|
||||||
"strconv"
|
"strconv"
|
||||||
"strings"
|
|
||||||
"time"
|
"time"
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -34,8 +32,6 @@ var (
|
|||||||
errRelativeXDGDataHome = errors.New("XDG_DATA_HOME must be an absolute path")
|
errRelativeXDGDataHome = errors.New("XDG_DATA_HOME must be an absolute path")
|
||||||
errUnsupportedOS = errors.New("unsupported operating system")
|
errUnsupportedOS = errors.New("unsupported operating system")
|
||||||
errInvalidPort = errors.New("PORT must be a whole number")
|
errInvalidPort = errors.New("PORT must be a whole number")
|
||||||
errInvalidProxy = errors.New(
|
|
||||||
"TRUSTED_PROXIES must list IP addresses and CIDR ranges, comma-separated")
|
|
||||||
)
|
)
|
||||||
|
|
||||||
// Config holds configuration for the entire application
|
// Config holds configuration for the entire application
|
||||||
@@ -46,10 +42,6 @@ type Config struct {
|
|||||||
// Port is the TCP port the HTTP server listens on
|
// Port is the TCP port the HTTP server listens on
|
||||||
Port int
|
Port int
|
||||||
|
|
||||||
// TrustedProxies are the reverse proxies whose X-Forwarded-For header
|
|
||||||
// the HTTP server believes
|
|
||||||
TrustedProxies []netip.Prefix
|
|
||||||
|
|
||||||
// MaxRuntime is the maximum runtime (0 = run forever)
|
// MaxRuntime is the maximum runtime (0 = run forever)
|
||||||
MaxRuntime time.Duration
|
MaxRuntime time.Duration
|
||||||
|
|
||||||
@@ -76,15 +68,9 @@ func New() (*Config, error) {
|
|||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
trustedProxies, err := getTrustedProxies()
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
|
|
||||||
return &Config{
|
return &Config{
|
||||||
StateDir: stateDir,
|
StateDir: stateDir,
|
||||||
Port: port,
|
Port: port,
|
||||||
TrustedProxies: trustedProxies,
|
|
||||||
// Run forever by default
|
// Run forever by default
|
||||||
MaxRuntime: 0,
|
MaxRuntime: 0,
|
||||||
// Enable batching by default
|
// Enable batching by default
|
||||||
@@ -159,38 +145,6 @@ func getPort() (int, error) {
|
|||||||
return int(port), nil
|
return int(port), nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// getTrustedProxies returns the addresses and CIDR ranges listed in
|
|
||||||
// TRUSTED_PROXIES, such as "10.0.0.0/8,192.0.2.1", or none when it is not set.
|
|
||||||
// An address is returned as a range holding only that address.
|
|
||||||
func getTrustedProxies() ([]netip.Prefix, error) {
|
|
||||||
value := os.Getenv("TRUSTED_PROXIES")
|
|
||||||
if value == "" {
|
|
||||||
return nil, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
var proxies []netip.Prefix
|
|
||||||
|
|
||||||
for entry := range strings.SplitSeq(value, ",") {
|
|
||||||
entry = strings.TrimSpace(entry)
|
|
||||||
|
|
||||||
addr, err := netip.ParseAddr(entry)
|
|
||||||
if err == nil {
|
|
||||||
proxies = append(proxies, netip.PrefixFrom(addr, addr.BitLen()))
|
|
||||||
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
|
|
||||||
prefix, err := netip.ParsePrefix(entry)
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("%w, got %q", errInvalidProxy, entry)
|
|
||||||
}
|
|
||||||
|
|
||||||
proxies = append(proxies, prefix)
|
|
||||||
}
|
|
||||||
|
|
||||||
return proxies, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// EnsureDirectories creates all necessary directories if they don't exist
|
// EnsureDirectories creates all necessary directories if they don't exist
|
||||||
func (c *Config) EnsureDirectories() error {
|
func (c *Config) EnsureDirectories() error {
|
||||||
// Ensure state directory exists
|
// Ensure state directory exists
|
||||||
|
|||||||
@@ -1,9 +1,7 @@
|
|||||||
package config
|
package config
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"net/netip"
|
|
||||||
"runtime"
|
"runtime"
|
||||||
"slices"
|
|
||||||
"testing"
|
"testing"
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -46,55 +44,6 @@ func TestNewRefusesInvalidPort(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestNewReadsTrustedProxies(t *testing.T) {
|
|
||||||
t.Setenv("PORT", "")
|
|
||||||
t.Setenv("XDG_DATA_HOME", "")
|
|
||||||
|
|
||||||
t.Setenv("TRUSTED_PROXIES", "")
|
|
||||||
|
|
||||||
cfg, err := New()
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("New() without TRUSTED_PROXIES: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
if len(cfg.TrustedProxies) != 0 {
|
|
||||||
t.Errorf("New() without TRUSTED_PROXIES: TrustedProxies = %v, want none",
|
|
||||||
cfg.TrustedProxies)
|
|
||||||
}
|
|
||||||
|
|
||||||
t.Setenv("TRUSTED_PROXIES", "10.0.0.0/8, 192.0.2.1,2001:db8::/32")
|
|
||||||
|
|
||||||
cfg, err = New()
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("New() with TRUSTED_PROXIES: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
want := []netip.Prefix{
|
|
||||||
netip.MustParsePrefix("10.0.0.0/8"),
|
|
||||||
netip.MustParsePrefix("192.0.2.1/32"),
|
|
||||||
netip.MustParsePrefix("2001:db8::/32"),
|
|
||||||
}
|
|
||||||
if !slices.Equal(cfg.TrustedProxies, want) {
|
|
||||||
t.Errorf("TrustedProxies = %v, want %v", cfg.TrustedProxies, want)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestNewRefusesInvalidTrustedProxies(t *testing.T) {
|
|
||||||
invalid := []string{"proxy.example", "10.0.0.0/33", "192.0.2.1:80", "192.0.2.1,"}
|
|
||||||
for _, value := range invalid {
|
|
||||||
t.Run(value, func(t *testing.T) {
|
|
||||||
t.Setenv("PORT", "")
|
|
||||||
t.Setenv("XDG_DATA_HOME", "")
|
|
||||||
t.Setenv("TRUSTED_PROXIES", value)
|
|
||||||
|
|
||||||
_, err := New()
|
|
||||||
if err == nil {
|
|
||||||
t.Errorf("New() with TRUSTED_PROXIES=%q returned no error", value)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestNewRefusesRelativeXDGDataHome(t *testing.T) {
|
func TestNewRefusesRelativeXDGDataHome(t *testing.T) {
|
||||||
if runtime.GOOS == "darwin" {
|
if runtime.GOOS == "darwin" {
|
||||||
t.Skip("macOS does not read XDG_DATA_HOME")
|
t.Skip("macOS does not read XDG_DATA_HOME")
|
||||||
|
|||||||
@@ -2,7 +2,6 @@ package database
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"fmt"
|
|
||||||
"slices"
|
"slices"
|
||||||
"sync"
|
"sync"
|
||||||
"testing"
|
"testing"
|
||||||
@@ -148,9 +147,7 @@ func TestLiveCountsTrackWritesInRealtime(t *testing.T) {
|
|||||||
func writeASNsPrefixesAndPeers(t *testing.T, db *Database, ts time.Time) {
|
func writeASNsPrefixesAndPeers(t *testing.T, db *Database, ts time.Time) {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
|
|
||||||
seen := SeenTimes{First: ts, Last: ts}
|
err := db.GetOrCreateASNBatch(map[int]time.Time{64500: ts, 64501: ts})
|
||||||
|
|
||||||
err := db.GetOrCreateASNBatch(map[int]SeenTimes{64500: seen, 64501: seen})
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("GetOrCreateASNBatch: %v", err)
|
t.Fatalf("GetOrCreateASNBatch: %v", err)
|
||||||
}
|
}
|
||||||
@@ -166,14 +163,14 @@ func writeASNsPrefixesAndPeers(t *testing.T, db *Database, ts time.Time) {
|
|||||||
err = db.UpdatePeerBatch(map[string]PeerUpdate{
|
err = db.UpdatePeerBatch(map[string]PeerUpdate{
|
||||||
testV4PeerIP: {
|
testV4PeerIP: {
|
||||||
PeerIP: testV4PeerIP, PeerASN: 64500,
|
PeerIP: testV4PeerIP, PeerASN: 64500,
|
||||||
MessageType: "UPDATE", Seen: seen,
|
MessageType: "UPDATE", Timestamp: ts,
|
||||||
},
|
},
|
||||||
})
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("UpdatePeerBatch: %v", err)
|
t.Fatalf("UpdatePeerBatch: %v", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
err = db.RecordPeering(64500, 64501, seen)
|
err = db.RecordPeering(64500, 64501, ts)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("RecordPeering: %v", err)
|
t.Fatalf("RecordPeering: %v", err)
|
||||||
}
|
}
|
||||||
@@ -194,10 +191,7 @@ func TestLiveCountsSeededFromDatabaseAtStartup(t *testing.T) {
|
|||||||
|
|
||||||
ts := time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC)
|
ts := time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC)
|
||||||
|
|
||||||
seen := SeenTimes{First: ts, Last: ts}
|
err = db.GetOrCreateASNBatch(map[int]time.Time{64500: ts, 64501: ts, 64502: ts})
|
||||||
|
|
||||||
err = db.GetOrCreateASNBatch(
|
|
||||||
map[int]SeenTimes{64500: seen, 64501: seen, 64502: seen})
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("GetOrCreateASNBatch: %v", err)
|
t.Fatalf("GetOrCreateASNBatch: %v", err)
|
||||||
}
|
}
|
||||||
@@ -500,124 +494,6 @@ func TestDeleteLiveRoutesOfPeer(t *testing.T) {
|
|||||||
checkOneLiveRoutePerIPVersion(t, db)
|
checkOneLiveRoutePerIPVersion(t, db)
|
||||||
}
|
}
|
||||||
|
|
||||||
// TestDeleteLiveRoutesOfPeerDeletesInChunks gives a peer one IPv4 route more
|
|
||||||
// than a chunk holds, two of them for prefixes another peer also has. One chunk
|
|
||||||
// deletes peerRouteChunkSize of the peer's routes and commits, leaving the last
|
|
||||||
// one. The routes are written again, and DeleteLiveRoutesOfPeer deletes all of
|
|
||||||
// them, a chunk and then the last one. After each step the route count and
|
|
||||||
// prefix distribution held in memory and those read from the route table are
|
|
||||||
// the ones wanted.
|
|
||||||
func TestDeleteLiveRoutesOfPeerDeletesInChunks(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
const (
|
|
||||||
peerIP = "192.0.2.1"
|
|
||||||
otherPeerIP = "192.0.2.2"
|
|
||||||
// How many of the peer's prefixes the other peer also has, the first
|
|
||||||
// ones in prefix order.
|
|
||||||
sharedPrefixes = 2
|
|
||||||
)
|
|
||||||
|
|
||||||
db, err := New(&config.Config{StateDir: t.TempDir()}, logger.New())
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("failed to create database: %v", err)
|
|
||||||
}
|
|
||||||
defer func() { _ = db.Close() }()
|
|
||||||
|
|
||||||
ts := time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC)
|
|
||||||
|
|
||||||
peerRoutes := make([]*LiveRoute, 0, peerRouteChunkSize+1)
|
|
||||||
otherPeerRoutes := make([]*LiveRoute, 0, sharedPrefixes)
|
|
||||||
|
|
||||||
for i := range peerRouteChunkSize + 1 {
|
|
||||||
prefix := fmt.Sprintf("10.%d.%d.0/24", i/256, i%256)
|
|
||||||
|
|
||||||
route := mkV4Route(t, prefix, 64500, ts)
|
|
||||||
route.PeerIP = peerIP
|
|
||||||
peerRoutes = append(peerRoutes, route)
|
|
||||||
|
|
||||||
if i < sharedPrefixes {
|
|
||||||
route := mkV4Route(t, prefix, 64500, ts)
|
|
||||||
route.PeerIP = otherPeerIP
|
|
||||||
otherPeerRoutes = append(otherPeerRoutes, route)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
err = db.UpsertLiveRouteBatch(slices.Concat(peerRoutes, otherPeerRoutes))
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("UpsertLiveRouteBatch: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
deletedV4, deletedV6, err := db.deletePeerRouteChunk(peerIP)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("deletePeerRouteChunk: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
if deletedV4 != peerRouteChunkSize || deletedV6 != 0 {
|
|
||||||
t.Errorf("deletePeerRouteChunk deleted (%d, %d) routes, want (%d, 0)",
|
|
||||||
deletedV4, deletedV6, peerRouteChunkSize)
|
|
||||||
}
|
|
||||||
|
|
||||||
// The shared prefixes went with the chunk, so each route left has a
|
|
||||||
// prefix of its own.
|
|
||||||
checkV4RoutesOfOwnPrefixes(t, db, "after one chunk", 1+sharedPrefixes)
|
|
||||||
|
|
||||||
err = db.UpsertLiveRouteBatch(peerRoutes)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("UpsertLiveRouteBatch: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
deleted, err := db.DeleteLiveRoutesOfPeer(peerIP)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("DeleteLiveRoutesOfPeer: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
if deleted != len(peerRoutes) {
|
|
||||||
t.Errorf("DeleteLiveRoutesOfPeer deleted %d routes, want %d",
|
|
||||||
deleted, len(peerRoutes))
|
|
||||||
}
|
|
||||||
|
|
||||||
checkV4RoutesOfOwnPrefixes(t, db, "after DeleteLiveRoutesOfPeer",
|
|
||||||
sharedPrefixes)
|
|
||||||
}
|
|
||||||
|
|
||||||
// checkV4RoutesOfOwnPrefixes checks that the database has wantRoutes IPv4 live
|
|
||||||
// routes, each for a /24 no other live route has, both as counted in memory and
|
|
||||||
// as read from the route table. when says which step of the test checks.
|
|
||||||
func checkV4RoutesOfOwnPrefixes(
|
|
||||||
t *testing.T, db *Database, when string, wantRoutes int,
|
|
||||||
) {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
ctx := context.Background()
|
|
||||||
|
|
||||||
stats, err := db.GetStatsContext(ctx)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("GetStatsContext: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
tableRoutes, _, err := db.GetLiveRouteCountsContext(ctx)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("GetLiveRouteCountsContext: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
if stats.IPv4Routes != wantRoutes || tableRoutes != wantRoutes {
|
|
||||||
t.Errorf("%s: IPv4 live routes: %d in memory, %d in the table, want %d",
|
|
||||||
when, stats.IPv4Routes, tableRoutes, wantRoutes)
|
|
||||||
}
|
|
||||||
|
|
||||||
queryDistribution, _, err := db.GetPrefixDistributionContext(ctx)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("GetPrefixDistributionContext: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
want := []PrefixDistribution{{MaskLength: 24, Count: wantRoutes}}
|
|
||||||
assertDistribution(t, when+": IPv4 distribution",
|
|
||||||
stats.IPv4PrefixDistribution, want)
|
|
||||||
assertDistribution(t, when+": IPv4 distribution query",
|
|
||||||
queryDistribution, want)
|
|
||||||
}
|
|
||||||
|
|
||||||
// checkOneLiveRoutePerIPVersion checks that the database has one IPv4 live
|
// checkOneLiveRoutePerIPVersion checks that the database has one IPv4 live
|
||||||
// route, for a /24, and one IPv6 live route, for a /32, both as counted in
|
// route, for a /24, and one IPv6 live route, for a /32, both as counted in
|
||||||
// memory and as read from the route tables.
|
// memory and as read from the route tables.
|
||||||
|
|||||||
+175
-428
@@ -21,7 +21,6 @@ import (
|
|||||||
|
|
||||||
"git.eeqj.de/sneak/routewatch/internal/config"
|
"git.eeqj.de/sneak/routewatch/internal/config"
|
||||||
"git.eeqj.de/sneak/routewatch/internal/logger"
|
"git.eeqj.de/sneak/routewatch/internal/logger"
|
||||||
"git.eeqj.de/sneak/routewatch/internal/ristypes"
|
|
||||||
"git.eeqj.de/sneak/routewatch/pkg/asinfo"
|
"git.eeqj.de/sneak/routewatch/pkg/asinfo"
|
||||||
_ "github.com/mattn/go-sqlite3" // CGO SQLite driver
|
_ "github.com/mattn/go-sqlite3" // CGO SQLite driver
|
||||||
)
|
)
|
||||||
@@ -82,23 +81,6 @@ const (
|
|||||||
sqliteHardHeapLimitBytes = 2147483648
|
sqliteHardHeapLimitBytes = 2147483648
|
||||||
)
|
)
|
||||||
|
|
||||||
// Settings that bound the WAL file's size once no read is open (see
|
|
||||||
// Checkpoint).
|
|
||||||
const (
|
|
||||||
// sqliteWALSizeLimitBytes (64 MiB) is the WAL file size past which
|
|
||||||
// Checkpoint runs a TRUNCATE checkpoint. It is also the
|
|
||||||
// journal_size_limit of the connection kept for writing: when SQLite
|
|
||||||
// starts the WAL again from the beginning, it cuts a larger WAL file down
|
|
||||||
// to this size.
|
|
||||||
sqliteWALSizeLimitBytes = 67108864
|
|
||||||
// truncateBusyTimeoutMs is the busy timeout of the TRUNCATE checkpoint:
|
|
||||||
// the longest it waits for open reads, and so the longest it keeps batch
|
|
||||||
// writes waiting beyond the copy itself.
|
|
||||||
truncateBusyTimeoutMs = 500
|
|
||||||
// walFileSuffix ends the name of the WAL file, next to the database file.
|
|
||||||
walFileSuffix = "-wal"
|
|
||||||
)
|
|
||||||
|
|
||||||
// Settings of the ANALYZE that Analyze runs.
|
// Settings of the ANALYZE that Analyze runs.
|
||||||
const (
|
const (
|
||||||
// analysisLimit is about how many rows of each index ANALYZE reads
|
// analysisLimit is about how many rows of each index ANALYZE reads
|
||||||
@@ -130,12 +112,6 @@ var (
|
|||||||
ErrNoRoute = errors.New("no route found")
|
ErrNoRoute = errors.New("no route found")
|
||||||
// ErrNoStaleASN is returned when no ASN needs WHOIS refresh
|
// ErrNoStaleASN is returned when no ASN needs WHOIS refresh
|
||||||
ErrNoStaleASN = errors.New("no stale ASN found")
|
ErrNoStaleASN = errors.New("no stale ASN found")
|
||||||
// ErrCheckpointIncomplete is returned by Checkpoint when it left part of
|
|
||||||
// the WAL uncopied, or left a WAL file over sqliteWALSizeLimitBytes
|
|
||||||
// untruncated, because reads still needed the WAL or SQLite reported the
|
|
||||||
// checkpoint busy. A later checkpoint that runs when no read is open
|
|
||||||
// finishes the work.
|
|
||||||
ErrCheckpointIncomplete = errors.New("WAL checkpoint incomplete")
|
|
||||||
|
|
||||||
errInvalidASN = errors.New("invalid ASN")
|
errInvalidASN = errors.New("invalid ASN")
|
||||||
errSameASNInPeering = errors.New("cannot create peering with same ASN")
|
errSameASNInPeering = errors.New("cannot create peering with same ASN")
|
||||||
@@ -170,20 +146,6 @@ func New(cfg *config.Config, logger *logger.Logger) (*Database, error) {
|
|||||||
return nil, fmt.Errorf("failed to create database directory: %w", err)
|
return nil, fmt.Errorf("failed to create database directory: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
// The size of the WAL file the previous run left, which Initialize logs
|
|
||||||
// with the start-up checkpoint. It is read before the database is opened,
|
|
||||||
// because opening it writes a frame to the WAL. There is no WAL file when
|
|
||||||
// the database is new or the last run closed the database, which deletes
|
|
||||||
// it.
|
|
||||||
var walBytes int64
|
|
||||||
|
|
||||||
wal, err := os.Stat(dbPath + walFileSuffix)
|
|
||||||
if err == nil {
|
|
||||||
walBytes = wal.Size()
|
|
||||||
} else if !errors.Is(err, os.ErrNotExist) {
|
|
||||||
logger.Warn("Failed to read the WAL file's size on startup", "error", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Per-connection SQLite settings go in the DSN so every pooled connection
|
// Per-connection SQLite settings go in the DSN so every pooled connection
|
||||||
// gets them, not just the one that runs the Initialize pragmas. _txlock=
|
// gets them, not just the one that runs the Initialize pragmas. _txlock=
|
||||||
// immediate makes every transaction take the write lock at BEGIN. Without it
|
// immediate makes every transaction take the write lock at BEGIN. Without it
|
||||||
@@ -232,19 +194,12 @@ func New(cfg *config.Config, logger *logger.Logger) (*Database, error) {
|
|||||||
err)
|
err)
|
||||||
}
|
}
|
||||||
|
|
||||||
_, err = writer.ExecContext(ctx,
|
|
||||||
fmt.Sprintf("PRAGMA journal_size_limit=%d", sqliteWALSizeLimitBytes))
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf(
|
|
||||||
"failed to set the writing connection's WAL size limit: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
database := &Database{
|
database := &Database{
|
||||||
db: db, writer: writer, logger: logger, path: dbPath,
|
db: db, writer: writer, logger: logger, path: dbPath,
|
||||||
counts: &liveCounts{},
|
counts: &liveCounts{},
|
||||||
}
|
}
|
||||||
|
|
||||||
err = database.Initialize(walBytes)
|
err = database.Initialize()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("failed to initialize database: %w", err)
|
return nil, fmt.Errorf("failed to initialize database: %w", err)
|
||||||
}
|
}
|
||||||
@@ -260,16 +215,8 @@ func New(cfg *config.Config, logger *logger.Logger) (*Database, error) {
|
|||||||
return database, nil
|
return database, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// The messages Initialize logs the start-up WAL checkpoint with.
|
// Initialize creates the database schema if it doesn't exist.
|
||||||
const (
|
func (d *Database) Initialize() error {
|
||||||
startupCheckpointMessage = "WAL checkpoint on startup"
|
|
||||||
startupCheckpointIncompleteMessage = "WAL checkpoint on startup incomplete"
|
|
||||||
)
|
|
||||||
|
|
||||||
// Initialize creates the database schema if it doesn't exist. walBytes is the
|
|
||||||
// size of the WAL file the previous run left, which New reads before it opens
|
|
||||||
// the database.
|
|
||||||
func (d *Database) Initialize(walBytes int64) error {
|
|
||||||
// Set SQLite pragmas for performance. Per-connection settings (cache_size,
|
// Set SQLite pragmas for performance. Per-connection settings (cache_size,
|
||||||
// synchronous, busy_timeout, journal_mode) live in the DSN, and New sets the
|
// synchronous, busy_timeout, journal_mode) live in the DSN, and New sets the
|
||||||
// writing connection's cache_size; temp_store is left at its default so
|
// writing connection's cache_size; temp_store is left at its default so
|
||||||
@@ -289,33 +236,19 @@ func (d *Database) Initialize(walBytes int64) error {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// The start-up checkpoint copies the WAL a previous run left into the
|
// Run WAL checkpoint on startup to consolidate any existing WAL
|
||||||
// database file and empties the WAL file; its frame counts then read 0,
|
var walPages, checkpointedPages, movedPages int
|
||||||
// so walBytes is logged with them.
|
|
||||||
var busy, walFrames, checkpointedFrames int
|
|
||||||
|
|
||||||
err := d.writer.QueryRowContext(context.Background(),
|
err := d.writer.QueryRowContext(context.Background(),
|
||||||
"PRAGMA wal_checkpoint(TRUNCATE)",
|
"PRAGMA wal_checkpoint(TRUNCATE)",
|
||||||
).Scan(&busy, &walFrames, &checkpointedFrames)
|
).Scan(&walPages, &checkpointedPages, &movedPages)
|
||||||
|
if err != nil {
|
||||||
switch {
|
|
||||||
case err != nil:
|
|
||||||
d.logger.Warn("Failed to checkpoint WAL on startup", "error", err)
|
d.logger.Warn("Failed to checkpoint WAL on startup", "error", err)
|
||||||
case busy != 0:
|
} else {
|
||||||
// Another process using the database kept the checkpoint from
|
d.logger.Info("WAL checkpoint on startup",
|
||||||
// copying or emptying all of the WAL.
|
"wal_pages", walPages,
|
||||||
d.logger.Warn(startupCheckpointIncompleteMessage,
|
"checkpointed", checkpointedPages,
|
||||||
"wal_bytes", walBytes,
|
"moved", movedPages,
|
||||||
"busy", busy,
|
|
||||||
"wal_frames", walFrames,
|
|
||||||
"checkpointed_frames", checkpointedFrames,
|
|
||||||
)
|
|
||||||
default:
|
|
||||||
d.logger.Info(startupCheckpointMessage,
|
|
||||||
"wal_bytes", walBytes,
|
|
||||||
"busy", busy,
|
|
||||||
"wal_frames", walFrames,
|
|
||||||
"checkpointed_frames", checkpointedFrames,
|
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -725,71 +658,48 @@ func (d *Database) DeleteLiveRouteBatch(deletions []LiveRouteDeletion) error {
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// peerRouteChunkSize is the most routes of a peer that one transaction of
|
// Deleting all live routes of a peer. The first query of each table selects the
|
||||||
// DeleteLiveRoutesOfPeer deletes from each route table. A peer can have a
|
// mask length of each of the peer's routes whose prefix no other peer has a
|
||||||
// million routes; deleted in one transaction, they would hold the database lock
|
// live route for: the delete takes those prefixes out of the prefix
|
||||||
// for most of a minute.
|
// distribution.
|
||||||
const peerRouteChunkSize = 10000
|
|
||||||
|
|
||||||
// Deleting a chunk of a peer's live routes. The first query of each table
|
|
||||||
// selects the peer's first routes in prefix order, through the
|
|
||||||
// UNIQUE(peer_ip, prefix) key, each with its mask length and whether another
|
|
||||||
// peer has a live route for its prefix: the delete takes the prefixes no other
|
|
||||||
// peer has out of the prefix distribution. The second query deletes the peer's
|
|
||||||
// routes up to the last prefix the first one selected.
|
|
||||||
const (
|
const (
|
||||||
peerRouteChunkV4SQL = `SELECT r.prefix, r.mask_length,
|
peerOnlyRouteMaskLengthsV4SQL = `SELECT r.mask_length
|
||||||
EXISTS (SELECT 1 FROM live_routes_v4 AS o
|
|
||||||
WHERE o.prefix = r.prefix AND o.peer_ip != r.peer_ip)
|
|
||||||
FROM live_routes_v4 AS r WHERE r.peer_ip = ?
|
FROM live_routes_v4 AS r WHERE r.peer_ip = ?
|
||||||
ORDER BY r.prefix LIMIT ?`
|
AND NOT EXISTS (SELECT 1 FROM live_routes_v4 AS o
|
||||||
deletePeerRouteChunkV4SQL = `DELETE FROM live_routes_v4
|
WHERE o.prefix = r.prefix AND o.peer_ip != r.peer_ip)`
|
||||||
WHERE peer_ip = ? AND prefix <= ?`
|
deletePeerLiveRoutesV4SQL = `DELETE FROM live_routes_v4 WHERE peer_ip = ?`
|
||||||
|
|
||||||
peerRouteChunkV6SQL = `SELECT r.prefix, r.mask_length,
|
peerOnlyRouteMaskLengthsV6SQL = `SELECT r.mask_length
|
||||||
EXISTS (SELECT 1 FROM live_routes_v6 AS o
|
|
||||||
WHERE o.prefix = r.prefix AND o.peer_ip != r.peer_ip)
|
|
||||||
FROM live_routes_v6 AS r WHERE r.peer_ip = ?
|
FROM live_routes_v6 AS r WHERE r.peer_ip = ?
|
||||||
ORDER BY r.prefix LIMIT ?`
|
AND NOT EXISTS (SELECT 1 FROM live_routes_v6 AS o
|
||||||
deletePeerRouteChunkV6SQL = `DELETE FROM live_routes_v6
|
WHERE o.prefix = r.prefix AND o.peer_ip != r.peer_ip)`
|
||||||
WHERE peer_ip = ? AND prefix <= ?`
|
deletePeerLiveRoutesV6SQL = `DELETE FROM live_routes_v6 WHERE peer_ip = ?`
|
||||||
)
|
)
|
||||||
|
|
||||||
// deletePeerRoutes deletes the first peerRouteChunkSize live routes of peerIP,
|
// deletePeerRoutes deletes all live routes of peerIP from one table, given that
|
||||||
// or as many as it has, from one table, given that table's two statements
|
// table's two statements above. It returns how many routes it deleted and the
|
||||||
// above. It returns how many routes it deleted and the mask length of each
|
// mask length of each prefix it left with no live route.
|
||||||
// prefix it left with no live route.
|
|
||||||
func deletePeerRoutes(
|
func deletePeerRoutes(
|
||||||
ctx context.Context, tx *loggingTx, chunkSQL, deleteSQL, peerIP string,
|
ctx context.Context, tx *loggingTx,
|
||||||
|
onlyRouteMaskLengthsSQL, deleteSQL, peerIP string,
|
||||||
) (int64, []int, error) {
|
) (int64, []int, error) {
|
||||||
rows, err := tx.QueryContext(ctx, chunkSQL, peerIP, peerRouteChunkSize)
|
rows, err := tx.QueryContext(ctx, onlyRouteMaskLengthsSQL, peerIP)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return 0, nil, err
|
return 0, nil, err
|
||||||
}
|
}
|
||||||
defer func() { _ = rows.Close() }()
|
defer func() { _ = rows.Close() }()
|
||||||
|
|
||||||
var (
|
var goneMaskLengths []int
|
||||||
selected int
|
|
||||||
lastPrefix string
|
|
||||||
goneMaskLengths []int
|
|
||||||
)
|
|
||||||
|
|
||||||
for rows.Next() {
|
for rows.Next() {
|
||||||
var (
|
var maskLength int
|
||||||
maskLength int
|
|
||||||
otherPeerHasARoute bool
|
|
||||||
)
|
|
||||||
|
|
||||||
err := rows.Scan(&lastPrefix, &maskLength, &otherPeerHasARoute)
|
err := rows.Scan(&maskLength)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return 0, nil, err
|
return 0, nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
selected++
|
goneMaskLengths = append(goneMaskLengths, maskLength)
|
||||||
|
|
||||||
if !otherPeerHasARoute {
|
|
||||||
goneMaskLengths = append(goneMaskLengths, maskLength)
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
err = rows.Err()
|
err = rows.Err()
|
||||||
@@ -797,11 +707,7 @@ func deletePeerRoutes(
|
|||||||
return 0, nil, err
|
return 0, nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
if selected == 0 {
|
res, err := tx.ExecContext(ctx, deleteSQL, peerIP)
|
||||||
return 0, nil, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
res, err := tx.ExecContext(ctx, deleteSQL, peerIP, lastPrefix)
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return 0, nil, err
|
return 0, nil, err
|
||||||
}
|
}
|
||||||
@@ -815,26 +721,43 @@ func deletePeerRoutes(
|
|||||||
}
|
}
|
||||||
|
|
||||||
// DeleteLiveRoutesOfPeer deletes every live route of the peer, IPv4 and IPv6,
|
// DeleteLiveRoutesOfPeer deletes every live route of the peer, IPv4 and IPv6,
|
||||||
// as when the peer's BGP session goes down. It deletes them a chunk at a time
|
// in one transaction, as when the peer's BGP session goes down. It returns how
|
||||||
// with deletePeerRouteChunk, each chunk in a transaction of its own, and
|
// many routes it deleted.
|
||||||
// releases the database lock between chunks so other writes go on. It returns
|
|
||||||
// how many routes it deleted; after an error, the chunks it deleted before stay
|
|
||||||
// deleted.
|
|
||||||
func (d *Database) DeleteLiveRoutesOfPeer(peerIP string) (int, error) {
|
func (d *Database) DeleteLiveRoutesOfPeer(peerIP string) (int, error) {
|
||||||
total := 0
|
d.lock("DeleteLiveRoutesOfPeer")
|
||||||
|
defer d.unlock()
|
||||||
|
|
||||||
for {
|
tx, err := d.beginTx()
|
||||||
deletedV4, deletedV6, err := d.deletePeerRouteChunk(peerIP)
|
if err != nil {
|
||||||
if err != nil {
|
return 0, fmt.Errorf("failed to begin transaction: %w", err)
|
||||||
return total, err
|
|
||||||
}
|
|
||||||
|
|
||||||
total += deletedV4 + deletedV6
|
|
||||||
|
|
||||||
if deletedV4 < peerRouteChunkSize && deletedV6 < peerRouteChunkSize {
|
|
||||||
return total, nil
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
defer d.rollback(tx)
|
||||||
|
|
||||||
|
ctx := context.Background()
|
||||||
|
|
||||||
|
deletedV4, goneMaskLengthsV4, err := deletePeerRoutes(ctx, tx,
|
||||||
|
peerOnlyRouteMaskLengthsV4SQL, deletePeerLiveRoutesV4SQL, peerIP)
|
||||||
|
if err != nil {
|
||||||
|
return 0, fmt.Errorf("failed to delete the IPv4 routes of peer %s: %w",
|
||||||
|
peerIP, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
deletedV6, goneMaskLengthsV6, err := deletePeerRoutes(ctx, tx,
|
||||||
|
peerOnlyRouteMaskLengthsV6SQL, deletePeerLiveRoutesV6SQL, peerIP)
|
||||||
|
if err != nil {
|
||||||
|
return 0, fmt.Errorf("failed to delete the IPv6 routes of peer %s: %w",
|
||||||
|
peerIP, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
err = tx.Commit()
|
||||||
|
if err != nil {
|
||||||
|
return 0, fmt.Errorf("failed to commit transaction: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
d.counts.addRoutes(-int(deletedV4), -int(deletedV6))
|
||||||
|
d.counts.addToDistribution(goneMaskLengthsV4, goneMaskLengthsV6, -1)
|
||||||
|
|
||||||
|
return int(deletedV4 + deletedV6), nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// insertPrefixesInTable inserts the prefixes, all of one IP version, that the
|
// insertPrefixesInTable inserts the prefixes, all of one IP version, that the
|
||||||
@@ -892,7 +815,7 @@ func insertPrefixesInTable(
|
|||||||
|
|
||||||
// UpdatePrefixesBatch stores, in a single transaction, the prefixes that are
|
// UpdatePrefixesBatch stores, in a single transaction, the prefixes that are
|
||||||
// not stored yet, each first seen at its time in prefixes: the IPv4 prefixes
|
// not stored yet, each first seen at its time in prefixes: the IPv4 prefixes
|
||||||
// first and then the IPv6 prefixes. A prefix that does not parse is not stored.
|
// first and then the IPv6 prefixes.
|
||||||
func (d *Database) UpdatePrefixesBatch(prefixes map[string]time.Time) error {
|
func (d *Database) UpdatePrefixesBatch(prefixes map[string]time.Time) error {
|
||||||
if len(prefixes) == 0 {
|
if len(prefixes) == 0 {
|
||||||
return nil
|
return nil
|
||||||
@@ -911,11 +834,6 @@ func (d *Database) UpdatePrefixesBatch(prefixes map[string]time.Time) error {
|
|||||||
prefixesV6 := make(map[string]time.Time)
|
prefixesV6 := make(map[string]time.Time)
|
||||||
|
|
||||||
for prefix, timestamp := range prefixes {
|
for prefix, timestamp := range prefixes {
|
||||||
_, err := netip.ParsePrefix(prefix)
|
|
||||||
if err != nil {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
|
|
||||||
if detectIPVersion(prefix) == ipVersionV4 {
|
if detectIPVersion(prefix) == ipVersionV4 {
|
||||||
prefixesV4[prefix] = timestamp
|
prefixesV4[prefix] = timestamp
|
||||||
} else {
|
} else {
|
||||||
@@ -945,34 +863,13 @@ func (d *Database) UpdatePrefixesBatch(prefixes map[string]time.Time) error {
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// These statements update a stored AS, peering or peer. Each keeps the earlier
|
// newASN returns a new ASN record first and last seen at timestamp, with the
|
||||||
// of the stored and the new first_seen, and the later of the stored and the new
|
// handle and description from the AS info lookup when it knows the number.
|
||||||
// last_seen. Some collectors send updates days old while they catch up on a
|
func newASN(number int, timestamp time.Time) ASN {
|
||||||
// backlog. MIN and MAX compare the times as text, which is time order here: the
|
|
||||||
// driver writes every time in one format, and every time is UTC.
|
|
||||||
const (
|
|
||||||
updateASNSQL = `UPDATE asns
|
|
||||||
SET first_seen = MIN(first_seen, ?), last_seen = MAX(last_seen, ?)
|
|
||||||
WHERE asn = ?`
|
|
||||||
|
|
||||||
updatePeeringSQL = `UPDATE peerings
|
|
||||||
SET first_seen = MIN(first_seen, ?), last_seen = MAX(last_seen, ?)
|
|
||||||
WHERE as_a = ? AND as_b = ?`
|
|
||||||
|
|
||||||
updatePeerSQL = `UPDATE bgp_peers
|
|
||||||
SET peer_asn = ?, last_message_type = ?,
|
|
||||||
first_seen = MIN(first_seen, ?), last_seen = MAX(last_seen, ?)
|
|
||||||
WHERE peer_ip = ?`
|
|
||||||
)
|
|
||||||
|
|
||||||
// newASN returns a new ASN record first seen at seen.First and last seen at
|
|
||||||
// seen.Last, with the handle and description from the AS info lookup when it
|
|
||||||
// knows the number.
|
|
||||||
func newASN(number int, seen SeenTimes) ASN {
|
|
||||||
asn := ASN{
|
asn := ASN{
|
||||||
ASN: number,
|
ASN: number,
|
||||||
FirstSeen: seen.First,
|
FirstSeen: timestamp,
|
||||||
LastSeen: seen.Last,
|
LastSeen: timestamp,
|
||||||
}
|
}
|
||||||
|
|
||||||
// Look up ASN info
|
// Look up ASN info
|
||||||
@@ -984,11 +881,12 @@ func newASN(number int, seen SeenTimes) ASN {
|
|||||||
return asn
|
return asn
|
||||||
}
|
}
|
||||||
|
|
||||||
// upsertASN stores the seen times of ASN number with the given statements,
|
// upsertASN updates the last_seen time of ASN number with the given
|
||||||
// inserting the ASN when it is new. It reports whether it inserted it.
|
// statements, inserting the ASN when it is new. It reports whether it inserted
|
||||||
|
// it.
|
||||||
func upsertASN(
|
func upsertASN(
|
||||||
ctx context.Context, selectStmt, updateStmt, insertStmt *sql.Stmt,
|
ctx context.Context, selectStmt, updateStmt, insertStmt *sql.Stmt,
|
||||||
number int, seen SeenTimes,
|
number int, timestamp time.Time,
|
||||||
) (bool, error) {
|
) (bool, error) {
|
||||||
var (
|
var (
|
||||||
asn ASN
|
asn ASN
|
||||||
@@ -998,8 +896,8 @@ func upsertASN(
|
|||||||
err := selectStmt.QueryRowContext(ctx, number).Scan(&asn.ASN, &handle,
|
err := selectStmt.QueryRowContext(ctx, number).Scan(&asn.ASN, &handle,
|
||||||
&description, &asn.FirstSeen, &asn.LastSeen)
|
&description, &asn.FirstSeen, &asn.LastSeen)
|
||||||
if err == nil {
|
if err == nil {
|
||||||
// ASN exists, update its seen times
|
// ASN exists, update last_seen
|
||||||
_, err = updateStmt.ExecContext(ctx, seen.First, seen.Last, number)
|
_, err = updateStmt.ExecContext(ctx, timestamp, number)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return false, fmt.Errorf("failed to update ASN %d: %w", number, err)
|
return false, fmt.Errorf("failed to update ASN %d: %w", number, err)
|
||||||
}
|
}
|
||||||
@@ -1012,7 +910,7 @@ func upsertASN(
|
|||||||
}
|
}
|
||||||
|
|
||||||
// ASN doesn't exist, create it
|
// ASN doesn't exist, create it
|
||||||
asn = newASN(number, seen)
|
asn = newASN(number, timestamp)
|
||||||
|
|
||||||
_, err = insertStmt.ExecContext(ctx, asn.ASN, asn.Handle,
|
_, err = insertStmt.ExecContext(ctx, asn.ASN, asn.Handle,
|
||||||
asn.Description, asn.FirstSeen, asn.LastSeen)
|
asn.Description, asn.FirstSeen, asn.LastSeen)
|
||||||
@@ -1024,7 +922,7 @@ func upsertASN(
|
|||||||
}
|
}
|
||||||
|
|
||||||
// GetOrCreateASNBatch creates or updates multiple ASNs in a single transaction
|
// GetOrCreateASNBatch creates or updates multiple ASNs in a single transaction
|
||||||
func (d *Database) GetOrCreateASNBatch(asns map[int]SeenTimes) error {
|
func (d *Database) GetOrCreateASNBatch(asns map[int]time.Time) error {
|
||||||
if len(asns) == 0 {
|
if len(asns) == 0 {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
@@ -1049,7 +947,8 @@ func (d *Database) GetOrCreateASNBatch(asns map[int]SeenTimes) error {
|
|||||||
}
|
}
|
||||||
defer func() { _ = selectStmt.Close() }()
|
defer func() { _ = selectStmt.Close() }()
|
||||||
|
|
||||||
updateStmt, err := tx.PrepareContext(ctx, updateASNSQL)
|
updateStmt, err := tx.PrepareContext(ctx,
|
||||||
|
"UPDATE asns SET last_seen = ? WHERE asn = ?")
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("failed to prepare update statement: %w", err)
|
return fmt.Errorf("failed to prepare update statement: %w", err)
|
||||||
}
|
}
|
||||||
@@ -1065,9 +964,9 @@ func (d *Database) GetOrCreateASNBatch(asns map[int]SeenTimes) error {
|
|||||||
|
|
||||||
var newASNs int
|
var newASNs int
|
||||||
|
|
||||||
for number, seen := range asns {
|
for number, timestamp := range asns {
|
||||||
inserted, err := upsertASN(ctx, selectStmt, updateStmt, insertStmt,
|
inserted, err := upsertASN(ctx, selectStmt, updateStmt, insertStmt,
|
||||||
number, seen)
|
number, timestamp)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
@@ -1108,19 +1007,17 @@ func (d *Database) GetOrCreateASN(number int, timestamp time.Time) (*ASN, error)
|
|||||||
"FROM asns WHERE asn = ?", number).
|
"FROM asns WHERE asn = ?", number).
|
||||||
Scan(&asn.ASN, &handle, &description, &asn.FirstSeen, &asn.LastSeen)
|
Scan(&asn.ASN, &handle, &description, &asn.FirstSeen, &asn.LastSeen)
|
||||||
if err == nil {
|
if err == nil {
|
||||||
// ASN exists, update its seen times
|
// ASN exists, update last_seen
|
||||||
asn.Handle = handle.String
|
asn.Handle = handle.String
|
||||||
asn.Description = description.String
|
asn.Description = description.String
|
||||||
|
|
||||||
_, err = tx.Exec(updateASNSQL, timestamp, timestamp, number)
|
_, err = tx.Exec("UPDATE asns SET last_seen = ? WHERE asn = ?",
|
||||||
|
timestamp, number)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
// Return the seen times the update stored.
|
asn.LastSeen = timestamp
|
||||||
seen := SeenTimes{First: asn.FirstSeen, Last: asn.LastSeen}
|
|
||||||
seen.Add(timestamp)
|
|
||||||
asn.FirstSeen, asn.LastSeen = seen.First, seen.Last
|
|
||||||
|
|
||||||
err = tx.Commit()
|
err = tx.Commit()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -1138,7 +1035,7 @@ func (d *Database) GetOrCreateASN(number int, timestamp time.Time) (*ASN, error)
|
|||||||
}
|
}
|
||||||
|
|
||||||
// ASN doesn't exist, create it with ASN info lookup
|
// ASN doesn't exist, create it with ASN info lookup
|
||||||
asn = newASN(number, SeenTimes{First: timestamp, Last: timestamp})
|
asn = newASN(number, timestamp)
|
||||||
|
|
||||||
_, err = tx.Exec("INSERT INTO asns "+
|
_, err = tx.Exec("INSERT INTO asns "+
|
||||||
"(asn, handle, description, first_seen, last_seen) "+
|
"(asn, handle, description, first_seen, last_seen) "+
|
||||||
@@ -1245,7 +1142,7 @@ func (d *Database) RecordAnnouncement(announcement *Announcement) error {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// RecordPeering records a peering relationship between two ASNs.
|
// RecordPeering records a peering relationship between two ASNs.
|
||||||
func (d *Database) RecordPeering(asA, asB int, seen SeenTimes) error {
|
func (d *Database) RecordPeering(asA, asB int, timestamp time.Time) error {
|
||||||
// Validate ASNs
|
// Validate ASNs
|
||||||
if asA <= 0 || asB <= 0 {
|
if asA <= 0 || asB <= 0 {
|
||||||
return fmt.Errorf("%w: asA=%d, asB=%d", errInvalidASN, asA, asB)
|
return fmt.Errorf("%w: asA=%d, asB=%d", errInvalidASN, asA, asB)
|
||||||
@@ -1279,12 +1176,14 @@ func (d *Database) RecordPeering(asA, asB int, seen SeenTimes) error {
|
|||||||
}
|
}
|
||||||
|
|
||||||
if exists {
|
if exists {
|
||||||
_, err = tx.Exec(updatePeeringSQL, seen.First, seen.Last, asA, asB)
|
_, err = tx.Exec(
|
||||||
|
"UPDATE peerings SET last_seen = ? WHERE as_a = ? AND as_b = ?",
|
||||||
|
timestamp, asA, asB)
|
||||||
} else {
|
} else {
|
||||||
_, err = tx.Exec(`
|
_, err = tx.Exec(`
|
||||||
INSERT INTO peerings (id, as_a, as_b, first_seen, last_seen)
|
INSERT INTO peerings (id, as_a, as_b, first_seen, last_seen)
|
||||||
VALUES (?, ?, ?, ?, ?)`,
|
VALUES (?, ?, ?, ?, ?)`,
|
||||||
generateUUID().String(), asA, asB, seen.First, seen.Last)
|
generateUUID().String(), asA, asB, timestamp, timestamp)
|
||||||
}
|
}
|
||||||
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -1334,7 +1233,9 @@ func (d *Database) UpdatePeerBatch(peers map[string]PeerUpdate) error {
|
|||||||
}
|
}
|
||||||
defer func() { _ = checkStmt.Close() }()
|
defer func() { _ = checkStmt.Close() }()
|
||||||
|
|
||||||
updateStmt, err := tx.PrepareContext(ctx, updatePeerSQL)
|
updateStmt, err := tx.PrepareContext(ctx,
|
||||||
|
"UPDATE bgp_peers SET peer_asn = ?, last_seen = ?, "+
|
||||||
|
"last_message_type = ? WHERE peer_ip = ?")
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("failed to prepare update statement: %w", err)
|
return fmt.Errorf("failed to prepare update statement: %w", err)
|
||||||
}
|
}
|
||||||
@@ -1360,12 +1261,11 @@ func (d *Database) UpdatePeerBatch(peers map[string]PeerUpdate) error {
|
|||||||
|
|
||||||
if exists {
|
if exists {
|
||||||
_, err = updateStmt.ExecContext(ctx, update.PeerASN,
|
_, err = updateStmt.ExecContext(ctx, update.PeerASN,
|
||||||
update.MessageType, update.Seen.First, update.Seen.Last,
|
update.Timestamp, update.MessageType, update.PeerIP)
|
||||||
update.PeerIP)
|
|
||||||
} else {
|
} else {
|
||||||
_, err = insertStmt.ExecContext(ctx,
|
_, err = insertStmt.ExecContext(ctx,
|
||||||
generateUUID().String(), update.PeerIP, update.PeerASN,
|
generateUUID().String(), update.PeerIP, update.PeerASN,
|
||||||
update.Seen.First, update.Seen.Last, update.MessageType)
|
update.Timestamp, update.Timestamp, update.MessageType)
|
||||||
}
|
}
|
||||||
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -1410,8 +1310,11 @@ func (d *Database) UpdatePeer(
|
|||||||
}
|
}
|
||||||
|
|
||||||
if exists {
|
if exists {
|
||||||
_, err = tx.Exec(updatePeerSQL,
|
_, err = tx.Exec(
|
||||||
peerASN, messageType, timestamp, timestamp, peerIP)
|
"UPDATE bgp_peers SET peer_asn = ?, last_seen = ?, "+
|
||||||
|
"last_message_type = ? WHERE peer_ip = ?",
|
||||||
|
peerASN, timestamp, messageType, peerIP,
|
||||||
|
)
|
||||||
} else {
|
} else {
|
||||||
_, err = tx.Exec(
|
_, err = tx.Exec(
|
||||||
"INSERT INTO bgp_peers (id, peer_ip, peer_asn, first_seen, "+
|
"INSERT INTO bgp_peers (id, peer_ip, peer_asn, first_seen, "+
|
||||||
@@ -1760,24 +1663,11 @@ func (d *Database) GetASDetails(asn int) (*ASN, []LiveRoute, error) {
|
|||||||
return d.GetASDetailsContext(context.Background(), asn)
|
return d.GetASDetailsContext(context.Background(), asn)
|
||||||
}
|
}
|
||||||
|
|
||||||
// GetASDetailsContext returns the AS and the prefixes it originates, the IPv4
|
// GetASDetailsContext returns detailed information about an AS including
|
||||||
// ones first, each once with the time of its newest route. It makes two index
|
// prefixes with context support
|
||||||
// searches per prefix, however many peers carry the prefix.
|
|
||||||
//
|
|
||||||
// The routes of an AS can be stored before its record in the asns table. When
|
|
||||||
// the AS has no record, its handle and description come from the AS info
|
|
||||||
// lookup, and it has no seen times. An AS with no record, no lookup entry and
|
|
||||||
// no prefix is ErrNoRoute.
|
|
||||||
//
|
|
||||||
// ristypes.NoOrigin means no known origin, not an AS, so it is ErrNoRoute
|
|
||||||
// whatever is stored.
|
|
||||||
func (d *Database) GetASDetailsContext(
|
func (d *Database) GetASDetailsContext(
|
||||||
ctx context.Context, asn int,
|
ctx context.Context, asn int,
|
||||||
) (*ASN, []LiveRoute, error) {
|
) (*ASN, []LiveRoute, error) {
|
||||||
if asn == ristypes.NoOrigin {
|
|
||||||
return nil, nil, fmt.Errorf("%w: AS%d", ErrNoRoute, asn)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Get AS information
|
// Get AS information
|
||||||
var (
|
var (
|
||||||
asnInfo ASN
|
asnInfo ASN
|
||||||
@@ -1790,45 +1680,43 @@ func (d *Database) GetASDetailsContext(
|
|||||||
asn,
|
asn,
|
||||||
).Scan(&asnInfo.ASN, &handle, &description, &asnInfo.FirstSeen,
|
).Scan(&asnInfo.ASN, &handle, &description, &asnInfo.FirstSeen,
|
||||||
&asnInfo.LastSeen)
|
&asnInfo.LastSeen)
|
||||||
|
if err != nil {
|
||||||
// known is whether the asns table or the AS info lookup holds the AS.
|
if errors.Is(err, sql.ErrNoRows) {
|
||||||
known := true
|
return nil, nil, fmt.Errorf("%w: AS%d", ErrNoRoute, asn)
|
||||||
|
|
||||||
switch {
|
|
||||||
case errors.Is(err, sql.ErrNoRows):
|
|
||||||
asnInfo.ASN = asn
|
|
||||||
|
|
||||||
info, ok := asinfo.Get(asn)
|
|
||||||
if ok {
|
|
||||||
asnInfo.Handle = info.Handle
|
|
||||||
asnInfo.Description = info.Description
|
|
||||||
}
|
}
|
||||||
|
|
||||||
known = ok
|
|
||||||
case err != nil:
|
|
||||||
return nil, nil, fmt.Errorf("failed to query AS: %w", err)
|
return nil, nil, fmt.Errorf("failed to query AS: %w", err)
|
||||||
default:
|
|
||||||
asnInfo.Handle = handle.String
|
|
||||||
asnInfo.Description = description.String
|
|
||||||
}
|
}
|
||||||
|
|
||||||
allPrefixes, err := d.originPrefixes(ctx, asn, ipVersionV4)
|
asnInfo.Handle = handle.String
|
||||||
|
asnInfo.Description = description.String
|
||||||
|
|
||||||
|
// Get prefixes announced by this AS from both tables
|
||||||
|
queryV4 := `
|
||||||
|
SELECT prefix, mask_length, MAX(last_updated) as last_updated
|
||||||
|
FROM live_routes_v4
|
||||||
|
WHERE origin_asn = ?
|
||||||
|
GROUP BY prefix, mask_length
|
||||||
|
`
|
||||||
|
|
||||||
|
allPrefixes, err := d.originPrefixes(ctx, queryV4, asn, ipVersionV4)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return &asnInfo, nil, err
|
return &asnInfo, nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
prefixesV6, err := d.originPrefixes(ctx, asn, ipVersionV6)
|
queryV6 := `
|
||||||
|
SELECT prefix, mask_length, MAX(last_updated) as last_updated
|
||||||
|
FROM live_routes_v6
|
||||||
|
WHERE origin_asn = ?
|
||||||
|
GROUP BY prefix, mask_length
|
||||||
|
`
|
||||||
|
|
||||||
|
prefixesV6, err := d.originPrefixes(ctx, queryV6, asn, ipVersionV6)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return &asnInfo, allPrefixes, err
|
return &asnInfo, allPrefixes, err
|
||||||
}
|
}
|
||||||
|
|
||||||
allPrefixes = append(allPrefixes, prefixesV6...)
|
return &asnInfo, append(allPrefixes, prefixesV6...), nil
|
||||||
|
|
||||||
if !known && len(allPrefixes) == 0 {
|
|
||||||
return nil, nil, fmt.Errorf("%w: AS%d", ErrNoRoute, asn)
|
|
||||||
}
|
|
||||||
|
|
||||||
return &asnInfo, allPrefixes, nil
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// ASPeer represents a peering relationship with another AS including handle,
|
// ASPeer represents a peering relationship with another AS including handle,
|
||||||
@@ -2357,24 +2245,9 @@ func (d *Database) Analyze(ctx context.Context) error {
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// Checkpoint copies the WAL into the database file. It holds the database
|
// Checkpoint runs a WAL checkpoint to transfer data from the WAL to the main database.
|
||||||
// lock, so batch writes wait while it runs.
|
// Uses TRUNCATE mode which blocks writers briefly but ensures complete checkpoint,
|
||||||
//
|
// keeping the WAL small for fast read performance. It holds the database lock.
|
||||||
// It runs a PASSIVE checkpoint, which never waits: it copies what no open read
|
|
||||||
// still needs, and leaves the file as large as it is. When the file is over
|
|
||||||
// sqliteWALSizeLimitBytes, Checkpoint runs one TRUNCATE checkpoint instead,
|
|
||||||
// which waits at most truncateBusyTimeoutMs for the open reads to end, then
|
|
||||||
// copies all of the WAL and empties the file, or gives up. It is logged with
|
|
||||||
// how long it took and whether it emptied the file.
|
|
||||||
//
|
|
||||||
// So a file over the limit is emptied by the first Checkpoint that finds no
|
|
||||||
// read open, at its start or at some moment of its wait. While reads follow
|
|
||||||
// one another with no pause between them, every TRUNCATE checkpoint can give
|
|
||||||
// up after its full wait, and the file can keep growing for as long as they
|
|
||||||
// do.
|
|
||||||
//
|
|
||||||
// It returns ErrCheckpointIncomplete when it leaves part of the WAL uncopied,
|
|
||||||
// or a file over the limit not emptied.
|
|
||||||
func (d *Database) Checkpoint(ctx context.Context) error {
|
func (d *Database) Checkpoint(ctx context.Context) error {
|
||||||
d.lock("Checkpoint")
|
d.lock("Checkpoint")
|
||||||
defer d.unlock()
|
defer d.unlock()
|
||||||
@@ -2384,32 +2257,11 @@ func (d *Database) Checkpoint(ctx context.Context) error {
|
|||||||
return fmt.Errorf("failed to checkpoint WAL: %w", err)
|
return fmt.Errorf("failed to checkpoint WAL: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
wal, err := os.Stat(d.path + walFileSuffix)
|
_, err = d.writer.ExecContext(ctx, "PRAGMA wal_checkpoint(TRUNCATE)")
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("failed to read the WAL file's size: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
if wal.Size() > sqliteWALSizeLimitBytes {
|
|
||||||
return d.truncateWAL(ctx, wal.Size())
|
|
||||||
}
|
|
||||||
|
|
||||||
var busy, walPages, copiedPages int
|
|
||||||
|
|
||||||
err = d.writer.QueryRowContext(ctx, "PRAGMA wal_checkpoint(PASSIVE)").
|
|
||||||
Scan(&busy, &walPages, &copiedPages)
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("failed to checkpoint WAL: %w", err)
|
return fmt.Errorf("failed to checkpoint WAL: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
if busy != 0 {
|
|
||||||
return fmt.Errorf("%w: SQLite reported it busy", ErrCheckpointIncomplete)
|
|
||||||
}
|
|
||||||
|
|
||||||
if copiedPages < walPages {
|
|
||||||
return fmt.Errorf("%w: copied %d of %d WAL pages",
|
|
||||||
ErrCheckpointIncomplete, copiedPages, walPages)
|
|
||||||
}
|
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -2425,50 +2277,6 @@ func (d *Database) Ping(ctx context.Context) error {
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// truncateWAL runs the TRUNCATE checkpoint of Checkpoint on a WAL file of
|
|
||||||
// walBytes, with the writing connection's busy timeout lowered to
|
|
||||||
// truncateBusyTimeoutMs for it, and logs how long it took. The caller holds the
|
|
||||||
// database lock.
|
|
||||||
func (d *Database) truncateWAL(ctx context.Context, walBytes int64) error {
|
|
||||||
_, err := d.writer.ExecContext(ctx,
|
|
||||||
fmt.Sprintf("PRAGMA busy_timeout=%d", truncateBusyTimeoutMs))
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("failed to lower the busy timeout to truncate WAL: %w",
|
|
||||||
err)
|
|
||||||
}
|
|
||||||
|
|
||||||
started := time.Now()
|
|
||||||
|
|
||||||
var busy, walPages, copiedPages int
|
|
||||||
|
|
||||||
checkpointErr := d.writer.QueryRowContext(ctx,
|
|
||||||
"PRAGMA wal_checkpoint(TRUNCATE)").Scan(&busy, &walPages, &copiedPages)
|
|
||||||
took := time.Since(started)
|
|
||||||
|
|
||||||
// Restored even when ctx has ended, so that no batch write runs with the
|
|
||||||
// short timeout.
|
|
||||||
_, restoreErr := d.writer.ExecContext(context.WithoutCancel(ctx),
|
|
||||||
fmt.Sprintf("PRAGMA busy_timeout=%d", sqliteBusyTimeoutMs))
|
|
||||||
if checkpointErr != nil || restoreErr != nil {
|
|
||||||
return fmt.Errorf("failed to truncate WAL: %w",
|
|
||||||
errors.Join(checkpointErr, restoreErr))
|
|
||||||
}
|
|
||||||
|
|
||||||
if busy != 0 {
|
|
||||||
d.logger.Warn("WAL file over its size limit not truncated",
|
|
||||||
"wal_bytes", walBytes, "wal_pages", walPages,
|
|
||||||
"copied_pages", copiedPages, "duration", took)
|
|
||||||
|
|
||||||
return fmt.Errorf("%w: WAL file of %d bytes not truncated",
|
|
||||||
ErrCheckpointIncomplete, walBytes)
|
|
||||||
}
|
|
||||||
|
|
||||||
d.logger.Info("WAL file over its size limit truncated",
|
|
||||||
"wal_bytes", walBytes, "duration", took)
|
|
||||||
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// The debug messages lock logs, with the operation's name, when it starts to
|
// The debug messages lock logs, with the operation's name, when it starts to
|
||||||
// wait for the database mutex and when it holds it.
|
// wait for the database mutex and when it holds it.
|
||||||
const (
|
const (
|
||||||
@@ -2524,46 +2332,6 @@ func (d *Database) rollback(tx *loggingTx) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// deletePeerRouteChunk deletes, in one transaction, the first
|
|
||||||
// peerRouteChunkSize IPv4 and the first peerRouteChunkSize IPv6 live routes of
|
|
||||||
// the peer, or as many as it has. It returns how many of each it deleted.
|
|
||||||
func (d *Database) deletePeerRouteChunk(peerIP string) (int, int, error) {
|
|
||||||
d.lock("DeleteLiveRoutesOfPeer")
|
|
||||||
defer d.unlock()
|
|
||||||
|
|
||||||
tx, err := d.beginTx()
|
|
||||||
if err != nil {
|
|
||||||
return 0, 0, fmt.Errorf("failed to begin transaction: %w", err)
|
|
||||||
}
|
|
||||||
defer d.rollback(tx)
|
|
||||||
|
|
||||||
ctx := context.Background()
|
|
||||||
|
|
||||||
deletedV4, goneMaskLengthsV4, err := deletePeerRoutes(ctx, tx,
|
|
||||||
peerRouteChunkV4SQL, deletePeerRouteChunkV4SQL, peerIP)
|
|
||||||
if err != nil {
|
|
||||||
return 0, 0, fmt.Errorf("failed to delete the IPv4 routes of peer %s: %w",
|
|
||||||
peerIP, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
deletedV6, goneMaskLengthsV6, err := deletePeerRoutes(ctx, tx,
|
|
||||||
peerRouteChunkV6SQL, deletePeerRouteChunkV6SQL, peerIP)
|
|
||||||
if err != nil {
|
|
||||||
return 0, 0, fmt.Errorf("failed to delete the IPv6 routes of peer %s: %w",
|
|
||||||
peerIP, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
err = tx.Commit()
|
|
||||||
if err != nil {
|
|
||||||
return 0, 0, fmt.Errorf("failed to commit transaction: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
d.counts.addRoutes(-int(deletedV4), -int(deletedV6))
|
|
||||||
d.counts.addToDistribution(goneMaskLengthsV4, goneMaskLengthsV6, -1)
|
|
||||||
|
|
||||||
return int(deletedV4), int(deletedV6), nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// routeTimestampRange returns the earliest and latest last_updated across both
|
// routeTimestampRange returns the earliest and latest last_updated across both
|
||||||
// live route tables, or nil values when both tables are empty. Each query reads
|
// live route tables, or nil values when both tables are empty. Each query reads
|
||||||
// one row from an end of the last_updated index rather than scanning the tables.
|
// one row from an end of the last_updated index rather than scanning the tables.
|
||||||
@@ -2634,87 +2402,66 @@ func (d *Database) scanRouteTimestamp(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// The AS page lists the prefixes an AS originates one at a time, each in two
|
// originPrefixes runs query, which selects the prefix, mask length and latest
|
||||||
// index searches that read no table row: one of the origin_asn and prefix
|
// last_updated of every prefix asn originates in the live route table of
|
||||||
// index for the AS's first prefix after the previous one, and one of the
|
// ipVersion. Rows that fail to scan or whose timestamp does not parse are
|
||||||
// prefix and last_updated index for the time of its newest route. Neither
|
// logged and skipped.
|
||||||
// reads more for a prefix that more peers carry.
|
|
||||||
const (
|
|
||||||
nextOriginPrefixV4SQL = `SELECT prefix FROM live_routes_v4
|
|
||||||
WHERE origin_asn = ? AND prefix > ? ORDER BY prefix LIMIT 1`
|
|
||||||
newestRouteTimeV4SQL = `SELECT last_updated FROM live_routes_v4
|
|
||||||
WHERE prefix = ? ORDER BY last_updated DESC LIMIT 1`
|
|
||||||
|
|
||||||
nextOriginPrefixV6SQL = `SELECT prefix FROM live_routes_v6
|
|
||||||
WHERE origin_asn = ? AND prefix > ? ORDER BY prefix LIMIT 1`
|
|
||||||
newestRouteTimeV6SQL = `SELECT last_updated FROM live_routes_v6
|
|
||||||
WHERE prefix = ? ORDER BY last_updated DESC LIMIT 1`
|
|
||||||
)
|
|
||||||
|
|
||||||
// originPrefixes returns every prefix asn originates in the live route table
|
|
||||||
// of ipVersion, in text order, each with its mask length and the time of the
|
|
||||||
// prefix's newest route, whichever AS that route names as origin. A prefix
|
|
||||||
// whose routes were all withdrawn between the two searches is left out.
|
|
||||||
func (d *Database) originPrefixes(
|
func (d *Database) originPrefixes(
|
||||||
ctx context.Context, asn, ipVersion int,
|
ctx context.Context, query string, asn, ipVersion int,
|
||||||
) ([]LiveRoute, error) {
|
) ([]LiveRoute, error) {
|
||||||
family, nextSQL, newestSQL := ipv4Family, nextOriginPrefixV4SQL,
|
family := ipv4Family
|
||||||
newestRouteTimeV4SQL
|
|
||||||
if ipVersion == ipVersionV6 {
|
if ipVersion == ipVersionV6 {
|
||||||
family, nextSQL, newestSQL = ipv6Family, nextOriginPrefixV6SQL,
|
family = ipv6Family
|
||||||
newestRouteTimeV6SQL
|
|
||||||
}
|
}
|
||||||
|
|
||||||
nextStmt, err := d.db.PrepareContext(ctx, nextSQL)
|
rows, err := d.db.QueryContext(ctx, query, asn)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("failed to prepare %s prefix query: %w",
|
return nil, fmt.Errorf("failed to query %s prefixes: %w", family, err)
|
||||||
family, err)
|
|
||||||
}
|
}
|
||||||
defer func() { _ = nextStmt.Close() }()
|
defer func() { _ = rows.Close() }()
|
||||||
|
|
||||||
newestStmt, err := d.db.PrepareContext(ctx, newestSQL)
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("failed to prepare %s route time query: %w",
|
|
||||||
family, err)
|
|
||||||
}
|
|
||||||
defer func() { _ = newestStmt.Close() }()
|
|
||||||
|
|
||||||
var routes []LiveRoute
|
var routes []LiveRoute
|
||||||
|
|
||||||
previous := "" // every prefix sorts after the empty string
|
for rows.Next() {
|
||||||
|
var (
|
||||||
for {
|
route LiveRoute
|
||||||
var prefix string
|
lastUpdatedStr string
|
||||||
|
)
|
||||||
err := nextStmt.QueryRowContext(ctx, asn, previous).Scan(&prefix)
|
|
||||||
if errors.Is(err, sql.ErrNoRows) {
|
|
||||||
return routes, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
|
err := rows.Scan(&route.Prefix, &route.MaskLength, &lastUpdatedStr)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("failed to list %s prefixes: %w", family, err)
|
d.logger.Error("Failed to scan "+family+" prefix row",
|
||||||
}
|
"error", err, "asn", asn)
|
||||||
|
|
||||||
previous = prefix
|
|
||||||
route := LiveRoute{Prefix: prefix, IPVersion: ipVersion, OriginASN: asn}
|
|
||||||
|
|
||||||
err = newestStmt.QueryRowContext(ctx, prefix).Scan(&route.LastUpdated)
|
|
||||||
if errors.Is(err, sql.ErrNoRows) {
|
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Parse the timestamp string
|
||||||
|
route.LastUpdated, err = time.Parse("2006-01-02 15:04:05-07:00",
|
||||||
|
lastUpdatedStr)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("failed to query route time of %s: %w",
|
// Try without timezone
|
||||||
prefix, err)
|
route.LastUpdated, err = time.Parse("2006-01-02 15:04:05",
|
||||||
}
|
lastUpdatedStr)
|
||||||
|
if err != nil {
|
||||||
|
d.logger.Error("Failed to parse timestamp",
|
||||||
|
"error", err, "timestamp", lastUpdatedStr)
|
||||||
|
|
||||||
route.MaskLength, err = prefixMaskLength(prefix)
|
continue
|
||||||
if err != nil {
|
}
|
||||||
return nil, err
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
route.OriginASN = asn
|
||||||
|
route.IPVersion = ipVersion
|
||||||
routes = append(routes, route)
|
routes = append(routes, route)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
err = rows.Err()
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("failed to read %s prefixes: %w", family, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
return routes, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// firstPrefixesByLength returns the first n prefixes, in text order, with the
|
// firstPrefixesByLength returns the first n prefixes, in text order, with the
|
||||||
|
|||||||
@@ -1,27 +1,21 @@
|
|||||||
package database
|
package database
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"bytes"
|
|
||||||
"context"
|
"context"
|
||||||
"database/sql"
|
"database/sql"
|
||||||
"encoding/json"
|
|
||||||
"errors"
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
"log/slog"
|
"log/slog"
|
||||||
"math/rand/v2"
|
"math/rand/v2"
|
||||||
"net/netip"
|
"net/netip"
|
||||||
"os"
|
|
||||||
"slices"
|
"slices"
|
||||||
"strconv"
|
"strconv"
|
||||||
"strings"
|
|
||||||
"sync"
|
"sync"
|
||||||
"testing"
|
"testing"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"git.eeqj.de/sneak/routewatch/internal/config"
|
"git.eeqj.de/sneak/routewatch/internal/config"
|
||||||
"git.eeqj.de/sneak/routewatch/internal/logger"
|
"git.eeqj.de/sneak/routewatch/internal/logger"
|
||||||
"git.eeqj.de/sneak/routewatch/internal/ristypes"
|
|
||||||
"git.eeqj.de/sneak/routewatch/pkg/asinfo"
|
|
||||||
"github.com/mattn/go-sqlite3"
|
"github.com/mattn/go-sqlite3"
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -73,15 +67,6 @@ const (
|
|||||||
lockWaitTimeout = time.Millisecond
|
lockWaitTimeout = time.Millisecond
|
||||||
// lockTestASN is the AS the batch write in the lock tests stores.
|
// lockTestASN is the AS the batch write in the lock tests stores.
|
||||||
lockTestASN = 64500
|
lockTestASN = 64500
|
||||||
// readerCheckpointLimit is how long a checkpoint, and a batch write
|
|
||||||
// waiting for it, may take while a reader is open: well under the busy
|
|
||||||
// timeout, which a checkpoint that waits for the reader takes.
|
|
||||||
readerCheckpointLimit = time.Second
|
|
||||||
// truncateCheckpointLimit is how long a checkpoint of a WAL file over
|
|
||||||
// sqliteWALSizeLimitBytes may take to give up while a reader is open: its
|
|
||||||
// wait for the reader plus a second, well under the busy timeout.
|
|
||||||
truncateCheckpointLimit = truncateBusyTimeoutMs*time.Millisecond +
|
|
||||||
time.Second
|
|
||||||
)
|
)
|
||||||
|
|
||||||
// Parameters for the prefix-length sample tests.
|
// Parameters for the prefix-length sample tests.
|
||||||
@@ -472,33 +457,6 @@ func TestWritingConnectionCache(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// TestWritingConnectionWALSizeLimit checks that the connection kept for
|
|
||||||
// writing, the one that writes the WAL, has the WAL size limit, which cuts the
|
|
||||||
// WAL file back when SQLite starts the WAL again from the beginning.
|
|
||||||
func TestWritingConnectionWALSizeLimit(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
db, err := New(&config.Config{StateDir: t.TempDir()}, logger.New())
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("failed to create database: %v", err)
|
|
||||||
}
|
|
||||||
defer func() { _ = db.Close() }()
|
|
||||||
|
|
||||||
var limit int64
|
|
||||||
|
|
||||||
err = db.writer.QueryRowContext(t.Context(), "PRAGMA journal_size_limit").
|
|
||||||
Scan(&limit)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("failed to read the writing connection's journal_size_limit: %v",
|
|
||||||
err)
|
|
||||||
}
|
|
||||||
|
|
||||||
if limit != sqliteWALSizeLimitBytes {
|
|
||||||
t.Errorf("writing connection: journal_size_limit = %d, want %d",
|
|
||||||
limit, sqliteWALSizeLimitBytes)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// readPragma reads the integer value of PRAGMA name on connection i, c.
|
// readPragma reads the integer value of PRAGMA name on connection i, c.
|
||||||
func readPragma(
|
func readPragma(
|
||||||
ctx context.Context, t *testing.T, c *sql.Conn, i int, name string,
|
ctx context.Context, t *testing.T, c *sql.Conn, i int, name string,
|
||||||
@@ -606,8 +564,11 @@ func TestBatchWriteDuringCheckpoint(t *testing.T) {
|
|||||||
ctx, cancel := context.WithCancel(context.Background())
|
ctx, cancel := context.WithCancel(context.Background())
|
||||||
|
|
||||||
var wg sync.WaitGroup
|
var wg sync.WaitGroup
|
||||||
|
wg.Add(1)
|
||||||
|
|
||||||
|
go func() {
|
||||||
|
defer wg.Done()
|
||||||
|
|
||||||
wg.Go(func() {
|
|
||||||
for {
|
for {
|
||||||
select {
|
select {
|
||||||
case <-ctx.Done():
|
case <-ctx.Done():
|
||||||
@@ -616,15 +577,13 @@ func TestBatchWriteDuringCheckpoint(t *testing.T) {
|
|||||||
_ = db.Checkpoint(ctx) // errors are the checkpoint's own to absorb
|
_ = db.Checkpoint(ctx) // errors are the checkpoint's own to absorb
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
})
|
}()
|
||||||
|
|
||||||
ts := time.Now().UTC()
|
ts := time.Now().UTC()
|
||||||
seen := SeenTimes{First: ts, Last: ts}
|
|
||||||
|
|
||||||
for i := range contentionIterations {
|
for i := range contentionIterations {
|
||||||
asns := map[int]SeenTimes{
|
asns := map[int]time.Time{
|
||||||
i % contendedASNCount: seen,
|
i % contendedASNCount: ts,
|
||||||
(i % contendedASNCount) + asnSecondBand: seen,
|
(i % contendedASNCount) + asnSecondBand: ts,
|
||||||
}
|
}
|
||||||
|
|
||||||
err := db.GetOrCreateASNBatch(asns)
|
err := db.GetOrCreateASNBatch(asns)
|
||||||
@@ -753,8 +712,7 @@ func TestBatchWriteWaitsForAnalyze(t *testing.T) {
|
|||||||
|
|
||||||
db, messages := newDatabaseWithLockMessages(t)
|
db, messages := newDatabaseWithLockMessages(t)
|
||||||
ctx := context.Background()
|
ctx := context.Background()
|
||||||
now := time.Now().UTC()
|
asns := map[int]time.Time{lockTestASN: time.Now().UTC()}
|
||||||
asns := map[int]SeenTimes{lockTestASN: {First: now, Last: now}}
|
|
||||||
|
|
||||||
// A stored AS gives ANALYZE a table to analyze and statistics to write.
|
// A stored AS gives ANALYZE a table to analyze and statistics to write.
|
||||||
err := db.GetOrCreateASNBatch(asns)
|
err := db.GetOrCreateASNBatch(asns)
|
||||||
@@ -844,388 +802,6 @@ func TestCheckpointLockWaitEndsWithContext(t *testing.T) {
|
|||||||
db.mu.Unlock()
|
db.mu.Unlock()
|
||||||
}
|
}
|
||||||
|
|
||||||
// TestCheckpointDoesNotWaitForReader keeps a read transaction on
|
|
||||||
// live_routes_v4 open while a route is committed, then starts Checkpoint and a
|
|
||||||
// batch write together. It checks that both end within readerCheckpointLimit,
|
|
||||||
// whichever of them takes the database lock first, and that Checkpoint reports
|
|
||||||
// the part of the WAL the reader still needs as left uncopied. A TRUNCATE
|
|
||||||
// checkpoint waited the busy timeout for the reader while holding the lock
|
|
||||||
// (https://git.eeqj.de/sneak/routewatch/issues/102).
|
|
||||||
func TestCheckpointDoesNotWaitForReader(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
db, err := New(&config.Config{StateDir: t.TempDir()}, logger.New())
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("failed to create database: %v", err)
|
|
||||||
}
|
|
||||||
defer func() { _ = db.Close() }()
|
|
||||||
|
|
||||||
ctx := context.Background()
|
|
||||||
ts := time.Now().UTC()
|
|
||||||
|
|
||||||
err = db.UpsertLiveRouteBatch(
|
|
||||||
[]*LiveRoute{mkV4Route(t, "198.51.100.0/24", lockTestASN, ts)})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("failed to store the route the reader reads: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
reader, err := db.db.Conn(ctx)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("failed to open the reading connection: %v", err)
|
|
||||||
}
|
|
||||||
defer func() { _ = reader.Close() }()
|
|
||||||
|
|
||||||
// A plain BEGIN starts a transaction that only reads, unlike the BEGIN
|
|
||||||
// IMMEDIATE of _txlock. It keeps the snapshot its first read takes until
|
|
||||||
// it ends.
|
|
||||||
_, err = reader.ExecContext(ctx, "BEGIN")
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("failed to begin the read transaction: %v", err)
|
|
||||||
}
|
|
||||||
defer func() { _, _ = reader.ExecContext(ctx, "ROLLBACK") }()
|
|
||||||
|
|
||||||
var routes int
|
|
||||||
|
|
||||||
err = reader.QueryRowContext(ctx,
|
|
||||||
"SELECT count(*) FROM live_routes_v4").Scan(&routes)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("failed to read the live routes: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
err = db.UpsertLiveRouteBatch(
|
|
||||||
[]*LiveRoute{mkV4Route(t, "203.0.113.0/24", lockTestASN, ts)})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("failed to store a route after the read: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
started := time.Now()
|
|
||||||
checkpointed := make(chan error, 1)
|
|
||||||
|
|
||||||
go func() { checkpointed <- db.Checkpoint(ctx) }()
|
|
||||||
|
|
||||||
// A route write, not an AS write: the first new AS loads the embedded AS
|
|
||||||
// names, which takes seconds under -race.
|
|
||||||
route := mkV4Route(t, "192.0.2.0/24", lockTestASN, ts)
|
|
||||||
written := make(chan error, 1)
|
|
||||||
|
|
||||||
go func() { written <- db.UpsertLiveRouteBatch([]*LiveRoute{route}) }()
|
|
||||||
|
|
||||||
err = <-checkpointed
|
|
||||||
if !errors.Is(err, ErrCheckpointIncomplete) {
|
|
||||||
t.Errorf("Checkpoint = %v with a reader open, want %v",
|
|
||||||
err, ErrCheckpointIncomplete)
|
|
||||||
}
|
|
||||||
|
|
||||||
took := time.Since(started)
|
|
||||||
if took >= readerCheckpointLimit {
|
|
||||||
t.Errorf("Checkpoint took %v with a reader open, want under %v",
|
|
||||||
took, readerCheckpointLimit)
|
|
||||||
}
|
|
||||||
|
|
||||||
err = <-written
|
|
||||||
if err != nil {
|
|
||||||
t.Errorf("batch write during Checkpoint: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
took = time.Since(started)
|
|
||||||
if took >= readerCheckpointLimit {
|
|
||||||
t.Errorf("batch write during Checkpoint ended after %v, want under %v",
|
|
||||||
took, readerCheckpointLimit)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestCheckpointEmptiesWALOnceNoReadIsOpen fills the WAL file past
|
|
||||||
// sqliteWALSizeLimitBytes while the test holds a read open. With the read
|
|
||||||
// open, it checks that Checkpoint gives up within truncateCheckpointLimit and
|
|
||||||
// reports the checkpoint incomplete, and that a batch write still succeeds.
|
|
||||||
// After the test closes the read, it checks that the next Checkpoint brings
|
|
||||||
// the WAL file under the limit and gives the writing connection its busy
|
|
||||||
// timeout back (https://git.eeqj.de/sneak/routewatch/issues/102).
|
|
||||||
func TestCheckpointEmptiesWALOnceNoReadIsOpen(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
db, err := New(&config.Config{StateDir: t.TempDir()}, logger.New())
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("failed to create database: %v", err)
|
|
||||||
}
|
|
||||||
defer func() { _ = db.Close() }()
|
|
||||||
|
|
||||||
ctx := t.Context()
|
|
||||||
|
|
||||||
reader, err := db.db.Conn(ctx)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("failed to open the reading connection: %v", err)
|
|
||||||
}
|
|
||||||
defer func() { _ = reader.Close() }()
|
|
||||||
|
|
||||||
// A plain BEGIN starts a transaction that only reads, unlike the BEGIN
|
|
||||||
// IMMEDIATE of _txlock. It keeps the snapshot its first read takes until
|
|
||||||
// it ends, so no checkpoint can copy what is written after that read.
|
|
||||||
_, err = reader.ExecContext(ctx, "BEGIN")
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("failed to begin the read transaction: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
var routes int
|
|
||||||
|
|
||||||
err = reader.QueryRowContext(ctx,
|
|
||||||
"SELECT count(*) FROM live_routes_v4").Scan(&routes)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("failed to read the live routes: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
fillWALPastLimit(ctx, t, db)
|
|
||||||
|
|
||||||
started := time.Now()
|
|
||||||
|
|
||||||
err = db.Checkpoint(ctx)
|
|
||||||
if !errors.Is(err, ErrCheckpointIncomplete) {
|
|
||||||
t.Errorf("Checkpoint = %v with a read open, want %v",
|
|
||||||
err, ErrCheckpointIncomplete)
|
|
||||||
}
|
|
||||||
|
|
||||||
took := time.Since(started)
|
|
||||||
if took >= truncateCheckpointLimit {
|
|
||||||
t.Errorf("Checkpoint took %v with a read open, want under %v",
|
|
||||||
took, truncateCheckpointLimit)
|
|
||||||
}
|
|
||||||
|
|
||||||
err = db.UpsertLiveRouteBatch([]*LiveRoute{
|
|
||||||
mkV4Route(t, "192.0.2.0/24", lockTestASN, time.Now().UTC()),
|
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
t.Errorf("batch write after Checkpoint gave up: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
_, err = reader.ExecContext(ctx, "ROLLBACK")
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("failed to end the read transaction: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
err = db.Checkpoint(ctx)
|
|
||||||
if err != nil {
|
|
||||||
t.Errorf("Checkpoint with no read open: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
size := walFileSize(t, db)
|
|
||||||
if size >= sqliteWALSizeLimitBytes {
|
|
||||||
t.Errorf("WAL file of %d bytes after Checkpoint with no read open, "+
|
|
||||||
"want under %d", size, sqliteWALSizeLimitBytes)
|
|
||||||
}
|
|
||||||
|
|
||||||
busyTimeout := readPragma(ctx, t, db.writer, 0, "busy_timeout")
|
|
||||||
if busyTimeout != sqliteBusyTimeoutMs {
|
|
||||||
t.Errorf("writing connection: busy_timeout = %d after the checkpoints, "+
|
|
||||||
"want %d", busyTimeout, sqliteBusyTimeoutMs)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// fillWALPastLimit writes sqliteWALSizeLimitBytes of random bytes to a table
|
|
||||||
// of the test's own, which with SQLite's page and frame headers leaves the WAL
|
|
||||||
// file of db over the limit, and checks that it does.
|
|
||||||
func fillWALPastLimit(ctx context.Context, t *testing.T, db *Database) {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
_, err := db.writer.ExecContext(ctx, "CREATE TABLE wal_fill (data BLOB)")
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("failed to create the table the test writes to: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
_, err = db.writer.ExecContext(ctx,
|
|
||||||
"INSERT INTO wal_fill VALUES (randomblob(?))", sqliteWALSizeLimitBytes)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("failed to fill the WAL: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
size := walFileSize(t, db)
|
|
||||||
if size <= sqliteWALSizeLimitBytes {
|
|
||||||
t.Fatalf("WAL file of %d bytes after the test's write, want over %d",
|
|
||||||
size, sqliteWALSizeLimitBytes)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// walFileSize returns the size of the WAL file of db.
|
|
||||||
func walFileSize(t *testing.T, db *Database) int64 {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
wal, err := os.Stat(db.path + walFileSuffix)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("failed to read the WAL file's size: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
return wal.Size()
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestStartupCheckpointLogsItsFigures opens a new database, writes a route,
|
|
||||||
// and opens the same file again while the first stays open, so that its WAL
|
|
||||||
// file stays. The first open finds no WAL file: it logs a size of 0 and no
|
|
||||||
// warning. The second logs the size of the WAL file the first left, and the
|
|
||||||
// checkpoint's busy flag and frame counts, each under its own name; the
|
|
||||||
// counts read 0 once the checkpoint has emptied the file.
|
|
||||||
// Initialize logged the busy flag as wal_pages and the counts under each
|
|
||||||
// other's names (https://git.eeqj.de/sneak/routewatch/issues/117).
|
|
||||||
func TestStartupCheckpointLogsItsFigures(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
cfg := &config.Config{StateDir: t.TempDir()}
|
|
||||||
|
|
||||||
db, lines := openLogged(t, cfg)
|
|
||||||
|
|
||||||
for _, line := range lines {
|
|
||||||
if line[slog.LevelKey] == slog.LevelWarn.String() {
|
|
||||||
t.Errorf("opening a new database logged a warning: %v", line)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
line := findLine(t, lines, startupCheckpointMessage)
|
|
||||||
if line["wal_bytes"] != 0.0 {
|
|
||||||
t.Errorf("new database: start-up checkpoint logged wal_bytes = %v, "+
|
|
||||||
"want 0", line["wal_bytes"])
|
|
||||||
}
|
|
||||||
|
|
||||||
err := db.UpsertLiveRouteBatch([]*LiveRoute{
|
|
||||||
mkV4Route(t, "192.0.2.0/24", lockTestASN, time.Now().UTC()),
|
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("failed to store a route: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
walBytes := walFileSize(t, db)
|
|
||||||
if walBytes == 0 {
|
|
||||||
t.Fatal("WAL file empty after the route write, want frames in it")
|
|
||||||
}
|
|
||||||
|
|
||||||
_, lines = openLogged(t, cfg)
|
|
||||||
line = findLine(t, lines, startupCheckpointMessage)
|
|
||||||
|
|
||||||
for _, key := range []string{"busy", "wal_frames", "checkpointed_frames"} {
|
|
||||||
if line[key] != 0.0 {
|
|
||||||
t.Errorf("start-up checkpoint logged %s = %v, want 0", key, line[key])
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if line["wal_bytes"] != float64(walBytes) {
|
|
||||||
t.Errorf("start-up checkpoint logged wal_bytes = %v, want the %d "+
|
|
||||||
"bytes the first database left", line["wal_bytes"], walBytes)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestStartupCheckpointWarnsWhenIncomplete keeps a read open on a database
|
|
||||||
// while a route is written after the read began, then opens the same file
|
|
||||||
// again. The second open's start-up checkpoint cannot copy the frames written
|
|
||||||
// after the read began, and gives up once the busy timeout has passed. It
|
|
||||||
// checks that the checkpoint is logged as a warning with busy 1 and fewer
|
|
||||||
// frames copied than the WAL holds.
|
|
||||||
func TestStartupCheckpointWarnsWhenIncomplete(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
cfg := &config.Config{StateDir: t.TempDir()}
|
|
||||||
db, _ := openLogged(t, cfg)
|
|
||||||
ctx := context.Background()
|
|
||||||
|
|
||||||
reader, err := db.db.Conn(ctx)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("failed to open the reading connection: %v", err)
|
|
||||||
}
|
|
||||||
defer func() { _ = reader.Close() }()
|
|
||||||
|
|
||||||
// A plain BEGIN starts a transaction that only reads, unlike the BEGIN
|
|
||||||
// IMMEDIATE of _txlock. It keeps the snapshot its first read takes until
|
|
||||||
// it ends.
|
|
||||||
_, err = reader.ExecContext(ctx, "BEGIN")
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("failed to begin the read transaction: %v", err)
|
|
||||||
}
|
|
||||||
defer func() { _, _ = reader.ExecContext(ctx, "ROLLBACK") }()
|
|
||||||
|
|
||||||
var routes int
|
|
||||||
|
|
||||||
err = reader.QueryRowContext(ctx,
|
|
||||||
"SELECT count(*) FROM live_routes_v4").Scan(&routes)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("failed to read the live routes: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
err = db.UpsertLiveRouteBatch([]*LiveRoute{
|
|
||||||
mkV4Route(t, "192.0.2.0/24", lockTestASN, time.Now().UTC()),
|
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("failed to store a route after the read: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
_, lines := openLogged(t, cfg)
|
|
||||||
line := findLine(t, lines, startupCheckpointIncompleteMessage)
|
|
||||||
|
|
||||||
if line[slog.LevelKey] != slog.LevelWarn.String() {
|
|
||||||
t.Errorf("start-up checkpoint logged at level %v, want %s",
|
|
||||||
line[slog.LevelKey], slog.LevelWarn)
|
|
||||||
}
|
|
||||||
|
|
||||||
if line["busy"] != 1.0 {
|
|
||||||
t.Errorf("start-up checkpoint logged busy = %v, want 1", line["busy"])
|
|
||||||
}
|
|
||||||
|
|
||||||
walFrames, _ := line["wal_frames"].(float64)
|
|
||||||
checkpointedFrames, _ := line["checkpointed_frames"].(float64)
|
|
||||||
|
|
||||||
if checkpointedFrames >= walFrames {
|
|
||||||
t.Errorf("start-up checkpoint logged checkpointed_frames = %v and "+
|
|
||||||
"wal_frames = %v with a read open, want fewer copied than the WAL "+
|
|
||||||
"holds", line["checkpointed_frames"], line["wal_frames"])
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// openLogged opens the database of cfg with a logger that writes JSON, and
|
|
||||||
// returns it with the lines it logged while it opened, decoded.
|
|
||||||
func openLogged(t *testing.T, cfg *config.Config) (*Database, []map[string]any) {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
var logged bytes.Buffer
|
|
||||||
|
|
||||||
db, err := New(cfg,
|
|
||||||
&logger.Logger{Logger: slog.New(slog.NewJSONHandler(&logged, nil))})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("failed to open the database: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
t.Cleanup(func() { _ = db.Close() })
|
|
||||||
|
|
||||||
var lines []map[string]any
|
|
||||||
|
|
||||||
decoder := json.NewDecoder(&logged)
|
|
||||||
for decoder.More() {
|
|
||||||
var line map[string]any
|
|
||||||
|
|
||||||
err := decoder.Decode(&line)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("failed to decode the log: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
lines = append(lines, line)
|
|
||||||
}
|
|
||||||
|
|
||||||
return db, lines
|
|
||||||
}
|
|
||||||
|
|
||||||
// findLine returns the first of lines logged with message.
|
|
||||||
func findLine(
|
|
||||||
t *testing.T,
|
|
||||||
lines []map[string]any,
|
|
||||||
message string,
|
|
||||||
) map[string]any {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
for _, line := range lines {
|
|
||||||
if line[slog.MessageKey] == message {
|
|
||||||
return line
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
t.Fatalf("no line %q in the log", message)
|
|
||||||
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestGetRandomPrefixesByLengthListsPrefixOnce stores one prefix of each
|
// TestGetRandomPrefixesByLengthListsPrefixOnce stores one prefix of each
|
||||||
// family learned from two peers in the same second and checks that the
|
// family learned from two peers in the same second and checks that the
|
||||||
// prefix-length sample lists each of them once.
|
// prefix-length sample lists each of them once.
|
||||||
@@ -1641,217 +1217,6 @@ func TestLiveRouteUpdateSetsOriginWhenItChanges(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// TestGetASDetailsListsEachPrefixOnce stores two IPv4 prefixes and an IPv6
|
|
||||||
// prefix an AS originates, two of them from two peers at different times, and
|
|
||||||
// a prefix another AS originates. The AS details list each of the AS's
|
|
||||||
// prefixes once, IPv4 first and in text order, with its mask length and the
|
|
||||||
// time of its newest route.
|
|
||||||
func TestGetASDetailsListsEachPrefixOnce(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
db, err := New(&config.Config{StateDir: t.TempDir()}, logger.New())
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("failed to create database: %v", err)
|
|
||||||
}
|
|
||||||
defer func() { _ = db.Close() }()
|
|
||||||
|
|
||||||
const asn, otherASN = 64500, 64501
|
|
||||||
|
|
||||||
ts := time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC)
|
|
||||||
|
|
||||||
_, err = db.GetOrCreateASN(asn, ts)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("GetOrCreateASN: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
v4Older := mkV4Route(t, "192.0.2.0/24", asn, ts)
|
|
||||||
v4Newer := mkV4Route(t, "192.0.2.0/24", asn, ts.Add(time.Minute))
|
|
||||||
v4Newer.PeerIP = secondV4PeerIP
|
|
||||||
v4Single := mkV4Route(t, "198.51.100.128/25", asn, ts)
|
|
||||||
v6Newer := mkV6Route(testV6Prefix, asn, ts.Add(time.Minute))
|
|
||||||
v6Older := mkV6Route(testV6Prefix, asn, ts)
|
|
||||||
v6Older.PeerIP = secondV6PeerIP
|
|
||||||
other := mkV4Route(t, "203.0.113.0/24", otherASN, ts)
|
|
||||||
|
|
||||||
err = db.UpsertLiveRouteBatch([]*LiveRoute{
|
|
||||||
v4Older, v4Newer, v4Single, v6Newer, v6Older, other,
|
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("UpsertLiveRouteBatch: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
_, prefixes, err := db.GetASDetails(asn)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("GetASDetails: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
want := []*LiveRoute{v4Newer, v4Single, v6Newer}
|
|
||||||
|
|
||||||
listedOnce := slices.EqualFunc(prefixes, want,
|
|
||||||
func(got LiveRoute, want *LiveRoute) bool {
|
|
||||||
return got.Prefix == want.Prefix &&
|
|
||||||
got.MaskLength == want.MaskLength &&
|
|
||||||
got.IPVersion == want.IPVersion &&
|
|
||||||
got.OriginASN == asn &&
|
|
||||||
got.LastUpdated.Equal(want.LastUpdated)
|
|
||||||
})
|
|
||||||
if !listedOnce {
|
|
||||||
t.Errorf("GetASDetails listed %+v, want the prefixes and times of %+v",
|
|
||||||
prefixes, want)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestGetASDetailsOfASNotInDatabase asks for ASes that the database has no
|
|
||||||
// record of, some with stored routes. Each gets the prefixes of its routes and
|
|
||||||
// no seen times. One that the AS info lookup knows gets its handle and
|
|
||||||
// description from it. An AS with no record, no lookup entry and no route is
|
|
||||||
// ErrNoRoute. Before the fix every AS with no record was ErrNoRoute.
|
|
||||||
func TestGetASDetailsOfASNotInDatabase(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
db, err := New(&config.Config{StateDir: t.TempDir()}, logger.New())
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("failed to create database: %v", err)
|
|
||||||
}
|
|
||||||
defer func() { _ = db.Close() }()
|
|
||||||
|
|
||||||
ts := time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC)
|
|
||||||
|
|
||||||
tests := []struct {
|
|
||||||
asn int
|
|
||||||
listed bool // in the AS info lookup
|
|
||||||
prefixes []string // of the routes stored for the AS
|
|
||||||
}{
|
|
||||||
{15169, true, []string{"192.0.2.0/24"}},
|
|
||||||
{13335, true, nil},
|
|
||||||
{4200000000, false, []string{"198.51.100.0/24"}},
|
|
||||||
}
|
|
||||||
|
|
||||||
var routes []*LiveRoute
|
|
||||||
|
|
||||||
for _, tt := range tests {
|
|
||||||
if _, ok := asinfo.Get(tt.asn); ok != tt.listed {
|
|
||||||
t.Fatalf("AS%d in the AS info data: %t, want %t", tt.asn, ok, tt.listed)
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, prefix := range tt.prefixes {
|
|
||||||
routes = append(routes, mkV4Route(t, prefix, tt.asn, ts))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
err = db.UpsertLiveRouteBatch(routes)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("UpsertLiveRouteBatch: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, tt := range tests {
|
|
||||||
got, prefixes, err := db.GetASDetails(tt.asn)
|
|
||||||
if err != nil {
|
|
||||||
t.Errorf("GetASDetails(%d): %v", tt.asn, err)
|
|
||||||
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
|
|
||||||
want := ASN{ASN: tt.asn}
|
|
||||||
if info, ok := asinfo.Get(tt.asn); ok {
|
|
||||||
want.Handle, want.Description = info.Handle, info.Description
|
|
||||||
}
|
|
||||||
|
|
||||||
var gotPrefixes []string
|
|
||||||
for _, route := range prefixes {
|
|
||||||
gotPrefixes = append(gotPrefixes, route.Prefix)
|
|
||||||
}
|
|
||||||
|
|
||||||
if *got != want || !slices.Equal(gotPrefixes, tt.prefixes) {
|
|
||||||
t.Errorf("GetASDetails(%d) = %+v with prefixes %v, want %+v with %v",
|
|
||||||
tt.asn, *got, gotPrefixes, want, tt.prefixes)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
const absentASN = 4200000001 // no record, no lookup entry and no route
|
|
||||||
|
|
||||||
if _, ok := asinfo.Get(absentASN); ok {
|
|
||||||
t.Fatalf("AS%d is in the AS info data", absentASN)
|
|
||||||
}
|
|
||||||
|
|
||||||
_, _, err = db.GetASDetails(absentASN)
|
|
||||||
if !errors.Is(err, ErrNoRoute) {
|
|
||||||
t.Errorf("GetASDetails(%d): error %v, want %v", absentASN, err, ErrNoRoute)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestASPrefixQueriesUseCoveringIndexes checks the plan of each query that
|
|
||||||
// lists the prefixes an AS originates: a search of a covering index and no
|
|
||||||
// temporary B-tree, so the list reads one index entry per prefix, not one
|
|
||||||
// table row per route. Grouping the AS's routes by prefix read every route's
|
|
||||||
// row and sorted them in a temporary B-tree
|
|
||||||
// (https://git.eeqj.de/sneak/routewatch/issues/104).
|
|
||||||
func TestASPrefixQueriesUseCoveringIndexes(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
db, err := New(&config.Config{StateDir: t.TempDir()}, logger.New())
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("failed to create database: %v", err)
|
|
||||||
}
|
|
||||||
defer func() { _ = db.Close() }()
|
|
||||||
|
|
||||||
tests := []struct {
|
|
||||||
query string
|
|
||||||
args []any
|
|
||||||
index string
|
|
||||||
}{
|
|
||||||
{nextOriginPrefixV4SQL, []any{0, ""}, "idx_live_routes_v4_origin_prefix"},
|
|
||||||
{newestRouteTimeV4SQL, []any{""}, "idx_live_routes_v4_prefix_last_updated"},
|
|
||||||
{nextOriginPrefixV6SQL, []any{0, ""}, "idx_live_routes_v6_origin_prefix"},
|
|
||||||
{newestRouteTimeV6SQL, []any{""}, "idx_live_routes_v6_prefix_last_updated"},
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, tt := range tests {
|
|
||||||
plan := queryPlan(t, db, tt.query, tt.args...)
|
|
||||||
|
|
||||||
if !strings.Contains(plan, "USING COVERING INDEX "+tt.index+" ") ||
|
|
||||||
strings.Contains(plan, "TEMP B-TREE") {
|
|
||||||
t.Errorf("plan of %s:\n%s\nwant a search of covering index %s "+
|
|
||||||
"and no temporary B-tree", tt.query, plan, tt.index)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// queryPlan returns the steps of EXPLAIN QUERY PLAN for query, one per line.
|
|
||||||
func queryPlan(t *testing.T, db *Database, query string, args ...any) string {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
rows, err := db.db.QueryContext(t.Context(), "EXPLAIN QUERY PLAN "+query,
|
|
||||||
args...)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("EXPLAIN QUERY PLAN %s: %v", query, err)
|
|
||||||
}
|
|
||||||
defer func() { _ = rows.Close() }()
|
|
||||||
|
|
||||||
var steps []string
|
|
||||||
|
|
||||||
for rows.Next() {
|
|
||||||
var (
|
|
||||||
id, parent, unused int
|
|
||||||
detail string
|
|
||||||
)
|
|
||||||
|
|
||||||
err := rows.Scan(&id, &parent, &unused, &detail)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("reading the plan of %s: %v", query, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
steps = append(steps, detail)
|
|
||||||
}
|
|
||||||
|
|
||||||
err = rows.Err()
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("reading the plan of %s: %v", query, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
return strings.Join(steps, "\n")
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestUpdatePrefixesBatchKeepsFirstSeen writes a prefix, then a batch with it
|
// TestUpdatePrefixesBatchKeepsFirstSeen writes a prefix, then a batch with it
|
||||||
// and a new prefix. Only the new prefix is inserted and counted, and the known
|
// and a new prefix. Only the new prefix is inserted and counted, and the known
|
||||||
// one keeps its first_seen time.
|
// one keeps its first_seen time.
|
||||||
@@ -1919,200 +1284,6 @@ func TestUpdatePrefixesBatchKeepsFirstSeen(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// TestUpdatePeerBatchStoresTheEarliestAndLatestTimesOfANewPeer writes a new
|
|
||||||
// peer whose updates span 5 seconds. Its row is first seen at the earliest time
|
|
||||||
// and last seen at the latest.
|
|
||||||
func TestUpdatePeerBatchStoresTheEarliestAndLatestTimesOfANewPeer(
|
|
||||||
t *testing.T,
|
|
||||||
) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
db, err := New(&config.Config{StateDir: t.TempDir()}, logger.New())
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("failed to create database: %v", err)
|
|
||||||
}
|
|
||||||
defer func() { _ = db.Close() }()
|
|
||||||
|
|
||||||
first := time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC)
|
|
||||||
last := first.Add(5 * time.Second)
|
|
||||||
|
|
||||||
err = db.UpdatePeerBatch(map[string]PeerUpdate{
|
|
||||||
testPeerIP: {
|
|
||||||
PeerIP: testPeerIP, PeerASN: 64500, MessageType: "UPDATE",
|
|
||||||
Seen: SeenTimes{First: first, Last: last},
|
|
||||||
},
|
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("UpdatePeerBatch: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
var firstSeen, lastSeen time.Time
|
|
||||||
|
|
||||||
err = db.db.QueryRowContext(t.Context(),
|
|
||||||
"SELECT first_seen, last_seen FROM bgp_peers WHERE peer_ip = ?",
|
|
||||||
testPeerIP).Scan(&firstSeen, &lastSeen)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("reading peer %s: %v", testPeerIP, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
if !firstSeen.Equal(first) || !lastSeen.Equal(last) {
|
|
||||||
t.Errorf("peer %s: first seen %s, last seen %s; want %s, %s",
|
|
||||||
testPeerIP, firstSeen, lastSeen, first, last)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// backlogAge is how much older one write is than the other in the tests of
|
|
||||||
// first_seen and last_seen, as when a collector catches up on a backlog.
|
|
||||||
const backlogAge = 24 * time.Hour
|
|
||||||
|
|
||||||
// batchSpan is how much earlier a batch's First is than its Last in the tests
|
|
||||||
// of first_seen and last_seen.
|
|
||||||
const batchSpan = time.Hour
|
|
||||||
|
|
||||||
// TestWritesKeepEarliestFirstSeenAndLatestLastSeen writes a row of each of
|
|
||||||
// asns, peerings and bgp_peers through each function that writes one, at a
|
|
||||||
// newer time and at an older one. Each row takes the older write's First as
|
|
||||||
// first_seen and the newer write's Last as last_seen. The test writes in both
|
|
||||||
// orders, each in a database of its own. Newer then older is a collector
|
|
||||||
// catching up on a backlog; older then newer is a feed that arrives in order.
|
|
||||||
// Each function writes a row of its own. A batch write's First is batchSpan
|
|
||||||
// before its Last, so the test catches the two swapped. A single-row write
|
|
||||||
// takes one time.
|
|
||||||
func TestWritesKeepEarliestFirstSeenAndLatestLastSeen(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
const (
|
|
||||||
batchASN, singleASN = 64500, 64501
|
|
||||||
peerASN = 64502
|
|
||||||
asnQuery = "SELECT first_seen, last_seen FROM asns WHERE asn = ?"
|
|
||||||
peeringQuery = "SELECT first_seen, last_seen FROM peerings " +
|
|
||||||
"WHERE as_a = ? AND as_b = ?"
|
|
||||||
peerQuery = "SELECT first_seen, last_seen FROM bgp_peers " +
|
|
||||||
"WHERE peer_ip = ?"
|
|
||||||
)
|
|
||||||
|
|
||||||
for name, olderFirst := range map[string]bool{
|
|
||||||
"newer then older": false,
|
|
||||||
"older then newer": true,
|
|
||||||
} {
|
|
||||||
t.Run(name, func(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
db, err := New(&config.Config{StateDir: t.TempDir()}, logger.New())
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("failed to create database: %v", err)
|
|
||||||
}
|
|
||||||
defer func() { _ = db.Close() }()
|
|
||||||
|
|
||||||
checkSeenTimes(t, db, olderFirst, func(seen SeenTimes) error {
|
|
||||||
return db.GetOrCreateASNBatch(map[int]SeenTimes{batchASN: seen})
|
|
||||||
}, batchSpan, asnQuery, batchASN)
|
|
||||||
|
|
||||||
checkSeenTimes(t, db, olderFirst, func(seen SeenTimes) error {
|
|
||||||
_, err := db.GetOrCreateASN(singleASN, seen.Last)
|
|
||||||
|
|
||||||
return err
|
|
||||||
}, 0, asnQuery, singleASN)
|
|
||||||
|
|
||||||
checkSeenTimes(t, db, olderFirst, func(seen SeenTimes) error {
|
|
||||||
return db.RecordPeering(batchASN, singleASN, seen)
|
|
||||||
}, batchSpan, peeringQuery, batchASN, singleASN)
|
|
||||||
|
|
||||||
checkSeenTimes(t, db, olderFirst, func(seen SeenTimes) error {
|
|
||||||
return db.UpdatePeerBatch(map[string]PeerUpdate{
|
|
||||||
testV4PeerIP: {
|
|
||||||
PeerIP: testV4PeerIP, PeerASN: peerASN,
|
|
||||||
MessageType: ristypes.MessageTypeUpdate,
|
|
||||||
Seen: seen,
|
|
||||||
},
|
|
||||||
})
|
|
||||||
}, batchSpan, peerQuery, testV4PeerIP)
|
|
||||||
|
|
||||||
checkSeenTimes(t, db, olderFirst, func(seen SeenTimes) error {
|
|
||||||
return db.UpdatePeer(testPeerIP, peerASN,
|
|
||||||
ristypes.MessageTypeUpdate, seen.Last)
|
|
||||||
}, 0, peerQuery, testPeerIP)
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// checkSeenTimes calls write twice, each time with First span before Last: at
|
|
||||||
// newer times and at times backlogAge older. It writes the older times first
|
|
||||||
// when olderFirst is true. It checks that query, given args, reads the older
|
|
||||||
// write's First as first_seen and the newer write's Last as last_seen.
|
|
||||||
func checkSeenTimes(
|
|
||||||
t *testing.T, db *Database, olderFirst bool,
|
|
||||||
write func(seen SeenTimes) error,
|
|
||||||
span time.Duration, query string, args ...any,
|
|
||||||
) {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
newerLast := time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC)
|
|
||||||
newer := SeenTimes{First: newerLast.Add(-span), Last: newerLast}
|
|
||||||
older := SeenTimes{
|
|
||||||
First: newer.First.Add(-backlogAge),
|
|
||||||
Last: newer.Last.Add(-backlogAge),
|
|
||||||
}
|
|
||||||
|
|
||||||
writes := []SeenTimes{newer, older}
|
|
||||||
if olderFirst {
|
|
||||||
writes = []SeenTimes{older, newer}
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, seen := range writes {
|
|
||||||
err := write(seen)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("writing first %s, last %s: %v", seen.First, seen.Last, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
var firstSeen, lastSeen time.Time
|
|
||||||
|
|
||||||
err := db.db.QueryRowContext(t.Context(), query, args...).
|
|
||||||
Scan(&firstSeen, &lastSeen)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("reading the row: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
if !firstSeen.Equal(older.First) || !lastSeen.Equal(newer.Last) {
|
|
||||||
t.Errorf("first seen %s, last seen %s; want %s, %s",
|
|
||||||
firstSeen, lastSeen, older.First, newer.Last)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestGetOrCreateASNReturnsStoredSeenTimes stores an AS at a newer time, then
|
|
||||||
// at an older one. The second call returns the times the row now holds: the
|
|
||||||
// older first_seen and the newer last_seen.
|
|
||||||
func TestGetOrCreateASNReturnsStoredSeenTimes(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
db, err := New(&config.Config{StateDir: t.TempDir()}, logger.New())
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("failed to create database: %v", err)
|
|
||||||
}
|
|
||||||
defer func() { _ = db.Close() }()
|
|
||||||
|
|
||||||
const asn = 64500
|
|
||||||
|
|
||||||
newer := time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC)
|
|
||||||
older := newer.Add(-backlogAge)
|
|
||||||
|
|
||||||
_, err = db.GetOrCreateASN(asn, newer)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("GetOrCreateASN at %s: %v", newer, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
got, err := db.GetOrCreateASN(asn, older)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("GetOrCreateASN at %s: %v", older, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
if !got.FirstSeen.Equal(older) || !got.LastSeen.Equal(newer) {
|
|
||||||
t.Errorf("first seen %s, last seen %s; want %s, %s",
|
|
||||||
got.FirstSeen, got.LastSeen, older, newer)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestUpsertLiveRouteBatchWritesInKeyOrder writes new routes of two peers out
|
// TestUpsertLiveRouteBatchWritesInKeyOrder writes new routes of two peers out
|
||||||
// of key order, the last one a second announcement of the first, and checks
|
// of key order, the last one a second announcement of the first, and checks
|
||||||
// that the rows were inserted in key order, peer then prefix (each new row's id
|
// that the rows were inserted in key order, peer then prefix (each new row's id
|
||||||
@@ -2350,66 +1521,3 @@ func writeBenchBatch(
|
|||||||
b.Fatalf("UpdatePrefixesBatch: %v", err)
|
b.Fatalf("UpdatePrefixesBatch: %v", err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Parameters for BenchmarkGetASDetailsWithFiveHundredPeers: the AS originates
|
|
||||||
// benchASPrefixes prefixes and every one of benchASPeers peers has a route for
|
|
||||||
// each, 2.5 million routes in all.
|
|
||||||
const (
|
|
||||||
benchASPrefixes = 5000
|
|
||||||
benchASPeers = 500
|
|
||||||
)
|
|
||||||
|
|
||||||
// BenchmarkGetASDetailsWithFiveHundredPeers times the read behind the AS page
|
|
||||||
// for an AS that originates 5,000 IPv4 prefixes, each from 500 peers. It
|
|
||||||
// builds that database first, one peer at a time, which takes some minutes and
|
|
||||||
// about a GB of disk under TMPDIR. It is run by hand, not by make test, for
|
|
||||||
// example in the test image:
|
|
||||||
//
|
|
||||||
// go test -run '^$' -bench GetASDetails -benchtime 10x ./internal/database/
|
|
||||||
func BenchmarkGetASDetailsWithFiveHundredPeers(b *testing.B) {
|
|
||||||
db, err := New(&config.Config{StateDir: b.TempDir()}, logger.New())
|
|
||||||
if err != nil {
|
|
||||||
b.Fatalf("failed to create database: %v", err)
|
|
||||||
}
|
|
||||||
defer func() { _ = db.Close() }()
|
|
||||||
|
|
||||||
ts := time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC)
|
|
||||||
|
|
||||||
_, err = db.GetOrCreateASN(benchFirstASN, ts)
|
|
||||||
if err != nil {
|
|
||||||
b.Fatalf("GetOrCreateASN: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
for peer := range benchASPeers {
|
|
||||||
peerIP := fmt.Sprintf("198.18.%d.%d", peer/256, peer%256)
|
|
||||||
|
|
||||||
routes := make([]*LiveRoute, 0, benchASPrefixes)
|
|
||||||
for prefix := range benchASPrefixes {
|
|
||||||
routes = append(routes, &LiveRoute{
|
|
||||||
Prefix: fmt.Sprintf("%d.%d.%d.0/%d",
|
|
||||||
1+prefix>>16, (prefix>>8)%256, prefix%256, benchMaskLength),
|
|
||||||
MaskLength: benchMaskLength,
|
|
||||||
IPVersion: ipVersionV4,
|
|
||||||
OriginASN: benchFirstASN,
|
|
||||||
PeerIP: peerIP,
|
|
||||||
ASPath: []int{benchFirstASN + 1 + peer, benchFirstASN},
|
|
||||||
NextHop: peerIP,
|
|
||||||
LastUpdated: ts,
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
writeBenchBatch(b, db, routes, nil, ts)
|
|
||||||
}
|
|
||||||
|
|
||||||
for b.Loop() {
|
|
||||||
_, prefixes, err := db.GetASDetails(benchFirstASN)
|
|
||||||
if err != nil {
|
|
||||||
b.Fatalf("GetASDetails: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
if len(prefixes) != benchASPrefixes {
|
|
||||||
b.Fatalf("GetASDetails listed %d prefixes, want %d",
|
|
||||||
len(prefixes), benchASPrefixes)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|||||||
@@ -1,61 +0,0 @@
|
|||||||
package database
|
|
||||||
|
|
||||||
import (
|
|
||||||
"os"
|
|
||||||
"path/filepath"
|
|
||||||
"strings"
|
|
||||||
"testing"
|
|
||||||
|
|
||||||
"git.eeqj.de/sneak/routewatch/internal/config"
|
|
||||||
"git.eeqj.de/sneak/routewatch/internal/logger"
|
|
||||||
)
|
|
||||||
|
|
||||||
// TestDockerignoreExcludesDatabaseFiles checks that .dockerignore keeps the
|
|
||||||
// database file, and the WAL and shared-memory files SQLite writes next to it,
|
|
||||||
// out of the build context wherever they sit in the clone. The image ships the
|
|
||||||
// whole build context as a source archive.
|
|
||||||
func TestDockerignoreExcludesDatabaseFiles(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
db, err := New(&config.Config{StateDir: t.TempDir()}, logger.New())
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("failed to create database: %v", err)
|
|
||||||
}
|
|
||||||
defer func() { _ = db.Close() }()
|
|
||||||
|
|
||||||
data, err := os.ReadFile("../../.dockerignore")
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("failed to read .dockerignore: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Docker reads `**/` as any number of directories and the rest of the
|
|
||||||
// pattern as filepath.Match does, so `**/` followed by a pattern that
|
|
||||||
// matches a file's name excludes that file at any depth.
|
|
||||||
var anyDepth []string
|
|
||||||
|
|
||||||
for line := range strings.Lines(string(data)) {
|
|
||||||
pattern, ok := strings.CutPrefix(strings.TrimSpace(line), "**/")
|
|
||||||
if ok {
|
|
||||||
anyDepth = append(anyDepth, pattern)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
name := filepath.Base(db.path)
|
|
||||||
|
|
||||||
for _, file := range []string{name, name + "-wal", name + "-shm"} {
|
|
||||||
excluded := false
|
|
||||||
|
|
||||||
for _, pattern := range anyDepth {
|
|
||||||
matched, err := filepath.Match(pattern, file)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("bad .dockerignore pattern **/%s: %v", pattern, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
excluded = excluded || matched
|
|
||||||
}
|
|
||||||
|
|
||||||
if !excluded {
|
|
||||||
t.Errorf(".dockerignore has no **/ pattern that excludes %s", file)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,14 +1,10 @@
|
|||||||
// Package database provides SQLite storage for BGP routing data including ASNs,
|
// Package database provides SQLite storage for BGP routing data including ASNs,
|
||||||
// prefixes, announcements, peerings, and live route tables. Its routes table
|
// prefixes, announcements, peerings, and live route tables.
|
||||||
// holds a copy of the whole routing table that internal/routetable keeps in
|
|
||||||
// memory.
|
|
||||||
package database
|
package database
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"git.eeqj.de/sneak/routewatch/internal/routetable"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
// Stats contains database statistics including counts for ASNs, prefixes,
|
// Stats contains database statistics including counts for ASNs, prefixes,
|
||||||
@@ -38,7 +34,7 @@ type Stats struct {
|
|||||||
type Store interface {
|
type Store interface {
|
||||||
// ASN operations
|
// ASN operations
|
||||||
GetOrCreateASN(number int, timestamp time.Time) (*ASN, error)
|
GetOrCreateASN(number int, timestamp time.Time) (*ASN, error)
|
||||||
GetOrCreateASNBatch(asns map[int]SeenTimes) error
|
GetOrCreateASNBatch(asns map[int]time.Time) error
|
||||||
|
|
||||||
// Prefix operations
|
// Prefix operations
|
||||||
GetOrCreatePrefix(prefix string, timestamp time.Time) (*Prefix, error)
|
GetOrCreatePrefix(prefix string, timestamp time.Time) (*Prefix, error)
|
||||||
@@ -48,7 +44,7 @@ type Store interface {
|
|||||||
RecordAnnouncement(announcement *Announcement) error
|
RecordAnnouncement(announcement *Announcement) error
|
||||||
|
|
||||||
// Peering operations
|
// Peering operations
|
||||||
RecordPeering(asA, asB int, seen SeenTimes) error
|
RecordPeering(asA, asB int, timestamp time.Time) error
|
||||||
|
|
||||||
// Statistics
|
// Statistics
|
||||||
GetStats() (Stats, error)
|
GetStats() (Stats, error)
|
||||||
@@ -71,10 +67,6 @@ type Store interface {
|
|||||||
GetLiveRouteCounts() (ipv4Count, ipv6Count int, err error)
|
GetLiveRouteCounts() (ipv4Count, ipv6Count int, err error)
|
||||||
GetLiveRouteCountsContext(ctx context.Context) (ipv4Count, ipv6Count int, err error)
|
GetLiveRouteCountsContext(ctx context.Context) (ipv4Count, ipv6Count int, err error)
|
||||||
|
|
||||||
// Routing table operations: the database's copy of the table in memory
|
|
||||||
WriteRouteChanges(changes []routetable.Change, newPeers map[string]uint) error
|
|
||||||
ReadRouteTable(ctx context.Context) (*routetable.Table, error)
|
|
||||||
|
|
||||||
// IP lookup operations
|
// IP lookup operations
|
||||||
GetIPInfo(ip string) (*IPInfo, error)
|
GetIPInfo(ip string) (*IPInfo, error)
|
||||||
GetIPInfoContext(ctx context.Context, ip string) (*IPInfo, error)
|
GetIPInfoContext(ctx context.Context, ip string) (*IPInfo, error)
|
||||||
|
|||||||
+11
-34
@@ -28,10 +28,9 @@ type ASN struct {
|
|||||||
RIRRegDate *time.Time `json:"rir_registration_date,omitempty"`
|
RIRRegDate *time.Time `json:"rir_registration_date,omitempty"`
|
||||||
RIRLastMod *time.Time `json:"rir_last_modified,omitempty"`
|
RIRLastMod *time.Time `json:"rir_last_modified,omitempty"`
|
||||||
WHOISRaw string `json:"whois_raw,omitempty"`
|
WHOISRaw string `json:"whois_raw,omitempty"`
|
||||||
// Timestamps. They are zero, and left out of JSON, for an AS that has no
|
// Timestamps
|
||||||
// record in the asns table.
|
FirstSeen time.Time `json:"first_seen"`
|
||||||
FirstSeen time.Time `json:"first_seen,omitzero"`
|
LastSeen time.Time `json:"last_seen"`
|
||||||
LastSeen time.Time `json:"last_seen,omitzero"`
|
|
||||||
WHOISUpdatedAt *time.Time `json:"whois_updated_at,omitempty"`
|
WHOISUpdatedAt *time.Time `json:"whois_updated_at,omitempty"`
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -92,11 +91,12 @@ type PrefixDistribution struct {
|
|||||||
//
|
//
|
||||||
//nolint:tagliatelle // field names are the existing JSON API
|
//nolint:tagliatelle // field names are the existing JSON API
|
||||||
type IPInfo struct {
|
type IPInfo struct {
|
||||||
IP string `json:"ip"`
|
IP string `json:"ip"`
|
||||||
Netblock string `json:"netblock"`
|
PTR []string `json:"ptr,omitempty"`
|
||||||
MaskLength int `json:"mask_length"`
|
Netblock string `json:"netblock"`
|
||||||
IPVersion int `json:"ip_version"`
|
MaskLength int `json:"mask_length"`
|
||||||
NumPeers int `json:"num_peers"`
|
IPVersion int `json:"ip_version"`
|
||||||
|
NumPeers int `json:"num_peers"`
|
||||||
// AS information
|
// AS information
|
||||||
ASN int `json:"asn"`
|
ASN int `json:"asn"`
|
||||||
ASName string `json:"as_name,omitempty"`
|
ASName string `json:"as_name,omitempty"`
|
||||||
@@ -122,35 +122,12 @@ type LiveRouteDeletion struct {
|
|||||||
IPVersion int
|
IPVersion int
|
||||||
}
|
}
|
||||||
|
|
||||||
// SeenTimes is the earliest and the latest time one batch of updates saw an AS,
|
// PeerUpdate represents parameters for updating a peer
|
||||||
// a peer or a peering. A row the batch adds is stored first seen at First and
|
|
||||||
// last seen at Last. A row already stored takes First as its first_seen only
|
|
||||||
// when First is earlier, and Last as its last_seen only when Last is later.
|
|
||||||
type SeenTimes struct {
|
|
||||||
First time.Time
|
|
||||||
Last time.Time
|
|
||||||
}
|
|
||||||
|
|
||||||
// Add makes t the earliest time of s when it is earlier than First, and the
|
|
||||||
// latest when it is later than Last. The zero SeenTimes holds no time yet, so
|
|
||||||
// the first Add sets both.
|
|
||||||
func (s *SeenTimes) Add(t time.Time) {
|
|
||||||
if s.First.IsZero() || t.Before(s.First) {
|
|
||||||
s.First = t
|
|
||||||
}
|
|
||||||
|
|
||||||
if s.Last.IsZero() || t.After(s.Last) {
|
|
||||||
s.Last = t
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// PeerUpdate represents parameters for updating a peer. PeerASN and
|
|
||||||
// MessageType are those of the peer's latest update.
|
|
||||||
type PeerUpdate struct {
|
type PeerUpdate struct {
|
||||||
PeerIP string
|
PeerIP string
|
||||||
PeerASN int
|
PeerASN int
|
||||||
MessageType string
|
MessageType string
|
||||||
Seen SeenTimes
|
Timestamp time.Time
|
||||||
}
|
}
|
||||||
|
|
||||||
// ASNWHOISUpdate contains WHOIS data for updating an ASN record.
|
// ASNWHOISUpdate contains WHOIS data for updating an ASN record.
|
||||||
|
|||||||
@@ -1,339 +0,0 @@
|
|||||||
package database
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"database/sql"
|
|
||||||
"encoding/json"
|
|
||||||
"errors"
|
|
||||||
"fmt"
|
|
||||||
"net/netip"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/bits-and-blooms/bitset"
|
|
||||||
|
|
||||||
"git.eeqj.de/sneak/routewatch/internal/routetable"
|
|
||||||
)
|
|
||||||
|
|
||||||
// keptRoutesPerRow is how many kept routes a row of the routes table has
|
|
||||||
// columns for: the newest and the older one.
|
|
||||||
const keptRoutesPerRow = 2
|
|
||||||
|
|
||||||
// errUnknownRoutePeer is returned for a peer that route_peers lacks: on a
|
|
||||||
// write, the peer of a kept route; on a read, a peer number in a row.
|
|
||||||
var errUnknownRoutePeer = errors.New("peer not in route_peers")
|
|
||||||
|
|
||||||
// The statements of the routes and route_peers tables (schema.sql).
|
|
||||||
const (
|
|
||||||
insertRoutePeerSQL = `INSERT INTO route_peers (number, address)
|
|
||||||
VALUES (?, ?)`
|
|
||||||
routePeerNumberSQL = `SELECT number FROM route_peers WHERE address = ?`
|
|
||||||
routePeersSQL = `SELECT number, address FROM route_peers`
|
|
||||||
|
|
||||||
upsertRouteSQL = `INSERT INTO routes (prefix, origin_asn, peers,
|
|
||||||
newest_peer, newest_as_path, newest_time,
|
|
||||||
older_peer, older_as_path, older_time)
|
|
||||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
||||||
ON CONFLICT (prefix, origin_asn) DO UPDATE SET peers = excluded.peers,
|
|
||||||
newest_peer = excluded.newest_peer,
|
|
||||||
newest_as_path = excluded.newest_as_path,
|
|
||||||
newest_time = excluded.newest_time,
|
|
||||||
older_peer = excluded.older_peer,
|
|
||||||
older_as_path = excluded.older_as_path,
|
|
||||||
older_time = excluded.older_time`
|
|
||||||
deleteRouteSQL = `DELETE FROM routes WHERE prefix = ? AND origin_asn = ?`
|
|
||||||
routesSQL = `SELECT prefix, origin_asn, peers,
|
|
||||||
newest_peer, newest_as_path, newest_time,
|
|
||||||
older_peer, older_as_path, older_time FROM routes`
|
|
||||||
)
|
|
||||||
|
|
||||||
// WriteRouteChanges writes, in one transaction, what one call of the routing
|
|
||||||
// table's TakeChanges returned. It stores each new peer with its number, writes
|
|
||||||
// the row of each changed prefix and origin AS pair, and deletes the row of
|
|
||||||
// each removed one. Route times are stored to the second, as the feed gives
|
|
||||||
// them.
|
|
||||||
//
|
|
||||||
// When it fails, it writes nothing. The caller then writes these changes and
|
|
||||||
// peers again, because later changes can name the peers it did not store.
|
|
||||||
func (d *Database) WriteRouteChanges(
|
|
||||||
changes []routetable.Change, newPeers map[string]uint,
|
|
||||||
) error {
|
|
||||||
if len(changes) == 0 && len(newPeers) == 0 {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
d.lock("WriteRouteChanges")
|
|
||||||
defer d.unlock()
|
|
||||||
|
|
||||||
tx, err := d.beginTx()
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("failed to begin transaction: %w", err)
|
|
||||||
}
|
|
||||||
defer d.rollback(tx)
|
|
||||||
|
|
||||||
ctx := context.Background()
|
|
||||||
|
|
||||||
// The new peers first, so that the rows can look up their numbers.
|
|
||||||
for address, number := range newPeers {
|
|
||||||
_, err := tx.ExecContext(ctx, insertRoutePeerSQL, number, address)
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("failed to store peer %s: %w", address, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
err = writeRouteRows(ctx, tx, changes)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
err = tx.Commit()
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("failed to commit transaction: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// writeRouteRows upserts the row of each change in tx, or deletes it for a
|
|
||||||
// removed pair.
|
|
||||||
func writeRouteRows(
|
|
||||||
ctx context.Context, tx *loggingTx, changes []routetable.Change,
|
|
||||||
) error {
|
|
||||||
upsert, err := tx.PrepareContext(ctx, upsertRouteSQL)
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("failed to prepare route upsert: %w", err)
|
|
||||||
}
|
|
||||||
defer func() { _ = upsert.Close() }()
|
|
||||||
|
|
||||||
del, err := tx.PrepareContext(ctx, deleteRouteSQL)
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("failed to prepare route delete: %w", err)
|
|
||||||
}
|
|
||||||
defer func() { _ = del.Close() }()
|
|
||||||
|
|
||||||
peerNumber, err := tx.PrepareContext(ctx, routePeerNumberSQL)
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("failed to prepare peer number lookup: %w", err)
|
|
||||||
}
|
|
||||||
defer func() { _ = peerNumber.Close() }()
|
|
||||||
|
|
||||||
for _, change := range changes {
|
|
||||||
prefix := change.Prefix.String()
|
|
||||||
|
|
||||||
if change.Removed {
|
|
||||||
_, err := del.ExecContext(ctx, prefix, change.OriginASN)
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("failed to delete route %s: %w", prefix, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
|
|
||||||
row, err := routeRow(ctx, peerNumber, change)
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("failed to write route %s: %w", prefix, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
_, err = upsert.ExecContext(ctx, row...)
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("failed to write route %s: %w", prefix, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// routeRow returns the values of the routes row of change, in the order of
|
|
||||||
// upsertRouteSQL. peerNumber looks up a peer's number by its address. The
|
|
||||||
// table keeps at most keptRoutesPerRow routes of a pair.
|
|
||||||
func routeRow(
|
|
||||||
ctx context.Context, peerNumber *sql.Stmt, change routetable.Change,
|
|
||||||
) ([]any, error) {
|
|
||||||
peers, err := change.Peers.MarshalBinary()
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
|
|
||||||
row := []any{change.Prefix.String(), change.OriginASN, peers}
|
|
||||||
|
|
||||||
for i := range keptRoutesPerRow {
|
|
||||||
if i >= len(change.Routes) {
|
|
||||||
row = append(row, nil, nil, nil)
|
|
||||||
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
|
|
||||||
r := change.Routes[i]
|
|
||||||
|
|
||||||
var number uint
|
|
||||||
|
|
||||||
err := peerNumber.QueryRowContext(ctx, r.Peer).Scan(&number)
|
|
||||||
if errors.Is(err, sql.ErrNoRows) {
|
|
||||||
return nil, fmt.Errorf("%w: %s", errUnknownRoutePeer, r.Peer)
|
|
||||||
}
|
|
||||||
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
|
|
||||||
path, err := json.Marshal(r.ASPath)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
|
|
||||||
row = append(row, number, string(path), r.Time.Unix())
|
|
||||||
}
|
|
||||||
|
|
||||||
return row, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// ReadRouteTable returns the routing table that the routes and route_peers
|
|
||||||
// tables hold, as startup reads it. It reads the peers first, so that the
|
|
||||||
// table keeps their numbers, then loads each row into the table as it reads
|
|
||||||
// it. The two reads are separate, so nothing may write the tables meanwhile.
|
|
||||||
// It fails on a row that names a peer number route_peers lacks.
|
|
||||||
func (d *Database) ReadRouteTable(ctx context.Context) (*routetable.Table, error) {
|
|
||||||
peerNumbers, err := d.readRoutePeers(ctx)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
|
|
||||||
addresses := make(map[uint]string, len(peerNumbers))
|
|
||||||
numbers := new(bitset.BitSet)
|
|
||||||
|
|
||||||
for address, number := range peerNumbers {
|
|
||||||
addresses[number] = address
|
|
||||||
numbers.Set(number)
|
|
||||||
}
|
|
||||||
|
|
||||||
table := routetable.New(routetable.Params{PeerNumbers: peerNumbers})
|
|
||||||
|
|
||||||
rows, err := d.db.QueryContext(ctx, routesSQL)
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("failed to read routes: %w", err)
|
|
||||||
}
|
|
||||||
defer func() { _ = rows.Close() }()
|
|
||||||
|
|
||||||
for rows.Next() {
|
|
||||||
change, err := scanRouteRow(rows, addresses, numbers)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
|
|
||||||
table.Load(change)
|
|
||||||
}
|
|
||||||
|
|
||||||
err = rows.Err()
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("failed to read routes: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
return table, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// readRoutePeers returns the number of each peer in route_peers, by address.
|
|
||||||
func (d *Database) readRoutePeers(ctx context.Context) (map[string]uint, error) {
|
|
||||||
rows, err := d.db.QueryContext(ctx, routePeersSQL)
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("failed to read route peers: %w", err)
|
|
||||||
}
|
|
||||||
defer func() { _ = rows.Close() }()
|
|
||||||
|
|
||||||
peerNumbers := make(map[string]uint)
|
|
||||||
|
|
||||||
for rows.Next() {
|
|
||||||
var (
|
|
||||||
number uint
|
|
||||||
address string
|
|
||||||
)
|
|
||||||
|
|
||||||
err := rows.Scan(&number, &address)
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("failed to read a route peer: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
peerNumbers[address] = number
|
|
||||||
}
|
|
||||||
|
|
||||||
err = rows.Err()
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("failed to read route peers: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
return peerNumbers, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// keptRouteColumns are the three columns of one kept route in a routes row.
|
|
||||||
type keptRouteColumns struct {
|
|
||||||
peer sql.Null[uint]
|
|
||||||
asPath sql.NullString
|
|
||||||
time sql.NullInt64
|
|
||||||
}
|
|
||||||
|
|
||||||
// scanRouteRow returns the content of the routes row rows is at, with each kept
|
|
||||||
// route's peer named by its address in addresses. It fails when the row names a
|
|
||||||
// peer number that addresses lacks. numbers holds the numbers of addresses as a
|
|
||||||
// set, against which a row's whole peer set is checked at once.
|
|
||||||
func scanRouteRow(
|
|
||||||
rows *sql.Rows, addresses map[uint]string, numbers *bitset.BitSet,
|
|
||||||
) (routetable.Change, error) {
|
|
||||||
var (
|
|
||||||
change routetable.Change
|
|
||||||
prefix string
|
|
||||||
peers []byte
|
|
||||||
kept [keptRoutesPerRow]keptRouteColumns
|
|
||||||
)
|
|
||||||
|
|
||||||
err := rows.Scan(&prefix, &change.OriginASN, &peers,
|
|
||||||
&kept[0].peer, &kept[0].asPath, &kept[0].time,
|
|
||||||
&kept[1].peer, &kept[1].asPath, &kept[1].time)
|
|
||||||
if err != nil {
|
|
||||||
return change, fmt.Errorf("failed to read a route: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
change.Prefix, err = netip.ParsePrefix(prefix)
|
|
||||||
if err != nil {
|
|
||||||
return change, fmt.Errorf("failed to read route %s: %w", prefix, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
change.Peers = new(bitset.BitSet)
|
|
||||||
|
|
||||||
err = change.Peers.UnmarshalBinary(peers)
|
|
||||||
if err != nil {
|
|
||||||
return change, fmt.Errorf("failed to read the peers of route %s: %w",
|
|
||||||
prefix, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
if !numbers.IsSuperSet(change.Peers) {
|
|
||||||
unknown, _ := change.Peers.Difference(numbers).NextSet(0)
|
|
||||||
|
|
||||||
return change, fmt.Errorf("%w: number %d in the peers of route %s",
|
|
||||||
errUnknownRoutePeer, unknown, prefix)
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, columns := range kept {
|
|
||||||
if !columns.peer.Valid {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
|
|
||||||
address, known := addresses[columns.peer.V]
|
|
||||||
if !known {
|
|
||||||
return change, fmt.Errorf("%w: number %d in route %s",
|
|
||||||
errUnknownRoutePeer, columns.peer.V, prefix)
|
|
||||||
}
|
|
||||||
|
|
||||||
r := routetable.Route{
|
|
||||||
Peer: address,
|
|
||||||
Time: time.Unix(columns.time.Int64, 0).UTC(),
|
|
||||||
}
|
|
||||||
|
|
||||||
err := json.Unmarshal([]byte(columns.asPath.String), &r.ASPath)
|
|
||||||
if err != nil {
|
|
||||||
return change, fmt.Errorf("failed to read an AS path of route %s: %w",
|
|
||||||
prefix, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
change.Routes = append(change.Routes, r)
|
|
||||||
}
|
|
||||||
|
|
||||||
return change, nil
|
|
||||||
}
|
|
||||||
@@ -1,527 +0,0 @@
|
|||||||
package database
|
|
||||||
|
|
||||||
import (
|
|
||||||
"errors"
|
|
||||||
"fmt"
|
|
||||||
"math"
|
|
||||||
"math/rand/v2"
|
|
||||||
"net/netip"
|
|
||||||
"os"
|
|
||||||
"reflect"
|
|
||||||
"slices"
|
|
||||||
"testing"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/bits-and-blooms/bitset"
|
|
||||||
|
|
||||||
"git.eeqj.de/sneak/routewatch/internal/config"
|
|
||||||
"git.eeqj.de/sneak/routewatch/internal/logger"
|
|
||||||
"git.eeqj.de/sneak/routewatch/internal/routetable"
|
|
||||||
)
|
|
||||||
|
|
||||||
// Peers, origin ASes and prefixes of the routes table tests.
|
|
||||||
const (
|
|
||||||
peerA = "192.0.2.1"
|
|
||||||
peerB = "192.0.2.2"
|
|
||||||
peerC = "2001:db8::3"
|
|
||||||
peerD = "2001:db8::4"
|
|
||||||
peerE = "2001:db8::5"
|
|
||||||
|
|
||||||
asA = 64500
|
|
||||||
asB = 64501
|
|
||||||
// transitAS is the first AS of every test path, the peer's own.
|
|
||||||
transitAS = 64496
|
|
||||||
|
|
||||||
tablePrefixV4 = "10.1.2.0/24"
|
|
||||||
tablePrefixShortV4 = "10.9.0.0/16"
|
|
||||||
tablePrefixV6 = "2001:db8:1::/48"
|
|
||||||
)
|
|
||||||
|
|
||||||
// tableTime returns the time second seconds after the start of every routes
|
|
||||||
// table test.
|
|
||||||
func tableTime(second int) time.Time {
|
|
||||||
return time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC).
|
|
||||||
Add(time.Duration(second) * time.Second)
|
|
||||||
}
|
|
||||||
|
|
||||||
// announceTo has peer announce prefix with origin AS asn second seconds after
|
|
||||||
// the start.
|
|
||||||
func announceTo(
|
|
||||||
table *routetable.Table, prefix string, asn int, peer string, second int,
|
|
||||||
) {
|
|
||||||
table.Announce(netip.MustParsePrefix(prefix), asn, routetable.Route{
|
|
||||||
Peer: peer, ASPath: []int{transitAS, asn}, Time: tableTime(second),
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
// newRoutesTestDatabase returns a new database in a temporary directory.
|
|
||||||
func newRoutesTestDatabase(t *testing.T) *Database {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
db, err := New(&config.Config{StateDir: t.TempDir()}, logger.New())
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("failed to create database: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
t.Cleanup(func() { _ = db.Close() })
|
|
||||||
|
|
||||||
return db
|
|
||||||
}
|
|
||||||
|
|
||||||
// writeTableChanges writes what table changed since the last call.
|
|
||||||
func writeTableChanges(t *testing.T, db *Database, table *routetable.Table) {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
changes, newPeers := table.TakeChanges(math.MaxInt)
|
|
||||||
|
|
||||||
err := db.WriteRouteChanges(changes, newPeers)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("WriteRouteChanges: %v", err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestReadRouteTableGivesBackWhatWasWritten(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
db := newRoutesTestDatabase(t)
|
|
||||||
source := routetable.New(routetable.Params{})
|
|
||||||
|
|
||||||
// Two origins of one prefix: the routes of peerC and peerB are kept for
|
|
||||||
// asA, peerA keeps only its bit.
|
|
||||||
announceTo(source, tablePrefixV4, asA, peerA, 10)
|
|
||||||
announceTo(source, tablePrefixV4, asA, peerB, 20)
|
|
||||||
announceTo(source, tablePrefixV4, asA, peerC, 30)
|
|
||||||
announceTo(source, tablePrefixV4, asB, peerD, 40)
|
|
||||||
// Two routes with the same time.
|
|
||||||
announceTo(source, tablePrefixV6, asA, peerA, 50)
|
|
||||||
announceTo(source, tablePrefixV6, asA, peerC, 50)
|
|
||||||
// An origin whose kept routes were withdrawn while peerA carries it.
|
|
||||||
announceTo(source, tablePrefixShortV4, asB, peerA, 10)
|
|
||||||
announceTo(source, tablePrefixShortV4, asB, peerB, 20)
|
|
||||||
announceTo(source, tablePrefixShortV4, asB, peerC, 30)
|
|
||||||
source.Withdraw(netip.MustParsePrefix(tablePrefixShortV4), peerB)
|
|
||||||
source.Withdraw(netip.MustParsePrefix(tablePrefixShortV4), peerC)
|
|
||||||
|
|
||||||
writeTableChanges(t, db, source)
|
|
||||||
checkReadTable(t, db, source)
|
|
||||||
|
|
||||||
// A peer moves to another origin, one goes down and a new one comes.
|
|
||||||
announceTo(source, tablePrefixV4, asB, peerA, 60)
|
|
||||||
source.PeerDown(peerC)
|
|
||||||
announceTo(source, tablePrefixV6, asB, peerE, 70)
|
|
||||||
|
|
||||||
writeTableChanges(t, db, source)
|
|
||||||
checkReadTable(t, db, source)
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestReadRouteTableGivesATableThatTakesLaterUpdates(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
db := newRoutesTestDatabase(t)
|
|
||||||
source := routetable.New(routetable.Params{})
|
|
||||||
|
|
||||||
// peerA and peerB carry tablePrefixV4, peerB and peerC tablePrefixV6.
|
|
||||||
announceTo(source, tablePrefixV4, asA, peerA, 10)
|
|
||||||
announceTo(source, tablePrefixV4, asA, peerB, 20)
|
|
||||||
announceTo(source, tablePrefixV6, asA, peerB, 30)
|
|
||||||
announceTo(source, tablePrefixV6, asA, peerC, 40)
|
|
||||||
|
|
||||||
writeTableChanges(t, db, source)
|
|
||||||
read := checkReadTable(t, db, source)
|
|
||||||
|
|
||||||
// The same updates to both tables: a loaded peer withdraws a prefix,
|
|
||||||
// another one goes down, and a new peer comes.
|
|
||||||
for _, table := range []*routetable.Table{source, read} {
|
|
||||||
table.Withdraw(netip.MustParsePrefix(tablePrefixV4), peerA)
|
|
||||||
table.PeerDown(peerB)
|
|
||||||
announceTo(table, tablePrefixShortV4, asB, peerD, 50)
|
|
||||||
}
|
|
||||||
|
|
||||||
checkSameReads(t, read, source)
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestReadRouteTableFailsOnAPeerWithoutANumber(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
db := newRoutesTestDatabase(t)
|
|
||||||
table := routetable.New(routetable.Params{})
|
|
||||||
|
|
||||||
// Only the routes of peerB and peerC are kept, so only the peer set names
|
|
||||||
// peerA.
|
|
||||||
announceTo(table, tablePrefixV4, asA, peerA, 10)
|
|
||||||
announceTo(table, tablePrefixV4, asA, peerB, 20)
|
|
||||||
announceTo(table, tablePrefixV4, asA, peerC, 30)
|
|
||||||
writeTableChanges(t, db, table)
|
|
||||||
|
|
||||||
_, err := db.db.ExecContext(t.Context(),
|
|
||||||
"DELETE FROM route_peers WHERE address = ?", peerA)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("deleting a route peer: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
_, err = db.ReadRouteTable(t.Context())
|
|
||||||
if !errors.Is(err, errUnknownRoutePeer) {
|
|
||||||
t.Fatalf("ReadRouteTable = %v, want %v", err, errUnknownRoutePeer)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// checkReadTable checks that the table ReadRouteTable reads answers every read
|
|
||||||
// as want does, and that reading it recorded no change. It returns that table.
|
|
||||||
func checkReadTable(
|
|
||||||
t *testing.T, db *Database, want *routetable.Table,
|
|
||||||
) *routetable.Table {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
got, err := db.ReadRouteTable(t.Context())
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("ReadRouteTable: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
checkSameReads(t, got, want)
|
|
||||||
|
|
||||||
changes, newPeers := got.TakeChanges(math.MaxInt)
|
|
||||||
if len(changes) != 0 || len(newPeers) != 0 {
|
|
||||||
t.Errorf("the read recorded changes %+v and new peers %v, want none",
|
|
||||||
changes, newPeers)
|
|
||||||
}
|
|
||||||
|
|
||||||
return got
|
|
||||||
}
|
|
||||||
|
|
||||||
// checkSameReads checks that got answers every read as want does, for the
|
|
||||||
// prefixes and the origin ASes the tests use.
|
|
||||||
func checkSameReads(t *testing.T, got, want *routetable.Table) {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
if !reflect.DeepEqual(got.Counts(), want.Counts()) {
|
|
||||||
t.Errorf("Counts() = %+v, want %+v", got.Counts(), want.Counts())
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, text := range []string{
|
|
||||||
tablePrefixV4, tablePrefixShortV4, tablePrefixV6,
|
|
||||||
} {
|
|
||||||
prefix := netip.MustParsePrefix(text)
|
|
||||||
|
|
||||||
if !reflect.DeepEqual(got.Origins(prefix), want.Origins(prefix)) {
|
|
||||||
t.Errorf("Origins(%s) = %+v, want %+v",
|
|
||||||
prefix, got.Origins(prefix), want.Origins(prefix))
|
|
||||||
}
|
|
||||||
|
|
||||||
gotSummary, gotFound := got.Lookup(prefix.Addr())
|
|
||||||
wantSummary, wantFound := want.Lookup(prefix.Addr())
|
|
||||||
|
|
||||||
if gotFound != wantFound || !reflect.DeepEqual(gotSummary, wantSummary) {
|
|
||||||
t.Errorf("Lookup(%s) = %+v, %v; want %+v, %v", prefix.Addr(),
|
|
||||||
gotSummary, gotFound, wantSummary, wantFound)
|
|
||||||
}
|
|
||||||
|
|
||||||
ipVersion := ipVersionV6
|
|
||||||
if prefix.Addr().Is4() {
|
|
||||||
ipVersion = ipVersionV4
|
|
||||||
}
|
|
||||||
|
|
||||||
gotSample := sortedSample(got, ipVersion, prefix.Bits())
|
|
||||||
wantSample := sortedSample(want, ipVersion, prefix.Bits())
|
|
||||||
|
|
||||||
if !reflect.DeepEqual(gotSample, wantSample) {
|
|
||||||
t.Errorf("SamplePrefixes(%d, %d) = %+v, want %+v",
|
|
||||||
ipVersion, prefix.Bits(), gotSample, wantSample)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, asn := range []int{asA, asB} {
|
|
||||||
if !reflect.DeepEqual(got.OriginatedPrefixes(asn),
|
|
||||||
want.OriginatedPrefixes(asn)) {
|
|
||||||
t.Errorf("OriginatedPrefixes(%d) = %+v, want %+v", asn,
|
|
||||||
got.OriginatedPrefixes(asn), want.OriginatedPrefixes(asn))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// sortedSample returns every prefix of the IP version and mask length in
|
|
||||||
// table, in prefix order.
|
|
||||||
func sortedSample(
|
|
||||||
table *routetable.Table, ipVersion, maskLength int,
|
|
||||||
) []routetable.PrefixSummary {
|
|
||||||
// More than the tests have of any mask length.
|
|
||||||
const all = 100
|
|
||||||
|
|
||||||
sample := table.SamplePrefixes(ipVersion, maskLength, all)
|
|
||||||
slices.SortFunc(sample, func(a, b routetable.PrefixSummary) int {
|
|
||||||
return a.Prefix.Compare(b.Prefix)
|
|
||||||
})
|
|
||||||
|
|
||||||
return sample
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestWriteRouteChangesDeletesTheRowOfARemovedPair(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
db := newRoutesTestDatabase(t)
|
|
||||||
table := routetable.New(routetable.Params{})
|
|
||||||
|
|
||||||
announceTo(table, tablePrefixV4, asA, peerA, 10)
|
|
||||||
announceTo(table, tablePrefixV4, asB, peerB, 20)
|
|
||||||
writeTableChanges(t, db, table)
|
|
||||||
|
|
||||||
// The prefix's only peer with asA withdraws it.
|
|
||||||
table.Withdraw(netip.MustParsePrefix(tablePrefixV4), peerA)
|
|
||||||
writeTableChanges(t, db, table)
|
|
||||||
|
|
||||||
var origins []int
|
|
||||||
|
|
||||||
rows, err := db.db.QueryContext(t.Context(),
|
|
||||||
"SELECT origin_asn FROM routes WHERE prefix = ?", tablePrefixV4)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("reading the routes: %v", err)
|
|
||||||
}
|
|
||||||
defer func() { _ = rows.Close() }()
|
|
||||||
|
|
||||||
for rows.Next() {
|
|
||||||
var origin int
|
|
||||||
|
|
||||||
err := rows.Scan(&origin)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("reading a route: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
origins = append(origins, origin)
|
|
||||||
}
|
|
||||||
|
|
||||||
err = rows.Err()
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("reading the routes: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
if !slices.Equal(origins, []int{asB}) {
|
|
||||||
t.Errorf("rows of %s have origins %v, want only %d",
|
|
||||||
tablePrefixV4, origins, asB)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// routePeer is one row of route_peers.
|
|
||||||
type routePeer struct {
|
|
||||||
number uint
|
|
||||||
address string
|
|
||||||
}
|
|
||||||
|
|
||||||
// readRoutePeerRows returns the rows of route_peers in number order.
|
|
||||||
func readRoutePeerRows(t *testing.T, db *Database) []routePeer {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
rows, err := db.db.QueryContext(t.Context(),
|
|
||||||
"SELECT number, address FROM route_peers ORDER BY number")
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("reading the route peers: %v", err)
|
|
||||||
}
|
|
||||||
defer func() { _ = rows.Close() }()
|
|
||||||
|
|
||||||
var peers []routePeer
|
|
||||||
|
|
||||||
for rows.Next() {
|
|
||||||
var peer routePeer
|
|
||||||
|
|
||||||
err := rows.Scan(&peer.number, &peer.address)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("reading a route peer: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
peers = append(peers, peer)
|
|
||||||
}
|
|
||||||
|
|
||||||
err = rows.Err()
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("reading the route peers: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
return peers
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestWriteRouteChangesStoresNewPeers(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
db := newRoutesTestDatabase(t)
|
|
||||||
table := routetable.New(routetable.Params{})
|
|
||||||
|
|
||||||
announceTo(table, tablePrefixV4, asA, peerA, 10)
|
|
||||||
announceTo(table, tablePrefixV6, asA, peerC, 20)
|
|
||||||
writeTableChanges(t, db, table)
|
|
||||||
|
|
||||||
// peerA is stored already; peerB is new.
|
|
||||||
announceTo(table, tablePrefixV4, asA, peerA, 30)
|
|
||||||
announceTo(table, tablePrefixV4, asA, peerB, 40)
|
|
||||||
writeTableChanges(t, db, table)
|
|
||||||
|
|
||||||
want := []routePeer{{0, peerA}, {1, peerC}, {2, peerB}}
|
|
||||||
|
|
||||||
got := readRoutePeerRows(t, db)
|
|
||||||
if !slices.Equal(got, want) {
|
|
||||||
t.Errorf("route_peers = %+v, want %+v", got, want)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestWriteRouteChangesWritesNothingWhenItFails(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
db := newRoutesTestDatabase(t)
|
|
||||||
table := routetable.New(routetable.Params{})
|
|
||||||
|
|
||||||
announceTo(table, tablePrefixV4, asA, peerA, 10)
|
|
||||||
announceTo(table, tablePrefixV6, asA, peerB, 20)
|
|
||||||
|
|
||||||
// peerB's number is missing, so its kept route cannot be written.
|
|
||||||
changes, newPeers := table.TakeChanges(math.MaxInt)
|
|
||||||
delete(newPeers, peerB)
|
|
||||||
|
|
||||||
err := db.WriteRouteChanges(changes, newPeers)
|
|
||||||
if !errors.Is(err, errUnknownRoutePeer) {
|
|
||||||
t.Fatalf("WriteRouteChanges = %v, want %v", err, errUnknownRoutePeer)
|
|
||||||
}
|
|
||||||
|
|
||||||
peers := readRoutePeerRows(t, db)
|
|
||||||
if len(peers) != 0 {
|
|
||||||
t.Errorf("route_peers = %+v after the failed write, want none", peers)
|
|
||||||
}
|
|
||||||
|
|
||||||
var routes int
|
|
||||||
|
|
||||||
err = db.db.QueryRowContext(t.Context(),
|
|
||||||
"SELECT COUNT(*) FROM routes").Scan(&routes)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("counting the routes: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
if routes != 0 {
|
|
||||||
t.Errorf("routes holds %d rows after the failed write, want none", routes)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Parameters for BenchmarkReadRouteTable.
|
|
||||||
const (
|
|
||||||
// benchTablePairs is how many prefix and origin AS pairs the table holds:
|
|
||||||
// one origin AS for each prefix. Every benchTableV6Every-th prefix is an
|
|
||||||
// IPv6 /48, the others IPv4 /24s.
|
|
||||||
benchTablePairs = 200000
|
|
||||||
benchTableV6Every = 5
|
|
||||||
// benchTablePeers is how many peers there are, about as many as RIS has.
|
|
||||||
// benchTablePeersPerPair is how many of them, picked at random, carry each
|
|
||||||
// pair: RIS's 460 million routes over 1.4 million pairs.
|
|
||||||
benchTablePeers = 1400
|
|
||||||
benchTablePeersPerPair = 330
|
|
||||||
// benchTableBatch is how many changes each write holds.
|
|
||||||
benchTableBatch = 10000
|
|
||||||
)
|
|
||||||
|
|
||||||
// BenchmarkReadRouteTable times the read of a routing table of 200,000 prefix
|
|
||||||
// and origin AS pairs, each carried by 330 of 1,400 peers and keeping two
|
|
||||||
// routes. It writes that table first. It reports the bytes of the database
|
|
||||||
// file per pair. It is run by hand, not by make test, for example in the test
|
|
||||||
// image:
|
|
||||||
//
|
|
||||||
// go test -run '^$' -bench ReadRouteTable -benchtime 5x ./internal/database/
|
|
||||||
func BenchmarkReadRouteTable(b *testing.B) {
|
|
||||||
db, err := New(&config.Config{StateDir: b.TempDir()}, logger.New())
|
|
||||||
if err != nil {
|
|
||||||
b.Fatalf("failed to create database: %v", err)
|
|
||||||
}
|
|
||||||
defer func() { _ = db.Close() }()
|
|
||||||
|
|
||||||
//nolint:gosec // The benchmark needs the same table every run.
|
|
||||||
rng := rand.New(rand.NewPCG(1, 2))
|
|
||||||
|
|
||||||
newPeers := make(map[string]uint, benchTablePeers)
|
|
||||||
for number := range uint(benchTablePeers) {
|
|
||||||
newPeers[benchPeerAddress(number)] = number
|
|
||||||
}
|
|
||||||
|
|
||||||
for first := 0; first < benchTablePairs; first += benchTableBatch {
|
|
||||||
changes := make([]routetable.Change, 0, benchTableBatch)
|
|
||||||
for pair := first; pair < first+benchTableBatch; pair++ {
|
|
||||||
changes = append(changes, benchTableChange(rng, pair))
|
|
||||||
}
|
|
||||||
|
|
||||||
err := db.WriteRouteChanges(changes, newPeers)
|
|
||||||
if err != nil {
|
|
||||||
b.Fatalf("WriteRouteChanges: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
newPeers = nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// The WAL is copied into the database file, whose size is reported.
|
|
||||||
err = db.Checkpoint(b.Context())
|
|
||||||
if err != nil {
|
|
||||||
b.Fatalf("Checkpoint: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
file, err := os.Stat(db.path)
|
|
||||||
if err != nil {
|
|
||||||
b.Fatalf("reading the database file's size: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
for b.Loop() {
|
|
||||||
table, err := db.ReadRouteTable(b.Context())
|
|
||||||
if err != nil {
|
|
||||||
b.Fatalf("ReadRouteTable: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
counts := table.Counts()
|
|
||||||
if counts.IPv4Prefixes+counts.IPv6Prefixes != benchTablePairs {
|
|
||||||
b.Fatalf("read %d prefixes, want %d",
|
|
||||||
counts.IPv4Prefixes+counts.IPv6Prefixes, benchTablePairs)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// After the loop: b.Loop drops the metrics reported before it starts.
|
|
||||||
b.ReportMetric(float64(file.Size())/benchTablePairs, "file-bytes/pair")
|
|
||||||
}
|
|
||||||
|
|
||||||
// benchPeerAddress returns the address of the peer numbered number.
|
|
||||||
func benchPeerAddress(number uint) string {
|
|
||||||
const addressesPerBlock = 256
|
|
||||||
|
|
||||||
return fmt.Sprintf("198.18.%d.%d",
|
|
||||||
number/addressesPerBlock, number%addressesPerBlock)
|
|
||||||
}
|
|
||||||
|
|
||||||
// benchTableChange returns the content of the pair numbered pair: its prefix,
|
|
||||||
// its origin AS, benchTablePeersPerPair peers picked at random, and the routes
|
|
||||||
// of two of them, each with an AS path of four ASes.
|
|
||||||
func benchTableChange(rng *rand.Rand, pair int) routetable.Change {
|
|
||||||
const (
|
|
||||||
firstOrigin = 64500
|
|
||||||
transitASes = 50000
|
|
||||||
olderSeconds = 10
|
|
||||||
)
|
|
||||||
|
|
||||||
text := fmt.Sprintf("%d.%d.%d.0/24", 1+pair>>16, (pair>>8)%256, pair%256)
|
|
||||||
if pair%benchTableV6Every == 0 {
|
|
||||||
text = fmt.Sprintf("2001:%x:%x::/48", pair>>16, pair%65536)
|
|
||||||
}
|
|
||||||
|
|
||||||
prefix := netip.MustParsePrefix(text)
|
|
||||||
origin := firstOrigin + pair%transitASes
|
|
||||||
peers := bitset.New(benchTablePeers)
|
|
||||||
|
|
||||||
for peers.Count() < benchTablePeersPerPair {
|
|
||||||
peers.Set(rng.UintN(benchTablePeers))
|
|
||||||
}
|
|
||||||
|
|
||||||
change := routetable.Change{Prefix: prefix, OriginASN: origin, Peers: peers}
|
|
||||||
|
|
||||||
// The routes of the first two peers, newest first.
|
|
||||||
for i, number := range slices.Collect(peers.EachSet())[:keptRoutesPerRow] {
|
|
||||||
change.Routes = append(change.Routes, routetable.Route{
|
|
||||||
Peer: benchPeerAddress(number),
|
|
||||||
ASPath: []int{
|
|
||||||
firstOrigin + rng.IntN(transitASes),
|
|
||||||
firstOrigin + rng.IntN(transitASes),
|
|
||||||
firstOrigin + rng.IntN(transitASes),
|
|
||||||
origin,
|
|
||||||
},
|
|
||||||
Time: tableTime(pair - i*olderSeconds),
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
return change
|
|
||||||
}
|
|
||||||
@@ -128,7 +128,7 @@ CREATE TABLE IF NOT EXISTS live_routes_v6 (
|
|||||||
-- prefix, last_updated: a prefix's routes, for the prefix page and its API
|
-- prefix, last_updated: a prefix's routes, for the prefix page and its API
|
||||||
-- answer, the IP lookup and its peer count, and the check whether a prefix
|
-- answer, the IP lookup and its peer count, and the check whether a prefix
|
||||||
-- still has a live route; and a prefix's newest route for the prefix-length
|
-- still has a live route; and a prefix's newest route for the prefix-length
|
||||||
-- pages and the AS page, in one search whatever the number of peers.
|
-- pages, in one search whatever the number of peers.
|
||||||
CREATE INDEX IF NOT EXISTS idx_live_routes_v4_prefix_last_updated ON live_routes_v4(prefix, last_updated);
|
CREATE INDEX IF NOT EXISTS idx_live_routes_v4_prefix_last_updated ON live_routes_v4(prefix, last_updated);
|
||||||
CREATE INDEX IF NOT EXISTS idx_live_routes_v6_prefix_last_updated ON live_routes_v6(prefix, last_updated);
|
CREATE INDEX IF NOT EXISTS idx_live_routes_v6_prefix_last_updated ON live_routes_v6(prefix, last_updated);
|
||||||
-- mask_length: the prefix-length pages' random sample, the row id range of a
|
-- mask_length: the prefix-length pages' random sample, the row id range of a
|
||||||
@@ -139,40 +139,10 @@ CREATE INDEX IF NOT EXISTS idx_live_routes_v6_mask_length ON live_routes_v6(mask
|
|||||||
-- prefixes in text order, and the prefix distribution read at startup.
|
-- prefixes in text order, and the prefix distribution read at startup.
|
||||||
CREATE INDEX IF NOT EXISTS idx_live_routes_v4_mask_prefix ON live_routes_v4(mask_length, prefix);
|
CREATE INDEX IF NOT EXISTS idx_live_routes_v4_mask_prefix ON live_routes_v4(mask_length, prefix);
|
||||||
CREATE INDEX IF NOT EXISTS idx_live_routes_v6_mask_prefix ON live_routes_v6(mask_length, prefix);
|
CREATE INDEX IF NOT EXISTS idx_live_routes_v6_mask_prefix ON live_routes_v6(mask_length, prefix);
|
||||||
-- origin_asn, prefix: the prefixes an AS originates, each in one search
|
-- origin_asn: the prefixes an AS originates, for the AS page and its API answer.
|
||||||
-- whatever the number of peers, for the AS page and its API answer.
|
CREATE INDEX IF NOT EXISTS idx_live_routes_v4_origin_asn ON live_routes_v4(origin_asn);
|
||||||
CREATE INDEX IF NOT EXISTS idx_live_routes_v4_origin_prefix ON live_routes_v4(origin_asn, prefix);
|
CREATE INDEX IF NOT EXISTS idx_live_routes_v6_origin_asn ON live_routes_v6(origin_asn);
|
||||||
CREATE INDEX IF NOT EXISTS idx_live_routes_v6_origin_prefix ON live_routes_v6(origin_asn, prefix);
|
|
||||||
-- last_updated: the oldest and newest route times in /api/v1/stats and
|
-- last_updated: the oldest and newest route times in /api/v1/stats and
|
||||||
-- /status.json.
|
-- /status.json.
|
||||||
CREATE INDEX IF NOT EXISTS idx_live_routes_v4_last_updated ON live_routes_v4(last_updated);
|
CREATE INDEX IF NOT EXISTS idx_live_routes_v4_last_updated ON live_routes_v4(last_updated);
|
||||||
CREATE INDEX IF NOT EXISTS idx_live_routes_v6_last_updated ON live_routes_v6(last_updated);
|
CREATE INDEX IF NOT EXISTS idx_live_routes_v6_last_updated ON live_routes_v6(last_updated);
|
||||||
|
|
||||||
-- The whole routing table, as internal/routetable keeps it in memory: one row
|
|
||||||
-- for each prefix and origin AS that some peer carries. Startup reads every
|
|
||||||
-- row, and the rows of the pairs that change are written back.
|
|
||||||
CREATE TABLE IF NOT EXISTS routes (
|
|
||||||
prefix TEXT NOT NULL, -- in the text form Go's net/netip prints
|
|
||||||
origin_asn INTEGER NOT NULL,
|
|
||||||
-- The peers whose current route to the prefix has this origin AS: a set
|
|
||||||
-- of their numbers in route_peers, as the bitset package's MarshalBinary
|
|
||||||
-- writes it.
|
|
||||||
peers BLOB NOT NULL,
|
|
||||||
-- The newest of those routes and the one before it, the two kept in full.
|
|
||||||
-- Each has its peer's number in route_peers, its AS path as a JSON array,
|
|
||||||
-- and its time in seconds since 1970, UTC. A route that is not kept has
|
|
||||||
-- NULL in its three columns.
|
|
||||||
newest_peer INTEGER,
|
|
||||||
newest_as_path TEXT,
|
|
||||||
newest_time INTEGER,
|
|
||||||
older_peer INTEGER,
|
|
||||||
older_as_path TEXT,
|
|
||||||
older_time INTEGER,
|
|
||||||
PRIMARY KEY (prefix, origin_asn)
|
|
||||||
) WITHOUT ROWID;
|
|
||||||
|
|
||||||
-- The peers of the routes table, each with the number it has there.
|
|
||||||
CREATE TABLE IF NOT EXISTS route_peers (
|
|
||||||
number INTEGER PRIMARY KEY,
|
|
||||||
address TEXT NOT NULL UNIQUE
|
|
||||||
);
|
|
||||||
@@ -34,7 +34,7 @@ type Tracker struct {
|
|||||||
// Announcement/withdrawal metrics
|
// Announcement/withdrawal metrics
|
||||||
announcementCounter metrics.Counter
|
announcementCounter metrics.Counter
|
||||||
withdrawalCounter metrics.Counter
|
withdrawalCounter metrics.Counter
|
||||||
churnPerMinute minuteCounter // prefix announcements and withdrawals
|
churnRate metrics.Meter // combined announcements + withdrawals per second
|
||||||
|
|
||||||
// BGP peer tracking
|
// BGP peer tracking
|
||||||
bgpPeerCount atomic.Int32
|
bgpPeerCount atomic.Int32
|
||||||
@@ -56,7 +56,7 @@ func New() *Tracker {
|
|||||||
ipv6UpdateRate: metrics.NewMeter(),
|
ipv6UpdateRate: metrics.NewMeter(),
|
||||||
announcementCounter: metrics.NewCounter(),
|
announcementCounter: metrics.NewCounter(),
|
||||||
withdrawalCounter: metrics.NewCounter(),
|
withdrawalCounter: metrics.NewCounter(),
|
||||||
churnPerMinute: minuteCounter{now: time.Now},
|
churnRate: metrics.NewMeter(),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -156,13 +156,13 @@ func (t *Tracker) RecordIPv6Update() {
|
|||||||
// RecordAnnouncement records a route announcement
|
// RecordAnnouncement records a route announcement
|
||||||
func (t *Tracker) RecordAnnouncement() {
|
func (t *Tracker) RecordAnnouncement() {
|
||||||
t.announcementCounter.Inc(1)
|
t.announcementCounter.Inc(1)
|
||||||
t.churnPerMinute.add()
|
t.churnRate.Mark(1)
|
||||||
}
|
}
|
||||||
|
|
||||||
// RecordWithdrawal records a route withdrawal
|
// RecordWithdrawal records a route withdrawal
|
||||||
func (t *Tracker) RecordWithdrawal() {
|
func (t *Tracker) RecordWithdrawal() {
|
||||||
t.withdrawalCounter.Inc(1)
|
t.withdrawalCounter.Inc(1)
|
||||||
t.churnPerMinute.add()
|
t.churnRate.Mark(1)
|
||||||
}
|
}
|
||||||
|
|
||||||
// SetBGPPeerCount updates the current BGP peer count
|
// SetBGPPeerCount updates the current BGP peer count
|
||||||
@@ -198,10 +198,9 @@ func (t *Tracker) GetWithdrawalCount() uint64 {
|
|||||||
return uint64(count)
|
return uint64(count)
|
||||||
}
|
}
|
||||||
|
|
||||||
// GetChurnPerMinute returns the number of prefix announcements and
|
// GetChurnRate returns the route churn rate per second
|
||||||
// withdrawals, from all peers, in the last full clock minute.
|
func (t *Tracker) GetChurnRate() float64 {
|
||||||
func (t *Tracker) GetChurnPerMinute() int {
|
return t.churnRate.Rate1()
|
||||||
return t.churnPerMinute.lastFullMinute()
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// GetRouteMetrics returns current route update metrics
|
// GetRouteMetrics returns current route update metrics
|
||||||
|
|||||||
@@ -1,59 +0,0 @@
|
|||||||
package metrics
|
|
||||||
|
|
||||||
import (
|
|
||||||
"sync"
|
|
||||||
"time"
|
|
||||||
)
|
|
||||||
|
|
||||||
// minuteCounter counts events in each clock minute and reports the count of
|
|
||||||
// the last full minute. It is safe for concurrent use.
|
|
||||||
type minuteCounter struct {
|
|
||||||
mu sync.Mutex
|
|
||||||
now func() time.Time
|
|
||||||
|
|
||||||
// minute is the start of the minute that count belongs to.
|
|
||||||
minute time.Time
|
|
||||||
count int
|
|
||||||
// lastMinuteCount is the count of the minute just before minute.
|
|
||||||
lastMinuteCount int
|
|
||||||
}
|
|
||||||
|
|
||||||
// add counts one event in the current minute.
|
|
||||||
func (c *minuteCounter) add() {
|
|
||||||
c.mu.Lock()
|
|
||||||
defer c.mu.Unlock()
|
|
||||||
|
|
||||||
c.moveToCurrentMinute()
|
|
||||||
c.count++
|
|
||||||
}
|
|
||||||
|
|
||||||
// lastFullMinute returns the number of events in the minute that ended most
|
|
||||||
// recently.
|
|
||||||
func (c *minuteCounter) lastFullMinute() int {
|
|
||||||
c.mu.Lock()
|
|
||||||
defer c.mu.Unlock()
|
|
||||||
|
|
||||||
c.moveToCurrentMinute()
|
|
||||||
|
|
||||||
return c.lastMinuteCount
|
|
||||||
}
|
|
||||||
|
|
||||||
// moveToCurrentMinute starts a new count when the clock has moved into
|
|
||||||
// another minute. The count it ends becomes the last full minute's count when
|
|
||||||
// its minute is the one just before the current minute. Otherwise the last
|
|
||||||
// full minute had no events. The caller holds mu.
|
|
||||||
func (c *minuteCounter) moveToCurrentMinute() {
|
|
||||||
minute := c.now().Truncate(time.Minute)
|
|
||||||
if minute.Equal(c.minute) {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
if minute.Sub(c.minute) == time.Minute {
|
|
||||||
c.lastMinuteCount = c.count
|
|
||||||
} else {
|
|
||||||
c.lastMinuteCount = 0
|
|
||||||
}
|
|
||||||
|
|
||||||
c.minute = minute
|
|
||||||
c.count = 0
|
|
||||||
}
|
|
||||||
@@ -1,86 +0,0 @@
|
|||||||
package metrics
|
|
||||||
|
|
||||||
import (
|
|
||||||
"sync"
|
|
||||||
"testing"
|
|
||||||
"time"
|
|
||||||
)
|
|
||||||
|
|
||||||
// TestChurnPerMinuteCountsLastFullMinute announces and withdraws prefixes on
|
|
||||||
// an injected clock and expects each minute's whole count once that minute
|
|
||||||
// has ended, and 0 for a minute with none.
|
|
||||||
func TestChurnPerMinuteCountsLastFullMinute(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
now := time.Date(2026, 10, 26, 12, 0, 30, 0, time.UTC)
|
|
||||||
tracker := New()
|
|
||||||
tracker.churnPerMinute.now = func() time.Time { return now }
|
|
||||||
|
|
||||||
expectChurn := func(want int, why string) {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
got := tracker.GetChurnPerMinute()
|
|
||||||
if got != want {
|
|
||||||
t.Fatalf("at %s churn = %d, want %d: %s",
|
|
||||||
now.Format(time.TimeOnly), got, want, why)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
const firstMinuteChurn = 2
|
|
||||||
|
|
||||||
tracker.RecordAnnouncement()
|
|
||||||
tracker.RecordWithdrawal()
|
|
||||||
expectChurn(0, "12:00 has not ended")
|
|
||||||
|
|
||||||
now = time.Date(2026, 10, 26, 12, 1, 59, 0, time.UTC)
|
|
||||||
|
|
||||||
tracker.RecordAnnouncement()
|
|
||||||
expectChurn(firstMinuteChurn, "12:00 had one announcement and one withdrawal")
|
|
||||||
|
|
||||||
now = time.Date(2026, 10, 26, 12, 2, 0, 0, time.UTC)
|
|
||||||
|
|
||||||
expectChurn(1, "12:01 had one announcement")
|
|
||||||
|
|
||||||
now = time.Date(2026, 10, 26, 12, 4, 0, 0, time.UTC)
|
|
||||||
|
|
||||||
expectChurn(0, "12:03 had none")
|
|
||||||
|
|
||||||
tracker.RecordWithdrawal()
|
|
||||||
|
|
||||||
now = time.Date(2026, 10, 26, 12, 6, 0, 0, time.UTC)
|
|
||||||
|
|
||||||
expectChurn(0, "12:05 had none, though 12:04 had one withdrawal")
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestChurnPerMinuteCountsConcurrentRecords announces prefixes from several
|
|
||||||
// goroutines at once and expects every one counted.
|
|
||||||
func TestChurnPerMinuteCountsConcurrentRecords(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
const (
|
|
||||||
goroutines = 8
|
|
||||||
recordsPerGoroutine = 1000
|
|
||||||
)
|
|
||||||
|
|
||||||
now := time.Date(2026, 10, 26, 12, 0, 0, 0, time.UTC)
|
|
||||||
tracker := New()
|
|
||||||
tracker.churnPerMinute.now = func() time.Time { return now }
|
|
||||||
|
|
||||||
var wg sync.WaitGroup
|
|
||||||
for range goroutines {
|
|
||||||
wg.Go(func() {
|
|
||||||
for range recordsPerGoroutine {
|
|
||||||
tracker.RecordAnnouncement()
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
wg.Wait()
|
|
||||||
|
|
||||||
now = now.Add(time.Minute)
|
|
||||||
|
|
||||||
want := goroutines * recordsPerGoroutine
|
|
||||||
if got := tracker.GetChurnPerMinute(); got != want {
|
|
||||||
t.Fatalf("churn = %d, want %d", got, want)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
+11
-33
@@ -4,7 +4,6 @@ package ristypes
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
"slices"
|
|
||||||
"time"
|
"time"
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -25,8 +24,7 @@ const (
|
|||||||
)
|
)
|
||||||
|
|
||||||
// ASPath is a BGP AS path as the feed writes it: an array of AS numbers in
|
// ASPath is a BGP AS path as the feed writes it: an array of AS numbers in
|
||||||
// which an AS set is an array of its members, such as [1, 2, [3, 4]]. A path
|
// which an AS set is an array of its members, such as [1, 2, [3, 4]].
|
||||||
// that holds AS 0 decodes as an empty one (UnmarshalJSON).
|
|
||||||
type ASPath struct {
|
type ASPath struct {
|
||||||
// Sequence is the path's AS numbers in order. A set of one member that
|
// Sequence is the path's AS numbers in order. A set of one member that
|
||||||
// ends the path counts as that AS and is its last element. The members of
|
// ends the path counts as that AS and is its last element. The members of
|
||||||
@@ -39,56 +37,34 @@ type ASPath struct {
|
|||||||
Set []int
|
Set []int
|
||||||
}
|
}
|
||||||
|
|
||||||
// NoOrigin is the origin AS of a route whose origin is not known. AS 0 is
|
|
||||||
// reserved, so no AS has this number.
|
|
||||||
const NoOrigin = 0
|
|
||||||
|
|
||||||
// Origin returns the AS that originated the route: the last AS of the
|
// Origin returns the AS that originated the route: the last AS of the
|
||||||
// sequence, or NoOrigin when the path ends in a set of several members or is
|
// sequence, or 0 (none) when the path ends in a set of several members or is
|
||||||
// empty.
|
// empty.
|
||||||
func (p *ASPath) Origin() int {
|
func (p *ASPath) Origin() int {
|
||||||
if len(p.Set) > 0 || len(p.Sequence) == 0 {
|
if len(p.Set) > 0 || len(p.Sequence) == 0 {
|
||||||
return NoOrigin
|
return 0
|
||||||
}
|
}
|
||||||
|
|
||||||
return p.Sequence[len(p.Sequence)-1]
|
return p.Sequence[len(p.Sequence)-1]
|
||||||
}
|
}
|
||||||
|
|
||||||
// UnmarshalJSON implements the json.Unmarshaler interface for ASPath. A path
|
// UnmarshalJSON implements the json.Unmarshaler interface for ASPath.
|
||||||
// that holds AS 0 anywhere, a member of an AS set included, decodes as an empty
|
|
||||||
// path: RFC 7607 makes such a path malformed and its announcements
|
|
||||||
// withdrawals. On an announcement with an empty path the prefix handler
|
|
||||||
// deletes the peer's route, and the AS and peering handlers record nothing.
|
|
||||||
func (p *ASPath) UnmarshalJSON(data []byte) error {
|
func (p *ASPath) UnmarshalJSON(data []byte) error {
|
||||||
path, err := decodeASPath(data)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
if slices.Contains(path.Sequence, 0) || slices.Contains(path.Set, 0) {
|
|
||||||
path = ASPath{}
|
|
||||||
}
|
|
||||||
|
|
||||||
*p = path
|
|
||||||
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// decodeASPath decodes an AS path as the feed writes it.
|
|
||||||
func decodeASPath(data []byte) (ASPath, error) {
|
|
||||||
// Most paths have no AS set and decode as a plain array of integers.
|
// Most paths have no AS set and decode as a plain array of integers.
|
||||||
var sequence []int
|
var sequence []int
|
||||||
|
|
||||||
err := json.Unmarshal(data, &sequence)
|
err := json.Unmarshal(data, &sequence)
|
||||||
if err == nil {
|
if err == nil {
|
||||||
return ASPath{Sequence: sequence}, nil
|
*p = ASPath{Sequence: sequence}
|
||||||
|
|
||||||
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
var raw []any
|
var raw []any
|
||||||
|
|
||||||
err = json.Unmarshal(data, &raw)
|
err = json.Unmarshal(data, &raw)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return ASPath{}, err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
path := ASPath{Sequence: make([]int, 0, len(raw))}
|
path := ASPath{Sequence: make([]int, 0, len(raw))}
|
||||||
@@ -114,7 +90,9 @@ func decodeASPath(data []byte) (ASPath, error) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
return path, nil
|
*p = path
|
||||||
|
|
||||||
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// RISLiveMessage represents the outer wrapper message from the RIPE RIS Live stream.
|
// RISLiveMessage represents the outer wrapper message from the RIPE RIS Live stream.
|
||||||
|
|||||||
@@ -145,33 +145,3 @@ func TestASPathKeepsATrailingSetApart(t *testing.T) {
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// TestASPathWithASZeroDecodesAsAnEmptyPath decodes AS paths that hold AS 0
|
|
||||||
// before their end, at their end, or in an AS set. RFC 7607 makes such a path
|
|
||||||
// malformed, so each decodes as an empty path.
|
|
||||||
func TestASPathWithASZeroDecodesAsAnEmptyPath(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
for _, path := range []string{
|
|
||||||
"[64496, 0, 64497]",
|
|
||||||
"[64496, 0]",
|
|
||||||
"[64496, [0, 64497], 64498]",
|
|
||||||
"[64496, [64497, 0]]",
|
|
||||||
} {
|
|
||||||
t.Run(path, func(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
var got ristypes.ASPath
|
|
||||||
|
|
||||||
err := json.Unmarshal([]byte(path), &got)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("unmarshal %s: %v", path, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
if len(got.Sequence) > 0 || len(got.Set) > 0 {
|
|
||||||
t.Errorf("%s decoded to sequence %v and set %v, want an empty path",
|
|
||||||
path, got.Sequence, got.Set)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|||||||
@@ -1,155 +0,0 @@
|
|||||||
package routetable_test
|
|
||||||
|
|
||||||
import (
|
|
||||||
"fmt"
|
|
||||||
"math"
|
|
||||||
"math/rand/v2"
|
|
||||||
"net/netip"
|
|
||||||
"runtime"
|
|
||||||
"testing"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"git.eeqj.de/sneak/routewatch/internal/routetable"
|
|
||||||
)
|
|
||||||
|
|
||||||
// Parameters for BenchmarkApplyMixedBatch.
|
|
||||||
const (
|
|
||||||
// benchPeers is how many peers there are, about as many as RIS has.
|
|
||||||
benchPeers = 1400
|
|
||||||
// benchPrefixes is how many IPv4 /24 prefixes the table holds, and
|
|
||||||
// benchPeersPerPrefix how many peers, picked at random, carry each.
|
|
||||||
benchPrefixes = 100000
|
|
||||||
benchPeersPerPrefix = 20
|
|
||||||
// benchBatchSize is how many updates the timed batch holds, as many as
|
|
||||||
// the prefix handler's largest batch (prefixBatchMaxSize in
|
|
||||||
// internal/routewatch).
|
|
||||||
benchBatchSize = 250000
|
|
||||||
// In the timed batch every benchWithdrawalEvery-th update is a
|
|
||||||
// withdrawal, and every benchNewOriginEvery-th announcement has an
|
|
||||||
// origin AS other than the prefix's.
|
|
||||||
benchWithdrawalEvery = 5
|
|
||||||
benchNewOriginEvery = 7
|
|
||||||
// benchOrigins is how many origin ASes the prefixes are spread over,
|
|
||||||
// from benchFirstASN on: 20 prefixes each, about as many as an origin AS
|
|
||||||
// has on average in the full table.
|
|
||||||
benchOrigins = 5000
|
|
||||||
benchFirstASN = 64500
|
|
||||||
// benchUpdatesPerSecond is how many updates share one time.
|
|
||||||
benchUpdatesPerSecond = 1000
|
|
||||||
benchMaskLength = 24
|
|
||||||
)
|
|
||||||
|
|
||||||
// benchUpdate is one update of the benchmark.
|
|
||||||
type benchUpdate struct {
|
|
||||||
prefix netip.Prefix
|
|
||||||
origin int
|
|
||||||
route routetable.Route
|
|
||||||
withdraw bool
|
|
||||||
}
|
|
||||||
|
|
||||||
// BenchmarkApplyMixedBatch times one batch of 250,000 updates of routes picked
|
|
||||||
// at random: announcements, withdrawals and announcements with a new origin AS.
|
|
||||||
// The table it applies them to holds 100,000 IPv4 prefixes from 5,000 origin
|
|
||||||
// ASes, each carried by 20 of 1,400 peers. It reports the heap bytes that table
|
|
||||||
// takes per prefix and origin AS pair. It is run by hand, not by make test, for
|
|
||||||
// example in the test image:
|
|
||||||
//
|
|
||||||
// go test -run '^$' -bench MixedBatch -benchtime 10x ./internal/routetable/
|
|
||||||
func BenchmarkApplyMixedBatch(b *testing.B) {
|
|
||||||
//nolint:gosec // The benchmark needs the same updates every run.
|
|
||||||
rng := rand.New(rand.NewPCG(1, 2))
|
|
||||||
peers := make([]string, benchPeers)
|
|
||||||
|
|
||||||
for i := range peers {
|
|
||||||
peers[i] = fmt.Sprintf("198.18.%d.%d", i/256, i%256)
|
|
||||||
}
|
|
||||||
|
|
||||||
before := heapBytes()
|
|
||||||
table := routetable.New(routetable.Params{})
|
|
||||||
updates := 0
|
|
||||||
|
|
||||||
for prefix := range benchPrefixes {
|
|
||||||
for range benchPeersPerPrefix {
|
|
||||||
peer := peers[rng.IntN(benchPeers)]
|
|
||||||
apply(table, benchAnnouncement(rng, prefix, peer, updates, false))
|
|
||||||
updates++
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// The record of changes is handed over every second while routewatch
|
|
||||||
// runs, so it is not part of the table's size.
|
|
||||||
table.TakeChanges(math.MaxInt)
|
|
||||||
|
|
||||||
bytesPerPair := float64(heapBytes()-before) / benchPrefixes
|
|
||||||
|
|
||||||
batch := make([]benchUpdate, 0, benchBatchSize)
|
|
||||||
for i := range benchBatchSize {
|
|
||||||
update := benchAnnouncement(rng, rng.IntN(benchPrefixes),
|
|
||||||
peers[rng.IntN(benchPeers)], updates+i, i%benchNewOriginEvery == 0)
|
|
||||||
update.withdraw = i%benchWithdrawalEvery == 0
|
|
||||||
batch = append(batch, update)
|
|
||||||
}
|
|
||||||
|
|
||||||
for b.Loop() {
|
|
||||||
for _, update := range batch {
|
|
||||||
apply(table, update)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// After the loop: b.Loop drops the metrics reported before it starts.
|
|
||||||
b.ReportMetric(bytesPerPair, "bytes/pair")
|
|
||||||
}
|
|
||||||
|
|
||||||
// benchAnnouncement returns the announcement of the prefix numbered prefix, a
|
|
||||||
// /24 from 1.0.0.0/24 on, by peer. It is the updates-th update of the
|
|
||||||
// benchmark, and its AS path holds four ASes from the peer's AS to the origin
|
|
||||||
// AS. newOrigin gives it an origin AS other than the prefix's.
|
|
||||||
func benchAnnouncement(
|
|
||||||
rng *rand.Rand, prefix int, peer string, updates int, newOrigin bool,
|
|
||||||
) benchUpdate {
|
|
||||||
origin := benchFirstASN + prefix%benchOrigins
|
|
||||||
if newOrigin {
|
|
||||||
origin += benchOrigins
|
|
||||||
}
|
|
||||||
|
|
||||||
start := time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC)
|
|
||||||
|
|
||||||
return benchUpdate{
|
|
||||||
prefix: netip.MustParsePrefix(fmt.Sprintf("%d.%d.%d.0/%d",
|
|
||||||
1+prefix>>16, (prefix>>8)%256, prefix%256, benchMaskLength)),
|
|
||||||
origin: origin,
|
|
||||||
route: routetable.Route{
|
|
||||||
Peer: peer,
|
|
||||||
ASPath: []int{
|
|
||||||
benchFirstASN + rng.IntN(benchOrigins),
|
|
||||||
benchFirstASN + rng.IntN(benchOrigins),
|
|
||||||
benchFirstASN + rng.IntN(benchOrigins),
|
|
||||||
origin,
|
|
||||||
},
|
|
||||||
Time: start.Add(time.Duration(updates/benchUpdatesPerSecond) *
|
|
||||||
time.Second),
|
|
||||||
},
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// apply applies update to table.
|
|
||||||
func apply(table *routetable.Table, update benchUpdate) {
|
|
||||||
if update.withdraw {
|
|
||||||
table.Withdraw(update.prefix, update.route.Peer)
|
|
||||||
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
table.Announce(update.prefix, update.origin, update.route)
|
|
||||||
}
|
|
||||||
|
|
||||||
// heapBytes returns the bytes of the live heap objects, after a collection.
|
|
||||||
func heapBytes() uint64 {
|
|
||||||
runtime.GC()
|
|
||||||
|
|
||||||
var stats runtime.MemStats
|
|
||||||
|
|
||||||
runtime.ReadMemStats(&stats)
|
|
||||||
|
|
||||||
return stats.HeapAlloc
|
|
||||||
}
|
|
||||||
@@ -1,233 +0,0 @@
|
|||||||
package routetable
|
|
||||||
|
|
||||||
import (
|
|
||||||
"net/netip"
|
|
||||||
"slices"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/bits-and-blooms/bitset"
|
|
||||||
)
|
|
||||||
|
|
||||||
// PrefixSummary is what the table answers about one prefix.
|
|
||||||
type PrefixSummary struct {
|
|
||||||
Prefix netip.Prefix
|
|
||||||
// OriginASN is the origin AS that the most peers carry the prefix with.
|
|
||||||
// When several have as many peers, it is the lowest AS number.
|
|
||||||
OriginASN int
|
|
||||||
// Peers is how many peers carry the prefix.
|
|
||||||
Peers int
|
|
||||||
// Newest is the time of the newest kept route of the prefix, or nil when
|
|
||||||
// it keeps none.
|
|
||||||
Newest *time.Time
|
|
||||||
}
|
|
||||||
|
|
||||||
// OriginatedPrefix is one prefix an AS originates.
|
|
||||||
type OriginatedPrefix struct {
|
|
||||||
Prefix netip.Prefix
|
|
||||||
// Newest is the time of the newest kept route of the prefix with that
|
|
||||||
// origin AS, or nil when it keeps none.
|
|
||||||
Newest *time.Time
|
|
||||||
}
|
|
||||||
|
|
||||||
// Counts are the table's figures for the stats.
|
|
||||||
type Counts struct {
|
|
||||||
// Pairs counts the prefix and origin AS pairs that some peer carries.
|
|
||||||
Pairs int
|
|
||||||
// Peers counts the peers that have a number: those given in
|
|
||||||
// Params.PeerNumbers and those numbered since.
|
|
||||||
Peers int
|
|
||||||
IPv4Prefixes int
|
|
||||||
IPv6Prefixes int
|
|
||||||
// IPv4PrefixesByLength[n] and IPv6PrefixesByLength[n] are how many
|
|
||||||
// prefixes of mask length n some peer carries.
|
|
||||||
IPv4PrefixesByLength []int
|
|
||||||
IPv6PrefixesByLength []int
|
|
||||||
// IPv4Routes and IPv6Routes count peer and prefix pairs: each peer that
|
|
||||||
// carries a prefix counts once for it.
|
|
||||||
IPv4Routes int
|
|
||||||
IPv6Routes int
|
|
||||||
// OriginsWithoutKeptRoute counts the prefix and origin AS pairs that peers
|
|
||||||
// carry but whose kept routes were all withdrawn.
|
|
||||||
OriginsWithoutKeptRoute int
|
|
||||||
// OldestKeptRoute and NewestKeptRoute are the times of the oldest and the
|
|
||||||
// newest kept routes, or nil when the table keeps none.
|
|
||||||
OldestKeptRoute *time.Time
|
|
||||||
NewestKeptRoute *time.Time
|
|
||||||
}
|
|
||||||
|
|
||||||
// Lookup returns the most specific prefix that holds addr and that some peer
|
|
||||||
// carries. It ignores prefixes shorter than /8 for IPv4 and /16 for IPv6. It
|
|
||||||
// makes one map lookup per mask length, and reports false when no prefix
|
|
||||||
// holds addr.
|
|
||||||
func (t *Table) Lookup(addr netip.Addr) (PrefixSummary, bool) {
|
|
||||||
t.mu.RLock()
|
|
||||||
defer t.mu.RUnlock()
|
|
||||||
|
|
||||||
// An IPv4 address written in IPv6 form (::ffff:192.0.2.1) is looked up as
|
|
||||||
// IPv4.
|
|
||||||
addr = addr.Unmap()
|
|
||||||
f := t.familyOf(addr)
|
|
||||||
|
|
||||||
for bits := addr.BitLen(); bits >= f.minLookupLength; bits-- {
|
|
||||||
prefix := netip.PrefixFrom(addr, bits).Masked()
|
|
||||||
|
|
||||||
origins, found := f.byLength[bits][prefix]
|
|
||||||
if found {
|
|
||||||
return summarize(prefix, origins), true
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return PrefixSummary{}, false
|
|
||||||
}
|
|
||||||
|
|
||||||
// OriginatedPrefixes returns the prefixes AS asn originates, in prefix order.
|
|
||||||
func (t *Table) OriginatedPrefixes(asn int) []OriginatedPrefix {
|
|
||||||
t.mu.RLock()
|
|
||||||
defer t.mu.RUnlock()
|
|
||||||
|
|
||||||
prefixes := make([]OriginatedPrefix, 0, len(t.byOrigin[asn]))
|
|
||||||
for prefix, o := range t.byOrigin[asn] {
|
|
||||||
prefixes = append(prefixes,
|
|
||||||
OriginatedPrefix{Prefix: prefix, Newest: o.newestTime()})
|
|
||||||
}
|
|
||||||
|
|
||||||
slices.SortFunc(prefixes, func(a, b OriginatedPrefix) int {
|
|
||||||
return a.Prefix.Compare(b.Prefix)
|
|
||||||
})
|
|
||||||
|
|
||||||
return prefixes
|
|
||||||
}
|
|
||||||
|
|
||||||
// Origins returns each origin AS of prefix, in AS number order, with its peers
|
|
||||||
// and its kept routes. It returns nil when no peer carries the prefix.
|
|
||||||
func (t *Table) Origins(prefix netip.Prefix) []Origin {
|
|
||||||
t.mu.RLock()
|
|
||||||
defer t.mu.RUnlock()
|
|
||||||
|
|
||||||
if !prefix.IsValid() {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
var origins []Origin
|
|
||||||
|
|
||||||
for _, o := range t.origins(prefix.Masked()) {
|
|
||||||
origins = append(origins, Origin{
|
|
||||||
ASN: o.asn,
|
|
||||||
Peers: t.addresses(&o.peers),
|
|
||||||
Routes: t.exportRoutes(o.routes),
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
return origins
|
|
||||||
}
|
|
||||||
|
|
||||||
// SamplePrefixes returns up to limit of the prefixes of IP version ipVersion
|
|
||||||
// (4 or 6) and mask length maskLength. It reads only the prefixes it returns.
|
|
||||||
// Go starts each walk over a map at a random place, so each call returns
|
|
||||||
// another sample.
|
|
||||||
func (t *Table) SamplePrefixes(ipVersion, maskLength, limit int) []PrefixSummary {
|
|
||||||
t.mu.RLock()
|
|
||||||
defer t.mu.RUnlock()
|
|
||||||
|
|
||||||
var f *family
|
|
||||||
|
|
||||||
switch ipVersion {
|
|
||||||
case ipVersion4:
|
|
||||||
f = &t.ipv4
|
|
||||||
case ipVersion6:
|
|
||||||
f = &t.ipv6
|
|
||||||
default:
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
if maskLength < 0 || maskLength >= len(f.byLength) || limit <= 0 {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
prefixes := f.byLength[maskLength]
|
|
||||||
sample := make([]PrefixSummary, 0, min(limit, len(prefixes)))
|
|
||||||
|
|
||||||
for prefix, origins := range prefixes {
|
|
||||||
if len(sample) == limit {
|
|
||||||
break
|
|
||||||
}
|
|
||||||
|
|
||||||
sample = append(sample, summarize(prefix, origins))
|
|
||||||
}
|
|
||||||
|
|
||||||
return sample
|
|
||||||
}
|
|
||||||
|
|
||||||
// Counts returns the table's figures for the stats. The writes keep them up to
|
|
||||||
// date, so this reads no prefix.
|
|
||||||
func (t *Table) Counts() Counts {
|
|
||||||
t.mu.RLock()
|
|
||||||
defer t.mu.RUnlock()
|
|
||||||
|
|
||||||
counts := Counts{
|
|
||||||
Peers: len(t.peerNumbers),
|
|
||||||
IPv4PrefixesByLength: t.ipv4.prefixesByLength(),
|
|
||||||
IPv6PrefixesByLength: t.ipv6.prefixesByLength(),
|
|
||||||
IPv4Routes: t.ipv4.routes,
|
|
||||||
IPv6Routes: t.ipv6.routes,
|
|
||||||
OriginsWithoutKeptRoute: t.originsWithoutKeptRoute,
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, n := range counts.IPv4PrefixesByLength {
|
|
||||||
counts.IPv4Prefixes += n
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, n := range counts.IPv6PrefixesByLength {
|
|
||||||
counts.IPv6Prefixes += n
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, prefixes := range t.byOrigin {
|
|
||||||
counts.Pairs += len(prefixes)
|
|
||||||
}
|
|
||||||
|
|
||||||
oldest, found := t.keptTimes.Min()
|
|
||||||
if found {
|
|
||||||
counts.OldestKeptRoute = &oldest.time
|
|
||||||
}
|
|
||||||
|
|
||||||
newest, found := t.keptTimes.Max()
|
|
||||||
if found {
|
|
||||||
counts.NewestKeptRoute = &newest.time
|
|
||||||
}
|
|
||||||
|
|
||||||
return counts
|
|
||||||
}
|
|
||||||
|
|
||||||
// addresses returns the addresses of the peers in peers, in number order.
|
|
||||||
func (t *Table) addresses(peers *bitset.BitSet) []string {
|
|
||||||
addresses := make([]string, 0, peers.Count())
|
|
||||||
for number := range peers.EachSet() {
|
|
||||||
addresses = append(addresses, t.peerAddresses[number])
|
|
||||||
}
|
|
||||||
|
|
||||||
return addresses
|
|
||||||
}
|
|
||||||
|
|
||||||
// summarize returns the summary of prefix, whose origins are in AS number
|
|
||||||
// order.
|
|
||||||
func summarize(prefix netip.Prefix, origins []*origin) PrefixSummary {
|
|
||||||
summary := PrefixSummary{Prefix: prefix}
|
|
||||||
mostPeers := 0
|
|
||||||
|
|
||||||
for _, o := range origins {
|
|
||||||
peers := int(o.peers.Count())
|
|
||||||
summary.Peers += peers
|
|
||||||
|
|
||||||
if peers > mostPeers {
|
|
||||||
mostPeers = peers
|
|
||||||
summary.OriginASN = o.asn
|
|
||||||
}
|
|
||||||
|
|
||||||
newest := o.newestTime()
|
|
||||||
if newest != nil && (summary.Newest == nil || newest.After(*summary.Newest)) {
|
|
||||||
summary.Newest = newest
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return summary
|
|
||||||
}
|
|
||||||
@@ -1,265 +0,0 @@
|
|||||||
package routetable_test
|
|
||||||
|
|
||||||
import (
|
|
||||||
"net/netip"
|
|
||||||
"reflect"
|
|
||||||
"testing"
|
|
||||||
|
|
||||||
"git.eeqj.de/sneak/routewatch/internal/routetable"
|
|
||||||
)
|
|
||||||
|
|
||||||
// announceAll has each listed peer announce prefix with origin AS asn, the
|
|
||||||
// peer at index i i*10 seconds after start.
|
|
||||||
func announceAll(
|
|
||||||
table *routetable.Table, prefix string, asn, start int, peers ...string,
|
|
||||||
) {
|
|
||||||
const secondsBetweenPeers = 10
|
|
||||||
|
|
||||||
for i, peer := range peers {
|
|
||||||
table.Announce(netip.MustParsePrefix(prefix), asn,
|
|
||||||
routeOf(peer, asn, start+i*secondsBetweenPeers))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestLookup(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
table := routetable.New(routetable.Params{})
|
|
||||||
announceAll(table, "10.0.0.0/8", asA, 10, peerA)
|
|
||||||
// Three peers with two origins: asB has more of them.
|
|
||||||
announceAll(table, "10.1.0.0/16", asA, 10, peerA)
|
|
||||||
announceAll(table, "10.1.0.0/16", asB, 20, peerB, peerC)
|
|
||||||
// Two origins with one peer each: the lower AS number wins.
|
|
||||||
announceAll(table, prefixV4, asB, 20, peerB)
|
|
||||||
announceAll(table, prefixV4, asA, 10, peerA)
|
|
||||||
// An origin whose kept routes were withdrawn: no newest time.
|
|
||||||
announceAll(table, "10.9.0.0/16", asA, 10, peerA, peerB, peerC)
|
|
||||||
table.Withdraw(netip.MustParsePrefix("10.9.0.0/16"), peerB)
|
|
||||||
table.Withdraw(netip.MustParsePrefix("10.9.0.0/16"), peerC)
|
|
||||||
announceAll(table, "2001:db8::/32", asA, 10, peerA)
|
|
||||||
announceAll(table, prefixV6, asB, 20, peerB)
|
|
||||||
announceAll(table, "3000::/16", asA, 10, peerA)
|
|
||||||
// Shorter than the lookup uses.
|
|
||||||
announceAll(table, "192.0.0.0/7", asA, 10, peerA)
|
|
||||||
announceAll(table, "2000::/15", asA, 10, peerA)
|
|
||||||
|
|
||||||
tests := []struct {
|
|
||||||
addr string
|
|
||||||
// want is the summary's prefix, or empty when none is found.
|
|
||||||
want string
|
|
||||||
originASN int
|
|
||||||
peers int
|
|
||||||
newest int // in seconds after the start; 0 for none
|
|
||||||
}{
|
|
||||||
{"10.1.2.3", prefixV4, asA, 2, 20},
|
|
||||||
{"::ffff:10.1.2.3", prefixV4, asA, 2, 20},
|
|
||||||
{"10.1.3.4", "10.1.0.0/16", asB, 3, 30},
|
|
||||||
{"10.2.0.1", "10.0.0.0/8", asA, 1, 10},
|
|
||||||
{"10.9.0.1", "10.9.0.0/16", asA, 1, 0},
|
|
||||||
{"192.0.2.1", "", 0, 0, 0},
|
|
||||||
{"2001:db8:1::5", prefixV6, asB, 1, 20},
|
|
||||||
{"2001:db8:2::1", "2001:db8::/32", asA, 1, 10},
|
|
||||||
{"3000::1", "3000::/16", asA, 1, 10},
|
|
||||||
{"2001:db9::1", "", 0, 0, 0},
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, tc := range tests {
|
|
||||||
t.Run(tc.addr, func(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
got, found := table.Lookup(netip.MustParseAddr(tc.addr))
|
|
||||||
if tc.want == "" {
|
|
||||||
if found {
|
|
||||||
t.Errorf("Lookup(%s) = %+v, want none", tc.addr, got)
|
|
||||||
}
|
|
||||||
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
want := routetable.PrefixSummary{
|
|
||||||
Prefix: netip.MustParsePrefix(tc.want),
|
|
||||||
OriginASN: tc.originASN,
|
|
||||||
Peers: tc.peers,
|
|
||||||
}
|
|
||||||
if tc.newest != 0 {
|
|
||||||
want.Newest = timeAt(tc.newest)
|
|
||||||
}
|
|
||||||
|
|
||||||
if !found || !reflect.DeepEqual(got, want) {
|
|
||||||
t.Errorf("Lookup(%s) = %+v, %v; want %+v", tc.addr, got, found, want)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestOriginatedPrefixes(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
table := routetable.New(routetable.Params{})
|
|
||||||
announceAll(table, "2001:db8::/32", asA, 30, peerA)
|
|
||||||
announceAll(table, prefixV4, asA, 10, peerA)
|
|
||||||
// asB's newer route for the same prefix does not count for asA.
|
|
||||||
announceAll(table, prefixV4, asB, 40, peerB)
|
|
||||||
announceAll(table, "10.1.0.0/16", asA, 10, peerA, peerB)
|
|
||||||
announceAll(table, "10.9.0.0/16", asA, 10, peerA, peerB, peerC)
|
|
||||||
table.Withdraw(netip.MustParsePrefix("10.9.0.0/16"), peerB)
|
|
||||||
table.Withdraw(netip.MustParsePrefix("10.9.0.0/16"), peerC)
|
|
||||||
|
|
||||||
want := []routetable.OriginatedPrefix{
|
|
||||||
{Prefix: netip.MustParsePrefix("10.1.0.0/16"), Newest: timeAt(20)},
|
|
||||||
{Prefix: netip.MustParsePrefix(prefixV4), Newest: timeAt(10)},
|
|
||||||
{Prefix: netip.MustParsePrefix("10.9.0.0/16")},
|
|
||||||
{Prefix: netip.MustParsePrefix("2001:db8::/32"), Newest: timeAt(30)},
|
|
||||||
}
|
|
||||||
|
|
||||||
got := table.OriginatedPrefixes(asA)
|
|
||||||
if !reflect.DeepEqual(got, want) {
|
|
||||||
t.Errorf("OriginatedPrefixes(%d) = %+v, want %+v", asA, got, want)
|
|
||||||
}
|
|
||||||
|
|
||||||
got = table.OriginatedPrefixes(asC)
|
|
||||||
if len(got) != 0 {
|
|
||||||
t.Errorf("OriginatedPrefixes(%d) = %+v, want none", asC, got)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestSamplePrefixes(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
table := routetable.New(routetable.Params{})
|
|
||||||
|
|
||||||
announced := []string{
|
|
||||||
"10.0.0.0/24", "10.0.1.0/24", "10.0.2.0/24", "10.0.3.0/24",
|
|
||||||
"10.0.4.0/24", "10.0.0.0/16",
|
|
||||||
"2001:db8::/48", "2001:db8:1::/48", "2001:db8:2::/48",
|
|
||||||
}
|
|
||||||
for _, prefix := range announced {
|
|
||||||
announceAll(table, prefix, asA, 10, peerA)
|
|
||||||
}
|
|
||||||
|
|
||||||
tests := []struct {
|
|
||||||
name string
|
|
||||||
ipVersion, maskLength, limit int
|
|
||||||
want int
|
|
||||||
}{
|
|
||||||
{"fewer than there are", 4, 24, 3, 3},
|
|
||||||
{"more than there are", 4, 24, 10, 5},
|
|
||||||
{"IPv6", 6, 48, 10, 3},
|
|
||||||
{"a mask length with none", 4, 20, 10, 0},
|
|
||||||
{"no such IP version", 5, 24, 10, 0},
|
|
||||||
{"a mask length too long", 4, 33, 10, 0},
|
|
||||||
{"a negative mask length", 4, -1, 10, 0},
|
|
||||||
{"a limit of zero", 4, 24, 0, 0},
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, tc := range tests {
|
|
||||||
t.Run(tc.name, func(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
sample := table.SamplePrefixes(tc.ipVersion, tc.maskLength, tc.limit)
|
|
||||||
if len(sample) != tc.want {
|
|
||||||
t.Fatalf("SamplePrefixes(%d, %d, %d) returned %d prefixes, want %d",
|
|
||||||
tc.ipVersion, tc.maskLength, tc.limit, len(sample), tc.want)
|
|
||||||
}
|
|
||||||
|
|
||||||
seen := make(map[netip.Prefix]bool)
|
|
||||||
|
|
||||||
for _, got := range sample {
|
|
||||||
want := routetable.PrefixSummary{
|
|
||||||
Prefix: got.Prefix, OriginASN: asA, Peers: 1, Newest: timeAt(10),
|
|
||||||
}
|
|
||||||
if seen[got.Prefix] || got.Prefix.Bits() != tc.maskLength ||
|
|
||||||
!reflect.DeepEqual(got, want) {
|
|
||||||
t.Errorf("SamplePrefixes(%d, %d, %d) returned %+v",
|
|
||||||
tc.ipVersion, tc.maskLength, tc.limit, got)
|
|
||||||
}
|
|
||||||
|
|
||||||
seen[got.Prefix] = true
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestCounts(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
table := routetable.New(routetable.Params{})
|
|
||||||
checkCounts(t, table, routetable.Counts{
|
|
||||||
IPv4PrefixesByLength: byLength(ipv4Bits, nil),
|
|
||||||
IPv6PrefixesByLength: byLength(ipv6Bits, nil),
|
|
||||||
})
|
|
||||||
|
|
||||||
announceAll(table, prefixV4, asA, 10, peerA, peerB)
|
|
||||||
announceAll(table, "10.2.0.0/16", asA, 15, peerA)
|
|
||||||
announceAll(table, "2001:db8::/32", asA, 5, peerA)
|
|
||||||
announceAll(table, "2001:db8::/32", asA, 25, peerC)
|
|
||||||
checkCounts(t, table, routetable.Counts{
|
|
||||||
Pairs: 3,
|
|
||||||
Peers: 3,
|
|
||||||
IPv4Prefixes: 2,
|
|
||||||
IPv6Prefixes: 1,
|
|
||||||
IPv4PrefixesByLength: byLength(ipv4Bits, map[int]int{16: 1, 24: 1}),
|
|
||||||
IPv6PrefixesByLength: byLength(ipv6Bits, map[int]int{32: 1}),
|
|
||||||
IPv4Routes: 3,
|
|
||||||
IPv6Routes: 2,
|
|
||||||
OldestKeptRoute: timeAt(5),
|
|
||||||
NewestKeptRoute: timeAt(25),
|
|
||||||
})
|
|
||||||
|
|
||||||
// peerC's route drops peerA's from the kept ones, and peerA leaves the
|
|
||||||
// IPv6 prefix.
|
|
||||||
announceAll(table, prefixV4, asA, 30, peerC)
|
|
||||||
table.Withdraw(netip.MustParsePrefix("2001:db8::/32"), peerA)
|
|
||||||
checkCounts(t, table, routetable.Counts{
|
|
||||||
Pairs: 3,
|
|
||||||
Peers: 3,
|
|
||||||
IPv4Prefixes: 2,
|
|
||||||
IPv6Prefixes: 1,
|
|
||||||
IPv4PrefixesByLength: byLength(ipv4Bits, map[int]int{16: 1, 24: 1}),
|
|
||||||
IPv6PrefixesByLength: byLength(ipv6Bits, map[int]int{32: 1}),
|
|
||||||
IPv4Routes: 4,
|
|
||||||
IPv6Routes: 1,
|
|
||||||
OldestKeptRoute: timeAt(15),
|
|
||||||
NewestKeptRoute: timeAt(30),
|
|
||||||
})
|
|
||||||
|
|
||||||
// peerC was the last peer of the IPv6 prefix. It keeps its number.
|
|
||||||
table.PeerDown(peerC)
|
|
||||||
checkCounts(t, table, routetable.Counts{
|
|
||||||
Pairs: 2,
|
|
||||||
Peers: 3,
|
|
||||||
IPv4Prefixes: 2,
|
|
||||||
IPv4PrefixesByLength: byLength(ipv4Bits, map[int]int{16: 1, 24: 1}),
|
|
||||||
IPv6PrefixesByLength: byLength(ipv6Bits, nil),
|
|
||||||
IPv4Routes: 3,
|
|
||||||
OldestKeptRoute: timeAt(15),
|
|
||||||
NewestKeptRoute: timeAt(20),
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
// The address lengths of the two IP versions.
|
|
||||||
const (
|
|
||||||
ipv4Bits = 32
|
|
||||||
ipv6Bits = 128
|
|
||||||
)
|
|
||||||
|
|
||||||
// byLength returns the prefix counts by mask length, for addresses of bits
|
|
||||||
// bits, with counts[n] prefixes of mask length n and none of any other.
|
|
||||||
func byLength(bits int, counts map[int]int) []int {
|
|
||||||
all := make([]int, bits+1)
|
|
||||||
for n, count := range counts {
|
|
||||||
all[n] = count
|
|
||||||
}
|
|
||||||
|
|
||||||
return all
|
|
||||||
}
|
|
||||||
|
|
||||||
// checkCounts checks that the table's counts are want.
|
|
||||||
func checkCounts(t *testing.T, table *routetable.Table, want routetable.Counts) {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
got := table.Counts()
|
|
||||||
if !reflect.DeepEqual(got, want) {
|
|
||||||
t.Errorf("Counts() = %+v, want %+v", got, want)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,677 +0,0 @@
|
|||||||
// Package routetable holds the whole routing table in memory. For each prefix
|
|
||||||
// and origin AS, it keeps the peers whose current route to the prefix has that
|
|
||||||
// origin AS. It also keeps the two newest of those routes in full. It applies
|
|
||||||
// announcements, withdrawals and peers' sessions going down. It answers the
|
|
||||||
// route reads that the pages and the API need.
|
|
||||||
package routetable
|
|
||||||
|
|
||||||
import (
|
|
||||||
"cmp"
|
|
||||||
"maps"
|
|
||||||
"net/netip"
|
|
||||||
"slices"
|
|
||||||
"sync"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/bits-and-blooms/bitset"
|
|
||||||
"github.com/google/btree"
|
|
||||||
)
|
|
||||||
|
|
||||||
const (
|
|
||||||
// keptRoutesPerOrigin is how many routes of each prefix and origin AS are
|
|
||||||
// kept in full: the newest ones. Every other peer that carries the prefix
|
|
||||||
// with that origin AS keeps only its bit in the peer set.
|
|
||||||
keptRoutesPerOrigin = 2
|
|
||||||
|
|
||||||
// keptTimesDegree is the degree of the B-tree that holds the times of the
|
|
||||||
// kept routes in order: each of its nodes holds up to 63 times.
|
|
||||||
keptTimesDegree = 32
|
|
||||||
|
|
||||||
ipv4Bits = 32
|
|
||||||
ipv6Bits = 128
|
|
||||||
|
|
||||||
// The address lookup ignores prefixes shorter than these mask lengths. A
|
|
||||||
// default route or a route this short only says where a peer sends traffic
|
|
||||||
// it has no better route for; it does not say who holds the address.
|
|
||||||
minLookupMaskLengthV4 = 8
|
|
||||||
minLookupMaskLengthV6 = 16
|
|
||||||
|
|
||||||
// The IP versions SamplePrefixes takes.
|
|
||||||
ipVersion4 = 4
|
|
||||||
ipVersion6 = 6
|
|
||||||
)
|
|
||||||
|
|
||||||
// Route is one peer's route to one prefix. The next hop is the peer itself, so
|
|
||||||
// it is not stored.
|
|
||||||
type Route struct {
|
|
||||||
// Peer is the peer's address.
|
|
||||||
Peer string
|
|
||||||
ASPath []int
|
|
||||||
// Time is when the peer announced the route.
|
|
||||||
Time time.Time
|
|
||||||
}
|
|
||||||
|
|
||||||
// Origin is one origin AS of a prefix: the peers whose current route to the
|
|
||||||
// prefix has that origin AS, and the newest of those routes in full.
|
|
||||||
type Origin struct {
|
|
||||||
// ASN is the origin AS. 0 means no known origin: the path ends in an AS
|
|
||||||
// set of several members.
|
|
||||||
ASN int
|
|
||||||
// Peers are the peers' addresses, in the order of their numbers: the
|
|
||||||
// order they were first seen in.
|
|
||||||
Peers []string
|
|
||||||
// Routes are the kept routes, newest first: at most two, and none when
|
|
||||||
// both were withdrawn while other peers still carry the prefix.
|
|
||||||
Routes []Route
|
|
||||||
}
|
|
||||||
|
|
||||||
// Change is the current content of one prefix and origin AS pair that changed:
|
|
||||||
// what a database row for the pair holds.
|
|
||||||
type Change struct {
|
|
||||||
Prefix netip.Prefix
|
|
||||||
OriginASN int
|
|
||||||
// Removed says that no peer carries the prefix with this origin AS any
|
|
||||||
// more. Peers and Routes are then nil.
|
|
||||||
Removed bool
|
|
||||||
// Peers holds the number of each peer that carries the prefix with this
|
|
||||||
// origin AS.
|
|
||||||
Peers *bitset.BitSet
|
|
||||||
// Routes are the kept routes, newest first.
|
|
||||||
Routes []Route
|
|
||||||
}
|
|
||||||
|
|
||||||
// Params configures New.
|
|
||||||
type Params struct {
|
|
||||||
// PeerNumbers holds the number each peer was given before, by address. A
|
|
||||||
// peer not in it gets the next number above all of them the first time it
|
|
||||||
// announces a route.
|
|
||||||
PeerNumbers map[string]uint
|
|
||||||
}
|
|
||||||
|
|
||||||
// Table is the routing table. Its methods are safe for concurrent use.
|
|
||||||
type Table struct {
|
|
||||||
mu sync.RWMutex
|
|
||||||
|
|
||||||
ipv4 family
|
|
||||||
ipv6 family
|
|
||||||
// byOrigin holds, for each origin AS, the prefixes it originates.
|
|
||||||
byOrigin map[int]map[netip.Prefix]*origin
|
|
||||||
|
|
||||||
// peerNumbers and peerAddresses give each peer's number by its address
|
|
||||||
// and its address by its number. nextPeerNumber is the number the next
|
|
||||||
// new peer gets.
|
|
||||||
peerNumbers map[string]uint
|
|
||||||
peerAddresses map[uint]string
|
|
||||||
nextPeerNumber uint
|
|
||||||
// peerPrefixes[n] counts the prefixes the peer numbered n carries, for
|
|
||||||
// each n below nextPeerNumber. It is a slice, not a map, because Load
|
|
||||||
// counts once for every peer of every pair it loads: with a map, reading
|
|
||||||
// a full table at startup took several times as long.
|
|
||||||
peerPrefixes []int
|
|
||||||
|
|
||||||
// newPeers and changed are what TakeChanges hands over: the peers
|
|
||||||
// numbered since it last ran, and the prefix and origin AS pairs changed
|
|
||||||
// since it last took them.
|
|
||||||
newPeers map[string]uint
|
|
||||||
changed map[pair]struct{}
|
|
||||||
|
|
||||||
// keptTimes counts the kept routes at each time, in time order, so its
|
|
||||||
// first and last items are the oldest and newest kept routes.
|
|
||||||
keptTimes *btree.BTreeG[keptTime]
|
|
||||||
// originsWithoutKeptRoute counts the origins that peers carry but that
|
|
||||||
// keep no route.
|
|
||||||
originsWithoutKeptRoute int
|
|
||||||
}
|
|
||||||
|
|
||||||
// New returns an empty table that numbers new peers on from
|
|
||||||
// params.PeerNumbers.
|
|
||||||
func New(params Params) *Table {
|
|
||||||
t := &Table{
|
|
||||||
ipv4: newFamily(ipv4Bits, minLookupMaskLengthV4),
|
|
||||||
ipv6: newFamily(ipv6Bits, minLookupMaskLengthV6),
|
|
||||||
byOrigin: make(map[int]map[netip.Prefix]*origin),
|
|
||||||
peerNumbers: make(map[string]uint, len(params.PeerNumbers)),
|
|
||||||
peerAddresses: make(map[uint]string, len(params.PeerNumbers)),
|
|
||||||
newPeers: make(map[string]uint),
|
|
||||||
changed: make(map[pair]struct{}),
|
|
||||||
keptTimes: btree.NewG(keptTimesDegree, func(a, b keptTime) bool {
|
|
||||||
return a.time.Before(b.time)
|
|
||||||
}),
|
|
||||||
}
|
|
||||||
|
|
||||||
for address, number := range params.PeerNumbers {
|
|
||||||
t.peerNumbers[address] = number
|
|
||||||
t.peerAddresses[number] = address
|
|
||||||
t.nextPeerNumber = max(t.nextPeerNumber, number+1)
|
|
||||||
}
|
|
||||||
|
|
||||||
t.peerPrefixes = make([]int, t.nextPeerNumber)
|
|
||||||
|
|
||||||
return t
|
|
||||||
}
|
|
||||||
|
|
||||||
// Announce applies an announcement: the peer's current route to prefix is now
|
|
||||||
// r, with origin AS originASN. A peer that carried the prefix with another
|
|
||||||
// origin AS leaves that origin's peer set, and its route there is dropped. r
|
|
||||||
// is kept in full when it is one of the two newest routes of its prefix and
|
|
||||||
// origin AS; otherwise only the peer's bit is.
|
|
||||||
//
|
|
||||||
// An announcement whose path is empty or holds AS 0 is a withdrawal, and the
|
|
||||||
// caller passes it to Withdraw instead.
|
|
||||||
func (t *Table) Announce(prefix netip.Prefix, originASN int, r Route) {
|
|
||||||
t.mu.Lock()
|
|
||||||
defer t.mu.Unlock()
|
|
||||||
|
|
||||||
if !prefix.IsValid() {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
prefix = prefix.Masked()
|
|
||||||
peer := t.numberPeer(r.Peer)
|
|
||||||
|
|
||||||
current := originWithPeer(t.origins(prefix), peer)
|
|
||||||
if current != nil && current.asn != originASN {
|
|
||||||
t.leave(prefix, current, peer)
|
|
||||||
}
|
|
||||||
|
|
||||||
t.join(prefix, t.findOrAddOrigin(prefix, originASN), route{
|
|
||||||
peer: peer,
|
|
||||||
path: slices.Clone(r.ASPath),
|
|
||||||
time: r.Time,
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
// Withdraw applies a withdrawal: the peer no longer carries prefix. It leaves
|
|
||||||
// the prefix's peer set, and its kept route, if any, is dropped.
|
|
||||||
func (t *Table) Withdraw(prefix netip.Prefix, peer string) {
|
|
||||||
t.mu.Lock()
|
|
||||||
defer t.mu.Unlock()
|
|
||||||
|
|
||||||
number, known := t.peerNumbers[peer]
|
|
||||||
if !known || !prefix.IsValid() {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
prefix = prefix.Masked()
|
|
||||||
|
|
||||||
o := originWithPeer(t.origins(prefix), number)
|
|
||||||
if o != nil {
|
|
||||||
t.leave(prefix, o, number)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// PeerDown applies a peer's session going down: the peer leaves every peer
|
|
||||||
// set, and every route of it is dropped. When the peer carries any prefix, it
|
|
||||||
// visits every prefix in the table.
|
|
||||||
func (t *Table) PeerDown(peer string) {
|
|
||||||
t.mu.Lock()
|
|
||||||
defer t.mu.Unlock()
|
|
||||||
|
|
||||||
number, known := t.peerNumbers[peer]
|
|
||||||
if !known || t.peerPrefixes[number] == 0 {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, f := range []*family{&t.ipv4, &t.ipv6} {
|
|
||||||
for _, prefixes := range f.byLength {
|
|
||||||
for prefix, origins := range prefixes {
|
|
||||||
o := originWithPeer(origins, number)
|
|
||||||
if o != nil {
|
|
||||||
t.leave(prefix, o, number)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Load puts one prefix and origin AS pair that the database holds into the
|
|
||||||
// table, as startup does with each row it reads. The change's peers join the
|
|
||||||
// pair's peer set, and its routes are kept as an announcement's would be.
|
|
||||||
// Unlike an update, Load records no change, so TakeChanges does not hand the
|
|
||||||
// pair back to be written again. Loading a pair twice changes nothing.
|
|
||||||
//
|
|
||||||
// The caller makes sure that every peer change names, in its peer set or in
|
|
||||||
// its routes, has a number in Params.PeerNumbers.
|
|
||||||
func (t *Table) Load(change Change) {
|
|
||||||
t.mu.Lock()
|
|
||||||
defer t.mu.Unlock()
|
|
||||||
|
|
||||||
// A removed pair has no peers, and a pair without peers is not in the
|
|
||||||
// table.
|
|
||||||
if change.Peers == nil || change.Peers.None() || !change.Prefix.IsValid() {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
prefix := change.Prefix.Masked()
|
|
||||||
o := t.findOrAddOrigin(prefix, change.OriginASN)
|
|
||||||
wasWithoutKeptRoute := o.withoutKeptRoute()
|
|
||||||
|
|
||||||
// The peers not yet in the peer set join it in one union, which takes
|
|
||||||
// less time than setting their bits one at a time.
|
|
||||||
joining := change.Peers.Difference(&o.peers)
|
|
||||||
o.peers.InPlaceUnion(joining)
|
|
||||||
t.familyOf(prefix.Addr()).routes += int(joining.Count())
|
|
||||||
|
|
||||||
for peer := range joining.EachSet() {
|
|
||||||
t.peerPrefixes[peer]++
|
|
||||||
}
|
|
||||||
|
|
||||||
// Oldest first, as they were announced, so that two routes with the same
|
|
||||||
// time are kept in the order they had.
|
|
||||||
for _, r := range slices.Backward(change.Routes) {
|
|
||||||
peer := t.peerNumbers[r.Peer]
|
|
||||||
|
|
||||||
t.dropKeptRoute(o, peer)
|
|
||||||
t.keepIfNewest(o, route{
|
|
||||||
peer: peer,
|
|
||||||
path: slices.Clone(r.ASPath),
|
|
||||||
time: r.Time,
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
t.countOriginWithoutKeptRoute(o, wasWithoutKeptRoute)
|
|
||||||
}
|
|
||||||
|
|
||||||
// TakeChanges returns at most maxChanges of the prefix and origin AS pairs that
|
|
||||||
// changed since they were last taken, in prefix order, each with its current
|
|
||||||
// content. The other changed pairs wait for a later call. It also returns every
|
|
||||||
// peer numbered since it last ran, by address, so a pair taken later names only
|
|
||||||
// peers already returned. It then forgets what it returned.
|
|
||||||
func (t *Table) TakeChanges(maxChanges int) ([]Change, map[string]uint) {
|
|
||||||
t.mu.Lock()
|
|
||||||
defer t.mu.Unlock()
|
|
||||||
|
|
||||||
changes := make([]Change, 0, min(maxChanges, len(t.changed)))
|
|
||||||
for key := range t.changed {
|
|
||||||
if len(changes) == maxChanges {
|
|
||||||
break
|
|
||||||
}
|
|
||||||
|
|
||||||
changes = append(changes, t.currentContent(key))
|
|
||||||
delete(t.changed, key)
|
|
||||||
}
|
|
||||||
|
|
||||||
// A map keeps the memory of its largest size, so an emptied one is
|
|
||||||
// replaced.
|
|
||||||
if len(t.changed) == 0 {
|
|
||||||
t.changed = make(map[pair]struct{})
|
|
||||||
}
|
|
||||||
|
|
||||||
slices.SortFunc(changes, func(a, b Change) int {
|
|
||||||
return cmp.Or(a.Prefix.Compare(b.Prefix),
|
|
||||||
cmp.Compare(a.OriginASN, b.OriginASN))
|
|
||||||
})
|
|
||||||
|
|
||||||
newPeers := t.newPeers
|
|
||||||
t.newPeers = make(map[string]uint)
|
|
||||||
|
|
||||||
return changes, newPeers
|
|
||||||
}
|
|
||||||
|
|
||||||
// PutBackChanges records again what one TakeChanges call returned, for a
|
|
||||||
// caller that could not write it. The next TakeChanges returns those peers
|
|
||||||
// again, and those pairs wait to be taken again like any changed pair. Each is
|
|
||||||
// then taken with its content at that time, so a later change of a pair
|
|
||||||
// replaces the earlier.
|
|
||||||
func (t *Table) PutBackChanges(changes []Change, newPeers map[string]uint) {
|
|
||||||
t.mu.Lock()
|
|
||||||
defer t.mu.Unlock()
|
|
||||||
|
|
||||||
for _, change := range changes {
|
|
||||||
t.changed[pair{prefix: change.Prefix, asn: change.OriginASN}] = struct{}{}
|
|
||||||
}
|
|
||||||
|
|
||||||
maps.Copy(t.newPeers, newPeers)
|
|
||||||
}
|
|
||||||
|
|
||||||
// ChangeCount returns how many changed prefix and origin AS pairs wait for
|
|
||||||
// TakeChanges.
|
|
||||||
func (t *Table) ChangeCount() int {
|
|
||||||
t.mu.RLock()
|
|
||||||
defer t.mu.RUnlock()
|
|
||||||
|
|
||||||
return len(t.changed)
|
|
||||||
}
|
|
||||||
|
|
||||||
// numberPeer returns the peer's number, and gives it the next one the first
|
|
||||||
// time the table sees it.
|
|
||||||
func (t *Table) numberPeer(address string) uint {
|
|
||||||
number, known := t.peerNumbers[address]
|
|
||||||
if known {
|
|
||||||
return number
|
|
||||||
}
|
|
||||||
|
|
||||||
number = t.nextPeerNumber
|
|
||||||
t.nextPeerNumber++
|
|
||||||
t.peerNumbers[address] = number
|
|
||||||
t.peerAddresses[number] = address
|
|
||||||
t.peerPrefixes = append(t.peerPrefixes, 0)
|
|
||||||
t.newPeers[address] = number
|
|
||||||
|
|
||||||
return number
|
|
||||||
}
|
|
||||||
|
|
||||||
// familyOf returns the part of the table for addr's IP version.
|
|
||||||
func (t *Table) familyOf(addr netip.Addr) *family {
|
|
||||||
if addr.Is4() {
|
|
||||||
return &t.ipv4
|
|
||||||
}
|
|
||||||
|
|
||||||
return &t.ipv6
|
|
||||||
}
|
|
||||||
|
|
||||||
// prefixesOfLength returns the prefixes of the IP version and the mask length
|
|
||||||
// of prefix, a valid masked prefix.
|
|
||||||
func (t *Table) prefixesOfLength(prefix netip.Prefix) map[netip.Prefix][]*origin {
|
|
||||||
return t.familyOf(prefix.Addr()).byLength[prefix.Bits()]
|
|
||||||
}
|
|
||||||
|
|
||||||
// origins returns the origins of prefix, a valid masked prefix.
|
|
||||||
func (t *Table) origins(prefix netip.Prefix) []*origin {
|
|
||||||
return t.prefixesOfLength(prefix)[prefix]
|
|
||||||
}
|
|
||||||
|
|
||||||
// findOrAddOrigin returns the origin of prefix with AS asn, added with no
|
|
||||||
// peers when the prefix has none with that AS.
|
|
||||||
func (t *Table) findOrAddOrigin(prefix netip.Prefix, asn int) *origin {
|
|
||||||
prefixes := t.prefixesOfLength(prefix)
|
|
||||||
origins := prefixes[prefix]
|
|
||||||
|
|
||||||
i, found := slices.BinarySearchFunc(origins, asn,
|
|
||||||
func(o *origin, asn int) int { return cmp.Compare(o.asn, asn) })
|
|
||||||
if found {
|
|
||||||
return origins[i]
|
|
||||||
}
|
|
||||||
|
|
||||||
o := &origin{asn: asn}
|
|
||||||
prefixes[prefix] = slices.Insert(origins, i, o)
|
|
||||||
|
|
||||||
if t.byOrigin[asn] == nil {
|
|
||||||
t.byOrigin[asn] = make(map[netip.Prefix]*origin)
|
|
||||||
}
|
|
||||||
|
|
||||||
t.byOrigin[asn][prefix] = o
|
|
||||||
|
|
||||||
return o
|
|
||||||
}
|
|
||||||
|
|
||||||
// removeOrigin removes o, which no peer carries any more, from prefix, and the
|
|
||||||
// prefix from the table when it has no origin left.
|
|
||||||
func (t *Table) removeOrigin(prefix netip.Prefix, o *origin) {
|
|
||||||
prefixes := t.prefixesOfLength(prefix)
|
|
||||||
|
|
||||||
origins := slices.DeleteFunc(prefixes[prefix],
|
|
||||||
func(other *origin) bool { return other == o })
|
|
||||||
if len(origins) == 0 {
|
|
||||||
delete(prefixes, prefix)
|
|
||||||
} else {
|
|
||||||
prefixes[prefix] = origins
|
|
||||||
}
|
|
||||||
|
|
||||||
delete(t.byOrigin[o.asn], prefix)
|
|
||||||
|
|
||||||
if len(t.byOrigin[o.asn]) == 0 {
|
|
||||||
delete(t.byOrigin, o.asn)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// join puts the peer of r in the peer set of o, an origin of prefix, and keeps
|
|
||||||
// r in full when it is one of the two newest routes of o. The peer's earlier
|
|
||||||
// route, if o keeps one, is replaced: it is no longer the peer's current route.
|
|
||||||
// The pair is recorded as changed only when its peer set or kept routes change.
|
|
||||||
func (t *Table) join(prefix netip.Prefix, o *origin, r route) {
|
|
||||||
if slices.ContainsFunc(o.routes, r.equal) {
|
|
||||||
// The peer announced its kept route again.
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
wasWithoutKeptRoute := o.withoutKeptRoute()
|
|
||||||
|
|
||||||
joined := !o.peers.Test(r.peer)
|
|
||||||
if joined {
|
|
||||||
o.peers.Set(r.peer)
|
|
||||||
|
|
||||||
t.familyOf(prefix.Addr()).routes++
|
|
||||||
t.peerPrefixes[r.peer]++
|
|
||||||
}
|
|
||||||
|
|
||||||
dropped := t.dropKeptRoute(o, r.peer)
|
|
||||||
kept := t.keepIfNewest(o, r)
|
|
||||||
|
|
||||||
if joined || dropped || kept {
|
|
||||||
t.recordChange(prefix, o, wasWithoutKeptRoute)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// leave takes peer out of the peer set of o, an origin of prefix, and drops
|
|
||||||
// its kept route, if any.
|
|
||||||
func (t *Table) leave(prefix netip.Prefix, o *origin, peer uint) {
|
|
||||||
wasWithoutKeptRoute := o.withoutKeptRoute()
|
|
||||||
|
|
||||||
o.peers.Clear(peer)
|
|
||||||
t.dropKeptRoute(o, peer)
|
|
||||||
|
|
||||||
t.familyOf(prefix.Addr()).routes--
|
|
||||||
t.peerPrefixes[peer]--
|
|
||||||
|
|
||||||
t.recordChange(prefix, o, wasWithoutKeptRoute)
|
|
||||||
}
|
|
||||||
|
|
||||||
// keepIfNewest keeps r in o unless both kept routes of o are newer, and says
|
|
||||||
// whether it kept r. When o already keeps two routes, the older one is
|
|
||||||
// dropped, and its peer keeps only its bit.
|
|
||||||
func (t *Table) keepIfNewest(o *origin, r route) bool {
|
|
||||||
// The kept routes stay newest first: r goes before the first one that is
|
|
||||||
// not newer than it.
|
|
||||||
i := 0
|
|
||||||
for i < len(o.routes) && o.routes[i].time.After(r.time) {
|
|
||||||
i++
|
|
||||||
}
|
|
||||||
|
|
||||||
if i == keptRoutesPerOrigin {
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
|
|
||||||
if len(o.routes) == keptRoutesPerOrigin {
|
|
||||||
t.dropKeptRoute(o, o.routes[keptRoutesPerOrigin-1].peer)
|
|
||||||
}
|
|
||||||
|
|
||||||
o.routes = slices.Insert(o.routes, i, r)
|
|
||||||
t.countKeptTime(r.time, 1)
|
|
||||||
|
|
||||||
return true
|
|
||||||
}
|
|
||||||
|
|
||||||
// dropKeptRoute drops the peer's route from the kept routes of o, if o keeps
|
|
||||||
// one, and says whether it did.
|
|
||||||
func (t *Table) dropKeptRoute(o *origin, peer uint) bool {
|
|
||||||
i := slices.IndexFunc(o.routes, func(r route) bool { return r.peer == peer })
|
|
||||||
if i < 0 {
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
|
|
||||||
t.countKeptTime(o.routes[i].time, -1)
|
|
||||||
o.routes = slices.Delete(o.routes, i, i+1)
|
|
||||||
|
|
||||||
return true
|
|
||||||
}
|
|
||||||
|
|
||||||
// recordChange records that the pair of prefix and the AS of o changed, and
|
|
||||||
// brings the count of origins without a kept route up to date.
|
|
||||||
// wasWithoutKeptRoute says whether o was such an origin before the change. An
|
|
||||||
// origin that no peer carries any more is removed.
|
|
||||||
func (t *Table) recordChange(
|
|
||||||
prefix netip.Prefix, o *origin, wasWithoutKeptRoute bool,
|
|
||||||
) {
|
|
||||||
t.changed[pair{prefix: prefix, asn: o.asn}] = struct{}{}
|
|
||||||
|
|
||||||
t.countOriginWithoutKeptRoute(o, wasWithoutKeptRoute)
|
|
||||||
|
|
||||||
if o.peers.None() {
|
|
||||||
t.removeOrigin(prefix, o)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// countOriginWithoutKeptRoute brings the count of origins without a kept route
|
|
||||||
// up to date after a change to o. wasWithoutKeptRoute says whether o was such
|
|
||||||
// an origin before the change.
|
|
||||||
func (t *Table) countOriginWithoutKeptRoute(o *origin, wasWithoutKeptRoute bool) {
|
|
||||||
if wasWithoutKeptRoute {
|
|
||||||
t.originsWithoutKeptRoute--
|
|
||||||
}
|
|
||||||
|
|
||||||
if o.withoutKeptRoute() {
|
|
||||||
t.originsWithoutKeptRoute++
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// countKeptTime adds delta to the number of kept routes with time at.
|
|
||||||
func (t *Table) countKeptTime(at time.Time, delta int) {
|
|
||||||
count, _ := t.keptTimes.Get(keptTime{time: at})
|
|
||||||
count.time = at
|
|
||||||
count.routes += delta
|
|
||||||
|
|
||||||
if count.routes == 0 {
|
|
||||||
t.keptTimes.Delete(count)
|
|
||||||
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
t.keptTimes.ReplaceOrInsert(count)
|
|
||||||
}
|
|
||||||
|
|
||||||
// currentContent returns what the table holds for the pair now.
|
|
||||||
func (t *Table) currentContent(key pair) Change {
|
|
||||||
o := t.byOrigin[key.asn][key.prefix]
|
|
||||||
if o == nil {
|
|
||||||
return Change{Prefix: key.prefix, OriginASN: key.asn, Removed: true}
|
|
||||||
}
|
|
||||||
|
|
||||||
return Change{
|
|
||||||
Prefix: key.prefix,
|
|
||||||
OriginASN: key.asn,
|
|
||||||
Peers: o.peers.Clone(),
|
|
||||||
Routes: t.exportRoutes(o.routes),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// exportRoutes returns copies of routes that name each peer by its address, or
|
|
||||||
// nil when there are none.
|
|
||||||
func (t *Table) exportRoutes(routes []route) []Route {
|
|
||||||
if len(routes) == 0 {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
exported := make([]Route, 0, len(routes))
|
|
||||||
for _, r := range routes {
|
|
||||||
exported = append(exported, Route{
|
|
||||||
Peer: t.peerAddresses[r.peer],
|
|
||||||
ASPath: slices.Clone(r.path),
|
|
||||||
Time: r.time,
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
return exported
|
|
||||||
}
|
|
||||||
|
|
||||||
// family is the part of the table for one IP version.
|
|
||||||
type family struct {
|
|
||||||
// byLength[n] holds the prefixes of mask length n that some peer carries,
|
|
||||||
// each with its origins in AS number order.
|
|
||||||
byLength []map[netip.Prefix][]*origin
|
|
||||||
// minLookupLength is the shortest mask length the address lookup uses.
|
|
||||||
minLookupLength int
|
|
||||||
// routes counts the peer and prefix pairs: each peer that carries a
|
|
||||||
// prefix counts once for it.
|
|
||||||
routes int
|
|
||||||
}
|
|
||||||
|
|
||||||
// newFamily returns an empty family for addresses of bits bits.
|
|
||||||
func newFamily(bits, minLookupLength int) family {
|
|
||||||
f := family{
|
|
||||||
byLength: make([]map[netip.Prefix][]*origin, bits+1),
|
|
||||||
minLookupLength: minLookupLength,
|
|
||||||
}
|
|
||||||
|
|
||||||
for n := range f.byLength {
|
|
||||||
f.byLength[n] = make(map[netip.Prefix][]*origin)
|
|
||||||
}
|
|
||||||
|
|
||||||
return f
|
|
||||||
}
|
|
||||||
|
|
||||||
// prefixesByLength returns how many prefixes of each mask length f holds.
|
|
||||||
func (f *family) prefixesByLength() []int {
|
|
||||||
counts := make([]int, len(f.byLength))
|
|
||||||
for n, prefixes := range f.byLength {
|
|
||||||
counts[n] = len(prefixes)
|
|
||||||
}
|
|
||||||
|
|
||||||
return counts
|
|
||||||
}
|
|
||||||
|
|
||||||
// origin is what the table holds for one prefix and origin AS.
|
|
||||||
type origin struct {
|
|
||||||
asn int
|
|
||||||
// peers holds the number of each peer whose current route to the prefix
|
|
||||||
// has this origin AS.
|
|
||||||
peers bitset.BitSet
|
|
||||||
// routes are the newest of those routes, newest first, at most
|
|
||||||
// keptRoutesPerOrigin.
|
|
||||||
routes []route
|
|
||||||
}
|
|
||||||
|
|
||||||
// withoutKeptRoute says whether peers carry o but o keeps no route.
|
|
||||||
func (o *origin) withoutKeptRoute() bool {
|
|
||||||
return o.peers.Any() && len(o.routes) == 0
|
|
||||||
}
|
|
||||||
|
|
||||||
// newestTime returns the time of the newest kept route of o, or nil when o
|
|
||||||
// keeps none.
|
|
||||||
func (o *origin) newestTime() *time.Time {
|
|
||||||
if len(o.routes) == 0 {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
newest := o.routes[0].time
|
|
||||||
|
|
||||||
return &newest
|
|
||||||
}
|
|
||||||
|
|
||||||
// originWithPeer returns the origin whose peer set holds peer, or nil when
|
|
||||||
// none does. A peer carries a prefix with one origin AS at most.
|
|
||||||
func originWithPeer(origins []*origin, peer uint) *origin {
|
|
||||||
for _, o := range origins {
|
|
||||||
if o.peers.Test(peer) {
|
|
||||||
return o
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// route is a kept route. peer is the peer's number.
|
|
||||||
type route struct {
|
|
||||||
peer uint
|
|
||||||
path []int
|
|
||||||
time time.Time
|
|
||||||
}
|
|
||||||
|
|
||||||
// equal says whether r and other are the same peer's route with the same path
|
|
||||||
// and time.
|
|
||||||
func (r route) equal(other route) bool {
|
|
||||||
return r.peer == other.peer && r.time.Equal(other.time) &&
|
|
||||||
slices.Equal(r.path, other.path)
|
|
||||||
}
|
|
||||||
|
|
||||||
// pair names one prefix and origin AS.
|
|
||||||
type pair struct {
|
|
||||||
prefix netip.Prefix
|
|
||||||
asn int
|
|
||||||
}
|
|
||||||
|
|
||||||
// keptTime counts the kept routes with one time.
|
|
||||||
type keptTime struct {
|
|
||||||
time time.Time
|
|
||||||
routes int
|
|
||||||
}
|
|
||||||
@@ -1,667 +0,0 @@
|
|||||||
package routetable_test
|
|
||||||
|
|
||||||
import (
|
|
||||||
"math"
|
|
||||||
"net/netip"
|
|
||||||
"reflect"
|
|
||||||
"slices"
|
|
||||||
"sync"
|
|
||||||
"testing"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/bits-and-blooms/bitset"
|
|
||||||
|
|
||||||
"git.eeqj.de/sneak/routewatch/internal/routetable"
|
|
||||||
)
|
|
||||||
|
|
||||||
const (
|
|
||||||
peerA = "192.0.2.1"
|
|
||||||
peerB = "192.0.2.2"
|
|
||||||
peerC = "2001:db8::3"
|
|
||||||
peerD = "2001:db8::4"
|
|
||||||
|
|
||||||
asA = 64500
|
|
||||||
asB = 64501
|
|
||||||
asC = 64502
|
|
||||||
// transitAS is the first AS of every test path, the peer's own.
|
|
||||||
transitAS = 64496
|
|
||||||
|
|
||||||
prefixV4 = "10.1.2.0/24"
|
|
||||||
prefixV6 = "2001:db8:1::/48"
|
|
||||||
)
|
|
||||||
|
|
||||||
// at returns the time second seconds after the start of every test.
|
|
||||||
func at(second int) time.Time {
|
|
||||||
return time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC).
|
|
||||||
Add(time.Duration(second) * time.Second)
|
|
||||||
}
|
|
||||||
|
|
||||||
// timeAt returns at(second) for a field that may be nil.
|
|
||||||
func timeAt(second int) *time.Time {
|
|
||||||
t := at(second)
|
|
||||||
|
|
||||||
return &t
|
|
||||||
}
|
|
||||||
|
|
||||||
// routeOf returns the route peer announces with origin AS asn second seconds
|
|
||||||
// after the start.
|
|
||||||
func routeOf(peer string, asn, second int) routetable.Route {
|
|
||||||
return routetable.Route{
|
|
||||||
Peer: peer, ASPath: []int{transitAS, asn}, Time: at(second),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// step is one update a test applies to a table for one prefix.
|
|
||||||
type step func(table *routetable.Table, prefix netip.Prefix)
|
|
||||||
|
|
||||||
// announce is the step of peer announcing prefix with origin AS asn second
|
|
||||||
// seconds after the start.
|
|
||||||
func announce(peer string, asn, second int) step {
|
|
||||||
return func(table *routetable.Table, prefix netip.Prefix) {
|
|
||||||
table.Announce(prefix, asn, routeOf(peer, asn, second))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// withdraw is the step of peer withdrawing prefix.
|
|
||||||
func withdraw(peer string) step {
|
|
||||||
return func(table *routetable.Table, prefix netip.Prefix) {
|
|
||||||
table.Withdraw(prefix, peer)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// peerDown is the step of peer's session going down.
|
|
||||||
func peerDown(peer string) step {
|
|
||||||
return func(table *routetable.Table, _ netip.Prefix) {
|
|
||||||
table.PeerDown(peer)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// rule is one case of the rule that decides what the table holds for a prefix:
|
|
||||||
// the steps applied to an empty table, and the prefix's origins after them.
|
|
||||||
// The peers are numbered in the order they first announce, which is the order
|
|
||||||
// Origin.Peers lists them in.
|
|
||||||
type rule struct {
|
|
||||||
name string
|
|
||||||
steps []step
|
|
||||||
want []routetable.Origin
|
|
||||||
}
|
|
||||||
|
|
||||||
// checkRules runs each case for an IPv4 and an IPv6 prefix.
|
|
||||||
func checkRules(t *testing.T, rules []rule) {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
for _, tc := range rules {
|
|
||||||
for _, text := range []string{prefixV4, prefixV6} {
|
|
||||||
t.Run(tc.name+" "+text, func(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
prefix := netip.MustParsePrefix(text)
|
|
||||||
table := routetable.New(routetable.Params{})
|
|
||||||
|
|
||||||
for _, apply := range tc.steps {
|
|
||||||
apply(table, prefix)
|
|
||||||
}
|
|
||||||
|
|
||||||
got := table.Origins(prefix)
|
|
||||||
if !reflect.DeepEqual(got, tc.want) {
|
|
||||||
t.Errorf("Origins(%s) = %+v, want %+v", prefix, got, tc.want)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestAnnouncementsKeepTheTwoNewestRoutes(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
checkRules(t, []rule{
|
|
||||||
{
|
|
||||||
name: "an announcement is kept",
|
|
||||||
steps: []step{announce(peerA, asA, 10)},
|
|
||||||
want: []routetable.Origin{{ASN: asA, Peers: []string{peerA},
|
|
||||||
Routes: []routetable.Route{routeOf(peerA, asA, 10)}}},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "a lagging peer's older route is not kept",
|
|
||||||
steps: []step{announce(peerA, asA, 20), announce(peerB, asA, 30),
|
|
||||||
announce(peerC, asA, 10)},
|
|
||||||
want: []routetable.Origin{{ASN: asA,
|
|
||||||
Peers: []string{peerA, peerB, peerC}, Routes: []routetable.Route{
|
|
||||||
routeOf(peerB, asA, 30), routeOf(peerA, asA, 20),
|
|
||||||
}}},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "a third newer route drops the older kept one, whose peer " +
|
|
||||||
"keeps its bit",
|
|
||||||
steps: []step{announce(peerA, asA, 10), announce(peerB, asA, 20),
|
|
||||||
announce(peerC, asA, 30)},
|
|
||||||
want: []routetable.Origin{{ASN: asA,
|
|
||||||
Peers: []string{peerA, peerB, peerC}, Routes: []routetable.Route{
|
|
||||||
routeOf(peerC, asA, 30), routeOf(peerB, asA, 20),
|
|
||||||
}}},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "a route as old as the older kept one replaces it",
|
|
||||||
steps: []step{announce(peerA, asA, 10), announce(peerB, asA, 20),
|
|
||||||
announce(peerC, asA, 10)},
|
|
||||||
want: []routetable.Origin{{ASN: asA,
|
|
||||||
Peers: []string{peerA, peerB, peerC}, Routes: []routetable.Route{
|
|
||||||
routeOf(peerB, asA, 20), routeOf(peerC, asA, 10),
|
|
||||||
}}},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "a peer's new route replaces its kept one",
|
|
||||||
steps: []step{announce(peerA, asA, 10), announce(peerB, asA, 20),
|
|
||||||
announce(peerA, asA, 30)},
|
|
||||||
want: []routetable.Origin{{ASN: asA, Peers: []string{peerA, peerB},
|
|
||||||
Routes: []routetable.Route{
|
|
||||||
routeOf(peerA, asA, 30), routeOf(peerB, asA, 20),
|
|
||||||
}}},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "a peer's new route replaces its kept one even when older",
|
|
||||||
steps: []step{announce(peerA, asA, 10), announce(peerB, asA, 20),
|
|
||||||
announce(peerB, asA, 5)},
|
|
||||||
want: []routetable.Origin{{ASN: asA, Peers: []string{peerA, peerB},
|
|
||||||
Routes: []routetable.Route{
|
|
||||||
routeOf(peerA, asA, 10), routeOf(peerB, asA, 5),
|
|
||||||
}}},
|
|
||||||
},
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestAPeerCarriesAPrefixWithOneOrigin(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
checkRules(t, []rule{
|
|
||||||
{
|
|
||||||
name: "a peer moving to a new origin leaves the old one",
|
|
||||||
steps: []step{announce(peerA, asA, 10), announce(peerB, asA, 20),
|
|
||||||
announce(peerA, asB, 30)},
|
|
||||||
want: []routetable.Origin{
|
|
||||||
{ASN: asA, Peers: []string{peerB},
|
|
||||||
Routes: []routetable.Route{routeOf(peerB, asA, 20)}},
|
|
||||||
{ASN: asB, Peers: []string{peerA},
|
|
||||||
Routes: []routetable.Route{routeOf(peerA, asB, 30)}},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "a peer whose route is not kept moves to a new origin",
|
|
||||||
steps: []step{announce(peerA, asA, 10), announce(peerB, asA, 20),
|
|
||||||
announce(peerC, asA, 30), announce(peerA, asB, 40)},
|
|
||||||
want: []routetable.Origin{
|
|
||||||
{ASN: asA, Peers: []string{peerB, peerC},
|
|
||||||
Routes: []routetable.Route{
|
|
||||||
routeOf(peerC, asA, 30), routeOf(peerB, asA, 20),
|
|
||||||
}},
|
|
||||||
{ASN: asB, Peers: []string{peerA},
|
|
||||||
Routes: []routetable.Route{routeOf(peerA, asB, 40)}},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "origins are listed in AS number order",
|
|
||||||
steps: []step{announce(peerA, asC, 10), announce(peerB, asA, 20),
|
|
||||||
announce(peerC, asB, 30)},
|
|
||||||
want: []routetable.Origin{
|
|
||||||
{ASN: asA, Peers: []string{peerB},
|
|
||||||
Routes: []routetable.Route{routeOf(peerB, asA, 20)}},
|
|
||||||
{ASN: asB, Peers: []string{peerC},
|
|
||||||
Routes: []routetable.Route{routeOf(peerC, asB, 30)}},
|
|
||||||
{ASN: asC, Peers: []string{peerA},
|
|
||||||
Routes: []routetable.Route{routeOf(peerA, asC, 10)}},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestWithdrawalsAndPeerDowns(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
// threePeers is peerA, peerB and peerC announcing in turn; the routes of
|
|
||||||
// peerC and peerB are kept.
|
|
||||||
threePeers := []step{announce(peerA, asA, 10), announce(peerB, asA, 20),
|
|
||||||
announce(peerC, asA, 30)}
|
|
||||||
|
|
||||||
checkRules(t, []rule{
|
|
||||||
{
|
|
||||||
name: "a withdrawal drops the peer's kept route",
|
|
||||||
steps: append(slices.Clone(threePeers), withdraw(peerC)),
|
|
||||||
want: []routetable.Origin{{ASN: asA, Peers: []string{peerA, peerB},
|
|
||||||
Routes: []routetable.Route{routeOf(peerB, asA, 20)}}},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "a withdrawal of a route that is not kept leaves the kept ones",
|
|
||||||
steps: append(slices.Clone(threePeers), withdraw(peerA)),
|
|
||||||
want: []routetable.Origin{{ASN: asA, Peers: []string{peerB, peerC},
|
|
||||||
Routes: []routetable.Route{
|
|
||||||
routeOf(peerC, asA, 30), routeOf(peerB, asA, 20),
|
|
||||||
}}},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "an origin whose kept routes are withdrawn keeps its other " +
|
|
||||||
"peers",
|
|
||||||
steps: append(slices.Clone(threePeers), withdraw(peerB),
|
|
||||||
withdraw(peerC)),
|
|
||||||
want: []routetable.Origin{{ASN: asA, Peers: []string{peerA}}},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "an origin without kept routes keeps the next update",
|
|
||||||
steps: append(slices.Clone(threePeers), withdraw(peerB),
|
|
||||||
withdraw(peerC), announce(peerA, asA, 5)),
|
|
||||||
want: []routetable.Origin{{ASN: asA, Peers: []string{peerA},
|
|
||||||
Routes: []routetable.Route{routeOf(peerA, asA, 5)}}},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "an origin no peer carries is removed",
|
|
||||||
steps: []step{announce(peerA, asA, 10), announce(peerB, asB, 20),
|
|
||||||
withdraw(peerA)},
|
|
||||||
want: []routetable.Origin{{ASN: asB, Peers: []string{peerB},
|
|
||||||
Routes: []routetable.Route{routeOf(peerB, asB, 20)}}},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "a prefix no peer carries is removed",
|
|
||||||
steps: []step{announce(peerA, asA, 10), withdraw(peerA)},
|
|
||||||
want: nil,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "a withdrawal from a peer that does not carry it changes nothing",
|
|
||||||
steps: []step{announce(peerA, asA, 10), withdraw(peerB)},
|
|
||||||
want: []routetable.Origin{{ASN: asA, Peers: []string{peerA},
|
|
||||||
Routes: []routetable.Route{routeOf(peerA, asA, 10)}}},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "a peer going down leaves the prefix",
|
|
||||||
steps: []step{announce(peerA, asA, 10), announce(peerB, asA, 20),
|
|
||||||
peerDown(peerB)},
|
|
||||||
want: []routetable.Origin{{ASN: asA, Peers: []string{peerA},
|
|
||||||
Routes: []routetable.Route{routeOf(peerA, asA, 10)}}},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "a peer going down after it withdrew changes nothing",
|
|
||||||
steps: []step{announce(peerA, asA, 10), announce(peerB, asA, 20),
|
|
||||||
withdraw(peerB), peerDown(peerB)},
|
|
||||||
want: []routetable.Origin{{ASN: asA, Peers: []string{peerA},
|
|
||||||
Routes: []routetable.Route{routeOf(peerA, asA, 10)}}},
|
|
||||||
},
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestPeerDownLeavesEveryPrefix(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
table := routetable.New(routetable.Params{})
|
|
||||||
prefixes := []netip.Prefix{
|
|
||||||
netip.MustParsePrefix(prefixV4),
|
|
||||||
netip.MustParsePrefix("10.2.0.0/16"),
|
|
||||||
netip.MustParsePrefix(prefixV6),
|
|
||||||
}
|
|
||||||
|
|
||||||
// peerA carries every prefix with origin asA, peerB the first one with
|
|
||||||
// origin asB.
|
|
||||||
for _, prefix := range prefixes {
|
|
||||||
table.Announce(prefix, asA, routeOf(peerA, asA, 10))
|
|
||||||
}
|
|
||||||
|
|
||||||
table.Announce(prefixes[0], asB, routeOf(peerB, asB, 20))
|
|
||||||
table.PeerDown(peerA)
|
|
||||||
|
|
||||||
want := []routetable.Origin{{ASN: asB, Peers: []string{peerB},
|
|
||||||
Routes: []routetable.Route{routeOf(peerB, asB, 20)}}}
|
|
||||||
|
|
||||||
got := table.Origins(prefixes[0])
|
|
||||||
if !reflect.DeepEqual(got, want) {
|
|
||||||
t.Errorf("Origins(%s) = %+v, want %+v", prefixes[0], got, want)
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, prefix := range prefixes[1:] {
|
|
||||||
got := table.Origins(prefix)
|
|
||||||
if got != nil {
|
|
||||||
t.Errorf("Origins(%s) = %+v after the only peer went down, want nil",
|
|
||||||
prefix, got)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
counts := table.Counts()
|
|
||||||
if counts.IPv4Prefixes != 1 || counts.IPv6Prefixes != 0 ||
|
|
||||||
counts.IPv4Routes != 1 || counts.IPv6Routes != 0 {
|
|
||||||
t.Errorf("Counts() = %+v, want one IPv4 prefix and route", counts)
|
|
||||||
}
|
|
||||||
|
|
||||||
if len(table.OriginatedPrefixes(asA)) != 0 {
|
|
||||||
t.Errorf("OriginatedPrefixes(%d) = %+v, want none",
|
|
||||||
asA, table.OriginatedPrefixes(asA))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestCountsOriginsWithoutKeptRoute(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
prefix := netip.MustParsePrefix(prefixV4)
|
|
||||||
table := routetable.New(routetable.Params{})
|
|
||||||
|
|
||||||
steps := []struct {
|
|
||||||
apply step
|
|
||||||
want int
|
|
||||||
}{
|
|
||||||
{announce(peerA, asA, 10), 0},
|
|
||||||
{announce(peerB, asA, 20), 0},
|
|
||||||
{announce(peerC, asA, 30), 0},
|
|
||||||
{withdraw(peerC), 0},
|
|
||||||
{withdraw(peerB), 1},
|
|
||||||
// An update from the last peer is kept.
|
|
||||||
{announce(peerA, asA, 40), 0},
|
|
||||||
{withdraw(peerA), 0},
|
|
||||||
// Once more without kept routes, then without peers: the origin
|
|
||||||
// is removed.
|
|
||||||
{announce(peerA, asA, 10), 0},
|
|
||||||
{announce(peerB, asA, 20), 0},
|
|
||||||
{announce(peerC, asA, 30), 0},
|
|
||||||
{withdraw(peerB), 0},
|
|
||||||
{withdraw(peerC), 1},
|
|
||||||
{peerDown(peerA), 0},
|
|
||||||
}
|
|
||||||
|
|
||||||
for i, s := range steps {
|
|
||||||
s.apply(table, prefix)
|
|
||||||
|
|
||||||
got := table.Counts().OriginsWithoutKeptRoute
|
|
||||||
if got != s.want {
|
|
||||||
t.Errorf("after step %d, OriginsWithoutKeptRoute = %d, want %d",
|
|
||||||
i, got, s.want)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// change is a routetable.Change with its peer set as a list of peer numbers.
|
|
||||||
type change struct {
|
|
||||||
Prefix string
|
|
||||||
OriginASN int
|
|
||||||
Removed bool
|
|
||||||
Peers []uint
|
|
||||||
Routes []routetable.Route
|
|
||||||
}
|
|
||||||
|
|
||||||
// peerNumbers returns the numbers in set, or nil for a nil set.
|
|
||||||
func peerNumbers(set *bitset.BitSet) []uint {
|
|
||||||
if set == nil {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
return slices.Collect(set.EachSet())
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestTakeChanges(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
// peerA was numbered 5 before; the next new peer is 6.
|
|
||||||
table := routetable.New(routetable.Params{
|
|
||||||
PeerNumbers: map[string]uint{peerA: 5},
|
|
||||||
})
|
|
||||||
v4 := netip.MustParsePrefix(prefixV4)
|
|
||||||
v6 := netip.MustParsePrefix(prefixV6)
|
|
||||||
|
|
||||||
table.Announce(v6, asA, routeOf(peerA, asA, 10))
|
|
||||||
table.Announce(v4, asA, routeOf(peerA, asA, 10))
|
|
||||||
table.Announce(v4, asA, routeOf(peerB, asA, 20))
|
|
||||||
|
|
||||||
checkChanges(t, table, []change{
|
|
||||||
{Prefix: prefixV4, OriginASN: asA, Peers: []uint{5, 6},
|
|
||||||
Routes: []routetable.Route{
|
|
||||||
routeOf(peerB, asA, 20), routeOf(peerA, asA, 10),
|
|
||||||
}},
|
|
||||||
{Prefix: prefixV6, OriginASN: asA, Peers: []uint{5},
|
|
||||||
Routes: []routetable.Route{routeOf(peerA, asA, 10)}},
|
|
||||||
}, map[string]uint{peerB: 6})
|
|
||||||
|
|
||||||
// TakeChanges cleared the record.
|
|
||||||
checkChanges(t, table, []change{}, map[string]uint{})
|
|
||||||
|
|
||||||
// peerB moves to asB; peerA withdraws, which removes the pair of v4 and
|
|
||||||
// asA.
|
|
||||||
table.Announce(v4, asB, routeOf(peerB, asB, 30))
|
|
||||||
table.Withdraw(v4, peerA)
|
|
||||||
|
|
||||||
checkChanges(t, table, []change{
|
|
||||||
{Prefix: prefixV4, OriginASN: asA, Removed: true},
|
|
||||||
{Prefix: prefixV4, OriginASN: asB, Peers: []uint{6},
|
|
||||||
Routes: []routetable.Route{routeOf(peerB, asB, 30)}},
|
|
||||||
}, map[string]uint{})
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestTakeChangesHoldsOnlyPairsThatChanged(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
table := routetable.New(routetable.Params{})
|
|
||||||
v4 := netip.MustParsePrefix(prefixV4)
|
|
||||||
v6 := netip.MustParsePrefix(prefixV6)
|
|
||||||
|
|
||||||
// peerA, peerB and peerC are numbered 0, 1 and 2. The routes of peerC and
|
|
||||||
// peerB to v4 are kept; peerA keeps only its bit.
|
|
||||||
announceAll(table, prefixV4, asA, 10, peerA, peerB, peerC)
|
|
||||||
table.Announce(v6, asA, routeOf(peerB, asA, 10))
|
|
||||||
table.TakeChanges(math.MaxInt)
|
|
||||||
|
|
||||||
// peerA announces again, with a time older than both kept routes.
|
|
||||||
table.Announce(v4, asA, routeOf(peerA, asA, 15))
|
|
||||||
checkChanges(t, table, []change{}, map[string]uint{})
|
|
||||||
|
|
||||||
// peerC announces its kept route again.
|
|
||||||
table.Announce(v4, asA, routeOf(peerC, asA, 30))
|
|
||||||
checkChanges(t, table, []change{}, map[string]uint{})
|
|
||||||
|
|
||||||
// peerC announces the same path with a newer time, so its kept route
|
|
||||||
// changes.
|
|
||||||
table.Announce(v4, asA, routeOf(peerC, asA, 40))
|
|
||||||
checkChanges(t, table, []change{
|
|
||||||
{Prefix: prefixV4, OriginASN: asA, Peers: []uint{0, 1, 2},
|
|
||||||
Routes: []routetable.Route{
|
|
||||||
routeOf(peerC, asA, 40), routeOf(peerB, asA, 20),
|
|
||||||
}},
|
|
||||||
}, map[string]uint{})
|
|
||||||
|
|
||||||
// peerA withdraws v6, which it does not carry.
|
|
||||||
table.Withdraw(v6, peerA)
|
|
||||||
checkChanges(t, table, []change{}, map[string]uint{})
|
|
||||||
|
|
||||||
// peerA going down changes v4, the one prefix it carries.
|
|
||||||
table.PeerDown(peerA)
|
|
||||||
checkChanges(t, table, []change{
|
|
||||||
{Prefix: prefixV4, OriginASN: asA, Peers: []uint{1, 2},
|
|
||||||
Routes: []routetable.Route{
|
|
||||||
routeOf(peerC, asA, 40), routeOf(peerB, asA, 20),
|
|
||||||
}},
|
|
||||||
}, map[string]uint{})
|
|
||||||
|
|
||||||
// peerA, which carries nothing now, goes down again.
|
|
||||||
table.PeerDown(peerA)
|
|
||||||
checkChanges(t, table, []change{}, map[string]uint{})
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestTakeChangesTakesAtMostMaxChanges changes three prefix and origin AS
|
|
||||||
// pairs, then takes the changes two at a time. The first call must return two
|
|
||||||
// pairs and both new peers, and the second the third pair and no peer.
|
|
||||||
func TestTakeChangesTakesAtMostMaxChanges(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
const maxChanges = 2
|
|
||||||
|
|
||||||
table := routetable.New(routetable.Params{})
|
|
||||||
v4 := netip.MustParsePrefix(prefixV4)
|
|
||||||
v6 := netip.MustParsePrefix(prefixV6)
|
|
||||||
|
|
||||||
table.Announce(v4, asA, routeOf(peerA, asA, 10))
|
|
||||||
table.Announce(v4, asB, routeOf(peerB, asB, 10))
|
|
||||||
table.Announce(v6, asA, routeOf(peerA, asA, 10))
|
|
||||||
|
|
||||||
first, firstPeers := table.TakeChanges(maxChanges)
|
|
||||||
second, secondPeers := table.TakeChanges(maxChanges)
|
|
||||||
|
|
||||||
if len(first) != maxChanges || len(second) != 1 {
|
|
||||||
t.Fatalf("TakeChanges(%d) returned %d and then %d pairs, want %d and 1",
|
|
||||||
maxChanges, len(first), len(second), maxChanges)
|
|
||||||
}
|
|
||||||
|
|
||||||
wantPeers := map[string]uint{peerA: 0, peerB: 1}
|
|
||||||
if !reflect.DeepEqual(firstPeers, wantPeers) || len(secondPeers) != 0 {
|
|
||||||
t.Errorf("TakeChanges(%d) returned the new peers %v and then %v, "+
|
|
||||||
"want %v and none", maxChanges, firstPeers, secondPeers, wantPeers)
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, c := range first {
|
|
||||||
if c.Prefix == second[0].Prefix && c.OriginASN == second[0].OriginASN {
|
|
||||||
t.Errorf("both calls returned %s with AS%d", c.Prefix, c.OriginASN)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestPutBackChanges takes a table's changes and puts them back, as a caller
|
|
||||||
// does whose write failed. The next TakeChanges must return them again, with
|
|
||||||
// each pair's content then, and with the changes since.
|
|
||||||
func TestPutBackChanges(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
table := routetable.New(routetable.Params{})
|
|
||||||
v4 := netip.MustParsePrefix(prefixV4)
|
|
||||||
v6 := netip.MustParsePrefix(prefixV6)
|
|
||||||
|
|
||||||
table.Announce(v4, asA, routeOf(peerA, asA, 10))
|
|
||||||
table.Announce(v6, asA, routeOf(peerA, asA, 10))
|
|
||||||
table.PutBackChanges(table.TakeChanges(math.MaxInt))
|
|
||||||
|
|
||||||
if got := table.ChangeCount(); got != 2 {
|
|
||||||
t.Errorf("ChangeCount() = %d after the changes were put back, want 2",
|
|
||||||
got)
|
|
||||||
}
|
|
||||||
|
|
||||||
// peerA withdraws v4, so v4's earlier content must not come back; peerB
|
|
||||||
// announces v6, and is a new peer as well.
|
|
||||||
table.Withdraw(v4, peerA)
|
|
||||||
table.Announce(v6, asA, routeOf(peerB, asA, 20))
|
|
||||||
|
|
||||||
checkChanges(t, table, []change{
|
|
||||||
{Prefix: prefixV4, OriginASN: asA, Removed: true},
|
|
||||||
{Prefix: prefixV6, OriginASN: asA, Peers: []uint{0, 1},
|
|
||||||
Routes: []routetable.Route{
|
|
||||||
routeOf(peerB, asA, 20), routeOf(peerA, asA, 10),
|
|
||||||
}},
|
|
||||||
}, map[string]uint{peerA: 0, peerB: 1})
|
|
||||||
|
|
||||||
if got := table.ChangeCount(); got != 0 {
|
|
||||||
t.Errorf("ChangeCount() = %d after TakeChanges, want 0", got)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestLoad(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
const otherV4 = "10.9.0.0/16"
|
|
||||||
|
|
||||||
v4 := netip.MustParsePrefix(prefixV4)
|
|
||||||
v6 := netip.MustParsePrefix(prefixV6)
|
|
||||||
other := netip.MustParsePrefix(otherV4)
|
|
||||||
|
|
||||||
// The routes of peerC and peerB to v4 are kept; peerA keeps only its bit.
|
|
||||||
// The two routes to v6 have the same time. The routes of other were
|
|
||||||
// withdrawn while peerA still carries it.
|
|
||||||
source := routetable.New(routetable.Params{})
|
|
||||||
announceAll(source, prefixV4, asA, 10, peerA, peerB, peerC)
|
|
||||||
announceAll(source, prefixV4, asB, 40, peerD)
|
|
||||||
source.Announce(v6, asA, routeOf(peerA, asA, 50))
|
|
||||||
source.Announce(v6, asA, routeOf(peerC, asA, 50))
|
|
||||||
announceAll(source, otherV4, asB, 10, peerA, peerB, peerC)
|
|
||||||
source.Withdraw(other, peerB)
|
|
||||||
source.Withdraw(other, peerC)
|
|
||||||
|
|
||||||
changes, peers := source.TakeChanges(math.MaxInt)
|
|
||||||
loaded := routetable.New(routetable.Params{PeerNumbers: peers})
|
|
||||||
|
|
||||||
// Loading a pair twice changes nothing.
|
|
||||||
for range 2 {
|
|
||||||
for _, change := range changes {
|
|
||||||
loaded.Load(change)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, prefix := range []netip.Prefix{v4, v6, other} {
|
|
||||||
got, want := loaded.Origins(prefix), source.Origins(prefix)
|
|
||||||
if !reflect.DeepEqual(got, want) {
|
|
||||||
t.Errorf("Origins(%s) = %+v after the load, want %+v", prefix, got, want)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if !reflect.DeepEqual(loaded.Counts(), source.Counts()) {
|
|
||||||
t.Errorf("Counts() = %+v after the load, want %+v",
|
|
||||||
loaded.Counts(), source.Counts())
|
|
||||||
}
|
|
||||||
|
|
||||||
checkChanges(t, loaded, []change{}, map[string]uint{})
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestConcurrentUse has goroutines update one table while others read it. The
|
|
||||||
// race detector, which make test runs, fails it when a method uses the table
|
|
||||||
// without its lock.
|
|
||||||
func TestConcurrentUse(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
const (
|
|
||||||
// rounds is how many times each goroutine calls its method.
|
|
||||||
rounds = 1000
|
|
||||||
ipVersion = 4
|
|
||||||
sampleSize = 10
|
|
||||||
)
|
|
||||||
|
|
||||||
table := routetable.New(routetable.Params{})
|
|
||||||
prefix := netip.MustParsePrefix(prefixV4)
|
|
||||||
|
|
||||||
uses := []func(round int){
|
|
||||||
func(round int) { table.Announce(prefix, asA, routeOf(peerA, asA, round)) },
|
|
||||||
func(round int) { table.Announce(prefix, asB, routeOf(peerB, asB, round)) },
|
|
||||||
func(int) { table.Withdraw(prefix, peerA) },
|
|
||||||
func(int) { table.PeerDown(peerB) },
|
|
||||||
func(int) { table.PutBackChanges(table.TakeChanges(math.MaxInt)) },
|
|
||||||
func(int) { table.ChangeCount() },
|
|
||||||
func(int) { table.Lookup(prefix.Addr()) },
|
|
||||||
func(int) { table.Origins(prefix) },
|
|
||||||
func(int) { table.OriginatedPrefixes(asA) },
|
|
||||||
func(int) { table.SamplePrefixes(ipVersion, prefix.Bits(), sampleSize) },
|
|
||||||
func(int) { table.Counts() },
|
|
||||||
}
|
|
||||||
|
|
||||||
var wg sync.WaitGroup
|
|
||||||
for _, use := range uses {
|
|
||||||
wg.Go(func() {
|
|
||||||
for round := range rounds {
|
|
||||||
use(round)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
wg.Wait()
|
|
||||||
}
|
|
||||||
|
|
||||||
// checkChanges checks that TakeChanges, taking every change, returns want and
|
|
||||||
// wantPeers.
|
|
||||||
func checkChanges(
|
|
||||||
t *testing.T, table *routetable.Table, want []change,
|
|
||||||
wantPeers map[string]uint,
|
|
||||||
) {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
changes, newPeers := table.TakeChanges(math.MaxInt)
|
|
||||||
|
|
||||||
got := make([]change, 0, len(changes))
|
|
||||||
for _, c := range changes {
|
|
||||||
got = append(got, change{
|
|
||||||
Prefix: c.Prefix.String(), OriginASN: c.OriginASN, Removed: c.Removed,
|
|
||||||
Peers: peerNumbers(c.Peers), Routes: c.Routes,
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
if !reflect.DeepEqual(got, want) {
|
|
||||||
t.Errorf("TakeChanges() changes = %+v, want %+v", got, want)
|
|
||||||
}
|
|
||||||
|
|
||||||
if !reflect.DeepEqual(newPeers, wantPeers) {
|
|
||||||
t.Errorf("TakeChanges() new peers = %v, want %v", newPeers, wantPeers)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
+24
-104
@@ -8,17 +8,13 @@ package routewatch
|
|||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"errors"
|
"errors"
|
||||||
"fmt"
|
|
||||||
"path/filepath"
|
|
||||||
"sync"
|
"sync"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"git.eeqj.de/sneak/routewatch/internal/blocklist"
|
|
||||||
"git.eeqj.de/sneak/routewatch/internal/config"
|
"git.eeqj.de/sneak/routewatch/internal/config"
|
||||||
"git.eeqj.de/sneak/routewatch/internal/database"
|
"git.eeqj.de/sneak/routewatch/internal/database"
|
||||||
"git.eeqj.de/sneak/routewatch/internal/logger"
|
"git.eeqj.de/sneak/routewatch/internal/logger"
|
||||||
"git.eeqj.de/sneak/routewatch/internal/metrics"
|
"git.eeqj.de/sneak/routewatch/internal/metrics"
|
||||||
"git.eeqj.de/sneak/routewatch/internal/routetable"
|
|
||||||
"git.eeqj.de/sneak/routewatch/internal/server"
|
"git.eeqj.de/sneak/routewatch/internal/server"
|
||||||
"git.eeqj.de/sneak/routewatch/internal/streamer"
|
"git.eeqj.de/sneak/routewatch/internal/streamer"
|
||||||
|
|
||||||
@@ -56,13 +52,8 @@ type RouteWatch struct {
|
|||||||
peeringHandler *PeeringHandler
|
peeringHandler *PeeringHandler
|
||||||
asnFetcher *ASNFetcher
|
asnFetcher *ASNFetcher
|
||||||
dbMaintainer *DBMaintainer
|
dbMaintainer *DBMaintainer
|
||||||
blocklists *blocklist.Blocklists
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// blocklistsDirName is the directory in the state directory that holds the
|
|
||||||
// blocklist copies.
|
|
||||||
const blocklistsDirName = "blocklists"
|
|
||||||
|
|
||||||
// New creates a new RouteWatch instance
|
// New creates a new RouteWatch instance
|
||||||
func New(deps Dependencies) *RouteWatch {
|
func New(deps Dependencies) *RouteWatch {
|
||||||
rw := &RouteWatch{
|
rw := &RouteWatch{
|
||||||
@@ -94,14 +85,6 @@ func (rw *RouteWatch) Run(ctx context.Context) error {
|
|||||||
// Register database handler to process BGP UPDATE messages
|
// Register database handler to process BGP UPDATE messages
|
||||||
if rw.config.EnableBatchedDatabaseWrites {
|
if rw.config.EnableBatchedDatabaseWrites {
|
||||||
rw.logger.Info("Using batched database handlers for improved performance")
|
rw.logger.Info("Using batched database handlers for improved performance")
|
||||||
|
|
||||||
// The routing table is read before the streamer connects, so that
|
|
||||||
// the prefix handler applies the feed's updates to all of it.
|
|
||||||
table, err := rw.readRouteTable(ctx)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
// ASHandler maintains the asns table
|
// ASHandler maintains the asns table
|
||||||
rw.dbHandler = NewASHandler(rw.db, rw.logger)
|
rw.dbHandler = NewASHandler(rw.db, rw.logger)
|
||||||
rw.streamer.RegisterHandler(rw.dbHandler)
|
rw.streamer.RegisterHandler(rw.dbHandler)
|
||||||
@@ -110,9 +93,8 @@ func (rw *RouteWatch) Run(ctx context.Context) error {
|
|||||||
rw.peerHandler = NewPeerHandler(rw.db, rw.logger)
|
rw.peerHandler = NewPeerHandler(rw.db, rw.logger)
|
||||||
rw.streamer.RegisterHandler(rw.peerHandler)
|
rw.streamer.RegisterHandler(rw.peerHandler)
|
||||||
|
|
||||||
// PrefixHandler maintains the prefixes and live_routes tables, and
|
// PrefixHandler maintains the prefixes and live_routes tables
|
||||||
// the routing table in memory and its copy in the routes table
|
rw.prefixHandler = NewPrefixHandler(rw.db, rw.logger)
|
||||||
rw.prefixHandler = NewPrefixHandler(rw.db, rw.logger, table)
|
|
||||||
rw.prefixHandler.SetMetricsTracker(rw.streamer.GetMetricsTracker())
|
rw.prefixHandler.SetMetricsTracker(rw.streamer.GetMetricsTracker())
|
||||||
rw.streamer.RegisterHandler(rw.prefixHandler)
|
rw.streamer.RegisterHandler(rw.prefixHandler)
|
||||||
|
|
||||||
@@ -134,30 +116,17 @@ func (rw *RouteWatch) Run(ctx context.Context) error {
|
|||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
// Start ASN WHOIS fetcher for background updates. The HTTP handlers read
|
|
||||||
// the server's fetcher without a lock, so it is set before they run.
|
|
||||||
rw.asnFetcher = NewASNFetcher(rw.db, rw.logger.Logger)
|
|
||||||
rw.asnFetcher.Start(ctx)
|
|
||||||
rw.server.SetASNFetcher(rw.asnFetcher)
|
|
||||||
|
|
||||||
// Load the blocklist copies from disk and start downloading the lists.
|
|
||||||
// The HTTP handlers read the server's blocklists without a lock, so they
|
|
||||||
// are set before the handlers run.
|
|
||||||
blocklists := blocklist.New(blocklist.Params{
|
|
||||||
Dir: filepath.Join(rw.config.StateDir, blocklistsDirName),
|
|
||||||
Lists: blocklist.Lists(),
|
|
||||||
Logger: rw.logger.Logger,
|
|
||||||
})
|
|
||||||
blocklists.Start(ctx)
|
|
||||||
rw.blocklists = blocklists
|
|
||||||
rw.server.SetBlocklists(blocklists)
|
|
||||||
|
|
||||||
// Start HTTP server
|
// Start HTTP server
|
||||||
err = rw.server.Start(ctx)
|
err = rw.server.Start(ctx)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Start ASN WHOIS fetcher for background updates
|
||||||
|
rw.asnFetcher = NewASNFetcher(rw.db, rw.logger.Logger)
|
||||||
|
rw.asnFetcher.Start(ctx)
|
||||||
|
rw.server.SetASNFetcher(rw.asnFetcher)
|
||||||
|
|
||||||
// Start database maintenance goroutine
|
// Start database maintenance goroutine
|
||||||
rw.dbMaintainer = NewDBMaintainer(rw.db, rw.logger.Logger)
|
rw.dbMaintainer = NewDBMaintainer(rw.db, rw.logger.Logger)
|
||||||
rw.dbMaintainer.Start(ctx)
|
rw.dbMaintainer.Start(ctx)
|
||||||
@@ -168,13 +137,7 @@ func (rw *RouteWatch) Run(ctx context.Context) error {
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// Shutdown stops all services in order: the streamer, the handlers, which
|
// Shutdown performs graceful shutdown of all services
|
||||||
// write what they hold, the WHOIS fetcher, the database maintainer, the
|
|
||||||
// blocklist downloads and the HTTP server, and last the database, which it
|
|
||||||
// checkpoints and closes. Each stage waits at most until ctx ends, so
|
|
||||||
// Shutdown returns soon after: a handler logs what it could not write by
|
|
||||||
// then, and the HTTP server closes the connections of requests still
|
|
||||||
// running.
|
|
||||||
func (rw *RouteWatch) Shutdown(ctx context.Context) {
|
func (rw *RouteWatch) Shutdown(ctx context.Context) {
|
||||||
rw.mu.Lock()
|
rw.mu.Lock()
|
||||||
if rw.shutdown {
|
if rw.shutdown {
|
||||||
@@ -187,52 +150,51 @@ func (rw *RouteWatch) Shutdown(ctx context.Context) {
|
|||||||
rw.mu.Unlock()
|
rw.mu.Unlock()
|
||||||
|
|
||||||
// Stop the streamer first: it returns once the handlers have been handed
|
// Stop the streamer first: it returns once the handlers have been handed
|
||||||
// every message it queued, or when ctx ends. Then each handler writes
|
// every message it queued. Then each handler writes what it holds.
|
||||||
// what it holds.
|
rw.streamer.Stop()
|
||||||
rw.streamer.Stop(ctx)
|
|
||||||
|
|
||||||
if rw.dbHandler != nil {
|
if rw.dbHandler != nil {
|
||||||
rw.logger.Info("Flushing database handler")
|
rw.logger.Info("Flushing database handler")
|
||||||
rw.dbHandler.Stop(ctx)
|
rw.dbHandler.Stop()
|
||||||
}
|
}
|
||||||
|
|
||||||
if rw.peerHandler != nil {
|
if rw.peerHandler != nil {
|
||||||
rw.logger.Info("Flushing peer handler")
|
rw.logger.Info("Flushing peer handler")
|
||||||
rw.peerHandler.Stop(ctx)
|
rw.peerHandler.Stop()
|
||||||
}
|
}
|
||||||
|
|
||||||
if rw.prefixHandler != nil {
|
if rw.prefixHandler != nil {
|
||||||
rw.logger.Info("Flushing prefix handler")
|
rw.logger.Info("Flushing prefix handler")
|
||||||
rw.prefixHandler.Stop(ctx)
|
rw.prefixHandler.Stop()
|
||||||
}
|
}
|
||||||
|
|
||||||
if rw.peeringHandler != nil {
|
if rw.peeringHandler != nil {
|
||||||
rw.logger.Info("Flushing peering handler")
|
rw.logger.Info("Flushing peering handler")
|
||||||
rw.peeringHandler.Stop(ctx)
|
rw.peeringHandler.Stop()
|
||||||
}
|
}
|
||||||
|
|
||||||
// Stop ASN WHOIS fetcher
|
// Stop ASN WHOIS fetcher
|
||||||
if rw.asnFetcher != nil {
|
if rw.asnFetcher != nil {
|
||||||
rw.asnFetcher.Stop(ctx)
|
rw.asnFetcher.Stop()
|
||||||
}
|
}
|
||||||
|
|
||||||
// Stop database maintainer
|
// Stop database maintainer
|
||||||
if rw.dbMaintainer != nil {
|
if rw.dbMaintainer != nil {
|
||||||
rw.dbMaintainer.Stop(ctx)
|
rw.dbMaintainer.Stop()
|
||||||
}
|
}
|
||||||
|
|
||||||
if rw.blocklists != nil {
|
// Stop HTTP server with a timeout of its own, however much of ctx is left
|
||||||
rw.blocklists.Stop(ctx)
|
const serverStopTimeout = 5 * time.Second
|
||||||
}
|
|
||||||
|
|
||||||
err := rw.server.Stop(ctx)
|
stopCtx, cancel := context.WithTimeout(
|
||||||
|
context.WithoutCancel(ctx), serverStopTimeout)
|
||||||
|
defer cancel()
|
||||||
|
|
||||||
|
err := rw.server.Stop(stopCtx)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
rw.logger.Error("Stopped the HTTP server before every request finished",
|
rw.logger.Error("Failed to stop HTTP server gracefully", "error", err)
|
||||||
"error", err)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
rw.closeDatabase(ctx)
|
|
||||||
|
|
||||||
// Log final metrics
|
// Log final metrics
|
||||||
metrics := rw.streamer.GetMetrics()
|
metrics := rw.streamer.GetMetrics()
|
||||||
rw.logger.Info("Final metrics",
|
rw.logger.Info("Final metrics",
|
||||||
@@ -244,48 +206,6 @@ func (rw *RouteWatch) Shutdown(ctx context.Context) {
|
|||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
// closeDatabase checkpoints and closes the database, unless ctx has ended: a
|
|
||||||
// write may then still be running, and both would wait for it. SQLite rolls
|
|
||||||
// back a write cut off by the exit when the database next opens.
|
|
||||||
func (rw *RouteWatch) closeDatabase(ctx context.Context) {
|
|
||||||
if ctx.Err() != nil {
|
|
||||||
rw.logger.Error("Stop deadline passed; the database is not closed")
|
|
||||||
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
err := rw.db.Checkpoint(ctx)
|
|
||||||
if err != nil {
|
|
||||||
rw.logger.Error("Failed to checkpoint the database", "error", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
err = rw.db.Close()
|
|
||||||
if err != nil {
|
|
||||||
rw.logger.Error("Failed to close the database", "error", err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// readRouteTable reads the routing table from the database and logs how many
|
|
||||||
// prefix and origin AS pairs and peers it holds and how long the read took.
|
|
||||||
// ctx is fx's start context, which ends 15 seconds after the start. Reading a
|
|
||||||
// full table can take longer, so the read does not stop with ctx.
|
|
||||||
func (rw *RouteWatch) readRouteTable(
|
|
||||||
ctx context.Context,
|
|
||||||
) (*routetable.Table, error) {
|
|
||||||
start := time.Now()
|
|
||||||
|
|
||||||
table, err := rw.db.ReadRouteTable(context.WithoutCancel(ctx))
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("failed to read the routing table: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
counts := table.Counts()
|
|
||||||
rw.logger.Info("Read the routing table", "pairs", counts.Pairs,
|
|
||||||
"peers", counts.Peers, "duration_ms", time.Since(start).Milliseconds())
|
|
||||||
|
|
||||||
return table, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// getModule provides all fx dependencies
|
// getModule provides all fx dependencies
|
||||||
func getModule() []fx.Option {
|
func getModule() []fx.Option {
|
||||||
return []fx.Option{
|
return []fx.Option{
|
||||||
|
|||||||
@@ -4,11 +4,8 @@ import (
|
|||||||
"context"
|
"context"
|
||||||
"fmt"
|
"fmt"
|
||||||
"io"
|
"io"
|
||||||
"net"
|
|
||||||
"net/http"
|
"net/http"
|
||||||
"net/http/httptest"
|
"net/http/httptest"
|
||||||
"slices"
|
|
||||||
"strconv"
|
|
||||||
"strings"
|
"strings"
|
||||||
"sync"
|
"sync"
|
||||||
"testing"
|
"testing"
|
||||||
@@ -19,7 +16,6 @@ import (
|
|||||||
"git.eeqj.de/sneak/routewatch/internal/logger"
|
"git.eeqj.de/sneak/routewatch/internal/logger"
|
||||||
"git.eeqj.de/sneak/routewatch/internal/metrics"
|
"git.eeqj.de/sneak/routewatch/internal/metrics"
|
||||||
"git.eeqj.de/sneak/routewatch/internal/ristypes"
|
"git.eeqj.de/sneak/routewatch/internal/ristypes"
|
||||||
"git.eeqj.de/sneak/routewatch/internal/routetable"
|
|
||||||
"git.eeqj.de/sneak/routewatch/internal/server"
|
"git.eeqj.de/sneak/routewatch/internal/server"
|
||||||
"git.eeqj.de/sneak/routewatch/internal/streamer"
|
"git.eeqj.de/sneak/routewatch/internal/streamer"
|
||||||
)
|
)
|
||||||
@@ -40,9 +36,6 @@ type mockStore struct {
|
|||||||
Prefixes map[string]*database.Prefix
|
Prefixes map[string]*database.Prefix
|
||||||
Peerings map[string]bool // key is "from_to"
|
Peerings map[string]bool // key is "from_to"
|
||||||
Routes map[string]bool // key is "prefix_origin_peer"
|
Routes map[string]bool // key is "prefix_origin_peer"
|
||||||
// TablePrefixes holds the prefixes that the written changes of the
|
|
||||||
// routing table leave some peer carrying.
|
|
||||||
TablePrefixes map[string]bool
|
|
||||||
|
|
||||||
// Track IP versions
|
// Track IP versions
|
||||||
IPv4Prefixes int
|
IPv4Prefixes int
|
||||||
@@ -52,11 +45,10 @@ type mockStore struct {
|
|||||||
// newMockStore creates a new mock store
|
// newMockStore creates a new mock store
|
||||||
func newMockStore() *mockStore {
|
func newMockStore() *mockStore {
|
||||||
return &mockStore{
|
return &mockStore{
|
||||||
ASNs: make(map[int]*database.ASN),
|
ASNs: make(map[int]*database.ASN),
|
||||||
Prefixes: make(map[string]*database.Prefix),
|
Prefixes: make(map[string]*database.Prefix),
|
||||||
Peerings: make(map[string]bool),
|
Peerings: make(map[string]bool),
|
||||||
Routes: make(map[string]bool),
|
Routes: make(map[string]bool),
|
||||||
TablePrefixes: make(map[string]bool),
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -157,7 +149,7 @@ func (m *mockStore) RecordAnnouncement(_ *database.Announcement) error {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// RecordPeering mock implementation
|
// RecordPeering mock implementation
|
||||||
func (m *mockStore) RecordPeering(asA, asB int, _ database.SeenTimes) error {
|
func (m *mockStore) RecordPeering(asA, asB int, _ time.Time) error {
|
||||||
m.mu.Lock()
|
m.mu.Lock()
|
||||||
defer m.mu.Unlock()
|
defer m.mu.Unlock()
|
||||||
|
|
||||||
@@ -400,41 +392,17 @@ func (m *mockStore) DeleteLiveRoutesOfPeer(_ string) (int, error) {
|
|||||||
return 0, nil
|
return 0, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// WriteRouteChanges records the prefixes the changes leave some peer carrying,
|
|
||||||
// whatever their origin AS.
|
|
||||||
func (m *mockStore) WriteRouteChanges(
|
|
||||||
changes []routetable.Change, _ map[string]uint,
|
|
||||||
) error {
|
|
||||||
m.mu.Lock()
|
|
||||||
defer m.mu.Unlock()
|
|
||||||
|
|
||||||
for _, change := range changes {
|
|
||||||
if change.Removed {
|
|
||||||
delete(m.TablePrefixes, change.Prefix.String())
|
|
||||||
} else {
|
|
||||||
m.TablePrefixes[change.Prefix.String()] = true
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// ReadRouteTable returns an empty routing table.
|
|
||||||
func (m *mockStore) ReadRouteTable(_ context.Context) (*routetable.Table, error) {
|
|
||||||
return routetable.New(routetable.Params{}), nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// GetOrCreateASNBatch mock implementation
|
// GetOrCreateASNBatch mock implementation
|
||||||
func (m *mockStore) GetOrCreateASNBatch(asns map[int]database.SeenTimes) error {
|
func (m *mockStore) GetOrCreateASNBatch(asns map[int]time.Time) error {
|
||||||
m.mu.Lock()
|
m.mu.Lock()
|
||||||
defer m.mu.Unlock()
|
defer m.mu.Unlock()
|
||||||
|
|
||||||
for number, seen := range asns {
|
for number, timestamp := range asns {
|
||||||
if _, exists := m.ASNs[number]; !exists {
|
if _, exists := m.ASNs[number]; !exists {
|
||||||
m.ASNs[number] = &database.ASN{
|
m.ASNs[number] = &database.ASN{
|
||||||
ASN: number,
|
ASN: number,
|
||||||
FirstSeen: seen.First,
|
FirstSeen: timestamp,
|
||||||
LastSeen: seen.Last,
|
LastSeen: timestamp,
|
||||||
}
|
}
|
||||||
m.ASNCount++
|
m.ASNCount++
|
||||||
}
|
}
|
||||||
@@ -495,9 +463,9 @@ func TestRouteWatchLiveFeed(t *testing.T) {
|
|||||||
// Create streamer
|
// Create streamer
|
||||||
s := streamer.New(logger, metricsTracker)
|
s := streamer.New(logger, metricsTracker)
|
||||||
|
|
||||||
// The state directory holds the blocklist copies Run downloads.
|
// Create test config with empty state dir (no snapshot loading)
|
||||||
cfg := &config.Config{
|
cfg := &config.Config{
|
||||||
StateDir: t.TempDir(),
|
StateDir: "",
|
||||||
MaxRuntime: 5 * time.Second,
|
MaxRuntime: 5 * time.Second,
|
||||||
EnableBatchedDatabaseWrites: true,
|
EnableBatchedDatabaseWrites: true,
|
||||||
}
|
}
|
||||||
@@ -595,8 +563,7 @@ func announcementFeed(prefixes []string) string {
|
|||||||
// handler's worker waits on the first one until the server sees the streamer
|
// handler's worker waits on the first one until the server sees the streamer
|
||||||
// close the connection, which only Shutdown does, so all three are still in
|
// close the connection, which only Shutdown does, so all three are still in
|
||||||
// the streamer when Shutdown starts. Each prefix must be in the database when
|
// the streamer when Shutdown starts. Each prefix must be in the database when
|
||||||
// Shutdown returns, both in the prefix tables and in the routing table's
|
// Shutdown returns.
|
||||||
// writes.
|
|
||||||
func TestShutdownWritesTheUpdatesQueuedInTheStreamer(t *testing.T) {
|
func TestShutdownWritesTheUpdatesQueuedInTheStreamer(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
@@ -630,7 +597,7 @@ func TestShutdownWritesTheUpdatesQueuedInTheStreamer(t *testing.T) {
|
|||||||
Logger: log,
|
Logger: log,
|
||||||
Config: cfg,
|
Config: cfg,
|
||||||
})
|
})
|
||||||
rw.prefixHandler = NewPrefixHandler(store, log, routetable.New(routetable.Params{}))
|
rw.prefixHandler = NewPrefixHandler(store, log)
|
||||||
s.RegisterHandler(gatedPrefixHandler{
|
s.RegisterHandler(gatedPrefixHandler{
|
||||||
PrefixHandler: rw.prefixHandler, release: release,
|
PrefixHandler: rw.prefixHandler, release: release,
|
||||||
})
|
})
|
||||||
@@ -670,197 +637,9 @@ func TestShutdownWritesTheUpdatesQueuedInTheStreamer(t *testing.T) {
|
|||||||
defer store.mu.Unlock()
|
defer store.mu.Unlock()
|
||||||
|
|
||||||
for _, prefix := range prefixes {
|
for _, prefix := range prefixes {
|
||||||
_, inPrefixTables := store.Prefixes[prefix]
|
if _, ok := store.Prefixes[prefix]; !ok {
|
||||||
if !inPrefixTables || !store.TablePrefixes[prefix] {
|
t.Errorf("%s was queued in the streamer when Shutdown started "+
|
||||||
t.Errorf("%s was queued in the streamer when Shutdown started; "+
|
"but is not in the database", prefix)
|
||||||
"written to the prefix tables %t, in the routing table's "+
|
|
||||||
"writes %t", prefix, inPrefixTables, store.TablePrefixes[prefix])
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// freePort returns a TCP port that was free a moment ago, for a server the
|
|
||||||
// test starts.
|
|
||||||
func freePort(t *testing.T) int {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
var listenConfig net.ListenConfig
|
|
||||||
|
|
||||||
listener, err := listenConfig.Listen(t.Context(), "tcp", ":0")
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("failed to find a free port: %v", err)
|
|
||||||
}
|
|
||||||
defer func() { _ = listener.Close() }()
|
|
||||||
|
|
||||||
addr, ok := listener.Addr().(*net.TCPAddr)
|
|
||||||
if !ok {
|
|
||||||
t.Fatalf("listener address %v is not a TCP address", listener.Addr())
|
|
||||||
}
|
|
||||||
|
|
||||||
return addr.Port
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestShutdownReturnsByItsDeadlineWhileAWriteIsBlocked starts the HTTP server
|
|
||||||
// and has the AS handler start a write that the store holds until the test
|
|
||||||
// ends, past Shutdown's deadline. Shutdown must return within its deadline
|
|
||||||
// plus a small margin, with the HTTP server stopped, and must neither
|
|
||||||
// checkpoint nor close the database: the real one would wait for the write.
|
|
||||||
// Before the fix Shutdown waited for the write.
|
|
||||||
func TestShutdownReturnsByItsDeadlineWhileAWriteIsBlocked(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
const (
|
|
||||||
deadline = 100 * time.Millisecond
|
|
||||||
// margin is how long Shutdown may take after its deadline, to stop
|
|
||||||
// the HTTP server.
|
|
||||||
margin = 2 * time.Second
|
|
||||||
)
|
|
||||||
|
|
||||||
blocking := newBlockingStore()
|
|
||||||
defer blocking.releaseWrites()
|
|
||||||
|
|
||||||
store := &closeOrderStore{Store: blocking}
|
|
||||||
log := logger.New()
|
|
||||||
port := freePort(t)
|
|
||||||
cfg := &config.Config{Port: port}
|
|
||||||
s := streamer.New(log, metrics.New())
|
|
||||||
|
|
||||||
rw := New(Dependencies{
|
|
||||||
DB: store,
|
|
||||||
Streamer: s,
|
|
||||||
Server: server.New(store, s, log, cfg),
|
|
||||||
Logger: log,
|
|
||||||
Config: cfg,
|
|
||||||
})
|
|
||||||
|
|
||||||
err := rw.server.Start(t.Context())
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("failed to start the HTTP server: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
rw.dbHandler = NewASHandler(store, log)
|
|
||||||
fill := ristypes.ASPath{Sequence: slices.Repeat([]int{testASNA}, asnBatchSize)}
|
|
||||||
startBlockedWrite(t, blocking, rw.dbHandler,
|
|
||||||
[]*ristypes.RISMessage{{Path: fill}})
|
|
||||||
|
|
||||||
ctx, cancel := context.WithTimeout(t.Context(), deadline)
|
|
||||||
defer cancel()
|
|
||||||
|
|
||||||
shutDown := make(chan struct{})
|
|
||||||
|
|
||||||
go func() {
|
|
||||||
rw.Shutdown(ctx)
|
|
||||||
close(shutDown)
|
|
||||||
}()
|
|
||||||
|
|
||||||
select {
|
|
||||||
case <-shutDown:
|
|
||||||
case <-time.After(deadline + margin):
|
|
||||||
t.Fatalf("Shutdown did not return within %s of its deadline while "+
|
|
||||||
"a write was blocked", margin)
|
|
||||||
}
|
|
||||||
|
|
||||||
var dialer net.Dialer
|
|
||||||
|
|
||||||
addr := net.JoinHostPort("127.0.0.1", strconv.Itoa(port))
|
|
||||||
|
|
||||||
conn, err := dialer.DialContext(t.Context(), "tcp", addr)
|
|
||||||
if err == nil {
|
|
||||||
_ = conn.Close()
|
|
||||||
|
|
||||||
t.Errorf("the HTTP server still accepts connections on %s after "+
|
|
||||||
"Shutdown returned", addr)
|
|
||||||
}
|
|
||||||
|
|
||||||
store.callsMu.Lock()
|
|
||||||
defer store.callsMu.Unlock()
|
|
||||||
|
|
||||||
if slices.Contains(store.calls, callCheckpoint) ||
|
|
||||||
slices.Contains(store.calls, callClose) {
|
|
||||||
t.Errorf("Shutdown made the calls %q past its deadline, want no %s "+
|
|
||||||
"and no %s", store.calls, callCheckpoint, callClose)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// closeOrderStore is a store that records, in order, the AS writes,
|
|
||||||
// checkpoints and closes it is given. It passes the AS writes on to Store.
|
|
||||||
type closeOrderStore struct {
|
|
||||||
database.Store
|
|
||||||
|
|
||||||
callsMu sync.Mutex
|
|
||||||
calls []string
|
|
||||||
}
|
|
||||||
|
|
||||||
// The calls closeOrderStore records.
|
|
||||||
const (
|
|
||||||
callWrite = "AS write"
|
|
||||||
callCheckpoint = "checkpoint"
|
|
||||||
callClose = "close"
|
|
||||||
)
|
|
||||||
|
|
||||||
func (s *closeOrderStore) GetOrCreateASNBatch(
|
|
||||||
asns map[int]database.SeenTimes,
|
|
||||||
) error {
|
|
||||||
s.record(callWrite)
|
|
||||||
|
|
||||||
return s.Store.GetOrCreateASNBatch(asns)
|
|
||||||
}
|
|
||||||
|
|
||||||
func (s *closeOrderStore) Checkpoint(_ context.Context) error {
|
|
||||||
s.record(callCheckpoint)
|
|
||||||
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (s *closeOrderStore) Close() error {
|
|
||||||
s.record(callClose)
|
|
||||||
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (s *closeOrderStore) record(call string) {
|
|
||||||
s.callsMu.Lock()
|
|
||||||
defer s.callsMu.Unlock()
|
|
||||||
|
|
||||||
s.calls = append(s.calls, call)
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestShutdownCheckpointsAndClosesTheDatabaseLast hands the AS handler one
|
|
||||||
// message and shuts down with time to spare. The handler's last write must
|
|
||||||
// come first, then a checkpoint, then the close. Before the fix Shutdown
|
|
||||||
// neither checkpointed nor closed the database.
|
|
||||||
func TestShutdownCheckpointsAndClosesTheDatabaseLast(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
const deadline = 10 * time.Second
|
|
||||||
|
|
||||||
store := &closeOrderStore{Store: newMockStore()}
|
|
||||||
log := logger.New()
|
|
||||||
cfg := &config.Config{}
|
|
||||||
s := streamer.New(log, metrics.New())
|
|
||||||
|
|
||||||
rw := New(Dependencies{
|
|
||||||
DB: store,
|
|
||||||
Streamer: s,
|
|
||||||
Server: server.New(store, s, log, cfg),
|
|
||||||
Logger: log,
|
|
||||||
Config: cfg,
|
|
||||||
})
|
|
||||||
rw.dbHandler = NewASHandler(store, log)
|
|
||||||
rw.dbHandler.HandleMessage(&ristypes.RISMessage{
|
|
||||||
Path: ristypes.ASPath{Sequence: []int{testASNA}},
|
|
||||||
})
|
|
||||||
|
|
||||||
ctx, cancel := context.WithTimeout(t.Context(), deadline)
|
|
||||||
defer cancel()
|
|
||||||
|
|
||||||
rw.Shutdown(ctx)
|
|
||||||
|
|
||||||
store.callsMu.Lock()
|
|
||||||
defer store.callsMu.Unlock()
|
|
||||||
|
|
||||||
want := []string{callWrite, callCheckpoint, callClose}
|
|
||||||
if !slices.Equal(store.calls, want) {
|
|
||||||
t.Errorf("Shutdown made the calls %q, want %q", store.calls, want)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|||||||
@@ -1,7 +1,6 @@
|
|||||||
package routewatch
|
package routewatch
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
|
||||||
"slices"
|
"slices"
|
||||||
"sync"
|
"sync"
|
||||||
"time"
|
"time"
|
||||||
@@ -40,23 +39,18 @@ type ASHandler struct {
|
|||||||
db database.Store
|
db database.Store
|
||||||
logger *logger.Logger
|
logger *logger.Logger
|
||||||
|
|
||||||
// Batching. mu guards batch and writing only: the flush goroutine takes
|
// Batching. mu guards batch only: the flush goroutine takes the batch
|
||||||
// the batch under mu and writes it after releasing mu, so HandleMessage
|
// under mu and writes it after releasing mu, so HandleMessage waits for
|
||||||
// waits for a database write only while batch is at asnBatchMaxSize.
|
// a database write only while batch is at asnBatchMaxSize.
|
||||||
mu sync.Mutex
|
mu sync.Mutex
|
||||||
batch []asnOp
|
batch []asnOp
|
||||||
// writing is how many operations the flush goroutine took from batch
|
|
||||||
// and has not finished writing.
|
|
||||||
writing int
|
|
||||||
// batchTaken is signalled when the flush goroutine takes batch; its L is
|
// batchTaken is signalled when the flush goroutine takes batch; its L is
|
||||||
// &mu.
|
// &mu.
|
||||||
batchTaken sync.Cond
|
batchTaken sync.Cond
|
||||||
batchFullCh chan struct{} // wakes the flush goroutine when batch is full
|
batchFullCh chan struct{} // wakes the flush goroutine when batch is full
|
||||||
lastFlush time.Time // used only by the flush goroutine
|
lastFlush time.Time // used only by the flush goroutine
|
||||||
// stopCh hands Stop's context to the flush goroutine. It holds one, so
|
stopCh chan struct{}
|
||||||
// Stop does not wait for a write in progress to hand it over.
|
wg sync.WaitGroup
|
||||||
stopCh chan context.Context
|
|
||||||
done chan struct{} // closed when the flush goroutine returns
|
|
||||||
}
|
}
|
||||||
|
|
||||||
type asnOp struct {
|
type asnOp struct {
|
||||||
@@ -76,12 +70,12 @@ func NewASHandler(db database.Store, logger *logger.Logger) *ASHandler {
|
|||||||
batch: make([]asnOp, 0, asnBatchSize),
|
batch: make([]asnOp, 0, asnBatchSize),
|
||||||
batchFullCh: make(chan struct{}, 1),
|
batchFullCh: make(chan struct{}, 1),
|
||||||
lastFlush: time.Now(),
|
lastFlush: time.Now(),
|
||||||
stopCh: make(chan context.Context, 1),
|
stopCh: make(chan struct{}),
|
||||||
done: make(chan struct{}),
|
|
||||||
}
|
}
|
||||||
h.batchTaken.L = &h.mu
|
h.batchTaken.L = &h.mu
|
||||||
|
|
||||||
// Start the flush timer goroutine
|
// Start the flush timer goroutine
|
||||||
|
h.wg.Add(1)
|
||||||
go h.flushLoop()
|
go h.flushLoop()
|
||||||
|
|
||||||
return h
|
return h
|
||||||
@@ -153,32 +147,20 @@ func (h *ASHandler) HandleMessage(msg *ristypes.RISMessage) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Stop stops the flush goroutine, which writes the operations the handler holds
|
// Stop gracefully shuts down the ASHandler by signaling the background flush
|
||||||
// before it returns, and waits for it until ctx ends. Once ctx has ended the
|
// goroutine to terminate and waiting for it to complete. Any pending ASN
|
||||||
// flush goroutine skips that last write, and a write still running is lost
|
// operations in the current batch are flushed to the database before Stop
|
||||||
// when the process exits, so Stop logs how many operations are not written.
|
// returns. This method should be called during application shutdown to ensure
|
||||||
func (h *ASHandler) Stop(ctx context.Context) {
|
// no data is lost.
|
||||||
h.stopCh <- ctx
|
func (h *ASHandler) Stop() {
|
||||||
|
close(h.stopCh)
|
||||||
select {
|
h.wg.Wait()
|
||||||
case <-h.done:
|
|
||||||
case <-ctx.Done():
|
|
||||||
}
|
|
||||||
|
|
||||||
h.mu.Lock()
|
|
||||||
unwritten := h.writing + len(h.batch)
|
|
||||||
h.mu.Unlock()
|
|
||||||
|
|
||||||
if unwritten > 0 {
|
|
||||||
h.logger.Error("Stop deadline passed before the AS handler wrote "+
|
|
||||||
"its updates", "updates", unwritten)
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// flushLoop runs in a goroutine, the only one that writes the batch: it
|
// flushLoop runs in a goroutine, the only one that writes the batch: it
|
||||||
// flushes when the batch is full, periodically, and once more on Stop.
|
// flushes when the batch is full, periodically, and once more on Stop.
|
||||||
func (h *ASHandler) flushLoop() {
|
func (h *ASHandler) flushLoop() {
|
||||||
defer close(h.done)
|
defer h.wg.Done()
|
||||||
|
|
||||||
ticker := time.NewTicker(asnBatchTimeout)
|
ticker := time.NewTicker(asnBatchTimeout)
|
||||||
defer ticker.Stop()
|
defer ticker.Stop()
|
||||||
@@ -191,11 +173,9 @@ func (h *ASHandler) flushLoop() {
|
|||||||
}
|
}
|
||||||
case <-h.batchFullCh:
|
case <-h.batchFullCh:
|
||||||
h.flushBatch()
|
h.flushBatch()
|
||||||
case ctx := <-h.stopCh:
|
case <-h.stopCh:
|
||||||
// Final flush, unless the stop deadline has passed
|
// Final flush
|
||||||
if ctx.Err() == nil {
|
h.flushBatch()
|
||||||
h.flushBatch()
|
|
||||||
}
|
|
||||||
|
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
@@ -203,8 +183,7 @@ func (h *ASHandler) flushLoop() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// takeBatch returns the pending operations and leaves an empty batch in their
|
// takeBatch returns the pending operations and leaves an empty batch in their
|
||||||
// place for HandleMessage to fill. They count in writing until flushBatch has
|
// place for HandleMessage to fill.
|
||||||
// written them.
|
|
||||||
func (h *ASHandler) takeBatch() []asnOp {
|
func (h *ASHandler) takeBatch() []asnOp {
|
||||||
h.mu.Lock()
|
h.mu.Lock()
|
||||||
defer h.mu.Unlock()
|
defer h.mu.Unlock()
|
||||||
@@ -214,7 +193,6 @@ func (h *ASHandler) takeBatch() []asnOp {
|
|||||||
}
|
}
|
||||||
|
|
||||||
batch := h.batch
|
batch := h.batch
|
||||||
h.writing = len(batch)
|
|
||||||
h.batch = make([]asnOp, 0, asnBatchSize)
|
h.batch = make([]asnOp, 0, asnBatchSize)
|
||||||
h.batchTaken.Broadcast()
|
h.batchTaken.Broadcast()
|
||||||
|
|
||||||
@@ -229,26 +207,20 @@ func (h *ASHandler) flushBatch() {
|
|||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
// The earliest and the latest time the batch saw each AS. The batch holds
|
// Process ASNs first (deduped)
|
||||||
// several updates for an AS, from many peers, so a new AS is stored first
|
asnMap := make(map[int]time.Time)
|
||||||
// seen at the earliest of them and last seen at the latest.
|
|
||||||
asns := make(map[int]database.SeenTimes)
|
|
||||||
for _, op := range batch {
|
for _, op := range batch {
|
||||||
seen := asns[op.number]
|
if existing, ok := asnMap[op.number]; !ok || op.timestamp.After(existing) {
|
||||||
seen.Add(op.timestamp)
|
asnMap[op.number] = op.timestamp
|
||||||
asns[op.number] = seen
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Process all ASNs in a single batch transaction
|
// Process all ASNs in a single batch transaction
|
||||||
err := h.db.GetOrCreateASNBatch(asns)
|
err := h.db.GetOrCreateASNBatch(asnMap)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
h.logger.Error("Failed to process ASN batch",
|
h.logger.Error("Failed to process ASN batch",
|
||||||
"error", err, "count", len(asns))
|
"error", err, "count", len(asnMap))
|
||||||
}
|
}
|
||||||
|
|
||||||
h.mu.Lock()
|
|
||||||
h.writing = 0
|
|
||||||
h.mu.Unlock()
|
|
||||||
|
|
||||||
h.lastFlush = time.Now()
|
h.lastFlush = time.Now()
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,68 +0,0 @@
|
|||||||
package routewatch
|
|
||||||
|
|
||||||
import (
|
|
||||||
"maps"
|
|
||||||
"slices"
|
|
||||||
"testing"
|
|
||||||
|
|
||||||
"git.eeqj.de/sneak/routewatch/internal/logger"
|
|
||||||
)
|
|
||||||
|
|
||||||
// TestASHandlerStoresTheEarliestAndLatestTimesOfANewAS handles two updates
|
|
||||||
// whose path carries an AS the database does not hold yet, the second 5
|
|
||||||
// seconds after the first, in one batch into a real database. The AS is stored
|
|
||||||
// first seen at the time of the first update and last seen at the time of the
|
|
||||||
// second.
|
|
||||||
func TestASHandlerStoresTheEarliestAndLatestTimesOfANewAS(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
db := newTestDatabase(t)
|
|
||||||
|
|
||||||
// Built without NewASHandler's flush goroutine, so the test flushes itself.
|
|
||||||
h := &ASHandler{db: db, logger: logger.New()}
|
|
||||||
|
|
||||||
first := peerAnnouncement(testPeerIP, testV4Prefix)
|
|
||||||
second := peerAnnouncement(testPeerIP, testV4Prefix)
|
|
||||||
second.ParsedTimestamp = first.ParsedTimestamp.Add(secondUpdateAfter)
|
|
||||||
|
|
||||||
h.HandleMessage(first)
|
|
||||||
h.HandleMessage(second)
|
|
||||||
h.flushBatch()
|
|
||||||
|
|
||||||
asn, _, err := db.GetASDetails(testASNA)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("GetASDetails(%d): %v", testASNA, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
if !asn.FirstSeen.Equal(first.ParsedTimestamp) {
|
|
||||||
t.Errorf("first_seen of AS%d = %s, want %s", testASNA,
|
|
||||||
asn.FirstSeen, first.ParsedTimestamp)
|
|
||||||
}
|
|
||||||
|
|
||||||
if !asn.LastSeen.Equal(second.ParsedTimestamp) {
|
|
||||||
t.Errorf("last_seen of AS%d = %s, want %s", testASNA,
|
|
||||||
asn.LastSeen, second.ParsedTimestamp)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestASHandlerRecordsNothingFromAPathWithASZero handles an update whose AS
|
|
||||||
// path holds AS 0 between two other ASes. RFC 7607 makes such a path
|
|
||||||
// malformed, so neither AS is recorded.
|
|
||||||
func TestASHandlerRecordsNothingFromAPathWithASZero(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
store := newMockStore()
|
|
||||||
|
|
||||||
// Built without NewASHandler's flush goroutine, so the test flushes itself.
|
|
||||||
h := &ASHandler{db: store, logger: logger.New()}
|
|
||||||
|
|
||||||
msg := peerAnnouncement(testPeerIP, testV4Prefix)
|
|
||||||
msg.Path = decodePath(t, testASNA, 0, testASNB)
|
|
||||||
h.HandleMessage(msg)
|
|
||||||
h.flushBatch()
|
|
||||||
|
|
||||||
if len(store.ASNs) > 0 {
|
|
||||||
t.Errorf("ASes recorded = %v, want none",
|
|
||||||
slices.Sorted(maps.Keys(store.ASNs)))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -21,16 +21,10 @@ const (
|
|||||||
minInterval = 1 * time.Second
|
minInterval = 1 * time.Second
|
||||||
|
|
||||||
// maxInterval is the maximum interval after failures (backoff cap).
|
// maxInterval is the maximum interval after failures (backoff cap).
|
||||||
maxInterval = time.Minute
|
maxInterval = 5 * time.Minute
|
||||||
|
|
||||||
// A failure multiplies the interval by backoffNumerator divided by
|
// backoffMultiplier is how much to multiply interval on failure.
|
||||||
// backoffDenominator, 1.5, up to maxInterval.
|
backoffMultiplier = 2
|
||||||
backoffNumerator = 3
|
|
||||||
backoffDenominator = 2
|
|
||||||
|
|
||||||
// A success divides the interval by successDivisor, down to
|
|
||||||
// minInterval.
|
|
||||||
successDivisor = 2
|
|
||||||
|
|
||||||
// whoisStaleThreshold is how old WHOIS data can be before refresh.
|
// whoisStaleThreshold is how old WHOIS data can be before refresh.
|
||||||
whoisStaleThreshold = 30 * 24 * time.Hour // 30 days
|
whoisStaleThreshold = 30 * 24 * time.Hour // 30 days
|
||||||
@@ -49,7 +43,7 @@ type ASNFetcher struct {
|
|||||||
logger *slog.Logger
|
logger *slog.Logger
|
||||||
immediateQueue chan int
|
immediateQueue chan int
|
||||||
stopCh chan struct{}
|
stopCh chan struct{}
|
||||||
done chan struct{} // closed when the fetcher goroutine returns
|
wg sync.WaitGroup
|
||||||
|
|
||||||
// fetchMu ensures only one fetch runs at a time
|
// fetchMu ensures only one fetch runs at a time
|
||||||
fetchMu sync.Mutex
|
fetchMu sync.Mutex
|
||||||
@@ -59,8 +53,7 @@ type ASNFetcher struct {
|
|||||||
currentInterval time.Duration
|
currentInterval time.Duration
|
||||||
consecutiveFails int
|
consecutiveFails int
|
||||||
|
|
||||||
// hourly stats tracking; now is the clock, time.Now except in tests
|
// hourly stats tracking
|
||||||
now func() time.Time
|
|
||||||
statsMu sync.Mutex
|
statsMu sync.Mutex
|
||||||
successTimes []time.Time
|
successTimes []time.Time
|
||||||
errorTimes []time.Time
|
errorTimes []time.Time
|
||||||
@@ -68,17 +61,13 @@ type ASNFetcher struct {
|
|||||||
|
|
||||||
// NewASNFetcher creates a new ASN fetcher.
|
// NewASNFetcher creates a new ASN fetcher.
|
||||||
func NewASNFetcher(db database.Store, logger *slog.Logger) *ASNFetcher {
|
func NewASNFetcher(db database.Store, logger *slog.Logger) *ASNFetcher {
|
||||||
logger = logger.With("component", "asn_fetcher")
|
|
||||||
|
|
||||||
return &ASNFetcher{
|
return &ASNFetcher{
|
||||||
db: db,
|
db: db,
|
||||||
whoisClient: whois.NewClient(logger),
|
whoisClient: whois.NewClient(),
|
||||||
logger: logger,
|
logger: logger.With("component", "asn_fetcher"),
|
||||||
immediateQueue: make(chan int, immediateQueueSize),
|
immediateQueue: make(chan int, immediateQueueSize),
|
||||||
stopCh: make(chan struct{}),
|
stopCh: make(chan struct{}),
|
||||||
done: make(chan struct{}),
|
|
||||||
currentInterval: baseInterval,
|
currentInterval: baseInterval,
|
||||||
now: time.Now,
|
|
||||||
successTimes: make([]time.Time, 0),
|
successTimes: make([]time.Time, 0),
|
||||||
errorTimes: make([]time.Time, 0),
|
errorTimes: make([]time.Time, 0),
|
||||||
}
|
}
|
||||||
@@ -87,6 +76,8 @@ func NewASNFetcher(db database.Store, logger *slog.Logger) *ASNFetcher {
|
|||||||
// Start begins the background ASN fetcher goroutine. It runs until Stop:
|
// Start begins the background ASN fetcher goroutine. It runs until Stop:
|
||||||
// the end of ctx does not stop it.
|
// the end of ctx does not stop it.
|
||||||
func (f *ASNFetcher) Start(ctx context.Context) {
|
func (f *ASNFetcher) Start(ctx context.Context) {
|
||||||
|
f.wg.Add(1)
|
||||||
|
|
||||||
go f.run(context.WithoutCancel(ctx))
|
go f.run(context.WithoutCancel(ctx))
|
||||||
|
|
||||||
f.logger.Info("ASN fetcher started",
|
f.logger.Info("ASN fetcher started",
|
||||||
@@ -96,17 +87,11 @@ func (f *ASNFetcher) Start(ctx context.Context) {
|
|||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Stop stops the fetcher and waits, until ctx ends, for a lookup in progress to
|
// Stop gracefully shuts down the fetcher.
|
||||||
// finish.
|
func (f *ASNFetcher) Stop() {
|
||||||
func (f *ASNFetcher) Stop(ctx context.Context) {
|
|
||||||
close(f.stopCh)
|
close(f.stopCh)
|
||||||
|
f.wg.Wait()
|
||||||
select {
|
f.logger.Info("ASN fetcher stopped")
|
||||||
case <-f.done:
|
|
||||||
f.logger.Info("ASN fetcher stopped")
|
|
||||||
case <-ctx.Done():
|
|
||||||
f.logger.Warn("Stop deadline passed before the ASN fetcher stopped")
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// QueueImmediate queues an ASN for immediate WHOIS lookup.
|
// QueueImmediate queues an ASN for immediate WHOIS lookup.
|
||||||
@@ -131,7 +116,7 @@ func (f *ASNFetcher) GetStats() server.ASNFetcherStats {
|
|||||||
f.intervalMu.Unlock()
|
f.intervalMu.Unlock()
|
||||||
|
|
||||||
// Prune old entries and count
|
// Prune old entries and count
|
||||||
cutoff := f.now().Add(-statsWindow)
|
cutoff := time.Now().Add(-statsWindow)
|
||||||
f.successTimes = pruneOldTimes(f.successTimes, cutoff)
|
f.successTimes = pruneOldTimes(f.successTimes, cutoff)
|
||||||
f.errorTimes = pruneOldTimes(f.errorTimes, cutoff)
|
f.errorTimes = pruneOldTimes(f.errorTimes, cutoff)
|
||||||
|
|
||||||
@@ -143,15 +128,16 @@ func (f *ASNFetcher) GetStats() server.ASNFetcherStats {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// pruneOldTimes drops the times at or before cutoff from the front of times,
|
// pruneOldTimes removes times older than cutoff and returns the pruned slice.
|
||||||
// which holds times in the order they were recorded, and returns the rest. It
|
|
||||||
// copies nothing, so it is cheap enough to run on every append.
|
|
||||||
func pruneOldTimes(times []time.Time, cutoff time.Time) []time.Time {
|
func pruneOldTimes(times []time.Time, cutoff time.Time) []time.Time {
|
||||||
for len(times) > 0 && !times[0].After(cutoff) {
|
result := make([]time.Time, 0, len(times))
|
||||||
times = times[1:]
|
for _, t := range times {
|
||||||
|
if t.After(cutoff) {
|
||||||
|
result = append(result, t)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
return times
|
return result
|
||||||
}
|
}
|
||||||
|
|
||||||
// getInterval returns the current fetch interval.
|
// getInterval returns the current fetch interval.
|
||||||
@@ -168,7 +154,7 @@ func (f *ASNFetcher) recordSuccess() {
|
|||||||
f.consecutiveFails = 0
|
f.consecutiveFails = 0
|
||||||
|
|
||||||
// Decrease interval by half, but not below minimum
|
// Decrease interval by half, but not below minimum
|
||||||
newInterval := max(f.currentInterval/successDivisor, minInterval)
|
newInterval := max(f.currentInterval/backoffMultiplier, minInterval)
|
||||||
|
|
||||||
if newInterval != f.currentInterval {
|
if newInterval != f.currentInterval {
|
||||||
f.logger.Debug("Decreased fetch interval",
|
f.logger.Debug("Decreased fetch interval",
|
||||||
@@ -179,11 +165,9 @@ func (f *ASNFetcher) recordSuccess() {
|
|||||||
}
|
}
|
||||||
f.intervalMu.Unlock()
|
f.intervalMu.Unlock()
|
||||||
|
|
||||||
// Record success time for stats, dropping those older than statsWindow
|
// Record success time for stats
|
||||||
f.statsMu.Lock()
|
f.statsMu.Lock()
|
||||||
now := f.now()
|
f.successTimes = append(f.successTimes, time.Now())
|
||||||
f.successTimes = pruneOldTimes(f.successTimes, now.Add(-statsWindow))
|
|
||||||
f.successTimes = append(f.successTimes, now)
|
|
||||||
f.statsMu.Unlock()
|
f.statsMu.Unlock()
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -192,9 +176,8 @@ func (f *ASNFetcher) recordFailure() {
|
|||||||
f.intervalMu.Lock()
|
f.intervalMu.Lock()
|
||||||
f.consecutiveFails++
|
f.consecutiveFails++
|
||||||
|
|
||||||
// Exponential backoff: multiply by 1.5, capped at max
|
// Exponential backoff: multiply by 2, capped at max
|
||||||
newInterval := min(
|
newInterval := min(f.currentInterval*backoffMultiplier, maxInterval)
|
||||||
f.currentInterval*backoffNumerator/backoffDenominator, maxInterval)
|
|
||||||
|
|
||||||
if newInterval != f.currentInterval {
|
if newInterval != f.currentInterval {
|
||||||
f.logger.Debug("Increased fetch interval due to failure",
|
f.logger.Debug("Increased fetch interval due to failure",
|
||||||
@@ -206,17 +189,15 @@ func (f *ASNFetcher) recordFailure() {
|
|||||||
}
|
}
|
||||||
f.intervalMu.Unlock()
|
f.intervalMu.Unlock()
|
||||||
|
|
||||||
// Record error time for stats, dropping those older than statsWindow
|
// Record error time for stats
|
||||||
f.statsMu.Lock()
|
f.statsMu.Lock()
|
||||||
now := f.now()
|
f.errorTimes = append(f.errorTimes, time.Now())
|
||||||
f.errorTimes = pruneOldTimes(f.errorTimes, now.Add(-statsWindow))
|
|
||||||
f.errorTimes = append(f.errorTimes, now)
|
|
||||||
f.statsMu.Unlock()
|
f.statsMu.Unlock()
|
||||||
}
|
}
|
||||||
|
|
||||||
// run is the main background loop.
|
// run is the main background loop.
|
||||||
func (f *ASNFetcher) run(ctx context.Context) {
|
func (f *ASNFetcher) run(ctx context.Context) {
|
||||||
defer close(f.done)
|
defer f.wg.Done()
|
||||||
|
|
||||||
timer := time.NewTimer(f.getInterval())
|
timer := time.NewTimer(f.getInterval())
|
||||||
defer timer.Stop()
|
defer timer.Stop()
|
||||||
@@ -283,10 +264,7 @@ func (f *ASNFetcher) fetchNextStale(ctx context.Context) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// fetchAndUpdate performs a WHOIS lookup and updates the database.
|
// fetchAndUpdate performs a WHOIS lookup and updates the database.
|
||||||
// Returns true if successful. A failed lookup leaves the AS's row as it was,
|
// Returns true if successful.
|
||||||
// so the AS stays stale. An AS IANA refers to no RIR is stored with no
|
|
||||||
// WHOIS data, so it is not looked up again until it is stale; that counts
|
|
||||||
// as neither a success nor a failure.
|
|
||||||
func (f *ASNFetcher) fetchAndUpdate(ctx context.Context, asn int) bool {
|
func (f *ASNFetcher) fetchAndUpdate(ctx context.Context, asn int) bool {
|
||||||
ctx, cancel := context.WithTimeout(ctx, time.Minute)
|
ctx, cancel := context.WithTimeout(ctx, time.Minute)
|
||||||
defer cancel()
|
defer cancel()
|
||||||
@@ -294,19 +272,6 @@ func (f *ASNFetcher) fetchAndUpdate(ctx context.Context, asn int) bool {
|
|||||||
f.logger.Info("Fetching WHOIS data", "asn", asn)
|
f.logger.Info("Fetching WHOIS data", "asn", asn)
|
||||||
|
|
||||||
info, err := f.whoisClient.LookupASN(ctx, asn)
|
info, err := f.whoisClient.LookupASN(ctx, asn)
|
||||||
if errors.Is(err, whois.ErrNoReferral) {
|
|
||||||
f.logger.Info("IANA refers the AS to no RIR", "asn", asn)
|
|
||||||
|
|
||||||
err = f.db.UpdateASNWHOIS(ctx, &database.ASNWHOISUpdate{ASN: asn})
|
|
||||||
if err != nil {
|
|
||||||
f.logger.Error("Failed to update ASN WHOIS data",
|
|
||||||
"asn", asn, "error", err)
|
|
||||||
f.recordFailure()
|
|
||||||
}
|
|
||||||
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
f.logger.Error("WHOIS lookup failed", "asn", asn, "error", err)
|
f.logger.Error("WHOIS lookup failed", "asn", asn, "error", err)
|
||||||
f.recordFailure()
|
f.recordFailure()
|
||||||
|
|||||||
@@ -1,278 +0,0 @@
|
|||||||
package routewatch
|
|
||||||
|
|
||||||
import (
|
|
||||||
"bufio"
|
|
||||||
"context"
|
|
||||||
"errors"
|
|
||||||
"io"
|
|
||||||
"net"
|
|
||||||
"sync"
|
|
||||||
"syscall"
|
|
||||||
"testing"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"git.eeqj.de/sneak/routewatch/internal/database"
|
|
||||||
"git.eeqj.de/sneak/routewatch/internal/logger"
|
|
||||||
)
|
|
||||||
|
|
||||||
// The WHOIS servers a lookup asks: IANA first, then the RIR IANA refers it
|
|
||||||
// to.
|
|
||||||
const (
|
|
||||||
ianaWHOISServer = "whois.iana.org:43"
|
|
||||||
ripeWHOISServer = "whois.ripe.net:43"
|
|
||||||
)
|
|
||||||
|
|
||||||
// ianaRefersToRIPE is IANA's answer for an AS in a block it gave to RIPE.
|
|
||||||
const ianaRefersToRIPE = `refer: whois.ripe.net
|
|
||||||
|
|
||||||
as-block: 64496-64511
|
|
||||||
organisation: Assigned by RIPE NCC
|
|
||||||
changed: 2002-09-01
|
|
||||||
source: IANA
|
|
||||||
`
|
|
||||||
|
|
||||||
// ripeAccessDenied is RIPE's answer to a host it no longer serves.
|
|
||||||
const ripeAccessDenied = `% This is the RIPE Database query service.
|
|
||||||
%ERROR:201: access denied for 192.0.2.1
|
|
||||||
`
|
|
||||||
|
|
||||||
// ianaPrivateUseBlock is IANA's answer for a private-use AS: its block and
|
|
||||||
// no referral.
|
|
||||||
const ianaPrivateUseBlock = `as-block: 64512-65534
|
|
||||||
organisation: Reserved for Private Use
|
|
||||||
source: IANA
|
|
||||||
`
|
|
||||||
|
|
||||||
// newFetcherWithWHOISAnswers returns an ASN fetcher on a new database that
|
|
||||||
// holds testASNA, never looked up. The fetcher's WHOIS client connects to a
|
|
||||||
// fake server for each address in answers, which reads the query and sends
|
|
||||||
// answers[address]; connecting to any other address is refused.
|
|
||||||
func newFetcherWithWHOISAnswers(
|
|
||||||
t *testing.T, answers map[string]string,
|
|
||||||
) (*ASNFetcher, *database.Database) {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
db := newTestDatabase(t)
|
|
||||||
|
|
||||||
_, err := db.GetOrCreateASN(testASNA, time.Now())
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("GetOrCreateASN: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
var servers sync.WaitGroup
|
|
||||||
|
|
||||||
t.Cleanup(servers.Wait)
|
|
||||||
|
|
||||||
f := NewASNFetcher(db, logger.New().Logger)
|
|
||||||
f.whoisClient.DialContext = func(
|
|
||||||
_ context.Context, _, address string,
|
|
||||||
) (net.Conn, error) {
|
|
||||||
answer, ok := answers[address]
|
|
||||||
if !ok {
|
|
||||||
return nil, syscall.ECONNREFUSED
|
|
||||||
}
|
|
||||||
|
|
||||||
client, server := net.Pipe()
|
|
||||||
|
|
||||||
servers.Go(func() {
|
|
||||||
defer func() { _ = server.Close() }()
|
|
||||||
|
|
||||||
_, _ = bufio.NewReader(server).ReadString('\n')
|
|
||||||
_, _ = io.WriteString(server, answer)
|
|
||||||
})
|
|
||||||
|
|
||||||
return client, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
return f, db
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestFetchAndUpdateKeepsTheRowWhenTheLookupFails looks up an AS IANA refers
|
|
||||||
// to RIPE when RIPE's server refuses the connection, and when it answers only
|
|
||||||
// with an error. The AS must stay never looked up, and the fetcher must back
|
|
||||||
// off.
|
|
||||||
func TestFetchAndUpdateKeepsTheRowWhenTheLookupFails(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
for name, answers := range map[string]map[string]string{
|
|
||||||
"connection refused": {ianaWHOISServer: ianaRefersToRIPE},
|
|
||||||
"only an error": {
|
|
||||||
ianaWHOISServer: ianaRefersToRIPE,
|
|
||||||
ripeWHOISServer: ripeAccessDenied,
|
|
||||||
},
|
|
||||||
} {
|
|
||||||
t.Run(name, func(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
f, db := newFetcherWithWHOISAnswers(t, answers)
|
|
||||||
|
|
||||||
if f.fetchAndUpdate(t.Context(), testASNA) {
|
|
||||||
t.Error("fetchAndUpdate reported success")
|
|
||||||
}
|
|
||||||
|
|
||||||
stats, err := db.GetWHOISStats(t.Context(), whoisStaleThreshold)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("GetWHOISStats: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
if stats.NeverFetched != 1 {
|
|
||||||
t.Errorf("never looked up = %d, want 1: the failed lookup "+
|
|
||||||
"was stored", stats.NeverFetched)
|
|
||||||
}
|
|
||||||
|
|
||||||
want := baseInterval * backoffNumerator / backoffDenominator
|
|
||||||
|
|
||||||
got := f.GetStats()
|
|
||||||
if got.CurrentInterval != want || got.ConsecutiveFails != 1 {
|
|
||||||
t.Errorf("interval %v after %d failures, want %v after 1",
|
|
||||||
got.CurrentInterval, got.ConsecutiveFails, want)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestRecordStepsTheInterval has the fetcher fail from its base interval of
|
|
||||||
// 15 seconds, then succeed. Each failure makes the interval 1.5 times as
|
|
||||||
// long, up to one minute, and each success halves it, down to one second.
|
|
||||||
func TestRecordStepsTheInterval(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
f := NewASNFetcher(newMockStore(), logger.New().Logger)
|
|
||||||
|
|
||||||
for i, want := range []time.Duration{
|
|
||||||
22500 * time.Millisecond,
|
|
||||||
33750 * time.Millisecond,
|
|
||||||
50625 * time.Millisecond,
|
|
||||||
time.Minute,
|
|
||||||
time.Minute,
|
|
||||||
} {
|
|
||||||
f.recordFailure()
|
|
||||||
|
|
||||||
if got := f.getInterval(); got != want {
|
|
||||||
t.Errorf("interval after %d failures = %v, want %v",
|
|
||||||
i+1, got, want)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
for i, want := range []time.Duration{
|
|
||||||
30 * time.Second,
|
|
||||||
15 * time.Second,
|
|
||||||
7500 * time.Millisecond,
|
|
||||||
3750 * time.Millisecond,
|
|
||||||
1875 * time.Millisecond,
|
|
||||||
time.Second,
|
|
||||||
time.Second,
|
|
||||||
} {
|
|
||||||
f.recordSuccess()
|
|
||||||
|
|
||||||
if got := f.getInterval(); got != want {
|
|
||||||
t.Errorf("interval after %d successes = %v, want %v",
|
|
||||||
i+1, got, want)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestFetchAndUpdateMarksAnASWithoutReferral looks up a private-use AS, which
|
|
||||||
// IANA refers to no RIR. The AS must not be looked up again until it is
|
|
||||||
// stale, and the lookup counts as neither a success nor a failure.
|
|
||||||
func TestFetchAndUpdateMarksAnASWithoutReferral(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
f, db := newFetcherWithWHOISAnswers(t,
|
|
||||||
map[string]string{ianaWHOISServer: ianaPrivateUseBlock})
|
|
||||||
|
|
||||||
f.fetchAndUpdate(t.Context(), testASNA)
|
|
||||||
|
|
||||||
asn, err := db.GetNextStaleASN(t.Context(), whoisStaleThreshold)
|
|
||||||
if !errors.Is(err, database.ErrNoStaleASN) {
|
|
||||||
t.Errorf("GetNextStaleASN = %d, %v; want ErrNoStaleASN", asn, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
got := f.GetStats()
|
|
||||||
if got.CurrentInterval != baseInterval || got.SuccessesLastHour != 0 ||
|
|
||||||
got.ErrorsLastHour != 0 {
|
|
||||||
t.Errorf("interval %v, %d successes, %d errors; want %v, 0, 0",
|
|
||||||
got.CurrentInterval, got.SuccessesLastHour, got.ErrorsLastHour,
|
|
||||||
baseInterval)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestASNFetcherStopReturnsByItsDeadline starts a fetcher, has it look up an
|
|
||||||
// AS whose WHOIS server does not answer until the test ends, and stops it with
|
|
||||||
// a short deadline while that lookup is in progress. Stop must return all the
|
|
||||||
// same instead of waiting for the lookup.
|
|
||||||
func TestASNFetcherStopReturnsByItsDeadline(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
const stopDeadline = 100 * time.Millisecond
|
|
||||||
|
|
||||||
lookupStarted := make(chan struct{})
|
|
||||||
release := make(chan struct{})
|
|
||||||
|
|
||||||
f := NewASNFetcher(newMockStore(), logger.New().Logger)
|
|
||||||
f.whoisClient.DialContext = func(
|
|
||||||
context.Context, string, string,
|
|
||||||
) (net.Conn, error) {
|
|
||||||
close(lookupStarted)
|
|
||||||
<-release
|
|
||||||
|
|
||||||
return nil, syscall.ECONNREFUSED
|
|
||||||
}
|
|
||||||
|
|
||||||
f.Start(t.Context())
|
|
||||||
f.QueueImmediate(testASNA)
|
|
||||||
|
|
||||||
select {
|
|
||||||
case <-lookupStarted:
|
|
||||||
case <-time.After(handleMessageTimeout):
|
|
||||||
t.Fatal("the fetcher did not start the lookup")
|
|
||||||
}
|
|
||||||
|
|
||||||
ctx, cancel := context.WithTimeout(t.Context(), stopDeadline)
|
|
||||||
defer cancel()
|
|
||||||
|
|
||||||
if !returnsInTime(func() { f.Stop(ctx) }) {
|
|
||||||
t.Fatalf("Stop with a deadline of %s did not return while a lookup "+
|
|
||||||
"was in progress", stopDeadline)
|
|
||||||
}
|
|
||||||
|
|
||||||
close(release)
|
|
||||||
|
|
||||||
select {
|
|
||||||
case <-f.done:
|
|
||||||
case <-time.After(handleMessageTimeout):
|
|
||||||
t.Fatal("the fetcher goroutine did not return after the lookup ended")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestRecordKeepsOnlyTheStatsWindow records a success and a failure every
|
|
||||||
// second, by a clock the test sets, far longer than statsWindow, and never
|
|
||||||
// asks for the stats. The fetcher must keep only the times within the last
|
|
||||||
// statsWindow.
|
|
||||||
func TestRecordKeepsOnlyTheStatsWindow(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
const (
|
|
||||||
records = 100_000
|
|
||||||
step = time.Second
|
|
||||||
want = int(statsWindow / step)
|
|
||||||
)
|
|
||||||
|
|
||||||
clock := time.Now()
|
|
||||||
|
|
||||||
f := NewASNFetcher(newMockStore(), logger.New().Logger)
|
|
||||||
f.now = func() time.Time { return clock }
|
|
||||||
|
|
||||||
for range records {
|
|
||||||
clock = clock.Add(step)
|
|
||||||
|
|
||||||
f.recordSuccess()
|
|
||||||
f.recordFailure()
|
|
||||||
}
|
|
||||||
|
|
||||||
if len(f.successTimes) != want || len(f.errorTimes) != want {
|
|
||||||
t.Errorf("%d success and %d error times kept, want %d of each, "+
|
|
||||||
"those of the last %s", len(f.successTimes), len(f.errorTimes),
|
|
||||||
want, statsWindow)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,10 +1,6 @@
|
|||||||
package routewatch
|
package routewatch
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"bytes"
|
|
||||||
"context"
|
|
||||||
"encoding/json"
|
|
||||||
"log/slog"
|
|
||||||
"slices"
|
"slices"
|
||||||
"sync"
|
"sync"
|
||||||
"testing"
|
"testing"
|
||||||
@@ -13,7 +9,6 @@ import (
|
|||||||
"git.eeqj.de/sneak/routewatch/internal/database"
|
"git.eeqj.de/sneak/routewatch/internal/database"
|
||||||
"git.eeqj.de/sneak/routewatch/internal/logger"
|
"git.eeqj.de/sneak/routewatch/internal/logger"
|
||||||
"git.eeqj.de/sneak/routewatch/internal/ristypes"
|
"git.eeqj.de/sneak/routewatch/internal/ristypes"
|
||||||
"git.eeqj.de/sneak/routewatch/internal/routetable"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
const (
|
const (
|
||||||
@@ -27,9 +22,8 @@ const (
|
|||||||
)
|
)
|
||||||
|
|
||||||
// blockingStore is a mockStore whose batch writes for the AS, peer and prefix
|
// blockingStore is a mockStore whose batch writes for the AS, peer and prefix
|
||||||
// handlers, delete of a peer's routes and peering write wait until the test
|
// handlers wait until the test calls releaseWrites. writeStarted is closed when
|
||||||
// calls releaseWrites. writeStarted is closed when the first of them starts. It
|
// the first of them starts. It records what each write was given.
|
||||||
// records what each batch write was given.
|
|
||||||
type blockingStore struct {
|
type blockingStore struct {
|
||||||
*mockStore
|
*mockStore
|
||||||
|
|
||||||
@@ -67,7 +61,7 @@ func (s *blockingStore) UpsertLiveRouteBatch(routes []*database.LiveRoute) error
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (s *blockingStore) GetOrCreateASNBatch(asns map[int]database.SeenTimes) error {
|
func (s *blockingStore) GetOrCreateASNBatch(asns map[int]time.Time) error {
|
||||||
s.waitForRelease()
|
s.waitForRelease()
|
||||||
s.mu.Lock()
|
s.mu.Lock()
|
||||||
defer s.mu.Unlock()
|
defer s.mu.Unlock()
|
||||||
@@ -91,20 +85,6 @@ func (s *blockingStore) UpdatePeerBatch(peers map[string]database.PeerUpdate) er
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (s *blockingStore) DeleteLiveRoutesOfPeer(peerIP string) (int, error) {
|
|
||||||
s.waitForRelease()
|
|
||||||
|
|
||||||
return s.mockStore.DeleteLiveRoutesOfPeer(peerIP)
|
|
||||||
}
|
|
||||||
|
|
||||||
func (s *blockingStore) RecordPeering(
|
|
||||||
asA, asB int, seen database.SeenTimes,
|
|
||||||
) error {
|
|
||||||
s.waitForRelease()
|
|
||||||
|
|
||||||
return s.mockStore.RecordPeering(asA, asB, seen)
|
|
||||||
}
|
|
||||||
|
|
||||||
func (s *blockingStore) waitForRelease() {
|
func (s *blockingStore) waitForRelease() {
|
||||||
s.startOnce.Do(func() { close(s.writeStarted) })
|
s.startOnce.Do(func() { close(s.writeStarted) })
|
||||||
<-s.release
|
<-s.release
|
||||||
@@ -116,41 +96,6 @@ func (s *blockingStore) releaseWrites() {
|
|||||||
s.releaseOnce.Do(func() { close(s.release) })
|
s.releaseOnce.Do(func() { close(s.release) })
|
||||||
}
|
}
|
||||||
|
|
||||||
// stopLog holds what a handler's Stop logs it did not write.
|
|
||||||
//
|
|
||||||
//nolint:tagliatelle // field names are the keys the handlers log
|
|
||||||
type stopLog struct {
|
|
||||||
Msg string `json:"msg"`
|
|
||||||
Updates int `json:"updates"`
|
|
||||||
PeersDown int `json:"peers_down"`
|
|
||||||
TableChanges int `json:"table_changes"`
|
|
||||||
Paths int `json:"paths"`
|
|
||||||
}
|
|
||||||
|
|
||||||
// newErrorLogger returns a logger that writes only errors, as JSON lines, to
|
|
||||||
// logs.
|
|
||||||
func newErrorLogger(logs *bytes.Buffer) *logger.Logger {
|
|
||||||
return &logger.Logger{Logger: slog.New(slog.NewJSONHandler(logs,
|
|
||||||
&slog.HandlerOptions{Level: slog.LevelError}))}
|
|
||||||
}
|
|
||||||
|
|
||||||
// lastLogged decodes the last line in logs, which is the one Stop logs: a
|
|
||||||
// handler logs nothing after it.
|
|
||||||
func lastLogged(t *testing.T, logs *bytes.Buffer) stopLog {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
lines := bytes.Split(bytes.TrimSpace(logs.Bytes()), []byte("\n"))
|
|
||||||
|
|
||||||
var logged stopLog
|
|
||||||
|
|
||||||
err := json.Unmarshal(lines[len(lines)-1], &logged)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("want the error Stop logs, got %q: %v", logs.String(), err)
|
|
||||||
}
|
|
||||||
|
|
||||||
return logged
|
|
||||||
}
|
|
||||||
|
|
||||||
// returnsInTime reports whether f returns within handleMessageTimeout.
|
// returnsInTime reports whether f returns within handleMessageTimeout.
|
||||||
func returnsInTime(f func()) bool {
|
func returnsInTime(f func()) bool {
|
||||||
done := make(chan struct{})
|
done := make(chan struct{})
|
||||||
@@ -206,14 +151,14 @@ func handleWhileAWriteIsBlocked(
|
|||||||
store *blockingStore,
|
store *blockingStore,
|
||||||
h interface {
|
h interface {
|
||||||
HandleMessage(msg *ristypes.RISMessage)
|
HandleMessage(msg *ristypes.RISMessage)
|
||||||
Stop(ctx context.Context)
|
Stop()
|
||||||
},
|
},
|
||||||
fill []*ristypes.RISMessage,
|
fill []*ristypes.RISMessage,
|
||||||
extra *ristypes.RISMessage,
|
extra *ristypes.RISMessage,
|
||||||
) {
|
) {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
|
|
||||||
defer h.Stop(t.Context())
|
defer h.Stop()
|
||||||
defer store.releaseWrites()
|
defer store.releaseWrites()
|
||||||
|
|
||||||
startBlockedWrite(t, store, h, fill)
|
startBlockedWrite(t, store, h, fill)
|
||||||
@@ -233,14 +178,14 @@ func handleAtMaxBatchSize(
|
|||||||
store *blockingStore,
|
store *blockingStore,
|
||||||
h interface {
|
h interface {
|
||||||
HandleMessage(msg *ristypes.RISMessage)
|
HandleMessage(msg *ristypes.RISMessage)
|
||||||
Stop(ctx context.Context)
|
Stop()
|
||||||
},
|
},
|
||||||
fill, toMax []*ristypes.RISMessage,
|
fill, toMax []*ristypes.RISMessage,
|
||||||
extra *ristypes.RISMessage,
|
extra *ristypes.RISMessage,
|
||||||
) {
|
) {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
|
|
||||||
defer h.Stop(t.Context())
|
defer h.Stop()
|
||||||
defer store.releaseWrites()
|
defer store.releaseWrites()
|
||||||
|
|
||||||
startBlockedWrite(t, store, h, fill)
|
startBlockedWrite(t, store, h, fill)
|
||||||
@@ -272,246 +217,6 @@ func handleAtMaxBatchSize(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// stopWhileAWriteIsBlocked starts a blocked write with the messages in fill,
|
|
||||||
// hands h the message extra, and stops h with a deadline that passes while
|
|
||||||
// store holds the write. Stop must return all the same. Then it lets the write
|
|
||||||
// finish and waits until done, closed when h's flush goroutine returns. Past
|
|
||||||
// the deadline that goroutine skips its last write, so extra is not written,
|
|
||||||
// and Stop logs as not written both extra and what the held write took.
|
|
||||||
func stopWhileAWriteIsBlocked(
|
|
||||||
t *testing.T,
|
|
||||||
store *blockingStore,
|
|
||||||
h interface {
|
|
||||||
HandleMessage(msg *ristypes.RISMessage)
|
|
||||||
Stop(ctx context.Context)
|
|
||||||
},
|
|
||||||
done <-chan struct{},
|
|
||||||
fill []*ristypes.RISMessage,
|
|
||||||
extra *ristypes.RISMessage,
|
|
||||||
) {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
const stopDeadline = 100 * time.Millisecond
|
|
||||||
|
|
||||||
defer store.releaseWrites()
|
|
||||||
|
|
||||||
startBlockedWrite(t, store, h, fill)
|
|
||||||
h.HandleMessage(extra)
|
|
||||||
|
|
||||||
ctx, cancel := context.WithTimeout(t.Context(), stopDeadline)
|
|
||||||
defer cancel()
|
|
||||||
|
|
||||||
if !returnsInTime(func() { h.Stop(ctx) }) {
|
|
||||||
t.Fatalf("Stop with a deadline of %s did not return while a write "+
|
|
||||||
"was blocked", stopDeadline)
|
|
||||||
}
|
|
||||||
|
|
||||||
store.releaseWrites()
|
|
||||||
|
|
||||||
select {
|
|
||||||
case <-done:
|
|
||||||
case <-time.After(handleMessageTimeout):
|
|
||||||
t.Fatal("the flush goroutine did not return after the write finished")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestPrefixHandlerSkipsItsLastWriteAfterTheStopDeadline also checks the
|
|
||||||
// routing table's changes: the flush whose delete is held wrote those of the
|
|
||||||
// fill, so Stop must log the one of the extra prefix as not written. The flush
|
|
||||||
// timer must not fire. A flush it started before the fill would be the one
|
|
||||||
// held, and it would not have written the fill's change.
|
|
||||||
func TestPrefixHandlerSkipsItsLastWriteAfterTheStopDeadline(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
const extraPrefix = "198.51.100.0/24"
|
|
||||||
|
|
||||||
var logs bytes.Buffer
|
|
||||||
|
|
||||||
store := newBlockingStore()
|
|
||||||
h := newPrefixHandler(store, newErrorLogger(&logs),
|
|
||||||
routetable.New(routetable.Params{}), longerThanAnyTest)
|
|
||||||
fill := slices.Repeat([]string{testV4Prefix}, prefixBatchSize)
|
|
||||||
// The peer's session goes down first, so the write held is the delete of
|
|
||||||
// its routes, and Stop must log that peer down too.
|
|
||||||
stopWhileAWriteIsBlocked(t, store, h, h.done,
|
|
||||||
[]*ristypes.RISMessage{sessionDown(), prefixAnnouncement(fill)},
|
|
||||||
prefixAnnouncement([]string{extraPrefix}))
|
|
||||||
|
|
||||||
store.mu.Lock()
|
|
||||||
defer store.mu.Unlock()
|
|
||||||
|
|
||||||
if store.prefixes[extraPrefix] {
|
|
||||||
t.Errorf("%s was written after the stop deadline", extraPrefix)
|
|
||||||
}
|
|
||||||
|
|
||||||
logged := lastLogged(t, &logs)
|
|
||||||
if logged.Updates != prefixBatchSize+1 || logged.PeersDown != 1 ||
|
|
||||||
logged.TableChanges != 1 {
|
|
||||||
t.Errorf("Stop logged %d updates, %d peer downs and %d table changes "+
|
|
||||||
"not written, want %d, 1 and 1", logged.Updates, logged.PeersDown,
|
|
||||||
logged.TableChanges, prefixBatchSize+1)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestPrefixHandlerSkipsAFullBatchAfterTheStopDeadline hands the prefix handler
|
|
||||||
// a full batch while a write is held, so the batch's wake-up of the flush
|
|
||||||
// goroutine waits. The stop deadline passes meanwhile. When the write ends, the
|
|
||||||
// flush goroutine must stop instead of writing the batch.
|
|
||||||
func TestPrefixHandlerSkipsAFullBatchAfterTheStopDeadline(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
const extraPrefix = "198.51.100.0/24"
|
|
||||||
|
|
||||||
store := newBlockingStore()
|
|
||||||
h := NewPrefixHandler(store, logger.New(), routetable.New(routetable.Params{}))
|
|
||||||
fill := slices.Repeat([]string{testV4Prefix}, prefixBatchSize)
|
|
||||||
full := slices.Repeat([]string{extraPrefix}, prefixBatchSize)
|
|
||||||
stopWhileAWriteIsBlocked(t, store, h, h.done,
|
|
||||||
[]*ristypes.RISMessage{prefixAnnouncement(fill)},
|
|
||||||
prefixAnnouncement(full))
|
|
||||||
|
|
||||||
store.mu.Lock()
|
|
||||||
defer store.mu.Unlock()
|
|
||||||
|
|
||||||
if store.prefixes[extraPrefix] {
|
|
||||||
t.Errorf("%s was written after the stop deadline", extraPrefix)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// tableWriteBlockingStore is a blockingStore whose writes of the routing
|
|
||||||
// table's changes also wait until the test calls releaseWrites.
|
|
||||||
type tableWriteBlockingStore struct {
|
|
||||||
*blockingStore
|
|
||||||
}
|
|
||||||
|
|
||||||
func (s tableWriteBlockingStore) WriteRouteChanges(
|
|
||||||
changes []routetable.Change, newPeers map[string]uint,
|
|
||||||
) error {
|
|
||||||
s.waitForRelease()
|
|
||||||
|
|
||||||
return s.mockStore.WriteRouteChanges(changes, newPeers)
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestPrefixHandlerStopCountsTheTableChangesOfAWriteInProgress holds the write
|
|
||||||
// of the routing table's changes that a full batch starts, then hands the
|
|
||||||
// prefix handler one more prefix and stops it with a deadline that passes
|
|
||||||
// meanwhile. Stop must log as not written both the change the held write took
|
|
||||||
// and the one since.
|
|
||||||
func TestPrefixHandlerStopCountsTheTableChangesOfAWriteInProgress(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
const extraPrefix = "198.51.100.0/24"
|
|
||||||
|
|
||||||
var logs bytes.Buffer
|
|
||||||
|
|
||||||
blocking := newBlockingStore()
|
|
||||||
h := NewPrefixHandler(tableWriteBlockingStore{blockingStore: blocking},
|
|
||||||
newErrorLogger(&logs), routetable.New(routetable.Params{}))
|
|
||||||
fill := slices.Repeat([]string{testV4Prefix}, prefixBatchSize)
|
|
||||||
stopWhileAWriteIsBlocked(t, blocking, h, h.done,
|
|
||||||
[]*ristypes.RISMessage{prefixAnnouncement(fill)},
|
|
||||||
prefixAnnouncement([]string{extraPrefix}))
|
|
||||||
|
|
||||||
if logged := lastLogged(t, &logs); logged.TableChanges != 2 {
|
|
||||||
t.Errorf("Stop logged %d table changes not written, want 2",
|
|
||||||
logged.TableChanges)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestASHandlerSkipsItsLastWriteAfterTheStopDeadline(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
// Each message gives an update for its origin AS and one for each AS
|
|
||||||
// in its path: fill asnBatchSize+1, extra 2.
|
|
||||||
const wantUpdates = asnBatchSize + 1 + 2
|
|
||||||
|
|
||||||
var logs bytes.Buffer
|
|
||||||
|
|
||||||
store := newBlockingStore()
|
|
||||||
h := NewASHandler(store, newErrorLogger(&logs))
|
|
||||||
fill := ristypes.ASPath{Sequence: slices.Repeat([]int{testASNA}, asnBatchSize)}
|
|
||||||
stopWhileAWriteIsBlocked(t, store, h, h.done,
|
|
||||||
[]*ristypes.RISMessage{{Path: fill}},
|
|
||||||
&ristypes.RISMessage{Path: ristypes.ASPath{Sequence: []int{testASNB}}})
|
|
||||||
|
|
||||||
store.mu.Lock()
|
|
||||||
defer store.mu.Unlock()
|
|
||||||
|
|
||||||
if store.asns[testASNB] {
|
|
||||||
t.Errorf("AS%d was written after the stop deadline", testASNB)
|
|
||||||
}
|
|
||||||
|
|
||||||
if logged := lastLogged(t, &logs); logged.Updates != wantUpdates {
|
|
||||||
t.Errorf("Stop logged %d updates not written, want %d",
|
|
||||||
logged.Updates, wantUpdates)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestPeerHandlerSkipsItsLastWriteAfterTheStopDeadline(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
const otherPeerIP = "2001:db8:ffff::2"
|
|
||||||
|
|
||||||
var logs bytes.Buffer
|
|
||||||
|
|
||||||
store := newBlockingStore()
|
|
||||||
h := NewPeerHandler(store, newErrorLogger(&logs))
|
|
||||||
stopWhileAWriteIsBlocked(t, store, h, h.done,
|
|
||||||
slices.Repeat([]*ristypes.RISMessage{{Peer: testPeerIP}}, peerBatchSize),
|
|
||||||
&ristypes.RISMessage{Peer: otherPeerIP})
|
|
||||||
|
|
||||||
store.mu.Lock()
|
|
||||||
defer store.mu.Unlock()
|
|
||||||
|
|
||||||
if store.peers[otherPeerIP] {
|
|
||||||
t.Errorf("peer %s was written after the stop deadline", otherPeerIP)
|
|
||||||
}
|
|
||||||
|
|
||||||
if logged := lastLogged(t, &logs); logged.Updates != peerBatchSize+1 {
|
|
||||||
t.Errorf("Stop logged %d updates not written, want %d",
|
|
||||||
logged.Updates, peerBatchSize+1)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestPeeringHandlerStopCountsTheWriteInProgress hands a running peering
|
|
||||||
// handler one AS path and stops it with a deadline that passes while store
|
|
||||||
// holds Stop's last write, of that path's peering. Stop must log that path as
|
|
||||||
// not written.
|
|
||||||
func TestPeeringHandlerStopCountsTheWriteInProgress(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
const stopDeadline = 100 * time.Millisecond
|
|
||||||
|
|
||||||
var logs bytes.Buffer
|
|
||||||
|
|
||||||
store := newBlockingStore()
|
|
||||||
defer store.releaseWrites()
|
|
||||||
|
|
||||||
h := NewPeeringHandler(store, newErrorLogger(&logs))
|
|
||||||
handle(h, time.Now().UTC(), testASNA, testASNB)
|
|
||||||
|
|
||||||
ctx, cancel := context.WithTimeout(t.Context(), stopDeadline)
|
|
||||||
defer cancel()
|
|
||||||
|
|
||||||
if !returnsInTime(func() { h.Stop(ctx) }) {
|
|
||||||
t.Fatalf("Stop with a deadline of %s did not return while a write "+
|
|
||||||
"was blocked", stopDeadline)
|
|
||||||
}
|
|
||||||
|
|
||||||
store.releaseWrites()
|
|
||||||
|
|
||||||
select {
|
|
||||||
case <-h.done:
|
|
||||||
case <-time.After(handleMessageTimeout):
|
|
||||||
t.Fatal("the processing goroutine did not return after the write " +
|
|
||||||
"finished")
|
|
||||||
}
|
|
||||||
|
|
||||||
if logged := lastLogged(t, &logs); logged.Paths != 1 {
|
|
||||||
t.Errorf("Stop logged %d paths not written, want 1", logged.Paths)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// prefixAnnouncement returns a message in which testPeerIP announces prefixes
|
// prefixAnnouncement returns a message in which testPeerIP announces prefixes
|
||||||
// with origin AS testASNA.
|
// with origin AS testASNA.
|
||||||
func prefixAnnouncement(prefixes []string) *ristypes.RISMessage {
|
func prefixAnnouncement(prefixes []string) *ristypes.RISMessage {
|
||||||
@@ -530,7 +235,7 @@ func TestPrefixHandlerHandlesMessagesWhileAWriteIsBlocked(t *testing.T) {
|
|||||||
const extraPrefix = "198.51.100.0/24"
|
const extraPrefix = "198.51.100.0/24"
|
||||||
|
|
||||||
store := newBlockingStore()
|
store := newBlockingStore()
|
||||||
h := NewPrefixHandler(store, logger.New(), routetable.New(routetable.Params{}))
|
h := NewPrefixHandler(store, logger.New())
|
||||||
fill := slices.Repeat([]string{testV4Prefix}, prefixBatchSize)
|
fill := slices.Repeat([]string{testV4Prefix}, prefixBatchSize)
|
||||||
handleWhileAWriteIsBlocked(t, store, h,
|
handleWhileAWriteIsBlocked(t, store, h,
|
||||||
[]*ristypes.RISMessage{prefixAnnouncement(fill)},
|
[]*ristypes.RISMessage{prefixAnnouncement(fill)},
|
||||||
@@ -590,7 +295,7 @@ func TestPrefixHandlerWaitsAtTheMaximumBatchSize(t *testing.T) {
|
|||||||
const extraPrefix = "198.51.100.0/24"
|
const extraPrefix = "198.51.100.0/24"
|
||||||
|
|
||||||
store := newBlockingStore()
|
store := newBlockingStore()
|
||||||
h := NewPrefixHandler(store, logger.New(), routetable.New(routetable.Params{}))
|
h := NewPrefixHandler(store, logger.New())
|
||||||
fill := slices.Repeat([]string{testV4Prefix}, prefixBatchSize)
|
fill := slices.Repeat([]string{testV4Prefix}, prefixBatchSize)
|
||||||
toMax := slices.Repeat([]string{testV4Prefix}, prefixBatchMaxSize)
|
toMax := slices.Repeat([]string{testV4Prefix}, prefixBatchMaxSize)
|
||||||
handleAtMaxBatchSize(t, store, h,
|
handleAtMaxBatchSize(t, store, h,
|
||||||
@@ -667,10 +372,7 @@ func TestHandlersWakeTheFlushGoroutineWhenTheBatchIsFull(t *testing.T) {
|
|||||||
name: "prefix",
|
name: "prefix",
|
||||||
batchSize: prefixBatchSize,
|
batchSize: prefixBatchSize,
|
||||||
handle: func(n int) chan struct{} {
|
handle: func(n int) chan struct{} {
|
||||||
h := &PrefixHandler{
|
h := &PrefixHandler{batchFullCh: make(chan struct{}, 1)}
|
||||||
batchFullCh: make(chan struct{}, 1),
|
|
||||||
table: routetable.New(routetable.Params{}),
|
|
||||||
}
|
|
||||||
h.HandleMessage(prefixAnnouncement(
|
h.HandleMessage(prefixAnnouncement(
|
||||||
slices.Repeat([]string{testV4Prefix}, n)))
|
slices.Repeat([]string{testV4Prefix}, n)))
|
||||||
|
|
||||||
|
|||||||
+23
-17
@@ -3,8 +3,10 @@ package routewatch
|
|||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"os"
|
"os"
|
||||||
|
"os/signal"
|
||||||
"runtime"
|
"runtime"
|
||||||
"strings"
|
"strings"
|
||||||
|
"syscall"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"git.eeqj.de/sneak/routewatch/internal/logger"
|
"git.eeqj.de/sneak/routewatch/internal/logger"
|
||||||
@@ -12,14 +14,8 @@ import (
|
|||||||
)
|
)
|
||||||
|
|
||||||
const (
|
const (
|
||||||
// shutdownTimeout is how long Shutdown has after a stop signal. upaas,
|
// shutdownTimeout is the maximum time allowed for graceful shutdown
|
||||||
// like docker stop by default, kills the container 10 seconds after the
|
shutdownTimeout = 60 * time.Second
|
||||||
// stop signal (README "Running under upaas").
|
|
||||||
shutdownTimeout = 8 * time.Second
|
|
||||||
// stopTimeout is fx's limit on the stop hook, past which fx exits with
|
|
||||||
// status 1: shutdownTimeout and time for Shutdown to close the HTTP
|
|
||||||
// server after its deadline.
|
|
||||||
stopTimeout = 9 * time.Second
|
|
||||||
// debugInterval is how often to log debug stats
|
// debugInterval is how often to log debug stats
|
||||||
debugInterval = 60 * time.Second
|
debugInterval = 60 * time.Second
|
||||||
// bytesPerMB is bytes per megabyte
|
// bytesPerMB is bytes per megabyte
|
||||||
@@ -60,14 +56,14 @@ func logDebugStats(logger *logger.Logger) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// CLIEntry is the main entry point for the routewatch command-line interface.
|
// CLIEntry is the main entry point for the routewatch command-line interface.
|
||||||
// It initializes the application using the fx dependency injection framework
|
// It initializes the application using the fx dependency injection framework,
|
||||||
// and starts the RouteWatch service. It blocks until fx receives SIGINT or
|
// sets up signal handling for graceful shutdown, and starts the RouteWatch
|
||||||
// SIGTERM or RouteWatch stops on an error; fx then runs the stop hook and sets
|
// service. This function blocks until the application receives a shutdown
|
||||||
// the exit code.
|
// signal or encounters a fatal error.
|
||||||
func CLIEntry() {
|
func CLIEntry() {
|
||||||
app := fx.New(
|
app := fx.New(
|
||||||
fx.Options(getModule()...),
|
fx.Options(getModule()...),
|
||||||
fx.StopTimeout(stopTimeout),
|
fx.StopTimeout(shutdownTimeout), // Allow 60 seconds for graceful shutdown
|
||||||
fx.Invoke(func(
|
fx.Invoke(func(
|
||||||
lc fx.Lifecycle,
|
lc fx.Lifecycle,
|
||||||
rw *RouteWatch,
|
rw *RouteWatch,
|
||||||
@@ -79,6 +75,20 @@ func CLIEntry() {
|
|||||||
// Start debug stats logging
|
// Start debug stats logging
|
||||||
go logDebugStats(logger)
|
go logDebugStats(logger)
|
||||||
|
|
||||||
|
// Handle shutdown signals
|
||||||
|
sigCh := make(chan os.Signal, 1)
|
||||||
|
signal.Notify(sigCh, os.Interrupt, syscall.SIGTERM)
|
||||||
|
|
||||||
|
go func() {
|
||||||
|
<-sigCh
|
||||||
|
logger.Info("Received shutdown signal")
|
||||||
|
|
||||||
|
err := shutdowner.Shutdown()
|
||||||
|
if err != nil {
|
||||||
|
logger.Error("Failed to shutdown gracefully", "error", err)
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
|
||||||
go func() {
|
go func() {
|
||||||
err := rw.Run(ctx)
|
err := rw.Run(ctx)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -96,10 +106,6 @@ func CLIEntry() {
|
|||||||
},
|
},
|
||||||
OnStop: func(ctx context.Context) error {
|
OnStop: func(ctx context.Context) error {
|
||||||
logger.Info("Shutting down RouteWatch")
|
logger.Info("Shutting down RouteWatch")
|
||||||
|
|
||||||
ctx, cancel := context.WithTimeout(ctx, shutdownTimeout)
|
|
||||||
defer cancel()
|
|
||||||
|
|
||||||
rw.Shutdown(ctx)
|
rw.Shutdown(ctx)
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
|
|||||||
@@ -2,7 +2,6 @@ package routewatch
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"errors"
|
|
||||||
"log/slog"
|
"log/slog"
|
||||||
"sync"
|
"sync"
|
||||||
"time"
|
"time"
|
||||||
@@ -35,18 +34,12 @@ const (
|
|||||||
analyzeTimeout = 5 * time.Minute
|
analyzeTimeout = 5 * time.Minute
|
||||||
)
|
)
|
||||||
|
|
||||||
// checkpointIncompleteMessage is logged, at debug level like a completed
|
|
||||||
// checkpoint, when a checkpoint leaves part of the WAL for a later one: while
|
|
||||||
// reads overlap, most do. The database warns when the WAL file stays over its
|
|
||||||
// size limit.
|
|
||||||
const checkpointIncompleteMessage = "WAL checkpoint incomplete"
|
|
||||||
|
|
||||||
// DBMaintainer handles background database maintenance tasks.
|
// DBMaintainer handles background database maintenance tasks.
|
||||||
type DBMaintainer struct {
|
type DBMaintainer struct {
|
||||||
db database.Store
|
db database.Store
|
||||||
logger *slog.Logger
|
logger *slog.Logger
|
||||||
stopCh chan struct{}
|
stopCh chan struct{}
|
||||||
done chan struct{} // closed when the maintenance goroutine returns
|
wg sync.WaitGroup
|
||||||
|
|
||||||
// Stats tracking
|
// Stats tracking
|
||||||
statsMu sync.Mutex
|
statsMu sync.Mutex
|
||||||
@@ -67,13 +60,14 @@ func NewDBMaintainer(db database.Store, logger *slog.Logger) *DBMaintainer {
|
|||||||
db: db,
|
db: db,
|
||||||
logger: logger.With("component", "db_maintainer"),
|
logger: logger.With("component", "db_maintainer"),
|
||||||
stopCh: make(chan struct{}),
|
stopCh: make(chan struct{}),
|
||||||
done: make(chan struct{}),
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Start begins the background maintenance goroutine. It runs until Stop:
|
// Start begins the background maintenance goroutine. It runs until Stop:
|
||||||
// the end of ctx does not stop it.
|
// the end of ctx does not stop it.
|
||||||
func (m *DBMaintainer) Start(ctx context.Context) {
|
func (m *DBMaintainer) Start(ctx context.Context) {
|
||||||
|
m.wg.Add(1)
|
||||||
|
|
||||||
go m.run(context.WithoutCancel(ctx))
|
go m.run(context.WithoutCancel(ctx))
|
||||||
|
|
||||||
m.logger.Info("Database maintainer started",
|
m.logger.Info("Database maintainer started",
|
||||||
@@ -83,23 +77,16 @@ func (m *DBMaintainer) Start(ctx context.Context) {
|
|||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Stop stops the maintainer and waits, until ctx ends, for a maintenance task
|
// Stop gracefully shuts down the maintainer.
|
||||||
// in progress to finish.
|
func (m *DBMaintainer) Stop() {
|
||||||
func (m *DBMaintainer) Stop(ctx context.Context) {
|
|
||||||
close(m.stopCh)
|
close(m.stopCh)
|
||||||
|
m.wg.Wait()
|
||||||
select {
|
m.logger.Info("Database maintainer stopped")
|
||||||
case <-m.done:
|
|
||||||
m.logger.Info("Database maintainer stopped")
|
|
||||||
case <-ctx.Done():
|
|
||||||
m.logger.Warn("Stop deadline passed before the database maintainer " +
|
|
||||||
"stopped")
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// run is the main background loop.
|
// run is the main background loop.
|
||||||
func (m *DBMaintainer) run(ctx context.Context) {
|
func (m *DBMaintainer) run(ctx context.Context) {
|
||||||
defer close(m.done)
|
defer m.wg.Done()
|
||||||
|
|
||||||
// Use different timers for each task
|
// Use different timers for each task
|
||||||
checkpointTimer := time.NewTimer(checkpointInterval)
|
checkpointTimer := time.NewTimer(checkpointInterval)
|
||||||
@@ -136,13 +123,9 @@ func (m *DBMaintainer) runCheckpoint(ctx context.Context) {
|
|||||||
duration, err := runWithTimeout(ctx, checkpointTimeout, m.db.Checkpoint)
|
duration, err := runWithTimeout(ctx, checkpointTimeout, m.db.Checkpoint)
|
||||||
m.recordResult(&m.lastCheckpoint, &m.lastCheckpointError, &m.checkpointCount, err)
|
m.recordResult(&m.lastCheckpoint, &m.lastCheckpointError, &m.checkpointCount, err)
|
||||||
|
|
||||||
switch {
|
if err != nil {
|
||||||
case errors.Is(err, database.ErrCheckpointIncomplete):
|
|
||||||
m.logger.Debug(checkpointIncompleteMessage,
|
|
||||||
"error", err, "duration", duration)
|
|
||||||
case err != nil:
|
|
||||||
m.logger.Error("WAL checkpoint failed", "error", err, "duration", duration)
|
m.logger.Error("WAL checkpoint failed", "error", err, "duration", duration)
|
||||||
default:
|
} else {
|
||||||
m.logger.Debug("WAL checkpoint completed", "duration", duration)
|
m.logger.Debug("WAL checkpoint completed", "duration", duration)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,120 +0,0 @@
|
|||||||
package routewatch
|
|
||||||
|
|
||||||
import (
|
|
||||||
"bytes"
|
|
||||||
"context"
|
|
||||||
"encoding/json"
|
|
||||||
"log/slog"
|
|
||||||
"testing"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"git.eeqj.de/sneak/routewatch/internal/database"
|
|
||||||
)
|
|
||||||
|
|
||||||
// incompleteCheckpointStore is a store whose every checkpoint leaves part of
|
|
||||||
// the WAL uncopied.
|
|
||||||
type incompleteCheckpointStore struct{ *mockStore }
|
|
||||||
|
|
||||||
// Checkpoint reports the checkpoint incomplete.
|
|
||||||
func (incompleteCheckpointStore) Checkpoint(context.Context) error {
|
|
||||||
return database.ErrCheckpointIncomplete
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestIncompleteCheckpointNotLoggedAsCompleted runs a checkpoint that leaves
|
|
||||||
// part of the WAL uncopied and checks that the maintainer logs it as
|
|
||||||
// incomplete, not as completed, and does not count it as a checkpoint that
|
|
||||||
// completed.
|
|
||||||
func TestIncompleteCheckpointNotLoggedAsCompleted(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
var logs bytes.Buffer
|
|
||||||
|
|
||||||
logger := slog.New(slog.NewJSONHandler(&logs,
|
|
||||||
&slog.HandlerOptions{Level: slog.LevelDebug}))
|
|
||||||
m := NewDBMaintainer(incompleteCheckpointStore{newMockStore()}, logger)
|
|
||||||
|
|
||||||
m.runCheckpoint(t.Context())
|
|
||||||
|
|
||||||
var logged struct {
|
|
||||||
Msg string `json:"msg"`
|
|
||||||
}
|
|
||||||
|
|
||||||
err := json.Unmarshal(logs.Bytes(), &logged)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("want one log message, got %q: %v", logs.String(), err)
|
|
||||||
}
|
|
||||||
|
|
||||||
if logged.Msg != checkpointIncompleteMessage {
|
|
||||||
t.Errorf("logged %q, want %q", logged.Msg, checkpointIncompleteMessage)
|
|
||||||
}
|
|
||||||
|
|
||||||
if m.checkpointCount != 0 {
|
|
||||||
t.Errorf("checkpointCount = %d after an incomplete checkpoint, want 0",
|
|
||||||
m.checkpointCount)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// heldCheckpointStore is a store whose checkpoint closes started and then
|
|
||||||
// waits until release is closed.
|
|
||||||
type heldCheckpointStore struct {
|
|
||||||
*mockStore
|
|
||||||
|
|
||||||
started chan struct{}
|
|
||||||
release chan struct{}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Checkpoint waits until release is closed.
|
|
||||||
func (s heldCheckpointStore) Checkpoint(context.Context) error {
|
|
||||||
close(s.started)
|
|
||||||
<-s.release
|
|
||||||
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestDBMaintainerStopReturnsByItsDeadline stops the maintainer with a short
|
|
||||||
// deadline while its goroutine runs a checkpoint that the store holds until
|
|
||||||
// the test ends. Stop must return all the same instead of waiting for the
|
|
||||||
// checkpoint.
|
|
||||||
func TestDBMaintainerStopReturnsByItsDeadline(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
const stopDeadline = 100 * time.Millisecond
|
|
||||||
|
|
||||||
store := heldCheckpointStore{
|
|
||||||
mockStore: newMockStore(),
|
|
||||||
started: make(chan struct{}),
|
|
||||||
release: make(chan struct{}),
|
|
||||||
}
|
|
||||||
m := NewDBMaintainer(store, slog.Default())
|
|
||||||
|
|
||||||
// Start would run the first checkpoint only after checkpointInterval.
|
|
||||||
// This goroutine stands for the one Start runs and checkpoints at once.
|
|
||||||
go func() {
|
|
||||||
defer close(m.done)
|
|
||||||
|
|
||||||
m.runCheckpoint(t.Context())
|
|
||||||
}()
|
|
||||||
|
|
||||||
select {
|
|
||||||
case <-store.started:
|
|
||||||
case <-time.After(handleMessageTimeout):
|
|
||||||
t.Fatal("the maintainer did not start the checkpoint")
|
|
||||||
}
|
|
||||||
|
|
||||||
ctx, cancel := context.WithTimeout(t.Context(), stopDeadline)
|
|
||||||
defer cancel()
|
|
||||||
|
|
||||||
if !returnsInTime(func() { m.Stop(ctx) }) {
|
|
||||||
t.Fatalf("Stop with a deadline of %s did not return while a "+
|
|
||||||
"checkpoint was in progress", stopDeadline)
|
|
||||||
}
|
|
||||||
|
|
||||||
close(store.release)
|
|
||||||
|
|
||||||
select {
|
|
||||||
case <-m.done:
|
|
||||||
case <-time.After(handleMessageTimeout):
|
|
||||||
t.Fatal("the maintenance goroutine did not return after the " +
|
|
||||||
"checkpoint ended")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -5,7 +5,6 @@ package routewatch
|
|||||||
// peer state in the database.
|
// peer state in the database.
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
|
||||||
"strconv"
|
"strconv"
|
||||||
"sync"
|
"sync"
|
||||||
"time"
|
"time"
|
||||||
@@ -37,30 +36,24 @@ const (
|
|||||||
|
|
||||||
// PeerHandler tracks BGP peers from UPDATE messages using batched operations.
|
// PeerHandler tracks BGP peers from UPDATE messages using batched operations.
|
||||||
// It maintains a queue of peer updates and periodically flushes them to the database
|
// It maintains a queue of peer updates and periodically flushes them to the database
|
||||||
// in batches to improve performance. The handler writes one row per peer IP
|
// in batches to improve performance. The handler deduplicates peer updates within
|
||||||
// address for each batch, with the earliest and the latest time of its updates.
|
// each batch, keeping only the most recent update for each peer IP address.
|
||||||
type PeerHandler struct {
|
type PeerHandler struct {
|
||||||
db database.Store
|
db database.Store
|
||||||
logger *logger.Logger
|
logger *logger.Logger
|
||||||
|
|
||||||
// Batching. mu guards peerBatch and writing only: the flush goroutine
|
// Batching. mu guards peerBatch only: the flush goroutine takes the batch
|
||||||
// takes the batch under mu and writes it after releasing mu, so
|
// under mu and writes it after releasing mu, so HandleMessage waits for
|
||||||
// HandleMessage waits for a database write only while peerBatch is at
|
// a database write only while peerBatch is at peerBatchMaxSize.
|
||||||
// peerBatchMaxSize.
|
|
||||||
mu sync.Mutex
|
mu sync.Mutex
|
||||||
peerBatch []peerUpdate
|
peerBatch []peerUpdate
|
||||||
// writing is how many updates the flush goroutine took from peerBatch
|
|
||||||
// and has not finished writing.
|
|
||||||
writing int
|
|
||||||
// batchTaken is signalled when the flush goroutine takes peerBatch; its L
|
// batchTaken is signalled when the flush goroutine takes peerBatch; its L
|
||||||
// is &mu.
|
// is &mu.
|
||||||
batchTaken sync.Cond
|
batchTaken sync.Cond
|
||||||
batchFullCh chan struct{} // wakes the flush goroutine when peerBatch is full
|
batchFullCh chan struct{} // wakes the flush goroutine when peerBatch is full
|
||||||
lastFlush time.Time // used only by the flush goroutine
|
lastFlush time.Time // used only by the flush goroutine
|
||||||
// stopCh hands Stop's context to the flush goroutine. It holds one, so
|
stopCh chan struct{}
|
||||||
// Stop does not wait for a write in progress to hand it over.
|
wg sync.WaitGroup
|
||||||
stopCh chan context.Context
|
|
||||||
done chan struct{} // closed when the flush goroutine returns
|
|
||||||
}
|
}
|
||||||
|
|
||||||
type peerUpdate struct {
|
type peerUpdate struct {
|
||||||
@@ -81,12 +74,12 @@ func NewPeerHandler(db database.Store, logger *logger.Logger) *PeerHandler {
|
|||||||
peerBatch: make([]peerUpdate, 0, peerBatchSize),
|
peerBatch: make([]peerUpdate, 0, peerBatchSize),
|
||||||
batchFullCh: make(chan struct{}, 1),
|
batchFullCh: make(chan struct{}, 1),
|
||||||
lastFlush: time.Now(),
|
lastFlush: time.Now(),
|
||||||
stopCh: make(chan context.Context, 1),
|
stopCh: make(chan struct{}),
|
||||||
done: make(chan struct{}),
|
|
||||||
}
|
}
|
||||||
h.batchTaken.L = &h.mu
|
h.batchTaken.L = &h.mu
|
||||||
|
|
||||||
// Start the flush timer goroutine
|
// Start the flush timer goroutine
|
||||||
|
h.wg.Add(1)
|
||||||
go h.flushLoop()
|
go h.flushLoop()
|
||||||
|
|
||||||
return h
|
return h
|
||||||
@@ -151,32 +144,16 @@ func (h *PeerHandler) HandleMessage(msg *ristypes.RISMessage) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Stop stops the flush goroutine, which writes the updates the handler holds
|
// Stop gracefully stops the handler and flushes remaining batches
|
||||||
// before it returns, and waits for it until ctx ends. Once ctx has ended the
|
func (h *PeerHandler) Stop() {
|
||||||
// flush goroutine skips that last write, and a write still running is lost
|
close(h.stopCh)
|
||||||
// when the process exits, so Stop logs how many updates are not written.
|
h.wg.Wait()
|
||||||
func (h *PeerHandler) Stop(ctx context.Context) {
|
|
||||||
h.stopCh <- ctx
|
|
||||||
|
|
||||||
select {
|
|
||||||
case <-h.done:
|
|
||||||
case <-ctx.Done():
|
|
||||||
}
|
|
||||||
|
|
||||||
h.mu.Lock()
|
|
||||||
unwritten := h.writing + len(h.peerBatch)
|
|
||||||
h.mu.Unlock()
|
|
||||||
|
|
||||||
if unwritten > 0 {
|
|
||||||
h.logger.Error("Stop deadline passed before the peer handler wrote "+
|
|
||||||
"its updates", "updates", unwritten)
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// flushLoop runs in a goroutine, the only one that writes the batch: it
|
// flushLoop runs in a goroutine, the only one that writes the batch: it
|
||||||
// flushes when the batch is full, periodically, and once more on Stop.
|
// flushes when the batch is full, periodically, and once more on Stop.
|
||||||
func (h *PeerHandler) flushLoop() {
|
func (h *PeerHandler) flushLoop() {
|
||||||
defer close(h.done)
|
defer h.wg.Done()
|
||||||
|
|
||||||
ticker := time.NewTicker(peerBatchTimeout)
|
ticker := time.NewTicker(peerBatchTimeout)
|
||||||
defer ticker.Stop()
|
defer ticker.Stop()
|
||||||
@@ -189,11 +166,9 @@ func (h *PeerHandler) flushLoop() {
|
|||||||
}
|
}
|
||||||
case <-h.batchFullCh:
|
case <-h.batchFullCh:
|
||||||
h.flushBatch()
|
h.flushBatch()
|
||||||
case ctx := <-h.stopCh:
|
case <-h.stopCh:
|
||||||
// Final flush, unless the stop deadline has passed
|
// Final flush
|
||||||
if ctx.Err() == nil {
|
h.flushBatch()
|
||||||
h.flushBatch()
|
|
||||||
}
|
|
||||||
|
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
@@ -201,8 +176,7 @@ func (h *PeerHandler) flushLoop() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// takeBatch returns the pending updates and leaves an empty batch in their
|
// takeBatch returns the pending updates and leaves an empty batch in their
|
||||||
// place for HandleMessage to fill. They count in writing until flushBatch has
|
// place for HandleMessage to fill.
|
||||||
// written them.
|
|
||||||
func (h *PeerHandler) takeBatch() []peerUpdate {
|
func (h *PeerHandler) takeBatch() []peerUpdate {
|
||||||
h.mu.Lock()
|
h.mu.Lock()
|
||||||
defer h.mu.Unlock()
|
defer h.mu.Unlock()
|
||||||
@@ -212,7 +186,6 @@ func (h *PeerHandler) takeBatch() []peerUpdate {
|
|||||||
}
|
}
|
||||||
|
|
||||||
batch := h.peerBatch
|
batch := h.peerBatch
|
||||||
h.writing = len(batch)
|
|
||||||
h.peerBatch = make([]peerUpdate, 0, peerBatchSize)
|
h.peerBatch = make([]peerUpdate, 0, peerBatchSize)
|
||||||
h.batchTaken.Broadcast()
|
h.batchTaken.Broadcast()
|
||||||
|
|
||||||
@@ -227,33 +200,32 @@ func (h *PeerHandler) flushBatch() {
|
|||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
// The earliest and the latest time the batch saw each peer, with the peer
|
// Deduplicate by peer IP, keeping the latest update for each peer
|
||||||
// AS and message type of its latest update. The batch holds several
|
peerMap := make(map[string]peerUpdate)
|
||||||
// updates for a peer, so a new peer is stored first seen at the earliest of
|
|
||||||
// them and last seen at the latest.
|
|
||||||
peers := make(map[string]database.PeerUpdate)
|
|
||||||
for _, update := range batch {
|
for _, update := range batch {
|
||||||
peer, ok := peers[update.peerIP]
|
existing, ok := peerMap[update.peerIP]
|
||||||
if !ok || update.timestamp.After(peer.Seen.Last) {
|
if !ok || update.timestamp.After(existing.timestamp) {
|
||||||
peer.PeerIP = update.peerIP
|
peerMap[update.peerIP] = update
|
||||||
peer.PeerASN = update.peerASN
|
|
||||||
peer.MessageType = update.messageType
|
|
||||||
}
|
}
|
||||||
|
}
|
||||||
|
|
||||||
peer.Seen.Add(update.timestamp)
|
// Convert to database format
|
||||||
peers[update.peerIP] = peer
|
dbPeerMap := make(map[string]database.PeerUpdate)
|
||||||
|
for peerIP, update := range peerMap {
|
||||||
|
dbPeerMap[peerIP] = database.PeerUpdate{
|
||||||
|
PeerIP: update.peerIP,
|
||||||
|
PeerASN: update.peerASN,
|
||||||
|
MessageType: update.messageType,
|
||||||
|
Timestamp: update.timestamp,
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Process all peers in a single batch transaction
|
// Process all peers in a single batch transaction
|
||||||
err := h.db.UpdatePeerBatch(peers)
|
err := h.db.UpdatePeerBatch(dbPeerMap)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
h.logger.Error("Failed to process peer batch",
|
h.logger.Error("Failed to process peer batch",
|
||||||
"error", err, "count", len(peers))
|
"error", err, "count", len(dbPeerMap))
|
||||||
}
|
}
|
||||||
|
|
||||||
h.mu.Lock()
|
|
||||||
h.writing = 0
|
|
||||||
h.mu.Unlock()
|
|
||||||
|
|
||||||
h.lastFlush = time.Now()
|
h.lastFlush = time.Now()
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,59 +0,0 @@
|
|||||||
package routewatch
|
|
||||||
|
|
||||||
import (
|
|
||||||
"testing"
|
|
||||||
|
|
||||||
"git.eeqj.de/sneak/routewatch/internal/database"
|
|
||||||
"git.eeqj.de/sneak/routewatch/internal/logger"
|
|
||||||
)
|
|
||||||
|
|
||||||
// peerBatchStore keeps the peers of the last UpdatePeerBatch call. The
|
|
||||||
// database has no read of the peer table, so the peer handler's test checks
|
|
||||||
// what the handler writes.
|
|
||||||
type peerBatchStore struct {
|
|
||||||
*mockStore
|
|
||||||
|
|
||||||
peers map[string]database.PeerUpdate
|
|
||||||
}
|
|
||||||
|
|
||||||
func (s *peerBatchStore) UpdatePeerBatch(
|
|
||||||
peers map[string]database.PeerUpdate,
|
|
||||||
) error {
|
|
||||||
s.peers = peers
|
|
||||||
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestPeerHandlerWritesTheEarliestAndLatestTimesOfANewPeer handles two updates
|
|
||||||
// from one peer, the second 5 seconds after the first, in one batch. The peer
|
|
||||||
// handler writes the peer first seen at the time of the first update and last
|
|
||||||
// seen at the time of the second.
|
|
||||||
func TestPeerHandlerWritesTheEarliestAndLatestTimesOfANewPeer(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
store := &peerBatchStore{mockStore: newMockStore()}
|
|
||||||
|
|
||||||
// Built without NewPeerHandler's flush goroutine, so the test flushes
|
|
||||||
// itself.
|
|
||||||
h := &PeerHandler{db: store, logger: logger.New()}
|
|
||||||
|
|
||||||
first := peerAnnouncement(testPeerIP, testV4Prefix)
|
|
||||||
second := peerAnnouncement(testPeerIP, testV4Prefix)
|
|
||||||
second.ParsedTimestamp = first.ParsedTimestamp.Add(secondUpdateAfter)
|
|
||||||
|
|
||||||
h.HandleMessage(first)
|
|
||||||
h.HandleMessage(second)
|
|
||||||
h.flushBatch()
|
|
||||||
|
|
||||||
seen := store.peers[testPeerIP].Seen
|
|
||||||
|
|
||||||
if !seen.First.Equal(first.ParsedTimestamp) {
|
|
||||||
t.Errorf("first_seen of peer %s = %s, want %s", testPeerIP,
|
|
||||||
seen.First, first.ParsedTimestamp)
|
|
||||||
}
|
|
||||||
|
|
||||||
if !seen.Last.Equal(second.ParsedTimestamp) {
|
|
||||||
t.Errorf("last_seen of peer %s = %s, want %s", testPeerIP,
|
|
||||||
seen.Last, second.ParsedTimestamp)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,7 +1,6 @@
|
|||||||
package routewatch
|
package routewatch
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
"sync"
|
"sync"
|
||||||
"time"
|
"time"
|
||||||
@@ -42,19 +41,13 @@ type PeeringHandler struct {
|
|||||||
db database.Store
|
db database.Store
|
||||||
logger *logger.Logger
|
logger *logger.Logger
|
||||||
|
|
||||||
// In-memory AS path tracking: each path, JSON-encoded, with the earliest
|
// In-memory AS path tracking
|
||||||
// and the latest time an update carried it.
|
|
||||||
mu sync.Mutex
|
mu sync.Mutex
|
||||||
asPaths map[string]database.SeenTimes
|
asPaths map[string]time.Time // key is JSON-encoded AS path
|
||||||
droppedPaths int // paths dropped since the last run because the map was full
|
droppedPaths int // paths dropped because the map was full
|
||||||
// writing is how many paths processPeerings took from asPaths and has
|
|
||||||
// not finished writing the peerings of.
|
|
||||||
writing int
|
|
||||||
|
|
||||||
// stopCh hands Stop's context to the processing goroutine. It holds one,
|
stopCh chan struct{}
|
||||||
// so Stop does not wait for a write in progress to hand it over.
|
wg sync.WaitGroup
|
||||||
stopCh chan context.Context
|
|
||||||
done chan struct{} // closed when the processing goroutine returns
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// NewPeeringHandler creates and initializes a new PeeringHandler with the
|
// NewPeeringHandler creates and initializes a new PeeringHandler with the
|
||||||
@@ -65,12 +58,13 @@ func NewPeeringHandler(db database.Store, logger *logger.Logger) *PeeringHandler
|
|||||||
h := &PeeringHandler{
|
h := &PeeringHandler{
|
||||||
db: db,
|
db: db,
|
||||||
logger: logger,
|
logger: logger,
|
||||||
asPaths: make(map[string]database.SeenTimes),
|
asPaths: make(map[string]time.Time),
|
||||||
stopCh: make(chan context.Context, 1),
|
stopCh: make(chan struct{}),
|
||||||
done: make(chan struct{}),
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// Start the periodic processing goroutine
|
// Start the periodic processing goroutine
|
||||||
|
h.wg.Add(1)
|
||||||
|
|
||||||
go h.processLoop()
|
go h.processLoop()
|
||||||
|
|
||||||
return h
|
return h
|
||||||
@@ -119,18 +113,16 @@ func (h *PeeringHandler) HandleMessage(msg *ristypes.RISMessage) {
|
|||||||
key := string(pathJSON)
|
key := string(pathJSON)
|
||||||
|
|
||||||
h.mu.Lock()
|
h.mu.Lock()
|
||||||
defer h.mu.Unlock()
|
if _, exists := h.asPaths[key]; exists {
|
||||||
|
// Already tracked: refresh its timestamp.
|
||||||
seen, exists := h.asPaths[key]
|
h.asPaths[key] = timestamp
|
||||||
if !exists && len(h.asPaths) >= maxTrackedPaths {
|
} else if len(h.asPaths) >= maxTrackedPaths {
|
||||||
// Map is full; drop this new path and count it.
|
// Map is full; drop this new path and count it.
|
||||||
h.droppedPaths++
|
h.droppedPaths++
|
||||||
|
} else {
|
||||||
return
|
h.asPaths[key] = timestamp
|
||||||
}
|
}
|
||||||
|
h.mu.Unlock()
|
||||||
seen.Add(timestamp)
|
|
||||||
h.asPaths[key] = seen
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// ProcessPeeringsNow triggers immediate processing of all accumulated AS
|
// ProcessPeeringsNow triggers immediate processing of all accumulated AS
|
||||||
@@ -140,33 +132,18 @@ func (h *PeeringHandler) ProcessPeeringsNow() {
|
|||||||
h.processPeerings()
|
h.processPeerings()
|
||||||
}
|
}
|
||||||
|
|
||||||
// Stop stops the processing goroutine, which writes the peerings of the AS
|
// Stop gracefully shuts down the handler by signaling the background
|
||||||
// paths the handler holds before it returns, and waits for it until ctx ends.
|
// goroutine to stop and waiting while it processes the remaining AS paths,
|
||||||
// Once ctx has ended the processing goroutine skips that last write, and a
|
// so no peering data is lost during shutdown.
|
||||||
// write still running is lost when the process exits, so Stop logs how many
|
func (h *PeeringHandler) Stop() {
|
||||||
// paths' peerings are not written.
|
close(h.stopCh)
|
||||||
func (h *PeeringHandler) Stop(ctx context.Context) {
|
h.wg.Wait()
|
||||||
h.stopCh <- ctx
|
|
||||||
|
|
||||||
select {
|
|
||||||
case <-h.done:
|
|
||||||
case <-ctx.Done():
|
|
||||||
}
|
|
||||||
|
|
||||||
h.mu.Lock()
|
|
||||||
unwritten := h.writing + len(h.asPaths)
|
|
||||||
h.mu.Unlock()
|
|
||||||
|
|
||||||
if unwritten > 0 {
|
|
||||||
h.logger.Error("Stop deadline passed before the peering handler wrote "+
|
|
||||||
"its peerings", "paths", unwritten)
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// processLoop runs periodically to process AS paths into peerings, and once
|
// processLoop runs periodically to process AS paths into peerings, and once
|
||||||
// more on Stop.
|
// more on Stop.
|
||||||
func (h *PeeringHandler) processLoop() {
|
func (h *PeeringHandler) processLoop() {
|
||||||
defer close(h.done)
|
defer h.wg.Done()
|
||||||
|
|
||||||
ticker := time.NewTicker(peeringProcessInterval)
|
ticker := time.NewTicker(peeringProcessInterval)
|
||||||
defer ticker.Stop()
|
defer ticker.Stop()
|
||||||
@@ -175,11 +152,9 @@ func (h *PeeringHandler) processLoop() {
|
|||||||
select {
|
select {
|
||||||
case <-ticker.C:
|
case <-ticker.C:
|
||||||
h.processPeerings()
|
h.processPeerings()
|
||||||
case ctx := <-h.stopCh:
|
case <-h.stopCh:
|
||||||
// Final processing, unless the stop deadline has passed
|
// Final processing
|
||||||
if ctx.Err() == nil {
|
h.processPeerings()
|
||||||
h.processPeerings()
|
|
||||||
}
|
|
||||||
|
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
@@ -190,14 +165,11 @@ func (h *PeeringHandler) processLoop() {
|
|||||||
func (h *PeeringHandler) processPeerings() {
|
func (h *PeeringHandler) processPeerings() {
|
||||||
// Take the accumulated paths and replace the map with a fresh empty one
|
// Take the accumulated paths and replace the map with a fresh empty one
|
||||||
// under the lock. Each path is processed exactly once and the memory is
|
// under the lock. Each path is processed exactly once and the memory is
|
||||||
// released, so the map never grows past a single interval's traffic. The
|
// released, so the map never grows past a single interval's traffic.
|
||||||
// count of dropped paths starts again with the new map.
|
|
||||||
h.mu.Lock()
|
h.mu.Lock()
|
||||||
paths := h.asPaths
|
paths := h.asPaths
|
||||||
h.asPaths = make(map[string]database.SeenTimes)
|
h.asPaths = make(map[string]time.Time)
|
||||||
h.writing = len(paths)
|
|
||||||
dropped := h.droppedPaths
|
dropped := h.droppedPaths
|
||||||
h.droppedPaths = 0
|
|
||||||
h.mu.Unlock()
|
h.mu.Unlock()
|
||||||
|
|
||||||
if len(paths) == 0 {
|
if len(paths) == 0 {
|
||||||
@@ -209,13 +181,9 @@ func (h *PeeringHandler) processPeerings() {
|
|||||||
low, high int
|
low, high int
|
||||||
}
|
}
|
||||||
|
|
||||||
// The earliest and the latest time an update carried each peering. Several
|
peerings := make(map[peeringKey]time.Time)
|
||||||
// paths, each carried at several times, can hold a peering, so a new
|
|
||||||
// peering is stored first seen at the earliest of them and last seen at the
|
|
||||||
// latest.
|
|
||||||
peerings := make(map[peeringKey]database.SeenTimes)
|
|
||||||
|
|
||||||
for pathJSON, pathSeen := range paths {
|
for pathJSON, timestamp := range paths {
|
||||||
var path []int
|
var path []int
|
||||||
|
|
||||||
err := json.Unmarshal([]byte(pathJSON), &path)
|
err := json.Unmarshal([]byte(pathJSON), &path)
|
||||||
@@ -242,11 +210,10 @@ func (h *PeeringHandler) processPeerings() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
key := peeringKey{low: low, high: high}
|
key := peeringKey{low: low, high: high}
|
||||||
|
// Update timestamp if this is newer
|
||||||
seen := peerings[key]
|
if existing, ok := peerings[key]; !ok || timestamp.After(existing) {
|
||||||
seen.Add(pathSeen.First)
|
peerings[key] = timestamp
|
||||||
seen.Add(pathSeen.Last)
|
}
|
||||||
peerings[key] = seen
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -254,8 +221,8 @@ func (h *PeeringHandler) processPeerings() {
|
|||||||
start := time.Now()
|
start := time.Now()
|
||||||
successCount := 0
|
successCount := 0
|
||||||
|
|
||||||
for key, seen := range peerings {
|
for key, ts := range peerings {
|
||||||
err := h.db.RecordPeering(key.low, key.high, seen)
|
err := h.db.RecordPeering(key.low, key.high, ts)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
h.logger.Error("Failed to record peering",
|
h.logger.Error("Failed to record peering",
|
||||||
"as_a", key.low,
|
"as_a", key.low,
|
||||||
@@ -267,10 +234,6 @@ func (h *PeeringHandler) processPeerings() {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
h.mu.Lock()
|
|
||||||
h.writing = 0
|
|
||||||
h.mu.Unlock()
|
|
||||||
|
|
||||||
h.logger.Info("Processed AS peerings",
|
h.logger.Info("Processed AS peerings",
|
||||||
"paths", len(paths),
|
"paths", len(paths),
|
||||||
"unique_peerings", len(peerings),
|
"unique_peerings", len(peerings),
|
||||||
|
|||||||
@@ -31,14 +31,12 @@ type recordingStore struct {
|
|||||||
calls int
|
calls int
|
||||||
}
|
}
|
||||||
|
|
||||||
func (r *recordingStore) RecordPeering(
|
func (r *recordingStore) RecordPeering(asA, asB int, ts time.Time) error {
|
||||||
asA, asB int, seen database.SeenTimes,
|
|
||||||
) error {
|
|
||||||
r.mu.Lock()
|
r.mu.Lock()
|
||||||
r.calls++
|
r.calls++
|
||||||
r.mu.Unlock()
|
r.mu.Unlock()
|
||||||
|
|
||||||
return r.mockStore.RecordPeering(asA, asB, seen)
|
return r.mockStore.RecordPeering(asA, asB, ts)
|
||||||
}
|
}
|
||||||
|
|
||||||
func (r *recordingStore) callCount() int {
|
func (r *recordingStore) callCount() int {
|
||||||
@@ -54,7 +52,8 @@ func newTestHandler(db database.Store) *PeeringHandler {
|
|||||||
return &PeeringHandler{
|
return &PeeringHandler{
|
||||||
db: db,
|
db: db,
|
||||||
logger: logger.New(),
|
logger: logger.New(),
|
||||||
asPaths: make(map[string]database.SeenTimes),
|
asPaths: make(map[string]time.Time),
|
||||||
|
stopCh: make(chan struct{}),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -153,43 +152,6 @@ func TestPeeringHandlerLeavesATrailingASSetOutOfThePeerings(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// TestPeeringHandlerRecordsNothingFromAPathWithASZero handles AS paths that
|
|
||||||
// hold AS 0. RFC 7607 makes such a path malformed, so no peering is recorded
|
|
||||||
// from it, not even one between two other ASes of the path.
|
|
||||||
func TestPeeringHandlerRecordsNothingFromAPathWithASZero(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
tests := []struct {
|
|
||||||
name string
|
|
||||||
path []any
|
|
||||||
}{
|
|
||||||
{name: "AS 0 between two ASes", path: []any{testASNA, 0, testASNB}},
|
|
||||||
{
|
|
||||||
name: "AS 0 after two ASes",
|
|
||||||
path: []any{testASNA, testASNB, 0, testASNC},
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, tt := range tests {
|
|
||||||
t.Run(tt.name, func(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
store := newMockStore()
|
|
||||||
h := newTestHandler(store)
|
|
||||||
h.HandleMessage(&ristypes.RISMessage{
|
|
||||||
Path: decodePath(t, tt.path...),
|
|
||||||
ParsedTimestamp: time.Now().UTC(),
|
|
||||||
})
|
|
||||||
h.ProcessPeeringsNow()
|
|
||||||
|
|
||||||
if len(store.Peerings) > 0 {
|
|
||||||
t.Errorf("peerings recorded = %v, want none",
|
|
||||||
slices.Sorted(maps.Keys(store.Peerings)))
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestPeeringHandlerProcessesEachRunAndEmpties verifies that a processing run
|
// TestPeeringHandlerProcessesEachRunAndEmpties verifies that a processing run
|
||||||
// empties the path map (the swap) and that a path seen again after a run is
|
// empties the path map (the swap) and that a path seen again after a run is
|
||||||
// recorded in the next run too.
|
// recorded in the next run too.
|
||||||
@@ -235,7 +197,7 @@ func TestPeeringHandlerStopWritesTheRemainingPaths(t *testing.T) {
|
|||||||
h := NewPeeringHandler(store, logger.New())
|
h := NewPeeringHandler(store, logger.New())
|
||||||
|
|
||||||
handle(h, time.Now().UTC(), testASNA, testASNB)
|
handle(h, time.Now().UTC(), testASNA, testASNB)
|
||||||
h.Stop(t.Context())
|
h.Stop()
|
||||||
|
|
||||||
if got := store.callCount(); got != 1 {
|
if got := store.callCount(); got != 1 {
|
||||||
t.Errorf("want 1 RecordPeering call when Stop returns, got %d", got)
|
t.Errorf("want 1 RecordPeering call when Stop returns, got %d", got)
|
||||||
@@ -259,10 +221,10 @@ func TestPeeringHandlerCapDropsAndCounts(t *testing.T) {
|
|||||||
existing := pathKey(t, testASNA, testASNB)
|
existing := pathKey(t, testASNA, testASNB)
|
||||||
|
|
||||||
h.mu.Lock()
|
h.mu.Lock()
|
||||||
h.asPaths[existing] = database.SeenTimes{First: now, Last: now}
|
h.asPaths[existing] = now
|
||||||
|
|
||||||
for i := 0; len(h.asPaths) < maxTrackedPaths; i++ {
|
for i := 0; len(h.asPaths) < maxTrackedPaths; i++ {
|
||||||
h.asPaths[strconv.Itoa(i)] = database.SeenTimes{First: now, Last: now}
|
h.asPaths[strconv.Itoa(i)] = now
|
||||||
}
|
}
|
||||||
h.mu.Unlock()
|
h.mu.Unlock()
|
||||||
|
|
||||||
@@ -292,83 +254,11 @@ func TestPeeringHandlerCapDropsAndCounts(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
h.mu.Lock()
|
h.mu.Lock()
|
||||||
gotSeen := h.asPaths[existing]
|
gotTS := h.asPaths[existing]
|
||||||
h.mu.Unlock()
|
h.mu.Unlock()
|
||||||
|
|
||||||
if !gotSeen.Last.Equal(refreshed) {
|
if !gotTS.Equal(refreshed) {
|
||||||
t.Fatalf("existing path last seen not refreshed: want %v, got %v",
|
t.Fatalf("existing path timestamp not refreshed: want %v, got %v",
|
||||||
refreshed, gotSeen.Last)
|
refreshed, gotTS)
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestPeeringHandlerRunResetsTheDroppedCount fills the map, has the handler
|
|
||||||
// drop one path, and runs the processing. The count of dropped paths is the
|
|
||||||
// run's own, so it is 0 again after the run.
|
|
||||||
func TestPeeringHandlerRunResetsTheDroppedCount(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
h := newTestHandler(newMockStore())
|
|
||||||
|
|
||||||
now := time.Now().UTC()
|
|
||||||
|
|
||||||
// Paths of one AS carry no peering, so the run records nothing.
|
|
||||||
h.mu.Lock()
|
|
||||||
for asn := 1; len(h.asPaths) < maxTrackedPaths; asn++ {
|
|
||||||
h.asPaths[pathKey(t, asn)] = database.SeenTimes{First: now, Last: now}
|
|
||||||
}
|
|
||||||
h.mu.Unlock()
|
|
||||||
|
|
||||||
handle(h, now, testASNA, testASNB)
|
|
||||||
|
|
||||||
if _, dropped := snapshot(h); dropped != 1 {
|
|
||||||
t.Fatalf("want dropped count 1 with the map full, got %d", dropped)
|
|
||||||
}
|
|
||||||
|
|
||||||
h.ProcessPeeringsNow()
|
|
||||||
|
|
||||||
if _, dropped := snapshot(h); dropped != 0 {
|
|
||||||
t.Errorf("want dropped count 0 after a run, got %d", dropped)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestPeeringHandlerStoresTheEarliestAndLatestTimesOfANewPeering handles two
|
|
||||||
// updates whose path carries a peering the database does not hold yet, the
|
|
||||||
// second 5 seconds after the first, in one run into a real database. The
|
|
||||||
// peering is stored first seen at the time of the first update and last seen
|
|
||||||
// at the time of the second.
|
|
||||||
func TestPeeringHandlerStoresTheEarliestAndLatestTimesOfANewPeering(
|
|
||||||
t *testing.T,
|
|
||||||
) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
db := newTestDatabase(t)
|
|
||||||
h := newTestHandler(db)
|
|
||||||
|
|
||||||
first := peerAnnouncement(testPeerIP, testV4Prefix)
|
|
||||||
second := peerAnnouncement(testPeerIP, testV4Prefix)
|
|
||||||
second.ParsedTimestamp = first.ParsedTimestamp.Add(secondUpdateAfter)
|
|
||||||
|
|
||||||
h.HandleMessage(first)
|
|
||||||
h.HandleMessage(second)
|
|
||||||
h.ProcessPeeringsNow()
|
|
||||||
|
|
||||||
peers, err := db.GetASPeers(testASNA)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("GetASPeers(%d): %v", testASNA, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
if len(peers) != 1 || peers[0].ASN != testASNB {
|
|
||||||
t.Fatalf("peers of AS%d = %+v, want AS%d alone", testASNA, peers,
|
|
||||||
testASNB)
|
|
||||||
}
|
|
||||||
|
|
||||||
if !peers[0].FirstSeen.Equal(first.ParsedTimestamp) {
|
|
||||||
t.Errorf("first_seen of peering AS%d-AS%d = %s, want %s", testASNA,
|
|
||||||
testASNB, peers[0].FirstSeen, first.ParsedTimestamp)
|
|
||||||
}
|
|
||||||
|
|
||||||
if !peers[0].LastSeen.Equal(second.ParsedTimestamp) {
|
|
||||||
t.Errorf("last_seen of peering AS%d-AS%d = %s, want %s", testASNA,
|
|
||||||
testASNB, peers[0].LastSeen, second.ParsedTimestamp)
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,7 +1,6 @@
|
|||||||
package routewatch
|
package routewatch
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
|
||||||
"net"
|
"net"
|
||||||
"net/netip"
|
"net/netip"
|
||||||
"slices"
|
"slices"
|
||||||
@@ -13,7 +12,6 @@ import (
|
|||||||
"git.eeqj.de/sneak/routewatch/internal/logger"
|
"git.eeqj.de/sneak/routewatch/internal/logger"
|
||||||
"git.eeqj.de/sneak/routewatch/internal/metrics"
|
"git.eeqj.de/sneak/routewatch/internal/metrics"
|
||||||
"git.eeqj.de/sneak/routewatch/internal/ristypes"
|
"git.eeqj.de/sneak/routewatch/internal/ristypes"
|
||||||
"git.eeqj.de/sneak/routewatch/internal/routetable"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
const (
|
const (
|
||||||
@@ -30,77 +28,41 @@ const (
|
|||||||
// prefixBatchMaxSize is the most prefix updates the batch holds while the
|
// prefixBatchMaxSize is the most prefix updates the batch holds while the
|
||||||
// flush goroutine writes the previous one: ten batches, about 60 MiB
|
// flush goroutine writes the previous one: ten batches, about 60 MiB
|
||||||
// (README "Memory"). At this size HandleMessage waits until the flush
|
// (README "Memory"). At this size HandleMessage waits until the flush
|
||||||
// goroutine takes the batch. On top of these, the handler holds at most
|
// goroutine takes the batch.
|
||||||
// prefixBatchSize updates, about 6 MiB, that a flush kept for the peers
|
|
||||||
// whose route delete failed (held).
|
|
||||||
prefixBatchMaxSize = 250000
|
prefixBatchMaxSize = 250000
|
||||||
|
|
||||||
// prefixBatchTimeout is the maximum time to wait before flushing a batch
|
// prefixBatchTimeout is the maximum time to wait before flushing a batch
|
||||||
// DO NOT reduce this timeout - larger batches are more efficient
|
// DO NOT reduce this timeout - larger batches are more efficient
|
||||||
prefixBatchTimeout = 1 * time.Second
|
prefixBatchTimeout = 1 * time.Second
|
||||||
|
|
||||||
// maxTableChangesPerWrite is the most changed prefix and origin AS pairs
|
|
||||||
// of the routing table that one flush writes; the rest wait for the next
|
|
||||||
// flushes. The change of one pair takes about 430 bytes of memory and 15
|
|
||||||
// microseconds to write. So a write holds about 9 MB and takes about a
|
|
||||||
// third of the one-second flush interval, which leaves the rest to the
|
|
||||||
// batch. The feed peaks at about 5,000 updates a second, each changing at
|
|
||||||
// most one pair. So the million pairs that a full-table peer's session
|
|
||||||
// going down changes are written in about a minute.
|
|
||||||
maxTableChangesPerWrite = 20000
|
|
||||||
|
|
||||||
// IP version constants
|
// IP version constants
|
||||||
ipv4Version = 4
|
ipv4Version = 4
|
||||||
ipv6Version = 6
|
ipv6Version = 6
|
||||||
)
|
)
|
||||||
|
|
||||||
// PrefixHandler tracks BGP prefixes and maintains a live routing table in the
|
// PrefixHandler tracks BGP prefixes and maintains a live routing table in the
|
||||||
// database. Routes are added on announcement and deleted on withdrawal or on an
|
// database. Routes are added on announcement and deleted on withdrawal, and all
|
||||||
// announcement whose AS path is empty or holds AS 0 anywhere, and all routes of
|
// routes of a peer are deleted when its BGP session goes down.
|
||||||
// a peer are deleted when its BGP session goes down.
|
|
||||||
//
|
|
||||||
// It applies the same updates to the routing table in memory as it handles
|
|
||||||
// them, and writes the table's changes to the database with each flush.
|
|
||||||
type PrefixHandler struct {
|
type PrefixHandler struct {
|
||||||
db database.Store
|
db database.Store
|
||||||
logger *logger.Logger
|
logger *logger.Logger
|
||||||
metrics *metrics.Tracker
|
metrics *metrics.Tracker
|
||||||
table *routetable.Table
|
|
||||||
|
|
||||||
// Batching. mu guards batch, held, peersDown, writing, deleting and
|
// Batching. mu guards batch and peersDown only: the flush goroutine takes
|
||||||
// writingTableChanges only: the flush goroutine takes them under mu and
|
// both under mu and writes them after releasing mu, so HandleMessage
|
||||||
// writes them after releasing mu, so HandleMessage waits for a database
|
// waits for a database write only while batch is at prefixBatchMaxSize.
|
||||||
// write only while batch is at prefixBatchMaxSize.
|
|
||||||
mu sync.Mutex
|
mu sync.Mutex
|
||||||
batch []prefixUpdate
|
batch []prefixUpdate
|
||||||
// held holds the updates that the peers whose route delete failed in the
|
// peersDown holds the peers whose BGP session went down since the batch
|
||||||
// last flush sent after their session went down, for the next flush to
|
// was taken.
|
||||||
// write after it retries the delete. They are not in batch, so they do
|
|
||||||
// not count toward a full batch: a delete that keeps failing is retried
|
|
||||||
// at the pace of the flush timer.
|
|
||||||
held []prefixUpdate
|
|
||||||
// peersDown holds, each once, the peers whose BGP session went down
|
|
||||||
// since the batch was taken, and those whose route delete failed in the
|
|
||||||
// last flush.
|
|
||||||
peersDown []string
|
peersDown []string
|
||||||
// writing is how many updates the flush goroutine took and has not
|
|
||||||
// finished writing.
|
|
||||||
writing int
|
|
||||||
// deleting is how many peers from peersDown the flush goroutine took and
|
|
||||||
// has not yet deleted the routes of.
|
|
||||||
deleting int
|
|
||||||
// writingTableChanges is how many changes the flush goroutine took from
|
|
||||||
// the routing table and has not finished writing.
|
|
||||||
writingTableChanges int
|
|
||||||
// batchTaken is signalled when the flush goroutine takes batch; its L is
|
// batchTaken is signalled when the flush goroutine takes batch; its L is
|
||||||
// &mu.
|
// &mu.
|
||||||
batchTaken sync.Cond
|
batchTaken sync.Cond
|
||||||
batchFullCh chan struct{} // wakes the flush goroutine when batch is full
|
batchFullCh chan struct{} // wakes the flush goroutine when batch is full
|
||||||
lastFlush time.Time // used only by the flush goroutine
|
lastFlush time.Time // used only by the flush goroutine
|
||||||
// stopCh hands Stop's context to the flush goroutine. It holds one, so
|
stopCh chan struct{}
|
||||||
// Stop does not wait for a write in progress to hand it over.
|
wg sync.WaitGroup
|
||||||
stopCh chan context.Context
|
|
||||||
done chan struct{} // closed when the flush goroutine returns
|
|
||||||
}
|
}
|
||||||
|
|
||||||
type prefixUpdate struct {
|
type prefixUpdate struct {
|
||||||
@@ -115,46 +77,21 @@ type prefixUpdate struct {
|
|||||||
endsInSet bool
|
endsInSet bool
|
||||||
}
|
}
|
||||||
|
|
||||||
// announcesRoute says whether u gives the peer a route to the prefix: an
|
// NewPrefixHandler creates a new batched prefix tracking handler
|
||||||
// announcement whose path gives an origin, or ends in an AS set of several
|
func NewPrefixHandler(db database.Store, logger *logger.Logger) *PrefixHandler {
|
||||||
// members, which gives origin 0 (none). Any other update removes the peer's
|
|
||||||
// route for the prefix: a withdrawal, or an announcement whose path is empty
|
|
||||||
// or holds AS 0 anywhere (the decoder empties such a path). Such an
|
|
||||||
// announcement replaces the peer's route, as any announcement does, with none
|
|
||||||
// (RFC 7607 treats AS 0 in a path as a withdrawal).
|
|
||||||
func (u prefixUpdate) announcesRoute() bool {
|
|
||||||
return u.messageType == "announcement" && (u.originASN > 0 || u.endsInSet)
|
|
||||||
}
|
|
||||||
|
|
||||||
// NewPrefixHandler creates a new batched prefix tracking handler, which applies
|
|
||||||
// the updates to table, the routing table read from db at startup.
|
|
||||||
func NewPrefixHandler(
|
|
||||||
db database.Store, logger *logger.Logger, table *routetable.Table,
|
|
||||||
) *PrefixHandler {
|
|
||||||
return newPrefixHandler(db, logger, table, prefixBatchTimeout)
|
|
||||||
}
|
|
||||||
|
|
||||||
// newPrefixHandler is NewPrefixHandler with the flush timer's interval,
|
|
||||||
// batchTimeout, in place of prefixBatchTimeout. A test that the timer must not
|
|
||||||
// disturb gives an interval longer than the test runs.
|
|
||||||
func newPrefixHandler(
|
|
||||||
db database.Store, logger *logger.Logger, table *routetable.Table,
|
|
||||||
batchTimeout time.Duration,
|
|
||||||
) *PrefixHandler {
|
|
||||||
h := &PrefixHandler{
|
h := &PrefixHandler{
|
||||||
db: db,
|
db: db,
|
||||||
logger: logger,
|
logger: logger,
|
||||||
table: table,
|
|
||||||
batch: make([]prefixUpdate, 0, prefixBatchSize),
|
batch: make([]prefixUpdate, 0, prefixBatchSize),
|
||||||
batchFullCh: make(chan struct{}, 1),
|
batchFullCh: make(chan struct{}, 1),
|
||||||
lastFlush: time.Now(),
|
lastFlush: time.Now(),
|
||||||
stopCh: make(chan context.Context, 1),
|
stopCh: make(chan struct{}),
|
||||||
done: make(chan struct{}),
|
|
||||||
}
|
}
|
||||||
h.batchTaken.L = &h.mu
|
h.batchTaken.L = &h.mu
|
||||||
|
|
||||||
// Start the flush timer goroutine
|
// Start the flush timer goroutine
|
||||||
go h.flushLoop(batchTimeout)
|
h.wg.Add(1)
|
||||||
|
go h.flushLoop()
|
||||||
|
|
||||||
return h
|
return h
|
||||||
}
|
}
|
||||||
@@ -208,22 +145,18 @@ func (h *PrefixHandler) HandleMessage(msg *ristypes.RISMessage) {
|
|||||||
h.batchTaken.Wait()
|
h.batchTaken.Wait()
|
||||||
}
|
}
|
||||||
|
|
||||||
// Withdrawals are queued, and applied to the routing table, before
|
// Withdrawals are queued before announcements, because the batch keeps
|
||||||
// announcements, because the batch keeps each peer's last update for a
|
// each peer's last update for a prefix: a message that lists a prefix as
|
||||||
// prefix: a message that lists a prefix as both withdrawn and announced
|
// both withdrawn and announced keeps the route (RFC 4271 section 4.3).
|
||||||
// keeps the route (RFC 4271 section 4.3). A withdrawal names only the
|
// A withdrawal names only the prefix and peer: the message's AS path, and
|
||||||
// prefix and peer: the message's AS path, and so its origin AS, belongs
|
// so its origin AS, belongs to the announced prefixes.
|
||||||
// to the announced prefixes.
|
|
||||||
for _, prefix := range msg.Withdrawals {
|
for _, prefix := range msg.Withdrawals {
|
||||||
update := prefixUpdate{
|
h.batch = append(h.batch, prefixUpdate{
|
||||||
prefix: canonicalPrefix(prefix),
|
prefix: canonicalPrefix(prefix),
|
||||||
peer: msg.Peer,
|
peer: msg.Peer,
|
||||||
messageType: "withdrawal",
|
messageType: "withdrawal",
|
||||||
timestamp: timestamp,
|
timestamp: timestamp,
|
||||||
}
|
})
|
||||||
h.batch = append(h.batch, update)
|
|
||||||
h.applyToTable(update)
|
|
||||||
|
|
||||||
// Record withdrawal in metrics
|
// Record withdrawal in metrics
|
||||||
if h.metrics != nil {
|
if h.metrics != nil {
|
||||||
h.metrics.RecordWithdrawal()
|
h.metrics.RecordWithdrawal()
|
||||||
@@ -232,7 +165,7 @@ func (h *PrefixHandler) HandleMessage(msg *ristypes.RISMessage) {
|
|||||||
|
|
||||||
for _, announcement := range msg.Announcements {
|
for _, announcement := range msg.Announcements {
|
||||||
for _, prefix := range announcement.Prefixes {
|
for _, prefix := range announcement.Prefixes {
|
||||||
update := prefixUpdate{
|
h.batch = append(h.batch, prefixUpdate{
|
||||||
prefix: canonicalPrefix(prefix),
|
prefix: canonicalPrefix(prefix),
|
||||||
originASN: originASN,
|
originASN: originASN,
|
||||||
peer: msg.Peer,
|
peer: msg.Peer,
|
||||||
@@ -240,10 +173,7 @@ func (h *PrefixHandler) HandleMessage(msg *ristypes.RISMessage) {
|
|||||||
timestamp: timestamp,
|
timestamp: timestamp,
|
||||||
path: path,
|
path: path,
|
||||||
endsInSet: len(msg.Path.Set) > 0,
|
endsInSet: len(msg.Path.Set) > 0,
|
||||||
}
|
})
|
||||||
h.batch = append(h.batch, update)
|
|
||||||
h.applyToTable(update)
|
|
||||||
|
|
||||||
// Record announcement in metrics
|
// Record announcement in metrics
|
||||||
if h.metrics != nil {
|
if h.metrics != nil {
|
||||||
h.metrics.RecordAnnouncement()
|
h.metrics.RecordAnnouncement()
|
||||||
@@ -262,131 +192,64 @@ func (h *PrefixHandler) HandleMessage(msg *ristypes.RISMessage) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Stop stops the flush goroutine, which makes one last flush before it returns,
|
// Stop gracefully stops the handler and flushes remaining batches
|
||||||
// and waits for it until ctx ends. That flush writes the updates and peer downs
|
func (h *PrefixHandler) Stop() {
|
||||||
// the handler holds, and at most maxTableChangesPerWrite of the routing table's
|
close(h.stopCh)
|
||||||
// changes. Once ctx has ended the flush goroutine skips that last flush, and a
|
h.wg.Wait()
|
||||||
// write still running is lost when the process exits. A peer whose route delete
|
|
||||||
// fails in the last flush is not retried either, nor is a failed write of the
|
|
||||||
// table's changes. Stop logs how many updates are not written, how many peer
|
|
||||||
// downs have not had their routes deleted, and how many of the table's changes
|
|
||||||
// are not written.
|
|
||||||
func (h *PrefixHandler) Stop(ctx context.Context) {
|
|
||||||
h.stopCh <- ctx
|
|
||||||
|
|
||||||
select {
|
|
||||||
case <-h.done:
|
|
||||||
case <-ctx.Done():
|
|
||||||
}
|
|
||||||
|
|
||||||
h.mu.Lock()
|
|
||||||
unwritten := h.writing + len(h.batch) + len(h.held)
|
|
||||||
peersDown := h.deleting + len(h.peersDown)
|
|
||||||
tableChanges := h.writingTableChanges + h.table.ChangeCount()
|
|
||||||
h.mu.Unlock()
|
|
||||||
|
|
||||||
if unwritten == 0 && peersDown == 0 && tableChanges == 0 {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
if ctx.Err() != nil {
|
|
||||||
h.logger.Error("Stop deadline passed before the prefix handler wrote "+
|
|
||||||
"its updates", "updates", unwritten, "peers_down", peersDown,
|
|
||||||
"table_changes", tableChanges)
|
|
||||||
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
h.logger.Error("The prefix handler stopped before it wrote everything",
|
|
||||||
"updates", unwritten, "peers_down", peersDown,
|
|
||||||
"table_changes", tableChanges)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// peerSessionDown notes that the peer's BGP session went down, so that the next
|
// peerSessionDown notes that the peer's BGP session went down, so that the next
|
||||||
// flush deletes all of the peer's live routes before it writes the batch. The
|
// flush deletes all of the peer's live routes before it writes the batch. The
|
||||||
// peer's updates still in the batch or held came before the down, which
|
// peer's updates still in the batch came before the down, which replaces them,
|
||||||
// replaces them, so they are dropped; updates the peer sends later stay in the
|
// so they are dropped; updates the peer sends later stay in the batch and are
|
||||||
// batch and are written after the delete. A peer already in peersDown is not
|
// written after the delete.
|
||||||
// added again: one delete removes all of its routes. In the routing table in
|
|
||||||
// memory the down takes effect at once.
|
|
||||||
func (h *PrefixHandler) peerSessionDown(peer string) {
|
func (h *PrefixHandler) peerSessionDown(peer string) {
|
||||||
h.mu.Lock()
|
h.mu.Lock()
|
||||||
defer h.mu.Unlock()
|
defer h.mu.Unlock()
|
||||||
|
|
||||||
h.table.PeerDown(peer)
|
h.batch = slices.DeleteFunc(h.batch, func(update prefixUpdate) bool {
|
||||||
|
return update.peer == peer
|
||||||
isPeers := func(update prefixUpdate) bool { return update.peer == peer }
|
})
|
||||||
h.batch = slices.DeleteFunc(h.batch, isPeers)
|
h.peersDown = append(h.peersDown, peer)
|
||||||
h.held = slices.DeleteFunc(h.held, isPeers)
|
|
||||||
|
|
||||||
if !slices.Contains(h.peersDown, peer) {
|
|
||||||
h.peersDown = append(h.peersDown, peer)
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// flushLoop runs in a goroutine, the only one that writes the batch: it
|
// flushLoop runs in a goroutine, the only one that writes the batch: it
|
||||||
// flushes when the batch is full, every batchTimeout, and once more on Stop.
|
// flushes when the batch is full, periodically, and once more on Stop.
|
||||||
func (h *PrefixHandler) flushLoop(batchTimeout time.Duration) {
|
func (h *PrefixHandler) flushLoop() {
|
||||||
defer close(h.done)
|
defer h.wg.Done()
|
||||||
|
|
||||||
ticker := time.NewTicker(batchTimeout)
|
ticker := time.NewTicker(prefixBatchTimeout)
|
||||||
defer ticker.Stop()
|
defer ticker.Stop()
|
||||||
|
|
||||||
for {
|
for {
|
||||||
// Stop goes first. A write can run past the stop deadline while a full
|
|
||||||
// batch's wake-up or a tick waits beside Stop's context. The select
|
|
||||||
// below picks one of them at random, but past the deadline no flush
|
|
||||||
// may start.
|
|
||||||
select {
|
|
||||||
case ctx := <-h.stopCh:
|
|
||||||
// Final flush, unless the stop deadline has passed
|
|
||||||
if ctx.Err() == nil {
|
|
||||||
h.flushBatch()
|
|
||||||
}
|
|
||||||
|
|
||||||
return
|
|
||||||
default:
|
|
||||||
}
|
|
||||||
|
|
||||||
select {
|
select {
|
||||||
case <-ticker.C:
|
case <-ticker.C:
|
||||||
if time.Since(h.lastFlush) >= batchTimeout {
|
if time.Since(h.lastFlush) >= prefixBatchTimeout {
|
||||||
h.flushBatch()
|
h.flushBatch()
|
||||||
}
|
}
|
||||||
case <-h.batchFullCh:
|
case <-h.batchFullCh:
|
||||||
h.flushBatch()
|
h.flushBatch()
|
||||||
case ctx := <-h.stopCh:
|
case <-h.stopCh:
|
||||||
// Final flush, unless the stop deadline has passed
|
// Final flush
|
||||||
if ctx.Err() == nil {
|
h.flushBatch()
|
||||||
h.flushBatch()
|
|
||||||
}
|
|
||||||
|
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// takeBatch returns the pending updates, the held ones first, and the peers in
|
// takeBatch returns the pending updates and the peers whose session went down
|
||||||
// peersDown, and leaves an empty batch, no held updates and no peers in their
|
// since the last flush, and leaves an empty batch and no peers in their place
|
||||||
// place for HandleMessage to fill. The updates count in writing until
|
// for HandleMessage to fill.
|
||||||
// flushBatch has written them, and the peers in deleting until their routes
|
|
||||||
// are deleted.
|
|
||||||
func (h *PrefixHandler) takeBatch() ([]prefixUpdate, []string) {
|
func (h *PrefixHandler) takeBatch() ([]prefixUpdate, []string) {
|
||||||
h.mu.Lock()
|
h.mu.Lock()
|
||||||
defer h.mu.Unlock()
|
defer h.mu.Unlock()
|
||||||
|
|
||||||
if len(h.held) == 0 && len(h.batch) == 0 && len(h.peersDown) == 0 {
|
if len(h.batch) == 0 && len(h.peersDown) == 0 {
|
||||||
return nil, nil
|
return nil, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
batch, peersDown := h.batch, h.peersDown
|
batch, peersDown := h.batch, h.peersDown
|
||||||
if len(h.held) > 0 {
|
|
||||||
batch = slices.Concat(h.held, h.batch)
|
|
||||||
}
|
|
||||||
|
|
||||||
h.writing = len(batch)
|
|
||||||
h.deleting = len(peersDown)
|
|
||||||
h.held = nil
|
|
||||||
h.batch = make([]prefixUpdate, 0, prefixBatchSize)
|
h.batch = make([]prefixUpdate, 0, prefixBatchSize)
|
||||||
h.peersDown = nil
|
h.peersDown = nil
|
||||||
h.batchTaken.Broadcast()
|
h.batchTaken.Broadcast()
|
||||||
@@ -394,11 +257,9 @@ func (h *PrefixHandler) takeBatch() ([]prefixUpdate, []string) {
|
|||||||
return batch, peersDown
|
return batch, peersDown
|
||||||
}
|
}
|
||||||
|
|
||||||
// flushBatch writes the routing table's changes, then takes the pending batch
|
// flushBatch takes the pending batch and writes it to the database, without
|
||||||
// and writes it to the database, without holding the mutex during the writes.
|
// holding the mutex during the write.
|
||||||
func (h *PrefixHandler) flushBatch() {
|
func (h *PrefixHandler) flushBatch() {
|
||||||
h.writeTableChanges()
|
|
||||||
|
|
||||||
batch, peersDown := h.takeBatch()
|
batch, peersDown := h.takeBatch()
|
||||||
if len(batch) == 0 && len(peersDown) == 0 {
|
if len(batch) == 0 && len(peersDown) == 0 {
|
||||||
return
|
return
|
||||||
@@ -408,12 +269,8 @@ func (h *PrefixHandler) flushBatch() {
|
|||||||
batchSize := len(batch)
|
batchSize := len(batch)
|
||||||
|
|
||||||
// The routes of a peer whose session went down are deleted first: the
|
// The routes of a peer whose session went down are deleted first: the
|
||||||
// batch holds only the updates the peer sent after the down. A peer whose
|
// batch holds only the updates the peer sent after the down.
|
||||||
// delete failed is kept, with those updates, for the next flush to retry.
|
h.deleteLiveRoutesOfPeers(peersDown)
|
||||||
failedPeers := h.deleteLiveRoutesOfPeers(peersDown)
|
|
||||||
if len(failedPeers) > 0 {
|
|
||||||
batch = h.keepForNextFlush(batch, failedPeers)
|
|
||||||
}
|
|
||||||
|
|
||||||
// For each prefix and peer, keep only the last update the feed delivered.
|
// For each prefix and peer, keep only the last update the feed delivered.
|
||||||
// Updates from different peers for the same prefix are all kept. Arrival
|
// Updates from different peers for the same prefix are all kept. Arrival
|
||||||
@@ -423,19 +280,8 @@ func (h *PrefixHandler) flushBatch() {
|
|||||||
|
|
||||||
lastUpdates := make(map[prefixAndPeer]prefixUpdate)
|
lastUpdates := make(map[prefixAndPeer]prefixUpdate)
|
||||||
|
|
||||||
// Each prefix the batch names, announced or withdrawn, with the earliest
|
|
||||||
// timestamp among all of its updates, not only each peer's last one: the
|
|
||||||
// prefix tables store only first_seen, written when the prefix is new and
|
|
||||||
// never changed after.
|
|
||||||
prefixesToUpdate := make(map[string]time.Time)
|
|
||||||
|
|
||||||
for _, update := range batch {
|
for _, update := range batch {
|
||||||
lastUpdates[prefixAndPeer{update.prefix, update.peer}] = update
|
lastUpdates[prefixAndPeer{update.prefix, update.peer}] = update
|
||||||
|
|
||||||
seen, exists := prefixesToUpdate[update.prefix]
|
|
||||||
if !exists || update.timestamp.Before(seen) {
|
|
||||||
prefixesToUpdate[update.prefix] = update.timestamp
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// Collect routes to upsert and delete
|
// Collect routes to upsert and delete
|
||||||
@@ -444,18 +290,31 @@ func (h *PrefixHandler) flushBatch() {
|
|||||||
routesToDelete []database.LiveRouteDeletion
|
routesToDelete []database.LiveRouteDeletion
|
||||||
)
|
)
|
||||||
|
|
||||||
|
// Collect unique prefixes to update
|
||||||
|
prefixesToUpdate := make(map[string]time.Time)
|
||||||
|
|
||||||
for _, update := range lastUpdates {
|
for _, update := range lastUpdates {
|
||||||
if update.announcesRoute() {
|
// Track prefix for both announcements and withdrawals
|
||||||
|
seen, exists := prefixesToUpdate[update.prefix]
|
||||||
|
if !exists || update.timestamp.After(seen) {
|
||||||
|
prefixesToUpdate[update.prefix] = update.timestamp
|
||||||
|
}
|
||||||
|
|
||||||
|
// An announcement is stored when its path gives an origin, or when it
|
||||||
|
// ends in an AS set of several members: that route is stored with
|
||||||
|
// origin 0 (none). An announcement with an empty path is not stored.
|
||||||
|
if update.messageType == "announcement" &&
|
||||||
|
(update.originASN > 0 || update.endsInSet) {
|
||||||
// Create live route for batch upsert
|
// Create live route for batch upsert
|
||||||
route := h.createLiveRoute(update)
|
route := h.createLiveRoute(update)
|
||||||
if route != nil {
|
if route != nil {
|
||||||
routesToUpsert = append(routesToUpsert, route)
|
routesToUpsert = append(routesToUpsert, route)
|
||||||
}
|
}
|
||||||
} else {
|
} else if update.messageType == "withdrawal" {
|
||||||
// Parse CIDR to get IP version
|
// Parse CIDR to get IP version
|
||||||
_, ipVersion, err := parseCIDR(update.prefix)
|
_, ipVersion, err := parseCIDR(update.prefix)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
h.logger.Error("Failed to parse CIDR for route deletion",
|
h.logger.Error("Failed to parse CIDR for withdrawal",
|
||||||
"prefix", update.prefix, "error", err)
|
"prefix", update.prefix, "error", err)
|
||||||
|
|
||||||
continue
|
continue
|
||||||
@@ -472,10 +331,6 @@ func (h *PrefixHandler) flushBatch() {
|
|||||||
|
|
||||||
successCount := h.writeBatch(routesToUpsert, routesToDelete, prefixesToUpdate)
|
successCount := h.writeBatch(routesToUpsert, routesToDelete, prefixesToUpdate)
|
||||||
|
|
||||||
h.mu.Lock()
|
|
||||||
h.writing = 0
|
|
||||||
h.mu.Unlock()
|
|
||||||
|
|
||||||
elapsed := time.Since(startTime)
|
elapsed := time.Since(startTime)
|
||||||
h.logger.Debug("Flushed prefix batch",
|
h.logger.Debug("Flushed prefix batch",
|
||||||
"batch_size", batchSize,
|
"batch_size", batchSize,
|
||||||
@@ -488,151 +343,26 @@ func (h *PrefixHandler) flushBatch() {
|
|||||||
h.lastFlush = time.Now()
|
h.lastFlush = time.Now()
|
||||||
}
|
}
|
||||||
|
|
||||||
// applyToTable applies update to the routing table in memory: as an
|
|
||||||
// announcement when it gives the peer a route, otherwise as a withdrawal. A
|
|
||||||
// prefix that does not parse is left out; flushBatch logs it.
|
|
||||||
func (h *PrefixHandler) applyToTable(update prefixUpdate) {
|
|
||||||
prefix, err := netip.ParsePrefix(update.prefix)
|
|
||||||
if err != nil {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
if !update.announcesRoute() {
|
|
||||||
h.table.Withdraw(prefix, update.peer)
|
|
||||||
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
h.table.Announce(prefix, update.originASN, routetable.Route{
|
|
||||||
Peer: update.peer,
|
|
||||||
ASPath: update.path,
|
|
||||||
Time: update.timestamp,
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
// writeTableChanges writes at most maxTableChangesPerWrite changes of the
|
|
||||||
// routing table in memory, and every peer the table numbered since the last
|
|
||||||
// write.
|
|
||||||
// When the write fails, they go back to the table, so that a later write writes
|
|
||||||
// them again, each pair with its content then.
|
|
||||||
func (h *PrefixHandler) writeTableChanges() {
|
|
||||||
h.mu.Lock()
|
|
||||||
changes, newPeers := h.table.TakeChanges(maxTableChangesPerWrite)
|
|
||||||
h.writingTableChanges = len(changes)
|
|
||||||
h.mu.Unlock()
|
|
||||||
|
|
||||||
if len(changes) == 0 && len(newPeers) == 0 {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
err := h.db.WriteRouteChanges(changes, newPeers)
|
|
||||||
|
|
||||||
h.mu.Lock()
|
|
||||||
defer h.mu.Unlock()
|
|
||||||
|
|
||||||
h.writingTableChanges = 0
|
|
||||||
|
|
||||||
if err != nil {
|
|
||||||
h.logger.Error("Failed to write the routing table's changes; the next "+
|
|
||||||
"flush retries", "changes", len(changes), "new_peers", len(newPeers),
|
|
||||||
"error", err)
|
|
||||||
|
|
||||||
h.table.PutBackChanges(changes, newPeers)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// deleteLiveRoutesOfPeers deletes all live routes of each of peers, the peers
|
// deleteLiveRoutesOfPeers deletes all live routes of each of peers, the peers
|
||||||
// whose BGP session went down, and returns those whose delete failed. Most
|
// whose BGP session went down. Most downs come from peers that have no routes,
|
||||||
// downs come from peers that have no routes, so only a delete that removed
|
// so only a delete that removed routes is logged.
|
||||||
// routes is logged. A peer whose delete succeeded no longer counts in deleting;
|
func (h *PrefixHandler) deleteLiveRoutesOfPeers(peers []string) {
|
||||||
// a failed one counts until keepForNextFlush puts it back in peersDown.
|
|
||||||
func (h *PrefixHandler) deleteLiveRoutesOfPeers(peers []string) []string {
|
|
||||||
var failed []string
|
|
||||||
|
|
||||||
for _, peer := range peers {
|
for _, peer := range peers {
|
||||||
deleted, err := h.db.DeleteLiveRoutesOfPeer(peer)
|
deleted, err := h.db.DeleteLiveRoutesOfPeer(peer)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
h.logger.Error(
|
h.logger.Error(
|
||||||
"Failed to delete the live routes of a peer whose session "+
|
"Failed to delete the live routes of a peer whose session went down",
|
||||||
"went down; the next flush retries",
|
"peer", peer, "error", err)
|
||||||
"peer", peer, "routes_deleted", deleted, "error", err)
|
|
||||||
|
|
||||||
failed = append(failed, peer)
|
|
||||||
|
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
|
||||||
h.mu.Lock()
|
|
||||||
h.deleting--
|
|
||||||
h.mu.Unlock()
|
|
||||||
|
|
||||||
if deleted > 0 {
|
if deleted > 0 {
|
||||||
h.logger.Info(
|
h.logger.Info(
|
||||||
"Deleted the live routes of a peer whose session went down",
|
"Deleted the live routes of a peer whose session went down",
|
||||||
"peer", peer, "routes", deleted)
|
"peer", peer, "routes", deleted)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
return failed
|
|
||||||
}
|
|
||||||
|
|
||||||
// keepForNextFlush puts the peers whose route delete failed back in peersDown,
|
|
||||||
// and their updates from batch in held, so that the next flush retries each
|
|
||||||
// delete before it writes the peer's updates; written now, they would be
|
|
||||||
// deleted by the retry. The updates of a peer whose session went down again
|
|
||||||
// since the batch was taken came before that down, so they are dropped, as
|
|
||||||
// peerSessionDown drops them, and the peer is already in peersDown. It returns
|
|
||||||
// the updates of the other peers, for this flush to write.
|
|
||||||
//
|
|
||||||
// While a delete keeps failing, the held updates would grow with every flush,
|
|
||||||
// so only the last prefixBatchSize of them are kept and the older ones are
|
|
||||||
// dropped. This never leaves in place a route the peer withdrew: a prefix with
|
|
||||||
// a kept update keeps its last one, and the retry deletes all of the peer's
|
|
||||||
// routes, so a prefix whose updates were all dropped has no route from the peer
|
|
||||||
// until the peer announces it again.
|
|
||||||
func (h *PrefixHandler) keepForNextFlush(
|
|
||||||
batch []prefixUpdate, failedPeers []string,
|
|
||||||
) []prefixUpdate {
|
|
||||||
var kept, toWrite []prefixUpdate
|
|
||||||
|
|
||||||
for _, update := range batch {
|
|
||||||
if slices.Contains(failedPeers, update.peer) {
|
|
||||||
kept = append(kept, update)
|
|
||||||
} else {
|
|
||||||
toWrite = append(toWrite, update)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
h.mu.Lock()
|
|
||||||
defer h.mu.Unlock()
|
|
||||||
|
|
||||||
kept = slices.DeleteFunc(kept, func(update prefixUpdate) bool {
|
|
||||||
return slices.Contains(h.peersDown, update.peer)
|
|
||||||
})
|
|
||||||
|
|
||||||
if len(kept) > prefixBatchSize {
|
|
||||||
h.logger.Error(
|
|
||||||
"Dropped the oldest updates of peers whose route delete failed",
|
|
||||||
"peers", failedPeers, "dropped", len(kept)-prefixBatchSize)
|
|
||||||
|
|
||||||
// A copy, so that the dropped updates are freed.
|
|
||||||
kept = slices.Clone(kept[len(kept)-prefixBatchSize:])
|
|
||||||
}
|
|
||||||
|
|
||||||
// takeBatch left held empty, and only this flush fills it. The kept
|
|
||||||
// updates now count in held, not in writing, and the failed peers in
|
|
||||||
// peersDown, not in deleting.
|
|
||||||
h.held = kept
|
|
||||||
h.writing = len(toWrite)
|
|
||||||
h.deleting = 0
|
|
||||||
|
|
||||||
for _, peer := range failedPeers {
|
|
||||||
if !slices.Contains(h.peersDown, peer) {
|
|
||||||
h.peersDown = append(h.peersDown, peer)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return toWrite
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// writeBatch writes the collected route upserts, route deletions and prefix
|
// writeBatch writes the collected route upserts, route deletions and prefix
|
||||||
|
|||||||
@@ -1,13 +1,10 @@
|
|||||||
package routewatch
|
package routewatch
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"bytes"
|
|
||||||
"context"
|
"context"
|
||||||
"errors"
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
"net/netip"
|
|
||||||
"slices"
|
"slices"
|
||||||
"strings"
|
|
||||||
"sync"
|
"sync"
|
||||||
"testing"
|
"testing"
|
||||||
"time"
|
"time"
|
||||||
@@ -16,7 +13,6 @@ import (
|
|||||||
"git.eeqj.de/sneak/routewatch/internal/database"
|
"git.eeqj.de/sneak/routewatch/internal/database"
|
||||||
"git.eeqj.de/sneak/routewatch/internal/logger"
|
"git.eeqj.de/sneak/routewatch/internal/logger"
|
||||||
"git.eeqj.de/sneak/routewatch/internal/ristypes"
|
"git.eeqj.de/sneak/routewatch/internal/ristypes"
|
||||||
"git.eeqj.de/sneak/routewatch/internal/routetable"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
const testPeerIP = "2001:db8:ffff::1"
|
const testPeerIP = "2001:db8:ffff::1"
|
||||||
@@ -27,10 +23,6 @@ const (
|
|||||||
testV6Prefix = "2001:db8:1::/48"
|
testV6Prefix = "2001:db8:1::/48"
|
||||||
)
|
)
|
||||||
|
|
||||||
// secondUpdateAfter is how long after the first update the tests of first_seen
|
|
||||||
// and last_seen send the second, in the same batch.
|
|
||||||
const secondUpdateAfter = 5 * time.Second
|
|
||||||
|
|
||||||
// TestPrefixHandlerStoresPrefixesTheIPLookupFinds runs announcements and
|
// TestPrefixHandlerStoresPrefixesTheIPLookupFinds runs announcements and
|
||||||
// withdrawals through the prefix handler into a real database and looks the
|
// withdrawals through the prefix handler into a real database and looks the
|
||||||
// addresses up. The prefixes are written the way the feed sends them: IPv6
|
// addresses up. The prefixes are written the way the feed sends them: IPv6
|
||||||
@@ -41,7 +33,9 @@ func TestPrefixHandlerStoresPrefixesTheIPLookupFinds(t *testing.T) {
|
|||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
db := newTestDatabase(t)
|
db := newTestDatabase(t)
|
||||||
h := prefixHandlerWithoutTimer(db)
|
|
||||||
|
// Built without NewPrefixHandler's flush timer, so the test flushes itself.
|
||||||
|
h := &PrefixHandler{db: db, logger: logger.New()}
|
||||||
|
|
||||||
ts := time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC)
|
ts := time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC)
|
||||||
h.HandleMessage(&ristypes.RISMessage{
|
h.HandleMessage(&ristypes.RISMessage{
|
||||||
@@ -86,49 +80,6 @@ func TestPrefixHandlerStoresPrefixesTheIPLookupFinds(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// TestPrefixHandlerStoresANewPrefixFirstSeenAtItsEarliestUpdate announces a
|
|
||||||
// prefix the database does not hold yet twice in one batch, the second time 5
|
|
||||||
// seconds after the first, through the prefix handler into a real database. The
|
|
||||||
// prefix is stored first seen at the time of the first announcement, whether
|
|
||||||
// the two come from two peers or from one, and the IP lookup answers with that
|
|
||||||
// time.
|
|
||||||
func TestPrefixHandlerStoresANewPrefixFirstSeenAtItsEarliestUpdate(
|
|
||||||
t *testing.T,
|
|
||||||
) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
const otherPeerIP = "2001:db8:ffff::2"
|
|
||||||
|
|
||||||
tests := []struct {
|
|
||||||
name string
|
|
||||||
firstPeer, secondPeer string
|
|
||||||
}{
|
|
||||||
{name: "two peers", firstPeer: testPeerIP, secondPeer: otherPeerIP},
|
|
||||||
{name: "one peer", firstPeer: testPeerIP, secondPeer: testPeerIP},
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, tt := range tests {
|
|
||||||
t.Run(tt.name, func(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
first := peerAnnouncement(tt.firstPeer, testV4Prefix)
|
|
||||||
second := peerAnnouncement(tt.secondPeer, testV4Prefix)
|
|
||||||
second.ParsedTimestamp = first.ParsedTimestamp.Add(secondUpdateAfter)
|
|
||||||
db := handleBatches(t, [][]*ristypes.RISMessage{{first, second}})
|
|
||||||
|
|
||||||
info, err := db.GetIPInfo("192.0.2.1")
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("GetIPInfo(192.0.2.1): %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
if !info.FirstSeen.Equal(first.ParsedTimestamp) {
|
|
||||||
t.Errorf("first_seen of %s = %s, want %s", testV4Prefix,
|
|
||||||
info.FirstSeen, first.ParsedTimestamp)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestPrefixHandlerAppliesEachPeersLastUpdateInABatch runs updates for one
|
// TestPrefixHandlerAppliesEachPeersLastUpdateInABatch runs updates for one
|
||||||
// prefix through the prefix handler into a real database. All updates carry the
|
// prefix through the prefix handler into a real database. All updates carry the
|
||||||
// same timestamp, as updates within one second of the feed do. Within a batch,
|
// same timestamp, as updates within one second of the feed do. Within a batch,
|
||||||
@@ -377,132 +328,6 @@ func TestPrefixHandlerDoesNotStoreAnAnnouncementWithAnEmptyPath(t *testing.T) {
|
|||||||
assertPeers(t, db, map[string][]string{testV4Prefix: nil})
|
assertPeers(t, db, map[string][]string{testV4Prefix: nil})
|
||||||
}
|
}
|
||||||
|
|
||||||
// TestPrefixHandlerRemovesTheRouteOnAnAnnouncementWithoutAnOrigin runs, through
|
|
||||||
// the prefix handler into a real database, a peer's announcement of a prefix
|
|
||||||
// with origin AS testASNB and then the peer's announcement of the prefix with
|
|
||||||
// an AS path that gives no origin: one whose last AS is 0, an empty one, or one
|
|
||||||
// that holds AS 0 before its end, which decodes as an empty path. As any later
|
|
||||||
// announcement from the peer does, the second replaces the peer's route, here
|
|
||||||
// with none, whether it comes in a later batch or follows a withdrawal in one
|
|
||||||
// batch. The in-memory route count and prefix distribution must match the
|
|
||||||
// route table.
|
|
||||||
func TestPrefixHandlerRemovesTheRouteOnAnAnnouncementWithoutAnOrigin(
|
|
||||||
t *testing.T,
|
|
||||||
) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
const prefixMaskLength = 24
|
|
||||||
|
|
||||||
noOrigin := func(path ristypes.ASPath) *ristypes.RISMessage {
|
|
||||||
msg := peerAnnouncement(testPeerIP, testV4Prefix)
|
|
||||||
msg.Path = path
|
|
||||||
|
|
||||||
return msg
|
|
||||||
}
|
|
||||||
endsInZero := ristypes.ASPath{Sequence: []int{testASNA, 0}}
|
|
||||||
withdrawal := &ristypes.RISMessage{
|
|
||||||
Type: ristypes.MessageTypeUpdate,
|
|
||||||
Peer: testPeerIP,
|
|
||||||
ParsedTimestamp: time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC),
|
|
||||||
Withdrawals: []string{testV4Prefix},
|
|
||||||
}
|
|
||||||
|
|
||||||
tests := []struct {
|
|
||||||
name string
|
|
||||||
// Handled and flushed as one batch after the announcement with origin
|
|
||||||
// AS testASNB.
|
|
||||||
batch []*ristypes.RISMessage
|
|
||||||
}{
|
|
||||||
{
|
|
||||||
name: "a path whose last AS is 0",
|
|
||||||
batch: []*ristypes.RISMessage{noOrigin(endsInZero)},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "an empty path",
|
|
||||||
batch: []*ristypes.RISMessage{noOrigin(ristypes.ASPath{})},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "a path with AS 0 before its end",
|
|
||||||
batch: []*ristypes.RISMessage{
|
|
||||||
noOrigin(decodePath(t, testASNA, 0, testASNB)),
|
|
||||||
},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "a withdrawal and then a path whose last AS is 0",
|
|
||||||
batch: []*ristypes.RISMessage{withdrawal, noOrigin(endsInZero)},
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, tt := range tests {
|
|
||||||
t.Run(tt.name, func(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
db := handleBatches(t, [][]*ristypes.RISMessage{
|
|
||||||
{peerAnnouncement(testPeerIP, testV4Prefix)},
|
|
||||||
tt.batch,
|
|
||||||
})
|
|
||||||
checkLivePeers(t, db, testV4Prefix, prefixMaskLength, nil)
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestPrefixHandlerDoesNotStoreAPrefixThatDoesNotParse runs an announcement of
|
|
||||||
// "not-a-prefix" and a withdrawal of "also/bad" through the prefix handler into
|
|
||||||
// a real database. Neither is stored in the prefix tables: the prefix count in
|
|
||||||
// the stats stays 0, both as kept in memory and as counted from the tables when
|
|
||||||
// the database is opened again.
|
|
||||||
func TestPrefixHandlerDoesNotStoreAPrefixThatDoesNotParse(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
cfg := &config.Config{StateDir: t.TempDir()}
|
|
||||||
|
|
||||||
db, err := database.New(cfg, logger.New())
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("failed to create database: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
h := prefixHandlerWithoutTimer(db)
|
|
||||||
|
|
||||||
withdrawal := &ristypes.RISMessage{
|
|
||||||
Type: ristypes.MessageTypeUpdate,
|
|
||||||
Peer: testPeerIP,
|
|
||||||
ParsedTimestamp: time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC),
|
|
||||||
Withdrawals: []string{"also/bad"},
|
|
||||||
}
|
|
||||||
h.HandleMessage(peerAnnouncement(testPeerIP, "not-a-prefix"))
|
|
||||||
h.HandleMessage(withdrawal)
|
|
||||||
h.flushBatch()
|
|
||||||
|
|
||||||
checkNoPrefixes := func(when string) {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
stats, err := db.GetStatsContext(context.Background())
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("GetStatsContext %s: %v", when, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
if stats.Prefixes != 0 {
|
|
||||||
t.Errorf("prefix count %s = %d, want 0", when, stats.Prefixes)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
checkNoPrefixes("after the flush")
|
|
||||||
|
|
||||||
err = db.Close()
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("Close: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
db, err = database.New(cfg, logger.New())
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("failed to open the database again: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
t.Cleanup(func() { _ = db.Close() })
|
|
||||||
|
|
||||||
checkNoPrefixes("after opening the database again")
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestStateMessagesGoToThePrefixHandlerNotThePeerHandler checks that STATE
|
// TestStateMessagesGoToThePrefixHandlerNotThePeerHandler checks that STATE
|
||||||
// messages go to the prefix handler, which acts on a down, and not to the peer
|
// messages go to the prefix handler, which acts on a down, and not to the peer
|
||||||
// handler, which would record a peer whose session never comes up as active.
|
// handler, which would record a peer whose session never comes up as active.
|
||||||
@@ -692,7 +517,8 @@ func TestPrefixHandlerHandlesADownWhileAFlushWrites(t *testing.T) {
|
|||||||
release: make(chan struct{}),
|
release: make(chan struct{}),
|
||||||
}
|
}
|
||||||
|
|
||||||
h := prefixHandlerWithoutTimer(store)
|
// Built without NewPrefixHandler's flush timer, so the test flushes itself.
|
||||||
|
h := &PrefixHandler{db: store, logger: logger.New()}
|
||||||
h.HandleMessage(peerAnnouncement(testPeerIP, beforeDown))
|
h.HandleMessage(peerAnnouncement(testPeerIP, beforeDown))
|
||||||
|
|
||||||
flushed := make(chan struct{})
|
flushed := make(chan struct{})
|
||||||
@@ -734,268 +560,6 @@ func TestPrefixHandlerHandlesADownWhileAFlushWrites(t *testing.T) {
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
// errTestPeerDelete is the error a test store's failing delete of a peer's
|
|
||||||
// routes returns.
|
|
||||||
var errTestPeerDelete = errors.New("test: the delete of a peer's routes failed")
|
|
||||||
|
|
||||||
// firstPeerDeleteFailsStore is a real database whose first delete of a peer's
|
|
||||||
// live routes fails, deleting nothing. deletes counts the deletes asked for.
|
|
||||||
type firstPeerDeleteFailsStore struct {
|
|
||||||
*database.Database
|
|
||||||
|
|
||||||
// whileFailing, if not nil, runs during the delete that fails.
|
|
||||||
whileFailing func()
|
|
||||||
deletes int
|
|
||||||
}
|
|
||||||
|
|
||||||
// DeleteLiveRoutesOfPeer fails the first call and deletes the peer's routes on
|
|
||||||
// every later one.
|
|
||||||
func (s *firstPeerDeleteFailsStore) DeleteLiveRoutesOfPeer(
|
|
||||||
peerIP string,
|
|
||||||
) (int, error) {
|
|
||||||
s.deletes++
|
|
||||||
if s.deletes > 1 {
|
|
||||||
return s.Database.DeleteLiveRoutesOfPeer(peerIP)
|
|
||||||
}
|
|
||||||
|
|
||||||
if s.whileFailing != nil {
|
|
||||||
s.whileFailing()
|
|
||||||
}
|
|
||||||
|
|
||||||
return 0, errTestPeerDelete
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestPrefixHandlerRetriesAFailedDeleteOfAPeersRoutes runs, through the prefix
|
|
||||||
// handler into a database whose first delete of a peer's routes fails, one flush
|
|
||||||
// after another: a peer's announcement, then its session going down and a new
|
|
||||||
// announcement from the peer, then nothing. The failed delete must be retried
|
|
||||||
// on the next flush, and the announcement the peer sent after the down written
|
|
||||||
// only after that retry, so it is kept. In the other cases the peer's session
|
|
||||||
// goes down again, and the peer announces another prefix, while the delete
|
|
||||||
// fails or after the flush whose delete failed: the announcement sent between
|
|
||||||
// the two downs must not be written, and the next flush must delete the peer's
|
|
||||||
// routes once, not once for the retry and once for the down.
|
|
||||||
func TestPrefixHandlerRetriesAFailedDeleteOfAPeersRoutes(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
const (
|
|
||||||
beforeDown = "192.0.2.0/24"
|
|
||||||
afterDown = "198.51.100.0/24"
|
|
||||||
afterSecondDown = "203.0.113.0/24"
|
|
||||||
maskLength = 24
|
|
||||||
// The delete that fails and its retry, which is also the delete for
|
|
||||||
// a second down.
|
|
||||||
wantDeletes = 2
|
|
||||||
)
|
|
||||||
|
|
||||||
tests := []struct {
|
|
||||||
name string
|
|
||||||
// Handled while the delete fails, if not nil.
|
|
||||||
whileFailing []*ristypes.RISMessage
|
|
||||||
// Handled after the flush whose delete fails, if not nil.
|
|
||||||
afterFailing []*ristypes.RISMessage
|
|
||||||
// The peers left with a live route for each prefix.
|
|
||||||
wantPeers map[string][]string
|
|
||||||
}{
|
|
||||||
{
|
|
||||||
name: "the delete fails once",
|
|
||||||
wantPeers: map[string][]string{
|
|
||||||
beforeDown: nil, afterDown: {testPeerIP}, afterSecondDown: nil,
|
|
||||||
},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "the session goes down again while the delete fails",
|
|
||||||
whileFailing: []*ristypes.RISMessage{
|
|
||||||
sessionDown(), peerAnnouncement(testPeerIP, afterSecondDown),
|
|
||||||
},
|
|
||||||
wantPeers: map[string][]string{
|
|
||||||
beforeDown: nil, afterDown: nil, afterSecondDown: {testPeerIP},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "the session goes down again after the delete failed",
|
|
||||||
afterFailing: []*ristypes.RISMessage{
|
|
||||||
sessionDown(), peerAnnouncement(testPeerIP, afterSecondDown),
|
|
||||||
},
|
|
||||||
wantPeers: map[string][]string{
|
|
||||||
beforeDown: nil, afterDown: nil, afterSecondDown: {testPeerIP},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, tt := range tests {
|
|
||||||
t.Run(tt.name, func(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
db := newTestDatabase(t)
|
|
||||||
store := &firstPeerDeleteFailsStore{Database: db}
|
|
||||||
h := prefixHandlerWithoutTimer(store)
|
|
||||||
store.whileFailing = func() {
|
|
||||||
for _, msg := range tt.whileFailing {
|
|
||||||
h.HandleMessage(msg)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
h.HandleMessage(peerAnnouncement(testPeerIP, beforeDown))
|
|
||||||
h.flushBatch()
|
|
||||||
h.HandleMessage(sessionDown())
|
|
||||||
h.HandleMessage(peerAnnouncement(testPeerIP, afterDown))
|
|
||||||
h.flushBatch() // the delete fails
|
|
||||||
|
|
||||||
for _, msg := range tt.afterFailing {
|
|
||||||
h.HandleMessage(msg)
|
|
||||||
}
|
|
||||||
|
|
||||||
h.flushBatch()
|
|
||||||
|
|
||||||
if store.deletes != wantDeletes {
|
|
||||||
t.Errorf("the store was asked for %d deletes of the peer's "+
|
|
||||||
"routes, want %d", store.deletes, wantDeletes)
|
|
||||||
}
|
|
||||||
|
|
||||||
assertPeers(t, db, tt.wantPeers)
|
|
||||||
assertCounts(t, db, ipv4Version, 1, []database.PrefixDistribution{
|
|
||||||
{MaskLength: maskLength, Count: 1},
|
|
||||||
})
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// peerDeleteAlwaysFailsStore is a real database whose every delete of a peer's
|
|
||||||
// live routes fails, deleting nothing.
|
|
||||||
type peerDeleteAlwaysFailsStore struct {
|
|
||||||
*database.Database
|
|
||||||
}
|
|
||||||
|
|
||||||
// DeleteLiveRoutesOfPeer fails.
|
|
||||||
func (peerDeleteAlwaysFailsStore) DeleteLiveRoutesOfPeer(string) (int, error) {
|
|
||||||
return 0, errTestPeerDelete
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestPrefixHandlerKeepsAtMostABatchOfUpdatesWhileADeleteKeepsFailing runs,
|
|
||||||
// through the prefix handler into a database whose every delete of a peer's
|
|
||||||
// routes fails, a peer's session going down and then three flushes. Before each
|
|
||||||
// flush the peer announces prefixBatchSize new prefixes and another peer
|
|
||||||
// announces one. After each flush the handler must hold only the peer's last
|
|
||||||
// prefixBatchSize updates, not every update since the down, and the other
|
|
||||||
// peer's route must be written. After the last flush, one more update must not
|
|
||||||
// wake the flush goroutine: held updates do not count toward a full batch.
|
|
||||||
func TestPrefixHandlerKeepsAtMostABatchOfUpdatesWhileADeleteKeepsFailing(
|
|
||||||
t *testing.T,
|
|
||||||
) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
const otherPeerIP = "2001:db8:ffff::2"
|
|
||||||
|
|
||||||
// One prefix the other peer announces before each flush.
|
|
||||||
otherPeersPrefixes := []string{testV4Prefix, testV6Prefix, "2001:db8:2::/48"}
|
|
||||||
|
|
||||||
db := newTestDatabase(t)
|
|
||||||
h := prefixHandlerWithoutTimer(peerDeleteAlwaysFailsStore{Database: db})
|
|
||||||
h.HandleMessage(sessionDown())
|
|
||||||
|
|
||||||
// The peer announces a /32 for each address from 10.0.0.0 on, each
|
|
||||||
// address once.
|
|
||||||
addr := netip.MustParseAddr("10.0.0.0")
|
|
||||||
|
|
||||||
for flush, otherPeersPrefix := range otherPeersPrefixes {
|
|
||||||
prefixes := make([]string, prefixBatchSize)
|
|
||||||
for i := range prefixes {
|
|
||||||
prefixes[i] = netip.PrefixFrom(addr, addr.BitLen()).String()
|
|
||||||
addr = addr.Next()
|
|
||||||
}
|
|
||||||
|
|
||||||
h.HandleMessage(peerAnnouncement(testPeerIP, prefixes...))
|
|
||||||
h.HandleMessage(peerAnnouncement(otherPeerIP, otherPeersPrefix))
|
|
||||||
h.flushBatch()
|
|
||||||
|
|
||||||
kept := make([]string, 0, len(h.held))
|
|
||||||
for _, update := range h.held {
|
|
||||||
kept = append(kept, update.prefix)
|
|
||||||
}
|
|
||||||
|
|
||||||
if !slices.Equal(kept, prefixes) {
|
|
||||||
t.Fatalf("after flush %d the handler holds %d updates, not just "+
|
|
||||||
"the peer's last %d", flush+1, len(kept), len(prefixes))
|
|
||||||
}
|
|
||||||
|
|
||||||
assertPeers(t, db, map[string][]string{otherPeersPrefix: {otherPeerIP}})
|
|
||||||
}
|
|
||||||
|
|
||||||
// A new channel, so that a wake-up in it comes from the update below.
|
|
||||||
h.batchFullCh = make(chan struct{}, 1)
|
|
||||||
h.HandleMessage(peerAnnouncement(otherPeerIP, testV4Prefix))
|
|
||||||
|
|
||||||
if len(h.batchFullCh) > 0 {
|
|
||||||
t.Error("one update after a flush that held a batch of updates " +
|
|
||||||
"woke the flush goroutine")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestPrefixHandlerStopLogsAFailedDeleteWithoutTheDeadline stops a prefix
|
|
||||||
// handler, with no deadline, after a peer's session went down and the peer
|
|
||||||
// announced a prefix, into a database whose every delete of a peer's routes
|
|
||||||
// fails. Stop must log the peer down and the announcement as not written, and
|
|
||||||
// must not say that the stop deadline passed.
|
|
||||||
func TestPrefixHandlerStopLogsAFailedDeleteWithoutTheDeadline(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
var logs bytes.Buffer
|
|
||||||
|
|
||||||
h := NewPrefixHandler(peerDeleteAlwaysFailsStore{Database: newTestDatabase(t)},
|
|
||||||
newErrorLogger(&logs), routetable.New(routetable.Params{}))
|
|
||||||
h.HandleMessage(sessionDown())
|
|
||||||
h.HandleMessage(peerAnnouncement(testPeerIP, testV4Prefix))
|
|
||||||
h.Stop(t.Context())
|
|
||||||
|
|
||||||
logged := lastLogged(t, &logs)
|
|
||||||
if strings.Contains(logged.Msg, "deadline") {
|
|
||||||
t.Errorf("Stop logged %q although its deadline had not passed",
|
|
||||||
logged.Msg)
|
|
||||||
}
|
|
||||||
|
|
||||||
if logged.Updates != 1 || logged.PeersDown != 1 {
|
|
||||||
t.Errorf("Stop logged %d updates and %d peer downs not written, "+
|
|
||||||
"want 1 and 1", logged.Updates, logged.PeersDown)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestPrefixHandlerStopLogsNothingAfterADeleteThatSucceeds stops a prefix
|
|
||||||
// handler, with no deadline, after a peer's session went down, into a database
|
|
||||||
// whose delete of the peer's routes succeeds. Stop must log nothing: a peer
|
|
||||||
// whose routes are deleted no longer counts as a peer down left undone.
|
|
||||||
func TestPrefixHandlerStopLogsNothingAfterADeleteThatSucceeds(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
var logs bytes.Buffer
|
|
||||||
|
|
||||||
h := NewPrefixHandler(newTestDatabase(t), newErrorLogger(&logs),
|
|
||||||
routetable.New(routetable.Params{}))
|
|
||||||
h.HandleMessage(sessionDown())
|
|
||||||
h.Stop(t.Context())
|
|
||||||
|
|
||||||
if logs.Len() != 0 {
|
|
||||||
t.Errorf("Stop logged %q after the routes of the peer whose session "+
|
|
||||||
"went down were deleted", logs.String())
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// longerThanAnyTest is a flush timer interval that no test lasts. A test whose
|
|
||||||
// prefix handler must flush only when its batch is full and on Stop gives it to
|
|
||||||
// newPrefixHandler.
|
|
||||||
const longerThanAnyTest = time.Hour
|
|
||||||
|
|
||||||
// prefixHandlerWithoutTimer returns a prefix handler that writes to store and
|
|
||||||
// applies the updates to an empty routing table. It is built without
|
|
||||||
// NewPrefixHandler's flush goroutine, so the test flushes itself.
|
|
||||||
func prefixHandlerWithoutTimer(store database.Store) *PrefixHandler {
|
|
||||||
return &PrefixHandler{
|
|
||||||
db: store,
|
|
||||||
logger: logger.New(),
|
|
||||||
table: routetable.New(routetable.Params{}),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// newTestDatabase opens a database in a new temporary directory. It is closed
|
// newTestDatabase opens a database in a new temporary directory. It is closed
|
||||||
// when the test ends.
|
// when the test ends.
|
||||||
func newTestDatabase(t *testing.T) *database.Database {
|
func newTestDatabase(t *testing.T) *database.Database {
|
||||||
@@ -1019,7 +583,9 @@ func handleBatches(
|
|||||||
t.Helper()
|
t.Helper()
|
||||||
|
|
||||||
db := newTestDatabase(t)
|
db := newTestDatabase(t)
|
||||||
h := prefixHandlerWithoutTimer(db)
|
|
||||||
|
// Built without NewPrefixHandler's flush timer, so the test flushes itself.
|
||||||
|
h := &PrefixHandler{db: db, logger: logger.New()}
|
||||||
|
|
||||||
for _, batch := range batches {
|
for _, batch := range batches {
|
||||||
for _, msg := range batch {
|
for _, msg := range batch {
|
||||||
|
|||||||
@@ -1,534 +0,0 @@
|
|||||||
package routewatch
|
|
||||||
|
|
||||||
import (
|
|
||||||
"bytes"
|
|
||||||
"encoding/json"
|
|
||||||
"errors"
|
|
||||||
"log/slog"
|
|
||||||
"net/netip"
|
|
||||||
"reflect"
|
|
||||||
"slices"
|
|
||||||
"testing"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"git.eeqj.de/sneak/routewatch/internal/database"
|
|
||||||
"git.eeqj.de/sneak/routewatch/internal/logger"
|
|
||||||
"git.eeqj.de/sneak/routewatch/internal/ristypes"
|
|
||||||
"git.eeqj.de/sneak/routewatch/internal/routetable"
|
|
||||||
)
|
|
||||||
|
|
||||||
// More peers and prefixes for the tests of the routing table in memory.
|
|
||||||
const (
|
|
||||||
secondPeerIP = "2001:db8:ffff::2"
|
|
||||||
thirdPeerIP = "2001:db8:ffff::3"
|
|
||||||
fourthPeerIP = "192.0.2.4"
|
|
||||||
fifthPeerIP = "192.0.2.5"
|
|
||||||
|
|
||||||
otherV4Prefix = "198.51.100.0/24"
|
|
||||||
setV4Prefix = "203.0.113.0/24"
|
|
||||||
|
|
||||||
// firstManyPrefix is the first prefix announceManyPrefixes announces.
|
|
||||||
firstManyPrefix = "10.0.0.0/32"
|
|
||||||
)
|
|
||||||
|
|
||||||
// tableWriteTimeout is how long a test waits for the flush timer to write the
|
|
||||||
// routing table's changes: a few ticks of it.
|
|
||||||
const tableWriteTimeout = 5 * prefixBatchTimeout
|
|
||||||
|
|
||||||
// tableCase is one test of the routing table in memory: the messages handed to
|
|
||||||
// the prefix handler, and for each prefix the peers that carry it with each
|
|
||||||
// origin AS after them.
|
|
||||||
type tableCase struct {
|
|
||||||
name string
|
|
||||||
messages []*ristypes.RISMessage
|
|
||||||
want map[string]map[int][]string
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestPrefixHandlerAppliesUpdatesToTheRoutingTable hands the prefix handler
|
|
||||||
// announcements, withdrawals and a peer's session going down.
|
|
||||||
func TestPrefixHandlerAppliesUpdatesToTheRoutingTable(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
announceAndWithdraw := peerAnnouncement(testPeerIP, testV4Prefix)
|
|
||||||
announceAndWithdraw.Withdrawals = []string{testV4Prefix}
|
|
||||||
|
|
||||||
checkTableCases(t, []tableCase{
|
|
||||||
{
|
|
||||||
name: "an announcement",
|
|
||||||
messages: []*ristypes.RISMessage{peerAnnouncement(testPeerIP, testV4Prefix)},
|
|
||||||
want: map[string]map[int][]string{testV4Prefix: {testASNB: {testPeerIP}}},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "a withdrawal",
|
|
||||||
messages: []*ristypes.RISMessage{
|
|
||||||
peerAnnouncement(testPeerIP, testV4Prefix),
|
|
||||||
peerWithdrawal(testPeerIP, testV4Prefix),
|
|
||||||
},
|
|
||||||
want: map[string]map[int][]string{testV4Prefix: {}},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
// RFC 4271 section 4.3: the announcement counts, not the withdrawal.
|
|
||||||
name: "a message that announces and withdraws the prefix",
|
|
||||||
messages: []*ristypes.RISMessage{announceAndWithdraw},
|
|
||||||
want: map[string]map[int][]string{testV4Prefix: {testASNB: {testPeerIP}}},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "a peer's session going down",
|
|
||||||
messages: []*ristypes.RISMessage{
|
|
||||||
peerAnnouncement(testPeerIP, testV4Prefix, testV6Prefix),
|
|
||||||
peerAnnouncement(secondPeerIP, testV4Prefix),
|
|
||||||
sessionDown(),
|
|
||||||
},
|
|
||||||
want: map[string]map[int][]string{
|
|
||||||
testV4Prefix: {testASNB: {secondPeerIP}},
|
|
||||||
testV6Prefix: {},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestPrefixHandlerAppliesAPathWithoutAnOriginToTheRoutingTable hands the
|
|
||||||
// prefix handler announcements whose AS path gives no origin. One whose path
|
|
||||||
// holds AS 0 removes the peer's route, as a withdrawal does; one whose path ends
|
|
||||||
// in an AS set of several members is kept with origin 0.
|
|
||||||
func TestPrefixHandlerAppliesAPathWithoutAnOriginToTheRoutingTable(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
announceWithPath := func(path ristypes.ASPath) *ristypes.RISMessage {
|
|
||||||
msg := peerAnnouncement(testPeerIP, testV4Prefix)
|
|
||||||
msg.Path = path
|
|
||||||
|
|
||||||
return msg
|
|
||||||
}
|
|
||||||
|
|
||||||
checkTableCases(t, []tableCase{
|
|
||||||
{
|
|
||||||
name: "a path whose last AS is 0",
|
|
||||||
messages: []*ristypes.RISMessage{
|
|
||||||
peerAnnouncement(testPeerIP, testV4Prefix),
|
|
||||||
announceWithPath(ristypes.ASPath{Sequence: []int{testASNA, 0}}),
|
|
||||||
},
|
|
||||||
want: map[string]map[int][]string{testV4Prefix: {}},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "a path with AS 0 before its end",
|
|
||||||
messages: []*ristypes.RISMessage{
|
|
||||||
peerAnnouncement(testPeerIP, testV4Prefix),
|
|
||||||
announceWithPath(decodePath(t, testASNA, 0, testASNB)),
|
|
||||||
},
|
|
||||||
want: map[string]map[int][]string{testV4Prefix: {}},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "a path that ends in an AS set",
|
|
||||||
messages: []*ristypes.RISMessage{
|
|
||||||
announceWithPath(decodePath(t, testASNA, []int{testASNC, testASND})),
|
|
||||||
},
|
|
||||||
want: map[string]map[int][]string{testV4Prefix: {0: {testPeerIP}}},
|
|
||||||
},
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
// checkTableCases hands the messages of each case to a new prefix handler and
|
|
||||||
// checks its routing table right after, with no flush.
|
|
||||||
func checkTableCases(t *testing.T, tests []tableCase) {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
for _, tt := range tests {
|
|
||||||
t.Run(tt.name, func(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
h := prefixHandlerWithoutTimer(newMockStore())
|
|
||||||
for _, msg := range tt.messages {
|
|
||||||
h.HandleMessage(msg)
|
|
||||||
}
|
|
||||||
|
|
||||||
for prefix, want := range tt.want {
|
|
||||||
got := tableOrigins(h.table, prefix)
|
|
||||||
if !reflect.DeepEqual(got, want) {
|
|
||||||
t.Errorf("peers of %s by origin AS = %v, want %v",
|
|
||||||
prefix, got, want)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// tableWritesStore is a real database that hands each write of the routing
|
|
||||||
// table's changes to written once it is made.
|
|
||||||
type tableWritesStore struct {
|
|
||||||
*database.Database
|
|
||||||
|
|
||||||
written chan []routetable.Change
|
|
||||||
}
|
|
||||||
|
|
||||||
func (s tableWritesStore) WriteRouteChanges(
|
|
||||||
changes []routetable.Change, newPeers map[string]uint,
|
|
||||||
) error {
|
|
||||||
err := s.Database.WriteRouteChanges(changes, newPeers)
|
|
||||||
s.written <- changes
|
|
||||||
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestPrefixHandlerWritesTheTableChangesEachSecond hands a running prefix
|
|
||||||
// handler an announcement and then its withdrawal, each far short of a full
|
|
||||||
// batch. The flush timer must write the change of each to the database.
|
|
||||||
func TestPrefixHandlerWritesTheTableChangesEachSecond(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
// The two writes the test waits for, and room for one more.
|
|
||||||
const writesBuffered = 3
|
|
||||||
|
|
||||||
store := tableWritesStore{
|
|
||||||
Database: newTestDatabase(t),
|
|
||||||
written: make(chan []routetable.Change, writesBuffered),
|
|
||||||
}
|
|
||||||
h := NewPrefixHandler(store, logger.New(), routetable.New(routetable.Params{}))
|
|
||||||
|
|
||||||
defer h.Stop(t.Context())
|
|
||||||
|
|
||||||
prefix := netip.MustParsePrefix(testV4Prefix)
|
|
||||||
|
|
||||||
for _, step := range []struct {
|
|
||||||
msg *ristypes.RISMessage
|
|
||||||
wantRemoved bool
|
|
||||||
}{
|
|
||||||
{msg: peerAnnouncement(testPeerIP, testV4Prefix)},
|
|
||||||
{msg: peerWithdrawal(testPeerIP, testV4Prefix), wantRemoved: true},
|
|
||||||
} {
|
|
||||||
h.HandleMessage(step.msg)
|
|
||||||
|
|
||||||
select {
|
|
||||||
case changes := <-store.written:
|
|
||||||
if len(changes) != 1 || changes[0].Prefix != prefix ||
|
|
||||||
changes[0].OriginASN != testASNB ||
|
|
||||||
changes[0].Removed != step.wantRemoved {
|
|
||||||
t.Fatalf("wrote %+v, want the pair of %s and AS%d with "+
|
|
||||||
"removed %t", changes, prefix, testASNB, step.wantRemoved)
|
|
||||||
}
|
|
||||||
case <-time.After(tableWriteTimeout):
|
|
||||||
t.Fatalf("the routing table's change was not written within %s",
|
|
||||||
tableWriteTimeout)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// errTestTableWrite is the error a test store's failing write of the routing
|
|
||||||
// table's changes returns.
|
|
||||||
var errTestTableWrite = errors.New("test: the write of the table's changes failed")
|
|
||||||
|
|
||||||
// firstTableWriteFailsStore is a real database whose first write of the
|
|
||||||
// routing table's changes fails, writing nothing. writes counts the writes
|
|
||||||
// asked for.
|
|
||||||
type firstTableWriteFailsStore struct {
|
|
||||||
*database.Database
|
|
||||||
|
|
||||||
writes int
|
|
||||||
}
|
|
||||||
|
|
||||||
func (s *firstTableWriteFailsStore) WriteRouteChanges(
|
|
||||||
changes []routetable.Change, newPeers map[string]uint,
|
|
||||||
) error {
|
|
||||||
s.writes++
|
|
||||||
if s.writes == 1 {
|
|
||||||
return errTestTableWrite
|
|
||||||
}
|
|
||||||
|
|
||||||
return s.Database.WriteRouteChanges(changes, newPeers)
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestPrefixHandlerWritesAFailedTableWriteAgain runs two flushes into a
|
|
||||||
// database whose first write of the routing table's changes fails. Before the
|
|
||||||
// first, two new peers announce a prefix each; before the second, one of them
|
|
||||||
// announces its prefix again with another path to the same origin AS. The
|
|
||||||
// second write must hold the changes and new peers of the first, with the later
|
|
||||||
// path, so the database holds the table that is in memory.
|
|
||||||
func TestPrefixHandlerWritesAFailedTableWriteAgain(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
db := newTestDatabase(t)
|
|
||||||
store := &firstTableWriteFailsStore{Database: db}
|
|
||||||
h := prefixHandlerWithoutTimer(store)
|
|
||||||
|
|
||||||
first := peerAnnouncement(testPeerIP, testV4Prefix)
|
|
||||||
h.HandleMessage(first)
|
|
||||||
h.HandleMessage(peerAnnouncement(secondPeerIP, otherV4Prefix))
|
|
||||||
h.flushBatch()
|
|
||||||
|
|
||||||
laterPath := []int{testASNC, testASNB}
|
|
||||||
later := peerAnnouncement(testPeerIP, testV4Prefix)
|
|
||||||
later.Path = ristypes.ASPath{Sequence: laterPath}
|
|
||||||
later.ParsedTimestamp = first.ParsedTimestamp.Add(secondUpdateAfter)
|
|
||||||
h.HandleMessage(later)
|
|
||||||
h.flushBatch()
|
|
||||||
|
|
||||||
if store.writes != 2 {
|
|
||||||
t.Fatalf("the store was asked for %d writes of the table's changes, "+
|
|
||||||
"want 2", store.writes)
|
|
||||||
}
|
|
||||||
|
|
||||||
read, err := db.ReadRouteTable(t.Context())
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("ReadRouteTable: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
checkSameTable(t, read, h.table, testV4Prefix, otherV4Prefix)
|
|
||||||
|
|
||||||
origins := read.Origins(netip.MustParsePrefix(testV4Prefix))
|
|
||||||
if len(origins) != 1 || len(origins[0].Routes) != 1 ||
|
|
||||||
!slices.Equal(origins[0].Routes[0].ASPath, laterPath) {
|
|
||||||
t.Errorf("Origins(%s) = %+v, want one route with the path %v",
|
|
||||||
testV4Prefix, origins, laterPath)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// tableWriteSizesStore is a real database that records how many changes and
|
|
||||||
// how many new peers each write of the routing table's changes holds.
|
|
||||||
type tableWriteSizesStore struct {
|
|
||||||
*database.Database
|
|
||||||
|
|
||||||
changes []int
|
|
||||||
newPeers []int
|
|
||||||
}
|
|
||||||
|
|
||||||
func (s *tableWriteSizesStore) WriteRouteChanges(
|
|
||||||
changes []routetable.Change, newPeers map[string]uint,
|
|
||||||
) error {
|
|
||||||
s.changes = append(s.changes, len(changes))
|
|
||||||
s.newPeers = append(s.newPeers, len(newPeers))
|
|
||||||
|
|
||||||
return s.Database.WriteRouteChanges(changes, newPeers)
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestPrefixHandlerWritesManyTableChangesOverSeveralFlushes has one new peer
|
|
||||||
// change one and a half times as many prefix and origin AS pairs as one write
|
|
||||||
// takes, then flushes three times. The first write must hold as many pairs as
|
|
||||||
// one write takes, and the peer. The second must hold the other pairs, and the
|
|
||||||
// third flush must write nothing. The database must then hold the table in
|
|
||||||
// memory.
|
|
||||||
func TestPrefixHandlerWritesManyTableChangesOverSeveralFlushes(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
const (
|
|
||||||
pairs = maxTableChangesPerWrite * 3 / 2
|
|
||||||
flushes = 3
|
|
||||||
)
|
|
||||||
|
|
||||||
db := newTestDatabase(t)
|
|
||||||
store := &tableWriteSizesStore{Database: db}
|
|
||||||
h := prefixHandlerWithoutTimer(store)
|
|
||||||
announceManyPrefixes(h.table, pairs)
|
|
||||||
|
|
||||||
for range flushes {
|
|
||||||
h.flushBatch()
|
|
||||||
}
|
|
||||||
|
|
||||||
wantChanges := []int{maxTableChangesPerWrite, pairs - maxTableChangesPerWrite}
|
|
||||||
wantNewPeers := []int{1, 0}
|
|
||||||
|
|
||||||
if !slices.Equal(store.changes, wantChanges) ||
|
|
||||||
!slices.Equal(store.newPeers, wantNewPeers) {
|
|
||||||
t.Errorf("the writes held %v pairs and %v new peers, want %v and %v",
|
|
||||||
store.changes, store.newPeers, wantChanges, wantNewPeers)
|
|
||||||
}
|
|
||||||
|
|
||||||
read, err := db.ReadRouteTable(t.Context())
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("ReadRouteTable: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
checkSameTable(t, read, h.table, firstManyPrefix)
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestPrefixHandlerStopLogsTheTableChangesLeftUnwritten stops a prefix handler,
|
|
||||||
// with no deadline, whose routing table holds more changed pairs than one write
|
|
||||||
// takes. Its last flush writes one write's worth, so Stop must log the pairs
|
|
||||||
// that no write took. The flush timer must not fire, or its writes could leave
|
|
||||||
// no pairs for Stop to log.
|
|
||||||
func TestPrefixHandlerStopLogsTheTableChangesLeftUnwritten(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
const pairs = maxTableChangesPerWrite + 1
|
|
||||||
|
|
||||||
var logs bytes.Buffer
|
|
||||||
|
|
||||||
store := &tableWriteSizesStore{Database: newTestDatabase(t)}
|
|
||||||
table := routetable.New(routetable.Params{})
|
|
||||||
announceManyPrefixes(table, pairs)
|
|
||||||
|
|
||||||
h := newPrefixHandler(store, newErrorLogger(&logs), table, longerThanAnyTest)
|
|
||||||
h.Stop(t.Context())
|
|
||||||
|
|
||||||
written := 0
|
|
||||||
for _, n := range store.changes {
|
|
||||||
written += n
|
|
||||||
}
|
|
||||||
|
|
||||||
if logged := lastLogged(t, &logs); logged.TableChanges != pairs-written {
|
|
||||||
t.Errorf("Stop logged %d table changes not written, want %d",
|
|
||||||
logged.TableChanges, pairs-written)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// readLog holds what readRouteTable logs.
|
|
||||||
//
|
|
||||||
//nolint:tagliatelle // field names are the keys readRouteTable logs
|
|
||||||
type readLog struct {
|
|
||||||
Msg string `json:"msg"`
|
|
||||||
Pairs int `json:"pairs"`
|
|
||||||
Peers int `json:"peers"`
|
|
||||||
DurationMs *int64 `json:"duration_ms"`
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestARestartReadsTheRoutingTableBack runs announcements, a withdrawal and a
|
|
||||||
// peer's session going down through a prefix handler into a database, then
|
|
||||||
// reads the table back as startup does. The table read must answer as the one
|
|
||||||
// in memory, and the log must give its pairs and peers. A new handler on the
|
|
||||||
// table read then takes later updates and writes them as it stops: the
|
|
||||||
// database must then hold what the first table holds after the same updates.
|
|
||||||
func TestARestartReadsTheRoutingTableBack(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
prefixes := []string{testV4Prefix, testV6Prefix, otherV4Prefix, setV4Prefix}
|
|
||||||
|
|
||||||
db := newTestDatabase(t)
|
|
||||||
h := prefixHandlerWithoutTimer(db)
|
|
||||||
|
|
||||||
viaOtherOrigin := peerAnnouncement(thirdPeerIP, testV4Prefix)
|
|
||||||
viaOtherOrigin.Path = ristypes.ASPath{Sequence: []int{testASNA, testASNC}}
|
|
||||||
endsInSet := peerAnnouncement(secondPeerIP, setV4Prefix)
|
|
||||||
endsInSet.Path = decodePath(t, testASNA, []int{testASNC, testASND})
|
|
||||||
|
|
||||||
for _, msg := range []*ristypes.RISMessage{
|
|
||||||
peerAnnouncement(testPeerIP, testV4Prefix, testV6Prefix),
|
|
||||||
peerAnnouncement(secondPeerIP, testV4Prefix),
|
|
||||||
viaOtherOrigin,
|
|
||||||
endsInSet,
|
|
||||||
peerAnnouncement(thirdPeerIP, otherV4Prefix),
|
|
||||||
peerWithdrawal(thirdPeerIP, otherV4Prefix),
|
|
||||||
peerAnnouncement(fourthPeerIP, otherV4Prefix),
|
|
||||||
sessionDownOf(fourthPeerIP),
|
|
||||||
} {
|
|
||||||
h.HandleMessage(msg)
|
|
||||||
}
|
|
||||||
|
|
||||||
h.flushBatch()
|
|
||||||
|
|
||||||
var logs bytes.Buffer
|
|
||||||
|
|
||||||
rw := &RouteWatch{
|
|
||||||
db: db,
|
|
||||||
logger: &logger.Logger{Logger: slog.New(slog.NewJSONHandler(&logs, nil))},
|
|
||||||
}
|
|
||||||
|
|
||||||
table, err := rw.readRouteTable(t.Context())
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("readRouteTable: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
checkSameTable(t, table, h.table, prefixes...)
|
|
||||||
|
|
||||||
// testV4Prefix with two origin ASes, testV6Prefix and setV4Prefix; four
|
|
||||||
// peers, the last of which carries nothing.
|
|
||||||
const wantPairs, wantPeers = 4, 4
|
|
||||||
|
|
||||||
var logged readLog
|
|
||||||
|
|
||||||
err = json.Unmarshal(bytes.TrimSpace(logs.Bytes()), &logged)
|
|
||||||
if err != nil || logged.Pairs != wantPairs || logged.Peers != wantPeers ||
|
|
||||||
logged.DurationMs == nil {
|
|
||||||
t.Errorf("readRouteTable logged %q, want %d pairs, %d peers and the "+
|
|
||||||
"duration", logs.String(), wantPairs, wantPeers)
|
|
||||||
}
|
|
||||||
|
|
||||||
restarted := NewPrefixHandler(db, logger.New(), table)
|
|
||||||
|
|
||||||
for _, msg := range []*ristypes.RISMessage{
|
|
||||||
peerWithdrawal(testPeerIP, testV6Prefix),
|
|
||||||
sessionDownOf(secondPeerIP),
|
|
||||||
peerAnnouncement(fourthPeerIP, otherV4Prefix),
|
|
||||||
peerAnnouncement(fifthPeerIP, setV4Prefix),
|
|
||||||
} {
|
|
||||||
// The first handler only applies the updates to its table.
|
|
||||||
h.HandleMessage(msg)
|
|
||||||
restarted.HandleMessage(msg)
|
|
||||||
}
|
|
||||||
|
|
||||||
restarted.Stop(t.Context())
|
|
||||||
|
|
||||||
read, err := db.ReadRouteTable(t.Context())
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("ReadRouteTable after the restart: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
checkSameTable(t, read, h.table, prefixes...)
|
|
||||||
}
|
|
||||||
|
|
||||||
// peerWithdrawal returns an UPDATE message in which peer withdraws prefixes.
|
|
||||||
func peerWithdrawal(peer string, prefixes ...string) *ristypes.RISMessage {
|
|
||||||
return &ristypes.RISMessage{
|
|
||||||
Type: ristypes.MessageTypeUpdate,
|
|
||||||
Peer: peer,
|
|
||||||
ParsedTimestamp: time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC),
|
|
||||||
Withdrawals: prefixes,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// announceManyPrefixes has testPeerIP announce n prefixes in table, with the
|
|
||||||
// path of peerAnnouncement: the /32 of each address from firstManyPrefix on.
|
|
||||||
func announceManyPrefixes(table *routetable.Table, n int) {
|
|
||||||
addr := netip.MustParsePrefix(firstManyPrefix).Addr()
|
|
||||||
route := routetable.Route{
|
|
||||||
Peer: testPeerIP,
|
|
||||||
ASPath: []int{testASNA, testASNB},
|
|
||||||
Time: time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC),
|
|
||||||
}
|
|
||||||
|
|
||||||
for range n {
|
|
||||||
table.Announce(netip.PrefixFrom(addr, addr.BitLen()), testASNB, route)
|
|
||||||
addr = addr.Next()
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// sessionDownOf returns the STATE message that says the BGP session of peer
|
|
||||||
// went down.
|
|
||||||
func sessionDownOf(peer string) *ristypes.RISMessage {
|
|
||||||
return &ristypes.RISMessage{
|
|
||||||
Type: ristypes.MessageTypeState,
|
|
||||||
Peer: peer,
|
|
||||||
State: ristypes.StateDown,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// tableOrigins returns, for each origin AS of prefix in table, the peers that
|
|
||||||
// carry the prefix with it.
|
|
||||||
func tableOrigins(table *routetable.Table, prefix string) map[int][]string {
|
|
||||||
origins := make(map[int][]string)
|
|
||||||
for _, o := range table.Origins(netip.MustParsePrefix(prefix)) {
|
|
||||||
origins[o.ASN] = o.Peers
|
|
||||||
}
|
|
||||||
|
|
||||||
return origins
|
|
||||||
}
|
|
||||||
|
|
||||||
// checkSameTable checks that got answers as want does for each of prefixes, and
|
|
||||||
// that the two tables have the same counts.
|
|
||||||
func checkSameTable(
|
|
||||||
t *testing.T, got, want *routetable.Table, prefixes ...string,
|
|
||||||
) {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
for _, text := range prefixes {
|
|
||||||
prefix := netip.MustParsePrefix(text)
|
|
||||||
|
|
||||||
gotOrigins, wantOrigins := got.Origins(prefix), want.Origins(prefix)
|
|
||||||
if !reflect.DeepEqual(gotOrigins, wantOrigins) {
|
|
||||||
t.Errorf("Origins(%s) = %+v, want %+v", text, gotOrigins, wantOrigins)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
gotCounts, wantCounts := got.Counts(), want.Counts()
|
|
||||||
if !reflect.DeepEqual(gotCounts, wantCounts) {
|
|
||||||
t.Errorf("Counts() = %+v, want %+v", gotCounts, wantCounts)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
+96
-343
@@ -1,7 +1,7 @@
|
|||||||
package server
|
package server
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"cmp"
|
"bytes"
|
||||||
"context"
|
"context"
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
"errors"
|
"errors"
|
||||||
@@ -9,16 +9,13 @@ import (
|
|||||||
"net/http"
|
"net/http"
|
||||||
"net/netip"
|
"net/netip"
|
||||||
"runtime"
|
"runtime"
|
||||||
"slices"
|
|
||||||
"sort"
|
"sort"
|
||||||
"strconv"
|
"strconv"
|
||||||
"strings"
|
"strings"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"git.eeqj.de/sneak/routewatch/internal/blocklist"
|
|
||||||
"git.eeqj.de/sneak/routewatch/internal/database"
|
"git.eeqj.de/sneak/routewatch/internal/database"
|
||||||
"git.eeqj.de/sneak/routewatch/internal/metrics"
|
"git.eeqj.de/sneak/routewatch/internal/metrics"
|
||||||
"git.eeqj.de/sneak/routewatch/internal/ristypes"
|
|
||||||
"git.eeqj.de/sneak/routewatch/internal/streamer"
|
"git.eeqj.de/sneak/routewatch/internal/streamer"
|
||||||
"git.eeqj.de/sneak/routewatch/internal/templates"
|
"git.eeqj.de/sneak/routewatch/internal/templates"
|
||||||
asinfo "git.eeqj.de/sneak/routewatch/pkg/asinfo"
|
asinfo "git.eeqj.de/sneak/routewatch/pkg/asinfo"
|
||||||
@@ -34,27 +31,6 @@ const (
|
|||||||
healthCheckTimeout = 2 * time.Second
|
healthCheckTimeout = 2 * time.Second
|
||||||
)
|
)
|
||||||
|
|
||||||
// Limits of one IP lookup.
|
|
||||||
const (
|
|
||||||
// maxLookupTargets is the most addresses and host names an IP lookup may
|
|
||||||
// list, and the most addresses it may have once the names are resolved:
|
|
||||||
// each address costs up to 113 database reads and a PTR lookup.
|
|
||||||
maxLookupTargets = 10
|
|
||||||
|
|
||||||
// ptrLookupTimeout is each PTR lookup's own time limit, so that ten
|
|
||||||
// addresses whose PTR lookups never answer fit in the 30-second limit.
|
|
||||||
ptrLookupTimeout = 2 * time.Second
|
|
||||||
)
|
|
||||||
|
|
||||||
// The Cache-Control header of an IP lookup answer, such as
|
|
||||||
// "public, max-age=900".
|
|
||||||
const (
|
|
||||||
cacheControlHeader = "Cache-Control"
|
|
||||||
cacheControlPublic = "public"
|
|
||||||
cacheControlPrivate = "private"
|
|
||||||
cacheControlMaxAge = ", max-age="
|
|
||||||
)
|
|
||||||
|
|
||||||
// Keys of the JSON envelope the handlers and middleware write, and the status
|
// Keys of the JSON envelope the handlers and middleware write, and the status
|
||||||
// value of an error answer.
|
// value of an error answer.
|
||||||
const (
|
const (
|
||||||
@@ -65,23 +41,6 @@ const (
|
|||||||
jsonStatusError = "error"
|
jsonStatusError = "error"
|
||||||
)
|
)
|
||||||
|
|
||||||
// Error messages an answer sends, in place of the error behind it where there
|
|
||||||
// is one.
|
|
||||||
const (
|
|
||||||
invalidPrefixMessage = "Invalid prefix"
|
|
||||||
hostNotResolvedMessage = "could not resolve host name"
|
|
||||||
hostNotPublicMessage = "host name has no public address"
|
|
||||||
tooManyTargetsMessage = "Too many addresses or host names"
|
|
||||||
searchQueryMissingMessage = "Missing search query"
|
|
||||||
)
|
|
||||||
|
|
||||||
// The AS name search reads the text it looks for from the URL query parameter
|
|
||||||
// asSearchQueryParam, and answers with at most maxASSearchResults matches.
|
|
||||||
const (
|
|
||||||
asSearchQueryParam = "q"
|
|
||||||
maxASSearchResults = 50
|
|
||||||
)
|
|
||||||
|
|
||||||
// IP versions as database.LiveRoute.IPVersion records them.
|
// IP versions as database.LiveRoute.IPVersion records them.
|
||||||
const (
|
const (
|
||||||
ipVersionV4 = 4
|
ipVersionV4 = 4
|
||||||
@@ -109,9 +68,7 @@ func (s *Server) handleHealthCheck() http.HandlerFunc {
|
|||||||
// Check database connectivity with lightweight ping
|
// Check database connectivity with lightweight ping
|
||||||
err := s.db.Ping(ctx)
|
err := s.db.Ping(ctx)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
s.logger.Error("Health check database ping failed", "error", err)
|
checks["database"] = "error: " + err.Error()
|
||||||
|
|
||||||
checks["database"] = jsonStatusError
|
|
||||||
healthy = false
|
healthy = false
|
||||||
} else {
|
} else {
|
||||||
checks["database"] = "ok"
|
checks["database"] = "ok"
|
||||||
@@ -151,15 +108,12 @@ func (s *Server) handleHealthCheck() http.HandlerFunc {
|
|||||||
|
|
||||||
// handleIndex returns a handler that serves the home page.
|
// handleIndex returns a handler that serves the home page.
|
||||||
func (s *Server) handleIndex() http.HandlerFunc {
|
func (s *Server) handleIndex() http.HandlerFunc {
|
||||||
return func(w http.ResponseWriter, r *http.Request) {
|
return func(w http.ResponseWriter, _ *http.Request) {
|
||||||
w.Header().Set("Content-Type", "text/html; charset=utf-8")
|
w.Header().Set("Content-Type", "text/html; charset=utf-8")
|
||||||
|
|
||||||
tmpl := templates.IndexTemplate()
|
tmpl := templates.IndexTemplate()
|
||||||
|
|
||||||
err := tmpl.Execute(w, indexPageData{
|
err := tmpl.Execute(w, nil)
|
||||||
CSPNonce: cspNonce(r.Context()),
|
|
||||||
ClientIP: extractClientIP(r),
|
|
||||||
})
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
s.logger.Error("Failed to render index template", "error", err)
|
s.logger.Error("Failed to render index template", "error", err)
|
||||||
http.Error(w, "Internal Server Error", http.StatusInternalServerError)
|
http.Error(w, "Internal Server Error", http.StatusInternalServerError)
|
||||||
@@ -167,21 +121,6 @@ func (s *Server) handleIndex() http.HandlerFunc {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// pageData is what the pages that show no data are rendered with: the nonce
|
|
||||||
// their script and style blocks carry.
|
|
||||||
type pageData struct {
|
|
||||||
CSPNonce string
|
|
||||||
}
|
|
||||||
|
|
||||||
// indexPageData is what the home page is rendered with.
|
|
||||||
type indexPageData struct {
|
|
||||||
// CSPNonce is the nonce the page's script and style blocks carry.
|
|
||||||
CSPNonce string
|
|
||||||
// ClientIP is the visitor's address, which the IP information API card
|
|
||||||
// looks up: the address /ip/ would look up for the same request.
|
|
||||||
ClientIP string
|
|
||||||
}
|
|
||||||
|
|
||||||
// writeJSONError writes a standardized JSON error response with the given
|
// writeJSONError writes a standardized JSON error response with the given
|
||||||
// status code and error message.
|
// status code and error message.
|
||||||
func (s *Server) writeJSONError(w http.ResponseWriter, statusCode int, message string) {
|
func (s *Server) writeJSONError(w http.ResponseWriter, statusCode int, message string) {
|
||||||
@@ -312,7 +251,7 @@ func (s *Server) handleStatusJSON() http.HandlerFunc {
|
|||||||
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
s.logger.Error("Failed to get database stats", "error", err)
|
s.logger.Error("Failed to get database stats", "error", err)
|
||||||
s.writeJSONError(w, http.StatusInternalServerError, internalErrorMessage)
|
s.writeJSONError(w, http.StatusInternalServerError, err.Error())
|
||||||
|
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
@@ -464,10 +403,10 @@ type GCStats struct {
|
|||||||
//
|
//
|
||||||
//nolint:tagliatelle // field names are the existing JSON API
|
//nolint:tagliatelle // field names are the existing JSON API
|
||||||
type StreamStats struct {
|
type StreamStats struct {
|
||||||
Announcements uint64 `json:"announcements"`
|
Announcements uint64 `json:"announcements"`
|
||||||
Withdrawals uint64 `json:"withdrawals"`
|
Withdrawals uint64 `json:"withdrawals"`
|
||||||
RouteChurnPerMin int `json:"route_churn_per_min"`
|
RouteChurnPerSec float64 `json:"route_churn_per_sec"`
|
||||||
BGPPeerCount int `json:"bgp_peer_count"`
|
BGPPeerCount int `json:"bgp_peer_count"`
|
||||||
}
|
}
|
||||||
|
|
||||||
// StatsResponse represents the API statistics response
|
// StatsResponse represents the API statistics response
|
||||||
@@ -568,7 +507,7 @@ func (s *Server) streamStats() StreamStats {
|
|||||||
return StreamStats{
|
return StreamStats{
|
||||||
Announcements: metricsTracker.GetAnnouncementCount(),
|
Announcements: metricsTracker.GetAnnouncementCount(),
|
||||||
Withdrawals: metricsTracker.GetWithdrawalCount(),
|
Withdrawals: metricsTracker.GetWithdrawalCount(),
|
||||||
RouteChurnPerMin: metricsTracker.GetChurnPerMinute(),
|
RouteChurnPerSec: metricsTracker.GetChurnRate(),
|
||||||
BGPPeerCount: metricsTracker.GetBGPPeerCount(),
|
BGPPeerCount: metricsTracker.GetBGPPeerCount(),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -602,7 +541,7 @@ func (s *Server) handleStats() http.HandlerFunc {
|
|||||||
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
s.logger.Error("Failed to get database stats", "error", err)
|
s.logger.Error("Failed to get database stats", "error", err)
|
||||||
s.writeJSONError(w, http.StatusInternalServerError, internalErrorMessage)
|
s.writeJSONError(w, http.StatusInternalServerError, err.Error())
|
||||||
|
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
@@ -683,12 +622,12 @@ func (s *Server) statsResponse(
|
|||||||
// handleStatusHTML returns a handler that serves the HTML status page,
|
// handleStatusHTML returns a handler that serves the HTML status page,
|
||||||
// which displays real-time statistics fetched via JavaScript.
|
// which displays real-time statistics fetched via JavaScript.
|
||||||
func (s *Server) handleStatusHTML() http.HandlerFunc {
|
func (s *Server) handleStatusHTML() http.HandlerFunc {
|
||||||
return func(w http.ResponseWriter, r *http.Request) {
|
return func(w http.ResponseWriter, _ *http.Request) {
|
||||||
w.Header().Set("Content-Type", "text/html; charset=utf-8")
|
w.Header().Set("Content-Type", "text/html; charset=utf-8")
|
||||||
|
|
||||||
tmpl := templates.StatusTemplate()
|
tmpl := templates.StatusTemplate()
|
||||||
|
|
||||||
err := tmpl.Execute(w, pageData{CSPNonce: cspNonce(r.Context())})
|
err := tmpl.Execute(w, nil)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
s.logger.Error("Failed to render template", "error", err)
|
s.logger.Error("Failed to render template", "error", err)
|
||||||
http.Error(w, "Internal Server Error", http.StatusInternalServerError)
|
http.Error(w, "Internal Server Error", http.StatusInternalServerError)
|
||||||
@@ -703,24 +642,9 @@ func (s *Server) handleIPLookup() http.HandlerFunc {
|
|||||||
|
|
||||||
// IPLookupResponse is the standard response for IP/hostname lookups.
|
// IPLookupResponse is the standard response for IP/hostname lookups.
|
||||||
type IPLookupResponse struct {
|
type IPLookupResponse struct {
|
||||||
Query string `json:"query"`
|
Query string `json:"query"`
|
||||||
Results []IPLookupResult `json:"results"`
|
Results []*database.IPInfo `json:"results"`
|
||||||
Errors []string `json:"errors,omitempty"`
|
Errors []string `json:"errors,omitempty"`
|
||||||
}
|
|
||||||
|
|
||||||
// IPLookupResult is the answer for one address: its PTR names, its live route
|
|
||||||
// and that route's AS, and the blocklists that list it. IPInfo is nil when
|
|
||||||
// the address has no live route, and the answer then has no route or AS
|
|
||||||
// fields. IP hides the IPInfo field of the same name, so that an address with
|
|
||||||
// no route has it too.
|
|
||||||
type IPLookupResult struct {
|
|
||||||
*database.IPInfo
|
|
||||||
|
|
||||||
IP string `json:"ip"`
|
|
||||||
PTR []string `json:"ptr,omitempty"`
|
|
||||||
|
|
||||||
// Blocklists has one entry for each blocklist the address is on.
|
|
||||||
Blocklists []blocklist.Listing `json:"blocklists"`
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// handleIPInfo returns a handler that provides comprehensive IP information.
|
// handleIPInfo returns a handler that provides comprehensive IP information.
|
||||||
@@ -729,7 +653,7 @@ type IPLookupResult struct {
|
|||||||
// Always returns the same response structure with PTR records for each IP.
|
// Always returns the same response structure with PTR records for each IP.
|
||||||
func (s *Server) handleIPInfo() http.HandlerFunc {
|
func (s *Server) handleIPInfo() http.HandlerFunc {
|
||||||
return func(w http.ResponseWriter, r *http.Request) {
|
return func(w http.ResponseWriter, r *http.Request) {
|
||||||
target, ownAddress := lookupTarget(r)
|
target := lookupTarget(r)
|
||||||
if target == "" {
|
if target == "" {
|
||||||
s.writeJSONError(w, http.StatusBadRequest, "Could not determine IP address")
|
s.writeJSONError(w, http.StatusBadRequest, "Could not determine IP address")
|
||||||
|
|
||||||
@@ -737,18 +661,15 @@ func (s *Server) handleIPInfo() http.HandlerFunc {
|
|||||||
}
|
}
|
||||||
|
|
||||||
ctx := r.Context()
|
ctx := r.Context()
|
||||||
response := IPLookupResponse{Query: target, Results: make([]IPLookupResult, 0)}
|
response := IPLookupResponse{
|
||||||
|
Query: target,
|
||||||
// Collect all IPs to look up
|
Results: make([]*database.IPInfo, 0),
|
||||||
ipsToLookup, resolveErrors, err := s.resolveLookupTargets(ctx, target)
|
|
||||||
if deadlinePassed(ctx) {
|
|
||||||
s.writeJSONError(w, http.StatusGatewayTimeout, requestTimeoutMessage)
|
|
||||||
|
|
||||||
return
|
|
||||||
}
|
}
|
||||||
|
|
||||||
if err != nil {
|
// Collect all IPs to look up
|
||||||
s.writeJSONError(w, http.StatusBadRequest, tooManyTargetsMessage)
|
ipsToLookup, resolveErrors := resolveLookupTargets(ctx, target)
|
||||||
|
if deadlinePassed(ctx) {
|
||||||
|
s.writeJSONError(w, http.StatusGatewayTimeout, requestTimeoutMessage)
|
||||||
|
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
@@ -756,7 +677,7 @@ func (s *Server) handleIPInfo() http.HandlerFunc {
|
|||||||
response.Errors = append(response.Errors, resolveErrors...)
|
response.Errors = append(response.Errors, resolveErrors...)
|
||||||
|
|
||||||
if len(ipsToLookup) == 0 {
|
if len(ipsToLookup) == 0 {
|
||||||
s.writeJSONError(w, http.StatusBadRequest, noTargetsMessage(resolveErrors))
|
s.writeJSONError(w, http.StatusBadRequest, "No valid IPs or hostnames provided")
|
||||||
|
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
@@ -773,37 +694,27 @@ func (s *Server) handleIPInfo() http.HandlerFunc {
|
|||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
switch {
|
if err != nil {
|
||||||
case errors.Is(err, database.ErrNoRoute):
|
|
||||||
// The address still gets a result, without route fields.
|
|
||||||
ipInfo = nil
|
|
||||||
case errors.Is(err, database.ErrInvalidIP):
|
|
||||||
response.Errors = append(response.Errors, ip+": "+err.Error())
|
response.Errors = append(response.Errors, ip+": "+err.Error())
|
||||||
|
|
||||||
continue
|
continue
|
||||||
case err != nil:
|
|
||||||
s.logger.Error("Failed to look up IP", "ip", ip, "error", err)
|
|
||||||
s.writeJSONError(w, http.StatusInternalServerError, internalErrorMessage)
|
|
||||||
|
|
||||||
return
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// Do PTR lookup for this IP
|
// Do PTR lookup for this IP
|
||||||
ptrs := s.lookupPTRNames(ctx, ip)
|
ptrs := lookupPTRNames(ctx, ip)
|
||||||
if deadlinePassed(ctx) {
|
if deadlinePassed(ctx) {
|
||||||
s.writeJSONError(w, http.StatusGatewayTimeout, requestTimeoutMessage)
|
s.writeJSONError(w, http.StatusGatewayTimeout, requestTimeoutMessage)
|
||||||
|
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
response.Results = append(response.Results, IPLookupResult{
|
if len(ptrs) > 0 {
|
||||||
IP: ip,
|
ipInfo.PTR = ptrs
|
||||||
PTR: ptrs,
|
}
|
||||||
IPInfo: ipInfo,
|
|
||||||
Blocklists: s.blocklistListings(ip),
|
|
||||||
})
|
|
||||||
|
|
||||||
if ipInfo != nil && ipInfo.NeedsWHOISRefresh {
|
response.Results = append(response.Results, ipInfo)
|
||||||
|
|
||||||
|
if ipInfo.NeedsWHOISRefresh {
|
||||||
refreshASNs[ipInfo.ASN] = true
|
refreshASNs[ipInfo.ASN] = true
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -811,70 +722,23 @@ func (s *Server) handleIPInfo() http.HandlerFunc {
|
|||||||
// Queue WHOIS refresh for stale ASNs (non-blocking)
|
// Queue WHOIS refresh for stale ASNs (non-blocking)
|
||||||
s.queueWHOISRefresh(refreshASNs)
|
s.queueWHOISRefresh(refreshASNs)
|
||||||
|
|
||||||
s.writeIPLookupAnswer(w, response, ownAddress)
|
// Return response (even if no results, include errors)
|
||||||
}
|
if len(response.Results) == 0 && len(response.Errors) > 0 {
|
||||||
}
|
s.writeJSONError(w, http.StatusNotFound, "No routes found: "+response.Errors[0])
|
||||||
|
|
||||||
// blocklistListings returns the blocklists that list ip. It returns an empty
|
return
|
||||||
// list, never nil, so that an answer's blocklists is [] rather than null.
|
}
|
||||||
func (s *Server) blocklistListings(ip string) []blocklist.Listing {
|
|
||||||
addr, err := netip.ParseAddr(ip)
|
|
||||||
if s.blocklists == nil || err != nil {
|
|
||||||
return []blocklist.Listing{}
|
|
||||||
}
|
|
||||||
|
|
||||||
return s.blocklists.Lookup(addr)
|
err := writeJSONSuccess(w, response)
|
||||||
}
|
if err != nil {
|
||||||
|
s.logger.Error("Failed to encode IP lookup response", "error", err)
|
||||||
// setIPLookupCacheControl lets caches keep an IP lookup answer for the
|
}
|
||||||
// shortest blocklist interval, the soonest a blocklist copy can change. The
|
|
||||||
// answer about the caller's own address is private, so that a shared cache
|
|
||||||
// does not give it to another caller. Without blocklists it sets nothing.
|
|
||||||
func (s *Server) setIPLookupCacheControl(w http.ResponseWriter, ownAddress bool) {
|
|
||||||
if s.blocklists == nil {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
visibility := cacheControlPublic
|
|
||||||
if ownAddress {
|
|
||||||
visibility = cacheControlPrivate
|
|
||||||
}
|
|
||||||
|
|
||||||
maxAgeSeconds := int(s.blocklists.ShortestInterval().Seconds())
|
|
||||||
w.Header().Set(cacheControlHeader,
|
|
||||||
visibility+cacheControlMaxAge+strconv.Itoa(maxAgeSeconds))
|
|
||||||
}
|
|
||||||
|
|
||||||
// noTargetsMessage returns the message of the 400 an IP lookup answers when
|
|
||||||
// no address is left to look up, followed by the first of resolveErrors, which
|
|
||||||
// says why, when there is one.
|
|
||||||
func noTargetsMessage(resolveErrors []string) string {
|
|
||||||
const message = "No valid IPs or hostnames provided"
|
|
||||||
|
|
||||||
if len(resolveErrors) == 0 {
|
|
||||||
return message
|
|
||||||
}
|
|
||||||
|
|
||||||
return message + ": " + resolveErrors[0]
|
|
||||||
}
|
|
||||||
|
|
||||||
// writeIPLookupAnswer sends response with its Cache-Control header.
|
|
||||||
// ownAddress says the request asked about the caller's own address.
|
|
||||||
func (s *Server) writeIPLookupAnswer(
|
|
||||||
w http.ResponseWriter, response IPLookupResponse, ownAddress bool,
|
|
||||||
) {
|
|
||||||
s.setIPLookupCacheControl(w, ownAddress)
|
|
||||||
|
|
||||||
err := writeJSONSuccess(w, response)
|
|
||||||
if err != nil {
|
|
||||||
s.logger.Error("Failed to encode IP lookup response", "error", err)
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// lookupTarget returns what an IP lookup request asks for: the ip or addr URL
|
// lookupTarget returns what an IP lookup request asks for: the ip or addr URL
|
||||||
// parameter, or else the client's own address. The bool is true for the
|
// parameter, or else the client's own address.
|
||||||
// client's own address.
|
func lookupTarget(r *http.Request) string {
|
||||||
func lookupTarget(r *http.Request) (string, bool) {
|
|
||||||
target := chi.URLParam(r, "ip")
|
target := chi.URLParam(r, "ip")
|
||||||
if target == "" {
|
if target == "" {
|
||||||
target = chi.URLParam(r, "addr")
|
target = chi.URLParam(r, "addr")
|
||||||
@@ -882,10 +746,10 @@ func lookupTarget(r *http.Request) (string, bool) {
|
|||||||
|
|
||||||
if target == "" {
|
if target == "" {
|
||||||
// Use client IP (RealIP middleware has already processed this)
|
// Use client IP (RealIP middleware has already processed this)
|
||||||
return extractClientIP(r), true
|
target = extractClientIP(r)
|
||||||
}
|
}
|
||||||
|
|
||||||
return target, false
|
return target
|
||||||
}
|
}
|
||||||
|
|
||||||
// queueWHOISRefresh asks the WHOIS fetcher, when there is one, to refresh
|
// queueWHOISRefresh asks the WHOIS fetcher, when there is one, to refresh
|
||||||
@@ -900,16 +764,12 @@ func (s *Server) queueWHOISRefresh(asns map[int]bool) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// errTooManyTargets is what resolveLookupTargets returns for a list of more
|
// resolveLookupTargets splits a comma-separated list of IP addresses and
|
||||||
// than maxLookupTargets addresses and host names, or of names that resolve to
|
// hostnames and returns the IP addresses to look up, resolving each hostname.
|
||||||
// more than maxLookupTargets addresses.
|
// A hostname that does not resolve is reported in the returned errors instead.
|
||||||
var errTooManyTargets = errors.New("too many addresses or host names")
|
// It stops once ctx's deadline has passed; the caller then answers 504.
|
||||||
|
func resolveLookupTargets(ctx context.Context, target string) ([]string, []string) {
|
||||||
// splitLookupTargets returns the addresses and host names in a comma-separated
|
var ipsToLookup, resolveErrors []string
|
||||||
// list, leaving out blank entries, or errTooManyTargets when there are more
|
|
||||||
// than maxLookupTargets.
|
|
||||||
func splitLookupTargets(target string) ([]string, error) {
|
|
||||||
var targets []string
|
|
||||||
|
|
||||||
for t := range strings.SplitSeq(target, ",") {
|
for t := range strings.SplitSeq(target, ",") {
|
||||||
t = strings.TrimSpace(t)
|
t = strings.TrimSpace(t)
|
||||||
@@ -917,36 +777,6 @@ func splitLookupTargets(target string) ([]string, error) {
|
|||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
|
||||||
if len(targets) == maxLookupTargets {
|
|
||||||
return nil, errTooManyTargets
|
|
||||||
}
|
|
||||||
|
|
||||||
targets = append(targets, t)
|
|
||||||
}
|
|
||||||
|
|
||||||
return targets, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// resolveLookupTargets splits a comma-separated list of IP addresses and
|
|
||||||
// hostnames and returns the IP addresses to look up, resolving each hostname.
|
|
||||||
// A hostname that does not resolve is reported in the returned errors instead,
|
|
||||||
// with the resolver's own error only in the log, since it names the DNS server.
|
|
||||||
// Only the public addresses a hostname resolves to are kept, so the answer
|
|
||||||
// does not show the deployment's private addresses; a hostname with none is
|
|
||||||
// reported in the errors. More than maxLookupTargets entries, or addresses once
|
|
||||||
// resolved, return errTooManyTargets. It stops once ctx's deadline has passed;
|
|
||||||
// the caller then answers 504.
|
|
||||||
func (s *Server) resolveLookupTargets(
|
|
||||||
ctx context.Context, target string,
|
|
||||||
) ([]string, []string, error) {
|
|
||||||
targets, err := splitLookupTargets(target)
|
|
||||||
if err != nil {
|
|
||||||
return nil, nil, err
|
|
||||||
}
|
|
||||||
|
|
||||||
var ipsToLookup, resolveErrors []string
|
|
||||||
|
|
||||||
for _, t := range targets {
|
|
||||||
// Check if this target is an IP address
|
// Check if this target is an IP address
|
||||||
if net.ParseIP(t) != nil {
|
if net.ParseIP(t) != nil {
|
||||||
ipsToLookup = append(ipsToLookup, t)
|
ipsToLookup = append(ipsToLookup, t)
|
||||||
@@ -955,57 +785,27 @@ func (s *Server) resolveLookupTargets(
|
|||||||
}
|
}
|
||||||
|
|
||||||
// It's a hostname - resolve it
|
// It's a hostname - resolve it
|
||||||
resolved, err := s.resolver.LookupHost(ctx, t)
|
resolved, err := net.DefaultResolver.LookupHost(ctx, t)
|
||||||
if deadlinePassed(ctx) {
|
if deadlinePassed(ctx) {
|
||||||
return nil, nil, nil
|
return nil, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
s.logger.Warn("Host name lookup failed", "host", t, "error", err)
|
resolveErrors = append(resolveErrors, t+": "+err.Error())
|
||||||
resolveErrors = append(resolveErrors, t+": "+hostNotResolvedMessage)
|
|
||||||
|
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
|
||||||
public := publicAddresses(resolved)
|
ipsToLookup = append(ipsToLookup, resolved...)
|
||||||
if len(public) == 0 {
|
|
||||||
resolveErrors = append(resolveErrors, t+": "+hostNotPublicMessage)
|
|
||||||
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
|
|
||||||
ipsToLookup = append(ipsToLookup, public...)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
if len(ipsToLookup) > maxLookupTargets {
|
return ipsToLookup, resolveErrors
|
||||||
return nil, nil, errTooManyTargets
|
|
||||||
}
|
|
||||||
|
|
||||||
return ipsToLookup, resolveErrors, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// publicAddresses returns the addresses in addrs that are global unicast and
|
|
||||||
// not private; Go counts the private ranges as global unicast.
|
|
||||||
func publicAddresses(addrs []string) []string {
|
|
||||||
var public []string
|
|
||||||
|
|
||||||
for _, a := range addrs {
|
|
||||||
ip, err := netip.ParseAddr(a)
|
|
||||||
if err == nil && ip.IsGlobalUnicast() && !ip.IsPrivate() {
|
|
||||||
public = append(public, a)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return public
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// lookupPTRNames returns the PTR records of ip without their trailing dots, or
|
// lookupPTRNames returns the PTR records of ip without their trailing dots, or
|
||||||
// nil when the lookup fails or runs past ptrLookupTimeout.
|
// nil when the lookup fails.
|
||||||
func (s *Server) lookupPTRNames(ctx context.Context, ip string) []string {
|
func lookupPTRNames(ctx context.Context, ip string) []string {
|
||||||
ctx, cancel := context.WithTimeout(ctx, ptrLookupTimeout)
|
ptrs, err := net.DefaultResolver.LookupAddr(ctx, ip)
|
||||||
defer cancel()
|
|
||||||
|
|
||||||
ptrs, err := s.resolver.LookupAddr(ctx, ip)
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
@@ -1019,7 +819,7 @@ func (s *Server) lookupPTRNames(ctx context.Context, ip string) []string {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// extractClientIP extracts the client IP from the request.
|
// extractClientIP extracts the client IP from the request.
|
||||||
// Works with RealIPMiddleware, which sets RemoteAddr.
|
// Works with chi's RealIP middleware which sets RemoteAddr.
|
||||||
func extractClientIP(r *http.Request) string {
|
func extractClientIP(r *http.Request) string {
|
||||||
// RemoteAddr is in the form "IP:port" or just "IP" for unix sockets
|
// RemoteAddr is in the form "IP:port" or just "IP" for unix sockets
|
||||||
host, _, err := net.SplitHostPort(r.RemoteAddr)
|
host, _, err := net.SplitHostPort(r.RemoteAddr)
|
||||||
@@ -1054,9 +854,7 @@ func (s *Server) handleASDetailJSON() http.HandlerFunc {
|
|||||||
if errors.Is(err, database.ErrNoRoute) {
|
if errors.Is(err, database.ErrNoRoute) {
|
||||||
s.writeJSONError(w, http.StatusNotFound, err.Error())
|
s.writeJSONError(w, http.StatusNotFound, err.Error())
|
||||||
} else {
|
} else {
|
||||||
s.logger.Error("Failed to get AS details", "asn", asn, "error", err)
|
s.writeJSONError(w, http.StatusInternalServerError, err.Error())
|
||||||
s.writeJSONError(w, http.StatusInternalServerError,
|
|
||||||
internalErrorMessage)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
return
|
return
|
||||||
@@ -1079,12 +877,11 @@ func (s *Server) handleASDetailJSON() http.HandlerFunc {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// splitRoutesByIPVersion groups routes into the IPv4 and the IPv6 ones. Neither
|
// splitRoutesByIPVersion groups routes into the IPv4 and the IPv6 ones.
|
||||||
// group is nil, so that a JSON answer holds [] for an empty one and not null.
|
|
||||||
func splitRoutesByIPVersion(
|
func splitRoutesByIPVersion(
|
||||||
routes []database.LiveRoute,
|
routes []database.LiveRoute,
|
||||||
) ([]database.LiveRoute, []database.LiveRoute) {
|
) ([]database.LiveRoute, []database.LiveRoute) {
|
||||||
ipv4Routes, ipv6Routes := []database.LiveRoute{}, []database.LiveRoute{}
|
var ipv4Routes, ipv6Routes []database.LiveRoute
|
||||||
|
|
||||||
for _, route := range routes {
|
for _, route := range routes {
|
||||||
if route.IPVersion == ipVersionV4 {
|
if route.IPVersion == ipVersionV4 {
|
||||||
@@ -1097,15 +894,24 @@ func splitRoutesByIPVersion(
|
|||||||
return ipv4Routes, ipv6Routes
|
return ipv4Routes, ipv6Routes
|
||||||
}
|
}
|
||||||
|
|
||||||
// sortRoutesByMaskLength sorts routes by mask length, shortest first, and
|
// sortRoutesByNetwork sorts routes by network address, then by mask length.
|
||||||
// routes with the same mask length by network address.
|
func sortRoutesByNetwork(routes []database.LiveRoute) {
|
||||||
func sortRoutesByMaskLength(routes []database.LiveRoute) {
|
sort.Slice(routes, func(i, j int) bool {
|
||||||
slices.SortFunc(routes, func(a, b database.LiveRoute) int {
|
// Parse the prefixes to compare network addresses
|
||||||
prefixA, _ := netip.ParsePrefix(a.Prefix)
|
ipI, netI, _ := net.ParseCIDR(routes[i].Prefix)
|
||||||
prefixB, _ := netip.ParsePrefix(b.Prefix)
|
ipJ, netJ, _ := net.ParseCIDR(routes[j].Prefix)
|
||||||
|
|
||||||
return cmp.Or(cmp.Compare(prefixA.Bits(), prefixB.Bits()),
|
// Compare by network address first
|
||||||
prefixA.Addr().Compare(prefixB.Addr()))
|
cmp := bytes.Compare(ipI.To16(), ipJ.To16())
|
||||||
|
if cmp != 0 {
|
||||||
|
return cmp < 0
|
||||||
|
}
|
||||||
|
|
||||||
|
// If network addresses are equal, compare by mask length
|
||||||
|
onesI, _ := netI.Mask.Size()
|
||||||
|
onesJ, _ := netJ.Mask.Size()
|
||||||
|
|
||||||
|
return onesI < onesJ
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1127,7 +933,7 @@ func (s *Server) handlePrefixDetailJSON() http.HandlerFunc {
|
|||||||
// tables store, so that any valid way of writing it finds the route.
|
// tables store, so that any valid way of writing it finds the route.
|
||||||
parsed, err := netip.ParsePrefix(prefixParam + "/" + lenParam)
|
parsed, err := netip.ParsePrefix(prefixParam + "/" + lenParam)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
s.writeJSONError(w, http.StatusBadRequest, invalidPrefixMessage)
|
s.writeJSONError(w, http.StatusBadRequest, err.Error())
|
||||||
|
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
@@ -1145,10 +951,7 @@ func (s *Server) handlePrefixDetailJSON() http.HandlerFunc {
|
|||||||
if errors.Is(err, database.ErrNoRoute) {
|
if errors.Is(err, database.ErrNoRoute) {
|
||||||
s.writeJSONError(w, http.StatusNotFound, err.Error())
|
s.writeJSONError(w, http.StatusNotFound, err.Error())
|
||||||
} else {
|
} else {
|
||||||
s.logger.Error("Failed to get prefix details",
|
s.writeJSONError(w, http.StatusInternalServerError, err.Error())
|
||||||
"prefix", prefix, "error", err)
|
|
||||||
s.writeJSONError(w, http.StatusInternalServerError,
|
|
||||||
internalErrorMessage)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
return
|
return
|
||||||
@@ -1175,56 +978,6 @@ func (s *Server) handlePrefixDetailJSON() http.HandlerFunc {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// ASSearchResponse is the answer to an AS name search: the text searched for,
|
|
||||||
// the number of ASes that match it, and the first maxASSearchResults of them in
|
|
||||||
// AS number order.
|
|
||||||
type ASSearchResponse struct {
|
|
||||||
Query string `json:"query"`
|
|
||||||
Total int `json:"total"`
|
|
||||||
Results []asinfo.Info `json:"results"`
|
|
||||||
}
|
|
||||||
|
|
||||||
// handleASSearch returns a handler that searches the AS list built into
|
|
||||||
// asinfo for the text in the q parameter, without its leading and trailing
|
|
||||||
// spaces. It answers 400 when no text is left.
|
|
||||||
func (s *Server) handleASSearch() http.HandlerFunc {
|
|
||||||
return func(w http.ResponseWriter, r *http.Request) {
|
|
||||||
query := strings.TrimSpace(r.URL.Query().Get(asSearchQueryParam))
|
|
||||||
if query == "" {
|
|
||||||
s.writeJSONError(w, http.StatusBadRequest, searchQueryMissingMessage)
|
|
||||||
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
// AS 0 means no known origin and has no AS page, so the answer leaves
|
|
||||||
// it out. The matches come in AS number order, so AS 0 can only be the
|
|
||||||
// first. The search asks for one match more than the answer holds, to
|
|
||||||
// take its place.
|
|
||||||
results, total := asinfo.Search(query, maxASSearchResults+1)
|
|
||||||
if len(results) > 0 && results[0].ASN == ristypes.NoOrigin {
|
|
||||||
results = results[1:]
|
|
||||||
total--
|
|
||||||
}
|
|
||||||
|
|
||||||
results = results[:min(len(results), maxASSearchResults)]
|
|
||||||
|
|
||||||
// results is an empty list rather than nil when nothing matches, so
|
|
||||||
// that the answer holds [] and not null.
|
|
||||||
if results == nil {
|
|
||||||
results = []asinfo.Info{}
|
|
||||||
}
|
|
||||||
|
|
||||||
err := writeJSONSuccess(w, ASSearchResponse{
|
|
||||||
Query: query,
|
|
||||||
Total: total,
|
|
||||||
Results: results,
|
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
s.logger.Error("Failed to encode AS search results", "error", err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// handleASDetail returns a handler that serves the AS detail HTML page
|
// handleASDetail returns a handler that serves the AS detail HTML page
|
||||||
func (s *Server) handleASDetail() http.HandlerFunc {
|
func (s *Server) handleASDetail() http.HandlerFunc {
|
||||||
return func(w http.ResponseWriter, r *http.Request) {
|
return func(w http.ResponseWriter, r *http.Request) {
|
||||||
@@ -1249,7 +1002,7 @@ func (s *Server) handleASDetail() http.HandlerFunc {
|
|||||||
http.Error(w, "AS not found", http.StatusNotFound)
|
http.Error(w, "AS not found", http.StatusNotFound)
|
||||||
} else {
|
} else {
|
||||||
s.logger.Error("Failed to get AS details", "error", err)
|
s.logger.Error("Failed to get AS details", "error", err)
|
||||||
http.Error(w, internalErrorMessage, http.StatusInternalServerError)
|
http.Error(w, "Internal server error", http.StatusInternalServerError)
|
||||||
}
|
}
|
||||||
|
|
||||||
return
|
return
|
||||||
@@ -1272,8 +1025,9 @@ func (s *Server) handleASDetail() http.HandlerFunc {
|
|||||||
// Group prefixes by IP version
|
// Group prefixes by IP version
|
||||||
ipv4Prefixes, ipv6Prefixes := splitRoutesByIPVersion(prefixes)
|
ipv4Prefixes, ipv6Prefixes := splitRoutesByIPVersion(prefixes)
|
||||||
|
|
||||||
sortRoutesByMaskLength(ipv4Prefixes)
|
// Sort prefixes by network address
|
||||||
sortRoutesByMaskLength(ipv6Prefixes)
|
sortRoutesByNetwork(ipv4Prefixes)
|
||||||
|
sortRoutesByNetwork(ipv6Prefixes)
|
||||||
|
|
||||||
// Prepare template data
|
// Prepare template data
|
||||||
data := struct {
|
data := struct {
|
||||||
@@ -1285,7 +1039,6 @@ func (s *Server) handleASDetail() http.HandlerFunc {
|
|||||||
IPv6Count int
|
IPv6Count int
|
||||||
Peers []database.ASPeer
|
Peers []database.ASPeer
|
||||||
PeerCount int
|
PeerCount int
|
||||||
CSPNonce string
|
|
||||||
}{
|
}{
|
||||||
ASN: asInfo,
|
ASN: asInfo,
|
||||||
IPv4Prefixes: ipv4Prefixes,
|
IPv4Prefixes: ipv4Prefixes,
|
||||||
@@ -1295,7 +1048,6 @@ func (s *Server) handleASDetail() http.HandlerFunc {
|
|||||||
IPv6Count: len(ipv6Prefixes),
|
IPv6Count: len(ipv6Prefixes),
|
||||||
Peers: peers,
|
Peers: peers,
|
||||||
PeerCount: len(peers),
|
PeerCount: len(peers),
|
||||||
CSPNonce: cspNonce(r.Context()),
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// Check if context is still valid before writing response
|
// Check if context is still valid before writing response
|
||||||
@@ -1306,7 +1058,9 @@ func (s *Server) handleASDetail() http.HandlerFunc {
|
|||||||
|
|
||||||
w.Header().Set("Content-Type", "text/html; charset=utf-8")
|
w.Header().Set("Content-Type", "text/html; charset=utf-8")
|
||||||
|
|
||||||
err = templates.ASDetailTemplate().Execute(w, data)
|
tmpl := templates.ASDetailTemplate()
|
||||||
|
|
||||||
|
err = tmpl.Execute(w, data)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
s.logger.Error("Failed to render AS detail template", "error", err)
|
s.logger.Error("Failed to render AS detail template", "error", err)
|
||||||
http.Error(w, "Internal Server Error", http.StatusInternalServerError)
|
http.Error(w, "Internal Server Error", http.StatusInternalServerError)
|
||||||
@@ -1331,7 +1085,7 @@ func (s *Server) handlePrefixDetail() http.HandlerFunc {
|
|||||||
// tables store, so that any valid way of writing it finds the route.
|
// tables store, so that any valid way of writing it finds the route.
|
||||||
parsed, err := netip.ParsePrefix(prefixParam + "/" + lenParam)
|
parsed, err := netip.ParsePrefix(prefixParam + "/" + lenParam)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
http.Error(w, invalidPrefixMessage, http.StatusBadRequest)
|
http.Error(w, "Invalid prefix", http.StatusBadRequest)
|
||||||
|
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
@@ -1350,7 +1104,7 @@ func (s *Server) handlePrefixDetail() http.HandlerFunc {
|
|||||||
http.Error(w, "Prefix not found", http.StatusNotFound)
|
http.Error(w, "Prefix not found", http.StatusNotFound)
|
||||||
} else {
|
} else {
|
||||||
s.logger.Error("Failed to get prefix details", "error", err)
|
s.logger.Error("Failed to get prefix details", "error", err)
|
||||||
http.Error(w, internalErrorMessage, http.StatusInternalServerError)
|
http.Error(w, "Internal server error", http.StatusInternalServerError)
|
||||||
}
|
}
|
||||||
|
|
||||||
return
|
return
|
||||||
@@ -1377,7 +1131,6 @@ func (s *Server) handlePrefixDetail() http.HandlerFunc {
|
|||||||
Origins []*ASNInfo
|
Origins []*ASNInfo
|
||||||
PeerCount int
|
PeerCount int
|
||||||
OriginCount int
|
OriginCount int
|
||||||
CSPNonce string
|
|
||||||
}{
|
}{
|
||||||
Prefix: prefix,
|
Prefix: prefix,
|
||||||
MaskLength: maskLength,
|
MaskLength: maskLength,
|
||||||
@@ -1386,7 +1139,6 @@ func (s *Server) handlePrefixDetail() http.HandlerFunc {
|
|||||||
Origins: origins,
|
Origins: origins,
|
||||||
PeerCount: len(routes),
|
PeerCount: len(routes),
|
||||||
OriginCount: len(origins),
|
OriginCount: len(origins),
|
||||||
CSPNonce: cspNonce(r.Context()),
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// Check if context is still valid before writing response
|
// Check if context is still valid before writing response
|
||||||
@@ -1399,7 +1151,9 @@ func (s *Server) handlePrefixDetail() http.HandlerFunc {
|
|||||||
|
|
||||||
w.Header().Set("Content-Type", "text/html; charset=utf-8")
|
w.Header().Set("Content-Type", "text/html; charset=utf-8")
|
||||||
|
|
||||||
err = templates.PrefixDetailTemplate().Execute(w, data)
|
tmpl := templates.PrefixDetailTemplate()
|
||||||
|
|
||||||
|
err = tmpl.Execute(w, data)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
s.logger.Error("Failed to render prefix detail template", "error", err)
|
s.logger.Error("Failed to render prefix detail template", "error", err)
|
||||||
http.Error(w, "Internal Server Error", http.StatusInternalServerError)
|
http.Error(w, "Internal Server Error", http.StatusInternalServerError)
|
||||||
@@ -1549,7 +1303,7 @@ func (s *Server) prefixLengthHandler(ipVersion, maxMaskLength int) http.HandlerF
|
|||||||
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
s.logger.Error("Failed to get prefixes by length", "error", err)
|
s.logger.Error("Failed to get prefixes by length", "error", err)
|
||||||
http.Error(w, internalErrorMessage, http.StatusInternalServerError)
|
http.Error(w, "Internal server error", http.StatusInternalServerError)
|
||||||
|
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
@@ -1570,7 +1324,6 @@ func (s *Server) prefixLengthHandler(ipVersion, maxMaskLength int) http.HandlerF
|
|||||||
"IPVersion": ipVersion,
|
"IPVersion": ipVersion,
|
||||||
"Prefixes": prefixesWithASDescriptions(prefixes),
|
"Prefixes": prefixesWithASDescriptions(prefixes),
|
||||||
"Count": len(prefixes),
|
"Count": len(prefixes),
|
||||||
"CSPNonce": cspNonce(r.Context()),
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// Check if context is still valid before writing response
|
// Check if context is still valid before writing response
|
||||||
|
|||||||
File diff suppressed because it is too large
Load Diff
@@ -3,19 +3,13 @@ package server
|
|||||||
import (
|
import (
|
||||||
"bytes"
|
"bytes"
|
||||||
"context"
|
"context"
|
||||||
"crypto/rand"
|
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
"errors"
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
"log/slog"
|
"log/slog"
|
||||||
"net/http"
|
"net/http"
|
||||||
"net/netip"
|
|
||||||
"slices"
|
|
||||||
"strings"
|
|
||||||
"sync"
|
"sync"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/go-chi/chi/v5"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
// responseWriter wraps http.ResponseWriter to capture the response
|
// responseWriter wraps http.ResponseWriter to capture the response
|
||||||
@@ -122,10 +116,6 @@ func JSONResponseMiddleware(next http.Handler) http.Handler {
|
|||||||
// requestTimeoutMessage is the error message sent with 504 Gateway Timeout.
|
// requestTimeoutMessage is the error message sent with 504 Gateway Timeout.
|
||||||
const requestTimeoutMessage = "Request timeout"
|
const requestTimeoutMessage = "Request timeout"
|
||||||
|
|
||||||
// internalErrorMessage is the error message sent with 500 Internal Server
|
|
||||||
// Error, whatever went wrong; the handler logs the detail.
|
|
||||||
const internalErrorMessage = "Internal server error"
|
|
||||||
|
|
||||||
// deadlinePassed reports whether ctx ended because its deadline passed.
|
// deadlinePassed reports whether ctx ended because its deadline passed.
|
||||||
//
|
//
|
||||||
// A handler calls it right after each database read or host name lookup,
|
// A handler calls it right after each database read or host name lookup,
|
||||||
@@ -142,9 +132,8 @@ func deadlinePassed(ctx context.Context) bool {
|
|||||||
//
|
//
|
||||||
// If a response is not valid JSON or is empty when JSON is expected, the middleware
|
// If a response is not valid JSON or is empty when JSON is expected, the middleware
|
||||||
// returns a properly formatted JSON error response. If the request's time limit
|
// returns a properly formatted JSON error response. If the request's time limit
|
||||||
// has passed, it answers 504 with the error message "Request timeout". A 4xx
|
// has passed, it answers 504 with the error message "Request timeout". For other
|
||||||
// keeps its status, with the status's standard text as the message. For other
|
// errors, it returns "Internal server error" with status 500 if the original
|
||||||
// errors, it returns "Internal server error", with status 500 if the original
|
|
||||||
// status was 200.
|
// status was 200.
|
||||||
//
|
//
|
||||||
// Non-JSON responses are passed through unchanged.
|
// Non-JSON responses are passed through unchanged.
|
||||||
@@ -203,14 +192,9 @@ func JSONValidationMiddleware(next http.Handler) http.Handler {
|
|||||||
|
|
||||||
w.WriteHeader(statusCode)
|
w.WriteHeader(statusCode)
|
||||||
|
|
||||||
errorMsg := internalErrorMessage
|
errorMsg := "Internal server error"
|
||||||
|
if statusCode == http.StatusGatewayTimeout {
|
||||||
switch {
|
|
||||||
case statusCode == http.StatusGatewayTimeout:
|
|
||||||
errorMsg = requestTimeoutMessage
|
errorMsg = requestTimeoutMessage
|
||||||
case statusCode >= http.StatusBadRequest &&
|
|
||||||
statusCode < http.StatusInternalServerError:
|
|
||||||
errorMsg = http.StatusText(statusCode)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
response := map[string]any{
|
response := map[string]any{
|
||||||
@@ -297,118 +281,3 @@ func RequestLoggerMiddleware(logger *slog.Logger) func(http.Handler) http.Handle
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// DecodedPathMiddleware makes the router match routes against the decoded
|
|
||||||
// path, so that the handlers read their URL parameters decoded. Without it,
|
|
||||||
// chi matches against the escaped path whenever the request has one and hands
|
|
||||||
// the handlers their parameters still escaped: /ip/2001%3Adb8%3A%3A1 would
|
|
||||||
// look up "2001%3Adb8%3A%3A1".
|
|
||||||
func DecodedPathMiddleware(next http.Handler) http.Handler {
|
|
||||||
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
||||||
chi.RouteContext(r.Context()).RoutePath = r.URL.Path
|
|
||||||
|
|
||||||
next.ServeHTTP(w, r)
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
// The values of the headers SecurityHeadersMiddleware sets.
|
|
||||||
const (
|
|
||||||
// strictTransportSecurity tells browsers to reach this host and its
|
|
||||||
// subdomains only over HTTPS for a year. The server listens on plain
|
|
||||||
// HTTP behind a TLS-terminating proxy, which passes the header on.
|
|
||||||
strictTransportSecurity = "max-age=31536000; includeSubDomains"
|
|
||||||
|
|
||||||
// contentSecurityPolicy lets a page load content only from this server,
|
|
||||||
// and run or apply an inline script or style block only when the block
|
|
||||||
// carries the nonce made for that answer, which replaces %[1]s. No other
|
|
||||||
// site may frame the pages.
|
|
||||||
contentSecurityPolicy = "default-src 'self'; " +
|
|
||||||
"script-src 'nonce-%[1]s'; style-src 'nonce-%[1]s'; " +
|
|
||||||
"frame-ancestors 'none'; base-uri 'none'; form-action 'self'"
|
|
||||||
|
|
||||||
frameOptions = "DENY"
|
|
||||||
contentTypeOptions = "nosniff"
|
|
||||||
referrerPolicy = "strict-origin-when-cross-origin"
|
|
||||||
|
|
||||||
// permissionsPolicy turns off browser features the pages do not use.
|
|
||||||
permissionsPolicy = "accelerometer=(), camera=(), geolocation=(), " +
|
|
||||||
"gyroscope=(), magnetometer=(), microphone=(), payment=(), usb=()"
|
|
||||||
)
|
|
||||||
|
|
||||||
// cspNonceKey is the request context key of the nonce SecurityHeadersMiddleware
|
|
||||||
// made for the answer.
|
|
||||||
type cspNonceKey struct{}
|
|
||||||
|
|
||||||
// SecurityHeadersMiddleware sets the security headers REPO_POLICIES.md requires
|
|
||||||
// on every answer. Each answer's Content-Security-Policy names a nonce of its
|
|
||||||
// own; a page reads it with cspNonce and writes it on its script and style
|
|
||||||
// blocks.
|
|
||||||
func SecurityHeadersMiddleware(next http.Handler) http.Handler {
|
|
||||||
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
||||||
nonce := rand.Text()
|
|
||||||
|
|
||||||
header := w.Header()
|
|
||||||
header.Set("Strict-Transport-Security", strictTransportSecurity)
|
|
||||||
header.Set("Content-Security-Policy",
|
|
||||||
fmt.Sprintf(contentSecurityPolicy, nonce))
|
|
||||||
header.Set("X-Frame-Options", frameOptions)
|
|
||||||
header.Set("X-Content-Type-Options", contentTypeOptions)
|
|
||||||
header.Set("Referrer-Policy", referrerPolicy)
|
|
||||||
header.Set("Permissions-Policy", permissionsPolicy)
|
|
||||||
|
|
||||||
ctx := context.WithValue(r.Context(), cspNonceKey{}, nonce)
|
|
||||||
next.ServeHTTP(w, r.WithContext(ctx))
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
// cspNonce returns the nonce SecurityHeadersMiddleware made for the answer to
|
|
||||||
// the request with ctx.
|
|
||||||
func cspNonce(ctx context.Context) string {
|
|
||||||
nonce, _ := ctx.Value(cspNonceKey{}).(string)
|
|
||||||
|
|
||||||
return nonce
|
|
||||||
}
|
|
||||||
|
|
||||||
// RealIPMiddleware gives a request from one of trustedProxies the address of
|
|
||||||
// the client the proxies forwarded it for as its RemoteAddr: the last
|
|
||||||
// X-Forwarded-For entry that is not a trusted proxy. Each proxy appends the
|
|
||||||
// address it took the request from, so the entries left of that one are
|
|
||||||
// whatever the client sent. A request from any other peer keeps the peer's
|
|
||||||
// address, whatever headers it carries.
|
|
||||||
func RealIPMiddleware(trustedProxies []netip.Prefix) func(http.Handler) http.Handler {
|
|
||||||
isTrusted := func(addr netip.Addr) bool {
|
|
||||||
return slices.ContainsFunc(trustedProxies, func(proxy netip.Prefix) bool {
|
|
||||||
return proxy.Contains(addr.Unmap())
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
return func(next http.Handler) http.Handler {
|
|
||||||
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
||||||
peer, err := netip.ParseAddrPort(r.RemoteAddr)
|
|
||||||
if err != nil || !isTrusted(peer.Addr()) {
|
|
||||||
next.ServeHTTP(w, r)
|
|
||||||
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
// A header sent on several lines is one list.
|
|
||||||
entries := strings.Split(
|
|
||||||
strings.Join(r.Header.Values("X-Forwarded-For"), ","), ",")
|
|
||||||
|
|
||||||
for _, entry := range slices.Backward(entries) {
|
|
||||||
addr, err := netip.ParseAddr(strings.TrimSpace(entry))
|
|
||||||
if err != nil {
|
|
||||||
break
|
|
||||||
}
|
|
||||||
|
|
||||||
r.RemoteAddr = addr.String()
|
|
||||||
|
|
||||||
if !isTrusted(addr) {
|
|
||||||
break
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
next.ServeHTTP(w, r)
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|||||||
@@ -1,234 +0,0 @@
|
|||||||
package server
|
|
||||||
|
|
||||||
import (
|
|
||||||
"encoding/json"
|
|
||||||
"net/http"
|
|
||||||
"net/http/httptest"
|
|
||||||
"net/netip"
|
|
||||||
"strings"
|
|
||||||
"testing"
|
|
||||||
"time"
|
|
||||||
)
|
|
||||||
|
|
||||||
// The addresses the forwarding tests use: the client a proxy names in
|
|
||||||
// X-Forwarded-For, the range of trusted proxies, a peer inside it, and a peer
|
|
||||||
// outside it, which lies in testPrefixV4 so that an IP lookup of it finds a
|
|
||||||
// route.
|
|
||||||
const (
|
|
||||||
forwardedClient = "203.0.113.9"
|
|
||||||
trustedProxyRange = "10.0.0.0/8"
|
|
||||||
trustedPeer = "10.0.0.2:1234"
|
|
||||||
untrustedPeerAddr = "192.0.2.1"
|
|
||||||
untrustedPeer = untrustedPeerAddr + ":1234"
|
|
||||||
)
|
|
||||||
|
|
||||||
// TestResponsesCarrySecurityHeaders checks that a page, a JSON route and a
|
|
||||||
// path no route matches each answer with the security headers REPO_POLICIES.md
|
|
||||||
// requires, and that the Content-Security-Policy allows no inline content
|
|
||||||
// without a nonce (https://git.eeqj.de/sneak/routewatch/issues/108).
|
|
||||||
func TestResponsesCarrySecurityHeaders(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
s := newServerWithTestRoutes(t)
|
|
||||||
|
|
||||||
headers := []string{
|
|
||||||
"Strict-Transport-Security",
|
|
||||||
"Content-Security-Policy",
|
|
||||||
"X-Frame-Options",
|
|
||||||
"X-Content-Type-Options",
|
|
||||||
"Referrer-Policy",
|
|
||||||
"Permissions-Policy",
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, path := range []string{"/", statsPath, "/no-such-page"} {
|
|
||||||
rec := httptest.NewRecorder()
|
|
||||||
s.router.ServeHTTP(rec, httptest.NewRequestWithContext(
|
|
||||||
t.Context(), http.MethodGet, path, nil))
|
|
||||||
|
|
||||||
for _, name := range headers {
|
|
||||||
if rec.Header().Get(name) == "" {
|
|
||||||
t.Errorf("%s: no %s header", path, name)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
policy := rec.Header().Get("Content-Security-Policy")
|
|
||||||
if strings.Contains(policy, "'unsafe-") {
|
|
||||||
t.Errorf("%s: Content-Security-Policy %q allows unsafe content",
|
|
||||||
path, policy)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestJSONValidationKeeps4xxWithoutBody wraps a handler that answers 400
|
|
||||||
// without a body in JSONValidationMiddleware and checks that the JSON error
|
|
||||||
// body keeps the 400 and gives the status's own text. Before the fix it said
|
|
||||||
// "Internal server error"
|
|
||||||
// (https://git.eeqj.de/sneak/routewatch/issues/111).
|
|
||||||
func TestJSONValidationKeeps4xxWithoutBody(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
handler := JSONValidationMiddleware(http.HandlerFunc(
|
|
||||||
func(w http.ResponseWriter, _ *http.Request) {
|
|
||||||
w.WriteHeader(http.StatusBadRequest)
|
|
||||||
}))
|
|
||||||
|
|
||||||
rec := httptest.NewRecorder()
|
|
||||||
handler.ServeHTTP(rec, httptest.NewRequestWithContext(
|
|
||||||
t.Context(), http.MethodGet, "/", nil))
|
|
||||||
|
|
||||||
const name = "400 without a body"
|
|
||||||
|
|
||||||
if rec.Code != http.StatusBadRequest {
|
|
||||||
t.Errorf("%s: status %d, want %d; body %s",
|
|
||||||
name, rec.Code, http.StatusBadRequest, rec.Body)
|
|
||||||
}
|
|
||||||
|
|
||||||
checkJSONErrorCode(t, name, rec, http.StatusBadRequest)
|
|
||||||
checkJSONErrorMessage(t, name, rec, http.StatusText(http.StatusBadRequest))
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestPagesCarryTheCSPNonce checks that every script and style block of each
|
|
||||||
// page carries the nonce its Content-Security-Policy header names, so that the
|
|
||||||
// browser runs and applies them, that each answer has a nonce of its own, and
|
|
||||||
// that no page uses a style attribute, which the policy does not allow.
|
|
||||||
func TestPagesCarryTheCSPNonce(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
s := newServerWithTestRoutes(t)
|
|
||||||
|
|
||||||
_, err := s.db.GetOrCreateASN(64500, time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC))
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("GetOrCreateASN: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
paths := []string{
|
|
||||||
"/", "/status", asPagePath, "/prefix/" + testPrefixV4,
|
|
||||||
prefixLengthPagePath, "/prefixlength6/32",
|
|
||||||
}
|
|
||||||
seen := make(map[string]bool)
|
|
||||||
|
|
||||||
for _, path := range paths {
|
|
||||||
rec := httptest.NewRecorder()
|
|
||||||
s.router.ServeHTTP(rec, httptest.NewRequestWithContext(
|
|
||||||
t.Context(), http.MethodGet, path, nil))
|
|
||||||
|
|
||||||
if rec.Code != http.StatusOK {
|
|
||||||
t.Errorf("%s: status %d, want %d; body %s",
|
|
||||||
path, rec.Code, http.StatusOK, rec.Body)
|
|
||||||
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
|
|
||||||
policy := rec.Header().Get("Content-Security-Policy")
|
|
||||||
|
|
||||||
_, nonce, found := strings.Cut(policy, "'nonce-")
|
|
||||||
if !found {
|
|
||||||
t.Errorf("%s: Content-Security-Policy %q names no nonce", path, policy)
|
|
||||||
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
|
|
||||||
nonce, _, _ = strings.Cut(nonce, "'")
|
|
||||||
if seen[nonce] {
|
|
||||||
t.Errorf("%s: nonce %q was already used for another answer", path, nonce)
|
|
||||||
}
|
|
||||||
|
|
||||||
seen[nonce] = true
|
|
||||||
page := rec.Body.String()
|
|
||||||
|
|
||||||
for _, tag := range []string{"<script", "<style"} {
|
|
||||||
withNonce := tag + ` nonce="` + nonce + `"`
|
|
||||||
if strings.Count(page, tag) != strings.Count(page, withNonce) {
|
|
||||||
t.Errorf("%s: a %s tag does not carry the nonce %q", path, tag, nonce)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if strings.Contains(page, "style=") {
|
|
||||||
t.Errorf("%s: the page uses a style attribute", path)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestForwardedForFromUntrustedPeerIsIgnored asks /ip/ for the caller's own
|
|
||||||
// address from a peer that is not a trusted proxy, with forwarding headers
|
|
||||||
// naming another address, and checks that the lookup is for the TCP peer.
|
|
||||||
// Before the fix any client could choose the address it was looked up and
|
|
||||||
// logged as (https://git.eeqj.de/sneak/routewatch/issues/108).
|
|
||||||
func TestForwardedForFromUntrustedPeerIsIgnored(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
s := newServerWithTestRoutes(t)
|
|
||||||
|
|
||||||
req := httptest.NewRequestWithContext(t.Context(), http.MethodGet, "/ip/", nil)
|
|
||||||
req.RemoteAddr = untrustedPeer
|
|
||||||
req.Header.Set("X-Forwarded-For", forwardedClient)
|
|
||||||
req.Header.Set("X-Real-IP", forwardedClient)
|
|
||||||
req.Header.Set("True-Client-IP", forwardedClient)
|
|
||||||
|
|
||||||
rec := httptest.NewRecorder()
|
|
||||||
s.router.ServeHTTP(rec, req)
|
|
||||||
|
|
||||||
var body struct {
|
|
||||||
Data IPLookupResponse `json:"data"`
|
|
||||||
}
|
|
||||||
|
|
||||||
err := json.Unmarshal(rec.Body.Bytes(), &body)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("decoding the answer %q: %v", rec.Body, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
if body.Data.Query != untrustedPeerAddr {
|
|
||||||
t.Errorf("looked up %q, want the TCP peer %s; answer %s",
|
|
||||||
body.Data.Query, untrustedPeerAddr, rec.Body)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestRealIPMiddlewareTrustsOnlyConfiguredProxies checks which address
|
|
||||||
// RealIPMiddleware gives a request: the TCP peer, unless the peer is a trusted
|
|
||||||
// proxy; then the last X-Forwarded-For entry that is not a trusted proxy,
|
|
||||||
// since the entries left of it are whatever the client sent.
|
|
||||||
func TestRealIPMiddlewareTrustsOnlyConfiguredProxies(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
trusted := []netip.Prefix{netip.MustParsePrefix(trustedProxyRange)}
|
|
||||||
|
|
||||||
tests := []struct {
|
|
||||||
name string
|
|
||||||
peer string
|
|
||||||
forwardedFor string
|
|
||||||
want string
|
|
||||||
}{
|
|
||||||
{"untrusted peer", untrustedPeer, forwardedClient, untrustedPeer},
|
|
||||||
{"trusted peer", trustedPeer, forwardedClient, forwardedClient},
|
|
||||||
{
|
|
||||||
"entries the client wrote", trustedPeer,
|
|
||||||
"198.51.100.7, " + forwardedClient, forwardedClient,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"two trusted proxies", trustedPeer,
|
|
||||||
forwardedClient + ", 10.0.0.3", forwardedClient,
|
|
||||||
},
|
|
||||||
{"no header", trustedPeer, "", trustedPeer},
|
|
||||||
{"entry not an address", trustedPeer, "client", trustedPeer},
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, tt := range tests {
|
|
||||||
var got string
|
|
||||||
|
|
||||||
handler := RealIPMiddleware(trusted)(http.HandlerFunc(
|
|
||||||
func(_ http.ResponseWriter, r *http.Request) { got = r.RemoteAddr }))
|
|
||||||
|
|
||||||
req := httptest.NewRequestWithContext(t.Context(), http.MethodGet, "/", nil)
|
|
||||||
req.RemoteAddr = tt.peer
|
|
||||||
|
|
||||||
if tt.forwardedFor != "" {
|
|
||||||
req.Header.Set("X-Forwarded-For", tt.forwardedFor)
|
|
||||||
}
|
|
||||||
|
|
||||||
handler.ServeHTTP(httptest.NewRecorder(), req)
|
|
||||||
|
|
||||||
if got != tt.want {
|
|
||||||
t.Errorf("%s: RemoteAddr %q, want %q", tt.name, got, tt.want)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,36 +1,19 @@
|
|||||||
package server
|
package server
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"net/http"
|
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/go-chi/chi/v5"
|
"github.com/go-chi/chi/v5"
|
||||||
"github.com/go-chi/chi/v5/middleware"
|
"github.com/go-chi/chi/v5/middleware"
|
||||||
)
|
)
|
||||||
|
|
||||||
// The error messages the JSON routes under /api/v1 and /ip answer with when no
|
|
||||||
// route matches the path, and when none takes the method.
|
|
||||||
const (
|
|
||||||
notFoundMessage = "Not found"
|
|
||||||
methodNotAllowedMessage = "Method not allowed"
|
|
||||||
|
|
||||||
// jsonRoutesAllow is the Allow header of their 405 answer: every one of
|
|
||||||
// them takes GET and HEAD, and no other method.
|
|
||||||
jsonRoutesAllow = "GET, HEAD"
|
|
||||||
)
|
|
||||||
|
|
||||||
// setupRoutes configures the HTTP routes
|
// setupRoutes configures the HTTP routes
|
||||||
func (s *Server) setupRoutes() {
|
func (s *Server) setupRoutes() {
|
||||||
r := chi.NewRouter()
|
r := chi.NewRouter()
|
||||||
|
|
||||||
// Middleware
|
// Middleware
|
||||||
r.Use(DecodedPathMiddleware)
|
|
||||||
// GetHead serves HEAD with the GET handler. It looks the route up by the
|
|
||||||
// path DecodedPathMiddleware sets, so it comes after it.
|
|
||||||
r.Use(middleware.GetHead)
|
|
||||||
r.Use(SecurityHeadersMiddleware)
|
|
||||||
r.Use(middleware.RequestID)
|
r.Use(middleware.RequestID)
|
||||||
r.Use(RealIPMiddleware(s.trustedProxies))
|
r.Use(middleware.RealIP)
|
||||||
r.Use(RequestLoggerMiddleware(s.logger.Logger)) // Structured request logging
|
r.Use(RequestLoggerMiddleware(s.logger.Logger)) // Structured request logging
|
||||||
r.Use(middleware.Recoverer)
|
r.Use(middleware.Recoverer)
|
||||||
r.Use(JSONResponseMiddleware)
|
r.Use(JSONResponseMiddleware)
|
||||||
@@ -55,39 +38,20 @@ func (s *Server) setupRoutes() {
|
|||||||
r.Get("/prefixlength/{length}", s.handlePrefixLength())
|
r.Get("/prefixlength/{length}", s.handlePrefixLength())
|
||||||
r.Get("/prefixlength6/{length}", s.handlePrefixLength6())
|
r.Get("/prefixlength6/{length}", s.handlePrefixLength6())
|
||||||
|
|
||||||
// The JSON routes answer a path or method they do not take with the JSON
|
|
||||||
// error body, as they answer everything else.
|
|
||||||
jsonNotFound := func(w http.ResponseWriter, _ *http.Request) {
|
|
||||||
s.writeJSONError(w, http.StatusNotFound, notFoundMessage)
|
|
||||||
}
|
|
||||||
jsonMethodNotAllowed := func(w http.ResponseWriter, _ *http.Request) {
|
|
||||||
// chi's own 405 answer sets Allow; one set in its place must too.
|
|
||||||
w.Header().Set("Allow", jsonRoutesAllow)
|
|
||||||
s.writeJSONError(w, http.StatusMethodNotAllowed, methodNotAllowedMessage)
|
|
||||||
}
|
|
||||||
|
|
||||||
// IP info JSON endpoints (replaces old /ip redirect)
|
// IP info JSON endpoints (replaces old /ip redirect)
|
||||||
r.Route("/ip", func(r chi.Router) {
|
r.Route("/ip", func(r chi.Router) {
|
||||||
r.Use(JSONValidationMiddleware)
|
r.Use(JSONValidationMiddleware)
|
||||||
r.NotFound(jsonNotFound)
|
r.Get("/", s.handleIPInfo()) // Client IP
|
||||||
r.MethodNotAllowed(jsonMethodNotAllowed)
|
|
||||||
r.Get("/", s.handleIPInfo()) // Client IP
|
|
||||||
// GetHead does not reach this route: mounting the group registers
|
|
||||||
// /ip and /ip/ for every method, so HEAD arrives here as HEAD.
|
|
||||||
r.Head("/", s.handleIPInfo())
|
|
||||||
r.Get("/{addr}", s.handleIPInfo()) // Specified IP
|
r.Get("/{addr}", s.handleIPInfo()) // Specified IP
|
||||||
})
|
})
|
||||||
|
|
||||||
// API routes
|
// API routes
|
||||||
r.Route("/api/v1", func(r chi.Router) {
|
r.Route("/api/v1", func(r chi.Router) {
|
||||||
r.Use(JSONValidationMiddleware)
|
r.Use(JSONValidationMiddleware)
|
||||||
r.NotFound(jsonNotFound)
|
|
||||||
r.MethodNotAllowed(jsonMethodNotAllowed)
|
|
||||||
r.Get("/stats", s.handleStats())
|
r.Get("/stats", s.handleStats())
|
||||||
r.Get("/ip/{ip}", s.handleIPLookup())
|
r.Get("/ip/{ip}", s.handleIPLookup())
|
||||||
r.Get("/as/{asn}", s.handleASDetailJSON())
|
r.Get("/as/{asn}", s.handleASDetailJSON())
|
||||||
r.Get("/prefix/{prefix}/{len}", s.handlePrefixDetailJSON())
|
r.Get("/prefix/{prefix}/{len}", s.handlePrefixDetailJSON())
|
||||||
r.Get("/search/as", s.handleASSearch())
|
|
||||||
})
|
})
|
||||||
|
|
||||||
s.router = r
|
s.router = r
|
||||||
|
|||||||
@@ -1,100 +0,0 @@
|
|||||||
package server
|
|
||||||
|
|
||||||
import (
|
|
||||||
"encoding/json"
|
|
||||||
"net/http"
|
|
||||||
"net/http/httptest"
|
|
||||||
"testing"
|
|
||||||
)
|
|
||||||
|
|
||||||
// TestRoutesAnswerHEAD sends HEAD for a page and JSON routes through the
|
|
||||||
// server's router and checks that each answers as GET does. /ip and /ip/, the
|
|
||||||
// lookup of the caller's own address, are in the list because chi's GetHead
|
|
||||||
// does not reach them. Before the fix every route answered HEAD with 405
|
|
||||||
// (https://git.eeqj.de/sneak/routewatch/issues/111).
|
|
||||||
func TestRoutesAnswerHEAD(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
s := newServerWithTestRoutes(t)
|
|
||||||
|
|
||||||
for _, path := range []string{"/", statsPath, "/ip", "/ip/"} {
|
|
||||||
rec := httptest.NewRecorder()
|
|
||||||
s.router.ServeHTTP(rec, httptest.NewRequestWithContext(
|
|
||||||
t.Context(), http.MethodHead, path, nil))
|
|
||||||
|
|
||||||
if rec.Code != http.StatusOK {
|
|
||||||
t.Errorf("HEAD %s: status %d, want %d; body %s",
|
|
||||||
path, rec.Code, http.StatusOK, rec.Body)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestJSONRoutesAnswerUnmatchedRequestsWithJSONErrors sends a method the JSON
|
|
||||||
// routes do not take, and a path none of them matches, to each group of JSON
|
|
||||||
// routes through the server's router. Each must answer with the JSON error
|
|
||||||
// body, and a 405 must name the methods the routes take in its Allow header.
|
|
||||||
// Before the fix a 405 said "Internal server error" and a 404 was chi's plain
|
|
||||||
// text answer (https://git.eeqj.de/sneak/routewatch/issues/111).
|
|
||||||
func TestJSONRoutesAnswerUnmatchedRequestsWithJSONErrors(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
s := newServerWithTestRoutes(t)
|
|
||||||
|
|
||||||
tests := []struct {
|
|
||||||
method string
|
|
||||||
path string
|
|
||||||
wantCode int
|
|
||||||
wantMsg string
|
|
||||||
}{
|
|
||||||
{http.MethodPost, statsPath, http.StatusMethodNotAllowed, methodNotAllowedMessage},
|
|
||||||
{http.MethodPost, "/ip/", http.StatusMethodNotAllowed, methodNotAllowedMessage},
|
|
||||||
{http.MethodGet, "/api/v1/nothing", http.StatusNotFound, notFoundMessage},
|
|
||||||
{http.MethodGet, "/ip/192.0.2.1/nothing", http.StatusNotFound, notFoundMessage},
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, tt := range tests {
|
|
||||||
name := tt.method + " " + tt.path
|
|
||||||
|
|
||||||
rec := httptest.NewRecorder()
|
|
||||||
s.router.ServeHTTP(rec, httptest.NewRequestWithContext(
|
|
||||||
t.Context(), tt.method, tt.path, nil))
|
|
||||||
|
|
||||||
if rec.Code != tt.wantCode {
|
|
||||||
t.Errorf("%s: status %d, want %d; body %s",
|
|
||||||
name, rec.Code, tt.wantCode, rec.Body)
|
|
||||||
}
|
|
||||||
|
|
||||||
checkJSONErrorCode(t, name, rec, tt.wantCode)
|
|
||||||
checkJSONErrorMessage(t, name, rec, tt.wantMsg)
|
|
||||||
|
|
||||||
allow := rec.Header().Get("Allow")
|
|
||||||
if tt.wantCode == http.StatusMethodNotAllowed && allow != jsonRoutesAllow {
|
|
||||||
t.Errorf("%s: Allow %q, want %q", name, allow, jsonRoutesAllow)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// checkJSONErrorMessage checks that rec holds the JSON error body with the
|
|
||||||
// given message.
|
|
||||||
func checkJSONErrorMessage(
|
|
||||||
t *testing.T, name string, rec *httptest.ResponseRecorder, want string,
|
|
||||||
) {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
var body struct {
|
|
||||||
Error struct {
|
|
||||||
Msg string `json:"msg"`
|
|
||||||
} `json:"error"`
|
|
||||||
}
|
|
||||||
|
|
||||||
err := json.Unmarshal(rec.Body.Bytes(), &body)
|
|
||||||
if err != nil {
|
|
||||||
t.Errorf("%s: decoding the answer %q: %v", name, rec.Body, err)
|
|
||||||
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
if body.Error.Msg != want {
|
|
||||||
t.Errorf("%s: error message %q, want %q", name, body.Error.Msg, want)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,8 +1,4 @@
|
|||||||
// Package server serves RouteWatch's web pages and JSON API:
|
// Package server provides HTTP endpoints for status monitoring and statistics
|
||||||
// - the home page, the status figures and the health check
|
|
||||||
// - the AS, prefix and prefix length pages, and the JSON answers behind them
|
|
||||||
// - the IP lookups
|
|
||||||
// - the AS name search
|
|
||||||
package server
|
package server
|
||||||
|
|
||||||
import (
|
import (
|
||||||
@@ -11,11 +7,9 @@ import (
|
|||||||
"fmt"
|
"fmt"
|
||||||
"net"
|
"net"
|
||||||
"net/http"
|
"net/http"
|
||||||
"net/netip"
|
|
||||||
"strconv"
|
"strconv"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"git.eeqj.de/sneak/routewatch/internal/blocklist"
|
|
||||||
"git.eeqj.de/sneak/routewatch/internal/config"
|
"git.eeqj.de/sneak/routewatch/internal/config"
|
||||||
"git.eeqj.de/sneak/routewatch/internal/database"
|
"git.eeqj.de/sneak/routewatch/internal/database"
|
||||||
"git.eeqj.de/sneak/routewatch/internal/logger"
|
"git.eeqj.de/sneak/routewatch/internal/logger"
|
||||||
@@ -37,20 +31,6 @@ type ASNFetcher interface {
|
|||||||
GetStats() ASNFetcherStats
|
GetStats() ASNFetcherStats
|
||||||
}
|
}
|
||||||
|
|
||||||
// Blocklists tells which public blocklists list an address, from copies it
|
|
||||||
// keeps in memory, as *blocklist.Blocklists does.
|
|
||||||
type Blocklists interface {
|
|
||||||
Lookup(addr netip.Addr) []blocklist.Listing
|
|
||||||
ShortestInterval() time.Duration
|
|
||||||
}
|
|
||||||
|
|
||||||
// dnsResolver looks up the addresses of host names and the names of addresses
|
|
||||||
// in DNS, as *net.Resolver does.
|
|
||||||
type dnsResolver interface {
|
|
||||||
LookupHost(ctx context.Context, host string) ([]string, error)
|
|
||||||
LookupAddr(ctx context.Context, addr string) ([]string, error)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Server provides HTTP endpoints for status monitoring
|
// Server provides HTTP endpoints for status monitoring
|
||||||
type Server struct {
|
type Server struct {
|
||||||
router *chi.Mux
|
router *chi.Mux
|
||||||
@@ -60,15 +40,6 @@ type Server struct {
|
|||||||
port int
|
port int
|
||||||
srv *http.Server
|
srv *http.Server
|
||||||
asnFetcher ASNFetcher
|
asnFetcher ASNFetcher
|
||||||
blocklists Blocklists
|
|
||||||
|
|
||||||
// trustedProxies are the reverse proxies whose X-Forwarded-For header
|
|
||||||
// RealIPMiddleware believes
|
|
||||||
trustedProxies []netip.Prefix
|
|
||||||
|
|
||||||
// resolver makes the IP lookup's DNS lookups: net.DefaultResolver, or a
|
|
||||||
// fake in tests
|
|
||||||
resolver dnsResolver
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// New creates a new HTTP server
|
// New creates a new HTTP server
|
||||||
@@ -79,12 +50,10 @@ func New(
|
|||||||
cfg *config.Config,
|
cfg *config.Config,
|
||||||
) *Server {
|
) *Server {
|
||||||
s := &Server{
|
s := &Server{
|
||||||
db: db,
|
db: db,
|
||||||
streamer: streamer,
|
streamer: streamer,
|
||||||
logger: logger,
|
logger: logger,
|
||||||
port: cfg.Port,
|
port: cfg.Port,
|
||||||
trustedProxies: cfg.TrustedProxies,
|
|
||||||
resolver: net.DefaultResolver,
|
|
||||||
}
|
}
|
||||||
|
|
||||||
s.setupRoutes()
|
s.setupRoutes()
|
||||||
@@ -134,9 +103,7 @@ func (s *Server) Start(ctx context.Context) error {
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// Stop stops the HTTP server. It waits, until ctx ends, for the requests in
|
// Stop gracefully stops the HTTP server
|
||||||
// progress to finish, and then closes the connections of those still running
|
|
||||||
// and returns ctx's error.
|
|
||||||
func (s *Server) Stop(ctx context.Context) error {
|
func (s *Server) Stop(ctx context.Context) error {
|
||||||
if s.srv == nil {
|
if s.srv == nil {
|
||||||
return nil
|
return nil
|
||||||
@@ -144,22 +111,10 @@ func (s *Server) Stop(ctx context.Context) error {
|
|||||||
|
|
||||||
s.logger.Info("Stopping HTTP server")
|
s.logger.Info("Stopping HTTP server")
|
||||||
|
|
||||||
err := s.srv.Shutdown(ctx)
|
return s.srv.Shutdown(ctx)
|
||||||
if err != nil {
|
|
||||||
return errors.Join(err, s.srv.Close())
|
|
||||||
}
|
|
||||||
|
|
||||||
return nil
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// SetASNFetcher sets the ASN WHOIS fetcher for on-demand lookups. Call it
|
// SetASNFetcher sets the ASN WHOIS fetcher for on-demand lookups.
|
||||||
// before Start: the handlers read the fetcher without a lock.
|
|
||||||
func (s *Server) SetASNFetcher(fetcher ASNFetcher) {
|
func (s *Server) SetASNFetcher(fetcher ASNFetcher) {
|
||||||
s.asnFetcher = fetcher
|
s.asnFetcher = fetcher
|
||||||
}
|
}
|
||||||
|
|
||||||
// SetBlocklists sets the blocklists the IP lookups report. Call it before
|
|
||||||
// Start: the handlers read the blocklists without a lock.
|
|
||||||
func (s *Server) SetBlocklists(blocklists Blocklists) {
|
|
||||||
s.blocklists = blocklists
|
|
||||||
}
|
|
||||||
|
|||||||
@@ -1,92 +0,0 @@
|
|||||||
package server
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"net"
|
|
||||||
"net/http"
|
|
||||||
"testing"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"git.eeqj.de/sneak/routewatch/internal/logger"
|
|
||||||
)
|
|
||||||
|
|
||||||
// TestStopClosesTheConnectionOfARequestStillRunning serves a request whose
|
|
||||||
// handler does not return until the test ends, and stops the server with a
|
|
||||||
// deadline that passes while the request runs. The client must then see its
|
|
||||||
// connection closed. Before the fix Stop returned at the deadline and left the
|
|
||||||
// connection open until the handler returned.
|
|
||||||
func TestStopClosesTheConnectionOfARequestStillRunning(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
const (
|
|
||||||
stopDeadline = 100 * time.Millisecond
|
|
||||||
// waitTimeout is how long the test waits for the request to reach
|
|
||||||
// the handler, and then for the client to see its connection closed.
|
|
||||||
waitTimeout = 2 * time.Second
|
|
||||||
// readHeaderTimeout bounds the test server's read of the request
|
|
||||||
// header, as every http.Server must.
|
|
||||||
readHeaderTimeout = time.Second
|
|
||||||
)
|
|
||||||
|
|
||||||
release := make(chan struct{})
|
|
||||||
defer close(release)
|
|
||||||
|
|
||||||
started := make(chan struct{})
|
|
||||||
|
|
||||||
var listenConfig net.ListenConfig
|
|
||||||
|
|
||||||
listener, err := listenConfig.Listen(t.Context(), "tcp", "127.0.0.1:0")
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("failed to open the test listener: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
s := &Server{logger: logger.New(), srv: &http.Server{
|
|
||||||
Handler: http.HandlerFunc(func(http.ResponseWriter, *http.Request) {
|
|
||||||
close(started)
|
|
||||||
<-release
|
|
||||||
}),
|
|
||||||
ReadHeaderTimeout: readHeaderTimeout,
|
|
||||||
}}
|
|
||||||
|
|
||||||
go func() { _ = s.srv.Serve(listener) }()
|
|
||||||
|
|
||||||
answered := make(chan error, 1)
|
|
||||||
|
|
||||||
go func() {
|
|
||||||
req, err := http.NewRequestWithContext(t.Context(), http.MethodGet,
|
|
||||||
"http://"+listener.Addr().String()+"/", nil)
|
|
||||||
if err != nil {
|
|
||||||
answered <- err
|
|
||||||
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
resp, err := http.DefaultClient.Do(req)
|
|
||||||
if err == nil {
|
|
||||||
_ = resp.Body.Close()
|
|
||||||
}
|
|
||||||
|
|
||||||
answered <- err
|
|
||||||
}()
|
|
||||||
|
|
||||||
select {
|
|
||||||
case <-started:
|
|
||||||
case <-time.After(waitTimeout):
|
|
||||||
t.Fatal("the request did not reach the handler")
|
|
||||||
}
|
|
||||||
|
|
||||||
ctx, cancel := context.WithTimeout(t.Context(), stopDeadline)
|
|
||||||
defer cancel()
|
|
||||||
|
|
||||||
_ = s.Stop(ctx)
|
|
||||||
|
|
||||||
select {
|
|
||||||
case err := <-answered:
|
|
||||||
if err == nil {
|
|
||||||
t.Error("the request still running at the deadline was answered")
|
|
||||||
}
|
|
||||||
case <-time.After(waitTimeout):
|
|
||||||
t.Fatal("the connection of the request still running at the " +
|
|
||||||
"deadline was open after Stop returned")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -106,12 +106,12 @@ type RawMessageHandler func(line string)
|
|||||||
// handlerMetrics tracks performance metrics for a handler
|
// handlerMetrics tracks performance metrics for a handler
|
||||||
type handlerMetrics struct {
|
type handlerMetrics struct {
|
||||||
processedCount uint64 // Total messages processed
|
processedCount uint64 // Total messages processed
|
||||||
droppedCount atomic.Uint64 // Total messages dropped; not under mu
|
droppedCount uint64 // Total messages dropped
|
||||||
totalTime time.Duration // Total processing time (for average calculation)
|
totalTime time.Duration // Total processing time (for average calculation)
|
||||||
minTime time.Duration // Minimum processing time
|
minTime time.Duration // Minimum processing time
|
||||||
maxTime time.Duration // Maximum processing time
|
maxTime time.Duration // Maximum processing time
|
||||||
queueHighWaterMark int // Maximum queue length seen
|
queueHighWaterMark int // Maximum queue length seen
|
||||||
mu sync.Mutex // Protects the other metrics
|
mu sync.Mutex // Protects the metrics
|
||||||
}
|
}
|
||||||
|
|
||||||
// handlerInfo wraps a handler with its queue and metrics
|
// handlerInfo wraps a handler with its queue and metrics
|
||||||
@@ -253,10 +253,9 @@ func (s *Streamer) Start(ctx context.Context) error {
|
|||||||
|
|
||||||
// Stop ends the stream and stops the handler workers. It returns once each
|
// Stop ends the stream and stops the handler workers. It returns once each
|
||||||
// worker has handed its handler every message left on its queue and then the
|
// worker has handed its handler every message left on its queue and then the
|
||||||
// STATE messages held for it (see queueMessage), or once ctx ends, and then
|
// STATE messages held for it (see queueMessage). This method is safe to call
|
||||||
// logs how many messages were not handed over. This method is safe to call
|
|
||||||
// multiple times.
|
// multiple times.
|
||||||
func (s *Streamer) Stop(ctx context.Context) {
|
func (s *Streamer) Stop() {
|
||||||
s.mu.Lock()
|
s.mu.Lock()
|
||||||
if s.cancel == nil {
|
if s.cancel == nil {
|
||||||
// Not started, or already stopped: closing the queues again would panic.
|
// Not started, or already stopped: closing the queues again would panic.
|
||||||
@@ -265,56 +264,28 @@ func (s *Streamer) Stop(ctx context.Context) {
|
|||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
// Once the stream's context is cancelled the read loop queues and holds
|
// Once ctx is cancelled the read loop queues and holds nothing more (see
|
||||||
// nothing more (see dispatchMessage).
|
// dispatchMessage).
|
||||||
s.cancel()
|
s.cancel()
|
||||||
s.cancel = nil
|
s.cancel = nil
|
||||||
|
|
||||||
notHandedOver := 0
|
|
||||||
|
|
||||||
for _, info := range s.handlers {
|
for _, info := range s.handlers {
|
||||||
// The held messages came after everything on the queue. Each waits
|
// The held messages came after everything on the queue. Each waits
|
||||||
// for the worker, which does not take s.mu, to make room, until ctx
|
// for the worker, which does not take s.mu, to make room.
|
||||||
// ends.
|
for _, msg := range info.heldState {
|
||||||
for len(info.heldState) > 0 && ctx.Err() == nil {
|
info.queue <- msg
|
||||||
select {
|
|
||||||
case info.queue <- info.heldState[0]:
|
|
||||||
info.heldState = info.heldState[1:]
|
|
||||||
case <-ctx.Done():
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
notHandedOver += len(info.heldState)
|
|
||||||
info.heldState = nil
|
info.heldState = nil
|
||||||
|
|
||||||
// The worker returns once it has emptied the closed queue.
|
// The worker returns once it has emptied the closed queue.
|
||||||
close(info.queue)
|
close(info.queue)
|
||||||
}
|
}
|
||||||
|
|
||||||
handlers := s.handlers
|
|
||||||
s.running = false
|
s.running = false
|
||||||
s.mu.Unlock()
|
s.mu.Unlock()
|
||||||
s.metrics.SetConnected(false)
|
s.metrics.SetConnected(false)
|
||||||
|
s.workers.Wait()
|
||||||
workersDone := make(chan struct{})
|
|
||||||
|
|
||||||
go func() {
|
|
||||||
s.workers.Wait()
|
|
||||||
close(workersDone)
|
|
||||||
}()
|
|
||||||
|
|
||||||
select {
|
|
||||||
case <-workersDone:
|
|
||||||
case <-ctx.Done():
|
|
||||||
for _, info := range handlers {
|
|
||||||
notHandedOver += len(info.queue)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if notHandedOver > 0 {
|
|
||||||
s.logger.Error("Stop deadline passed before the handlers took every "+
|
|
||||||
"message", "messages", notHandedOver)
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// IsRunning reports whether the streamer is currently connected and processing
|
// IsRunning reports whether the streamer is currently connected and processing
|
||||||
@@ -371,7 +342,7 @@ func (s *Streamer) GetHandlerStats() []HandlerStats {
|
|||||||
QueueCapacity: cap(info.queue),
|
QueueCapacity: cap(info.queue),
|
||||||
QueueHighWaterMark: info.metrics.queueHighWaterMark,
|
QueueHighWaterMark: info.metrics.queueHighWaterMark,
|
||||||
ProcessedCount: info.metrics.processedCount,
|
ProcessedCount: info.metrics.processedCount,
|
||||||
DroppedCount: info.metrics.droppedCount.Load(),
|
DroppedCount: info.metrics.droppedCount,
|
||||||
MinProcessTime: info.metrics.minTime,
|
MinProcessTime: info.metrics.minTime,
|
||||||
MaxProcessTime: info.metrics.maxTime,
|
MaxProcessTime: info.metrics.maxTime,
|
||||||
}
|
}
|
||||||
@@ -488,7 +459,7 @@ func (s *Streamer) logMetrics() {
|
|||||||
"queue_len", len(info.queue),
|
"queue_len", len(info.queue),
|
||||||
"queue_cap", cap(info.queue),
|
"queue_cap", cap(info.queue),
|
||||||
"processed", info.metrics.processedCount,
|
"processed", info.metrics.processedCount,
|
||||||
"dropped", info.metrics.droppedCount.Load(),
|
"dropped", info.metrics.droppedCount,
|
||||||
"avg_time", avgTime,
|
"avg_time", avgTime,
|
||||||
"min_time", info.metrics.minTime,
|
"min_time", info.metrics.minTime,
|
||||||
"max_time", info.metrics.maxTime,
|
"max_time", info.metrics.maxTime,
|
||||||
@@ -858,7 +829,7 @@ func (s *Streamer) dispatchMessage(
|
|||||||
}
|
}
|
||||||
|
|
||||||
if !s.queueMessage(info, msg) {
|
if !s.queueMessage(info, msg) {
|
||||||
info.metrics.droppedCount.Add(1)
|
atomic.AddUint64(&info.metrics.droppedCount, 1)
|
||||||
s.totalDropped.Add(1)
|
s.totalDropped.Add(1)
|
||||||
|
|
||||||
continue
|
continue
|
||||||
|
|||||||
@@ -1,12 +1,9 @@
|
|||||||
package streamer
|
package streamer
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"bytes"
|
|
||||||
"context"
|
"context"
|
||||||
"encoding/json"
|
|
||||||
"errors"
|
"errors"
|
||||||
"io"
|
"io"
|
||||||
"log/slog"
|
|
||||||
"net/http"
|
"net/http"
|
||||||
"net/http/httptest"
|
"net/http/httptest"
|
||||||
"runtime"
|
"runtime"
|
||||||
@@ -380,7 +377,7 @@ func TestStopQueuesTheHeldStateMessages(t *testing.T) {
|
|||||||
stopped := make(chan struct{})
|
stopped := make(chan struct{})
|
||||||
|
|
||||||
go func() {
|
go func() {
|
||||||
s.Stop(t.Context())
|
s.Stop()
|
||||||
close(stopped)
|
close(stopped)
|
||||||
}()
|
}()
|
||||||
|
|
||||||
@@ -415,68 +412,6 @@ func TestStopQueuesTheHeldStateMessages(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// TestStopReturnsByItsDeadline stops a streamer whose handler has a full queue
|
|
||||||
// and a STATE message held for it, and a worker that stays busy until the test
|
|
||||||
// ends, as one does whose handler waits for a database write. Stop must return
|
|
||||||
// within its deadline plus a small margin instead of waiting for room on the
|
|
||||||
// queue and for the worker, and log as not handed over the held message and
|
|
||||||
// every message on the queue.
|
|
||||||
func TestStopReturnsByItsDeadline(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
const (
|
|
||||||
deadline = 100 * time.Millisecond
|
|
||||||
margin = 2 * time.Second
|
|
||||||
)
|
|
||||||
|
|
||||||
var logs bytes.Buffer
|
|
||||||
|
|
||||||
s := newStreamerWithFullStateQueue("")
|
|
||||||
s.logger = &logger.Logger{Logger: slog.New(slog.NewJSONHandler(&logs,
|
|
||||||
&slog.HandlerOptions{Level: slog.LevelError}))}
|
|
||||||
s.handlers[0].heldState = []*ristypes.RISMessage{
|
|
||||||
{Type: ristypes.MessageTypeState},
|
|
||||||
}
|
|
||||||
wantNotHandedOver := 1 + len(s.handlers[0].queue)
|
|
||||||
// Stop cancels the stream Start would have started; there is none here.
|
|
||||||
s.cancel = func() {}
|
|
||||||
|
|
||||||
release := make(chan struct{})
|
|
||||||
defer close(release)
|
|
||||||
|
|
||||||
s.workers.Go(func() { <-release })
|
|
||||||
|
|
||||||
ctx, cancel := context.WithTimeout(t.Context(), deadline)
|
|
||||||
defer cancel()
|
|
||||||
|
|
||||||
stopped := make(chan struct{})
|
|
||||||
|
|
||||||
go func() {
|
|
||||||
s.Stop(ctx)
|
|
||||||
close(stopped)
|
|
||||||
}()
|
|
||||||
|
|
||||||
select {
|
|
||||||
case <-stopped:
|
|
||||||
case <-time.After(deadline + margin):
|
|
||||||
t.Fatalf("Stop did not return within %s of its deadline", margin)
|
|
||||||
}
|
|
||||||
|
|
||||||
var logged struct {
|
|
||||||
Messages int `json:"messages"`
|
|
||||||
}
|
|
||||||
|
|
||||||
err := json.Unmarshal(logs.Bytes(), &logged)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("want the error Stop logs, got %q: %v", logs.String(), err)
|
|
||||||
}
|
|
||||||
|
|
||||||
if logged.Messages != wantNotHandedOver {
|
|
||||||
t.Errorf("Stop logged %d messages not handed over, want %d",
|
|
||||||
logged.Messages, wantNotHandedOver)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestStopBeforeMessageReachesQueues stops the streamer after the read loop
|
// TestStopBeforeMessageReachesQueues stops the streamer after the read loop
|
||||||
// has checked for cancellation but before it hands the message to the handler
|
// has checked for cancellation but before it hands the message to the handler
|
||||||
// queues. That is the gap a stop from another goroutine can land in, and it
|
// queues. That is the gap a stop from another goroutine can land in, and it
|
||||||
@@ -494,7 +429,7 @@ func TestStopBeforeMessageReachesQueues(t *testing.T) {
|
|||||||
s := New(logger.New(), metrics.New())
|
s := New(logger.New(), metrics.New())
|
||||||
s.url = srv.URL
|
s.url = srv.URL
|
||||||
s.RegisterHandler(updateHandler{})
|
s.RegisterHandler(updateHandler{})
|
||||||
s.RegisterRawHandler(func(string) { s.Stop(t.Context()) })
|
s.RegisterRawHandler(func(string) { s.Stop() })
|
||||||
|
|
||||||
// Start would run the stream in the background, where the test cannot
|
// Start would run the stream in the background, where the test cannot
|
||||||
// wait for it. Setting cancel as Start does lets Stop cancel the stream
|
// wait for it. Setting cancel as Start does lets Stop cancel the stream
|
||||||
@@ -508,7 +443,7 @@ func TestStopBeforeMessageReachesQueues(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// A second Stop must not close the queues again.
|
// A second Stop must not close the queues again.
|
||||||
s.Stop(t.Context())
|
s.Stop()
|
||||||
}
|
}
|
||||||
|
|
||||||
// TestReconnectWhenStreamEnds has the server end the first response
|
// TestReconnectWhenStreamEnds has the server end the first response
|
||||||
@@ -621,68 +556,6 @@ func TestStreamReadsLongLine(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// blockedHandler wants UPDATE messages, has room on its queue for one, and
|
|
||||||
// does not return from HandleMessage until release is closed.
|
|
||||||
type blockedHandler struct {
|
|
||||||
release chan struct{}
|
|
||||||
}
|
|
||||||
|
|
||||||
func (blockedHandler) WantsMessage(messageType string) bool {
|
|
||||||
return messageType == ristypes.MessageTypeUpdate
|
|
||||||
}
|
|
||||||
|
|
||||||
func (blockedHandler) QueueCapacity() int { return 1 }
|
|
||||||
|
|
||||||
func (h blockedHandler) HandleMessage(*ristypes.RISMessage) { <-h.release }
|
|
||||||
|
|
||||||
// TestHandlerStatsReadWhileMessagesAreDropped has the server send a few UPDATE
|
|
||||||
// lines to a blockedHandler, so the read loop drops all but the one the
|
|
||||||
// handler holds and the one on its queue, while the test reads the handler's
|
|
||||||
// statistics. The drop count must reach the number dropped, and the race
|
|
||||||
// detector must find the count read and written safely.
|
|
||||||
func TestHandlerStatsReadWhileMessagesAreDropped(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
const (
|
|
||||||
lines = 5
|
|
||||||
wantDropped = lines - 2
|
|
||||||
maxWait = 5 * time.Second
|
|
||||||
)
|
|
||||||
|
|
||||||
srv := httptest.NewServer(http.HandlerFunc(
|
|
||||||
func(w http.ResponseWriter, r *http.Request) {
|
|
||||||
_, _ = io.WriteString(w, strings.Repeat(updateLine, lines))
|
|
||||||
|
|
||||||
flush(w)
|
|
||||||
<-r.Context().Done()
|
|
||||||
}))
|
|
||||||
defer srv.Close()
|
|
||||||
|
|
||||||
s := New(logger.New(), metrics.New())
|
|
||||||
s.url = srv.URL
|
|
||||||
handler := blockedHandler{release: make(chan struct{})}
|
|
||||||
s.RegisterHandler(handler)
|
|
||||||
|
|
||||||
err := s.Start(t.Context())
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("Start returned error: %v", err)
|
|
||||||
}
|
|
||||||
defer s.Stop(t.Context())
|
|
||||||
defer close(handler.release)
|
|
||||||
|
|
||||||
// The test polls instead of waiting for a signal from the read loop: a
|
|
||||||
// signal would order the drops before the reads here, and the race
|
|
||||||
// detector would have nothing to report.
|
|
||||||
deadline := time.Now().Add(maxWait)
|
|
||||||
|
|
||||||
for s.GetHandlerStats()[0].DroppedCount < wantDropped {
|
|
||||||
if time.Now().After(deadline) {
|
|
||||||
t.Fatalf("fewer than %d messages counted as dropped within %s",
|
|
||||||
wantDropped, maxWait)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestPeerCountCountsEveryPeerHeardFrom has the server send an UPDATE from
|
// TestPeerCountCountsEveryPeerHeardFrom has the server send an UPDATE from
|
||||||
// stateDownPeer and a KEEPALIVE from another peer, with no OPEN from either,
|
// stateDownPeer and a KEEPALIVE from another peer, with no OPEN from either,
|
||||||
// and then a message that ends stateDownPeer's session: a NOTIFICATION, or a
|
// and then a message that ends stateDownPeer's session: a NOTIFICATION, or a
|
||||||
@@ -803,7 +676,7 @@ func checkReconnect(
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("Start returned error: %v", err)
|
t.Fatalf("Start returned error: %v", err)
|
||||||
}
|
}
|
||||||
defer s.Stop(t.Context())
|
defer s.Stop()
|
||||||
|
|
||||||
select {
|
select {
|
||||||
case <-handler.handled:
|
case <-handler.handled:
|
||||||
|
|||||||
@@ -4,7 +4,7 @@
|
|||||||
<meta charset="UTF-8">
|
<meta charset="UTF-8">
|
||||||
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
||||||
<title>AS{{.ASN.ASN}} - {{.ASN.Handle}} - RouteWatch</title>
|
<title>AS{{.ASN.ASN}} - {{.ASN.Handle}} - RouteWatch</title>
|
||||||
<style nonce="{{.CSPNonce}}">
|
<style>
|
||||||
* {
|
* {
|
||||||
box-sizing: border-box;
|
box-sizing: border-box;
|
||||||
}
|
}
|
||||||
@@ -126,26 +126,6 @@
|
|||||||
margin: 0;
|
margin: 0;
|
||||||
color: #2c3e50;
|
color: #2c3e50;
|
||||||
}
|
}
|
||||||
/* A click on a prefix list's heading folds or unfolds the list. Not
|
|
||||||
every browser shows its own marker on a flex element, so the page
|
|
||||||
hides it and draws a triangle before the heading. The triangle
|
|
||||||
points down while the list is open. */
|
|
||||||
summary.prefix-header {
|
|
||||||
cursor: pointer;
|
|
||||||
list-style: none;
|
|
||||||
}
|
|
||||||
summary.prefix-header::-webkit-details-marker {
|
|
||||||
display: none;
|
|
||||||
}
|
|
||||||
summary.prefix-header h2::before {
|
|
||||||
content: "\25B8";
|
|
||||||
display: inline-block;
|
|
||||||
margin-right: 10px;
|
|
||||||
transition: transform 0.2s;
|
|
||||||
}
|
|
||||||
details[open] > summary.prefix-header h2::before {
|
|
||||||
transform: rotate(90deg);
|
|
||||||
}
|
|
||||||
.prefix-count {
|
.prefix-count {
|
||||||
background: #e74c3c;
|
background: #e74c3c;
|
||||||
color: white;
|
color: white;
|
||||||
@@ -253,20 +233,18 @@
|
|||||||
<div class="info-label">Peer ASNs</div>
|
<div class="info-label">Peer ASNs</div>
|
||||||
<div class="info-value">{{.PeerCount}}</div>
|
<div class="info-value">{{.PeerCount}}</div>
|
||||||
</div>
|
</div>
|
||||||
{{if not .ASN.FirstSeen.IsZero}}
|
|
||||||
<div class="info-card">
|
<div class="info-card">
|
||||||
<div class="info-label">First Seen</div>
|
<div class="info-label">First Seen</div>
|
||||||
<div class="info-value">{{.ASN.FirstSeen.Format "2006-01-02"}}</div>
|
<div class="info-value">{{.ASN.FirstSeen.Format "2006-01-02"}}</div>
|
||||||
</div>
|
</div>
|
||||||
{{end}}
|
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
{{if .IPv4Prefixes}}
|
{{if .IPv4Prefixes}}
|
||||||
<details class="prefix-section" open>
|
<div class="prefix-section">
|
||||||
<summary class="prefix-header">
|
<div class="prefix-header">
|
||||||
<h2>IPv4 Prefixes</h2>
|
<h2>IPv4 Prefixes</h2>
|
||||||
<span class="prefix-count">{{.IPv4Count}}</span>
|
<span class="prefix-count">{{.IPv4Count}}</span>
|
||||||
</summary>
|
</div>
|
||||||
<table class="prefix-table">
|
<table class="prefix-table">
|
||||||
<thead>
|
<thead>
|
||||||
<tr>
|
<tr>
|
||||||
@@ -287,15 +265,15 @@
|
|||||||
{{end}}
|
{{end}}
|
||||||
</tbody>
|
</tbody>
|
||||||
</table>
|
</table>
|
||||||
</details>
|
</div>
|
||||||
{{end}}
|
{{end}}
|
||||||
|
|
||||||
{{if .IPv6Prefixes}}
|
{{if .IPv6Prefixes}}
|
||||||
<details class="prefix-section" open>
|
<div class="prefix-section">
|
||||||
<summary class="prefix-header">
|
<div class="prefix-header">
|
||||||
<h2>IPv6 Prefixes</h2>
|
<h2>IPv6 Prefixes</h2>
|
||||||
<span class="prefix-count">{{.IPv6Count}}</span>
|
<span class="prefix-count">{{.IPv6Count}}</span>
|
||||||
</summary>
|
</div>
|
||||||
<table class="prefix-table">
|
<table class="prefix-table">
|
||||||
<thead>
|
<thead>
|
||||||
<tr>
|
<tr>
|
||||||
@@ -316,7 +294,7 @@
|
|||||||
{{end}}
|
{{end}}
|
||||||
</tbody>
|
</tbody>
|
||||||
</table>
|
</table>
|
||||||
</details>
|
</div>
|
||||||
{{end}}
|
{{end}}
|
||||||
|
|
||||||
{{if eq .TotalCount 0}}
|
{{if eq .TotalCount 0}}
|
||||||
|
|||||||
+21
-210
@@ -4,7 +4,7 @@
|
|||||||
<meta charset="UTF-8">
|
<meta charset="UTF-8">
|
||||||
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
||||||
<title>RouteWatch - BGP Route Monitor</title>
|
<title>RouteWatch - BGP Route Monitor</title>
|
||||||
<style nonce="{{.CSPNonce}}">
|
<style>
|
||||||
* {
|
* {
|
||||||
box-sizing: border-box;
|
box-sizing: border-box;
|
||||||
}
|
}
|
||||||
@@ -107,15 +107,9 @@
|
|||||||
.stat-card.connected .stat-value {
|
.stat-card.connected .stat-value {
|
||||||
color: #27ae60;
|
color: #27ae60;
|
||||||
}
|
}
|
||||||
.stat-card.disconnected .stat-value,
|
.stat-card.disconnected .stat-value {
|
||||||
.stale {
|
|
||||||
color: #e74c3c;
|
color: #e74c3c;
|
||||||
}
|
}
|
||||||
/* The card has no room beside "Connected", so the spinner and "stale" go under it */
|
|
||||||
#freshness {
|
|
||||||
display: block;
|
|
||||||
font-size: 20px;
|
|
||||||
}
|
|
||||||
|
|
||||||
/* Search section */
|
/* Search section */
|
||||||
.search-section {
|
.search-section {
|
||||||
@@ -173,45 +167,6 @@
|
|||||||
color: #95a5a6;
|
color: #95a5a6;
|
||||||
margin-top: 8px;
|
margin-top: 8px;
|
||||||
}
|
}
|
||||||
#asname-results p {
|
|
||||||
color: #7f8c8d;
|
|
||||||
margin-top: 12px;
|
|
||||||
}
|
|
||||||
#asname-results .search-message {
|
|
||||||
font-size: 13px;
|
|
||||||
}
|
|
||||||
#asname-results .error {
|
|
||||||
color: #c00;
|
|
||||||
}
|
|
||||||
.asname-list {
|
|
||||||
list-style: none;
|
|
||||||
margin: 12px 0 0 0;
|
|
||||||
padding: 0;
|
|
||||||
max-height: 400px;
|
|
||||||
overflow-y: auto;
|
|
||||||
}
|
|
||||||
.asname-list li {
|
|
||||||
border-bottom: 1px solid #eee;
|
|
||||||
}
|
|
||||||
.asname-list a {
|
|
||||||
display: block;
|
|
||||||
padding: 8px 4px;
|
|
||||||
color: #2c3e50;
|
|
||||||
text-decoration: none;
|
|
||||||
font-size: 14px;
|
|
||||||
}
|
|
||||||
.asname-list a:hover {
|
|
||||||
background: #f5f5f5;
|
|
||||||
}
|
|
||||||
.asname-list .asn {
|
|
||||||
font-weight: 600;
|
|
||||||
color: #3498db;
|
|
||||||
}
|
|
||||||
.asname-list .description {
|
|
||||||
display: block;
|
|
||||||
color: #7f8c8d;
|
|
||||||
font-size: 12px;
|
|
||||||
}
|
|
||||||
|
|
||||||
/* IP Lookup result */
|
/* IP Lookup result */
|
||||||
.ip-result {
|
.ip-result {
|
||||||
@@ -239,8 +194,7 @@
|
|||||||
cursor: pointer;
|
cursor: pointer;
|
||||||
font-size: 12px;
|
font-size: 12px;
|
||||||
}
|
}
|
||||||
.ip-result pre,
|
.ip-result pre {
|
||||||
#ip-api pre {
|
|
||||||
background: #2c3e50;
|
background: #2c3e50;
|
||||||
color: #ecf0f1;
|
color: #ecf0f1;
|
||||||
padding: 16px;
|
padding: 16px;
|
||||||
@@ -252,30 +206,18 @@
|
|||||||
max-height: 400px;
|
max-height: 400px;
|
||||||
overflow-y: auto;
|
overflow-y: auto;
|
||||||
}
|
}
|
||||||
.ip-result .error,
|
.ip-result .error {
|
||||||
#ip-api .error {
|
|
||||||
background: #fee;
|
background: #fee;
|
||||||
color: #c00;
|
color: #c00;
|
||||||
padding: 12px;
|
padding: 12px;
|
||||||
border-radius: 6px;
|
border-radius: 6px;
|
||||||
font-size: 14px;
|
font-size: 14px;
|
||||||
}
|
}
|
||||||
.ip-result .loading,
|
.ip-result .loading {
|
||||||
#ip-api .loading {
|
|
||||||
color: #7f8c8d;
|
color: #7f8c8d;
|
||||||
font-style: italic;
|
font-style: italic;
|
||||||
}
|
}
|
||||||
|
|
||||||
/* IP information API card: the command, then its answer */
|
|
||||||
#ip-api p {
|
|
||||||
margin: 0;
|
|
||||||
color: #7f8c8d;
|
|
||||||
font-size: 14px;
|
|
||||||
}
|
|
||||||
#ip-api pre {
|
|
||||||
margin-top: 12px;
|
|
||||||
}
|
|
||||||
|
|
||||||
/* Footer */
|
/* Footer */
|
||||||
.footer {
|
.footer {
|
||||||
margin-top: 40px;
|
margin-top: 40px;
|
||||||
@@ -316,10 +258,7 @@
|
|||||||
<main class="main-content">
|
<main class="main-content">
|
||||||
<div class="stats-grid">
|
<div class="stats-grid">
|
||||||
<div class="stat-card" id="status-card">
|
<div class="stat-card" id="status-card">
|
||||||
<div class="stat-value">
|
<div class="stat-value" id="stat-status">-</div>
|
||||||
<span id="stat-status">-</span>
|
|
||||||
<span id="freshness"></span>
|
|
||||||
</div>
|
|
||||||
<div class="stat-label">Status</div>
|
<div class="stat-label">Status</div>
|
||||||
</div>
|
</div>
|
||||||
<div class="stat-card">
|
<div class="stat-card">
|
||||||
@@ -360,7 +299,7 @@
|
|||||||
<input type="text" id="asname-input" placeholder="e.g., Google, Cloudflare" autocomplete="off">
|
<input type="text" id="asname-input" placeholder="e.g., Google, Cloudflare" autocomplete="off">
|
||||||
<button type="submit">Search</button>
|
<button type="submit">Search</button>
|
||||||
</form>
|
</form>
|
||||||
<p class="search-hint">Search for autonomous systems by organization name or handle</p>
|
<p class="search-hint">Search for autonomous systems by organization name</p>
|
||||||
<div id="asname-results"></div>
|
<div id="asname-results"></div>
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
@@ -380,13 +319,6 @@
|
|||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
<div class="search-card" id="ip-api">
|
|
||||||
<h2>IP Information API</h2>
|
|
||||||
<p>The API answers what RouteWatch knows about an IP address, such as its AS number and country. <code>/ip/</code> with no address answers for the caller's own address.</p>
|
|
||||||
<pre id="ip-api-command" data-client-ip="{{.ClientIP}}"></pre>
|
|
||||||
<pre id="ip-api-answer" class="loading">Loading...</pre>
|
|
||||||
</div>
|
|
||||||
</main>
|
</main>
|
||||||
|
|
||||||
<footer class="footer">
|
<footer class="footer">
|
||||||
@@ -397,45 +329,7 @@
|
|||||||
<span><a href="{{appGitCommitURL}}">{{appGitRevision}}</a></span>
|
<span><a href="{{appGitCommitURL}}">{{appGitRevision}}</a></span>
|
||||||
</footer>
|
</footer>
|
||||||
|
|
||||||
<script nonce="{{.CSPNonce}}">
|
<script>
|
||||||
// The stats are fetched every REFRESH_PERIOD_MS. Next to the status, a
|
|
||||||
// spinner turns one frame each time an answer counts more messages
|
|
||||||
// than the one before. After STALE_AFTER_REFRESHES refreshes without
|
|
||||||
// such an answer, the spinner gives way to "stale" and the time since.
|
|
||||||
const REFRESH_PERIOD_MS = 5000;
|
|
||||||
const STALE_AFTER_REFRESHES = 3;
|
|
||||||
const MS_PER_SECOND = 1000;
|
|
||||||
const SPINNER_FRAMES = ['⠋', '⠙', '⠹', '⠸', '⠼', '⠴', '⠦', '⠧', '⠇', '⠏'];
|
|
||||||
const SPINNER_COLOURS = ['#e74c3c', '#e67e22', '#f1c40f', '#27ae60', '#3498db', '#9b59b6'];
|
|
||||||
|
|
||||||
let spinnerFrame = 0;
|
|
||||||
let lastMessageCount = null;
|
|
||||||
let lastFreshAt = Date.now();
|
|
||||||
|
|
||||||
// Turn the spinner when the message count has risen since the last answer.
|
|
||||||
function noteMessageCount(count) {
|
|
||||||
if (lastMessageCount !== null && count > lastMessageCount) {
|
|
||||||
spinnerFrame = (spinnerFrame + 1) % SPINNER_FRAMES.length;
|
|
||||||
lastFreshAt = Date.now();
|
|
||||||
}
|
|
||||||
lastMessageCount = count;
|
|
||||||
showFreshness();
|
|
||||||
}
|
|
||||||
|
|
||||||
function showFreshness() {
|
|
||||||
const freshness = document.getElementById('freshness');
|
|
||||||
const sinceFreshMs = Date.now() - lastFreshAt;
|
|
||||||
const stale = sinceFreshMs >= STALE_AFTER_REFRESHES * REFRESH_PERIOD_MS;
|
|
||||||
freshness.classList.toggle('stale', stale);
|
|
||||||
if (stale) {
|
|
||||||
freshness.textContent = 'stale for ' + Math.floor(sinceFreshMs / MS_PER_SECOND) + 's';
|
|
||||||
freshness.style.color = '';
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
freshness.textContent = SPINNER_FRAMES[spinnerFrame];
|
|
||||||
freshness.style.color = SPINNER_COLOURS[spinnerFrame % SPINNER_COLOURS.length];
|
|
||||||
}
|
|
||||||
|
|
||||||
function formatNumber(num) {
|
function formatNumber(num) {
|
||||||
if (num >= 1000000) {
|
if (num >= 1000000) {
|
||||||
return (num / 1000000).toFixed(1) + 'M';
|
return (num / 1000000).toFixed(1) + 'M';
|
||||||
@@ -457,7 +351,6 @@
|
|||||||
const statusEl = document.getElementById('stat-status');
|
const statusEl = document.getElementById('stat-status');
|
||||||
statusEl.textContent = data.connected ? 'Connected' : 'Disconnected';
|
statusEl.textContent = data.connected ? 'Connected' : 'Disconnected';
|
||||||
statusCard.className = 'stat-card ' + (data.connected ? 'connected' : 'disconnected');
|
statusCard.className = 'stat-card ' + (data.connected ? 'connected' : 'disconnected');
|
||||||
noteMessageCount(data.total_messages);
|
|
||||||
|
|
||||||
document.getElementById('stat-routes').textContent = formatNumber(data.live_routes);
|
document.getElementById('stat-routes').textContent = formatNumber(data.live_routes);
|
||||||
document.getElementById('stat-asns').textContent = formatNumber(data.asns);
|
document.getElementById('stat-asns').textContent = formatNumber(data.asns);
|
||||||
@@ -468,7 +361,7 @@
|
|||||||
}
|
}
|
||||||
|
|
||||||
const totalUpdates = data.ipv4_updates_per_sec + data.ipv6_updates_per_sec;
|
const totalUpdates = data.ipv4_updates_per_sec + data.ipv6_updates_per_sec;
|
||||||
document.getElementById('stat-updates').textContent = Math.round(totalUpdates).toLocaleString();
|
document.getElementById('stat-updates').textContent = totalUpdates.toFixed(1);
|
||||||
})
|
})
|
||||||
.catch(() => {
|
.catch(() => {
|
||||||
document.getElementById('stat-status').textContent = 'Error';
|
document.getElementById('stat-status').textContent = 'Error';
|
||||||
@@ -487,82 +380,24 @@
|
|||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
|
||||||
// AS name search. The handles and descriptions come from outside data,
|
// AS name search
|
||||||
// so the results are built with textContent, never innerHTML.
|
|
||||||
const asNameResults = document.getElementById('asname-results');
|
|
||||||
|
|
||||||
// Replace the search results with one line of text.
|
|
||||||
function showASNameMessage(text, className) {
|
|
||||||
const line = document.createElement('p');
|
|
||||||
line.className = className;
|
|
||||||
line.textContent = text;
|
|
||||||
asNameResults.replaceChildren(line);
|
|
||||||
}
|
|
||||||
|
|
||||||
// List the matches of a search answer, each as a link to its AS page,
|
|
||||||
// and say how many more matched than the answer holds.
|
|
||||||
function showASNameMatches(data) {
|
|
||||||
if (data.total === 0) {
|
|
||||||
showASNameMessage('No AS matches "' + data.query + '".', 'search-message');
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
const list = document.createElement('ul');
|
|
||||||
list.className = 'asname-list';
|
|
||||||
for (const match of data.results) {
|
|
||||||
const asn = document.createElement('span');
|
|
||||||
asn.className = 'asn';
|
|
||||||
asn.textContent = 'AS' + match.asn;
|
|
||||||
|
|
||||||
const description = document.createElement('span');
|
|
||||||
description.className = 'description';
|
|
||||||
description.textContent = match.description;
|
|
||||||
|
|
||||||
const link = document.createElement('a');
|
|
||||||
link.href = '/as/' + match.asn;
|
|
||||||
link.append(asn, ' ' + match.handle, description);
|
|
||||||
|
|
||||||
const item = document.createElement('li');
|
|
||||||
item.append(link);
|
|
||||||
list.append(item);
|
|
||||||
}
|
|
||||||
asNameResults.replaceChildren(list);
|
|
||||||
|
|
||||||
const more = data.total - data.results.length;
|
|
||||||
if (more > 0) {
|
|
||||||
const line = document.createElement('p');
|
|
||||||
line.className = 'search-message';
|
|
||||||
line.textContent = more.toLocaleString() + ' more matched. Narrow the search to see them.';
|
|
||||||
asNameResults.append(line);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
document.getElementById('asname-form').addEventListener('submit', function(e) {
|
document.getElementById('asname-form').addEventListener('submit', function(e) {
|
||||||
e.preventDefault();
|
e.preventDefault();
|
||||||
const query = document.getElementById('asname-input').value.trim();
|
const query = document.getElementById('asname-input').value.trim();
|
||||||
if (!query) return;
|
if (!query) return;
|
||||||
|
|
||||||
// A number, or AS and a number, opens that AS's page
|
const resultsDiv = document.getElementById('asname-results');
|
||||||
const asn = query.replace(/^AS/i, '');
|
resultsDiv.innerHTML = '<p class="loading" style="color: #7f8c8d; margin-top: 12px;">Searching...</p>';
|
||||||
if (/^\d+$/.test(asn)) {
|
|
||||||
window.location.href = '/as/' + asn;
|
// Use a simple client-side search against the asinfo data
|
||||||
|
// For now, redirect to AS page if it looks like an ASN
|
||||||
|
if (/^\d+$/.test(query)) {
|
||||||
|
window.location.href = '/as/' + query;
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
showASNameMessage('Searching...', 'loading');
|
// Show a message that server-side search is coming
|
||||||
|
resultsDiv.innerHTML = '<p style="color: #7f8c8d; margin-top: 12px; font-size: 13px;">AS name search coming soon. For now, try an AS number.</p>';
|
||||||
fetch('/api/v1/search/as?q=' + encodeURIComponent(query))
|
|
||||||
.then(response => response.json())
|
|
||||||
.then(response => {
|
|
||||||
if (response.status === 'error') {
|
|
||||||
showASNameMessage('Error: ' + response.error.msg, 'error');
|
|
||||||
} else {
|
|
||||||
showASNameMatches(response.data);
|
|
||||||
}
|
|
||||||
})
|
|
||||||
.catch(error => {
|
|
||||||
showASNameMessage('Error: ' + error.message, 'error');
|
|
||||||
});
|
|
||||||
});
|
});
|
||||||
|
|
||||||
// IP lookup
|
// IP lookup
|
||||||
@@ -604,33 +439,9 @@
|
|||||||
document.getElementById('ip-input').value = '';
|
document.getElementById('ip-input').value = '';
|
||||||
});
|
});
|
||||||
|
|
||||||
// IP information API card: show the curl command that asks /ip/ about
|
// Initial load and refresh stats every 5 seconds
|
||||||
// the visitor's own address, then make that same request and show the
|
|
||||||
// JSON it answers, an error answer included. The answer holds outside
|
|
||||||
// data, so it is set with textContent, never innerHTML.
|
|
||||||
const ipAPICommand = document.getElementById('ip-api-command');
|
|
||||||
const ipAPIAnswer = document.getElementById('ip-api-answer');
|
|
||||||
const ipAPIURL = window.location.origin + '/ip/' + ipAPICommand.dataset.clientIp;
|
|
||||||
|
|
||||||
ipAPICommand.textContent = 'curl ' + ipAPIURL;
|
|
||||||
|
|
||||||
fetch(ipAPIURL)
|
|
||||||
.then(response => response.json())
|
|
||||||
.then(answer => {
|
|
||||||
ipAPIAnswer.className = '';
|
|
||||||
ipAPIAnswer.textContent = JSON.stringify(answer, null, 2);
|
|
||||||
})
|
|
||||||
.catch(error => {
|
|
||||||
ipAPIAnswer.className = 'error';
|
|
||||||
ipAPIAnswer.textContent = 'Error: ' + error.message;
|
|
||||||
});
|
|
||||||
|
|
||||||
// Initial load and refresh stats every REFRESH_PERIOD_MS. The freshness
|
|
||||||
// check runs every second on its own, so a request that never returns
|
|
||||||
// still turns the spinner into "stale".
|
|
||||||
updateStats();
|
updateStats();
|
||||||
setInterval(updateStats, REFRESH_PERIOD_MS);
|
setInterval(updateStats, 5000);
|
||||||
setInterval(showFreshness, MS_PER_SECOND);
|
|
||||||
</script>
|
</script>
|
||||||
</body>
|
</body>
|
||||||
</html>
|
</html>
|
||||||
|
|||||||
@@ -4,7 +4,7 @@
|
|||||||
<meta charset="UTF-8">
|
<meta charset="UTF-8">
|
||||||
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
||||||
<title>{{.Prefix}} - RouteWatch</title>
|
<title>{{.Prefix}} - RouteWatch</title>
|
||||||
<style nonce="{{.CSPNonce}}">
|
<style>
|
||||||
* {
|
* {
|
||||||
box-sizing: border-box;
|
box-sizing: border-box;
|
||||||
}
|
}
|
||||||
@@ -221,10 +221,6 @@
|
|||||||
border-radius: 4px;
|
border-radius: 4px;
|
||||||
border: 1px solid #e0e0e0;
|
border: 1px solid #e0e0e0;
|
||||||
}
|
}
|
||||||
.origin-item .peer-count {
|
|
||||||
color: #7f8c8d;
|
|
||||||
margin-left: 10px;
|
|
||||||
}
|
|
||||||
.empty-state {
|
.empty-state {
|
||||||
text-align: center;
|
text-align: center;
|
||||||
padding: 40px;
|
padding: 40px;
|
||||||
@@ -292,7 +288,7 @@
|
|||||||
{{else}}
|
{{else}}
|
||||||
No known origin
|
No known origin
|
||||||
{{end}}
|
{{end}}
|
||||||
<span class="peer-count">{{.PeerCount}} peer{{if ne .PeerCount 1}}s{{end}}</span>
|
<span style="color: #7f8c8d; margin-left: 10px;">{{.PeerCount}} peer{{if ne .PeerCount 1}}s{{end}}</span>
|
||||||
</div>
|
</div>
|
||||||
{{end}}
|
{{end}}
|
||||||
</div>
|
</div>
|
||||||
|
|||||||
@@ -4,7 +4,7 @@
|
|||||||
<meta charset="UTF-8">
|
<meta charset="UTF-8">
|
||||||
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
||||||
<title>Prefixes with /{{ .MaskLength }} - RouteWatch</title>
|
<title>Prefixes with /{{ .MaskLength }} - RouteWatch</title>
|
||||||
<style nonce="{{.CSPNonce}}">
|
<style>
|
||||||
* {
|
* {
|
||||||
box-sizing: border-box;
|
box-sizing: border-box;
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -4,7 +4,7 @@
|
|||||||
<meta charset="UTF-8">
|
<meta charset="UTF-8">
|
||||||
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
||||||
<title>RouteWatch Status</title>
|
<title>RouteWatch Status</title>
|
||||||
<style nonce="{{.CSPNonce}}">
|
<style>
|
||||||
* {
|
* {
|
||||||
box-sizing: border-box;
|
box-sizing: border-box;
|
||||||
}
|
}
|
||||||
@@ -126,12 +126,10 @@
|
|||||||
.connected {
|
.connected {
|
||||||
color: #22c55e;
|
color: #22c55e;
|
||||||
}
|
}
|
||||||
.disconnected,
|
.disconnected {
|
||||||
.stale {
|
|
||||||
color: #ef4444;
|
color: #ef4444;
|
||||||
}
|
}
|
||||||
.error {
|
.error {
|
||||||
display: none;
|
|
||||||
background: #fee;
|
background: #fee;
|
||||||
color: #c00;
|
color: #c00;
|
||||||
padding: 10px;
|
padding: 10px;
|
||||||
@@ -175,16 +173,13 @@
|
|||||||
</nav>
|
</nav>
|
||||||
|
|
||||||
<main class="main-content">
|
<main class="main-content">
|
||||||
<div id="error" class="error"></div>
|
<div id="error" class="error" style="display: none;"></div>
|
||||||
<div class="status-grid">
|
<div class="status-grid">
|
||||||
<div class="status-card">
|
<div class="status-card">
|
||||||
<h2>RouteWatch</h2>
|
<h2>RouteWatch</h2>
|
||||||
<div class="metric">
|
<div class="metric">
|
||||||
<span class="metric-label">Status</span>
|
<span class="metric-label">Status</span>
|
||||||
<span>
|
<span class="metric-value" id="connected">-</span>
|
||||||
<span class="metric-value" id="connected">-</span>
|
|
||||||
<span class="metric-value" id="freshness"></span>
|
|
||||||
</span>
|
|
||||||
</div>
|
</div>
|
||||||
<div class="metric">
|
<div class="metric">
|
||||||
<span class="metric-label">Uptime</span>
|
<span class="metric-label">Uptime</span>
|
||||||
@@ -235,8 +230,8 @@
|
|||||||
<span class="metric-value" id="withdrawals">-</span>
|
<span class="metric-value" id="withdrawals">-</span>
|
||||||
</div>
|
</div>
|
||||||
<div class="metric">
|
<div class="metric">
|
||||||
<span class="metric-label">Route Churn/min</span>
|
<span class="metric-label">Route Churn/sec</span>
|
||||||
<span class="metric-value" id="route_churn_per_min">-</span>
|
<span class="metric-value" id="route_churn_per_sec">-</span>
|
||||||
</div>
|
</div>
|
||||||
<div class="metric">
|
<div class="metric">
|
||||||
<span class="metric-label">Total Data</span>
|
<span class="metric-label">Total Data</span>
|
||||||
@@ -388,46 +383,8 @@
|
|||||||
<div id="handler-stats-container" class="status-grid">
|
<div id="handler-stats-container" class="status-grid">
|
||||||
<!-- Handler stats will be dynamically added here -->
|
<!-- Handler stats will be dynamically added here -->
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
<script nonce="{{.CSPNonce}}">
|
<script>
|
||||||
// The stats are fetched every REFRESH_PERIOD_MS. Next to the status, a
|
|
||||||
// spinner turns one frame each time an answer counts more messages
|
|
||||||
// than the one before. After STALE_AFTER_REFRESHES refreshes without
|
|
||||||
// such an answer, the spinner gives way to "stale" and the time since.
|
|
||||||
const REFRESH_PERIOD_MS = 2000;
|
|
||||||
const STALE_AFTER_REFRESHES = 3;
|
|
||||||
const MS_PER_SECOND = 1000;
|
|
||||||
const SPINNER_FRAMES = ['⠋', '⠙', '⠹', '⠸', '⠼', '⠴', '⠦', '⠧', '⠇', '⠏'];
|
|
||||||
const SPINNER_COLOURS = ['#e74c3c', '#e67e22', '#f1c40f', '#27ae60', '#3498db', '#9b59b6'];
|
|
||||||
|
|
||||||
let spinnerFrame = 0;
|
|
||||||
let lastMessageCount = null;
|
|
||||||
let lastFreshAt = Date.now();
|
|
||||||
|
|
||||||
// Turn the spinner when the message count has risen since the last answer.
|
|
||||||
function noteMessageCount(count) {
|
|
||||||
if (lastMessageCount !== null && count > lastMessageCount) {
|
|
||||||
spinnerFrame = (spinnerFrame + 1) % SPINNER_FRAMES.length;
|
|
||||||
lastFreshAt = Date.now();
|
|
||||||
}
|
|
||||||
lastMessageCount = count;
|
|
||||||
showFreshness();
|
|
||||||
}
|
|
||||||
|
|
||||||
function showFreshness() {
|
|
||||||
const freshness = document.getElementById('freshness');
|
|
||||||
const sinceFreshMs = Date.now() - lastFreshAt;
|
|
||||||
const stale = sinceFreshMs >= STALE_AFTER_REFRESHES * REFRESH_PERIOD_MS;
|
|
||||||
freshness.classList.toggle('stale', stale);
|
|
||||||
if (stale) {
|
|
||||||
freshness.textContent = 'stale for ' + Math.floor(sinceFreshMs / MS_PER_SECOND) + 's';
|
|
||||||
freshness.style.color = '';
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
freshness.textContent = SPINNER_FRAMES[spinnerFrame];
|
|
||||||
freshness.style.color = SPINNER_COLOURS[spinnerFrame % SPINNER_COLOURS.length];
|
|
||||||
}
|
|
||||||
|
|
||||||
function formatBytes(bytes) {
|
function formatBytes(bytes) {
|
||||||
if (bytes === 0) return '0 B';
|
if (bytes === 0) return '0 B';
|
||||||
const k = 1024;
|
const k = 1024;
|
||||||
@@ -553,7 +510,7 @@
|
|||||||
document.getElementById('messages_per_sec').textContent = '-';
|
document.getElementById('messages_per_sec').textContent = '-';
|
||||||
document.getElementById('announcements').textContent = '-';
|
document.getElementById('announcements').textContent = '-';
|
||||||
document.getElementById('withdrawals').textContent = '-';
|
document.getElementById('withdrawals').textContent = '-';
|
||||||
document.getElementById('route_churn_per_min').textContent = '-';
|
document.getElementById('route_churn_per_sec').textContent = '-';
|
||||||
document.getElementById('total_wire_bytes').textContent = '-';
|
document.getElementById('total_wire_bytes').textContent = '-';
|
||||||
document.getElementById('wire_mbits_per_sec').textContent = '-';
|
document.getElementById('wire_mbits_per_sec').textContent = '-';
|
||||||
document.getElementById('gc_num').textContent = '-';
|
document.getElementById('gc_num').textContent = '-';
|
||||||
@@ -611,8 +568,7 @@
|
|||||||
const connectedEl = document.getElementById('connected');
|
const connectedEl = document.getElementById('connected');
|
||||||
connectedEl.textContent = data.connected ? 'Connected' : 'Disconnected';
|
connectedEl.textContent = data.connected ? 'Connected' : 'Disconnected';
|
||||||
connectedEl.className = 'metric-value ' + (data.connected ? 'connected' : 'disconnected');
|
connectedEl.className = 'metric-value ' + (data.connected ? 'connected' : 'disconnected');
|
||||||
noteMessageCount(data.total_messages);
|
|
||||||
|
|
||||||
// Update all metrics
|
// Update all metrics
|
||||||
document.getElementById('uptime').textContent = data.uptime;
|
document.getElementById('uptime').textContent = data.uptime;
|
||||||
document.getElementById('go_version').textContent = data.go_version;
|
document.getElementById('go_version').textContent = data.go_version;
|
||||||
@@ -621,7 +577,7 @@
|
|||||||
document.getElementById('connection_duration').textContent = data.connection_duration;
|
document.getElementById('connection_duration').textContent = data.connection_duration;
|
||||||
document.getElementById('reconnect_count').textContent = formatNumber(data.reconnect_count);
|
document.getElementById('reconnect_count').textContent = formatNumber(data.reconnect_count);
|
||||||
document.getElementById('total_messages').textContent = formatNumber(data.total_messages);
|
document.getElementById('total_messages').textContent = formatNumber(data.total_messages);
|
||||||
document.getElementById('messages_per_sec').textContent = formatNumber(Math.round(data.messages_per_sec));
|
document.getElementById('messages_per_sec').textContent = data.messages_per_sec.toFixed(1);
|
||||||
document.getElementById('total_wire_bytes').textContent = formatBytes(data.total_wire_bytes);
|
document.getElementById('total_wire_bytes').textContent = formatBytes(data.total_wire_bytes);
|
||||||
document.getElementById('wire_mbits_per_sec').textContent = data.wire_mbits_per_sec.toFixed(2) + ' Mbps';
|
document.getElementById('wire_mbits_per_sec').textContent = data.wire_mbits_per_sec.toFixed(2) + ' Mbps';
|
||||||
document.getElementById('asns').textContent = formatNumber(data.asns);
|
document.getElementById('asns').textContent = formatNumber(data.asns);
|
||||||
@@ -634,8 +590,8 @@
|
|||||||
document.getElementById('live_routes').textContent = formatNumber(data.live_routes);
|
document.getElementById('live_routes').textContent = formatNumber(data.live_routes);
|
||||||
document.getElementById('ipv4_routes').textContent = formatNumber(data.ipv4_routes);
|
document.getElementById('ipv4_routes').textContent = formatNumber(data.ipv4_routes);
|
||||||
document.getElementById('ipv6_routes').textContent = formatNumber(data.ipv6_routes);
|
document.getElementById('ipv6_routes').textContent = formatNumber(data.ipv6_routes);
|
||||||
document.getElementById('ipv4_updates_per_sec').textContent = formatNumber(Math.round(data.ipv4_updates_per_sec));
|
document.getElementById('ipv4_updates_per_sec').textContent = data.ipv4_updates_per_sec.toFixed(1);
|
||||||
document.getElementById('ipv6_updates_per_sec').textContent = formatNumber(Math.round(data.ipv6_updates_per_sec));
|
document.getElementById('ipv6_updates_per_sec').textContent = data.ipv6_updates_per_sec.toFixed(1);
|
||||||
document.getElementById('oldest_route').textContent = formatRelativeTime(data.oldest_route);
|
document.getElementById('oldest_route').textContent = formatRelativeTime(data.oldest_route);
|
||||||
document.getElementById('newest_route').textContent = formatRelativeTime(data.newest_route);
|
document.getElementById('newest_route').textContent = formatRelativeTime(data.newest_route);
|
||||||
|
|
||||||
@@ -644,7 +600,7 @@
|
|||||||
document.getElementById('bgp_peer_count').textContent = formatNumber(data.stream.bgp_peer_count);
|
document.getElementById('bgp_peer_count').textContent = formatNumber(data.stream.bgp_peer_count);
|
||||||
document.getElementById('announcements').textContent = formatNumber(data.stream.announcements);
|
document.getElementById('announcements').textContent = formatNumber(data.stream.announcements);
|
||||||
document.getElementById('withdrawals').textContent = formatNumber(data.stream.withdrawals);
|
document.getElementById('withdrawals').textContent = formatNumber(data.stream.withdrawals);
|
||||||
document.getElementById('route_churn_per_min').textContent = formatNumber(data.stream.route_churn_per_min);
|
document.getElementById('route_churn_per_sec').textContent = data.stream.route_churn_per_sec.toFixed(1);
|
||||||
}
|
}
|
||||||
|
|
||||||
// Update GC stats
|
// Update GC stats
|
||||||
@@ -686,12 +642,9 @@
|
|||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
// Update immediately and then every REFRESH_PERIOD_MS. The freshness
|
// Update immediately and then every 2 seconds
|
||||||
// check runs every second on its own, so a request that never returns
|
|
||||||
// still turns the spinner into "stale".
|
|
||||||
updateStatus();
|
updateStatus();
|
||||||
setInterval(updateStatus, REFRESH_PERIOD_MS);
|
setInterval(updateStatus, 2000);
|
||||||
setInterval(showFreshness, MS_PER_SECOND);
|
|
||||||
</script>
|
</script>
|
||||||
</main>
|
</main>
|
||||||
|
|
||||||
|
|||||||
@@ -7,9 +7,9 @@ import (
|
|||||||
"net/url"
|
"net/url"
|
||||||
"strings"
|
"strings"
|
||||||
"sync"
|
"sync"
|
||||||
|
"time"
|
||||||
|
|
||||||
"git.eeqj.de/sneak/routewatch/internal/version"
|
"git.eeqj.de/sneak/routewatch/internal/version"
|
||||||
"github.com/dustin/go-humanize"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
//go:embed index.html
|
//go:embed index.html
|
||||||
@@ -48,7 +48,48 @@ var (
|
|||||||
once sync.Once
|
once sync.Once
|
||||||
)
|
)
|
||||||
|
|
||||||
const cidrPartCount = 2 // A CIDR has two parts: prefix and length
|
const (
|
||||||
|
hoursPerDay = 24
|
||||||
|
daysPerMonth = 30
|
||||||
|
cidrPartCount = 2 // A CIDR has two parts: prefix and length
|
||||||
|
)
|
||||||
|
|
||||||
|
// timeSince returns a human-readable duration since the given time
|
||||||
|
func timeSince(t time.Time) string {
|
||||||
|
duration := time.Since(t)
|
||||||
|
if duration < time.Minute {
|
||||||
|
return "just now"
|
||||||
|
}
|
||||||
|
|
||||||
|
if duration < time.Hour {
|
||||||
|
minutes := int(duration.Minutes())
|
||||||
|
if minutes == 1 {
|
||||||
|
return "1 minute ago"
|
||||||
|
}
|
||||||
|
|
||||||
|
return duration.Truncate(time.Minute).String() + " ago"
|
||||||
|
}
|
||||||
|
|
||||||
|
if duration < hoursPerDay*time.Hour {
|
||||||
|
hours := int(duration.Hours())
|
||||||
|
if hours == 1 {
|
||||||
|
return "1 hour ago"
|
||||||
|
}
|
||||||
|
|
||||||
|
return duration.Truncate(time.Hour).String() + " ago"
|
||||||
|
}
|
||||||
|
|
||||||
|
days := int(duration.Hours() / hoursPerDay)
|
||||||
|
if days == 1 {
|
||||||
|
return "1 day ago"
|
||||||
|
}
|
||||||
|
|
||||||
|
if days < daysPerMonth {
|
||||||
|
return duration.Truncate(hoursPerDay*time.Hour).String() + " ago"
|
||||||
|
}
|
||||||
|
|
||||||
|
return t.Format("2006-01-02")
|
||||||
|
}
|
||||||
|
|
||||||
// prefixURL generates a URL path for a prefix in CIDR notation.
|
// prefixURL generates a URL path for a prefix in CIDR notation.
|
||||||
// Takes a prefix like "192.168.1.0/24" and returns "/prefix/192.168.1.0/24"
|
// Takes a prefix like "192.168.1.0/24" and returns "/prefix/192.168.1.0/24"
|
||||||
@@ -72,7 +113,7 @@ func initTemplates() {
|
|||||||
|
|
||||||
// Create common template functions
|
// Create common template functions
|
||||||
funcs := template.FuncMap{
|
funcs := template.FuncMap{
|
||||||
"timeSince": humanize.Time,
|
"timeSince": timeSince,
|
||||||
"urlEncode": url.QueryEscape,
|
"urlEncode": url.QueryEscape,
|
||||||
"prefixURL": prefixURL,
|
"prefixURL": prefixURL,
|
||||||
"appName": func() string { return version.Name },
|
"appName": func() string { return version.Name },
|
||||||
|
|||||||
@@ -1,181 +0,0 @@
|
|||||||
package templates_test
|
|
||||||
|
|
||||||
import (
|
|
||||||
"bytes"
|
|
||||||
"html/template"
|
|
||||||
"strings"
|
|
||||||
"testing"
|
|
||||||
|
|
||||||
"git.eeqj.de/sneak/routewatch/internal/database"
|
|
||||||
"git.eeqj.de/sneak/routewatch/internal/templates"
|
|
||||||
)
|
|
||||||
|
|
||||||
// renderPage renders a page with the data every page needs, and the visitor's
|
|
||||||
// address, which the home page also needs.
|
|
||||||
func renderPage(t *testing.T, tmpl *template.Template) string {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
var page bytes.Buffer
|
|
||||||
|
|
||||||
err := tmpl.Execute(&page, struct{ CSPNonce, ClientIP string }{})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("rendering the page: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
return page.String()
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestFooterLicense renders the home and status pages and expects each footer
|
|
||||||
// to name the licence in LICENSE, MIT, and not WTFPL.
|
|
||||||
func TestFooterLicense(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
const (
|
|
||||||
license = "MIT"
|
|
||||||
wrongLicense = "WTFPL"
|
|
||||||
)
|
|
||||||
|
|
||||||
pages := map[string]*template.Template{
|
|
||||||
"index": templates.IndexTemplate(),
|
|
||||||
"status": templates.StatusTemplate(),
|
|
||||||
}
|
|
||||||
|
|
||||||
for name, tmpl := range pages {
|
|
||||||
t.Run(name, func(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
_, footer, found := strings.Cut(renderPage(t, tmpl), "<footer")
|
|
||||||
if !found {
|
|
||||||
t.Fatal("the page has no footer")
|
|
||||||
}
|
|
||||||
|
|
||||||
footer, _, _ = strings.Cut(footer, "</footer>")
|
|
||||||
|
|
||||||
if !strings.Contains(footer, license) {
|
|
||||||
t.Errorf("footer does not name %s: %q", license, footer)
|
|
||||||
}
|
|
||||||
|
|
||||||
if strings.Contains(footer, wrongLicense) {
|
|
||||||
t.Errorf("footer names %s: %q", wrongLicense, footer)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestFreshnessIndicator renders the home and status pages and expects each to
|
|
||||||
// have, right after the status element, the element where the page's script
|
|
||||||
// shows its spinner or "stale".
|
|
||||||
func TestFreshnessIndicator(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
pages := map[string]struct {
|
|
||||||
tmpl *template.Template
|
|
||||||
status string
|
|
||||||
indicator string
|
|
||||||
}{
|
|
||||||
"index": {
|
|
||||||
tmpl: templates.IndexTemplate(),
|
|
||||||
status: `<span id="stat-status">-</span>`,
|
|
||||||
indicator: `<span id="freshness"></span>`,
|
|
||||||
},
|
|
||||||
"status": {
|
|
||||||
tmpl: templates.StatusTemplate(),
|
|
||||||
status: `<span class="metric-value" id="connected">-</span>`,
|
|
||||||
indicator: `<span class="metric-value" id="freshness"></span>`,
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
for name, page := range pages {
|
|
||||||
t.Run(name, func(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
_, afterStatus, found := strings.Cut(renderPage(t, page.tmpl), page.status)
|
|
||||||
if !found {
|
|
||||||
t.Fatalf("the page has no status element %s", page.status)
|
|
||||||
}
|
|
||||||
|
|
||||||
if !strings.HasPrefix(strings.TrimSpace(afterStatus), page.indicator) {
|
|
||||||
t.Errorf("the status element is not followed by %s", page.indicator)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestASDetailPrefixListsFold renders the AS page with one IPv4 and one IPv6
|
|
||||||
// prefix. It expects each prefix list inside its own <details> element that
|
|
||||||
// starts open and whose <summary> holds the list's heading, so that a click on
|
|
||||||
// the heading folds the list.
|
|
||||||
func TestASDetailPrefixListsFold(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
const (
|
|
||||||
asn = 64500
|
|
||||||
prefixV4 = "192.0.2.0/24"
|
|
||||||
prefixV6 = "2001:db8::/32"
|
|
||||||
)
|
|
||||||
|
|
||||||
ipv4 := []database.LiveRoute{{Prefix: prefixV4}}
|
|
||||||
ipv6 := []database.LiveRoute{{Prefix: prefixV6}}
|
|
||||||
|
|
||||||
// The fields the AS page handler gives the template.
|
|
||||||
data := struct {
|
|
||||||
ASN *database.ASN
|
|
||||||
IPv4Prefixes []database.LiveRoute
|
|
||||||
IPv6Prefixes []database.LiveRoute
|
|
||||||
TotalCount int
|
|
||||||
IPv4Count int
|
|
||||||
IPv6Count int
|
|
||||||
Peers []database.ASPeer
|
|
||||||
PeerCount int
|
|
||||||
CSPNonce string
|
|
||||||
}{
|
|
||||||
ASN: &database.ASN{ASN: asn},
|
|
||||||
IPv4Prefixes: ipv4,
|
|
||||||
IPv6Prefixes: ipv6,
|
|
||||||
TotalCount: len(ipv4) + len(ipv6),
|
|
||||||
IPv4Count: len(ipv4),
|
|
||||||
IPv6Count: len(ipv6),
|
|
||||||
}
|
|
||||||
|
|
||||||
var rendered bytes.Buffer
|
|
||||||
|
|
||||||
err := templates.ASDetailTemplate().Execute(&rendered, data)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("rendering the page: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
page := rendered.String()
|
|
||||||
|
|
||||||
lists := []struct{ heading, prefix string }{
|
|
||||||
{"IPv4 Prefixes", prefixV4},
|
|
||||||
{"IPv6 Prefixes", prefixV6},
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, list := range lists {
|
|
||||||
before, _, found := strings.Cut(page, list.prefix)
|
|
||||||
if !found {
|
|
||||||
t.Fatalf("the page does not list %s", list.prefix)
|
|
||||||
}
|
|
||||||
|
|
||||||
// The list's section starts at the last <details before its prefix.
|
|
||||||
start := strings.LastIndex(before, "<details")
|
|
||||||
if start < 0 || strings.Contains(before[start:], "</details>") {
|
|
||||||
t.Errorf("%s: the list is not inside a <details> element", list.heading)
|
|
||||||
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
|
|
||||||
tag, section, _ := strings.Cut(before[start:], ">")
|
|
||||||
if !strings.Contains(tag, " open") {
|
|
||||||
t.Errorf("%s: the <details> element does not start open: %s>",
|
|
||||||
list.heading, tag)
|
|
||||||
}
|
|
||||||
|
|
||||||
_, summary, _ := strings.Cut(section, "<summary")
|
|
||||||
summary, _, _ = strings.Cut(summary, "</summary>")
|
|
||||||
|
|
||||||
if !strings.Contains(summary, list.heading) {
|
|
||||||
t.Errorf("%s: the <summary> does not hold the heading", list.heading)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -19,8 +19,8 @@ const (
|
|||||||
Author = "@sneak"
|
Author = "@sneak"
|
||||||
// AuthorURL is the author's website
|
// AuthorURL is the author's website
|
||||||
AuthorURL = "https://sneak.berlin"
|
AuthorURL = "https://sneak.berlin"
|
||||||
// License is the program license, as LICENSE grants it
|
// License is the program license
|
||||||
License = "MIT"
|
License = "WTFPL"
|
||||||
// RepoURL is the git repository URL
|
// RepoURL is the git repository URL
|
||||||
RepoURL = "https://git.eeqj.de/sneak/routewatch"
|
RepoURL = "https://git.eeqj.de/sneak/routewatch"
|
||||||
)
|
)
|
||||||
|
|||||||
+86
-357
@@ -4,18 +4,11 @@ package whois
|
|||||||
import (
|
import (
|
||||||
"bufio"
|
"bufio"
|
||||||
"context"
|
"context"
|
||||||
"errors"
|
|
||||||
"fmt"
|
"fmt"
|
||||||
"io"
|
|
||||||
"log/slog"
|
|
||||||
"net"
|
"net"
|
||||||
"regexp"
|
"regexp"
|
||||||
"strings"
|
"strings"
|
||||||
"sync"
|
|
||||||
"time"
|
"time"
|
||||||
"unicode/utf8"
|
|
||||||
|
|
||||||
"golang.org/x/text/encoding/charmap"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
// Timeout constants for WHOIS queries.
|
// Timeout constants for WHOIS queries.
|
||||||
@@ -25,24 +18,12 @@ const (
|
|||||||
writeTimeout = 5 * time.Second
|
writeTimeout = 5 * time.Second
|
||||||
)
|
)
|
||||||
|
|
||||||
// maxAnswerSize caps the bytes read of one WHOIS answer: a real one is a
|
|
||||||
// few KiB, so one this large is from a server that does not stop sending.
|
|
||||||
const maxAnswerSize = 1 << 20
|
|
||||||
|
|
||||||
// Parsing constants.
|
// Parsing constants.
|
||||||
const (
|
const (
|
||||||
|
keyValueParts = 2 // Expected parts when splitting "key: value"
|
||||||
lacnicDateFormatLen = 8 // Length of YYYYMMDD date format
|
lacnicDateFormatLen = 8 // Length of YYYYMMDD date format
|
||||||
)
|
)
|
||||||
|
|
||||||
// The classes of the two objects that describe an AS in an answer from
|
|
||||||
// RIPE, APNIC or AFRINIC, and the key of the aut-num object's line that
|
|
||||||
// gives the ID of the AS's organisation object.
|
|
||||||
const (
|
|
||||||
autNumClass = "aut-num"
|
|
||||||
organisationClass = "organisation"
|
|
||||||
orgKey = "org"
|
|
||||||
)
|
|
||||||
|
|
||||||
// WHOIS server addresses.
|
// WHOIS server addresses.
|
||||||
const (
|
const (
|
||||||
whoisServerIANA = "whois.iana.org:43"
|
whoisServerIANA = "whois.iana.org:43"
|
||||||
@@ -51,32 +32,8 @@ const (
|
|||||||
whoisServerAPNIC = "whois.apnic.net:43"
|
whoisServerAPNIC = "whois.apnic.net:43"
|
||||||
whoisServerLACNIC = "whois.lacnic.net:43"
|
whoisServerLACNIC = "whois.lacnic.net:43"
|
||||||
whoisServerAFRINIC = "whois.afrinic.net:43"
|
whoisServerAFRINIC = "whois.afrinic.net:43"
|
||||||
|
|
||||||
// NIC.br registers Brazil's ASes under LACNIC.
|
|
||||||
whoisServerNICbr = "whois.registro.br:43"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
// nicbrHeader starts the copyright notice at the top of NIC.br's WHOIS
|
|
||||||
// answers. LACNIC's answer for an AS that NIC.br registers is NIC.br's, and
|
|
||||||
// carries the same notice:
|
|
||||||
//
|
|
||||||
// % Copyright (c) Nic.br - Use of this data is governed by the Use and
|
|
||||||
// % Privacy Policy at https://registro.br/upp . Distribution,
|
|
||||||
const nicbrHeader = "% Copyright (c) Nic.br"
|
|
||||||
|
|
||||||
// nicbrLimitReached is in the line that marks NIC.br's reduced answer to a
|
|
||||||
// client over its query limit. NIC.br publishes no limit. Its Use and
|
|
||||||
// Privacy Policy, https://registro.br/upp, says only that it restricts
|
|
||||||
// queries to stop the data being taken in bulk.
|
|
||||||
const nicbrLimitReached = "Query rate limit exceeded"
|
|
||||||
|
|
||||||
// NICbrSkipTime is how long the client stops asking NIC.br after NIC.br
|
|
||||||
// answers that the client is over its query limit. NIC.br publishes
|
|
||||||
// neither the limit nor how long it lasts. An hour outlasts a limit
|
|
||||||
// counted per minute or per hour. The ASes looked up meanwhile get
|
|
||||||
// NIC.br's fields when they are next refreshed.
|
|
||||||
const NICbrSkipTime = time.Hour
|
|
||||||
|
|
||||||
// RIR identifiers.
|
// RIR identifiers.
|
||||||
const (
|
const (
|
||||||
RIRARIN = "ARIN"
|
RIRARIN = "ARIN"
|
||||||
@@ -86,45 +43,6 @@ const (
|
|||||||
RIRAFRNIC = "AFRINIC"
|
RIRAFRNIC = "AFRINIC"
|
||||||
)
|
)
|
||||||
|
|
||||||
// noPersonalFlag keeps person and role objects out of an answer. The
|
|
||||||
// organisation object stays in it. RIPE counts the person and role objects
|
|
||||||
// it returns against a daily limit per IP address. Role objects for abuse
|
|
||||||
// contacts do not count. RIPE refuses an address for good that passes the
|
|
||||||
// limit too often:
|
|
||||||
// https://docs.db.ripe.net/Types-of-Queries/Access-Control-for-Queries/
|
|
||||||
//
|
|
||||||
// RIPE documents the flag here:
|
|
||||||
// https://docs.db.ripe.net/Types-of-Queries/Referenced-Objects-in-Query-Response/
|
|
||||||
// APNIC and AFRINIC list it in their answers to the query "help". AFRINIC
|
|
||||||
// sends readers to that answer for its full list of flags:
|
|
||||||
// https://afrinic.net/whois-getting-started
|
|
||||||
const noPersonalFlag = "--no-personal"
|
|
||||||
|
|
||||||
// The error lines with which RIPE's database software refuses a query:
|
|
||||||
// access denied to the client's IP address, and that address's limit
|
|
||||||
// reached. APNIC and AFRINIC run the same software. The lines are listed
|
|
||||||
// under "Access Errors" on this page, whose URL is split over two lines:
|
|
||||||
// https://docs.db.ripe.net/Appendices/
|
|
||||||
// Appendix-C--RIPE-Database-Query-Server-Response-Codes-and-Messages/
|
|
||||||
const (
|
|
||||||
accessDeniedError = "%ERROR:201:"
|
|
||||||
accessLimitError = "%ERROR:202:"
|
|
||||||
)
|
|
||||||
|
|
||||||
// Errors LookupASN returns when a WHOIS answer has nothing to store.
|
|
||||||
var (
|
|
||||||
// ErrNoReferral is returned when IANA refers the AS to no RIR, as it
|
|
||||||
// does for a reserved, private-use or unallocated AS number.
|
|
||||||
ErrNoReferral = errors.New("IANA refers the AS to no RIR")
|
|
||||||
|
|
||||||
errNoASBlock = errors.New("IANA answered with neither a referral " +
|
|
||||||
"nor an AS block")
|
|
||||||
errNoFields = errors.New("RIR answered with no WHOIS fields")
|
|
||||||
errRefused = errors.New("RIR refused the query")
|
|
||||||
errUnknownServer = errors.New("IANA refers the AS to a server no RIR runs")
|
|
||||||
errAnswerTooLarge = errors.New("WHOIS answer reached the size cap")
|
|
||||||
)
|
|
||||||
|
|
||||||
// ASNInfo contains parsed WHOIS information for an ASN.
|
// ASNInfo contains parsed WHOIS information for an ASN.
|
||||||
type ASNInfo struct {
|
type ASNInfo struct {
|
||||||
ASN int
|
ASN int
|
||||||
@@ -145,46 +63,20 @@ type ASNInfo struct {
|
|||||||
|
|
||||||
// Client performs WHOIS lookups for ASNs.
|
// Client performs WHOIS lookups for ASNs.
|
||||||
type Client struct {
|
type Client struct {
|
||||||
// DialContext opens the connection to a WHOIS server, given as
|
// Dialer for creating connections (can be overridden for testing)
|
||||||
// host:port. NewClient sets a net.Dialer's; tests replace it to reach
|
dialer *net.Dialer
|
||||||
// local servers instead.
|
|
||||||
DialContext func(ctx context.Context, network, address string) (net.Conn, error)
|
|
||||||
|
|
||||||
// Now returns the current time. NewClient sets time.Now; tests replace
|
|
||||||
// it to move the clock.
|
|
||||||
Now func() time.Time
|
|
||||||
|
|
||||||
logger *slog.Logger
|
|
||||||
|
|
||||||
// mu guards nicbrSkipUntil.
|
|
||||||
mu sync.Mutex
|
|
||||||
|
|
||||||
// nicbrSkipUntil is when the client next asks NIC.br, after NIC.br
|
|
||||||
// answered that the client is over its query limit.
|
|
||||||
nicbrSkipUntil time.Time
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// NewClient creates a new WHOIS client that logs to logger.
|
// NewClient creates a new WHOIS client.
|
||||||
func NewClient(logger *slog.Logger) *Client {
|
func NewClient() *Client {
|
||||||
dialer := &net.Dialer{Timeout: dialTimeout}
|
|
||||||
|
|
||||||
return &Client{
|
return &Client{
|
||||||
DialContext: dialer.DialContext,
|
dialer: &net.Dialer{
|
||||||
Now: time.Now,
|
Timeout: dialTimeout,
|
||||||
logger: logger,
|
},
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// LookupASN asks IANA which RIR holds the given ASN, asks that RIR, and
|
// LookupASN queries WHOIS for the given ASN and returns parsed information.
|
||||||
// returns what the RIR's answer says. When LACNIC's answer is NIC.br's, it
|
|
||||||
// returns what NIC.br's own server says instead, unless that query fails or
|
|
||||||
// is skipped, as askNICbr says.
|
|
||||||
//
|
|
||||||
// It returns ErrNoReferral when IANA refers the AS to no RIR, and another
|
|
||||||
// error when the query to IANA or to the RIR fails, when the RIR refuses
|
|
||||||
// it, when IANA's answer has neither a referral nor the AS's block, when
|
|
||||||
// IANA refers the AS to a server none of the five RIRs run, or when the
|
|
||||||
// answer carries none of ASNInfo's fields.
|
|
||||||
func (c *Client) LookupASN(ctx context.Context, asn int) (*ASNInfo, error) {
|
func (c *Client) LookupASN(ctx context.Context, asn int) (*ASNInfo, error) {
|
||||||
// Query IANA first to find the authoritative RIR
|
// Query IANA first to find the authoritative RIR
|
||||||
query := fmt.Sprintf("AS%d", asn)
|
query := fmt.Sprintf("AS%d", asn)
|
||||||
@@ -195,145 +87,34 @@ func (c *Client) LookupASN(ctx context.Context, asn int) (*ASNInfo, error) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Determine RIR from IANA response
|
// Determine RIR from IANA response
|
||||||
rir, whoisServer, err := c.parseIANAReferral(ianaResp)
|
rir, whoisServer := c.parseIANAReferral(ianaResp)
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("%s: %w", query, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
if whoisServer == "" {
|
if whoisServer == "" {
|
||||||
// IANA names the block of an AS it refers to no RIR. An answer
|
// No referral, try to parse what we have
|
||||||
// without that, such as an empty one, says nothing about the AS.
|
return c.parseResponse(asn, rir, ianaResp), nil
|
||||||
if !strings.Contains(ianaResp, "as-block:") {
|
|
||||||
return nil, fmt.Errorf("%s: %w", query, errNoASBlock)
|
|
||||||
}
|
|
||||||
|
|
||||||
return nil, fmt.Errorf("%s: %w", query, ErrNoReferral)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// Query the authoritative RIR
|
// Query the authoritative RIR
|
||||||
rirResp, err := c.queryRIR(ctx, whoisServer, rirQuery(rir, query))
|
rirResp, err := c.query(ctx, whoisServer, query)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("RIR query failed: %w", err)
|
// Return partial data from IANA if RIR query fails
|
||||||
|
info := c.parseResponse(asn, rir, ianaResp)
|
||||||
|
info.RawResponse = fmt.Sprintf("%s\n--- RIR query failed: %v ---\n",
|
||||||
|
ianaResp, err)
|
||||||
|
|
||||||
|
return info, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// The fields never come from IANA's answer, which describes the block
|
// Combine responses and parse
|
||||||
// the AS is in.
|
fullResponse := ianaResp + "\n" + rirResp
|
||||||
info := c.parseResponse(asn, rir, rirResp)
|
info := c.parseResponse(asn, rir, fullResponse)
|
||||||
|
info.RawResponse = fullResponse
|
||||||
if rir == RIRLACNIC && strings.Contains(rirResp, nicbrHeader) {
|
|
||||||
info = c.askNICbr(ctx, query, info)
|
|
||||||
}
|
|
||||||
|
|
||||||
if info.isEmpty() {
|
|
||||||
return nil, fmt.Errorf("%s from %s: %w", query, whoisServer, errNoFields)
|
|
||||||
}
|
|
||||||
|
|
||||||
info.RawResponse = ianaResp + "\n" + info.RawResponse
|
|
||||||
|
|
||||||
return info, nil
|
return info, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// askNICbr asks NIC.br's server about an AS whose answer from LACNIC is
|
// query performs a raw WHOIS query to the specified server.
|
||||||
// NIC.br's, and returns what NIC.br's answer says. LACNIC replaces each
|
|
||||||
// accented letter in such an answer with U+FFFD. NIC.br's own server keeps
|
|
||||||
// them, in Latin-1.
|
|
||||||
//
|
|
||||||
// askNICbr returns lacnicInfo, what LACNIC's answer says, and logs a
|
|
||||||
// warning when the query fails, when NIC.br sends the reduced answer for a
|
|
||||||
// client over its limit, or when NIC.br's answer carries none of ASNInfo's
|
|
||||||
// fields. After the reduced answer, it returns lacnicInfo without asking
|
|
||||||
// NIC.br until NICbrSkipTime has passed.
|
|
||||||
func (c *Client) askNICbr(
|
|
||||||
ctx context.Context, asQuery string, lacnicInfo *ASNInfo,
|
|
||||||
) *ASNInfo {
|
|
||||||
asn := lacnicInfo.ASN
|
|
||||||
|
|
||||||
c.mu.Lock()
|
|
||||||
skipping := c.Now().Before(c.nicbrSkipUntil)
|
|
||||||
c.mu.Unlock()
|
|
||||||
|
|
||||||
if skipping {
|
|
||||||
return lacnicInfo
|
|
||||||
}
|
|
||||||
|
|
||||||
answer, err := c.query(ctx, whoisServerNICbr, asQuery)
|
|
||||||
if err != nil {
|
|
||||||
c.logger.Warn("NIC.br query failed, keeping LACNIC's fields",
|
|
||||||
"asn", asn, "error", err)
|
|
||||||
|
|
||||||
return lacnicInfo
|
|
||||||
}
|
|
||||||
|
|
||||||
if strings.Contains(answer, nicbrLimitReached) {
|
|
||||||
c.mu.Lock()
|
|
||||||
c.nicbrSkipUntil = c.Now().Add(NICbrSkipTime)
|
|
||||||
c.mu.Unlock()
|
|
||||||
|
|
||||||
c.logger.Warn("NIC.br answered that the client is over its query "+
|
|
||||||
"limit, keeping LACNIC's fields and skipping NIC.br",
|
|
||||||
"asn", asn, "skip_for", NICbrSkipTime)
|
|
||||||
|
|
||||||
return lacnicInfo
|
|
||||||
}
|
|
||||||
|
|
||||||
info := c.parseResponse(asn, RIRLACNIC, answer)
|
|
||||||
if info.isEmpty() {
|
|
||||||
c.logger.Warn("NIC.br answered with no WHOIS fields, "+
|
|
||||||
"keeping LACNIC's fields", "asn", asn)
|
|
||||||
|
|
||||||
return lacnicInfo
|
|
||||||
}
|
|
||||||
|
|
||||||
return info
|
|
||||||
}
|
|
||||||
|
|
||||||
// isEmpty reports whether info holds none of the fields a WHOIS answer
|
|
||||||
// fills, as when the answer is only comment and error lines.
|
|
||||||
func (info *ASNInfo) isEmpty() bool {
|
|
||||||
return info.ASName == "" && info.OrgName == "" && info.OrgID == "" &&
|
|
||||||
info.Address == "" && info.CountryCode == "" &&
|
|
||||||
info.AbuseEmail == "" && info.AbusePhone == "" &&
|
|
||||||
info.TechEmail == "" && info.TechPhone == "" &&
|
|
||||||
info.RegDate == nil && info.LastMod == nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// rirQuery returns the query sent to rir's server for asQuery, an AS
|
|
||||||
// number such as "AS3333". RIPE, APNIC and AFRINIC are asked to keep person
|
|
||||||
// and role objects out of the answer.
|
|
||||||
func rirQuery(rir, asQuery string) string {
|
|
||||||
switch rir {
|
|
||||||
case RIRRIPE, RIRAPNIC, RIRAFRNIC:
|
|
||||||
return noPersonalFlag + " " + asQuery
|
|
||||||
default:
|
|
||||||
return asQuery
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// queryRIR sends query to an RIR's server and returns the answer. An answer
|
|
||||||
// holding an error line with which the server refuses the query is an
|
|
||||||
// error, and the error holds that line.
|
|
||||||
func (c *Client) queryRIR(
|
|
||||||
ctx context.Context, server, query string,
|
|
||||||
) (string, error) {
|
|
||||||
answer, err := c.query(ctx, server, query)
|
|
||||||
if err != nil {
|
|
||||||
return "", err
|
|
||||||
}
|
|
||||||
|
|
||||||
for line := range strings.SplitSeq(answer, "\n") {
|
|
||||||
if strings.HasPrefix(line, accessDeniedError) ||
|
|
||||||
strings.HasPrefix(line, accessLimitError) {
|
|
||||||
return "", fmt.Errorf("%w: %s", errRefused, line)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return answer, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// query performs a raw WHOIS query to the specified server and returns the
|
|
||||||
// answer in UTF-8. An answer that reaches maxAnswerSize is an error.
|
|
||||||
func (c *Client) query(ctx context.Context, server, query string) (string, error) {
|
func (c *Client) query(ctx context.Context, server, query string) (string, error) {
|
||||||
conn, err := c.DialContext(ctx, "tcp", server)
|
conn, err := c.dialer.DialContext(ctx, "tcp", server)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return "", fmt.Errorf("dial %s: %w", server, err)
|
return "", fmt.Errorf("dial %s: %w", server, err)
|
||||||
}
|
}
|
||||||
@@ -359,11 +140,7 @@ func (c *Client) query(ctx context.Context, server, query string) (string, error
|
|||||||
|
|
||||||
var sb strings.Builder
|
var sb strings.Builder
|
||||||
|
|
||||||
// The reader ends at the cap; N, the bytes it has left, is 0 once the
|
scanner := bufio.NewScanner(conn)
|
||||||
// answer has reached it.
|
|
||||||
answer := &io.LimitedReader{R: conn, N: maxAnswerSize}
|
|
||||||
|
|
||||||
scanner := bufio.NewScanner(answer)
|
|
||||||
for scanner.Scan() {
|
for scanner.Scan() {
|
||||||
sb.WriteString(scanner.Text())
|
sb.WriteString(scanner.Text())
|
||||||
sb.WriteString("\n")
|
sb.WriteString("\n")
|
||||||
@@ -374,36 +151,13 @@ func (c *Client) query(ctx context.Context, server, query string) (string, error
|
|||||||
return sb.String(), fmt.Errorf("read response: %w", err)
|
return sb.String(), fmt.Errorf("read response: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
if answer.N == 0 {
|
return sb.String(), nil
|
||||||
return "", fmt.Errorf("read response: %w", errAnswerTooLarge)
|
|
||||||
}
|
|
||||||
|
|
||||||
return toUTF8(sb.String())
|
|
||||||
}
|
|
||||||
|
|
||||||
// toUTF8 returns a WHOIS answer in UTF-8. An answer that is valid UTF-8 is
|
|
||||||
// returned as it is. Any other answer is decoded as Latin-1, which RIPE,
|
|
||||||
// APNIC and NIC.br send. LACNIC, ARIN and AFRINIC send UTF-8. RIPE's
|
|
||||||
// database working group lists the five RIRs here:
|
|
||||||
// https://www.ripe.net/ripe/mail/archives/db-wg/2024-May/008075.html
|
|
||||||
func toUTF8(answer string) (string, error) {
|
|
||||||
if utf8.ValidString(answer) {
|
|
||||||
return answer, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
decoded, err := charmap.ISO8859_1.NewDecoder().String(answer)
|
|
||||||
if err != nil {
|
|
||||||
return "", fmt.Errorf("decode Latin-1 answer: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
return decoded, nil
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// parseIANAReferral extracts the RIR and WHOIS server from an IANA response.
|
// parseIANAReferral extracts the RIR and WHOIS server from an IANA response.
|
||||||
// The server is "" when the response has no referral. A referral to a server
|
func (c *Client) parseIANAReferral(response string) (string, string) {
|
||||||
// that is none of the five RIRs' is an error, so a forged or broken answer
|
var rir string
|
||||||
// cannot send the client to any other host or port.
|
|
||||||
func (c *Client) parseIANAReferral(response string) (string, string, error) {
|
|
||||||
for line := range strings.SplitSeq(response, "\n") {
|
for line := range strings.SplitSeq(response, "\n") {
|
||||||
line = strings.TrimSpace(line)
|
line = strings.TrimSpace(line)
|
||||||
|
|
||||||
@@ -414,27 +168,57 @@ func (c *Client) parseIANAReferral(response string) (string, string, error) {
|
|||||||
|
|
||||||
switch {
|
switch {
|
||||||
case strings.Contains(server, "arin"):
|
case strings.Contains(server, "arin"):
|
||||||
return RIRARIN, whoisServerARIN, nil
|
return RIRARIN, whoisServerARIN
|
||||||
case strings.Contains(server, "ripe"):
|
case strings.Contains(server, "ripe"):
|
||||||
return RIRRIPE, whoisServerRIPE, nil
|
return RIRRIPE, whoisServerRIPE
|
||||||
case strings.Contains(server, "apnic"):
|
case strings.Contains(server, "apnic"):
|
||||||
return RIRAPNIC, whoisServerAPNIC, nil
|
return RIRAPNIC, whoisServerAPNIC
|
||||||
case strings.Contains(server, "lacnic"):
|
case strings.Contains(server, "lacnic"):
|
||||||
return RIRLACNIC, whoisServerLACNIC, nil
|
return RIRLACNIC, whoisServerLACNIC
|
||||||
case strings.Contains(server, "afrinic"):
|
case strings.Contains(server, "afrinic"):
|
||||||
return RIRAFRNIC, whoisServerAFRINIC, nil
|
return RIRAFRNIC, whoisServerAFRINIC
|
||||||
default:
|
default:
|
||||||
return "", "", fmt.Errorf("%w: %q", errUnknownServer, server)
|
// Unknown server, add port if missing
|
||||||
|
if !strings.Contains(server, ":") {
|
||||||
|
server += ":43"
|
||||||
|
}
|
||||||
|
|
||||||
|
return "", server
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Also check organisation line for RIR hints
|
||||||
|
if strings.HasPrefix(strings.ToLower(line), "organisation:") {
|
||||||
|
orgRIR := rirInOrganisationLine(strings.ToLower(line))
|
||||||
|
if orgRIR != "" {
|
||||||
|
rir = orgRIR
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
return "", "", nil
|
return rir, ""
|
||||||
}
|
}
|
||||||
|
|
||||||
// parseResponse extracts ASN information from a WHOIS response. Of an
|
// rirInOrganisationLine returns the RIR a lowercased IANA "organisation:"
|
||||||
// answer from RIPE, APNIC or AFRINIC it reads only the lines asObjectLines
|
// line names, or "" when it names none.
|
||||||
// returns. Of an answer from ARIN or LACNIC it reads every line.
|
func rirInOrganisationLine(org string) string {
|
||||||
|
switch {
|
||||||
|
case strings.Contains(org, "arin"):
|
||||||
|
return RIRARIN
|
||||||
|
case strings.Contains(org, "ripe"):
|
||||||
|
return RIRRIPE
|
||||||
|
case strings.Contains(org, "apnic"):
|
||||||
|
return RIRAPNIC
|
||||||
|
case strings.Contains(org, "lacnic"):
|
||||||
|
return RIRLACNIC
|
||||||
|
case strings.Contains(org, "afrinic"):
|
||||||
|
return RIRAFRNIC
|
||||||
|
default:
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// parseResponse extracts ASN information from a WHOIS response.
|
||||||
func (c *Client) parseResponse(asn int, rir, response string) *ASNInfo {
|
func (c *Client) parseResponse(asn int, rir, response string) *ASNInfo {
|
||||||
info := &ASNInfo{
|
info := &ASNInfo{
|
||||||
ASN: asn,
|
ASN: asn,
|
||||||
@@ -443,15 +227,26 @@ func (c *Client) parseResponse(asn int, rir, response string) *ASNInfo {
|
|||||||
}
|
}
|
||||||
|
|
||||||
lines := strings.Split(response, "\n")
|
lines := strings.Split(response, "\n")
|
||||||
if rir == RIRRIPE || rir == RIRAPNIC || rir == RIRAFRNIC {
|
|
||||||
lines = asObjectLines(lines)
|
|
||||||
}
|
|
||||||
|
|
||||||
var addressLines []string
|
var addressLines []string
|
||||||
|
|
||||||
for _, line := range lines {
|
for _, line := range lines {
|
||||||
key, value, ok := splitField(line)
|
line = strings.TrimSpace(line)
|
||||||
if !ok || value == "" {
|
if line == "" || strings.HasPrefix(line, "%") ||
|
||||||
|
strings.HasPrefix(line, "#") {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
// Split on first colon
|
||||||
|
parts := strings.SplitN(line, ":", keyValueParts)
|
||||||
|
if len(parts) != keyValueParts {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
key := strings.TrimSpace(strings.ToLower(parts[0]))
|
||||||
|
value := strings.TrimSpace(parts[1])
|
||||||
|
|
||||||
|
if value == "" {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -478,72 +273,6 @@ func (c *Client) parseResponse(asn int, rir, response string) *ASNInfo {
|
|||||||
return info
|
return info
|
||||||
}
|
}
|
||||||
|
|
||||||
// splitField splits a "key: value" line of a WHOIS answer into its key, in
|
|
||||||
// lower case, and its value. The third result is false for a line that
|
|
||||||
// holds no field: a blank line, a comment line or a line without a colon.
|
|
||||||
func splitField(line string) (string, string, bool) {
|
|
||||||
line = strings.TrimSpace(line)
|
|
||||||
if strings.HasPrefix(line, "%") || strings.HasPrefix(line, "#") {
|
|
||||||
return "", "", false
|
|
||||||
}
|
|
||||||
|
|
||||||
key, value, ok := strings.Cut(line, ":")
|
|
||||||
|
|
||||||
return strings.ToLower(strings.TrimSpace(key)), strings.TrimSpace(value), ok
|
|
||||||
}
|
|
||||||
|
|
||||||
// asObjectLines returns the lines of the aut-num object in an answer from
|
|
||||||
// RIPE, APNIC or AFRINIC, followed by the lines of the organisation object
|
|
||||||
// whose ID the aut-num object gives on its "org:" line. The answer's other
|
|
||||||
// objects are left out. RIPE and APNIC start their answer with the
|
|
||||||
// as-block object of the block the AS is in, whose dates are not the AS's.
|
|
||||||
//
|
|
||||||
// These answers are in RPSL. A blank line ends an object. An object's first
|
|
||||||
// line has the object's class as its key and the object's ID as its value,
|
|
||||||
// as in "organisation: ORG-APNI1-AP".
|
|
||||||
func asObjectLines(lines []string) []string {
|
|
||||||
var autNum []string
|
|
||||||
|
|
||||||
// The lines of each organisation object, by its ID.
|
|
||||||
organisations := make(map[string][]string)
|
|
||||||
|
|
||||||
// The class and ID of the object the line is in, "" between objects.
|
|
||||||
var class, id string
|
|
||||||
|
|
||||||
// The ID of the organisation object the aut-num object refers to.
|
|
||||||
var orgID string
|
|
||||||
|
|
||||||
for _, line := range lines {
|
|
||||||
if strings.TrimSpace(line) == "" {
|
|
||||||
class = ""
|
|
||||||
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
|
|
||||||
key, value, ok := splitField(line)
|
|
||||||
if !ok {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
|
|
||||||
if class == "" {
|
|
||||||
class, id = key, value
|
|
||||||
}
|
|
||||||
|
|
||||||
switch class {
|
|
||||||
case autNumClass:
|
|
||||||
autNum = append(autNum, line)
|
|
||||||
|
|
||||||
if key == orgKey {
|
|
||||||
orgID = value
|
|
||||||
}
|
|
||||||
case organisationClass:
|
|
||||||
organisations[id] = append(organisations[id], line)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return append(autNum, organisations[orgID]...)
|
|
||||||
}
|
|
||||||
|
|
||||||
// setResponseField stores the value of one "key: value" line of a WHOIS
|
// setResponseField stores the value of one "key: value" line of a WHOIS
|
||||||
// response in info, keeping the first value seen for each field.
|
// response in info, keeping the first value seen for each field.
|
||||||
func (c *Client) setResponseField(info *ASNInfo, key, value string) {
|
func (c *Client) setResponseField(info *ASNInfo, key, value string) {
|
||||||
|
|||||||
@@ -1,881 +0,0 @@
|
|||||||
package whois_test
|
|
||||||
|
|
||||||
import (
|
|
||||||
"bufio"
|
|
||||||
"bytes"
|
|
||||||
"context"
|
|
||||||
"encoding/json"
|
|
||||||
"errors"
|
|
||||||
"io"
|
|
||||||
"log/slog"
|
|
||||||
"maps"
|
|
||||||
"net"
|
|
||||||
"slices"
|
|
||||||
"strings"
|
|
||||||
"sync"
|
|
||||||
"syscall"
|
|
||||||
"testing"
|
|
||||||
"time"
|
|
||||||
"unicode/utf8"
|
|
||||||
|
|
||||||
"git.eeqj.de/sneak/routewatch/internal/whois"
|
|
||||||
)
|
|
||||||
|
|
||||||
// The servers LookupASN asks: IANA first, then the RIR IANA refers it to.
|
|
||||||
const (
|
|
||||||
ianaServer = "whois.iana.org:43"
|
|
||||||
ripeServer = "whois.ripe.net:43"
|
|
||||||
)
|
|
||||||
|
|
||||||
// testASN is the AS the tests look up, and asQuery the query for it.
|
|
||||||
const (
|
|
||||||
testASN = 3333
|
|
||||||
asQuery = "AS3333"
|
|
||||||
)
|
|
||||||
|
|
||||||
// The fake servers' answers are keyed by a server's address, a space and a
|
|
||||||
// query. These are the keys of the queries LookupASN sends for testASN:
|
|
||||||
// IANA's, then RIPE's, which asks RIPE to leave out person and role
|
|
||||||
// objects.
|
|
||||||
const (
|
|
||||||
ianaQuery = ianaServer + " " + asQuery
|
|
||||||
ripeQuery = ripeServer + " --no-personal " + asQuery
|
|
||||||
)
|
|
||||||
|
|
||||||
// ianaReferral is IANA's answer for an AS in a block it gave to RIPE.
|
|
||||||
const ianaReferral = `% IANA WHOIS server
|
|
||||||
|
|
||||||
refer: whois.ripe.net
|
|
||||||
|
|
||||||
as-block: 3154-3353
|
|
||||||
organisation: Assigned by RIPE NCC
|
|
||||||
whois: whois.ripe.net
|
|
||||||
changed: 2002-09-01
|
|
||||||
source: IANA
|
|
||||||
`
|
|
||||||
|
|
||||||
// lacnicReferral is IANA's answer for an AS in a block it gave to LACNIC,
|
|
||||||
// shortened to the referral.
|
|
||||||
const lacnicReferral = "refer: whois.lacnic.net\n"
|
|
||||||
|
|
||||||
// ripeRecord is RIPE's answer for the AS when asked to leave out person and
|
|
||||||
// role objects. It starts with the as-block object of the block the AS is
|
|
||||||
// in. Then come the abuse address as a comment, the AS's object and the
|
|
||||||
// organisation object it refers to.
|
|
||||||
const ripeRecord = `% This is the RIPE Database query service.
|
|
||||||
% The objects are in RPSL format.
|
|
||||||
|
|
||||||
% Note: --no-personal means ALL personal data has been filtered from this output.
|
|
||||||
|
|
||||||
% Information related to 'AS3209 - AS3353'
|
|
||||||
|
|
||||||
as-block: AS3209 - AS3353
|
|
||||||
descr: RIPE NCC ASN block
|
|
||||||
remarks: These AS Numbers are assigned to network operators
|
|
||||||
remarks: in the RIPE NCC service region.
|
|
||||||
mnt-by: RIPE-NCC-HM-MNT
|
|
||||||
created: 2018-11-22T15:27:19Z
|
|
||||||
last-modified: 2018-11-22T15:27:19Z
|
|
||||||
source: RIPE
|
|
||||||
|
|
||||||
% Information related to 'AS3333'
|
|
||||||
|
|
||||||
% Abuse contact for 'AS3333' is 'abuse@ripe.net'
|
|
||||||
|
|
||||||
aut-num: AS3333
|
|
||||||
as-name: RIPE-NCC-AS
|
|
||||||
org: ORG-RIEN1-RIPE
|
|
||||||
admin-c: BRD-RIPE
|
|
||||||
tech-c: OPS4-RIPE
|
|
||||||
abuse-c: OPS4-RIPE
|
|
||||||
mnt-by: RIPE-NCC-MNT
|
|
||||||
created: 2002-09-17T13:08:32Z
|
|
||||||
last-modified: 2024-01-09T08:55:41Z
|
|
||||||
source: RIPE
|
|
||||||
|
|
||||||
organisation: ORG-RIEN1-RIPE
|
|
||||||
org-name: Reseaux IP Europeens Network Coordination Centre (RIPE NCC)
|
|
||||||
country: NL
|
|
||||||
org-type: RIR
|
|
||||||
address: P.O. Box 10096
|
|
||||||
address: 1001EB Amsterdam
|
|
||||||
abuse-c: OPS4-RIPE
|
|
||||||
created: 2012-03-09T13:08:29Z
|
|
||||||
last-modified: 2024-03-26T10:55:38Z
|
|
||||||
source: RIPE
|
|
||||||
`
|
|
||||||
|
|
||||||
// The error lines with which RIPE refuses a query: access denied to the
|
|
||||||
// host, and the host's limit on returned objects reached.
|
|
||||||
const (
|
|
||||||
ripeAccessDenied = "%ERROR:201: access denied for 192.0.2.1"
|
|
||||||
ripeLimitReached = "%ERROR:202: access control limit reached"
|
|
||||||
)
|
|
||||||
|
|
||||||
// ripeRefusal is RIPE's answer to a host it no longer serves.
|
|
||||||
const ripeRefusal = `% This is the RIPE Database query service.
|
|
||||||
% The objects are in RPSL format.
|
|
||||||
%
|
|
||||||
` + ripeAccessDenied + `
|
|
||||||
%
|
|
||||||
% Sorry, access from your host has been permanently
|
|
||||||
% denied because of a repeated excessive querying.
|
|
||||||
`
|
|
||||||
|
|
||||||
// apnicAfrinicRecord is the answer of APNIC or AFRINIC for the AS when asked
|
|
||||||
// to leave out person and role objects. Like RIPE's, it holds the abuse
|
|
||||||
// address as a comment, the AS's object and the organisation object.
|
|
||||||
const apnicAfrinicRecord = `% Abuse contact for 'AS3333' is 'abuse@example.net'
|
|
||||||
|
|
||||||
aut-num: AS3333
|
|
||||||
as-name: EXAMPLE-AS
|
|
||||||
org: ORG-EXAMPLE1
|
|
||||||
admin-c: EX1-TEST
|
|
||||||
last-modified: 2026-08-26T00:40:00Z
|
|
||||||
|
|
||||||
organisation: ORG-EXAMPLE1
|
|
||||||
org-name: Example Networks
|
|
||||||
country: ZA
|
|
||||||
address: 1 Example Road
|
|
||||||
`
|
|
||||||
|
|
||||||
// apnicServer is APNIC's WHOIS server, and apnicASN an AS that APNIC holds.
|
|
||||||
const (
|
|
||||||
apnicServer = "whois.apnic.net:43"
|
|
||||||
apnicASN = 4608
|
|
||||||
)
|
|
||||||
|
|
||||||
// The keys of the queries LookupASN sends for apnicASN: IANA's, then
|
|
||||||
// APNIC's, which asks APNIC to leave out person and role objects.
|
|
||||||
const (
|
|
||||||
apnicIANAQuery = ianaServer + " AS4608"
|
|
||||||
apnicQuery = apnicServer + " --no-personal AS4608"
|
|
||||||
)
|
|
||||||
|
|
||||||
// apnicRecord is APNIC's answer for apnicASN when asked to leave out person
|
|
||||||
// and role objects, shortened. It starts with the as-block object of the
|
|
||||||
// block the AS is in. Then come the abuse address as a comment, the AS's
|
|
||||||
// object, and the irt and organisation objects it refers to.
|
|
||||||
const apnicRecord = `% [whois.apnic.net]
|
|
||||||
% Whois data copyright terms http://www.apnic.net/db/dbcopyright.html
|
|
||||||
|
|
||||||
% Note: --no-personal means ALL personal data has been filtered from this output.
|
|
||||||
|
|
||||||
% Information related to 'AS4608 - AS4865'
|
|
||||||
|
|
||||||
as-block: AS4608 - AS4865
|
|
||||||
descr: APNIC ASN block
|
|
||||||
remarks: These AS numbers are further assigned by APNIC
|
|
||||||
remarks: to APNIC members and end-users in the APNIC region
|
|
||||||
mnt-by: APNIC-HM
|
|
||||||
mnt-lower: APNIC-HM
|
|
||||||
last-modified: 2019-05-20T03:52:26Z
|
|
||||||
source: APNIC
|
|
||||||
|
|
||||||
% Information related to 'AS4608'
|
|
||||||
|
|
||||||
% Abuse contact for 'AS4608' is 'abuse@apnic.net'
|
|
||||||
|
|
||||||
aut-num: AS4608
|
|
||||||
as-name: APNIC-SERVICES
|
|
||||||
descr: Asia Pacific Network Information Centre
|
|
||||||
country: AU
|
|
||||||
org: ORG-APNI1-AP
|
|
||||||
admin-c: AIC1-AP
|
|
||||||
abuse-c: AA1589-AP
|
|
||||||
mnt-by: APNIC-HM
|
|
||||||
mnt-irt: IRT-APNIC-IS-AP
|
|
||||||
last-modified: 2026-08-26T00:40:00Z
|
|
||||||
source: APNIC
|
|
||||||
|
|
||||||
irt: IRT-APNIC-IS-AP
|
|
||||||
address: South Brisbane, Australia
|
|
||||||
e-mail: helpdesk@apnic.net
|
|
||||||
abuse-mailbox: abuse@apnic.net
|
|
||||||
mnt-by: MAINT-APNIC-IS-AP
|
|
||||||
last-modified: 2025-11-18T00:27:08Z
|
|
||||||
source: APNIC
|
|
||||||
|
|
||||||
organisation: ORG-APNI1-AP
|
|
||||||
org-name: Asia Pacific Network Information Centre
|
|
||||||
org-type: LIR
|
|
||||||
country: AU
|
|
||||||
address: PO Box 3646
|
|
||||||
address: South Brisbane, QLD 4101,
|
|
||||||
address: Australia
|
|
||||||
e-mail: noc@apnic.net
|
|
||||||
mnt-by: APNIC-HM
|
|
||||||
last-modified: 2026-08-24T12:58:43Z
|
|
||||||
source: APNIC
|
|
||||||
|
|
||||||
% This query was served by the APNIC Whois Service version 1.88.48 (WHOIS-UK2)
|
|
||||||
`
|
|
||||||
|
|
||||||
// ianaPrivateUse is IANA's answer for a private-use AS: its block and no
|
|
||||||
// referral.
|
|
||||||
const ianaPrivateUse = `% IANA WHOIS server
|
|
||||||
|
|
||||||
as-block: 64512-65534
|
|
||||||
organisation: Reserved for Private Use
|
|
||||||
descr: Reserved for Private Use
|
|
||||||
source: IANA
|
|
||||||
`
|
|
||||||
|
|
||||||
// ianaNoObject is IANA's answer when it has nothing for the query.
|
|
||||||
const ianaNoObject = `% IANA WHOIS server
|
|
||||||
% This query returned 0 objects.
|
|
||||||
`
|
|
||||||
|
|
||||||
// fillerLine is one of the lines a server that does not stop keeps sending.
|
|
||||||
const fillerLine = "remarks: and more\n"
|
|
||||||
|
|
||||||
// endlessAnswerSize is how many bytes of filler lines that server sends:
|
|
||||||
// four times the client's cap on an answer.
|
|
||||||
const endlessAnswerSize = 4 << 20
|
|
||||||
|
|
||||||
// fakeServers returns a DialContext for a whois.Client that connects to a
|
|
||||||
// fake server at each address answers names. Each key of answers is a
|
|
||||||
// server's address, a space and a query, and its value is that server's
|
|
||||||
// answer to that query. A server reads one query and sends its answer, or
|
|
||||||
// nothing for a query it has no answer for. Connecting to any other address
|
|
||||||
// is refused.
|
|
||||||
func fakeServers(
|
|
||||||
t *testing.T, answers map[string]string,
|
|
||||||
) func(context.Context, string, string) (net.Conn, error) {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
addresses := make(map[string]bool)
|
|
||||||
|
|
||||||
for key := range answers {
|
|
||||||
address, _, _ := strings.Cut(key, " ")
|
|
||||||
addresses[address] = true
|
|
||||||
}
|
|
||||||
|
|
||||||
var servers sync.WaitGroup
|
|
||||||
|
|
||||||
t.Cleanup(servers.Wait)
|
|
||||||
|
|
||||||
return func(_ context.Context, _, address string) (net.Conn, error) {
|
|
||||||
if !addresses[address] {
|
|
||||||
return nil, syscall.ECONNREFUSED
|
|
||||||
}
|
|
||||||
|
|
||||||
client, server := net.Pipe()
|
|
||||||
|
|
||||||
servers.Go(func() {
|
|
||||||
defer func() { _ = server.Close() }()
|
|
||||||
|
|
||||||
query, _ := bufio.NewReader(server).ReadString('\n')
|
|
||||||
answer := answers[address+" "+strings.TrimSpace(query)]
|
|
||||||
_, _ = io.WriteString(server, answer)
|
|
||||||
})
|
|
||||||
|
|
||||||
return client, nil
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// lookUp looks testASN up with fake servers sending answers.
|
|
||||||
func lookUp(
|
|
||||||
t *testing.T, answers map[string]string,
|
|
||||||
) (*whois.ASNInfo, error) {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
client := whois.NewClient(slog.New(slog.DiscardHandler))
|
|
||||||
client.DialContext = fakeServers(t, answers)
|
|
||||||
|
|
||||||
return client.LookupASN(t.Context(), testASN)
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestLookupASNReadsTheRIRAnswer looks up an AS IANA refers to RIPE. RIPE
|
|
||||||
// answers only a query that leaves out person and role objects, and its
|
|
||||||
// answer to that still fills the AS's fields: the abuse address from the
|
|
||||||
// comment line, and the organisation's name and country from its object.
|
|
||||||
// The dates are the AS's. IANA's "changed:" line dates the AS's block, and
|
|
||||||
// so do the dates of the as-block object at the start of RIPE's answer.
|
|
||||||
func TestLookupASNReadsTheRIRAnswer(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
info, err := lookUp(t, map[string]string{
|
|
||||||
ianaQuery: ianaReferral,
|
|
||||||
ripeQuery: ripeRecord,
|
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("LookupASN: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
got := []string{
|
|
||||||
info.ASName, info.OrgID, info.OrgName, info.CountryCode,
|
|
||||||
info.AbuseEmail, info.RIR,
|
|
||||||
}
|
|
||||||
want := []string{
|
|
||||||
"RIPE-NCC-AS", "ORG-RIEN1-RIPE",
|
|
||||||
"Reseaux IP Europeens Network Coordination Centre (RIPE NCC)", "NL",
|
|
||||||
"abuse@ripe.net", whois.RIRRIPE,
|
|
||||||
}
|
|
||||||
|
|
||||||
if !slices.Equal(got, want) {
|
|
||||||
t.Errorf("LookupASN = AS name, org ID, org name, country, abuse "+
|
|
||||||
"address, RIR %q; want %q", got, want)
|
|
||||||
}
|
|
||||||
|
|
||||||
wantRegDate := time.Date(2002, 9, 17, 13, 8, 32, 0, time.UTC)
|
|
||||||
if info.RegDate == nil || !info.RegDate.Equal(wantRegDate) {
|
|
||||||
t.Errorf("LookupASN registered = %v, want %v",
|
|
||||||
info.RegDate, wantRegDate)
|
|
||||||
}
|
|
||||||
|
|
||||||
wantLastMod := time.Date(2024, 1, 9, 8, 55, 41, 0, time.UTC)
|
|
||||||
if info.LastMod == nil || !info.LastMod.Equal(wantLastMod) {
|
|
||||||
t.Errorf("LookupASN last modified = %v, want %v",
|
|
||||||
info.LastMod, wantLastMod)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestLookupASNReadsOnlyTheASAndItsOrganisation looks up an AS at APNIC,
|
|
||||||
// whose answer starts with the as-block object of the block the AS is in.
|
|
||||||
// The last-modified date is the aut-num object's, not the block's. The
|
|
||||||
// address is the organisation object's alone, without the irt object's.
|
|
||||||
func TestLookupASNReadsOnlyTheASAndItsOrganisation(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
client := whois.NewClient(slog.New(slog.DiscardHandler))
|
|
||||||
client.DialContext = fakeServers(t, map[string]string{
|
|
||||||
apnicIANAQuery: "refer: whois.apnic.net\n",
|
|
||||||
apnicQuery: apnicRecord,
|
|
||||||
})
|
|
||||||
|
|
||||||
info, err := client.LookupASN(t.Context(), apnicASN)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("LookupASN: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
wantLastMod := time.Date(2026, 8, 26, 0, 40, 0, 0, time.UTC)
|
|
||||||
if info.LastMod == nil || !info.LastMod.Equal(wantLastMod) {
|
|
||||||
t.Errorf("LookupASN last modified = %v, want %v",
|
|
||||||
info.LastMod, wantLastMod)
|
|
||||||
}
|
|
||||||
|
|
||||||
got := []string{
|
|
||||||
info.ASName, info.OrgID, info.OrgName, info.CountryCode,
|
|
||||||
info.Address, info.AbuseEmail,
|
|
||||||
}
|
|
||||||
want := []string{
|
|
||||||
"APNIC-SERVICES", "ORG-APNI1-AP",
|
|
||||||
"Asia Pacific Network Information Centre", "AU",
|
|
||||||
"PO Box 3646\nSouth Brisbane, QLD 4101,\nAustralia", "abuse@apnic.net",
|
|
||||||
}
|
|
||||||
|
|
||||||
if !slices.Equal(got, want) {
|
|
||||||
t.Errorf("LookupASN = AS name, org ID, org name, country, address, "+
|
|
||||||
"abuse address %q; want %q", got, want)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestLookupASNAsksAPNICAndAFRINICToLeaveOutPersonalData looks up an AS at
|
|
||||||
// APNIC and at AFRINIC. Each answers only a query that leaves out person
|
|
||||||
// and role objects, as RIPE does. That one answer still holds the
|
|
||||||
// organisation object, so the organisation's name and country are filled.
|
|
||||||
func TestLookupASNAsksAPNICAndAFRINICToLeaveOutPersonalData(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
for _, server := range []string{"whois.apnic.net", "whois.afrinic.net"} {
|
|
||||||
t.Run(server, func(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
address := server + ":43"
|
|
||||||
|
|
||||||
info, err := lookUp(t, map[string]string{
|
|
||||||
ianaQuery: "refer: " + server + "\n",
|
|
||||||
address + " --no-personal " + asQuery: apnicAfrinicRecord,
|
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("LookupASN: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
got := []string{
|
|
||||||
info.ASName, info.OrgName, info.CountryCode, info.AbuseEmail,
|
|
||||||
}
|
|
||||||
want := []string{
|
|
||||||
"EXAMPLE-AS", "Example Networks", "ZA", "abuse@example.net",
|
|
||||||
}
|
|
||||||
|
|
||||||
if !slices.Equal(got, want) {
|
|
||||||
t.Errorf("LookupASN = AS name, org name, country, abuse "+
|
|
||||||
"address %q; want %q", got, want)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// The name of an organisation with accented letters, in Latin-1, one byte for
|
|
||||||
// each letter, and in UTF-8.
|
|
||||||
const (
|
|
||||||
latin1OrgName = "Fundaci\xf3n Ra\xedces de Espa\xf1a"
|
|
||||||
utf8OrgName = "Fundación Raíces de España"
|
|
||||||
)
|
|
||||||
|
|
||||||
// TestLookupASNStoresAnswersInUTF8 looks up an AS at RIPE, which answers in
|
|
||||||
// Latin-1, and at LACNIC, which answers in UTF-8. Both answers give the
|
|
||||||
// organisation's name with its accented letters in UTF-8, in the parsed field
|
|
||||||
// and in the stored answer. Before the fix the Latin-1 answer was kept as
|
|
||||||
// received, which is not valid UTF-8.
|
|
||||||
func TestLookupASNStoresAnswersInUTF8(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
for name, answers := range map[string]map[string]string{
|
|
||||||
"RIPE in Latin-1": {
|
|
||||||
ianaQuery: ianaReferral,
|
|
||||||
ripeQuery: "aut-num: AS3333\n" +
|
|
||||||
"org-name: " + latin1OrgName + "\n",
|
|
||||||
},
|
|
||||||
"LACNIC in UTF-8": {
|
|
||||||
ianaQuery: lacnicReferral,
|
|
||||||
"whois.lacnic.net:43 " + asQuery: "aut-num: AS3333\n" +
|
|
||||||
"owner: " + utf8OrgName + "\n",
|
|
||||||
},
|
|
||||||
} {
|
|
||||||
t.Run(name, func(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
info, err := lookUp(t, answers)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("LookupASN: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
if info.OrgName != utf8OrgName {
|
|
||||||
t.Errorf("LookupASN org name = %q, want %q",
|
|
||||||
info.OrgName, utf8OrgName)
|
|
||||||
}
|
|
||||||
|
|
||||||
if !utf8.ValidString(info.RawResponse) ||
|
|
||||||
!strings.Contains(info.RawResponse, utf8OrgName) {
|
|
||||||
t.Errorf("LookupASN answer = %q, want UTF-8 holding %q",
|
|
||||||
info.RawResponse, utf8OrgName)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// lacnicServer is LACNIC's WHOIS server and nicbrServer NIC.br's, which
|
|
||||||
// registers Brazil's ASes under LACNIC. brazilASN is one of those ASes.
|
|
||||||
const (
|
|
||||||
lacnicServer = "whois.lacnic.net:43"
|
|
||||||
nicbrServer = "whois.registro.br:43"
|
|
||||||
brazilASN = 52751
|
|
||||||
)
|
|
||||||
|
|
||||||
// The keys of the queries LookupASN sends for brazilASN: IANA's, LACNIC's,
|
|
||||||
// then NIC.br's.
|
|
||||||
const (
|
|
||||||
brazilIANAQuery = ianaServer + " AS52751"
|
|
||||||
brazilLACNICQuery = lacnicServer + " AS52751"
|
|
||||||
brazilNICbrQuery = nicbrServer + " AS52751"
|
|
||||||
)
|
|
||||||
|
|
||||||
// The name of brazilASN's organisation in three forms: as LACNIC sends it,
|
|
||||||
// with each accented letter replaced by U+FFFD; as NIC.br sends it, in
|
|
||||||
// Latin-1; and in UTF-8.
|
|
||||||
const (
|
|
||||||
lacnicBrazilOrgName = "Brasrede Telecomunica\U0000FFFD\U0000FFFDes LTDA"
|
|
||||||
nicbrBrazilOrgName = "Brasrede Telecomunica\xe7\xf5es LTDA"
|
|
||||||
brazilOrgName = "Brasrede Telecomunicações LTDA"
|
|
||||||
)
|
|
||||||
|
|
||||||
// nicbrCopyright is the copyright notice at the top of NIC.br's answers.
|
|
||||||
const nicbrCopyright = "% Copyright (c) Nic.br - " +
|
|
||||||
"Use of this data is governed by the Use and\n" +
|
|
||||||
"% Privacy Policy at https://registro.br/upp . Distribution,\n" +
|
|
||||||
"% commercialization, reproduction, and use for advertising or similar\n" +
|
|
||||||
"% purposes are expressly prohibited.\n"
|
|
||||||
|
|
||||||
// lacnicBrazilRecord is LACNIC's answer for brazilASN, recorded on
|
|
||||||
// 2026-10-26. It is NIC.br's answer, which LACNIC passes on after a line
|
|
||||||
// of its own and a space. It is shortened. The client's address is a
|
|
||||||
// documentation address. The contact's name, the contact's object and the
|
|
||||||
// footer are left out.
|
|
||||||
const lacnicBrazilRecord = "% IP Client: 192.0.2.1\n" +
|
|
||||||
" " + nicbrCopyright + `% 2026-10-26T23:12:22-03:00 - 192.0.2.1
|
|
||||||
|
|
||||||
aut-num: AS52751
|
|
||||||
owner: ` + lacnicBrazilOrgName + `
|
|
||||||
owner-c: AVL217
|
|
||||||
routing-c: AVL217
|
|
||||||
abuse-c: AVL217
|
|
||||||
created: 20120829
|
|
||||||
changed: 20130306
|
|
||||||
inetnum: 177.38.152.0/21
|
|
||||||
inetnum: 2804:b34::/32
|
|
||||||
inetnum: 191.243.4.0/22
|
|
||||||
inetnum: 131.72.248.0/22
|
|
||||||
inetnum: 138.94.228.0/22
|
|
||||||
inetnum: 170.233.8.0/22
|
|
||||||
inetnum: 170.246.4.0/22
|
|
||||||
`
|
|
||||||
|
|
||||||
// nicbrRecord is NIC.br's answer for brazilASN, in Latin-1, recorded on
|
|
||||||
// 2026-10-26 and shortened as lacnicBrazilRecord is.
|
|
||||||
const nicbrRecord = nicbrCopyright + `% 2026-10-26T23:14:41-03:00 - 192.0.2.1
|
|
||||||
|
|
||||||
aut-num: AS52751
|
|
||||||
owner: ` + nicbrBrazilOrgName + `
|
|
||||||
owner-c: AVL217
|
|
||||||
routing-c: AVL217
|
|
||||||
abuse-c: AVL217
|
|
||||||
created: 20120829
|
|
||||||
changed: 20130306
|
|
||||||
inetnum: 177.38.152.0/21
|
|
||||||
inetnum: 2804:b34::/32
|
|
||||||
inetnum: 191.243.4.0/22
|
|
||||||
inetnum: 131.72.248.0/22
|
|
||||||
inetnum: 138.94.228.0/22
|
|
||||||
inetnum: 170.233.8.0/22
|
|
||||||
inetnum: 170.246.4.0/22
|
|
||||||
`
|
|
||||||
|
|
||||||
// nicbrOverLimit stands for NIC.br's answer for brazilASN to a client over
|
|
||||||
// its query limit, and is not recorded. Such an answer leaves information
|
|
||||||
// out, here the owner, and says so in its first line.
|
|
||||||
const nicbrOverLimit = `% Query rate limit exceeded. Reduced information.
|
|
||||||
|
|
||||||
aut-num: AS52751
|
|
||||||
created: 20120829
|
|
||||||
changed: 20130306
|
|
||||||
`
|
|
||||||
|
|
||||||
// newNICbrClient returns a client whose fake servers send answers and
|
|
||||||
// whose logger writes JSON lines to logs. It adds one to *nicbrAsks each
|
|
||||||
// time it connects to NIC.br's server.
|
|
||||||
func newNICbrClient(
|
|
||||||
t *testing.T, answers map[string]string, logs io.Writer, nicbrAsks *int,
|
|
||||||
) *whois.Client {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
dial := fakeServers(t, answers)
|
|
||||||
|
|
||||||
client := whois.NewClient(slog.New(slog.NewJSONHandler(logs, nil)))
|
|
||||||
client.DialContext = func(
|
|
||||||
ctx context.Context, network, address string,
|
|
||||||
) (net.Conn, error) {
|
|
||||||
if address == nicbrServer {
|
|
||||||
*nicbrAsks++
|
|
||||||
}
|
|
||||||
|
|
||||||
return dial(ctx, network, address)
|
|
||||||
}
|
|
||||||
|
|
||||||
return client
|
|
||||||
}
|
|
||||||
|
|
||||||
// lookUpBrazilianAS looks brazilASN up with fake servers sending answers,
|
|
||||||
// and reports whether the client connected to NIC.br's server.
|
|
||||||
func lookUpBrazilianAS(
|
|
||||||
t *testing.T, answers map[string]string,
|
|
||||||
) (*whois.ASNInfo, bool, error) {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
nicbrAsks := 0
|
|
||||||
client := newNICbrClient(t, answers, io.Discard, &nicbrAsks)
|
|
||||||
|
|
||||||
info, err := client.LookupASN(t.Context(), brazilASN)
|
|
||||||
|
|
||||||
return info, nicbrAsks > 0, err
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestLookupASNTakesABrazilianASFromNICbr looks up an AS that NIC.br
|
|
||||||
// registers. LACNIC's answer is NIC.br's with each accented letter replaced
|
|
||||||
// by U+FFFD, so the client also asks NIC.br's server, which answers in
|
|
||||||
// Latin-1. The organisation's name has its accented letters, in the parsed
|
|
||||||
// field and in the stored answer.
|
|
||||||
func TestLookupASNTakesABrazilianASFromNICbr(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
info, _, err := lookUpBrazilianAS(t, map[string]string{
|
|
||||||
brazilIANAQuery: lacnicReferral,
|
|
||||||
brazilLACNICQuery: lacnicBrazilRecord,
|
|
||||||
brazilNICbrQuery: nicbrRecord,
|
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("LookupASN: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
if info.OrgName != brazilOrgName || info.RIR != whois.RIRLACNIC {
|
|
||||||
t.Errorf("LookupASN org name, RIR = %q, %q; want %q, %q",
|
|
||||||
info.OrgName, info.RIR, brazilOrgName, whois.RIRLACNIC)
|
|
||||||
}
|
|
||||||
|
|
||||||
if !strings.Contains(info.RawResponse, brazilOrgName) {
|
|
||||||
t.Errorf("LookupASN answer = %q, want one holding %q",
|
|
||||||
info.RawResponse, brazilOrgName)
|
|
||||||
}
|
|
||||||
|
|
||||||
wantRegDate := time.Date(2012, 8, 29, 0, 0, 0, 0, time.UTC)
|
|
||||||
if info.RegDate == nil || !info.RegDate.Equal(wantRegDate) {
|
|
||||||
t.Errorf("LookupASN registered = %v, want %v",
|
|
||||||
info.RegDate, wantRegDate)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestLookupASNAsksNICbrOnlyForItsASes looks up an AS at LACNIC whose
|
|
||||||
// answer does not carry NIC.br's copyright notice, as for an AS that LACNIC
|
|
||||||
// registers itself. The client does not connect to NIC.br's server, which
|
|
||||||
// would answer with a record, and keeps LACNIC's fields.
|
|
||||||
func TestLookupASNAsksNICbrOnlyForItsASes(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
info, askedNICbr, err := lookUpBrazilianAS(t, map[string]string{
|
|
||||||
brazilIANAQuery: lacnicReferral,
|
|
||||||
brazilLACNICQuery: "aut-num: AS52751\n" +
|
|
||||||
"owner: " + utf8OrgName + "\n",
|
|
||||||
brazilNICbrQuery: nicbrRecord,
|
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("LookupASN: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
if askedNICbr {
|
|
||||||
t.Error("LookupASN asked NIC.br's server about an AS whose " +
|
|
||||||
"answer from LACNIC is not NIC.br's")
|
|
||||||
}
|
|
||||||
|
|
||||||
if info.OrgName != utf8OrgName {
|
|
||||||
t.Errorf("LookupASN org name = %q, want LACNIC's %q",
|
|
||||||
info.OrgName, utf8OrgName)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestLookupASNKeepsLACNICsAnswerWhenNICbrFails looks up an AS that NIC.br
|
|
||||||
// registers when NIC.br's server refuses the connection, sends nothing, or
|
|
||||||
// sends the reduced answer for a client over its query limit. The client
|
|
||||||
// asks NIC.br, and the lookup still succeeds with LACNIC's fields. The
|
|
||||||
// client logs one warning, which names the AS and the reason.
|
|
||||||
func TestLookupASNKeepsLACNICsAnswerWhenNICbrFails(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
for name, test := range map[string]struct {
|
|
||||||
nicbrAnswer map[string]string
|
|
||||||
reason string
|
|
||||||
}{
|
|
||||||
"connection refused": {
|
|
||||||
nicbrAnswer: map[string]string{},
|
|
||||||
reason: syscall.ECONNREFUSED.Error(),
|
|
||||||
},
|
|
||||||
"no answer": {
|
|
||||||
nicbrAnswer: map[string]string{brazilNICbrQuery: ""},
|
|
||||||
reason: "no WHOIS fields",
|
|
||||||
},
|
|
||||||
"over the query limit": {
|
|
||||||
nicbrAnswer: map[string]string{brazilNICbrQuery: nicbrOverLimit},
|
|
||||||
reason: "over its query limit",
|
|
||||||
},
|
|
||||||
} {
|
|
||||||
t.Run(name, func(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
answers := map[string]string{
|
|
||||||
brazilIANAQuery: lacnicReferral,
|
|
||||||
brazilLACNICQuery: lacnicBrazilRecord,
|
|
||||||
}
|
|
||||||
maps.Copy(answers, test.nicbrAnswer)
|
|
||||||
|
|
||||||
var logs bytes.Buffer
|
|
||||||
|
|
||||||
nicbrAsks := 0
|
|
||||||
client := newNICbrClient(t, answers, &logs, &nicbrAsks)
|
|
||||||
|
|
||||||
info, err := client.LookupASN(t.Context(), brazilASN)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("LookupASN: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
if nicbrAsks == 0 {
|
|
||||||
t.Error("LookupASN did not ask NIC.br's server")
|
|
||||||
}
|
|
||||||
|
|
||||||
if info.OrgName != lacnicBrazilOrgName {
|
|
||||||
t.Errorf("LookupASN org name = %q, want LACNIC's %q",
|
|
||||||
info.OrgName, lacnicBrazilOrgName)
|
|
||||||
}
|
|
||||||
|
|
||||||
var logged struct {
|
|
||||||
Level string `json:"level"`
|
|
||||||
ASN int `json:"asn"`
|
|
||||||
}
|
|
||||||
|
|
||||||
err = json.Unmarshal(logs.Bytes(), &logged)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("want one log line, got %q: %v", logs.String(), err)
|
|
||||||
}
|
|
||||||
|
|
||||||
if logged.Level != slog.LevelWarn.String() ||
|
|
||||||
logged.ASN != brazilASN ||
|
|
||||||
!strings.Contains(logs.String(), test.reason) {
|
|
||||||
t.Errorf("logged %q, want a warning for AS%d holding %q",
|
|
||||||
logs.String(), brazilASN, test.reason)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestLookupASNSkipsNICbrAfterItsLimit looks up an AS that NIC.br
|
|
||||||
// registers three times with one client, while NIC.br answers that the
|
|
||||||
// client is over its query limit. The first lookup asks NIC.br. The
|
|
||||||
// second, a second before NICbrSkipTime has passed, does not. The third,
|
|
||||||
// once NICbrSkipTime has passed, asks again. Each keeps LACNIC's fields.
|
|
||||||
func TestLookupASNSkipsNICbrAfterItsLimit(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
nicbrAsks := 0
|
|
||||||
client := newNICbrClient(t, map[string]string{
|
|
||||||
brazilIANAQuery: lacnicReferral,
|
|
||||||
brazilLACNICQuery: lacnicBrazilRecord,
|
|
||||||
brazilNICbrQuery: nicbrOverLimit,
|
|
||||||
}, io.Discard, &nicbrAsks)
|
|
||||||
|
|
||||||
start := time.Date(2026, 10, 27, 0, 0, 0, 0, time.UTC)
|
|
||||||
now := start
|
|
||||||
client.Now = func() time.Time { return now }
|
|
||||||
|
|
||||||
for _, step := range []struct {
|
|
||||||
after time.Duration
|
|
||||||
wantAsks int
|
|
||||||
}{
|
|
||||||
{after: 0, wantAsks: 1},
|
|
||||||
{after: whois.NICbrSkipTime - time.Second, wantAsks: 1},
|
|
||||||
{after: whois.NICbrSkipTime, wantAsks: 2},
|
|
||||||
} {
|
|
||||||
now = start.Add(step.after)
|
|
||||||
|
|
||||||
info, err := client.LookupASN(t.Context(), brazilASN)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("LookupASN after %v: %v", step.after, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
if nicbrAsks != step.wantAsks {
|
|
||||||
t.Errorf("after %v NIC.br's server was asked %d times, want %d",
|
|
||||||
step.after, nicbrAsks, step.wantAsks)
|
|
||||||
}
|
|
||||||
|
|
||||||
if info.OrgName != lacnicBrazilOrgName {
|
|
||||||
t.Errorf("LookupASN after %v org name = %q, want LACNIC's %q",
|
|
||||||
step.after, info.OrgName, lacnicBrazilOrgName)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestLookupASNReportsARefusal looks up an AS IANA refers to RIPE when RIPE
|
|
||||||
// refuses the query, with each of the error lines RIPE refuses with. The
|
|
||||||
// error holds RIPE's line, so the log says why the lookup failed.
|
|
||||||
func TestLookupASNReportsARefusal(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
for _, line := range []string{ripeAccessDenied, ripeLimitReached} {
|
|
||||||
t.Run(line, func(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
_, err := lookUp(t, map[string]string{
|
|
||||||
ianaQuery: ianaReferral,
|
|
||||||
ripeQuery: "% This is the RIPE Database query service.\n" +
|
|
||||||
line + "\n",
|
|
||||||
})
|
|
||||||
if err == nil || !strings.Contains(err.Error(), line) {
|
|
||||||
t.Errorf("LookupASN error = %v, want one holding %q",
|
|
||||||
err, line)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestLookupASNFailsWhenTheRIRGivesNoAnswer looks up an AS IANA refers to
|
|
||||||
// RIPE when RIPE's server refuses the connection, and when it answers only
|
|
||||||
// with an error. Neither is a lookup that found the AS.
|
|
||||||
func TestLookupASNFailsWhenTheRIRGivesNoAnswer(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
for name, answers := range map[string]map[string]string{
|
|
||||||
"connection refused": {ianaQuery: ianaReferral},
|
|
||||||
"only an error": {ianaQuery: ianaReferral, ripeQuery: ripeRefusal},
|
|
||||||
} {
|
|
||||||
t.Run(name, func(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
_, err := lookUp(t, answers)
|
|
||||||
if err == nil || errors.Is(err, whois.ErrNoReferral) {
|
|
||||||
t.Errorf("LookupASN error = %v, want one that is not "+
|
|
||||||
"ErrNoReferral", err)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestLookupASNReportsNoReferral looks up a private-use AS, which IANA
|
|
||||||
// refers to no RIR.
|
|
||||||
func TestLookupASNReportsNoReferral(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
_, err := lookUp(t, map[string]string{ianaQuery: ianaPrivateUse})
|
|
||||||
if !errors.Is(err, whois.ErrNoReferral) {
|
|
||||||
t.Errorf("LookupASN error = %v, want ErrNoReferral", err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestLookupASNFailsWhenIANAAnswersNoBlock looks up an AS when IANA answers
|
|
||||||
// with neither a referral nor the AS's block. That is not IANA saying no
|
|
||||||
// RIR holds the AS.
|
|
||||||
func TestLookupASNFailsWhenIANAAnswersNoBlock(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
_, err := lookUp(t, map[string]string{ianaQuery: ianaNoObject})
|
|
||||||
if err == nil || errors.Is(err, whois.ErrNoReferral) {
|
|
||||||
t.Errorf("LookupASN error = %v, want one that is not ErrNoReferral",
|
|
||||||
err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestLookupASNFailsWhenTheRIRAnswerDoesNotEnd looks up an AS whose RIR
|
|
||||||
// server follows the record with filler lines far past any real answer.
|
|
||||||
// The client stops reading at its cap and fails instead of keeping them.
|
|
||||||
func TestLookupASNFailsWhenTheRIRAnswerDoesNotEnd(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
endless := ripeRecord +
|
|
||||||
strings.Repeat(fillerLine, endlessAnswerSize/len(fillerLine))
|
|
||||||
|
|
||||||
_, err := lookUp(t, map[string]string{
|
|
||||||
ianaQuery: ianaReferral,
|
|
||||||
ripeQuery: endless,
|
|
||||||
})
|
|
||||||
if err == nil || errors.Is(err, whois.ErrNoReferral) {
|
|
||||||
t.Errorf("LookupASN error = %v, want one that is not ErrNoReferral",
|
|
||||||
err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestLookupASNFailsOnAReferralToAnotherServer looks up an AS when IANA's
|
|
||||||
// answer refers it to a server none of the five RIRs run, with a port and
|
|
||||||
// without. The client follows a referral only to an RIR's server, so it
|
|
||||||
// fails instead of asking that server, which would answer with a record.
|
|
||||||
func TestLookupASNFailsOnAReferralToAnotherServer(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
// Each case maps the server on the "refer:" line to the address the
|
|
||||||
// client would dial to follow it. IANA's answer also names the AS's
|
|
||||||
// block, so a client that dropped the referral would report no RIR.
|
|
||||||
for refer, address := range map[string]string{
|
|
||||||
"whois.example.net:8080": "whois.example.net:8080",
|
|
||||||
"whois.example.net": "whois.example.net:43",
|
|
||||||
} {
|
|
||||||
t.Run(refer, func(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
_, err := lookUp(t, map[string]string{
|
|
||||||
ianaQuery: "refer: " + refer + "\n\n" +
|
|
||||||
"as-block: 3154-3353\n",
|
|
||||||
address + " " + asQuery: ripeRecord,
|
|
||||||
})
|
|
||||||
if err == nil || errors.Is(err, whois.ErrNoReferral) {
|
|
||||||
t.Errorf("LookupASN error = %v, want one that is not "+
|
|
||||||
"ErrNoReferral", err)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
Binary file not shown.
+23
-49
@@ -2,13 +2,10 @@ package asinfo
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"bytes"
|
"bytes"
|
||||||
"cmp"
|
|
||||||
"compress/gzip"
|
"compress/gzip"
|
||||||
_ "embed"
|
_ "embed"
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
"io"
|
"io"
|
||||||
"slices"
|
|
||||||
"strings"
|
|
||||||
"sync"
|
"sync"
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -26,17 +23,6 @@ type Info struct {
|
|||||||
type Registry struct {
|
type Registry struct {
|
||||||
mu sync.RWMutex
|
mu sync.RWMutex
|
||||||
byASN map[int]*Info
|
byASN map[int]*Info
|
||||||
// searchEntries holds every entry in AS number order, for Search.
|
|
||||||
searchEntries []searchEntry
|
|
||||||
}
|
|
||||||
|
|
||||||
// searchEntry is an entry with its handle and description in lower case. They
|
|
||||||
// are put in lower case once, when the data loads, so that Search only
|
|
||||||
// compares.
|
|
||||||
type searchEntry struct {
|
|
||||||
info *Info
|
|
||||||
lowerHandle string
|
|
||||||
lowerDescription string
|
|
||||||
}
|
}
|
||||||
|
|
||||||
var (
|
var (
|
||||||
@@ -73,21 +59,9 @@ func initRegistry() {
|
|||||||
panic("failed to unmarshal embedded AS data: " + err.Error())
|
panic("failed to unmarshal embedded AS data: " + err.Error())
|
||||||
}
|
}
|
||||||
|
|
||||||
slices.SortFunc(asInfos, func(a, b Info) int {
|
|
||||||
return cmp.Compare(a.ASN, b.ASN)
|
|
||||||
})
|
|
||||||
|
|
||||||
defaultRegistry.searchEntries = make([]searchEntry, 0, len(asInfos))
|
|
||||||
|
|
||||||
for i := range asInfos {
|
for i := range asInfos {
|
||||||
info := &asInfos[i]
|
info := &asInfos[i]
|
||||||
defaultRegistry.byASN[info.ASN] = info
|
defaultRegistry.byASN[info.ASN] = info
|
||||||
defaultRegistry.searchEntries = append(defaultRegistry.searchEntries,
|
|
||||||
searchEntry{
|
|
||||||
info: info,
|
|
||||||
lowerHandle: strings.ToLower(info.Handle),
|
|
||||||
lowerDescription: strings.ToLower(info.Description),
|
|
||||||
})
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -155,36 +129,36 @@ func All() []Info {
|
|||||||
return result
|
return result
|
||||||
}
|
}
|
||||||
|
|
||||||
// Search finds the AS entries whose handle or description contains query,
|
// Search searches for AS entries where the handle or description contains the query
|
||||||
// ignoring case. It returns the first limit of them in AS number order, and the
|
// This is a simple case-sensitive substring search
|
||||||
// number of entries that match. An empty query matches nothing.
|
func Search(query string) []Info {
|
||||||
func Search(query string, limit int) ([]Info, int) {
|
|
||||||
once.Do(initRegistry)
|
once.Do(initRegistry)
|
||||||
|
|
||||||
if query == "" {
|
|
||||||
return nil, 0
|
|
||||||
}
|
|
||||||
|
|
||||||
query = strings.ToLower(query)
|
|
||||||
|
|
||||||
defaultRegistry.mu.RLock()
|
defaultRegistry.mu.RLock()
|
||||||
defer defaultRegistry.mu.RUnlock()
|
defer defaultRegistry.mu.RUnlock()
|
||||||
|
|
||||||
var (
|
var results []Info
|
||||||
results []Info
|
|
||||||
total int
|
|
||||||
)
|
|
||||||
|
|
||||||
for _, entry := range defaultRegistry.searchEntries {
|
for _, info := range defaultRegistry.byASN {
|
||||||
if strings.Contains(entry.lowerHandle, query) ||
|
if contains(info.Handle, query) || contains(info.Description, query) {
|
||||||
strings.Contains(entry.lowerDescription, query) {
|
results = append(results, *info)
|
||||||
if total < limit {
|
|
||||||
results = append(results, *entry.info)
|
|
||||||
}
|
|
||||||
|
|
||||||
total++
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
return results, total
|
return results
|
||||||
|
}
|
||||||
|
|
||||||
|
// contains is a simple substring search
|
||||||
|
func contains(s, substr string) bool {
|
||||||
|
return len(substr) > 0 && len(s) >= len(substr) && containsImpl(s, substr)
|
||||||
|
}
|
||||||
|
|
||||||
|
func containsImpl(s, substr string) bool {
|
||||||
|
for i := 0; i <= len(s)-len(substr); i++ {
|
||||||
|
if s[i:i+len(substr)] == substr {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return false
|
||||||
}
|
}
|
||||||
|
|||||||
+20
-112
@@ -1,8 +1,6 @@
|
|||||||
package asinfo_test
|
package asinfo_test
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"cmp"
|
|
||||||
"slices"
|
|
||||||
"strings"
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
@@ -22,7 +20,7 @@ func TestGet(t *testing.T) {
|
|||||||
name: "AS1 Level 3",
|
name: "AS1 Level 3",
|
||||||
asn: 1,
|
asn: 1,
|
||||||
wantOK: true,
|
wantOK: true,
|
||||||
wantDesc: "Level 3 Parent LLC",
|
wantDesc: "Level 3 Parent, LLC",
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
name: "AS15169 Google",
|
name: "AS15169 Google",
|
||||||
@@ -36,10 +34,10 @@ func TestGet(t *testing.T) {
|
|||||||
wantOK: false,
|
wantOK: false,
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
name: "AS1251 with accented letters",
|
name: "AS0 IANA Reserved",
|
||||||
asn: 1251,
|
asn: 0,
|
||||||
wantOK: true,
|
wantOK: true,
|
||||||
wantDesc: "FUNDAÇÃO DE AMPARO À PESQUISA DO ESTADO SÃO PAULO",
|
wantDesc: "Internet Assigned Numbers Authority",
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -69,7 +67,7 @@ func TestGetDescription(t *testing.T) {
|
|||||||
asn int
|
asn int
|
||||||
want string
|
want string
|
||||||
}{
|
}{
|
||||||
{1, "Level 3 Parent LLC"},
|
{1, "Level 3 Parent, LLC"},
|
||||||
{999999999, ""},
|
{999999999, ""},
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -92,7 +90,8 @@ func TestGetHandle(t *testing.T) {
|
|||||||
asn int
|
asn int
|
||||||
want string
|
want string
|
||||||
}{
|
}{
|
||||||
{1, "LVLT-1"},
|
{1, "LVLT"},
|
||||||
|
{0, "IANA-RSVD"},
|
||||||
{999999999, ""},
|
{999999999, ""},
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -142,9 +141,9 @@ func TestSearch(t *testing.T) {
|
|||||||
wantMin: 10,
|
wantMin: 10,
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
name: "Search for São Paulo",
|
name: "Empty search",
|
||||||
query: "São Paulo",
|
query: "",
|
||||||
wantMin: 1,
|
wantMin: 0,
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
name: "Non-existent org",
|
name: "Non-existent org",
|
||||||
@@ -157,115 +156,26 @@ func TestSearch(t *testing.T) {
|
|||||||
t.Run(tt.name, func(t *testing.T) {
|
t.Run(tt.name, func(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
results, _ := asinfo.Search(tt.query, asinfo.Total())
|
results := asinfo.Search(tt.query)
|
||||||
if len(results) < tt.wantMin {
|
if len(results) < tt.wantMin {
|
||||||
t.Errorf("Search(%q) returned %d results, want at least %d",
|
t.Errorf("Search(%q) returned %d results, want at least %d",
|
||||||
tt.query, len(results), tt.wantMin)
|
tt.query, len(results), tt.wantMin)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Verify all results contain the query, ignoring case
|
// Verify all results contain the query
|
||||||
query := strings.ToLower(tt.query)
|
if tt.query != "" {
|
||||||
for _, r := range results {
|
for _, r := range results {
|
||||||
if !strings.Contains(strings.ToLower(r.Handle), query) &&
|
if !strings.Contains(r.Handle, tt.query) &&
|
||||||
!strings.Contains(strings.ToLower(r.Description), query) {
|
!strings.Contains(r.Description, tt.query) {
|
||||||
t.Errorf("Search result ASN %d doesn't contain query %q",
|
t.Errorf("Search result ASN %d doesn't contain query %q",
|
||||||
r.ASN, tt.query)
|
r.ASN, tt.query)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// TestSearchEmptyQueryMatchesNothing expects an empty query to match no AS,
|
|
||||||
// although every handle contains the empty string.
|
|
||||||
func TestSearchEmptyQueryMatchesNothing(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
results, total := asinfo.Search("", asinfo.Total())
|
|
||||||
if len(results) != 0 || total != 0 {
|
|
||||||
t.Errorf(`Search("") returned %d results and a total of %d, want none`,
|
|
||||||
len(results), total)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestSearchIgnoresCase searches for Cloudflare's handle in lower case and its
|
|
||||||
// description in upper case, and expects both searches to find AS13335.
|
|
||||||
func TestSearchIgnoresCase(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
const cloudflareASN = 13335 // handle CLOUDFLARENET, "Cloudflare Inc."
|
|
||||||
|
|
||||||
isCloudflare := func(info asinfo.Info) bool {
|
|
||||||
return info.ASN == cloudflareASN
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, query := range []string{"cloudflarenet", "CLOUDFLARE INC."} {
|
|
||||||
results, _ := asinfo.Search(query, asinfo.Total())
|
|
||||||
if !slices.ContainsFunc(results, isCloudflare) {
|
|
||||||
t.Errorf("Search(%q) did not find AS%d", query, cloudflareASN)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestSearchOrdersByASN expects the matches of a search with many of them to
|
|
||||||
// come in AS number order.
|
|
||||||
func TestSearchOrdersByASN(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
const query = "University"
|
|
||||||
|
|
||||||
results, _ := asinfo.Search(query, asinfo.Total())
|
|
||||||
|
|
||||||
for i := 1; i < len(results); i++ {
|
|
||||||
if results[i-1].ASN >= results[i].ASN {
|
|
||||||
t.Fatalf("Search(%q): AS%d comes after AS%d",
|
|
||||||
query, results[i].ASN, results[i-1].ASN)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestSearchStopsAtLimit searches for a word that matches more entries than
|
|
||||||
// the limit. It expects exactly limit entries, the first matches in AS number
|
|
||||||
// order, and the number of all matches, counted here from All.
|
|
||||||
func TestSearchStopsAtLimit(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
const (
|
|
||||||
query = "university"
|
|
||||||
limit = 10
|
|
||||||
)
|
|
||||||
|
|
||||||
var matches []asinfo.Info
|
|
||||||
|
|
||||||
for _, info := range asinfo.All() {
|
|
||||||
if strings.Contains(strings.ToLower(info.Handle), query) ||
|
|
||||||
strings.Contains(strings.ToLower(info.Description), query) {
|
|
||||||
matches = append(matches, info)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if len(matches) <= limit {
|
|
||||||
t.Fatalf("%q matches %d entries; the test needs more than %d",
|
|
||||||
query, len(matches), limit)
|
|
||||||
}
|
|
||||||
|
|
||||||
slices.SortFunc(matches, func(a, b asinfo.Info) int {
|
|
||||||
return cmp.Compare(a.ASN, b.ASN)
|
|
||||||
})
|
|
||||||
|
|
||||||
results, total := asinfo.Search(query, limit)
|
|
||||||
|
|
||||||
if total != len(matches) {
|
|
||||||
t.Errorf("Search(%q, %d): total %d, want %d",
|
|
||||||
query, limit, total, len(matches))
|
|
||||||
}
|
|
||||||
|
|
||||||
if !slices.Equal(results, matches[:limit]) {
|
|
||||||
t.Errorf("Search(%q, %d) = %v, want %v",
|
|
||||||
query, limit, results, matches[:limit])
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestDataIntegrity(t *testing.T) {
|
func TestDataIntegrity(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
@@ -305,13 +215,11 @@ func BenchmarkGet(b *testing.B) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func BenchmarkSearch(b *testing.B) {
|
func BenchmarkSearch(b *testing.B) {
|
||||||
const limit = 50
|
|
||||||
|
|
||||||
queries := []string{"Google", "Amazon", "Microsoft", "University"}
|
queries := []string{"Google", "Amazon", "Microsoft", "University"}
|
||||||
|
|
||||||
b.ResetTimer()
|
b.ResetTimer()
|
||||||
|
|
||||||
for i := range b.N {
|
for i := range b.N {
|
||||||
asinfo.Search(queries[i%len(queries)], limit)
|
asinfo.Search(queries[i%len(queries)])
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
+3
-5
@@ -2,7 +2,7 @@
|
|||||||
Package asinfo provides information about Autonomous Systems (AS).
|
Package asinfo provides information about Autonomous Systems (AS).
|
||||||
|
|
||||||
The package embeds a comprehensive database of AS numbers, handles, and descriptions
|
The package embeds a comprehensive database of AS numbers, handles, and descriptions
|
||||||
sourced from https://github.com/ipverse/as-metadata. The data is embedded at compile
|
sourced from https://github.com/ipverse/asn-info. The data is embedded at compile
|
||||||
time and provides fast, offline lookups.
|
time and provides fast, offline lookups.
|
||||||
|
|
||||||
Basic usage:
|
Basic usage:
|
||||||
@@ -16,10 +16,8 @@ Basic usage:
|
|||||||
// Get just the description
|
// Get just the description
|
||||||
desc := asinfo.GetDescription(15169) // "Google LLC"
|
desc := asinfo.GetDescription(15169) // "Google LLC"
|
||||||
|
|
||||||
// Search the handles and descriptions, ignoring case, for the first 10
|
// Search for AS entries
|
||||||
// matches in AS number order and the number of all matches
|
results := asinfo.Search("University")
|
||||||
results, total := asinfo.Search("university", 10)
|
|
||||||
fmt.Printf("%d matches\n", total)
|
|
||||||
for _, as := range results {
|
for _, as := range results {
|
||||||
fmt.Printf("AS%d: %s\n", as.ASN, as.Description)
|
fmt.Printf("AS%d: %s\n", as.ASN, as.Description)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -17,20 +17,13 @@ func ExampleGet() {
|
|||||||
func ExampleGetDescription() {
|
func ExampleGetDescription() {
|
||||||
desc := asinfo.GetDescription(13335)
|
desc := asinfo.GetDescription(13335)
|
||||||
fmt.Println(desc)
|
fmt.Println(desc)
|
||||||
// Output: Cloudflare Inc.
|
// Output: Cloudflare, Inc.
|
||||||
}
|
}
|
||||||
|
|
||||||
func ExampleSearch() {
|
func ExampleSearch() {
|
||||||
const limit = 10
|
results := asinfo.Search("MIT-GATEWAY")
|
||||||
|
|
||||||
results, total := asinfo.Search("mit-gateway", limit)
|
|
||||||
|
|
||||||
fmt.Printf("%d of %d matches\n", len(results), total)
|
|
||||||
|
|
||||||
for _, info := range results {
|
for _, info := range results {
|
||||||
fmt.Printf("AS%d: %s - %s\n", info.ASN, info.Handle, info.Description)
|
fmt.Printf("AS%d: %s - %s\n", info.ASN, info.Handle, info.Description)
|
||||||
}
|
}
|
||||||
// Output:
|
// Output: AS3: MIT-GATEWAYS - Massachusetts Institute of Technology
|
||||||
// 1 of 1 matches
|
|
||||||
// AS3: MIT-GATEWAYS - Massachusetts Institute of Technology
|
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,34 +0,0 @@
|
|||||||
#!/bin/sh
|
|
||||||
# script/asupdate: rebuild the AS list built into RouteWatch from
|
|
||||||
# as.csv with cmd/asinfo-gen, which fails when it finds no AS. The new
|
|
||||||
# list replaces the old one only when the generator succeeds. Otherwise
|
|
||||||
# the old list stays as it was and the script exits non-zero.
|
|
||||||
set -eu
|
|
||||||
|
|
||||||
ROOT="$(cd "$(dirname "$0")/.." && pwd -P)"
|
|
||||||
|
|
||||||
AS_DATA="pkg/asinfo/asdata.json.gz"
|
|
||||||
# Next to AS_DATA, so the final mv is a rename on one file system.
|
|
||||||
AS_DATA_TMP="$AS_DATA.tmp"
|
|
||||||
|
|
||||||
main() {
|
|
||||||
cd "$ROOT"
|
|
||||||
|
|
||||||
json="$(mktemp)"
|
|
||||||
trap 'rm -f "$json" "$AS_DATA_TMP"' EXIT
|
|
||||||
# A signal exits through the EXIT trap too.
|
|
||||||
trap 'exit 1' HUP INT TERM
|
|
||||||
|
|
||||||
echo "Updating AS info data..."
|
|
||||||
if ! go run ./cmd/asinfo-gen >"$json"; then
|
|
||||||
echo "asupdate: the generator failed; $AS_DATA is unchanged" >&2
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
|
|
||||||
# -n leaves the time out of the file, so an unchanged list gives the
|
|
||||||
# same bytes.
|
|
||||||
gzip -n -c <"$json" >"$AS_DATA_TMP"
|
|
||||||
mv "$AS_DATA_TMP" "$AS_DATA"
|
|
||||||
}
|
|
||||||
|
|
||||||
main "$@"
|
|
||||||
Reference in New Issue
Block a user