Compare commits
1
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
 sneak
|
dfc3945bc0 |
No files matched your search
+1
-1
@@ -105,7 +105,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
|
||||
ARG CACHEBUST
|
||||
|
||||
# Create non-root user
|
||||
RUN useradd -u 1000 -m routewatch
|
||||
RUN useradd -r -u 1000 -m routewatch
|
||||
|
||||
RUN mkdir -p /var/lib/berlin.sneak.app.routewatch && chown routewatch:routewatch /var/lib/berlin.sneak.app.routewatch
|
||||
|
||||
|
||||
@@ -143,9 +143,9 @@ and origin AS can have two kept routes, one or none.
|
||||
AS path, a next hop and a time. The next hop is the router the peer sends the
|
||||
prefix's traffic to. RouteWatch records the peer itself as the next hop. The
|
||||
page also lists the prefix's origin changes, newest first. An origin change is
|
||||
recorded when the first peer starts to carry the prefix with an origin AS, or
|
||||
when the last peer stops. The page shows at most the newest 5,000. Each row
|
||||
shows:
|
||||
an AS starting to announce the prefix, when its first peer carries it, or
|
||||
stopping, when its last peer no longer does. The page shows at most the newest
|
||||
5,000. Each row shows:
|
||||
- the time
|
||||
- announced or withdrawn
|
||||
- the origin AS
|
||||
@@ -197,9 +197,8 @@ also carries an `@meta` object:
|
||||
- `api_version`: `1`
|
||||
- `execution_time_ms`: the number of milliseconds the request took
|
||||
|
||||
Every value with a unit is a number. The unit ends the field name, as in `_ms`,
|
||||
`_seconds`, `_bytes`, `_per_sec` and `_per_min`. `latitude` and `longitude` are
|
||||
in degrees, and their names do not say so.
|
||||
Every value with a unit is a number, and the unit ends the field name. Examples
|
||||
are `_ms`, `_seconds`, `_bytes`, `_per_sec` and `_per_min`.
|
||||
|
||||
The paths:
|
||||
|
||||
@@ -288,7 +287,6 @@ address. Every entry has the same fields:
|
||||
- `is_tor`: true when the address is on the `tor_exits` list
|
||||
- `is_vpn`: true when the address is on any list of VPN servers (see IP
|
||||
lists)
|
||||
- `is_relay`: true when the address is on the `icloud_private_relay` list
|
||||
- `is_bogon`: true when the address is reserved, so that no network should
|
||||
announce it on the internet
|
||||
- `vpn_providers`: the names of the lists of VPN servers the address is on, or
|
||||
@@ -424,9 +422,6 @@ The lists:
|
||||
last 30 days. From
|
||||
[FireHOL](https://iplists.firehol.org/?ipset=firehol_level3), downloaded every
|
||||
3 hours.
|
||||
- `icloud_private_relay`: the networks where iCloud Private Relay traffic
|
||||
leaves. From [Apple](https://mask-api.icloud.com/egress-ip-ranges.csv),
|
||||
downloaded once a day.
|
||||
- `ivpn`: VPN servers run by IVPN. From
|
||||
[IVPN](https://api.ivpn.net/v5/servers.json), downloaded once a day.
|
||||
- `mullvad`: VPN servers run by Mullvad. From
|
||||
@@ -503,13 +498,11 @@ How RouteWatch keeps the answers:
|
||||
there.
|
||||
- It writes new answers every 5 seconds to `cache.db` in the state directory
|
||||
(see Configuration), and loads them from there at startup.
|
||||
- Each DNS record carries a TTL, the time for which it may be kept. An answer
|
||||
expires after the shortest TTL of its records. RouteWatch keeps it for at
|
||||
least 60 seconds and at most 24 hours.
|
||||
- When an address has no PTR names, the answer's SOA record says how long that
|
||||
answer may be kept. The answer expires after the SOA record's minimum field,
|
||||
or the record's own TTL when that is shorter. RouteWatch keeps it for at most
|
||||
1 hour.
|
||||
- An answer expires after the TTL of its records, the shortest one when there
|
||||
are several. RouteWatch keeps it for at least 60 seconds and at most 24 hours.
|
||||
- When an address has no PTR names, that answer expires after the SOA record's
|
||||
minimum field, or the record's own TTL when that is shorter. RouteWatch keeps
|
||||
it for at most 1 hour.
|
||||
- An answer with no PTR names and no SOA record is not kept. Neither is one
|
||||
whose SOA record has a TTL or minimum field of 0.
|
||||
- A failed lookup is not kept, so the next lookup of the address asks again.
|
||||
@@ -523,8 +516,7 @@ An IP lookup finds where each address is, on a best-effort basis. It uses two
|
||||
sources:
|
||||
|
||||
- [geojs](https://www.geojs.io/) is asked first. One request carries every
|
||||
address of the lookup that the cache does not hold. geojs has 1 second to
|
||||
answer.
|
||||
address of the lookup, and it has 1 second to answer.
|
||||
- [DB-IP City Lite](https://db-ip.com/db/download/ip-to-city-lite) answers when
|
||||
geojs has no answer for the address. It is a database file that RouteWatch
|
||||
reads itself.
|
||||
@@ -684,13 +676,6 @@ The ceilings add up to 3.7 GiB:
|
||||
limit caps the SQLite library.
|
||||
- About 0.2 GiB for everything else in the runtime.
|
||||
|
||||
The `icloud_private_relay` list has about 285,000 networks. Neighbouring
|
||||
networks merge, which leaves about 6,900 address ranges. Loaded, the list adds
|
||||
about 0.3 MiB to the Go heap and under 6 MiB to the process's resident memory.
|
||||
Parsing the file allocates about 94 MiB in about 0.2 seconds. The next garbage
|
||||
collection frees all of it but the 0.3 MiB above. Parsing happens at startup and
|
||||
whenever a download brings a new version.
|
||||
|
||||
The SQLite library gets its memory from glibc, the C library. glibc keeps freed
|
||||
memory in pools called arenas, and hands it back to the kernel slowly. It
|
||||
creates up to eight arenas per processor core. So on a host with many cores the
|
||||
@@ -706,10 +691,8 @@ What happens at each limit:
|
||||
allocating more.
|
||||
- The SQLite hard heap limit: a database statement fails with an out-of-memory
|
||||
error, which the handler logs. The prefix handler writes the routing table's
|
||||
changes and its origin changes again in a later write. Everything else in a
|
||||
handler's failed write is dropped. The PTR cache and the geolocation part
|
||||
write their answers to `cache.db` again in a later write. The process keeps
|
||||
running.
|
||||
changes again in a later write. Everything else in the failed write is
|
||||
dropped. The process keeps running.
|
||||
- The handler queues: the handlers are the four parts that take the feed's
|
||||
messages (see Design). The prefix handler's queue holds at most 100,000
|
||||
messages, about 100 MiB. Each of the other three holds at most 20,000. When a
|
||||
@@ -1076,8 +1059,8 @@ calls one of them. The scripts:
|
||||
- `script/projectname`: print the project name, `routewatch`, which
|
||||
`script/docker` uses as the image tag.
|
||||
- `script/test`: build the `test` phase of the `Dockerfile` alone. It runs
|
||||
`go test -short -count=1 -timeout 90s -race -cover ./...`. On failure it runs
|
||||
the tests again with `-v` and without `-cover`.
|
||||
`go test -short -count=1 -timeout 90s -race -cover ./...`, and reruns it with
|
||||
`-v` on failure.
|
||||
- `script/lint`: build the `lint` phase of the `Dockerfile` alone. It runs
|
||||
`golangci-lint` v2.14.0 with `.golangci.yml`.
|
||||
- `script/fmt`: format the Go code with `go fmt` and the markdown with
|
||||
@@ -1100,7 +1083,8 @@ calls one of them. The scripts:
|
||||
|
||||
## TODO
|
||||
|
||||
`TODO.md` holds the working list: the next step and the steps planned.
|
||||
`TODO.md` holds the working list: the next step, the steps done and the steps
|
||||
planned.
|
||||
|
||||
## License
|
||||
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
|
||||
- branch (from `main`)
|
||||
- do the work in Next Step
|
||||
- delete Next Step from `TODO.md`
|
||||
- move Next Step to the top of Completed Steps
|
||||
- move the top item of Future Steps into Next Step
|
||||
- commit (`TODO.md` changes in the same commit as the work)
|
||||
- merge to `main` if the branch is not protected, otherwise open a PR
|
||||
@@ -10,31 +10,460 @@
|
||||
|
||||
# Status
|
||||
|
||||
RouteWatch is pre-1.0. The order of the remaining work is kept on
|
||||
RouteWatch is pre-1.0, with no git tags. `main` last took `next` on 2026-10-27,
|
||||
through https://git.eeqj.de/sneak/routewatch/pulls/166.
|
||||
https://git.eeqj.de/sneak/routewatch/pulls/202 carries the commits since then
|
||||
into `main`. It is labelled `merge-ready` and waits for sneak.
|
||||
|
||||
The linter and the tests are phases of the Docker build. The Gitea workflow runs
|
||||
`script/cibuild` on every push. That script runs the two phases and checks the
|
||||
formatting of the Go code and the markdown. Then it builds the image.
|
||||
|
||||
The image sets memory ceilings for a 5 GiB container (README "Memory"). README
|
||||
"Running under upaas" says how to run it.
|
||||
|
||||
A five-day run of `1c65656` on the live feed, from 2026-10-27 to 2026-11-01,
|
||||
found three problems:
|
||||
|
||||
- The prefix handler dropped updates while a peer's session went down
|
||||
(https://git.eeqj.de/sneak/routewatch/issues/216).
|
||||
- The AS handler dropped updates when the feed burst
|
||||
(https://git.eeqj.de/sneak/routewatch/issues/217).
|
||||
- The WHOIS fetcher kept asking a registry that had refused it
|
||||
(https://git.eeqj.de/sneak/routewatch/issues/218).
|
||||
|
||||
All three are fixed on `next`. So is
|
||||
https://git.eeqj.de/sneak/routewatch/issues/222, which the third fix brought up.
|
||||
A run of `26d939a` on the live feed showed the prefix handler still dropping
|
||||
updates while the home page's cards were worked out.
|
||||
https://git.eeqj.de/sneak/routewatch/issues/226 fixed that on `next`.
|
||||
https://git.eeqj.de/sneak/routewatch/issues/225, also on `next`, takes the peers
|
||||
whose sessions go down together out of the table in one walk. A 24-hour run of
|
||||
`9bd0e91` on the live feed is checking whether any handler still drops updates.
|
||||
|
||||
The order of the project's remaining work is kept on
|
||||
https://git.eeqj.de/sneak/routewatch/issues/86.
|
||||
|
||||
# Next Step
|
||||
|
||||
Post the outcome of the run of `e58c663` on the live feed on
|
||||
https://git.eeqj.de/sneak/routewatch/issues/232. Work it out from the run's
|
||||
data:
|
||||
Post the outcome of the 24-hour run of `9bd0e91` on
|
||||
https://git.eeqj.de/sneak/routewatch/issues/226: whether any handler still drops
|
||||
updates, and the process's peak memory against 5 GiB. File an issue for each
|
||||
problem the run finds.
|
||||
|
||||
- the log of the stopped container `routewatch-run-e58c663`, read with
|
||||
`docker logs`
|
||||
- the database in the Docker volume `routewatch-run-e58c663`
|
||||
# Completed Steps
|
||||
|
||||
Keep the container until the outcome is posted. The outcome says:
|
||||
- 2026-11-10: each result of the IP lookup has `geo`, where the address is. It
|
||||
comes from geojs, with DB-IP's City Lite database as the fallback. The geojs
|
||||
answers stay in memory and in `cache.db` for 7 days (closes
|
||||
https://git.eeqj.de/sneak/routewatch/issues/239).
|
||||
- 2026-11-10: the IP lookup checks the `protonvpn` list of ProtonVPN's /24
|
||||
networks and the `protonvpn_entry` list of its servers' IPv4 and IPv6
|
||||
addresses, from a public mirror of Proton's server list (closes
|
||||
https://git.eeqj.de/sneak/routewatch/issues/236).
|
||||
- 2026-11-10: the IP lookup finds PTR names over DNS-over-HTTPS and caches them
|
||||
for the record's TTL, in memory and in `cache.db` (closes
|
||||
https://git.eeqj.de/sneak/routewatch/issues/238).
|
||||
- 2026-11-10: the tests no longer fail when a busy host stalls for several
|
||||
seconds. Most such waits are now 15 seconds. The test of the stream's idle
|
||||
timeout runs on a fake clock (closes #240)
|
||||
- 2026-11-10: the IP lookup checks the public VPN server lists of Mullvad,
|
||||
NordVPN, Private Internet Access and IVPN, and X4BNet's community list. Each
|
||||
result has `flags.is_vpn` and `vpn_providers` (closes
|
||||
https://git.eeqj.de/sneak/routewatch/issues/235).
|
||||
- 2026-11-10: `tor_exits` comes from Onionoo, so it holds IPv6 exits and
|
||||
`is_tor` is true for them (closes
|
||||
https://git.eeqj.de/sneak/routewatch/issues/234).
|
||||
- 2026-11-10: each result of the IP lookup has a `flags` object with `is_tor`
|
||||
and `is_bogon`. The public IP lists are named `ip_lists` in the answer,
|
||||
`internal/iplist` in the code and `iplists` in the state directory. A list
|
||||
file may be up to 200 MiB (closes
|
||||
https://git.eeqj.de/sneak/routewatch/issues/233).
|
||||
- 2026-11-05: the prefix page lists the prefix's origin changes, newest first.
|
||||
The routing table records each change, and the prefix handler writes them to a
|
||||
new `origin_changes` table. The database maintainer prunes each prefix to its
|
||||
newest 500 rows or its rows from the last 72 hours, whichever keeps more. The
|
||||
page shows at most the newest 5,000. Each row shows:
|
||||
- the time
|
||||
- announced or withdrawn
|
||||
- the origin AS
|
||||
- the peer count
|
||||
|
||||
- whether the feed stalled or reconnected
|
||||
- whether any handler dropped an update, and the most messages the prefix
|
||||
handler's queue held at once
|
||||
- the most messages the prefix handler held back at once
|
||||
- the size of the `origin_changes` table on disk
|
||||
- the process's peak memory against 5 GiB
|
||||
A row's peer count is how many peers carry the prefix just after the change,
|
||||
with any origin AS (closes #161).
|
||||
|
||||
The run's data does not show the most messages the prefix handler held back at
|
||||
once, or the process's peak memory.
|
||||
|
||||
File an issue for each problem the run found.
|
||||
- 2026-11-03: the streamer drops an update for a handler only when that
|
||||
handler's queue is full. It used to drop a random share of the updates once
|
||||
the queue was half full. The prefix handler's queue now holds 100,000
|
||||
messages, about 100 MiB. It used to hold 20,000. While the routing table's
|
||||
`Overview` walks every prefix, the prefix handler applies updates more slowly.
|
||||
In a run on the live feed, the feed paused during such a walk and then sent
|
||||
its backlog. The streamer then dropped about 10,000 of the prefix handler's
|
||||
updates (closes #229)
|
||||
- 2026-11-02: `README.md` is rewritten from scratch, sentence by sentence,
|
||||
against the rules of https://git.eeqj.de/sneak/routewatch/issues/157. Every
|
||||
statement was checked against the code on `next` (closes #157)
|
||||
- 2026-11-02: `TODO.md` is rewritten from scratch, sentence by sentence, against
|
||||
the rules of https://git.eeqj.de/sneak/routewatch/issues/157. Every Completed
|
||||
Steps entry keeps its date, its facts and its issue. Status and Next Step
|
||||
describe the repo as the tracker shows it (part of #157)
|
||||
- 2026-11-02: when the sessions of several peers go down, the prefix handler
|
||||
takes the peers out of the routing table together. The table's `PeersDown`
|
||||
takes several peers out of every prefix in one walk. One walk runs at a time.
|
||||
The next one takes every peer whose session went down since the previous one
|
||||
began. Each peer used to have a walk of its own. A peer's messages stayed held
|
||||
back while the walks queued ahead of its own ran. The handler holds back at
|
||||
most 150,000 messages, about 150 MiB. It used to hold back at most 20,000. The
|
||||
status page shows how many messages the handler holds back (closes #225)
|
||||
- 2026-11-02: working out the home page's cards no longer stops the prefix
|
||||
handler. The walk over the routing table holds the table's lock for 10,000
|
||||
prefixes at a time, then lets the lock go for as long again, as `PeerDown`
|
||||
does. It used to take the lock back at once. The prefix handler then applied
|
||||
about one update per 10,000 prefixes walked. During a walk, its queue could
|
||||
fill, and the streamer then dropped updates for it (closes #226)
|
||||
- 2026-11-02: A skipped WHOIS lookup now makes the fetcher wait longer before
|
||||
the next one, as a failed lookup does. A lookup is skipped when its RIR is
|
||||
paused. An RIR is one of the five regional Internet registries. The longest
|
||||
wait is one minute. The status page does not count a skip as a failure. A skip
|
||||
used to leave the wait as it was. Each lookup asks IANA which RIR holds the AS
|
||||
before it finds that RIR paused. So while most stale ASes belonged to a paused
|
||||
RIR, the fetcher could ask IANA every second (closes #222)
|
||||
- 2026-11-02: A peer's session going down no longer stops the prefix handler.
|
||||
The handler runs the routing table's `PeerDown` on a goroutine of its own, one
|
||||
peer at a time. Meanwhile it goes on taking messages. `PeerDown` holds the
|
||||
table's lock for 10,000 prefixes at a time, then lets the lock go for as long
|
||||
again. The handler holds back the down peer's later messages until that peer's
|
||||
`PeerDown` returns. Then the handler applies them in order, one at a time, and
|
||||
takes new messages in between. It holds back at most 20,000 messages. At that
|
||||
number, `HandleMessage` waits until one of them is applied. When `Stop` runs
|
||||
past its deadline, it logs how many held-back messages were not applied
|
||||
(closes #216)
|
||||
- 2026-11-02: RouteWatch pauses an RIR's or NIC.br's WHOIS server for an hour
|
||||
after the server refuses a query. NIC.br is Brazil's registry. A refused or
|
||||
reset connection is a refusal. So is an answer with an `%ERROR:201:` or
|
||||
`%ERROR:202:` line. Lookups at the other servers go on. An AS whose RIR is
|
||||
paused is looked up after the pause. While NIC.br is paused, a Brazilian AS
|
||||
gets the details from LACNIC's answer. IANA is never paused, because a pause
|
||||
of IANA would stop every lookup. A refusal by IANA is an ordinary failure.
|
||||
NIC.br is also paused for an hour after it says RouteWatch has passed its
|
||||
query limit. Before, the fetcher asked a server again within seconds of a
|
||||
refusal (closes #218)
|
||||
- 2026-11-02: The AS handler never waits for a database write. Its batch holds
|
||||
one entry per AS, with the earliest and the latest time of its updates. So the
|
||||
batch no longer grows with the feed's rate. It used to hold one entry for each
|
||||
AS of each path. At 300,000 entries it waited for the write in progress. In a
|
||||
burst of the feed its queue then filled, and the streamer dropped messages for
|
||||
it (closes #217)
|
||||
- 2026-10-27: The tests of `internal/routewatch` no longer fail on a loaded
|
||||
host. Five of them gave `HandleMessage` two seconds to apply a full prefix
|
||||
batch. Under the race detector that can take longer. A test now waits up to 15
|
||||
seconds for each step it expects. The tests of the handlers' limits now give
|
||||
the handlers small limits. So the package no longer runs past its 90-second
|
||||
limit (closes #213, #215)
|
||||
- 2026-10-27: The server works out the home page's cards every minute until it
|
||||
stops. It used to work them out only once, at startup. The end of the context
|
||||
given to `Start` stopped the refreshes (closes #211)
|
||||
- 2026-10-27: README "Rationale" says why RouteWatch exists. RouteWatch answers
|
||||
our other services' questions about IP addresses, from its own routing table
|
||||
and the WHOIS details it keeps. The section still says what RIS Live does not
|
||||
give (closes #204)
|
||||
- 2026-10-27: Every JSON answer carries `@meta`. A panic on a JSON route answers
|
||||
500 with the JSON error body. `/status.json` and the health check answer 405
|
||||
with the JSON error body to any method they do not take. The 500 and the 405
|
||||
used to have an empty body. The home page's IP lookup shows the whole JSON
|
||||
answer, `@meta` included. It used to show only the answer's `data`, or only
|
||||
the message of an error answer. A test checks `@meta` on every JSON route, in
|
||||
its successes and its errors (closes #201)
|
||||
- 2026-10-27: Every value with a unit in a JSON answer is a number. The unit is
|
||||
at the end of the field name. `@meta.execution_time` is now
|
||||
`execution_time_ms`. In the stats, `uptime` is now `uptime_seconds`,
|
||||
`connection_duration` is now `connection_duration_seconds`, and `memory_usage`
|
||||
is now `memory_usage_bytes`. `whois_stats.current_interval` is now
|
||||
`whois_stats.current_interval_seconds`. They used to be strings such as "12
|
||||
ms", "1h18m34s" or "1.1 GB". `connection_duration_seconds` is null while the
|
||||
feed is not connected. `API_POLICIES.md` is vendored from `sneak/prompts`, and
|
||||
`REPO_POLICIES.md` is re-vendored (closes #199)
|
||||
- 2026-10-27: Every result of the IP lookup has the same fields. A field without
|
||||
a value is null. It used to be left out. An address with no route in the
|
||||
routing table has every route and AS field set to null. `ptr` is an empty list
|
||||
when there are no PTR names. The new field `num_origins` gives the number of
|
||||
origin ASes of the netblock, and `origin_asns` gives which they are (closes
|
||||
#200)
|
||||
- 2026-10-27: With `TRUSTED_PROXIES` unset, the server trusts reverse proxies at
|
||||
the private addresses of RFC 1918. `/ip/`, the home page's `curl` card and the
|
||||
request log then give the client's address from `X-Forwarded-For`. They used
|
||||
to give the proxy's address. A list in `TRUSTED_PROXIES` replaces the default.
|
||||
`TRUSTED_PROXIES` set but empty trusts no proxy (closes #198)
|
||||
- 2026-10-27: The home page has cards with figures of the routing table. One
|
||||
card shows the newest and the oldest kept route. One shows the newest and the
|
||||
oldest updated AS. One shows the AS with the most prefixes. One shows the ASes
|
||||
with a single prefix. One shows fifteen well-known networks. Two cards count
|
||||
the bogon announcements and the prefixes with more than one origin AS. They
|
||||
link to the new pages `/bogons` and `/multiple-origins`, which list them. A
|
||||
new package, `internal/bogon`, tells which prefixes and AS numbers are
|
||||
reserved. The server works the figures out once a minute (closes #154)
|
||||
- 2026-10-27: The per-peer route tables are gone, with the code that wrote,
|
||||
counted and read them. The prefix handler applies each update to the routing
|
||||
table in memory. It writes the table's changes every second. It also stores
|
||||
each new prefix with its first-seen time. The image's memory limits fit a full
|
||||
routing table of about 1 GB. `GOMEMLIMIT` is 2.5 GiB. The writing connection's
|
||||
page cache is 512 MiB. SQLite's soft and hard heap limits are 768 MiB and 1
|
||||
GiB (closes #122)
|
||||
- 2026-10-27: The pages, the API and the stats read every route from the routing
|
||||
table in memory. For a prefix with several origin ASes, the IP lookup gives
|
||||
the one most peers carry. Its `last_seen` is the newest kept route's time. The
|
||||
prefix page and API list every peer that carries the prefix. Only the kept
|
||||
routes show an AS path, a next hop and a time. The stats count the origins
|
||||
whose kept routes were all withdrawn, in `origins_without_kept_route` (part of
|
||||
#122)
|
||||
- 2026-10-27: The prefix, AS and peer handlers make a timed write on every tick
|
||||
of their timers. The prefix handler's timer ticks every second. The other two
|
||||
tick every two seconds. They used to skip every other tick. A full batch's
|
||||
write restarts the timer. So the next timed write comes one interval after
|
||||
that write began (closes #192)
|
||||
- 2026-10-27: Once the stop deadline has passed, the AS, peer and peering
|
||||
handlers start no write either. Before, a full batch or a timer tick could
|
||||
wait while a write ran past the deadline. Either could then start one more
|
||||
write. The flush goroutines of the prefix, AS and peer handlers now run the
|
||||
same code (closes #189)
|
||||
- 2026-10-27: Once the stop deadline has passed, the prefix handler starts no
|
||||
flush. Before, a batch that filled during a late write could still be written
|
||||
after the deadline (closes #188)
|
||||
- 2026-10-27: The prefix handler applies each update to the routing table in
|
||||
memory. It writes the table's changes to the database every second. A write
|
||||
takes at most 20,000 changed pairs of prefix and origin AS. The rest follow in
|
||||
the next writes. A failed write is written again later. Startup reads the
|
||||
table before the streamer connects. The pages still read the per-peer route
|
||||
tables (part of #122)
|
||||
- 2026-10-27: The database keeps a copy of the whole routing table in a new
|
||||
`routes` table, one row per prefix and origin AS. The peers' numbers are in a
|
||||
new `route_peers` table. One function writes the pairs that changed in memory.
|
||||
Another reads the whole table back into memory at startup. Nothing uses them
|
||||
yet (part of #122)
|
||||
- 2026-10-27: A new package, `internal/routetable`, holds the whole routing
|
||||
table in memory. For each prefix and origin AS it keeps the peers that carry
|
||||
it, one bit per peer. It also keeps the two newest of their routes. Nothing
|
||||
uses it yet (part of #122)
|
||||
- 2026-10-27: Each result of the IP lookup lists the public blocklists the
|
||||
address is on, in `blocklists`. The lists are DShield, Feodo Tracker, two
|
||||
FireHOL lists, two Spamhaus DROP lists and the Tor exit list. RouteWatch
|
||||
downloads each list on its own schedule and answers from copies in memory. The
|
||||
copies are also kept in the state directory's `blocklists` directory. IP
|
||||
lookup answers carry `Cache-Control`. An address with no route now has a
|
||||
result too, without route fields. So the IP lookup no longer answers 404
|
||||
(closes #158)
|
||||
- 2026-10-27: The home page has an IP information API card. It shows the `curl`
|
||||
command that looks up the visitor's own address at `/ip/{addr}`. It also shows
|
||||
that command's live answer (closes #156)
|
||||
- 2026-10-27: On the AS page, a click on the heading of the IPv4 or the IPv6
|
||||
prefix list folds or unfolds the list. Both lists start open (closes #155)
|
||||
- 2026-10-27: The AS page lists its IPv4 and its IPv6 prefixes by mask length,
|
||||
shortest first. Prefixes of one mask length are listed by address. They used
|
||||
to be sorted by address first, then by mask length (closes #153)
|
||||
- 2026-10-27: The connection status on `/` and `/status` gets a spinner. It
|
||||
turns one frame each time the stats show more messages than the time before.
|
||||
After three refreshes with no new messages, the page shows "stale" in red
|
||||
instead. Next to it is the time since the last fresh answer (closes #152)
|
||||
- 2026-10-27: The WHOIS details of a Brazilian AS keep their accented letters.
|
||||
LACNIC's answer for such an AS is NIC.br's answer with each accented letter
|
||||
replaced. NIC.br is Brazil's registry. The details now come from NIC.br's own
|
||||
server, `whois.registro.br`. When that query fails, the details from LACNIC's
|
||||
answer are kept and a warning is logged. RouteWatch stops asking NIC.br for an
|
||||
hour after NIC.br says RouteWatch is over its query limit (closes #174)
|
||||
- 2026-10-27: `make asupdate` stops with an error when the download of `as.csv`
|
||||
breaks off. The generator used to retry the failed read forever. A record that
|
||||
does not parse is still logged and skipped (closes #177)
|
||||
- 2026-10-27: `make asupdate` runs the new `script/asupdate`. It replaces the AS
|
||||
list built into RouteWatch only when the generator succeeds and finds at least
|
||||
one AS. A failed download used to replace the list with an empty file. The
|
||||
generator reads `as.csv` from `ipverse/as-metadata`, the source's new name
|
||||
(closes #173)
|
||||
- 2026-10-27: The WHOIS details of an AS at RIPE, APNIC or AFRINIC come only
|
||||
from its `aut-num` object and the organisation object it names. RIPE and APNIC
|
||||
start their answer with the `as-block` object of the AS's block. Its dates
|
||||
were stored as the AS's (closes #168)
|
||||
- 2026-10-27: AS names keep their accented letters. The AS list built into
|
||||
RouteWatch is rebuilt from today's `as.csv`, which added a country code
|
||||
column. That file no longer lists AS 0 or the documentation AS numbers. A
|
||||
WHOIS answer that is not valid UTF-8 is decoded as Latin-1 before it is parsed
|
||||
and stored. RIPE and APNIC send Latin-1 (closes #165)
|
||||
- 2026-10-27: The Age column on the AS and prefix pages reads in words, such as
|
||||
"5 minutes ago" or "2 weeks ago". It used to print Go durations such as
|
||||
"360h0m0s ago". An age under a minute reads "now" or in seconds. An age over a
|
||||
month reads in months or years instead of a date (closes #163)
|
||||
- 2026-10-26: A write to a stored AS, peering or peer keeps the earlier of the
|
||||
two first-seen times. It keeps the later of the two last-seen times. An old
|
||||
update from a collector catching up on a backlog no longer moves last seen
|
||||
back (closes #162)
|
||||
- 2026-10-26: The status page shows "Route Churn/min". That is the number of
|
||||
prefix announcements and withdrawals from all peers in the last full clock
|
||||
minute. `/api/v1/stats` reports it as `route_churn_per_min` in `stream`. It
|
||||
replaces the moving average `route_churn_per_sec`. The messages and updates
|
||||
per second on `/` and `/status` are whole numbers (closes #160)
|
||||
- 2026-10-26: The WHOIS fetcher backs off more slowly. A failure makes its wait
|
||||
1.5 times as long, up to one minute instead of five. Its queries to RIPE,
|
||||
APNIC and AFRINIC leave out person and role objects. RIPE counts those objects
|
||||
against a daily limit per address. A refused query is logged with the RIR's
|
||||
error line (closes #159)
|
||||
- 2026-10-26: The home page's AS name search works. It lists the ASes whose
|
||||
handle or description holds the text, as links to their pages. The new
|
||||
`/api/v1/search/as` answers it. The search ignores case. When the database has
|
||||
no record of an AS, its page takes the handle and description from the AS list
|
||||
built into RouteWatch. It still lists the AS's prefixes (closes #151)
|
||||
- 2026-10-19: README "HTTP Endpoints" lists every route with what it answers.
|
||||
README "Database Schema" matches `schema.sql`. The rest of the README matches
|
||||
the code. This file is brought up to date (closes #72)
|
||||
- 2026-10-19: Route writes are cheaper. The live route and prefix tables lose
|
||||
their random UUID keys. The prefix tables lose their duplicate index. A route
|
||||
update sets only the columns that change. Every write goes through one
|
||||
connection with a page cache of 1 GiB (README "Memory"). A benchmark writes
|
||||
the prefix handler's largest batch faster than the feed sends it (closes #96)
|
||||
- 2026-10-19: The checkpoint of the WAL (SQLite's write-ahead log), the
|
||||
incremental vacuum and `ANALYZE` take the database lock that every batch write
|
||||
holds. So a vacuum no longer fails with "database is locked". An `ANALYZE` no
|
||||
longer makes a batch write lose its updates. `ANALYZE` runs as
|
||||
`PRAGMA optimize` with an analysis limit (closes #94)
|
||||
- 2026-10-19: Shutdown stops the streamer first. Each handler then writes
|
||||
everything left on its queue before it stops. So the updates waiting there are
|
||||
no longer lost (closes #69)
|
||||
- 2026-10-19: A path that ends in an AS set of several members has origin AS 0,
|
||||
which means no known origin. It used to have the set's last member as its
|
||||
origin. The set adds no peerings. The pages show such a route as having no
|
||||
known origin (closes #75)
|
||||
- 2026-10-19: `bgp_peer_count` drops to 0 when the last peer goes down (closes
|
||||
#92)
|
||||
- 2026-10-19: `bgp_peer_count` counts every peer the feed has heard from. It
|
||||
used to count only the peers whose session opened since startup. It drops a
|
||||
peer whose session goes down (closes #74)
|
||||
- 2026-10-19: The shared files are re-vendored from `sneak/prompts` at
|
||||
`0653579`. The linter is `golangci-lint` v2.14.0 with its configuration. Lint
|
||||
and test are `Dockerfile` phases that keep Go's and `golangci-lint`'s caches.
|
||||
`make fmt` also formats the markdown with `prettier`. The agent guidance moves
|
||||
to `AGENTS.md`. The code is fixed for what the new linter configuration finds
|
||||
(closes #52)
|
||||
- 2026-10-19: The streamer reads feed lines up to 1 MiB. A line over 64 KiB used
|
||||
to end the connection (closes #76)
|
||||
- 2026-10-19: A peer's live routes are removed when its BGP session goes down.
|
||||
`STATE` messages are never dropped when a queue is full. One that finds its
|
||||
queue full waits for room ahead of the handler's other messages (closes #56)
|
||||
- 2026-10-19: The prefix-length pages pick their sample with a bounded number of
|
||||
index lookups. They used to read every route of that length. They list each
|
||||
prefix once (closes #66)
|
||||
- 2026-10-19: A request past its time limit answers 504 instead of 408.
|
||||
`/api/v1/stats` answers 504 instead of 500 when its read runs out of time
|
||||
(closes #70)
|
||||
- 2026-10-19: The AS, peer and prefix handlers write each batch outside the lock
|
||||
their message handling needs. So messages keep coming off their queues during
|
||||
a write. Meanwhile the next batch grows to at most ten batch sizes (closes
|
||||
#65)
|
||||
- 2026-10-19: A live route is identified by prefix and peer. So an announcement
|
||||
with a new origin AS replaces the peer's route. A withdrawal removes it
|
||||
whatever its origin AS (closes #55)
|
||||
- 2026-10-14: The streamer reconnects when RIS Live ends the stream or sends
|
||||
nothing for 60 seconds. The health check reports the feed disconnected
|
||||
meanwhile (closes #53)
|
||||
- 2026-10-14: The prefix page takes the names of the origin ASes from the prefix
|
||||
query. It used to read every route of each origin AS (closes #67)
|
||||
- 2026-10-14: The daemon exits with status 1 when its web server cannot listen
|
||||
on its port. It used to run on without a web server (closes #73)
|
||||
- 2026-10-14: The prefix pages parse the prefix before looking it up. One that
|
||||
does not parse answers 400. Any valid way of writing a prefix finds its routes
|
||||
(closes #71)
|
||||
- 2026-10-14: The IP lookup ignores routes shorter than /8 for IPv4 and /16 for
|
||||
IPv6. So a default route is no longer named as an address's owner (closes #57)
|
||||
- 2026-10-14: A prefix batch keeps each peer's last update for a prefix. It used
|
||||
to keep one update per prefix across all peers (closes #54)
|
||||
- 2026-10-14: `reconnect_count` in `/api/v1/stats` counts every reconnect
|
||||
(closes #68)
|
||||
- 2026-10-03: `/api/v1/stats` serves the prefix distribution from memory. The
|
||||
distribution is seeded at startup and adjusted on every live-route write. So a
|
||||
request no longer reads every live route (closes #30)
|
||||
- 2026-10-03: A new database is created with `auto_vacuum` set to incremental.
|
||||
The setting goes in the connection string, so it is set before the file is
|
||||
first written. Each periodic incremental vacuum now returns up to 1000 free
|
||||
pages. The late `PRAGMA auto_vacuum` in `Initialize`, which SQLite ignored, is
|
||||
gone (closes #43)
|
||||
- 2026-10-03: Looking up an IP address no longer reads every IPv6 route. Both
|
||||
families find the most specific live route with at most 33 or 129 lookups on
|
||||
the prefix index. The IPv4 range columns are gone. Prefixes from the feed are
|
||||
stored in one text form. So an IPv6 withdrawal, which the feed sends
|
||||
uncompressed, now removes its route (closes #48)
|
||||
- 2026-10-02: A plain `docker build .` stamps the commit's tag or short commit
|
||||
into the page footer. It used to stamp `unknown`. The value comes from
|
||||
`git describe --tags --always`. `.dockerignore` sends `.git` without
|
||||
`.git/config`. A `VERSION` build argument takes precedence. `make build`
|
||||
stamps the same value (closes #46)
|
||||
- 2026-09-29: The entrypoint creates the data directory if it is missing. It
|
||||
stops the start if a step fails. README "Running under upaas" no longer asks
|
||||
for the host directory to be created first (closes #42)
|
||||
- 2026-09-29: `.dockerignore` keeps `.git`, local build output, local databases
|
||||
and `.env` out of the Docker build context. So they stay out of the source
|
||||
archive in the image (closes #39)
|
||||
- 2026-09-29: The README's first line names the MIT licence and the author. The
|
||||
License section says MIT and links `LICENSE`. An Author section is added. This
|
||||
file is brought up to date (closes #38)
|
||||
- 2026-09-29: `LICENSE` is added, with the MIT licence (closes #1)
|
||||
- 2026-09-28: Stopping the daemon while the feed is flowing no longer panics
|
||||
with "send on closed channel". The read loop checks for a stop just before
|
||||
handing a message to the handler queues. A second `Stop` no longer closes the
|
||||
queues again (closes #34)
|
||||
- 2026-09-28: `docker stop` no longer kills the daemon 2 seconds after the stop
|
||||
signal. The entrypoint switches to the `routewatch` user with `setpriv`
|
||||
instead of `runuser`. So the daemon receives the signal itself. It gets the
|
||||
whole wait `docker stop` allows, up to its own 60-second limit (closes #33)
|
||||
- 2026-09-28: RouteWatch is ready to run under upaas. A set but invalid `PORT`,
|
||||
`XDG_DATA_HOME` or `MALLOC_ARENA_MAX` stops the start. The health check
|
||||
follows `PORT`. README gains a "Running under upaas" section (closes #31)
|
||||
- 2026-09-22: The database statistics are kept in memory. The counts are seeded
|
||||
at startup and adjusted on every write. The oldest and newest route times come
|
||||
from a lookup at each end of an index. `/api/v1/stats` no longer scans the
|
||||
tables (closes #27)
|
||||
- 2026-09-21: Batch writes take the write lock when their transaction begins
|
||||
(`_txlock=immediate`). So they wait out a WAL checkpoint instead of failing
|
||||
with "database is locked" (closes #25)
|
||||
- 2026-09-21: `MALLOC_ARENA_MAX=2` in the image caps the number of glibc malloc
|
||||
arenas. So memory outside the Go runtime no longer grows with the core count
|
||||
(closes #23)
|
||||
- 2026-09-21: The image sets `GOMEMLIMIT=1536MiB`. README "Memory" gives the
|
||||
memory budget and the 5 GiB container limit (closes #13)
|
||||
- 2026-09-21: The four handler queues hold at most 20,000 messages each, down
|
||||
from 100,000 (closes #11)
|
||||
- 2026-09-21: Two goroutine leaks are fixed. The stats handlers leaked after a
|
||||
timeout. The streamer's tickers leaked on every reconnect (closes #12)
|
||||
- 2026-09-21: Parsed RIS messages no longer keep the unused `Community` and
|
||||
`Raw` fields (closes #9)
|
||||
- 2026-09-21: `.editorconfig` is added. A Gitea workflow runs `script/cibuild`
|
||||
on every push (closes #14)
|
||||
- 2026-09-21: The peering handler's AS-path map holds at most 500,000 paths.
|
||||
Every 30 seconds it is swapped for an empty one instead of copied (closes #10)
|
||||
- 2026-09-21: SQLite's memory is bounded across the whole connection pool. Each
|
||||
connection has a 64 MiB page cache. The heap limits are 1 GiB soft and 1.5 GiB
|
||||
hard (closes #8)
|
||||
- 2026-09-21: The Docker build runs the format check, the linter and the tests.
|
||||
Linting runs in a separate stage on a `golangci-lint` image pinned by digest
|
||||
(closes #5)
|
||||
- 2026-09-21: `make test` skips the live feed test, with `-short`. So
|
||||
`make check` no longer depends on the network (closes #2)
|
||||
- 2026-07-07: The scripts-to-rule-them-all layout is adopted. `script/` holds
|
||||
the entrypoints. The `Makefile` targets `check`, `test`, `lint`, `fmt` and
|
||||
`fmt-check` call them. So do `bootstrap`, `setup`, `docker` and `hooks`.
|
||||
README gains an "Entrypoints" section
|
||||
- 2026-02-22: The repo is brought in line with the repo policies. The required
|
||||
policy files are added. `.gitignore` is updated. The `Makefile` gains
|
||||
`fmt-check`, `check`, `docker` and `hooks` targets. The Go code is formatted
|
||||
with `gofmt`. The work was on the branch `repo-policies-compliance`, unmerged
|
||||
when this entry was written
|
||||
- 2026-01-01: The WAL is checkpointed every 5 seconds, and at startup with a log
|
||||
line. Checkpoints run in `TRUNCATE` mode. This fixed slow queries
|
||||
- 2025-12-31: The pages get a navigation bar and a home page with a search. The
|
||||
status page shows the oldest and newest route times. The WHOIS stats query
|
||||
handles `NULL`. The container runs as the `routewatch` user with the right
|
||||
state directory
|
||||
- 2025-12-30: HTTP requests are logged as structured log lines. The HTTP
|
||||
timeouts are longer. Prefix URLs with a slash in them route correctly. The
|
||||
status page gets more metrics and a footer
|
||||
- 2025-12-29: The `Dockerfile` builds in stages and puts a source archive in the
|
||||
image. SQLite reclaims space with incremental vacuum, which does not block
|
||||
- 2025-07-27: The first commit: RouteWatch, a BGP stream monitor
|
||||
|
||||
# Future Steps
|
||||
@@ -9,7 +9,7 @@ require (
|
||||
github.com/google/btree v1.1.3
|
||||
github.com/google/uuid v1.6.0
|
||||
github.com/hashicorp/golang-lru/v2 v2.0.7
|
||||
github.com/mattn/go-sqlite3 v1.14.53
|
||||
github.com/mattn/go-sqlite3 v1.14.29
|
||||
github.com/maxmind/mmdbwriter v1.2.0
|
||||
github.com/miekg/dns v1.1.73
|
||||
github.com/oschwald/maxminddb-golang/v2 v2.9.0
|
||||
|
||||
@@ -10,8 +10,8 @@ github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
|
||||
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
|
||||
github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k=
|
||||
github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM=
|
||||
github.com/mattn/go-sqlite3 v1.14.53 h1:mnTM0owzUMrKuLo+0w51NUiB0ir2Fo7kFXlPt/l3BGs=
|
||||
github.com/mattn/go-sqlite3 v1.14.53/go.mod h1:6JTjA44L93a0QCyJef5YvlPoKXntQPjzWv5gtm9sB6w=
|
||||
github.com/mattn/go-sqlite3 v1.14.29 h1:1O6nRLJKvsi1H2Sj0Hzdfojwt8GiGKm+LOfLaBFaouQ=
|
||||
github.com/mattn/go-sqlite3 v1.14.29/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y=
|
||||
github.com/maxmind/mmdbwriter v1.2.0 h1:hyvDopImmgvle3aR8AaddxXnT0iQH2KWJX3vNfkwzYM=
|
||||
github.com/maxmind/mmdbwriter v1.2.0/go.mod h1:EQmKHhk2y9DRVvyNxwCLKC5FrkXZLx4snc5OlLY5XLE=
|
||||
github.com/miekg/dns v1.1.73 h1:uhT8nJxmTrPJYClxVxTCX+CVn6qnzSiybRk72Z6DgrE=
|
||||
|
||||
@@ -90,12 +90,14 @@ func parseAddresses(t *testing.T, addresses ...string) []netip.Addr {
|
||||
return addrs
|
||||
}
|
||||
|
||||
// fakeGeoJS is a geojs server. It sends body, or status when that is set. It
|
||||
// keeps each request's address list.
|
||||
// fakeGeoJS is a geojs server. It sends body, or status when that is set, or
|
||||
// waits for the request to end when hang is true. It keeps each request's
|
||||
// address list.
|
||||
type fakeGeoJS struct {
|
||||
server *httptest.Server
|
||||
body string
|
||||
status int
|
||||
hang bool
|
||||
|
||||
mu sync.Mutex
|
||||
requests []string
|
||||
@@ -117,6 +119,12 @@ func (f *fakeGeoJS) serve(w http.ResponseWriter, r *http.Request) {
|
||||
f.requests = append(f.requests, r.URL.Query().Get("ip"))
|
||||
f.mu.Unlock()
|
||||
|
||||
if f.hang {
|
||||
<-r.Context().Done()
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
if f.status != 0 {
|
||||
w.WriteHeader(f.status)
|
||||
|
||||
@@ -134,20 +142,6 @@ func (f *fakeGeoJS) requestsSeen() []string {
|
||||
return append([]string{}, f.requests...)
|
||||
}
|
||||
|
||||
// hangingTransport never answers: each request waits until its context ends.
|
||||
// It counts the requests as they are sent, so the count does not depend on
|
||||
// whether one reaches a server before its time limit runs out.
|
||||
type hangingTransport struct {
|
||||
requests atomic.Int32
|
||||
}
|
||||
|
||||
func (h *hangingTransport) RoundTrip(r *http.Request) (*http.Response, error) {
|
||||
h.requests.Add(1)
|
||||
<-r.Context().Done()
|
||||
|
||||
return nil, r.Context().Err()
|
||||
}
|
||||
|
||||
// openCacheDB opens a new cache.db in a temporary directory.
|
||||
func openCacheDB(t *testing.T, dir string) *sql.DB {
|
||||
t.Helper()
|
||||
@@ -391,15 +385,13 @@ func TestLookupFallsBackToTheDatabase(t *testing.T) {
|
||||
|
||||
fake := newFakeGeoJS(t, tt.body)
|
||||
fake.status = tt.status
|
||||
fake.hang = tt.hang
|
||||
dir := t.TempDir()
|
||||
locator := newTestLocator(t, fake, openCacheDB(t, dir), dir)
|
||||
requestCount := func() int { return len(fake.requestsSeen()) }
|
||||
locator.timeLimit = shortTimeLimit
|
||||
|
||||
if tt.hang {
|
||||
hanging := &hangingTransport{}
|
||||
locator.client = &http.Client{Transport: hanging}
|
||||
locator.timeLimit = shortTimeLimit
|
||||
requestCount = func() int { return int(hanging.requests.Load()) }
|
||||
if !tt.hang {
|
||||
locator.timeLimit = testTimeLimit
|
||||
}
|
||||
|
||||
giveDatabase(t, locator)
|
||||
@@ -421,8 +413,8 @@ func TestLookupFallsBackToTheDatabase(t *testing.T) {
|
||||
// asks geojs again.
|
||||
lookupOne(t, locator, testAddress)
|
||||
|
||||
if requests := requestCount(); requests != 2 {
|
||||
t.Errorf("%d requests, want 2", requests)
|
||||
if requests := fake.requestsSeen(); len(requests) != 2 {
|
||||
t.Errorf("%d requests, want 2", len(requests))
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
@@ -21,7 +21,7 @@ const (
|
||||
downloadTimeout = time.Minute
|
||||
|
||||
// maxDownloadSize is the size of the largest list file accepted, in
|
||||
// bytes. The largest list, iCloud Private Relay, is about 12 MiB.
|
||||
// bytes. The largest list, FireHOL abusers, is about 2.5 MiB.
|
||||
maxDownloadSize = 200 << 20
|
||||
)
|
||||
|
||||
|
||||
@@ -56,11 +56,6 @@ const (
|
||||
// that start with #. Unlike FormatLines, a file with no addresses or
|
||||
// networks fails.
|
||||
FormatProtonVPN
|
||||
|
||||
// FormatICloudPrivateRelayCSV is Apple's CSV file of iCloud Private Relay
|
||||
// networks. Each line starts with a network, followed by its country,
|
||||
// region and city.
|
||||
FormatICloudPrivateRelayCSV
|
||||
)
|
||||
|
||||
// List is one public IP list.
|
||||
|
||||
@@ -83,18 +83,6 @@ const protonVPNEntryBody = "#\n# protonvpn_entry_ips.txt\n" +
|
||||
"# Entry IPs clients connect to, resolved from ProtonVPN hostnames.\n#\n" +
|
||||
"192.0.2.80\n2001:db8::80\n"
|
||||
|
||||
// iCloudPrivateRelayAddr is an address on iCloudPrivateRelayBody.
|
||||
const iCloudPrivateRelayAddr = "2001:db8:1::5"
|
||||
|
||||
// iCloudPrivateRelayBody is a small file in the format of Apple's iCloud
|
||||
// Private Relay file.
|
||||
const iCloudPrivateRelayBody = "192.0.2.0/31,GB,GB-EN,London,\n" +
|
||||
"198.51.100.32/27,US,US-CA,Los Angeles,\n" +
|
||||
"2001:db8:1::/64,US,US-AK,Anchorage,\n"
|
||||
|
||||
// emptyFileCase names the case of a wrong document that is an empty file.
|
||||
const emptyFileCase = "an empty file"
|
||||
|
||||
// errorPage is a file that is no list, as a publisher's error page is.
|
||||
const errorPage = "<!DOCTYPE html>\n<html></html>\n"
|
||||
|
||||
@@ -255,15 +243,6 @@ func TestLookupFindsListedAddresses(t *testing.T) {
|
||||
},
|
||||
unlisted: []string{"192.0.2.12", "2001:db8::2"},
|
||||
},
|
||||
{
|
||||
name: "iCloud Private Relay networks",
|
||||
format: FormatICloudPrivateRelayCSV,
|
||||
body: iCloudPrivateRelayBody,
|
||||
listed: []string{
|
||||
"192.0.2.1", "198.51.100.63", iCloudPrivateRelayAddr,
|
||||
},
|
||||
unlisted: []string{"192.0.2.2", "198.51.100.64", "2001:db8:2::1"},
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
@@ -506,12 +485,12 @@ func TestWrongVPNDocumentFailsToParse(t *testing.T) {
|
||||
|
||||
for _, f := range formats {
|
||||
bodies := map[string]string{
|
||||
"not JSON": errorPage,
|
||||
"other JSON": `{"status":"ok","data":{"count":1}}`,
|
||||
"no addresses": f.empty,
|
||||
"bad address": f.badAddress,
|
||||
"cut off": f.good[:len(f.good)/2],
|
||||
emptyFileCase: "",
|
||||
"not JSON": errorPage,
|
||||
"other JSON": `{"status":"ok","data":{"count":1}}`,
|
||||
"no addresses": f.empty,
|
||||
"bad address": f.badAddress,
|
||||
"cut off": f.good[:len(f.good)/2],
|
||||
"an empty file": "",
|
||||
}
|
||||
|
||||
for name, body := range bodies {
|
||||
@@ -537,7 +516,7 @@ func TestWrongProtonVPNDocumentFailsToParse(t *testing.T) {
|
||||
"JSON": `{"status":"ok"}`,
|
||||
"comments only": "#\n# protonvpn_ip_ranges.txt\n#\n",
|
||||
"bad network": "192.0.2.0/24\nexample\n",
|
||||
emptyFileCase: "",
|
||||
"an empty file": "",
|
||||
}
|
||||
|
||||
for name, body := range bodies {
|
||||
@@ -552,34 +531,6 @@ func TestWrongProtonVPNDocumentFailsToParse(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// TestWrongICloudPrivateRelayDocumentFailsToParse parses documents that are
|
||||
// not Apple's iCloud Private Relay file. Each fails, so none can replace a
|
||||
// good copy.
|
||||
func TestWrongICloudPrivateRelayDocumentFailsToParse(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
bodies := map[string]string{
|
||||
"error page": errorPage,
|
||||
"JSON": `{"status":"ok","count":1}`,
|
||||
"no other field": "192.0.2.0/24\n",
|
||||
"bad network": "192.0.2.0/24,GB,GB-EN,London,\nexample,GB,GB-EN,London,\n",
|
||||
"an address": "192.0.2.1,GB,GB-EN,London,\n",
|
||||
emptyFileCase: "",
|
||||
}
|
||||
|
||||
for name, body := range bodies {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
addresses, err := parse(strings.NewReader(body),
|
||||
FormatICloudPrivateRelayCSV)
|
||||
if err == nil {
|
||||
t.Errorf("parse returned %v and no error", addresses.Ranges())
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestLookupMarksVPNLists looks up an address on a VPN list and on another
|
||||
// list. Only the VPN list's listing says it is one.
|
||||
func TestLookupMarksVPNLists(t *testing.T) {
|
||||
@@ -934,33 +885,15 @@ func TestStopCancelsADownloadInProgress(t *testing.T) {
|
||||
func TestTorExitsListHoldsIPv6Exits(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
checkPublicListHolds(t, TorExitsList, onionooBody, onionooIPv6ExitAddr)
|
||||
}
|
||||
|
||||
// TestICloudPrivateRelayListHoldsItsNetworks downloads a file in Apple's
|
||||
// format as the public iCloud Private Relay list, which is what an address's
|
||||
// is_relay answer comes from.
|
||||
func TestICloudPrivateRelayListHoldsItsNetworks(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
checkPublicListHolds(t, ICloudPrivateRelayList, iCloudPrivateRelayBody,
|
||||
iCloudPrivateRelayAddr)
|
||||
}
|
||||
|
||||
// checkPublicListHolds downloads body as the public list called name and
|
||||
// checks that addr has one listing, which names that list.
|
||||
func checkPublicListHolds(t *testing.T, name, body, addr string) {
|
||||
t.Helper()
|
||||
|
||||
index := slices.IndexFunc(Lists(), func(list List) bool {
|
||||
return list.Name == name
|
||||
return list.Name == TorExitsList
|
||||
})
|
||||
if index < 0 {
|
||||
t.Fatalf("no list named %s", name)
|
||||
t.Fatalf("no list named %s", TorExitsList)
|
||||
}
|
||||
|
||||
list := Lists()[index]
|
||||
_, list.URL = newPublisher(t, body)
|
||||
_, list.URL = newPublisher(t, onionooBody)
|
||||
b := newTestIPLists(t.TempDir(), list)
|
||||
|
||||
err := b.download(t.Context(), list)
|
||||
@@ -968,9 +901,9 @@ func checkPublicListHolds(t *testing.T, name, body, addr string) {
|
||||
t.Fatalf("download: %v", err)
|
||||
}
|
||||
|
||||
listings := lookup(t, b, addr)
|
||||
if len(listings) != 1 || listings[0].Name != name {
|
||||
t.Errorf("listings %+v, want one for %s", listings, name)
|
||||
listings := lookup(t, b, onionooIPv6ExitAddr)
|
||||
if len(listings) != 1 || listings[0].Name != TorExitsList {
|
||||
t.Errorf("listings %+v, want one for %s", listings, TorExitsList)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -8,10 +8,6 @@ import (
|
||||
// TorExitsList is the name of the list of Tor exit nodes.
|
||||
const TorExitsList = "tor_exits"
|
||||
|
||||
// ICloudPrivateRelayList is the name of the list of iCloud Private Relay's
|
||||
// networks.
|
||||
const ICloudPrivateRelayList = "icloud_private_relay"
|
||||
|
||||
// protonVPNIPsURL is where the ProtonVPN-IPs mirror publishes its files.
|
||||
const protonVPNIPsURL = "https://raw.githubusercontent.com/tn3w/ProtonVPN-IPs/master/"
|
||||
|
||||
@@ -34,9 +30,6 @@ const (
|
||||
fireHOLAbusersInterval = 5 * time.Hour
|
||||
fireHOLLevel3Interval = 3 * time.Hour
|
||||
|
||||
// Apple's iCloud Private Relay file is 12 MB, so once a day.
|
||||
iCloudPrivateRelayInterval = 24 * time.Hour
|
||||
|
||||
// Spamhaus asks that DROP be downloaded at most once a day:
|
||||
// https://www.spamhaus.org/blocklists/do-not-route-or-peer/
|
||||
spamhausDROPInterval = 24 * time.Hour
|
||||
@@ -91,13 +84,6 @@ func nonVPNLists() []List {
|
||||
Interval: fireHOLLevel3Interval,
|
||||
Format: FormatLines,
|
||||
},
|
||||
{
|
||||
Name: ICloudPrivateRelayList,
|
||||
Description: "In a network where iCloud Private Relay traffic leaves",
|
||||
URL: "https://mask-api.icloud.com/egress-ip-ranges.csv",
|
||||
Interval: iCloudPrivateRelayInterval,
|
||||
Format: FormatICloudPrivateRelayCSV,
|
||||
},
|
||||
{
|
||||
Name: "spamhaus_drop",
|
||||
Description: "In an IPv4 network that Spamhaus says spammers " +
|
||||
|
||||
@@ -25,10 +25,6 @@ const (
|
||||
// dshieldFieldSeparator separates the fields of a DShield line.
|
||||
dshieldFieldSeparator = "\t"
|
||||
|
||||
// iCloudPrivateRelayFieldSeparator separates the fields of a line of
|
||||
// Apple's iCloud Private Relay file.
|
||||
iCloudPrivateRelayFieldSeparator = ","
|
||||
|
||||
// spamhausMetadataType is the type of the line that ends a Spamhaus
|
||||
// file. That line holds the file's metadata and no network.
|
||||
spamhausMetadataType = "metadata"
|
||||
@@ -63,9 +59,7 @@ func parse(r io.Reader, format Format) (*netipx.IPSet, error) {
|
||||
case FormatIVPNJSON:
|
||||
return parseIVPN(r)
|
||||
case FormatProtonVPN:
|
||||
return parseLinesNotEmpty(r, addAddressOrNetwork)
|
||||
case FormatICloudPrivateRelayCSV:
|
||||
return parseLinesNotEmpty(r, addICloudPrivateRelayNetwork)
|
||||
return parseProtonVPN(r)
|
||||
default:
|
||||
return nil, errUnknownFormat
|
||||
}
|
||||
@@ -337,12 +331,11 @@ func parseIVPN(r io.Reader) (*netipx.IPSet, error) {
|
||||
return addressSet(texts)
|
||||
}
|
||||
|
||||
// parseLinesNotEmpty is parseLines for a list that always holds addresses, so
|
||||
// a file with none fails.
|
||||
func parseLinesNotEmpty(
|
||||
r io.Reader, addLine func(*netipx.IPSetBuilder, string) error,
|
||||
) (*netipx.IPSet, error) {
|
||||
addresses, err := parseLines(r, addLine)
|
||||
// parseProtonVPN returns the addresses or networks in a ProtonVPN-IPs list. A
|
||||
// file with no addresses or networks fails, because the service always has
|
||||
// servers.
|
||||
func parseProtonVPN(r io.Reader) (*netipx.IPSet, error) {
|
||||
addresses, err := parseLines(r, addAddressOrNetwork)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -354,24 +347,6 @@ func parseLinesNotEmpty(
|
||||
return addresses, nil
|
||||
}
|
||||
|
||||
// addICloudPrivateRelayNetwork adds the network at the start of a line of
|
||||
// Apple's iCloud Private Relay file to builder.
|
||||
func addICloudPrivateRelayNetwork(builder *netipx.IPSetBuilder, line string) error {
|
||||
field, _, found := strings.Cut(line, iCloudPrivateRelayFieldSeparator)
|
||||
if !found {
|
||||
return errNoNetwork
|
||||
}
|
||||
|
||||
network, err := netip.ParsePrefix(field)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
builder.AddPrefix(network)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// readJSON reads r to its end and decodes it, as one JSON document, into v.
|
||||
func readJSON(r io.Reader, v any) error {
|
||||
data, err := io.ReadAll(r)
|
||||
|
||||
@@ -801,9 +801,6 @@ type IPLookupFlags struct {
|
||||
// IsVPN says that the address is on a list of VPN servers.
|
||||
IsVPN bool `json:"is_vpn"`
|
||||
|
||||
// IsRelay says that the address is on the iCloud Private Relay list.
|
||||
IsRelay bool `json:"is_relay"`
|
||||
|
||||
// IsBogon says that the address is reserved, as package bogon tells.
|
||||
IsBogon bool `json:"is_bogon"`
|
||||
}
|
||||
@@ -1036,9 +1033,6 @@ func ipLookupFlags(ip string, listings []iplist.Listing) IPLookupFlags {
|
||||
IsVPN: slices.ContainsFunc(listings, func(l iplist.Listing) bool {
|
||||
return l.VPN
|
||||
}),
|
||||
IsRelay: slices.ContainsFunc(listings, func(l iplist.Listing) bool {
|
||||
return l.Name == iplist.ICloudPrivateRelayList
|
||||
}),
|
||||
IsBogon: isBogon(ip),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1506,10 +1506,6 @@ const listedUnroutedAddr = "203.0.113.1"
|
||||
// torExitAddr is the address that fakeIPLists lists on the Tor exit list.
|
||||
const torExitAddr = "203.0.113.2"
|
||||
|
||||
// relayAddr is the address that fakeIPLists lists on the iCloud Private Relay
|
||||
// list.
|
||||
const relayAddr = "203.0.113.5"
|
||||
|
||||
// The addresses that fakeIPLists lists on one and on two VPN lists, and the
|
||||
// names of those lists.
|
||||
const (
|
||||
@@ -1519,11 +1515,10 @@ const (
|
||||
otherTestVPN = "other_test_vpn"
|
||||
)
|
||||
|
||||
// fakeIPLists has five lists. The test list holds addrInTestPrefixV4,
|
||||
// fakeIPLists has four lists. The test list holds addrInTestPrefixV4,
|
||||
// listedUnroutedAddr and twoVPNListsAddr. The Tor exit list holds torExitAddr.
|
||||
// The iCloud Private Relay list holds relayAddr. The test VPN list holds
|
||||
// oneVPNListAddr and twoVPNListsAddr, and the other test VPN list holds
|
||||
// twoVPNListsAddr.
|
||||
// The test VPN list holds oneVPNListAddr and twoVPNListsAddr, and the other
|
||||
// test VPN list holds twoVPNListsAddr.
|
||||
type fakeIPLists struct{}
|
||||
|
||||
func testListing() iplist.Listing {
|
||||
@@ -1544,10 +1539,6 @@ func (fakeIPLists) Lookup(addr netip.Addr) []iplist.Listing {
|
||||
return []iplist.Listing{{Name: iplist.TorExitsList}}
|
||||
}
|
||||
|
||||
if addr == netip.MustParseAddr(relayAddr) {
|
||||
return []iplist.Listing{{Name: iplist.ICloudPrivateRelayList}}
|
||||
}
|
||||
|
||||
if addr == netip.MustParseAddr(oneVPNListAddr) {
|
||||
return []iplist.Listing{{Name: testVPNList, VPN: true}}
|
||||
}
|
||||
@@ -1580,22 +1571,20 @@ func TestIPLookupFlags(t *testing.T) {
|
||||
name string
|
||||
addr string
|
||||
tor bool
|
||||
relay bool
|
||||
bogon bool
|
||||
}{
|
||||
{"on the Tor exit list", torExitAddr, true, false, true},
|
||||
{"on the iCloud Private Relay list", relayAddr, false, true, true},
|
||||
{"on another list", listedUnroutedAddr, false, false, true},
|
||||
{"routed IPv4", addrInTestPrefixV4, false, false, true},
|
||||
{"routed IPv6", addrInTestPrefixV6, false, false, true},
|
||||
{"private IPv4", "10.0.0.1", false, false, true},
|
||||
{"loopback IPv4", "127.0.0.1", false, false, true},
|
||||
{"public IPv4", "1.1.1.1", false, false, false},
|
||||
{"private IPv6", "fd00::1", false, false, true},
|
||||
{"loopback IPv6", "::1", false, false, true},
|
||||
{"public IPv6", "2606:4700:4700::1111", false, false, false},
|
||||
{"public IPv4 in IPv6 form", "::ffff:1.1.1.1", false, false, false},
|
||||
{"private IPv4 in IPv6 form", "::ffff:10.0.0.1", false, false, true},
|
||||
{"on the Tor exit list", torExitAddr, true, true},
|
||||
{"on another list", listedUnroutedAddr, false, true},
|
||||
{"routed IPv4", addrInTestPrefixV4, false, true},
|
||||
{"routed IPv6", addrInTestPrefixV6, false, true},
|
||||
{"private IPv4", "10.0.0.1", false, true},
|
||||
{"loopback IPv4", "127.0.0.1", false, true},
|
||||
{"public IPv4", "1.1.1.1", false, false},
|
||||
{"private IPv6", "fd00::1", false, true},
|
||||
{"loopback IPv6", "::1", false, true},
|
||||
{"public IPv6", "2606:4700:4700::1111", false, false},
|
||||
{"public IPv4 in IPv6 form", "::ffff:1.1.1.1", false, false},
|
||||
{"private IPv4 in IPv6 form", "::ffff:10.0.0.1", false, true},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
@@ -1617,8 +1606,7 @@ func TestIPLookupFlags(t *testing.T) {
|
||||
}
|
||||
|
||||
want := map[string]bool{
|
||||
"is_tor": tt.tor, "is_vpn": false, "is_relay": tt.relay,
|
||||
"is_bogon": tt.bogon,
|
||||
"is_tor": tt.tor, "is_vpn": false, "is_bogon": tt.bogon,
|
||||
}
|
||||
if !maps.Equal(got, want) {
|
||||
t.Errorf("%s: flags %v, want %v", path, got, want)
|
||||
|
||||
@@ -275,66 +275,49 @@ func TestPagesServeTheCopyOfTheCards(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// TestKeepHomeCardsFreshWorksThemOutEveryInterval runs keepHomeCardsFresh. It
|
||||
// works the cards out at once, again an interval after the table changes, and
|
||||
// stops when its context ends. It runs in a synctest bubble, whose clock is
|
||||
// fake, so a stall on the host cannot fail it.
|
||||
// TestKeepHomeCardsFreshWorksThemOutEveryInterval runs keepHomeCardsFresh with
|
||||
// a short interval. It works the cards out at once, again after the table
|
||||
// changes, and stops when its context ends.
|
||||
func TestKeepHomeCardsFreshWorksThemOutEveryInterval(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
const interval = time.Minute
|
||||
const (
|
||||
interval = 10 * time.Millisecond
|
||||
maxWait = 15 * time.Second
|
||||
)
|
||||
|
||||
table := emptyTable()
|
||||
announce(table, "1.0.0.0/24", twoPrefixASN, testPeer(1), testRouteTime(0))
|
||||
|
||||
s := newServerWithTable(t, table)
|
||||
|
||||
// newestRouteOrigin returns the origin AS of the newest route on the
|
||||
// cards, or 0 when there is none.
|
||||
newestRouteOrigin := func() int {
|
||||
cards := s.cards.Load()
|
||||
if cards == nil || cards.NewestRoute == nil {
|
||||
return 0
|
||||
}
|
||||
ctx, cancel := context.WithCancel(t.Context())
|
||||
stopped := make(chan struct{})
|
||||
|
||||
return cards.NewestRoute.OriginASN
|
||||
go func() {
|
||||
s.keepHomeCardsFresh(ctx, interval)
|
||||
close(stopped)
|
||||
}()
|
||||
|
||||
newestRouteASNIs := func(asn int) bool {
|
||||
cards := s.cards.Load()
|
||||
|
||||
return cards != nil && cards.NewestRoute != nil &&
|
||||
cards.NewestRoute.OriginASN == asn
|
||||
}
|
||||
|
||||
synctest.Test(t, func(t *testing.T) {
|
||||
ctx, cancel := context.WithCancel(t.Context())
|
||||
stopped := make(chan struct{})
|
||||
waitUntil(t, maxWait, func() bool { return newestRouteASNIs(twoPrefixASN) })
|
||||
|
||||
go func() {
|
||||
s.keepHomeCardsFresh(ctx, interval)
|
||||
close(stopped)
|
||||
}()
|
||||
announce(table, "8.8.8.0/24", newestRouteASN, testPeer(2), testRouteTime(1))
|
||||
waitUntil(t, maxWait, func() bool { return newestRouteASNIs(newestRouteASN) })
|
||||
|
||||
synctest.Wait()
|
||||
cancel()
|
||||
|
||||
if got := newestRouteOrigin(); got != twoPrefixASN {
|
||||
t.Fatalf("at the start, the newest route's AS is %d, want %d",
|
||||
got, twoPrefixASN)
|
||||
}
|
||||
|
||||
announce(table, "8.8.8.0/24", newestRouteASN, testPeer(2),
|
||||
testRouteTime(1))
|
||||
time.Sleep(interval)
|
||||
synctest.Wait()
|
||||
|
||||
if got := newestRouteOrigin(); got != newestRouteASN {
|
||||
t.Fatalf("an interval after the table changed, the newest "+
|
||||
"route's AS is %d, want %d", got, newestRouteASN)
|
||||
}
|
||||
|
||||
cancel()
|
||||
synctest.Wait()
|
||||
|
||||
select {
|
||||
case <-stopped:
|
||||
default:
|
||||
t.Fatal("keepHomeCardsFresh did not stop when its context ended")
|
||||
}
|
||||
})
|
||||
select {
|
||||
case <-stopped:
|
||||
case <-time.After(maxWait):
|
||||
t.Fatal("keepHomeCardsFresh did not stop when its context ended")
|
||||
}
|
||||
}
|
||||
|
||||
// TestStartedServerWorksOutTheCards starts a server and waits until its home
|
||||
|
||||
Reference in new issue
Block a user