Compare commits
1
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
 sneak
|
dfc3945bc0 |
No files matched your search
+1
-1
@@ -105,7 +105,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
|
||||
ARG CACHEBUST
|
||||
|
||||
# Create non-root user
|
||||
RUN useradd -u 1000 -m routewatch
|
||||
RUN useradd -r -u 1000 -m routewatch
|
||||
|
||||
RUN mkdir -p /var/lib/berlin.sneak.app.routewatch && chown routewatch:routewatch /var/lib/berlin.sneak.app.routewatch
|
||||
|
||||
|
||||
@@ -143,9 +143,9 @@ and origin AS can have two kept routes, one or none.
|
||||
AS path, a next hop and a time. The next hop is the router the peer sends the
|
||||
prefix's traffic to. RouteWatch records the peer itself as the next hop. The
|
||||
page also lists the prefix's origin changes, newest first. An origin change is
|
||||
recorded when the first peer starts to carry the prefix with an origin AS, or
|
||||
when the last peer stops. The page shows at most the newest 5,000. Each row
|
||||
shows:
|
||||
an AS starting to announce the prefix, when its first peer carries it, or
|
||||
stopping, when its last peer no longer does. The page shows at most the newest
|
||||
5,000. Each row shows:
|
||||
- the time
|
||||
- announced or withdrawn
|
||||
- the origin AS
|
||||
@@ -197,9 +197,8 @@ also carries an `@meta` object:
|
||||
- `api_version`: `1`
|
||||
- `execution_time_ms`: the number of milliseconds the request took
|
||||
|
||||
Every value with a unit is a number. The unit ends the field name, as in `_ms`,
|
||||
`_seconds`, `_bytes`, `_per_sec` and `_per_min`. `latitude` and `longitude` are
|
||||
in degrees, and their names do not say so.
|
||||
Every value with a unit is a number, and the unit ends the field name. Examples
|
||||
are `_ms`, `_seconds`, `_bytes`, `_per_sec` and `_per_min`.
|
||||
|
||||
The paths:
|
||||
|
||||
@@ -288,7 +287,6 @@ address. Every entry has the same fields:
|
||||
- `is_tor`: true when the address is on the `tor_exits` list
|
||||
- `is_vpn`: true when the address is on any list of VPN servers (see IP
|
||||
lists)
|
||||
- `is_relay`: true when the address is on the `icloud_private_relay` list
|
||||
- `is_bogon`: true when the address is reserved, so that no network should
|
||||
announce it on the internet
|
||||
- `vpn_providers`: the names of the lists of VPN servers the address is on, or
|
||||
@@ -424,9 +422,6 @@ The lists:
|
||||
last 30 days. From
|
||||
[FireHOL](https://iplists.firehol.org/?ipset=firehol_level3), downloaded every
|
||||
3 hours.
|
||||
- `icloud_private_relay`: the networks where iCloud Private Relay traffic
|
||||
leaves. From [Apple](https://mask-api.icloud.com/egress-ip-ranges.csv),
|
||||
downloaded once a day.
|
||||
- `ivpn`: VPN servers run by IVPN. From
|
||||
[IVPN](https://api.ivpn.net/v5/servers.json), downloaded once a day.
|
||||
- `mullvad`: VPN servers run by Mullvad. From
|
||||
@@ -503,13 +498,11 @@ How RouteWatch keeps the answers:
|
||||
there.
|
||||
- It writes new answers every 5 seconds to `cache.db` in the state directory
|
||||
(see Configuration), and loads them from there at startup.
|
||||
- Each DNS record carries a TTL, the time for which it may be kept. An answer
|
||||
expires after the shortest TTL of its records. RouteWatch keeps it for at
|
||||
least 60 seconds and at most 24 hours.
|
||||
- When an address has no PTR names, the answer's SOA record says how long that
|
||||
answer may be kept. The answer expires after the SOA record's minimum field,
|
||||
or the record's own TTL when that is shorter. RouteWatch keeps it for at most
|
||||
1 hour.
|
||||
- An answer expires after the TTL of its records, the shortest one when there
|
||||
are several. RouteWatch keeps it for at least 60 seconds and at most 24 hours.
|
||||
- When an address has no PTR names, that answer expires after the SOA record's
|
||||
minimum field, or the record's own TTL when that is shorter. RouteWatch keeps
|
||||
it for at most 1 hour.
|
||||
- An answer with no PTR names and no SOA record is not kept. Neither is one
|
||||
whose SOA record has a TTL or minimum field of 0.
|
||||
- A failed lookup is not kept, so the next lookup of the address asks again.
|
||||
@@ -523,8 +516,7 @@ An IP lookup finds where each address is, on a best-effort basis. It uses two
|
||||
sources:
|
||||
|
||||
- [geojs](https://www.geojs.io/) is asked first. One request carries every
|
||||
address of the lookup that the cache does not hold. geojs has 1 second to
|
||||
answer.
|
||||
address of the lookup, and it has 1 second to answer.
|
||||
- [DB-IP City Lite](https://db-ip.com/db/download/ip-to-city-lite) answers when
|
||||
geojs has no answer for the address. It is a database file that RouteWatch
|
||||
reads itself.
|
||||
@@ -684,13 +676,6 @@ The ceilings add up to 3.7 GiB:
|
||||
limit caps the SQLite library.
|
||||
- About 0.2 GiB for everything else in the runtime.
|
||||
|
||||
The `icloud_private_relay` list has about 285,000 networks. Neighbouring
|
||||
networks merge, which leaves about 6,900 address ranges. Loaded, the list adds
|
||||
about 0.3 MiB to the Go heap and under 6 MiB to the process's resident memory.
|
||||
Parsing the file allocates about 94 MiB in about 0.2 seconds. The next garbage
|
||||
collection frees all of it but the 0.3 MiB above. Parsing happens at startup and
|
||||
whenever a download brings a new version.
|
||||
|
||||
The SQLite library gets its memory from glibc, the C library. glibc keeps freed
|
||||
memory in pools called arenas, and hands it back to the kernel slowly. It
|
||||
creates up to eight arenas per processor core. So on a host with many cores the
|
||||
@@ -706,10 +691,8 @@ What happens at each limit:
|
||||
allocating more.
|
||||
- The SQLite hard heap limit: a database statement fails with an out-of-memory
|
||||
error, which the handler logs. The prefix handler writes the routing table's
|
||||
changes and its origin changes again in a later write. Everything else in a
|
||||
handler's failed write is dropped. The PTR cache and the geolocation part
|
||||
write their answers to `cache.db` again in a later write. The process keeps
|
||||
running.
|
||||
changes again in a later write. Everything else in the failed write is
|
||||
dropped. The process keeps running.
|
||||
- The handler queues: the handlers are the four parts that take the feed's
|
||||
messages (see Design). The prefix handler's queue holds at most 100,000
|
||||
messages, about 100 MiB. Each of the other three holds at most 20,000. When a
|
||||
@@ -1076,8 +1059,8 @@ calls one of them. The scripts:
|
||||
- `script/projectname`: print the project name, `routewatch`, which
|
||||
`script/docker` uses as the image tag.
|
||||
- `script/test`: build the `test` phase of the `Dockerfile` alone. It runs
|
||||
`go test -short -count=1 -timeout 90s -race -cover ./...`. On failure it runs
|
||||
the tests again with `-v` and without `-cover`.
|
||||
`go test -short -count=1 -timeout 90s -race -cover ./...`, and reruns it with
|
||||
`-v` on failure.
|
||||
- `script/lint`: build the `lint` phase of the `Dockerfile` alone. It runs
|
||||
`golangci-lint` v2.14.0 with `.golangci.yml`.
|
||||
- `script/fmt`: format the Go code with `go fmt` and the markdown with
|
||||
|
||||
@@ -22,76 +22,51 @@ formatting of the Go code and the markdown. Then it builds the image.
|
||||
The image sets memory ceilings for a 5 GiB container (README "Memory"). README
|
||||
"Running under upaas" says how to run it.
|
||||
|
||||
Five runs of `next` on the live feed have ended since 2026-10-27, from three
|
||||
hours to five days long. The runs found the handlers dropping updates for five
|
||||
causes, and the WHOIS fetcher asking a registry that had refused it. Each is
|
||||
fixed on `next`, and its Completed Steps entry below names its issue. A 21-hour
|
||||
run of `742e759`, which has every fix but one, dropped no update except after
|
||||
the feed stalled. That cause, https://git.eeqj.de/sneak/routewatch/issues/229,
|
||||
is fixed on `next`.
|
||||
A five-day run of `1c65656` on the live feed, from 2026-10-27 to 2026-11-01,
|
||||
found three problems:
|
||||
|
||||
A run of `e58c663` on the live feed began at 09:19 UTC on 2026-11-10
|
||||
(https://git.eeqj.de/sneak/routewatch/issues/232). It is to confirm that fix.
|
||||
Next Step lists what its outcome reports. The seven IP lookup additions of
|
||||
https://git.eeqj.de/sneak/routewatch/issues/203 landed after it began, so it
|
||||
does not exercise them.
|
||||
- The prefix handler dropped updates while a peer's session went down
|
||||
(https://git.eeqj.de/sneak/routewatch/issues/216).
|
||||
- The AS handler dropped updates when the feed burst
|
||||
(https://git.eeqj.de/sneak/routewatch/issues/217).
|
||||
- The WHOIS fetcher kept asking a registry that had refused it
|
||||
(https://git.eeqj.de/sneak/routewatch/issues/218).
|
||||
|
||||
All three are fixed on `next`. So is
|
||||
https://git.eeqj.de/sneak/routewatch/issues/222, which the third fix brought up.
|
||||
A run of `26d939a` on the live feed showed the prefix handler still dropping
|
||||
updates while the home page's cards were worked out.
|
||||
https://git.eeqj.de/sneak/routewatch/issues/226 fixed that on `next`.
|
||||
https://git.eeqj.de/sneak/routewatch/issues/225, also on `next`, takes the peers
|
||||
whose sessions go down together out of the table in one walk. A 24-hour run of
|
||||
`9bd0e91` on the live feed is checking whether any handler still drops updates.
|
||||
|
||||
The order of the project's remaining work is kept on
|
||||
https://git.eeqj.de/sneak/routewatch/issues/86.
|
||||
|
||||
# Next Step
|
||||
|
||||
Post the outcome of the run of `e58c663` on
|
||||
https://git.eeqj.de/sneak/routewatch/issues/232. The run lasts at least 24
|
||||
hours. If the feed has not stalled or reconnected by then, the run goes on until
|
||||
it does, up to 72 hours in all. The outcome says:
|
||||
|
||||
- whether the feed stalled or reconnected
|
||||
- whether any handler dropped an update, and the most messages the prefix
|
||||
handler's queue held at once
|
||||
- the most messages the prefix handler held back at once
|
||||
- the size of the `origin_changes` table on disk
|
||||
- the process's peak memory against 5 GiB
|
||||
|
||||
File an issue for each problem the run finds.
|
||||
Post the outcome of the 24-hour run of `9bd0e91` on
|
||||
https://git.eeqj.de/sneak/routewatch/issues/226: whether any handler still drops
|
||||
updates, and the process's peak memory against 5 GiB. File an issue for each
|
||||
problem the run finds.
|
||||
|
||||
# Completed Steps
|
||||
|
||||
- 2026-11-11: two more tests no longer fail on a busy host. The test of the home
|
||||
page's card refresh runs on a fake clock. The geo test in which geojs never
|
||||
answers counts each request as it is sent, not when it reaches the server
|
||||
(closes https://git.eeqj.de/sneak/routewatch/issues/256).
|
||||
- 2026-11-10: `github.com/mattn/go-sqlite3` is at v1.14.53, so building on the
|
||||
host no longer prints its two C compiler warnings (closes
|
||||
https://git.eeqj.de/sneak/routewatch/issues/245).
|
||||
- 2026-11-10: `README.md` and this file are read whole against the rules of
|
||||
https://git.eeqj.de/sneak/routewatch/issues/157, and every statement except
|
||||
measured figures and what outside services do is checked against the code on
|
||||
`next`. Four README sentences failed the prose rules. Five disagreed with the
|
||||
code. Each is rewritten. Status and Next Step describe the live-feed run in
|
||||
progress (closes https://git.eeqj.de/sneak/routewatch/issues/251).
|
||||
- 2026-11-10: the image creates the `routewatch` user as a regular user with UID
|
||||
1000, so the build no longer warns about the system UID range (closes
|
||||
https://git.eeqj.de/sneak/routewatch/issues/248).
|
||||
- 2026-11-10: each result of the IP lookup has `geo`, where the address is. It
|
||||
comes from geojs, with DB-IP's City Lite database as the fallback. The geojs
|
||||
answers stay in memory and in `cache.db` for 7 days (closes
|
||||
https://git.eeqj.de/sneak/routewatch/issues/239).
|
||||
- 2026-11-10: the IP lookup checks the `icloud_private_relay` list of Apple's
|
||||
iCloud Private Relay networks, downloaded once a day. Each result has
|
||||
`flags.is_relay` (closes https://git.eeqj.de/sneak/routewatch/issues/237).
|
||||
- 2026-11-10: the IP lookup checks two ProtonVPN lists from a public mirror of
|
||||
Proton's server list. `protonvpn` holds the /24 networks of ProtonVPN's
|
||||
servers, and `protonvpn_entry` holds the servers' IPv4 and IPv6 addresses
|
||||
(closes https://git.eeqj.de/sneak/routewatch/issues/236).
|
||||
- 2026-11-10: the IP lookup checks the `protonvpn` list of ProtonVPN's /24
|
||||
networks and the `protonvpn_entry` list of its servers' IPv4 and IPv6
|
||||
addresses, from a public mirror of Proton's server list (closes
|
||||
https://git.eeqj.de/sneak/routewatch/issues/236).
|
||||
- 2026-11-10: the IP lookup finds PTR names over DNS-over-HTTPS and caches them
|
||||
in memory and in `cache.db` for the record's TTL. A TTL is the time for which
|
||||
a DNS record may be kept (closes
|
||||
for the record's TTL, in memory and in `cache.db` (closes
|
||||
https://git.eeqj.de/sneak/routewatch/issues/238).
|
||||
- 2026-11-10: the tests no longer fail when a busy host stalls for several
|
||||
seconds. Most waits for something that should happen are now 15 seconds. Tests
|
||||
that depend on a short idle timeout or a request limit run on a fake clock
|
||||
(closes https://git.eeqj.de/sneak/routewatch/issues/240)
|
||||
seconds. Most such waits are now 15 seconds. The test of the stream's idle
|
||||
timeout runs on a fake clock (closes #240)
|
||||
- 2026-11-10: the IP lookup checks the public VPN server lists of Mullvad,
|
||||
NordVPN, Private Internet Access and IVPN, and X4BNet's community list. Each
|
||||
result has `flags.is_vpn` and `vpn_providers` (closes
|
||||
|
||||
@@ -9,7 +9,7 @@ require (
|
||||
github.com/google/btree v1.1.3
|
||||
github.com/google/uuid v1.6.0
|
||||
github.com/hashicorp/golang-lru/v2 v2.0.7
|
||||
github.com/mattn/go-sqlite3 v1.14.53
|
||||
github.com/mattn/go-sqlite3 v1.14.29
|
||||
github.com/maxmind/mmdbwriter v1.2.0
|
||||
github.com/miekg/dns v1.1.73
|
||||
github.com/oschwald/maxminddb-golang/v2 v2.9.0
|
||||
|
||||
@@ -10,8 +10,8 @@ github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
|
||||
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
|
||||
github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k=
|
||||
github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM=
|
||||
github.com/mattn/go-sqlite3 v1.14.53 h1:mnTM0owzUMrKuLo+0w51NUiB0ir2Fo7kFXlPt/l3BGs=
|
||||
github.com/mattn/go-sqlite3 v1.14.53/go.mod h1:6JTjA44L93a0QCyJef5YvlPoKXntQPjzWv5gtm9sB6w=
|
||||
github.com/mattn/go-sqlite3 v1.14.29 h1:1O6nRLJKvsi1H2Sj0Hzdfojwt8GiGKm+LOfLaBFaouQ=
|
||||
github.com/mattn/go-sqlite3 v1.14.29/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y=
|
||||
github.com/maxmind/mmdbwriter v1.2.0 h1:hyvDopImmgvle3aR8AaddxXnT0iQH2KWJX3vNfkwzYM=
|
||||
github.com/maxmind/mmdbwriter v1.2.0/go.mod h1:EQmKHhk2y9DRVvyNxwCLKC5FrkXZLx4snc5OlLY5XLE=
|
||||
github.com/miekg/dns v1.1.73 h1:uhT8nJxmTrPJYClxVxTCX+CVn6qnzSiybRk72Z6DgrE=
|
||||
|
||||
@@ -90,12 +90,14 @@ func parseAddresses(t *testing.T, addresses ...string) []netip.Addr {
|
||||
return addrs
|
||||
}
|
||||
|
||||
// fakeGeoJS is a geojs server. It sends body, or status when that is set. It
|
||||
// keeps each request's address list.
|
||||
// fakeGeoJS is a geojs server. It sends body, or status when that is set, or
|
||||
// waits for the request to end when hang is true. It keeps each request's
|
||||
// address list.
|
||||
type fakeGeoJS struct {
|
||||
server *httptest.Server
|
||||
body string
|
||||
status int
|
||||
hang bool
|
||||
|
||||
mu sync.Mutex
|
||||
requests []string
|
||||
@@ -117,6 +119,12 @@ func (f *fakeGeoJS) serve(w http.ResponseWriter, r *http.Request) {
|
||||
f.requests = append(f.requests, r.URL.Query().Get("ip"))
|
||||
f.mu.Unlock()
|
||||
|
||||
if f.hang {
|
||||
<-r.Context().Done()
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
if f.status != 0 {
|
||||
w.WriteHeader(f.status)
|
||||
|
||||
@@ -134,20 +142,6 @@ func (f *fakeGeoJS) requestsSeen() []string {
|
||||
return append([]string{}, f.requests...)
|
||||
}
|
||||
|
||||
// hangingTransport never answers: each request waits until its context ends.
|
||||
// It counts the requests as they are sent, so the count does not depend on
|
||||
// whether one reaches a server before its time limit runs out.
|
||||
type hangingTransport struct {
|
||||
requests atomic.Int32
|
||||
}
|
||||
|
||||
func (h *hangingTransport) RoundTrip(r *http.Request) (*http.Response, error) {
|
||||
h.requests.Add(1)
|
||||
<-r.Context().Done()
|
||||
|
||||
return nil, r.Context().Err()
|
||||
}
|
||||
|
||||
// openCacheDB opens a new cache.db in a temporary directory.
|
||||
func openCacheDB(t *testing.T, dir string) *sql.DB {
|
||||
t.Helper()
|
||||
@@ -391,15 +385,13 @@ func TestLookupFallsBackToTheDatabase(t *testing.T) {
|
||||
|
||||
fake := newFakeGeoJS(t, tt.body)
|
||||
fake.status = tt.status
|
||||
fake.hang = tt.hang
|
||||
dir := t.TempDir()
|
||||
locator := newTestLocator(t, fake, openCacheDB(t, dir), dir)
|
||||
requestCount := func() int { return len(fake.requestsSeen()) }
|
||||
locator.timeLimit = shortTimeLimit
|
||||
|
||||
if tt.hang {
|
||||
hanging := &hangingTransport{}
|
||||
locator.client = &http.Client{Transport: hanging}
|
||||
locator.timeLimit = shortTimeLimit
|
||||
requestCount = func() int { return int(hanging.requests.Load()) }
|
||||
if !tt.hang {
|
||||
locator.timeLimit = testTimeLimit
|
||||
}
|
||||
|
||||
giveDatabase(t, locator)
|
||||
@@ -421,8 +413,8 @@ func TestLookupFallsBackToTheDatabase(t *testing.T) {
|
||||
// asks geojs again.
|
||||
lookupOne(t, locator, testAddress)
|
||||
|
||||
if requests := requestCount(); requests != 2 {
|
||||
t.Errorf("%d requests, want 2", requests)
|
||||
if requests := fake.requestsSeen(); len(requests) != 2 {
|
||||
t.Errorf("%d requests, want 2", len(requests))
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
@@ -21,7 +21,7 @@ const (
|
||||
downloadTimeout = time.Minute
|
||||
|
||||
// maxDownloadSize is the size of the largest list file accepted, in
|
||||
// bytes. The largest list, iCloud Private Relay, is about 12 MiB.
|
||||
// bytes. The largest list, FireHOL abusers, is about 2.5 MiB.
|
||||
maxDownloadSize = 200 << 20
|
||||
)
|
||||
|
||||
|
||||
@@ -56,11 +56,6 @@ const (
|
||||
// that start with #. Unlike FormatLines, a file with no addresses or
|
||||
// networks fails.
|
||||
FormatProtonVPN
|
||||
|
||||
// FormatICloudPrivateRelayCSV is Apple's CSV file of iCloud Private Relay
|
||||
// networks. Each line starts with a network, followed by its country,
|
||||
// region and city.
|
||||
FormatICloudPrivateRelayCSV
|
||||
)
|
||||
|
||||
// List is one public IP list.
|
||||
|
||||
@@ -83,18 +83,6 @@ const protonVPNEntryBody = "#\n# protonvpn_entry_ips.txt\n" +
|
||||
"# Entry IPs clients connect to, resolved from ProtonVPN hostnames.\n#\n" +
|
||||
"192.0.2.80\n2001:db8::80\n"
|
||||
|
||||
// iCloudPrivateRelayAddr is an address on iCloudPrivateRelayBody.
|
||||
const iCloudPrivateRelayAddr = "2001:db8:1::5"
|
||||
|
||||
// iCloudPrivateRelayBody is a small file in the format of Apple's iCloud
|
||||
// Private Relay file.
|
||||
const iCloudPrivateRelayBody = "192.0.2.0/31,GB,GB-EN,London,\n" +
|
||||
"198.51.100.32/27,US,US-CA,Los Angeles,\n" +
|
||||
"2001:db8:1::/64,US,US-AK,Anchorage,\n"
|
||||
|
||||
// emptyFileCase names the case of a wrong document that is an empty file.
|
||||
const emptyFileCase = "an empty file"
|
||||
|
||||
// errorPage is a file that is no list, as a publisher's error page is.
|
||||
const errorPage = "<!DOCTYPE html>\n<html></html>\n"
|
||||
|
||||
@@ -255,15 +243,6 @@ func TestLookupFindsListedAddresses(t *testing.T) {
|
||||
},
|
||||
unlisted: []string{"192.0.2.12", "2001:db8::2"},
|
||||
},
|
||||
{
|
||||
name: "iCloud Private Relay networks",
|
||||
format: FormatICloudPrivateRelayCSV,
|
||||
body: iCloudPrivateRelayBody,
|
||||
listed: []string{
|
||||
"192.0.2.1", "198.51.100.63", iCloudPrivateRelayAddr,
|
||||
},
|
||||
unlisted: []string{"192.0.2.2", "198.51.100.64", "2001:db8:2::1"},
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
@@ -506,12 +485,12 @@ func TestWrongVPNDocumentFailsToParse(t *testing.T) {
|
||||
|
||||
for _, f := range formats {
|
||||
bodies := map[string]string{
|
||||
"not JSON": errorPage,
|
||||
"other JSON": `{"status":"ok","data":{"count":1}}`,
|
||||
"no addresses": f.empty,
|
||||
"bad address": f.badAddress,
|
||||
"cut off": f.good[:len(f.good)/2],
|
||||
emptyFileCase: "",
|
||||
"not JSON": errorPage,
|
||||
"other JSON": `{"status":"ok","data":{"count":1}}`,
|
||||
"no addresses": f.empty,
|
||||
"bad address": f.badAddress,
|
||||
"cut off": f.good[:len(f.good)/2],
|
||||
"an empty file": "",
|
||||
}
|
||||
|
||||
for name, body := range bodies {
|
||||
@@ -537,7 +516,7 @@ func TestWrongProtonVPNDocumentFailsToParse(t *testing.T) {
|
||||
"JSON": `{"status":"ok"}`,
|
||||
"comments only": "#\n# protonvpn_ip_ranges.txt\n#\n",
|
||||
"bad network": "192.0.2.0/24\nexample\n",
|
||||
emptyFileCase: "",
|
||||
"an empty file": "",
|
||||
}
|
||||
|
||||
for name, body := range bodies {
|
||||
@@ -552,34 +531,6 @@ func TestWrongProtonVPNDocumentFailsToParse(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// TestWrongICloudPrivateRelayDocumentFailsToParse parses documents that are
|
||||
// not Apple's iCloud Private Relay file. Each fails, so none can replace a
|
||||
// good copy.
|
||||
func TestWrongICloudPrivateRelayDocumentFailsToParse(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
bodies := map[string]string{
|
||||
"error page": errorPage,
|
||||
"JSON": `{"status":"ok","count":1}`,
|
||||
"no other field": "192.0.2.0/24\n",
|
||||
"bad network": "192.0.2.0/24,GB,GB-EN,London,\nexample,GB,GB-EN,London,\n",
|
||||
"an address": "192.0.2.1,GB,GB-EN,London,\n",
|
||||
emptyFileCase: "",
|
||||
}
|
||||
|
||||
for name, body := range bodies {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
addresses, err := parse(strings.NewReader(body),
|
||||
FormatICloudPrivateRelayCSV)
|
||||
if err == nil {
|
||||
t.Errorf("parse returned %v and no error", addresses.Ranges())
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestLookupMarksVPNLists looks up an address on a VPN list and on another
|
||||
// list. Only the VPN list's listing says it is one.
|
||||
func TestLookupMarksVPNLists(t *testing.T) {
|
||||
@@ -934,33 +885,15 @@ func TestStopCancelsADownloadInProgress(t *testing.T) {
|
||||
func TestTorExitsListHoldsIPv6Exits(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
checkPublicListHolds(t, TorExitsList, onionooBody, onionooIPv6ExitAddr)
|
||||
}
|
||||
|
||||
// TestICloudPrivateRelayListHoldsItsNetworks downloads a file in Apple's
|
||||
// format as the public iCloud Private Relay list, which is what an address's
|
||||
// is_relay answer comes from.
|
||||
func TestICloudPrivateRelayListHoldsItsNetworks(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
checkPublicListHolds(t, ICloudPrivateRelayList, iCloudPrivateRelayBody,
|
||||
iCloudPrivateRelayAddr)
|
||||
}
|
||||
|
||||
// checkPublicListHolds downloads body as the public list called name and
|
||||
// checks that addr has one listing, which names that list.
|
||||
func checkPublicListHolds(t *testing.T, name, body, addr string) {
|
||||
t.Helper()
|
||||
|
||||
index := slices.IndexFunc(Lists(), func(list List) bool {
|
||||
return list.Name == name
|
||||
return list.Name == TorExitsList
|
||||
})
|
||||
if index < 0 {
|
||||
t.Fatalf("no list named %s", name)
|
||||
t.Fatalf("no list named %s", TorExitsList)
|
||||
}
|
||||
|
||||
list := Lists()[index]
|
||||
_, list.URL = newPublisher(t, body)
|
||||
_, list.URL = newPublisher(t, onionooBody)
|
||||
b := newTestIPLists(t.TempDir(), list)
|
||||
|
||||
err := b.download(t.Context(), list)
|
||||
@@ -968,9 +901,9 @@ func checkPublicListHolds(t *testing.T, name, body, addr string) {
|
||||
t.Fatalf("download: %v", err)
|
||||
}
|
||||
|
||||
listings := lookup(t, b, addr)
|
||||
if len(listings) != 1 || listings[0].Name != name {
|
||||
t.Errorf("listings %+v, want one for %s", listings, name)
|
||||
listings := lookup(t, b, onionooIPv6ExitAddr)
|
||||
if len(listings) != 1 || listings[0].Name != TorExitsList {
|
||||
t.Errorf("listings %+v, want one for %s", listings, TorExitsList)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -8,10 +8,6 @@ import (
|
||||
// TorExitsList is the name of the list of Tor exit nodes.
|
||||
const TorExitsList = "tor_exits"
|
||||
|
||||
// ICloudPrivateRelayList is the name of the list of iCloud Private Relay's
|
||||
// networks.
|
||||
const ICloudPrivateRelayList = "icloud_private_relay"
|
||||
|
||||
// protonVPNIPsURL is where the ProtonVPN-IPs mirror publishes its files.
|
||||
const protonVPNIPsURL = "https://raw.githubusercontent.com/tn3w/ProtonVPN-IPs/master/"
|
||||
|
||||
@@ -34,9 +30,6 @@ const (
|
||||
fireHOLAbusersInterval = 5 * time.Hour
|
||||
fireHOLLevel3Interval = 3 * time.Hour
|
||||
|
||||
// Apple's iCloud Private Relay file is 12 MB, so once a day.
|
||||
iCloudPrivateRelayInterval = 24 * time.Hour
|
||||
|
||||
// Spamhaus asks that DROP be downloaded at most once a day:
|
||||
// https://www.spamhaus.org/blocklists/do-not-route-or-peer/
|
||||
spamhausDROPInterval = 24 * time.Hour
|
||||
@@ -91,13 +84,6 @@ func nonVPNLists() []List {
|
||||
Interval: fireHOLLevel3Interval,
|
||||
Format: FormatLines,
|
||||
},
|
||||
{
|
||||
Name: ICloudPrivateRelayList,
|
||||
Description: "In a network where iCloud Private Relay traffic leaves",
|
||||
URL: "https://mask-api.icloud.com/egress-ip-ranges.csv",
|
||||
Interval: iCloudPrivateRelayInterval,
|
||||
Format: FormatICloudPrivateRelayCSV,
|
||||
},
|
||||
{
|
||||
Name: "spamhaus_drop",
|
||||
Description: "In an IPv4 network that Spamhaus says spammers " +
|
||||
|
||||
@@ -25,10 +25,6 @@ const (
|
||||
// dshieldFieldSeparator separates the fields of a DShield line.
|
||||
dshieldFieldSeparator = "\t"
|
||||
|
||||
// iCloudPrivateRelayFieldSeparator separates the fields of a line of
|
||||
// Apple's iCloud Private Relay file.
|
||||
iCloudPrivateRelayFieldSeparator = ","
|
||||
|
||||
// spamhausMetadataType is the type of the line that ends a Spamhaus
|
||||
// file. That line holds the file's metadata and no network.
|
||||
spamhausMetadataType = "metadata"
|
||||
@@ -63,9 +59,7 @@ func parse(r io.Reader, format Format) (*netipx.IPSet, error) {
|
||||
case FormatIVPNJSON:
|
||||
return parseIVPN(r)
|
||||
case FormatProtonVPN:
|
||||
return parseLinesNotEmpty(r, addAddressOrNetwork)
|
||||
case FormatICloudPrivateRelayCSV:
|
||||
return parseLinesNotEmpty(r, addICloudPrivateRelayNetwork)
|
||||
return parseProtonVPN(r)
|
||||
default:
|
||||
return nil, errUnknownFormat
|
||||
}
|
||||
@@ -337,12 +331,11 @@ func parseIVPN(r io.Reader) (*netipx.IPSet, error) {
|
||||
return addressSet(texts)
|
||||
}
|
||||
|
||||
// parseLinesNotEmpty is parseLines for a list that always holds addresses, so
|
||||
// a file with none fails.
|
||||
func parseLinesNotEmpty(
|
||||
r io.Reader, addLine func(*netipx.IPSetBuilder, string) error,
|
||||
) (*netipx.IPSet, error) {
|
||||
addresses, err := parseLines(r, addLine)
|
||||
// parseProtonVPN returns the addresses or networks in a ProtonVPN-IPs list. A
|
||||
// file with no addresses or networks fails, because the service always has
|
||||
// servers.
|
||||
func parseProtonVPN(r io.Reader) (*netipx.IPSet, error) {
|
||||
addresses, err := parseLines(r, addAddressOrNetwork)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -354,24 +347,6 @@ func parseLinesNotEmpty(
|
||||
return addresses, nil
|
||||
}
|
||||
|
||||
// addICloudPrivateRelayNetwork adds the network at the start of a line of
|
||||
// Apple's iCloud Private Relay file to builder.
|
||||
func addICloudPrivateRelayNetwork(builder *netipx.IPSetBuilder, line string) error {
|
||||
field, _, found := strings.Cut(line, iCloudPrivateRelayFieldSeparator)
|
||||
if !found {
|
||||
return errNoNetwork
|
||||
}
|
||||
|
||||
network, err := netip.ParsePrefix(field)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
builder.AddPrefix(network)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// readJSON reads r to its end and decodes it, as one JSON document, into v.
|
||||
func readJSON(r io.Reader, v any) error {
|
||||
data, err := io.ReadAll(r)
|
||||
|
||||
@@ -801,9 +801,6 @@ type IPLookupFlags struct {
|
||||
// IsVPN says that the address is on a list of VPN servers.
|
||||
IsVPN bool `json:"is_vpn"`
|
||||
|
||||
// IsRelay says that the address is on the iCloud Private Relay list.
|
||||
IsRelay bool `json:"is_relay"`
|
||||
|
||||
// IsBogon says that the address is reserved, as package bogon tells.
|
||||
IsBogon bool `json:"is_bogon"`
|
||||
}
|
||||
@@ -1036,9 +1033,6 @@ func ipLookupFlags(ip string, listings []iplist.Listing) IPLookupFlags {
|
||||
IsVPN: slices.ContainsFunc(listings, func(l iplist.Listing) bool {
|
||||
return l.VPN
|
||||
}),
|
||||
IsRelay: slices.ContainsFunc(listings, func(l iplist.Listing) bool {
|
||||
return l.Name == iplist.ICloudPrivateRelayList
|
||||
}),
|
||||
IsBogon: isBogon(ip),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1506,10 +1506,6 @@ const listedUnroutedAddr = "203.0.113.1"
|
||||
// torExitAddr is the address that fakeIPLists lists on the Tor exit list.
|
||||
const torExitAddr = "203.0.113.2"
|
||||
|
||||
// relayAddr is the address that fakeIPLists lists on the iCloud Private Relay
|
||||
// list.
|
||||
const relayAddr = "203.0.113.5"
|
||||
|
||||
// The addresses that fakeIPLists lists on one and on two VPN lists, and the
|
||||
// names of those lists.
|
||||
const (
|
||||
@@ -1519,11 +1515,10 @@ const (
|
||||
otherTestVPN = "other_test_vpn"
|
||||
)
|
||||
|
||||
// fakeIPLists has five lists. The test list holds addrInTestPrefixV4,
|
||||
// fakeIPLists has four lists. The test list holds addrInTestPrefixV4,
|
||||
// listedUnroutedAddr and twoVPNListsAddr. The Tor exit list holds torExitAddr.
|
||||
// The iCloud Private Relay list holds relayAddr. The test VPN list holds
|
||||
// oneVPNListAddr and twoVPNListsAddr, and the other test VPN list holds
|
||||
// twoVPNListsAddr.
|
||||
// The test VPN list holds oneVPNListAddr and twoVPNListsAddr, and the other
|
||||
// test VPN list holds twoVPNListsAddr.
|
||||
type fakeIPLists struct{}
|
||||
|
||||
func testListing() iplist.Listing {
|
||||
@@ -1544,10 +1539,6 @@ func (fakeIPLists) Lookup(addr netip.Addr) []iplist.Listing {
|
||||
return []iplist.Listing{{Name: iplist.TorExitsList}}
|
||||
}
|
||||
|
||||
if addr == netip.MustParseAddr(relayAddr) {
|
||||
return []iplist.Listing{{Name: iplist.ICloudPrivateRelayList}}
|
||||
}
|
||||
|
||||
if addr == netip.MustParseAddr(oneVPNListAddr) {
|
||||
return []iplist.Listing{{Name: testVPNList, VPN: true}}
|
||||
}
|
||||
@@ -1580,22 +1571,20 @@ func TestIPLookupFlags(t *testing.T) {
|
||||
name string
|
||||
addr string
|
||||
tor bool
|
||||
relay bool
|
||||
bogon bool
|
||||
}{
|
||||
{"on the Tor exit list", torExitAddr, true, false, true},
|
||||
{"on the iCloud Private Relay list", relayAddr, false, true, true},
|
||||
{"on another list", listedUnroutedAddr, false, false, true},
|
||||
{"routed IPv4", addrInTestPrefixV4, false, false, true},
|
||||
{"routed IPv6", addrInTestPrefixV6, false, false, true},
|
||||
{"private IPv4", "10.0.0.1", false, false, true},
|
||||
{"loopback IPv4", "127.0.0.1", false, false, true},
|
||||
{"public IPv4", "1.1.1.1", false, false, false},
|
||||
{"private IPv6", "fd00::1", false, false, true},
|
||||
{"loopback IPv6", "::1", false, false, true},
|
||||
{"public IPv6", "2606:4700:4700::1111", false, false, false},
|
||||
{"public IPv4 in IPv6 form", "::ffff:1.1.1.1", false, false, false},
|
||||
{"private IPv4 in IPv6 form", "::ffff:10.0.0.1", false, false, true},
|
||||
{"on the Tor exit list", torExitAddr, true, true},
|
||||
{"on another list", listedUnroutedAddr, false, true},
|
||||
{"routed IPv4", addrInTestPrefixV4, false, true},
|
||||
{"routed IPv6", addrInTestPrefixV6, false, true},
|
||||
{"private IPv4", "10.0.0.1", false, true},
|
||||
{"loopback IPv4", "127.0.0.1", false, true},
|
||||
{"public IPv4", "1.1.1.1", false, false},
|
||||
{"private IPv6", "fd00::1", false, true},
|
||||
{"loopback IPv6", "::1", false, true},
|
||||
{"public IPv6", "2606:4700:4700::1111", false, false},
|
||||
{"public IPv4 in IPv6 form", "::ffff:1.1.1.1", false, false},
|
||||
{"private IPv4 in IPv6 form", "::ffff:10.0.0.1", false, true},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
@@ -1617,8 +1606,7 @@ func TestIPLookupFlags(t *testing.T) {
|
||||
}
|
||||
|
||||
want := map[string]bool{
|
||||
"is_tor": tt.tor, "is_vpn": false, "is_relay": tt.relay,
|
||||
"is_bogon": tt.bogon,
|
||||
"is_tor": tt.tor, "is_vpn": false, "is_bogon": tt.bogon,
|
||||
}
|
||||
if !maps.Equal(got, want) {
|
||||
t.Errorf("%s: flags %v, want %v", path, got, want)
|
||||
|
||||
@@ -275,66 +275,49 @@ func TestPagesServeTheCopyOfTheCards(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// TestKeepHomeCardsFreshWorksThemOutEveryInterval runs keepHomeCardsFresh. It
|
||||
// works the cards out at once, again an interval after the table changes, and
|
||||
// stops when its context ends. It runs in a synctest bubble, whose clock is
|
||||
// fake, so a stall on the host cannot fail it.
|
||||
// TestKeepHomeCardsFreshWorksThemOutEveryInterval runs keepHomeCardsFresh with
|
||||
// a short interval. It works the cards out at once, again after the table
|
||||
// changes, and stops when its context ends.
|
||||
func TestKeepHomeCardsFreshWorksThemOutEveryInterval(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
const interval = time.Minute
|
||||
const (
|
||||
interval = 10 * time.Millisecond
|
||||
maxWait = 15 * time.Second
|
||||
)
|
||||
|
||||
table := emptyTable()
|
||||
announce(table, "1.0.0.0/24", twoPrefixASN, testPeer(1), testRouteTime(0))
|
||||
|
||||
s := newServerWithTable(t, table)
|
||||
|
||||
// newestRouteOrigin returns the origin AS of the newest route on the
|
||||
// cards, or 0 when there is none.
|
||||
newestRouteOrigin := func() int {
|
||||
cards := s.cards.Load()
|
||||
if cards == nil || cards.NewestRoute == nil {
|
||||
return 0
|
||||
}
|
||||
ctx, cancel := context.WithCancel(t.Context())
|
||||
stopped := make(chan struct{})
|
||||
|
||||
return cards.NewestRoute.OriginASN
|
||||
go func() {
|
||||
s.keepHomeCardsFresh(ctx, interval)
|
||||
close(stopped)
|
||||
}()
|
||||
|
||||
newestRouteASNIs := func(asn int) bool {
|
||||
cards := s.cards.Load()
|
||||
|
||||
return cards != nil && cards.NewestRoute != nil &&
|
||||
cards.NewestRoute.OriginASN == asn
|
||||
}
|
||||
|
||||
synctest.Test(t, func(t *testing.T) {
|
||||
ctx, cancel := context.WithCancel(t.Context())
|
||||
stopped := make(chan struct{})
|
||||
waitUntil(t, maxWait, func() bool { return newestRouteASNIs(twoPrefixASN) })
|
||||
|
||||
go func() {
|
||||
s.keepHomeCardsFresh(ctx, interval)
|
||||
close(stopped)
|
||||
}()
|
||||
announce(table, "8.8.8.0/24", newestRouteASN, testPeer(2), testRouteTime(1))
|
||||
waitUntil(t, maxWait, func() bool { return newestRouteASNIs(newestRouteASN) })
|
||||
|
||||
synctest.Wait()
|
||||
cancel()
|
||||
|
||||
if got := newestRouteOrigin(); got != twoPrefixASN {
|
||||
t.Fatalf("at the start, the newest route's AS is %d, want %d",
|
||||
got, twoPrefixASN)
|
||||
}
|
||||
|
||||
announce(table, "8.8.8.0/24", newestRouteASN, testPeer(2),
|
||||
testRouteTime(1))
|
||||
time.Sleep(interval)
|
||||
synctest.Wait()
|
||||
|
||||
if got := newestRouteOrigin(); got != newestRouteASN {
|
||||
t.Fatalf("an interval after the table changed, the newest "+
|
||||
"route's AS is %d, want %d", got, newestRouteASN)
|
||||
}
|
||||
|
||||
cancel()
|
||||
synctest.Wait()
|
||||
|
||||
select {
|
||||
case <-stopped:
|
||||
default:
|
||||
t.Fatal("keepHomeCardsFresh did not stop when its context ended")
|
||||
}
|
||||
})
|
||||
select {
|
||||
case <-stopped:
|
||||
case <-time.After(maxWait):
|
||||
t.Fatal("keepHomeCardsFresh did not stop when its context ended")
|
||||
}
|
||||
}
|
||||
|
||||
// TestStartedServerWorksOutTheCards starts a server and waits until its home
|
||||
|
||||
Reference in new issue
Block a user