Compare commits
1
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
 sneak
|
6e903d511f |
No files matched your search
+1
-1
@@ -105,7 +105,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
|
||||
ARG CACHEBUST
|
||||
|
||||
# Create non-root user
|
||||
RUN useradd -u 1000 -m routewatch
|
||||
RUN useradd -r -u 1000 -m routewatch
|
||||
|
||||
RUN mkdir -p /var/lib/berlin.sneak.app.routewatch && chown routewatch:routewatch /var/lib/berlin.sneak.app.routewatch
|
||||
|
||||
|
||||
@@ -143,9 +143,9 @@ and origin AS can have two kept routes, one or none.
|
||||
AS path, a next hop and a time. The next hop is the router the peer sends the
|
||||
prefix's traffic to. RouteWatch records the peer itself as the next hop. The
|
||||
page also lists the prefix's origin changes, newest first. An origin change is
|
||||
recorded when the first peer starts to carry the prefix with an origin AS, or
|
||||
when the last peer stops. The page shows at most the newest 5,000. Each row
|
||||
shows:
|
||||
an AS starting to announce the prefix, when its first peer carries it, or
|
||||
stopping, when its last peer no longer does. The page shows at most the newest
|
||||
5,000. Each row shows:
|
||||
- the time
|
||||
- announced or withdrawn
|
||||
- the origin AS
|
||||
@@ -197,9 +197,8 @@ also carries an `@meta` object:
|
||||
- `api_version`: `1`
|
||||
- `execution_time_ms`: the number of milliseconds the request took
|
||||
|
||||
Every value with a unit is a number. The unit ends the field name, as in `_ms`,
|
||||
`_seconds`, `_bytes`, `_per_sec` and `_per_min`. `latitude` and `longitude` are
|
||||
in degrees, and their names do not say so.
|
||||
Every value with a unit is a number, and the unit ends the field name. Examples
|
||||
are `_ms`, `_seconds`, `_bytes`, `_per_sec` and `_per_min`.
|
||||
|
||||
The paths:
|
||||
|
||||
@@ -288,7 +287,6 @@ address. Every entry has the same fields:
|
||||
- `is_tor`: true when the address is on the `tor_exits` list
|
||||
- `is_vpn`: true when the address is on any list of VPN servers (see IP
|
||||
lists)
|
||||
- `is_relay`: true when the address is on the `icloud_private_relay` list
|
||||
- `is_bogon`: true when the address is reserved, so that no network should
|
||||
announce it on the internet
|
||||
- `vpn_providers`: the names of the lists of VPN servers the address is on, or
|
||||
@@ -424,9 +422,6 @@ The lists:
|
||||
last 30 days. From
|
||||
[FireHOL](https://iplists.firehol.org/?ipset=firehol_level3), downloaded every
|
||||
3 hours.
|
||||
- `icloud_private_relay`: the networks where iCloud Private Relay traffic
|
||||
leaves. From [Apple](https://mask-api.icloud.com/egress-ip-ranges.csv),
|
||||
downloaded once a day.
|
||||
- `ivpn`: VPN servers run by IVPN. From
|
||||
[IVPN](https://api.ivpn.net/v5/servers.json), downloaded once a day.
|
||||
- `mullvad`: VPN servers run by Mullvad. From
|
||||
@@ -437,14 +432,6 @@ The lists:
|
||||
- `pia`: VPN servers run by Private Internet Access. From
|
||||
[Private Internet Access](https://serverlist.piaservers.net/vpninfo/servers/v6),
|
||||
downloaded once a day.
|
||||
- `protonvpn`: the /24 networks that hold ProtonVPN servers. The whole /24 is
|
||||
listed because the ProtonVPN-IPs README says traffic leaves from unpublished
|
||||
addresses in it. From the public
|
||||
[ProtonVPN-IPs](https://github.com/tn3w/ProtonVPN-IPs) mirror of Proton's
|
||||
server list, downloaded once a day.
|
||||
- `protonvpn_entry`: the IPv4 and IPv6 addresses of ProtonVPN servers. From the
|
||||
same [ProtonVPN-IPs](https://github.com/tn3w/ProtonVPN-IPs) mirror, downloaded
|
||||
once a day.
|
||||
- `spamhaus_drop` and `spamhaus_dropv6`: IPv4 and IPv6 networks hijacked or
|
||||
leased by spammers or cyber criminals, according to Spamhaus. From
|
||||
[Spamhaus DROP](https://www.spamhaus.org/blocklists/do-not-route-or-peer/),
|
||||
@@ -503,13 +490,11 @@ How RouteWatch keeps the answers:
|
||||
there.
|
||||
- It writes new answers every 5 seconds to `cache.db` in the state directory
|
||||
(see Configuration), and loads them from there at startup.
|
||||
- Each DNS record carries a TTL, the time for which it may be kept. An answer
|
||||
expires after the shortest TTL of its records. RouteWatch keeps it for at
|
||||
least 60 seconds and at most 24 hours.
|
||||
- When an address has no PTR names, the answer's SOA record says how long that
|
||||
answer may be kept. The answer expires after the SOA record's minimum field,
|
||||
or the record's own TTL when that is shorter. RouteWatch keeps it for at most
|
||||
1 hour.
|
||||
- An answer expires after the TTL of its records, the shortest one when there
|
||||
are several. RouteWatch keeps it for at least 60 seconds and at most 24 hours.
|
||||
- When an address has no PTR names, that answer expires after the SOA record's
|
||||
minimum field, or the record's own TTL when that is shorter. RouteWatch keeps
|
||||
it for at most 1 hour.
|
||||
- An answer with no PTR names and no SOA record is not kept. Neither is one
|
||||
whose SOA record has a TTL or minimum field of 0.
|
||||
- A failed lookup is not kept, so the next lookup of the address asks again.
|
||||
@@ -523,16 +508,11 @@ An IP lookup finds where each address is, on a best-effort basis. It uses two
|
||||
sources:
|
||||
|
||||
- [geojs](https://www.geojs.io/) is asked first. One request carries every
|
||||
address of the lookup that the cache does not hold. geojs has 1 second to
|
||||
answer.
|
||||
address of the lookup, and it has 1 second to answer.
|
||||
- [DB-IP City Lite](https://db-ip.com/db/download/ip-to-city-lite) answers when
|
||||
geojs has no answer for the address. It is a database file that RouteWatch
|
||||
geojs fails or does not answer in time. It is a database file that RouteWatch
|
||||
reads itself.
|
||||
|
||||
geojs has no answer when its request fails or runs out of time. An answer with
|
||||
no country, city or coordinates also counts as no answer, and RouteWatch does
|
||||
not cache it.
|
||||
|
||||
The `geo` object of a result has these fields:
|
||||
|
||||
- `country_code`: the two-letter country code
|
||||
@@ -554,13 +534,13 @@ goes to neither source, so its `geo` is null.
|
||||
|
||||
What goes outside: geojs receives each address that the cache does not hold. For
|
||||
`/ip/` and the home page, that address is the visitor's own. A lookup answered
|
||||
from the cache sends nothing. DB-IP receives only the monthly request for its
|
||||
file.
|
||||
from the cache or from the database sends nothing. DB-IP receives only the
|
||||
monthly request for its file.
|
||||
|
||||
How RouteWatch keeps the answers of geojs:
|
||||
|
||||
- It holds at most 250,000 addresses in memory, and drops the least recently
|
||||
used first. That takes about 100 MB of the Go heap.
|
||||
used first. That takes about 75 MB of the Go heap.
|
||||
- It writes new answers every 5 seconds to `cache.db` in the state directory.
|
||||
- It loads the newest 250,000 answers from `cache.db` at startup.
|
||||
- An address not in memory is read from `cache.db` before geojs is asked.
|
||||
@@ -581,8 +561,8 @@ How RouteWatch keeps the database:
|
||||
The kernel loads the parts that are read, and can drop them again when memory
|
||||
is short.
|
||||
|
||||
The home page's footer says "IP Geolocation by DB-IP" with a link to
|
||||
[db-ip.com](https://db-ip.com).
|
||||
DB-IP's licence requires an attribution, so the home page's footer says "IP
|
||||
Geolocation by DB-IP" with a link to [db-ip.com](https://db-ip.com).
|
||||
|
||||
## Configuration
|
||||
|
||||
@@ -684,13 +664,6 @@ The ceilings add up to 3.7 GiB:
|
||||
limit caps the SQLite library.
|
||||
- About 0.2 GiB for everything else in the runtime.
|
||||
|
||||
The `icloud_private_relay` list has about 285,000 networks. Neighbouring
|
||||
networks merge, which leaves about 6,900 address ranges. Loaded, the list adds
|
||||
about 0.3 MiB to the Go heap and under 6 MiB to the process's resident memory.
|
||||
Parsing the file allocates about 94 MiB in about 0.2 seconds. The next garbage
|
||||
collection frees all of it but the 0.3 MiB above. Parsing happens at startup and
|
||||
whenever a download brings a new version.
|
||||
|
||||
The SQLite library gets its memory from glibc, the C library. glibc keeps freed
|
||||
memory in pools called arenas, and hands it back to the kernel slowly. It
|
||||
creates up to eight arenas per processor core. So on a host with many cores the
|
||||
@@ -706,10 +679,8 @@ What happens at each limit:
|
||||
allocating more.
|
||||
- The SQLite hard heap limit: a database statement fails with an out-of-memory
|
||||
error, which the handler logs. The prefix handler writes the routing table's
|
||||
changes and its origin changes again in a later write. Everything else in a
|
||||
handler's failed write is dropped. The PTR cache and the geolocation part
|
||||
write their answers to `cache.db` again in a later write. The process keeps
|
||||
running.
|
||||
changes again in a later write. Everything else in the failed write is
|
||||
dropped. The process keeps running.
|
||||
- The handler queues: the handlers are the four parts that take the feed's
|
||||
messages (see Design). The prefix handler's queue holds at most 100,000
|
||||
messages, about 100 MiB. Each of the other three holds at most 20,000. When a
|
||||
@@ -1076,8 +1047,8 @@ calls one of them. The scripts:
|
||||
- `script/projectname`: print the project name, `routewatch`, which
|
||||
`script/docker` uses as the image tag.
|
||||
- `script/test`: build the `test` phase of the `Dockerfile` alone. It runs
|
||||
`go test -short -count=1 -timeout 90s -race -cover ./...`. On failure it runs
|
||||
the tests again with `-v` and without `-cover`.
|
||||
`go test -short -count=1 -timeout 90s -race -cover ./...`, and reruns it with
|
||||
`-v` on failure.
|
||||
- `script/lint`: build the `lint` phase of the `Dockerfile` alone. It runs
|
||||
`golangci-lint` v2.14.0 with `.golangci.yml`.
|
||||
- `script/fmt`: format the Go code with `go fmt` and the markdown with
|
||||
|
||||
@@ -22,76 +22,47 @@ formatting of the Go code and the markdown. Then it builds the image.
|
||||
The image sets memory ceilings for a 5 GiB container (README "Memory"). README
|
||||
"Running under upaas" says how to run it.
|
||||
|
||||
Five runs of `next` on the live feed have ended since 2026-10-27, from three
|
||||
hours to five days long. The runs found the handlers dropping updates for five
|
||||
causes, and the WHOIS fetcher asking a registry that had refused it. Each is
|
||||
fixed on `next`, and its Completed Steps entry below names its issue. A 21-hour
|
||||
run of `742e759`, which has every fix but one, dropped no update except after
|
||||
the feed stalled. That cause, https://git.eeqj.de/sneak/routewatch/issues/229,
|
||||
is fixed on `next`.
|
||||
A five-day run of `1c65656` on the live feed, from 2026-10-27 to 2026-11-01,
|
||||
found three problems:
|
||||
|
||||
A run of `e58c663` on the live feed began at 09:19 UTC on 2026-11-10
|
||||
(https://git.eeqj.de/sneak/routewatch/issues/232). It is to confirm that fix.
|
||||
Next Step lists what its outcome reports. The seven IP lookup additions of
|
||||
https://git.eeqj.de/sneak/routewatch/issues/203 landed after it began, so it
|
||||
does not exercise them.
|
||||
- The prefix handler dropped updates while a peer's session went down
|
||||
(https://git.eeqj.de/sneak/routewatch/issues/216).
|
||||
- The AS handler dropped updates when the feed burst
|
||||
(https://git.eeqj.de/sneak/routewatch/issues/217).
|
||||
- The WHOIS fetcher kept asking a registry that had refused it
|
||||
(https://git.eeqj.de/sneak/routewatch/issues/218).
|
||||
|
||||
All three are fixed on `next`. So is
|
||||
https://git.eeqj.de/sneak/routewatch/issues/222, which the third fix brought up.
|
||||
A run of `26d939a` on the live feed showed the prefix handler still dropping
|
||||
updates while the home page's cards were worked out.
|
||||
https://git.eeqj.de/sneak/routewatch/issues/226 fixed that on `next`.
|
||||
https://git.eeqj.de/sneak/routewatch/issues/225, also on `next`, takes the peers
|
||||
whose sessions go down together out of the table in one walk. A 24-hour run of
|
||||
`9bd0e91` on the live feed is checking whether any handler still drops updates.
|
||||
|
||||
The order of the project's remaining work is kept on
|
||||
https://git.eeqj.de/sneak/routewatch/issues/86.
|
||||
|
||||
# Next Step
|
||||
|
||||
Post the outcome of the run of `e58c663` on
|
||||
https://git.eeqj.de/sneak/routewatch/issues/232. The run lasts at least 24
|
||||
hours. If the feed has not stalled or reconnected by then, the run goes on until
|
||||
it does, up to 72 hours in all. The outcome says:
|
||||
|
||||
- whether the feed stalled or reconnected
|
||||
- whether any handler dropped an update, and the most messages the prefix
|
||||
handler's queue held at once
|
||||
- the most messages the prefix handler held back at once
|
||||
- the size of the `origin_changes` table on disk
|
||||
- the process's peak memory against 5 GiB
|
||||
|
||||
File an issue for each problem the run finds.
|
||||
Post the outcome of the 24-hour run of `9bd0e91` on
|
||||
https://git.eeqj.de/sneak/routewatch/issues/226: whether any handler still drops
|
||||
updates, and the process's peak memory against 5 GiB. File an issue for each
|
||||
problem the run finds.
|
||||
|
||||
# Completed Steps
|
||||
|
||||
- 2026-11-11: two more tests no longer fail on a busy host. The test of the home
|
||||
page's card refresh runs on a fake clock. The geo test in which geojs never
|
||||
answers counts each request as it is sent, not when it reaches the server
|
||||
(closes https://git.eeqj.de/sneak/routewatch/issues/256).
|
||||
- 2026-11-10: `github.com/mattn/go-sqlite3` is at v1.14.53, so building on the
|
||||
host no longer prints its two C compiler warnings (closes
|
||||
https://git.eeqj.de/sneak/routewatch/issues/245).
|
||||
- 2026-11-10: `README.md` and this file are read whole against the rules of
|
||||
https://git.eeqj.de/sneak/routewatch/issues/157, and every statement except
|
||||
measured figures and what outside services do is checked against the code on
|
||||
`next`. Four README sentences failed the prose rules. Five disagreed with the
|
||||
code. Each is rewritten. Status and Next Step describe the live-feed run in
|
||||
progress (closes https://git.eeqj.de/sneak/routewatch/issues/251).
|
||||
- 2026-11-10: the image creates the `routewatch` user as a regular user with UID
|
||||
1000, so the build no longer warns about the system UID range (closes
|
||||
https://git.eeqj.de/sneak/routewatch/issues/248).
|
||||
- 2026-11-10: each result of the IP lookup has `geo`, where the address is. It
|
||||
comes from geojs, with DB-IP's City Lite database as the fallback. The geojs
|
||||
answers stay in memory and in `cache.db` for 7 days (closes
|
||||
https://git.eeqj.de/sneak/routewatch/issues/239).
|
||||
- 2026-11-10: the IP lookup checks the `icloud_private_relay` list of Apple's
|
||||
iCloud Private Relay networks, downloaded once a day. Each result has
|
||||
`flags.is_relay` (closes https://git.eeqj.de/sneak/routewatch/issues/237).
|
||||
- 2026-11-10: the IP lookup checks two ProtonVPN lists from a public mirror of
|
||||
Proton's server list. `protonvpn` holds the /24 networks of ProtonVPN's
|
||||
servers, and `protonvpn_entry` holds the servers' IPv4 and IPv6 addresses
|
||||
(closes https://git.eeqj.de/sneak/routewatch/issues/236).
|
||||
- 2026-11-10: the IP lookup finds PTR names over DNS-over-HTTPS and caches them
|
||||
in memory and in `cache.db` for the record's TTL. A TTL is the time for which
|
||||
a DNS record may be kept (closes
|
||||
for the record's TTL, in memory and in `cache.db` (closes
|
||||
https://git.eeqj.de/sneak/routewatch/issues/238).
|
||||
- 2026-11-10: the tests no longer fail when a busy host stalls for several
|
||||
seconds. Most waits for something that should happen are now 15 seconds. Tests
|
||||
that depend on a short idle timeout or a request limit run on a fake clock
|
||||
(closes https://git.eeqj.de/sneak/routewatch/issues/240)
|
||||
seconds. Most such waits are now 15 seconds. The test of the stream's idle
|
||||
timeout runs on a fake clock (closes #240)
|
||||
- 2026-11-10: the IP lookup checks the public VPN server lists of Mullvad,
|
||||
NordVPN, Private Internet Access and IVPN, and X4BNet's community list. Each
|
||||
result has `flags.is_vpn` and `vpn_providers` (closes
|
||||
|
||||
@@ -9,7 +9,7 @@ require (
|
||||
github.com/google/btree v1.1.3
|
||||
github.com/google/uuid v1.6.0
|
||||
github.com/hashicorp/golang-lru/v2 v2.0.7
|
||||
github.com/mattn/go-sqlite3 v1.14.53
|
||||
github.com/mattn/go-sqlite3 v1.14.29
|
||||
github.com/maxmind/mmdbwriter v1.2.0
|
||||
github.com/miekg/dns v1.1.73
|
||||
github.com/oschwald/maxminddb-golang/v2 v2.9.0
|
||||
|
||||
@@ -10,8 +10,8 @@ github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
|
||||
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
|
||||
github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k=
|
||||
github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM=
|
||||
github.com/mattn/go-sqlite3 v1.14.53 h1:mnTM0owzUMrKuLo+0w51NUiB0ir2Fo7kFXlPt/l3BGs=
|
||||
github.com/mattn/go-sqlite3 v1.14.53/go.mod h1:6JTjA44L93a0QCyJef5YvlPoKXntQPjzWv5gtm9sB6w=
|
||||
github.com/mattn/go-sqlite3 v1.14.29 h1:1O6nRLJKvsi1H2Sj0Hzdfojwt8GiGKm+LOfLaBFaouQ=
|
||||
github.com/mattn/go-sqlite3 v1.14.29/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y=
|
||||
github.com/maxmind/mmdbwriter v1.2.0 h1:hyvDopImmgvle3aR8AaddxXnT0iQH2KWJX3vNfkwzYM=
|
||||
github.com/maxmind/mmdbwriter v1.2.0/go.mod h1:EQmKHhk2y9DRVvyNxwCLKC5FrkXZLx4snc5OlLY5XLE=
|
||||
github.com/miekg/dns v1.1.73 h1:uhT8nJxmTrPJYClxVxTCX+CVn6qnzSiybRk72Z6DgrE=
|
||||
|
||||
@@ -107,8 +107,7 @@ func nilIfEmpty(text string) *string {
|
||||
}
|
||||
|
||||
// fromDatabase returns where the copy of the database places addr. It returns
|
||||
// nil when there is no copy yet or the copy has no record for addr. It also
|
||||
// returns nil, and logs the error, when reading the copy fails.
|
||||
// nil when there is no copy yet or the copy has no record for addr.
|
||||
func (l *Locator) fromDatabase(addr netip.Addr) *Location {
|
||||
l.databaseMu.RLock()
|
||||
defer l.databaseMu.RUnlock()
|
||||
@@ -118,22 +117,13 @@ func (l *Locator) fromDatabase(addr netip.Addr) *Location {
|
||||
}
|
||||
|
||||
result := l.database.Lookup(addr)
|
||||
|
||||
err := result.Err()
|
||||
if err != nil {
|
||||
l.logger.Error("Failed to look up an address in the database",
|
||||
"address", addr, "error", err)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
if !result.Found() {
|
||||
return nil
|
||||
}
|
||||
|
||||
var record dbipRecord
|
||||
|
||||
err = result.Decode(&record)
|
||||
err := result.Decode(&record)
|
||||
if err != nil {
|
||||
l.logger.Error("Failed to read a database record",
|
||||
"address", addr, "error", err)
|
||||
|
||||
+2
-12
@@ -51,8 +51,7 @@ type geojsAnswer struct {
|
||||
}
|
||||
|
||||
// askGeoJS asks geojs about addresses in one request, and returns its answers
|
||||
// by address. An address geojs left out, or gave no country, city or
|
||||
// coordinates for, has no entry.
|
||||
// by address. An address geojs left out has no entry.
|
||||
func (l *Locator) askGeoJS(
|
||||
ctx context.Context, addresses []string,
|
||||
) (map[string]*Location, error) {
|
||||
@@ -98,21 +97,12 @@ func (l *Locator) askGeoJS(
|
||||
continue
|
||||
}
|
||||
|
||||
location := answer.location(fetched)
|
||||
if hasLocation(location) {
|
||||
answers[plainAddress(addr)] = location
|
||||
}
|
||||
answers[addr.Unmap().String()] = answer.location(fetched)
|
||||
}
|
||||
|
||||
return answers, nil
|
||||
}
|
||||
|
||||
// hasLocation says whether found has a country, a city or a coordinate.
|
||||
func hasLocation(found *Location) bool {
|
||||
return found.CountryCode != nil || found.Country != nil ||
|
||||
found.City != nil || found.Latitude != nil || found.Longitude != nil
|
||||
}
|
||||
|
||||
// location returns the answer as a Location fetched at the time given.
|
||||
func (a geojsAnswer) location(fetched time.Time) *Location {
|
||||
return &Location{
|
||||
|
||||
+4
-12
@@ -1,7 +1,6 @@
|
||||
// Package geo finds where IP addresses are. It asks geojs.io, and keeps those
|
||||
// answers in memory and in cache.db. When geojs.io gives no location for an
|
||||
// address, it reads a copy of DB-IP's City Lite database, which it downloads
|
||||
// once a month.
|
||||
// answers in memory and in cache.db. When geojs.io does not answer in time, it
|
||||
// reads a copy of DB-IP's City Lite database, which it downloads once a month.
|
||||
package geo
|
||||
|
||||
import (
|
||||
@@ -142,7 +141,7 @@ func (l *Locator) Lookup(ctx context.Context, addrs []netip.Addr) []*Location {
|
||||
var unanswered []string
|
||||
|
||||
for i, addr := range addrs {
|
||||
address := plainAddress(addr)
|
||||
address := addr.Unmap().String()
|
||||
|
||||
locations[i] = l.cached(ctx, address)
|
||||
if locations[i] == nil {
|
||||
@@ -164,7 +163,7 @@ func (l *Locator) Lookup(ctx context.Context, addrs []netip.Addr) []*Location {
|
||||
continue
|
||||
}
|
||||
|
||||
address := plainAddress(addr)
|
||||
address := addr.Unmap().String()
|
||||
|
||||
answer, ok := answers[address]
|
||||
if ok {
|
||||
@@ -180,13 +179,6 @@ func (l *Locator) Lookup(ctx context.Context, addrs []netip.Addr) []*Location {
|
||||
return locations
|
||||
}
|
||||
|
||||
// plainAddress returns addr as the requests to geojs and the cache name it: an
|
||||
// IPv4 address mapped into IPv6 as plain IPv4, and with no zone. A client
|
||||
// chooses the zone, so it must not reach the request.
|
||||
func plainAddress(addr netip.Addr) string {
|
||||
return addr.Unmap().WithZone("").String()
|
||||
}
|
||||
|
||||
// Start loads the cached answers that have not expired and opens the copy of
|
||||
// the database, if there is one. It then starts the goroutines that write new
|
||||
// answers, delete expired ones and download the database. They run until Stop:
|
||||
|
||||
@@ -53,18 +53,9 @@ const (
|
||||
"country": "Germany", "country_code": "DE", "ip": "` + testAddress + `",
|
||||
"latitude": "52.6171", "longitude": "13.1207",
|
||||
"region": "Brandenburg", "timezone": "Europe/Berlin"},
|
||||
{"country": "Germany", "ip": "` + otherAddress + `",
|
||||
"latitude": "nil", "longitude": "nil"}
|
||||
{"ip": "` + otherAddress + `", "latitude": "nil", "longitude": "nil"}
|
||||
]`
|
||||
geojsOneAddress = `[{"ip": "` + testAddress + `", "city": "` + testCity + `"}]`
|
||||
|
||||
// geojsNoLocation has no country, city or coordinates, as geojs sends for
|
||||
// some addresses. Its other parts do not count as a location.
|
||||
geojsNoLocation = `[{"accuracy": 1000, "area_code": "0", "asn": 64496,
|
||||
"continent_code": "EU", "ip": "` + testAddress + `",
|
||||
"latitude": "nil", "longitude": "nil",
|
||||
"organization": "AS64496 Example", "organization_name": "Example",
|
||||
"region": "Brandenburg", "timezone": "Europe/Berlin"}]`
|
||||
)
|
||||
|
||||
// fixedTime returns the time a test's clock starts at.
|
||||
@@ -90,12 +81,14 @@ func parseAddresses(t *testing.T, addresses ...string) []netip.Addr {
|
||||
return addrs
|
||||
}
|
||||
|
||||
// fakeGeoJS is a geojs server. It sends body, or status when that is set. It
|
||||
// keeps each request's address list.
|
||||
// fakeGeoJS is a geojs server. It sends body, or status when that is set, or
|
||||
// waits for the request to end when hang is true. It keeps each request's
|
||||
// address list.
|
||||
type fakeGeoJS struct {
|
||||
server *httptest.Server
|
||||
body string
|
||||
status int
|
||||
hang bool
|
||||
|
||||
mu sync.Mutex
|
||||
requests []string
|
||||
@@ -117,6 +110,12 @@ func (f *fakeGeoJS) serve(w http.ResponseWriter, r *http.Request) {
|
||||
f.requests = append(f.requests, r.URL.Query().Get("ip"))
|
||||
f.mu.Unlock()
|
||||
|
||||
if f.hang {
|
||||
<-r.Context().Done()
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
if f.status != 0 {
|
||||
w.WriteHeader(f.status)
|
||||
|
||||
@@ -134,20 +133,6 @@ func (f *fakeGeoJS) requestsSeen() []string {
|
||||
return append([]string{}, f.requests...)
|
||||
}
|
||||
|
||||
// hangingTransport never answers: each request waits until its context ends.
|
||||
// It counts the requests as they are sent, so the count does not depend on
|
||||
// whether one reaches a server before its time limit runs out.
|
||||
type hangingTransport struct {
|
||||
requests atomic.Int32
|
||||
}
|
||||
|
||||
func (h *hangingTransport) RoundTrip(r *http.Request) (*http.Response, error) {
|
||||
h.requests.Add(1)
|
||||
<-r.Context().Done()
|
||||
|
||||
return nil, r.Context().Err()
|
||||
}
|
||||
|
||||
// openCacheDB opens a new cache.db in a temporary directory.
|
||||
func openCacheDB(t *testing.T, dir string) *sql.DB {
|
||||
t.Helper()
|
||||
@@ -330,8 +315,9 @@ func TestLookupAsksGeoJSOnceForEveryAddress(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// TestLookupGivesUnknownPartsAsNil checks an answer with only a country name,
|
||||
// whose coordinates are the string "nil": the other parts are nil.
|
||||
// TestLookupGivesUnknownPartsAsNil checks an answer whose coordinates are the
|
||||
// string "nil" and which has no other part: the parts are nil, and the answer
|
||||
// is still an answer.
|
||||
func TestLookupGivesUnknownPartsAsNil(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
@@ -382,7 +368,6 @@ func TestLookupFallsBackToTheDatabase(t *testing.T) {
|
||||
{name: "error status", status: http.StatusBadGateway},
|
||||
{name: "unreadable answer", body: "not JSON"},
|
||||
{name: "address left out", body: "[]"},
|
||||
{name: "answer with no location", body: geojsNoLocation},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
@@ -391,15 +376,13 @@ func TestLookupFallsBackToTheDatabase(t *testing.T) {
|
||||
|
||||
fake := newFakeGeoJS(t, tt.body)
|
||||
fake.status = tt.status
|
||||
fake.hang = tt.hang
|
||||
dir := t.TempDir()
|
||||
locator := newTestLocator(t, fake, openCacheDB(t, dir), dir)
|
||||
requestCount := func() int { return len(fake.requestsSeen()) }
|
||||
locator.timeLimit = shortTimeLimit
|
||||
|
||||
if tt.hang {
|
||||
hanging := &hangingTransport{}
|
||||
locator.client = &http.Client{Transport: hanging}
|
||||
locator.timeLimit = shortTimeLimit
|
||||
requestCount = func() int { return int(hanging.requests.Load()) }
|
||||
if !tt.hang {
|
||||
locator.timeLimit = testTimeLimit
|
||||
}
|
||||
|
||||
giveDatabase(t, locator)
|
||||
@@ -421,100 +404,13 @@ func TestLookupFallsBackToTheDatabase(t *testing.T) {
|
||||
// asks geojs again.
|
||||
lookupOne(t, locator, testAddress)
|
||||
|
||||
if requests := requestCount(); requests != 2 {
|
||||
t.Errorf("%d requests, want 2", requests)
|
||||
if requests := fake.requestsSeen(); len(requests) != 2 {
|
||||
t.Errorf("%d requests, want 2", len(requests))
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestAnyCountryCityOrCoordinateIsAnAnswer checks that each of those parts,
|
||||
// alone, makes an answer from geojs that is returned and kept.
|
||||
func TestAnyCountryCityOrCoordinateIsAnAnswer(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
parts := []string{
|
||||
`"country": "Germany"`,
|
||||
`"country_code": "DE"`,
|
||||
`"city": "` + testCity + `"`,
|
||||
`"latitude": "52.5"`,
|
||||
`"longitude": "13.5"`,
|
||||
}
|
||||
|
||||
for _, part := range parts {
|
||||
t.Run(part, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
fake := newFakeGeoJS(t, `[{"ip": "`+testAddress+`", `+part+`}]`)
|
||||
dir := t.TempDir()
|
||||
locator := newTestLocator(t, fake, openCacheDB(t, dir), dir)
|
||||
|
||||
got := lookupOne(t, locator, testAddress)
|
||||
if got == nil || got.Source != sourceGeoJS {
|
||||
t.Fatalf("location %+v, want the answer from geojs", got)
|
||||
}
|
||||
|
||||
if !locator.entries.Contains(testAddress) {
|
||||
t.Error("the answer is not in memory")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestLookupSendsNoZoneToGeoJS looks up an address with a zone, which is text
|
||||
// a client chooses. geojs receives only the plain address, and the answer is
|
||||
// kept under it.
|
||||
func TestLookupSendsNoZoneToGeoJS(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
fake := newFakeGeoJS(t, geojsTwoAddresses)
|
||||
dir := t.TempDir()
|
||||
locator := newTestLocator(t, fake, openCacheDB(t, dir), dir)
|
||||
|
||||
got := lookupOne(t, locator, otherAddress+"%x&callback=evil")
|
||||
|
||||
requests := fake.requestsSeen()
|
||||
if len(requests) != 1 || requests[0] != otherAddress {
|
||||
t.Errorf("requests %v, want one naming only %s", requests, otherAddress)
|
||||
}
|
||||
|
||||
if got == nil || got.Source != sourceGeoJS {
|
||||
t.Errorf("location %+v, want the answer from geojs", got)
|
||||
}
|
||||
|
||||
if !locator.entries.Contains(otherAddress) {
|
||||
t.Error("the answer is not kept under the plain address")
|
||||
}
|
||||
}
|
||||
|
||||
// TestFailedDatabaseLookupIsLogged closes the copy under the locator, so that
|
||||
// the lookup in it fails. The location is nil and the failure is logged.
|
||||
func TestFailedDatabaseLookupIsLogged(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
fake := newFakeGeoJS(t, "[]")
|
||||
dir := t.TempDir()
|
||||
locator := newTestLocator(t, fake, openCacheDB(t, dir), dir)
|
||||
giveDatabase(t, locator)
|
||||
|
||||
err := locator.database.Close()
|
||||
if err != nil {
|
||||
t.Fatalf("closing the copy: %v", err)
|
||||
}
|
||||
|
||||
var logged bytes.Buffer
|
||||
|
||||
locator.logger = slog.New(slog.NewTextHandler(&logged, nil))
|
||||
|
||||
if got := locator.fromDatabase(parseAddresses(t, testAddress)[0]); got != nil {
|
||||
t.Errorf("location %+v from a failed lookup, want nil", got)
|
||||
}
|
||||
|
||||
if !strings.Contains(logged.String(), "address="+testAddress) {
|
||||
t.Errorf("the failure was not logged: %q", logged.String())
|
||||
}
|
||||
}
|
||||
|
||||
// TestLookupIsNilWhenNeitherSourceAnswers makes geojs fail with no database
|
||||
// copy, and then with a copy that does not cover the address.
|
||||
func TestLookupIsNilWhenNeitherSourceAnswers(t *testing.T) {
|
||||
|
||||
@@ -21,7 +21,7 @@ const (
|
||||
downloadTimeout = time.Minute
|
||||
|
||||
// maxDownloadSize is the size of the largest list file accepted, in
|
||||
// bytes. The largest list, iCloud Private Relay, is about 12 MiB.
|
||||
// bytes. The largest list, FireHOL abusers, is about 2.5 MiB.
|
||||
maxDownloadSize = 200 << 20
|
||||
)
|
||||
|
||||
|
||||
@@ -51,16 +51,6 @@ const (
|
||||
// FormatIVPNJSON is IVPN's JSON server list. The hosts of its wireguard
|
||||
// and openvpn gateways have their addresses in host and v2ray.
|
||||
FormatIVPNJSON
|
||||
|
||||
// FormatProtonVPN has one address or network per line, with comment lines
|
||||
// that start with #. Unlike FormatLines, a file with no addresses or
|
||||
// networks fails.
|
||||
FormatProtonVPN
|
||||
|
||||
// FormatICloudPrivateRelayCSV is Apple's CSV file of iCloud Private Relay
|
||||
// networks. Each line starts with a network, followed by its country,
|
||||
// region and city.
|
||||
FormatICloudPrivateRelayCSV
|
||||
)
|
||||
|
||||
// List is one public IP list.
|
||||
|
||||
@@ -70,31 +70,6 @@ const (
|
||||
"config":{"api":{"ips":["192.0.2.73"]}}}`
|
||||
)
|
||||
|
||||
// protonVPNBody is a small file in the format of the ProtonVPN-IPs list.
|
||||
const protonVPNBody = "#\n# protonvpn_ip_ranges.txt\n" +
|
||||
"# https://github.com/tn3w/ProtonVPN-IPs\n#\n" +
|
||||
"# The /24 block of every advertised IPv4.\n#\n" +
|
||||
"192.0.2.0/24\n198.51.100.0/24\n"
|
||||
|
||||
// protonVPNEntryBody is a small file in the format of the ProtonVPN-IPs list
|
||||
// of server addresses.
|
||||
const protonVPNEntryBody = "#\n# protonvpn_entry_ips.txt\n" +
|
||||
"# https://github.com/tn3w/ProtonVPN-IPs\n#\n" +
|
||||
"# Entry IPs clients connect to, resolved from ProtonVPN hostnames.\n#\n" +
|
||||
"192.0.2.80\n2001:db8::80\n"
|
||||
|
||||
// iCloudPrivateRelayAddr is an address on iCloudPrivateRelayBody.
|
||||
const iCloudPrivateRelayAddr = "2001:db8:1::5"
|
||||
|
||||
// iCloudPrivateRelayBody is a small file in the format of Apple's iCloud
|
||||
// Private Relay file.
|
||||
const iCloudPrivateRelayBody = "192.0.2.0/31,GB,GB-EN,London,\n" +
|
||||
"198.51.100.32/27,US,US-CA,Los Angeles,\n" +
|
||||
"2001:db8:1::/64,US,US-AK,Anchorage,\n"
|
||||
|
||||
// emptyFileCase names the case of a wrong document that is an empty file.
|
||||
const emptyFileCase = "an empty file"
|
||||
|
||||
// errorPage is a file that is no list, as a publisher's error page is.
|
||||
const errorPage = "<!DOCTYPE html>\n<html></html>\n"
|
||||
|
||||
@@ -255,15 +230,6 @@ func TestLookupFindsListedAddresses(t *testing.T) {
|
||||
},
|
||||
unlisted: []string{"192.0.2.12", "2001:db8::2"},
|
||||
},
|
||||
{
|
||||
name: "iCloud Private Relay networks",
|
||||
format: FormatICloudPrivateRelayCSV,
|
||||
body: iCloudPrivateRelayBody,
|
||||
listed: []string{
|
||||
"192.0.2.1", "198.51.100.63", iCloudPrivateRelayAddr,
|
||||
},
|
||||
unlisted: []string{"192.0.2.2", "198.51.100.64", "2001:db8:2::1"},
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
@@ -301,20 +267,6 @@ func TestLookupFindsVPNServers(t *testing.T) {
|
||||
listed: []string{"192.0.2.70", "192.0.2.71", "192.0.2.72"},
|
||||
unlisted: []string{"192.0.2.73", "2001:db8::70"},
|
||||
},
|
||||
{
|
||||
name: "ProtonVPN networks",
|
||||
format: FormatProtonVPN,
|
||||
body: protonVPNBody,
|
||||
listed: []string{"192.0.2.0", "192.0.2.255", "198.51.100.7"},
|
||||
unlisted: []string{"192.0.3.1", "2001:db8::90"},
|
||||
},
|
||||
{
|
||||
name: "ProtonVPN server addresses",
|
||||
format: FormatProtonVPN,
|
||||
body: protonVPNEntryBody,
|
||||
listed: []string{"192.0.2.80", "2001:db8::80"},
|
||||
unlisted: []string{"192.0.2.81", "2001:db8::81"},
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
@@ -506,12 +458,12 @@ func TestWrongVPNDocumentFailsToParse(t *testing.T) {
|
||||
|
||||
for _, f := range formats {
|
||||
bodies := map[string]string{
|
||||
"not JSON": errorPage,
|
||||
"other JSON": `{"status":"ok","data":{"count":1}}`,
|
||||
"no addresses": f.empty,
|
||||
"bad address": f.badAddress,
|
||||
"cut off": f.good[:len(f.good)/2],
|
||||
emptyFileCase: "",
|
||||
"not JSON": errorPage,
|
||||
"other JSON": `{"status":"ok","data":{"count":1}}`,
|
||||
"no addresses": f.empty,
|
||||
"bad address": f.badAddress,
|
||||
"cut off": f.good[:len(f.good)/2],
|
||||
"an empty file": "",
|
||||
}
|
||||
|
||||
for name, body := range bodies {
|
||||
@@ -527,59 +479,6 @@ func TestWrongVPNDocumentFailsToParse(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// TestWrongProtonVPNDocumentFailsToParse parses documents that are not a
|
||||
// ProtonVPN-IPs list. Each fails, so none can replace a good copy.
|
||||
func TestWrongProtonVPNDocumentFailsToParse(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
bodies := map[string]string{
|
||||
"error page": errorPage,
|
||||
"JSON": `{"status":"ok"}`,
|
||||
"comments only": "#\n# protonvpn_ip_ranges.txt\n#\n",
|
||||
"bad network": "192.0.2.0/24\nexample\n",
|
||||
emptyFileCase: "",
|
||||
}
|
||||
|
||||
for name, body := range bodies {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
addresses, err := parse(strings.NewReader(body), FormatProtonVPN)
|
||||
if err == nil {
|
||||
t.Errorf("parse returned %v and no error", addresses.Ranges())
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestWrongICloudPrivateRelayDocumentFailsToParse parses documents that are
|
||||
// not Apple's iCloud Private Relay file. Each fails, so none can replace a
|
||||
// good copy.
|
||||
func TestWrongICloudPrivateRelayDocumentFailsToParse(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
bodies := map[string]string{
|
||||
"error page": errorPage,
|
||||
"JSON": `{"status":"ok","count":1}`,
|
||||
"no other field": "192.0.2.0/24\n",
|
||||
"bad network": "192.0.2.0/24,GB,GB-EN,London,\nexample,GB,GB-EN,London,\n",
|
||||
"an address": "192.0.2.1,GB,GB-EN,London,\n",
|
||||
emptyFileCase: "",
|
||||
}
|
||||
|
||||
for name, body := range bodies {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
addresses, err := parse(strings.NewReader(body),
|
||||
FormatICloudPrivateRelayCSV)
|
||||
if err == nil {
|
||||
t.Errorf("parse returned %v and no error", addresses.Ranges())
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestLookupMarksVPNLists looks up an address on a VPN list and on another
|
||||
// list. Only the VPN list's listing says it is one.
|
||||
func TestLookupMarksVPNLists(t *testing.T) {
|
||||
@@ -624,10 +523,7 @@ func TestPublicListsMarkTheVPNLists(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
want := []string{
|
||||
"ivpn", "mullvad", "nordvpn", "pia", "protonvpn", "protonvpn_entry",
|
||||
"x4bnet", "x4bnet_v6",
|
||||
}
|
||||
want := []string{"ivpn", "mullvad", "nordvpn", "pia", "x4bnet", "x4bnet_v6"}
|
||||
if !slices.Equal(got, want) {
|
||||
t.Errorf("VPN lists %v, want %v", got, want)
|
||||
}
|
||||
@@ -934,33 +830,15 @@ func TestStopCancelsADownloadInProgress(t *testing.T) {
|
||||
func TestTorExitsListHoldsIPv6Exits(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
checkPublicListHolds(t, TorExitsList, onionooBody, onionooIPv6ExitAddr)
|
||||
}
|
||||
|
||||
// TestICloudPrivateRelayListHoldsItsNetworks downloads a file in Apple's
|
||||
// format as the public iCloud Private Relay list, which is what an address's
|
||||
// is_relay answer comes from.
|
||||
func TestICloudPrivateRelayListHoldsItsNetworks(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
checkPublicListHolds(t, ICloudPrivateRelayList, iCloudPrivateRelayBody,
|
||||
iCloudPrivateRelayAddr)
|
||||
}
|
||||
|
||||
// checkPublicListHolds downloads body as the public list called name and
|
||||
// checks that addr has one listing, which names that list.
|
||||
func checkPublicListHolds(t *testing.T, name, body, addr string) {
|
||||
t.Helper()
|
||||
|
||||
index := slices.IndexFunc(Lists(), func(list List) bool {
|
||||
return list.Name == name
|
||||
return list.Name == TorExitsList
|
||||
})
|
||||
if index < 0 {
|
||||
t.Fatalf("no list named %s", name)
|
||||
t.Fatalf("no list named %s", TorExitsList)
|
||||
}
|
||||
|
||||
list := Lists()[index]
|
||||
_, list.URL = newPublisher(t, body)
|
||||
_, list.URL = newPublisher(t, onionooBody)
|
||||
b := newTestIPLists(t.TempDir(), list)
|
||||
|
||||
err := b.download(t.Context(), list)
|
||||
@@ -968,9 +846,9 @@ func checkPublicListHolds(t *testing.T, name, body, addr string) {
|
||||
t.Fatalf("download: %v", err)
|
||||
}
|
||||
|
||||
listings := lookup(t, b, addr)
|
||||
if len(listings) != 1 || listings[0].Name != name {
|
||||
t.Errorf("listings %+v, want one for %s", listings, name)
|
||||
listings := lookup(t, b, onionooIPv6ExitAddr)
|
||||
if len(listings) != 1 || listings[0].Name != TorExitsList {
|
||||
t.Errorf("listings %+v, want one for %s", listings, TorExitsList)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -8,13 +8,6 @@ import (
|
||||
// TorExitsList is the name of the list of Tor exit nodes.
|
||||
const TorExitsList = "tor_exits"
|
||||
|
||||
// ICloudPrivateRelayList is the name of the list of iCloud Private Relay's
|
||||
// networks.
|
||||
const ICloudPrivateRelayList = "icloud_private_relay"
|
||||
|
||||
// protonVPNIPsURL is where the ProtonVPN-IPs mirror publishes its files.
|
||||
const protonVPNIPsURL = "https://raw.githubusercontent.com/tn3w/ProtonVPN-IPs/master/"
|
||||
|
||||
// How often each list is downloaded. No list is downloaded more often than
|
||||
// its publisher updates it or allows.
|
||||
const (
|
||||
@@ -34,9 +27,6 @@ const (
|
||||
fireHOLAbusersInterval = 5 * time.Hour
|
||||
fireHOLLevel3Interval = 3 * time.Hour
|
||||
|
||||
// Apple's iCloud Private Relay file is 12 MB, so once a day.
|
||||
iCloudPrivateRelayInterval = 24 * time.Hour
|
||||
|
||||
// Spamhaus asks that DROP be downloaded at most once a day:
|
||||
// https://www.spamhaus.org/blocklists/do-not-route-or-peer/
|
||||
spamhausDROPInterval = 24 * time.Hour
|
||||
@@ -91,13 +81,6 @@ func nonVPNLists() []List {
|
||||
Interval: fireHOLLevel3Interval,
|
||||
Format: FormatLines,
|
||||
},
|
||||
{
|
||||
Name: ICloudPrivateRelayList,
|
||||
Description: "In a network where iCloud Private Relay traffic leaves",
|
||||
URL: "https://mask-api.icloud.com/egress-ip-ranges.csv",
|
||||
Interval: iCloudPrivateRelayInterval,
|
||||
Format: FormatICloudPrivateRelayCSV,
|
||||
},
|
||||
{
|
||||
Name: "spamhaus_drop",
|
||||
Description: "In an IPv4 network that Spamhaus says spammers " +
|
||||
@@ -163,28 +146,6 @@ func vpnLists() []List {
|
||||
Format: FormatPIAJSON,
|
||||
VPN: true,
|
||||
},
|
||||
{
|
||||
// Proton's own server list needs a logged-in session, so this is
|
||||
// the public ProtonVPN-IPs mirror of it. Proton publishes only
|
||||
// the addresses clients connect to. The mirror's README says
|
||||
// traffic leaves from others in the same /24, so this file of
|
||||
// /24 networks is the one it says to use for detecting exits.
|
||||
Name: "protonvpn",
|
||||
Description: "In a /24 network that holds a server of the " +
|
||||
"ProtonVPN service",
|
||||
URL: protonVPNIPsURL + "protonvpn_ip_ranges.txt",
|
||||
Interval: vpnListInterval,
|
||||
Format: FormatProtonVPN,
|
||||
VPN: true,
|
||||
},
|
||||
{
|
||||
Name: "protonvpn_entry",
|
||||
Description: "A server of the ProtonVPN service",
|
||||
URL: protonVPNIPsURL + "protonvpn_entry_ips.txt",
|
||||
Interval: vpnListInterval,
|
||||
Format: FormatProtonVPN,
|
||||
VPN: true,
|
||||
},
|
||||
{
|
||||
Name: "x4bnet",
|
||||
Description: "In an IPv4 network of a VPN service, according to " +
|
||||
|
||||
@@ -25,10 +25,6 @@ const (
|
||||
// dshieldFieldSeparator separates the fields of a DShield line.
|
||||
dshieldFieldSeparator = "\t"
|
||||
|
||||
// iCloudPrivateRelayFieldSeparator separates the fields of a line of
|
||||
// Apple's iCloud Private Relay file.
|
||||
iCloudPrivateRelayFieldSeparator = ","
|
||||
|
||||
// spamhausMetadataType is the type of the line that ends a Spamhaus
|
||||
// file. That line holds the file's metadata and no network.
|
||||
spamhausMetadataType = "metadata"
|
||||
@@ -62,10 +58,6 @@ func parse(r io.Reader, format Format) (*netipx.IPSet, error) {
|
||||
return parsePIA(r)
|
||||
case FormatIVPNJSON:
|
||||
return parseIVPN(r)
|
||||
case FormatProtonVPN:
|
||||
return parseLinesNotEmpty(r, addAddressOrNetwork)
|
||||
case FormatICloudPrivateRelayCSV:
|
||||
return parseLinesNotEmpty(r, addICloudPrivateRelayNetwork)
|
||||
default:
|
||||
return nil, errUnknownFormat
|
||||
}
|
||||
@@ -337,41 +329,6 @@ func parseIVPN(r io.Reader) (*netipx.IPSet, error) {
|
||||
return addressSet(texts)
|
||||
}
|
||||
|
||||
// parseLinesNotEmpty is parseLines for a list that always holds addresses, so
|
||||
// a file with none fails.
|
||||
func parseLinesNotEmpty(
|
||||
r io.Reader, addLine func(*netipx.IPSetBuilder, string) error,
|
||||
) (*netipx.IPSet, error) {
|
||||
addresses, err := parseLines(r, addLine)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if len(addresses.Ranges()) == 0 {
|
||||
return nil, errNoAddresses
|
||||
}
|
||||
|
||||
return addresses, nil
|
||||
}
|
||||
|
||||
// addICloudPrivateRelayNetwork adds the network at the start of a line of
|
||||
// Apple's iCloud Private Relay file to builder.
|
||||
func addICloudPrivateRelayNetwork(builder *netipx.IPSetBuilder, line string) error {
|
||||
field, _, found := strings.Cut(line, iCloudPrivateRelayFieldSeparator)
|
||||
if !found {
|
||||
return errNoNetwork
|
||||
}
|
||||
|
||||
network, err := netip.ParsePrefix(field)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
builder.AddPrefix(network)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// readJSON reads r to its end and decodes it, as one JSON document, into v.
|
||||
func readJSON(r io.Reader, v any) error {
|
||||
data, err := io.ReadAll(r)
|
||||
|
||||
@@ -801,9 +801,6 @@ type IPLookupFlags struct {
|
||||
// IsVPN says that the address is on a list of VPN servers.
|
||||
IsVPN bool `json:"is_vpn"`
|
||||
|
||||
// IsRelay says that the address is on the iCloud Private Relay list.
|
||||
IsRelay bool `json:"is_relay"`
|
||||
|
||||
// IsBogon says that the address is reserved, as package bogon tells.
|
||||
IsBogon bool `json:"is_bogon"`
|
||||
}
|
||||
@@ -1036,9 +1033,6 @@ func ipLookupFlags(ip string, listings []iplist.Listing) IPLookupFlags {
|
||||
IsVPN: slices.ContainsFunc(listings, func(l iplist.Listing) bool {
|
||||
return l.VPN
|
||||
}),
|
||||
IsRelay: slices.ContainsFunc(listings, func(l iplist.Listing) bool {
|
||||
return l.Name == iplist.ICloudPrivateRelayList
|
||||
}),
|
||||
IsBogon: isBogon(ip),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1506,10 +1506,6 @@ const listedUnroutedAddr = "203.0.113.1"
|
||||
// torExitAddr is the address that fakeIPLists lists on the Tor exit list.
|
||||
const torExitAddr = "203.0.113.2"
|
||||
|
||||
// relayAddr is the address that fakeIPLists lists on the iCloud Private Relay
|
||||
// list.
|
||||
const relayAddr = "203.0.113.5"
|
||||
|
||||
// The addresses that fakeIPLists lists on one and on two VPN lists, and the
|
||||
// names of those lists.
|
||||
const (
|
||||
@@ -1519,11 +1515,10 @@ const (
|
||||
otherTestVPN = "other_test_vpn"
|
||||
)
|
||||
|
||||
// fakeIPLists has five lists. The test list holds addrInTestPrefixV4,
|
||||
// fakeIPLists has four lists. The test list holds addrInTestPrefixV4,
|
||||
// listedUnroutedAddr and twoVPNListsAddr. The Tor exit list holds torExitAddr.
|
||||
// The iCloud Private Relay list holds relayAddr. The test VPN list holds
|
||||
// oneVPNListAddr and twoVPNListsAddr, and the other test VPN list holds
|
||||
// twoVPNListsAddr.
|
||||
// The test VPN list holds oneVPNListAddr and twoVPNListsAddr, and the other
|
||||
// test VPN list holds twoVPNListsAddr.
|
||||
type fakeIPLists struct{}
|
||||
|
||||
func testListing() iplist.Listing {
|
||||
@@ -1544,10 +1539,6 @@ func (fakeIPLists) Lookup(addr netip.Addr) []iplist.Listing {
|
||||
return []iplist.Listing{{Name: iplist.TorExitsList}}
|
||||
}
|
||||
|
||||
if addr == netip.MustParseAddr(relayAddr) {
|
||||
return []iplist.Listing{{Name: iplist.ICloudPrivateRelayList}}
|
||||
}
|
||||
|
||||
if addr == netip.MustParseAddr(oneVPNListAddr) {
|
||||
return []iplist.Listing{{Name: testVPNList, VPN: true}}
|
||||
}
|
||||
@@ -1580,22 +1571,20 @@ func TestIPLookupFlags(t *testing.T) {
|
||||
name string
|
||||
addr string
|
||||
tor bool
|
||||
relay bool
|
||||
bogon bool
|
||||
}{
|
||||
{"on the Tor exit list", torExitAddr, true, false, true},
|
||||
{"on the iCloud Private Relay list", relayAddr, false, true, true},
|
||||
{"on another list", listedUnroutedAddr, false, false, true},
|
||||
{"routed IPv4", addrInTestPrefixV4, false, false, true},
|
||||
{"routed IPv6", addrInTestPrefixV6, false, false, true},
|
||||
{"private IPv4", "10.0.0.1", false, false, true},
|
||||
{"loopback IPv4", "127.0.0.1", false, false, true},
|
||||
{"public IPv4", "1.1.1.1", false, false, false},
|
||||
{"private IPv6", "fd00::1", false, false, true},
|
||||
{"loopback IPv6", "::1", false, false, true},
|
||||
{"public IPv6", "2606:4700:4700::1111", false, false, false},
|
||||
{"public IPv4 in IPv6 form", "::ffff:1.1.1.1", false, false, false},
|
||||
{"private IPv4 in IPv6 form", "::ffff:10.0.0.1", false, false, true},
|
||||
{"on the Tor exit list", torExitAddr, true, true},
|
||||
{"on another list", listedUnroutedAddr, false, true},
|
||||
{"routed IPv4", addrInTestPrefixV4, false, true},
|
||||
{"routed IPv6", addrInTestPrefixV6, false, true},
|
||||
{"private IPv4", "10.0.0.1", false, true},
|
||||
{"loopback IPv4", "127.0.0.1", false, true},
|
||||
{"public IPv4", "1.1.1.1", false, false},
|
||||
{"private IPv6", "fd00::1", false, true},
|
||||
{"loopback IPv6", "::1", false, true},
|
||||
{"public IPv6", "2606:4700:4700::1111", false, false},
|
||||
{"public IPv4 in IPv6 form", "::ffff:1.1.1.1", false, false},
|
||||
{"private IPv4 in IPv6 form", "::ffff:10.0.0.1", false, true},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
@@ -1617,8 +1606,7 @@ func TestIPLookupFlags(t *testing.T) {
|
||||
}
|
||||
|
||||
want := map[string]bool{
|
||||
"is_tor": tt.tor, "is_vpn": false, "is_relay": tt.relay,
|
||||
"is_bogon": tt.bogon,
|
||||
"is_tor": tt.tor, "is_vpn": false, "is_bogon": tt.bogon,
|
||||
}
|
||||
if !maps.Equal(got, want) {
|
||||
t.Errorf("%s: flags %v, want %v", path, got, want)
|
||||
|
||||
@@ -275,66 +275,49 @@ func TestPagesServeTheCopyOfTheCards(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// TestKeepHomeCardsFreshWorksThemOutEveryInterval runs keepHomeCardsFresh. It
|
||||
// works the cards out at once, again an interval after the table changes, and
|
||||
// stops when its context ends. It runs in a synctest bubble, whose clock is
|
||||
// fake, so a stall on the host cannot fail it.
|
||||
// TestKeepHomeCardsFreshWorksThemOutEveryInterval runs keepHomeCardsFresh with
|
||||
// a short interval. It works the cards out at once, again after the table
|
||||
// changes, and stops when its context ends.
|
||||
func TestKeepHomeCardsFreshWorksThemOutEveryInterval(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
const interval = time.Minute
|
||||
const (
|
||||
interval = 10 * time.Millisecond
|
||||
maxWait = 15 * time.Second
|
||||
)
|
||||
|
||||
table := emptyTable()
|
||||
announce(table, "1.0.0.0/24", twoPrefixASN, testPeer(1), testRouteTime(0))
|
||||
|
||||
s := newServerWithTable(t, table)
|
||||
|
||||
// newestRouteOrigin returns the origin AS of the newest route on the
|
||||
// cards, or 0 when there is none.
|
||||
newestRouteOrigin := func() int {
|
||||
cards := s.cards.Load()
|
||||
if cards == nil || cards.NewestRoute == nil {
|
||||
return 0
|
||||
}
|
||||
ctx, cancel := context.WithCancel(t.Context())
|
||||
stopped := make(chan struct{})
|
||||
|
||||
return cards.NewestRoute.OriginASN
|
||||
go func() {
|
||||
s.keepHomeCardsFresh(ctx, interval)
|
||||
close(stopped)
|
||||
}()
|
||||
|
||||
newestRouteASNIs := func(asn int) bool {
|
||||
cards := s.cards.Load()
|
||||
|
||||
return cards != nil && cards.NewestRoute != nil &&
|
||||
cards.NewestRoute.OriginASN == asn
|
||||
}
|
||||
|
||||
synctest.Test(t, func(t *testing.T) {
|
||||
ctx, cancel := context.WithCancel(t.Context())
|
||||
stopped := make(chan struct{})
|
||||
waitUntil(t, maxWait, func() bool { return newestRouteASNIs(twoPrefixASN) })
|
||||
|
||||
go func() {
|
||||
s.keepHomeCardsFresh(ctx, interval)
|
||||
close(stopped)
|
||||
}()
|
||||
announce(table, "8.8.8.0/24", newestRouteASN, testPeer(2), testRouteTime(1))
|
||||
waitUntil(t, maxWait, func() bool { return newestRouteASNIs(newestRouteASN) })
|
||||
|
||||
synctest.Wait()
|
||||
cancel()
|
||||
|
||||
if got := newestRouteOrigin(); got != twoPrefixASN {
|
||||
t.Fatalf("at the start, the newest route's AS is %d, want %d",
|
||||
got, twoPrefixASN)
|
||||
}
|
||||
|
||||
announce(table, "8.8.8.0/24", newestRouteASN, testPeer(2),
|
||||
testRouteTime(1))
|
||||
time.Sleep(interval)
|
||||
synctest.Wait()
|
||||
|
||||
if got := newestRouteOrigin(); got != newestRouteASN {
|
||||
t.Fatalf("an interval after the table changed, the newest "+
|
||||
"route's AS is %d, want %d", got, newestRouteASN)
|
||||
}
|
||||
|
||||
cancel()
|
||||
synctest.Wait()
|
||||
|
||||
select {
|
||||
case <-stopped:
|
||||
default:
|
||||
t.Fatal("keepHomeCardsFresh did not stop when its context ended")
|
||||
}
|
||||
})
|
||||
select {
|
||||
case <-stopped:
|
||||
case <-time.After(maxWait):
|
||||
t.Fatal("keepHomeCardsFresh did not stop when its context ended")
|
||||
}
|
||||
}
|
||||
|
||||
// TestStartedServerWorksOutTheCards starts a server and waits until its home
|
||||
|
||||
@@ -67,8 +67,8 @@ func TestFooterLicense(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// TestHomePageCreditsDBIP expects the home page's footer to carry the credit
|
||||
// to DB-IP.
|
||||
// TestHomePageCreditsDBIP expects the home page's footer to carry the
|
||||
// attribution the DB-IP licence requires.
|
||||
func TestHomePageCreditsDBIP(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
|
||||
Reference in new issue
Block a user