Compare commits
4
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
368fca9955 | ||
|
|
28ef0275ba | ||
|
|
3204b7ee4b | ||
|
|
cf50d195ad |
No files matched your search
@@ -285,8 +285,12 @@ address. Every entry has the same fields:
|
|||||||
empty list when it has none
|
empty list when it has none
|
||||||
- `flags`: yes-or-no facts about the address:
|
- `flags`: yes-or-no facts about the address:
|
||||||
- `is_tor`: true when the address is on the `tor_exits` list
|
- `is_tor`: true when the address is on the `tor_exits` list
|
||||||
|
- `is_vpn`: true when the address is on any list of VPN servers (see IP
|
||||||
|
lists)
|
||||||
- `is_bogon`: true when the address is reserved, so that no network should
|
- `is_bogon`: true when the address is reserved, so that no network should
|
||||||
announce it on the internet
|
announce it on the internet
|
||||||
|
- `vpn_providers`: the names of the lists of VPN servers the address is on, or
|
||||||
|
an empty list when it is on none
|
||||||
- `ip_lists`: the public IP lists the address is on (see IP lists), or an empty
|
- `ip_lists`: the public IP lists the address is on (see IP lists), or an empty
|
||||||
list when it is on none
|
list when it is on none
|
||||||
|
|
||||||
@@ -420,13 +424,27 @@ The lists:
|
|||||||
last 30 days. From
|
last 30 days. From
|
||||||
[FireHOL](https://iplists.firehol.org/?ipset=firehol_level3), downloaded every
|
[FireHOL](https://iplists.firehol.org/?ipset=firehol_level3), downloaded every
|
||||||
3 hours.
|
3 hours.
|
||||||
|
- `ivpn`: VPN servers run by IVPN. From
|
||||||
|
[IVPN](https://api.ivpn.net/v5/servers.json), downloaded once a day.
|
||||||
|
- `mullvad`: VPN servers run by Mullvad. From
|
||||||
|
[Mullvad](https://api.mullvad.net/www/relays/all/), downloaded once a day.
|
||||||
|
- `nordvpn`: VPN servers run by NordVPN. From
|
||||||
|
[NordVPN](https://api.nordvpn.com/v1/servers?limit=0&fields[servers.station]&fields[servers.ipv6_station]&fields[servers.hostname]),
|
||||||
|
downloaded once a day.
|
||||||
|
- `pia`: VPN servers run by Private Internet Access. From
|
||||||
|
[Private Internet Access](https://serverlist.piaservers.net/vpninfo/servers/v6),
|
||||||
|
downloaded once a day.
|
||||||
- `spamhaus_drop` and `spamhaus_dropv6`: IPv4 and IPv6 networks hijacked or
|
- `spamhaus_drop` and `spamhaus_dropv6`: IPv4 and IPv6 networks hijacked or
|
||||||
leased by spammers or cyber criminals, according to Spamhaus. From
|
leased by spammers or cyber criminals, according to Spamhaus. From
|
||||||
[Spamhaus DROP](https://www.spamhaus.org/blocklists/do-not-route-or-peer/),
|
[Spamhaus DROP](https://www.spamhaus.org/blocklists/do-not-route-or-peer/),
|
||||||
downloaded once a day.
|
downloaded once a day.
|
||||||
- `tor_exits`: Tor exit nodes, where traffic from the Tor network leaves. From
|
- `tor_exits`: the IPv4 and IPv6 addresses of Tor exit nodes, where traffic from
|
||||||
the [Tor Project](https://check.torproject.org/torbulkexitlist), downloaded
|
the Tor network leaves. From the Tor Project's
|
||||||
|
[Onionoo](https://metrics.torproject.org/onionoo.html) service, downloaded
|
||||||
every hour.
|
every hour.
|
||||||
|
- `x4bnet` and `x4bnet_v6`: IPv4 and IPv6 networks of VPN services, according to
|
||||||
|
a community list that names no service. From
|
||||||
|
[X4BNet](https://github.com/X4BNet/lists_vpn), downloaded once a day.
|
||||||
|
|
||||||
No list is downloaded more often than its publisher updates it or allows. Each
|
No list is downloaded more often than its publisher updates it or allows. Each
|
||||||
can be downloaded whole without an account or a key.
|
can be downloaded whole without an account or a key.
|
||||||
|
|||||||
@@ -53,6 +53,16 @@ problem the run finds.
|
|||||||
|
|
||||||
# Completed Steps
|
# Completed Steps
|
||||||
|
|
||||||
|
- 2026-11-10: the tests no longer fail when a busy host stalls for several
|
||||||
|
seconds. Most such waits are now 15 seconds. The test of the stream's idle
|
||||||
|
timeout runs on a fake clock (closes #240)
|
||||||
|
- 2026-11-10: the IP lookup checks the public VPN server lists of Mullvad,
|
||||||
|
NordVPN, Private Internet Access and IVPN, and X4BNet's community list. Each
|
||||||
|
result has `flags.is_vpn` and `vpn_providers` (closes
|
||||||
|
https://git.eeqj.de/sneak/routewatch/issues/235).
|
||||||
|
- 2026-11-10: `tor_exits` comes from Onionoo, so it holds IPv6 exits and
|
||||||
|
`is_tor` is true for them (closes
|
||||||
|
https://git.eeqj.de/sneak/routewatch/issues/234).
|
||||||
- 2026-11-10: each result of the IP lookup has a `flags` object with `is_tor`
|
- 2026-11-10: each result of the IP lookup has a `flags` object with `is_tor`
|
||||||
and `is_bogon`. The public IP lists are named `ip_lists` in the answer,
|
and `is_bogon`. The public IP lists are named `ip_lists` in the answer,
|
||||||
`internal/iplist` in the code and `iplists` in the state directory. A list
|
`internal/iplist` in the code and `iplists` in the state directory. A list
|
||||||
|
|||||||
@@ -12,7 +12,7 @@ import (
|
|||||||
|
|
||||||
// parseTimeLimit is how long a test waits for parseCSV to return before it
|
// parseTimeLimit is how long a test waits for parseCSV to return before it
|
||||||
// fails.
|
// fails.
|
||||||
const parseTimeLimit = 2 * time.Second
|
const parseTimeLimit = 15 * time.Second
|
||||||
|
|
||||||
// TestParseCSVReadsTodaysLayout parses an as.csv with the country code column
|
// TestParseCSVReadsTodaysLayout parses an as.csv with the country code column
|
||||||
// that the source added after the description. It expects each AS with its
|
// that the source added after the description. It expects each AS with its
|
||||||
|
|||||||
@@ -72,17 +72,34 @@ const (
|
|||||||
// lockTestPrefix is the prefix the batch writes in the checkpoint tests
|
// lockTestPrefix is the prefix the batch writes in the checkpoint tests
|
||||||
// store.
|
// store.
|
||||||
lockTestPrefix = "192.0.2.0/24"
|
lockTestPrefix = "192.0.2.0/24"
|
||||||
|
// longBusyTimeoutMs is the busy timeout setLongBusyTimeout gives the
|
||||||
|
// writing connection. A checkpoint that waits for an open read takes this
|
||||||
|
// long.
|
||||||
|
longBusyTimeoutMs = 30000
|
||||||
// readerCheckpointLimit is how long a checkpoint, and a batch write
|
// readerCheckpointLimit is how long a checkpoint, and a batch write
|
||||||
// waiting for it, may take while a reader is open: well under the busy
|
// waiting for it, may take while a reader is open: well under
|
||||||
// timeout, which a checkpoint that waits for the reader takes.
|
// longBusyTimeoutMs, with room for a stall on a busy host.
|
||||||
readerCheckpointLimit = time.Second
|
readerCheckpointLimit = 15 * time.Second
|
||||||
// truncateCheckpointLimit is how long a checkpoint of a WAL file over
|
// truncateCheckpointLimit is how long a checkpoint of a WAL file over
|
||||||
// sqliteWALSizeLimitBytes may take to give up while a reader is open: its
|
// sqliteWALSizeLimitBytes may take to give up while a reader is open: its
|
||||||
// wait for the reader plus a second, well under the busy timeout.
|
// wait for the reader plus readerCheckpointLimit.
|
||||||
truncateCheckpointLimit = truncateBusyTimeoutMs*time.Millisecond +
|
truncateCheckpointLimit = truncateBusyTimeoutMs*time.Millisecond +
|
||||||
time.Second
|
readerCheckpointLimit
|
||||||
)
|
)
|
||||||
|
|
||||||
|
// setLongBusyTimeout gives the writing connection of db longBusyTimeoutMs, so
|
||||||
|
// that a checkpoint that waits for an open read takes far longer than a stall
|
||||||
|
// on a busy host.
|
||||||
|
func setLongBusyTimeout(ctx context.Context, t *testing.T, db *Database) {
|
||||||
|
t.Helper()
|
||||||
|
|
||||||
|
_, err := db.writer.ExecContext(ctx,
|
||||||
|
fmt.Sprintf("PRAGMA busy_timeout=%d", longBusyTimeoutMs))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("failed to set the busy timeout: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// TestConnectionPoolPragmas holds several pooled connections open at once and
|
// TestConnectionPoolPragmas holds several pooled connections open at once and
|
||||||
// checks each one carries the per-connection settings from the DSN, plus the
|
// checks each one carries the per-connection settings from the DSN, plus the
|
||||||
// process-wide heap limits, and sees the new file with auto_vacuum
|
// process-wide heap limits, and sees the new file with auto_vacuum
|
||||||
@@ -579,6 +596,8 @@ func TestCheckpointDoesNotWaitForReader(t *testing.T) {
|
|||||||
ctx := context.Background()
|
ctx := context.Background()
|
||||||
ts := time.Now().UTC()
|
ts := time.Now().UTC()
|
||||||
|
|
||||||
|
setLongBusyTimeout(ctx, t, db)
|
||||||
|
|
||||||
err = db.UpdatePrefixesBatch(map[string]time.Time{"198.51.100.0/24": ts})
|
err = db.UpdatePrefixesBatch(map[string]time.Time{"198.51.100.0/24": ts})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("failed to store the prefix the reader reads: %v", err)
|
t.Fatalf("failed to store the prefix the reader reads: %v", err)
|
||||||
@@ -690,6 +709,7 @@ func TestCheckpointEmptiesWALOnceNoReadIsOpen(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
fillWALPastLimit(ctx, t, db)
|
fillWALPastLimit(ctx, t, db)
|
||||||
|
setLongBusyTimeout(ctx, t, db)
|
||||||
|
|
||||||
started := time.Now()
|
started := time.Now()
|
||||||
|
|
||||||
|
|||||||
@@ -29,6 +29,28 @@ const (
|
|||||||
// start with the network's first and last address. Lines that start
|
// start with the network's first and last address. Lines that start
|
||||||
// with # are comments.
|
// with # are comments.
|
||||||
FormatDShield
|
FormatDShield
|
||||||
|
|
||||||
|
// FormatOnionooJSON is one Onionoo details document. Its relays have
|
||||||
|
// addresses in two fields: exit_addresses, which has IPv4 only, and
|
||||||
|
// or_addresses, which has IPv4 and IPv6, each with a port.
|
||||||
|
FormatOnionooJSON
|
||||||
|
|
||||||
|
// FormatMullvadJSON is Mullvad's JSON list of relays. A relay's addresses
|
||||||
|
// are in ipv4_addr_in, ipv6_addr_in and ipv4_v2ray.
|
||||||
|
FormatMullvadJSON
|
||||||
|
|
||||||
|
// FormatNordVPNJSON is NordVPN's JSON list of servers. A server's
|
||||||
|
// addresses are in station and ipv6_station.
|
||||||
|
FormatNordVPNJSON
|
||||||
|
|
||||||
|
// FormatPIAJSON is Private Internet Access's JSON server list, then a
|
||||||
|
// blank line and a signature. Each region's servers has lists of servers
|
||||||
|
// by protocol, and each server has its address in ip.
|
||||||
|
FormatPIAJSON
|
||||||
|
|
||||||
|
// FormatIVPNJSON is IVPN's JSON server list. The hosts of its wireguard
|
||||||
|
// and openvpn gateways have their addresses in host and v2ray.
|
||||||
|
FormatIVPNJSON
|
||||||
)
|
)
|
||||||
|
|
||||||
// List is one public IP list.
|
// List is one public IP list.
|
||||||
@@ -48,6 +70,9 @@ type List struct {
|
|||||||
|
|
||||||
// Format is how the list's file is written.
|
// Format is how the list's file is written.
|
||||||
Format Format
|
Format Format
|
||||||
|
|
||||||
|
// VPN says that the list holds servers of VPN services.
|
||||||
|
VPN bool
|
||||||
}
|
}
|
||||||
|
|
||||||
// Listing says that a list contains an address.
|
// Listing says that a list contains an address.
|
||||||
@@ -58,6 +83,10 @@ type Listing struct {
|
|||||||
// Updated is when routewatch last downloaded the list, or last heard from
|
// Updated is when routewatch last downloaded the list, or last heard from
|
||||||
// its publisher that its copy was still current.
|
// its publisher that its copy was still current.
|
||||||
Updated time.Time `json:"updated"`
|
Updated time.Time `json:"updated"`
|
||||||
|
|
||||||
|
// VPN says that the list holds servers of VPN services. A lookup answer
|
||||||
|
// reports it through is_vpn and vpn_providers.
|
||||||
|
VPN bool `json:"-"`
|
||||||
}
|
}
|
||||||
|
|
||||||
// listCopy is a list's last good copy in memory.
|
// listCopy is a list's last good copy in memory.
|
||||||
@@ -124,6 +153,7 @@ func (b *IPLists) Lookup(addr netip.Addr) []Listing {
|
|||||||
Name: list.Name,
|
Name: list.Name,
|
||||||
Description: list.Description,
|
Description: list.Description,
|
||||||
Updated: c.updated,
|
Updated: c.updated,
|
||||||
|
VPN: list.VPN,
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -11,6 +11,7 @@ import (
|
|||||||
"net/netip"
|
"net/netip"
|
||||||
"os"
|
"os"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
|
"slices"
|
||||||
"strings"
|
"strings"
|
||||||
"sync"
|
"sync"
|
||||||
"testing"
|
"testing"
|
||||||
@@ -28,6 +29,50 @@ const (
|
|||||||
newListBody = "# A test list\n" + newListedAddr + "\n"
|
newListBody = "# A test list\n" + newListedAddr + "\n"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
// A small Onionoo details document: a relay with an exit address and IPv4 and
|
||||||
|
// IPv6 relay addresses, one without exit addresses, and one with a list of
|
||||||
|
// ports.
|
||||||
|
const (
|
||||||
|
onionooExitAddr = "192.0.2.10"
|
||||||
|
onionooIPv6ExitAddr = "2001:db8::1"
|
||||||
|
onionooBody = `{"version":"8.0","relays":[
|
||||||
|
{"exit_addresses":["` + onionooExitAddr + `"],
|
||||||
|
"or_addresses":["192.0.2.11:443","[` + onionooIPv6ExitAddr + `]:9001"]},
|
||||||
|
{"or_addresses":["192.0.2.20:443"]},
|
||||||
|
{"or_addresses":["192.0.2.30:80,443"]}
|
||||||
|
],"bridges":[]}`
|
||||||
|
)
|
||||||
|
|
||||||
|
// Small documents in the VPN services' formats. PIA's and IVPN's have no
|
||||||
|
// public IPv6 addresses, and IVPN's holds addresses that are not its servers,
|
||||||
|
// which must not be read.
|
||||||
|
const (
|
||||||
|
mullvadBody = `[
|
||||||
|
{"hostname":"a-wg-001","ipv4_addr_in":"192.0.2.40","ipv6_addr_in":"2001:db8::40"},
|
||||||
|
{"hostname":"a-br-001","ipv4_addr_in":"192.0.2.41","ipv6_addr_in":null,
|
||||||
|
"ipv4_v2ray":"192.0.2.42"}
|
||||||
|
]`
|
||||||
|
nordVPNBody = `[
|
||||||
|
{"hostname":"a1.example.com","station":"192.0.2.50","ipv6_station":""},
|
||||||
|
{"hostname":"a2.example.com","station":"192.0.2.51","ipv6_station":"2001:db8::51"}
|
||||||
|
]`
|
||||||
|
piaBody = `{"groups":{"wg":[{"name":"wireguard","ports":[1337]}]},"regions":[
|
||||||
|
{"id":"a","servers":{
|
||||||
|
"wg":[{"ip":"192.0.2.60","cn":"a1"}],
|
||||||
|
"ikev2":[{"ip":"192.0.2.61","cn":"a1"}]}},
|
||||||
|
{"id":"b","servers":{"meta":[{"ip":"192.0.2.62","cn":"b1","van":true}]}}
|
||||||
|
]}` + "\n\nc2lnbmF0dXJl\nbGluZSB0d28=\n"
|
||||||
|
ivpnBody = `{"wireguard":[{"gateway":"a.wg.example.com","hosts":[
|
||||||
|
{"hostname":"a1","host":"192.0.2.70","v2ray":"192.0.2.71",
|
||||||
|
"ipv6":{"local_ip":"2001:db8::70/96"}}]}],
|
||||||
|
"openvpn":[{"gateway":"a.gw.example.com","hosts":[
|
||||||
|
{"hostname":"a1","host":"192.0.2.72"}]}],
|
||||||
|
"config":{"api":{"ips":["192.0.2.73"]}}}`
|
||||||
|
)
|
||||||
|
|
||||||
|
// errorPage is a file that is no list, as a publisher's error page is.
|
||||||
|
const errorPage = "<!DOCTYPE html>\n<html></html>\n"
|
||||||
|
|
||||||
// testETag is the ETag a publisher sends with a version of the test list.
|
// testETag is the ETag a publisher sends with a version of the test list.
|
||||||
const testETag = `"v1"`
|
const testETag = `"v1"`
|
||||||
|
|
||||||
@@ -143,13 +188,7 @@ func lookup(t *testing.T, b *IPLists, addr string) []Listing {
|
|||||||
func TestLookupFindsListedAddresses(t *testing.T) {
|
func TestLookupFindsListedAddresses(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
tests := []struct {
|
runLookupCases(t, []lookupCase{
|
||||||
name string
|
|
||||||
format Format
|
|
||||||
body string
|
|
||||||
listed []string
|
|
||||||
unlisted []string
|
|
||||||
}{
|
|
||||||
{
|
{
|
||||||
name: "single addresses",
|
name: "single addresses",
|
||||||
format: FormatLines,
|
format: FormatLines,
|
||||||
@@ -181,9 +220,70 @@ func TestLookupFindsListedAddresses(t *testing.T) {
|
|||||||
listed: []string{"192.0.2.0", "192.0.2.77"},
|
listed: []string{"192.0.2.0", "192.0.2.77"},
|
||||||
unlisted: []string{"192.0.3.1", "2001:db8::1"},
|
unlisted: []string{"192.0.3.1", "2001:db8::1"},
|
||||||
},
|
},
|
||||||
}
|
{
|
||||||
|
name: "Onionoo relays",
|
||||||
|
format: FormatOnionooJSON,
|
||||||
|
body: onionooBody,
|
||||||
|
listed: []string{
|
||||||
|
onionooExitAddr, "192.0.2.11", "192.0.2.20", "192.0.2.30",
|
||||||
|
onionooIPv6ExitAddr,
|
||||||
|
},
|
||||||
|
unlisted: []string{"192.0.2.12", "2001:db8::2"},
|
||||||
|
},
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
for _, tt := range tests {
|
// TestLookupFindsVPNServers downloads a list in each VPN service's format and
|
||||||
|
// looks up addresses on it and off it.
|
||||||
|
func TestLookupFindsVPNServers(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
runLookupCases(t, []lookupCase{
|
||||||
|
{
|
||||||
|
name: "Mullvad relays",
|
||||||
|
format: FormatMullvadJSON,
|
||||||
|
body: mullvadBody,
|
||||||
|
listed: []string{"192.0.2.40", "192.0.2.41", "192.0.2.42", "2001:db8::40"},
|
||||||
|
unlisted: []string{"192.0.2.43", "2001:db8::41"},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "NordVPN servers",
|
||||||
|
format: FormatNordVPNJSON,
|
||||||
|
body: nordVPNBody,
|
||||||
|
listed: []string{"192.0.2.50", "192.0.2.51", "2001:db8::51"},
|
||||||
|
unlisted: []string{"192.0.2.52", "2001:db8::50"},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "PIA servers, with the signature after the JSON",
|
||||||
|
format: FormatPIAJSON,
|
||||||
|
body: piaBody,
|
||||||
|
listed: []string{"192.0.2.60", "192.0.2.61", "192.0.2.62"},
|
||||||
|
unlisted: []string{"192.0.2.63", "2001:db8::60"},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "IVPN hosts",
|
||||||
|
format: FormatIVPNJSON,
|
||||||
|
body: ivpnBody,
|
||||||
|
listed: []string{"192.0.2.70", "192.0.2.71", "192.0.2.72"},
|
||||||
|
unlisted: []string{"192.0.2.73", "2001:db8::70"},
|
||||||
|
},
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
// lookupCase is a list file, the addresses it holds and some it does not.
|
||||||
|
type lookupCase struct {
|
||||||
|
name string
|
||||||
|
format Format
|
||||||
|
body string
|
||||||
|
listed []string
|
||||||
|
unlisted []string
|
||||||
|
}
|
||||||
|
|
||||||
|
// runLookupCases downloads each case's list file and checks its addresses.
|
||||||
|
func runLookupCases(t *testing.T, cases []lookupCase) {
|
||||||
|
t.Helper()
|
||||||
|
|
||||||
|
for _, tt := range cases {
|
||||||
t.Run(tt.name, func(t *testing.T) {
|
t.Run(tt.name, func(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
@@ -195,26 +295,34 @@ func TestLookupFindsListedAddresses(t *testing.T) {
|
|||||||
t.Fatalf("download: %v", err)
|
t.Fatalf("download: %v", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
for _, addr := range tt.listed {
|
checkLookups(t, b, tt.listed, tt.unlisted)
|
||||||
listings := lookup(t, b, addr)
|
|
||||||
if len(listings) != 1 || listings[0].Name != testListName ||
|
|
||||||
listings[0].Description != testListDescription ||
|
|
||||||
listings[0].Updated.IsZero() {
|
|
||||||
t.Errorf("%s: listings %+v, want one for %s", addr,
|
|
||||||
listings, testListName)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, addr := range tt.unlisted {
|
|
||||||
listings := lookup(t, b, addr)
|
|
||||||
if listings == nil || len(listings) != 0 {
|
|
||||||
t.Errorf("%s: listings %#v, want an empty list", addr, listings)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// checkLookups checks that each of listed has one listing that names the test
|
||||||
|
// list, and each of unlisted has an empty list of listings.
|
||||||
|
func checkLookups(t *testing.T, b *IPLists, listed, unlisted []string) {
|
||||||
|
t.Helper()
|
||||||
|
|
||||||
|
for _, addr := range listed {
|
||||||
|
listings := lookup(t, b, addr)
|
||||||
|
if len(listings) != 1 || listings[0].Name != testListName ||
|
||||||
|
listings[0].Description != testListDescription ||
|
||||||
|
listings[0].Updated.IsZero() {
|
||||||
|
t.Errorf("%s: listings %+v, want one for %s", addr,
|
||||||
|
listings, testListName)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, addr := range unlisted {
|
||||||
|
listings := lookup(t, b, addr)
|
||||||
|
if listings == nil || len(listings) != 0 {
|
||||||
|
t.Errorf("%s: listings %#v, want an empty list", addr, listings)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// TestFailedDownloadKeepsThePreviousCopy downloads a good version of a list,
|
// TestFailedDownloadKeepsThePreviousCopy downloads a good version of a list,
|
||||||
// then one that fails. The good copy stays in memory and on disk, and no
|
// then one that fails. The good copy stays in memory and on disk, and no
|
||||||
// part of the failed download is left beside it.
|
// part of the failed download is left beside it.
|
||||||
@@ -227,7 +335,7 @@ func TestFailedDownloadKeepsThePreviousCopy(t *testing.T) {
|
|||||||
body string
|
body string
|
||||||
}{
|
}{
|
||||||
{"server error", http.StatusInternalServerError, "unavailable"},
|
{"server error", http.StatusInternalServerError, "unavailable"},
|
||||||
{"not a list", http.StatusOK, "<!DOCTYPE html>\n<html></html>\n"},
|
{"not a list", http.StatusOK, errorPage},
|
||||||
}
|
}
|
||||||
|
|
||||||
for _, tt := range tests {
|
for _, tt := range tests {
|
||||||
@@ -263,6 +371,164 @@ func TestFailedDownloadKeepsThePreviousCopy(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// TestFileThatIsNotAnOnionooDocumentKeepsThePreviousCopy downloads a good
|
||||||
|
// Onionoo document, then files that are not one. Each fails the parse, and
|
||||||
|
// the good copy stays.
|
||||||
|
func TestFileThatIsNotAnOnionooDocumentKeepsThePreviousCopy(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
body string
|
||||||
|
}{
|
||||||
|
{"not JSON", errorPage},
|
||||||
|
{"other JSON", `{"status":"ok","data":{"relays_count":1}}`},
|
||||||
|
{"JSON array", `[{"or_addresses":["192.0.2.1:443"]}]`},
|
||||||
|
{"no relays", `{"relays":[],"bridges":[]}`},
|
||||||
|
{"relays without addresses", `{"relays":[{"nickname":"example"}]}`},
|
||||||
|
{"relay address without a port", `{"relays":[{"or_addresses":["192.0.2.1"]}]}`},
|
||||||
|
{"exit address that is not one", `{"relays":[{"exit_addresses":["example"]}]}`},
|
||||||
|
{"text after the document", `{"relays":[{"or_addresses":["192.0.2.1:1"]}]} x`},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tt := range tests {
|
||||||
|
t.Run(tt.name, func(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
dir := t.TempDir()
|
||||||
|
p, url := newPublisher(t, onionooBody)
|
||||||
|
b := newTestIPLists(dir, testList(url, FormatOnionooJSON))
|
||||||
|
|
||||||
|
err := b.download(t.Context(), b.lists[0])
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("first download: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
p.answer(http.StatusOK, tt.body)
|
||||||
|
|
||||||
|
err = b.download(t.Context(), b.lists[0])
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("second download returned no error")
|
||||||
|
}
|
||||||
|
|
||||||
|
if len(lookup(t, b, onionooExitAddr)) != 1 {
|
||||||
|
t.Errorf("%s is no longer listed", onionooExitAddr)
|
||||||
|
}
|
||||||
|
|
||||||
|
onDisk, err := fs.ReadFile(os.DirFS(dir), testListName)
|
||||||
|
if err != nil || string(onDisk) != onionooBody {
|
||||||
|
t.Errorf("copy on disk %q (%v), want the good version", onDisk, err)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestWrongVPNDocumentFailsToParse parses documents that are not a VPN
|
||||||
|
// service's list. Each fails, so none can replace a good copy.
|
||||||
|
func TestWrongVPNDocumentFailsToParse(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
formats := []struct {
|
||||||
|
name string
|
||||||
|
format Format
|
||||||
|
good string
|
||||||
|
empty string
|
||||||
|
badAddress string
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
"Mullvad", FormatMullvadJSON, mullvadBody,
|
||||||
|
`[{"hostname":"a"}]`, `[{"ipv4_addr_in":"example"}]`,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"NordVPN", FormatNordVPNJSON, nordVPNBody,
|
||||||
|
`[{"hostname":"a","station":"","ipv6_station":""}]`,
|
||||||
|
`[{"station":"example"}]`,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"PIA", FormatPIAJSON, piaBody,
|
||||||
|
`{"regions":[{"id":"a","servers":{}}]}`,
|
||||||
|
`{"regions":[{"servers":{"wg":[{"ip":"example"}]}}]}`,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"IVPN", FormatIVPNJSON, ivpnBody,
|
||||||
|
`{"wireguard":[{"hosts":[]}],"openvpn":[]}`,
|
||||||
|
`{"wireguard":[{"hosts":[{"host":"example"}]}]}`,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, f := range formats {
|
||||||
|
bodies := map[string]string{
|
||||||
|
"not JSON": errorPage,
|
||||||
|
"other JSON": `{"status":"ok","data":{"count":1}}`,
|
||||||
|
"no addresses": f.empty,
|
||||||
|
"bad address": f.badAddress,
|
||||||
|
"cut off": f.good[:len(f.good)/2],
|
||||||
|
"an empty file": "",
|
||||||
|
}
|
||||||
|
|
||||||
|
for name, body := range bodies {
|
||||||
|
t.Run(f.name+" "+name, func(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
addresses, err := parse(strings.NewReader(body), f.format)
|
||||||
|
if err == nil {
|
||||||
|
t.Errorf("parse returned %v and no error", addresses.Ranges())
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestLookupMarksVPNLists looks up an address on a VPN list and on another
|
||||||
|
// list. Only the VPN list's listing says it is one.
|
||||||
|
func TestLookupMarksVPNLists(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
_, url := newPublisher(t, goodListBody)
|
||||||
|
vpnList := testList(url, FormatLines)
|
||||||
|
vpnList.Name = "test_vpn_list"
|
||||||
|
vpnList.VPN = true
|
||||||
|
otherList := testList(url, FormatLines)
|
||||||
|
|
||||||
|
b := New(Params{
|
||||||
|
Dir: t.TempDir(),
|
||||||
|
Lists: []List{vpnList, otherList},
|
||||||
|
Logger: slog.New(slog.DiscardHandler),
|
||||||
|
})
|
||||||
|
|
||||||
|
for _, list := range b.lists {
|
||||||
|
err := b.download(t.Context(), list)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("download of %s: %v", list.Name, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
listings := lookup(t, b, listedAddr)
|
||||||
|
if len(listings) != 2 || !listings[0].VPN || listings[1].VPN {
|
||||||
|
t.Errorf("listings %+v, want the first of two marked as a VPN list",
|
||||||
|
listings)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestPublicListsMarkTheVPNLists checks which public lists are VPN lists,
|
||||||
|
// since their names are what vpn_providers reports.
|
||||||
|
func TestPublicListsMarkTheVPNLists(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
var got []string
|
||||||
|
|
||||||
|
for _, list := range Lists() {
|
||||||
|
if list.VPN {
|
||||||
|
got = append(got, list.Name)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
want := []string{"ivpn", "mullvad", "nordvpn", "pia", "x4bnet", "x4bnet_v6"}
|
||||||
|
if !slices.Equal(got, want) {
|
||||||
|
t.Errorf("VPN lists %v, want %v", got, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// checkGoodCopyOnDisk checks that dir holds the good version of the test list
|
// checkGoodCopyOnDisk checks that dir holds the good version of the test list
|
||||||
// and no part of a failed download.
|
// and no part of a failed download.
|
||||||
func checkGoodCopyOnDisk(t *testing.T, dir string) {
|
func checkGoodCopyOnDisk(t *testing.T, dir string) {
|
||||||
@@ -380,9 +646,13 @@ func TestRestartLoadsCopiesFromDisk(t *testing.T) {
|
|||||||
second.Start(t.Context())
|
second.Start(t.Context())
|
||||||
second.Stop(t.Context())
|
second.Stop(t.Context())
|
||||||
|
|
||||||
|
// The download takes its time after writing the file, and the restart
|
||||||
|
// reads the file's time. A busy host can put seconds between the two.
|
||||||
|
const maxUpdatedDifference = 15 * time.Second
|
||||||
|
|
||||||
listings := lookup(t, second, listedAddr)
|
listings := lookup(t, second, listedAddr)
|
||||||
if len(listings) != 1 ||
|
if len(listings) != 1 ||
|
||||||
listings[0].Updated.Sub(downloaded).Abs() > time.Second {
|
listings[0].Updated.Sub(downloaded).Abs() > maxUpdatedDifference {
|
||||||
t.Errorf("after the restart, listings %+v, want one updated at %s",
|
t.Errorf("after the restart, listings %+v, want one updated at %s",
|
||||||
listings, downloaded)
|
listings, downloaded)
|
||||||
}
|
}
|
||||||
@@ -555,6 +825,33 @@ func TestStopCancelsADownloadInProgress(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// TestTorExitsListHoldsIPv6Exits downloads an Onionoo document as the public
|
||||||
|
// Tor exit list, which is what an IPv6 address's is_tor answer comes from.
|
||||||
|
func TestTorExitsListHoldsIPv6Exits(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
index := slices.IndexFunc(Lists(), func(list List) bool {
|
||||||
|
return list.Name == TorExitsList
|
||||||
|
})
|
||||||
|
if index < 0 {
|
||||||
|
t.Fatalf("no list named %s", TorExitsList)
|
||||||
|
}
|
||||||
|
|
||||||
|
list := Lists()[index]
|
||||||
|
_, list.URL = newPublisher(t, onionooBody)
|
||||||
|
b := newTestIPLists(t.TempDir(), list)
|
||||||
|
|
||||||
|
err := b.download(t.Context(), list)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("download: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
listings := lookup(t, b, onionooIPv6ExitAddr)
|
||||||
|
if len(listings) != 1 || listings[0].Name != TorExitsList {
|
||||||
|
t.Errorf("listings %+v, want one for %s", listings, TorExitsList)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// TestPublicListsAreComplete checks that every public list has what a
|
// TestPublicListsAreComplete checks that every public list has what a
|
||||||
// download and a lookup answer need, and a unique name.
|
// download and a lookup answer need, and a unique name.
|
||||||
func TestPublicListsAreComplete(t *testing.T) {
|
func TestPublicListsAreComplete(t *testing.T) {
|
||||||
|
|||||||
@@ -1,6 +1,9 @@
|
|||||||
package iplist
|
package iplist
|
||||||
|
|
||||||
import "time"
|
import (
|
||||||
|
"slices"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
// TorExitsList is the name of the list of Tor exit nodes.
|
// TorExitsList is the name of the list of Tor exit nodes.
|
||||||
const TorExitsList = "tor_exits"
|
const TorExitsList = "tor_exits"
|
||||||
@@ -28,15 +31,24 @@ const (
|
|||||||
// https://www.spamhaus.org/blocklists/do-not-route-or-peer/
|
// https://www.spamhaus.org/blocklists/do-not-route-or-peer/
|
||||||
spamhausDROPInterval = 24 * time.Hour
|
spamhausDROPInterval = 24 * time.Hour
|
||||||
|
|
||||||
// Tor publishes a new exit list every hour and sets no download limit:
|
// Onionoo's relay data follows Tor's hourly network consensus, and
|
||||||
// https://collector.torproject.org/recent/exit-lists/
|
// Onionoo sets no download limit:
|
||||||
|
// https://metrics.torproject.org/onionoo.html
|
||||||
torExitsInterval = time.Hour
|
torExitsInterval = time.Hour
|
||||||
|
|
||||||
|
// A VPN service's servers change slowly, so once a day is enough.
|
||||||
|
vpnListInterval = 24 * time.Hour
|
||||||
)
|
)
|
||||||
|
|
||||||
// Lists returns the public IP lists routewatch downloads. Each can be
|
// Lists returns the public IP lists routewatch downloads. Each can be
|
||||||
// downloaded whole without an account or a key. Spamhaus merged its EDROP
|
// downloaded whole without an account or a key. Spamhaus merged its EDROP
|
||||||
// list into DROP, so DROP covers both.
|
// list into DROP, so DROP covers both.
|
||||||
func Lists() []List {
|
func Lists() []List {
|
||||||
|
return slices.Concat(nonVPNLists(), vpnLists())
|
||||||
|
}
|
||||||
|
|
||||||
|
// nonVPNLists returns the public lists that are not lists of VPN servers.
|
||||||
|
func nonVPNLists() []List {
|
||||||
return []List{
|
return []List{
|
||||||
{
|
{
|
||||||
Name: "dshield",
|
Name: "dshield",
|
||||||
@@ -88,9 +100,71 @@ func Lists() []List {
|
|||||||
{
|
{
|
||||||
Name: TorExitsList,
|
Name: TorExitsList,
|
||||||
Description: "A Tor exit node, where traffic from the Tor network leaves",
|
Description: "A Tor exit node, where traffic from the Tor network leaves",
|
||||||
URL: "https://check.torproject.org/torbulkexitlist",
|
URL: "https://onionoo.torproject.org/details?" +
|
||||||
Interval: torExitsInterval,
|
"flag=Exit&running=true&fields=exit_addresses,or_addresses",
|
||||||
Format: FormatLines,
|
Interval: torExitsInterval,
|
||||||
|
Format: FormatOnionooJSON,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// vpnLists returns the public lists of VPN servers. Their names are what
|
||||||
|
// vpn_providers reports.
|
||||||
|
func vpnLists() []List {
|
||||||
|
return []List{
|
||||||
|
{
|
||||||
|
Name: "ivpn",
|
||||||
|
Description: "A server of the IVPN service",
|
||||||
|
URL: "https://api.ivpn.net/v5/servers.json",
|
||||||
|
Interval: vpnListInterval,
|
||||||
|
Format: FormatIVPNJSON,
|
||||||
|
VPN: true,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
Name: "mullvad",
|
||||||
|
Description: "A server of the Mullvad VPN service",
|
||||||
|
URL: "https://api.mullvad.net/www/relays/all/",
|
||||||
|
Interval: vpnListInterval,
|
||||||
|
Format: FormatMullvadJSON,
|
||||||
|
VPN: true,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
Name: "nordvpn",
|
||||||
|
Description: "A server of the NordVPN service",
|
||||||
|
URL: "https://api.nordvpn.com/v1/servers?limit=0" +
|
||||||
|
"&fields[servers.station]&fields[servers.ipv6_station]" +
|
||||||
|
"&fields[servers.hostname]",
|
||||||
|
Interval: vpnListInterval,
|
||||||
|
Format: FormatNordVPNJSON,
|
||||||
|
VPN: true,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
Name: "pia",
|
||||||
|
Description: "A server of the Private Internet Access VPN service",
|
||||||
|
URL: "https://serverlist.piaservers.net/vpninfo/servers/v6",
|
||||||
|
Interval: vpnListInterval,
|
||||||
|
Format: FormatPIAJSON,
|
||||||
|
VPN: true,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
Name: "x4bnet",
|
||||||
|
Description: "In an IPv4 network of a VPN service, according to " +
|
||||||
|
"X4BNet's community list",
|
||||||
|
URL: "https://raw.githubusercontent.com/X4BNet/lists_vpn/main/" +
|
||||||
|
"output/vpn/ipv4.txt",
|
||||||
|
Interval: vpnListInterval,
|
||||||
|
Format: FormatLines,
|
||||||
|
VPN: true,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
Name: "x4bnet_v6",
|
||||||
|
Description: "In an IPv6 network of a VPN service, according to " +
|
||||||
|
"X4BNet's community list",
|
||||||
|
URL: "https://raw.githubusercontent.com/X4BNet/lists_vpn/main/" +
|
||||||
|
"output/vpn/ipv6.txt",
|
||||||
|
Interval: vpnListInterval,
|
||||||
|
Format: FormatLines,
|
||||||
|
VPN: true,
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
+250
-16
@@ -6,7 +6,9 @@ import (
|
|||||||
"errors"
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
"io"
|
"io"
|
||||||
|
"net"
|
||||||
"net/netip"
|
"net/netip"
|
||||||
|
"slices"
|
||||||
"strings"
|
"strings"
|
||||||
|
|
||||||
"go4.org/netipx"
|
"go4.org/netipx"
|
||||||
@@ -32,12 +34,40 @@ var (
|
|||||||
errUnknownFormat = errors.New("unknown list format")
|
errUnknownFormat = errors.New("unknown list format")
|
||||||
errNoNetwork = errors.New("no network on the line")
|
errNoNetwork = errors.New("no network on the line")
|
||||||
errBadRange = errors.New("first and last address make no range")
|
errBadRange = errors.New("first and last address make no range")
|
||||||
|
errNoAddresses = errors.New("no addresses in the document")
|
||||||
)
|
)
|
||||||
|
|
||||||
// parse reads a list file written in format and returns the addresses it
|
// parse reads a list file written in format and returns the addresses it
|
||||||
// holds. A line it cannot read fails the whole file, so a file that is not
|
// holds. Anything it cannot read fails the whole file, so a file that is not
|
||||||
// the list, such as an error page, never replaces a good copy.
|
// the list, such as an error page, never replaces a good copy.
|
||||||
func parse(r io.Reader, format Format) (*netipx.IPSet, error) {
|
func parse(r io.Reader, format Format) (*netipx.IPSet, error) {
|
||||||
|
switch format {
|
||||||
|
case FormatLines:
|
||||||
|
return parseLines(r, addAddressOrNetwork)
|
||||||
|
case FormatSpamhausJSON:
|
||||||
|
return parseLines(r, addSpamhausNetwork)
|
||||||
|
case FormatDShield:
|
||||||
|
return parseLines(r, addDShieldNetwork)
|
||||||
|
case FormatOnionooJSON:
|
||||||
|
return parseOnionoo(r)
|
||||||
|
case FormatMullvadJSON:
|
||||||
|
return parseMullvad(r)
|
||||||
|
case FormatNordVPNJSON:
|
||||||
|
return parseNordVPN(r)
|
||||||
|
case FormatPIAJSON:
|
||||||
|
return parsePIA(r)
|
||||||
|
case FormatIVPNJSON:
|
||||||
|
return parseIVPN(r)
|
||||||
|
default:
|
||||||
|
return nil, errUnknownFormat
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// parseLines reads a list file line by line, skipping blank lines and
|
||||||
|
// comments, and calls addLine with each other line.
|
||||||
|
func parseLines(
|
||||||
|
r io.Reader, addLine func(*netipx.IPSetBuilder, string) error,
|
||||||
|
) (*netipx.IPSet, error) {
|
||||||
var builder netipx.IPSetBuilder
|
var builder netipx.IPSetBuilder
|
||||||
|
|
||||||
scanner := bufio.NewScanner(r)
|
scanner := bufio.NewScanner(r)
|
||||||
@@ -48,7 +78,7 @@ func parse(r io.Reader, format Format) (*netipx.IPSet, error) {
|
|||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
|
||||||
err := addLine(&builder, line, format)
|
err := addLine(&builder, line)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("line %d: %w", lineNumber, err)
|
return nil, fmt.Errorf("line %d: %w", lineNumber, err)
|
||||||
}
|
}
|
||||||
@@ -62,20 +92,6 @@ func parse(r io.Reader, format Format) (*netipx.IPSet, error) {
|
|||||||
return builder.IPSet()
|
return builder.IPSet()
|
||||||
}
|
}
|
||||||
|
|
||||||
// addLine adds the address or network on one line of a list file to builder.
|
|
||||||
func addLine(builder *netipx.IPSetBuilder, line string, format Format) error {
|
|
||||||
switch format {
|
|
||||||
case FormatLines:
|
|
||||||
return addAddressOrNetwork(builder, line)
|
|
||||||
case FormatSpamhausJSON:
|
|
||||||
return addSpamhausNetwork(builder, line)
|
|
||||||
case FormatDShield:
|
|
||||||
return addDShieldNetwork(builder, line)
|
|
||||||
default:
|
|
||||||
return errUnknownFormat
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// addAddressOrNetwork adds an address, such as 192.0.2.1, or a network, such
|
// addAddressOrNetwork adds an address, such as 192.0.2.1, or a network, such
|
||||||
// as 192.0.2.0/24, to builder.
|
// as 192.0.2.0/24, to builder.
|
||||||
func addAddressOrNetwork(builder *netipx.IPSetBuilder, text string) error {
|
func addAddressOrNetwork(builder *netipx.IPSetBuilder, text string) error {
|
||||||
@@ -156,3 +172,221 @@ func addDShieldNetwork(builder *netipx.IPSetBuilder, line string) error {
|
|||||||
|
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// parseOnionoo returns every address in the exit_addresses and or_addresses
|
||||||
|
// fields of the relays in an Onionoo details document. A document with no
|
||||||
|
// addresses fails, because the Tor network always has relays.
|
||||||
|
func parseOnionoo(r io.Reader) (*netipx.IPSet, error) {
|
||||||
|
//nolint:tagliatelle // Onionoo's own field names
|
||||||
|
var details struct {
|
||||||
|
Relays []struct {
|
||||||
|
ExitAddresses []string `json:"exit_addresses"`
|
||||||
|
ORAddresses []string `json:"or_addresses"`
|
||||||
|
} `json:"relays"`
|
||||||
|
}
|
||||||
|
|
||||||
|
err := readJSON(r, &details)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
var builder netipx.IPSetBuilder
|
||||||
|
|
||||||
|
for _, relay := range details.Relays {
|
||||||
|
for _, text := range relay.ExitAddresses {
|
||||||
|
err = addAddress(&builder, text)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, text := range relay.ORAddresses {
|
||||||
|
err = addAddressWithPort(&builder, text)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
addresses, err := builder.IPSet()
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
if len(addresses.Ranges()) == 0 {
|
||||||
|
return nil, errNoAddresses
|
||||||
|
}
|
||||||
|
|
||||||
|
return addresses, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// parseMullvad returns the addresses of the relays in a Mullvad document. A
|
||||||
|
// relay with no IPv6 address has null for it, and only bridges have
|
||||||
|
// ipv4_v2ray.
|
||||||
|
func parseMullvad(r io.Reader) (*netipx.IPSet, error) {
|
||||||
|
//nolint:tagliatelle // Mullvad's own field names
|
||||||
|
var relays []struct {
|
||||||
|
IPv4 string `json:"ipv4_addr_in"`
|
||||||
|
IPv6 string `json:"ipv6_addr_in"`
|
||||||
|
IPv4V2Ray string `json:"ipv4_v2ray"`
|
||||||
|
}
|
||||||
|
|
||||||
|
err := readJSON(r, &relays)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
var texts []string
|
||||||
|
|
||||||
|
for _, relay := range relays {
|
||||||
|
texts = append(texts, relay.IPv4, relay.IPv6, relay.IPv4V2Ray)
|
||||||
|
}
|
||||||
|
|
||||||
|
return addressSet(texts)
|
||||||
|
}
|
||||||
|
|
||||||
|
// parseNordVPN returns the addresses of the servers in a NordVPN document. A
|
||||||
|
// server with no IPv6 address has an empty ipv6_station.
|
||||||
|
func parseNordVPN(r io.Reader) (*netipx.IPSet, error) {
|
||||||
|
//nolint:tagliatelle // NordVPN's own field names
|
||||||
|
var servers []struct {
|
||||||
|
Station string `json:"station"`
|
||||||
|
IPv6Station string `json:"ipv6_station"`
|
||||||
|
}
|
||||||
|
|
||||||
|
err := readJSON(r, &servers)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
var texts []string
|
||||||
|
|
||||||
|
for _, server := range servers {
|
||||||
|
texts = append(texts, server.Station, server.IPv6Station)
|
||||||
|
}
|
||||||
|
|
||||||
|
return addressSet(texts)
|
||||||
|
}
|
||||||
|
|
||||||
|
// parsePIA returns the addresses of the servers in a Private Internet Access
|
||||||
|
// document. json.Decoder stops at the end of the JSON, so the signature after
|
||||||
|
// it is not read.
|
||||||
|
func parsePIA(r io.Reader) (*netipx.IPSet, error) {
|
||||||
|
var list struct {
|
||||||
|
Regions []struct {
|
||||||
|
Servers map[string][]struct {
|
||||||
|
IP string `json:"ip"`
|
||||||
|
} `json:"servers"`
|
||||||
|
} `json:"regions"`
|
||||||
|
}
|
||||||
|
|
||||||
|
err := json.NewDecoder(r).Decode(&list)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
var texts []string
|
||||||
|
|
||||||
|
for _, region := range list.Regions {
|
||||||
|
for _, servers := range region.Servers {
|
||||||
|
for _, server := range servers {
|
||||||
|
texts = append(texts, server.IP)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return addressSet(texts)
|
||||||
|
}
|
||||||
|
|
||||||
|
// parseIVPN returns the addresses of the hosts in an IVPN document. A host's
|
||||||
|
// ipv6 object holds only an address inside the VPN, so it is not read.
|
||||||
|
func parseIVPN(r io.Reader) (*netipx.IPSet, error) {
|
||||||
|
type gateway struct {
|
||||||
|
Hosts []struct {
|
||||||
|
Host string `json:"host"`
|
||||||
|
V2Ray string `json:"v2ray"`
|
||||||
|
} `json:"hosts"`
|
||||||
|
}
|
||||||
|
|
||||||
|
var list struct {
|
||||||
|
WireGuard []gateway `json:"wireguard"`
|
||||||
|
OpenVPN []gateway `json:"openvpn"`
|
||||||
|
}
|
||||||
|
|
||||||
|
err := readJSON(r, &list)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
var texts []string
|
||||||
|
|
||||||
|
for _, gw := range slices.Concat(list.WireGuard, list.OpenVPN) {
|
||||||
|
for _, host := range gw.Hosts {
|
||||||
|
texts = append(texts, host.Host, host.V2Ray)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return addressSet(texts)
|
||||||
|
}
|
||||||
|
|
||||||
|
// readJSON reads r to its end and decodes it, as one JSON document, into v.
|
||||||
|
func readJSON(r io.Reader, v any) error {
|
||||||
|
data, err := io.ReadAll(r)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
return json.Unmarshal(data, v)
|
||||||
|
}
|
||||||
|
|
||||||
|
// addressSet returns the set of the addresses in texts, skipping empty ones.
|
||||||
|
// A list with no addresses fails, because every VPN service has servers.
|
||||||
|
func addressSet(texts []string) (*netipx.IPSet, error) {
|
||||||
|
var builder netipx.IPSetBuilder
|
||||||
|
|
||||||
|
for _, text := range texts {
|
||||||
|
if text == "" {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
err := addAddress(&builder, text)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
addresses, err := builder.IPSet()
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
if len(addresses.Ranges()) == 0 {
|
||||||
|
return nil, errNoAddresses
|
||||||
|
}
|
||||||
|
|
||||||
|
return addresses, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// addAddress adds an address, such as 192.0.2.1, to builder.
|
||||||
|
func addAddress(builder *netipx.IPSetBuilder, text string) error {
|
||||||
|
addr, err := netip.ParseAddr(text)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
builder.Add(addr)
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// addAddressWithPort adds the address of text, such as 192.0.2.1:443 or
|
||||||
|
// [2001:db8::1]:443, to builder. Onionoo allows a list of ports after the
|
||||||
|
// address, so the part after the address is not read.
|
||||||
|
func addAddressWithPort(builder *netipx.IPSetBuilder, text string) error {
|
||||||
|
host, _, err := net.SplitHostPort(text)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
return addAddress(builder, host)
|
||||||
|
}
|
||||||
@@ -781,6 +781,10 @@ type IPLookupResult struct {
|
|||||||
|
|
||||||
Flags IPLookupFlags `json:"flags"`
|
Flags IPLookupFlags `json:"flags"`
|
||||||
|
|
||||||
|
// VPNProviders names the VPN lists the address is on. It is empty, never
|
||||||
|
// nil, when there are none.
|
||||||
|
VPNProviders []string `json:"vpn_providers"`
|
||||||
|
|
||||||
// IPLists has one entry for each IP list the address is on.
|
// IPLists has one entry for each IP list the address is on.
|
||||||
IPLists []iplist.Listing `json:"ip_lists"`
|
IPLists []iplist.Listing `json:"ip_lists"`
|
||||||
}
|
}
|
||||||
@@ -793,6 +797,9 @@ type IPLookupFlags struct {
|
|||||||
// IsTor says that the address is on the Tor exit list.
|
// IsTor says that the address is on the Tor exit list.
|
||||||
IsTor bool `json:"is_tor"`
|
IsTor bool `json:"is_tor"`
|
||||||
|
|
||||||
|
// IsVPN says that the address is on a list of VPN servers.
|
||||||
|
IsVPN bool `json:"is_vpn"`
|
||||||
|
|
||||||
// IsBogon says that the address is reserved, as package bogon tells.
|
// IsBogon says that the address is reserved, as package bogon tells.
|
||||||
IsBogon bool `json:"is_bogon"`
|
IsBogon bool `json:"is_bogon"`
|
||||||
}
|
}
|
||||||
@@ -977,14 +984,30 @@ func (s *Server) ipLookupResult(ip string, ptrs []string, info IPInfo) IPLookupR
|
|||||||
listings := s.ipListListings(ip)
|
listings := s.ipListListings(ip)
|
||||||
|
|
||||||
return IPLookupResult{
|
return IPLookupResult{
|
||||||
IP: ip,
|
IP: ip,
|
||||||
PTR: ptrs,
|
PTR: ptrs,
|
||||||
IPInfo: info,
|
IPInfo: info,
|
||||||
Flags: ipLookupFlags(ip, listings),
|
Flags: ipLookupFlags(ip, listings),
|
||||||
IPLists: listings,
|
VPNProviders: vpnProviders(listings),
|
||||||
|
IPLists: listings,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// vpnProviders returns the names of the VPN lists among listings. It returns
|
||||||
|
// an empty list, never nil, so that an answer's vpn_providers is [] rather
|
||||||
|
// than null.
|
||||||
|
func vpnProviders(listings []iplist.Listing) []string {
|
||||||
|
names := []string{}
|
||||||
|
|
||||||
|
for _, listing := range listings {
|
||||||
|
if listing.VPN {
|
||||||
|
names = append(names, listing.Name)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return names
|
||||||
|
}
|
||||||
|
|
||||||
// ipListListings returns the IP lists that contain ip. It returns an empty
|
// ipListListings returns the IP lists that contain ip. It returns an empty
|
||||||
// list, never nil, so that an answer's ip_lists is [] rather than null.
|
// list, never nil, so that an answer's ip_lists is [] rather than null.
|
||||||
func (s *Server) ipListListings(ip string) []iplist.Listing {
|
func (s *Server) ipListListings(ip string) []iplist.Listing {
|
||||||
@@ -1002,6 +1025,9 @@ func ipLookupFlags(ip string, listings []iplist.Listing) IPLookupFlags {
|
|||||||
IsTor: slices.ContainsFunc(listings, func(l iplist.Listing) bool {
|
IsTor: slices.ContainsFunc(listings, func(l iplist.Listing) bool {
|
||||||
return l.Name == iplist.TorExitsList
|
return l.Name == iplist.TorExitsList
|
||||||
}),
|
}),
|
||||||
|
IsVPN: slices.ContainsFunc(listings, func(l iplist.Listing) bool {
|
||||||
|
return l.VPN
|
||||||
|
}),
|
||||||
IsBogon: isBogon(ip),
|
IsBogon: isBogon(ip),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -17,6 +17,7 @@ import (
|
|||||||
"strings"
|
"strings"
|
||||||
"sync/atomic"
|
"sync/atomic"
|
||||||
"testing"
|
"testing"
|
||||||
|
"testing/synctest"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"git.eeqj.de/sneak/routewatch/internal/config"
|
"git.eeqj.de/sneak/routewatch/internal/config"
|
||||||
@@ -173,6 +174,9 @@ type tableStats struct {
|
|||||||
// carried 203.0.113.0/24, the two with kept routes withdrew it, so its origin
|
// carried 203.0.113.0/24, the two with kept routes withdrew it, so its origin
|
||||||
// keeps no route. The oldest and newest routes are those of the other two
|
// keeps no route. The oldest and newest routes are those of the other two
|
||||||
// prefixes.
|
// prefixes.
|
||||||
|
//
|
||||||
|
// The requests run in a synctest bubble, whose clock is fake, so a stall on
|
||||||
|
// the host does not use up the handlers' 4-second limit.
|
||||||
func TestStatsHandlersAnswerFromMemory(t *testing.T) {
|
func TestStatsHandlersAnswerFromMemory(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
@@ -218,7 +222,11 @@ func TestStatsHandlersAnswerFromMemory(t *testing.T) {
|
|||||||
|
|
||||||
for name, handler := range handlers {
|
for name, handler := range handlers {
|
||||||
rec := httptest.NewRecorder()
|
rec := httptest.NewRecorder()
|
||||||
handler(rec, httptest.NewRequestWithContext(t.Context(), http.MethodGet, "/", nil))
|
|
||||||
|
synctest.Test(t, func(t *testing.T) {
|
||||||
|
handler(rec,
|
||||||
|
httptest.NewRequestWithContext(t.Context(), http.MethodGet, "/", nil))
|
||||||
|
})
|
||||||
|
|
||||||
if rec.Code != http.StatusOK {
|
if rec.Code != http.StatusOK {
|
||||||
t.Errorf("%s: status %d, want %d; body %s",
|
t.Errorf("%s: status %d, want %d; body %s",
|
||||||
@@ -253,15 +261,20 @@ func TestStatsHandlersAnswerFromMemory(t *testing.T) {
|
|||||||
// TestStatsAnswerCarriesChurnPerMinute checks that the stream object of the
|
// TestStatsAnswerCarriesChurnPerMinute checks that the stream object of the
|
||||||
// /api/v1/stats answer holds route_churn_per_min as a whole number, and no
|
// /api/v1/stats answer holds route_churn_per_min as a whole number, and no
|
||||||
// longer route_churn_per_sec, the moving average it replaces
|
// longer route_churn_per_sec, the moving average it replaces
|
||||||
// (https://git.eeqj.de/sneak/routewatch/issues/160).
|
// (https://git.eeqj.de/sneak/routewatch/issues/160). The request runs in a
|
||||||
|
// synctest bubble, whose clock is fake, so a stall on the host does not use up
|
||||||
|
// the handler's 4-second limit.
|
||||||
func TestStatsAnswerCarriesChurnPerMinute(t *testing.T) {
|
func TestStatsAnswerCarriesChurnPerMinute(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
s := newServerWithTestRoutes(t)
|
s := newServerWithTestRoutes(t)
|
||||||
|
|
||||||
rec := httptest.NewRecorder()
|
rec := httptest.NewRecorder()
|
||||||
s.handleStats()(rec,
|
|
||||||
httptest.NewRequestWithContext(t.Context(), http.MethodGet, statsPath, nil))
|
synctest.Test(t, func(t *testing.T) {
|
||||||
|
s.handleStats()(rec,
|
||||||
|
httptest.NewRequestWithContext(t.Context(), http.MethodGet, statsPath, nil))
|
||||||
|
})
|
||||||
|
|
||||||
if rec.Code != http.StatusOK {
|
if rec.Code != http.StatusOK {
|
||||||
t.Fatalf("status %d, want %d; body %s", rec.Code, http.StatusOK, rec.Body)
|
t.Fatalf("status %d, want %d; body %s", rec.Code, http.StatusOK, rec.Body)
|
||||||
@@ -309,18 +322,26 @@ func (heldBackHandler) HeldBackCount() int { return heldBackMess
|
|||||||
|
|
||||||
// TestStatsAnswerGivesHowManyMessagesAHandlerHoldsBack registers a handler
|
// TestStatsAnswerGivesHowManyMessagesAHandlerHoldsBack registers a handler
|
||||||
// that holds back messages. Its figures in /api/v1/stats, which the status
|
// that holds back messages. Its figures in /api/v1/stats, which the status
|
||||||
// page shows, must give how many.
|
// page shows, must give how many. The request runs in a synctest bubble, whose
|
||||||
|
// clock is fake, so a stall on the host does not use up the handler's 4-second
|
||||||
|
// limit.
|
||||||
func TestStatsAnswerGivesHowManyMessagesAHandlerHoldsBack(t *testing.T) {
|
func TestStatsAnswerGivesHowManyMessagesAHandlerHoldsBack(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
s := newServerWithTestRoutes(t)
|
s := newServerWithTestRoutes(t)
|
||||||
s.streamer.RegisterHandler(heldBackHandler{})
|
s.streamer.RegisterHandler(heldBackHandler{})
|
||||||
|
|
||||||
|
var answer []byte
|
||||||
|
|
||||||
|
synctest.Test(t, func(t *testing.T) {
|
||||||
|
answer = get(t, s, statsPath).Body.Bytes()
|
||||||
|
})
|
||||||
|
|
||||||
var body struct {
|
var body struct {
|
||||||
Data StatsResponse `json:"data"`
|
Data StatsResponse `json:"data"`
|
||||||
}
|
}
|
||||||
|
|
||||||
err := json.Unmarshal(get(t, s, statsPath).Body.Bytes(), &body)
|
err := json.Unmarshal(answer, &body)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("decoding the answer: %v", err)
|
t.Fatalf("decoding the answer: %v", err)
|
||||||
}
|
}
|
||||||
@@ -364,7 +385,9 @@ type unitStats struct {
|
|||||||
// names. They used to be strings with the unit in them, such as "1h18m34s" or
|
// names. They used to be strings with the unit in them, such as "1h18m34s" or
|
||||||
// "1.1 GB" (https://git.eeqj.de/sneak/routewatch/issues/199). The stream has
|
// "1.1 GB" (https://git.eeqj.de/sneak/routewatch/issues/199). The stream has
|
||||||
// never connected, so the uptime is 0. The memory use in /api/v1/stats is the
|
// never connected, so the uptime is 0. The memory use in /api/v1/stats is the
|
||||||
// same number of bytes as its gc.heap_alloc_bytes.
|
// same number of bytes as its gc.heap_alloc_bytes. The requests run in a
|
||||||
|
// synctest bubble, whose clock is fake, so a stall on the host does not use up
|
||||||
|
// the handlers' 4-second limit.
|
||||||
func TestStatsAnswersGiveUnitsInFieldNames(t *testing.T) {
|
func TestStatsAnswersGiveUnitsInFieldNames(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
@@ -374,7 +397,11 @@ func TestStatsAnswersGiveUnitsInFieldNames(t *testing.T) {
|
|||||||
s.SetASNFetcher(fixedIntervalASNFetcher{interval: fetchInterval})
|
s.SetASNFetcher(fixedIntervalASNFetcher{interval: fetchInterval})
|
||||||
|
|
||||||
for _, path := range []string{statusJSONPath, statsPath} {
|
for _, path := range []string{statusJSONPath, statsPath} {
|
||||||
answer := get(t, s, path).Body.Bytes()
|
var answer []byte
|
||||||
|
|
||||||
|
synctest.Test(t, func(t *testing.T) {
|
||||||
|
answer = get(t, s, path).Body.Bytes()
|
||||||
|
})
|
||||||
|
|
||||||
var body struct {
|
var body struct {
|
||||||
Data unitStats `json:"data"`
|
Data unitStats `json:"data"`
|
||||||
@@ -415,82 +442,89 @@ func TestStatsAnswersGiveUnitsInFieldNames(t *testing.T) {
|
|||||||
// connection_duration_seconds of /api/v1/stats is null while the stream is not
|
// connection_duration_seconds of /api/v1/stats is null while the stream is not
|
||||||
// connected, and a whole number of seconds once it is. It used to be the
|
// connected, and a whole number of seconds once it is. It used to be the
|
||||||
// string "disconnected", or a Go duration such as "1h18m34s"
|
// string "disconnected", or a Go duration such as "1h18m34s"
|
||||||
// (https://git.eeqj.de/sneak/routewatch/issues/199).
|
// (https://git.eeqj.de/sneak/routewatch/issues/199). The test runs in a
|
||||||
|
// synctest bubble, whose clock is fake, so a stall on the host does not use up
|
||||||
|
// the handler's 4-second limit.
|
||||||
func TestStatsAnswerGivesConnectionDurationInSeconds(t *testing.T) {
|
func TestStatsAnswerGivesConnectionDurationInSeconds(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
s := newServerWithTestRoutes(t)
|
s := newServerWithTestRoutes(t)
|
||||||
|
|
||||||
connectionDuration := func() json.RawMessage {
|
synctest.Test(t, func(t *testing.T) {
|
||||||
answer := get(t, s, statsPath).Body.Bytes()
|
connectionDuration := func() json.RawMessage {
|
||||||
|
answer := get(t, s, statsPath).Body.Bytes()
|
||||||
|
|
||||||
var body struct {
|
var body struct {
|
||||||
Data map[string]json.RawMessage `json:"data"`
|
Data map[string]json.RawMessage `json:"data"`
|
||||||
|
}
|
||||||
|
|
||||||
|
err := json.Unmarshal(answer, &body)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("decoding the answer: %v; answer %s", err, answer)
|
||||||
|
}
|
||||||
|
|
||||||
|
duration, ok := body.Data["connection_duration_seconds"]
|
||||||
|
if !ok {
|
||||||
|
t.Fatalf("no connection_duration_seconds: %s", answer)
|
||||||
|
}
|
||||||
|
|
||||||
|
return duration
|
||||||
}
|
}
|
||||||
|
|
||||||
err := json.Unmarshal(answer, &body)
|
disconnected := connectionDuration()
|
||||||
if err != nil {
|
if string(disconnected) != "null" {
|
||||||
t.Fatalf("decoding the answer: %v; answer %s", err, answer)
|
t.Errorf("disconnected: connection_duration_seconds = %s, want null",
|
||||||
|
disconnected)
|
||||||
}
|
}
|
||||||
|
|
||||||
duration, ok := body.Data["connection_duration_seconds"]
|
s.streamer.GetMetricsTracker().SetConnected(true)
|
||||||
if !ok {
|
|
||||||
t.Fatalf("no connection_duration_seconds: %s", answer)
|
var seconds int64
|
||||||
|
|
||||||
|
connected := connectionDuration()
|
||||||
|
|
||||||
|
err := json.Unmarshal(connected, &seconds)
|
||||||
|
if err != nil || seconds != 0 {
|
||||||
|
t.Errorf("just connected: connection_duration_seconds = %s, want 0",
|
||||||
|
connected)
|
||||||
}
|
}
|
||||||
|
})
|
||||||
return duration
|
|
||||||
}
|
|
||||||
|
|
||||||
disconnected := connectionDuration()
|
|
||||||
if string(disconnected) != "null" {
|
|
||||||
t.Errorf("disconnected: connection_duration_seconds = %s, want null",
|
|
||||||
disconnected)
|
|
||||||
}
|
|
||||||
|
|
||||||
s.streamer.GetMetricsTracker().SetConnected(true)
|
|
||||||
|
|
||||||
var seconds int64
|
|
||||||
|
|
||||||
connected := connectionDuration()
|
|
||||||
|
|
||||||
err := json.Unmarshal(connected, &seconds)
|
|
||||||
if err != nil || seconds != 0 {
|
|
||||||
t.Errorf("just connected: connection_duration_seconds = %s, want 0",
|
|
||||||
connected)
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// TestStreamDurationsCountSeconds checks that the uptime and the connection
|
// TestStreamDurationsCountSeconds checks that the uptime and the connection
|
||||||
// duration of a stream that connected 90 seconds ago are both 90. A stream
|
// duration of a stream that connected 90 seconds ago are both 90. A stream
|
||||||
// that has never connected has an uptime of 0, and one that is not connected
|
// that has never connected has an uptime of 0, and one that is not connected
|
||||||
// has no connection duration.
|
// has no connection duration. The test runs in a synctest bubble, whose clock
|
||||||
|
// is fake, so a stall on the host does not add to the 90 seconds.
|
||||||
func TestStreamDurationsCountSeconds(t *testing.T) {
|
func TestStreamDurationsCountSeconds(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
const connectedFor = 90 * time.Second
|
synctest.Test(t, func(t *testing.T) {
|
||||||
|
const connectedFor = 90 * time.Second
|
||||||
|
|
||||||
connectedSince := time.Now().Add(-connectedFor)
|
connectedSince := time.Now().Add(-connectedFor)
|
||||||
want := int64(connectedFor / time.Second)
|
want := int64(connectedFor / time.Second)
|
||||||
|
|
||||||
if got := uptimeSeconds(connectedSince); got != want {
|
if got := uptimeSeconds(connectedSince); got != want {
|
||||||
t.Errorf("uptimeSeconds = %d, want %d", got, want)
|
t.Errorf("uptimeSeconds = %d, want %d", got, want)
|
||||||
}
|
}
|
||||||
|
|
||||||
if got := uptimeSeconds(time.Time{}); got != 0 {
|
if got := uptimeSeconds(time.Time{}); got != 0 {
|
||||||
t.Errorf("uptimeSeconds before any connection = %d, want 0", got)
|
t.Errorf("uptimeSeconds before any connection = %d, want 0", got)
|
||||||
}
|
}
|
||||||
|
|
||||||
switch got := connectionDurationSeconds(true, connectedSince); {
|
switch got := connectionDurationSeconds(true, connectedSince); {
|
||||||
case got == nil:
|
case got == nil:
|
||||||
t.Errorf("connectionDurationSeconds = nil, want %d", want)
|
t.Errorf("connectionDurationSeconds = nil, want %d", want)
|
||||||
case *got != want:
|
case *got != want:
|
||||||
t.Errorf("connectionDurationSeconds = %d, want %d", *got, want)
|
t.Errorf("connectionDurationSeconds = %d, want %d", *got, want)
|
||||||
}
|
}
|
||||||
|
|
||||||
if got := connectionDurationSeconds(false, connectedSince); got != nil {
|
if got := connectionDurationSeconds(false, connectedSince); got != nil {
|
||||||
t.Errorf("connectionDurationSeconds while not connected = %d, want nil",
|
t.Errorf("connectionDurationSeconds while not connected = %d, want nil",
|
||||||
*got)
|
*got)
|
||||||
}
|
}
|
||||||
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
// newTestTable returns a routing table that holds a route for testPrefixV4,
|
// newTestTable returns a routing table that holds a route for testPrefixV4,
|
||||||
@@ -1134,7 +1168,9 @@ func (failingDB) GetPrefixFirstSeen(
|
|||||||
// with the driver's error text. It also sends a prefix that does not parse.
|
// with the driver's error text. It also sends a prefix that does not parse.
|
||||||
// No answer may carry the driver's or the prefix parser's text. Before the fix
|
// No answer may carry the driver's or the prefix parser's text. Before the fix
|
||||||
// the answers sent it to the client
|
// the answers sent it to the client
|
||||||
// (https://git.eeqj.de/sneak/routewatch/issues/108).
|
// (https://git.eeqj.de/sneak/routewatch/issues/108). The requests run in a
|
||||||
|
// synctest bubble, whose clock is fake, so a stall on the host does not use up
|
||||||
|
// the stats handlers' 4-second limit.
|
||||||
func TestJSONRoutesDoNotSendErrorDetails(t *testing.T) {
|
func TestJSONRoutesDoNotSendErrorDetails(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
@@ -1155,8 +1191,11 @@ func TestJSONRoutesDoNotSendErrorDetails(t *testing.T) {
|
|||||||
|
|
||||||
for _, tt := range tests {
|
for _, tt := range tests {
|
||||||
rec := httptest.NewRecorder()
|
rec := httptest.NewRecorder()
|
||||||
s.router.ServeHTTP(rec, httptest.NewRequestWithContext(
|
|
||||||
t.Context(), http.MethodGet, tt.path, nil))
|
synctest.Test(t, func(t *testing.T) {
|
||||||
|
s.router.ServeHTTP(rec, httptest.NewRequestWithContext(
|
||||||
|
t.Context(), http.MethodGet, tt.path, nil))
|
||||||
|
})
|
||||||
|
|
||||||
if rec.Code != tt.wantCode {
|
if rec.Code != tt.wantCode {
|
||||||
t.Errorf("%s: status %d, want %d; body %s",
|
t.Errorf("%s: status %d, want %d; body %s",
|
||||||
@@ -1397,8 +1436,11 @@ func TestIPLookupKeepsRouteWhenPTRLookupDoesNotAnswer(t *testing.T) {
|
|||||||
s := newServerWithTestRoutes(t)
|
s := newServerWithTestRoutes(t)
|
||||||
s.resolver = &fakeResolver{ptrNeverAnswers: true}
|
s.resolver = &fakeResolver{ptrNeverAnswers: true}
|
||||||
|
|
||||||
// The request's time limit passes after the PTR lookup's.
|
// The request's time limit passes well after the PTR lookup's, so that a
|
||||||
ctx, cancel := context.WithTimeout(t.Context(), 2*ptrLookupTimeout)
|
// stall on a busy host does not use up the difference.
|
||||||
|
const requestTimeLimit = ptrLookupTimeout + 15*time.Second
|
||||||
|
|
||||||
|
ctx, cancel := context.WithTimeout(t.Context(), requestTimeLimit)
|
||||||
defer cancel()
|
defer cancel()
|
||||||
|
|
||||||
rec := httptest.NewRecorder()
|
rec := httptest.NewRecorder()
|
||||||
@@ -1441,8 +1483,19 @@ const listedUnroutedAddr = "203.0.113.1"
|
|||||||
// torExitAddr is the address that fakeIPLists lists on the Tor exit list.
|
// torExitAddr is the address that fakeIPLists lists on the Tor exit list.
|
||||||
const torExitAddr = "203.0.113.2"
|
const torExitAddr = "203.0.113.2"
|
||||||
|
|
||||||
// fakeIPLists has two lists. The test list holds addrInTestPrefixV4 and
|
// The addresses that fakeIPLists lists on one and on two VPN lists, and the
|
||||||
// listedUnroutedAddr, and the Tor exit list holds torExitAddr.
|
// names of those lists.
|
||||||
|
const (
|
||||||
|
oneVPNListAddr = "203.0.113.3"
|
||||||
|
twoVPNListsAddr = "203.0.113.4"
|
||||||
|
testVPNList = "test_vpn"
|
||||||
|
otherTestVPN = "other_test_vpn"
|
||||||
|
)
|
||||||
|
|
||||||
|
// fakeIPLists has four lists. The test list holds addrInTestPrefixV4,
|
||||||
|
// listedUnroutedAddr and twoVPNListsAddr. The Tor exit list holds torExitAddr.
|
||||||
|
// The test VPN list holds oneVPNListAddr and twoVPNListsAddr, and the other
|
||||||
|
// test VPN list holds twoVPNListsAddr.
|
||||||
type fakeIPLists struct{}
|
type fakeIPLists struct{}
|
||||||
|
|
||||||
func testListing() iplist.Listing {
|
func testListing() iplist.Listing {
|
||||||
@@ -1463,6 +1516,18 @@ func (fakeIPLists) Lookup(addr netip.Addr) []iplist.Listing {
|
|||||||
return []iplist.Listing{{Name: iplist.TorExitsList}}
|
return []iplist.Listing{{Name: iplist.TorExitsList}}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if addr == netip.MustParseAddr(oneVPNListAddr) {
|
||||||
|
return []iplist.Listing{{Name: testVPNList, VPN: true}}
|
||||||
|
}
|
||||||
|
|
||||||
|
if addr == netip.MustParseAddr(twoVPNListsAddr) {
|
||||||
|
return []iplist.Listing{
|
||||||
|
{Name: otherTestVPN, VPN: true},
|
||||||
|
testListing(),
|
||||||
|
{Name: testVPNList, VPN: true},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
return []iplist.Listing{}
|
return []iplist.Listing{}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1471,7 +1536,7 @@ func (fakeIPLists) ShortestInterval() time.Duration {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// TestIPLookupFlags looks up addresses with and without a route. Every result
|
// TestIPLookupFlags looks up addresses with and without a route. Every result
|
||||||
// has both flags, under the JSON keys the API documents.
|
// has every flag, under the JSON keys the API documents.
|
||||||
func TestIPLookupFlags(t *testing.T) {
|
func TestIPLookupFlags(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
@@ -1517,7 +1582,9 @@ func TestIPLookupFlags(t *testing.T) {
|
|||||||
t.Fatalf("%s: decoding flags %s: %v", path, results[0]["flags"], err)
|
t.Fatalf("%s: decoding flags %s: %v", path, results[0]["flags"], err)
|
||||||
}
|
}
|
||||||
|
|
||||||
want := map[string]bool{"is_tor": tt.tor, "is_bogon": tt.bogon}
|
want := map[string]bool{
|
||||||
|
"is_tor": tt.tor, "is_vpn": false, "is_bogon": tt.bogon,
|
||||||
|
}
|
||||||
if !maps.Equal(got, want) {
|
if !maps.Equal(got, want) {
|
||||||
t.Errorf("%s: flags %v, want %v", path, got, want)
|
t.Errorf("%s: flags %v, want %v", path, got, want)
|
||||||
}
|
}
|
||||||
@@ -1525,6 +1592,61 @@ func TestIPLookupFlags(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// TestIPLookupReportsVPNProviders looks up addresses on no list, on a list
|
||||||
|
// that is not a VPN list, on one VPN list and on two. is_vpn says whether any
|
||||||
|
// VPN list holds the address, and vpn_providers names only those lists. An
|
||||||
|
// address on none has [], not null.
|
||||||
|
func TestIPLookupReportsVPNProviders(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
s := newServerWithTestRoutes(t)
|
||||||
|
s.resolver = &fakeResolver{}
|
||||||
|
s.SetIPLists(fakeIPLists{})
|
||||||
|
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
addr string
|
||||||
|
isVPN string
|
||||||
|
providers string
|
||||||
|
}{
|
||||||
|
{"on no list", "1.1.1.1", "false", `[]`},
|
||||||
|
{"on a list that is not a VPN list", listedUnroutedAddr, "false", `[]`},
|
||||||
|
{"on one VPN list", oneVPNListAddr, "true", `["` + testVPNList + `"]`},
|
||||||
|
{
|
||||||
|
"on two VPN lists", twoVPNListsAddr, "true",
|
||||||
|
`["` + otherTestVPN + `","` + testVPNList + `"]`,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tt := range tests {
|
||||||
|
t.Run(tt.name, func(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
path := "/api/v1/ip/" + tt.addr
|
||||||
|
|
||||||
|
results := resultFields(t, get(t, s, path).Body.Bytes())
|
||||||
|
if len(results) != 1 {
|
||||||
|
t.Fatalf("%s: %d results, want 1", path, len(results))
|
||||||
|
}
|
||||||
|
|
||||||
|
var flags map[string]json.RawMessage
|
||||||
|
|
||||||
|
err := json.Unmarshal(results[0]["flags"], &flags)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("%s: decoding flags %s: %v", path, results[0]["flags"], err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if got := string(flags["is_vpn"]); got != tt.isVPN {
|
||||||
|
t.Errorf("%s: is_vpn %s, want %s", path, got, tt.isVPN)
|
||||||
|
}
|
||||||
|
|
||||||
|
if got := string(results[0]["vpn_providers"]); got != tt.providers {
|
||||||
|
t.Errorf("%s: vpn_providers %s, want %s", path, got, tt.providers)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// TestIPLookupReportsIPLists looks up two routed addresses, one of them on an
|
// TestIPLookupReportsIPLists looks up two routed addresses, one of them on an
|
||||||
// IP list. The listed address's result names the list, says what it means and
|
// IP list. The listed address's result names the list, says what it means and
|
||||||
// when it was updated. The other's result has an empty list.
|
// when it was updated. The other's result has an empty list.
|
||||||
@@ -2287,9 +2409,13 @@ func settledGoroutineCount() int {
|
|||||||
return prev
|
return prev
|
||||||
}
|
}
|
||||||
|
|
||||||
// waitForGoroutines waits until the goroutine count drops to target or below.
|
// waitForGoroutines waits up to 15 seconds for the goroutine count to drop to
|
||||||
|
// target or below.
|
||||||
func waitForGoroutines(target int) bool {
|
func waitForGoroutines(target int) bool {
|
||||||
for range 100 {
|
const maxWait = 15 * time.Second
|
||||||
|
|
||||||
|
deadline := time.Now().Add(maxWait)
|
||||||
|
for time.Now().Before(deadline) {
|
||||||
if runtime.NumGoroutine() <= target {
|
if runtime.NumGoroutine() <= target {
|
||||||
return true
|
return true
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -76,7 +76,9 @@ func asLink(asn int) string {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// TestHomePageCardsShowTheTableFigures fills a routing table and the asns
|
// TestHomePageCardsShowTheTableFigures fills a routing table and the asns
|
||||||
// table, works out the cards, and checks that each card shows its figure.
|
// table, works out the cards, and checks that each card shows its figure. The
|
||||||
|
// cards are worked out in a synctest bubble, whose clock is fake, so a stall on
|
||||||
|
// the host does not use up the 4-second limit on their database read.
|
||||||
func TestHomePageCardsShowTheTableFigures(t *testing.T) {
|
func TestHomePageCardsShowTheTableFigures(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
@@ -95,7 +97,9 @@ func TestHomePageCardsShowTheTableFigures(t *testing.T) {
|
|||||||
t.Fatalf("GetOrCreateASNBatch: %v", err)
|
t.Fatalf("GetOrCreateASNBatch: %v", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
s.refreshHomeCards(t.Context())
|
synctest.Test(t, func(t *testing.T) {
|
||||||
|
s.refreshHomeCards(t.Context())
|
||||||
|
})
|
||||||
|
|
||||||
page := get(t, s, "/").Body.String()
|
page := get(t, s, "/").Body.String()
|
||||||
|
|
||||||
@@ -279,7 +283,7 @@ func TestKeepHomeCardsFreshWorksThemOutEveryInterval(t *testing.T) {
|
|||||||
|
|
||||||
const (
|
const (
|
||||||
interval = 10 * time.Millisecond
|
interval = 10 * time.Millisecond
|
||||||
maxWait = 5 * time.Second
|
maxWait = 15 * time.Second
|
||||||
)
|
)
|
||||||
|
|
||||||
table := emptyTable()
|
table := emptyTable()
|
||||||
@@ -321,7 +325,7 @@ func TestKeepHomeCardsFreshWorksThemOutEveryInterval(t *testing.T) {
|
|||||||
func TestStartedServerWorksOutTheCards(t *testing.T) {
|
func TestStartedServerWorksOutTheCards(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
const maxWait = 5 * time.Second
|
const maxWait = 15 * time.Second
|
||||||
|
|
||||||
s := newServerWithTable(t, newCardsTable())
|
s := newServerWithTable(t, newCardsTable())
|
||||||
|
|
||||||
|
|||||||
@@ -14,6 +14,7 @@ import (
|
|||||||
"regexp"
|
"regexp"
|
||||||
"strings"
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
|
"testing/synctest"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"git.eeqj.de/sneak/routewatch/internal/config"
|
"git.eeqj.de/sneak/routewatch/internal/config"
|
||||||
@@ -163,6 +164,10 @@ func TestJSONAnswersPutNoUnitInAString(t *testing.T) {
|
|||||||
// router and the middleware write. Before the fix a panic in a handler, and a
|
// router and the middleware write. Before the fix a panic in a handler, and a
|
||||||
// method /status.json or the health check does not take, answered with an
|
// method /status.json or the health check does not take, answered with an
|
||||||
// empty body (https://git.eeqj.de/sneak/routewatch/issues/201).
|
// empty body (https://git.eeqj.de/sneak/routewatch/issues/201).
|
||||||
|
//
|
||||||
|
// The requests run in a synctest bubble, whose clock is fake, so a stall on
|
||||||
|
// the host does not use up the 2-second limit of the health check or the
|
||||||
|
// 4-second limit of the stats handlers.
|
||||||
func TestJSONAnswersCarryMeta(t *testing.T) {
|
func TestJSONAnswersCarryMeta(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
@@ -181,6 +186,10 @@ func TestJSONAnswersCarryMeta(t *testing.T) {
|
|||||||
s.router.Get(panicsPath, JSONValidationMiddleware(http.HandlerFunc(
|
s.router.Get(panicsPath, JSONValidationMiddleware(http.HandlerFunc(
|
||||||
func(http.ResponseWriter, *http.Request) { panic(errDiskIO) })).ServeHTTP)
|
func(http.ResponseWriter, *http.Request) { panic(errDiskIO) })).ServeHTTP)
|
||||||
|
|
||||||
|
failing := New(failingDB{}, newTestTable(),
|
||||||
|
streamer.New(logger.New(), metrics.New()), logger.New(), &config.Config{})
|
||||||
|
failing.resolver = &fakeResolver{}
|
||||||
|
|
||||||
tests := []struct {
|
tests := []struct {
|
||||||
method string
|
method string
|
||||||
path string
|
path string
|
||||||
@@ -216,42 +225,40 @@ func TestJSONAnswersCarryMeta(t *testing.T) {
|
|||||||
{http.MethodGet, panicsPath, http.StatusInternalServerError},
|
{http.MethodGet, panicsPath, http.StatusInternalServerError},
|
||||||
}
|
}
|
||||||
|
|
||||||
for _, tt := range tests {
|
synctest.Test(t, func(t *testing.T) {
|
||||||
checkAnswerCarriesMeta(t, s, httptest.NewRequestWithContext(
|
for _, tt := range tests {
|
||||||
t.Context(), tt.method, tt.path, nil), tt.wantCode)
|
checkAnswerCarriesMeta(t, s, httptest.NewRequestWithContext(
|
||||||
}
|
t.Context(), tt.method, tt.path, nil), tt.wantCode)
|
||||||
|
}
|
||||||
|
|
||||||
// A request whose time limit has already passed answers 504. The handlers
|
// A request whose time limit has already passed answers 504. The
|
||||||
// write the answers of the first five routes, and the JSON check on the
|
// handlers write the answers of the first five routes, and the JSON
|
||||||
// route writes the last.
|
// check on the route writes the last.
|
||||||
pastTimeLimit, cancel := context.WithDeadline(t.Context(), time.Now())
|
pastTimeLimit, cancel := context.WithDeadline(t.Context(), time.Now())
|
||||||
defer cancel()
|
defer cancel()
|
||||||
|
|
||||||
for _, path := range []string{
|
for _, path := range []string{
|
||||||
statusJSONPath, statsPath, "/ip/" + addrInTestPrefixV4,
|
statusJSONPath, statsPath, "/ip/" + addrInTestPrefixV4,
|
||||||
"/api/v1/ip/" + addrInTestPrefixV4, asJSONPath, writesNothingPath,
|
"/api/v1/ip/" + addrInTestPrefixV4, asJSONPath, writesNothingPath,
|
||||||
} {
|
} {
|
||||||
checkAnswerCarriesMeta(t, s, httptest.NewRequestWithContext(
|
checkAnswerCarriesMeta(t, s, httptest.NewRequestWithContext(
|
||||||
pastTimeLimit, http.MethodGet, path, nil), http.StatusGatewayTimeout)
|
pastTimeLimit, http.MethodGet, path, nil), http.StatusGatewayTimeout)
|
||||||
}
|
}
|
||||||
|
|
||||||
// With a store whose reads fail, the health check answers 503, and the
|
// With a store whose reads fail, the health check answers 503, and
|
||||||
// handlers of the other routes 500.
|
// the handlers of the other routes 500.
|
||||||
failing := New(failingDB{}, newTestTable(),
|
|
||||||
streamer.New(logger.New(), metrics.New()), logger.New(), &config.Config{})
|
|
||||||
failing.resolver = &fakeResolver{}
|
|
||||||
|
|
||||||
checkAnswerCarriesMeta(t, failing, httptest.NewRequestWithContext(
|
|
||||||
t.Context(), http.MethodGet, healthCheckPath, nil),
|
|
||||||
http.StatusServiceUnavailable)
|
|
||||||
|
|
||||||
for _, path := range []string{
|
|
||||||
statusJSONPath, statsPath, asJSONPath,
|
|
||||||
"/api/v1/ip/" + addrInTestPrefixV4, "/ip/" + addrInTestPrefixV4,
|
|
||||||
} {
|
|
||||||
checkAnswerCarriesMeta(t, failing, httptest.NewRequestWithContext(
|
checkAnswerCarriesMeta(t, failing, httptest.NewRequestWithContext(
|
||||||
t.Context(), http.MethodGet, path, nil), http.StatusInternalServerError)
|
t.Context(), http.MethodGet, healthCheckPath, nil),
|
||||||
}
|
http.StatusServiceUnavailable)
|
||||||
|
|
||||||
|
for _, path := range []string{
|
||||||
|
statusJSONPath, statsPath, asJSONPath,
|
||||||
|
"/api/v1/ip/" + addrInTestPrefixV4, "/ip/" + addrInTestPrefixV4,
|
||||||
|
} {
|
||||||
|
checkAnswerCarriesMeta(t, failing, httptest.NewRequestWithContext(
|
||||||
|
t.Context(), http.MethodGet, path, nil), http.StatusInternalServerError)
|
||||||
|
}
|
||||||
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
// checkAnswerCarriesMeta serves req through the server's router, and checks
|
// checkAnswerCarriesMeta serves req through the server's router, and checks
|
||||||
|
|||||||
@@ -5,6 +5,7 @@ import (
|
|||||||
"net/http"
|
"net/http"
|
||||||
"net/http/httptest"
|
"net/http/httptest"
|
||||||
"testing"
|
"testing"
|
||||||
|
"testing/synctest"
|
||||||
)
|
)
|
||||||
|
|
||||||
// TestRoutesAnswerHEAD sends HEAD for a page and JSON routes through the
|
// TestRoutesAnswerHEAD sends HEAD for a page and JSON routes through the
|
||||||
@@ -12,15 +13,23 @@ import (
|
|||||||
// lookup of the caller's own address, are in the list because chi's GetHead
|
// lookup of the caller's own address, are in the list because chi's GetHead
|
||||||
// does not reach them. Before the fix every route answered HEAD with 405
|
// does not reach them. Before the fix every route answered HEAD with 405
|
||||||
// (https://git.eeqj.de/sneak/routewatch/issues/111).
|
// (https://git.eeqj.de/sneak/routewatch/issues/111).
|
||||||
|
//
|
||||||
|
// The requests run in a synctest bubble, whose clock is fake, so a stall on
|
||||||
|
// the host does not use up the stats handlers' 4-second limit. The resolver
|
||||||
|
// is fake because no time limit would cut off a real DNS lookup in a bubble.
|
||||||
func TestRoutesAnswerHEAD(t *testing.T) {
|
func TestRoutesAnswerHEAD(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
s := newServerWithTestRoutes(t)
|
s := newServerWithTestRoutes(t)
|
||||||
|
s.resolver = &fakeResolver{}
|
||||||
|
|
||||||
for _, path := range []string{"/", statusJSONPath, statsPath, "/ip", "/ip/"} {
|
for _, path := range []string{"/", statusJSONPath, statsPath, "/ip", "/ip/"} {
|
||||||
rec := httptest.NewRecorder()
|
rec := httptest.NewRecorder()
|
||||||
s.router.ServeHTTP(rec, httptest.NewRequestWithContext(
|
|
||||||
t.Context(), http.MethodHead, path, nil))
|
synctest.Test(t, func(t *testing.T) {
|
||||||
|
s.router.ServeHTTP(rec, httptest.NewRequestWithContext(
|
||||||
|
t.Context(), http.MethodHead, path, nil))
|
||||||
|
})
|
||||||
|
|
||||||
if rec.Code != http.StatusOK {
|
if rec.Code != http.StatusOK {
|
||||||
t.Errorf("HEAD %s: status %d, want %d; body %s",
|
t.Errorf("HEAD %s: status %d, want %d; body %s",
|
||||||
|
|||||||
@@ -22,10 +22,10 @@ func TestStopClosesTheConnectionOfARequestStillRunning(t *testing.T) {
|
|||||||
stopDeadline = 100 * time.Millisecond
|
stopDeadline = 100 * time.Millisecond
|
||||||
// waitTimeout is how long the test waits for the request to reach
|
// waitTimeout is how long the test waits for the request to reach
|
||||||
// the handler, and then for the client to see its connection closed.
|
// the handler, and then for the client to see its connection closed.
|
||||||
waitTimeout = 2 * time.Second
|
waitTimeout = 15 * time.Second
|
||||||
// readHeaderTimeout bounds the test server's read of the request
|
// readHeaderTimeout bounds the test server's read of the request
|
||||||
// header, as every http.Server must.
|
// header, as every http.Server must.
|
||||||
readHeaderTimeout = time.Second
|
readHeaderTimeout = waitTimeout
|
||||||
)
|
)
|
||||||
|
|
||||||
release := make(chan struct{})
|
release := make(chan struct{})
|
||||||
|
|||||||
@@ -14,6 +14,7 @@ import (
|
|||||||
"strings"
|
"strings"
|
||||||
"sync/atomic"
|
"sync/atomic"
|
||||||
"testing"
|
"testing"
|
||||||
|
"testing/synctest"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"git.eeqj.de/sneak/routewatch/internal/logger"
|
"git.eeqj.de/sneak/routewatch/internal/logger"
|
||||||
@@ -398,7 +399,7 @@ func TestStopQueuesTheHeldStateMessages(t *testing.T) {
|
|||||||
const (
|
const (
|
||||||
held = 2
|
held = 2
|
||||||
lines = held + 1
|
lines = held + 1
|
||||||
maxWait = 5 * time.Second
|
maxWait = 15 * time.Second
|
||||||
)
|
)
|
||||||
|
|
||||||
srv := httptest.NewServer(http.HandlerFunc(
|
srv := httptest.NewServer(http.HandlerFunc(
|
||||||
@@ -478,15 +479,15 @@ func TestStopQueuesTheHeldStateMessages(t *testing.T) {
|
|||||||
// TestStopReturnsByItsDeadline stops a streamer whose handler has a full queue
|
// TestStopReturnsByItsDeadline stops a streamer whose handler has a full queue
|
||||||
// and a STATE message held for it, and a worker that stays busy until the test
|
// and a STATE message held for it, and a worker that stays busy until the test
|
||||||
// ends, as one does whose handler waits for a database write. Stop must return
|
// ends, as one does whose handler waits for a database write. Stop must return
|
||||||
// within its deadline plus a small margin instead of waiting for room on the
|
// within its deadline plus a margin instead of waiting for room on the queue
|
||||||
// queue and for the worker, and log as not handed over the held message and
|
// and for the worker, and log as not handed over the held message and every
|
||||||
// every message on the queue.
|
// message on the queue.
|
||||||
func TestStopReturnsByItsDeadline(t *testing.T) {
|
func TestStopReturnsByItsDeadline(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
const (
|
const (
|
||||||
deadline = 100 * time.Millisecond
|
deadline = 100 * time.Millisecond
|
||||||
margin = 2 * time.Second
|
margin = 15 * time.Second
|
||||||
)
|
)
|
||||||
|
|
||||||
var logs bytes.Buffer
|
var logs bytes.Buffer
|
||||||
@@ -577,7 +578,7 @@ func TestStopBeforeMessageReachesQueues(t *testing.T) {
|
|||||||
func TestReconnectWhenStreamEnds(t *testing.T) {
|
func TestReconnectWhenStreamEnds(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
endResponse := func(http.ResponseWriter, *http.Request) {}
|
endResponse := pipeTransport{write: func(context.Context, io.Writer) {}}
|
||||||
checkReconnect(t, endResponse, streamIdleTimeout)
|
checkReconnect(t, endResponse, streamIdleTimeout)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -589,9 +590,8 @@ func TestReconnectWhenStreamGoesIdle(t *testing.T) {
|
|||||||
|
|
||||||
const idleTimeout = 50 * time.Millisecond
|
const idleTimeout = 50 * time.Millisecond
|
||||||
|
|
||||||
holdResponse := func(w http.ResponseWriter, r *http.Request) {
|
holdResponse := pipeTransport{
|
||||||
flush(w)
|
write: func(ctx context.Context, _ io.Writer) { <-ctx.Done() },
|
||||||
<-r.Context().Done()
|
|
||||||
}
|
}
|
||||||
checkReconnect(t, holdResponse, idleTimeout)
|
checkReconnect(t, holdResponse, idleTimeout)
|
||||||
}
|
}
|
||||||
@@ -604,16 +604,51 @@ func TestReconnectWhenServerNeverAnswers(t *testing.T) {
|
|||||||
|
|
||||||
const idleTimeout = 50 * time.Millisecond
|
const idleTimeout = 50 * time.Millisecond
|
||||||
|
|
||||||
neverAnswer := func(_ http.ResponseWriter, r *http.Request) {
|
neverAnswer := roundTripFunc(
|
||||||
<-r.Context().Done()
|
func(req *http.Request) (*http.Response, error) {
|
||||||
}
|
<-req.Context().Done()
|
||||||
|
|
||||||
|
return nil, req.Context().Err()
|
||||||
|
},
|
||||||
|
)
|
||||||
checkReconnect(t, neverAnswer, idleTimeout)
|
checkReconnect(t, neverAnswer, idleTimeout)
|
||||||
}
|
}
|
||||||
|
|
||||||
// TestStreamNotDroppedWhileLinesArrive has the server send lines well within
|
// pipeTransport answers every request with a response whose body write fills.
|
||||||
// the idle timeout for several times the timeout, then end the response. Each
|
// The body fails when the request's context ends, as a connection's does.
|
||||||
// line must start the idle timeout over, so the stream lasts until the server
|
// write gets that context, so that it can end when the request does.
|
||||||
// ends it instead of being dropped as idle.
|
type pipeTransport struct {
|
||||||
|
write func(context.Context, io.Writer)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (p pipeTransport) RoundTrip(req *http.Request) (*http.Response, error) {
|
||||||
|
body, w := io.Pipe()
|
||||||
|
|
||||||
|
context.AfterFunc(req.Context(), func() {
|
||||||
|
_ = w.CloseWithError(req.Context().Err())
|
||||||
|
})
|
||||||
|
|
||||||
|
go func(ctx context.Context) {
|
||||||
|
p.write(ctx, w)
|
||||||
|
|
||||||
|
_ = w.Close()
|
||||||
|
}(req.Context())
|
||||||
|
|
||||||
|
return &http.Response{StatusCode: http.StatusOK, Body: body}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// roundTripFunc makes a function an http.RoundTripper.
|
||||||
|
type roundTripFunc func(*http.Request) (*http.Response, error)
|
||||||
|
|
||||||
|
func (f roundTripFunc) RoundTrip(req *http.Request) (*http.Response, error) {
|
||||||
|
return f(req)
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestStreamNotDroppedWhileLinesArrive has a response send lines well within
|
||||||
|
// the idle timeout for several times the timeout, then end. Each line must
|
||||||
|
// start the idle timeout over, so the stream lasts until the response ends
|
||||||
|
// instead of being dropped as idle. It runs in a synctest bubble, whose clock
|
||||||
|
// is fake, so a stall on the host does not use up the timeout.
|
||||||
func TestStreamNotDroppedWhileLinesArrive(t *testing.T) {
|
func TestStreamNotDroppedWhileLinesArrive(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
@@ -623,26 +658,35 @@ func TestStreamNotDroppedWhileLinesArrive(t *testing.T) {
|
|||||||
sendFor = 5 * idleTimeout
|
sendFor = 5 * idleTimeout
|
||||||
)
|
)
|
||||||
|
|
||||||
srv := httptest.NewServer(http.HandlerFunc(
|
// The streamer is made outside the bubble. Its metrics start a goroutine
|
||||||
func(w http.ResponseWriter, _ *http.Request) {
|
// that never returns, and a bubble waits for each of its goroutines to
|
||||||
|
// return.
|
||||||
|
s := New(logger.New(), metrics.New())
|
||||||
|
s.idleTimeout = idleTimeout
|
||||||
|
s.client = &http.Client{Transport: pipeTransport{
|
||||||
|
write: func(ctx context.Context, w io.Writer) {
|
||||||
deadline := time.Now().Add(sendFor)
|
deadline := time.Now().Add(sendFor)
|
||||||
for time.Now().Before(deadline) {
|
for time.Now().Before(deadline) {
|
||||||
_, _ = io.WriteString(w, updateLine)
|
_, _ = io.WriteString(w, updateLine)
|
||||||
|
|
||||||
flush(w)
|
// The wait ends with the stream. A bubble's clock stops when
|
||||||
time.Sleep(lineInterval)
|
// the test function returns, and a goroutine then still
|
||||||
|
// asleep panics the test binary.
|
||||||
|
select {
|
||||||
|
case <-ctx.Done():
|
||||||
|
return
|
||||||
|
case <-time.After(lineInterval):
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}))
|
},
|
||||||
defer srv.Close()
|
}}
|
||||||
|
|
||||||
s := New(logger.New(), metrics.New())
|
synctest.Test(t, func(t *testing.T) {
|
||||||
s.url = srv.URL
|
err := s.stream(context.Background())
|
||||||
s.idleTimeout = idleTimeout
|
if !errors.Is(err, errStreamEnded) {
|
||||||
|
t.Fatalf("stream returned %v, want %v", err, errStreamEnded)
|
||||||
err := s.stream(context.Background())
|
}
|
||||||
if !errors.Is(err, errStreamEnded) {
|
})
|
||||||
t.Fatalf("stream returned %v, want %v", err, errStreamEnded)
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// TestStreamReadsLongLine has the server send one UPDATE line of about
|
// TestStreamReadsLongLine has the server send one UPDATE line of about
|
||||||
@@ -706,7 +750,7 @@ func TestHandlerStatsReadWhileMessagesAreDropped(t *testing.T) {
|
|||||||
const (
|
const (
|
||||||
lines = 5
|
lines = 5
|
||||||
wantDropped = lines - 2
|
wantDropped = lines - 2
|
||||||
maxWait = 5 * time.Second
|
maxWait = 15 * time.Second
|
||||||
)
|
)
|
||||||
|
|
||||||
srv := httptest.NewServer(http.HandlerFunc(
|
srv := httptest.NewServer(http.HandlerFunc(
|
||||||
@@ -820,61 +864,70 @@ func streamLines(t *testing.T, s *Streamer, lines string) {
|
|||||||
// line and then stays open. The streamer must come back for a second
|
// line and then stays open. The streamer must come back for a second
|
||||||
// response, must not be marked connected when it does, and must handle the
|
// response, must not be marked connected when it does, and must handle the
|
||||||
// UPDATE in it.
|
// UPDATE in it.
|
||||||
|
//
|
||||||
|
// It runs in a synctest bubble, whose clock is fake, so a stall on the host
|
||||||
|
// does not use up the idle timeout. The fake server gets the requests in the
|
||||||
|
// order the streamer makes them, because no real connection carries them.
|
||||||
func checkReconnect(
|
func checkReconnect(
|
||||||
t *testing.T, firstResponse http.HandlerFunc, idleTimeout time.Duration,
|
t *testing.T, firstResponse http.RoundTripper, idleTimeout time.Duration,
|
||||||
) {
|
) {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
|
|
||||||
const maxWait = 5 * time.Second
|
const maxWait = 15 * time.Second
|
||||||
|
|
||||||
|
// The tracker is made outside the bubble. It starts a goroutine that never
|
||||||
|
// returns, and a bubble waits for each of its goroutines to return.
|
||||||
tracker := metrics.New()
|
tracker := metrics.New()
|
||||||
connectedAtReconnect := make(chan bool, 1)
|
|
||||||
|
|
||||||
var requests atomic.Int32
|
synctest.Test(t, func(t *testing.T) {
|
||||||
|
connectedAtReconnect := make(chan bool, 1)
|
||||||
|
laterResponse := pipeTransport{
|
||||||
|
write: func(ctx context.Context, w io.Writer) {
|
||||||
|
_, _ = io.WriteString(w, updateLine)
|
||||||
|
|
||||||
srv := httptest.NewServer(http.HandlerFunc(
|
<-ctx.Done()
|
||||||
func(w http.ResponseWriter, r *http.Request) {
|
},
|
||||||
if requests.Add(1) == 1 {
|
}
|
||||||
firstResponse(w, r)
|
|
||||||
|
|
||||||
return
|
var requests atomic.Int32
|
||||||
}
|
|
||||||
|
|
||||||
select {
|
s := New(logger.New(), tracker)
|
||||||
case connectedAtReconnect <- tracker.IsConnected():
|
s.client = &http.Client{Transport: roundTripFunc(
|
||||||
default:
|
func(req *http.Request) (*http.Response, error) {
|
||||||
}
|
if requests.Add(1) == 1 {
|
||||||
|
return firstResponse.RoundTrip(req)
|
||||||
|
}
|
||||||
|
|
||||||
_, _ = io.WriteString(w, updateLine)
|
select {
|
||||||
|
case connectedAtReconnect <- tracker.IsConnected():
|
||||||
|
default:
|
||||||
|
}
|
||||||
|
|
||||||
flush(w)
|
return laterResponse.RoundTrip(req)
|
||||||
<-r.Context().Done()
|
},
|
||||||
}))
|
)}
|
||||||
defer srv.Close()
|
s.idleTimeout = idleTimeout
|
||||||
|
s.minBackoff = time.Millisecond
|
||||||
|
handler := updateHandler{handled: make(chan struct{}, 1)}
|
||||||
|
s.RegisterHandler(handler)
|
||||||
|
|
||||||
s := New(logger.New(), tracker)
|
err := s.Start(t.Context())
|
||||||
s.url = srv.URL
|
if err != nil {
|
||||||
s.idleTimeout = idleTimeout
|
t.Fatalf("Start returned error: %v", err)
|
||||||
s.minBackoff = time.Millisecond
|
}
|
||||||
handler := updateHandler{handled: make(chan struct{}, 1)}
|
defer s.Stop(t.Context())
|
||||||
s.RegisterHandler(handler)
|
|
||||||
|
|
||||||
err := s.Start(t.Context())
|
select {
|
||||||
if err != nil {
|
case <-handler.handled:
|
||||||
t.Fatalf("Start returned error: %v", err)
|
case <-time.After(maxWait):
|
||||||
}
|
t.Fatalf("no message from a second response handled within %s", maxWait)
|
||||||
defer s.Stop(t.Context())
|
}
|
||||||
|
|
||||||
select {
|
if <-connectedAtReconnect {
|
||||||
case <-handler.handled:
|
t.Error("streamer was still marked connected when it came back " +
|
||||||
case <-time.After(maxWait):
|
"for a second response")
|
||||||
t.Fatalf("no message from a second response handled within %s", maxWait)
|
}
|
||||||
}
|
})
|
||||||
|
|
||||||
if <-connectedAtReconnect {
|
|
||||||
t.Error("streamer was still marked connected when it came back " +
|
|
||||||
"for a second response")
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// settledGoroutineCount lets transient goroutines finish, then reports the
|
// settledGoroutineCount lets transient goroutines finish, then reports the
|
||||||
@@ -896,9 +949,13 @@ func settledGoroutineCount() int {
|
|||||||
return prev
|
return prev
|
||||||
}
|
}
|
||||||
|
|
||||||
// waitForGoroutines waits until the goroutine count drops to target or below.
|
// waitForGoroutines waits up to 15 seconds for the goroutine count to drop to
|
||||||
|
// target or below.
|
||||||
func waitForGoroutines(target int) bool {
|
func waitForGoroutines(target int) bool {
|
||||||
for range 100 {
|
const maxWait = 15 * time.Second
|
||||||
|
|
||||||
|
deadline := time.Now().Add(maxWait)
|
||||||
|
for time.Now().Before(deadline) {
|
||||||
if runtime.NumGoroutine() <= target {
|
if runtime.NumGoroutine() <= target {
|
||||||
return true
|
return true
|
||||||
}
|
}
|
||||||
|
|||||||
Reference in new issue
Block a user