Author SHA1 Message Date
clawbot eea0382b37 Add the ProtonVPN server networks and addresses to the IP lookup (closes #236)
check / check (push) Successful in 5m8s
Adds the VPN lists `protonvpn`, the servers' /24 networks, IPv4 only, and `protonvpn_entry`, the servers' IPv4 and IPv6 addresses. Each is downloaded once a day and shows in `flags.is_vpn` and `vpn_providers` with no other change.

Proton's `/vpn/logicals` answers 401 without a logged-in session, even with a current client's `x-pm-appversion`, so both come from the public ProtonVPN-IPs mirror. No request header is needed.

Judgement call: the mirror's /24 file as well as its exact addresses. Its README says traffic leaves from unpublished addresses in those networks. The cost is that neighbouring addresses are listed too.

Model: opus-5-5
2026-11-10 12:34:28 +00:00
clawbot 641798bb22 Make the tests pass on a busy host (closes #240)
check / check (push) Successful in 1m51s
Tests that wait on the real clock failed when the host stalled for a few seconds, although the code was right. Waits for something that should happen are now mostly 15 seconds. Tests whose result depends on a short idle timeout or a production request limit run under `synctest.Test`, on a fake clock, with a fake transport where a real connection would stop that clock. Test code only.

Judgement call: the 5-second waits in three `internal/streamer` tests also became 15 seconds.

Model: opus-5-5
2026-11-10 12:48:07 +01:00
14 changed files with 453 additions and 188 deletions

No files matched your search

+8
View File
@@ -434,6 +434,14 @@ The lists:
- `pia`: VPN servers run by Private Internet Access. From
[Private Internet Access](https://serverlist.piaservers.net/vpninfo/servers/v6),
downloaded once a day.
- `protonvpn`: the /24 networks that hold ProtonVPN servers. The whole /24 is
listed because the ProtonVPN-IPs README says traffic leaves from unpublished
addresses in it. From the public
[ProtonVPN-IPs](https://github.com/tn3w/ProtonVPN-IPs) mirror of Proton's
server list, downloaded once a day.
- `protonvpn_entry`: the IPv4 and IPv6 addresses of ProtonVPN servers. From the
same [ProtonVPN-IPs](https://github.com/tn3w/ProtonVPN-IPs) mirror, downloaded
once a day.
- `spamhaus_drop` and `spamhaus_dropv6`: IPv4 and IPv6 networks hijacked or
leased by spammers or cyber criminals, according to Spamhaus. From
[Spamhaus DROP](https://www.spamhaus.org/blocklists/do-not-route-or-peer/),
+7
View File
@@ -53,6 +53,13 @@ problem the run finds.
# Completed Steps
- 2026-11-10: the IP lookup checks the `protonvpn` list of ProtonVPN's /24
networks and the `protonvpn_entry` list of its servers' IPv4 and IPv6
addresses, from a public mirror of Proton's server list (closes
https://git.eeqj.de/sneak/routewatch/issues/236).
- 2026-11-10: the tests no longer fail when a busy host stalls for several
seconds. Most such waits are now 15 seconds. The test of the stream's idle
timeout runs on a fake clock (closes #240)
- 2026-11-10: the IP lookup checks the public VPN server lists of Mullvad,
NordVPN, Private Internet Access and IVPN, and X4BNet's community list. Each
result has `flags.is_vpn` and `vpn_providers` (closes
+1 -1
View File
@@ -12,7 +12,7 @@ import (
// parseTimeLimit is how long a test waits for parseCSV to return before it
// fails.
const parseTimeLimit = 2 * time.Second
const parseTimeLimit = 15 * time.Second
// TestParseCSVReadsTodaysLayout parses an as.csv with the country code column
// that the source added after the description. It expects each AS with its
+25 -5
View File
@@ -72,17 +72,34 @@ const (
// lockTestPrefix is the prefix the batch writes in the checkpoint tests
// store.
lockTestPrefix = "192.0.2.0/24"
// longBusyTimeoutMs is the busy timeout setLongBusyTimeout gives the
// writing connection. A checkpoint that waits for an open read takes this
// long.
longBusyTimeoutMs = 30000
// readerCheckpointLimit is how long a checkpoint, and a batch write
// waiting for it, may take while a reader is open: well under the busy
// timeout, which a checkpoint that waits for the reader takes.
readerCheckpointLimit = time.Second
// waiting for it, may take while a reader is open: well under
// longBusyTimeoutMs, with room for a stall on a busy host.
readerCheckpointLimit = 15 * time.Second
// truncateCheckpointLimit is how long a checkpoint of a WAL file over
// sqliteWALSizeLimitBytes may take to give up while a reader is open: its
// wait for the reader plus a second, well under the busy timeout.
// wait for the reader plus readerCheckpointLimit.
truncateCheckpointLimit = truncateBusyTimeoutMs*time.Millisecond +
time.Second
readerCheckpointLimit
)
// setLongBusyTimeout gives the writing connection of db longBusyTimeoutMs, so
// that a checkpoint that waits for an open read takes far longer than a stall
// on a busy host.
func setLongBusyTimeout(ctx context.Context, t *testing.T, db *Database) {
t.Helper()
_, err := db.writer.ExecContext(ctx,
fmt.Sprintf("PRAGMA busy_timeout=%d", longBusyTimeoutMs))
if err != nil {
t.Fatalf("failed to set the busy timeout: %v", err)
}
}
// TestConnectionPoolPragmas holds several pooled connections open at once and
// checks each one carries the per-connection settings from the DSN, plus the
// process-wide heap limits, and sees the new file with auto_vacuum
@@ -579,6 +596,8 @@ func TestCheckpointDoesNotWaitForReader(t *testing.T) {
ctx := context.Background()
ts := time.Now().UTC()
setLongBusyTimeout(ctx, t, db)
err = db.UpdatePrefixesBatch(map[string]time.Time{"198.51.100.0/24": ts})
if err != nil {
t.Fatalf("failed to store the prefix the reader reads: %v", err)
@@ -690,6 +709,7 @@ func TestCheckpointEmptiesWALOnceNoReadIsOpen(t *testing.T) {
}
fillWALPastLimit(ctx, t, db)
setLongBusyTimeout(ctx, t, db)
started := time.Now()
+5
View File
@@ -51,6 +51,11 @@ const (
// FormatIVPNJSON is IVPN's JSON server list. The hosts of its wireguard
// and openvpn gateways have their addresses in host and v2ray.
FormatIVPNJSON
// FormatProtonVPN has one address or network per line, with comment lines
// that start with #. Unlike FormatLines, a file with no addresses or
// networks fails.
FormatProtonVPN
)
// List is one public IP list.
+61 -2
View File
@@ -70,6 +70,19 @@ const (
"config":{"api":{"ips":["192.0.2.73"]}}}`
)
// protonVPNBody is a small file in the format of the ProtonVPN-IPs list.
const protonVPNBody = "#\n# protonvpn_ip_ranges.txt\n" +
"# https://github.com/tn3w/ProtonVPN-IPs\n#\n" +
"# The /24 block of every advertised IPv4.\n#\n" +
"192.0.2.0/24\n198.51.100.0/24\n"
// protonVPNEntryBody is a small file in the format of the ProtonVPN-IPs list
// of server addresses.
const protonVPNEntryBody = "#\n# protonvpn_entry_ips.txt\n" +
"# https://github.com/tn3w/ProtonVPN-IPs\n#\n" +
"# Entry IPs clients connect to, resolved from ProtonVPN hostnames.\n#\n" +
"192.0.2.80\n2001:db8::80\n"
// errorPage is a file that is no list, as a publisher's error page is.
const errorPage = "<!DOCTYPE html>\n<html></html>\n"
@@ -267,6 +280,20 @@ func TestLookupFindsVPNServers(t *testing.T) {
listed: []string{"192.0.2.70", "192.0.2.71", "192.0.2.72"},
unlisted: []string{"192.0.2.73", "2001:db8::70"},
},
{
name: "ProtonVPN networks",
format: FormatProtonVPN,
body: protonVPNBody,
listed: []string{"192.0.2.0", "192.0.2.255", "198.51.100.7"},
unlisted: []string{"192.0.3.1", "2001:db8::90"},
},
{
name: "ProtonVPN server addresses",
format: FormatProtonVPN,
body: protonVPNEntryBody,
listed: []string{"192.0.2.80", "2001:db8::80"},
unlisted: []string{"192.0.2.81", "2001:db8::81"},
},
})
}
@@ -479,6 +506,31 @@ func TestWrongVPNDocumentFailsToParse(t *testing.T) {
}
}
// TestWrongProtonVPNDocumentFailsToParse parses documents that are not a
// ProtonVPN-IPs list. Each fails, so none can replace a good copy.
func TestWrongProtonVPNDocumentFailsToParse(t *testing.T) {
t.Parallel()
bodies := map[string]string{
"error page": errorPage,
"JSON": `{"status":"ok"}`,
"comments only": "#\n# protonvpn_ip_ranges.txt\n#\n",
"bad network": "192.0.2.0/24\nexample\n",
"an empty file": "",
}
for name, body := range bodies {
t.Run(name, func(t *testing.T) {
t.Parallel()
addresses, err := parse(strings.NewReader(body), FormatProtonVPN)
if err == nil {
t.Errorf("parse returned %v and no error", addresses.Ranges())
}
})
}
}
// TestLookupMarksVPNLists looks up an address on a VPN list and on another
// list. Only the VPN list's listing says it is one.
func TestLookupMarksVPNLists(t *testing.T) {
@@ -523,7 +575,10 @@ func TestPublicListsMarkTheVPNLists(t *testing.T) {
}
}
want := []string{"ivpn", "mullvad", "nordvpn", "pia", "x4bnet", "x4bnet_v6"}
want := []string{
"ivpn", "mullvad", "nordvpn", "pia", "protonvpn", "protonvpn_entry",
"x4bnet", "x4bnet_v6",
}
if !slices.Equal(got, want) {
t.Errorf("VPN lists %v, want %v", got, want)
}
@@ -646,9 +701,13 @@ func TestRestartLoadsCopiesFromDisk(t *testing.T) {
second.Start(t.Context())
second.Stop(t.Context())
// The download takes its time after writing the file, and the restart
// reads the file's time. A busy host can put seconds between the two.
const maxUpdatedDifference = 15 * time.Second
listings := lookup(t, second, listedAddr)
if len(listings) != 1 ||
listings[0].Updated.Sub(downloaded).Abs() > time.Second {
listings[0].Updated.Sub(downloaded).Abs() > maxUpdatedDifference {
t.Errorf("after the restart, listings %+v, want one updated at %s",
listings, downloaded)
}
+25
View File
@@ -8,6 +8,9 @@ import (
// TorExitsList is the name of the list of Tor exit nodes.
const TorExitsList = "tor_exits"
// protonVPNIPsURL is where the ProtonVPN-IPs mirror publishes its files.
const protonVPNIPsURL = "https://raw.githubusercontent.com/tn3w/ProtonVPN-IPs/master/"
// How often each list is downloaded. No list is downloaded more often than
// its publisher updates it or allows.
const (
@@ -146,6 +149,28 @@ func vpnLists() []List {
Format: FormatPIAJSON,
VPN: true,
},
{
// Proton's own server list needs a logged-in session, so this is
// the public ProtonVPN-IPs mirror of it. Proton publishes only
// the addresses clients connect to. The mirror's README says
// traffic leaves from others in the same /24, so this file of
// /24 networks is the one it says to use for detecting exits.
Name: "protonvpn",
Description: "In a /24 network that holds a server of the " +
"ProtonVPN service",
URL: protonVPNIPsURL + "protonvpn_ip_ranges.txt",
Interval: vpnListInterval,
Format: FormatProtonVPN,
VPN: true,
},
{
Name: "protonvpn_entry",
Description: "A server of the ProtonVPN service",
URL: protonVPNIPsURL + "protonvpn_entry_ips.txt",
Interval: vpnListInterval,
Format: FormatProtonVPN,
VPN: true,
},
{
Name: "x4bnet",
Description: "In an IPv4 network of a VPN service, according to " +
+18
View File
@@ -58,6 +58,8 @@ func parse(r io.Reader, format Format) (*netipx.IPSet, error) {
return parsePIA(r)
case FormatIVPNJSON:
return parseIVPN(r)
case FormatProtonVPN:
return parseProtonVPN(r)
default:
return nil, errUnknownFormat
}
@@ -329,6 +331,22 @@ func parseIVPN(r io.Reader) (*netipx.IPSet, error) {
return addressSet(texts)
}
// parseProtonVPN returns the addresses or networks in a ProtonVPN-IPs list. A
// file with no addresses or networks fails, because the service always has
// servers.
func parseProtonVPN(r io.Reader) (*netipx.IPSet, error) {
addresses, err := parseLines(r, addAddressOrNetwork)
if err != nil {
return nil, err
}
if len(addresses.Ranges()) == 0 {
return nil, errNoAddresses
}
return addresses, nil
}
// readJSON reads r to its end and decodes it, as one JSON document, into v.
func readJSON(r io.Reader, v any) error {
data, err := io.ReadAll(r)
+113 -67
View File
@@ -17,6 +17,7 @@ import (
"strings"
"sync/atomic"
"testing"
"testing/synctest"
"time"
"git.eeqj.de/sneak/routewatch/internal/config"
@@ -173,6 +174,9 @@ type tableStats struct {
// carried 203.0.113.0/24, the two with kept routes withdrew it, so its origin
// keeps no route. The oldest and newest routes are those of the other two
// prefixes.
//
// The requests run in a synctest bubble, whose clock is fake, so a stall on
// the host does not use up the handlers' 4-second limit.
func TestStatsHandlersAnswerFromMemory(t *testing.T) {
t.Parallel()
@@ -218,7 +222,11 @@ func TestStatsHandlersAnswerFromMemory(t *testing.T) {
for name, handler := range handlers {
rec := httptest.NewRecorder()
handler(rec, httptest.NewRequestWithContext(t.Context(), http.MethodGet, "/", nil))
synctest.Test(t, func(t *testing.T) {
handler(rec,
httptest.NewRequestWithContext(t.Context(), http.MethodGet, "/", nil))
})
if rec.Code != http.StatusOK {
t.Errorf("%s: status %d, want %d; body %s",
@@ -253,15 +261,20 @@ func TestStatsHandlersAnswerFromMemory(t *testing.T) {
// TestStatsAnswerCarriesChurnPerMinute checks that the stream object of the
// /api/v1/stats answer holds route_churn_per_min as a whole number, and no
// longer route_churn_per_sec, the moving average it replaces
// (https://git.eeqj.de/sneak/routewatch/issues/160).
// (https://git.eeqj.de/sneak/routewatch/issues/160). The request runs in a
// synctest bubble, whose clock is fake, so a stall on the host does not use up
// the handler's 4-second limit.
func TestStatsAnswerCarriesChurnPerMinute(t *testing.T) {
t.Parallel()
s := newServerWithTestRoutes(t)
rec := httptest.NewRecorder()
s.handleStats()(rec,
httptest.NewRequestWithContext(t.Context(), http.MethodGet, statsPath, nil))
synctest.Test(t, func(t *testing.T) {
s.handleStats()(rec,
httptest.NewRequestWithContext(t.Context(), http.MethodGet, statsPath, nil))
})
if rec.Code != http.StatusOK {
t.Fatalf("status %d, want %d; body %s", rec.Code, http.StatusOK, rec.Body)
@@ -309,18 +322,26 @@ func (heldBackHandler) HeldBackCount() int { return heldBackMess
// TestStatsAnswerGivesHowManyMessagesAHandlerHoldsBack registers a handler
// that holds back messages. Its figures in /api/v1/stats, which the status
// page shows, must give how many.
// page shows, must give how many. The request runs in a synctest bubble, whose
// clock is fake, so a stall on the host does not use up the handler's 4-second
// limit.
func TestStatsAnswerGivesHowManyMessagesAHandlerHoldsBack(t *testing.T) {
t.Parallel()
s := newServerWithTestRoutes(t)
s.streamer.RegisterHandler(heldBackHandler{})
var answer []byte
synctest.Test(t, func(t *testing.T) {
answer = get(t, s, statsPath).Body.Bytes()
})
var body struct {
Data StatsResponse `json:"data"`
}
err := json.Unmarshal(get(t, s, statsPath).Body.Bytes(), &body)
err := json.Unmarshal(answer, &body)
if err != nil {
t.Fatalf("decoding the answer: %v", err)
}
@@ -364,7 +385,9 @@ type unitStats struct {
// names. They used to be strings with the unit in them, such as "1h18m34s" or
// "1.1 GB" (https://git.eeqj.de/sneak/routewatch/issues/199). The stream has
// never connected, so the uptime is 0. The memory use in /api/v1/stats is the
// same number of bytes as its gc.heap_alloc_bytes.
// same number of bytes as its gc.heap_alloc_bytes. The requests run in a
// synctest bubble, whose clock is fake, so a stall on the host does not use up
// the handlers' 4-second limit.
func TestStatsAnswersGiveUnitsInFieldNames(t *testing.T) {
t.Parallel()
@@ -374,7 +397,11 @@ func TestStatsAnswersGiveUnitsInFieldNames(t *testing.T) {
s.SetASNFetcher(fixedIntervalASNFetcher{interval: fetchInterval})
for _, path := range []string{statusJSONPath, statsPath} {
answer := get(t, s, path).Body.Bytes()
var answer []byte
synctest.Test(t, func(t *testing.T) {
answer = get(t, s, path).Body.Bytes()
})
var body struct {
Data unitStats `json:"data"`
@@ -415,82 +442,89 @@ func TestStatsAnswersGiveUnitsInFieldNames(t *testing.T) {
// connection_duration_seconds of /api/v1/stats is null while the stream is not
// connected, and a whole number of seconds once it is. It used to be the
// string "disconnected", or a Go duration such as "1h18m34s"
// (https://git.eeqj.de/sneak/routewatch/issues/199).
// (https://git.eeqj.de/sneak/routewatch/issues/199). The test runs in a
// synctest bubble, whose clock is fake, so a stall on the host does not use up
// the handler's 4-second limit.
func TestStatsAnswerGivesConnectionDurationInSeconds(t *testing.T) {
t.Parallel()
s := newServerWithTestRoutes(t)
connectionDuration := func() json.RawMessage {
answer := get(t, s, statsPath).Body.Bytes()
synctest.Test(t, func(t *testing.T) {
connectionDuration := func() json.RawMessage {
answer := get(t, s, statsPath).Body.Bytes()
var body struct {
Data map[string]json.RawMessage `json:"data"`
var body struct {
Data map[string]json.RawMessage `json:"data"`
}
err := json.Unmarshal(answer, &body)
if err != nil {
t.Fatalf("decoding the answer: %v; answer %s", err, answer)
}
duration, ok := body.Data["connection_duration_seconds"]
if !ok {
t.Fatalf("no connection_duration_seconds: %s", answer)
}
return duration
}
err := json.Unmarshal(answer, &body)
if err != nil {
t.Fatalf("decoding the answer: %v; answer %s", err, answer)
disconnected := connectionDuration()
if string(disconnected) != "null" {
t.Errorf("disconnected: connection_duration_seconds = %s, want null",
disconnected)
}
duration, ok := body.Data["connection_duration_seconds"]
if !ok {
t.Fatalf("no connection_duration_seconds: %s", answer)
s.streamer.GetMetricsTracker().SetConnected(true)
var seconds int64
connected := connectionDuration()
err := json.Unmarshal(connected, &seconds)
if err != nil || seconds != 0 {
t.Errorf("just connected: connection_duration_seconds = %s, want 0",
connected)
}
return duration
}
disconnected := connectionDuration()
if string(disconnected) != "null" {
t.Errorf("disconnected: connection_duration_seconds = %s, want null",
disconnected)
}
s.streamer.GetMetricsTracker().SetConnected(true)
var seconds int64
connected := connectionDuration()
err := json.Unmarshal(connected, &seconds)
if err != nil || seconds != 0 {
t.Errorf("just connected: connection_duration_seconds = %s, want 0",
connected)
}
})
}
// TestStreamDurationsCountSeconds checks that the uptime and the connection
// duration of a stream that connected 90 seconds ago are both 90. A stream
// that has never connected has an uptime of 0, and one that is not connected
// has no connection duration.
// has no connection duration. The test runs in a synctest bubble, whose clock
// is fake, so a stall on the host does not add to the 90 seconds.
func TestStreamDurationsCountSeconds(t *testing.T) {
t.Parallel()
const connectedFor = 90 * time.Second
synctest.Test(t, func(t *testing.T) {
const connectedFor = 90 * time.Second
connectedSince := time.Now().Add(-connectedFor)
want := int64(connectedFor / time.Second)
connectedSince := time.Now().Add(-connectedFor)
want := int64(connectedFor / time.Second)
if got := uptimeSeconds(connectedSince); got != want {
t.Errorf("uptimeSeconds = %d, want %d", got, want)
}
if got := uptimeSeconds(connectedSince); got != want {
t.Errorf("uptimeSeconds = %d, want %d", got, want)
}
if got := uptimeSeconds(time.Time{}); got != 0 {
t.Errorf("uptimeSeconds before any connection = %d, want 0", got)
}
if got := uptimeSeconds(time.Time{}); got != 0 {
t.Errorf("uptimeSeconds before any connection = %d, want 0", got)
}
switch got := connectionDurationSeconds(true, connectedSince); {
case got == nil:
t.Errorf("connectionDurationSeconds = nil, want %d", want)
case *got != want:
t.Errorf("connectionDurationSeconds = %d, want %d", *got, want)
}
switch got := connectionDurationSeconds(true, connectedSince); {
case got == nil:
t.Errorf("connectionDurationSeconds = nil, want %d", want)
case *got != want:
t.Errorf("connectionDurationSeconds = %d, want %d", *got, want)
}
if got := connectionDurationSeconds(false, connectedSince); got != nil {
t.Errorf("connectionDurationSeconds while not connected = %d, want nil",
*got)
}
if got := connectionDurationSeconds(false, connectedSince); got != nil {
t.Errorf("connectionDurationSeconds while not connected = %d, want nil",
*got)
}
})
}
// newTestTable returns a routing table that holds a route for testPrefixV4,
@@ -1134,7 +1168,9 @@ func (failingDB) GetPrefixFirstSeen(
// with the driver's error text. It also sends a prefix that does not parse.
// No answer may carry the driver's or the prefix parser's text. Before the fix
// the answers sent it to the client
// (https://git.eeqj.de/sneak/routewatch/issues/108).
// (https://git.eeqj.de/sneak/routewatch/issues/108). The requests run in a
// synctest bubble, whose clock is fake, so a stall on the host does not use up
// the stats handlers' 4-second limit.
func TestJSONRoutesDoNotSendErrorDetails(t *testing.T) {
t.Parallel()
@@ -1155,8 +1191,11 @@ func TestJSONRoutesDoNotSendErrorDetails(t *testing.T) {
for _, tt := range tests {
rec := httptest.NewRecorder()
s.router.ServeHTTP(rec, httptest.NewRequestWithContext(
t.Context(), http.MethodGet, tt.path, nil))
synctest.Test(t, func(t *testing.T) {
s.router.ServeHTTP(rec, httptest.NewRequestWithContext(
t.Context(), http.MethodGet, tt.path, nil))
})
if rec.Code != tt.wantCode {
t.Errorf("%s: status %d, want %d; body %s",
@@ -1397,8 +1436,11 @@ func TestIPLookupKeepsRouteWhenPTRLookupDoesNotAnswer(t *testing.T) {
s := newServerWithTestRoutes(t)
s.resolver = &fakeResolver{ptrNeverAnswers: true}
// The request's time limit passes after the PTR lookup's.
ctx, cancel := context.WithTimeout(t.Context(), 2*ptrLookupTimeout)
// The request's time limit passes well after the PTR lookup's, so that a
// stall on a busy host does not use up the difference.
const requestTimeLimit = ptrLookupTimeout + 15*time.Second
ctx, cancel := context.WithTimeout(t.Context(), requestTimeLimit)
defer cancel()
rec := httptest.NewRecorder()
@@ -2367,9 +2409,13 @@ func settledGoroutineCount() int {
return prev
}
// waitForGoroutines waits until the goroutine count drops to target or below.
// waitForGoroutines waits up to 15 seconds for the goroutine count to drop to
// target or below.
func waitForGoroutines(target int) bool {
for range 100 {
const maxWait = 15 * time.Second
deadline := time.Now().Add(maxWait)
for time.Now().Before(deadline) {
if runtime.NumGoroutine() <= target {
return true
}
+8 -4
View File
@@ -76,7 +76,9 @@ func asLink(asn int) string {
}
// TestHomePageCardsShowTheTableFigures fills a routing table and the asns
// table, works out the cards, and checks that each card shows its figure.
// table, works out the cards, and checks that each card shows its figure. The
// cards are worked out in a synctest bubble, whose clock is fake, so a stall on
// the host does not use up the 4-second limit on their database read.
func TestHomePageCardsShowTheTableFigures(t *testing.T) {
t.Parallel()
@@ -95,7 +97,9 @@ func TestHomePageCardsShowTheTableFigures(t *testing.T) {
t.Fatalf("GetOrCreateASNBatch: %v", err)
}
s.refreshHomeCards(t.Context())
synctest.Test(t, func(t *testing.T) {
s.refreshHomeCards(t.Context())
})
page := get(t, s, "/").Body.String()
@@ -279,7 +283,7 @@ func TestKeepHomeCardsFreshWorksThemOutEveryInterval(t *testing.T) {
const (
interval = 10 * time.Millisecond
maxWait = 5 * time.Second
maxWait = 15 * time.Second
)
table := emptyTable()
@@ -321,7 +325,7 @@ func TestKeepHomeCardsFreshWorksThemOutEveryInterval(t *testing.T) {
func TestStartedServerWorksOutTheCards(t *testing.T) {
t.Parallel()
const maxWait = 5 * time.Second
const maxWait = 15 * time.Second
s := newServerWithTable(t, newCardsTable())
+39 -32
View File
@@ -14,6 +14,7 @@ import (
"regexp"
"strings"
"testing"
"testing/synctest"
"time"
"git.eeqj.de/sneak/routewatch/internal/config"
@@ -163,6 +164,10 @@ func TestJSONAnswersPutNoUnitInAString(t *testing.T) {
// router and the middleware write. Before the fix a panic in a handler, and a
// method /status.json or the health check does not take, answered with an
// empty body (https://git.eeqj.de/sneak/routewatch/issues/201).
//
// The requests run in a synctest bubble, whose clock is fake, so a stall on
// the host does not use up the 2-second limit of the health check or the
// 4-second limit of the stats handlers.
func TestJSONAnswersCarryMeta(t *testing.T) {
t.Parallel()
@@ -181,6 +186,10 @@ func TestJSONAnswersCarryMeta(t *testing.T) {
s.router.Get(panicsPath, JSONValidationMiddleware(http.HandlerFunc(
func(http.ResponseWriter, *http.Request) { panic(errDiskIO) })).ServeHTTP)
failing := New(failingDB{}, newTestTable(),
streamer.New(logger.New(), metrics.New()), logger.New(), &config.Config{})
failing.resolver = &fakeResolver{}
tests := []struct {
method string
path string
@@ -216,42 +225,40 @@ func TestJSONAnswersCarryMeta(t *testing.T) {
{http.MethodGet, panicsPath, http.StatusInternalServerError},
}
for _, tt := range tests {
checkAnswerCarriesMeta(t, s, httptest.NewRequestWithContext(
t.Context(), tt.method, tt.path, nil), tt.wantCode)
}
synctest.Test(t, func(t *testing.T) {
for _, tt := range tests {
checkAnswerCarriesMeta(t, s, httptest.NewRequestWithContext(
t.Context(), tt.method, tt.path, nil), tt.wantCode)
}
// A request whose time limit has already passed answers 504. The handlers
// write the answers of the first five routes, and the JSON check on the
// route writes the last.
pastTimeLimit, cancel := context.WithDeadline(t.Context(), time.Now())
defer cancel()
// A request whose time limit has already passed answers 504. The
// handlers write the answers of the first five routes, and the JSON
// check on the route writes the last.
pastTimeLimit, cancel := context.WithDeadline(t.Context(), time.Now())
defer cancel()
for _, path := range []string{
statusJSONPath, statsPath, "/ip/" + addrInTestPrefixV4,
"/api/v1/ip/" + addrInTestPrefixV4, asJSONPath, writesNothingPath,
} {
checkAnswerCarriesMeta(t, s, httptest.NewRequestWithContext(
pastTimeLimit, http.MethodGet, path, nil), http.StatusGatewayTimeout)
}
for _, path := range []string{
statusJSONPath, statsPath, "/ip/" + addrInTestPrefixV4,
"/api/v1/ip/" + addrInTestPrefixV4, asJSONPath, writesNothingPath,
} {
checkAnswerCarriesMeta(t, s, httptest.NewRequestWithContext(
pastTimeLimit, http.MethodGet, path, nil), http.StatusGatewayTimeout)
}
// With a store whose reads fail, the health check answers 503, and the
// handlers of the other routes 500.
failing := New(failingDB{}, newTestTable(),
streamer.New(logger.New(), metrics.New()), logger.New(), &config.Config{})
failing.resolver = &fakeResolver{}
checkAnswerCarriesMeta(t, failing, httptest.NewRequestWithContext(
t.Context(), http.MethodGet, healthCheckPath, nil),
http.StatusServiceUnavailable)
for _, path := range []string{
statusJSONPath, statsPath, asJSONPath,
"/api/v1/ip/" + addrInTestPrefixV4, "/ip/" + addrInTestPrefixV4,
} {
// With a store whose reads fail, the health check answers 503, and
// the handlers of the other routes 500.
checkAnswerCarriesMeta(t, failing, httptest.NewRequestWithContext(
t.Context(), http.MethodGet, path, nil), http.StatusInternalServerError)
}
t.Context(), http.MethodGet, healthCheckPath, nil),
http.StatusServiceUnavailable)
for _, path := range []string{
statusJSONPath, statsPath, asJSONPath,
"/api/v1/ip/" + addrInTestPrefixV4, "/ip/" + addrInTestPrefixV4,
} {
checkAnswerCarriesMeta(t, failing, httptest.NewRequestWithContext(
t.Context(), http.MethodGet, path, nil), http.StatusInternalServerError)
}
})
}
// checkAnswerCarriesMeta serves req through the server's router, and checks
+11 -2
View File
@@ -5,6 +5,7 @@ import (
"net/http"
"net/http/httptest"
"testing"
"testing/synctest"
)
// TestRoutesAnswerHEAD sends HEAD for a page and JSON routes through the
@@ -12,15 +13,23 @@ import (
// lookup of the caller's own address, are in the list because chi's GetHead
// does not reach them. Before the fix every route answered HEAD with 405
// (https://git.eeqj.de/sneak/routewatch/issues/111).
//
// The requests run in a synctest bubble, whose clock is fake, so a stall on
// the host does not use up the stats handlers' 4-second limit. The resolver
// is fake because no time limit would cut off a real DNS lookup in a bubble.
func TestRoutesAnswerHEAD(t *testing.T) {
t.Parallel()
s := newServerWithTestRoutes(t)
s.resolver = &fakeResolver{}
for _, path := range []string{"/", statusJSONPath, statsPath, "/ip", "/ip/"} {
rec := httptest.NewRecorder()
s.router.ServeHTTP(rec, httptest.NewRequestWithContext(
t.Context(), http.MethodHead, path, nil))
synctest.Test(t, func(t *testing.T) {
s.router.ServeHTTP(rec, httptest.NewRequestWithContext(
t.Context(), http.MethodHead, path, nil))
})
if rec.Code != http.StatusOK {
t.Errorf("HEAD %s: status %d, want %d; body %s",
+2 -2
View File
@@ -22,10 +22,10 @@ func TestStopClosesTheConnectionOfARequestStillRunning(t *testing.T) {
stopDeadline = 100 * time.Millisecond
// waitTimeout is how long the test waits for the request to reach
// the handler, and then for the client to see its connection closed.
waitTimeout = 2 * time.Second
waitTimeout = 15 * time.Second
// readHeaderTimeout bounds the test server's read of the request
// header, as every http.Server must.
readHeaderTimeout = time.Second
readHeaderTimeout = waitTimeout
)
release := make(chan struct{})
+130 -73
View File
@@ -14,6 +14,7 @@ import (
"strings"
"sync/atomic"
"testing"
"testing/synctest"
"time"
"git.eeqj.de/sneak/routewatch/internal/logger"
@@ -398,7 +399,7 @@ func TestStopQueuesTheHeldStateMessages(t *testing.T) {
const (
held = 2
lines = held + 1
maxWait = 5 * time.Second
maxWait = 15 * time.Second
)
srv := httptest.NewServer(http.HandlerFunc(
@@ -478,15 +479,15 @@ func TestStopQueuesTheHeldStateMessages(t *testing.T) {
// TestStopReturnsByItsDeadline stops a streamer whose handler has a full queue
// and a STATE message held for it, and a worker that stays busy until the test
// ends, as one does whose handler waits for a database write. Stop must return
// within its deadline plus a small margin instead of waiting for room on the
// queue and for the worker, and log as not handed over the held message and
// every message on the queue.
// within its deadline plus a margin instead of waiting for room on the queue
// and for the worker, and log as not handed over the held message and every
// message on the queue.
func TestStopReturnsByItsDeadline(t *testing.T) {
t.Parallel()
const (
deadline = 100 * time.Millisecond
margin = 2 * time.Second
margin = 15 * time.Second
)
var logs bytes.Buffer
@@ -577,7 +578,7 @@ func TestStopBeforeMessageReachesQueues(t *testing.T) {
func TestReconnectWhenStreamEnds(t *testing.T) {
t.Parallel()
endResponse := func(http.ResponseWriter, *http.Request) {}
endResponse := pipeTransport{write: func(context.Context, io.Writer) {}}
checkReconnect(t, endResponse, streamIdleTimeout)
}
@@ -589,9 +590,8 @@ func TestReconnectWhenStreamGoesIdle(t *testing.T) {
const idleTimeout = 50 * time.Millisecond
holdResponse := func(w http.ResponseWriter, r *http.Request) {
flush(w)
<-r.Context().Done()
holdResponse := pipeTransport{
write: func(ctx context.Context, _ io.Writer) { <-ctx.Done() },
}
checkReconnect(t, holdResponse, idleTimeout)
}
@@ -604,16 +604,51 @@ func TestReconnectWhenServerNeverAnswers(t *testing.T) {
const idleTimeout = 50 * time.Millisecond
neverAnswer := func(_ http.ResponseWriter, r *http.Request) {
<-r.Context().Done()
}
neverAnswer := roundTripFunc(
func(req *http.Request) (*http.Response, error) {
<-req.Context().Done()
return nil, req.Context().Err()
},
)
checkReconnect(t, neverAnswer, idleTimeout)
}
// TestStreamNotDroppedWhileLinesArrive has the server send lines well within
// the idle timeout for several times the timeout, then end the response. Each
// line must start the idle timeout over, so the stream lasts until the server
// ends it instead of being dropped as idle.
// pipeTransport answers every request with a response whose body write fills.
// The body fails when the request's context ends, as a connection's does.
// write gets that context, so that it can end when the request does.
type pipeTransport struct {
write func(context.Context, io.Writer)
}
func (p pipeTransport) RoundTrip(req *http.Request) (*http.Response, error) {
body, w := io.Pipe()
context.AfterFunc(req.Context(), func() {
_ = w.CloseWithError(req.Context().Err())
})
go func(ctx context.Context) {
p.write(ctx, w)
_ = w.Close()
}(req.Context())
return &http.Response{StatusCode: http.StatusOK, Body: body}, nil
}
// roundTripFunc makes a function an http.RoundTripper.
type roundTripFunc func(*http.Request) (*http.Response, error)
func (f roundTripFunc) RoundTrip(req *http.Request) (*http.Response, error) {
return f(req)
}
// TestStreamNotDroppedWhileLinesArrive has a response send lines well within
// the idle timeout for several times the timeout, then end. Each line must
// start the idle timeout over, so the stream lasts until the response ends
// instead of being dropped as idle. It runs in a synctest bubble, whose clock
// is fake, so a stall on the host does not use up the timeout.
func TestStreamNotDroppedWhileLinesArrive(t *testing.T) {
t.Parallel()
@@ -623,26 +658,35 @@ func TestStreamNotDroppedWhileLinesArrive(t *testing.T) {
sendFor = 5 * idleTimeout
)
srv := httptest.NewServer(http.HandlerFunc(
func(w http.ResponseWriter, _ *http.Request) {
// The streamer is made outside the bubble. Its metrics start a goroutine
// that never returns, and a bubble waits for each of its goroutines to
// return.
s := New(logger.New(), metrics.New())
s.idleTimeout = idleTimeout
s.client = &http.Client{Transport: pipeTransport{
write: func(ctx context.Context, w io.Writer) {
deadline := time.Now().Add(sendFor)
for time.Now().Before(deadline) {
_, _ = io.WriteString(w, updateLine)
flush(w)
time.Sleep(lineInterval)
// The wait ends with the stream. A bubble's clock stops when
// the test function returns, and a goroutine then still
// asleep panics the test binary.
select {
case <-ctx.Done():
return
case <-time.After(lineInterval):
}
}
}))
defer srv.Close()
},
}}
s := New(logger.New(), metrics.New())
s.url = srv.URL
s.idleTimeout = idleTimeout
err := s.stream(context.Background())
if !errors.Is(err, errStreamEnded) {
t.Fatalf("stream returned %v, want %v", err, errStreamEnded)
}
synctest.Test(t, func(t *testing.T) {
err := s.stream(context.Background())
if !errors.Is(err, errStreamEnded) {
t.Fatalf("stream returned %v, want %v", err, errStreamEnded)
}
})
}
// TestStreamReadsLongLine has the server send one UPDATE line of about
@@ -706,7 +750,7 @@ func TestHandlerStatsReadWhileMessagesAreDropped(t *testing.T) {
const (
lines = 5
wantDropped = lines - 2
maxWait = 5 * time.Second
maxWait = 15 * time.Second
)
srv := httptest.NewServer(http.HandlerFunc(
@@ -820,61 +864,70 @@ func streamLines(t *testing.T, s *Streamer, lines string) {
// line and then stays open. The streamer must come back for a second
// response, must not be marked connected when it does, and must handle the
// UPDATE in it.
//
// It runs in a synctest bubble, whose clock is fake, so a stall on the host
// does not use up the idle timeout. The fake server gets the requests in the
// order the streamer makes them, because no real connection carries them.
func checkReconnect(
t *testing.T, firstResponse http.HandlerFunc, idleTimeout time.Duration,
t *testing.T, firstResponse http.RoundTripper, idleTimeout time.Duration,
) {
t.Helper()
const maxWait = 5 * time.Second
const maxWait = 15 * time.Second
// The tracker is made outside the bubble. It starts a goroutine that never
// returns, and a bubble waits for each of its goroutines to return.
tracker := metrics.New()
connectedAtReconnect := make(chan bool, 1)
var requests atomic.Int32
synctest.Test(t, func(t *testing.T) {
connectedAtReconnect := make(chan bool, 1)
laterResponse := pipeTransport{
write: func(ctx context.Context, w io.Writer) {
_, _ = io.WriteString(w, updateLine)
srv := httptest.NewServer(http.HandlerFunc(
func(w http.ResponseWriter, r *http.Request) {
if requests.Add(1) == 1 {
firstResponse(w, r)
<-ctx.Done()
},
}
return
}
var requests atomic.Int32
select {
case connectedAtReconnect <- tracker.IsConnected():
default:
}
s := New(logger.New(), tracker)
s.client = &http.Client{Transport: roundTripFunc(
func(req *http.Request) (*http.Response, error) {
if requests.Add(1) == 1 {
return firstResponse.RoundTrip(req)
}
_, _ = io.WriteString(w, updateLine)
select {
case connectedAtReconnect <- tracker.IsConnected():
default:
}
flush(w)
<-r.Context().Done()
}))
defer srv.Close()
return laterResponse.RoundTrip(req)
},
)}
s.idleTimeout = idleTimeout
s.minBackoff = time.Millisecond
handler := updateHandler{handled: make(chan struct{}, 1)}
s.RegisterHandler(handler)
s := New(logger.New(), tracker)
s.url = srv.URL
s.idleTimeout = idleTimeout
s.minBackoff = time.Millisecond
handler := updateHandler{handled: make(chan struct{}, 1)}
s.RegisterHandler(handler)
err := s.Start(t.Context())
if err != nil {
t.Fatalf("Start returned error: %v", err)
}
defer s.Stop(t.Context())
err := s.Start(t.Context())
if err != nil {
t.Fatalf("Start returned error: %v", err)
}
defer s.Stop(t.Context())
select {
case <-handler.handled:
case <-time.After(maxWait):
t.Fatalf("no message from a second response handled within %s", maxWait)
}
select {
case <-handler.handled:
case <-time.After(maxWait):
t.Fatalf("no message from a second response handled within %s", maxWait)
}
if <-connectedAtReconnect {
t.Error("streamer was still marked connected when it came back " +
"for a second response")
}
if <-connectedAtReconnect {
t.Error("streamer was still marked connected when it came back " +
"for a second response")
}
})
}
// settledGoroutineCount lets transient goroutines finish, then reports the
@@ -896,9 +949,13 @@ func settledGoroutineCount() int {
return prev
}
// waitForGoroutines waits until the goroutine count drops to target or below.
// waitForGoroutines waits up to 15 seconds for the goroutine count to drop to
// target or below.
func waitForGoroutines(target int) bool {
for range 100 {
const maxWait = 15 * time.Second
deadline := time.Now().Add(maxWait)
for time.Now().Before(deadline) {
if runtime.NumGoroutine() <= target {
return true
}