Make the tests pass on a busy host (closes #240)

Tests that waited a few seconds for something to happen now wait 15 seconds.
The wait ends as soon as the condition holds, so a passing run takes no longer.

TestStreamNotDroppedWhileLinesArrive, the reconnect tests, the tests whose
requests pass a 2- or 4-second limit in the server, and
TestStreamDurationsCountSeconds run on a fake clock, so a stall on the host
cannot change their result. The reconnect tests also use a fake transport, so
a stall cannot reorder their requests.

The checkpoint tests give the writing connection a 30-second busy timeout, so
their limit can be 15 seconds and a checkpoint that waits for a read still
fails them.

Model: opus-5-5

internal/server/handlers_internal_test.go

@@ -17,6 +17,7 @@ import (
 	"strings"
 	"sync/atomic"
 	"testing"
+	"testing/synctest"
 	"time"
 
 	"git.eeqj.de/sneak/routewatch/internal/config"
@@ -173,6 +174,9 @@ type tableStats struct {
 // carried 203.0.113.0/24, the two with kept routes withdrew it, so its origin
 // keeps no route. The oldest and newest routes are those of the other two
 // prefixes.
+//
+// The requests run in a synctest bubble, whose clock is fake, so a stall on
+// the host does not use up the handlers' 4-second limit.
 func TestStatsHandlersAnswerFromMemory(t *testing.T) {
 	t.Parallel()
 
@@ -218,7 +222,11 @@ func TestStatsHandlersAnswerFromMemory(t *testing.T) {
 
 	for name, handler := range handlers {
 		rec := httptest.NewRecorder()
-		handler(rec, httptest.NewRequestWithContext(t.Context(), http.MethodGet, "/", nil))
+
+		synctest.Test(t, func(t *testing.T) {
+			handler(rec,
+				httptest.NewRequestWithContext(t.Context(), http.MethodGet, "/", nil))
+		})
 
 		if rec.Code != http.StatusOK {
 			t.Errorf("%s: status %d, want %d; body %s",
@@ -253,15 +261,20 @@ func TestStatsHandlersAnswerFromMemory(t *testing.T) {
 // TestStatsAnswerCarriesChurnPerMinute checks that the stream object of the
 // /api/v1/stats answer holds route_churn_per_min as a whole number, and no
 // longer route_churn_per_sec, the moving average it replaces
-// (https://git.eeqj.de/sneak/routewatch/issues/160).
+// (https://git.eeqj.de/sneak/routewatch/issues/160). The request runs in a
+// synctest bubble, whose clock is fake, so a stall on the host does not use up
+// the handler's 4-second limit.
 func TestStatsAnswerCarriesChurnPerMinute(t *testing.T) {
 	t.Parallel()
 
 	s := newServerWithTestRoutes(t)
 
 	rec := httptest.NewRecorder()
-	s.handleStats()(rec,
-		httptest.NewRequestWithContext(t.Context(), http.MethodGet, statsPath, nil))
+
+	synctest.Test(t, func(t *testing.T) {
+		s.handleStats()(rec,
+			httptest.NewRequestWithContext(t.Context(), http.MethodGet, statsPath, nil))
+	})
 
 	if rec.Code != http.StatusOK {
 		t.Fatalf("status %d, want %d; body %s", rec.Code, http.StatusOK, rec.Body)
@@ -309,18 +322,26 @@ func (heldBackHandler) HeldBackCount() int                 { return heldBackMess
 
 // TestStatsAnswerGivesHowManyMessagesAHandlerHoldsBack registers a handler
 // that holds back messages. Its figures in /api/v1/stats, which the status
-// page shows, must give how many.
+// page shows, must give how many. The request runs in a synctest bubble, whose
+// clock is fake, so a stall on the host does not use up the handler's 4-second
+// limit.
 func TestStatsAnswerGivesHowManyMessagesAHandlerHoldsBack(t *testing.T) {
 	t.Parallel()
 
 	s := newServerWithTestRoutes(t)
 	s.streamer.RegisterHandler(heldBackHandler{})
 
+	var answer []byte
+
+	synctest.Test(t, func(t *testing.T) {
+		answer = get(t, s, statsPath).Body.Bytes()
+	})
+
 	var body struct {
 		Data StatsResponse `json:"data"`
 	}
 
-	err := json.Unmarshal(get(t, s, statsPath).Body.Bytes(), &body)
+	err := json.Unmarshal(answer, &body)
 	if err != nil {
 		t.Fatalf("decoding the answer: %v", err)
 	}
@@ -364,7 +385,9 @@ type unitStats struct {
 // names. They used to be strings with the unit in them, such as "1h18m34s" or
 // "1.1 GB" (https://git.eeqj.de/sneak/routewatch/issues/199). The stream has
 // never connected, so the uptime is 0. The memory use in /api/v1/stats is the
-// same number of bytes as its gc.heap_alloc_bytes.
+// same number of bytes as its gc.heap_alloc_bytes. The requests run in a
+// synctest bubble, whose clock is fake, so a stall on the host does not use up
+// the handlers' 4-second limit.
 func TestStatsAnswersGiveUnitsInFieldNames(t *testing.T) {
 	t.Parallel()
 
@@ -374,7 +397,11 @@ func TestStatsAnswersGiveUnitsInFieldNames(t *testing.T) {
 	s.SetASNFetcher(fixedIntervalASNFetcher{interval: fetchInterval})
 
 	for _, path := range []string{statusJSONPath, statsPath} {
-		answer := get(t, s, path).Body.Bytes()
+		var answer []byte
+
+		synctest.Test(t, func(t *testing.T) {
+			answer = get(t, s, path).Body.Bytes()
+		})
 
 		var body struct {
 			Data unitStats `json:"data"`
@@ -415,82 +442,89 @@ func TestStatsAnswersGiveUnitsInFieldNames(t *testing.T) {
 // connection_duration_seconds of /api/v1/stats is null while the stream is not
 // connected, and a whole number of seconds once it is. It used to be the
 // string "disconnected", or a Go duration such as "1h18m34s"
-// (https://git.eeqj.de/sneak/routewatch/issues/199).
+// (https://git.eeqj.de/sneak/routewatch/issues/199). The test runs in a
+// synctest bubble, whose clock is fake, so a stall on the host does not use up
+// the handler's 4-second limit.
 func TestStatsAnswerGivesConnectionDurationInSeconds(t *testing.T) {
 	t.Parallel()
 
 	s := newServerWithTestRoutes(t)
 
-	connectionDuration := func() json.RawMessage {
-		answer := get(t, s, statsPath).Body.Bytes()
+	synctest.Test(t, func(t *testing.T) {
+		connectionDuration := func() json.RawMessage {
+			answer := get(t, s, statsPath).Body.Bytes()
 
-		var body struct {
-			Data map[string]json.RawMessage `json:"data"`
+			var body struct {
+				Data map[string]json.RawMessage `json:"data"`
+			}
+
+			err := json.Unmarshal(answer, &body)
+			if err != nil {
+				t.Fatalf("decoding the answer: %v; answer %s", err, answer)
+			}
+
+			duration, ok := body.Data["connection_duration_seconds"]
+			if !ok {
+				t.Fatalf("no connection_duration_seconds: %s", answer)
+			}
+
+			return duration
 		}
 
-		err := json.Unmarshal(answer, &body)
-		if err != nil {
-			t.Fatalf("decoding the answer: %v; answer %s", err, answer)
+		disconnected := connectionDuration()
+		if string(disconnected) != "null" {
+			t.Errorf("disconnected: connection_duration_seconds = %s, want null",
+				disconnected)
 		}
 
-		duration, ok := body.Data["connection_duration_seconds"]
-		if !ok {
-			t.Fatalf("no connection_duration_seconds: %s", answer)
+		s.streamer.GetMetricsTracker().SetConnected(true)
+
+		var seconds int64
+
+		connected := connectionDuration()
+
+		err := json.Unmarshal(connected, &seconds)
+		if err != nil || seconds != 0 {
+			t.Errorf("just connected: connection_duration_seconds = %s, want 0",
+				connected)
 		}
-
-		return duration
-	}
-
-	disconnected := connectionDuration()
-	if string(disconnected) != "null" {
-		t.Errorf("disconnected: connection_duration_seconds = %s, want null",
-			disconnected)
-	}
-
-	s.streamer.GetMetricsTracker().SetConnected(true)
-
-	var seconds int64
-
-	connected := connectionDuration()
-
-	err := json.Unmarshal(connected, &seconds)
-	if err != nil || seconds != 0 {
-		t.Errorf("just connected: connection_duration_seconds = %s, want 0",
-			connected)
-	}
+	})
 }
 
 // TestStreamDurationsCountSeconds checks that the uptime and the connection
 // duration of a stream that connected 90 seconds ago are both 90. A stream
 // that has never connected has an uptime of 0, and one that is not connected
-// has no connection duration.
+// has no connection duration. The test runs in a synctest bubble, whose clock
+// is fake, so a stall on the host does not add to the 90 seconds.
 func TestStreamDurationsCountSeconds(t *testing.T) {
 	t.Parallel()
 
-	const connectedFor = 90 * time.Second
+	synctest.Test(t, func(t *testing.T) {
+		const connectedFor = 90 * time.Second
 
-	connectedSince := time.Now().Add(-connectedFor)
-	want := int64(connectedFor / time.Second)
+		connectedSince := time.Now().Add(-connectedFor)
+		want := int64(connectedFor / time.Second)
 
-	if got := uptimeSeconds(connectedSince); got != want {
-		t.Errorf("uptimeSeconds = %d, want %d", got, want)
-	}
+		if got := uptimeSeconds(connectedSince); got != want {
+			t.Errorf("uptimeSeconds = %d, want %d", got, want)
+		}
 
-	if got := uptimeSeconds(time.Time{}); got != 0 {
-		t.Errorf("uptimeSeconds before any connection = %d, want 0", got)
-	}
+		if got := uptimeSeconds(time.Time{}); got != 0 {
+			t.Errorf("uptimeSeconds before any connection = %d, want 0", got)
+		}
 
-	switch got := connectionDurationSeconds(true, connectedSince); {
-	case got == nil:
-		t.Errorf("connectionDurationSeconds = nil, want %d", want)
-	case *got != want:
-		t.Errorf("connectionDurationSeconds = %d, want %d", *got, want)
-	}
+		switch got := connectionDurationSeconds(true, connectedSince); {
+		case got == nil:
+			t.Errorf("connectionDurationSeconds = nil, want %d", want)
+		case *got != want:
+			t.Errorf("connectionDurationSeconds = %d, want %d", *got, want)
+		}
 
-	if got := connectionDurationSeconds(false, connectedSince); got != nil {
-		t.Errorf("connectionDurationSeconds while not connected = %d, want nil",
-			*got)
-	}
+		if got := connectionDurationSeconds(false, connectedSince); got != nil {
+			t.Errorf("connectionDurationSeconds while not connected = %d, want nil",
+				*got)
+		}
+	})
 }
 
 // newTestTable returns a routing table that holds a route for testPrefixV4,
@@ -1134,7 +1168,9 @@ func (failingDB) GetPrefixFirstSeen(
 // with the driver's error text. It also sends a prefix that does not parse.
 // No answer may carry the driver's or the prefix parser's text. Before the fix
 // the answers sent it to the client
-// (https://git.eeqj.de/sneak/routewatch/issues/108).
+// (https://git.eeqj.de/sneak/routewatch/issues/108). The requests run in a
+// synctest bubble, whose clock is fake, so a stall on the host does not use up
+// the stats handlers' 4-second limit.
 func TestJSONRoutesDoNotSendErrorDetails(t *testing.T) {
 	t.Parallel()
 
@@ -1155,8 +1191,11 @@ func TestJSONRoutesDoNotSendErrorDetails(t *testing.T) {
 
 	for _, tt := range tests {
 		rec := httptest.NewRecorder()
-		s.router.ServeHTTP(rec, httptest.NewRequestWithContext(
-			t.Context(), http.MethodGet, tt.path, nil))
+
+		synctest.Test(t, func(t *testing.T) {
+			s.router.ServeHTTP(rec, httptest.NewRequestWithContext(
+				t.Context(), http.MethodGet, tt.path, nil))
+		})
 
 		if rec.Code != tt.wantCode {
 			t.Errorf("%s: status %d, want %d; body %s",
@@ -1397,8 +1436,11 @@ func TestIPLookupKeepsRouteWhenPTRLookupDoesNotAnswer(t *testing.T) {
 	s := newServerWithTestRoutes(t)
 	s.resolver = &fakeResolver{ptrNeverAnswers: true}
 
-	// The request's time limit passes after the PTR lookup's.
-	ctx, cancel := context.WithTimeout(t.Context(), 2*ptrLookupTimeout)
+	// The request's time limit passes well after the PTR lookup's, so that a
+	// stall on a busy host does not use up the difference.
+	const requestTimeLimit = ptrLookupTimeout + 15*time.Second
+
+	ctx, cancel := context.WithTimeout(t.Context(), requestTimeLimit)
 	defer cancel()
 
 	rec := httptest.NewRecorder()
@@ -2367,9 +2409,13 @@ func settledGoroutineCount() int {
 	return prev
 }
 
-// waitForGoroutines waits until the goroutine count drops to target or below.
+// waitForGoroutines waits up to 15 seconds for the goroutine count to drop to
+// target or below.
 func waitForGoroutines(target int) bool {
-	for range 100 {
+	const maxWait = 15 * time.Second
+
+	deadline := time.Now().Add(maxWait)
+	for time.Now().Before(deadline) {
 		if runtime.NumGoroutine() <= target {
 			return true
 		}