From 123f34679fdbc3b141881b1728ac50d0315dd71d Mon Sep 17 00:00:00 2001 From: clawbot Date: Thu, 19 Nov 2026 18:30:44 +0000 Subject: [PATCH] rproxy, sww: keep the breaker and stop tests from failing under load (closes #474) The circuit breaker reads the time through a clock the handler holds, which is time.Now outside tests. The test that answers waf's questions with answers rproxy cannot pass on stops that clock, so how long its requests take on a slow host no longer decides whether the breaker opens or turns the next request away. The test that stops rproxy mid-request sets its read deadline from the stop, not from the send, so a slow wait for the app no longer shortens it. Model: opus-5-5 --- internal/rproxy/export_test.go | 6 ++++++ internal/rproxy/rproxy.go | 7 +++++-- internal/rproxy/rproxy_test.go | 32 +++++++++++++++++++++++++++----- internal/rproxy/waf.go | 5 ++--- internal/sww/rproxy_test.go | 11 ++++------- 5 files changed, 44 insertions(+), 17 deletions(-) diff --git a/internal/rproxy/export_test.go b/internal/rproxy/export_test.go index 975298b..1228f37 100644 --- a/internal/rproxy/export_test.go +++ b/internal/rproxy/export_test.go @@ -2,6 +2,12 @@ package rproxy import "time" +// StopBreakerClock has the circuit breaker of s read at as the time, so that how +// long requests take cannot matter to it. It is called before s serves. +func (s *Server) StopBreakerClock(at time.Time) { + s.h.breakerClock = func() time.Time { return at } +} + // Breaker is the circuit breaker, for tests that give it the time. type Breaker struct { b breaker diff --git a/internal/rproxy/rproxy.go b/internal/rproxy/rproxy.go index 489c4c6..b6a7d10 100644 --- a/internal/rproxy/rproxy.go +++ b/internal/rproxy/rproxy.go @@ -94,6 +94,7 @@ func New(params Params) *Server { reportURL: (&url.URL{ Scheme: backend.Scheme, Host: backend.Host, Path: waf.ReportPath, }).String(), + breakerClock: time.Now, } return &Server{ @@ -133,8 +134,10 @@ type handler struct { wafTransport http.RoundTripper // reportURL is where the reports go. reportURL string - // breaker is the circuit breaker for asks to waf. - breaker breaker + // breaker is the circuit breaker for asks to waf, and breakerClock gives it + // the time. + breaker breaker + breakerClock func() time.Time // reports waits for the reports being sent. reportsMu guards starting one // against reportsStopped, which is set once nothing is to wait for another. reports sync.WaitGroup diff --git a/internal/rproxy/rproxy_test.go b/internal/rproxy/rproxy_test.go index 74e8eae..74578f3 100644 --- a/internal/rproxy/rproxy_test.go +++ b/internal/rproxy/rproxy_test.go @@ -356,6 +356,19 @@ func startServerLogging( ) (string, *rproxy.Server) { t.Helper() + server := newServer(t, backend, appURL, env, requestLog, processLog) + + return serve(t, server), server +} + +// newServer returns rproxy, not yet serving, with its request lines going to +// requestLog and its process lines to processLog. +func newServer( + t *testing.T, backend, appURL string, env map[string]string, + requestLog, processLog io.Writer, +) *rproxy.Server { + t.Helper() + settings := map[string]string{ "SWW_RPROXY_WAF_BACKEND": backend, "SWW_RPROXY_UPSTREAM_URL": appURL, wafBackendTimeout: "60000", @@ -367,7 +380,7 @@ func startServerLogging( t.Fatalf("settings %v: %v", settings, err) } - server := rproxy.New(rproxy.Params{ + return rproxy.New(rproxy.Params{ Config: cfg, RequestLog: requestLog, ProcessLog: requestlog.NewProcessLogger( @@ -376,6 +389,11 @@ func startServerLogging( slog.LevelInfo, ), }) +} + +// serve has server serve until the test ends, and returns its address. +func serve(t *testing.T, server *rproxy.Server) string { + t.Helper() listener, err := (&net.ListenConfig{}).Listen( t.Context(), @@ -396,7 +414,7 @@ func startServerLogging( server.WaitForReports() }) - return listener.Addr().String(), server + return listener.Addr().String() } // answered is the visitor's answer. @@ -1532,9 +1550,13 @@ func TestAnswersThatCannotBePassedOnOpenTheBreaker(t *testing.T) { tooLarge(w, r) }) app := startApp(t, greet) - addr, out := startRProxy(t, standIn.URL, app.URL, map[string]string{ - wafBackendTimeout: "60000", - }) + out := &output{} + server := newServer(t, standIn.URL, app.URL, nil, out, out) + // The breaker's clock stands still. On a slow host, the five failures could + // otherwise take more than the five seconds that open the breaker. The next + // request could come more than a second later, when it may ask waf again. + server.StopBreakerClock(time.Now()) + addr := serve(t, server) for range 5 { if got := get(t, addr, "/"); got.status != http.StatusBadGateway { diff --git a/internal/rproxy/waf.go b/internal/rproxy/waf.go index 630329b..5f33bf1 100644 --- a/internal/rproxy/waf.go +++ b/internal/rproxy/waf.go @@ -10,7 +10,6 @@ import ( "maps" "net/http" "strings" - "time" "sneak.berlin/go/smallwebwaf/internal/waf" ) @@ -97,7 +96,7 @@ func (rq *request) askUnlessBreakerOpen( ctx context.Context, ) (*wafAnswer, error) { b := &rq.h.breaker - if !b.mayAsk(time.Now()) { + if !b.mayAsk(rq.h.breakerClock()) { return nil, errBreakerOpen } @@ -107,7 +106,7 @@ func (rq *request) askUnlessBreakerOpen( case err == nil: b.noteAnswer() case ctx.Err() == nil: - b.noteFailure(time.Now()) + b.noteFailure(rq.h.breakerClock()) } return answer, err diff --git a/internal/sww/rproxy_test.go b/internal/sww/rproxy_test.go index 8d54d66..8196b69 100644 --- a/internal/sww/rproxy_test.go +++ b/internal/sww/rproxy_test.go @@ -755,8 +755,7 @@ func TestRProxyStoppingWaitsForAReportBeingSent(t *testing.T) { } } -// sendAndKeepOpen sends request to rproxy at addr, and returns the connection. A -// read from it ends with an error once waitLimit is up. +// sendAndKeepOpen sends request to rproxy at addr, and returns the connection. func sendAndKeepOpen(t *testing.T, addr, request string) net.Conn { t.Helper() @@ -769,11 +768,6 @@ func sendAndKeepOpen(t *testing.T, addr, request string) net.Conn { _ = conn.Close() }) - err = conn.SetDeadline(time.Now().Add(waitLimit)) - if err != nil { - t.Fatalf("set the deadline: %v", err) - } - _, err = io.WriteString(conn, request) if err != nil { t.Fatalf("send: %v", err) @@ -838,6 +832,9 @@ func TestRProxyStoppingCutsOffARequestAfterFiveSeconds(t *testing.T) { stop() + // Counted from the stop, so a slow wait for the app cannot shorten it. + _ = conn.SetReadDeadline(stopped.Add(cutOffAfter + waitLimit)) + text, err := io.ReadAll(conn) if err != nil || len(text) != 0 { t.Errorf("read %q (%v), want the connection cut off", text, err) -- 2.54.0