rproxy, sww: keep the breaker and stop tests from failing under load (closes #474) #475

Merged
clawbot merged 1 commits from issue-474-breaker-and-stop-tests-under-load into next 2026-11-19 19:59:34 +01:00
5 changed files with 44 additions and 17 deletions

No files matched your search

+6
View File
@@ -2,6 +2,12 @@ package rproxy
import "time" import "time"
// StopBreakerClock has the circuit breaker of s read at as the time, so that how
// long requests take cannot matter to it. It is called before s serves.
func (s *Server) StopBreakerClock(at time.Time) {
s.h.breakerClock = func() time.Time { return at }
}
// Breaker is the circuit breaker, for tests that give it the time. // Breaker is the circuit breaker, for tests that give it the time.
type Breaker struct { type Breaker struct {
b breaker b breaker
+5 -2
View File
@@ -94,6 +94,7 @@ func New(params Params) *Server {
reportURL: (&url.URL{ reportURL: (&url.URL{
Scheme: backend.Scheme, Host: backend.Host, Path: waf.ReportPath, Scheme: backend.Scheme, Host: backend.Host, Path: waf.ReportPath,
}).String(), }).String(),
breakerClock: time.Now,
} }
return &Server{ return &Server{
@@ -133,8 +134,10 @@ type handler struct {
wafTransport http.RoundTripper wafTransport http.RoundTripper
// reportURL is where the reports go. // reportURL is where the reports go.
reportURL string reportURL string
// breaker is the circuit breaker for asks to waf. // breaker is the circuit breaker for asks to waf, and breakerClock gives it
breaker breaker // the time.
breaker breaker
breakerClock func() time.Time
// reports waits for the reports being sent. reportsMu guards starting one // reports waits for the reports being sent. reportsMu guards starting one
// against reportsStopped, which is set once nothing is to wait for another. // against reportsStopped, which is set once nothing is to wait for another.
reports sync.WaitGroup reports sync.WaitGroup
+27 -5
View File
@@ -356,6 +356,19 @@ func startServerLogging(
) (string, *rproxy.Server) { ) (string, *rproxy.Server) {
t.Helper() t.Helper()
server := newServer(t, backend, appURL, env, requestLog, processLog)
return serve(t, server), server
}
// newServer returns rproxy, not yet serving, with its request lines going to
// requestLog and its process lines to processLog.
func newServer(
t *testing.T, backend, appURL string, env map[string]string,
requestLog, processLog io.Writer,
) *rproxy.Server {
t.Helper()
settings := map[string]string{ settings := map[string]string{
"SWW_RPROXY_WAF_BACKEND": backend, "SWW_RPROXY_UPSTREAM_URL": appURL, "SWW_RPROXY_WAF_BACKEND": backend, "SWW_RPROXY_UPSTREAM_URL": appURL,
wafBackendTimeout: "60000", wafBackendTimeout: "60000",
@@ -367,7 +380,7 @@ func startServerLogging(
t.Fatalf("settings %v: %v", settings, err) t.Fatalf("settings %v: %v", settings, err)
} }
server := rproxy.New(rproxy.Params{ return rproxy.New(rproxy.Params{
Config: cfg, Config: cfg,
RequestLog: requestLog, RequestLog: requestLog,
ProcessLog: requestlog.NewProcessLogger( ProcessLog: requestlog.NewProcessLogger(
@@ -376,6 +389,11 @@ func startServerLogging(
slog.LevelInfo, slog.LevelInfo,
), ),
}) })
}
// serve has server serve until the test ends, and returns its address.
func serve(t *testing.T, server *rproxy.Server) string {
t.Helper()
listener, err := (&net.ListenConfig{}).Listen( listener, err := (&net.ListenConfig{}).Listen(
t.Context(), t.Context(),
@@ -396,7 +414,7 @@ func startServerLogging(
server.WaitForReports() server.WaitForReports()
}) })
return listener.Addr().String(), server return listener.Addr().String()
} }
// answered is the visitor's answer. // answered is the visitor's answer.
@@ -1532,9 +1550,13 @@ func TestAnswersThatCannotBePassedOnOpenTheBreaker(t *testing.T) {
tooLarge(w, r) tooLarge(w, r)
}) })
app := startApp(t, greet) app := startApp(t, greet)
addr, out := startRProxy(t, standIn.URL, app.URL, map[string]string{ out := &output{}
wafBackendTimeout: "60000", server := newServer(t, standIn.URL, app.URL, nil, out, out)
}) // The breaker's clock stands still. On a slow host, the five failures could
// otherwise take more than the five seconds that open the breaker. The next
// request could come more than a second later, when it may ask waf again.
server.StopBreakerClock(time.Now())
addr := serve(t, server)
for range 5 { for range 5 {
if got := get(t, addr, "/"); got.status != http.StatusBadGateway { if got := get(t, addr, "/"); got.status != http.StatusBadGateway {
+2 -3
View File
@@ -10,7 +10,6 @@ import (
"maps" "maps"
"net/http" "net/http"
"strings" "strings"
"time"
"sneak.berlin/go/smallwebwaf/internal/waf" "sneak.berlin/go/smallwebwaf/internal/waf"
) )
@@ -97,7 +96,7 @@ func (rq *request) askUnlessBreakerOpen(
ctx context.Context, ctx context.Context,
) (*wafAnswer, error) { ) (*wafAnswer, error) {
b := &rq.h.breaker b := &rq.h.breaker
if !b.mayAsk(time.Now()) { if !b.mayAsk(rq.h.breakerClock()) {
return nil, errBreakerOpen return nil, errBreakerOpen
} }
@@ -107,7 +106,7 @@ func (rq *request) askUnlessBreakerOpen(
case err == nil: case err == nil:
b.noteAnswer() b.noteAnswer()
case ctx.Err() == nil: case ctx.Err() == nil:
b.noteFailure(time.Now()) b.noteFailure(rq.h.breakerClock())
} }
return answer, err return answer, err
+4 -7
View File
@@ -755,8 +755,7 @@ func TestRProxyStoppingWaitsForAReportBeingSent(t *testing.T) {
} }
} }
// sendAndKeepOpen sends request to rproxy at addr, and returns the connection. A // sendAndKeepOpen sends request to rproxy at addr, and returns the connection.
// read from it ends with an error once waitLimit is up.
func sendAndKeepOpen(t *testing.T, addr, request string) net.Conn { func sendAndKeepOpen(t *testing.T, addr, request string) net.Conn {
t.Helper() t.Helper()
@@ -769,11 +768,6 @@ func sendAndKeepOpen(t *testing.T, addr, request string) net.Conn {
_ = conn.Close() _ = conn.Close()
}) })
err = conn.SetDeadline(time.Now().Add(waitLimit))
if err != nil {
t.Fatalf("set the deadline: %v", err)
}
_, err = io.WriteString(conn, request) _, err = io.WriteString(conn, request)
if err != nil { if err != nil {
t.Fatalf("send: %v", err) t.Fatalf("send: %v", err)
@@ -838,6 +832,9 @@ func TestRProxyStoppingCutsOffARequestAfterFiveSeconds(t *testing.T) {
stop() stop()
// Counted from the stop, so a slow wait for the app cannot shorten it.
_ = conn.SetReadDeadline(stopped.Add(cutOffAfter + waitLimit))
text, err := io.ReadAll(conn) text, err := io.ReadAll(conn)
if err != nil || len(text) != 0 { if err != nil || len(text) != 0 {
t.Errorf("read %q (%v), want the connection cut off", text, err) t.Errorf("read %q (%v), want the connection cut off", text, err)