rproxy, sww: keep the breaker and stop tests from failing under load (closes #474)
check / check (push) Waiting to run
check / check (push) Waiting to run
The circuit breaker reads the time through a clock the handler holds, which is time.Now outside tests. The test that answers waf's questions with answers rproxy cannot pass on stops that clock, so how long its requests take on a slow host no longer decides whether the breaker opens or turns the next request away. The test that stops rproxy mid-request sets its read deadline from the stop, not from the send, so a slow wait for the app no longer shortens it. Model: opus-5-5
This commit was merged in pull request #475.
This commit is contained in:
1 parent
98d7432077
commit
aaa28f541b
5 files changed
+44
-17
No files matched your search
@@ -2,6 +2,12 @@ package rproxy
|
||||
|
||||
import "time"
|
||||
|
||||
// StopBreakerClock has the circuit breaker of s read at as the time, so that how
|
||||
// long requests take cannot matter to it. It is called before s serves.
|
||||
func (s *Server) StopBreakerClock(at time.Time) {
|
||||
s.h.breakerClock = func() time.Time { return at }
|
||||
}
|
||||
|
||||
// Breaker is the circuit breaker, for tests that give it the time.
|
||||
type Breaker struct {
|
||||
b breaker
|
||||
|
||||
@@ -94,6 +94,7 @@ func New(params Params) *Server {
|
||||
reportURL: (&url.URL{
|
||||
Scheme: backend.Scheme, Host: backend.Host, Path: waf.ReportPath,
|
||||
}).String(),
|
||||
breakerClock: time.Now,
|
||||
}
|
||||
|
||||
return &Server{
|
||||
@@ -133,8 +134,10 @@ type handler struct {
|
||||
wafTransport http.RoundTripper
|
||||
// reportURL is where the reports go.
|
||||
reportURL string
|
||||
// breaker is the circuit breaker for asks to waf.
|
||||
breaker breaker
|
||||
// breaker is the circuit breaker for asks to waf, and breakerClock gives it
|
||||
// the time.
|
||||
breaker breaker
|
||||
breakerClock func() time.Time
|
||||
// reports waits for the reports being sent. reportsMu guards starting one
|
||||
// against reportsStopped, which is set once nothing is to wait for another.
|
||||
reports sync.WaitGroup
|
||||
|
||||
@@ -356,6 +356,19 @@ func startServerLogging(
|
||||
) (string, *rproxy.Server) {
|
||||
t.Helper()
|
||||
|
||||
server := newServer(t, backend, appURL, env, requestLog, processLog)
|
||||
|
||||
return serve(t, server), server
|
||||
}
|
||||
|
||||
// newServer returns rproxy, not yet serving, with its request lines going to
|
||||
// requestLog and its process lines to processLog.
|
||||
func newServer(
|
||||
t *testing.T, backend, appURL string, env map[string]string,
|
||||
requestLog, processLog io.Writer,
|
||||
) *rproxy.Server {
|
||||
t.Helper()
|
||||
|
||||
settings := map[string]string{
|
||||
"SWW_RPROXY_WAF_BACKEND": backend, "SWW_RPROXY_UPSTREAM_URL": appURL,
|
||||
wafBackendTimeout: "60000",
|
||||
@@ -367,7 +380,7 @@ func startServerLogging(
|
||||
t.Fatalf("settings %v: %v", settings, err)
|
||||
}
|
||||
|
||||
server := rproxy.New(rproxy.Params{
|
||||
return rproxy.New(rproxy.Params{
|
||||
Config: cfg,
|
||||
RequestLog: requestLog,
|
||||
ProcessLog: requestlog.NewProcessLogger(
|
||||
@@ -376,6 +389,11 @@ func startServerLogging(
|
||||
slog.LevelInfo,
|
||||
),
|
||||
})
|
||||
}
|
||||
|
||||
// serve has server serve until the test ends, and returns its address.
|
||||
func serve(t *testing.T, server *rproxy.Server) string {
|
||||
t.Helper()
|
||||
|
||||
listener, err := (&net.ListenConfig{}).Listen(
|
||||
t.Context(),
|
||||
@@ -396,7 +414,7 @@ func startServerLogging(
|
||||
server.WaitForReports()
|
||||
})
|
||||
|
||||
return listener.Addr().String(), server
|
||||
return listener.Addr().String()
|
||||
}
|
||||
|
||||
// answered is the visitor's answer.
|
||||
@@ -1532,9 +1550,13 @@ func TestAnswersThatCannotBePassedOnOpenTheBreaker(t *testing.T) {
|
||||
tooLarge(w, r)
|
||||
})
|
||||
app := startApp(t, greet)
|
||||
addr, out := startRProxy(t, standIn.URL, app.URL, map[string]string{
|
||||
wafBackendTimeout: "60000",
|
||||
})
|
||||
out := &output{}
|
||||
server := newServer(t, standIn.URL, app.URL, nil, out, out)
|
||||
// The breaker's clock stands still. On a slow host, the five failures could
|
||||
// otherwise take more than the five seconds that open the breaker. The next
|
||||
// request could come more than a second later, when it may ask waf again.
|
||||
server.StopBreakerClock(time.Now())
|
||||
addr := serve(t, server)
|
||||
|
||||
for range 5 {
|
||||
if got := get(t, addr, "/"); got.status != http.StatusBadGateway {
|
||||
|
||||
@@ -10,7 +10,6 @@ import (
|
||||
"maps"
|
||||
"net/http"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"sneak.berlin/go/smallwebwaf/internal/waf"
|
||||
)
|
||||
@@ -97,7 +96,7 @@ func (rq *request) askUnlessBreakerOpen(
|
||||
ctx context.Context,
|
||||
) (*wafAnswer, error) {
|
||||
b := &rq.h.breaker
|
||||
if !b.mayAsk(time.Now()) {
|
||||
if !b.mayAsk(rq.h.breakerClock()) {
|
||||
return nil, errBreakerOpen
|
||||
}
|
||||
|
||||
@@ -107,7 +106,7 @@ func (rq *request) askUnlessBreakerOpen(
|
||||
case err == nil:
|
||||
b.noteAnswer()
|
||||
case ctx.Err() == nil:
|
||||
b.noteFailure(time.Now())
|
||||
b.noteFailure(rq.h.breakerClock())
|
||||
}
|
||||
|
||||
return answer, err
|
||||
|
||||
Reference in new issue
Block a user