Compare commits

...
Author SHA1 Message Date
sneak 69e62c3e37 rproxy: one type for the goroutines sending reports (closes #433)
check / check (push) Waiting to run
handler held a WaitGroup, a mutex and a stopped flag for the reports,
and the rule that no report starts once waiting has begun was split
between sendReport and Server.WaitForReports. reportGoroutines now
holds all three, and its two methods are the only code that touches
the lock and the flag.

Model: opus-5-5
2026-11-19 19:09:27 +00:00
clawbot aaa28f541b rproxy, sww: keep the breaker and stop tests from failing under load (closes #474)
check / check (push) Waiting to run
The circuit breaker reads the time through a clock the handler holds,
which is time.Now outside tests. The test that answers waf's
questions with answers rproxy cannot pass on stops that clock, so how
long its requests take on a slow host no longer decides whether the
breaker opens or turns the next request away.

The test that stops rproxy mid-request sets its read deadline from the
stop, not from the send, so a slow wait for the app no longer shortens
it.

Model: opus-5-5
2026-11-19 19:59:34 +01:00
clawbot 98d7432077 waf: the steps after the answer as plain statements (closes #445)
check / check (push) Waiting to run
banAndCount runs the heuristics that only ban, then countAnomalies,
then addToHistory. A refused request's decision and an allowed
request's report both call it, each with its own list. answerQuestion
keeps one defer, which writes the log line and holds the decision for
the report.

A check that panics now leaves the request out of its client's history
and the anomaly counts, as the issue states.

Model: opus-5-5
2026-11-19 19:45:13 +01:00
8 changed files with 111 additions and 70 deletions

No files matched your search

+6
View File
@@ -2,6 +2,12 @@ package rproxy
import "time"
// StopBreakerClock has the circuit breaker of s read at as the time, so that how
// long requests take cannot matter to it. It is called before s serves.
func (s *Server) StopBreakerClock(at time.Time) {
s.h.breakerClock = func() time.Time { return at }
}
// Breaker is the circuit breaker, for tests that give it the time.
type Breaker struct {
b breaker
+8 -13
View File
@@ -20,7 +20,6 @@ import (
"net/url"
"slices"
"strings"
"sync"
"time"
"sneak.berlin/go/smallwebwaf/internal/config"
@@ -94,6 +93,7 @@ func New(params Params) *Server {
reportURL: (&url.URL{
Scheme: backend.Scheme, Host: backend.Host, Path: waf.ReportPath,
}).String(),
breakerClock: time.Now,
}
return &Server{
@@ -113,11 +113,7 @@ func New(params Params) *Server {
// which SWW_RPROXY_WAF_BACKEND_TIMEOUT_MS bounds. Any report after this call
// is dropped: that of a WebSocket that closes while rproxy stops, for one.
func (s *Server) WaitForReports() {
s.h.reportsMu.Lock()
s.h.reportsStopped = true
s.h.reportsMu.Unlock()
s.h.reports.Wait()
s.h.reports.stopAndWait()
}
// handler is the proxy. It holds what every request shares; what belongs to
@@ -133,13 +129,12 @@ type handler struct {
wafTransport http.RoundTripper
// reportURL is where the reports go.
reportURL string
// breaker is the circuit breaker for asks to waf.
breaker breaker
// reports waits for the reports being sent. reportsMu guards starting one
// against reportsStopped, which is set once nothing is to wait for another.
reports sync.WaitGroup
reportsMu sync.Mutex
reportsStopped bool
// breaker is the circuit breaker for asks to waf, and breakerClock gives it
// the time.
breaker breaker
breakerClock func() time.Time
// reports are the goroutines sending reports to waf.
reports reportGoroutines
}
// newTransport returns what carries requests to the app. It never goes through
+27 -5
View File
@@ -356,6 +356,19 @@ func startServerLogging(
) (string, *rproxy.Server) {
t.Helper()
server := newServer(t, backend, appURL, env, requestLog, processLog)
return serve(t, server), server
}
// newServer returns rproxy, not yet serving, with its request lines going to
// requestLog and its process lines to processLog.
func newServer(
t *testing.T, backend, appURL string, env map[string]string,
requestLog, processLog io.Writer,
) *rproxy.Server {
t.Helper()
settings := map[string]string{
"SWW_RPROXY_WAF_BACKEND": backend, "SWW_RPROXY_UPSTREAM_URL": appURL,
wafBackendTimeout: "60000",
@@ -367,7 +380,7 @@ func startServerLogging(
t.Fatalf("settings %v: %v", settings, err)
}
server := rproxy.New(rproxy.Params{
return rproxy.New(rproxy.Params{
Config: cfg,
RequestLog: requestLog,
ProcessLog: requestlog.NewProcessLogger(
@@ -376,6 +389,11 @@ func startServerLogging(
slog.LevelInfo,
),
})
}
// serve has server serve until the test ends, and returns its address.
func serve(t *testing.T, server *rproxy.Server) string {
t.Helper()
listener, err := (&net.ListenConfig{}).Listen(
t.Context(),
@@ -396,7 +414,7 @@ func startServerLogging(
server.WaitForReports()
})
return listener.Addr().String(), server
return listener.Addr().String()
}
// answered is the visitor's answer.
@@ -1532,9 +1550,13 @@ func TestAnswersThatCannotBePassedOnOpenTheBreaker(t *testing.T) {
tooLarge(w, r)
})
app := startApp(t, greet)
addr, out := startRProxy(t, standIn.URL, app.URL, map[string]string{
wafBackendTimeout: "60000",
})
out := &output{}
server := newServer(t, standIn.URL, app.URL, nil, out, out)
// The breaker's clock stands still. On a slow host, the five failures could
// otherwise take more than the five seconds that open the breaker. The next
// request could come more than a second later, when it may ask waf again.
server.StopBreakerClock(time.Now())
addr := serve(t, server)
for range 5 {
if got := get(t, addr, "/"); got.status != http.StatusBadGateway {
+35 -11
View File
@@ -10,7 +10,7 @@ import (
"maps"
"net/http"
"strings"
"time"
"sync"
"sneak.berlin/go/smallwebwaf/internal/waf"
)
@@ -97,7 +97,7 @@ func (rq *request) askUnlessBreakerOpen(
ctx context.Context,
) (*wafAnswer, error) {
b := &rq.h.breaker
if !b.mayAsk(time.Now()) {
if !b.mayAsk(rq.h.breakerClock()) {
return nil, errBreakerOpen
}
@@ -107,7 +107,7 @@ func (rq *request) askUnlessBreakerOpen(
case err == nil:
b.noteAnswer()
case ctx.Err() == nil:
b.noteFailure(time.Now())
b.noteFailure(rq.h.breakerClock())
}
return answer, err
@@ -254,14 +254,7 @@ func (rq *request) answerWith(answer *wafAnswer) {
// visitor does not wait for it. The report goes on after ctx, the request's own
// context, has ended.
func (h *handler) sendReport(ctx context.Context, report *waf.Report) {
h.reportsMu.Lock()
defer h.reportsMu.Unlock()
if h.reportsStopped {
return
}
h.reports.Go(func() {
h.reports.startUnlessStopped(func() {
err := h.postReport(context.WithoutCancel(ctx), report)
if err != nil {
h.processLog.Warn("a report is dropped", "error", err.Error())
@@ -269,6 +262,37 @@ func (h *handler) sendReport(ctx context.Context, report *waf.Report) {
})
}
// reportGoroutines are the goroutines sending reports to waf. None starts once
// stopAndWait has been called.
type reportGoroutines struct {
mu sync.Mutex
stopped bool
running sync.WaitGroup
}
// startUnlessStopped runs send on a goroutine of its own, or drops it once
// stopAndWait has been called.
func (g *reportGoroutines) startUnlessStopped(send func()) {
g.mu.Lock()
defer g.mu.Unlock()
if g.stopped {
return
}
g.running.Go(send)
}
// stopAndWait stops any more goroutines from starting, and returns once those
// running have ended.
func (g *reportGoroutines) stopAndWait() {
g.mu.Lock()
g.stopped = true
g.mu.Unlock()
g.running.Wait()
}
// postReport posts report to waf, and waits for its answer for up to
// SWW_RPROXY_WAF_BACKEND_TIMEOUT_MS.
func (h *handler) postReport(ctx context.Context, report *waf.Report) error {
+4 -7
View File
@@ -755,8 +755,7 @@ func TestRProxyStoppingWaitsForAReportBeingSent(t *testing.T) {
}
}
// sendAndKeepOpen sends request to rproxy at addr, and returns the connection. A
// read from it ends with an error once waitLimit is up.
// sendAndKeepOpen sends request to rproxy at addr, and returns the connection.
func sendAndKeepOpen(t *testing.T, addr, request string) net.Conn {
t.Helper()
@@ -769,11 +768,6 @@ func sendAndKeepOpen(t *testing.T, addr, request string) net.Conn {
_ = conn.Close()
})
err = conn.SetDeadline(time.Now().Add(waitLimit))
if err != nil {
t.Fatalf("set the deadline: %v", err)
}
_, err = io.WriteString(conn, request)
if err != nil {
t.Fatalf("send: %v", err)
@@ -838,6 +832,9 @@ func TestRProxyStoppingCutsOffARequestAfterFiveSeconds(t *testing.T) {
stop()
// Counted from the stop, so a slow wait for the app cannot shorten it.
_ = conn.SetReadDeadline(stopped.Add(cutOffAfter + waitLimit))
text, err := io.ReadAll(conn)
if err != nil || len(text) != 0 {
t.Errorf("read %q (%v), want the connection cut off", text, err)
-6
View File
@@ -104,12 +104,6 @@ func onReport() []heuristic {
}
}
// runAfterAnswer runs the heuristics of afterAnswer on the request, at the time
// it is called.
func (rq *request) runAfterAnswer(ctx context.Context) {
rq.runBanningHeuristics(ctx, afterAnswer())
}
// runHeuristics runs list, one of those checkClient runs, on the request at now,
// and returns the first verdict, or nil when none gives one. The log line's
// heuristic and heuristic_figures name the heuristic that gave it, and give its
+10
View File
@@ -345,6 +345,16 @@ func timing(start, end time.Time) *float64 {
return new(requestlog.Milliseconds(end.Sub(start)))
}
// banAndCount runs list, afterAnswer or onReport, whose heuristics only ban.
// It then counts the request for the anomaly thresholds and adds it to its
// client's history. list runs first, so that a refusal limit it finds broken
// is in that history. ctx is the request's own context.
func (rq *request) banAndCount(ctx context.Context, list []heuristic) {
rq.runBanningHeuristics(ctx, list)
rq.countAnomalies()
rq.addToHistory()
}
// addToHistory adds the request, which has ended, to its client's
// history, and counts its offences in the metrics, and then the lookup's
// answer about the client, as answerAtTheEnd gives it, to that history and
+21 -28
View File
@@ -177,11 +177,13 @@ func (h *handler) answerQuestion(w http.ResponseWriter, r *http.Request) {
peer.Unmap(),
true,
)
// The last deferred runs first, so that what is held has the finished log
// line. Go's server sends the answer only once this function has
// returned, so the report cannot come before it is held.
defer rq.holdForReport()
defer rq.finish()
// These run even when a check panics. finish runs first, so that what is
// held has the finished log line. Go's server sends the answer only once
// this function has returned, so the report cannot come before it is held.
defer func() {
rq.finish()
rq.holdForReport()
}()
rq.site = r.Host
rq.line.WAFLine = requestlog.WAFLineDecision
@@ -190,35 +192,28 @@ func (h *handler) answerQuestion(w http.ResponseWriter, r *http.Request) {
underOwnPrefix := strings.HasPrefix(r.URL.Path, ownEndpointPrefix)
if refused == nil && !underOwnPrefix {
// In observe mode, what enforce mode would have refused counts here.
defer rq.runAfterAnswer(r.Context())
rq.allow()
// In observe mode, what enforce mode would have refused counts here.
rq.runBanningHeuristics(r.Context(), afterAnswer())
return
}
// The last deferred runs first: runAfterAnswer before addToHistory, so that
// a broken refusal limit is in the client's history.
defer rq.addToHistory()
defer rq.countAnomalies()
defer rq.runAfterAnswer(r.Context())
if refused != nil {
rq.deny(*refused)
return
} else {
// The app never sees this prefix. waf's own endpoints are at its
// own address, so this answers as a path with no endpoint does.
rq.line.Action = requestlog.ActionOwnEndpoint
rq.out.Header().Set(DispositionHeader, DispositionDeny)
http.Error(
rq.out,
http.StatusText(http.StatusNotFound),
http.StatusNotFound,
)
}
// The app never sees this prefix. waf's own endpoints are at its
// own address, so this answers as a path with no endpoint does.
rq.line.Action = requestlog.ActionOwnEndpoint
rq.out.Header().Set(DispositionHeader, DispositionDeny)
http.Error(
rq.out,
http.StatusText(http.StatusNotFound),
http.StatusNotFound,
)
rq.banAndCount(r.Context(), afterAnswer())
}
// readQuestion returns the visitor's address and the body a question carries,
@@ -491,9 +486,7 @@ func (h *handler) finishWith(
if rq.allowed {
rq.line.DurationUpstreamTotalMS = &report.DurationUpstreamTotalMS
rq.runBanningHeuristics(ctx, onReport())
rq.countAnomalies()
rq.addToHistory()
rq.banAndCount(ctx, onReport())
rq.h.metrics.RequestReported(
&rq.line,