From bb3c8f9351607bf1bd2af8d62588eeb0f520feff Mon Sep 17 00:00:00 2001 From: sneak Date: Thu, 19 Nov 2026 23:02:16 +0000 Subject: [PATCH] rproxy: the two time limits get a type and a file of their own (closes #432) timeLimits in internal/rproxy/timelimits.go owns the mutex, the two timers and the end of the request; request holds one field of it. It is given refuse, whether the visitor's body is being waited on, and a function that ends that read. Tests of the type alone run in a synctest bubble. Model: opus-5-5 --- internal/rproxy/request.go | 192 +++----------------- internal/rproxy/timelimits.go | 165 +++++++++++++++++ internal/rproxy/timelimits_internal_test.go | 171 +++++++++++++++++ internal/rproxy/waf.go | 2 +- 4 files changed, 367 insertions(+), 163 deletions(-) create mode 100644 internal/rproxy/timelimits.go create mode 100644 internal/rproxy/timelimits_internal_test.go diff --git a/internal/rproxy/request.go b/internal/rproxy/request.go index 1c9f8c2..a22aa35 100644 --- a/internal/rproxy/request.go +++ b/internal/rproxy/request.go @@ -12,7 +12,6 @@ import ( "net/url" "os" "strconv" - "sync" "sync/atomic" "time" @@ -82,17 +81,9 @@ type request struct { // upgraded is the connection to the app once the app has switched protocols, // as for a WebSocket, and nil otherwise. upgraded *upgradedConn - - // mu guards what follows. The time limits run on goroutines of their own, - // and the transport starts and stops them, and notes requestEnd, from its - // own; once timersStopped is set, neither time limit acts any more. - mu sync.Mutex - timersStopped bool - requestTimer *time.Timer - responseTimer *time.Timer - // requestEnd is when the app had been sent the whole request, or the answer - // started, if that was sooner. - requestEnd time.Time + // limits are the request's SWW_RPROXY_MAX_REQUEST_TIME_MS and + // SWW_RPROXY_MAX_RESPONSE_TIME_MS. + limits *timeLimits } // newRequest starts handling r: it notes the time, works out the visitor from @@ -127,12 +118,25 @@ func (h *handler) newRequest( } } + rq.limits = &timeLimits{ + maxRequestTime: h.config.MaxRequestTime, + maxResponseTime: h.config.MaxResponseTime, + start: rq.start, + refuse: rq.refuse, + waitingForVisitor: func() bool { + return rq.body != nil && rq.body.waiting.Load() + }, + endVisitorRead: func() { + _ = rq.rc.SetReadDeadline(time.Now()) + }, + } + return rq } // answerHealth answers the health check. func (rq *request) answerHealth() { - rq.prepareOwnAnswer(rq.requestDeadline()) + rq.prepareOwnAnswer(rq.limits.requestDeadline()) rq.out.Header().Set("Content-Type", plainText) rq.send(http.StatusOK, []byte("ok\n")) } @@ -148,7 +152,7 @@ func (rq *request) readFirstPartOfBody() bool { return true } - _ = rq.rc.SetReadDeadline(rq.requestDeadline()) + _ = rq.rc.SetReadDeadline(rq.limits.requestDeadline()) held, err := io.ReadAll( io.LimitReader(rq.body, wafprotocol.QuestionBodyMaxBytes), ) @@ -188,7 +192,7 @@ func (rq *request) forward(ctx context.Context) { rq.cancel = cancel ctx = httptrace.WithClientTrace(ctx, &httptrace.ClientTrace{ - WroteRequest: rq.wroteRequest, + WroteRequest: rq.limits.wroteRequest, }) out := rq.in.WithContext(ctx) @@ -205,7 +209,7 @@ func (rq *request) forward(ctx context.Context) { ErrorHandler: rq.answerError, } - rq.startRequestTimer() + rq.limits.startRequestTimer() rq.upstreamStart = time.Now() reverseProxy.ServeHTTP(rq.out, out) } @@ -248,7 +252,7 @@ func (rq *request) modifyResponse(res *http.Response) error { // switch, as when the app's Upgrade is not the visitor's, so the status // is noted only once it has taken over the connection, by rq.out's // Hijack. - rq.stopTimers() + rq.limits.stop() conn, ok := res.Body.(io.ReadWriteCloser) if ok { @@ -259,7 +263,7 @@ func (rq *request) modifyResponse(res *http.Response) error { return nil } - requestEnd := rq.endRequest() + requestEnd := rq.limits.endRequest() // An answer that arrives once SWW_RPROXY_MAX_RESPONSE_TIME_MS has run out is not // passed on: the visitor is answered 504. @@ -289,12 +293,12 @@ func (rq *request) modifyResponse(res *http.Response) error { // An answer that arrives before the app has been sent the whole request ends // the request, and SWW_RPROXY_MAX_RESPONSE_TIME_MS starts. - rq.startResponseTimer(requestEnd) + rq.limits.startResponseTimer(requestEnd) res.Body = &responseBody{ body: limitBody(res.Body, maxBytes), refuse: rq.refuse, - responseReceived: rq.responseReceived, + responseReceived: rq.limits.stop, ctx: rq.in.Context(), } rq.out.afterStatus = rq.setResponseDeadline @@ -347,7 +351,7 @@ func (rq *request) answer(status int) { // A visitor found too slow is read no more; any other may go on sending until // its time is up, so that the rest of the body is read and the request ends // cleanly: by Go's server, or by readRestOfBody. - deadline := rq.requestDeadline() + deadline := rq.limits.requestDeadline() if status == http.StatusRequestTimeout { deadline = time.Now() } @@ -393,7 +397,7 @@ func (rq *request) closeAfterAnswer() { // used that time up first, and at least minSendTime. func (rq *request) prepareOwnAnswer(deadline time.Time) { rq.stopReadingBody(deadline) - rq.endRequest() + rq.limits.endRequest() limit := rq.h.config.MaxResponseTime if limit > 0 { @@ -414,7 +418,7 @@ func (rq *request) refuse(status int) { // finish ends the request's time limits, writes its log line and, if waf // answered the question, sends the report. ctx is the request's own context. func (rq *request) finish(ctx context.Context) { - rq.stopTimers() + rq.limits.stop() // After a cut-off answer, the reading of a body that has not arrived whole // ends now. Go's server would otherwise keep the connection open while it @@ -432,10 +436,10 @@ func (rq *request) finish(ctx context.Context) { errors.Is(rq.rc.Flush(), os.ErrDeadlineExceeded): rq.stopPassingOnBody(time.Now()) case rq.refused.Load() == 0: - rq.stopPassingOnBody(rq.requestDeadline()) + rq.stopPassingOnBody(rq.limits.requestDeadline()) } - requestEnd := rq.endRequest() + requestEnd := rq.limits.endRequest() end := time.Now() requestBytes, responseBytes := rq.bytesEachWay() @@ -509,17 +513,6 @@ func (rq *request) bytesEachWay() (int64, int64) { return requestBytes, responseBytes } -// requestDeadline is when the visitor must have sent its whole request, or zero -// when SWW_RPROXY_MAX_REQUEST_TIME_MS is off. -func (rq *request) requestDeadline() time.Time { - limit := rq.h.config.MaxRequestTime - if limit == 0 { - return time.Time{} - } - - return rq.start.Add(limit) -} - // stopReadingBody ends, at deadline, the reading of a visitor's body that has not // arrived whole: Go's server then reads no more of it, and closes the connection // after the answer. @@ -576,100 +569,6 @@ func (rq *request) stopPassingOnBody(deadline time.Time) { rq.stopReadingBody(deadline) } -// startRequestTimer starts SWW_RPROXY_MAX_REQUEST_TIME_MS, which runs from the -// request's headers being read until the app has been sent the whole request. -func (rq *request) startRequestTimer() { - deadline := rq.requestDeadline() - if deadline.IsZero() { - return - } - - rq.mu.Lock() - defer rq.mu.Unlock() - - rq.requestTimer = time.AfterFunc( - time.Until(deadline), - rq.requestTimedOut, - ) -} - -// requestTimedOut is called when SWW_RPROXY_MAX_REQUEST_TIME_MS runs out while the -// request is still on its way to the app. The answer names the side rproxy was -// waiting on at that moment: 408 when it was waiting for the visitor to send more -// of its body, 504 when it was waiting for the app to be reached or to take what -// it had. -func (rq *request) requestTimedOut() { - rq.mu.Lock() - defer rq.mu.Unlock() - - if rq.timersStopped { - return - } - - if rq.body == nil || !rq.body.waiting.Load() { - rq.refuse(http.StatusGatewayTimeout) - - return - } - - rq.refuse(http.StatusRequestTimeout) - // The transport gives up on the app only once its Read of the visitor's body - // returns, so that Read is ended now. The lock keeps this from reaching the - // connection after the request is handled. - _ = rq.rc.SetReadDeadline(time.Now()) -} - -// wroteRequest is called once the app has been sent the whole request: -// SWW_RPROXY_MAX_REQUEST_TIME_MS ends and SWW_RPROXY_MAX_RESPONSE_TIME_MS starts. -func (rq *request) wroteRequest(info httptrace.WroteRequestInfo) { - if info.Err != nil { - return // the transport gives up, or tries again - } - - requestEnd := rq.endRequest() - - rq.mu.Lock() - stopTimer(rq.requestTimer) - rq.mu.Unlock() - - rq.startResponseTimer(requestEnd) -} - -// startResponseTimer starts SWW_RPROXY_MAX_RESPONSE_TIME_MS, which runs from -// requestEnd, unless it has started already. -func (rq *request) startResponseTimer(requestEnd time.Time) { - limit := rq.h.config.MaxResponseTime - if limit == 0 { - return - } - - rq.mu.Lock() - defer rq.mu.Unlock() - - if rq.timersStopped || rq.responseTimer != nil { - return - } - - rq.responseTimer = time.AfterFunc(time.Until(requestEnd.Add(limit)), - rq.responseTimedOut) -} - -// responseTimedOut is called when SWW_RPROXY_MAX_RESPONSE_TIME_MS runs out before the -// app has sent its whole answer. -func (rq *request) responseTimedOut() { - rq.mu.Lock() - defer rq.mu.Unlock() - - if !rq.timersStopped { - rq.refuse(http.StatusGatewayTimeout) - } -} - -// responseReceived is called once the app has sent its whole answer. -func (rq *request) responseReceived() { - rq.stopTimers() -} - // setResponseDeadline sends the status and headers of the app's answer, with // at least minSendTime, before ReverseProxy reads the answer's body, and then // the rest has until SWW_RPROXY_MAX_RESPONSE_TIME_MS after the end of the request. @@ -679,7 +578,7 @@ func (rq *request) setResponseDeadline() { return } - deadline := rq.endRequest().Add(limit) + deadline := rq.limits.endRequest().Add(limit) _ = rq.rc.SetWriteDeadline( time.Now().Add(max(time.Until(deadline), minSendTime)), @@ -687,34 +586,3 @@ func (rq *request) setResponseDeadline() { _ = rq.rc.Flush() _ = rq.rc.SetWriteDeadline(deadline) } - -// endRequest notes now as the end of the request, unless an end is noted, and -// returns the end noted. -func (rq *request) endRequest() time.Time { - rq.mu.Lock() - defer rq.mu.Unlock() - - if rq.requestEnd.IsZero() { - rq.requestEnd = time.Now() - } - - return rq.requestEnd -} - -// stopTimers stops the request's time limits and keeps any from starting later: -// the app's answer is complete, the connection upgraded, or the request handled. -func (rq *request) stopTimers() { - rq.mu.Lock() - defer rq.mu.Unlock() - - rq.timersStopped = true - stopTimer(rq.requestTimer) - stopTimer(rq.responseTimer) -} - -// stopTimer stops t, which is nil when its time limit is off. -func stopTimer(t *time.Timer) { - if t != nil { - t.Stop() - } -} diff --git a/internal/rproxy/timelimits.go b/internal/rproxy/timelimits.go new file mode 100644 index 0000000..98e570f --- /dev/null +++ b/internal/rproxy/timelimits.go @@ -0,0 +1,165 @@ +package rproxy + +import ( + "net/http" + "net/http/httptrace" + "sync" + "time" +) + +// timeLimits are one request's two time limits: SWW_RPROXY_MAX_REQUEST_TIME_MS, +// which runs from the request's headers being read until the app has been sent +// the whole request, and SWW_RPROXY_MAX_RESPONSE_TIME_MS, which runs from then +// until the app has sent its whole answer. A limit of zero is off. Each limit +// runs on a goroutine of its own, and the transport starts and stops them from +// its own. +type timeLimits struct { + maxRequestTime time.Duration + maxResponseTime time.Duration + // start is when the request's headers had been read. + start time.Time + // refuse refuses the request with 408 or 504 when a limit runs out. + refuse func(status int) + // waitingForVisitor reports whether a read of the visitor's body is waiting + // for the visitor to send more. + waitingForVisitor func() bool + // endVisitorRead ends at once a read of the visitor's body that is waiting. + endVisitorRead func() + + // mu guards what follows. Once stopped is set, neither limit acts any more. + mu sync.Mutex + stopped bool + requestTimer *time.Timer + responseTimer *time.Timer + // requestEnd is when the app had been sent the whole request, or the answer + // started, if that was sooner. + requestEnd time.Time +} + +// requestDeadline is when the visitor must have sent its whole request, or zero +// when SWW_RPROXY_MAX_REQUEST_TIME_MS is off. +func (l *timeLimits) requestDeadline() time.Time { + if l.maxRequestTime == 0 { + return time.Time{} + } + + return l.start.Add(l.maxRequestTime) +} + +// startRequestTimer starts SWW_RPROXY_MAX_REQUEST_TIME_MS. +func (l *timeLimits) startRequestTimer() { + deadline := l.requestDeadline() + if deadline.IsZero() { + return + } + + l.mu.Lock() + defer l.mu.Unlock() + + l.requestTimer = time.AfterFunc(time.Until(deadline), l.requestTimedOut) +} + +// requestTimedOut is called when SWW_RPROXY_MAX_REQUEST_TIME_MS runs out while +// the request is still on its way to the app. The answer names the side rproxy +// was waiting on at that moment: 408 when it was waiting for the visitor to send +// more of its body, 504 when it was waiting for the app to be reached or to take +// what it had. +func (l *timeLimits) requestTimedOut() { + l.mu.Lock() + defer l.mu.Unlock() + + if l.stopped { + return + } + + if !l.waitingForVisitor() { + l.refuse(http.StatusGatewayTimeout) + + return + } + + l.refuse(http.StatusRequestTimeout) + // The transport gives up on the app only once its Read of the visitor's body + // returns, so that Read is ended now. The lock keeps this from reaching the + // connection after the request is handled. + l.endVisitorRead() +} + +// wroteRequest is called once the app has been sent the whole request: +// SWW_RPROXY_MAX_REQUEST_TIME_MS ends and SWW_RPROXY_MAX_RESPONSE_TIME_MS +// starts. +func (l *timeLimits) wroteRequest(info httptrace.WroteRequestInfo) { + if info.Err != nil { + return // the transport gives up, or tries again + } + + requestEnd := l.endRequest() + + l.mu.Lock() + stopTimer(l.requestTimer) + l.mu.Unlock() + + l.startResponseTimer(requestEnd) +} + +// startResponseTimer starts SWW_RPROXY_MAX_RESPONSE_TIME_MS, which runs from +// requestEnd, unless it has started already. +func (l *timeLimits) startResponseTimer(requestEnd time.Time) { + if l.maxResponseTime == 0 { + return + } + + l.mu.Lock() + defer l.mu.Unlock() + + if l.stopped || l.responseTimer != nil { + return + } + + l.responseTimer = time.AfterFunc( + time.Until(requestEnd.Add(l.maxResponseTime)), + l.responseTimedOut, + ) +} + +// responseTimedOut is called when SWW_RPROXY_MAX_RESPONSE_TIME_MS runs out +// before the app has sent its whole answer. +func (l *timeLimits) responseTimedOut() { + l.mu.Lock() + defer l.mu.Unlock() + + if !l.stopped { + l.refuse(http.StatusGatewayTimeout) + } +} + +// endRequest notes now as the end of the request, unless an end is noted, and +// returns the end noted. +func (l *timeLimits) endRequest() time.Time { + l.mu.Lock() + defer l.mu.Unlock() + + if l.requestEnd.IsZero() { + l.requestEnd = time.Now() + } + + return l.requestEnd +} + +// stop stops both limits and keeps either from starting later: the app's +// answer is complete, the connection upgraded, or the request handled. +func (l *timeLimits) stop() { + l.mu.Lock() + defer l.mu.Unlock() + + l.stopped = true + stopTimer(l.requestTimer) + stopTimer(l.responseTimer) +} + +// stopTimer stops t, which is nil when its limit is off. +func stopTimer(t *time.Timer) { + if t != nil { + t.Stop() + } +} diff --git a/internal/rproxy/timelimits_internal_test.go b/internal/rproxy/timelimits_internal_test.go new file mode 100644 index 0000000..bad3579 --- /dev/null +++ b/internal/rproxy/timelimits_internal_test.go @@ -0,0 +1,171 @@ +package rproxy + +import ( + "net/http" + "net/http/httptrace" + "sync/atomic" + "testing" + "testing/synctest" + "time" +) + +// The tests below run timeLimits in a synctest bubble, where time is a clock of +// the test's own: time.Sleep moves it on at once, and synctest.Wait returns once +// a limit that ran out has refused the request. + +const ( + testRequestTime = 3 * time.Second + testResponseTime = 5 * time.Second +) + +// limitsSeen is what a timeLimits under test did to its request. +type limitsSeen struct { + // refused is the status of the last refusal, or zero. + refused atomic.Int64 + // readEnded is true once the read of the visitor's body was ended. + readEnded atomic.Bool + // waiting is what waitingForVisitor reports. + waiting atomic.Bool +} + +// newTestLimits returns time limits that started now, with what they do noted +// in seen. +func newTestLimits(seen *limitsSeen) *timeLimits { + return &timeLimits{ + maxRequestTime: testRequestTime, + maxResponseTime: testResponseTime, + start: time.Now(), + refuse: func(status int) { + seen.refused.Store(int64(status)) + }, + waitingForVisitor: seen.waiting.Load, + endVisitorRead: func() { + seen.readEnded.Store(true) + }, + } +} + +// sleepThenWait moves the bubble's clock on by d and waits for what is then +// due. +func sleepThenWait(d time.Duration) { + time.Sleep(d) + synctest.Wait() +} + +func TestRequestTimeRunningOutWhileTheVisitorIsWaitedOnRefuses408( + t *testing.T, +) { + t.Parallel() + + synctest.Test(t, func(t *testing.T) { + seen := &limitsSeen{} + seen.waiting.Store(true) + + limits := newTestLimits(seen) + limits.startRequestTimer() + + sleepThenWait(testRequestTime - time.Millisecond) + + if got := seen.refused.Load(); got != 0 { + t.Fatalf( + "refused %d before the limit ran out, want nothing", + got, + ) + } + + sleepThenWait(time.Millisecond) + + if got := seen.refused.Load(); got != http.StatusRequestTimeout { + t.Errorf("refused %d, want 408", got) + } + + if !seen.readEnded.Load() { + t.Error("the read of the visitor's body goes on, want it ended") + } + }) +} + +func TestRequestTimeRunningOutWhileTheAppIsWaitedOnRefuses504( + t *testing.T, +) { + t.Parallel() + + synctest.Test(t, func(t *testing.T) { + seen := &limitsSeen{} + + limits := newTestLimits(seen) + limits.startRequestTimer() + + sleepThenWait(testRequestTime) + + if got := seen.refused.Load(); got != http.StatusGatewayTimeout { + t.Errorf("refused %d, want 504", got) + } + + if seen.readEnded.Load() { + t.Error("the read of the visitor's body was ended, want it not") + } + }) +} + +func TestResponseTimeRunsFromTheEndOfTheRequest(t *testing.T) { + t.Parallel() + + synctest.Test(t, func(t *testing.T) { + seen := &limitsSeen{} + + limits := newTestLimits(seen) + limits.startRequestTimer() + + sleepThenWait(time.Second) + + end := time.Now() + + limits.wroteRequest(httptrace.WroteRequestInfo{}) + + if got := limits.endRequest(); !got.Equal(end) { + t.Errorf("the request ended at %v, want %v", got, end) + } + + // Past SWW_RPROXY_MAX_REQUEST_TIME_MS, which the end of the request + // stopped, and just short of SWW_RPROXY_MAX_RESPONSE_TIME_MS. + sleepThenWait(testResponseTime - time.Millisecond) + + if got := seen.refused.Load(); got != 0 { + t.Fatalf( + "refused %d before the limit ran out, want nothing", + got, + ) + } + + sleepThenWait(time.Millisecond) + + if got := seen.refused.Load(); got != http.StatusGatewayTimeout { + t.Errorf("refused %d, want 504", got) + } + }) +} + +func TestStoppedLimitsRefuseNothing(t *testing.T) { + t.Parallel() + + synctest.Test(t, func(t *testing.T) { + seen := &limitsSeen{} + seen.waiting.Store(true) + + limits := newTestLimits(seen) + limits.startRequestTimer() + limits.stop() + limits.startResponseTimer(limits.endRequest()) + + sleepThenWait(testRequestTime + testResponseTime) + + if got := seen.refused.Load(); got != 0 { + t.Errorf("refused %d, want nothing", got) + } + + if seen.readEnded.Load() { + t.Error("the read of the visitor's body was ended, want it not") + } + }) +} diff --git a/internal/rproxy/waf.go b/internal/rproxy/waf.go index 3bca666..d5917df 100644 --- a/internal/rproxy/waf.go +++ b/internal/rproxy/waf.go @@ -188,7 +188,7 @@ func canBeAsked(r *http.Request) bool { // answerWith gives the visitor waf's answer, less its disposition. func (rq *request) answerWith(answer *wafAnswer) { - rq.prepareOwnAnswer(rq.requestDeadline()) + rq.prepareOwnAnswer(rq.limits.requestDeadline()) header := rq.out.Header() maps.Copy(header, answer.header)