sneak
37419716ea
check / check (push) Failing after 6m19s
Five failed asks to waf within five seconds open the breaker. An ask fails when it gets no answer in time, slow answers included, or an answer rproxy cannot pass on. Every usable answer clears the failures and closes the breaker. While open, a request takes SWW_RPROXY_WAF_BACKEND_NO_ANSWER_DISPOSITION without asking waf, except one ask a second. SPEC.md describes it. The sww test that stops and restarts waf now waits for the breaker to close before it expects the ban again. Model: opus-5-5
83 lines
2.0 KiB
Go
83 lines
2.0 KiB
Go
package rproxy
|
|
|
|
import (
|
|
"errors"
|
|
"sync"
|
|
"time"
|
|
)
|
|
|
|
// What opens the circuit breaker, and how often a request asks waf while it is
|
|
// open.
|
|
const (
|
|
breakerFailures = 5
|
|
breakerWindow = 5 * time.Second
|
|
breakerAskInterval = time.Second
|
|
)
|
|
|
|
// errBreakerOpen is why a request did not ask waf.
|
|
var errBreakerOpen = errors.New(
|
|
"waf is not asked while the circuit breaker is open",
|
|
)
|
|
|
|
// breaker is the circuit breaker for asks to waf. It opens when
|
|
// breakerFailures asks fail within breakerWindow. An ask fails when it gets no
|
|
// answer in time, or an answer rproxy cannot pass on. While it is open, at
|
|
// most one request each breakerAskInterval asks waf. Any usable answer closes
|
|
// it, whatever it decides, and forgets the failures before it.
|
|
type breaker struct {
|
|
mu sync.Mutex
|
|
// failures holds the times of the latest failed asks, oldest first, at most
|
|
// breakerFailures of them.
|
|
failures []time.Time
|
|
open bool
|
|
// lastAsk is when a request last asked waf while the breaker was open.
|
|
lastAsk time.Time
|
|
}
|
|
|
|
// mayAsk reports whether a request may ask waf at now. While the breaker is
|
|
// open, it says yes once each breakerAskInterval.
|
|
func (b *breaker) mayAsk(now time.Time) bool {
|
|
b.mu.Lock()
|
|
defer b.mu.Unlock()
|
|
|
|
if !b.open {
|
|
return true
|
|
}
|
|
|
|
if now.Sub(b.lastAsk) < breakerAskInterval {
|
|
return false
|
|
}
|
|
|
|
b.lastAsk = now
|
|
|
|
return true
|
|
}
|
|
|
|
// noteAnswer notes that an ask got an answer rproxy can use. It closes the
|
|
// breaker and forgets the failures before it.
|
|
func (b *breaker) noteAnswer() {
|
|
b.mu.Lock()
|
|
defer b.mu.Unlock()
|
|
|
|
b.open = false
|
|
b.failures = nil
|
|
}
|
|
|
|
// noteFailure notes that an ask that ended at now got no answer in time, and
|
|
// opens the breaker if breakerFailures asks have failed within breakerWindow.
|
|
func (b *breaker) noteFailure(now time.Time) {
|
|
b.mu.Lock()
|
|
defer b.mu.Unlock()
|
|
|
|
b.failures = append(b.failures, now)
|
|
if len(b.failures) > breakerFailures {
|
|
b.failures = b.failures[1:]
|
|
}
|
|
|
|
if !b.open && len(b.failures) == breakerFailures &&
|
|
now.Sub(b.failures[0]) <= breakerWindow {
|
|
b.open = true
|
|
b.lastAsk = now
|
|
}
|
|
}
|