Files
webhooker/internal/delivery/circuit_breaker.go
T
clawbot e0b211f960
check / check (push) Successful in 5m0s
Make deliveries refused while half-open wait a cooldown (closes #306)
While the breaker was half-open, Allow refused every delivery but the
probe and CooldownRemaining returned zero, so each queued task for the
target went straight back onto the retry channel and rewrote its status
on every pass until the probe finished.

CooldownRemaining now returns the whole cooldown while half-open, so a
refused delivery waits that long. A refused delivery already at
retrying is not written again, so the retry counter now moves only
when a refusal moves a delivery into retrying.

Model: opus-5-5
2026-09-29 05:48:20 +02:00

161 lines
3.3 KiB
Go

package delivery
import (
"sync"
"time"
)
// CircuitState represents the current state of a circuit
// breaker.
type CircuitState int
const (
// CircuitClosed is the normal operating state.
CircuitClosed CircuitState = iota
// CircuitOpen means the circuit has tripped.
CircuitOpen
// CircuitHalfOpen allows a single probe delivery to
// test whether the target has recovered.
CircuitHalfOpen
)
const (
// defaultFailureThreshold is the number of consecutive
// failures before a circuit breaker trips open.
defaultFailureThreshold = 5
// defaultCooldown is how long a circuit stays open
// before transitioning to half-open.
defaultCooldown = 30 * time.Second
)
// CircuitBreaker implements the circuit breaker pattern
// for a single delivery target.
type CircuitBreaker struct {
mu sync.Mutex
state CircuitState
failures int
threshold int
cooldown time.Duration
lastFailure time.Time
}
// NewCircuitBreaker creates a circuit breaker with default
// settings.
func NewCircuitBreaker() *CircuitBreaker {
return &CircuitBreaker{
state: CircuitClosed,
threshold: defaultFailureThreshold,
cooldown: defaultCooldown,
}
}
// Allow checks whether a delivery attempt should proceed.
func (cb *CircuitBreaker) Allow() bool {
cb.mu.Lock()
defer cb.mu.Unlock()
switch cb.state {
case CircuitClosed:
return true
case CircuitOpen:
if time.Since(cb.lastFailure) >= cb.cooldown {
cb.state = CircuitHalfOpen
return true
}
return false
case CircuitHalfOpen:
return false
default:
return true
}
}
// CooldownRemaining returns how long a delivery that Allow refused
// should wait before it is tried again. Closed, it returns zero.
// Open, it returns what is left of the cooldown, or zero once that
// has passed. Half-open, it returns the whole cooldown: the one
// probe delivery is still in flight, and if it fails the circuit
// reopens for that long.
func (cb *CircuitBreaker) CooldownRemaining() time.Duration {
cb.mu.Lock()
defer cb.mu.Unlock()
if cb.state == CircuitHalfOpen {
return cb.cooldown
}
if cb.state != CircuitOpen {
return 0
}
remaining := cb.cooldown - time.Since(cb.lastFailure)
if remaining < 0 {
return 0
}
return remaining
}
// RecordSuccess records a successful delivery and resets
// the circuit breaker to closed state.
func (cb *CircuitBreaker) RecordSuccess() {
cb.mu.Lock()
defer cb.mu.Unlock()
cb.failures = 0
cb.state = CircuitClosed
}
// RecordFailure records a failed delivery. If the failure
// count reaches the threshold, the circuit trips open.
func (cb *CircuitBreaker) RecordFailure() {
cb.mu.Lock()
defer cb.mu.Unlock()
cb.failures++
cb.lastFailure = time.Now()
switch cb.state {
case CircuitClosed:
if cb.failures >= cb.threshold {
cb.state = CircuitOpen
}
case CircuitOpen:
// Already open; no state change needed.
case CircuitHalfOpen:
// Probe failed -- reopen immediately.
cb.state = CircuitOpen
}
}
// State returns the current circuit state.
func (cb *CircuitBreaker) State() CircuitState {
cb.mu.Lock()
defer cb.mu.Unlock()
return cb.state
}
// String returns the human-readable name of a circuit
// state.
func (s CircuitState) String() string {
switch s {
case CircuitClosed:
return "closed"
case CircuitOpen:
return "open"
case CircuitHalfOpen:
return "half-open"
default:
return "unknown"
}
}