check / check (push) Successful in 5m0s
While the breaker was half-open, Allow refused every delivery but the probe and CooldownRemaining returned zero, so each queued task for the target went straight back onto the retry channel and rewrote its status on every pass until the probe finished. CooldownRemaining now returns the whole cooldown while half-open, so a refused delivery waits that long. A refused delivery already at retrying is not written again, so the retry counter now moves only when a refusal moves a delivery into retrying. Model: opus-5-5
161 lines
3.3 KiB
Go
161 lines
3.3 KiB
Go
package delivery
|
|
|
|
import (
|
|
"sync"
|
|
"time"
|
|
)
|
|
|
|
// CircuitState represents the current state of a circuit
|
|
// breaker.
|
|
type CircuitState int
|
|
|
|
const (
|
|
// CircuitClosed is the normal operating state.
|
|
CircuitClosed CircuitState = iota
|
|
// CircuitOpen means the circuit has tripped.
|
|
CircuitOpen
|
|
// CircuitHalfOpen allows a single probe delivery to
|
|
// test whether the target has recovered.
|
|
CircuitHalfOpen
|
|
)
|
|
|
|
const (
|
|
// defaultFailureThreshold is the number of consecutive
|
|
// failures before a circuit breaker trips open.
|
|
defaultFailureThreshold = 5
|
|
|
|
// defaultCooldown is how long a circuit stays open
|
|
// before transitioning to half-open.
|
|
defaultCooldown = 30 * time.Second
|
|
)
|
|
|
|
// CircuitBreaker implements the circuit breaker pattern
|
|
// for a single delivery target.
|
|
type CircuitBreaker struct {
|
|
mu sync.Mutex
|
|
state CircuitState
|
|
failures int
|
|
threshold int
|
|
cooldown time.Duration
|
|
lastFailure time.Time
|
|
}
|
|
|
|
// NewCircuitBreaker creates a circuit breaker with default
|
|
// settings.
|
|
func NewCircuitBreaker() *CircuitBreaker {
|
|
return &CircuitBreaker{
|
|
state: CircuitClosed,
|
|
threshold: defaultFailureThreshold,
|
|
cooldown: defaultCooldown,
|
|
}
|
|
}
|
|
|
|
// Allow checks whether a delivery attempt should proceed.
|
|
func (cb *CircuitBreaker) Allow() bool {
|
|
cb.mu.Lock()
|
|
defer cb.mu.Unlock()
|
|
|
|
switch cb.state {
|
|
case CircuitClosed:
|
|
return true
|
|
|
|
case CircuitOpen:
|
|
if time.Since(cb.lastFailure) >= cb.cooldown {
|
|
cb.state = CircuitHalfOpen
|
|
|
|
return true
|
|
}
|
|
|
|
return false
|
|
|
|
case CircuitHalfOpen:
|
|
return false
|
|
|
|
default:
|
|
return true
|
|
}
|
|
}
|
|
|
|
// CooldownRemaining returns how long a delivery that Allow refused
|
|
// should wait before it is tried again. Closed, it returns zero.
|
|
// Open, it returns what is left of the cooldown, or zero once that
|
|
// has passed. Half-open, it returns the whole cooldown: the one
|
|
// probe delivery is still in flight, and if it fails the circuit
|
|
// reopens for that long.
|
|
func (cb *CircuitBreaker) CooldownRemaining() time.Duration {
|
|
cb.mu.Lock()
|
|
defer cb.mu.Unlock()
|
|
|
|
if cb.state == CircuitHalfOpen {
|
|
return cb.cooldown
|
|
}
|
|
|
|
if cb.state != CircuitOpen {
|
|
return 0
|
|
}
|
|
|
|
remaining := cb.cooldown - time.Since(cb.lastFailure)
|
|
if remaining < 0 {
|
|
return 0
|
|
}
|
|
|
|
return remaining
|
|
}
|
|
|
|
// RecordSuccess records a successful delivery and resets
|
|
// the circuit breaker to closed state.
|
|
func (cb *CircuitBreaker) RecordSuccess() {
|
|
cb.mu.Lock()
|
|
defer cb.mu.Unlock()
|
|
|
|
cb.failures = 0
|
|
cb.state = CircuitClosed
|
|
}
|
|
|
|
// RecordFailure records a failed delivery. If the failure
|
|
// count reaches the threshold, the circuit trips open.
|
|
func (cb *CircuitBreaker) RecordFailure() {
|
|
cb.mu.Lock()
|
|
defer cb.mu.Unlock()
|
|
|
|
cb.failures++
|
|
cb.lastFailure = time.Now()
|
|
|
|
switch cb.state {
|
|
case CircuitClosed:
|
|
if cb.failures >= cb.threshold {
|
|
cb.state = CircuitOpen
|
|
}
|
|
|
|
case CircuitOpen:
|
|
// Already open; no state change needed.
|
|
|
|
case CircuitHalfOpen:
|
|
// Probe failed -- reopen immediately.
|
|
cb.state = CircuitOpen
|
|
}
|
|
}
|
|
|
|
// State returns the current circuit state.
|
|
func (cb *CircuitBreaker) State() CircuitState {
|
|
cb.mu.Lock()
|
|
defer cb.mu.Unlock()
|
|
|
|
return cb.state
|
|
}
|
|
|
|
// String returns the human-readable name of a circuit
|
|
// state.
|
|
func (s CircuitState) String() string {
|
|
switch s {
|
|
case CircuitClosed:
|
|
return "closed"
|
|
case CircuitOpen:
|
|
return "open"
|
|
case CircuitHalfOpen:
|
|
return "half-open"
|
|
default:
|
|
return "unknown"
|
|
}
|
|
}
|