Files
webhooker/internal/delivery/circuit_breaker.go
T
clawbotandsneak 5f84d891cf
check / check (push) Successful in 3m32s
Deploy: main into prod (#343)
Brings `prod`, which upaas deploys, up to `main` at `9cf9cdd`, the merge of #321. `prod` was cut from `main` at `251cb3d` (1.0.0b1).

What it deploys is everything listed in #321. For running it:

- With `WEBHOOKER_ENVIRONMENT` unset, the instance runs as `prod` and sends no `Access-Control-Allow-Origin: *`.
- Each event database gains its new indexes the first time it is opened after the upgrade.
- `webhooker_delivery_retries_total` no longer counts a circuit breaker holding back a delivery that is already `retrying`.

Not in this PR yet: #340, in which the container sets its own data directory owner and mode before start. It is in progress on `next`. Once it reaches `main`, this PR carries it, because the PR follows `main`.

Model: opus-5-5
Co-authored-by: Jeffrey Paul <1+sneak@noreply.example.org>
Reviewed-on: #343
2026-09-29 02:48:03 -07:00

161 lines
3.3 KiB
Go

package delivery
import (
"sync"
"time"
)
// CircuitState represents the current state of a circuit
// breaker.
type CircuitState int
const (
// CircuitClosed is the normal operating state.
CircuitClosed CircuitState = iota
// CircuitOpen means the circuit has tripped.
CircuitOpen
// CircuitHalfOpen allows a single probe delivery to
// test whether the target has recovered.
CircuitHalfOpen
)
const (
// defaultFailureThreshold is the number of consecutive
// failures before a circuit breaker trips open.
defaultFailureThreshold = 5
// defaultCooldown is how long a circuit stays open
// before transitioning to half-open.
defaultCooldown = 30 * time.Second
)
// CircuitBreaker implements the circuit breaker pattern
// for a single delivery target.
type CircuitBreaker struct {
mu sync.Mutex
state CircuitState
failures int
threshold int
cooldown time.Duration
lastFailure time.Time
}
// NewCircuitBreaker creates a circuit breaker with default
// settings.
func NewCircuitBreaker() *CircuitBreaker {
return &CircuitBreaker{
state: CircuitClosed,
threshold: defaultFailureThreshold,
cooldown: defaultCooldown,
}
}
// Allow checks whether a delivery attempt should proceed.
func (cb *CircuitBreaker) Allow() bool {
cb.mu.Lock()
defer cb.mu.Unlock()
switch cb.state {
case CircuitClosed:
return true
case CircuitOpen:
if time.Since(cb.lastFailure) >= cb.cooldown {
cb.state = CircuitHalfOpen
return true
}
return false
case CircuitHalfOpen:
return false
default:
return true
}
}
// CooldownRemaining returns how long a delivery that Allow refused
// should wait before it is tried again. Closed, it returns zero.
// Open, it returns what is left of the cooldown, or zero once that
// has passed. Half-open, it returns the whole cooldown: the one
// probe delivery is still in flight, and if it fails the circuit
// reopens for that long.
func (cb *CircuitBreaker) CooldownRemaining() time.Duration {
cb.mu.Lock()
defer cb.mu.Unlock()
if cb.state == CircuitHalfOpen {
return cb.cooldown
}
if cb.state != CircuitOpen {
return 0
}
remaining := cb.cooldown - time.Since(cb.lastFailure)
if remaining < 0 {
return 0
}
return remaining
}
// RecordSuccess records a successful delivery and resets
// the circuit breaker to closed state.
func (cb *CircuitBreaker) RecordSuccess() {
cb.mu.Lock()
defer cb.mu.Unlock()
cb.failures = 0
cb.state = CircuitClosed
}
// RecordFailure records a failed delivery. If the failure
// count reaches the threshold, the circuit trips open.
func (cb *CircuitBreaker) RecordFailure() {
cb.mu.Lock()
defer cb.mu.Unlock()
cb.failures++
cb.lastFailure = time.Now()
switch cb.state {
case CircuitClosed:
if cb.failures >= cb.threshold {
cb.state = CircuitOpen
}
case CircuitOpen:
// Already open; no state change needed.
case CircuitHalfOpen:
// Probe failed -- reopen immediately.
cb.state = CircuitOpen
}
}
// State returns the current circuit state.
func (cb *CircuitBreaker) State() CircuitState {
cb.mu.Lock()
defer cb.mu.Unlock()
return cb.state
}
// String returns the human-readable name of a circuit
// state.
func (s CircuitState) String() string {
switch s {
case CircuitClosed:
return "closed"
case CircuitOpen:
return "open"
case CircuitHalfOpen:
return "half-open"
default:
return "unknown"
}
}