Files
webhooker/internal/delivery/circuit_breaker_test.go
clawbot 536e5682d6
All checks were successful
check / check (push) Successful in 1m48s
test: add comprehensive delivery engine and circuit breaker tests
Add unit tests for internal/delivery/ package covering:

Circuit breaker tests (circuit_breaker_test.go):
- Closed state allows deliveries
- Failure counting below threshold
- Open transition after threshold failures
- Cooldown blocks during cooldown period
- Half-open transition after cooldown expires
- Probe success closes circuit
- Probe failure reopens circuit
- Success resets failure counter
- Concurrent access safety (race-safe)
- CooldownRemaining for all states
- CircuitState String() output

Engine tests (engine_test.go):
- Non-blocking Notify when channel is full
- HTTP target success and failure delivery
- Database target immediate success
- Log target immediate success
- Retry target success with circuit breaker
- Max retries exhausted marks delivery failed
- Retry scheduling on failure
- Exponential backoff duration verification
- Backoff cap at shift 30
- Body pointer semantics (inline <16KB, nil >=16KB)
- Worker pool bounded concurrency
- Circuit breaker blocks delivery attempts
- Circuit breaker per-target creation
- HTTP config parsing (valid, empty, missing URL)
- scheduleRetry sends to retry channel
- scheduleRetry drops when channel full
- Header forwarding (forwardable vs hop-by-hop)
- processDelivery routing to correct handler
- Truncate helper function

All tests use real SQLite databases and httptest servers.
All tests pass with -race flag.
2026-03-01 23:16:30 -08:00

244 lines
6.7 KiB
Go

package delivery
import (
"sync"
"testing"
"time"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestCircuitBreaker_ClosedState_AllowsDeliveries(t *testing.T) {
t.Parallel()
cb := NewCircuitBreaker()
assert.Equal(t, CircuitClosed, cb.State())
assert.True(t, cb.Allow(), "closed circuit should allow deliveries")
// Multiple calls should all succeed
for i := 0; i < 10; i++ {
assert.True(t, cb.Allow())
}
}
func TestCircuitBreaker_FailureCounting(t *testing.T) {
t.Parallel()
cb := NewCircuitBreaker()
// Record failures below threshold — circuit should stay closed
for i := 0; i < defaultFailureThreshold-1; i++ {
cb.RecordFailure()
assert.Equal(t, CircuitClosed, cb.State(),
"circuit should remain closed after %d failures", i+1)
assert.True(t, cb.Allow(), "should still allow after %d failures", i+1)
}
}
func TestCircuitBreaker_OpenTransition(t *testing.T) {
t.Parallel()
cb := NewCircuitBreaker()
// Record exactly threshold failures
for i := 0; i < defaultFailureThreshold; i++ {
cb.RecordFailure()
}
assert.Equal(t, CircuitOpen, cb.State(), "circuit should be open after threshold failures")
assert.False(t, cb.Allow(), "open circuit should reject deliveries")
}
func TestCircuitBreaker_Cooldown_StaysOpen(t *testing.T) {
t.Parallel()
// Use a circuit with a known short cooldown for testing
cb := &CircuitBreaker{
state: CircuitClosed,
threshold: defaultFailureThreshold,
cooldown: 200 * time.Millisecond,
}
// Trip the circuit open
for i := 0; i < defaultFailureThreshold; i++ {
cb.RecordFailure()
}
require.Equal(t, CircuitOpen, cb.State())
// During cooldown, Allow should return false
assert.False(t, cb.Allow(), "should be blocked during cooldown")
// CooldownRemaining should be positive
remaining := cb.CooldownRemaining()
assert.Greater(t, remaining, time.Duration(0), "cooldown should have remaining time")
}
func TestCircuitBreaker_HalfOpen_AfterCooldown(t *testing.T) {
t.Parallel()
cb := &CircuitBreaker{
state: CircuitClosed,
threshold: defaultFailureThreshold,
cooldown: 50 * time.Millisecond,
}
// Trip the circuit open
for i := 0; i < defaultFailureThreshold; i++ {
cb.RecordFailure()
}
require.Equal(t, CircuitOpen, cb.State())
// Wait for cooldown to expire
time.Sleep(60 * time.Millisecond)
// CooldownRemaining should be zero after cooldown
assert.Equal(t, time.Duration(0), cb.CooldownRemaining())
// First Allow after cooldown should succeed (probe)
assert.True(t, cb.Allow(), "should allow one probe after cooldown")
assert.Equal(t, CircuitHalfOpen, cb.State(), "should be half-open after probe allowed")
// Second Allow should be rejected (only one probe at a time)
assert.False(t, cb.Allow(), "should reject additional probes while half-open")
}
func TestCircuitBreaker_ProbeSuccess_ClosesCircuit(t *testing.T) {
t.Parallel()
cb := &CircuitBreaker{
state: CircuitClosed,
threshold: defaultFailureThreshold,
cooldown: 50 * time.Millisecond,
}
// Trip open → wait for cooldown → allow probe
for i := 0; i < defaultFailureThreshold; i++ {
cb.RecordFailure()
}
time.Sleep(60 * time.Millisecond)
require.True(t, cb.Allow()) // probe allowed, state → half-open
// Probe succeeds → circuit should close
cb.RecordSuccess()
assert.Equal(t, CircuitClosed, cb.State(), "successful probe should close circuit")
// Should allow deliveries again
assert.True(t, cb.Allow(), "closed circuit should allow deliveries")
}
func TestCircuitBreaker_ProbeFailure_ReopensCircuit(t *testing.T) {
t.Parallel()
cb := &CircuitBreaker{
state: CircuitClosed,
threshold: defaultFailureThreshold,
cooldown: 50 * time.Millisecond,
}
// Trip open → wait for cooldown → allow probe
for i := 0; i < defaultFailureThreshold; i++ {
cb.RecordFailure()
}
time.Sleep(60 * time.Millisecond)
require.True(t, cb.Allow()) // probe allowed, state → half-open
// Probe fails → circuit should reopen
cb.RecordFailure()
assert.Equal(t, CircuitOpen, cb.State(), "failed probe should reopen circuit")
assert.False(t, cb.Allow(), "reopened circuit should reject deliveries")
}
func TestCircuitBreaker_SuccessResetsFailures(t *testing.T) {
t.Parallel()
cb := NewCircuitBreaker()
// Accumulate failures just below threshold
for i := 0; i < defaultFailureThreshold-1; i++ {
cb.RecordFailure()
}
require.Equal(t, CircuitClosed, cb.State())
// Success should reset the failure counter
cb.RecordSuccess()
assert.Equal(t, CircuitClosed, cb.State())
// Now we should need another full threshold of failures to trip
for i := 0; i < defaultFailureThreshold-1; i++ {
cb.RecordFailure()
}
assert.Equal(t, CircuitClosed, cb.State(),
"circuit should still be closed — success reset the counter")
// One more failure should trip it
cb.RecordFailure()
assert.Equal(t, CircuitOpen, cb.State())
}
func TestCircuitBreaker_ConcurrentAccess(t *testing.T) {
t.Parallel()
cb := NewCircuitBreaker()
const goroutines = 100
var wg sync.WaitGroup
wg.Add(goroutines * 3)
// Concurrent Allow calls
for i := 0; i < goroutines; i++ {
go func() {
defer wg.Done()
cb.Allow()
}()
}
// Concurrent RecordFailure calls
for i := 0; i < goroutines; i++ {
go func() {
defer wg.Done()
cb.RecordFailure()
}()
}
// Concurrent RecordSuccess calls
for i := 0; i < goroutines; i++ {
go func() {
defer wg.Done()
cb.RecordSuccess()
}()
}
wg.Wait()
// No panic or data race — the test passes if -race doesn't flag anything.
// State should be one of the valid states.
state := cb.State()
assert.Contains(t, []CircuitState{CircuitClosed, CircuitOpen, CircuitHalfOpen}, state,
"state should be valid after concurrent access")
}
func TestCircuitBreaker_CooldownRemaining_ClosedReturnsZero(t *testing.T) {
t.Parallel()
cb := NewCircuitBreaker()
assert.Equal(t, time.Duration(0), cb.CooldownRemaining(),
"closed circuit should have zero cooldown remaining")
}
func TestCircuitBreaker_CooldownRemaining_HalfOpenReturnsZero(t *testing.T) {
t.Parallel()
cb := &CircuitBreaker{
state: CircuitClosed,
threshold: defaultFailureThreshold,
cooldown: 50 * time.Millisecond,
}
// Trip open, wait, transition to half-open
for i := 0; i < defaultFailureThreshold; i++ {
cb.RecordFailure()
}
time.Sleep(60 * time.Millisecond)
require.True(t, cb.Allow()) // → half-open
assert.Equal(t, time.Duration(0), cb.CooldownRemaining(),
"half-open circuit should have zero cooldown remaining")
}
func TestCircuitState_String(t *testing.T) {
t.Parallel()
assert.Equal(t, "closed", CircuitClosed.String())
assert.Equal(t, "open", CircuitOpen.String())
assert.Equal(t, "half-open", CircuitHalfOpen.String())
assert.Equal(t, "unknown", CircuitState(99).String())
}