test: add comprehensive delivery engine and circuit breaker tests
All checks were successful
check / check (push) Successful in 1m48s
All checks were successful
check / check (push) Successful in 1m48s
Add unit tests for internal/delivery/ package covering: Circuit breaker tests (circuit_breaker_test.go): - Closed state allows deliveries - Failure counting below threshold - Open transition after threshold failures - Cooldown blocks during cooldown period - Half-open transition after cooldown expires - Probe success closes circuit - Probe failure reopens circuit - Success resets failure counter - Concurrent access safety (race-safe) - CooldownRemaining for all states - CircuitState String() output Engine tests (engine_test.go): - Non-blocking Notify when channel is full - HTTP target success and failure delivery - Database target immediate success - Log target immediate success - Retry target success with circuit breaker - Max retries exhausted marks delivery failed - Retry scheduling on failure - Exponential backoff duration verification - Backoff cap at shift 30 - Body pointer semantics (inline <16KB, nil >=16KB) - Worker pool bounded concurrency - Circuit breaker blocks delivery attempts - Circuit breaker per-target creation - HTTP config parsing (valid, empty, missing URL) - scheduleRetry sends to retry channel - scheduleRetry drops when channel full - Header forwarding (forwardable vs hop-by-hop) - processDelivery routing to correct handler - Truncate helper function All tests use real SQLite databases and httptest servers. All tests pass with -race flag.
This commit is contained in:
243
internal/delivery/circuit_breaker_test.go
Normal file
243
internal/delivery/circuit_breaker_test.go
Normal file
@@ -0,0 +1,243 @@
|
||||
package delivery
|
||||
|
||||
import (
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestCircuitBreaker_ClosedState_AllowsDeliveries(t *testing.T) {
|
||||
t.Parallel()
|
||||
cb := NewCircuitBreaker()
|
||||
|
||||
assert.Equal(t, CircuitClosed, cb.State())
|
||||
assert.True(t, cb.Allow(), "closed circuit should allow deliveries")
|
||||
// Multiple calls should all succeed
|
||||
for i := 0; i < 10; i++ {
|
||||
assert.True(t, cb.Allow())
|
||||
}
|
||||
}
|
||||
|
||||
func TestCircuitBreaker_FailureCounting(t *testing.T) {
|
||||
t.Parallel()
|
||||
cb := NewCircuitBreaker()
|
||||
|
||||
// Record failures below threshold — circuit should stay closed
|
||||
for i := 0; i < defaultFailureThreshold-1; i++ {
|
||||
cb.RecordFailure()
|
||||
assert.Equal(t, CircuitClosed, cb.State(),
|
||||
"circuit should remain closed after %d failures", i+1)
|
||||
assert.True(t, cb.Allow(), "should still allow after %d failures", i+1)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCircuitBreaker_OpenTransition(t *testing.T) {
|
||||
t.Parallel()
|
||||
cb := NewCircuitBreaker()
|
||||
|
||||
// Record exactly threshold failures
|
||||
for i := 0; i < defaultFailureThreshold; i++ {
|
||||
cb.RecordFailure()
|
||||
}
|
||||
|
||||
assert.Equal(t, CircuitOpen, cb.State(), "circuit should be open after threshold failures")
|
||||
assert.False(t, cb.Allow(), "open circuit should reject deliveries")
|
||||
}
|
||||
|
||||
func TestCircuitBreaker_Cooldown_StaysOpen(t *testing.T) {
|
||||
t.Parallel()
|
||||
// Use a circuit with a known short cooldown for testing
|
||||
cb := &CircuitBreaker{
|
||||
state: CircuitClosed,
|
||||
threshold: defaultFailureThreshold,
|
||||
cooldown: 200 * time.Millisecond,
|
||||
}
|
||||
|
||||
// Trip the circuit open
|
||||
for i := 0; i < defaultFailureThreshold; i++ {
|
||||
cb.RecordFailure()
|
||||
}
|
||||
require.Equal(t, CircuitOpen, cb.State())
|
||||
|
||||
// During cooldown, Allow should return false
|
||||
assert.False(t, cb.Allow(), "should be blocked during cooldown")
|
||||
|
||||
// CooldownRemaining should be positive
|
||||
remaining := cb.CooldownRemaining()
|
||||
assert.Greater(t, remaining, time.Duration(0), "cooldown should have remaining time")
|
||||
}
|
||||
|
||||
func TestCircuitBreaker_HalfOpen_AfterCooldown(t *testing.T) {
|
||||
t.Parallel()
|
||||
cb := &CircuitBreaker{
|
||||
state: CircuitClosed,
|
||||
threshold: defaultFailureThreshold,
|
||||
cooldown: 50 * time.Millisecond,
|
||||
}
|
||||
|
||||
// Trip the circuit open
|
||||
for i := 0; i < defaultFailureThreshold; i++ {
|
||||
cb.RecordFailure()
|
||||
}
|
||||
require.Equal(t, CircuitOpen, cb.State())
|
||||
|
||||
// Wait for cooldown to expire
|
||||
time.Sleep(60 * time.Millisecond)
|
||||
|
||||
// CooldownRemaining should be zero after cooldown
|
||||
assert.Equal(t, time.Duration(0), cb.CooldownRemaining())
|
||||
|
||||
// First Allow after cooldown should succeed (probe)
|
||||
assert.True(t, cb.Allow(), "should allow one probe after cooldown")
|
||||
assert.Equal(t, CircuitHalfOpen, cb.State(), "should be half-open after probe allowed")
|
||||
|
||||
// Second Allow should be rejected (only one probe at a time)
|
||||
assert.False(t, cb.Allow(), "should reject additional probes while half-open")
|
||||
}
|
||||
|
||||
func TestCircuitBreaker_ProbeSuccess_ClosesCircuit(t *testing.T) {
|
||||
t.Parallel()
|
||||
cb := &CircuitBreaker{
|
||||
state: CircuitClosed,
|
||||
threshold: defaultFailureThreshold,
|
||||
cooldown: 50 * time.Millisecond,
|
||||
}
|
||||
|
||||
// Trip open → wait for cooldown → allow probe
|
||||
for i := 0; i < defaultFailureThreshold; i++ {
|
||||
cb.RecordFailure()
|
||||
}
|
||||
time.Sleep(60 * time.Millisecond)
|
||||
require.True(t, cb.Allow()) // probe allowed, state → half-open
|
||||
|
||||
// Probe succeeds → circuit should close
|
||||
cb.RecordSuccess()
|
||||
assert.Equal(t, CircuitClosed, cb.State(), "successful probe should close circuit")
|
||||
|
||||
// Should allow deliveries again
|
||||
assert.True(t, cb.Allow(), "closed circuit should allow deliveries")
|
||||
}
|
||||
|
||||
func TestCircuitBreaker_ProbeFailure_ReopensCircuit(t *testing.T) {
|
||||
t.Parallel()
|
||||
cb := &CircuitBreaker{
|
||||
state: CircuitClosed,
|
||||
threshold: defaultFailureThreshold,
|
||||
cooldown: 50 * time.Millisecond,
|
||||
}
|
||||
|
||||
// Trip open → wait for cooldown → allow probe
|
||||
for i := 0; i < defaultFailureThreshold; i++ {
|
||||
cb.RecordFailure()
|
||||
}
|
||||
time.Sleep(60 * time.Millisecond)
|
||||
require.True(t, cb.Allow()) // probe allowed, state → half-open
|
||||
|
||||
// Probe fails → circuit should reopen
|
||||
cb.RecordFailure()
|
||||
assert.Equal(t, CircuitOpen, cb.State(), "failed probe should reopen circuit")
|
||||
assert.False(t, cb.Allow(), "reopened circuit should reject deliveries")
|
||||
}
|
||||
|
||||
func TestCircuitBreaker_SuccessResetsFailures(t *testing.T) {
|
||||
t.Parallel()
|
||||
cb := NewCircuitBreaker()
|
||||
|
||||
// Accumulate failures just below threshold
|
||||
for i := 0; i < defaultFailureThreshold-1; i++ {
|
||||
cb.RecordFailure()
|
||||
}
|
||||
require.Equal(t, CircuitClosed, cb.State())
|
||||
|
||||
// Success should reset the failure counter
|
||||
cb.RecordSuccess()
|
||||
assert.Equal(t, CircuitClosed, cb.State())
|
||||
|
||||
// Now we should need another full threshold of failures to trip
|
||||
for i := 0; i < defaultFailureThreshold-1; i++ {
|
||||
cb.RecordFailure()
|
||||
}
|
||||
assert.Equal(t, CircuitClosed, cb.State(),
|
||||
"circuit should still be closed — success reset the counter")
|
||||
|
||||
// One more failure should trip it
|
||||
cb.RecordFailure()
|
||||
assert.Equal(t, CircuitOpen, cb.State())
|
||||
}
|
||||
|
||||
func TestCircuitBreaker_ConcurrentAccess(t *testing.T) {
|
||||
t.Parallel()
|
||||
cb := NewCircuitBreaker()
|
||||
|
||||
const goroutines = 100
|
||||
var wg sync.WaitGroup
|
||||
wg.Add(goroutines * 3)
|
||||
|
||||
// Concurrent Allow calls
|
||||
for i := 0; i < goroutines; i++ {
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
cb.Allow()
|
||||
}()
|
||||
}
|
||||
|
||||
// Concurrent RecordFailure calls
|
||||
for i := 0; i < goroutines; i++ {
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
cb.RecordFailure()
|
||||
}()
|
||||
}
|
||||
|
||||
// Concurrent RecordSuccess calls
|
||||
for i := 0; i < goroutines; i++ {
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
cb.RecordSuccess()
|
||||
}()
|
||||
}
|
||||
|
||||
wg.Wait()
|
||||
// No panic or data race — the test passes if -race doesn't flag anything.
|
||||
// State should be one of the valid states.
|
||||
state := cb.State()
|
||||
assert.Contains(t, []CircuitState{CircuitClosed, CircuitOpen, CircuitHalfOpen}, state,
|
||||
"state should be valid after concurrent access")
|
||||
}
|
||||
|
||||
func TestCircuitBreaker_CooldownRemaining_ClosedReturnsZero(t *testing.T) {
|
||||
t.Parallel()
|
||||
cb := NewCircuitBreaker()
|
||||
assert.Equal(t, time.Duration(0), cb.CooldownRemaining(),
|
||||
"closed circuit should have zero cooldown remaining")
|
||||
}
|
||||
|
||||
func TestCircuitBreaker_CooldownRemaining_HalfOpenReturnsZero(t *testing.T) {
|
||||
t.Parallel()
|
||||
cb := &CircuitBreaker{
|
||||
state: CircuitClosed,
|
||||
threshold: defaultFailureThreshold,
|
||||
cooldown: 50 * time.Millisecond,
|
||||
}
|
||||
|
||||
// Trip open, wait, transition to half-open
|
||||
for i := 0; i < defaultFailureThreshold; i++ {
|
||||
cb.RecordFailure()
|
||||
}
|
||||
time.Sleep(60 * time.Millisecond)
|
||||
require.True(t, cb.Allow()) // → half-open
|
||||
|
||||
assert.Equal(t, time.Duration(0), cb.CooldownRemaining(),
|
||||
"half-open circuit should have zero cooldown remaining")
|
||||
}
|
||||
|
||||
func TestCircuitState_String(t *testing.T) {
|
||||
t.Parallel()
|
||||
assert.Equal(t, "closed", CircuitClosed.String())
|
||||
assert.Equal(t, "open", CircuitOpen.String())
|
||||
assert.Equal(t, "half-open", CircuitHalfOpen.String())
|
||||
assert.Equal(t, "unknown", CircuitState(99).String())
|
||||
}
|
||||
Reference in New Issue
Block a user