All checks were successful
check / check (push) Successful in 1m48s
Add unit tests for internal/delivery/ package covering: Circuit breaker tests (circuit_breaker_test.go): - Closed state allows deliveries - Failure counting below threshold - Open transition after threshold failures - Cooldown blocks during cooldown period - Half-open transition after cooldown expires - Probe success closes circuit - Probe failure reopens circuit - Success resets failure counter - Concurrent access safety (race-safe) - CooldownRemaining for all states - CircuitState String() output Engine tests (engine_test.go): - Non-blocking Notify when channel is full - HTTP target success and failure delivery - Database target immediate success - Log target immediate success - Retry target success with circuit breaker - Max retries exhausted marks delivery failed - Retry scheduling on failure - Exponential backoff duration verification - Backoff cap at shift 30 - Body pointer semantics (inline <16KB, nil >=16KB) - Worker pool bounded concurrency - Circuit breaker blocks delivery attempts - Circuit breaker per-target creation - HTTP config parsing (valid, empty, missing URL) - scheduleRetry sends to retry channel - scheduleRetry drops when channel full - Header forwarding (forwardable vs hop-by-hop) - processDelivery routing to correct handler - Truncate helper function All tests use real SQLite databases and httptest servers. All tests pass with -race flag.
244 lines
6.7 KiB
Go
244 lines
6.7 KiB
Go
package delivery
|
|
|
|
import (
|
|
"sync"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/stretchr/testify/assert"
|
|
"github.com/stretchr/testify/require"
|
|
)
|
|
|
|
func TestCircuitBreaker_ClosedState_AllowsDeliveries(t *testing.T) {
|
|
t.Parallel()
|
|
cb := NewCircuitBreaker()
|
|
|
|
assert.Equal(t, CircuitClosed, cb.State())
|
|
assert.True(t, cb.Allow(), "closed circuit should allow deliveries")
|
|
// Multiple calls should all succeed
|
|
for i := 0; i < 10; i++ {
|
|
assert.True(t, cb.Allow())
|
|
}
|
|
}
|
|
|
|
func TestCircuitBreaker_FailureCounting(t *testing.T) {
|
|
t.Parallel()
|
|
cb := NewCircuitBreaker()
|
|
|
|
// Record failures below threshold — circuit should stay closed
|
|
for i := 0; i < defaultFailureThreshold-1; i++ {
|
|
cb.RecordFailure()
|
|
assert.Equal(t, CircuitClosed, cb.State(),
|
|
"circuit should remain closed after %d failures", i+1)
|
|
assert.True(t, cb.Allow(), "should still allow after %d failures", i+1)
|
|
}
|
|
}
|
|
|
|
func TestCircuitBreaker_OpenTransition(t *testing.T) {
|
|
t.Parallel()
|
|
cb := NewCircuitBreaker()
|
|
|
|
// Record exactly threshold failures
|
|
for i := 0; i < defaultFailureThreshold; i++ {
|
|
cb.RecordFailure()
|
|
}
|
|
|
|
assert.Equal(t, CircuitOpen, cb.State(), "circuit should be open after threshold failures")
|
|
assert.False(t, cb.Allow(), "open circuit should reject deliveries")
|
|
}
|
|
|
|
func TestCircuitBreaker_Cooldown_StaysOpen(t *testing.T) {
|
|
t.Parallel()
|
|
// Use a circuit with a known short cooldown for testing
|
|
cb := &CircuitBreaker{
|
|
state: CircuitClosed,
|
|
threshold: defaultFailureThreshold,
|
|
cooldown: 200 * time.Millisecond,
|
|
}
|
|
|
|
// Trip the circuit open
|
|
for i := 0; i < defaultFailureThreshold; i++ {
|
|
cb.RecordFailure()
|
|
}
|
|
require.Equal(t, CircuitOpen, cb.State())
|
|
|
|
// During cooldown, Allow should return false
|
|
assert.False(t, cb.Allow(), "should be blocked during cooldown")
|
|
|
|
// CooldownRemaining should be positive
|
|
remaining := cb.CooldownRemaining()
|
|
assert.Greater(t, remaining, time.Duration(0), "cooldown should have remaining time")
|
|
}
|
|
|
|
func TestCircuitBreaker_HalfOpen_AfterCooldown(t *testing.T) {
|
|
t.Parallel()
|
|
cb := &CircuitBreaker{
|
|
state: CircuitClosed,
|
|
threshold: defaultFailureThreshold,
|
|
cooldown: 50 * time.Millisecond,
|
|
}
|
|
|
|
// Trip the circuit open
|
|
for i := 0; i < defaultFailureThreshold; i++ {
|
|
cb.RecordFailure()
|
|
}
|
|
require.Equal(t, CircuitOpen, cb.State())
|
|
|
|
// Wait for cooldown to expire
|
|
time.Sleep(60 * time.Millisecond)
|
|
|
|
// CooldownRemaining should be zero after cooldown
|
|
assert.Equal(t, time.Duration(0), cb.CooldownRemaining())
|
|
|
|
// First Allow after cooldown should succeed (probe)
|
|
assert.True(t, cb.Allow(), "should allow one probe after cooldown")
|
|
assert.Equal(t, CircuitHalfOpen, cb.State(), "should be half-open after probe allowed")
|
|
|
|
// Second Allow should be rejected (only one probe at a time)
|
|
assert.False(t, cb.Allow(), "should reject additional probes while half-open")
|
|
}
|
|
|
|
func TestCircuitBreaker_ProbeSuccess_ClosesCircuit(t *testing.T) {
|
|
t.Parallel()
|
|
cb := &CircuitBreaker{
|
|
state: CircuitClosed,
|
|
threshold: defaultFailureThreshold,
|
|
cooldown: 50 * time.Millisecond,
|
|
}
|
|
|
|
// Trip open → wait for cooldown → allow probe
|
|
for i := 0; i < defaultFailureThreshold; i++ {
|
|
cb.RecordFailure()
|
|
}
|
|
time.Sleep(60 * time.Millisecond)
|
|
require.True(t, cb.Allow()) // probe allowed, state → half-open
|
|
|
|
// Probe succeeds → circuit should close
|
|
cb.RecordSuccess()
|
|
assert.Equal(t, CircuitClosed, cb.State(), "successful probe should close circuit")
|
|
|
|
// Should allow deliveries again
|
|
assert.True(t, cb.Allow(), "closed circuit should allow deliveries")
|
|
}
|
|
|
|
func TestCircuitBreaker_ProbeFailure_ReopensCircuit(t *testing.T) {
|
|
t.Parallel()
|
|
cb := &CircuitBreaker{
|
|
state: CircuitClosed,
|
|
threshold: defaultFailureThreshold,
|
|
cooldown: 50 * time.Millisecond,
|
|
}
|
|
|
|
// Trip open → wait for cooldown → allow probe
|
|
for i := 0; i < defaultFailureThreshold; i++ {
|
|
cb.RecordFailure()
|
|
}
|
|
time.Sleep(60 * time.Millisecond)
|
|
require.True(t, cb.Allow()) // probe allowed, state → half-open
|
|
|
|
// Probe fails → circuit should reopen
|
|
cb.RecordFailure()
|
|
assert.Equal(t, CircuitOpen, cb.State(), "failed probe should reopen circuit")
|
|
assert.False(t, cb.Allow(), "reopened circuit should reject deliveries")
|
|
}
|
|
|
|
func TestCircuitBreaker_SuccessResetsFailures(t *testing.T) {
|
|
t.Parallel()
|
|
cb := NewCircuitBreaker()
|
|
|
|
// Accumulate failures just below threshold
|
|
for i := 0; i < defaultFailureThreshold-1; i++ {
|
|
cb.RecordFailure()
|
|
}
|
|
require.Equal(t, CircuitClosed, cb.State())
|
|
|
|
// Success should reset the failure counter
|
|
cb.RecordSuccess()
|
|
assert.Equal(t, CircuitClosed, cb.State())
|
|
|
|
// Now we should need another full threshold of failures to trip
|
|
for i := 0; i < defaultFailureThreshold-1; i++ {
|
|
cb.RecordFailure()
|
|
}
|
|
assert.Equal(t, CircuitClosed, cb.State(),
|
|
"circuit should still be closed — success reset the counter")
|
|
|
|
// One more failure should trip it
|
|
cb.RecordFailure()
|
|
assert.Equal(t, CircuitOpen, cb.State())
|
|
}
|
|
|
|
func TestCircuitBreaker_ConcurrentAccess(t *testing.T) {
|
|
t.Parallel()
|
|
cb := NewCircuitBreaker()
|
|
|
|
const goroutines = 100
|
|
var wg sync.WaitGroup
|
|
wg.Add(goroutines * 3)
|
|
|
|
// Concurrent Allow calls
|
|
for i := 0; i < goroutines; i++ {
|
|
go func() {
|
|
defer wg.Done()
|
|
cb.Allow()
|
|
}()
|
|
}
|
|
|
|
// Concurrent RecordFailure calls
|
|
for i := 0; i < goroutines; i++ {
|
|
go func() {
|
|
defer wg.Done()
|
|
cb.RecordFailure()
|
|
}()
|
|
}
|
|
|
|
// Concurrent RecordSuccess calls
|
|
for i := 0; i < goroutines; i++ {
|
|
go func() {
|
|
defer wg.Done()
|
|
cb.RecordSuccess()
|
|
}()
|
|
}
|
|
|
|
wg.Wait()
|
|
// No panic or data race — the test passes if -race doesn't flag anything.
|
|
// State should be one of the valid states.
|
|
state := cb.State()
|
|
assert.Contains(t, []CircuitState{CircuitClosed, CircuitOpen, CircuitHalfOpen}, state,
|
|
"state should be valid after concurrent access")
|
|
}
|
|
|
|
func TestCircuitBreaker_CooldownRemaining_ClosedReturnsZero(t *testing.T) {
|
|
t.Parallel()
|
|
cb := NewCircuitBreaker()
|
|
assert.Equal(t, time.Duration(0), cb.CooldownRemaining(),
|
|
"closed circuit should have zero cooldown remaining")
|
|
}
|
|
|
|
func TestCircuitBreaker_CooldownRemaining_HalfOpenReturnsZero(t *testing.T) {
|
|
t.Parallel()
|
|
cb := &CircuitBreaker{
|
|
state: CircuitClosed,
|
|
threshold: defaultFailureThreshold,
|
|
cooldown: 50 * time.Millisecond,
|
|
}
|
|
|
|
// Trip open, wait, transition to half-open
|
|
for i := 0; i < defaultFailureThreshold; i++ {
|
|
cb.RecordFailure()
|
|
}
|
|
time.Sleep(60 * time.Millisecond)
|
|
require.True(t, cb.Allow()) // → half-open
|
|
|
|
assert.Equal(t, time.Duration(0), cb.CooldownRemaining(),
|
|
"half-open circuit should have zero cooldown remaining")
|
|
}
|
|
|
|
func TestCircuitState_String(t *testing.T) {
|
|
t.Parallel()
|
|
assert.Equal(t, "closed", CircuitClosed.String())
|
|
assert.Equal(t, "open", CircuitOpen.String())
|
|
assert.Equal(t, "half-open", CircuitHalfOpen.String())
|
|
assert.Equal(t, "unknown", CircuitState(99).String())
|
|
}
|