webhooker/internal/delivery/circuit_breaker_test.go

package delivery_test

import (
	"sync"
	"testing"
	"time"

	"github.com/stretchr/testify/assert"
	"github.com/stretchr/testify/require"
	"sneak.berlin/go/webhooker/internal/delivery"
)

func TestCircuitBreaker_ClosedState_AllowsDeliveries(
	t *testing.T,
) {
	t.Parallel()

	cb := delivery.NewCircuitBreaker()

	assert.Equal(t, delivery.CircuitClosed, cb.State())
	assert.True(t, cb.Allow(),
		"closed circuit should allow deliveries",
	)

	for range 10 {
		assert.True(t, cb.Allow())
	}
}

func TestCircuitBreaker_FailureCounting(t *testing.T) {
	t.Parallel()

	cb := delivery.NewCircuitBreaker()

	for i := range delivery.ExportDefaultFailureThreshold - 1 {
		cb.RecordFailure()

		assert.Equal(t,
			delivery.CircuitClosed, cb.State(),
			"circuit should remain closed after %d failures",
			i+1,
		)

		assert.True(t, cb.Allow(),
			"should still allow after %d failures",
			i+1,
		)
	}
}

func TestCircuitBreaker_OpenTransition(t *testing.T) {
	t.Parallel()

	cb := delivery.NewCircuitBreaker()

	for range delivery.ExportDefaultFailureThreshold {
		cb.RecordFailure()
	}

	assert.Equal(t, delivery.CircuitOpen, cb.State(),
		"circuit should be open after threshold failures",
	)

	assert.False(t, cb.Allow(),
		"open circuit should reject deliveries",
	)
}

func TestCircuitBreaker_Cooldown_StaysOpen(t *testing.T) {
	t.Parallel()

	cb := delivery.NewCircuitBreaker()

	for range delivery.ExportDefaultFailureThreshold {
		cb.RecordFailure()
	}

	require.Equal(t, delivery.CircuitOpen, cb.State())

	assert.False(t, cb.Allow(),
		"should be blocked during cooldown",
	)

	remaining := cb.CooldownRemaining()

	assert.Greater(t, remaining, time.Duration(0),
		"cooldown should have remaining time",
	)
}

func TestCircuitBreaker_HalfOpen_AfterCooldown(
	t *testing.T,
) {
	t.Parallel()

	cb := newShortCooldownCB(t)

	for range delivery.ExportDefaultFailureThreshold {
		cb.RecordFailure()
	}

	require.Equal(t, delivery.CircuitOpen, cb.State())

	time.Sleep(60 * time.Millisecond)

	assert.Equal(t, time.Duration(0),
		cb.CooldownRemaining(),
	)

	assert.True(t, cb.Allow(),
		"should allow one probe after cooldown",
	)

	assert.Equal(t,
		delivery.CircuitHalfOpen, cb.State(),
		"should be half-open after probe allowed",
	)

	assert.False(t, cb.Allow(),
		"should reject additional probes while half-open",
	)
}

func TestCircuitBreaker_ProbeSuccess_ClosesCircuit(
	t *testing.T,
) {
	t.Parallel()

	cb := newShortCooldownCB(t)

	for range delivery.ExportDefaultFailureThreshold {
		cb.RecordFailure()
	}

	time.Sleep(60 * time.Millisecond)

	require.True(t, cb.Allow())

	cb.RecordSuccess()

	assert.Equal(t, delivery.CircuitClosed, cb.State(),
		"successful probe should close circuit",
	)

	assert.True(t, cb.Allow(),
		"closed circuit should allow deliveries",
	)
}

func TestCircuitBreaker_ProbeFailure_ReopensCircuit(
	t *testing.T,
) {
	t.Parallel()

	cb := newShortCooldownCB(t)

	for range delivery.ExportDefaultFailureThreshold {
		cb.RecordFailure()
	}

	time.Sleep(60 * time.Millisecond)

	require.True(t, cb.Allow())

	cb.RecordFailure()

	assert.Equal(t, delivery.CircuitOpen, cb.State(),
		"failed probe should reopen circuit",
	)

	assert.False(t, cb.Allow(),
		"reopened circuit should reject deliveries",
	)
}

func TestCircuitBreaker_SuccessResetsFailures(
	t *testing.T,
) {
	t.Parallel()

	cb := delivery.NewCircuitBreaker()

	for range delivery.ExportDefaultFailureThreshold - 1 {
		cb.RecordFailure()
	}

	require.Equal(t, delivery.CircuitClosed, cb.State())

	cb.RecordSuccess()

	assert.Equal(t, delivery.CircuitClosed, cb.State())

	for range delivery.ExportDefaultFailureThreshold - 1 {
		cb.RecordFailure()
	}

	assert.Equal(t, delivery.CircuitClosed, cb.State(),
		"circuit should still be closed -- "+
			"success reset the counter",
	)

	cb.RecordFailure()

	assert.Equal(t, delivery.CircuitOpen, cb.State())
}

func TestCircuitBreaker_ConcurrentAccess(t *testing.T) {
	t.Parallel()

	cb := delivery.NewCircuitBreaker()

	const goroutines = 100

	var wg sync.WaitGroup

	wg.Add(goroutines * 3)

	for range goroutines {
		go func() {
			defer wg.Done()

			cb.Allow()
		}()
	}

	for range goroutines {
		go func() {
			defer wg.Done()

			cb.RecordFailure()
		}()
	}

	for range goroutines {
		go func() {
			defer wg.Done()

			cb.RecordSuccess()
		}()
	}

	wg.Wait()

	state := cb.State()

	assert.Contains(t,
		[]delivery.CircuitState{
			delivery.CircuitClosed,
			delivery.CircuitOpen,
			delivery.CircuitHalfOpen,
		},
		state,
		"state should be valid after concurrent access",
	)
}

func TestCircuitBreaker_CooldownRemaining_ClosedReturnsZero(
	t *testing.T,
) {
	t.Parallel()

	cb := delivery.NewCircuitBreaker()

	assert.Equal(t, time.Duration(0),
		cb.CooldownRemaining(),
		"closed circuit should have zero cooldown remaining",
	)
}

func TestCircuitBreaker_CooldownRemaining_HalfOpenReturnsCooldown(
	t *testing.T,
) {
	t.Parallel()

	cb := newShortCooldownCB(t)

	for range delivery.ExportDefaultFailureThreshold {
		cb.RecordFailure()
	}

	time.Sleep(60 * time.Millisecond)

	require.True(t, cb.Allow())

	// The cooldown newShortCooldownCB gives the breaker.
	assert.Equal(t, 50*time.Millisecond,
		cb.CooldownRemaining(),
		"a delivery refused while half-open should wait "+
			"a whole cooldown",
	)
}

func TestCircuitState_String(t *testing.T) {
	t.Parallel()

	assert.Equal(t, "closed", delivery.CircuitClosed.String())
	assert.Equal(t, "open", delivery.CircuitOpen.String())
	assert.Equal(t, "half-open", delivery.CircuitHalfOpen.String())
	assert.Equal(t, "unknown", delivery.CircuitState(99).String())
}

// newShortCooldownCB creates a CircuitBreaker with a short
// cooldown for testing. We use NewCircuitBreaker and
// manipulate through the public API.
func newShortCooldownCB(t *testing.T) *delivery.CircuitBreaker {
	t.Helper()

	return delivery.NewTestCircuitBreaker(
		delivery.ExportDefaultFailureThreshold,
		50*time.Millisecond,
	)
}