Show a target paused by its circuit breaker (closes #385)
check / check (push) Successful in 3m16s
check / check (push) Successful in 3m16s
A target whose circuit breaker had tripped still showed as Active, and its deliveries sat at a bare "retrying" with no attempts. Its row on the webhook page now says deliveries are paused after repeated failures and until when the cooldown ends, adding that one waiting delivery is then sent to test the target; while half-open it says deliveries are held while one tests it, with no time. Waiting deliveries show "next try no earlier than" the later of the cooldown and their own backoff, with the date when not today. The engine gains one read of a breaker's state and remaining cooldown under one lock, and shares the backoff formula. Model: opus-5-5
This commit was merged in pull request #478.
This commit is contained in:
@@ -1018,6 +1018,62 @@ func TestGetCircuitBreaker_CreatesOnDemand(t *testing.T) {
|
||||
)
|
||||
}
|
||||
|
||||
// TestStateAndCooldown_ReadsHTTPAndSlackBreakers proves the engine
|
||||
// reads the state of an http or a slack target's circuit breaker, with
|
||||
// what is left of its cooldown while it is open, and no cooldown while
|
||||
// it is half-open, once it closes, or for a target with no breaker.
|
||||
func TestStateAndCooldown_ReadsHTTPAndSlackBreakers(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
e := testEngine(t, 1)
|
||||
|
||||
httpID := uuid.New().String()
|
||||
slackID := uuid.New().String()
|
||||
|
||||
state, cooldown := e.StateAndCooldown(httpID)
|
||||
assert.Equal(t, delivery.CircuitClosed, state, "no breaker")
|
||||
assert.Zero(t, cooldown, "no breaker")
|
||||
|
||||
httpCB := delivery.NewTestCircuitBreaker(1, time.Hour)
|
||||
e.ExportSetCircuitBreaker(httpID, httpCB)
|
||||
|
||||
slackCB := delivery.NewTestCircuitBreaker(1, time.Hour)
|
||||
e.ExportSetSlackCircuitBreaker(slackID, slackCB)
|
||||
|
||||
httpCB.RecordFailure()
|
||||
slackCB.RecordFailure()
|
||||
|
||||
for _, id := range []string{httpID, slackID} {
|
||||
state, cooldown := e.StateAndCooldown(id)
|
||||
assert.Equal(t, delivery.CircuitOpen, state)
|
||||
assert.Greater(t, cooldown, 59*time.Minute)
|
||||
assert.LessOrEqual(t, cooldown, time.Hour)
|
||||
}
|
||||
|
||||
httpCB.RecordSuccess()
|
||||
slackCB.RecordSuccess()
|
||||
|
||||
for _, id := range []string{httpID, slackID} {
|
||||
state, cooldown := e.StateAndCooldown(id)
|
||||
assert.Equal(t, delivery.CircuitClosed, state, "closed")
|
||||
assert.Zero(t, cooldown, "closed")
|
||||
}
|
||||
|
||||
// A breaker with no cooldown goes half-open on the first Allow
|
||||
// after it trips, letting that one delivery through to test the
|
||||
// target.
|
||||
halfOpenID := uuid.New().String()
|
||||
halfOpenCB := delivery.NewTestCircuitBreaker(1, 0)
|
||||
e.ExportSetCircuitBreaker(halfOpenID, halfOpenCB)
|
||||
|
||||
halfOpenCB.RecordFailure()
|
||||
require.True(t, halfOpenCB.Allow())
|
||||
|
||||
state, cooldown = e.StateAndCooldown(halfOpenID)
|
||||
assert.Equal(t, delivery.CircuitHalfOpen, state)
|
||||
assert.Zero(t, cooldown, "half-open")
|
||||
}
|
||||
|
||||
func TestParseHTTPConfig_Valid(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
|
||||
Reference in New Issue
Block a user