Show a target paused by its circuit breaker (closes #385)
check / check (push) Successful in 3m21s

While an http or slack target's circuit breaker is open, the target's
row on the webhook page says its deliveries are paused until the
cooldown ends, in UTC and from now. Each of its retrying deliveries
shows as waiting in the event log and on the event's page, until the
later of the cooldown's end and the end of its own backoff. While the
breaker is half-open, the row says deliveries are held while one
delivery tests the target, with no time, and deliveries keep their
plain status.

The engine gains one read, StateAndCooldown(targetID), taking a
breaker's state and remaining cooldown under one lock; the handlers
reach it through a one-method interface wired like Archives.

Model: opus-5-5
This commit is contained in:
2026-10-02 23:16:44 +00:00
parent 3489d6909a
commit d66292df43
19 changed files with 520 additions and 28 deletions
+56
View File
@@ -1018,6 +1018,62 @@ func TestGetCircuitBreaker_CreatesOnDemand(t *testing.T) {
)
}
// TestStateAndCooldown_ReadsHTTPAndSlackBreakers proves the engine
// reads the state of an http or a slack target's circuit breaker, with
// what is left of its cooldown while it is open, and no cooldown while
// it is half-open, once it closes, or for a target with no breaker.
func TestStateAndCooldown_ReadsHTTPAndSlackBreakers(t *testing.T) {
t.Parallel()
e := testEngine(t, 1)
httpID := uuid.New().String()
slackID := uuid.New().String()
state, cooldown := e.StateAndCooldown(httpID)
assert.Equal(t, delivery.CircuitClosed, state, "no breaker")
assert.Zero(t, cooldown, "no breaker")
httpCB := delivery.NewTestCircuitBreaker(1, time.Hour)
e.ExportSetCircuitBreaker(httpID, httpCB)
slackCB := delivery.NewTestCircuitBreaker(1, time.Hour)
e.ExportSetSlackCircuitBreaker(slackID, slackCB)
httpCB.RecordFailure()
slackCB.RecordFailure()
for _, id := range []string{httpID, slackID} {
state, cooldown := e.StateAndCooldown(id)
assert.Equal(t, delivery.CircuitOpen, state)
assert.Greater(t, cooldown, 59*time.Minute)
assert.LessOrEqual(t, cooldown, time.Hour)
}
httpCB.RecordSuccess()
slackCB.RecordSuccess()
for _, id := range []string{httpID, slackID} {
state, cooldown := e.StateAndCooldown(id)
assert.Equal(t, delivery.CircuitClosed, state, "closed")
assert.Zero(t, cooldown, "closed")
}
// A breaker with no cooldown goes half-open on the first Allow
// after it trips, letting that one delivery through to test the
// target.
halfOpenID := uuid.New().String()
halfOpenCB := delivery.NewTestCircuitBreaker(1, 0)
e.ExportSetCircuitBreaker(halfOpenID, halfOpenCB)
halfOpenCB.RecordFailure()
require.True(t, halfOpenCB.Allow())
state, cooldown = e.StateAndCooldown(halfOpenID)
assert.Equal(t, delivery.CircuitHalfOpen, state)
assert.Zero(t, cooldown, "half-open")
}
func TestParseHTTPConfig_Valid(t *testing.T) {
t.Parallel()