Show a target paused by its circuit breaker (closes #385)
check / check (push) Successful in 3m20s

While an http or slack target's circuit breaker is open, the target's
row on the webhook page says its deliveries are paused until the
cooldown ends, then one is sent to test the target. Each of its
retrying deliveries shows as waiting in the event log and on the
event's page, with the earliest it can be tried next: the later of the
cooldown's end and the end of its own backoff, dated when not today
in UTC. While the breaker is half-open, the row says deliveries are
held while one delivery tests the target, with no time, and deliveries
keep their plain status.

The engine gains one read, StateAndCooldown(targetID), taking a
breaker's state and remaining cooldown under one lock; the handlers
reach it through a one-method interface wired like Archives.

Model: opus-5-5
This commit is contained in:
2026-10-02 23:49:34 +00:00
parent 22fa502638
commit f448acabe1
19 changed files with 539 additions and 28 deletions
+56
View File
@@ -1018,6 +1018,62 @@ func TestGetCircuitBreaker_CreatesOnDemand(t *testing.T) {
)
}
// TestStateAndCooldown_ReadsHTTPAndSlackBreakers proves the engine
// reads the state of an http or a slack target's circuit breaker, with
// what is left of its cooldown while it is open, and no cooldown while
// it is half-open, once it closes, or for a target with no breaker.
func TestStateAndCooldown_ReadsHTTPAndSlackBreakers(t *testing.T) {
t.Parallel()
e := testEngine(t, 1)
httpID := uuid.New().String()
slackID := uuid.New().String()
state, cooldown := e.StateAndCooldown(httpID)
assert.Equal(t, delivery.CircuitClosed, state, "no breaker")
assert.Zero(t, cooldown, "no breaker")
httpCB := delivery.NewTestCircuitBreaker(1, time.Hour)
e.ExportSetCircuitBreaker(httpID, httpCB)
slackCB := delivery.NewTestCircuitBreaker(1, time.Hour)
e.ExportSetSlackCircuitBreaker(slackID, slackCB)
httpCB.RecordFailure()
slackCB.RecordFailure()
for _, id := range []string{httpID, slackID} {
state, cooldown := e.StateAndCooldown(id)
assert.Equal(t, delivery.CircuitOpen, state)
assert.Greater(t, cooldown, 59*time.Minute)
assert.LessOrEqual(t, cooldown, time.Hour)
}
httpCB.RecordSuccess()
slackCB.RecordSuccess()
for _, id := range []string{httpID, slackID} {
state, cooldown := e.StateAndCooldown(id)
assert.Equal(t, delivery.CircuitClosed, state, "closed")
assert.Zero(t, cooldown, "closed")
}
// A breaker with no cooldown goes half-open on the first Allow
// after it trips, letting that one delivery through to test the
// target.
halfOpenID := uuid.New().String()
halfOpenCB := delivery.NewTestCircuitBreaker(1, 0)
e.ExportSetCircuitBreaker(halfOpenID, halfOpenCB)
halfOpenCB.RecordFailure()
require.True(t, halfOpenCB.Allow())
state, cooldown = e.StateAndCooldown(halfOpenID)
assert.Equal(t, delivery.CircuitHalfOpen, state)
assert.Zero(t, cooldown, "half-open")
}
func TestParseHTTPConfig_Valid(t *testing.T) {
t.Parallel()