Show a target paused by its circuit breaker (closes #385)
check / check (push) Successful in 3m21s

While an http or slack target's circuit breaker turns its deliveries
away, the target's row on the webhook page says its deliveries are
paused and until when, in UTC and from now. Each of its retrying
deliveries shows as waiting, with the reason and the same time, in the
event log and on the event's page, instead of a bare "retrying".

The engine gains one read, CooldownRemaining(targetID), over the
breakers it already keeps; the handlers reach it through a one-method
interface wired like Archives.

Model: opus-5-5
This commit is contained in:
2026-10-02 22:04:53 +00:00
parent 9305af4f85
commit f531cf9cdf
16 changed files with 358 additions and 22 deletions
+48
View File
@@ -9,6 +9,7 @@ import (
"net/http/httptest"
"sync"
"testing"
"time"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
@@ -181,6 +182,47 @@ func (r *recordingArchives) Renames() []archiveRename {
return out
}
// testCircuitBreakers is a delivery.CircuitBreakers over real circuit
// breakers, one per target a test asks for, so a test can trip a
// target's breaker with RecordFailure and close it with RecordSuccess.
type testCircuitBreakers struct {
mu sync.Mutex
breakers map[string]*delivery.CircuitBreaker
}
// Breaker returns the target's circuit breaker, making it on first
// use.
func (b *testCircuitBreakers) Breaker(
targetID string,
) *delivery.CircuitBreaker {
b.mu.Lock()
defer b.mu.Unlock()
if b.breakers == nil {
b.breakers = map[string]*delivery.CircuitBreaker{}
}
if b.breakers[targetID] == nil {
b.breakers[targetID] = delivery.NewCircuitBreaker()
}
return b.breakers[targetID]
}
func (b *testCircuitBreakers) CooldownRemaining(
targetID string,
) time.Duration {
b.mu.Lock()
defer b.mu.Unlock()
cb := b.breakers[targetID]
if cb == nil {
return 0
}
return cb.CooldownRemaining()
}
// newTestApp returns an app whose RequireStart fails the test when
// starting takes longer than fx's default start timeout of 15s. That
// limit catches a start that hangs, not a busy host: measured with make
@@ -231,6 +273,12 @@ func newTestAppWithConfig(
func(r *recordingArchives) delivery.Archives {
return r
},
func() *testCircuitBreakers {
return &testCircuitBreakers{}
},
func(b *testCircuitBreakers) delivery.CircuitBreakers {
return b
},
metrics.NewRegistry,
metrics.New,
middleware.New,