Show a target paused by its circuit breaker (closes #385)

While an http or slack target's circuit breaker turns its deliveries
away, the target's row on the webhook page says its deliveries are
paused and until when, in UTC and from now. Each of its retrying
deliveries shows as waiting, with the reason and the same time, in the
event log and on the event's page, instead of a bare "retrying".

The engine gains one read, CooldownRemaining(targetID), over the
breakers it already keeps; the handlers reach it through a one-method
interface wired like Archives.

Model: opus-5-5

internal/handlers/target_paused_test.go

@@ -0,0 +1,129 @@
+package handlers_test
+
+import (
+	"net/http"
+	"strings"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"gorm.io/gorm/clause"
+	"sneak.berlin/go/webhooker/internal/database"
+	"sneak.berlin/go/webhooker/internal/delivery"
+	"sneak.berlin/go/webhooker/internal/handlers"
+	"sneak.berlin/go/webhooker/internal/session"
+)
+
+// resumesAt is how the pages write when a paused target's deliveries
+// resume: the time in UTC, then how long that is from now.
+const resumesAt = `\d\d:\d\d:\d\d UTC \(\d+ seconds from now\)`
+
+// TestPausedTarget_ShownUntilBreakerCloses trips the circuit breaker of
+// an http target, then checks that its row on the webhook page says its
+// deliveries are paused and until when, and that its retrying delivery
+// says it is waiting and why in the event log and on the event's page,
+// while its delivered delivery and the log target are shown as before.
+// Once the breaker closes, none of the pages says so any more.
+func TestPausedTarget_ShownUntilBreakerCloses(t *testing.T) {
+	t.Parallel()
+
+	var (
+		h        *handlers.Handlers
+		sess     *session.Session
+		db       *database.Database
+		dbMgr    *database.WebhookDBManager
+		breakers *testCircuitBreakers
+	)
+
+	app := newTestApp(t, &h, &sess, &db, &dbMgr, &breakers)
+	app.RequireStart()
+
+	t.Cleanup(app.RequireStop)
+
+	wh := seedWebhook(t, db)
+	target := seedTarget(t, db, wh.ID, database.TargetTypeHTTP)
+	seedTarget(t, db, wh.ID, database.TargetTypeLog)
+
+	retrying := seedStoredEvent(t, dbMgr, wh.ID, `{"n":1}`)
+	addDelivery(t, dbMgr, wh.ID, retrying.ID, target.ID,
+		database.DeliveryStatusRetrying)
+
+	delivered := seedStoredEvent(t, dbMgr, wh.ID, `{"n":2}`)
+	addDelivery(t, dbMgr, wh.ID, delivered.ID, target.ID,
+		database.DeliveryStatusDelivered)
+
+	cb := breakers.Breaker(target.ID)
+	for cb.State() != delivery.CircuitOpen {
+		cb.RecordFailure()
+	}
+
+	const waiting = "waiting: target paused after repeated failures, " +
+		"resumes " + resumesAt
+
+	list := targetList(t, renderSourceDetailPage(t, h, sess, wh.ID))
+	assert.Regexp(t, "t-http http Active Edit Deactivate Delete "+
+		"Deliveries Paused: after repeated failures, until "+resumesAt,
+		list)
+	assert.Equal(t, 1, strings.Count(list, "Paused"))
+
+	log := renderSourceLogsPage(t, h, sess, wh.ID)
+	assert.Contains(t, log, "t-http: waiting")
+	assert.Contains(t, log, "t-http: delivered")
+	assert.Regexp(t, waiting, log)
+	assert.NotContains(t, log, "retrying")
+
+	page := eventPage(t, h, sess, wh.ID, retrying.ID)
+	assert.Regexp(t, waiting, page)
+	assert.NotContains(t, page, "retrying")
+
+	cb.RecordSuccess()
+
+	list = targetList(t, renderSourceDetailPage(t, h, sess, wh.ID))
+	assert.NotContains(t, list, "Paused")
+
+	log = renderSourceLogsPage(t, h, sess, wh.ID)
+	assert.Contains(t, log, "t-http: retrying")
+	assert.NotContains(t, log, "waiting")
+
+	page = eventPage(t, h, sess, wh.ID, retrying.ID)
+	assert.Contains(t, page, ">retrying</span>")
+	assert.NotContains(t, page, "waiting")
+}
+
+// addDelivery records a delivery of the event to the target, with the
+// given status, in the webhook's own database.
+func addDelivery(
+	t *testing.T,
+	dbMgr *database.WebhookDBManager,
+	webhookID, eventID, targetID string,
+	status database.DeliveryStatus,
+) {
+	t.Helper()
+
+	webhookDB, err := dbMgr.GetDB(webhookID)
+	require.NoError(t, err)
+
+	require.NoError(t, webhookDB.Omit(clause.Associations).Create(
+		&database.Delivery{
+			EventID:  eventID,
+			TargetID: targetID,
+			Status:   status,
+		},
+	).Error)
+}
+
+// eventPage runs the real event page handler and returns the
+// rendered HTML.
+func eventPage(
+	t *testing.T,
+	h *handlers.Handlers,
+	sess *session.Session,
+	webhookID, eventID string,
+) string {
+	t.Helper()
+
+	w := serveEventPage(t, h, sess, webhookID, eventID)
+	require.Equal(t, http.StatusOK, w.Code)
+
+	return w.Body.String()
+}