Add per-delivery replay to the event log (closes #203)

A delivery that exhausted max_retries was failed forever. The event
body is durably stored, so the only way to get it delivered was to
download it and re-POST by hand.

The event log now offers a Replay action on any finished delivery.
Replay creates a NEW pending delivery for the same event and target
and hands it to the delivery engine through the same Notifier the
receiver uses, so it is retried, SSRF-guarded and circuit-broken
exactly as a first attempt. The original delivery's status,
timestamps and recorded attempts are never touched, and what is
re-sent is the stored event body, not the response the original
attempt received.

The target is read as it stands now, including soft-deleted rows so
that a deleted target refuses the replay with a message on the page
instead of erroring or delivering from stale configuration. A
deactivated target and a target id that names nothing refuse the same
way, as does a replay of a delivery the engine has not finished.

Two bounds on replay storms: the route carries a per-client POST rate
limit of 30 per minute, and the handler refuses a replay while an
earlier one for the same event and target is still pending or
retrying.

One new metric, webhooker_delivery_replays_total, on the existing
target_type label. A replay is a real delivery and moves the attempt,
outcome and duration series like any other; this counter is what
separates it from ordinary traffic without adding a dimension to
every existing series.

The delivery row is written with associations omitted and with
neither Event nor Target populated, so no target row reaches the
per-webhook event database.

internal/server/routes.go

@@ -201,6 +201,18 @@ func (s *Server) setupSourceRoutes() {
 			"/logs/{eventID}/body",
 			s.h.HandleEventBodyDownload(),
 		)
+		// Replay is the one page action that queues outbound work:
+		// it creates a delivery from a stored event and hands it to
+		// the delivery engine. The rate limit is what bounds a
+		// held-down button or a scripted loop; the handler
+		// separately refuses a replay while an earlier one for the
+		// same event and target is still in flight. POST only, so
+		// the action cannot be taken by a link, a prefetch or an
+		// image tag.
+		r.With(s.mw.ReplayRateLimit()).Post(
+			"/deliveries/{deliveryID}/replay",
+			s.h.HandleDeliveryReplay(),
+		)
 		r.Post(
 			"/entrypoints",
 			s.h.HandleEntrypointCreate(),

internal/server/routes_test.go

@@ -310,6 +310,75 @@ func (e *testEnv) seedEvent(
 	return event
 }
 
+// seedTarget creates an active HTTP target for a webhook.
+func (e *testEnv) seedTarget(
+	t *testing.T,
+	webhookID string,
+) *database.Target {
+	t.Helper()
+
+	tgt := &database.Target{
+		WebhookID: webhookID,
+		Name:      "routed-target",
+		Type:      database.TargetTypeHTTP,
+		Active:    true,
+		Config:    `{"url":"http://93.184.216.34/hook"}`,
+	}
+
+	require.NoError(
+		t,
+		e.db.DB().Omit(clause.Associations).Create(tgt).Error,
+	)
+
+	return tgt
+}
+
+// seedFailedDelivery records a terminally failed delivery of an event
+// to a target in the webhook's own database.
+func (e *testEnv) seedFailedDelivery(
+	t *testing.T,
+	webhookID, eventID, targetID string,
+) *database.Delivery {
+	t.Helper()
+
+	webhookDB, err := e.dbMgr.GetDB(webhookID)
+	require.NoError(t, err)
+
+	dlv := &database.Delivery{
+		EventID:  eventID,
+		TargetID: targetID,
+		Status:   database.DeliveryStatusFailed,
+	}
+
+	require.NoError(
+		t,
+		webhookDB.Omit(clause.Associations).Create(dlv).Error,
+	)
+
+	return dlv
+}
+
+// countDeliveries reports how many deliveries a webhook's database
+// holds.
+func (e *testEnv) countDeliveries(
+	t *testing.T, webhookID string,
+) int64 {
+	t.Helper()
+
+	webhookDB, err := e.dbMgr.GetDB(webhookID)
+	require.NoError(t, err)
+
+	var count int64
+
+	require.NoError(
+		t,
+		webhookDB.Model(&database.Delivery{}).
+			Count(&count).Error,
+	)
+
+	return count
+}
+
 // storedHash reads the current password hash for a username.
 func (e *testEnv) storedHash(t *testing.T, username string) string {
 	t.Helper()
@@ -674,6 +743,85 @@ func TestSourceLogsBody_OtherUser404s(t *testing.T) {
 	assert.Equal(t, "/pages/login", anon.Header().Get("Location"))
 }
 
+// TestDeliveryReplay_PostOnlyAndCSRFProtected walks the replay action
+// through the production router rather than a forged route context,
+// which is the only way to prove what the route group actually gives
+// it: a GET cannot trigger a replay, an unauthenticated request never
+// reaches the handler, a POST without the token is refused by CSRF,
+// and the form the template emits — token and action URL both — works
+// as rendered.
+func TestDeliveryReplay_PostOnlyAndCSRFProtected(t *testing.T) {
+	t.Parallel()
+
+	env := newTestEnv(t)
+
+	userID, _ := env.seedUser(t, "replayer", "somepassword")
+	cookies := env.authCookies(t, userID, "replayer")
+
+	wh := env.seedWebhook(t, userID)
+	tgt := env.seedTarget(t, wh.ID)
+	evt := env.seedEvent(t, wh.ID, `{"replay":"me"}`)
+	dlv := env.seedFailedDelivery(t, wh.ID, evt.ID, tgt.ID)
+
+	path := "/source/" + wh.ID + "/deliveries/" + dlv.ID +
+		"/replay"
+
+	assert.Equal(
+		t, http.StatusMethodNotAllowed,
+		env.get(path, cookies).Code,
+		"a replay must not be reachable by GET",
+	)
+
+	assert.Equal(
+		t, http.StatusForbidden,
+		env.post(path, url.Values{}, cookies).Code,
+		"a replay POST without a CSRF token must be refused",
+	)
+
+	anon := env.post(path, url.Values{}, nil)
+	assert.Equal(t, http.StatusForbidden, anon.Code)
+
+	require.Equal(
+		t, int64(1), env.countDeliveries(t, wh.ID),
+		"no refused request may have created a delivery",
+	)
+
+	// The token and the action URL both come out of the rendered
+	// page, so a typo in either the route pattern or the template
+	// fails here.
+	logsPath := "/source/" + wh.ID + "/logs"
+
+	token, cookies := env.csrfFrom(t, logsPath, cookies)
+
+	page := env.get(logsPath, cookies)
+	require.Equal(t, http.StatusOK, page.Code)
+
+	action := regexp.MustCompile(
+		`action="(/source/[^"]+/replay)"`,
+	).FindStringSubmatch(page.Body.String())
+	require.Len(
+		t, action, 2,
+		"a finished delivery should render a replay form",
+	)
+
+	form := url.Values{}
+	form.Set("csrf_token", token)
+
+	w := env.post(
+		html.UnescapeString(action[1]), form, cookies,
+	)
+
+	require.Equal(t, http.StatusSeeOther, w.Code)
+	assert.Equal(
+		t, logsPath+"?replay=queued",
+		w.Header().Get("Location"),
+	)
+	assert.Equal(
+		t, int64(2), env.countDeliveries(t, wh.ID),
+		"the replay appends a delivery",
+	)
+}
+
 // metricsConfig is a Config differing from the routing default only
 // in the two /metrics credentials.
 func metricsConfig(