Add per-delivery replay to the event log (closes #203) (#240)

There was no redelivery path anywhere: once a delivery exhausted
max_retries it was failed permanently, even though the event body is
durably stored. Storing an event and being unable to re-send it defeats
the reason it is stored, and the ordinary case is a destination that was
down longer than the backoff ladder.

Adds POST /source/{sourceID}/deliveries/{deliveryID}/replay, inside the
authenticated group so it inherits MaxBodySize, CSRF, NoCache and
RequireAuth. Replay creates a NEW pending delivery against the target's
CURRENT config and hands it to the engine through the same notifier the
receiver uses, so it runs the normal path with the retry ladder, the
SSRF-guarded transport and the circuit breaker. The original delivery's
rows are never touched, and the stored event body is re-sent, never the
recorded response.

Replay is refused, with a distinct message, for a non-terminal delivery, a
deleted target, a deactivated target, and when an earlier replay of the
same event and target is still in flight. Bounded by a per-client rate
limit and by that in-flight check.

The new delivery row is written with Omit(clause.Associations) and with
neither Event nor Target populated, so it cannot upsert a targets row into
the per-webhook event database (#206).

Counted by webhooker_delivery_replays_total on the existing target_type
label. A replay also moves the ordinary attempt, outcome and duration
series, because it is a real delivery.

internal/server/routes.go

@@ -201,6 +201,18 @@ func (s *Server) setupSourceRoutes() {
 			"/logs/{eventID}/body",
 			s.h.HandleEventBodyDownload(),
 		)
+		// Replay is the one page action that queues outbound work:
+		// it creates a delivery from a stored event and hands it to
+		// the delivery engine. The rate limit is what bounds a
+		// held-down button or a scripted loop; the handler
+		// separately refuses a replay while an earlier one for the
+		// same event and target is still in flight. POST only, so
+		// the action cannot be taken by a link, a prefetch or an
+		// image tag.
+		r.With(s.mw.ReplayRateLimit()).Post(
+			"/deliveries/{deliveryID}/replay",
+			s.h.HandleDeliveryReplay(),
+		)
 		r.Post(
 			"/entrypoints",
 			s.h.HandleEntrypointCreate(),

internal/server/routes_test.go

@@ -310,6 +310,75 @@ func (e *testEnv) seedEvent(
 	return event
 }
 
+// seedTarget creates an active HTTP target for a webhook.
+func (e *testEnv) seedTarget(
+	t *testing.T,
+	webhookID string,
+) *database.Target {
+	t.Helper()
+
+	tgt := &database.Target{
+		WebhookID: webhookID,
+		Name:      "routed-target",
+		Type:      database.TargetTypeHTTP,
+		Active:    true,
+		Config:    `{"url":"http://93.184.216.34/hook"}`,
+	}
+
+	require.NoError(
+		t,
+		e.db.DB().Omit(clause.Associations).Create(tgt).Error,
+	)
+
+	return tgt
+}
+
+// seedFailedDelivery records a terminally failed delivery of an event
+// to a target in the webhook's own database.
+func (e *testEnv) seedFailedDelivery(
+	t *testing.T,
+	webhookID, eventID, targetID string,
+) *database.Delivery {
+	t.Helper()
+
+	webhookDB, err := e.dbMgr.GetDB(webhookID)
+	require.NoError(t, err)
+
+	dlv := &database.Delivery{
+		EventID:  eventID,
+		TargetID: targetID,
+		Status:   database.DeliveryStatusFailed,
+	}
+
+	require.NoError(
+		t,
+		webhookDB.Omit(clause.Associations).Create(dlv).Error,
+	)
+
+	return dlv
+}
+
+// countDeliveries reports how many deliveries a webhook's database
+// holds.
+func (e *testEnv) countDeliveries(
+	t *testing.T, webhookID string,
+) int64 {
+	t.Helper()
+
+	webhookDB, err := e.dbMgr.GetDB(webhookID)
+	require.NoError(t, err)
+
+	var count int64
+
+	require.NoError(
+		t,
+		webhookDB.Model(&database.Delivery{}).
+			Count(&count).Error,
+	)
+
+	return count
+}
+
 // storedHash reads the current password hash for a username.
 func (e *testEnv) storedHash(t *testing.T, username string) string {
 	t.Helper()
@@ -674,6 +743,85 @@ func TestSourceLogsBody_OtherUser404s(t *testing.T) {
 	assert.Equal(t, "/pages/login", anon.Header().Get("Location"))
 }
 
+// TestDeliveryReplay_PostOnlyAndCSRFProtected walks the replay action
+// through the production router rather than a forged route context,
+// which is the only way to prove what the route group actually gives
+// it: a GET cannot trigger a replay, an unauthenticated request never
+// reaches the handler, a POST without the token is refused by CSRF,
+// and the form the template emits — token and action URL both — works
+// as rendered.
+func TestDeliveryReplay_PostOnlyAndCSRFProtected(t *testing.T) {
+	t.Parallel()
+
+	env := newTestEnv(t)
+
+	userID, _ := env.seedUser(t, "replayer", "somepassword")
+	cookies := env.authCookies(t, userID, "replayer")
+
+	wh := env.seedWebhook(t, userID)
+	tgt := env.seedTarget(t, wh.ID)
+	evt := env.seedEvent(t, wh.ID, `{"replay":"me"}`)
+	dlv := env.seedFailedDelivery(t, wh.ID, evt.ID, tgt.ID)
+
+	path := "/source/" + wh.ID + "/deliveries/" + dlv.ID +
+		"/replay"
+
+	assert.Equal(
+		t, http.StatusMethodNotAllowed,
+		env.get(path, cookies).Code,
+		"a replay must not be reachable by GET",
+	)
+
+	assert.Equal(
+		t, http.StatusForbidden,
+		env.post(path, url.Values{}, cookies).Code,
+		"a replay POST without a CSRF token must be refused",
+	)
+
+	anon := env.post(path, url.Values{}, nil)
+	assert.Equal(t, http.StatusForbidden, anon.Code)
+
+	require.Equal(
+		t, int64(1), env.countDeliveries(t, wh.ID),
+		"no refused request may have created a delivery",
+	)
+
+	// The token and the action URL both come out of the rendered
+	// page, so a typo in either the route pattern or the template
+	// fails here.
+	logsPath := "/source/" + wh.ID + "/logs"
+
+	token, cookies := env.csrfFrom(t, logsPath, cookies)
+
+	page := env.get(logsPath, cookies)
+	require.Equal(t, http.StatusOK, page.Code)
+
+	action := regexp.MustCompile(
+		`action="(/source/[^"]+/replay)"`,
+	).FindStringSubmatch(page.Body.String())
+	require.Len(
+		t, action, 2,
+		"a finished delivery should render a replay form",
+	)
+
+	form := url.Values{}
+	form.Set("csrf_token", token)
+
+	w := env.post(
+		html.UnescapeString(action[1]), form, cookies,
+	)
+
+	require.Equal(t, http.StatusSeeOther, w.Code)
+	assert.Equal(
+		t, logsPath+"?replay=queued",
+		w.Header().Get("Location"),
+	)
+	assert.Equal(
+		t, int64(2), env.countDeliveries(t, wh.ID),
+		"the replay appends a delivery",
+	)
+}
+
 // metricsConfig is a Config differing from the routing default only
 // in the two /metrics credentials.
 func metricsConfig(