Add a statistics pane to the webhook page (closes #368)

Each webhook's event database keeps running totals: one row for its
events, with when the newest arrived, and one row per target for its
deliveries, delivered and failed, each with what retention removed.
Every write to them shares the transaction of the rows it counts, and
a delivery already delivered or failed is not settled again.
Deliveries get a finished_at column; it and target_id end the status
index, so each target's deliveries finished in a window come from one
index-range query grouped by target. Retention deletes 1000 expired
events per transaction, pausing 200 ms between them so other writers
get in, and stops between them on shutdown. The pane is its own
template, its figures in tables.

The schema changes in place with nothing back-filled, so an existing
database must be recreated.

Model: opus-5-5

internal/handlers/webhook_stats_test.go

@@ -121,15 +121,15 @@ type statsHistory struct {
 	first, second string
 }
 
-// seedStatsHistory builds the webhook the statistics test checks: one
+// seedStatsHistory builds the webhook the statistics test checks: 14
-// day of retention, five entrypoints (one inactive) and three targets
+// days of retention, eleven entrypoints (one inactive) and five targets
-// (one inactive). Six events arrive through the receiver, and so each
+// (three inactive). Nine events arrive through the receiver, and so
-// has a delivery to the two active targets. The oldest event is past
+// each has a delivery to the two active targets. The oldest event is
-// retention, the next six hours old, the other four just in. Five
+// past retention, the next 30 hours old, the next six hours old, the
-// deliveries are settled as the delivery engine would, the rest left
+// other six just in. Five deliveries are settled as the delivery engine
-// pending, and a replay adds a pending delivery to the oldest event.
+// would, the rest left pending, and a replay adds a pending delivery to
-// Once retention has removed the oldest event, no two figures in the
+// the oldest event. Once retention has removed the oldest event, every
-// pane's top row, or in the same table, are equal.
+// figure in the pane differs from every other.
 func seedStatsHistory(
 	t *testing.T,
 	h *handlers.Handlers,
@@ -140,12 +140,12 @@ func seedStatsHistory(
 	t.Helper()
 
 	wh := &database.Webhook{
-		UserID: deleteTestUserID, Name: "stats", RetentionDays: 1,
+		UserID: deleteTestUserID, Name: "stats", RetentionDays: 14,
 	}
 	require.NoError(t, db.DB().Omit(clause.Associations).Create(wh).Error)
 
 	path := statsEntrypoint(t, db, wh.ID, true)
-	for range 3 {
+	for range 9 {
 		statsEntrypoint(t, db, wh.ID, true)
 	}
 
@@ -156,13 +156,16 @@ func seedStatsHistory(
 		`{"url":"`+replayTargetURL+`"}`,
 	)
 	second := seedTarget(t, db, wh.ID, database.TargetTypeLog)
-	inactive := seedTarget(t, db, wh.ID, database.TargetTypeLog)
+
-	require.NoError(t, db.DB().Model(inactive).
+	for range 3 {
-		Update("active", false).Error)
+		inactive := seedTarget(t, db, wh.ID, database.TargetTypeLog)
+		require.NoError(t, db.DB().Model(inactive).
+			Update("active", false).Error)
+	}
 
 	router := receiverRouter(h)
 
-	for range 6 {
+	for range 9 {
 		require.Equal(t, http.StatusOK, postReceiver(t, router, path))
 	}
 
@@ -170,26 +173,27 @@ func seedStatsHistory(
 	require.NoError(t, err)
 
 	events := listEvents(t, webhookDB)
-	require.Len(t, events, 6)
+	require.Len(t, events, 9)
 
-	oldest, middle, newest := events[0], events[1], events[5]
+	oldest, yesterday, middle, newest := events[0], events[1], events[2], events[8]
 	now := time.Now()
 
-	statsAge(t, webhookDB, oldest.ID, now.Add(-50*time.Hour))
+	statsAge(t, webhookDB, oldest.ID, now.Add(-15*24*time.Hour))
+	statsAge(t, webhookDB, yesterday.ID, now.Add(-30*time.Hour))
 	statsAge(t, webhookDB, middle.ID, now.Add(-6*time.Hour))
 
 	oldestFailure := statsDelivery(t, webhookDB, oldest.ID, first.ID)
 	statsFinish(t, webhookDB, oldestFailure,
-		database.DeliveryStatusFailed, now.Add(-49*time.Hour))
+		database.DeliveryStatusFailed, now.Add(-14*24*time.Hour))
 	statsFinish(t, webhookDB,
-		statsDelivery(t, webhookDB, oldest.ID, second.ID),
+		statsDelivery(t, webhookDB, yesterday.ID, first.ID),
-		database.DeliveryStatusDelivered, now.Add(-49*time.Hour))
+		database.DeliveryStatusFailed, now.Add(-29*time.Hour))
 	statsFinish(t, webhookDB,
-		statsDelivery(t, webhookDB, middle.ID, first.ID),
+		statsDelivery(t, webhookDB, yesterday.ID, second.ID),
-		database.DeliveryStatusFailed, now.Add(-5*time.Hour))
+		database.DeliveryStatusFailed, now.Add(-29*time.Hour))
 	statsFinish(t, webhookDB,
 		statsDelivery(t, webhookDB, middle.ID, second.ID),
-		database.DeliveryStatusFailed, now.Add(-time.Minute))
+		database.DeliveryStatusFailed, now.Add(-5*time.Hour))
 	statsFinish(t, webhookDB,
 		statsDelivery(t, webhookDB, newest.ID, first.ID),
 		database.DeliveryStatusDelivered, now.Add(-2*time.Minute))
@@ -259,18 +263,16 @@ func statsPane(t *testing.T, page string) string {
 
 // assertStatsTargets checks, for the history seedStatsHistory builds,
 // each target's totals and its deliveries finished in the last 24
-// hours. The first target has six deliveries and the replay, the
+// hours. The first target has nine deliveries and the replay, the
-// second six; the inactive target has none and so no row.
+// second nine; the inactive targets have none and so no row.
 func assertStatsTargets(t *testing.T, hist statsHistory) {
 	t.Helper()
 
 	first, second := hist.first, hist.second
 
 	assert.Equal(t, map[string]database.TargetTotals{
-		first: {TargetID: first, Deliveries: 7, Delivered: 1, Failed: 2},
+		first:  {TargetID: first, Deliveries: 10, Delivered: 1, Failed: 2},
-		second: {
+		second: {TargetID: second, Deliveries: 9, Failed: 2},
-			TargetID: second, Deliveries: 6, Delivered: 1, Failed: 1,
-		},
 	}, statsTargetTotals(t, hist.webhookDB))
 
 	lastDay, err := handlers.FinishedByTargetForTest(
@@ -278,7 +280,7 @@ func assertStatsTargets(t *testing.T, hist statsHistory) {
 	)
 	require.NoError(t, err)
 	assert.ElementsMatch(t, []handlers.TargetFinished{
-		{TargetID: first, Delivered: 1, Failed: 1},
+		{TargetID: first, Delivered: 1},
 		{TargetID: second, Failed: 1},
 	}, lastDay)
 }
@@ -295,21 +297,21 @@ func assertStatsPaneAfterPrune(
 	t.Helper()
 
 	pane := statsPane(t, renderSourceDetailPage(t, h, sess, hist.webhook.ID))
-	lastEvent := hist.newest.CreatedAt.Format("2006-01-02 15:04:05 UTC")
+	lastEvent := hist.newest.CreatedAt.UTC().Format("2006-01-02 15:04:05 UTC")
 
-	assert.Contains(t, pane, "Entrypoints 5 (4 active) "+
+	assert.Contains(t, pane, "Entrypoints 11 (10 active) "+
-		"Targets 3 (2 active) "+
+		"Targets 5 (2 active) "+
-		"Deliveries in progress 7 "+
+		"Deliveries in progress 12 "+
 		"Last event "+lastEvent+" "+
-		"Retention 1 day")
+		"Retention 14 days")
 	assert.Contains(t, pane, "Lifetime Within retention "+
-		"Events 6 5 "+
+		"Events 9 8 "+
-		"Deliveries 13 10 "+
+		"Deliveries 19 16 "+
-		"Failures 3 2")
+		"Failures 4 3")
 	assert.Contains(t, pane, "Last 10 minutes Last 24 hours "+
-		"Events 4 5 "+
+		"Events 6 7 "+
-		"Failures 1 2 "+
+		"Failures 0 1 "+
-		"Failure percentage 50.0% 66.7%")
+		"Failure percentage 0.0% 50.0%")
 }
 
 // TestWebhookStats_EveryFigureAcrossRetentionPrune checks every figure
@@ -338,50 +340,49 @@ func TestWebhookStats_EveryFigureAcrossRetentionPrune(t *testing.T) {
 	stats := h.WebhookStatsForTest(hist.webhook.ID)
 	require.NotNil(t, stats)
 
-	assert.Equal(t, 5, stats.Entrypoints)
+	assert.Equal(t, 11, stats.Entrypoints)
-	assert.Equal(t, 4, stats.ActiveEntrypoints)
+	assert.Equal(t, 10, stats.ActiveEntrypoints)
-	assert.Equal(t, 3, stats.Targets)
+	assert.Equal(t, 5, stats.Targets)
 	assert.Equal(t, 2, stats.ActiveTargets)
-	assert.Equal(t, handlers.Counts{Events: 6, Deliveries: 13, Failures: 3},
+	assert.Equal(t, handlers.Counts{Events: 9, Deliveries: 19, Failures: 4},
 		stats.Lifetime)
 	assert.Equal(t, stats.Lifetime, stats.WithinRetention)
-	assert.Equal(t, int64(8), stats.InProgress)
+	assert.Equal(t, int64(14), stats.InProgress)
 	require.NotNil(t, stats.LastEventAt)
 	assert.True(t, hist.newest.CreatedAt.Equal(*stats.LastEventAt))
+	assert.Equal(t, handlers.RecentWindow{Events: 6, Delivered: 1},
+		stats.Last10Minutes)
 	assert.Equal(t, handlers.RecentWindow{
-		Events: 4, Delivered: 1, Failed: 1,
+		Events: 7, Delivered: 1, Failed: 1,
-	}, stats.Last10Minutes)
-	assert.Equal(t, handlers.RecentWindow{
-		Events: 5, Delivered: 1, Failed: 2,
 	}, stats.Last24Hours)
-	assert.Equal(t, "50.0%", stats.Last10Minutes.FailurePercent())
+	assert.Equal(t, "0.0%", stats.Last10Minutes.FailurePercent())
-	assert.Equal(t, "66.7%", stats.Last24Hours.FailurePercent())
+	assert.Equal(t, "50.0%", stats.Last24Hours.FailurePercent())
 
 	assertStatsTargets(t, hist)
 
 	// Retention removes the oldest event with its three deliveries:
 	// the first target's failed one and the pending replay, and the
-	// second target's delivered one.
+	// second target's pending one.
 	statsPrune(t, db, dbMgr, log, hist.webhookDB)
 
 	after := h.WebhookStatsForTest(hist.webhook.ID)
 	require.NotNil(t, after)
 
 	assert.Equal(t, stats.Lifetime, after.Lifetime)
-	assert.Equal(t, handlers.Counts{Events: 5, Deliveries: 10, Failures: 2},
+	assert.Equal(t, handlers.Counts{Events: 8, Deliveries: 16, Failures: 3},
 		after.WithinRetention)
-	assert.Equal(t, int64(7), after.InProgress)
+	assert.Equal(t, int64(12), after.InProgress)
 	assert.Equal(t, stats.LastEventAt, after.LastEventAt)
 	assert.Equal(t, stats.Last10Minutes, after.Last10Minutes)
 	assert.Equal(t, stats.Last24Hours, after.Last24Hours)
 
 	assert.Equal(t, map[string]database.TargetTotals{
 		first: {
-			TargetID: first, Deliveries: 7, Delivered: 1, Failed: 2,
+			TargetID: first, Deliveries: 10, Delivered: 1, Failed: 2,
 			DeliveriesRemoved: 2, FailedRemoved: 1,
 		},
 		second: {
-			TargetID: second, Deliveries: 6, Delivered: 1, Failed: 1,
+			TargetID: second, Deliveries: 9, Failed: 2,
 			DeliveriesRemoved: 1,
 		},
 	}, statsTargetTotals(t, hist.webhookDB))
@@ -436,7 +437,40 @@ func TestWebhookStats_LastEventSurvivesPruningEveryEvent(t *testing.T) {
 
 	pane := statsPane(t, renderSourceDetailPage(t, h, sess, wh.ID))
 	assert.Contains(t, pane,
-		"Last event "+arrived.Format("2006-01-02 15:04:05 UTC"))
+		"Last event "+arrived.UTC().Format("2006-01-02 15:04:05 UTC"))
+}
+
+// TestWebhookStats_LastEventInUTC checks that the pane shows when the
+// last event arrived in UTC, as the event list does, when the time was
+// stored in another zone, as it is on a host whose local time is not
+// UTC.
+func TestWebhookStats_LastEventInUTC(t *testing.T) {
+	t.Parallel()
+
+	var (
+		h     *handlers.Handlers
+		sess  *session.Session
+		db    *database.Database
+		dbMgr *database.WebhookDBManager
+	)
+
+	app := newTestApp(t, &h, &sess, &db, &dbMgr)
+	app.RequireStart()
+
+	t.Cleanup(app.RequireStop)
+
+	wh := seedWebhook(t, db)
+
+	webhookDB, err := dbMgr.GetDB(wh.ID)
+	require.NoError(t, err)
+
+	arrived := time.Date(2026, time.March, 4, 22, 30, 0, 0,
+		time.FixedZone("EST", -5*60*60))
+	require.NoError(t, database.AddEventTotals(webhookDB,
+		database.EventTotals{Events: 1, LastEventAt: &arrived}))
+
+	pane := statsPane(t, renderSourceDetailPage(t, h, sess, wh.ID))
+	assert.Contains(t, pane, "Last event 2026-03-05 03:30:00 UTC")
 }
 
 // TestWebhookStats_PaneShowsRetentionPeriod checks that the statistics

templates/webhook_stats.html

@@ -22,7 +22,7 @@
         </div>
         <div>
             <span class="text-gray-500">Last event</span>
-            <span class="font-medium text-gray-900">{{with .LastEventAt}}{{.Format "2006-01-02 15:04:05 UTC"}}{{else}}none{{end}}</span>
+            <span class="font-medium text-gray-900">{{with .LastEventAt}}{{.UTC.Format "2006-01-02 15:04:05 UTC"}}{{else}}none{{end}}</span>
         </div>
         <div>
             <span class="text-gray-500">Retention</span>