Add a statistics pane to the webhook page (closes #368)

Each webhook's event database keeps running totals: one row for its
events, with when the newest arrived, and one row per target for its
deliveries, delivered and failed, each with what retention removed.
Every write to them shares the transaction of the rows it counts, and
a delivery already delivered or failed is not settled again.
Deliveries get a finished_at column; it and target_id end the status
index, so each target's deliveries finished in a window come from one
index-range query grouped by target. Retention deletes 1000 expired
events per transaction, pausing 200 ms between them so other writers
get in, and stops between them on shutdown. The pane is its own
template, its figures in tables.

The schema changes in place with nothing back-filled, so an existing
database must be recreated.

Model: opus-5-5

internal/handlers/webhook_stats_test.go

@@ -122,16 +122,14 @@ type statsHistory struct {
 }
 
 // seedStatsHistory builds the webhook the statistics test checks: 14
-// days of retention, twelve entrypoints (one inactive) and six targets
-// (four inactive). Ten events arrive through the receiver, and so each
-// has a delivery to the two active targets. The oldest event is past
-// retention, the next 30 hours old, the next six hours old, the other
-// seven just in. Six deliveries are settled as the delivery engine
-// would, two of them inside a recent window though their event arrived
-// before it. The newest event's delivery to the second target is
-// retrying, the rest are left pending, and a replay adds a pending
-// delivery to the oldest event. Once retention has removed the oldest
-// event, every figure in the pane differs from every other.
+// days of retention, eleven entrypoints (one inactive) and five targets
+// (three inactive). Nine events arrive through the receiver, and so
+// each has a delivery to the two active targets. The oldest event is
+// past retention, the next 30 hours old, the next six hours old, the
+// other six just in. Five deliveries are settled as the delivery engine
+// would, the rest left pending, and a replay adds a pending delivery to
+// the oldest event. Once retention has removed the oldest event, every
+// figure in the pane differs from every other.
 func seedStatsHistory(
 	t *testing.T,
 	h *handlers.Handlers,
@@ -141,11 +139,13 @@ func seedStatsHistory(
 ) statsHistory {
 	t.Helper()
 
-	wh := &database.Webhook{UserID: deleteTestUserID, Name: "stats", RetentionDays: 14}
+	wh := &database.Webhook{
+		UserID: deleteTestUserID, Name: "stats", RetentionDays: 14,
+	}
 	require.NoError(t, db.DB().Omit(clause.Associations).Create(wh).Error)
 
 	path := statsEntrypoint(t, db, wh.ID, true)
-	for range 10 {
+	for range 9 {
 		statsEntrypoint(t, db, wh.ID, true)
 	}
 
@@ -157,7 +157,7 @@ func seedStatsHistory(
 	)
 	second := seedTarget(t, db, wh.ID, database.TargetTypeLog)
 
-	for range 4 {
+	for range 3 {
 		inactive := seedTarget(t, db, wh.ID, database.TargetTypeLog)
 		require.NoError(t, db.DB().Model(inactive).
 			Update("active", false).Error)
@@ -165,7 +165,7 @@ func seedStatsHistory(
 
 	router := receiverRouter(h)
 
-	for range 10 {
+	for range 9 {
 		require.Equal(t, http.StatusOK, postReceiver(t, router, path))
 	}
 
@@ -173,9 +173,9 @@ func seedStatsHistory(
 	require.NoError(t, err)
 
 	events := listEvents(t, webhookDB)
-	require.Len(t, events, 10)
+	require.Len(t, events, 9)
 
-	oldest, yesterday, middle, newest := events[0], events[1], events[2], events[9]
+	oldest, yesterday, middle, newest := events[0], events[1], events[2], events[8]
 	now := time.Now()
 
 	statsAge(t, webhookDB, oldest.ID, now.Add(-15*24*time.Hour))
@@ -190,27 +190,23 @@ func seedStatsHistory(
 		database.DeliveryStatusFailed, now.Add(-29*time.Hour))
 	statsFinish(t, webhookDB,
 		statsDelivery(t, webhookDB, yesterday.ID, second.ID),
-		database.DeliveryStatusFailed, now.Add(-23*time.Hour))
+		database.DeliveryStatusFailed, now.Add(-29*time.Hour))
 	statsFinish(t, webhookDB,
 		statsDelivery(t, webhookDB, middle.ID, second.ID),
 		database.DeliveryStatusFailed, now.Add(-5*time.Hour))
-	statsFinish(t, webhookDB,
-		statsDelivery(t, webhookDB, middle.ID, first.ID),
-		database.DeliveryStatusFailed, now.Add(-time.Minute))
 	statsFinish(t, webhookDB,
 		statsDelivery(t, webhookDB, newest.ID, first.ID),
 		database.DeliveryStatusDelivered, now.Add(-2*time.Minute))
 
-	retrying := statsDelivery(t, webhookDB, newest.ID, second.ID)
-	require.NoError(t, webhookDB.Model(&retrying).
-		Update("status", database.DeliveryStatusRetrying).Error)
-
 	require.Equal(t, http.StatusSeeOther,
 		postReplay(t, h, sess, wh.ID, oldestFailure.ID).Code)
 
 	return statsHistory{
-		webhook: wh, webhookDB: webhookDB, newest: newest,
-		first: first.ID, second: second.ID,
+		webhook:   wh,
+		webhookDB: webhookDB,
+		newest:    newest,
+		first:     first.ID,
+		second:    second.ID,
 	}
 }
 
@@ -267,16 +263,16 @@ func statsPane(t *testing.T, page string) string {
 
 // assertStatsTargets checks, for the history seedStatsHistory builds,
 // each target's totals and its deliveries finished in the last 24
-// hours. The first target has ten deliveries and the replay, the
-// second ten; the inactive targets have none and so no row.
+// hours. The first target has nine deliveries and the replay, the
+// second nine; the inactive targets have none and so no row.
 func assertStatsTargets(t *testing.T, hist statsHistory) {
 	t.Helper()
 
 	first, second := hist.first, hist.second
 
 	assert.Equal(t, map[string]database.TargetTotals{
-		first:  {TargetID: first, Deliveries: 11, Delivered: 1, Failed: 3},
-		second: {TargetID: second, Deliveries: 10, Failed: 2},
+		first:  {TargetID: first, Deliveries: 10, Delivered: 1, Failed: 2},
+		second: {TargetID: second, Deliveries: 9, Failed: 2},
 	}, statsTargetTotals(t, hist.webhookDB))
 
 	lastDay, err := handlers.FinishedByTargetForTest(
@@ -284,8 +280,8 @@ func assertStatsTargets(t *testing.T, hist statsHistory) {
 	)
 	require.NoError(t, err)
 	assert.ElementsMatch(t, []handlers.TargetFinished{
-		{TargetID: first, Delivered: 1, Failed: 1},
-		{TargetID: second, Failed: 2},
+		{TargetID: first, Delivered: 1},
+		{TargetID: second, Failed: 1},
 	}, lastDay)
 }
 
@@ -303,19 +299,19 @@ func assertStatsPaneAfterPrune(
 	pane := statsPane(t, renderSourceDetailPage(t, h, sess, hist.webhook.ID))
 	lastEvent := hist.newest.CreatedAt.UTC().Format("2006-01-02 15:04:05 UTC")
 
-	assert.Contains(t, pane, "Entrypoints 12 (11 active) "+
-		"Targets 6 (2 active) "+
-		"Deliveries in progress 13 "+
+	assert.Contains(t, pane, "Entrypoints 11 (10 active) "+
+		"Targets 5 (2 active) "+
+		"Deliveries in progress 12 "+
 		"Last event "+lastEvent+" "+
 		"Retention 14 days")
 	assert.Contains(t, pane, "Lifetime Within retention "+
-		"Events 10 9 "+
-		"Deliveries 21 18 "+
-		"Failures 5 4")
+		"Events 9 8 "+
+		"Deliveries 19 16 "+
+		"Failures 4 3")
 	assert.Contains(t, pane, "Last 10 minutes Last 24 hours "+
-		"Events 7 8 "+
-		"Failures 1 3 "+
-		"Failure percentage 50.0% 75.0%")
+		"Events 6 7 "+
+		"Failures 0 1 "+
+		"Failure percentage 0.0% 50.0%")
 }
 
 // TestWebhookStats_EveryFigureAcrossRetentionPrune checks every figure
@@ -344,24 +340,23 @@ func TestWebhookStats_EveryFigureAcrossRetentionPrune(t *testing.T) {
 	stats := h.WebhookStatsForTest(hist.webhook.ID)
 	require.NotNil(t, stats)
 
-	assert.Equal(t, 12, stats.Entrypoints)
-	assert.Equal(t, 11, stats.ActiveEntrypoints)
-	assert.Equal(t, 6, stats.Targets)
+	assert.Equal(t, 11, stats.Entrypoints)
+	assert.Equal(t, 10, stats.ActiveEntrypoints)
+	assert.Equal(t, 5, stats.Targets)
 	assert.Equal(t, 2, stats.ActiveTargets)
-	assert.Equal(t, handlers.Counts{Events: 10, Deliveries: 21, Failures: 5},
+	assert.Equal(t, handlers.Counts{Events: 9, Deliveries: 19, Failures: 4},
 		stats.Lifetime)
 	assert.Equal(t, stats.Lifetime, stats.WithinRetention)
-	assert.Equal(t, int64(15), stats.InProgress)
+	assert.Equal(t, int64(14), stats.InProgress)
 	require.NotNil(t, stats.LastEventAt)
 	assert.True(t, hist.newest.CreatedAt.Equal(*stats.LastEventAt))
+	assert.Equal(t, handlers.RecentWindow{Events: 6, Delivered: 1},
+		stats.Last10Minutes)
 	assert.Equal(t, handlers.RecentWindow{
 		Events: 7, Delivered: 1, Failed: 1,
-	}, stats.Last10Minutes)
-	assert.Equal(t, handlers.RecentWindow{
-		Events: 8, Delivered: 1, Failed: 3,
 	}, stats.Last24Hours)
-	assert.Equal(t, "50.0%", stats.Last10Minutes.FailurePercent())
-	assert.Equal(t, "75.0%", stats.Last24Hours.FailurePercent())
+	assert.Equal(t, "0.0%", stats.Last10Minutes.FailurePercent())
+	assert.Equal(t, "50.0%", stats.Last24Hours.FailurePercent())
 
 	assertStatsTargets(t, hist)
 
@@ -374,20 +369,20 @@ func TestWebhookStats_EveryFigureAcrossRetentionPrune(t *testing.T) {
 	require.NotNil(t, after)
 
 	assert.Equal(t, stats.Lifetime, after.Lifetime)
-	assert.Equal(t, handlers.Counts{Events: 9, Deliveries: 18, Failures: 4},
+	assert.Equal(t, handlers.Counts{Events: 8, Deliveries: 16, Failures: 3},
 		after.WithinRetention)
-	assert.Equal(t, int64(13), after.InProgress)
+	assert.Equal(t, int64(12), after.InProgress)
 	assert.Equal(t, stats.LastEventAt, after.LastEventAt)
 	assert.Equal(t, stats.Last10Minutes, after.Last10Minutes)
 	assert.Equal(t, stats.Last24Hours, after.Last24Hours)
 
 	assert.Equal(t, map[string]database.TargetTotals{
 		first: {
-			TargetID: first, Deliveries: 11, Delivered: 1, Failed: 3,
+			TargetID: first, Deliveries: 10, Delivered: 1, Failed: 2,
 			DeliveriesRemoved: 2, FailedRemoved: 1,
 		},
 		second: {
-			TargetID: second, Deliveries: 10, Failed: 2,
+			TargetID: second, Deliveries: 9, Failed: 2,
 			DeliveriesRemoved: 1,
 		},
 	}, statsTargetTotals(t, hist.webhookDB))