Add a statistics pane to the webhook page (closes #368)

Each webhook's event database keeps running totals: one row for its
events, with when the newest arrived, and one row per target for its
deliveries, delivered and failed, each with what retention removed.
Every write to them shares the transaction of the rows it counts, and
a delivery already delivered or failed is not settled again.
Deliveries get a finished_at column; it and target_id end the status
index, so each target's deliveries finished in a window come from one
index-range query grouped by target. Retention deletes 1000 expired
events per transaction, pausing 200 ms between them so other writers
get in, and stops between them on shutdown. The pane is its own
template, its figures in tables.

The schema changes in place with nothing back-filled, so an existing
database must be recreated.

Model: opus-5-5

internal/handlers/webhook_stats_test.go

@@ -122,14 +122,16 @@ type statsHistory struct {
 }
 
 // seedStatsHistory builds the webhook the statistics test checks: 14
-// days of retention, eleven entrypoints (one inactive) and five targets
+// days of retention, twelve entrypoints (one inactive) and six targets
-// (three inactive). Nine events arrive through the receiver, and so
+// (four inactive). Ten events arrive through the receiver, and so each
-// each has a delivery to the two active targets. The oldest event is
+// has a delivery to the two active targets. The oldest event is past
-// past retention, the next 30 hours old, the next six hours old, the
+// retention, the next 30 hours old, the next six hours old, the other
-// other six just in. Five deliveries are settled as the delivery engine
+// seven just in. Six deliveries are settled as the delivery engine
-// would, the rest left pending, and a replay adds a pending delivery to
+// would, two of them inside a recent window though their event arrived
-// the oldest event. Once retention has removed the oldest event, every
+// before it. The newest event's delivery to the second target is
-// figure in the pane differs from every other.
+// retrying, the rest are left pending, and a replay adds a pending
+// delivery to the oldest event. Once retention has removed the oldest
+// event, every figure in the pane differs from every other.
 func seedStatsHistory(
 	t *testing.T,
 	h *handlers.Handlers,
@@ -139,13 +141,11 @@ func seedStatsHistory(
 ) statsHistory {
 	t.Helper()
 
-	wh := &database.Webhook{
+	wh := &database.Webhook{UserID: deleteTestUserID, Name: "stats", RetentionDays: 14}
-		UserID: deleteTestUserID, Name: "stats", RetentionDays: 14,
-	}
 	require.NoError(t, db.DB().Omit(clause.Associations).Create(wh).Error)
 
 	path := statsEntrypoint(t, db, wh.ID, true)
-	for range 9 {
+	for range 10 {
 		statsEntrypoint(t, db, wh.ID, true)
 	}
 
@@ -157,7 +157,7 @@ func seedStatsHistory(
 	)
 	second := seedTarget(t, db, wh.ID, database.TargetTypeLog)
 
-	for range 3 {
+	for range 4 {
 		inactive := seedTarget(t, db, wh.ID, database.TargetTypeLog)
 		require.NoError(t, db.DB().Model(inactive).
 			Update("active", false).Error)
@@ -165,7 +165,7 @@ func seedStatsHistory(
 
 	router := receiverRouter(h)
 
-	for range 9 {
+	for range 10 {
 		require.Equal(t, http.StatusOK, postReceiver(t, router, path))
 	}
 
@@ -173,9 +173,9 @@ func seedStatsHistory(
 	require.NoError(t, err)
 
 	events := listEvents(t, webhookDB)
-	require.Len(t, events, 9)
+	require.Len(t, events, 10)
 
-	oldest, yesterday, middle, newest := events[0], events[1], events[2], events[8]
+	oldest, yesterday, middle, newest := events[0], events[1], events[2], events[9]
 	now := time.Now()
 
 	statsAge(t, webhookDB, oldest.ID, now.Add(-15*24*time.Hour))
@@ -190,23 +190,27 @@ func seedStatsHistory(
 		database.DeliveryStatusFailed, now.Add(-29*time.Hour))
 	statsFinish(t, webhookDB,
 		statsDelivery(t, webhookDB, yesterday.ID, second.ID),
-		database.DeliveryStatusFailed, now.Add(-29*time.Hour))
+		database.DeliveryStatusFailed, now.Add(-23*time.Hour))
 	statsFinish(t, webhookDB,
 		statsDelivery(t, webhookDB, middle.ID, second.ID),
 		database.DeliveryStatusFailed, now.Add(-5*time.Hour))
+	statsFinish(t, webhookDB,
+		statsDelivery(t, webhookDB, middle.ID, first.ID),
+		database.DeliveryStatusFailed, now.Add(-time.Minute))
 	statsFinish(t, webhookDB,
 		statsDelivery(t, webhookDB, newest.ID, first.ID),
 		database.DeliveryStatusDelivered, now.Add(-2*time.Minute))
 
+	retrying := statsDelivery(t, webhookDB, newest.ID, second.ID)
+	require.NoError(t, webhookDB.Model(&retrying).
+		Update("status", database.DeliveryStatusRetrying).Error)
+
 	require.Equal(t, http.StatusSeeOther,
 		postReplay(t, h, sess, wh.ID, oldestFailure.ID).Code)
 
 	return statsHistory{
-		webhook:   wh,
+		webhook: wh, webhookDB: webhookDB, newest: newest,
-		webhookDB: webhookDB,
+		first: first.ID, second: second.ID,
-		newest:    newest,
-		first:     first.ID,
-		second:    second.ID,
 	}
 }
 
@@ -263,16 +267,16 @@ func statsPane(t *testing.T, page string) string {
 
 // assertStatsTargets checks, for the history seedStatsHistory builds,
 // each target's totals and its deliveries finished in the last 24
-// hours. The first target has nine deliveries and the replay, the
+// hours. The first target has ten deliveries and the replay, the
-// second nine; the inactive targets have none and so no row.
+// second ten; the inactive targets have none and so no row.
 func assertStatsTargets(t *testing.T, hist statsHistory) {
 	t.Helper()
 
 	first, second := hist.first, hist.second
 
 	assert.Equal(t, map[string]database.TargetTotals{
-		first:  {TargetID: first, Deliveries: 10, Delivered: 1, Failed: 2},
+		first:  {TargetID: first, Deliveries: 11, Delivered: 1, Failed: 3},
-		second: {TargetID: second, Deliveries: 9, Failed: 2},
+		second: {TargetID: second, Deliveries: 10, Failed: 2},
 	}, statsTargetTotals(t, hist.webhookDB))
 
 	lastDay, err := handlers.FinishedByTargetForTest(
@@ -280,8 +284,8 @@ func assertStatsTargets(t *testing.T, hist statsHistory) {
 	)
 	require.NoError(t, err)
 	assert.ElementsMatch(t, []handlers.TargetFinished{
-		{TargetID: first, Delivered: 1},
+		{TargetID: first, Delivered: 1, Failed: 1},
-		{TargetID: second, Failed: 1},
+		{TargetID: second, Failed: 2},
 	}, lastDay)
 }
 
@@ -299,19 +303,19 @@ func assertStatsPaneAfterPrune(
 	pane := statsPane(t, renderSourceDetailPage(t, h, sess, hist.webhook.ID))
 	lastEvent := hist.newest.CreatedAt.UTC().Format("2006-01-02 15:04:05 UTC")
 
-	assert.Contains(t, pane, "Entrypoints 11 (10 active) "+
+	assert.Contains(t, pane, "Entrypoints 12 (11 active) "+
-		"Targets 5 (2 active) "+
+		"Targets 6 (2 active) "+
-		"Deliveries in progress 12 "+
+		"Deliveries in progress 13 "+
 		"Last event "+lastEvent+" "+
 		"Retention 14 days")
 	assert.Contains(t, pane, "Lifetime Within retention "+
-		"Events 9 8 "+
+		"Events 10 9 "+
-		"Deliveries 19 16 "+
+		"Deliveries 21 18 "+
-		"Failures 4 3")
+		"Failures 5 4")
 	assert.Contains(t, pane, "Last 10 minutes Last 24 hours "+
-		"Events 6 7 "+
+		"Events 7 8 "+
-		"Failures 0 1 "+
+		"Failures 1 3 "+
-		"Failure percentage 0.0% 50.0%")
+		"Failure percentage 50.0% 75.0%")
 }
 
 // TestWebhookStats_EveryFigureAcrossRetentionPrune checks every figure
@@ -340,23 +344,24 @@ func TestWebhookStats_EveryFigureAcrossRetentionPrune(t *testing.T) {
 	stats := h.WebhookStatsForTest(hist.webhook.ID)
 	require.NotNil(t, stats)
 
-	assert.Equal(t, 11, stats.Entrypoints)
+	assert.Equal(t, 12, stats.Entrypoints)
-	assert.Equal(t, 10, stats.ActiveEntrypoints)
+	assert.Equal(t, 11, stats.ActiveEntrypoints)
-	assert.Equal(t, 5, stats.Targets)
+	assert.Equal(t, 6, stats.Targets)
 	assert.Equal(t, 2, stats.ActiveTargets)
-	assert.Equal(t, handlers.Counts{Events: 9, Deliveries: 19, Failures: 4},
+	assert.Equal(t, handlers.Counts{Events: 10, Deliveries: 21, Failures: 5},
 		stats.Lifetime)
 	assert.Equal(t, stats.Lifetime, stats.WithinRetention)
-	assert.Equal(t, int64(14), stats.InProgress)
+	assert.Equal(t, int64(15), stats.InProgress)
 	require.NotNil(t, stats.LastEventAt)
 	assert.True(t, hist.newest.CreatedAt.Equal(*stats.LastEventAt))
-	assert.Equal(t, handlers.RecentWindow{Events: 6, Delivered: 1},
-		stats.Last10Minutes)
 	assert.Equal(t, handlers.RecentWindow{
 		Events: 7, Delivered: 1, Failed: 1,
+	}, stats.Last10Minutes)
+	assert.Equal(t, handlers.RecentWindow{
+		Events: 8, Delivered: 1, Failed: 3,
 	}, stats.Last24Hours)
-	assert.Equal(t, "0.0%", stats.Last10Minutes.FailurePercent())
+	assert.Equal(t, "50.0%", stats.Last10Minutes.FailurePercent())
-	assert.Equal(t, "50.0%", stats.Last24Hours.FailurePercent())
+	assert.Equal(t, "75.0%", stats.Last24Hours.FailurePercent())
 
 	assertStatsTargets(t, hist)
 
@@ -369,20 +374,20 @@ func TestWebhookStats_EveryFigureAcrossRetentionPrune(t *testing.T) {
 	require.NotNil(t, after)
 
 	assert.Equal(t, stats.Lifetime, after.Lifetime)
-	assert.Equal(t, handlers.Counts{Events: 8, Deliveries: 16, Failures: 3},
+	assert.Equal(t, handlers.Counts{Events: 9, Deliveries: 18, Failures: 4},
 		after.WithinRetention)
-	assert.Equal(t, int64(12), after.InProgress)
+	assert.Equal(t, int64(13), after.InProgress)
 	assert.Equal(t, stats.LastEventAt, after.LastEventAt)
 	assert.Equal(t, stats.Last10Minutes, after.Last10Minutes)
 	assert.Equal(t, stats.Last24Hours, after.Last24Hours)
 
 	assert.Equal(t, map[string]database.TargetTotals{
 		first: {
-			TargetID: first, Deliveries: 10, Delivered: 1, Failed: 2,
+			TargetID: first, Deliveries: 11, Delivered: 1, Failed: 3,
 			DeliveriesRemoved: 2, FailedRemoved: 1,
 		},
 		second: {
-			TargetID: second, Deliveries: 9, Failed: 2,
+			TargetID: second, Deliveries: 10, Failed: 2,
 			DeliveriesRemoved: 1,
 		},
 	}, statsTargetTotals(t, hist.webhookDB))