Add a statistics pane to the webhook page (closes #368)

Each webhook's event database keeps running totals: one row for its
events, with when the newest arrived, and one row per target for its
deliveries, delivered and failed, each with what retention removed.
Every write to them shares the transaction of the rows it counts, and
a delivery already delivered or failed is not settled again.
Deliveries get a finished_at column; it and target_id end the status
index, so each target's deliveries finished in a window come from one
index-range query grouped by target. Retention deletes 1000 expired
events per transaction, pausing 200 ms between them so other writers
get in, and stops between them on shutdown. The pane is its own
template, its figures in tables.

The schema changes in place with nothing back-filled, so an existing
database must be recreated.

Model: opus-5-5

README.md

@@ -1811,7 +1811,8 @@ range only on the last column it uses.
 Every entity except `Setting`, `EventTotals` and `TargetTotals` includes
 these fields from `BaseModel`. `Setting` is a bare key-value row with no
 `id`, no timestamps and no soft delete, and the two totals tables hold
-only counts, keyed by a numeric `id` and by `target_id`:
+counts, plus `last_event_at` in `event_totals`, keyed by a numeric `id`
+and by `target_id`:
 
 | Field        | Type      | Description |
 | ------------ | --------- | ----------- |
@@ -3099,7 +3100,8 @@ check, see [The login endpoint](#the-login-endpoint).
   before the app starts; the image's health check; and `docker exec`,
   unless given `--user`
 - GORM soft deletes on every entity that carries `BaseModel`, which is
-  all of them but `Setting` (data preserved for audit)
+  all of them but `Setting`, `EventTotals` and `TargetTotals` (data
+  preserved for audit)
 
 ### Shutdown
 

internal/database/event_tier_indexes_test.go

@@ -207,18 +207,6 @@ func assertPlanUses(
 ) {
 	t.Helper()
 
-	plan := queryPlan(t, db, built)
-
-	for _, index := range indexes {
-		assert.Contains(t, plan, index, built.Statement.SQL.String())
-	}
-}
-
-// queryPlan returns SQLite's plan for a statement GORM built in a dry
-// run, run with the same SQL and arguments GORM would send.
-func queryPlan(t *testing.T, db, built *gorm.DB) string {
-	t.Helper()
-
 	var plan []struct{ Detail string }
 
 	require.NoError(t, db.Raw(
@@ -226,5 +214,8 @@ func queryPlan(t *testing.T, db, built *gorm.DB) string {
 		built.Statement.Vars...,
 	).Scan(&plan).Error)
 
-	return fmt.Sprint(plan)
+	for _, index := range indexes {
+		assert.Contains(t, fmt.Sprint(plan), index,
+			built.Statement.SQL.String())
+	}
 }

internal/database/export_test.go

@@ -5,6 +5,7 @@ import (
 	"io"
 	"log/slog"
 	"os"
+	"testing"
 	"time"
 
 	"go.uber.org/fx"
@@ -83,3 +84,14 @@ func (d *Database) ExportSetBannerOut(w io.Writer) {
 func DummyPasswordHashForTest() string {
 	return dummyPasswordHash()
 }
+
+// HashAtShippedCostForTest makes HashPassword hash at the shipped
+// memory cost until t ends. t must not run in parallel with other
+// tests, which would hash at that cost alongside it.
+func HashAtShippedCostForTest(t *testing.T) {
+	t.Helper()
+
+	hashAtShippedCostInTest = true
+
+	t.Cleanup(func() { hashAtShippedCostInTest = false })
+}

internal/database/password.go

@@ -9,6 +9,7 @@ import (
 	"math/big"
 	"strings"
 	"sync"
+	"testing"
 
 	"golang.org/x/crypto/argon2"
 )
@@ -63,10 +64,30 @@ func DefaultPasswordConfig() *PasswordConfig {
 	}
 }
 
-// HashPassword generates an Argon2id hash of the password
+// testArgon2Memory is the Argon2id memory cost, in KiB, that a test
+// binary hashes with: 1 MB instead of the shipped 64 MB. Every test
+// that starts a database hashes the bootstrap admin password, dozens
+// of them run in parallel, and under the race detector each 64 MB hash
+// holds about 150 MB. VerifyPassword reads the cost from the hash it
+// checks, so verification follows.
+const testArgon2Memory = 1024
+
+// hashAtShippedCostInTest makes a test binary hash at the shipped
+// memory cost. Only TestHashPassword_ShippedParameters sets it.
+//
+//nolint:gochecknoglobals // set by one test, see above
+var hashAtShippedCostInTest bool
+
+// HashPassword generates an Argon2id hash of the password. A binary
+// built by go test hashes at testArgon2Memory; one built by go build
+// always hashes at the defaults.
 func HashPassword(password string) (string, error) {
 	config := DefaultPasswordConfig()
 
+	if testing.Testing() && !hashAtShippedCostInTest {
+		config.Memory = testArgon2Memory
+	}
+
 	// Generate a salt
 	salt := make([]byte, config.SaltLen)
 

internal/database/password_test.go

@@ -192,6 +192,39 @@ func TestHashPasswordUniqueness(t *testing.T) {
 	}
 }
 
+// TestHashPassword_ShippedParameters hashes and verifies through
+// HashPassword at the shipped Argon2id parameters. Every other test
+// hashes at the lower memory cost a test binary uses, so this is the
+// one that keeps production hashing covered. One hash and one
+// verification: each costs 64 MB.
+//
+//nolint:paralleltest // changes the hashing cost for the whole binary
+func TestHashPassword_ShippedParameters(t *testing.T) {
+	database.HashAtShippedCostForTest(t)
+
+	password := "correct horse battery staple"
+
+	hash, err := database.HashPassword(password)
+	if err != nil {
+		t.Fatalf("hashing with the shipped parameters: %v", err)
+	}
+
+	const shipped = "$argon2id$v=19$m=65536,t=1,p=4$"
+
+	if !strings.HasPrefix(hash, shipped) {
+		t.Errorf("hash = %q, want prefix %q", hash, shipped)
+	}
+
+	valid, err := database.VerifyPassword(password, hash)
+	if err != nil {
+		t.Fatalf("VerifyPassword() error = %v", err)
+	}
+
+	if !valid {
+		t.Error("VerifyPassword() returned false for correct password")
+	}
+}
+
 // TestVerifyDummyPassword_DoesRealWork covers the anti-enumeration
 // path. Login charges an unknown username a verification against a
 // dummy hash so that a nonexistent account is not answered in

internal/handlers/webhook_stats_test.go

@@ -122,12 +122,14 @@ type statsHistory struct {
 }
 
 // seedStatsHistory builds the webhook the statistics test checks: one
-// day of retention, two entrypoints (one inactive) and three targets
-// (one inactive). Three events arrive through the receiver, and so
-// each has a delivery to the two active targets. The oldest event is
-// past retention, the middle one six hours old, the newest just in.
-// Their deliveries are settled as the delivery engine would, and a
-// replay adds a pending delivery to the oldest event.
+// day of retention, five entrypoints (one inactive) and three targets
+// (one inactive). Six events arrive through the receiver, and so each
+// has a delivery to the two active targets. The oldest event is past
+// retention, the next six hours old, the other four just in. Five
+// deliveries are settled as the delivery engine would, the rest left
+// pending, and a replay adds a pending delivery to the oldest event.
+// Once retention has removed the oldest event, no two figures in the
+// pane's top row, or in the same table, are equal.
 func seedStatsHistory(
 	t *testing.T,
 	h *handlers.Handlers,
@@ -143,6 +145,10 @@ func seedStatsHistory(
 	require.NoError(t, db.DB().Omit(clause.Associations).Create(wh).Error)
 
 	path := statsEntrypoint(t, db, wh.ID, true)
+	for range 3 {
+		statsEntrypoint(t, db, wh.ID, true)
+	}
+
 	statsEntrypoint(t, db, wh.ID, false)
 
 	first := seedConfiguredTarget(
@@ -156,7 +162,7 @@ func seedStatsHistory(
 
 	router := receiverRouter(h)
 
-	for range 3 {
+	for range 6 {
 		require.Equal(t, http.StatusOK, postReceiver(t, router, path))
 	}
 
@@ -164,9 +170,9 @@ func seedStatsHistory(
 	require.NoError(t, err)
 
 	events := listEvents(t, webhookDB)
-	require.Len(t, events, 3)
+	require.Len(t, events, 6)
 
-	oldest, middle, newest := events[0], events[1], events[2]
+	oldest, middle, newest := events[0], events[1], events[5]
 	now := time.Now()
 
 	statsAge(t, webhookDB, oldest.ID, now.Add(-50*time.Hour))
@@ -253,17 +259,17 @@ func statsPane(t *testing.T, page string) string {
 
 // assertStatsTargets checks, for the history seedStatsHistory builds,
 // each target's totals and its deliveries finished in the last 24
-// hours. The first target has three deliveries and the replay, the
-// second three; the inactive target has none and so no row.
+// hours. The first target has six deliveries and the replay, the
+// second six; the inactive target has none and so no row.
 func assertStatsTargets(t *testing.T, hist statsHistory) {
 	t.Helper()
 
 	first, second := hist.first, hist.second
 
 	assert.Equal(t, map[string]database.TargetTotals{
-		first: {TargetID: first, Deliveries: 4, Delivered: 1, Failed: 2},
+		first: {TargetID: first, Deliveries: 7, Delivered: 1, Failed: 2},
 		second: {
-			TargetID: second, Deliveries: 3, Delivered: 1, Failed: 1,
+			TargetID: second, Deliveries: 6, Delivered: 1, Failed: 1,
 		},
 	}, statsTargetTotals(t, hist.webhookDB))
 
@@ -291,17 +297,17 @@ func assertStatsPaneAfterPrune(
 	pane := statsPane(t, renderSourceDetailPage(t, h, sess, hist.webhook.ID))
 	lastEvent := hist.newest.CreatedAt.Format("2006-01-02 15:04:05 UTC")
 
-	assert.Contains(t, pane, "Entrypoints 2 (1 active) "+
+	assert.Contains(t, pane, "Entrypoints 5 (4 active) "+
 		"Targets 3 (2 active) "+
-		"Deliveries in progress 1 "+
+		"Deliveries in progress 7 "+
 		"Last event "+lastEvent+" "+
 		"Retention 1 day")
 	assert.Contains(t, pane, "Lifetime Within retention "+
-		"Events 3 2 "+
-		"Deliveries 7 4 "+
+		"Events 6 5 "+
+		"Deliveries 13 10 "+
 		"Failures 3 2")
 	assert.Contains(t, pane, "Last 10 minutes Last 24 hours "+
-		"Events 1 2 "+
+		"Events 4 5 "+
 		"Failures 1 2 "+
 		"Failure percentage 50.0% 66.7%")
 }
@@ -332,21 +338,21 @@ func TestWebhookStats_EveryFigureAcrossRetentionPrune(t *testing.T) {
 	stats := h.WebhookStatsForTest(hist.webhook.ID)
 	require.NotNil(t, stats)
 
-	assert.Equal(t, 2, stats.Entrypoints)
-	assert.Equal(t, 1, stats.ActiveEntrypoints)
+	assert.Equal(t, 5, stats.Entrypoints)
+	assert.Equal(t, 4, stats.ActiveEntrypoints)
 	assert.Equal(t, 3, stats.Targets)
 	assert.Equal(t, 2, stats.ActiveTargets)
-	assert.Equal(t, handlers.Counts{Events: 3, Deliveries: 7, Failures: 3},
+	assert.Equal(t, handlers.Counts{Events: 6, Deliveries: 13, Failures: 3},
 		stats.Lifetime)
 	assert.Equal(t, stats.Lifetime, stats.WithinRetention)
-	assert.Equal(t, int64(2), stats.InProgress)
+	assert.Equal(t, int64(8), stats.InProgress)
 	require.NotNil(t, stats.LastEventAt)
 	assert.True(t, hist.newest.CreatedAt.Equal(*stats.LastEventAt))
 	assert.Equal(t, handlers.RecentWindow{
-		Events: 1, Delivered: 1, Failed: 1,
+		Events: 4, Delivered: 1, Failed: 1,
 	}, stats.Last10Minutes)
 	assert.Equal(t, handlers.RecentWindow{
-		Events: 2, Delivered: 1, Failed: 2,
+		Events: 5, Delivered: 1, Failed: 2,
 	}, stats.Last24Hours)
 	assert.Equal(t, "50.0%", stats.Last10Minutes.FailurePercent())
 	assert.Equal(t, "66.7%", stats.Last24Hours.FailurePercent())
@@ -362,20 +368,20 @@ func TestWebhookStats_EveryFigureAcrossRetentionPrune(t *testing.T) {
 	require.NotNil(t, after)
 
 	assert.Equal(t, stats.Lifetime, after.Lifetime)
-	assert.Equal(t, handlers.Counts{Events: 2, Deliveries: 4, Failures: 2},
+	assert.Equal(t, handlers.Counts{Events: 5, Deliveries: 10, Failures: 2},
 		after.WithinRetention)
-	assert.Equal(t, int64(1), after.InProgress)
+	assert.Equal(t, int64(7), after.InProgress)
 	assert.Equal(t, stats.LastEventAt, after.LastEventAt)
 	assert.Equal(t, stats.Last10Minutes, after.Last10Minutes)
 	assert.Equal(t, stats.Last24Hours, after.Last24Hours)
 
 	assert.Equal(t, map[string]database.TargetTotals{
 		first: {
-			TargetID: first, Deliveries: 4, Delivered: 1, Failed: 2,
+			TargetID: first, Deliveries: 7, Delivered: 1, Failed: 2,
 			DeliveriesRemoved: 2, FailedRemoved: 1,
 		},
 		second: {
-			TargetID: second, Deliveries: 3, Delivered: 1, Failed: 1,
+			TargetID: second, Deliveries: 6, Delivered: 1, Failed: 1,
 			DeliveriesRemoved: 1,
 		},
 	}, statsTargetTotals(t, hist.webhookDB))

internal/resetpw/resetpw_test.go

@@ -140,7 +140,7 @@ func (n *noopEvictor) EvictWebhook(string) {}
 // and the database, exactly as internal/handlers builds them.
 //
 // One application per test function, not per case: every start that
-// finds no account seeds one at 64 MB of Argon2id, and this package's
+// finds no account seeds one with an Argon2id hash, and this package's
 // budget is not the place to spend that repeatedly.
 func newServerApp(
 	t *testing.T, dir string,

script/test

@@ -22,6 +22,11 @@
 # The one figure above 90s is GOMAXPROCS 1, a synthetic core floor rather than
 # a condition CI runs under. If a CPU-limited runner ever puts a real run near
 # 67s, that is the datum to revisit the org figure with.
+#
+# -p 4 -parallel 8 keep the run under 2 GB of memory: at most four test
+# binaries build or run at once, each with at most eight parallel tests. Under
+# -race every test binary and every link costs a few hundred MB, so the
+# defaults (one per core) add up to several GB on a many-core host.
 set -eu
 
 ROOT="$(cd "$(dirname "$0")/.." && pwd -P)"
@@ -29,7 +34,7 @@ ROOT="$(cd "$(dirname "$0")/.." && pwd -P)"
 main() {
     cd "$ROOT"
     "$ROOT/script/assets"
-    go test -v -race -timeout 90s ./...
+    go test -v -race -p 4 -parallel 8 -timeout 90s ./...
 }
 
 main "$@"

templates/webhook_stats.html

@@ -30,32 +30,34 @@
         </div>
     </div>
     <div class="p-4 grid grid-cols-1 lg:grid-cols-2 gap-6 text-sm">
-        <table class="w-full text-center text-gray-900">
-            <thead>
-                <tr class="border-b border-gray-200 text-xs text-gray-500 uppercase tracking-wide">
-                    <th></th>
-                    <th class="py-2 font-medium">Lifetime</th>
-                    <th class="py-2 font-medium">Within retention</th>
-                </tr>
-            </thead>
-            <tbody>
-                <tr>
-                    <td class="py-2 text-left text-gray-600">Events</td>
-                    <td class="py-2">{{.Lifetime.Events}}</td>
-                    <td class="py-2">{{.WithinRetention.Events}}</td>
-                </tr>
-                <tr>
-                    <td class="py-2 text-left text-gray-600">Deliveries</td>
-                    <td class="py-2">{{.Lifetime.Deliveries}}</td>
-                    <td class="py-2">{{.WithinRetention.Deliveries}}</td>
-                </tr>
-                <tr>
-                    <td class="py-2 text-left text-gray-600">Failures</td>
-                    <td class="py-2">{{.Lifetime.Failures}}</td>
-                    <td class="py-2">{{.WithinRetention.Failures}}</td>
-                </tr>
-            </tbody>
-        </table>
+        <div>
+            <table class="w-full text-center text-gray-900">
+                <thead>
+                    <tr class="border-b border-gray-200 text-xs text-gray-500 uppercase tracking-wide">
+                        <th></th>
+                        <th class="py-2 font-medium">Lifetime</th>
+                        <th class="py-2 font-medium">Within retention</th>
+                    </tr>
+                </thead>
+                <tbody>
+                    <tr>
+                        <td class="py-2 text-left text-gray-600">Events</td>
+                        <td class="py-2">{{.Lifetime.Events}}</td>
+                        <td class="py-2">{{.WithinRetention.Events}}</td>
+                    </tr>
+                    <tr>
+                        <td class="py-2 text-left text-gray-600">Deliveries</td>
+                        <td class="py-2">{{.Lifetime.Deliveries}}</td>
+                        <td class="py-2">{{.WithinRetention.Deliveries}}</td>
+                    </tr>
+                    <tr>
+                        <td class="py-2 text-left text-gray-600">Failures</td>
+                        <td class="py-2">{{.Lifetime.Failures}}</td>
+                        <td class="py-2">{{.WithinRetention.Failures}}</td>
+                    </tr>
+                </tbody>
+            </table>
+        </div>
         <div>
             <table class="w-full text-center text-gray-900">
                 <thead>