Show each entrypoint's last event and event count on the webhook page (closes #393)

Each entrypoint in the webhook page's entrypoint list now shows when its last event arrived (relative, with the full UTC time on hover), or "never", and how many events arrived through it within the webhook's retention period. Both come from one query per page, grouped by entrypoint, over a new events index on entrypoint_id, deleted_at and created_at, so the page reads only the index entries it counts. Pre-1.0: the index goes into the schema in place. A test checks the database's plan for the statement as the code builds it; another shows two entrypoints with different traffic each with their own figures, an event older than retention left out, and an unused entrypoint reading "never".

Model: opus-5-5

README.md

@@ -1639,6 +1639,10 @@ different event sources that all feed into the same processing pipeline
 (e.g., one entrypoint for GitHub, another for Stripe, both routing to
 the same targets).
 
+The webhook page shows, for each entrypoint, how many events arrived
+through it within the webhook's retention period and when the newest of
+them arrived, or "never".
+
 #### Target
 
 A delivery destination for events. Each target defines where and how
@@ -1903,6 +1907,7 @@ tags, so `AutoMigrate` creates them on a fresh database:
 | `delivery_results` | `delivery_id`, `deleted_at` | The event log, which loads the attempts of a page's deliveries, and retention, which deletes the attempts of expired events |
 | `events`           | `deleted_at`, `created_at`  | The webhook page's statistics, which count recent events |
 | `events`           | `resubmitted_from_id`, `deleted_at` | The event log, which counts the events resubmitted from each event on a page |
+| `events`           | `entrypoint_id`, `deleted_at`, `created_at` | The webhook page's entrypoint list, which counts each entrypoint's events within the retention period and reads when the newest arrived |
 | `events`           | `created_at`                | Retention, which selects expired events by age |
 
 GORM's soft delete adds `deleted_at IS NULL` to these queries; retention
@@ -1912,7 +1917,9 @@ a column matched against several values or compared with a range. So every
 index but the last also covers `deleted_at`. It comes second in the `event_id`
 and `delivery_id` indexes, so that retention can use them without it. The event
 log's count, the one query on the `resubmitted_from_id` index, always carries
-`deleted_at IS NULL` and uses both columns. In the statistics' `events` index
+`deleted_at IS NULL` and uses both columns. The entrypoint list's query, the one
+on the `entrypoint_id` index, uses all three, with `created_at` last because it
+compares it with a range (`>=`). In the statistics' `events` index
 `deleted_at` comes first, because they compare `created_at` with a range (`>=`)
 and SQLite narrows by a range only on the last column it uses.
 

internal/database/event_tier_indexes_test.go

@@ -233,6 +233,42 @@ func TestResubmitCountUsesItsIndex(t *testing.T) {
 			"(resubmitted_from_id=? AND deleted_at=?)")
 }
 
+// TestEntrypointEventsUseTheirIndex does the same for the webhook
+// page's figures for each entrypoint, the events since the retention
+// cutoff and when the newest arrived (addEntrypointEvents in the
+// handlers), which must come from the index alone. It passes 25
+// entrypoints, as TestResubmitCountUsesItsIndex passes 25 events.
+func TestEntrypointEventsUseTheirIndex(t *testing.T) {
+	t.Parallel()
+
+	mgr, lc := setupTestWebhookDBManager(t)
+	ctx := context.Background()
+	require.NoError(t, lc.Start(ctx))
+
+	defer func() { require.NoError(t, lc.Stop(ctx)) }()
+
+	db, err := mgr.GetDB(uuid.New().String())
+	require.NoError(t, err)
+
+	dry := db.Session(&gorm.Session{DryRun: true})
+
+	entrypoints := make([]string, 25)
+	for i := range entrypoints {
+		entrypoints[i] = uuid.New().String()
+	}
+
+	var rows []struct{ Events int }
+
+	assertPlanUses(t, db, dry.Model(&database.Event{}).
+		Select("entrypoint_id, count(*) AS events, "+
+			"max(created_at), created_at AS last_event_at").
+		Where("entrypoint_id IN ?", entrypoints).
+		Where("created_at >= ?", time.Now()).
+		Group("entrypoint_id").Find(&rows),
+		"COVERING INDEX idx_events_entrypoint_id "+
+			"(entrypoint_id=? AND deleted_at=? AND created_at>?)")
+}
+
 // assertPlanUses asserts that SQLite's plan for a statement GORM built
 // in a dry run, run with the same SQL and arguments GORM would send,
 // names each of the given indexes.

internal/database/model_event.go

@@ -20,12 +20,14 @@ type Event struct {
 	// has no deleted_at condition and uses the index on created_at
 	// alone. The other tables keep the unindexed BaseModel created_at.
 	// DeletedAt is also the second column of the resubmitted_from_id
-	// index, for the reason DeliveryResult gives.
+	// index, for the reason DeliveryResult gives. The entrypoint_id
-	CreatedAt time.Time      `gorm:"index;index:idx_events_deleted_at_created_at,priority:2"                                           json:"createdAt"`
+	// index, for the webhook page's entrypoint list, has it second too,
-	DeletedAt gorm.DeletedAt `gorm:"index:idx_events_deleted_at_created_at,priority:1;index:idx_events_resubmitted_from_id,priority:2" json:"deletedAt,omitzero"`
+	// and created_at last, which the list compares with a range.
+	CreatedAt time.Time      `gorm:"index;index:idx_events_deleted_at_created_at,priority:2;index:idx_events_entrypoint_id,priority:3"                                           json:"createdAt"`
+	DeletedAt gorm.DeletedAt `gorm:"index:idx_events_deleted_at_created_at,priority:1;index:idx_events_resubmitted_from_id,priority:2;index:idx_events_entrypoint_id,priority:2" json:"deletedAt,omitzero"`
 
 	WebhookID    string `gorm:"type:uuid;not null"                                           json:"webhookId"`
-	EntrypointID string `gorm:"type:uuid;not null" json:"entrypointId"`
+	EntrypointID string `gorm:"type:uuid;not null;index:idx_events_entrypoint_id,priority:1" json:"entrypointId"`
 
 	// Request data
 	Method      string `gorm:"not null"    json:"method"`

internal/database/model_webhook.go

@@ -111,6 +111,13 @@ func (w *Webhook) RetainsForever() bool {
 	return retainsForever(w.RetentionDays)
 }
 
+// RetentionCutoff returns the time before which this webhook's events
+// have expired, as the reaper computes it, and false when the webhook
+// retains them forever.
+func (w *Webhook) RetentionCutoff(now time.Time) (time.Time, bool) {
+	return retentionCutoff(now, w.RetentionDays)
+}
+
 // RetentionLabel returns the webhook's retention policy as display
 // text, so that no template has to know about the sentinel value.
 func (w *Webhook) RetentionLabel() string {

internal/handlers/entrypoint_view.go

@@ -1,6 +1,11 @@
 package handlers
 
 import (
+	"fmt"
+	"time"
+
+	"github.com/dustin/go-humanize"
+	"gorm.io/gorm"
 	"sneak.berlin/go/webhooker/internal/database"
 )
 
@@ -11,6 +16,21 @@ type EntrypointView struct {
 	Path        string
 	Description string
 	Active      bool
+
+	// Events is how many events arrived through the entrypoint within
+	// the webhook's retention period. LastEvent is when the newest of
+	// them arrived, relative, and LastEventUTC the full time; both are
+	// empty when none did.
+	Events       int64
+	LastEvent    string
+	LastEventUTC string
+}
+
+// entrypointEvents is one entrypoint's row read by addEntrypointEvents.
+type entrypointEvents struct {
+	EntrypointID string
+	Events       int64
+	LastEventAt  time.Time
 }
 
 // NewEntrypointViews projects entrypoints for rendering.
@@ -32,3 +52,59 @@ func NewEntrypointViews(
 
 	return views
 }
+
+// addEntrypointEvents fills in each view's event figures from the
+// webhook's event database, in one query over the events'
+// entrypoint_id index. Only events received since the webhook's
+// retention cutoff are read.
+func addEntrypointEvents(
+	webhookDB *gorm.DB,
+	webhook *database.Webhook,
+	views []EntrypointView,
+	now time.Time,
+) error {
+	ids := make([]string, len(views))
+	for i := range views {
+		ids[i] = views[i].ID
+	}
+
+	// max(created_at) picks each entrypoint's newest row, and SQLite
+	// takes the bare created_at beside it from that row. Unlike max(),
+	// the plain column keeps its datetime type, so the driver returns
+	// it as a time.
+	query := webhookDB.Model(&database.Event{}).
+		Select("entrypoint_id, count(*) AS events, "+
+			"max(created_at), created_at AS last_event_at").
+		Where("entrypoint_id IN ?", ids)
+
+	cutoff, finite := webhook.RetentionCutoff(now)
+	if finite {
+		query = query.Where("created_at >= ?", cutoff)
+	}
+
+	var rows []entrypointEvents
+
+	err := query.Group("entrypoint_id").Find(&rows).Error
+	if err != nil {
+		return fmt.Errorf("counting events by entrypoint: %w", err)
+	}
+
+	byID := make(map[string]entrypointEvents, len(rows))
+	for _, row := range rows {
+		byID[row.EntrypointID] = row
+	}
+
+	for i := range views {
+		row, found := byID[views[i].ID]
+		if !found {
+			continue
+		}
+
+		views[i].Events = row.Events
+		views[i].LastEvent = humanize.Time(row.LastEventAt)
+		views[i].LastEventUTC =
+			row.LastEventAt.UTC().Format(time.DateTime) + " UTC"
+	}
+
+	return nil
+}

internal/handlers/entrypoint_view_test.go

@@ -0,0 +1,106 @@
+package handlers_test
+
+import (
+	"net/http"
+	"strconv"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/google/uuid"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"gorm.io/gorm/clause"
+	"sneak.berlin/go/webhooker/internal/database"
+	"sneak.berlin/go/webhooker/internal/handlers"
+	"sneak.berlin/go/webhooker/internal/session"
+)
+
+// TestHandleSourceDetail_ShowsEntrypointEvents proves each entrypoint
+// on the webhook page shows its own figures: how many events arrived
+// through it within the webhook's retention period, leaving out one
+// older than that, and when the newest arrived, or "never" for an
+// entrypoint with none.
+func TestHandleSourceDetail_ShowsEntrypointEvents(t *testing.T) {
+	t.Parallel()
+
+	var (
+		h     *handlers.Handlers
+		sess  *session.Session
+		db    *database.Database
+		dbMgr *database.WebhookDBManager
+	)
+
+	app := newTestApp(t, &h, &sess, &db, &dbMgr)
+	app.RequireStart()
+
+	t.Cleanup(app.RequireStop)
+
+	wh := &database.Webhook{
+		UserID: deleteTestUserID, Name: "figures", RetentionDays: 7,
+	}
+	require.NoError(t, db.DB().Omit(clause.Associations).Create(wh).Error)
+
+	webhookDB, err := dbMgr.GetDB(wh.ID)
+	require.NoError(t, err)
+
+	entrypoint := func() *database.Entrypoint {
+		ep := &database.Entrypoint{
+			WebhookID: wh.ID, Path: uuid.New().String(), Active: true,
+		}
+		require.NoError(t,
+			db.DB().Omit(clause.Associations).Create(ep).Error)
+
+		return ep
+	}
+
+	event := func(ep *database.Entrypoint, age time.Duration) time.Time {
+		e := &database.Event{
+			WebhookID:    wh.ID,
+			EntrypointID: ep.ID,
+			Method:       http.MethodPost,
+		}
+		e.CreatedAt = time.Now().Add(-age)
+		require.NoError(t,
+			webhookDB.Omit(clause.Associations).Create(e).Error)
+
+		return e.CreatedAt
+	}
+
+	busy, quiet, unused := entrypoint(), entrypoint(), entrypoint()
+
+	event(busy, 8*24*time.Hour) // older than the 7 days kept
+	event(busy, 3*time.Hour)
+	busyLast := event(busy, time.Hour)
+	quietLast := event(quiet, 2*24*time.Hour)
+
+	body := renderSourceDetailPage(t, h, sess, wh.ID)
+
+	// rowOf returns the part of the page from an entrypoint's URL to
+	// the next entrypoint's, which holds its figures.
+	rowOf := func(ep *database.Entrypoint) string {
+		_, row, found := strings.Cut(body, `id="entrypoint-url-`+ep.ID+`"`)
+		require.True(t, found)
+
+		row, _, _ = strings.Cut(row, `id="entrypoint-url-`)
+
+		return row
+	}
+
+	lastEvent := func(at time.Time) string {
+		return `Last Event:</span>\s*<span title="` +
+			at.UTC().Format(time.DateTime) + ` UTC">[^<]+ ago</span>`
+	}
+
+	count := func(n int) string {
+		return `Events Within Retention:</span>\s*<span>` +
+			strconv.Itoa(n) + `</span>`
+	}
+
+	assert.Regexp(t, lastEvent(busyLast), rowOf(busy))
+	assert.Regexp(t, count(2), rowOf(busy))
+	assert.Regexp(t, lastEvent(quietLast), rowOf(quiet))
+	assert.Regexp(t, count(1), rowOf(quiet))
+	assert.Regexp(t, `Last Event:</span>\s*<span>never</span>`, rowOf(unused))
+	assert.Regexp(t, count(0), rowOf(unused))
+}

internal/handlers/source_management.go

@@ -482,6 +482,8 @@ func (h *Handlers) renderSourceDetail(
 		"webhook_id = ?", webhook.ID,
 	).Find(&targets)
 
+	entrypointViews := NewEntrypointViews(entrypoints)
+
 	var events []RecentEventView
 
 	if h.dbMgr.DBExists(webhook.ID) {
@@ -500,6 +502,15 @@ func (h *Handlers) renderSourceDetail(
 
 			return
 		}
+
+		err = addEntrypointEvents(
+			webhookDB, &webhook, entrypointViews, time.Now(),
+		)
+		if err != nil {
+			h.serverError(w, r, "failed to count entrypoint events", err)
+
+			return
+		}
 	}
 
 	scheme := "http"
@@ -520,7 +531,7 @@ func (h *Handlers) renderSourceDetail(
 		// Targets are projected to a display-safe view: a
 		// target's stored config blob holds a credential, and it
 		// must never reach a template.
-		"Entrypoints": NewEntrypointViews(entrypoints),
+		"Entrypoints": entrypointViews,
 		"Targets":     h.targetRows(&webhook, targets),
 		"Events":      events,
 		"BaseURL":     baseURL,

templates/source_detail.html

@@ -97,6 +97,18 @@
                     </div>
                     <!-- The URL above is the entrypoint's credential:
                          anyone holding it can submit events. -->
+                    <div class="text-xs text-gray-500 mt-1">
+                        <span class="font-medium text-gray-700">Last Event:</span>
+                        {{if .LastEvent}}
+                        <span title="{{.LastEventUTC}}">{{.LastEvent}}</span>
+                        {{else}}
+                        <span>never</span>
+                        {{end}}
+                    </div>
+                    <div class="text-xs text-gray-500 mt-1">
+                        <span class="font-medium text-gray-700">Events Within Retention:</span>
+                        <span>{{.Events}}</span>
+                    </div>
                 </div>
                 {{else}}
                 <div class="p-4 text-sm text-gray-500">No entrypoints configured.</div>