Show each entrypoint's last event and event count on the webhook page (closes #393)

Each entrypoint in the webhook page's entrypoint list shows when the last event arrived on its URL (relative, with the full UTC time on hover), or "never" if none ever has, and how many events arrived on it within the webhook's retention period. The last event is kept per entrypoint in a new EntrypointTotals row in the event database, written in the transaction that stores the event, so retention leaves it in place. The count is one query per page, grouped by entrypoint, over a new events index on entrypoint_id, deleted_at, resubmitted_from_id and created_at. Resubmitted copies did not arrive on the URL and count in neither. Pre-1.0: the table and index go into the schema in place.

Model: opus-5-5

internal/database/event_tier_indexes_test.go

@@ -233,6 +233,43 @@ func TestResubmitCountUsesItsIndex(t *testing.T) {
 			"(resubmitted_from_id=? AND deleted_at=?)")
 }
 
+// TestEntrypointEventsUseTheirIndex does the same for the webhook
+// page's count, for each entrypoint, of the events that arrived on its
+// URL since the retention cutoff (addEntrypointEvents in the
+// handlers), which must come from the index alone. It passes 25
+// entrypoints, as TestResubmitCountUsesItsIndex passes 25 events.
+func TestEntrypointEventsUseTheirIndex(t *testing.T) {
+	t.Parallel()
+
+	mgr, lc := setupTestWebhookDBManager(t)
+	ctx := context.Background()
+	require.NoError(t, lc.Start(ctx))
+
+	defer func() { require.NoError(t, lc.Stop(ctx)) }()
+
+	db, err := mgr.GetDB(uuid.New().String())
+	require.NoError(t, err)
+
+	dry := db.Session(&gorm.Session{DryRun: true})
+
+	entrypoints := make([]string, 25)
+	for i := range entrypoints {
+		entrypoints[i] = uuid.New().String()
+	}
+
+	var rows []struct{ Events int }
+
+	assertPlanUses(t, db, dry.Model(&database.Event{}).
+		Select("entrypoint_id, count(*) AS events").
+		Where("entrypoint_id IN ? AND resubmitted_from_id IS NULL",
+			entrypoints).
+		Where("created_at >= ?", time.Now()).
+		Group("entrypoint_id").Find(&rows),
+		"COVERING INDEX idx_events_entrypoint_id "+
+			"(entrypoint_id=? AND deleted_at=? AND "+
+			"resubmitted_from_id=? AND created_at>?)")
+}
+
 // assertPlanUses asserts that SQLite's plan for a statement GORM built
 // in a dry run, run with the same SQL and arguments GORM would send,
 // names each of the given indexes.

internal/database/model_event.go

@@ -20,12 +20,15 @@ type Event struct {
 	// has no deleted_at condition and uses the index on created_at
 	// alone. The other tables keep the unindexed BaseModel created_at.
 	// DeletedAt is also the second column of the resubmitted_from_id
-	// index, for the reason DeliveryResult gives.
-	CreatedAt time.Time      `gorm:"index;index:idx_events_deleted_at_created_at,priority:2"                                           json:"createdAt"`
-	DeletedAt gorm.DeletedAt `gorm:"index:idx_events_deleted_at_created_at,priority:1;index:idx_events_resubmitted_from_id,priority:2" json:"deletedAt,omitzero"`
+	// index, for the reason DeliveryResult gives. The entrypoint_id
+	// index, for the webhook page's entrypoint list, has it second too,
+	// resubmitted_from_id third, and created_at last, which the list
+	// compares with a range.
+	CreatedAt time.Time      `gorm:"index;index:idx_events_deleted_at_created_at,priority:2;index:idx_events_entrypoint_id,priority:4"                                           json:"createdAt"`
+	DeletedAt gorm.DeletedAt `gorm:"index:idx_events_deleted_at_created_at,priority:1;index:idx_events_resubmitted_from_id,priority:2;index:idx_events_entrypoint_id,priority:2" json:"deletedAt,omitzero"`
 
-	WebhookID    string `gorm:"type:uuid;not null" json:"webhookId"`
-	EntrypointID string `gorm:"type:uuid;not null" json:"entrypointId"`
+	WebhookID    string `gorm:"type:uuid;not null"                                           json:"webhookId"`
+	EntrypointID string `gorm:"type:uuid;not null;index:idx_events_entrypoint_id,priority:1" json:"entrypointId"`
 
 	// Request data
 	Method      string `gorm:"not null"    json:"method"`
@@ -44,7 +47,7 @@ type Event struct {
 	// existed. It is not a foreign key: the source event can be
 	// reaped by retention while its copies remain, and the id is
 	// kept as the record of where the copy came from either way.
-	ResubmittedFromID *string `gorm:"type:uuid;index:idx_events_resubmitted_from_id,priority:1" json:"resubmittedFromId,omitempty"`
+	ResubmittedFromID *string `gorm:"type:uuid;index:idx_events_resubmitted_from_id,priority:1;index:idx_events_entrypoint_id,priority:3" json:"resubmittedFromId,omitempty"`
 
 	// Relations. No model marshals the record it belongs to, so
 	// Webhook and Entrypoint are left out of the JSON.

internal/database/model_totals.go

@@ -52,6 +52,21 @@ func (TargetTotals) TableName() string {
 	return "target_totals"
 }
 
+// EntrypointTotals is one row per entrypoint, created by the first
+// event that arrives on its URL: when the newest such event arrived,
+// which retention leaves as it is. A resubmitted copy did not arrive
+// on the URL and does not change it.
+type EntrypointTotals struct {
+	EntrypointID string `gorm:"type:uuid;primaryKey"`
+
+	LastEventAt time.Time `gorm:"not null"`
+}
+
+// TableName names the table AddEntrypointTotals updates.
+func (EntrypointTotals) TableName() string {
+	return "entrypoint_totals"
+}
+
 // AddEventTotals adds each count in add to the webhook's event totals,
 // and records add.LastEventAt as when the newest event arrived if it is
 // set. Call it on the transaction that writes or deletes the events it
@@ -97,3 +112,25 @@ func AddTargetTotals(tx *gorm.DB, add TargetTotals) error {
 
 	return nil
 }
+
+// AddEntrypointTotals records add.LastEventAt as when the newest event
+// arrived on the URL of the entrypoint add.EntrypointID names, creating
+// its row the first time. Call it on the transaction that stores the
+// event.
+func AddEntrypointTotals(tx *gorm.DB, add EntrypointTotals) error {
+	err := tx.Exec(
+		`INSERT INTO entrypoint_totals (entrypoint_id, last_event_at)
+		VALUES (?, ?)
+		ON CONFLICT (entrypoint_id) DO UPDATE SET
+			last_event_at = excluded.last_event_at`,
+		add.EntrypointID, add.LastEventAt,
+	).Error
+	if err != nil {
+		return fmt.Errorf(
+			"adding to totals of entrypoint %s: %w",
+			add.EntrypointID, err,
+		)
+	}
+
+	return nil
+}

internal/database/model_webhook.go

@@ -111,6 +111,13 @@ func (w *Webhook) RetainsForever() bool {
 	return retainsForever(w.RetentionDays)
 }
 
+// RetentionCutoff returns the time before which this webhook's events
+// have expired, as the reaper computes it, and false when the webhook
+// retains them forever.
+func (w *Webhook) RetentionCutoff(now time.Time) (time.Time, bool) {
+	return retentionCutoff(now, w.RetentionDays)
+}
+
 // RetentionLabel returns the webhook's retention policy as display
 // text, so that no template has to know about the sentinel value.
 func (w *Webhook) RetentionLabel() string {

internal/database/models.go

@@ -3,7 +3,7 @@ package database
 // Migrate runs database migrations for the main application database.
 // Only configuration-tier models are stored in the main database.
 // Event-tier models (Event, Delivery, DeliveryResult, EventTotals,
-// TargetTotals) live in
+// TargetTotals, EntrypointTotals) live in
 // per-webhook dedicated databases managed by WebhookDBManager.
 func (d *Database) Migrate() error {
 	return d.db.AutoMigrate(

internal/database/webhook_db_manager.go

@@ -49,7 +49,7 @@ var ErrSidecarNotRemoved = errors.New(
 // WebhookDBManager manages per-webhook SQLite database files
 // for event storage. Each webhook gets its own dedicated
 // database containing Events, Deliveries, DeliveryResults and the
-// running totals of them (EventTotals, TargetTotals).
+// running totals of them (EventTotals, TargetTotals, EntrypointTotals).
 // Database connections are opened lazily and cached.
 type WebhookDBManager struct {
 	dataDir string
@@ -381,7 +381,7 @@ func (m *WebhookDBManager) openDB(
 	// Run migrations for event-tier models only
 	err = db.AutoMigrate(
 		&Event{}, &Delivery{}, &DeliveryResult{},
-		&EventTotals{}, &TargetTotals{},
+		&EventTotals{}, &TargetTotals{}, &EntrypointTotals{},
 	)
 	if err != nil {
 		_ = sqlDB.Close()

internal/handlers/entrypoint_view.go

@@ -1,6 +1,11 @@
 package handlers
 
 import (
+	"fmt"
+	"time"
+
+	"github.com/dustin/go-humanize"
+	"gorm.io/gorm"
 	"sneak.berlin/go/webhooker/internal/database"
 )
 
@@ -11,6 +16,21 @@ type EntrypointView struct {
 	Path        string
 	Description string
 	Active      bool
+
+	// Events is how many events arrived on the entrypoint's URL within
+	// the webhook's retention period. LastEvent is when the newest
+	// event ever to arrive on it did, relative, and LastEventUTC the
+	// full time; both are empty when none ever did.
+	Events       int64
+	LastEvent    string
+	LastEventUTC string
+}
+
+// entrypointEvents is one entrypoint's count read by
+// addEntrypointEvents.
+type entrypointEvents struct {
+	EntrypointID string
+	Events       int64
 }
 
 // NewEntrypointViews projects entrypoints for rendering.
@@ -32,3 +52,60 @@ func NewEntrypointViews(
 
 	return views
 }
+
+// addEntrypointEvents fills in each view's event figures from the
+// webhook's event database: when the last event arrived on its URL,
+// from its EntrypointTotals row, and how many events arrived on it
+// since the webhook's retention cutoff, counted in one query over the
+// events' entrypoint_id index. Resubmitted copies did not arrive on
+// the URL and are left out of both.
+func addEntrypointEvents(
+	webhookDB *gorm.DB,
+	webhook *database.Webhook,
+	views []EntrypointView,
+	now time.Time,
+) error {
+	ids := make([]string, len(views))
+	byID := make(map[string]*EntrypointView, len(views))
+
+	for i := range views {
+		ids[i] = views[i].ID
+		byID[views[i].ID] = &views[i]
+	}
+
+	var totals []database.EntrypointTotals
+
+	err := webhookDB.Where("entrypoint_id IN ?", ids).Find(&totals).Error
+	if err != nil {
+		return fmt.Errorf("reading entrypoint totals: %w", err)
+	}
+
+	query := webhookDB.Model(&database.Event{}).
+		Select("entrypoint_id, count(*) AS events").
+		Where("entrypoint_id IN ? AND resubmitted_from_id IS NULL", ids)
+
+	cutoff, finite := webhook.RetentionCutoff(now)
+	if finite {
+		query = query.Where("created_at >= ?", cutoff)
+	}
+
+	var counts []entrypointEvents
+
+	err = query.Group("entrypoint_id").Find(&counts).Error
+	if err != nil {
+		return fmt.Errorf("counting events by entrypoint: %w", err)
+	}
+
+	for _, row := range totals {
+		view := byID[row.EntrypointID]
+		view.LastEvent = humanize.Time(row.LastEventAt)
+		view.LastEventUTC =
+			row.LastEventAt.UTC().Format(time.DateTime) + " UTC"
+	}
+
+	for _, row := range counts {
+		byID[row.EntrypointID].Events = row.Events
+	}
+
+	return nil
+}

internal/handlers/entrypoint_view_test.go

@@ -0,0 +1,197 @@
+package handlers_test
+
+import (
+	"net/http"
+	"strconv"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/google/uuid"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"gorm.io/gorm/clause"
+	"sneak.berlin/go/webhooker/internal/database"
+	"sneak.berlin/go/webhooker/internal/handlers"
+	"sneak.berlin/go/webhooker/internal/logger"
+	"sneak.berlin/go/webhooker/internal/session"
+)
+
+// entrypointRow returns the part of a rendered webhook page from an
+// entrypoint's URL to the next entrypoint's, which holds its figures.
+func entrypointRow(t *testing.T, page, entrypointID string) string {
+	t.Helper()
+
+	_, row, found := strings.Cut(page, `id="entrypoint-url-`+entrypointID+`"`)
+	require.True(t, found)
+
+	row, _, _ = strings.Cut(row, `id="entrypoint-url-`)
+
+	return row
+}
+
+// lastEventShown matches an entrypoint row's last event arriving at at.
+func lastEventShown(at time.Time) string {
+	return `Last Event:</span>\s*<span title="` +
+		at.UTC().Format(time.DateTime) + ` UTC">[^<]+</span>`
+}
+
+// eventsShown matches an entrypoint row's count of n events.
+func eventsShown(n int) string {
+	return `Events Within Retention:</span>\s*<span>` +
+		strconv.Itoa(n) + `</span>`
+}
+
+// TestHandleSourceDetail_ShowsEntrypointEvents proves each entrypoint
+// on the webhook page shows its own figures: how many events arrived
+// through it within the webhook's retention period, leaving out one
+// older than that, and when the newest arrived, or "never" for an
+// entrypoint with none.
+func TestHandleSourceDetail_ShowsEntrypointEvents(t *testing.T) {
+	t.Parallel()
+
+	var (
+		h     *handlers.Handlers
+		sess  *session.Session
+		db    *database.Database
+		dbMgr *database.WebhookDBManager
+	)
+
+	app := newTestApp(t, &h, &sess, &db, &dbMgr)
+	app.RequireStart()
+
+	t.Cleanup(app.RequireStop)
+
+	wh := &database.Webhook{
+		UserID: deleteTestUserID, Name: "figures", RetentionDays: 7,
+	}
+	require.NoError(t, db.DB().Omit(clause.Associations).Create(wh).Error)
+
+	webhookDB, err := dbMgr.GetDB(wh.ID)
+	require.NoError(t, err)
+
+	entrypoint := func() *database.Entrypoint {
+		ep := &database.Entrypoint{
+			WebhookID: wh.ID, Path: uuid.New().String(), Active: true,
+		}
+		require.NoError(t,
+			db.DB().Omit(clause.Associations).Create(ep).Error)
+
+		return ep
+	}
+
+	// event stores an event that arrived on ep's URL age ago and
+	// records it as ep's last event, as the receiver does.
+	event := func(ep *database.Entrypoint, age time.Duration) time.Time {
+		e := &database.Event{
+			WebhookID:    wh.ID,
+			EntrypointID: ep.ID,
+			Method:       http.MethodPost,
+		}
+		e.CreatedAt = time.Now().Add(-age)
+		require.NoError(t,
+			webhookDB.Omit(clause.Associations).Create(e).Error)
+		require.NoError(t, database.AddEntrypointTotals(webhookDB,
+			database.EntrypointTotals{
+				EntrypointID: ep.ID, LastEventAt: e.CreatedAt,
+			}))
+
+		return e.CreatedAt
+	}
+
+	busy, quiet, unused := entrypoint(), entrypoint(), entrypoint()
+
+	event(busy, 8*24*time.Hour) // older than the 7 days kept
+	event(busy, 3*time.Hour)
+	busyLast := event(busy, time.Hour)
+	quietLast := event(quiet, 2*24*time.Hour)
+
+	body := renderSourceDetailPage(t, h, sess, wh.ID)
+
+	assert.Regexp(t, lastEventShown(busyLast), entrypointRow(t, body, busy.ID))
+	assert.Regexp(t, eventsShown(2), entrypointRow(t, body, busy.ID))
+	assert.Regexp(t, lastEventShown(quietLast), entrypointRow(t, body, quiet.ID))
+	assert.Regexp(t, eventsShown(1), entrypointRow(t, body, quiet.ID))
+	assert.Regexp(t, `Last Event:</span>\s*<span>never</span>`,
+		entrypointRow(t, body, unused.ID))
+	assert.Regexp(t, eventsShown(0), entrypointRow(t, body, unused.ID))
+}
+
+// TestHandleSourceDetail_EntrypointLastEventSurvivesRetention checks
+// that once retention has removed every event that arrived on an
+// entrypoint's URL, the entrypoint still shows when the last one
+// arrived rather than "never".
+func TestHandleSourceDetail_EntrypointLastEventSurvivesRetention(
+	t *testing.T,
+) {
+	t.Parallel()
+
+	var (
+		h     *handlers.Handlers
+		sess  *session.Session
+		db    *database.Database
+		dbMgr *database.WebhookDBManager
+		log   *logger.Logger
+	)
+
+	app := newTestApp(t, &h, &sess, &db, &dbMgr, &log)
+	app.RequireStart()
+
+	t.Cleanup(app.RequireStop)
+
+	wh := &database.Webhook{
+		UserID: deleteTestUserID, Name: "swept", RetentionDays: 1,
+	}
+	require.NoError(t, db.DB().Omit(clause.Associations).Create(wh).Error)
+
+	ep := seedEntrypoint(t, db, wh.ID)
+	webhookDB, events := receiveEvents(t, h, dbMgr, wh.ID, ep.Path, 1)
+	arrived := events[0].CreatedAt
+
+	statsAge(t, webhookDB, events[0].ID, time.Now().Add(-50*time.Hour))
+	statsPrune(t, db, dbMgr, log, webhookDB)
+	require.Empty(t, listEvents(t, webhookDB))
+
+	row := entrypointRow(t, renderSourceDetailPage(t, h, sess, wh.ID), ep.ID)
+	assert.Regexp(t, lastEventShown(arrived), row)
+	assert.Regexp(t, eventsShown(0), row)
+}
+
+// TestHandleSourceDetail_ResubmitLeavesEntrypointFigures checks that a
+// resubmitted copy, which did not arrive on the entrypoint's URL,
+// changes neither the entrypoint's last event nor its count.
+func TestHandleSourceDetail_ResubmitLeavesEntrypointFigures(
+	t *testing.T,
+) {
+	t.Parallel()
+
+	var (
+		h     *handlers.Handlers
+		sess  *session.Session
+		db    *database.Database
+		dbMgr *database.WebhookDBManager
+	)
+
+	app := newTestApp(t, &h, &sess, &db, &dbMgr)
+	app.RequireStart()
+
+	t.Cleanup(app.RequireStop)
+
+	wh := seedWebhook(t, db)
+	ep := seedEntrypoint(t, db, wh.ID)
+	webhookDB, events := receiveEvents(t, h, dbMgr, wh.ID, ep.Path, 1)
+	arrived := events[0].CreatedAt
+
+	require.Equal(t, http.StatusSeeOther,
+		postResubmit(t, h, sess, wh.ID, events[0].ID).Code)
+	require.Len(t, listEvents(t, webhookDB), 2)
+
+	var totals database.EntrypointTotals
+
+	require.NoError(t, webhookDB.Take(&totals).Error)
+	assert.True(t, arrived.Equal(totals.LastEventAt))
+
+	row := entrypointRow(t, renderSourceDetailPage(t, h, sess, wh.ID), ep.ID)
+	assert.Regexp(t, lastEventShown(arrived), row)
+	assert.Regexp(t, eventsShown(1), row)
+}

internal/handlers/source_management.go

@@ -485,6 +485,8 @@ func (h *Handlers) renderSourceDetail(
 		"webhook_id = ?", webhook.ID,
 	).Find(&targets)
 
+	entrypointViews := NewEntrypointViews(entrypoints)
+
 	var events []RecentEventView
 
 	if h.dbMgr.DBExists(webhook.ID) {
@@ -503,6 +505,15 @@ func (h *Handlers) renderSourceDetail(
 
 			return
 		}
+
+		err = addEntrypointEvents(
+			webhookDB, &webhook, entrypointViews, time.Now(),
+		)
+		if err != nil {
+			h.serverError(w, r, "failed to count entrypoint events", err)
+
+			return
+		}
 	}
 
 	scheme := "http"
@@ -523,7 +534,7 @@ func (h *Handlers) renderSourceDetail(
 		// Targets are projected to a display-safe view: a
 		// target's stored config blob holds a credential, and it
 		// must never reach a template.
-		"Entrypoints": NewEntrypointViews(entrypoints),
+		"Entrypoints": entrypointViews,
 		"Targets":     h.targetRows(&webhook, targets),
 		"Events":      events,
 		"BaseURL":     baseURL,

internal/handlers/webhook.go

@@ -326,6 +326,19 @@ func (h *Handlers) createAndFanOut(
 		return nil, nil, err
 	}
 
+	// A resubmitted copy did not arrive on its entrypoint's URL, so it
+	// leaves the entrypoint's last event as it is.
+	if src.ResubmittedFromID == nil {
+		err = database.AddEntrypointTotals(tx, database.EntrypointTotals{
+			EntrypointID: event.EntrypointID, LastEventAt: event.CreatedAt,
+		})
+		if err != nil {
+			tx.Rollback()
+
+			return nil, nil, err
+		}
+	}
+
 	err = tx.Commit().Error
 	if err != nil {
 		return nil, nil, fmt.Errorf(