diff --git a/README.md b/README.md index 3d0c5f8..d5b461f 100644 --- a/README.md +++ b/README.md @@ -1639,6 +1639,10 @@ different event sources that all feed into the same processing pipeline (e.g., one entrypoint for GitHub, another for Stripe, both routing to the same targets). +The webhook page shows, for each entrypoint, how many events arrived +through it within the webhook's retention period and when the newest of +them arrived, or "never". + #### Target A delivery destination for events. Each target defines where and how @@ -1903,6 +1907,7 @@ tags, so `AutoMigrate` creates them on a fresh database: | `delivery_results` | `delivery_id`, `deleted_at` | The event log, which loads the attempts of a page's deliveries, and retention, which deletes the attempts of expired events | | `events` | `deleted_at`, `created_at` | The webhook page's statistics, which count recent events | | `events` | `resubmitted_from_id`, `deleted_at` | The event log, which counts the events resubmitted from each event on a page | +| `events` | `entrypoint_id`, `deleted_at`, `created_at` | The webhook page's entrypoint list, which counts each entrypoint's events within the retention period and reads when the newest arrived | | `events` | `created_at` | Retention, which selects expired events by age | GORM's soft delete adds `deleted_at IS NULL` to these queries; retention @@ -1912,7 +1917,9 @@ a column matched against several values or compared with a range. So every index but the last also covers `deleted_at`. It comes second in the `event_id` and `delivery_id` indexes, so that retention can use them without it. The event log's count, the one query on the `resubmitted_from_id` index, always carries -`deleted_at IS NULL` and uses both columns. In the statistics' `events` index +`deleted_at IS NULL` and uses both columns. The entrypoint list's query, the one +on the `entrypoint_id` index, uses all three, with `created_at` last because it +compares it with a range (`>=`). In the statistics' `events` index `deleted_at` comes first, because they compare `created_at` with a range (`>=`) and SQLite narrows by a range only on the last column it uses. diff --git a/internal/database/event_tier_indexes_test.go b/internal/database/event_tier_indexes_test.go index 143c52f..687bcb9 100644 --- a/internal/database/event_tier_indexes_test.go +++ b/internal/database/event_tier_indexes_test.go @@ -233,6 +233,42 @@ func TestResubmitCountUsesItsIndex(t *testing.T) { "(resubmitted_from_id=? AND deleted_at=?)") } +// TestEntrypointEventsUseTheirIndex does the same for the webhook +// page's figures for each entrypoint, the events since the retention +// cutoff and when the newest arrived (addEntrypointEvents in the +// handlers), which must come from the index alone. It passes 25 +// entrypoints, as TestResubmitCountUsesItsIndex passes 25 events. +func TestEntrypointEventsUseTheirIndex(t *testing.T) { + t.Parallel() + + mgr, lc := setupTestWebhookDBManager(t) + ctx := context.Background() + require.NoError(t, lc.Start(ctx)) + + defer func() { require.NoError(t, lc.Stop(ctx)) }() + + db, err := mgr.GetDB(uuid.New().String()) + require.NoError(t, err) + + dry := db.Session(&gorm.Session{DryRun: true}) + + entrypoints := make([]string, 25) + for i := range entrypoints { + entrypoints[i] = uuid.New().String() + } + + var rows []struct{ Events int } + + assertPlanUses(t, db, dry.Model(&database.Event{}). + Select("entrypoint_id, count(*) AS events, "+ + "max(created_at), created_at AS last_event_at"). + Where("entrypoint_id IN ?", entrypoints). + Where("created_at >= ?", time.Now()). + Group("entrypoint_id").Find(&rows), + "COVERING INDEX idx_events_entrypoint_id "+ + "(entrypoint_id=? AND deleted_at=? AND created_at>?)") +} + // assertPlanUses asserts that SQLite's plan for a statement GORM built // in a dry run, run with the same SQL and arguments GORM would send, // names each of the given indexes. diff --git a/internal/database/model_event.go b/internal/database/model_event.go index d990de9..b607500 100644 --- a/internal/database/model_event.go +++ b/internal/database/model_event.go @@ -20,12 +20,14 @@ type Event struct { // has no deleted_at condition and uses the index on created_at // alone. The other tables keep the unindexed BaseModel created_at. // DeletedAt is also the second column of the resubmitted_from_id - // index, for the reason DeliveryResult gives. - CreatedAt time.Time `gorm:"index;index:idx_events_deleted_at_created_at,priority:2" json:"createdAt"` - DeletedAt gorm.DeletedAt `gorm:"index:idx_events_deleted_at_created_at,priority:1;index:idx_events_resubmitted_from_id,priority:2" json:"deletedAt,omitzero"` + // index, for the reason DeliveryResult gives. The entrypoint_id + // index, for the webhook page's entrypoint list, has it second too, + // and created_at last, which the list compares with a range. + CreatedAt time.Time `gorm:"index;index:idx_events_deleted_at_created_at,priority:2;index:idx_events_entrypoint_id,priority:3" json:"createdAt"` + DeletedAt gorm.DeletedAt `gorm:"index:idx_events_deleted_at_created_at,priority:1;index:idx_events_resubmitted_from_id,priority:2;index:idx_events_entrypoint_id,priority:2" json:"deletedAt,omitzero"` - WebhookID string `gorm:"type:uuid;not null" json:"webhookId"` - EntrypointID string `gorm:"type:uuid;not null" json:"entrypointId"` + WebhookID string `gorm:"type:uuid;not null" json:"webhookId"` + EntrypointID string `gorm:"type:uuid;not null;index:idx_events_entrypoint_id,priority:1" json:"entrypointId"` // Request data Method string `gorm:"not null" json:"method"` diff --git a/internal/database/model_webhook.go b/internal/database/model_webhook.go index 896579d..c6835e1 100644 --- a/internal/database/model_webhook.go +++ b/internal/database/model_webhook.go @@ -111,6 +111,13 @@ func (w *Webhook) RetainsForever() bool { return retainsForever(w.RetentionDays) } +// RetentionCutoff returns the time before which this webhook's events +// have expired, as the reaper computes it, and false when the webhook +// retains them forever. +func (w *Webhook) RetentionCutoff(now time.Time) (time.Time, bool) { + return retentionCutoff(now, w.RetentionDays) +} + // RetentionLabel returns the webhook's retention policy as display // text, so that no template has to know about the sentinel value. func (w *Webhook) RetentionLabel() string { diff --git a/internal/handlers/entrypoint_view.go b/internal/handlers/entrypoint_view.go index c679b3f..062e8ae 100644 --- a/internal/handlers/entrypoint_view.go +++ b/internal/handlers/entrypoint_view.go @@ -1,6 +1,11 @@ package handlers import ( + "fmt" + "time" + + "github.com/dustin/go-humanize" + "gorm.io/gorm" "sneak.berlin/go/webhooker/internal/database" ) @@ -11,6 +16,21 @@ type EntrypointView struct { Path string Description string Active bool + + // Events is how many events arrived through the entrypoint within + // the webhook's retention period. LastEvent is when the newest of + // them arrived, relative, and LastEventUTC the full time; both are + // empty when none did. + Events int64 + LastEvent string + LastEventUTC string +} + +// entrypointEvents is one entrypoint's row read by addEntrypointEvents. +type entrypointEvents struct { + EntrypointID string + Events int64 + LastEventAt time.Time } // NewEntrypointViews projects entrypoints for rendering. @@ -32,3 +52,59 @@ func NewEntrypointViews( return views } + +// addEntrypointEvents fills in each view's event figures from the +// webhook's event database, in one query over the events' +// entrypoint_id index. Only events received since the webhook's +// retention cutoff are read. +func addEntrypointEvents( + webhookDB *gorm.DB, + webhook *database.Webhook, + views []EntrypointView, + now time.Time, +) error { + ids := make([]string, len(views)) + for i := range views { + ids[i] = views[i].ID + } + + // max(created_at) picks each entrypoint's newest row, and SQLite + // takes the bare created_at beside it from that row. Unlike max(), + // the plain column keeps its datetime type, so the driver returns + // it as a time. + query := webhookDB.Model(&database.Event{}). + Select("entrypoint_id, count(*) AS events, "+ + "max(created_at), created_at AS last_event_at"). + Where("entrypoint_id IN ?", ids) + + cutoff, finite := webhook.RetentionCutoff(now) + if finite { + query = query.Where("created_at >= ?", cutoff) + } + + var rows []entrypointEvents + + err := query.Group("entrypoint_id").Find(&rows).Error + if err != nil { + return fmt.Errorf("counting events by entrypoint: %w", err) + } + + byID := make(map[string]entrypointEvents, len(rows)) + for _, row := range rows { + byID[row.EntrypointID] = row + } + + for i := range views { + row, found := byID[views[i].ID] + if !found { + continue + } + + views[i].Events = row.Events + views[i].LastEvent = humanize.Time(row.LastEventAt) + views[i].LastEventUTC = + row.LastEventAt.UTC().Format(time.DateTime) + " UTC" + } + + return nil +} diff --git a/internal/handlers/entrypoint_view_test.go b/internal/handlers/entrypoint_view_test.go new file mode 100644 index 0000000..f96939e --- /dev/null +++ b/internal/handlers/entrypoint_view_test.go @@ -0,0 +1,106 @@ +package handlers_test + +import ( + "net/http" + "strconv" + "strings" + "testing" + "time" + + "github.com/google/uuid" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "gorm.io/gorm/clause" + "sneak.berlin/go/webhooker/internal/database" + "sneak.berlin/go/webhooker/internal/handlers" + "sneak.berlin/go/webhooker/internal/session" +) + +// TestHandleSourceDetail_ShowsEntrypointEvents proves each entrypoint +// on the webhook page shows its own figures: how many events arrived +// through it within the webhook's retention period, leaving out one +// older than that, and when the newest arrived, or "never" for an +// entrypoint with none. +func TestHandleSourceDetail_ShowsEntrypointEvents(t *testing.T) { + t.Parallel() + + var ( + h *handlers.Handlers + sess *session.Session + db *database.Database + dbMgr *database.WebhookDBManager + ) + + app := newTestApp(t, &h, &sess, &db, &dbMgr) + app.RequireStart() + + t.Cleanup(app.RequireStop) + + wh := &database.Webhook{ + UserID: deleteTestUserID, Name: "figures", RetentionDays: 7, + } + require.NoError(t, db.DB().Omit(clause.Associations).Create(wh).Error) + + webhookDB, err := dbMgr.GetDB(wh.ID) + require.NoError(t, err) + + entrypoint := func() *database.Entrypoint { + ep := &database.Entrypoint{ + WebhookID: wh.ID, Path: uuid.New().String(), Active: true, + } + require.NoError(t, + db.DB().Omit(clause.Associations).Create(ep).Error) + + return ep + } + + event := func(ep *database.Entrypoint, age time.Duration) time.Time { + e := &database.Event{ + WebhookID: wh.ID, + EntrypointID: ep.ID, + Method: http.MethodPost, + } + e.CreatedAt = time.Now().Add(-age) + require.NoError(t, + webhookDB.Omit(clause.Associations).Create(e).Error) + + return e.CreatedAt + } + + busy, quiet, unused := entrypoint(), entrypoint(), entrypoint() + + event(busy, 8*24*time.Hour) // older than the 7 days kept + event(busy, 3*time.Hour) + busyLast := event(busy, time.Hour) + quietLast := event(quiet, 2*24*time.Hour) + + body := renderSourceDetailPage(t, h, sess, wh.ID) + + // rowOf returns the part of the page from an entrypoint's URL to + // the next entrypoint's, which holds its figures. + rowOf := func(ep *database.Entrypoint) string { + _, row, found := strings.Cut(body, `id="entrypoint-url-`+ep.ID+`"`) + require.True(t, found) + + row, _, _ = strings.Cut(row, `id="entrypoint-url-`) + + return row + } + + lastEvent := func(at time.Time) string { + return `Last Event:\s*[^<]+ ago` + } + + count := func(n int) string { + return `Events Within Retention:\s*` + + strconv.Itoa(n) + `` + } + + assert.Regexp(t, lastEvent(busyLast), rowOf(busy)) + assert.Regexp(t, count(2), rowOf(busy)) + assert.Regexp(t, lastEvent(quietLast), rowOf(quiet)) + assert.Regexp(t, count(1), rowOf(quiet)) + assert.Regexp(t, `Last Event:\s*never`, rowOf(unused)) + assert.Regexp(t, count(0), rowOf(unused)) +} diff --git a/internal/handlers/source_management.go b/internal/handlers/source_management.go index b4830c5..f2506cb 100644 --- a/internal/handlers/source_management.go +++ b/internal/handlers/source_management.go @@ -482,6 +482,8 @@ func (h *Handlers) renderSourceDetail( "webhook_id = ?", webhook.ID, ).Find(&targets) + entrypointViews := NewEntrypointViews(entrypoints) + var events []RecentEventView if h.dbMgr.DBExists(webhook.ID) { @@ -500,6 +502,15 @@ func (h *Handlers) renderSourceDetail( return } + + err = addEntrypointEvents( + webhookDB, &webhook, entrypointViews, time.Now(), + ) + if err != nil { + h.serverError(w, r, "failed to count entrypoint events", err) + + return + } } scheme := "http" @@ -520,7 +531,7 @@ func (h *Handlers) renderSourceDetail( // Targets are projected to a display-safe view: a // target's stored config blob holds a credential, and it // must never reach a template. - "Entrypoints": NewEntrypointViews(entrypoints), + "Entrypoints": entrypointViews, "Targets": h.targetRows(&webhook, targets), "Events": events, "BaseURL": baseURL, diff --git a/templates/source_detail.html b/templates/source_detail.html index 2608de0..3c9e2a4 100644 --- a/templates/source_detail.html +++ b/templates/source_detail.html @@ -97,6 +97,18 @@ +
+ Last Event: + {{if .LastEvent}} + {{.LastEvent}} + {{else}} + never + {{end}} +
+
+ Events Within Retention: + {{.Events}} +
{{else}}
No entrypoints configured.