diff --git a/README.md b/README.md index cd3e0ff..fcabb6e 100644 --- a/README.md +++ b/README.md @@ -1752,7 +1752,7 @@ retries) is individually logged for full observability. #### EventTotals and TargetTotals Running counts in each event database, read by the statistics pane at the -top of the webhook page. `EventTotals` is one row: +top of the webhook page and by the webhook list. `EventTotals` is one row: | Field | Type | Description | | ---------------- | --------- | ----------- | @@ -1784,6 +1784,14 @@ target. Its failure percentage for a window is the deliveries that became `failed` in it out of all that became `delivered` or `failed` in it, and a dash when none did. +The webhook list at `/hooks` shows three of the pane's figures for each +webhook: its events within retention and its last event, both from +`EventTotals`, and its deliveries that failed in the last 24 hours, +counted with the pane's query. It opens each webhook's event database once +(the handle stays open) and runs those two reads there, so its cost grows +with the number of webhooks and, for each, with the deliveries that +finished in the last 24 hours, never with the events stored. + #### Event-tier indexes These indexes on the per-webhook event databases are declared in the model @@ -1791,7 +1799,7 @@ tags, so `AutoMigrate` creates them on a fresh database: | Table | Columns | Serves | | ------------------ | --------------------------- | ------ | -| `deliveries` | `status`, `deleted_at`, `finished_at`, `target_id` | Startup recovery, the retry and pending sweeps every 60 seconds and the queue-depth sampler every 30 seconds, which select deliveries by status, and the webhook page's statistics, which count each target's deliveries by status and when they finished | +| `deliveries` | `status`, `deleted_at`, `finished_at`, `target_id` | Startup recovery, the retry and pending sweeps every 60 seconds and the queue-depth sampler every 30 seconds, which select deliveries by status, and the webhook page's statistics and the webhook list, which count each target's deliveries by status and when they finished | | `deliveries` | `event_id`, `deleted_at` | The event log, which loads each event's deliveries, and retention, which counts and deletes the deliveries of expired events | | `delivery_results` | `delivery_id`, `deleted_at` | The event log, which loads the attempts of a page's deliveries, and retention, which deletes the attempts of expired events | | `events` | `deleted_at`, `created_at` | The webhook page's statistics, which count recent events | diff --git a/internal/handlers/source_list_test.go b/internal/handlers/source_list_test.go new file mode 100644 index 0000000..29d79bc --- /dev/null +++ b/internal/handlers/source_list_test.go @@ -0,0 +1,267 @@ +package handlers_test + +import ( + "net/http" + "net/http/httptest" + "regexp" + "strings" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "gorm.io/gorm" + "gorm.io/gorm/clause" + "sneak.berlin/go/webhooker/internal/database" + "sneak.berlin/go/webhooker/internal/handlers" + "sneak.berlin/go/webhooker/internal/session" +) + +// failedHighlight is how the list marks a number of failed deliveries +// that is not zero. +const failedHighlight = `class="font-medium text-red-600"` + +// listWebhook adds a webhook with the given name, owned by the test +// user. +func listWebhook( + t *testing.T, db *database.Database, name string, +) *database.Webhook { + t.Helper() + + wh := &database.Webhook{UserID: deleteTestUserID, Name: name} + require.NoError(t, db.DB().Omit(clause.Associations).Create(wh).Error) + + return wh +} + +// renderWebhookList runs the real webhook list handler as the test user +// and returns the rendered page. +func renderWebhookList( + t *testing.T, h *handlers.Handlers, sess *session.Session, +) string { + t.Helper() + + cookies := authenticatedCookies( + t, sess, deleteTestUserID, deleteTestUsername, + ) + + w := httptest.NewRecorder() + h.HandleSourceList().ServeHTTP( + w, getRequest(t, "/hooks", cookies, nil), + ) + require.Equal(t, http.StatusOK, w.Code) + + return w.Body.String() +} + +// listCard returns one webhook's entry in a rendered webhook list, its +// markup as rendered and its text with the markup taken out and each +// run of space made one space. +func listCard(t *testing.T, page, webhookID string) (string, string) { + t.Helper() + + _, card, found := strings.Cut(page, `href="/hook/`+webhookID+`"`) + require.True(t, found, "the list has no entry for %s", webhookID) + + card, _, _ = strings.Cut(card, "") + text := regexp.MustCompile(`<[^>]*>`).ReplaceAllString(card, " ") + + return card, strings.Join(strings.Fields(text), " ") +} + +// receiveEvents posts the given number of events to an entrypoint +// through the real receiver, and returns the webhook's event database +// and its events, oldest first. +func receiveEvents( + t *testing.T, + h *handlers.Handlers, + dbMgr *database.WebhookDBManager, + webhookID, path string, + count int, +) (*gorm.DB, []database.Event) { + t.Helper() + + router := receiverRouter(h) + + for range count { + require.Equal(t, http.StatusOK, postReceiver(t, router, path)) + } + + webhookDB, err := dbMgr.GetDB(webhookID) + require.NoError(t, err) + + events := listEvents(t, webhookDB) + require.Len(t, events, count) + + return webhookDB, events +} + +// seedFailingWebhook adds a webhook with two entrypoints, one inactive, +// and four targets, one inactive. Three events each reach the three +// active targets. Two deliveries failed in the last 24 hours, one 30 +// hours ago, and one was delivered. It returns the webhook and its +// newest event. +func seedFailingWebhook( + t *testing.T, + h *handlers.Handlers, + db *database.Database, + dbMgr *database.WebhookDBManager, +) (*database.Webhook, database.Event) { + t.Helper() + + wh := listWebhook(t, db, "failing") + path := statsEntrypoint(t, db, wh.ID, true) + + statsEntrypoint(t, db, wh.ID, false) + + first := seedTarget(t, db, wh.ID, database.TargetTypeLog) + second := seedTarget(t, db, wh.ID, database.TargetTypeLog) + + seedTarget(t, db, wh.ID, database.TargetTypeLog) + + inactive := seedTarget(t, db, wh.ID, database.TargetTypeLog) + require.NoError(t, db.DB().Model(inactive). + Update("active", false).Error) + + webhookDB, events := receiveEvents(t, h, dbMgr, wh.ID, path, 3) + now := time.Now() + + statsFinish(t, webhookDB, + statsDelivery(t, webhookDB, events[0].ID, first.ID), + database.DeliveryStatusFailed, now.Add(-30*time.Hour)) + statsFinish(t, webhookDB, + statsDelivery(t, webhookDB, events[1].ID, first.ID), + database.DeliveryStatusFailed, now.Add(-time.Hour)) + statsFinish(t, webhookDB, + statsDelivery(t, webhookDB, events[2].ID, first.ID), + database.DeliveryStatusFailed, now.Add(-time.Minute)) + statsFinish(t, webhookDB, + statsDelivery(t, webhookDB, events[2].ID, second.ID), + database.DeliveryStatusDelivered, now.Add(-time.Minute)) + + return wh, events[2] +} + +// seedHealthyWebhook adds a webhook with one entrypoint and one target, +// both active, and two events, both delivered. It returns the webhook +// and its newest event. +func seedHealthyWebhook( + t *testing.T, + h *handlers.Handlers, + db *database.Database, + dbMgr *database.WebhookDBManager, +) (*database.Webhook, database.Event) { + t.Helper() + + wh := listWebhook(t, db, "healthy") + path := statsEntrypoint(t, db, wh.ID, true) + target := seedTarget(t, db, wh.ID, database.TargetTypeLog) + + webhookDB, events := receiveEvents(t, h, dbMgr, wh.ID, path, 2) + + for _, ev := range events { + statsFinish(t, webhookDB, + statsDelivery(t, webhookDB, ev.ID, target.ID), + database.DeliveryStatusDelivered, time.Now()) + } + + return wh, events[1] +} + +// lastEventText is how the list shows the arrival of an event. +func lastEventText(ev database.Event) string { + return ev.CreatedAt.UTC().Format("2006-01-02 15:04:05 UTC") +} + +// TestSourceList_ShowsActivityOfEachWebhook checks the figures the list +// shows for a webhook with recent failures, a healthy one, and one that +// has never received an event. +func TestSourceList_ShowsActivityOfEachWebhook(t *testing.T) { + t.Parallel() + + var ( + h *handlers.Handlers + sess *session.Session + db *database.Database + dbMgr *database.WebhookDBManager + ) + + app := newTestApp(t, &h, &sess, &db, &dbMgr) + app.RequireStart() + + t.Cleanup(app.RequireStop) + + failing, failingNewest := seedFailingWebhook(t, h, db, dbMgr) + healthy, healthyNewest := seedHealthyWebhook(t, h, db, dbMgr) + quiet := listWebhook(t, db, "quiet") + + page := renderWebhookList(t, h, sess) + + card, text := listCard(t, page, failing.ID) + assert.Contains(t, text, "2 entrypoints, 1 inactive "+ + "4 targets, 1 inactive "+ + "3 events "+ + "Last event "+lastEventText(failingNewest)+" "+ + "2 failed deliveries in the last 24 hours") + assert.Contains(t, card, + failedHighlight+">2 failed deliveries in the last 24 hours<") + + card, text = listCard(t, page, healthy.ID) + assert.Contains(t, text, "1 entrypoint "+ + "1 target "+ + "2 events "+ + "Last event "+lastEventText(healthyNewest)+" "+ + "0 failed deliveries in the last 24 hours") + assert.NotContains(t, text, "inactive") + assert.NotContains(t, card, failedHighlight) + + card, text = listCard(t, page, quiet.ID) + assert.Contains(t, text, "0 entrypoints "+ + "0 targets "+ + "0 events "+ + "No events yet "+ + "0 failed deliveries in the last 24 hours") + assert.NotContains(t, card, failedHighlight) + assert.False(t, dbMgr.DBExists(quiet.ID), + "showing the list must not create an event database") +} + +// TestSourceList_UnreadableEventDatabase checks that a webhook whose +// event database cannot be read says so in its entry instead of +// showing zeros, and that the rest of the list is still shown. +func TestSourceList_UnreadableEventDatabase(t *testing.T) { + t.Parallel() + + var ( + h *handlers.Handlers + sess *session.Session + db *database.Database + dbMgr *database.WebhookDBManager + ) + + app := newTestApp(t, &h, &sess, &db, &dbMgr) + app.RequireStart() + + t.Cleanup(app.RequireStop) + + broken := listWebhook(t, db, "broken") + statsEntrypoint(t, db, broken.ID, true) + + brokenDB, err := dbMgr.GetDB(broken.ID) + require.NoError(t, err) + require.NoError(t, + brokenDB.Migrator().DropTable(&database.EventTotals{})) + + quiet := listWebhook(t, db, "quiet") + + page := renderWebhookList(t, h, sess) + + _, text := listCard(t, page, broken.ID) + assert.Contains(t, text, + "1 entrypoint 0 targets The event figures could not be read.") + assert.NotContains(t, text, "events") + assert.NotContains(t, text, "failed") + + _, text = listCard(t, page, quiet.ID) + assert.Contains(t, text, "No events yet") +} diff --git a/internal/handlers/source_management.go b/internal/handlers/source_management.go index 7a26b94..519a22e 100644 --- a/internal/handlers/source_management.go +++ b/internal/handlers/source_management.go @@ -3,10 +3,12 @@ package handlers import ( "encoding/json" "errors" + "fmt" "net/http" "slices" "strconv" "strings" + "time" "github.com/go-chi/chi" "github.com/google/uuid" @@ -20,9 +22,20 @@ import ( type WebhookListItem struct { database.Webhook - EntrypointCount int64 - TargetCount int64 - EventCount int64 + EntrypointCount int + InactiveEntrypointCount int + TargetCount int + InactiveTargetCount int + + // EventCount is how many events the webhook holds, LastEventAt + // when the newest arrived (nil before the first), and + // FailedLast24Hours how many of its deliveries failed in the last + // 24 hours. When the webhook's event database could not be read, + // EventsUnreadable is set and these three are not known. + EventCount int64 + LastEventAt *time.Time + FailedLast24Hours int64 + EventsUnreadable bool } // errMissingURL signals that a required URL was not provided. @@ -160,7 +173,18 @@ func (h *Handlers) HandleSourceList() http.HandlerFunc { return } - items := h.buildWebhookListItems(webhooks) + items, err := h.buildWebhookListItems(webhooks) + if err != nil { + h.log.Error( + "failed to list webhooks", "error", err, + ) + http.Error( + w, "Internal server error", + http.StatusInternalServerError, + ) + + return + } data := map[string]any{ "Webhooks": items, @@ -170,36 +194,115 @@ func (h *Handlers) HandleSourceList() http.HandlerFunc { } } -// buildWebhookListItems builds list items with counts. +// buildWebhookListItems builds the list's entry for each webhook. It +// fails when the main database cannot be read. A webhook whose event +// database cannot be read is marked on its own entry, and the error is +// logged. func (h *Handlers) buildWebhookListItems( webhooks []database.Webhook, -) []WebhookListItem { +) ([]WebhookListItem, error) { items := make([]WebhookListItem, len(webhooks)) + since := time.Now().Add(-longWindow) for i := range webhooks { - items[i].Webhook = webhooks[i] + item := &items[i] + item.Webhook = webhooks[i] - h.db.DB().Model(&database.Entrypoint{}).Where( - "webhook_id = ?", webhooks[i].ID, - ).Count(&items[i].EntrypointCount) + var err error - h.db.DB().Model(&database.Target{}).Where( - "webhook_id = ?", webhooks[i].ID, - ).Count(&items[i].TargetCount) + item.EntrypointCount, item.InactiveEntrypointCount, err = + h.countWithInactive(&database.Entrypoint{}, item.ID) + if err != nil { + return nil, err + } - if h.dbMgr.DBExists(webhooks[i].ID) { - webhookDB, err := h.dbMgr.GetDB( - webhooks[i].ID, + item.TargetCount, item.InactiveTargetCount, err = + h.countWithInactive(&database.Target{}, item.ID) + if err != nil { + return nil, err + } + + // Opening an event database that does not exist would create + // it, and it would hold nothing to count. + if !h.dbMgr.DBExists(item.ID) { + continue + } + + err = h.readListEventFigures(item, since) + if err != nil { + h.log.Error( + "failed to read webhook list figures", + "webhook_id", item.ID, + "error", err, ) - if err == nil { - webhookDB.Model( - &database.Event{}, - ).Count(&items[i].EventCount) - } + + item.EventsUnreadable = true } } - return items + return items, nil +} + +// countWithInactive returns how many entrypoints or targets, as model +// says, a webhook has, and how many of them are inactive. +func (h *Handlers) countWithInactive( + model any, webhookID string, +) (int, int, error) { + var active []bool + + err := h.db.DB().Model(model). + Where("webhook_id = ?", webhookID). + Pluck("active", &active).Error + if err != nil { + return 0, 0, fmt.Errorf( + "reading active flags of webhook %s: %w", webhookID, err, + ) + } + + inactive := 0 + + for _, a := range active { + if !a { + inactive++ + } + } + + return len(active), inactive, nil +} + +// readListEventFigures fills in the figures the list shows from the +// webhook's event database, with the statistics pane's own queries: +// the event count and last arrival from the event totals row, and the +// deliveries that failed since the given time from the deliveries' +// status index. +func (h *Handlers) readListEventFigures( + item *WebhookListItem, since time.Time, +) error { + webhookDB, err := h.dbMgr.GetDB(item.ID) + if err != nil { + return err + } + + var totals database.EventTotals + + err = webhookDB.Take(&totals).Error + if err != nil { + return fmt.Errorf("reading event totals: %w", err) + } + + item.EventCount = totals.Events - totals.EventsRemoved + item.LastEventAt = totals.LastEventAt + + byTarget, err := finishedByTarget(webhookDB, since) + if err != nil { + return err + } + + for _, f := range byTarget { + item.FailedLast24Hours += f.Failed + } + + return nil } // HandleSourceCreate shows the form to create a new webhook. diff --git a/templates/sources_list.html b/templates/sources_list.html index 44fae4f..87a441e 100644 --- a/templates/sources_list.html +++ b/templates/sources_list.html @@ -27,10 +27,16 @@ Retention: {{.RetentionLabel}} -
- {{.EntrypointCount}} entrypoint{{if ne .EntrypointCount 1}}s{{end}} - {{.TargetCount}} target{{if ne .TargetCount 1}}s{{end}} +
+ {{.EntrypointCount}} entrypoint{{if ne .EntrypointCount 1}}s{{end}}{{if .InactiveEntrypointCount}}, {{.InactiveEntrypointCount}} inactive{{end}} + {{.TargetCount}} target{{if ne .TargetCount 1}}s{{end}}{{if .InactiveTargetCount}}, {{.InactiveTargetCount}} inactive{{end}} + {{if .EventsUnreadable}} + The event figures could not be read. + {{else}} {{.EventCount}} event{{if ne .EventCount 1}}s{{end}} + {{with .LastEventAt}}Last event {{.UTC.Format "2006-01-02 15:04:05 UTC"}}{{else}}No events yet{{end}} + {{.FailedLast24Hours}} failed deliver{{if eq .FailedLast24Hours 1}}y{{else}}ies{{end}} in the last 24 hours + {{end}}
{{end}}