From 8328016bec2f02397dfdc30d6383c94d9d82582b Mon Sep 17 00:00:00 2001 From: sneak Date: Mon, 21 Sep 2026 07:52:21 +0000 Subject: [PATCH] Index the event-tier columns the sweeps, event log and retention scan (closes #314) The per-webhook tables declared no secondary indexes, so the recovery and sweep queries (by delivery status, every minute), the event log (deliveries by event, results by delivery) and retention (events by age) each scanned a whole table. Add indexes through GORM model tags so AutoMigrate creates them on a fresh and on an existing per-webhook database. events.created_at is indexed by overriding the embedded BaseModel field on Event alone, leaving the other tables' created_at unindexed. A test drops the indexes from an opened database, reopens it, and asserts the open recreated them. The README Data Model section lists the indexes. Model: opus-4-8 --- README.md | 16 +++++ internal/database/event_tier_indexes_test.go | 72 ++++++++++++++++++++ internal/database/model_delivery.go | 6 +- internal/database/model_delivery_result.go | 2 +- internal/database/model_event.go | 8 +++ 5 files changed, 100 insertions(+), 4 deletions(-) create mode 100644 internal/database/event_tier_indexes_test.go diff --git a/README.md b/README.md index 319989c..8fbaea5 100644 --- a/README.md +++ b/README.md @@ -1700,6 +1700,22 @@ retries) is individually logged for full observability. **Relations:** Belongs to Delivery. +#### Event-tier indexes + +Beyond the primary keys, the per-webhook event databases carry secondary +indexes on the columns the background work reads by, each created by +`AutoMigrate` on a fresh and on an existing database: + +| Column | Serves | +| ------------------------------ | ------ | +| `deliveries.status` | The recovery and sweep queries that select deliveries by status once a minute | +| `deliveries.event_id` | Loading a page of the event log, which reads deliveries by event | +| `delivery_results.delivery_id` | Loading a page of the event log, which reads results by delivery | +| `events.created_at` | Retention, which deletes events by age | + +The `events.resubmitted_from_id` column is also indexed, to resolve the +resubmit relationship both ways in the event log. + #### Common Fields Every entity except `Setting` includes these fields from `BaseModel`. diff --git a/internal/database/event_tier_indexes_test.go b/internal/database/event_tier_indexes_test.go new file mode 100644 index 0000000..2999464 --- /dev/null +++ b/internal/database/event_tier_indexes_test.go @@ -0,0 +1,72 @@ +package database_test + +import ( + "context" + "testing" + + "github.com/google/uuid" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "sneak.berlin/go/webhooker/internal/database" +) + +// indexedColumn names a secondary index by the model and struct field +// GORM derives the index name from. +type indexedColumn struct { + model any + field string +} + +// eventTierIndexes are the columns the background work reads by: the +// recovery and sweep queries (status), the event log (event_id and +// delivery_id) and retention (created_at). +var eventTierIndexes = []indexedColumn{ + {&database.Delivery{}, "Status"}, + {&database.Delivery{}, "EventID"}, + {&database.DeliveryResult{}, "DeliveryID"}, + {&database.Event{}, "CreatedAt"}, +} + +// TestWebhookDBManager_OpenAddsEventTierIndexes verifies that opening a +// per-webhook database that predates these indexes creates them, so the +// queries above stop scanning whole tables. It stands in for an older +// database file by dropping the indexes AutoMigrate just created, then +// reopening the same file. +func TestWebhookDBManager_OpenAddsEventTierIndexes(t *testing.T) { + t.Parallel() + + mgr, lc := setupTestWebhookDBManager(t) + ctx := context.Background() + require.NoError(t, lc.Start(ctx)) + + defer func() { require.NoError(t, lc.Stop(ctx)) }() + + webhookID := uuid.New().String() + + db, err := mgr.GetDB(webhookID) + require.NoError(t, err) + + // A fresh database has them. + for _, ix := range eventTierIndexes { + require.True(t, db.Migrator().HasIndex(ix.model, ix.field)) + } + + // Stand in for a database file created before the indexes existed. + for _, ix := range eventTierIndexes { + require.NoError(t, db.Migrator().DropIndex(ix.model, ix.field)) + require.False(t, db.Migrator().HasIndex(ix.model, ix.field)) + } + + // Drop the cached connection so the next open reopens the file and + // runs AutoMigrate against it, as a restart would. + require.NoError(t, mgr.CloseAll()) + + db, err = mgr.GetDB(webhookID) + require.NoError(t, err) + + for _, ix := range eventTierIndexes { + assert.True(t, db.Migrator().HasIndex(ix.model, ix.field), + "opening the existing database should create the index on %s", + ix.field) + } +} diff --git a/internal/database/model_delivery.go b/internal/database/model_delivery.go index 71f6b6d..3c280f7 100644 --- a/internal/database/model_delivery.go +++ b/internal/database/model_delivery.go @@ -32,9 +32,9 @@ func (s DeliveryStatus) Terminal() bool { type Delivery struct { BaseModel - EventID string `gorm:"type:uuid;not null" json:"eventId"` - TargetID string `gorm:"type:uuid;not null" json:"targetId"` - Status DeliveryStatus `gorm:"not null;default:'pending'" json:"status"` + EventID string `gorm:"type:uuid;not null;index" json:"eventId"` + TargetID string `gorm:"type:uuid;not null" json:"targetId"` + Status DeliveryStatus `gorm:"not null;default:'pending';index" json:"status"` // Relations Event Event `json:"event,omitzero"` diff --git a/internal/database/model_delivery_result.go b/internal/database/model_delivery_result.go index 56c9cc2..d701a95 100644 --- a/internal/database/model_delivery_result.go +++ b/internal/database/model_delivery_result.go @@ -4,7 +4,7 @@ package database type DeliveryResult struct { BaseModel - DeliveryID string `gorm:"type:uuid;not null" json:"deliveryId"` + DeliveryID string `gorm:"type:uuid;not null;index" json:"deliveryId"` AttemptNum int `gorm:"not null" json:"attemptNum"` Success bool `json:"success"` StatusCode int `json:"statusCode,omitempty"` diff --git a/internal/database/model_event.go b/internal/database/model_event.go index dafc235..70126f2 100644 --- a/internal/database/model_event.go +++ b/internal/database/model_event.go @@ -1,9 +1,17 @@ package database +import "time" + // Event represents a captured webhook event type Event struct { BaseModel + // CreatedAt overrides BaseModel.CreatedAt only to add an index: + // retention deletes events by age, so events.created_at is queried + // on every sweep. The other tables keep the unindexed BaseModel + // field. + CreatedAt time.Time `gorm:"index" json:"createdAt"` + WebhookID string `gorm:"type:uuid;not null" json:"webhookId"` EntrypointID string `gorm:"type:uuid;not null" json:"entrypointId"` -- 2.54.0