diff --git a/README.md b/README.md index 319989c..8fbaea5 100644 --- a/README.md +++ b/README.md @@ -1700,6 +1700,22 @@ retries) is individually logged for full observability. **Relations:** Belongs to Delivery. +#### Event-tier indexes + +Beyond the primary keys, the per-webhook event databases carry secondary +indexes on the columns the background work reads by, each created by +`AutoMigrate` on a fresh and on an existing database: + +| Column | Serves | +| ------------------------------ | ------ | +| `deliveries.status` | The recovery and sweep queries that select deliveries by status once a minute | +| `deliveries.event_id` | Loading a page of the event log, which reads deliveries by event | +| `delivery_results.delivery_id` | Loading a page of the event log, which reads results by delivery | +| `events.created_at` | Retention, which deletes events by age | + +The `events.resubmitted_from_id` column is also indexed, to resolve the +resubmit relationship both ways in the event log. + #### Common Fields Every entity except `Setting` includes these fields from `BaseModel`. diff --git a/internal/database/event_tier_indexes_test.go b/internal/database/event_tier_indexes_test.go new file mode 100644 index 0000000..2999464 --- /dev/null +++ b/internal/database/event_tier_indexes_test.go @@ -0,0 +1,72 @@ +package database_test + +import ( + "context" + "testing" + + "github.com/google/uuid" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "sneak.berlin/go/webhooker/internal/database" +) + +// indexedColumn names a secondary index by the model and struct field +// GORM derives the index name from. +type indexedColumn struct { + model any + field string +} + +// eventTierIndexes are the columns the background work reads by: the +// recovery and sweep queries (status), the event log (event_id and +// delivery_id) and retention (created_at). +var eventTierIndexes = []indexedColumn{ + {&database.Delivery{}, "Status"}, + {&database.Delivery{}, "EventID"}, + {&database.DeliveryResult{}, "DeliveryID"}, + {&database.Event{}, "CreatedAt"}, +} + +// TestWebhookDBManager_OpenAddsEventTierIndexes verifies that opening a +// per-webhook database that predates these indexes creates them, so the +// queries above stop scanning whole tables. It stands in for an older +// database file by dropping the indexes AutoMigrate just created, then +// reopening the same file. +func TestWebhookDBManager_OpenAddsEventTierIndexes(t *testing.T) { + t.Parallel() + + mgr, lc := setupTestWebhookDBManager(t) + ctx := context.Background() + require.NoError(t, lc.Start(ctx)) + + defer func() { require.NoError(t, lc.Stop(ctx)) }() + + webhookID := uuid.New().String() + + db, err := mgr.GetDB(webhookID) + require.NoError(t, err) + + // A fresh database has them. + for _, ix := range eventTierIndexes { + require.True(t, db.Migrator().HasIndex(ix.model, ix.field)) + } + + // Stand in for a database file created before the indexes existed. + for _, ix := range eventTierIndexes { + require.NoError(t, db.Migrator().DropIndex(ix.model, ix.field)) + require.False(t, db.Migrator().HasIndex(ix.model, ix.field)) + } + + // Drop the cached connection so the next open reopens the file and + // runs AutoMigrate against it, as a restart would. + require.NoError(t, mgr.CloseAll()) + + db, err = mgr.GetDB(webhookID) + require.NoError(t, err) + + for _, ix := range eventTierIndexes { + assert.True(t, db.Migrator().HasIndex(ix.model, ix.field), + "opening the existing database should create the index on %s", + ix.field) + } +} diff --git a/internal/database/model_delivery.go b/internal/database/model_delivery.go index 71f6b6d..3c280f7 100644 --- a/internal/database/model_delivery.go +++ b/internal/database/model_delivery.go @@ -32,9 +32,9 @@ func (s DeliveryStatus) Terminal() bool { type Delivery struct { BaseModel - EventID string `gorm:"type:uuid;not null" json:"eventId"` - TargetID string `gorm:"type:uuid;not null" json:"targetId"` - Status DeliveryStatus `gorm:"not null;default:'pending'" json:"status"` + EventID string `gorm:"type:uuid;not null;index" json:"eventId"` + TargetID string `gorm:"type:uuid;not null" json:"targetId"` + Status DeliveryStatus `gorm:"not null;default:'pending';index" json:"status"` // Relations Event Event `json:"event,omitzero"` diff --git a/internal/database/model_delivery_result.go b/internal/database/model_delivery_result.go index 56c9cc2..d701a95 100644 --- a/internal/database/model_delivery_result.go +++ b/internal/database/model_delivery_result.go @@ -4,7 +4,7 @@ package database type DeliveryResult struct { BaseModel - DeliveryID string `gorm:"type:uuid;not null" json:"deliveryId"` + DeliveryID string `gorm:"type:uuid;not null;index" json:"deliveryId"` AttemptNum int `gorm:"not null" json:"attemptNum"` Success bool `json:"success"` StatusCode int `json:"statusCode,omitempty"` diff --git a/internal/database/model_event.go b/internal/database/model_event.go index dafc235..70126f2 100644 --- a/internal/database/model_event.go +++ b/internal/database/model_event.go @@ -1,9 +1,17 @@ package database +import "time" + // Event represents a captured webhook event type Event struct { BaseModel + // CreatedAt overrides BaseModel.CreatedAt only to add an index: + // retention deletes events by age, so events.created_at is queried + // on every sweep. The other tables keep the unindexed BaseModel + // field. + CreatedAt time.Time `gorm:"index" json:"createdAt"` + WebhookID string `gorm:"type:uuid;not null" json:"webhookId"` EntrypointID string `gorm:"type:uuid;not null" json:"entrypointId"`