diff --git a/README.md b/README.md index eb5ee27..7db1199 100644 --- a/README.md +++ b/README.md @@ -1759,6 +1759,28 @@ retries) is individually logged for full observability. **Relations:** Belongs to Delivery. +#### Event-tier indexes + +These indexes on the per-webhook event databases are declared in the model +tags, so `AutoMigrate` creates them on a fresh and on an existing database: + +| Table | Columns | Serves | +| ------------------ | --------------------------- | ------ | +| `deliveries` | `status`, `deleted_at` | Startup recovery, the retry and pending sweeps every 60 seconds and the queue-depth sampler every 30 seconds, which select deliveries by status | +| `deliveries` | `event_id`, `deleted_at` | The event log, which loads each event's deliveries, and retention, which selects and deletes the deliveries of expired events | +| `delivery_results` | `delivery_id`, `deleted_at` | The event log, which loads the attempts of a page's deliveries, and retention, which deletes the attempts of expired events | +| `events` | `deleted_at`, `created_at` | Retention, which selects expired events by age | +| `events` | `created_at` | Retention's delete of the expired events themselves | + +GORM's soft delete adds `deleted_at IS NULL` to these queries; retention's +deletes leave it out, but their lookups of expired rows keep it. SQLite keeps +no statistics on these tables, and without them it rates the `deleted_at` +index, which every live row matches, above an index on a column matched +against several values or compared with `<`. So every index but the last also +covers `deleted_at`. It comes second, so that retention's deletes can use the +index without it, except in `events`, where `created_at` is compared with `<` +and SQLite narrows by a `<` only on the last column it uses. + #### Common Fields Every entity except `Setting` includes these fields from `BaseModel`. diff --git a/internal/database/event_tier_indexes_test.go b/internal/database/event_tier_indexes_test.go new file mode 100644 index 0000000..d25dca5 --- /dev/null +++ b/internal/database/event_tier_indexes_test.go @@ -0,0 +1,166 @@ +package database_test + +import ( + "context" + "fmt" + "testing" + "time" + + "github.com/google/uuid" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "gorm.io/gorm" + "sneak.berlin/go/webhooker/internal/database" +) + +// TestWebhookDBManager_OpenAddsEventTierIndexes verifies that opening a +// per-webhook database that predates these indexes creates them. It +// stands in for an older database file by dropping the indexes +// AutoMigrate just created, then reopening the same file. +func TestWebhookDBManager_OpenAddsEventTierIndexes(t *testing.T) { + t.Parallel() + + indexes := []struct { + model any + name string + }{ + {&database.Delivery{}, "idx_deliveries_status"}, + {&database.Delivery{}, "idx_deliveries_event_id"}, + {&database.DeliveryResult{}, "idx_delivery_results_delivery_id"}, + {&database.Event{}, "idx_events_deleted_at_created_at"}, + {&database.Event{}, "idx_events_created_at"}, + } + + mgr, lc := setupTestWebhookDBManager(t) + ctx := context.Background() + require.NoError(t, lc.Start(ctx)) + + defer func() { require.NoError(t, lc.Stop(ctx)) }() + + webhookID := uuid.New().String() + + db, err := mgr.GetDB(webhookID) + require.NoError(t, err) + + // A fresh database has them. + for _, ix := range indexes { + require.True(t, db.Migrator().HasIndex(ix.model, ix.name)) + } + + // Stand in for a database file created before the indexes existed. + for _, ix := range indexes { + require.NoError(t, db.Migrator().DropIndex(ix.model, ix.name)) + require.False(t, db.Migrator().HasIndex(ix.model, ix.name)) + } + + // Drop the cached connection so the next open reopens the file and + // runs AutoMigrate against it, as a restart would. + require.NoError(t, mgr.CloseAll()) + + db, err = mgr.GetDB(webhookID) + require.NoError(t, err) + + for _, ix := range indexes { + assert.True(t, db.Migrator().HasIndex(ix.model, ix.name), + "opening the existing database should create %s", ix.name) + } +} + +// TestEventTierQueriesUseTheirIndexes verifies that the statements the +// indexes are for use them. GORM builds each statement in a dry run as +// the code named above it does, soft-delete condition included, and +// SQLite, which keeps no statistics on these tables, must plan to seek +// on each index listed by the columns in parentheses. +func TestEventTierQueriesUseTheirIndexes(t *testing.T) { + t.Parallel() + + mgr, lc := setupTestWebhookDBManager(t) + ctx := context.Background() + require.NoError(t, lc.Start(ctx)) + + defer func() { require.NoError(t, lc.Stop(ctx)) }() + + db, err := mgr.GetDB(uuid.New().String()) + require.NoError(t, err) + + dry := db.Session(&gorm.Session{DryRun: true}) + ids := []string{ + uuid.New().String(), uuid.New().String(), uuid.New().String(), + } + cutoff := time.Now() + + var ( + deliveries []database.Delivery + results []database.DeliveryResult + depths []struct{ Depth int } + ) + + byStatus := "idx_deliveries_status (status=? AND deleted_at=?)" + byEvent := "idx_deliveries_event_id (event_id=? AND deleted_at=?)" + byAge := "idx_events_deleted_at_created_at (deleted_at=? AND created_at