Index the event log's resubmit count with deleted_at (closes #325)
check / check (push) Failing after 1m10s
check / check (push) Failing after 1m10s
The resubmit count matches resubmitted_from_id against a page of event ids, and GORM adds deleted_at IS NULL. With no statistics, SQLite preferred the deleted_at index once the page held five or more ids, so every event log page read every live event. deleted_at now joins resubmitted_from_id in idx_events_resubmitted_from_id, as for the other event-tier indexes, and the plan test checks the statement with a full page of ids. Model: opus-5-5
This commit is contained in:
@@ -1876,6 +1876,7 @@ tags, so `AutoMigrate` creates them on a fresh database:
|
|||||||
| `deliveries` | `event_id`, `deleted_at` | The event log, which loads each event's deliveries, and retention, which counts and deletes the deliveries of expired events |
|
| `deliveries` | `event_id`, `deleted_at` | The event log, which loads each event's deliveries, and retention, which counts and deletes the deliveries of expired events |
|
||||||
| `delivery_results` | `delivery_id`, `deleted_at` | The event log, which loads the attempts of a page's deliveries, and retention, which deletes the attempts of expired events |
|
| `delivery_results` | `delivery_id`, `deleted_at` | The event log, which loads the attempts of a page's deliveries, and retention, which deletes the attempts of expired events |
|
||||||
| `events` | `deleted_at`, `created_at` | The webhook page's statistics, which count recent events |
|
| `events` | `deleted_at`, `created_at` | The webhook page's statistics, which count recent events |
|
||||||
|
| `events` | `resubmitted_from_id`, `deleted_at` | The event log, which counts the events resubmitted from each event on a page |
|
||||||
| `events` | `created_at` | Retention, which selects expired events by age |
|
| `events` | `created_at` | Retention, which selects expired events by age |
|
||||||
|
|
||||||
GORM's soft delete adds `deleted_at IS NULL` to these queries; retention
|
GORM's soft delete adds `deleted_at IS NULL` to these queries; retention
|
||||||
@@ -1883,9 +1884,9 @@ leaves it out. SQLite keeps no statistics on these tables, and without them it
|
|||||||
rates the `deleted_at` index, which every live row matches, above an index on
|
rates the `deleted_at` index, which every live row matches, above an index on
|
||||||
a column matched against several values or compared with a range. So every
|
a column matched against several values or compared with a range. So every
|
||||||
index but the last also covers `deleted_at`. It comes second, so that
|
index but the last also covers `deleted_at`. It comes second, so that
|
||||||
retention can use the index without it, except in `events`, where the
|
retention can use the index without it, except in the statistics' `events`
|
||||||
statistics compare `created_at` with a range (`>=`) and SQLite narrows by a
|
index, where they compare `created_at` with a range (`>=`) and SQLite narrows
|
||||||
range only on the last column it uses.
|
by a range only on the last column it uses.
|
||||||
|
|
||||||
#### Common Fields
|
#### Common Fields
|
||||||
|
|
||||||
|
|||||||
@@ -93,6 +93,7 @@ func TestEventTierQueriesUseTheirIndexes(t *testing.T) {
|
|||||||
deliveries []database.Delivery
|
deliveries []database.Delivery
|
||||||
results []database.DeliveryResult
|
results []database.DeliveryResult
|
||||||
depths []struct{ Depth int }
|
depths []struct{ Depth int }
|
||||||
|
resubmits []struct{ Total int }
|
||||||
removed []database.TargetTotals
|
removed []database.TargetTotals
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -116,13 +117,29 @@ func TestEventTierQueriesUseTheirIndexes(t *testing.T) {
|
|||||||
}).Group("target_id, status").Find(&depths), byStatus)
|
}).Group("target_id, status").Find(&depths), byStatus)
|
||||||
|
|
||||||
// The event log: each event's deliveries, then their attempts
|
// The event log: each event's deliveries, then their attempts
|
||||||
// (loadEventsWithDeliveries, loadDeliveryResults).
|
// (loadEventsWithDeliveries, loadDeliveryResults), and how many
|
||||||
|
// events were resubmitted from each of the page's (resubmitCounts).
|
||||||
assertPlanUses(t, db, dry.Where("event_id = ?", ids[0]).
|
assertPlanUses(t, db, dry.Where("event_id = ?", ids[0]).
|
||||||
Find(&deliveries), byEvent)
|
Find(&deliveries), byEvent)
|
||||||
assertPlanUses(t, db, dry.Where("delivery_id IN ?", ids).
|
assertPlanUses(t, db, dry.Where("delivery_id IN ?", ids).
|
||||||
Order("attempt_num ASC").Find(&results),
|
Order("attempt_num ASC").Find(&results),
|
||||||
"idx_delivery_results_delivery_id (delivery_id=? AND deleted_at=?)")
|
"idx_delivery_results_delivery_id (delivery_id=? AND deleted_at=?)")
|
||||||
|
|
||||||
|
// The resubmit count gets a full page of 25 ids: with an index on
|
||||||
|
// resubmitted_from_id alone, SQLite still uses it for three ids and
|
||||||
|
// turns to the deleted_at index only from five.
|
||||||
|
page := make([]string, 25)
|
||||||
|
for i := range page {
|
||||||
|
page[i] = uuid.New().String()
|
||||||
|
}
|
||||||
|
|
||||||
|
assertPlanUses(t, db, dry.Model(&database.Event{}).
|
||||||
|
Select("resubmitted_from_id, count(*) AS total").
|
||||||
|
Where("resubmitted_from_id IN ?", page).
|
||||||
|
Group("resubmitted_from_id").Find(&resubmits),
|
||||||
|
"idx_events_resubmitted_from_id "+
|
||||||
|
"(resubmitted_from_id=? AND deleted_at=?)")
|
||||||
|
|
||||||
// Retention (reapExpired, deleteEvents): one batch of expired
|
// Retention (reapExpired, deleteEvents): one batch of expired
|
||||||
// events, then their attempts, deliveries and the events.
|
// events, then their attempts, deliveries and the events.
|
||||||
var expired []string
|
var expired []string
|
||||||
|
|||||||
@@ -19,8 +19,10 @@ type Event struct {
|
|||||||
// narrows by a < only on the last column it uses. Its final delete
|
// narrows by a < only on the last column it uses. Its final delete
|
||||||
// has no deleted_at condition and uses the index on created_at
|
// has no deleted_at condition and uses the index on created_at
|
||||||
// alone. The other tables keep the unindexed BaseModel created_at.
|
// alone. The other tables keep the unindexed BaseModel created_at.
|
||||||
CreatedAt time.Time `gorm:"index;index:idx_events_deleted_at_created_at,priority:2" json:"createdAt"`
|
// DeletedAt is also the second column of the resubmitted_from_id
|
||||||
DeletedAt gorm.DeletedAt `gorm:"index:idx_events_deleted_at_created_at,priority:1" json:"deletedAt,omitzero"`
|
// index, for the reason DeliveryResult gives.
|
||||||
|
CreatedAt time.Time `gorm:"index;index:idx_events_deleted_at_created_at,priority:2" json:"createdAt"`
|
||||||
|
DeletedAt gorm.DeletedAt `gorm:"index:idx_events_deleted_at_created_at,priority:1;index:idx_events_resubmitted_from_id,priority:2" json:"deletedAt,omitzero"`
|
||||||
|
|
||||||
WebhookID string `gorm:"type:uuid;not null" json:"webhookId"`
|
WebhookID string `gorm:"type:uuid;not null" json:"webhookId"`
|
||||||
EntrypointID string `gorm:"type:uuid;not null" json:"entrypointId"`
|
EntrypointID string `gorm:"type:uuid;not null" json:"entrypointId"`
|
||||||
@@ -42,7 +44,7 @@ type Event struct {
|
|||||||
// existed. It is not a foreign key: the source event can be
|
// existed. It is not a foreign key: the source event can be
|
||||||
// reaped by retention while its copies remain, and the id is
|
// reaped by retention while its copies remain, and the id is
|
||||||
// kept as the record of where the copy came from either way.
|
// kept as the record of where the copy came from either way.
|
||||||
ResubmittedFromID *string `gorm:"type:uuid;index" json:"resubmittedFromId,omitempty"`
|
ResubmittedFromID *string `gorm:"type:uuid;index:idx_events_resubmitted_from_id,priority:1" json:"resubmittedFromId,omitempty"`
|
||||||
|
|
||||||
// Relations. No model marshals the record it belongs to, so
|
// Relations. No model marshals the record it belongs to, so
|
||||||
// Webhook and Entrypoint are left out of the JSON.
|
// Webhook and Entrypoint are left out of the JSON.
|
||||||
|
|||||||
Reference in New Issue
Block a user