027f0898e7
All checks were successful
check / check (push) Successful in 3m38s
An operator running `sqlite3 <db> .dump` against their own per-webhook database wedged it: 60 of 60 inbound webhooks rejected with HTTP 500, 206 delivered webhooks stranded at `pending`, and every one of them POSTed a second time on the next restart while the event log recorded a single attempt. Durability. Every SQLite file — main, per-webhook, and archive — now opens through one path, `internal/database/sqlite_open.go`, in WAL journal mode with a 10-second busy timeout, `BEGIN IMMEDIATE` transactions, and a bounded connection pool. WAL is what stops a reader blocking writers at all. `_txlock=immediate` is what stops a `COMMIT` failing while its transaction stays open on a pooled connection, which is how four `database is locked` errors became 593 `cannot start a transaction within a transaction`: a deferred transaction that upgrades to a write lock mid-flight gets SQLITE_BUSY without the busy handler being consulted. `cache=shared` is gone, because under it an in-process conflict is SQLITE_LOCKED, which the busy handler does not retry. Delivery. `recordResult` and `updateDeliveryStatus` return their errors instead of logging and dropping them, and a caller whose bookkeeping write failed writes nothing at all — the delivery keeps whichever non-terminal status it already held, and both sweeps recover it. Recovery and the sweep now reconcile before re-sending: a pending delivery that already holds a successful `DeliveryResult` is marked delivered rather than sent again, which is the state that did not previously exist. A delivery handed back out is claimed by compare-and-set so successive sweeps cannot send it repeatedly, and it continues its own attempt numbering instead of restarting at 1. The sweep gains a `pending`-with-age-bound arm, so a stranded delivery no longer waits for a restart. Docs. WAL produces `-wal`/`-shm` sidecars, so the backup and restore procedures in README.md are corrected: both documented procedures were re-run against a live instance, and a `-wal` left by a crash carries data the `.db` alone does not. Verified by reproducing the failure on unmodified `next` first — 6 targets, 60 events at 5/s, a concurrent `.dump` reader — which gave 38 HTTP 500s and 112 duplicate POSTs at the sinks across a restart. Both arms of the matched pair now show 0 inbound 500s, 0 engine write errors, and 0 new requests at the sinks after a restart, counted by payload.
412 lines
9.8 KiB
Go
412 lines
9.8 KiB
Go
package delivery_test
|
|
|
|
import (
|
|
"context"
|
|
"net/http"
|
|
"net/http/httptest"
|
|
"sync/atomic"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/google/uuid"
|
|
"github.com/stretchr/testify/assert"
|
|
"github.com/stretchr/testify/require"
|
|
"gorm.io/gorm"
|
|
"sneak.berlin/go/webhooker/internal/database"
|
|
"sneak.berlin/go/webhooker/internal/delivery"
|
|
)
|
|
|
|
// These tests cover the delivery half of
|
|
// https://git.eeqj.de/sneak/webhooker/issues/256: a delivery that
|
|
// reached its receiver but whose bookkeeping write failed used to be
|
|
// left at pending and re-sent on the next restart, giving the receiver
|
|
// a second copy while the event log recorded one attempt.
|
|
|
|
// rSeedResult records a DeliveryResult against a delivery, standing in
|
|
// for the attempt row the send path writes before the status.
|
|
func rSeedResult(
|
|
t *testing.T,
|
|
db *gorm.DB,
|
|
deliveryID string,
|
|
attemptNum int,
|
|
success bool,
|
|
) {
|
|
t.Helper()
|
|
|
|
require.NoError(t, db.Create(&database.DeliveryResult{
|
|
DeliveryID: deliveryID,
|
|
AttemptNum: attemptNum,
|
|
Success: success,
|
|
}).Error)
|
|
}
|
|
|
|
// rAgePending backdates a delivery past the sweep's age bound, which is
|
|
// what separates a stranded delivery from one a worker still holds.
|
|
func rAgePending(
|
|
t *testing.T, db *gorm.DB, deliveryID string,
|
|
) {
|
|
t.Helper()
|
|
|
|
old := time.Now().Add(
|
|
-2 * delivery.ExportPendingSweepMinAge,
|
|
)
|
|
|
|
require.NoError(t, db.Model(&database.Delivery{}).
|
|
Where("id = ?", deliveryID).
|
|
UpdateColumn("updated_at", old).Error)
|
|
}
|
|
|
|
func TestRecoverySkipsPendingWithSuccessfulResult(
|
|
t *testing.T,
|
|
) {
|
|
t.Parallel()
|
|
|
|
s := newISetup(t)
|
|
targetID := uuid.New().String()
|
|
|
|
iCreateTarget(t, s.MainDB, targetID,
|
|
s.WebhookID, "already-delivered",
|
|
database.TargetTypeLog, "", 0,
|
|
)
|
|
|
|
event := iSeedEvent(
|
|
t, s.WebhookDB, s.WebhookID, `{"delivered":true}`,
|
|
)
|
|
|
|
// The delivery whose send succeeded and whose result row landed:
|
|
// only the status write failed, so it sits at pending.
|
|
done := iSeedDelivery(
|
|
t, s.WebhookDB, event.ID, targetID,
|
|
database.DeliveryStatusPending,
|
|
)
|
|
rSeedResult(t, s.WebhookDB, done.ID, 1, true)
|
|
|
|
// A delivery that was genuinely never attempted.
|
|
fresh := iSeedDelivery(
|
|
t, s.WebhookDB, event.ID, targetID,
|
|
database.DeliveryStatusPending,
|
|
)
|
|
|
|
s.Engine.ExportRecoverPendingDeliveries(
|
|
context.Background(), s.WebhookDB, s.WebhookID,
|
|
)
|
|
|
|
select {
|
|
case task := <-s.Engine.ExportDeliveryCh():
|
|
assert.Equal(
|
|
t, fresh.ID, task.DeliveryID,
|
|
"only the unattempted delivery may be re-sent",
|
|
)
|
|
case <-time.After(2 * time.Second):
|
|
t.Fatal("expected the unattempted delivery")
|
|
}
|
|
|
|
select {
|
|
case task := <-s.Engine.ExportDeliveryCh():
|
|
t.Fatalf(
|
|
"re-sent an already delivered delivery: %s",
|
|
task.DeliveryID,
|
|
)
|
|
case <-time.After(200 * time.Millisecond):
|
|
}
|
|
|
|
// It is settled rather than merely skipped: leaving it pending
|
|
// would strand it again on the next sweep.
|
|
iAssertStatus(
|
|
t, s.WebhookDB, done.ID,
|
|
database.DeliveryStatusDelivered,
|
|
)
|
|
}
|
|
|
|
// TestRecoveryContinuesTheAttemptNumbering pins the audit trail: a
|
|
// recovered delivery that already recorded two attempts is re-sent as
|
|
// attempt three, not as attempt one again.
|
|
func TestRecoveryContinuesTheAttemptNumbering(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
s := newISetup(t)
|
|
targetID := uuid.New().String()
|
|
|
|
iCreateTarget(t, s.MainDB, targetID,
|
|
s.WebhookID, "numbering",
|
|
database.TargetTypeLog, "", 0,
|
|
)
|
|
|
|
event := iSeedEvent(
|
|
t, s.WebhookDB, s.WebhookID, `{"numbering":true}`,
|
|
)
|
|
|
|
d := iSeedDelivery(
|
|
t, s.WebhookDB, event.ID, targetID,
|
|
database.DeliveryStatusPending,
|
|
)
|
|
|
|
rSeedResult(t, s.WebhookDB, d.ID, 1, false)
|
|
rSeedResult(t, s.WebhookDB, d.ID, 2, false)
|
|
|
|
s.Engine.ExportRecoverPendingDeliveries(
|
|
context.Background(), s.WebhookDB, s.WebhookID,
|
|
)
|
|
|
|
select {
|
|
case task := <-s.Engine.ExportDeliveryCh():
|
|
assert.Equal(t, d.ID, task.DeliveryID)
|
|
assert.Equal(t, 3, task.AttemptNum)
|
|
case <-time.After(2 * time.Second):
|
|
t.Fatal("expected the delivery to be recovered")
|
|
}
|
|
}
|
|
|
|
// TestSweepRecoversStrandedPending is the half that removes the
|
|
// restart requirement: a delivery left at pending is picked up by the
|
|
// periodic sweep.
|
|
func TestSweepRecoversStrandedPending(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
s := newISetup(t)
|
|
targetID := uuid.New().String()
|
|
|
|
iCreateTarget(t, s.MainDB, targetID,
|
|
s.WebhookID, "stranded",
|
|
database.TargetTypeLog, "", 0,
|
|
)
|
|
|
|
require.NoError(t, s.MainDB.Create(&database.Webhook{
|
|
BaseModel: database.BaseModel{ID: s.WebhookID},
|
|
UserID: uuid.New().String(),
|
|
Name: "stranded",
|
|
}).Error)
|
|
|
|
event := iSeedEvent(
|
|
t, s.WebhookDB, s.WebhookID, `{"stranded":true}`,
|
|
)
|
|
|
|
stranded := iSeedDelivery(
|
|
t, s.WebhookDB, event.ID, targetID,
|
|
database.DeliveryStatusPending,
|
|
)
|
|
rAgePending(t, s.WebhookDB, stranded.ID)
|
|
|
|
// A delivery a worker may still be holding: young, and therefore
|
|
// none of the sweep's business.
|
|
inFlight := iSeedDelivery(
|
|
t, s.WebhookDB, event.ID, targetID,
|
|
database.DeliveryStatusPending,
|
|
)
|
|
|
|
s.Engine.ExportSweepWebhookRetries(
|
|
context.Background(), s.WebhookID,
|
|
)
|
|
|
|
select {
|
|
case task := <-s.Engine.ExportDeliveryCh():
|
|
assert.Equal(t, stranded.ID, task.DeliveryID)
|
|
case <-time.After(2 * time.Second):
|
|
t.Fatal("expected the stranded delivery")
|
|
}
|
|
|
|
select {
|
|
case task := <-s.Engine.ExportDeliveryCh():
|
|
t.Fatalf(
|
|
"swept an in-flight delivery: %s",
|
|
task.DeliveryID,
|
|
)
|
|
case <-time.After(200 * time.Millisecond):
|
|
}
|
|
|
|
iAssertStatus(
|
|
t, s.WebhookDB, inFlight.ID,
|
|
database.DeliveryStatusPending,
|
|
)
|
|
}
|
|
|
|
// TestSweepClaimsAStrandedDeliveryOnlyOnce guards the repeat the sweep
|
|
// would otherwise be: the row stays pending for as long as the attempt
|
|
// runs, and a sweep a minute later must not send it a second time.
|
|
func TestSweepClaimsAStrandedDeliveryOnlyOnce(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
s := newISetup(t)
|
|
targetID := uuid.New().String()
|
|
|
|
iCreateTarget(t, s.MainDB, targetID,
|
|
s.WebhookID, "claimed",
|
|
database.TargetTypeLog, "", 0,
|
|
)
|
|
|
|
require.NoError(t, s.MainDB.Create(&database.Webhook{
|
|
BaseModel: database.BaseModel{ID: s.WebhookID},
|
|
UserID: uuid.New().String(),
|
|
Name: "claimed",
|
|
}).Error)
|
|
|
|
event := iSeedEvent(
|
|
t, s.WebhookDB, s.WebhookID, `{"claimed":true}`,
|
|
)
|
|
|
|
d := iSeedDelivery(
|
|
t, s.WebhookDB, event.ID, targetID,
|
|
database.DeliveryStatusPending,
|
|
)
|
|
rAgePending(t, s.WebhookDB, d.ID)
|
|
|
|
ctx := context.Background()
|
|
|
|
s.Engine.ExportSweepWebhookRetries(ctx, s.WebhookID)
|
|
|
|
select {
|
|
case task := <-s.Engine.ExportDeliveryCh():
|
|
assert.Equal(t, d.ID, task.DeliveryID)
|
|
case <-time.After(2 * time.Second):
|
|
t.Fatal("expected the stranded delivery")
|
|
}
|
|
|
|
// The delivery is still pending — nothing has run it yet — but
|
|
// the claim must keep the next sweep off it.
|
|
iAssertStatus(
|
|
t, s.WebhookDB, d.ID,
|
|
database.DeliveryStatusPending,
|
|
)
|
|
|
|
s.Engine.ExportSweepWebhookRetries(ctx, s.WebhookID)
|
|
|
|
select {
|
|
case task := <-s.Engine.ExportDeliveryCh():
|
|
t.Fatalf(
|
|
"sent a claimed delivery again: %s",
|
|
task.DeliveryID,
|
|
)
|
|
case <-time.After(200 * time.Millisecond):
|
|
}
|
|
}
|
|
|
|
// TestSweepSettlesStrandedPendingWithoutResending is the sweep's own
|
|
// version of the reconcile: a stranded delivery holding a successful
|
|
// result is settled where it stands, and the receiver hears nothing.
|
|
func TestSweepSettlesStrandedPendingWithoutResending(
|
|
t *testing.T,
|
|
) {
|
|
t.Parallel()
|
|
|
|
s := newISetup(t)
|
|
targetID := uuid.New().String()
|
|
|
|
iCreateTarget(t, s.MainDB, targetID,
|
|
s.WebhookID, "settled",
|
|
database.TargetTypeLog, "", 0,
|
|
)
|
|
|
|
require.NoError(t, s.MainDB.Create(&database.Webhook{
|
|
BaseModel: database.BaseModel{ID: s.WebhookID},
|
|
UserID: uuid.New().String(),
|
|
Name: "settled",
|
|
}).Error)
|
|
|
|
event := iSeedEvent(
|
|
t, s.WebhookDB, s.WebhookID, `{"settled":true}`,
|
|
)
|
|
|
|
d := iSeedDelivery(
|
|
t, s.WebhookDB, event.ID, targetID,
|
|
database.DeliveryStatusPending,
|
|
)
|
|
rSeedResult(t, s.WebhookDB, d.ID, 1, true)
|
|
rAgePending(t, s.WebhookDB, d.ID)
|
|
|
|
s.Engine.ExportSweepWebhookRetries(
|
|
context.Background(), s.WebhookID,
|
|
)
|
|
|
|
select {
|
|
case task := <-s.Engine.ExportDeliveryCh():
|
|
t.Fatalf(
|
|
"re-sent a delivery that already succeeded: %s",
|
|
task.DeliveryID,
|
|
)
|
|
case <-time.After(200 * time.Millisecond):
|
|
}
|
|
|
|
iAssertStatus(
|
|
t, s.WebhookDB, d.ID,
|
|
database.DeliveryStatusDelivered,
|
|
)
|
|
|
|
var attempts int64
|
|
|
|
require.NoError(t, s.WebhookDB.
|
|
Model(&database.DeliveryResult{}).
|
|
Where("delivery_id = ?", d.ID).
|
|
Count(&attempts).Error)
|
|
assert.Equal(
|
|
t, int64(1), attempts,
|
|
"settling must not invent an attempt",
|
|
)
|
|
}
|
|
|
|
// TestFailedResultWriteLeavesDeliveryRecoverable is the rule the
|
|
// targets now follow: a bookkeeping write that fails must not advance
|
|
// the status, because pending and retrying are the states the sweeps
|
|
// recover and delivered is a claim the database refused to record.
|
|
func TestFailedResultWriteLeavesDeliveryRecoverable(
|
|
t *testing.T,
|
|
) {
|
|
t.Parallel()
|
|
|
|
s := newISetup(t)
|
|
targetID := uuid.New().String()
|
|
|
|
var hits atomic.Int64
|
|
|
|
ts := httptest.NewServer(http.HandlerFunc(
|
|
func(w http.ResponseWriter, _ *http.Request) {
|
|
hits.Add(1)
|
|
w.WriteHeader(http.StatusOK)
|
|
},
|
|
))
|
|
defer ts.Close()
|
|
|
|
event := iSeedEvent(
|
|
t, s.WebhookDB, s.WebhookID, `{"unwritable":true}`,
|
|
)
|
|
|
|
d := iSeedDelivery(
|
|
t, s.WebhookDB, event.ID, targetID,
|
|
database.DeliveryStatusPending,
|
|
)
|
|
|
|
// Drop the table the attempt row goes in, so the send succeeds
|
|
// and only the bookkeeping write fails.
|
|
require.NoError(
|
|
t,
|
|
s.WebhookDB.Exec("drop table delivery_results").Error,
|
|
)
|
|
|
|
full := &database.Delivery{
|
|
EventID: event.ID,
|
|
TargetID: targetID,
|
|
Status: database.DeliveryStatusPending,
|
|
Event: event,
|
|
Target: database.Target{
|
|
Name: "unwritable",
|
|
Type: database.TargetTypeHTTP,
|
|
Config: iHTTPConfig(ts.URL),
|
|
},
|
|
}
|
|
full.ID = d.ID
|
|
|
|
s.Engine.ExportDeliverHTTP(
|
|
context.Background(), s.WebhookDB, full,
|
|
&delivery.Task{DeliveryID: d.ID, AttemptNum: 1},
|
|
)
|
|
|
|
assert.Equal(
|
|
t, int64(1), hits.Load(),
|
|
"the send itself must still happen",
|
|
)
|
|
|
|
iAssertStatus(
|
|
t, s.WebhookDB, d.ID,
|
|
database.DeliveryStatusPending,
|
|
)
|
|
}
|