Make SQLite durable under concurrent readers and stop re-delivering stranded webhooks (closes #256)

An operator running `sqlite3 <db> .dump` against their own per-webhook
database wedged it: inbound webhooks rejected with HTTP 500, delivered
webhooks stranded at `pending`, and every one of them POSTed a second
time on the next restart while the event log recorded a single attempt.

Durability. Every SQLite file — main, per-webhook, and archive — now
opens through one path, `internal/database/sqlite_open.go`, in WAL
journal mode with a 10-second busy timeout, `BEGIN IMMEDIATE`
transactions, and a bounded connection pool. WAL is what stops a reader
blocking writers at all. `_txlock=immediate` is what stops a `COMMIT`
failing while its transaction stays open on a pooled connection, which
is how four `database is locked` errors became 593 `cannot start a
transaction within a transaction`. `cache=shared` is gone, because
under it an in-process conflict is SQLITE_LOCKED, which the busy
handler does not retry. The busy timeout is applied before
journal_mode: the driver runs DSN pragmas in order on every new
connection, and `PRAGMA journal_mode` takes a lock, so the reverse
order leaves the one pragma that can block uncovered by the handler
meant to cover it.

Eligibility. `internal/delivery/inflight.go` holds the set of
deliveries the engine owns — taken when a task is queued, when a
target schedules a retry, and by every recovery path before it
re-dispatches; dropped when the worker that ran the task returns.
Recovery and both sweep arms re-dispatch only what the set does not
hold. Nothing decides that from a row's age: a delivery waiting in a
10000-deep channel is arbitrarily old and perfectly healthy, and
reasoning from age re-sends it. `takeForRedispatch` is the single gate
every re-dispatch goes through — ownership first, then a conditional
update confirming the row is still in the status the batch read.

Bookkeeping. `recordResult` and `updateDeliveryStatus` return their
errors instead of logging and dropping them, and a caller whose
bookkeeping write failed writes nothing at all: the delivery keeps
whichever non-terminal status it already held, and the sweeps recover
it. Every recovery path — pending and retrying alike — first settles
any delivery that already holds a successful `DeliveryResult` rather
than sending it again. Recovery continues each delivery's own attempt
numbering instead of restarting at 1. The sweep gains a
`pending`-with-age-bound arm, so a stranded delivery no longer waits
for a restart.

Docs. WAL produces `-wal`/`-shm` sidecars, so the backup and restore
procedures in README.md are corrected against measurement: both
documented procedures were re-run against a live instance, a `-wal`
left by a crash carries data the `.db` alone does not, and an archive
file normally holds its rows in a `-wal` rather than in the `.db`.

internal/database/database.go

@@ -4,7 +4,6 @@ package database
 import (
 	"context"
 	"crypto/rand"
-	"database/sql"
 	"encoding/base64"
 	"errors"
 	"fmt"
@@ -16,7 +15,6 @@ import (
 	"go.uber.org/fx"
 	"gorm.io/driver/sqlite"
 	"gorm.io/gorm"
-	_ "modernc.org/sqlite" // Pure Go SQLite driver
 	"sneak.berlin/go/webhooker/internal/banner"
 	"sneak.berlin/go/webhooker/internal/config"
 	"sneak.berlin/go/webhooker/internal/gormlog"
@@ -198,13 +196,11 @@ func (d *Database) connectTo(dataDir string) error {
 
 	// Construct the main application database path inside DATA_DIR.
 	dbPath := filepath.Join(dataDir, MainDBFileName)
-	dbURL := fmt.Sprintf(
-		"file:%s?cache=shared&mode=rwc",
-		dbPath,
-	)
 
-	// Open the database with the pure Go SQLite driver
-	sqlDB, err := sql.Open("sqlite", dbURL)
+	// Opened through OpenSQLite so this handle carries the same WAL
+	// journaling, busy timeout, immediate-transaction locking, and pool
+	// bounds as every other database file. See sqlite_open.go.
+	sqlDB, err := OpenSQLite(dbPath, SQLiteModeCreate)
 	if err != nil {
 		d.log.Error(
 			"failed to open database",

internal/database/sqlite_open.go

@@ -0,0 +1,157 @@
+package database
+
+import (
+	"database/sql"
+	"fmt"
+	"net/url"
+	"time"
+
+	_ "modernc.org/sqlite" // Pure Go SQLite driver
+)
+
+// Every SQLite file this service opens — the main database, the
+// per-webhook event databases, and the archive databases — is opened
+// through OpenSQLite, so the durability settings below are properties
+// of the service rather than of one call site.
+//
+// modernc.org/sqlite installs no busy handler and issues no pragmas of
+// its own: it executes only the pragmas named in explicit `_pragma=`
+// DSN parameters, and gorm.io/driver/sqlite adds none when it is
+// handed an existing *sql.DB. Every setting therefore has to be
+// spelled out here or it is simply not in effect.
+// SQLite URI open modes.
+const (
+	// SQLiteModeCreate creates the database file when it is missing.
+	SQLiteModeCreate = "rwc"
+
+	// SQLiteModeExisting requires the file to exist already.
+	SQLiteModeExisting = "rw"
+)
+
+const (
+	// SQLiteBusyTimeout is how long SQLite retries a lock conflict
+	// before returning SQLITE_BUSY.
+	//
+	// Under WAL a reader never blocks a writer, so the only conflict
+	// left is writer against writer: this process's delivery workers
+	// against each other, or against another process holding the write
+	// lock. Those clear in milliseconds. Ten seconds is far above that
+	// and still well inside the receiver's request budget, so an
+	// inbound webhook waits rather than being rejected with a 500.
+	SQLiteBusyTimeout = 10 * time.Second
+
+	// sqliteMaxOpenConns bounds the connection pool for one database
+	// file.
+	//
+	// The pool needs a bound at all because database/sql cannot detect
+	// a connection left mid-transaction: modernc.org/sqlite implements
+	// neither driver.Validator nor driver.SessionResetter, so a
+	// connection whose COMMIT failed is returned to the pool with its
+	// transaction still open and handed out again indefinitely. That is
+	// what turned four `database is locked` errors into 593
+	// `cannot start a transaction within a transaction` in
+	// https://git.eeqj.de/sneak/webhooker/issues/256.
+	//
+	// Four is above the one writer SQLite allows at a time, so reads
+	// still proceed while a write is in flight, and low enough that
+	// contention is resolved by the busy handler rather than by piling
+	// up connections against a lock only one of them can hold.
+	sqliteMaxOpenConns = 4
+
+	// sqliteMaxIdleConns keeps the pool warm without holding every
+	// connection open through an idle period.
+	sqliteMaxIdleConns = 2
+
+	// sqliteConnMaxLifetime and sqliteConnMaxIdleTime retire pooled
+	// connections on a schedule. With _txlock=immediate a failed
+	// COMMIT should no longer be reachable, but these bound the damage
+	// if one happens anyway: a poisoned connection is closed and
+	// replaced within the lifetime instead of wedging the file until
+	// the process restarts.
+	sqliteConnMaxLifetime = 5 * time.Minute
+	sqliteConnMaxIdleTime = time.Minute
+)
+
+// SQLiteDSN builds the connection string for one database file.
+//
+// mode is the SQLite URI open mode: "rwc" to create the file when it
+// is missing, "rw" to require that it already exists.
+//
+// Three settings carry the fix for
+// https://git.eeqj.de/sneak/webhooker/issues/256 and none of them is
+// optional:
+//
+//   - journal_mode=WAL, so a reader — an operator running
+//     `sqlite3 <db> .dump` over their own data — takes a snapshot
+//     instead of blocking every writer behind it.
+//
+//   - busy_timeout, so a writer that does meet a lock waits for it.
+//     Without one SQLite gives up immediately; nothing above it
+//     retries.
+//
+//   - _txlock=immediate, so every transaction takes the write lock at
+//     BEGIN. A deferred transaction acquires it lazily on its first
+//     write, and that upgrade returns SQLITE_BUSY *without* consulting
+//     the busy handler, because SQLite cannot block a transaction that
+//     may already hold a read snapshot. Such a COMMIT then fails while
+//     the transaction stays open on the connection. A busy timeout
+//     alone does not prevent this; BEGIN IMMEDIATE does, by putting
+//     the wait somewhere the handler applies.
+//
+// Note what is absent: `cache=shared`. Under a shared cache an
+// in-process conflict is reported as SQLITE_LOCKED rather than
+// SQLITE_BUSY, and the busy handler does not retry SQLITE_LOCKED — so
+// leaving it in would have defeated the busy timeout for exactly the
+// contention this service generates. Dropping it is part of the fix,
+// not housekeeping.
+//
+// synchronous is deliberately left at SQLite's default of FULL: this
+// is a webhook receiver whose one promise is that an event it answered
+// 200 for is durable.
+// The order of the _pragma parameters is load-bearing.
+// modernc.org/sqlite executes them in the order they appear, on every
+// new connection, before the connection is handed to the pool. Setting
+// journal_mode first means that pragma itself runs with no busy
+// handler installed: the pool opens connections lazily, so the moment
+// a new one is created is a moment the database is under load, and
+// PRAGMA journal_mode takes a lock. It would fail immediately with
+// SQLITE_BUSY and fail the query that caused the connection to be
+// opened. busy_timeout is therefore set first, so every pragma after
+// it — and the whole life of the connection — is covered.
+func SQLiteDSN(path, mode string) string {
+	q := url.Values{}
+	q.Set("mode", mode)
+	q.Set("_txlock", "immediate")
+	q.Add(
+		"_pragma",
+		fmt.Sprintf(
+			"busy_timeout(%d)",
+			SQLiteBusyTimeout.Milliseconds(),
+		),
+	)
+	q.Add("_pragma", "journal_mode(WAL)")
+
+	return "file:" + path + "?" + q.Encode()
+}
+
+// OpenSQLite opens the SQLite file at path with the service's
+// durability settings and pool bounds applied. mode is the SQLite URI
+// open mode ("rwc" or "rw").
+//
+// The handle is returned rather than a *gorm.DB because the callers
+// wrap it in gorm themselves with their own logger.
+func OpenSQLite(path, mode string) (*sql.DB, error) {
+	sqlDB, err := sql.Open("sqlite", SQLiteDSN(path, mode))
+	if err != nil {
+		return nil, fmt.Errorf(
+			"opening sqlite database %s: %w", path, err,
+		)
+	}
+
+	sqlDB.SetMaxOpenConns(sqliteMaxOpenConns)
+	sqlDB.SetMaxIdleConns(sqliteMaxIdleConns)
+	sqlDB.SetConnMaxLifetime(sqliteConnMaxLifetime)
+	sqlDB.SetConnMaxIdleTime(sqliteConnMaxIdleTime)
+
+	return sqlDB, nil
+}

internal/database/sqlite_open_test.go

@@ -0,0 +1,178 @@
+package database_test
+
+import (
+	"context"
+	"path/filepath"
+	"strings"
+	"testing"
+
+	"github.com/google/uuid"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"gorm.io/gorm"
+	"sneak.berlin/go/webhooker/internal/database"
+)
+
+// livePragma reads a pragma off a live handle. Reading the DSN back
+// would prove only that the string was built; these tests assert that
+// SQLite actually applied it.
+func livePragma(t *testing.T, db *gorm.DB, name string) string {
+	t.Helper()
+
+	var v string
+
+	row := db.Raw("pragma " + name).Row()
+	require.NoError(t, row.Scan(&v))
+
+	return v
+}
+
+func TestSQLiteDSNCarriesTheDurabilitySettings(t *testing.T) {
+	t.Parallel()
+
+	dsn := database.SQLiteDSN(
+		"/var/lib/webhooker/webhooker.db",
+		database.SQLiteModeCreate,
+	)
+
+	assert.Contains(t, dsn, "journal_mode%28WAL%29")
+	assert.Contains(t, dsn, "busy_timeout%2810000%29")
+	assert.Contains(t, dsn, "_txlock=immediate")
+	assert.Contains(t, dsn, "mode=rwc")
+
+	// busy_timeout must come first. The driver runs these in order on
+	// every new connection, and PRAGMA journal_mode takes a lock — a
+	// connection opened while the database is busy would fail on that
+	// pragma, with no busy handler yet installed to wait it out.
+	assert.Less(
+		t,
+		strings.Index(dsn, "busy_timeout"),
+		strings.Index(dsn, "journal_mode"),
+		"busy_timeout must be applied before journal_mode",
+	)
+
+	// cache=shared turns an in-process conflict into SQLITE_LOCKED,
+	// which the busy handler does not retry. It must never come back.
+	// See https://git.eeqj.de/sneak/webhooker/issues/256.
+	assert.NotContains(t, strings.ToLower(dsn), "cache=shared")
+}
+
+// TestPerWebhookDBAppliesPragmasOnALiveHandle is the check the issue
+// asks for by name: the settings are confirmed by querying the running
+// database, not by inspecting the connection string.
+func TestPerWebhookDBAppliesPragmasOnALiveHandle(t *testing.T) {
+	t.Parallel()
+
+	mgr, lc := setupTestWebhookDBManager(t)
+	ctx := context.Background()
+	require.NoError(t, lc.Start(ctx))
+
+	defer func() { require.NoError(t, lc.Stop(ctx)) }()
+
+	webhookID := uuid.New().String()
+
+	db, err := mgr.GetDB(webhookID)
+	require.NoError(t, err)
+
+	assert.Equal(
+		t, "wal",
+		strings.ToLower(livePragma(t, db, "journal_mode")),
+	)
+	assert.Equal(
+		t, "10000", livePragma(t, db, "busy_timeout"),
+	)
+}
+
+func TestMainDBAppliesPragmasOnALiveHandle(t *testing.T) {
+	t.Parallel()
+
+	ctx := context.Background()
+	dir := t.TempDir()
+
+	sqlDB, err := database.OpenSQLite(
+		filepath.Join(dir, database.MainDBFileName),
+		database.SQLiteModeCreate,
+	)
+	require.NoError(t, err)
+
+	defer func() { require.NoError(t, sqlDB.Close()) }()
+
+	var journal string
+
+	require.NoError(t, sqlDB.
+		QueryRowContext(ctx, "pragma journal_mode").
+		Scan(&journal))
+	assert.Equal(t, "wal", strings.ToLower(journal))
+
+	var busy string
+
+	require.NoError(t, sqlDB.
+		QueryRowContext(ctx, "pragma busy_timeout").
+		Scan(&busy))
+	assert.Equal(t, "10000", busy)
+}
+
+// TestConcurrentReaderDoesNotBlockWrites is the unit-scale form of the
+// reproduction in
+// https://git.eeqj.de/sneak/webhooker/issues/256: an operator's
+// long-held read of their own data used to make every concurrent write
+// fail. Under WAL the reader takes a snapshot and the writes proceed.
+func TestConcurrentReaderDoesNotBlockWrites(t *testing.T) {
+	t.Parallel()
+
+	mgr, lc := setupTestWebhookDBManager(t)
+	ctx := context.Background()
+	require.NoError(t, lc.Start(ctx))
+
+	defer func() { require.NoError(t, lc.Stop(ctx)) }()
+
+	webhookID := uuid.New().String()
+
+	db, err := mgr.GetDB(webhookID)
+	require.NoError(t, err)
+
+	// A second handle on the same file, holding a read transaction
+	// open across every write below — what `sqlite3 <db> .dump` is.
+	readerSQL, err := database.OpenSQLite(
+		mgr.DBPath(webhookID), database.SQLiteModeExisting,
+	)
+	require.NoError(t, err)
+
+	defer func() { require.NoError(t, readerSQL.Close()) }()
+
+	readerConn, err := readerSQL.Conn(ctx)
+	require.NoError(t, err)
+
+	defer func() { require.NoError(t, readerConn.Close()) }()
+
+	_, err = readerConn.ExecContext(ctx, "begin deferred")
+	require.NoError(t, err)
+
+	_, err = readerConn.ExecContext(
+		ctx, "select count(*) from events",
+	)
+	require.NoError(t, err)
+
+	for range 25 {
+		err = db.Transaction(func(tx *gorm.DB) error {
+			return tx.Create(&database.Event{
+				WebhookID:    webhookID,
+				EntrypointID: uuid.New().String(),
+				Method:       "POST",
+				Body:         "{}",
+			}).Error
+		})
+		require.NoError(t, err)
+	}
+
+	_, err = readerConn.ExecContext(ctx, "commit")
+	require.NoError(t, err)
+
+	var count int64
+
+	require.NoError(
+		t,
+		db.Model(&database.Event{}).Count(&count).Error,
+	)
+	assert.Equal(t, int64(25), count)
+}

internal/database/webhook_db_manager.go

@@ -2,7 +2,6 @@ package database
 
 import (
 	"context"
-	"database/sql"
 	"errors"
 	"fmt"
 	"log/slog"
@@ -234,12 +233,11 @@ func (m *WebhookDBManager) openDB(
 	webhookID string,
 ) (*gorm.DB, error) {
 	path := m.dbPath(webhookID)
-	dbURL := fmt.Sprintf(
-		"file:%s?cache=shared&mode=rwc",
-		path,
-	)
 
-	sqlDB, err := sql.Open("sqlite", dbURL)
+	// See sqlite_open.go: WAL, a busy timeout, immediate-transaction
+	// locking, and a bounded pool, all of which this file needs most —
+	// it is the one every delivery worker writes to concurrently.
+	sqlDB, err := OpenSQLite(path, SQLiteModeCreate)
 	if err != nil {
 		return nil, fmt.Errorf(
 			"opening webhook database %s: %w",