Make SQLite durable under concurrent readers and stop re-delivering stranded webhooks (closes #256)

An operator running `sqlite3 <db> .dump` against their own per-webhook
database wedged it: inbound webhooks rejected with HTTP 500, delivered
webhooks stranded at `pending`, and every one of them POSTed a second
time on the next restart while the event log recorded a single attempt.

Durability. Every SQLite file — main, per-webhook, and archive — now
opens through one path, `internal/database/sqlite_open.go`, in WAL
journal mode with a 10-second busy timeout, `BEGIN IMMEDIATE`
transactions, and a bounded connection pool. WAL is what stops a reader
blocking writers at all. `_txlock=immediate` is what stops a `COMMIT`
failing while its transaction stays open on a pooled connection, which
is how four `database is locked` errors became 593 `cannot start a
transaction within a transaction`. `cache=shared` is gone, because
under it an in-process conflict is SQLITE_LOCKED, which the busy
handler does not retry. The busy timeout is applied before
journal_mode: the driver runs DSN pragmas in order on every new
connection, and `PRAGMA journal_mode` takes a lock, so the reverse
order leaves the one pragma that can block uncovered by the handler
meant to cover it.

Eligibility. `internal/delivery/inflight.go` holds the set of
deliveries the engine owns — taken when a task is queued, when a
target schedules a retry, and by every recovery path before it
re-dispatches; dropped when the worker that ran the task returns.
Recovery and both sweep arms re-dispatch only what the set does not
hold. Nothing decides that from a row's age: a delivery waiting in a
10000-deep channel is arbitrarily old and perfectly healthy, and
reasoning from age re-sends it. `takeForRedispatch` is the single gate
every re-dispatch goes through — ownership first, then a conditional
update confirming the row is still in the status the batch read.

Bookkeeping. `recordResult` and `updateDeliveryStatus` return their
errors instead of logging and dropping them, and a caller whose
bookkeeping write failed writes nothing at all: the delivery keeps
whichever non-terminal status it already held, and the sweeps recover
it. Every recovery path — pending and retrying alike — first settles
any delivery that already holds a successful `DeliveryResult` rather
than sending it again. Recovery continues each delivery's own attempt
numbering instead of restarting at 1. The sweep gains a
`pending`-with-age-bound arm, so a stranded delivery no longer waits
for a restart.

Docs. WAL produces `-wal`/`-shm` sidecars, so the backup and restore
procedures in README.md are corrected against measurement: both
documented procedures were re-run against a live instance, a `-wal`
left by a crash carries data the `.db` alone does not, and an archive
file normally holds its rows in a `-wal` rather than in the `.db`.

internal/database/sqlite_open_test.go

@@ -0,0 +1,178 @@
+package database_test
+
+import (
+	"context"
+	"path/filepath"
+	"strings"
+	"testing"
+
+	"github.com/google/uuid"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"gorm.io/gorm"
+	"sneak.berlin/go/webhooker/internal/database"
+)
+
+// livePragma reads a pragma off a live handle. Reading the DSN back
+// would prove only that the string was built; these tests assert that
+// SQLite actually applied it.
+func livePragma(t *testing.T, db *gorm.DB, name string) string {
+	t.Helper()
+
+	var v string
+
+	row := db.Raw("pragma " + name).Row()
+	require.NoError(t, row.Scan(&v))
+
+	return v
+}
+
+func TestSQLiteDSNCarriesTheDurabilitySettings(t *testing.T) {
+	t.Parallel()
+
+	dsn := database.SQLiteDSN(
+		"/var/lib/webhooker/webhooker.db",
+		database.SQLiteModeCreate,
+	)
+
+	assert.Contains(t, dsn, "journal_mode%28WAL%29")
+	assert.Contains(t, dsn, "busy_timeout%2810000%29")
+	assert.Contains(t, dsn, "_txlock=immediate")
+	assert.Contains(t, dsn, "mode=rwc")
+
+	// busy_timeout must come first. The driver runs these in order on
+	// every new connection, and PRAGMA journal_mode takes a lock — a
+	// connection opened while the database is busy would fail on that
+	// pragma, with no busy handler yet installed to wait it out.
+	assert.Less(
+		t,
+		strings.Index(dsn, "busy_timeout"),
+		strings.Index(dsn, "journal_mode"),
+		"busy_timeout must be applied before journal_mode",
+	)
+
+	// cache=shared turns an in-process conflict into SQLITE_LOCKED,
+	// which the busy handler does not retry. It must never come back.
+	// See https://git.eeqj.de/sneak/webhooker/issues/256.
+	assert.NotContains(t, strings.ToLower(dsn), "cache=shared")
+}
+
+// TestPerWebhookDBAppliesPragmasOnALiveHandle is the check the issue
+// asks for by name: the settings are confirmed by querying the running
+// database, not by inspecting the connection string.
+func TestPerWebhookDBAppliesPragmasOnALiveHandle(t *testing.T) {
+	t.Parallel()
+
+	mgr, lc := setupTestWebhookDBManager(t)
+	ctx := context.Background()
+	require.NoError(t, lc.Start(ctx))
+
+	defer func() { require.NoError(t, lc.Stop(ctx)) }()
+
+	webhookID := uuid.New().String()
+
+	db, err := mgr.GetDB(webhookID)
+	require.NoError(t, err)
+
+	assert.Equal(
+		t, "wal",
+		strings.ToLower(livePragma(t, db, "journal_mode")),
+	)
+	assert.Equal(
+		t, "10000", livePragma(t, db, "busy_timeout"),
+	)
+}
+
+func TestMainDBAppliesPragmasOnALiveHandle(t *testing.T) {
+	t.Parallel()
+
+	ctx := context.Background()
+	dir := t.TempDir()
+
+	sqlDB, err := database.OpenSQLite(
+		filepath.Join(dir, database.MainDBFileName),
+		database.SQLiteModeCreate,
+	)
+	require.NoError(t, err)
+
+	defer func() { require.NoError(t, sqlDB.Close()) }()
+
+	var journal string
+
+	require.NoError(t, sqlDB.
+		QueryRowContext(ctx, "pragma journal_mode").
+		Scan(&journal))
+	assert.Equal(t, "wal", strings.ToLower(journal))
+
+	var busy string
+
+	require.NoError(t, sqlDB.
+		QueryRowContext(ctx, "pragma busy_timeout").
+		Scan(&busy))
+	assert.Equal(t, "10000", busy)
+}
+
+// TestConcurrentReaderDoesNotBlockWrites is the unit-scale form of the
+// reproduction in
+// https://git.eeqj.de/sneak/webhooker/issues/256: an operator's
+// long-held read of their own data used to make every concurrent write
+// fail. Under WAL the reader takes a snapshot and the writes proceed.
+func TestConcurrentReaderDoesNotBlockWrites(t *testing.T) {
+	t.Parallel()
+
+	mgr, lc := setupTestWebhookDBManager(t)
+	ctx := context.Background()
+	require.NoError(t, lc.Start(ctx))
+
+	defer func() { require.NoError(t, lc.Stop(ctx)) }()
+
+	webhookID := uuid.New().String()
+
+	db, err := mgr.GetDB(webhookID)
+	require.NoError(t, err)
+
+	// A second handle on the same file, holding a read transaction
+	// open across every write below — what `sqlite3 <db> .dump` is.
+	readerSQL, err := database.OpenSQLite(
+		mgr.DBPath(webhookID), database.SQLiteModeExisting,
+	)
+	require.NoError(t, err)
+
+	defer func() { require.NoError(t, readerSQL.Close()) }()
+
+	readerConn, err := readerSQL.Conn(ctx)
+	require.NoError(t, err)
+
+	defer func() { require.NoError(t, readerConn.Close()) }()
+
+	_, err = readerConn.ExecContext(ctx, "begin deferred")
+	require.NoError(t, err)
+
+	_, err = readerConn.ExecContext(
+		ctx, "select count(*) from events",
+	)
+	require.NoError(t, err)
+
+	for range 25 {
+		err = db.Transaction(func(tx *gorm.DB) error {
+			return tx.Create(&database.Event{
+				WebhookID:    webhookID,
+				EntrypointID: uuid.New().String(),
+				Method:       "POST",
+				Body:         "{}",
+			}).Error
+		})
+		require.NoError(t, err)
+	}
+
+	_, err = readerConn.ExecContext(ctx, "commit")
+	require.NoError(t, err)
+
+	var count int64
+
+	require.NoError(
+		t,
+		db.Model(&database.Event{}).Count(&count).Error,
+	)
+	assert.Equal(t, int64(25), count)
+}