Make SQLite durable under concurrent readers and stop re-delivering stranded webhooks (closes #256)

An operator running `sqlite3 <db> .dump` against their own per-webhook
database wedged it: 60 of 60 inbound webhooks rejected with HTTP 500,
206 delivered webhooks stranded at `pending`, and every one of them
POSTed a second time on the next restart while the event log recorded a
single attempt.

Durability. Every SQLite file — main, per-webhook, and archive — now
opens through one path, `internal/database/sqlite_open.go`, in WAL
journal mode with a 10-second busy timeout, `BEGIN IMMEDIATE`
transactions, and a bounded connection pool. WAL is what stops a
reader blocking writers at all. `_txlock=immediate` is what stops a
`COMMIT` failing while its transaction stays open on a pooled
connection, which is how four `database is locked` errors became 593
`cannot start a transaction within a transaction`: a deferred
transaction that upgrades to a write lock mid-flight gets SQLITE_BUSY
without the busy handler being consulted. `cache=shared` is gone,
because under it an in-process conflict is SQLITE_LOCKED, which the
busy handler does not retry.

Delivery. `recordResult` and `updateDeliveryStatus` return their
errors instead of logging and dropping them, and a caller whose
bookkeeping write failed writes nothing at all — the delivery keeps
whichever non-terminal status it already held, and both sweeps recover
it. Recovery and the sweep now reconcile before re-sending: a pending
delivery that already holds a successful `DeliveryResult` is marked
delivered rather than sent again, which is the state that did not
previously exist. A delivery handed back out is claimed by
compare-and-set so successive sweeps cannot send it repeatedly, and it
continues its own attempt numbering instead of restarting at 1. The
sweep gains a `pending`-with-age-bound arm, so a stranded delivery no
longer waits for a restart.

Docs. WAL produces `-wal`/`-shm` sidecars, so the backup and restore
procedures in README.md are corrected: both documented procedures were
re-run against a live instance, and a `-wal` left by a crash carries
data the `.db` alone does not.

Verified by reproducing the failure on unmodified `next` first — 6
targets, 60 events at 5/s, a concurrent `.dump` reader — which gave 38
HTTP 500s and 112 duplicate POSTs at the sinks across a restart. Both
arms of the matched pair now show 0 inbound 500s, 0 engine write
errors, and 0 new requests at the sinks after a restart, counted by
payload.

internal/database/sqlite_open_test.go

@@ -0,0 +1,167 @@
+package database_test
+
+import (
+	"context"
+	"path/filepath"
+	"strings"
+	"testing"
+
+	"github.com/google/uuid"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"gorm.io/gorm"
+	"sneak.berlin/go/webhooker/internal/database"
+)
+
+// livePragma reads a pragma off a live handle. Reading the DSN back
+// would prove only that the string was built; these tests assert that
+// SQLite actually applied it.
+func livePragma(t *testing.T, db *gorm.DB, name string) string {
+	t.Helper()
+
+	var v string
+
+	row := db.Raw("pragma " + name).Row()
+	require.NoError(t, row.Scan(&v))
+
+	return v
+}
+
+func TestSQLiteDSNCarriesTheDurabilitySettings(t *testing.T) {
+	t.Parallel()
+
+	dsn := database.SQLiteDSN(
+		"/var/lib/webhooker/webhooker.db",
+		database.SQLiteModeCreate,
+	)
+
+	assert.Contains(t, dsn, "journal_mode%28WAL%29")
+	assert.Contains(t, dsn, "busy_timeout%2810000%29")
+	assert.Contains(t, dsn, "_txlock=immediate")
+	assert.Contains(t, dsn, "mode=rwc")
+
+	// cache=shared turns an in-process conflict into SQLITE_LOCKED,
+	// which the busy handler does not retry. It must never come back.
+	// See https://git.eeqj.de/sneak/webhooker/issues/256.
+	assert.NotContains(t, strings.ToLower(dsn), "cache=shared")
+}
+
+// TestPerWebhookDBAppliesPragmasOnALiveHandle is the check the issue
+// asks for by name: the settings are confirmed by querying the running
+// database, not by inspecting the connection string.
+func TestPerWebhookDBAppliesPragmasOnALiveHandle(t *testing.T) {
+	t.Parallel()
+
+	mgr, lc := setupTestWebhookDBManager(t)
+	ctx := context.Background()
+	require.NoError(t, lc.Start(ctx))
+
+	defer func() { require.NoError(t, lc.Stop(ctx)) }()
+
+	webhookID := uuid.New().String()
+
+	db, err := mgr.GetDB(webhookID)
+	require.NoError(t, err)
+
+	assert.Equal(
+		t, "wal",
+		strings.ToLower(livePragma(t, db, "journal_mode")),
+	)
+	assert.Equal(
+		t, "10000", livePragma(t, db, "busy_timeout"),
+	)
+}
+
+func TestMainDBAppliesPragmasOnALiveHandle(t *testing.T) {
+	t.Parallel()
+
+	ctx := context.Background()
+	dir := t.TempDir()
+
+	sqlDB, err := database.OpenSQLite(
+		filepath.Join(dir, database.MainDBFileName),
+		database.SQLiteModeCreate,
+	)
+	require.NoError(t, err)
+
+	defer func() { require.NoError(t, sqlDB.Close()) }()
+
+	var journal string
+
+	require.NoError(t, sqlDB.
+		QueryRowContext(ctx, "pragma journal_mode").
+		Scan(&journal))
+	assert.Equal(t, "wal", strings.ToLower(journal))
+
+	var busy string
+
+	require.NoError(t, sqlDB.
+		QueryRowContext(ctx, "pragma busy_timeout").
+		Scan(&busy))
+	assert.Equal(t, "10000", busy)
+}
+
+// TestConcurrentReaderDoesNotBlockWrites is the unit-scale form of the
+// reproduction in
+// https://git.eeqj.de/sneak/webhooker/issues/256: an operator's
+// long-held read of their own data used to make every concurrent write
+// fail. Under WAL the reader takes a snapshot and the writes proceed.
+func TestConcurrentReaderDoesNotBlockWrites(t *testing.T) {
+	t.Parallel()
+
+	mgr, lc := setupTestWebhookDBManager(t)
+	ctx := context.Background()
+	require.NoError(t, lc.Start(ctx))
+
+	defer func() { require.NoError(t, lc.Stop(ctx)) }()
+
+	webhookID := uuid.New().String()
+
+	db, err := mgr.GetDB(webhookID)
+	require.NoError(t, err)
+
+	// A second handle on the same file, holding a read transaction
+	// open across every write below — what `sqlite3 <db> .dump` is.
+	readerSQL, err := database.OpenSQLite(
+		mgr.DBPath(webhookID), database.SQLiteModeExisting,
+	)
+	require.NoError(t, err)
+
+	defer func() { require.NoError(t, readerSQL.Close()) }()
+
+	readerConn, err := readerSQL.Conn(ctx)
+	require.NoError(t, err)
+
+	defer func() { require.NoError(t, readerConn.Close()) }()
+
+	_, err = readerConn.ExecContext(ctx, "begin deferred")
+	require.NoError(t, err)
+
+	_, err = readerConn.ExecContext(
+		ctx, "select count(*) from events",
+	)
+	require.NoError(t, err)
+
+	for range 25 {
+		err = db.Transaction(func(tx *gorm.DB) error {
+			return tx.Create(&database.Event{
+				WebhookID:    webhookID,
+				EntrypointID: uuid.New().String(),
+				Method:       "POST",
+				Body:         "{}",
+			}).Error
+		})
+		require.NoError(t, err)
+	}
+
+	_, err = readerConn.ExecContext(ctx, "commit")
+	require.NoError(t, err)
+
+	var count int64
+
+	require.NoError(
+		t,
+		db.Model(&database.Event{}).Count(&count).Error,
+	)
+	assert.Equal(t, int64(25), count)
+}