Make SQLite durable under concurrent readers and stop re-delivering stranded webhooks (closes #256)

An operator running `sqlite3 <db> .dump` against their own per-webhook
database wedged it: inbound webhooks rejected with HTTP 500, delivered
webhooks stranded at `pending`, and every one of them POSTed a second
time on the next restart while the event log recorded a single attempt.

Durability. Every SQLite file — main, per-webhook, and archive — now
opens through one path, `internal/database/sqlite_open.go`, in WAL
journal mode with a 10-second busy timeout, `BEGIN IMMEDIATE`
transactions, and a bounded connection pool. WAL is what stops a reader
blocking writers at all. `_txlock=immediate` is what stops a `COMMIT`
failing while its transaction stays open on a pooled connection, which
is how four `database is locked` errors became 593 `cannot start a
transaction within a transaction`. `cache=shared` is gone, because
under it an in-process conflict is SQLITE_LOCKED, which the busy
handler does not retry. The busy timeout is applied before
journal_mode: the driver runs DSN pragmas in order on every new
connection, and `PRAGMA journal_mode` takes a lock, so the reverse
order leaves the one pragma that can block uncovered by the handler
meant to cover it.

Eligibility. `internal/delivery/inflight.go` holds the set of
deliveries the engine owns — taken when a task is queued, when a
target schedules a retry, and by every recovery path before it
re-dispatches; dropped when the worker that ran the task returns.
Recovery and both sweep arms re-dispatch only what the set does not
hold. Nothing decides that from a row's age: a delivery waiting in a
10000-deep channel is arbitrarily old and perfectly healthy, and
reasoning from age re-sends it. `takeForRedispatch` is the single gate
every re-dispatch goes through — ownership first, then a conditional
update confirming the row is still in the status the batch read.

Bookkeeping. `recordResult` and `updateDeliveryStatus` return their
errors instead of logging and dropping them, and a caller whose
bookkeeping write failed writes nothing at all: the delivery keeps
whichever non-terminal status it already held, and the sweeps recover
it. Every recovery path — pending and retrying alike — first settles
any delivery that already holds a successful `DeliveryResult` rather
than sending it again. Recovery continues each delivery's own attempt
numbering instead of restarting at 1. The sweep gains a
`pending`-with-age-bound arm, so a stranded delivery no longer waits
for a restart.

Docs. WAL produces `-wal`/`-shm` sidecars, so the backup and restore
procedures in README.md are corrected against measurement: both
documented procedures were re-run against a live instance, a `-wal`
left by a crash carries data the `.db` alone does not, and an archive
file normally holds its rows in a `-wal` rather than in the `.db`.

internal/delivery/target_http.go

@@ -77,14 +77,19 @@ func (c *httpCore) fireAndForget(
 ) {
 	c.eng.observeAttempt(d.Target.Type, res.elapsed())
 
-	c.eng.recordResult(
+	err := c.eng.recordResult(
 		webhookDB, d, 1, res.success,
 		res.statusCode, res.respBody, res.errMsg,
 		res.duration,
 	)
+	if err != nil {
+		c.eng.bookkeepingFailed(d, err)
+
+		return
+	}
 
 	if res.success {
-		c.eng.updateDeliveryStatus(
+		c.eng.settleStatus(
 			webhookDB, d, d.Target.Type,
 			database.DeliveryStatusDelivered,
 		)
@@ -92,7 +97,7 @@ func (c *httpCore) fireAndForget(
 		return
 	}
 
-	c.eng.updateDeliveryStatus(
+	c.eng.settleStatus(
 		webhookDB, d, d.Target.Type,
 		database.DeliveryStatusFailed,
 	)
@@ -122,16 +127,25 @@ func (c *httpCore) withRetry(
 
 	c.eng.observeAttempt(d.Target.Type, res.elapsed())
 
-	c.eng.recordResult(
+	err := c.eng.recordResult(
 		webhookDB, d, attemptNum, res.success,
 		res.statusCode, res.respBody, res.errMsg,
 		res.duration,
 	)
+	if err != nil {
+		// The breaker still learns the outcome: it describes the
+		// target's health, which is unaffected by this database's.
+		c.recordCircuitOutcome(cb, res.success)
+
+		c.eng.bookkeepingFailed(d, err)
+
+		return
+	}
 
 	if res.success {
 		cb.RecordSuccess()
 
-		c.eng.updateDeliveryStatus(
+		c.eng.settleStatus(
 			webhookDB, d, d.Target.Type,
 			database.DeliveryStatusDelivered,
 		)
@@ -146,6 +160,20 @@ func (c *httpCore) withRetry(
 	)
 }
 
+// recordCircuitOutcome feeds one attempt's outcome to the target's
+// circuit breaker.
+func (c *httpCore) recordCircuitOutcome(
+	cb *CircuitBreaker, success bool,
+) {
+	if success {
+		cb.RecordSuccess()
+
+		return
+	}
+
+	cb.RecordFailure()
+}
+
 func (c *httpCore) circuitBreakerBlock(
 	webhookDB *gorm.DB,
 	d *database.Delivery,
@@ -169,7 +197,7 @@ func (c *httpCore) circuitBreakerBlock(
 		"cooldown_remaining", remaining,
 	)
 
-	c.eng.updateDeliveryStatus(
+	c.eng.settleStatus(
 		webhookDB, d, d.Target.Type,
 		database.DeliveryStatusRetrying,
 	)
@@ -189,7 +217,7 @@ func (c *httpCore) handleRetry(
 	attemptNum int,
 ) {
 	if attemptNum >= maxRetries {
-		c.eng.updateDeliveryStatus(
+		c.eng.settleStatus(
 			webhookDB, d, d.Target.Type,
 			database.DeliveryStatusFailed,
 		)
@@ -197,7 +225,7 @@ func (c *httpCore) handleRetry(
 		return
 	}
 
-	c.eng.updateDeliveryStatus(
+	c.eng.settleStatus(
 		webhookDB, d, d.Target.Type,
 		database.DeliveryStatusRetrying,
 	)
@@ -332,12 +360,17 @@ func (t *httpTarget) Deliver(
 			"error", err,
 		)
 
-		t.eng.recordResult(
+		recErr := t.eng.recordResult(
 			webhookDB, d, task.AttemptNum,
 			false, 0, "", err.Error(), 0,
 		)
+		if recErr != nil {
+			t.eng.bookkeepingFailed(d, recErr)
 
-		t.eng.updateDeliveryStatus(
+			return
+		}
+
+		t.eng.settleStatus(
 			webhookDB, d, d.Target.Type,
 			database.DeliveryStatusFailed,
 		)