Store a target's settings as columns (closes #543)

A target's settings move from the per-type JSON string in
targets.config to columns: url, headers, timeout_seconds,
forward_query, archive_expiry and archive_rotation. A column a type
does not use stays empty. headers holds the header map as JSON text
through GORM's JSON serializer, and archive_expiry holds a
time.Duration, 0 meaning never, so nothing that reads a target parses
it. The forms check each setting once, with the same refusals and
messages, and the edit form is filled from the columns, so the
per-type config structs, their parsers and the "could not be read"
branches are gone. A delivery task carries the six settings in place
of the JSON string.

Model: opus-5-5

.dockerignore

@@ -72,8 +72,10 @@
 **/*.sublime-*
 
 # This repository's own host-built artifacts: the binary `make build`
-# writes.
+# writes, and the Alpine.js file `make assets` extracts from 3p/ (the
+# build extracts its own).
 /bin
+/static/js/alpine.min.js
 
 # SQLite databases, which hold the session key and webhook payloads, at
 # any depth.

.gitignore

@@ -77,3 +77,7 @@ data/
 # Temporary files
 tmp/
 temp/
+
+# Alpine.js, extracted by `make assets` from its tarball in 3p/, which is
+# what is committed.
+/static/js/alpine.min.js

.prettierignore

@@ -1,3 +1,2 @@
 node_modules/
 yarn.lock
-static/js/alpine.min.js

Dockerfile

@@ -92,6 +92,10 @@ COPY . .
 # fmt-check` is the markdown-check stage above.
 RUN if [ -n "$(gofmt -s -l .)" ]; then echo "gofmt needed on:"; gofmt -s -l .; exit 1; fi
 
+# static/static.go embeds the Alpine.js file this extracts from 3p/; without
+# it the static package does not compile and cannot be linted.
+RUN script/assets
+
 # The golangci-lint steps run with --network=none. `golangci-lint config
 # verify` is documented as fetching its JSON schema over HTTPS; this pinned
 # image resolves the schema without any network, and --network=none enforces
@@ -107,16 +111,11 @@ RUN --network=none golangci-lint run --config .golangci.yml --build-tags browser
 # Nothing is wanted from js-lint; the copy is what makes this phase run it.
 COPY --from=js-lint /src/yarn.lock /dev/null
 
-# Test phase. The tests run with cgo off and so without Go's race detector,
+# Test phase. -race needs cgo and so a C compiler, which the Debian Go image
-# which needs cgo on Linux; sneak ruled the race detector out in
+# ships and the alpine one does not. `make test` (script/test) builds this
-# https://git.eeqj.de/sneak/webhooker/issues/538#issuecomment-153273.
+# stage alone; the build stage below depends on it.
-# `make test` (script/test) builds this stage alone; the build stage below
-# depends on it.
 #
-# golang:1.26.1-bookworm (Debian-based), 2026-03-17: the builder stage's
+# golang:1.26.1-bookworm (Debian-based), 2026-03-17
-# image, which that stage needs for its git and make. Its test step runs
-# under the image's bash, and the tests in internal/versionscript use its
-# git, make and sh, and skip when they are missing.
 FROM golang:1.26.1-bookworm@sha256:4465644228bc2857a954b092167e12aa59c006a3492282a6c820bf4755fd64a4 AS test
 
 WORKDIR /src
@@ -126,11 +125,14 @@ RUN go mod download
 
 COPY . .
 
+# static/static.go embeds the Alpine.js file this extracts from 3p/.
+RUN script/assets
+
 # -timeout applies to each package on its own, so 90s has only to clear the
 # slowest one. -p 4 -parallel 8 keep the run under 2 GB of memory: at most
 # four test binaries build or run at once, each with at most eight parallel
-# tests. The defaults are one per core, so without these the memory the run
+# tests. Under -race every test binary and every link costs a few hundred MB,
-# needs grows with the host's core count.
+# so the defaults (one per core) add up to several GB on a many-core host.
 #
 # The first run has no -v: go test then prints one result line per package,
 # with its coverage, and for a package that fails, everything its tests
@@ -152,12 +154,12 @@ COPY . .
 SHELL ["/bin/bash", "-o", "pipefail", "-c"]
 RUN --mount=type=tmpfs,target=/tmp/tests,size=512m \
     export TMPDIR=/tmp/tests GOTMPDIR=/tmp; \
-    CGO_ENABLED=0 go test -cover -p 4 -parallel 8 -timeout 90s ./... 2>&1 | tee /tmp/go-test.log && exit 0; \
+    go test -race -cover -p 4 -parallel 8 -timeout 90s ./... 2>&1 | tee /tmp/go-test.log && exit 0; \
     tests="$(awk '/^--- FAIL: / { print $3 }' /tmp/go-test.log | paste -s -d '|' -)"; \
     packages="$(awk '/^FAIL\t/ { print $2 }' /tmp/go-test.log)"; \
     if [ -n "$tests" ]; then \
         echo "--- Rerunning the failed tests with -v for details ---"; \
-        CGO_ENABLED=0 go test -v -p 4 -parallel 8 -timeout 90s -run "^($tests)\$" $packages; \
+        go test -race -v -p 4 -parallel 8 -timeout 90s -run "^($tests)\$" $packages; \
     fi; \
     exit 1
 
@@ -204,8 +206,10 @@ RUN version="$(make version VERSION="$VERSION")"; \
         esac; \
     fi
 
-# The Makefile's build target builds with cgo off, so the binary is
+# Builds through the Makefile's build target, which runs script/assets
-# statically linked and runs on the Alpine runtime image below.
+# (Alpine.js, extracted from its tarball in 3p/) first. The target builds
+# with cgo off, so the binary is statically linked and runs on the Alpine
+# runtime image below as it is.
 RUN make build VERSION="$VERSION"
 
 # Runtime stage

Dockerfile.browser

@@ -15,9 +15,9 @@ RUN go mod download
 COPY . .
 
 # The test binary embeds the templates and static files, so the browser
-# stage needs nothing else. It is built with cgo off and -p 4 keeps the
+# stage needs nothing else. -p 4 keeps the compile's memory down, as in
-# compile's memory down, both as in the test phase of Dockerfile.
+# the test phase of Dockerfile.
-RUN CGO_ENABLED=0 go test -c -p 4 -tags browser -o /browser.test ./internal/server
+RUN make assets && go test -c -p 4 -tags browser -o /browser.test ./internal/server
 
 # chromedp/headless-shell:151.0.7922.109 (Debian trixie), 2026-08-11. The
 # browser is on PATH as headless-shell, where the test's browser library

Makefile

@@ -1,4 +1,4 @@
-.PHONY: bootstrap setup test test-browser lint fmt fmt-check check build run dev deps docker clean hooks css css-check version
+.PHONY: bootstrap setup assets test test-browser lint fmt fmt-check check build run dev deps docker clean hooks css css-check version
 
 # Default target
 .DEFAULT_GOAL := check
@@ -22,6 +22,9 @@ bootstrap:
 setup:
 	@script/setup
 
+assets:
+	@script/assets
+
 test:
 	@script/test
 
@@ -44,13 +47,13 @@ check:
 # REPO_POLICIES.md does: the binary carries no build paths, symbol table or
 # debug information, and on Linux, the image's included, it is statically
 # linked.
-build:
+build: assets
 	CGO_ENABLED=0 go build -trimpath -ldflags '-s -w -X main.version=$(VERSION)' -o bin/webhooker ./cmd/webhooker
 
 run: build
 	./bin/webhooker
 
-dev:
+dev: assets
 	go run ./cmd/webhooker
 
 deps:

TODO.md

@@ -1,77 +1,72 @@
 # Workflow
 
-One issue per unit of work, and one branch and one PR per issue:
+One issue per unit of work, one branch and one PR per issue:
 
-- make sure a tracked issue exists with a definition of done
+- ensure a tracked issue exists with a definition of done
-- branch from `next`, never from `main`
+- branch from `next` (never from `main`)
-- do the work and push the branch
+- do the work; open a PR based on `next` (never on `main`)
-- open a PR with base `next`, never `main`
+- pass an independent review, then the manager squash-merges into `next`
-- pass an independent review
+- push; nothing stays local-only
-- the repository manager squash-merges it into `next`
 
-The repository manager is the role that merges each issue's PR into `next`.
+`next` is the branch for the next milestone and must stay green and mergeable to
-Nothing stays local-only: every commit is pushed.
+`main` without notice. One `next` -> `main` PR accumulates the milestone;
+releases are cut from `main` separately.
 
-`next` is the branch for the next milestone. It must stay green and mergeable
+The tracker (https://git.eeqj.de/sneak/webhooker/issues) is the record of what
-into `main` at all times. A milestone's work reaches `main` through one PR from
+is done and what is under way. This file keeps no list of either.
-`next`. The owner, @sneak, merges that PR. Cutting a release from `main` is a
-separate step after that.
-
-The tracker (https://git.eeqj.de/sneak/webhooker/issues) records what is done
-and what is under way. This file keeps no list of either.
 
 # Status
 
 What is still open for 1.0.0 is in its milestone
-(https://git.eeqj.de/sneak/webhooker/milestone/9). `v1.0.0` is not tagged. Only
+(https://git.eeqj.de/sneak/webhooker/milestone/9). `v1.0.0` is not tagged, and
-@sneak tags it.
+only @sneak tags it.
 
-Delivery is at least once by design. When webhooker cannot record the outcome of
+Delivery is at-least-once by design, not by accident: a send whose result row
-a delivery, it sends that delivery again. The target can then receive the same
+does not land is attempted again, so a receiver can see a duplicate. That is
-event twice. The alternative is a lost delivery that nothing reports. The README
+deliberate — the alternative is a silent lost delivery — and the README says so
-says so. Such a duplicate is not a defect to re-file.
+under Rationale. It is not a defect to re-file.
 
 # Next Step
 
 @sneak runs webhooker in limited production, then tags `v1.0.0`. Merging `next`
-into `main` is a separate act. It waits on neither the production run nor the
+into `main` is a separate act from tagging and waits on neither of those: `next`
-tag.
+is kept mergeable at all times, which is the point of the branch.
 
 # Future Steps
 
-- Per-webhook rate limits, set in the web UI and enforced at the entrypoint.
+- Per-webhook rate limits, set in the web UI and enforced in the receiver
-  Today `RECEIVER_RATE_LIMIT` is the only rate-limit setting. It is how many
+  handler, on top of the receiver's existing limits, which `RECEIVER_RATE_LIMIT`
-  requests per minute one client address may send to one entrypoint. The same
+  sets for the whole server (see Rate Limiting in the README)
-  address is also capped at ten times that across all entrypoints together.
+- API key authentication for programmatic access (the `APIKey` model exists, but
-- API key authentication for programmatic access. The `APIKey` model and its
+  nothing creates a key and no middleware checks one)
-  `api_keys` table exist. Nothing creates a key and nothing checks one.
+- REST API v1
-- REST API v1. `/api/v1` is mounted with no routes today.
+    - CRUD for webhooks, entrypoints, targets
-    - CRUD for webhooks, entrypoints and targets
     - event viewing and filtering endpoints
     - event redelivery endpoint
     - OpenAPI specification
 - Analytics in the web UI: response times summarised per webhook or per target,
-  and figures for all webhooks together. Today the webhook page shows event,
+  and figures for all webhooks together. The webhook page already shows event,
-  delivery and failure counts and failure percentages. The target list counts
+  delivery and failure counts and failure percentages, the target list counts
-  each target's deliveries. The event log shows every attempt's duration.
+  each target's deliveries, the event log and each event's page show every
-  `/metrics` carries server-wide counts and delivery durations.
+  attempt's duration, and `/metrics` carries server-wide counts and delivery
-- A remember-me option at login. Today every session survives a browser restart.
+  durations
-  It lasts up to 7 days. It ends sooner after `SESSION_IDLE_TIMEOUT` (default 24
+- A remember-me option at login. Every session already survives a browser
-  hours) without use.
+  restart: it lasts up to 7 days, and ends sooner after `SESSION_IDLE_TIMEOUT`
-- Password reset over the web for a forgotten password. Today a signed-in user
+  (default 24 hours) without activity
-  can change their password. A lost password is reset from the command line with
+- Password reset flow for a forgotten password over the web. The authenticated
-  `webhooker resetpw` (https://git.eeqj.de/sneak/webhooker/issues/208).
+  password _change_ flow already landed, and a lost password is recoverable from
+  the console with `webhooker resetpw`
+  (https://git.eeqj.de/sneak/webhooker/issues/208)
 - Later, nice to have
     - email delivery target type
     - SNS and S3 delivery targets
-    - data transformations, such as a configurable Slack message format. The
+    - data transformations (e.g. a configurable Slack message format; the Slack
-      Slack target sends one fixed format today.
+      target sends one fixed format today)
-    - JSONL file delivery with periodic S3 upload. Today an archive target
+    - JSONL file delivery with periodic S3 upload (the `database` target already
-      writes events to SQLite files. An archive target can rotate those files
+      archives events to SQLite files, rotated hourly, daily or monthly if set,
-      hourly, daily or monthly. The webhook page offers them as one gzipped JSON
+      and the webhook page offers them as one gzipped JSON download)
-      download.
     - webhook event search, and filtering beyond the event log's failed and
       pending views
-    - multi-user with role-based access control. Today each webhook belongs to
+    - multi-user with role-based access control (each webhook already belongs to
-      one account. Only the account made at first boot exists. There is no way
+      one account, but there is only the account made at first boot and no way
-      to add another.
+      to add another)

cmd/webhooker/main.go

@@ -211,6 +211,9 @@ func newApp() *fx.App {
 			delivery.New,
 			archive.NewWriters,
 			archive.NewSweeper,
+			// Wire *delivery.Engine as delivery.Notifier so the
+			// webhook handler can notify the engine of new deliveries.
+			func(e *delivery.Engine) delivery.Notifier { return e },
 			// Wire *archive.Writers as archive.Archives so deleting
 			// or renaming a webhook or target reaches its archive
 			// files.

eslint.config.mjs

@@ -5,7 +5,7 @@
 // REPO_POLICIES.md states that a linter can check: const for everything,
 // let only for a variable that is reassigned, never var.
 export default [
-    // Alpine.js, committed as published; not ours to lint.
+    // Alpine.js, extracted from 3p/ by make assets; not ours to lint.
     { ignores: ["static/js/alpine.min.js"] },
     {
         // The pages load static/js/app.js as a classic script, not a module.

internal/archive/evict_test.go

@@ -303,7 +303,7 @@ func (r *racingWrites) wait() (bool, error) {
 // contending for the writer's mutex when the eviction takes it.
 // The write that wins the mutex after the eviction must abandon
 // its work rather than reopen the archive, leaving the writer
-// permanently handle-free.
+// permanently handle-free. Run under -race.
 func TestEvictWebhook_RacingWriteDoesNotReopenHandle(
 	t *testing.T,
 ) {

internal/archive/sweeper_test.go

@@ -959,11 +959,9 @@ func TestArchiveSweep_SkipsDeletedWebhookTargets(t *testing.T) {
 	)
 }
 
-// TestArchiveSweep_ConcurrentWrites runs the sweep while writes to
+// TestArchiveSweep_ConcurrentWrites proves the sweep serialises
-// the same target are under way. The sweep serialises against them
+// against writes through the target's writer mutex. Run under
-// through the target's writer mutex; the suite runs without Go's race
+// -race, an unsynchronised sweep would be caught here.
-// detector, so a sweep that did not would fail this test only by
-// panicking, hanging or leaving no archive file.
 func TestArchiveSweep_ConcurrentWrites(t *testing.T) {
 	t.Parallel()
 

internal/database/event_tier_indexes_test.go

@@ -79,9 +79,8 @@ func TestEventTierQueriesUseTheirIndexes(t *testing.T) {
 	byStatus := "idx_deliveries_status (status=? AND deleted_at=?)"
 	byEvent := "idx_deliveries_event_id (event_id=? AND deleted_at=?)"
 
-	// The delivery engine: the recovery routine, which reads the
+	// The delivery engine: recovery and the retry sweep, the sweep for
-	// retrying deliveries and, on a tick of the sweep, the pending ones
+	// stranded pending deliveries, and the queue depth count.
-	// untouched for a while; and the queue depth count.
 	assertPlanUses(t, db, dry.Where(
 		"status = ?", database.DeliveryStatusRetrying,
 	).Find(&deliveries), byStatus)

internal/database/password.go

@@ -67,10 +67,9 @@ func DefaultPasswordConfig() *PasswordConfig {
 // testArgon2Memory is the Argon2id memory cost, in KiB, that a test
 // binary hashes with: 1 MB instead of the shipped 64 MB. Every test
 // that starts a database hashes the bootstrap admin password, dozens
-// of them run in parallel, and each 64 MB hash holds at least 64 MB
+// of them run in parallel, and under the race detector each 64 MB hash
-// while it runs (about 150 MB when the suite ran under the race
+// holds about 150 MB. VerifyPassword reads the cost from the hash it
-// detector). VerifyPassword reads the cost from the hash it checks, so
+// checks, so verification follows.
-// verification follows.
 const testArgon2Memory = 1024
 
 // hashAtShippedCostInTest makes a test binary hash at the shipped

internal/database/retention.go

@@ -184,9 +184,8 @@ func (r *RetentionReaper) sweep(ctx context.Context) {
 
 		wh := webhooks[i]
 
-		// A missing database has nothing to reap. The delivery
+		// A missing database has nothing to reap. Restart recovery
-		// engine's recovery routine reports a lost one at start (see
+		// reports a lost one (see WebhookDBManager.GetDB).
-		// WebhookDBManager.GetDB).
 		if !r.dbManager.DBExists(wh.ID) {
 			continue
 		}

internal/delivery/engine.go

@@ -1,5 +1,5 @@
-// Package delivery stores each event with its deliveries and delivers
+// Package delivery manages asynchronous event delivery
-// them, asynchronously, to configured targets.
+// to configured targets.
 package delivery
 
 import (
@@ -23,11 +23,10 @@ import (
 
 const (
 	// deliveryChannelSize is the buffer size for the delivery
-	// channel. The Tasks of newly stored deliveries, and of the
+	// channel. New Tasks from the webhook handler are sent
-	// pending deliveries the recovery routine hands back, are
+	// here. Workers drain this channel. Sized large enough
-	// sent here. Workers drain this channel. Sized large enough
+	// that the webhook handler should never block under
-	// that no task should be dropped under normal load: one that
+	// normal load.
-	// does not fit is left for the sweep.
 	deliveryChannelSize = 10000
 
 	// retryChannelSize is the buffer size for the retry
@@ -41,10 +40,9 @@ const (
 	// explosions regardless of queue depth.
 	defaultWorkers = 10
 
-	// sweepInterval is how often the recovery routine runs
+	// retrySweepInterval is how often the periodic retry
-	// once it has run at start. Each of those runs is a tick of
+	// sweep runs.
-	// the sweep.
+	retrySweepInterval = 60 * time.Second
-	sweepInterval = 60 * time.Second
 
 	// pendingSweepMinAge is how long a delivery must have sat
 	// untouched at pending before the sweep will look at it.
@@ -65,10 +63,10 @@ const (
 	pendingSweepMinAge = 15 * time.Minute
 
 	// pendingSweepBatch bounds how many stranded pending deliveries
-	// one tick of the sweep re-dispatches for one webhook. The sweep
+	// one sweep of one webhook re-dispatches. The sweep runs every
-	// ticks every sweepInterval, so a larger backlog drains
+	// retrySweepInterval, so a larger backlog drains across
-	// across successive ticks instead of arriving as one burst against
+	// successive sweeps instead of arriving as one burst against a
-	// a database that was already struggling to accept writes.
+	// database that was already struggling to accept writes.
 	pendingSweepBatch = 500
 
 	// MaxInlineBodySize is the maximum event body size that
@@ -128,11 +126,17 @@ type Task struct {
 	AttemptNum int
 }
 
+// Notifier is the interface for notifying the delivery
+// engine about new deliveries.
+type Notifier interface {
+	Notify(tasks []Task)
+}
+
 // CircuitBreakers is how the handlers read a target's circuit
 // breaker, so the webhook page and the event log can say that
-// deliveries to the target are paused and until when. The handlers
+// deliveries to the target are paused and until when. It keeps the
-// hold the engine itself as well; they read breakers through this
+// handlers free of the engine's internals and is trivially faked in
-// interface so that a test can make one read as open.
+// tests.
 type CircuitBreakers interface {
 	StateAndCooldown(targetID string) (CircuitState, time.Duration)
 }
@@ -152,13 +156,12 @@ type EngineParams struct {
 }
 
 // Engine processes queued deliveries in the background
-// using a bounded worker pool architecture. It also stores each
+// using a bounded worker pool architecture. It owns only
-// event with its deliveries and creates replay deliveries (see
+// the cross-target machinery: the worker pool, the queue
-// store.go). Beyond that it owns only the cross-target machinery:
+// and retry channels, restart recovery, and the persistence
-// the worker pool, the delivery and retry channels, the recovery
+// helpers and Scheduler that individual targets rely on.
-// routine, and the persistence helpers and Scheduler that
+// Each target type owns its own delivery, including retries,
-// individual targets rely on. Each target type owns its own
+// backoff, and circuit breaking.
-// delivery, including retries, backoff, and circuit breaking.
 type Engine struct {
 	database   *database.Database
 	dbManager  *database.WebhookDBManager
@@ -194,7 +197,7 @@ type Engine struct {
 	archives *archive.Writers
 
 	// inflight is the set of deliveries this engine currently owns.
-	// The recovery routine re-dispatches only what it does not
+	// Recovery and the sweeps re-dispatch only what it does not
 	// hold. Held by value: its zero value works, so no constructor
 	// can leave it out. See inflight.go.
 	inflight inflightSet
@@ -228,6 +231,38 @@ func New(
 	return e
 }
 
+// Notify signals the delivery engine that new deliveries
+// are ready.
+func (e *Engine) Notify(tasks []Task) {
+	for i := range tasks {
+		// Owned before it is queued, and until the worker that runs
+		// it returns. A task can sit in a 10000-deep channel for a
+		// long time on a healthy system, and nothing may re-send it
+		// while it waits. See inflight.go.
+		if !e.inflight.retainIdle(tasks[i].DeliveryID) {
+			e.log.Warn(
+				"delivery already in flight, not queued again",
+				"delivery_id", tasks[i].DeliveryID,
+				"event_id", tasks[i].EventID,
+			)
+
+			continue
+		}
+
+		select {
+		case e.deliveryCh <- tasks[i]:
+		default:
+			e.inflight.release(tasks[i].DeliveryID)
+			e.log.Warn(
+				"delivery channel full, "+
+					"task will be recovered by the sweep",
+				"delivery_id", tasks[i].DeliveryID,
+				"event_id", tasks[i].EventID,
+			)
+		}
+	}
+}
+
 // StateAndCooldown implements CircuitBreakers. It returns the state of
 // the target's circuit breaker and, while the breaker is open, what is
 // left of its cooldown; the cooldown is zero once that has passed and
@@ -288,38 +323,6 @@ func (e *Engine) ScheduleRetry(
 	})
 }
 
-// notify queues the tasks of deliveries already stored. StoreEvent and
-// CreateReplay call it once their transaction has committed.
-func (e *Engine) notify(tasks []Task) {
-	for i := range tasks {
-		// Owned before it is queued, and until the worker that runs
-		// it returns. A task can sit in a 10000-deep channel for a
-		// long time on a healthy system, and nothing may re-send it
-		// while it waits. See inflight.go.
-		if !e.inflight.retainIdle(tasks[i].DeliveryID) {
-			e.log.Warn(
-				"delivery already in flight, not queued again",
-				"delivery_id", tasks[i].DeliveryID,
-				"event_id", tasks[i].EventID,
-			)
-
-			continue
-		}
-
-		select {
-		case e.deliveryCh <- tasks[i]:
-		default:
-			e.inflight.release(tasks[i].DeliveryID)
-			e.log.Warn(
-				"delivery channel full, "+
-					"task will be recovered by the sweep",
-				"delivery_id", tasks[i].DeliveryID,
-				"event_id", tasks[i].EventID,
-			)
-		}
-	}
-}
-
 // userAgent is the User-Agent header of every http and slack
 // delivery request: the program name and the running build's
 // version.
@@ -346,8 +349,8 @@ func (e *Engine) registerHooks(lc fx.Lifecycle) {
 	})
 }
 
-// start launches the worker pool and the recovery routine, which
+// start launches the worker pool, restart recovery, and the
-// runs now and then on every tick of the sweep (see runRecovery).
+// periodic retry sweep.
 //
 // Their context is derived from context.Background(), NOT from
 // the fx OnStart hook context. The hook context carries fx's
@@ -370,7 +373,11 @@ func (e *Engine) start() {
 
 	e.wg.Add(1)
 
-	go e.runRecovery(ctx)
+	go e.recoverPending(ctx)
+
+	e.wg.Add(1)
+
+	go e.retrySweep(ctx)
 
 	e.wg.Add(1)
 
@@ -453,6 +460,12 @@ func (e *Engine) runTask(
 	run(ctx, task)
 }
 
+func (e *Engine) recoverPending(ctx context.Context) {
+	defer e.wg.Done()
+
+	e.recoverInFlight(ctx)
+}
+
 func (e *Engine) processNewTask(
 	ctx context.Context, task *Task,
 ) {
@@ -467,10 +480,9 @@ func (e *Engine) processNewTask(
 		return
 	}
 
-	// The recovery routine's run at start can send and release this
+	// Restart recovery can send and release this delivery before the
-	// delivery before StoreEvent or CreateReplay queues it. Ownership
+	// receiver's Notify queues it. Ownership cannot refuse a delivery
-	// cannot refuse a delivery nobody holds, so the row decides whether
+	// nobody holds, so the row decides whether it still needs sending.
-	// it still needs sending.
 	row, err := e.loadDelivery(webhookDB, task.DeliveryID)
 	if err != nil {
 		e.log.Error(
@@ -600,9 +612,9 @@ func (e *Engine) processRetryTask(
 // else on this path reads the target row. Without this check a
 // deletion stops nothing: the timer keeps firing and keeps sending to
 // the destination the operator removed, for the whole remaining
-// backoff chain. Terminalising in the recovery routine alone is not
+// backoff chain. Terminalising in the recovery and sweep paths alone
-// enough, because it only sees the delivery once nothing holds it in
+// is not enough, because those only see the delivery once nothing
-// memory — which is to say after a restart.
+// holds it in memory — which is to say after a restart.
 //
 // The worker already owns this delivery, so the terminal write happens
 // here directly, exactly as a target's own Deliver fails one. Claiming
@@ -650,47 +662,7 @@ func (e *Engine) abandonRetryForMissingTarget(
 	return true
 }
 
-// runRecovery runs the recovery routine, recoverDeliveries, once when
+func (e *Engine) recoverInFlight(ctx context.Context) {
-// the engine starts and then on every tick of the sweep, every
-// sweepInterval, until the engine stops.
-func (e *Engine) runRecovery(ctx context.Context) {
-	defer e.wg.Done()
-
-	ticker := time.NewTicker(sweepInterval)
-	defer ticker.Stop()
-
-	e.recoverDeliveries(ctx, true)
-
-	for {
-		select {
-		case <-ctx.Done():
-			return
-		case <-ticker.C:
-			e.recoverDeliveries(ctx, false)
-		}
-	}
-}
-
-// recoverDeliveries is the engine's one recovery routine: atStart is
-// true for its run when the engine starts and false on a tick of the
-// sweep.
-//
-// For every webhook it finds the pending and retrying deliveries the
-// engine is not working on, such as those a stop or a crash cut off, a
-// full channel dropped, or a failed bookkeeping write left behind. One
-// that already has a successful attempt is marked delivered. A retrying
-// one is scheduled after what is left of its backoff, or now when
-// nothing is left. A pending one goes to a worker now.
-//
-// Ownership keeps it off a delivery the engine is working on, has
-// queued, or holds a retry timer for, the same at start as on a tick:
-// every delivery it acts on is taken through the inflight set first.
-// See inflight.go.
-//
-// A run at start differs from a tick in which webhooks it opens (see
-// recoverWebhookDeliveries) and which pending deliveries it takes (see
-// recoverPendingDeliveries).
-func (e *Engine) recoverDeliveries(ctx context.Context, atStart bool) {
 	var webhookIDs []string
 
 	err := e.database.DB().
@@ -712,27 +684,18 @@ func (e *Engine) recoverDeliveries(ctx context.Context, atStart bool) {
 		default:
 		}
 
-		e.recoverWebhookDeliveries(ctx, webhookID, atStart)
+		// Opened even when its file is missing, so that a lost
+		// database is reported at start, not when the webhook next
+		// receives an event, which for a quiet webhook may be never.
+		e.recoverWebhookDeliveries(ctx, webhookID)
 	}
 }
 
-// recoverWebhookDeliveries runs the recovery routine over one webhook.
-//
-// At start the webhook's database is opened even when its file is
-// missing, so that a lost database is reported at start, not when the
-// webhook next receives an event, which for a quiet webhook may be
-// never. A tick leaves a missing file alone rather than create one: a
-// webhook being created has its row saved a moment before its file,
-// and a tick in between would report that file as lost.
 func (e *Engine) recoverWebhookDeliveries(
-	ctx context.Context, webhookID string, atStart bool,
+	ctx context.Context, webhookID string,
 ) {
-	if !atStart && !e.dbManager.DBExists(webhookID) {
+	// The web interface is already serving, so the webhook may have
-		return
+	// been deleted since the list was read. Opening its database then
-	}
-
-	// The web interface is serving, so the webhook may have been
-	// deleted since the list was read. Opening its database then
 	// would create the file again after the delete removed it.
 	stillExists := func() (bool, error) {
 		var count int64
@@ -764,18 +727,16 @@ func (e *Engine) recoverWebhookDeliveries(
 	}
 
 	e.recoverPendingDeliveries(
-		ctx, webhookDB, webhookID, atStart,
+		ctx, webhookDB, webhookID,
 	)
 
 	e.recoverRetryingDeliveries(
-		ctx, webhookDB, webhookID,
+		webhookDB, webhookID,
 	)
 }
 
-// recoverRetryingDeliveries recovers every retrying delivery of one
-// webhook, at start and on a tick alike.
 func (e *Engine) recoverRetryingDeliveries(
-	ctx context.Context, webhookDB *gorm.DB, webhookID string,
+	webhookDB *gorm.DB, webhookID string,
 ) {
 	var retrying []database.Delivery
 
@@ -802,12 +763,6 @@ func (e *Engine) recoverRetryingDeliveries(
 	)
 
 	for i := range retrying {
-		select {
-		case <-ctx.Done():
-			return
-		default:
-		}
-
 		if _, ok := settled[retrying[i].ID]; ok {
 			continue
 		}
@@ -819,9 +774,8 @@ func (e *Engine) recoverRetryingDeliveries(
 }
 
 // recoverSingleRetry hands an orphaned retrying delivery back
-// to its target to work out what is left of its backoff, then
+// to its target to recompute the remaining backoff, then
-// schedules its next attempt after that, or now when nothing is
+// reschedules it. Targets that do not own durable retries
-// left. Targets that do not own durable retries
 // (fire-and-forget) never produce retrying deliveries, so a
 // delivery found in that state has had its target's type
 // changed underneath it and is terminally failed.
@@ -869,19 +823,6 @@ func (e *Engine) recoverSingleRetry(
 		webhookDB, d.ID, attemptNum,
 	)
 
-	// Taken before the event is read, so that a delivery the engine
-	// already holds, most often one waiting out its backoff on a
-	// timer, costs a tick no read of its body.
-	if !e.takeForRedispatch(
-		webhookDB, d.ID, database.DeliveryStatusRetrying,
-	) {
-		return
-	}
-
-	// ScheduleRetry takes a reference of its own before it returns,
-	// and that one keeps the delivery owned through the backoff.
-	defer e.inflight.release(d.ID)
-
 	event, err := e.loadEvent(webhookDB, d.EventID)
 	if err != nil {
 		e.log.Error(
@@ -895,9 +836,13 @@ func (e *Engine) recoverSingleRetry(
 		return
 	}
 
-	e.ScheduleRetry(buildTask(
+	task := buildRecoveryTask(
 		d, webhookID, &event, &target, attemptNum+1,
-	), remaining)
+	)
+
+	if !e.rescheduleRecovered(webhookDB, task, remaining) {
+		return
+	}
 
 	e.log.Info(
 		"recovering retrying delivery",
@@ -908,54 +853,28 @@ func (e *Engine) recoverSingleRetry(
 	)
 }
 
-// recoverPendingDeliveries recovers the pending deliveries of one
-// webhook: it marks delivered those that already have a successful
-// attempt and sends the rest.
-//
-// A delivery is created pending and leaves that state only when its
-// outcome is written, so a pending row the engine does not own is one
-// it stopped working on before then: for example one whose bookkeeping
-// write failed, whose attempt a stop cut off, that was still queued
-// when the engine last stopped, whose task was dropped because the
-// delivery channel was full (by notify or by queueRecovered), or whose
-// worker could not open the webhook's database.
-//
-// At start every pending delivery is taken, however recent, so one a
-// stop cut off moments ago is sent again now. A tick takes only those
-// untouched for pendingSweepMinAge, at most pendingSweepBatch of them.
-// The age bound is not what keeps a tick off live work: a delivery
-// waiting in deliveryCh is pending and arbitrarily old. Ownership is,
-// checked per delivery in takeForRedispatch. See inflight.go.
 func (e *Engine) recoverPendingDeliveries(
 	ctx context.Context,
 	webhookDB *gorm.DB,
 	webhookID string,
-	atStart bool,
 ) {
-	query := webhookDB.Where(
-		"status = ?", database.DeliveryStatusPending,
-	)
-
-	if !atStart {
-		query = query.
-			Where(
-				"updated_at < ?",
-				time.Now().Add(-pendingSweepMinAge),
-			).
-			Limit(pendingSweepBatch)
-	}
-
 	var deliveries []database.Delivery
 
 	// No Preload: event bodies are read one at a time in
 	// sendRecoveredDeliveries, and only for the deliveries actually
 	// being sent.
-	err := query.Find(&deliveries).Error
+	result := webhookDB.
-	if err != nil {
+		Where(
+			"status = ?",
+			database.DeliveryStatusPending,
+		).
+		Find(&deliveries)
+
+	if result.Error != nil {
 		e.log.Error(
-			"failed to query pending deliveries for recovery",
+			"failed to query pending deliveries",
 			"webhook_id", webhookID,
-			"error", err,
+			"error", result.Error,
 		)
 
 		return
@@ -971,6 +890,21 @@ func (e *Engine) recoverPendingDeliveries(
 		"count", len(deliveries),
 	)
 
+	e.recoverPendingBatch(
+		ctx, webhookDB, webhookID, deliveries,
+	)
+}
+
+// recoverPendingBatch settles every delivery in the batch that was
+// already delivered, and re-dispatches only the rest. Both the
+// restart-time recovery and the periodic sweep go through it, so a
+// pending delivery is treated the same however it was found.
+func (e *Engine) recoverPendingBatch(
+	ctx context.Context,
+	webhookDB *gorm.DB,
+	webhookID string,
+	deliveries []database.Delivery,
+) {
 	targetMap := e.loadTargetMap(deliveries)
 
 	settled := e.reconcileDelivered(
@@ -998,11 +932,11 @@ func (e *Engine) recoverPendingDeliveries(
 // wire I/O happened and was recorded, and all that is missing is the
 // status.
 //
-// The recovery routine runs this for retrying deliveries as well as
+// Every recovery path runs this, not only the pending one. A delivery
-// pending ones. A delivery abandoned at retrying can hold a successful
+// abandoned at retrying can hold a successful result just as a pending
-// result just as a pending one can — a second attempt that reached the
+// one can — a second attempt that reached the receiver and whose status
-// receiver and whose status write then failed sits at retrying with
+// write then failed sits at retrying with success recorded — and
-// success recorded — and re-sending it is the same duplicate.
+// re-sending it is the same duplicate.
 //
 // Deliveries whose result row itself never landed are not in the
 // returned set and are re-sent, recorded as the further attempt they
@@ -1089,8 +1023,247 @@ func (e *Engine) reconcileDelivered(
 	return settled
 }
 
+func (e *Engine) retrySweep(ctx context.Context) {
+	defer e.wg.Done()
+
+	ticker := time.NewTicker(retrySweepInterval)
+	defer ticker.Stop()
+
+	for {
+		select {
+		case <-ctx.Done():
+			return
+		case <-ticker.C:
+			e.sweepOrphanedRetries(ctx)
+		}
+	}
+}
+
+func (e *Engine) sweepOrphanedRetries(
+	ctx context.Context,
+) {
+	var webhookIDs []string
+
+	err := e.database.DB().
+		Model(&database.Webhook{}).
+		Pluck("id", &webhookIDs).Error
+	if err != nil {
+		e.log.Error(
+			"retry sweep: failed to query webhook IDs",
+			"error", err,
+		)
+
+		return
+	}
+
+	for _, webhookID := range webhookIDs {
+		select {
+		case <-ctx.Done():
+			return
+		default:
+		}
+
+		if !e.dbManager.DBExists(webhookID) {
+			continue
+		}
+
+		e.sweepWebhookRetries(ctx, webhookID)
+	}
+}
+
+func (e *Engine) sweepWebhookRetries(
+	ctx context.Context, webhookID string,
+) {
+	webhookDB, err := e.dbManager.GetDB(webhookID)
+	if err != nil {
+		e.log.Error(
+			"retry sweep: failed to get webhook database",
+			"webhook_id", webhookID,
+			"error", err,
+		)
+
+		return
+	}
+
+	var retrying []database.Delivery
+
+	err = webhookDB.
+		Where(
+			"status = ?",
+			database.DeliveryStatusRetrying,
+		).
+		Find(&retrying).Error
+	if err != nil {
+		e.log.Error(
+			"retry sweep: "+
+				"failed to query retrying deliveries",
+			"webhook_id", webhookID,
+			"error", err,
+		)
+
+		return
+	}
+
+	settled := e.reconcileDelivered(
+		webhookDB, webhookID, retrying,
+		e.loadTargetMap(retrying),
+	)
+
+	for i := range retrying {
+		select {
+		case <-ctx.Done():
+			return
+		default:
+		}
+
+		if _, ok := settled[retrying[i].ID]; ok {
+			continue
+		}
+
+		e.sweepSingleRetry(
+			webhookDB, webhookID, &retrying[i],
+		)
+	}
+
+	e.sweepWebhookPending(ctx, webhookDB, webhookID)
+}
+
+// sweepWebhookPending recovers deliveries stranded at pending.
+//
+// A delivery is created pending and leaves that state only when its
+// outcome is written, so a pending row the engine does not own is one
+// whose bookkeeping write failed — the state that used to sit there
+// until a restart, and then produce a duplicate at the receiver. The
+// sweep gives it the same reconcile-then-dispatch treatment restart
+// recovery gets, so it costs a minute rather than an operator
+// noticing.
+//
+// What keeps the sweep off live work is ownership, checked per
+// delivery in takeForRedispatch, not the age bound in this query.
+// A delivery waiting in deliveryCh is pending and arbitrarily old —
+// the channel holds 10000 tasks and 10 workers drain it — so
+// reasoning from the row's age alone re-sends it. See inflight.go.
+func (e *Engine) sweepWebhookPending(
+	ctx context.Context,
+	webhookDB *gorm.DB,
+	webhookID string,
+) {
+	var pending []database.Delivery
+
+	err := webhookDB.
+		Where(
+			"status = ? AND updated_at < ?",
+			database.DeliveryStatusPending,
+			time.Now().Add(-pendingSweepMinAge),
+		).
+		Limit(pendingSweepBatch).
+		Find(&pending).Error
+	if err != nil {
+		e.log.Error(
+			"retry sweep: "+
+				"failed to query pending deliveries",
+			"webhook_id", webhookID,
+			"error", err,
+		)
+
+		return
+	}
+
+	if len(pending) == 0 {
+		return
+	}
+
+	e.log.Info(
+		"retry sweep: recovering stranded pending deliveries",
+		"webhook_id", webhookID,
+		"count", len(pending),
+	)
+
+	e.recoverPendingBatch(ctx, webhookDB, webhookID, pending)
+}
+
+// sweepSingleRetry re-enqueues an orphaned retrying delivery
+// whose backoff window has elapsed, delegating the backoff
+// decision to the delivery's target. A delivery whose target
+// no longer owns durable retries is terminally failed.
+func (e *Engine) sweepSingleRetry(
+	webhookDB *gorm.DB,
+	webhookID string,
+	d *database.Delivery,
+) {
+	target, err := e.loadTarget(d.TargetID)
+	if err != nil {
+		// Deleted is terminal, unreadable is not; see
+		// recoverSingleRetry.
+		if errors.Is(err, gorm.ErrRecordNotFound) {
+			e.failMissingTarget(webhookDB, webhookID, d)
+
+			return
+		}
+
+		e.log.Error(
+			"retry sweep: failed to load target",
+			"delivery_id", d.ID,
+			"target_id", d.TargetID,
+			"error", err,
+		)
+
+		return
+	}
+
+	rs, ok := e.targets[target.Type].(rescheduler)
+	if !ok {
+		e.failUnretryableRetry(
+			webhookDB, webhookID, d, &target,
+		)
+
+		return
+	}
+
+	attemptNum := e.countAttempts(webhookDB, d.ID)
+
+	if !rs.backoffElapsed(
+		webhookDB, d.ID, attemptNum,
+	) {
+		return
+	}
+
+	event, err := e.loadEvent(webhookDB, d.EventID)
+	if err != nil {
+		e.log.Error(
+			"retry sweep: failed to load event",
+			"delivery_id", d.ID,
+			"event_id", d.EventID,
+			"error", err,
+		)
+
+		return
+	}
+
+	task := buildRecoveryTask(
+		d, webhookID, &event, &target, attemptNum+1,
+	)
+
+	if !e.redispatch(
+		e.retryCh, webhookDB, task,
+		database.DeliveryStatusRetrying,
+	) {
+		return
+	}
+
+	e.log.Info(
+		"retry sweep: "+
+			"recovered orphaned retrying delivery",
+		"delivery_id", d.ID,
+		"webhook_id", webhookID,
+		"attempt", attemptNum+1,
+	)
+}
+
 // failUnretryableRetry terminally fails an orphaned retrying
-// delivery whose target type no longer supports retries.
+// delivery whose target type no longer supports retries. Both
+// restart recovery and the periodic sweep call it, so the
+// terminal transition exists once.
 //
 // This is only reachable when a target's type has been changed
 // out from under an in-flight retrying delivery (or the type is
@@ -1138,8 +1311,8 @@ func (e *Engine) failUnretryableRetry(
 }
 
 // failMissingTarget terminally fails a recovered delivery, pending or
-// retrying, whose target row is gone. The recovery routine calls it for
+// retrying, whose target row is gone. Restart recovery and the periodic
-// both statuses.
+// sweep call it for both statuses, so the transition exists once.
 //
 // Until it existed those paths logged the failed lookup and moved on,
 // which left the delivery where it was for the life of the database and
@@ -1669,11 +1842,11 @@ func (e *Engine) countAttempts(
 // answer cannot go stale between asking and acting.
 //
 // Stamping updated_at is the same statement, and it is a cadence
-// control rather than a claim: a tick of the sweep selects pending
+// control rather than a claim: the pending sweep selects on that
-// deliveries on that column, so one handed out now is not selected
+// column, so a delivery handed out now is not selected again on the
-// again on the next tick a minute later but after pendingSweepMinAge.
+// next tick a minute later but after pendingSweepMinAge. A delivery
-// A delivery the database refuses to settle is therefore retried on
+// the database refuses to settle is therefore retried on that
-// that interval instead of every tick.
+// interval instead of every tick.
 //
 // A failed write is a refusal. It means the database is not accepting
 // writes, which is the condition that stranded the delivery in the
@@ -1721,11 +1894,14 @@ func (e *Engine) takeForRedispatch(
 	return true
 }
 
-// queueRecovered puts an owned pending delivery's task on the delivery
+// queueRecovered puts an owned delivery's task on a worker channel,
-// channel, dropping the ownership the gate took if it does not fit.
+// dropping the ownership the gate took if it does not fit.
-func (e *Engine) queueRecovered(task Task) {
+func (e *Engine) queueRecovered(
+	ch chan<- Task, task Task,
+) bool {
 	select {
-	case e.deliveryCh <- task:
+	case ch <- task:
+		return true
 	default:
 		e.inflight.release(task.DeliveryID)
 		e.log.Warn(
@@ -1734,9 +1910,53 @@ func (e *Engine) queueRecovered(task Task) {
 			"delivery_id", task.DeliveryID,
 			"webhook_id", task.WebhookID,
 		)
+
+		return false
 	}
 }
 
+// redispatch hands a recovered delivery to a worker channel through
+// takeForRedispatch, and reports whether the task was queued.
+func (e *Engine) redispatch(
+	ch chan<- Task,
+	webhookDB *gorm.DB,
+	task Task,
+	eligible database.DeliveryStatus,
+) bool {
+	if !e.takeForRedispatch(
+		webhookDB, task.DeliveryID, eligible,
+	) {
+		return false
+	}
+
+	return e.queueRecovered(ch, task)
+}
+
+// rescheduleRecovered hands an orphaned retrying delivery back to the
+// retry timer, through the same gate. It reports whether the delivery
+// was rescheduled.
+//
+// The reference taken by the gate is dropped as soon as ScheduleRetry
+// has taken its own, which it does before returning: what keeps the
+// delivery owned through the backoff window is ScheduleRetry's
+// reference, not this one.
+func (e *Engine) rescheduleRecovered(
+	webhookDB *gorm.DB, task Task, delay time.Duration,
+) bool {
+	if !e.takeForRedispatch(
+		webhookDB, task.DeliveryID,
+		database.DeliveryStatusRetrying,
+	) {
+		return false
+	}
+
+	defer e.inflight.release(task.DeliveryID)
+
+	e.ScheduleRetry(task, delay)
+
+	return true
+}
+
 // countAttemptsBatch counts the recorded attempts of every delivery
 // in a batch with one grouped query, keyed by delivery id. Deliveries
 // with no attempts are simply absent from the result, which reads back
@@ -1818,11 +2038,7 @@ func (e *Engine) loadTarget(
 	return target, nil
 }
 
-// buildTask builds the task that carries delivery d of event to target
+func buildRecoveryTask(
-// through the engine's channels, as attempt attemptNum. The body is
-// carried only when it is under MaxInlineBodySize; a larger one is read
-// from the event row when the attempt runs (see hydrateEvent).
-func buildTask(
 	d *database.Delivery,
 	webhookID string,
 	event *database.Event,
@@ -1998,11 +2214,11 @@ func (e *Engine) sendRecoveredDeliveries(
 			continue
 		}
 
-		task := buildTask(
+		task := buildRecoveryTask(
 			&deliveries[i], webhookID, &event, &target,
 			attempts[deliveries[i].ID]+1,
 		)
 
-		e.queueRecovered(task)
+		e.queueRecovered(e.deliveryCh, task)
 	}
 }

internal/delivery/engine_integration_test.go

@@ -492,7 +492,7 @@ func TestWorkerLifecycle_StartStop(t *testing.T) {
 	)
 	task.TargetType = database.TargetTypeLog
 
-	s.Engine.ExportNotify([]delivery.Task{task})
+	s.Engine.Notify([]delivery.Task{task})
 
 	iWaitForDelivered(t, s.WebhookDB, d.ID)
 
@@ -640,7 +640,7 @@ func TestRecoverPendingDeliveries(t *testing.T) {
 
 	s.Engine.ExportRecoverPendingDeliveries(
 		context.Background(), s.WebhookDB,
-		s.WebhookID, atStart,
+		s.WebhookID,
 	)
 
 	for i := range 3 {
@@ -737,7 +737,7 @@ func TestRecoverWebhookDeliveries_RetryingDeliveries(
 	)
 
 	s.Engine.ExportRecoverWebhookDeliveries(
-		context.Background(), s.WebhookID, atStart,
+		context.Background(), s.WebhookID,
 	)
 
 	select {
@@ -861,7 +861,7 @@ func TestRecoverSingleRetry_TypeNoLongerRetries(
 	)
 
 	s.Engine.ExportRecoverWebhookDeliveries(
-		context.Background(), s.WebhookID, atStart,
+		context.Background(), s.WebhookID,
 	)
 
 	iAssertTerminallyFailed(
@@ -869,7 +869,7 @@ func TestRecoverSingleRetry_TypeNoLongerRetries(
 	)
 }
 
-func TestRecoverSingleRetry_OnTick_TypeNoLongerRetries(
+func TestSweepSingleRetry_TypeNoLongerRetries(
 	t *testing.T,
 ) {
 	t.Parallel()
@@ -884,8 +884,8 @@ func TestRecoverSingleRetry_OnTick_TypeNoLongerRetries(
 		t, s, database.TargetTypeDatabase,
 	)
 
-	s.Engine.ExportRecoverWebhookDeliveries(
+	s.Engine.ExportSweepWebhookRetries(
-		context.Background(), s.WebhookID, onTick,
+		context.Background(), s.WebhookID,
 	)
 
 	iAssertTerminallyFailed(
@@ -936,8 +936,8 @@ func TestFailUnretryableRetry_WritesNoTargetRow(
 
 	iSeedFailedResult(t, s.WebhookDB, d.ID)
 
-	s.Engine.ExportRecoverWebhookDeliveries(
+	s.Engine.ExportSweepWebhookRetries(
-		context.Background(), s.WebhookID, onTick,
+		context.Background(), s.WebhookID,
 	)
 
 	iAssertStatus(t, s.WebhookDB, d.ID,
@@ -968,13 +968,13 @@ func TestRecoverSingleRetry_UnknownTargetType(
 	deliveryID := iSeedRetryingWithType(t, s, unknown)
 
 	s.Engine.ExportRecoverWebhookDeliveries(
-		context.Background(), s.WebhookID, atStart,
+		context.Background(), s.WebhookID,
 	)
 
 	iAssertTerminallyFailed(t, s, deliveryID, unknown)
 }
 
-func TestRecoverSingleRetry_OnTick_UnknownTargetType(
+func TestSweepSingleRetry_UnknownTargetType(
 	t *testing.T,
 ) {
 	t.Parallel()
@@ -989,8 +989,8 @@ func TestRecoverSingleRetry_OnTick_UnknownTargetType(
 
 	deliveryID := iSeedRetryingWithType(t, s, unknown)
 
-	s.Engine.ExportRecoverWebhookDeliveries(
+	s.Engine.ExportSweepWebhookRetries(
-		context.Background(), s.WebhookID, onTick,
+		context.Background(), s.WebhookID,
 	)
 
 	iAssertTerminallyFailed(t, s, deliveryID, unknown)
@@ -1032,19 +1032,19 @@ func iCreateWebhook(
 	require.NoError(t, mainDB.Create(&webhook).Error)
 }
 
-// --- recoverDeliveries Tests ---
+// --- recoverInFlight Tests ---
 
-func TestRecoverDeliveries_NoWebhooks(t *testing.T) {
+func TestRecoverInFlight_NoWebhooks(t *testing.T) {
 	t.Parallel()
 
 	s := newISetup(t)
 
-	s.Engine.ExportRecoverDeliveries(
+	s.Engine.ExportRecoverInFlight(
-		context.Background(), atStart,
+		context.Background(),
 	)
 }
 
-func TestRecoverDeliveries_WithPendingDeliveries(
+func TestRecoverInFlight_WithPendingDeliveries(
 	t *testing.T,
 ) {
 	t.Parallel()
@@ -1071,8 +1071,8 @@ func TestRecoverDeliveries_WithPendingDeliveries(
 		database.DeliveryStatusPending,
 	)
 
-	s.Engine.ExportRecoverDeliveries(
+	s.Engine.ExportRecoverInFlight(
-		context.Background(), atStart,
+		context.Background(),
 	)
 
 	select {
@@ -1083,16 +1083,16 @@ func TestRecoverDeliveries_WithPendingDeliveries(
 			database.TargetTypeLog, task.TargetType,
 		)
 	case <-time.After(2 * time.Second):
-		t.Fatal("expected task from recovery at start")
+		t.Fatal("expected task from recoverInFlight")
 	}
 }
 
-// TestRecoverDeliveries_ReportsAMissingWebhookDatabase covers a webhook
+// TestRecoverInFlight_ReportsAMissingWebhookDatabase covers a webhook
-// whose database file is gone, after a partial restore say. Recovery at
+// whose database file is gone, after a partial restore say. Restart
-// start opens every webhook's database, so the empty one made in its
+// recovery opens every webhook's database, so the empty one made in its
 // place is reported at start, naming the file
 // (https://git.eeqj.de/sneak/webhooker/issues/290).
-func TestRecoverDeliveries_ReportsAMissingWebhookDatabase(t *testing.T) {
+func TestRecoverInFlight_ReportsAMissingWebhookDatabase(t *testing.T) {
 	t.Parallel()
 
 	mainDB := iMainDB(t)
@@ -1112,7 +1112,7 @@ func TestRecoverDeliveries_ReportsAMissingWebhookDatabase(t *testing.T) {
 		&http.Client{Timeout: 5 * time.Second}, 1,
 	)
 
-	engine.ExportRecoverDeliveries(context.Background(), atStart)
+	engine.ExportRecoverInFlight(context.Background())
 
 	assert.Contains(
 		t, logs.String(),
@@ -1121,11 +1121,10 @@ func TestRecoverDeliveries_ReportsAMissingWebhookDatabase(t *testing.T) {
 	)
 }
 
-// TestRecoverDeliveries_SkipsAWebhookDeletedAfterTheListIsRead covers a
+// TestRecoverInFlight_SkipsAWebhookDeletedAfterTheListIsRead covers a
-// webhook deleted from the web interface while the recovery routine
+// webhook deleted from the web interface while restart recovery runs.
-// runs. Its database file is gone, and recovery must not create it
+// Its database file is gone, and recovery must not create it again.
-// again.
+func TestRecoverInFlight_SkipsAWebhookDeletedAfterTheListIsRead(
-func TestRecoverDeliveries_SkipsAWebhookDeletedAfterTheListIsRead(
 	t *testing.T,
 ) {
 	t.Parallel()
@@ -1161,7 +1160,7 @@ func TestRecoverDeliveries_SkipsAWebhookDeletedAfterTheListIsRead(
 		&http.Client{Timeout: 5 * time.Second}, 1,
 	)
 
-	engine.ExportRecoverDeliveries(context.Background(), atStart)
+	engine.ExportRecoverInFlight(context.Background())
 
 	require.True(t, deleted)
 	assert.False(t, dbMgr.DBExists(webhookID))
@@ -1221,14 +1220,11 @@ func TestDeliverHTTP_CustomTargetHeaders(t *testing.T) {
 
 // --- HTTP delivery with custom timeout ---
 
-// TestDeliverHTTP_TargetTimeout proves a task's TimeoutSeconds reaches
-// the delivery through the engine. The destination answers after two
-// seconds, inside the engine's own five-second client timeout, so only
-// the task's one second can fail the delivery.
 func TestDeliverHTTP_TargetTimeout(t *testing.T) {
 	t.Parallel()
 
-	s := newISetup(t)
+	db := testWebhookDB(t)
+	e := iEngine(t, 1)
 
 	ts := httptest.NewServer(
 		http.HandlerFunc(
@@ -1240,30 +1236,24 @@ func TestDeliverHTTP_TargetTimeout(t *testing.T) {
 	)
 	defer ts.Close()
 
-	event := iSeedEvent(
+	event, del := iSeedEventAndDelivery(
-		t, s.WebhookDB, s.WebhookID, `{"timeout":"test"}`,
+		t, db, `{"timeout":"test"}`,
-	)
-	targetID := uuid.New().String()
-
-	d := iSeedDelivery(
-		t, s.WebhookDB, event.ID, targetID,
-		database.DeliveryStatusPending,
 	)
 
-	bodyStr := event.Body
+	task, d := iHTTPTaskAndDelivery(
-	task := iTask(
+		event, del, "timeout-target",
-		d, event, s.WebhookID, targetID,
+		ts.URL, 1, 1,
-		"timeout-target", ts.URL, 1, 1, &bodyStr,
 	)
 	task.TimeoutSeconds = 1
+	d.Target.TimeoutSeconds = 1
 
-	s.Engine.ExportProcessNewTask(context.TODO(), &task)
+	e.ExportDeliverHTTP(context.TODO(), db, d, task)
 
-	iAssertStatus(t, s.WebhookDB, d.ID,
+	iAssertStatus(t, db, del.ID,
 		database.DeliveryStatusFailed,
 	)
 
-	iAssertResultFailed(t, s.WebhookDB, d.ID)
+	iAssertResultFailed(t, db, del.ID)
 }
 
 // TestDeliverHTTP_CutsStoredResponseAtMaxBodyLog pins the size
@@ -1452,7 +1442,7 @@ func TestDeliverHTTP_NoURL(t *testing.T) {
 	iAssertResultFailed(t, db, del.ID)
 }
 
-// --- notify batching ---
+// --- Notify batching ---
 
 func TestNotify_MultipleTasks(t *testing.T) {
 	t.Parallel()
@@ -1467,7 +1457,7 @@ func TestNotify_MultipleTasks(t *testing.T) {
 		}
 	}
 
-	e.ExportNotify(tasks)
+	e.Notify(tasks)
 
 	for i := range 5 {
 		select {

internal/delivery/engine_lifecycle_test.go

@@ -29,7 +29,7 @@ const (
 	// OnStart before the caller may enqueue work. A worker pool
 	// wrongly rooted in the already-done hook context has nothing
 	// but ctx.Done() ready in its select, so it is deterministically
-	// gone by the end of this window. Without the wait, notify would
+	// gone by the end of this window. Without the wait, Notify would
 	// race the pool's very first select, in which a ready ctx.Done()
 	// and a ready deliveryCh are chosen between at random and a
 	// doomed pool still delivers.
@@ -99,7 +99,7 @@ func requireStopHookExpires(
 // the caller can drive OnStop too.
 //
 // Callers must not seed pending or retrying deliveries before
-// calling this: the recovery routine's run at start enqueues those,
+// calling this: restart recovery enqueues those during startup,
 // which would put work in the queue while the pool is still
 // racing its first select.
 func startEngineViaHook(
@@ -178,11 +178,11 @@ func TestEngine_WorkersOutliveStartHookContext(t *testing.T) {
 		_ = lc.hooks[0].OnStop(context.Background())
 	})
 
-	// Seeded only after the pool has settled, so the recovery
+	// Seeded only after the pool has settled, so restart recovery
-	// routine's run at start cannot enqueue it.
+	// cannot enqueue it during startup.
 	deliveryID, task := seedLogTask(t, s)
 
-	s.Engine.ExportNotify([]delivery.Task{task})
+	s.Engine.Notify([]delivery.Task{task})
 
 	iWaitForDelivered(t, s.WebhookDB, deliveryID)
 }
@@ -202,7 +202,7 @@ func TestEngine_StopHookStopsWorkers(t *testing.T) {
 	// Let the pool prove it is running before stopping it, so a
 	// fast OnStop cannot pass by stopping something already dead.
 	firstID, firstTask := seedLogTask(t, s)
-	s.Engine.ExportNotify([]delivery.Task{firstTask})
+	s.Engine.Notify([]delivery.Task{firstTask})
 	iWaitForDelivered(t, s.WebhookDB, firstID)
 
 	var stopErr error
@@ -232,7 +232,7 @@ func TestEngine_StopHookStopsWorkers(t *testing.T) {
 	// With every worker gone, a freshly notified task must sit
 	// untouched in the queue rather than being delivered.
 	secondID, secondTask := seedLogTask(t, s)
-	s.Engine.ExportNotify([]delivery.Task{secondTask})
+	s.Engine.Notify([]delivery.Task{secondTask})
 
 	time.Sleep(200 * time.Millisecond)
 
@@ -344,15 +344,15 @@ func stopDuringAttempt(
 	require.NoError(t, lc.hooks[0].OnStop(context.Background()))
 }
 
-// requireRequeued runs the recovery routine as at start on the stopped
+// requireRequeued runs restart recovery on the stopped engine and
-// engine and requires it to queue task's attempt again on ch.
+// requires it to queue task's attempt again on ch.
 func requireRequeued(
 	t *testing.T, s iSetup, ch <-chan delivery.Task, task delivery.Task,
 ) {
 	t.Helper()
 
 	s.Engine.ExportRecoverWebhookDeliveries(
-		context.Background(), s.WebhookID, atStart,
+		context.Background(), s.WebhookID,
 	)
 
 	select {
@@ -360,7 +360,7 @@ func requireRequeued(
 		assert.Equal(t, task.DeliveryID, got.DeliveryID)
 		assert.Equal(t, task.AttemptNum, got.AttemptNum)
 	case <-time.After(5 * time.Second):
-		t.Fatal("recovery at start did not queue the delivery again")
+		t.Fatal("restart recovery did not queue the delivery again")
 	}
 }
 
@@ -370,8 +370,7 @@ func requireRequeued(
 // no retries that was the delivery's only attempt, recorded as failed
 // with "context canceled", and nothing ever sent it again. The
 // delivery must instead stay pending, with no result and no count in
-// the attempt metrics, for the recovery routine to send at the next
+// the attempt metrics, for restart recovery to send.
-// start.
 func TestEngine_StopLeavesFireAndForgetAttemptPending(t *testing.T) {
 	t.Parallel()
 
@@ -383,7 +382,7 @@ func TestEngine_StopLeavesFireAndForgetAttemptPending(t *testing.T) {
 		t, s, 1, database.DeliveryStatusPending, 1,
 	)
 
-	s.Engine.ExportNotify([]delivery.Task{task})
+	s.Engine.Notify([]delivery.Task{task})
 	stopDuringAttempt(t, lc, arrived)
 
 	iAssertStatus(
@@ -399,8 +398,8 @@ func TestEngine_StopLeavesFireAndForgetAttemptPending(t *testing.T) {
 // TestEngine_StopLeavesLastRetryAttemptRetrying is the same regression
 // for a target with retries, on its last attempt: the cancelled attempt
 // used to exhaust the retries and end the delivery failed. It must stay
-// retrying with only its earlier attempt recorded, and the recovery
+// retrying with only its earlier attempt recorded, and restart recovery
-// routine at the next start must send that last attempt again.
+// must send that last attempt again.
 func TestEngine_StopLeavesLastRetryAttemptRetrying(t *testing.T) {
 	t.Parallel()
 
@@ -448,7 +447,7 @@ func deliverToArchive(t *testing.T, s iSetup) string {
 	task.TargetID = tgt.ID
 	task.TargetType = database.TargetTypeDatabase
 
-	s.Engine.ExportNotify([]delivery.Task{task})
+	s.Engine.Notify([]delivery.Task{task})
 
 	iWaitForDelivered(t, s.WebhookDB, deliveryID)
 

internal/delivery/engine_test.go

@@ -218,7 +218,7 @@ func TestNotify_NonBlocking(t *testing.T) {
 	done := make(chan struct{})
 
 	go func() {
-		e.ExportNotify([]delivery.Task{
+		e.Notify([]delivery.Task{
 			{DeliveryID: "overflow-1"},
 			{DeliveryID: "overflow-2"},
 		})
@@ -229,7 +229,7 @@ func TestNotify_NonBlocking(t *testing.T) {
 	case <-done:
 	case <-time.After(2 * time.Second):
 		t.Fatal(
-			"notify blocked when delivery channel was full",
+			"Notify blocked when delivery channel was full",
 		)
 	}
 }

internal/delivery/export_test.go

@@ -158,11 +158,6 @@ func (e *Engine) ExportUpdateDeliveryStatus(
 	return e.updateDeliveryStatus(webhookDB, d, "", status)
 }
 
-// ExportNotify exposes notify.
-func (e *Engine) ExportNotify(tasks []Task) {
-	e.notify(tasks)
-}
-
 // ExportProcessNewTask exposes processNewTask.
 func (e *Engine) ExportProcessNewTask(
 	ctx context.Context, task *Task,
@@ -272,27 +267,32 @@ func (e *Engine) ExportRecoverPendingDeliveries(
 	ctx context.Context,
 	webhookDB *gorm.DB,
 	webhookID string,
-	atStart bool,
 ) {
 	e.recoverPendingDeliveries(
-		ctx, webhookDB, webhookID, atStart,
+		ctx, webhookDB, webhookID,
 	)
 }
 
 // ExportRecoverWebhookDeliveries exposes
 // recoverWebhookDeliveries.
 func (e *Engine) ExportRecoverWebhookDeliveries(
-	ctx context.Context, webhookID string, atStart bool,
+	ctx context.Context, webhookID string,
 ) {
-	e.recoverWebhookDeliveries(ctx, webhookID, atStart)
+	e.recoverWebhookDeliveries(ctx, webhookID)
 }
 
-// ExportRecoverDeliveries exposes recoverDeliveries, the recovery
+// ExportRecoverInFlight exposes recoverInFlight.
-// routine.
+func (e *Engine) ExportRecoverInFlight(
-func (e *Engine) ExportRecoverDeliveries(
+	ctx context.Context,
-	ctx context.Context, atStart bool,
 ) {
-	e.recoverDeliveries(ctx, atStart)
+	e.recoverInFlight(ctx)
+}
+
+// ExportSweepWebhookRetries exposes sweepWebhookRetries.
+func (e *Engine) ExportSweepWebhookRetries(
+	ctx context.Context, webhookID string,
+) {
+	e.sweepWebhookRetries(ctx, webhookID)
 }
 
 // ExportStart exposes start for testing.
@@ -339,9 +339,9 @@ func (e *Engine) ExportRetainDelivery(deliveryID string) bool {
 
 // ExportRecoverRetryingDeliveries exposes recoverRetryingDeliveries.
 func (e *Engine) ExportRecoverRetryingDeliveries(
-	ctx context.Context, webhookDB *gorm.DB, webhookID string,
+	webhookDB *gorm.DB, webhookID string,
 ) {
-	e.recoverRetryingDeliveries(ctx, webhookDB, webhookID)
+	e.recoverRetryingDeliveries(webhookDB, webhookID)
 }
 
 // ExportFailMissingTarget exposes failMissingTarget, so a test can hand

internal/delivery/inflight.go

@@ -6,10 +6,10 @@ import "sync"
 //
 // A delivery is owned from the moment a task for it is handed to a
 // channel or to a retry timer until the engine has no further plan for
-// it in memory. The recovery routine, at start and on every tick of the
+// it in memory. Restart recovery and both arms of the periodic sweep
-// sweep, re-dispatches only deliveries the set does not hold, which is
+// re-dispatch only deliveries the set does not hold, which is what
-// what makes it exact rather than a guess about how long a row has sat
+// makes them exact rather than a guess about how long a row has sat at
-// at pending.
+// pending.
 //
 // This replaces reasoning from timestamps. A delivery's row says
 // pending from creation until its outcome is written, which covers
@@ -25,8 +25,8 @@ import "sync"
 // In-memory state is sufficient because a data directory admits one
 // process: internal/datadir takes an flock on it at startup and a
 // second instance refuses to run. Deliveries owned by a process that
-// died are not in any successor's set, and the recovery routine's run
+// died are not in any successor's set, and restart recovery is what
-// at start is what picks those up.
+// picks those up.
 //
 // References are counted rather than held as a plain set because
 // ownership outlives the worker that took it. A target that schedules
@@ -61,9 +61,9 @@ func (s *inflightSet) retain(deliveryID string) {
 
 // retainIdle takes the first reference on a delivery, and reports
 // whether it got it. It fails when the engine already owns the
-// delivery, which is what makes two claimants — notify and the
+// delivery, which is what makes two claimants — restart recovery and
-// recovery routine, say — mutually exclusive rather than merely
+// the sweep run concurrently, or two sweep arms — mutually exclusive
-// atomic.
+// rather than merely atomic.
 func (s *inflightSet) retainIdle(deliveryID string) bool {
 	s.mu.Lock()
 	defer s.mu.Unlock()

internal/delivery/inflight_test.go

@@ -84,9 +84,9 @@ func TestSweepLeavesAQueuedDeliveryAlone(t *testing.T) {
 	)
 	rAgePending(t, s.WebhookDB, d.ID)
 
-	// Queued exactly as StoreEvent queues it, and never dequeued:
+	// Queued exactly as the receiver queues it, and never dequeued:
 	// no workers are running in this engine.
-	s.Engine.ExportNotify([]delivery.Task{{
+	s.Engine.Notify([]delivery.Task{{
 		DeliveryID: d.ID,
 		EventID:    event.ID,
 		WebhookID:  s.WebhookID,
@@ -95,8 +95,8 @@ func TestSweepLeavesAQueuedDeliveryAlone(t *testing.T) {
 
 	require.Equal(t, 1, s.Engine.ExportInflightHeld())
 
-	s.Engine.ExportRecoverWebhookDeliveries(
+	s.Engine.ExportSweepWebhookRetries(
-		context.Background(), s.WebhookID, onTick,
+		context.Background(), s.WebhookID,
 	)
 
 	tasks := fDrain(s.Engine)
@@ -107,11 +107,9 @@ func TestSweepLeavesAQueuedDeliveryAlone(t *testing.T) {
 	)
 }
 
-// TestRecoveryAndSweepDoNotDoubleDispatch drives the recovery routine's
+// TestRecoveryAndSweepDoNotDoubleDispatch drives the two entry points
-// run at start and a tick of the sweep at once against one aged pending
+// the engine starts concurrently against one aged pending row. Before
-// row. The engine runs them one after the other, but notify can race
+// ownership they both dispatched it.
-// either, and ownership is what keeps any two claimants from both
-// sending. Before ownership the two runs both dispatched the row.
 func TestRecoveryAndSweepDoNotDoubleDispatch(t *testing.T) {
 	t.Parallel()
 
@@ -135,12 +133,12 @@ func TestRecoveryAndSweepDoNotDoubleDispatch(t *testing.T) {
 
 		wg.Go(func() {
 			s.Engine.ExportRecoverPendingDeliveries(
-				ctx, s.WebhookDB, s.WebhookID, atStart,
+				ctx, s.WebhookDB, s.WebhookID,
 			)
 		})
 		wg.Go(func() {
-			s.Engine.ExportRecoverWebhookDeliveries(
+			s.Engine.ExportSweepWebhookRetries(
-				ctx, s.WebhookID, onTick,
+				ctx, s.WebhookID,
 			)
 		})
 		wg.Wait()
@@ -224,7 +222,7 @@ func TestOwnershipIsReleasedAfterDelivery(t *testing.T) {
 
 	body := `{"released":true}`
 
-	s.Engine.ExportNotify([]delivery.Task{{
+	s.Engine.Notify([]delivery.Task{{
 		DeliveryID:   d.ID,
 		EventID:      event.ID,
 		WebhookID:    s.WebhookID,
@@ -248,11 +246,10 @@ func TestOwnershipIsReleasedAfterDelivery(t *testing.T) {
 }
 
 // TestNotifyAfterRecoveryDoesNotSendAgain is the startup race of
-// https://git.eeqj.de/sneak/webhooker/issues/299. StoreEvent has
+// https://git.eeqj.de/sneak/webhooker/issues/299. The receiver has
-// written a delivery, the recovery routine's run at start finds it
+// written a delivery, restart recovery finds it pending, sends it and
-// pending, sends it and releases it, and only then does StoreEvent's
+// releases it, and only then does the receiver's Notify for it arrive.
-// notify for it arrive. Nothing owns the delivery by then, so notify
+// Nothing owns the delivery by then, so Notify takes it.
-// takes it.
 func TestNotifyAfterRecoveryDoesNotSendAgain(t *testing.T) {
 	t.Parallel()
 
@@ -276,8 +273,7 @@ func TestNotifyAfterRecoveryDoesNotSendAgain(t *testing.T) {
 		)
 	}()
 
-	// The recovery routine's run at start sends the delivery and lets
+	// Restart recovery sends the delivery and lets it go.
-	// it go.
 	iWaitForDelivered(t, s.WebhookDB, d.ID)
 	require.Eventually(
 		t,
@@ -289,7 +285,7 @@ func TestNotifyAfterRecoveryDoesNotSendAgain(t *testing.T) {
 
 	body := event.Body
 
-	s.Engine.ExportNotify([]delivery.Task{{
+	s.Engine.Notify([]delivery.Task{{
 		DeliveryID:   d.ID,
 		EventID:      event.ID,
 		WebhookID:    s.WebhookID,
@@ -300,7 +296,7 @@ func TestNotifyAfterRecoveryDoesNotSendAgain(t *testing.T) {
 		EntrypointID: event.EntrypointID,
 	}})
 
-	// notify took the delivery, and a worker releases it once it has
+	// Notify took the delivery, and a worker releases it once it has
 	// run the task.
 	require.Eventually(
 		t,
@@ -338,7 +334,7 @@ func TestRetryingRecoverySkipsASuccessfulResult(t *testing.T) {
 	rSeedResult(t, s.WebhookDB, d.ID, 2, true)
 
 	s.Engine.ExportRecoverRetryingDeliveries(
-		context.Background(), s.WebhookDB, s.WebhookID,
+		s.WebhookDB, s.WebhookID,
 	)
 
 	assert.Empty(
@@ -353,8 +349,8 @@ func TestRetryingRecoverySkipsASuccessfulResult(t *testing.T) {
 	)
 }
 
-// TestRetryingSweepSkipsASuccessfulResult is the same rule on a tick of
+// TestRetryingSweepSkipsASuccessfulResult is the same rule on the
-// the sweep.
+// periodic sweep's retrying arm.
 func TestRetryingSweepSkipsASuccessfulResult(t *testing.T) {
 	t.Parallel()
 
@@ -372,8 +368,8 @@ func TestRetryingSweepSkipsASuccessfulResult(t *testing.T) {
 	rSeedResult(t, s.WebhookDB, d.ID, 1, false)
 	rSeedResult(t, s.WebhookDB, d.ID, 2, true)
 
-	s.Engine.ExportRecoverWebhookDeliveries(
+	s.Engine.ExportSweepWebhookRetries(
-		context.Background(), s.WebhookID, onTick,
+		context.Background(), s.WebhookID,
 	)
 
 	assert.Empty(t, fDrain(s.Engine))
@@ -424,8 +420,8 @@ func TestScheduledRetryIsNotSweptDuringBackoff(t *testing.T) {
 
 	require.Equal(t, 1, s.Engine.ExportInflightHeld())
 
-	s.Engine.ExportRecoverWebhookDeliveries(
+	s.Engine.ExportSweepWebhookRetries(
-		context.Background(), s.WebhookID, onTick,
+		context.Background(), s.WebhookID,
 	)
 
 	assert.Empty(
@@ -457,7 +453,7 @@ func TestRedispatchStampsTheRow(t *testing.T) {
 
 	ctx := context.Background()
 
-	s.Engine.ExportRecoverWebhookDeliveries(ctx, s.WebhookID, onTick)
+	s.Engine.ExportSweepWebhookRetries(ctx, s.WebhookID)
 	require.Len(t, fDrain(s.Engine), 1)
 
 	var row database.Delivery

internal/delivery/metrics_test.go

@@ -441,8 +441,8 @@ func TestDeliveryMetrics_OrphanedRetryFailureLabelled(
 		t, s, database.TargetTypeLog,
 	)
 
-	s.Engine.ExportRecoverWebhookDeliveries(
+	s.Engine.ExportSweepWebhookRetries(
-		context.Background(), s.WebhookID, onTick,
+		context.Background(), s.WebhookID,
 	)
 
 	iAssertStatus(t, s.WebhookDB, deliveryID,

internal/delivery/queue_depth.go

@@ -20,8 +20,7 @@ const queueDepthSampleInterval = 30 * time.Second
 // to be seeded correctly at startup from rows written by a previous
 // process, and would drift permanently on any transition that failed
 // to persist. Counting is the measurement that cannot go wrong, and
-// it is the same whole-database walk each tick of the sweep already
+// it is the same whole-database walk the retry sweep already makes.
-// makes.
 func (e *Engine) queueDepthSampler(ctx context.Context) {
 	defer e.wg.Done()
 

internal/delivery/recovery_durability_test.go

@@ -88,7 +88,7 @@ func TestRecoverySkipsPendingWithSuccessfulResult(
 	)
 
 	s.Engine.ExportRecoverPendingDeliveries(
-		context.Background(), s.WebhookDB, s.WebhookID, atStart,
+		context.Background(), s.WebhookDB, s.WebhookID,
 	)
 
 	select {
@@ -145,7 +145,7 @@ func TestRecoveryContinuesTheAttemptNumbering(t *testing.T) {
 	rSeedResult(t, s.WebhookDB, d.ID, 2, false)
 
 	s.Engine.ExportRecoverPendingDeliveries(
-		context.Background(), s.WebhookDB, s.WebhookID, atStart,
+		context.Background(), s.WebhookDB, s.WebhookID,
 	)
 
 	select {
@@ -183,8 +183,8 @@ func TestSweepRecoversStrandedPending(t *testing.T) {
 		database.DeliveryStatusPending,
 	)
 
-	s.Engine.ExportRecoverWebhookDeliveries(
+	s.Engine.ExportSweepWebhookRetries(
-		context.Background(), s.WebhookID, onTick,
+		context.Background(), s.WebhookID,
 	)
 
 	select {
@@ -230,7 +230,7 @@ func TestSweepClaimsAStrandedDeliveryOnlyOnce(t *testing.T) {
 
 	ctx := context.Background()
 
-	s.Engine.ExportRecoverWebhookDeliveries(ctx, s.WebhookID, onTick)
+	s.Engine.ExportSweepWebhookRetries(ctx, s.WebhookID)
 
 	select {
 	case task := <-s.Engine.ExportDeliveryCh():
@@ -246,7 +246,7 @@ func TestSweepClaimsAStrandedDeliveryOnlyOnce(t *testing.T) {
 		database.DeliveryStatusPending,
 	)
 
-	s.Engine.ExportRecoverWebhookDeliveries(ctx, s.WebhookID, onTick)
+	s.Engine.ExportSweepWebhookRetries(ctx, s.WebhookID)
 
 	select {
 	case task := <-s.Engine.ExportDeliveryCh():
@@ -258,9 +258,9 @@ func TestSweepClaimsAStrandedDeliveryOnlyOnce(t *testing.T) {
 	}
 }
 
-// TestSweepSettlesStrandedPendingWithoutResending is the reconcile on a
+// TestSweepSettlesStrandedPendingWithoutResending is the sweep's own
-// tick of the sweep: a stranded delivery holding a successful result is
+// version of the reconcile: a stranded delivery holding a successful
-// settled where it stands, and the receiver hears nothing.
+// result is settled where it stands, and the receiver hears nothing.
 func TestSweepSettlesStrandedPendingWithoutResending(
 	t *testing.T,
 ) {
@@ -280,8 +280,8 @@ func TestSweepSettlesStrandedPendingWithoutResending(
 	rSeedResult(t, s.WebhookDB, d.ID, 1, true)
 	rAgePending(t, s.WebhookDB, d.ID)
 
-	s.Engine.ExportRecoverWebhookDeliveries(
+	s.Engine.ExportSweepWebhookRetries(
-		context.Background(), s.WebhookID, onTick,
+		context.Background(), s.WebhookID,
 	)
 
 	select {
@@ -312,9 +312,8 @@ func TestSweepSettlesStrandedPendingWithoutResending(
 
 // TestFailedResultWriteLeavesDeliveryRecoverable is the rule the
 // targets now follow: a bookkeeping write that fails must not advance
-// the status, because pending and retrying are the states the recovery
+// the status, because pending and retrying are the states the sweeps
-// routine recovers and delivered is a claim the database refused to
+// recover and delivered is a claim the database refused to record.
-// record.
 func TestFailedResultWriteLeavesDeliveryRecoverable(
 	t *testing.T,
 ) {

internal/delivery/recovery_test.go

@@ -1,389 +0,0 @@
-package delivery_test
-
-import (
-	"context"
-	"log/slog"
-	"net/http"
-	"testing"
-	"time"
-
-	"github.com/google/uuid"
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
-	"sneak.berlin/go/webhooker/internal/database"
-	"sneak.berlin/go/webhooker/internal/database/databasetest"
-	"sneak.berlin/go/webhooker/internal/delivery"
-)
-
-// These tests cover the recovery routine, which the engine runs once
-// when it starts and then on every tick of the sweep
-// (https://git.eeqj.de/sneak/webhooker/issues/542). Every case runs it
-// both ways, each on a webhook of its own.
-
-// atStart and onTick are the two values of the recovery routine's
-// atStart parameter: its run when the engine starts, and a tick of the
-// sweep.
-const (
-	atStart = true
-	onTick  = false
-)
-
-// recoveryRun names one of the two runs of the recovery routine.
-type recoveryRun struct {
-	name    string
-	atStart bool
-}
-
-// recoveryRuns returns the routine's run at start and a tick.
-func recoveryRuns() []recoveryRun {
-	return []recoveryRun{
-		{"at start", atStart},
-		{"on a tick", onTick},
-	}
-}
-
-// recoverySetup gives a fresh webhook its main-database row and an http
-// target with retries, and returns the setup and the target's ID. No
-// worker runs, so whatever the routine queues stays in the channels for
-// the test to read, and nothing is sent to the target.
-func recoverySetup(t *testing.T) (iSetup, string) {
-	t.Helper()
-
-	s := newISetup(t)
-	targetID := uuid.New().String()
-
-	iCreateWebhook(t, s.MainDB, s.WebhookID, "recovery")
-	iCreateTarget(
-		t, s.MainDB, targetID, s.WebhookID, "recovery",
-		database.TargetTypeHTTP,
-		"http://example.com/hook", 20,
-	)
-
-	return s, targetID
-}
-
-// seedRecoveryDelivery seeds a delivery at status with the given
-// recorded attempts, failed but for the last when lastSucceeded is set,
-// all recorded now.
-func seedRecoveryDelivery(
-	t *testing.T,
-	s iSetup,
-	targetID string,
-	status database.DeliveryStatus,
-	attempts int,
-	lastSucceeded bool,
-) string {
-	t.Helper()
-
-	event := iSeedEvent(t, s.WebhookDB, s.WebhookID, `{"recovery":true}`)
-	d := iSeedDelivery(t, s.WebhookDB, event.ID, targetID, status)
-
-	for n := 1; n <= attempts; n++ {
-		rSeedResult(
-			t, s.WebhookDB, d.ID, n, lastSucceeded && n == attempts,
-		)
-	}
-
-	return d.ID
-}
-
-// backdateAttempts moves every recorded attempt of a delivery an hour
-// into the past, long past the backoff after its first attempt.
-func backdateAttempts(t *testing.T, s iSetup, deliveryID string) {
-	t.Helper()
-
-	require.NoError(t, s.WebhookDB.Model(&database.DeliveryResult{}).
-		Where("delivery_id = ?", deliveryID).
-		UpdateColumn("created_at", s.WebhookDB.NowFunc().Add(-time.Hour)).
-		Error)
-}
-
-// requireQueued waits for a task on ch. A retrying delivery reaches the
-// retry channel through a timer, even one due now, so it is waited for.
-func requireQueued(
-	t *testing.T, ch <-chan delivery.Task,
-) delivery.Task {
-	t.Helper()
-
-	select {
-	case task := <-ch:
-		return task
-	case <-time.After(5 * time.Second):
-		t.Fatal("the recovery routine queued nothing")
-
-		return delivery.Task{}
-	}
-}
-
-// assertNothingQueued asserts the engine queues nothing, giving a retry
-// timer that is due now the moment it needs to fire.
-func assertNothingQueued(t *testing.T, e *delivery.Engine) {
-	t.Helper()
-
-	select {
-	case task := <-e.ExportDeliveryCh():
-		t.Errorf("queued pending delivery %s", task.DeliveryID)
-	case task := <-e.ExportRetryCh():
-		t.Errorf("queued retrying delivery %s", task.DeliveryID)
-	case <-time.After(200 * time.Millisecond):
-	}
-}
-
-// TestRecovery_RetryingBackoffPassed: a retrying delivery whose backoff
-// is over is queued for its next attempt at once.
-func TestRecovery_RetryingBackoffPassed(t *testing.T) {
-	t.Parallel()
-
-	for _, run := range recoveryRuns() {
-		t.Run(run.name, func(t *testing.T) {
-			t.Parallel()
-
-			s, targetID := recoverySetup(t)
-			id := seedRecoveryDelivery(
-				t, s, targetID, database.DeliveryStatusRetrying, 1, false,
-			)
-			backdateAttempts(t, s, id)
-
-			s.Engine.ExportRecoverDeliveries(context.Background(), run.atStart)
-
-			task := requireQueued(t, s.Engine.ExportRetryCh())
-			assert.Equal(t, id, task.DeliveryID)
-			assert.Equal(t, 2, task.AttemptNum)
-			iAssertStatus(t, s.WebhookDB, id, database.DeliveryStatusRetrying)
-		})
-	}
-}
-
-// TestRecovery_RetryingBackoffNotPassed: a retrying delivery whose last
-// attempt was just now waits out the rest of its backoff, ten attempts
-// in, eight and a half minutes. It is not sent now, and the engine holds
-// it, so a later tick leaves it to the timer.
-func TestRecovery_RetryingBackoffNotPassed(t *testing.T) {
-	t.Parallel()
-
-	for _, run := range recoveryRuns() {
-		t.Run(run.name, func(t *testing.T) {
-			t.Parallel()
-
-			s, targetID := recoverySetup(t)
-			id := seedRecoveryDelivery(
-				t, s, targetID, database.DeliveryStatusRetrying, 10, false,
-			)
-
-			s.Engine.ExportRecoverDeliveries(context.Background(), run.atStart)
-
-			assertNothingQueued(t, s.Engine)
-			iAssertStatus(t, s.WebhookDB, id, database.DeliveryStatusRetrying)
-			assert.Equal(t, 1, s.Engine.ExportInflightHeld(),
-				"no retry timer holds the delivery",
-			)
-		})
-	}
-}
-
-// TestRecovery_PendingOlderThanTheAgeBound: a pending delivery untouched
-// for longer than the sweep's age bound is sent by either run.
-func TestRecovery_PendingOlderThanTheAgeBound(t *testing.T) {
-	t.Parallel()
-
-	for _, run := range recoveryRuns() {
-		t.Run(run.name, func(t *testing.T) {
-			t.Parallel()
-
-			s, targetID := recoverySetup(t)
-			id := seedRecoveryDelivery(
-				t, s, targetID, database.DeliveryStatusPending, 0, false,
-			)
-			rAgePending(t, s.WebhookDB, id)
-
-			s.Engine.ExportRecoverDeliveries(context.Background(), run.atStart)
-
-			task := requireQueued(t, s.Engine.ExportDeliveryCh())
-			assert.Equal(t, id, task.DeliveryID)
-			assert.Equal(t, 1, task.AttemptNum)
-		})
-	}
-}
-
-// TestRecovery_TaskCarriesEveryTargetSetting: the task the recovery
-// routine queues carries every one of the target's settings, all six
-// set here.
-// They are copied into the task one field at a time, and a field left
-// out is sent empty without any error.
-func TestRecovery_TaskCarriesEveryTargetSetting(t *testing.T) {
-	t.Parallel()
-
-	for _, run := range recoveryRuns() {
-		t.Run(run.name, func(t *testing.T) {
-			t.Parallel()
-
-			s := newISetup(t)
-			iCreateWebhook(t, s.MainDB, s.WebhookID, "recovery")
-
-			target := database.Target{
-				WebhookID:       s.WebhookID,
-				Name:            "recovery",
-				Type:            database.TargetTypeHTTP,
-				Active:          true,
-				URL:             "http://example.com/hook",
-				Headers:         map[string]string{headerAuthorization: "Bearer t"},
-				TimeoutSeconds:  7,
-				ForwardQuery:    true,
-				ArchiveExpiry:   time.Hour,
-				ArchiveRotation: "daily",
-				MaxAttempts:     1,
-			}
-			require.NoError(t, s.MainDB.Create(&target).Error)
-
-			id := seedRecoveryDelivery(
-				t, s, target.ID, database.DeliveryStatusPending, 0, false,
-			)
-			rAgePending(t, s.WebhookDB, id)
-
-			s.Engine.ExportRecoverDeliveries(context.Background(), run.atStart)
-
-			task := requireQueued(t, s.Engine.ExportDeliveryCh())
-			assert.Equal(t, target.ID, task.TargetID)
-			assert.Equal(t, target.URL, task.TargetURL)
-			assert.Equal(t, target.Headers, task.TargetHeaders)
-			assert.Equal(t, target.TimeoutSeconds, task.TimeoutSeconds)
-			assert.Equal(t, target.ForwardQuery, task.ForwardQuery)
-			assert.Equal(t, target.ArchiveExpiry, task.ArchiveExpiry)
-			assert.Equal(t, target.ArchiveRotation, task.ArchiveRotation)
-		})
-	}
-}
-
-// TestRecovery_PendingYoungerThanTheAgeBound: a pending delivery touched
-// a moment ago, as one a stop cut off is, is sent at start, which is how
-// it reaches its target on the next start
-// (https://git.eeqj.de/sneak/webhooker/issues/527). A tick leaves it.
-func TestRecovery_PendingYoungerThanTheAgeBound(t *testing.T) {
-	t.Parallel()
-
-	for _, run := range recoveryRuns() {
-		t.Run(run.name, func(t *testing.T) {
-			t.Parallel()
-
-			s, targetID := recoverySetup(t)
-			id := seedRecoveryDelivery(
-				t, s, targetID, database.DeliveryStatusPending, 0, false,
-			)
-
-			s.Engine.ExportRecoverDeliveries(context.Background(), run.atStart)
-
-			if run.atStart {
-				task := requireQueued(t, s.Engine.ExportDeliveryCh())
-				assert.Equal(t, id, task.DeliveryID)
-
-				return
-			}
-
-			assertNothingQueued(t, s.Engine)
-			iAssertStatus(t, s.WebhookDB, id, database.DeliveryStatusPending)
-			assert.Zero(t, s.Engine.ExportInflightHeld())
-		})
-	}
-}
-
-// TestRecovery_AlreadySucceeded: a pending or retrying delivery whose
-// success is recorded but whose status write was lost is marked
-// delivered where it stands, and not sent again.
-func TestRecovery_AlreadySucceeded(t *testing.T) {
-	t.Parallel()
-
-	for _, run := range recoveryRuns() {
-		t.Run(run.name, func(t *testing.T) {
-			t.Parallel()
-
-			s, targetID := recoverySetup(t)
-			pending := seedRecoveryDelivery(
-				t, s, targetID, database.DeliveryStatusPending, 1, true,
-			)
-			rAgePending(t, s.WebhookDB, pending)
-
-			retrying := seedRecoveryDelivery(
-				t, s, targetID, database.DeliveryStatusRetrying, 2, true,
-			)
-			backdateAttempts(t, s, retrying)
-
-			s.Engine.ExportRecoverDeliveries(context.Background(), run.atStart)
-
-			assertNothingQueued(t, s.Engine)
-
-			for id, attempts := range map[string]int{pending: 1, retrying: 2} {
-				iAssertStatus(
-					t, s.WebhookDB, id, database.DeliveryStatusDelivered,
-				)
-				assert.Len(t, iResults(t, s.WebhookDB, id), attempts,
-					"marking the delivery delivered recorded an attempt",
-				)
-			}
-		})
-	}
-}
-
-// TestRecovery_MissingDatabase: a tick leaves a webhook whose database
-// file is missing alone, and the run at start opens it, which creates an
-// empty one and reports the loss (see
-// TestRecoverDeliveries_ReportsAMissingWebhookDatabase).
-func TestRecovery_MissingDatabase(t *testing.T) {
-	t.Parallel()
-
-	mainDB := iMainDB(t)
-	webhookID := uuid.New().String()
-	iCreateWebhook(t, mainDB.DB(), webhookID, "missing-database")
-
-	dbMgr := databasetest.NewWebhookDBManager(t, t.TempDir())
-	t.Cleanup(func() { _ = dbMgr.CloseAll() })
-
-	engine := delivery.NewTestEngineWithDB(
-		mainDB, dbMgr,
-		slog.New(slog.DiscardHandler),
-		&http.Client{Timeout: 5 * time.Second}, 1,
-	)
-
-	engine.ExportRecoverDeliveries(context.Background(), onTick)
-	assert.False(t, dbMgr.DBExists(webhookID),
-		"a tick created the missing database",
-	)
-
-	engine.ExportRecoverDeliveries(context.Background(), atStart)
-	assert.True(t, dbMgr.DBExists(webhookID),
-		"the run at start did not open the missing database",
-	)
-}
-
-// TestRecovery_LeavesADeliveryTheEngineHolds: a delivery the engine is
-// working on, has queued, or holds a retry timer for is left alone by
-// either run, pending and retrying alike, however old.
-func TestRecovery_LeavesADeliveryTheEngineHolds(t *testing.T) {
-	t.Parallel()
-
-	for _, run := range recoveryRuns() {
-		t.Run(run.name, func(t *testing.T) {
-			t.Parallel()
-
-			s, targetID := recoverySetup(t)
-			pending := seedRecoveryDelivery(
-				t, s, targetID, database.DeliveryStatusPending, 0, false,
-			)
-			rAgePending(t, s.WebhookDB, pending)
-
-			retrying := seedRecoveryDelivery(
-				t, s, targetID, database.DeliveryStatusRetrying, 1, false,
-			)
-			backdateAttempts(t, s, retrying)
-
-			require.True(t, s.Engine.ExportRetainDelivery(pending))
-			require.True(t, s.Engine.ExportRetainDelivery(retrying))
-
-			s.Engine.ExportRecoverDeliveries(context.Background(), run.atStart)
-
-			assertNothingQueued(t, s.Engine)
-			iAssertStatus(t, s.WebhookDB, pending, database.DeliveryStatusPending)
-			iAssertStatus(t, s.WebhookDB, retrying, database.DeliveryStatusRetrying)
-			assert.Equal(t, 2, s.Engine.ExportInflightHeld())
-		})
-	}
-}

internal/delivery/store.go

@@ -1,164 +0,0 @@
-package delivery
-
-import (
-	"fmt"
-
-	"gorm.io/gorm"
-	"sneak.berlin/go/webhooker/internal/database"
-)
-
-// StoreEvent stores event with one pending delivery for each of its
-// webhook's active targets, and adds them to the totals, in one
-// transaction, then queues the deliveries. It returns how many
-// deliveries it created.
-//
-// The receiver stores every event it accepts through here, and a
-// resubmit stores its copy of a stored event, so every event is
-// stored, retried, SSRF-guarded and circuit-broken the same way however
-// it entered the system. CreateReplay is the only other place a
-// delivery is created.
-//
-// The targets are the webhook's active ones as they stand now: a
-// target created after a resubmitted event first arrived is included,
-// and an inactive one is skipped. An event with no active target is
-// stored with no deliveries.
-func (e *Engine) StoreEvent(event *database.Event) (int, error) {
-	var targets []database.Target
-
-	err := e.database.DB().Where(
-		"webhook_id = ? AND active = ?", event.WebhookID, true,
-	).Find(&targets).Error
-	if err != nil {
-		return 0, fmt.Errorf("loading active targets: %w", err)
-	}
-
-	webhookDB, err := e.dbManager.GetDB(event.WebhookID)
-	if err != nil {
-		return 0, fmt.Errorf("getting webhook database: %w", err)
-	}
-
-	event.BodyBytes = int64(len(event.Body))
-	tasks := make([]Task, 0, len(targets))
-
-	err = webhookDB.Transaction(func(tx *gorm.DB) error {
-		err := tx.Create(event).Error
-		if err != nil {
-			return fmt.Errorf("creating event: %w", err)
-		}
-
-		for i := range targets {
-			dlv, err := createDelivery(tx, event.ID, targets[i].ID, false)
-			if err != nil {
-				return err
-			}
-
-			tasks = append(tasks, buildTask(
-				dlv, event.WebhookID, event, &targets[i], 1,
-			))
-		}
-
-		err = database.AddEventTotals(tx, database.EventTotals{
-			Events: 1, LastEventAt: &event.CreatedAt,
-		})
-		if err != nil {
-			return err
-		}
-
-		// A resubmitted copy did not arrive on its entrypoint's URL, so
-		// it leaves the entrypoint's last event as it is.
-		if event.ResubmittedFromID != nil {
-			return nil
-		}
-
-		return database.AddEntrypointTotals(tx, database.EntrypointTotals{
-			EntrypointID: event.EntrypointID, LastEventAt: event.CreatedAt,
-		})
-	})
-	if err != nil {
-		return 0, err
-	}
-
-	// Counted here, after the commit: an event exists once it is
-	// durably stored, which is what the delivery counters are compared
-	// against on a dashboard. A resubmitted event counts too, because
-	// it produces deliveries that the delivery side counts; the
-	// resubmit counter is what separates the two.
-	e.mtr.EventReceived()
-
-	e.notify(tasks)
-
-	return len(tasks), nil
-}
-
-// CreateReplay creates one more pending delivery of the stored event
-// eventID to target, marked as a replay, adds it to the target's totals
-// in the same transaction, and queues it. It returns the new
-// delivery's id.
-//
-// The caller passes the target as it stands now and has already
-// decided the replay may go ahead. The delivery being replayed is not
-// touched.
-func (e *Engine) CreateReplay(
-	webhookID, eventID string, target *database.Target,
-) (string, error) {
-	webhookDB, err := e.dbManager.GetDB(webhookID)
-	if err != nil {
-		return "", fmt.Errorf("getting webhook database: %w", err)
-	}
-
-	event, err := e.loadEvent(webhookDB, eventID)
-	if err != nil {
-		return "", err
-	}
-
-	var dlv *database.Delivery
-
-	err = webhookDB.Transaction(func(tx *gorm.DB) error {
-		var err error
-
-		dlv, err = createDelivery(tx, event.ID, target.ID, true)
-
-		return err
-	})
-	if err != nil {
-		return "", err
-	}
-
-	e.notify([]Task{buildTask(dlv, webhookID, &event, target, 1)})
-
-	return dlv.ID, nil
-}
-
-// createDelivery writes a pending delivery of the event to the target
-// on the caller's transaction, and adds it to the target's totals.
-//
-// Neither Event nor Target is set on the row, and the per-webhook
-// database omits associations on every write in any case (see
-// internal/database/event_db_isolation.go), so the target's settings
-// are never copied into the event database.
-func createDelivery(
-	tx *gorm.DB, eventID, targetID string, replay bool,
-) (*database.Delivery, error) {
-	dlv := &database.Delivery{
-		EventID:  eventID,
-		TargetID: targetID,
-		Status:   database.DeliveryStatusPending,
-		Replay:   replay,
-	}
-
-	err := tx.Create(dlv).Error
-	if err != nil {
-		return nil, fmt.Errorf(
-			"creating delivery for target %s: %w", targetID, err,
-		)
-	}
-
-	err = database.AddTargetTotals(tx, database.TargetTotals{
-		TargetID: targetID, Deliveries: 1,
-	})
-	if err != nil {
-		return nil, err
-	}
-
-	return dlv, nil
-}

internal/delivery/store_test.go

@@ -1,386 +0,0 @@
-package delivery_test
-
-import (
-	"net/http"
-	"strings"
-	"testing"
-	"time"
-
-	"github.com/google/uuid"
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
-	"gorm.io/gorm"
-	"sneak.berlin/go/webhooker/internal/database"
-	"sneak.berlin/go/webhooker/internal/delivery"
-)
-
-// storeTestTarget creates an active http target of the setup's webhook
-// with every one of a target's settings set, so a task that drops one
-// of them differs from the target.
-func storeTestTarget(
-	t *testing.T, s iSetup, name string,
-) database.Target {
-	t.Helper()
-
-	target := database.Target{
-		WebhookID:       s.WebhookID,
-		Name:            name,
-		Type:            database.TargetTypeHTTP,
-		Active:          true,
-		URL:             "http://93.184.216.34/" + name,
-		Headers:         map[string]string{"X-Api-Key": "k"},
-		TimeoutSeconds:  7,
-		ForwardQuery:    true,
-		ArchiveExpiry:   time.Hour,
-		ArchiveRotation: rotationDaily,
-		MaxAttempts:     3,
-	}
-
-	require.NoError(t, s.MainDB.Create(&target).Error)
-
-	return target
-}
-
-// newStoreTestEvent returns an event of the setup's webhook as the
-// receiver builds one, not yet stored.
-func newStoreTestEvent(s iSetup, body string) *database.Event {
-	return &database.Event{
-		WebhookID:    s.WebhookID,
-		EntrypointID: uuid.New().String(),
-		Method:       http.MethodPost,
-		RawQuery:     "a=1&b=2",
-		Headers:      `{"X-Test":["yes"]}`,
-		ContentType:  testContentType,
-		Body:         body,
-	}
-}
-
-// queuedTasks takes every task the engine has queued so far. The
-// engine in these tests is never started, so nothing else takes them.
-func queuedTasks(e *delivery.Engine) []delivery.Task {
-	var tasks []delivery.Task
-
-	for {
-		select {
-		case task := <-e.ExportDeliveryCh():
-			tasks = append(tasks, task)
-		default:
-			return tasks
-		}
-	}
-}
-
-// assertTaskFor proves task carries delivery deliveryID of event to
-// target: every one of the target's settings and every event field.
-// They are copied into the task one field at a time, and a field left
-// out is sent empty without any error.
-func assertTaskFor(
-	t *testing.T,
-	task delivery.Task,
-	deliveryID string,
-	event *database.Event,
-	target *database.Target,
-) {
-	t.Helper()
-
-	assert.Equal(t, deliveryID, task.DeliveryID)
-	assert.Equal(t, event.ID, task.EventID)
-	assert.Equal(t, event.WebhookID, task.WebhookID)
-	assert.Equal(t, event.EntrypointID, task.EntrypointID)
-
-	assert.Equal(t, target.ID, task.TargetID)
-	assert.Equal(t, target.Name, task.TargetName)
-	assert.Equal(t, target.Type, task.TargetType)
-	assert.Equal(t, target.URL, task.TargetURL)
-	assert.Equal(t, target.Headers, task.TargetHeaders)
-	assert.Equal(t, target.TimeoutSeconds, task.TimeoutSeconds)
-	assert.Equal(t, target.ForwardQuery, task.ForwardQuery)
-	assert.Equal(t, target.ArchiveExpiry, task.ArchiveExpiry)
-	assert.Equal(t, target.ArchiveRotation, task.ArchiveRotation)
-	assert.Equal(t, target.MaxAttempts, task.MaxAttempts)
-
-	assert.Equal(t, event.Method, task.Method)
-	assert.Equal(t, event.RawQuery, task.RawQuery)
-	assert.Equal(t, event.Headers, task.Headers)
-	assert.Equal(t, event.ContentType, task.ContentType)
-	require.NotNil(t, task.Body)
-	assert.Equal(t, event.Body, *task.Body)
-
-	assert.Equal(t, 1, task.AttemptNum)
-}
-
-// storedDeliveries reads every delivery of an event.
-func storedDeliveries(
-	t *testing.T, db *gorm.DB, eventID string,
-) []database.Delivery {
-	t.Helper()
-
-	var deliveries []database.Delivery
-
-	require.NoError(t, db.Where("event_id = ?", eventID).
-		Find(&deliveries).Error)
-
-	return deliveries
-}
-
-// storedEventTotals reads the webhook's event totals.
-func storedEventTotals(t *testing.T, db *gorm.DB) database.EventTotals {
-	t.Helper()
-
-	var totals database.EventTotals
-
-	require.NoError(t, db.First(&totals).Error)
-
-	return totals
-}
-
-// countRows counts the rows of model in db.
-func countRows(t *testing.T, db *gorm.DB, model any) int64 {
-	t.Helper()
-
-	var n int64
-
-	require.NoError(t, db.Model(model).Count(&n).Error)
-
-	return n
-}
-
-// TestStoreEvent_StoresDeliveriesToActiveTargets proves StoreEvent
-// stores the event with one pending delivery to each active target and
-// none to an inactive one, adds them to the totals, and queues one task
-// per delivery.
-func TestStoreEvent_StoresDeliveriesToActiveTargets(t *testing.T) {
-	t.Parallel()
-
-	s := newISetup(t)
-	targets := []database.Target{
-		storeTestTarget(t, s, "first"),
-		storeTestTarget(t, s, "second"),
-	}
-	off := storeTestTarget(t, s, "off")
-
-	require.NoError(t, s.MainDB.Model(&database.Target{}).
-		Where("id = ?", off.ID).Update("active", false).Error)
-
-	event := newStoreTestEvent(s, `{"hello":"world"}`)
-
-	created, err := s.Engine.StoreEvent(event)
-	require.NoError(t, err)
-	assert.Equal(t, 2, created)
-
-	var stored database.Event
-
-	require.NoError(t, s.WebhookDB.First(&stored, "id = ?", event.ID).Error)
-	assert.Equal(t, event.Body, stored.Body)
-	assert.Equal(t, int64(len(event.Body)), stored.BodyBytes)
-
-	rows := storedDeliveries(t, s.WebhookDB, event.ID)
-	queued := queuedTasks(s.Engine)
-
-	require.Len(t, rows, 2)
-	require.Len(t, queued, 2)
-
-	deliveries := map[string]database.Delivery{}
-	for _, dlv := range rows {
-		deliveries[dlv.TargetID] = dlv
-	}
-
-	tasks := map[string]delivery.Task{}
-	for _, task := range queued {
-		tasks[task.TargetID] = task
-	}
-
-	for i := range targets {
-		dlv, ok := deliveries[targets[i].ID]
-		require.True(t, ok, "no delivery to %s", targets[i].Name)
-		assert.Equal(t, database.DeliveryStatusPending, dlv.Status)
-		assert.False(t, dlv.Replay)
-		assert.Equal(t, int64(1),
-			targetTotals(t, s.WebhookDB, targets[i].ID).Deliveries)
-
-		assertTaskFor(t, tasks[targets[i].ID], dlv.ID, event, &targets[i])
-	}
-
-	totals := storedEventTotals(t, s.WebhookDB)
-	assert.Equal(t, int64(1), totals.Events)
-	require.NotNil(t, totals.LastEventAt)
-	assert.True(t, totals.LastEventAt.Equal(stored.CreatedAt))
-
-	var entrypoint database.EntrypointTotals
-
-	require.NoError(t, s.WebhookDB.First(
-		&entrypoint, "entrypoint_id = ?", event.EntrypointID,
-	).Error)
-	assert.True(t, entrypoint.LastEventAt.Equal(stored.CreatedAt))
-}
-
-// TestStoreEvent_ResubmittedCopyLeavesTheEntrypoint proves a
-// resubmitted copy is counted among the webhook's events but leaves its
-// entrypoint's last event as it is: it did not arrive on that URL.
-func TestStoreEvent_ResubmittedCopyLeavesTheEntrypoint(t *testing.T) {
-	t.Parallel()
-
-	s := newISetup(t)
-	storeTestTarget(t, s, "only")
-
-	sourceID := uuid.New().String()
-	event := newStoreTestEvent(s, `{"copy":true}`)
-	event.ResubmittedFromID = &sourceID
-
-	created, err := s.Engine.StoreEvent(event)
-	require.NoError(t, err)
-	assert.Equal(t, 1, created)
-	assert.Len(t, queuedTasks(s.Engine), 1)
-
-	assert.Equal(t, int64(1), storedEventTotals(t, s.WebhookDB).Events)
-	assert.Zero(t, countRows(t, s.WebhookDB, &database.EntrypointTotals{}))
-}
-
-// TestStoreEvent_NoActiveTarget proves an event of a webhook with no
-// active target is stored with no delivery and nothing queued.
-func TestStoreEvent_NoActiveTarget(t *testing.T) {
-	t.Parallel()
-
-	s := newISetup(t)
-	event := newStoreTestEvent(s, `{"to":"nobody"}`)
-
-	created, err := s.Engine.StoreEvent(event)
-	require.NoError(t, err)
-	assert.Zero(t, created)
-
-	assert.Equal(t, int64(1), countRows(t, s.WebhookDB, &database.Event{}))
-	assert.Empty(t, storedDeliveries(t, s.WebhookDB, event.ID))
-	assert.Empty(t, queuedTasks(s.Engine))
-}
-
-// TestStoreEvent_LargeBodyIsNotCarried proves a body at the inline
-// limit is stored whole but left off the task, for the attempt to read
-// from the event row.
-func TestStoreEvent_LargeBodyIsNotCarried(t *testing.T) {
-	t.Parallel()
-
-	s := newISetup(t)
-	storeTestTarget(t, s, "only")
-
-	event := newStoreTestEvent(
-		s, strings.Repeat("x", delivery.MaxInlineBodySize),
-	)
-
-	_, err := s.Engine.StoreEvent(event)
-	require.NoError(t, err)
-
-	tasks := queuedTasks(s.Engine)
-	require.Len(t, tasks, 1)
-	assert.Nil(t, tasks[0].Body)
-
-	var stored database.Event
-
-	require.NoError(t, s.WebhookDB.First(&stored, "id = ?", event.ID).Error)
-	assert.Equal(t, event.Body, stored.Body)
-}
-
-// TestStoreEvent_RollsBackWhenAWriteFails proves the event, its
-// deliveries and its totals are one transaction: when the entrypoint
-// totals write fails, the last one, after the event, its delivery, the
-// target totals and the event totals were written, none of them is left
-// and nothing is queued.
-func TestStoreEvent_RollsBackWhenAWriteFails(t *testing.T) {
-	t.Parallel()
-
-	s := newISetup(t)
-	storeTestTarget(t, s, "only")
-
-	require.NoError(t,
-		s.WebhookDB.Migrator().DropTable(&database.EntrypointTotals{}))
-
-	_, err := s.Engine.StoreEvent(newStoreTestEvent(s, `{"lost":true}`))
-	require.Error(t, err)
-
-	assert.Zero(t, countRows(t, s.WebhookDB, &database.Event{}))
-	assert.Zero(t, countRows(t, s.WebhookDB, &database.Delivery{}))
-	assert.Zero(t, countRows(t, s.WebhookDB, &database.TargetTotals{}))
-
-	totals := storedEventTotals(t, s.WebhookDB)
-	assert.Zero(t, totals.Events)
-	assert.Nil(t, totals.LastEventAt)
-
-	assert.Empty(t, queuedTasks(s.Engine))
-	assert.Zero(t, s.Engine.ExportInflightHeld())
-}
-
-// TestCreateReplay_CreatesAndQueuesAReplay proves CreateReplay adds a
-// new pending delivery marked as a replay, counts it in the target's
-// totals, queues its task built from the target it was given and the
-// stored event, and leaves the delivery it repeats as it was. The
-// stored event has every event field set, so a task that drops one of
-// them differs from the event.
-func TestCreateReplay_CreatesAndQueuesAReplay(t *testing.T) {
-	t.Parallel()
-
-	s := newISetup(t)
-	target := storeTestTarget(t, s, "replayed")
-	event := newStoreTestEvent(s, `{"replay":"me"}`)
-	require.NoError(t, s.WebhookDB.Create(event).Error)
-
-	original := iSeedDelivery(
-		t, s.WebhookDB, event.ID, target.ID,
-		database.DeliveryStatusFailed,
-	)
-
-	deliveryID, err := s.Engine.CreateReplay(s.WebhookID, event.ID, &target)
-	require.NoError(t, err)
-
-	replay := storedDelivery(t, s.WebhookDB, deliveryID)
-	assert.Equal(t, event.ID, replay.EventID)
-	assert.Equal(t, target.ID, replay.TargetID)
-	assert.Equal(t, database.DeliveryStatusPending, replay.Status)
-	assert.True(t, replay.Replay)
-
-	assert.Equal(t, database.DeliveryStatusFailed,
-		storedDelivery(t, s.WebhookDB, original.ID).Status)
-	assert.Equal(t, int64(1),
-		targetTotals(t, s.WebhookDB, target.ID).Deliveries)
-
-	tasks := queuedTasks(s.Engine)
-	require.Len(t, tasks, 1)
-	assertTaskFor(t, tasks[0], deliveryID, event, &target)
-}
-
-// TestCreateReplay_RollsBackWhenAWriteFails proves the replay delivery
-// and its target totals are one transaction: when the totals write
-// fails, after the delivery was written, no new delivery is left and
-// nothing is queued.
-func TestCreateReplay_RollsBackWhenAWriteFails(t *testing.T) {
-	t.Parallel()
-
-	s := newISetup(t)
-	target := storeTestTarget(t, s, "replayed")
-	event := iSeedEvent(t, s.WebhookDB, s.WebhookID, `{"replay":"me"}`)
-	iSeedDelivery(
-		t, s.WebhookDB, event.ID, target.ID,
-		database.DeliveryStatusFailed,
-	)
-
-	require.NoError(t,
-		s.WebhookDB.Migrator().DropTable(&database.TargetTotals{}))
-
-	_, err := s.Engine.CreateReplay(s.WebhookID, event.ID, &target)
-	require.Error(t, err)
-
-	assert.Len(t, storedDeliveries(t, s.WebhookDB, event.ID), 1)
-	assert.Empty(t, queuedTasks(s.Engine))
-	assert.Zero(t, s.Engine.ExportInflightHeld())
-}
-
-// storedDelivery reads one delivery back.
-func storedDelivery(
-	t *testing.T, db *gorm.DB, deliveryID string,
-) database.Delivery {
-	t.Helper()
-
-	var dlv database.Delivery
-
-	require.NoError(t, db.First(&dlv, "id = ?", deliveryID).Error)
-
-	return dlv
-}

internal/delivery/target.go

@@ -13,8 +13,8 @@ import (
 // The engine provides one to each target so a target can own
 // its retries durably: it records the attempt, marks the
 // delivery retrying, and asks the Scheduler to deliver the
-// next attempt after delay — exactly what the engine's
+// next attempt after delay — exactly what the engine does for
-// recovery routine does for a retrying delivery.
+// its own restart recovery.
 type Scheduler interface {
 	ScheduleRetry(task Task, delay time.Duration)
 }
@@ -45,20 +45,29 @@ type Target interface {
 }
 
 // rescheduler is implemented by targets that own durable
-// retries. The engine's recovery routine uses it to let the
+// retries. The engine's restart recovery and periodic sweep
-// target recompute the schedule for an orphaned retrying
+// use it to let the target recompute the schedule for an
-// delivery, keeping the retry schedule target-owned.
+// orphaned retrying delivery, keeping the retry schedule
-// Fire-and-forget targets do not implement it, and a retrying
+// target-owned. Fire-and-forget targets do not implement it
-// delivery found for one is terminally failed.
+// and their (never-occurring) retrying deliveries are
+// skipped.
 type rescheduler interface {
 	// remainingBackoff returns how long to wait before the
-	// next attempt of a recovered retrying delivery: zero once
+	// next attempt of a recovered retrying delivery.
-	// its backoff has passed.
 	remainingBackoff(
 		webhookDB *gorm.DB,
 		deliveryID string,
 		attemptNum int,
 	) time.Duration
+
+	// backoffElapsed reports whether the backoff window for
+	// the last attempt has already passed, so the periodic
+	// sweep can re-enqueue the delivery now.
+	backoffElapsed(
+		webhookDB *gorm.DB,
+		deliveryID string,
+		attemptNum int,
+	) bool
 }
 
 // attemptResult is the outcome of a single delivery attempt,

internal/delivery/target_database_test.go

@@ -1,7 +1,6 @@
 package delivery_test
 
 import (
-	"context"
 	"database/sql"
 	"fmt"
 	"log/slog"
@@ -21,7 +20,6 @@ import (
 	"sneak.berlin/go/webhooker/internal/archive"
 	"sneak.berlin/go/webhooker/internal/database"
 	"sneak.berlin/go/webhooker/internal/database/databasetest"
-	"sneak.berlin/go/webhooker/internal/db/migrations"
 	"sneak.berlin/go/webhooker/internal/delivery"
 	"sneak.berlin/go/webhooker/internal/gormlog"
 )
@@ -309,87 +307,3 @@ func TestDeliverDatabase_ArchiveFailureFailsDelivery(
 		"no archive file should exist for a failed write",
 	)
 }
-
-// seedArchivedRow creates the archive file at path holding one row, for
-// eventID, archived at archivedAt.
-func seedArchivedRow(
-	t *testing.T, path, eventID string, archivedAt time.Time,
-) {
-	t.Helper()
-
-	sqlDB, err := sql.Open(
-		"sqlite", fmt.Sprintf("file:%s?mode=rwc", path),
-	)
-	require.NoError(t, err)
-
-	defer func() { _ = sqlDB.Close() }()
-
-	gdb, err := gorm.Open(
-		sqlite.Dialector{Conn: sqlDB},
-		&gorm.Config{Logger: gormlog.New(slog.New(slog.DiscardHandler))},
-	)
-	require.NoError(t, err)
-
-	require.NoError(t, migrations.Apply(gdb, migrations.Archive))
-	require.NoError(t, gdb.Exec(
-		"INSERT INTO archived_events (event_id, archived_at) VALUES (?, ?)",
-		eventID, archivedAt,
-	).Error)
-}
-
-// TestDeliverDatabase_TaskArchiveSettings proves a task's ArchiveExpiry
-// and ArchiveRotation reach the archive through the engine. The
-// target's row has neither, so only the task can bring them: with daily
-// rotation the event goes to the file for its day, and opening that
-// file with a one-hour expiry prunes the row it already held, archived
-// two hours ago.
-func TestDeliverDatabase_TaskArchiveSettings(t *testing.T) {
-	t.Parallel()
-
-	s := newISetup(t)
-
-	webhook := database.Webhook{UserID: uuid.NewString(), Name: "settings"}
-	webhook.ID = s.WebhookID
-	require.NoError(t, s.MainDB.Create(&webhook).Error)
-
-	target := database.Target{
-		WebhookID:   s.WebhookID,
-		Name:        "archive",
-		Type:        database.TargetTypeDatabase,
-		Active:      true,
-		MaxAttempts: 1,
-	}
-	require.NoError(t, s.MainDB.Create(&target).Error)
-
-	event := iSeedEvent(t, s.WebhookDB, s.WebhookID, `{"archived":true}`)
-	d := iSeedDelivery(
-		t, s.WebhookDB, event.ID, target.ID,
-		database.DeliveryStatusPending,
-	)
-
-	task := iTask(
-		d, event, s.WebhookID, target.ID, target.Name, "", 1, 1,
-		&event.Body,
-	)
-	task.TargetType = database.TargetTypeDatabase
-	task.ArchiveExpiry = time.Hour
-	task.ArchiveRotation = "daily"
-
-	daily := target
-	daily.ArchiveRotation = task.ArchiveRotation
-	dailyFile := archive.PathAt(s.DBMgr, &webhook, &daily, event.CreatedAt)
-	seedArchivedRow(t, dailyFile, "expired", time.Now().Add(-2*time.Hour))
-
-	s.Engine.ExportProcessNewTask(context.TODO(), &task)
-
-	iAssertStatus(t, s.WebhookDB, d.ID, database.DeliveryStatusDelivered)
-
-	var rows []archivedRow
-
-	require.NoError(t, openArchiveDBForRead(t, dailyFile).
-		Table("archived_events").Find(&rows).Error)
-	require.Len(t, rows, 1)
-	assert.Equal(t, event.ID, rows[0].EventID,
-		"the event goes to its day's file and the expired row is pruned",
-	)
-}

internal/delivery/target_http.go

@@ -60,8 +60,8 @@ func (c *httpCore) deliver(
 // engine is stopping, which cancels the context its request is made
 // with. Such an attempt is not recorded: no DeliveryResult, no circuit
 // breaker outcome, no status change, no count in the attempt metrics.
-// The delivery stays pending or retrying, and the recovery routine
+// The delivery stays pending or retrying, and startup recovery sends
-// sends it again on the next start.
+// it again on the next start.
 //
 // A target's own timeout leaves that context alone, so its attempt is
 // still recorded as the failure it is.
@@ -310,8 +310,7 @@ func (c *httpCore) publishCircuitState(
 
 // remainingBackoff returns how long remains of the backoff
 // window for the last attempt of a recovered retrying
-// delivery: zero once it has passed, or when the last attempt
+// delivery. It implements rescheduler.
-// cannot be read. It implements rescheduler.
 func (c *httpCore) remainingBackoff(
 	webhookDB *gorm.DB,
 	deliveryID string,
@@ -334,6 +333,29 @@ func (c *httpCore) remainingBackoff(
 	return max(remaining, 0)
 }
 
+// backoffElapsed reports whether the backoff window for the
+// last attempt of a retrying delivery has passed. It
+// implements rescheduler.
+func (c *httpCore) backoffElapsed(
+	webhookDB *gorm.DB,
+	deliveryID string,
+	attemptNum int,
+) bool {
+	var lastResult database.DeliveryResult
+
+	err := webhookDB.
+		Where("delivery_id = ?", deliveryID).
+		Order("created_at DESC").
+		First(&lastResult).Error
+	if err != nil {
+		return true
+	}
+
+	backoff := Backoff(attemptNum)
+
+	return time.Since(lastResult.CreatedAt) >= backoff
+}
+
 // Backoff is how long an http or slack target with retries waits after
 // a delivery's failed attempt attemptNum before trying it again.
 func Backoff(attemptNum int) time.Duration {

internal/delivery/terminal_state_test.go

@@ -178,7 +178,7 @@ func TestRecoverSingleRetry_TargetDeleted(t *testing.T) {
 	)
 
 	s.Engine.ExportRecoverWebhookDeliveries(
-		context.Background(), s.WebhookID, atStart,
+		context.Background(), s.WebhookID,
 	)
 
 	iAssertStatus(
@@ -201,7 +201,7 @@ func TestRecoverSingleRetry_TargetDeleted(t *testing.T) {
 	)
 }
 
-func TestRecoverSingleRetry_OnTick_TargetDeleted(t *testing.T) {
+func TestSweepSingleRetry_TargetDeleted(t *testing.T) {
 	t.Parallel()
 
 	s := newISetup(t)
@@ -218,11 +218,11 @@ func TestRecoverSingleRetry_OnTick_TargetDeleted(t *testing.T) {
 	// Twice, because the bug was an error the sweep repeated every
 	// minute for the life of the database: the second sweep must
 	// find nothing left to do.
-	s.Engine.ExportRecoverWebhookDeliveries(
+	s.Engine.ExportSweepWebhookRetries(
-		context.Background(), s.WebhookID, onTick,
+		context.Background(), s.WebhookID,
 	)
-	s.Engine.ExportRecoverWebhookDeliveries(
+	s.Engine.ExportSweepWebhookRetries(
-		context.Background(), s.WebhookID, onTick,
+		context.Background(), s.WebhookID,
 	)
 
 	iAssertStatus(
@@ -239,11 +239,10 @@ func TestRecoverSingleRetry_OnTick_TargetDeleted(t *testing.T) {
 	assert.Zero(t, s.Engine.ExportInflightHeld())
 }
 
-// TestRecoverSingleRetry_OnTick_TargetNeverExisted covers the other
+// TestSweepSingleRetry_TargetNeverExisted covers the other half of the
-// half of the soft-delete distinction: an id with no row at all,
+// soft-delete distinction: an id with no row at all, deleted or
-// deleted or otherwise, must not be reported as something the operator
+// otherwise, must not be reported as something the operator deleted.
-// deleted.
+func TestSweepSingleRetry_TargetNeverExisted(t *testing.T) {
-func TestRecoverSingleRetry_OnTick_TargetNeverExisted(t *testing.T) {
 	t.Parallel()
 
 	s := newISetup(t)
@@ -265,8 +264,8 @@ func TestRecoverSingleRetry_OnTick_TargetNeverExisted(t *testing.T) {
 
 	iSeedFailedResult(t, s.WebhookDB, d.ID)
 
-	s.Engine.ExportRecoverWebhookDeliveries(
+	s.Engine.ExportSweepWebhookRetries(
-		context.Background(), s.WebhookID, onTick,
+		context.Background(), s.WebhookID,
 	)
 
 	iAssertStatus(
@@ -304,8 +303,8 @@ func TestFailMissingTarget_WritesNoTargetRow(
 		database.DeliveryStatusRetrying,
 	)
 
-	s.Engine.ExportRecoverWebhookDeliveries(
+	s.Engine.ExportSweepWebhookRetries(
-		context.Background(), s.WebhookID, onTick,
+		context.Background(), s.WebhookID,
 	)
 
 	iAssertStatus(
@@ -546,7 +545,7 @@ func TestRecoverSingleRetry_TargetUnreadable_LeavesDeliveryAlone(
 	require.NoError(t, sqlDB.Close())
 
 	s.Engine.ExportRecoverRetryingDeliveries(
-		context.Background(), s.WebhookDB, s.WebhookID,
+		s.WebhookDB, s.WebhookID,
 	)
 
 	iAssertStatus(
@@ -564,7 +563,7 @@ func TestRecoverSingleRetry_TargetUnreadable_LeavesDeliveryAlone(
 
 // --- 4. A pending delivery whose target is gone ---
 
-func TestRecoverPendingDeliveries_TargetDeleted(t *testing.T) {
+func TestRecoverPending_TargetDeleted(t *testing.T) {
 	t.Parallel()
 
 	s := newISetup(t)
@@ -577,7 +576,7 @@ func TestRecoverPendingDeliveries_TargetDeleted(t *testing.T) {
 	)
 
 	s.Engine.ExportRecoverWebhookDeliveries(
-		context.Background(), s.WebhookID, atStart,
+		context.Background(), s.WebhookID,
 	)
 
 	iAssertStatus(
@@ -600,10 +599,10 @@ func TestRecoverPendingDeliveries_TargetDeleted(t *testing.T) {
 	)
 }
 
-// TestRecoverPendingDeliveries_TargetDeleted_LeavesAnOwnedDeliveryAlone:
+// TestRecoverPending_TargetDeleted_LeavesAnOwnedDeliveryAlone: the
-// the terminal write takes ownership like every other recovery write, so
+// terminal write takes ownership like every other recovery write, so a
-// a delivery the engine still holds is not failed underneath its worker.
+// delivery the engine still holds is not failed underneath its worker.
-func TestRecoverPendingDeliveries_TargetDeleted_LeavesAnOwnedDeliveryAlone(
+func TestRecoverPending_TargetDeleted_LeavesAnOwnedDeliveryAlone(
 	t *testing.T,
 ) {
 	t.Parallel()
@@ -620,7 +619,7 @@ func TestRecoverPendingDeliveries_TargetDeleted_LeavesAnOwnedDeliveryAlone(
 	require.True(t, s.Engine.ExportRetainDelivery(deliveryID))
 
 	s.Engine.ExportRecoverWebhookDeliveries(
-		context.Background(), s.WebhookID, atStart,
+		context.Background(), s.WebhookID,
 	)
 
 	iAssertStatus(
@@ -675,18 +674,18 @@ func TestFailMissingTarget_LeavesASettledDeliveryAlone(
 	assert.Zero(t, s.Engine.ExportInflightHeld())
 }
 
-// TestRecoverPendingDeliveries_OnTick_TargetDeleted sweeps twice over a
+// TestSweepPending_TargetDeleted sweeps twice over a batch that also
-// batch that also holds a healthy stranded delivery. The one whose
+// holds a healthy stranded delivery. The one whose target is gone is
-// target is gone is failed once and then left alone; the healthy one is
+// failed once and then left alone; the healthy one is queued by the
-// queued by the first sweep and not again by the second.
+// first sweep and not again by the second.
-func TestRecoverPendingDeliveries_OnTick_TargetDeleted(t *testing.T) {
+func TestSweepPending_TargetDeleted(t *testing.T) {
 	t.Parallel()
 
 	liveTargetID := uuid.New().String()
 	s := fSweepSetup(t, liveTargetID, "still-there")
 
 	deliveryID := tSeedDeletedTarget(
-		t, s, "gone-while-pending-on-tick", "http://example.com/hook",
+		t, s, "gone-on-pending-sweep", "http://example.com/hook",
 		database.DeliveryStatusPending,
 	)
 	rAgePending(t, s.WebhookDB, deliveryID)
@@ -703,7 +702,7 @@ func TestRecoverPendingDeliveries_OnTick_TargetDeleted(t *testing.T) {
 
 	ctx := context.Background()
 
-	s.Engine.ExportRecoverWebhookDeliveries(ctx, s.WebhookID, onTick)
+	s.Engine.ExportSweepWebhookRetries(ctx, s.WebhookID)
 
 	tasks := fDrain(s.Engine)
 	require.Len(t, tasks, 1,
@@ -711,7 +710,7 @@ func TestRecoverPendingDeliveries_OnTick_TargetDeleted(t *testing.T) {
 	)
 	assert.Equal(t, healthy.ID, tasks[0].DeliveryID)
 
-	s.Engine.ExportRecoverWebhookDeliveries(ctx, s.WebhookID, onTick)
+	s.Engine.ExportSweepWebhookRetries(ctx, s.WebhookID)
 
 	assert.Empty(t, fDrain(s.Engine),
 		"the second sweep queued a delivery again",
@@ -724,7 +723,7 @@ func TestRecoverPendingDeliveries_OnTick_TargetDeleted(t *testing.T) {
 
 	last := tLastResult(t, s, deliveryID, 2)
 
-	assert.Contains(t, last.Error, "gone-while-pending-on-tick")
+	assert.Contains(t, last.Error, "gone-on-pending-sweep")
 	assert.Contains(t, last.Error, "was deleted")
 }
 
@@ -775,11 +774,11 @@ func TestSendRecoveredDeliveries_TargetMissingFromMap(
 	)
 }
 
-// TestRecoverPendingDeliveries_TargetUnreadable_LeavesDeliveryAlone: a
+// TestRecoverPending_TargetUnreadable_LeavesDeliveryAlone: a failed
-// failed read of the main database is not a deleted target. At start the
+// read of the main database is not a deleted target. Restart recovery
-// recovery routine holds every pending delivery of the webhook in one
+// holds every pending delivery of the webhook in one batch, so failing
-// batch, so failing on this would fail all of them.
+// on this would fail all of them.
-func TestRecoverPendingDeliveries_TargetUnreadable_LeavesDeliveryAlone(
+func TestRecoverPending_TargetUnreadable_LeavesDeliveryAlone(
 	t *testing.T,
 ) {
 	t.Parallel()
@@ -807,7 +806,7 @@ func TestRecoverPendingDeliveries_TargetUnreadable_LeavesDeliveryAlone(
 	require.NoError(t, sqlDB.Close())
 
 	s.Engine.ExportRecoverPendingDeliveries(
-		context.Background(), s.WebhookDB, s.WebhookID, atStart,
+		context.Background(), s.WebhookDB, s.WebhookID,
 	)
 
 	iAssertStatus(

internal/handlers/delivery_replay.go

@@ -5,7 +5,9 @@ import (
 
 	"github.com/go-chi/chi"
 	"gorm.io/gorm"
+	"gorm.io/gorm/clause"
 	"sneak.berlin/go/webhooker/internal/database"
+	"sneak.berlin/go/webhooker/internal/delivery"
 )
 
 // The outcomes of a replay POST, as the notice codes its redirect
@@ -44,12 +46,12 @@ const (
 // HandleDeliveryReplay re-sends a finished delivery's event to its
 // target.
 //
-// A replay never touches the delivery it repeats. The delivery engine
+// A replay never touches the delivery it repeats. It creates a NEW
-// creates a NEW pending delivery row for the same event and target and
+// pending delivery row for the same event and target and hands it to
-// queues it as it queues the receiver's, so the original's status,
+// the delivery engine through the same Notifier the receiver uses, so
-// attempts and timestamps stand as the record of what actually
+// the original's status, attempts and timestamps stand as the record
-// happened, and the replay is retried, SSRF-guarded and circuit-broken
+// of what actually happened, and the replay is retried, SSRF-guarded
-// exactly as a first attempt is.
+// and circuit-broken exactly as a first attempt is.
 //
 // What is re-sent is the stored EVENT body, never the response the
 // original delivery received.
@@ -143,8 +145,7 @@ func (h *Handlers) loadReplaySource(
 	return &original, true
 }
 
-// queueReplay has the delivery engine create and queue the new
+// queueReplay writes the new delivery and hands it to the engine.
-// delivery, unless an earlier replay is still in flight.
 func (h *Handlers) queueReplay(
 	w http.ResponseWriter,
 	r *http.Request,
@@ -170,8 +171,18 @@ func (h *Handlers) queueReplay(
 		return
 	}
 
-	deliveryID, err := h.engine.CreateReplay(
+	var event database.Event
-		webhook.ID, original.EventID, target,
+
+	err = webhookDB.
+		First(&event, "id = ?", original.EventID).Error
+	if err != nil {
+		h.serverError(w, r, "failed to load event for replay", err)
+
+		return
+	}
+
+	task, err := createReplayDelivery(
+		webhookDB, webhook.ID, &event, target,
 	)
 	if err != nil {
 		h.serverError(
@@ -182,14 +193,15 @@ func (h *Handlers) queueReplay(
 	}
 
 	h.mtr.DeliveryReplayed(target.Type)
+	h.notifier.Notify([]delivery.Task{task})
 
 	h.log.Info(
 		"delivery replay queued",
 		"webhook_id", webhook.ID,
-		"event_id", original.EventID,
+		"event_id", event.ID,
 		"target_id", target.ID,
 		"replayed_delivery_id", original.ID,
-		"delivery_id", deliveryID,
+		"delivery_id", task.DeliveryID,
 	)
 
 	redirectToEventLog(w, r, webhook, replayQueued)
@@ -251,6 +263,77 @@ func countInFlightDeliveries(
 	return count, err
 }
 
+// createReplayDelivery writes the new pending delivery row, adds it to
+// its target's totals in the same transaction, and returns the task
+// that carries it to the delivery engine.
+//
+// The row is written with associations omitted, and neither Event nor
+// Target is populated on it: GORM's SaveBeforeAssociations would
+// otherwise upsert the whole target row — plaintext config, which for a
+// Slack target is the credential — into the per-webhook event database.
+// See https://git.eeqj.de/sneak/webhooker/issues/206.
+func createReplayDelivery(
+	webhookDB *gorm.DB,
+	webhookID string,
+	event *database.Event,
+	target *database.Target,
+) (delivery.Task, error) {
+	dlv := &database.Delivery{
+		EventID:  event.ID,
+		TargetID: target.ID,
+		Status:   database.DeliveryStatusPending,
+		Replay:   true,
+	}
+
+	err := webhookDB.Transaction(func(tx *gorm.DB) error {
+		err := tx.Omit(clause.Associations).Create(dlv).Error
+		if err != nil {
+			return err
+		}
+
+		return database.AddTargetTotals(tx, database.TargetTotals{
+			TargetID: dlv.TargetID, Deliveries: 1,
+		})
+	})
+	if err != nil {
+		return delivery.Task{}, err
+	}
+
+	return delivery.Task{
+		DeliveryID:      dlv.ID,
+		EventID:         event.ID,
+		WebhookID:       webhookID,
+		EntrypointID:    event.EntrypointID,
+		TargetID:        target.ID,
+		TargetName:      target.Name,
+		TargetType:      target.Type,
+		TargetURL:       target.URL,
+		TargetHeaders:   target.Headers,
+		TimeoutSeconds:  target.TimeoutSeconds,
+		ForwardQuery:    target.ForwardQuery,
+		ArchiveExpiry:   target.ArchiveExpiry,
+		ArchiveRotation: target.ArchiveRotation,
+		MaxAttempts:     target.MaxAttempts,
+		Method:          event.Method,
+		RawQuery:        event.RawQuery,
+		Headers:         event.Headers,
+		ContentType:     event.ContentType,
+		Body:            replayBody(event.Body),
+		AttemptNum:      1,
+	}, nil
+}
+
+// replayBody returns the stored event body for a replay task to carry
+// inline, or nil when it is large enough that the engine should fetch
+// it from the per-webhook database instead.
+func replayBody(body string) *string {
+	if len(body) >= delivery.MaxInlineBodySize {
+		return nil
+	}
+
+	return &body
+}
+
 // redirectToEventLog redirects a replay or resubmit back to the event
 // log it was triggered from, carrying the outcome as its notice. A
 // Replay form carries the list it was pressed in as show, so a replay

internal/handlers/delivery_replay_test.go

@@ -12,6 +12,7 @@ import (
 	"gorm.io/gorm"
 	"gorm.io/gorm/clause"
 	"sneak.berlin/go/webhooker/internal/database"
+	"sneak.berlin/go/webhooker/internal/delivery"
 	"sneak.berlin/go/webhooker/internal/handlers"
 	"sneak.berlin/go/webhooker/internal/session"
 )
@@ -22,7 +23,7 @@ const paramDeliveryID = "deliveryID"
 
 // replayTargetURL is a public destination, so a target configured with
 // it is one the SSRF guard would accept. Nothing in these tests
-// dispatches to it: the delivery engine is never started.
+// dispatches to it: the notifier is recorded, not run.
 const replayTargetURL = "http://93.184.216.34/hook"
 
 // replayEventQuery is the query string a seeded event arrived with.
@@ -162,8 +163,14 @@ func postReplay(
 
 // TestHandleDeliveryReplay_AppendsDeliveryAndLeavesOriginal is the
 // core requirement: replaying a failed delivery succeeds, appends a
-// new delivery marked as a replay, and leaves the original row and its
+// new delivery, and leaves the original row and its recorded attempt
-// recorded attempt exactly as they were.
+// exactly as they were.
+//
+// It also pins the two things a replay would be wrong to get from the
+// original: the task carries the target's CURRENT configuration, which
+// this test changes between the failure and the replay, and it carries
+// the stored EVENT body rather than anything the failed attempt
+// received back.
 func TestHandleDeliveryReplay_AppendsDeliveryAndLeavesOriginal(
 	t *testing.T,
 ) {
@@ -174,9 +181,10 @@ func TestHandleDeliveryReplay_AppendsDeliveryAndLeavesOriginal(
 		sess  *session.Session
 		db    *database.Database
 		dbMgr *database.WebhookDBManager
+		notif *recordingNotifier
 	)
 
-	app := newTestApp(t, &h, &sess, &db, &dbMgr)
+	app := newTestApp(t, &h, &sess, &db, &dbMgr, &notif)
 	app.RequireStart()
 
 	t.Cleanup(app.RequireStop)
@@ -196,6 +204,15 @@ func TestHandleDeliveryReplay_AppendsDeliveryAndLeavesOriginal(
 
 	before := loadDelivery(t, webhookDB, original.ID)
 
+	// The operator fixes the destination, which is the whole reason
+	// to replay. The replay must use this, not the URL the original
+	// delivery ran against.
+	const fixedURL = "http://93.184.216.34/fixed"
+
+	require.NoError(t, db.DB().Model(&database.Target{}).
+		Where("id = ?", tgt.ID).
+		Update("url", fixedURL).Error)
+
 	w := postReplay(t, h, sess, wh.ID, original.ID)
 
 	require.Equal(t, http.StatusSeeOther, w.Code)
@@ -217,9 +234,14 @@ func TestHandleDeliveryReplay_AppendsDeliveryAndLeavesOriginal(
 	assert.Equal(
 		t, database.DeliveryStatusPending, replayed.Status,
 	)
-	assert.True(t, replayed.Replay)
 
 	assertDeliveryUntouched(t, webhookDB, before)
+
+	tasks := notif.Tasks()
+	require.Len(t, tasks, 1)
+	assertReplayTask(
+		t, tasks[0], wh.ID, event, tgt, replayed.ID, fixedURL,
+	)
 	assertNoLeakedTarget(t, webhookDB)
 }
 
@@ -253,6 +275,45 @@ func assertDeliveryUntouched(
 	)
 }
 
+// assertReplayTask proves the task handed to the delivery engine is
+// the one the receiver would build for this event and this target, and
+// that it carries wantURL — the target's URL as it stands now rather
+// than as the original delivery ran against it.
+func assertReplayTask(
+	t *testing.T,
+	task delivery.Task,
+	webhookID string,
+	event *database.Event,
+	target *database.Target,
+	wantDeliveryID, wantURL string,
+) {
+	t.Helper()
+
+	assert.Equal(t, wantDeliveryID, task.DeliveryID)
+	assert.Equal(t, event.ID, task.EventID)
+	assert.Equal(t, webhookID, task.WebhookID)
+	assert.Equal(t, event.EntrypointID, task.EntrypointID)
+	assert.Equal(t, target.ID, task.TargetID)
+	assert.Equal(t, target.Type, task.TargetType)
+	assert.Equal(
+		t, wantURL, task.TargetURL,
+		"replay must use the target's current URL",
+	)
+	assert.Equal(t, event.Method, task.Method)
+	assert.Equal(
+		t, replayEventQuery, task.RawQuery,
+		"replay re-sends the stored query string",
+	)
+	assert.Equal(t, event.Headers, task.Headers)
+	assert.Equal(t, event.ContentType, task.ContentType)
+	assert.Equal(t, 1, task.AttemptNum)
+	require.NotNil(t, task.Body)
+	assert.Equal(
+		t, event.Body, *task.Body,
+		"replay re-sends the stored event body",
+	)
+}
+
 // assertNoLeakedTarget proves the per-webhook database holds no
 // target: an association write would have upserted the whole target,
 // plaintext config and all, into the event database. See
@@ -266,49 +327,9 @@ func assertNoLeakedTarget(t *testing.T, webhookDB *gorm.DB) {
 	)
 }
 
-// TestHandleDeliveryReplay_FailedStoreIsAnError proves a replay the
-// engine cannot store answers with the 500 error page and adds no
-// delivery.
-func TestHandleDeliveryReplay_FailedStoreIsAnError(t *testing.T) {
-	t.Parallel()
-
-	var (
-		h     *handlers.Handlers
-		sess  *session.Session
-		db    *database.Database
-		dbMgr *database.WebhookDBManager
-	)
-
-	app := newTestApp(t, &h, &sess, &db, &dbMgr)
-	app.RequireStart()
-
-	t.Cleanup(app.RequireStop)
-
-	wh := seedWebhook(t, db)
-	tgt := seedConfiguredTarget(
-		t, db, wh.ID, database.TargetTypeHTTP,
-		database.Target{URL: replayTargetURL},
-	)
-
-	event, original := seedFailedDelivery(t, dbMgr, wh.ID, tgt.ID)
-
-	// Without its target totals table the webhook's database refuses
-	// the totals write, which comes after the delivery row.
-	webhookDB, err := dbMgr.GetDB(wh.ID)
-	require.NoError(t, err)
-	require.NoError(t,
-		webhookDB.Migrator().DropTable(&database.TargetTotals{}))
-
-	w := postReplay(t, h, sess, wh.ID, original.ID)
-
-	assert.Equal(t, http.StatusInternalServerError, w.Code)
-	assert.Contains(t, w.Body.String(), "500 Internal Server Error")
-	assert.Len(t, listDeliveries(t, webhookDB, event.ID), 1)
-}
-
 // TestHandleDeliveryReplay_RefusesDeletedTarget proves the required
 // refusal: a target deleted since the delivery ran is reported as
-// deleted rather than erroring, and nothing is created.
+// deleted rather than erroring, and nothing is created or queued.
 func TestHandleDeliveryReplay_RefusesDeletedTarget(t *testing.T) {
 	t.Parallel()
 
@@ -317,9 +338,10 @@ func TestHandleDeliveryReplay_RefusesDeletedTarget(t *testing.T) {
 		sess  *session.Session
 		db    *database.Database
 		dbMgr *database.WebhookDBManager
+		notif *recordingNotifier
 	)
 
-	app := newTestApp(t, &h, &sess, &db, &dbMgr)
+	app := newTestApp(t, &h, &sess, &db, &dbMgr, &notif)
 	app.RequireStart()
 
 	t.Cleanup(app.RequireStop)
@@ -355,6 +377,10 @@ func TestHandleDeliveryReplay_RefusesDeletedTarget(t *testing.T) {
 		t, listDeliveries(t, webhookDB, event.ID), 1,
 		"a refused replay must create no delivery",
 	)
+	assert.Empty(
+		t, notif.Tasks(),
+		"a refused replay must queue nothing",
+	)
 
 	// The refusal is specific, which is why the target is looked up
 	// including soft-deleted rows: an id that never named a target
@@ -387,9 +413,10 @@ func TestHandleDeliveryReplay_RefusesWhileEarlierReplayInFlight(
 		sess  *session.Session
 		db    *database.Database
 		dbMgr *database.WebhookDBManager
+		notif *recordingNotifier
 	)
 
-	app := newTestApp(t, &h, &sess, &db, &dbMgr)
+	app := newTestApp(t, &h, &sess, &db, &dbMgr, &notif)
 	app.RequireStart()
 
 	t.Cleanup(app.RequireStop)
@@ -427,6 +454,10 @@ func TestHandleDeliveryReplay_RefusesWhileEarlierReplayInFlight(
 		t, listDeliveries(t, webhookDB, event.ID), 2,
 		"the refused second replay must add nothing",
 	)
+	assert.Len(
+		t, notif.Tasks(), 1,
+		"only the first replay reaches the delivery engine",
+	)
 
 	// A delivery the engine has not finished is not replayable
 	// either, which is the same rule seen from the other side.

internal/handlers/event_resubmit.go

@@ -171,29 +171,37 @@ func loadResubmitSource(
 	return src, true, nil
 }
 
-// queueResubmit has the delivery engine store the copy, with a delivery
+// queueResubmit stores the copy and fans it out to the webhook's
-// for each of the webhook's active targets as they stand now, through
+// active targets.
-// the same StoreEvent the receiver uses. The one difference is that the
-// copy leaves its entrypoint's last event as it is, since it did not
-// arrive on that URL.
 func (h *Handlers) queueResubmit(
 	w http.ResponseWriter,
 	r *http.Request,
 	webhook database.Webhook,
 	src resubmitSource,
 ) {
-	event := &database.Event{
+	// The receiver's own query, run now: an active target created
+	// after the original event arrived is included, and an
+	// inactive one is skipped rather than refused.
+	targets, err := h.loadActiveTargets(webhook.ID)
+	if err != nil {
+		h.serverError(w, r, "failed to query targets", err)
+
+		return
+	}
+
+	event, tasks, err := h.createAndFanOut(
+		eventSource{
 			WebhookID:         webhook.ID,
 			EntrypointID:      src.EntrypointID,
 			Method:            src.Method,
 			RawQuery:          src.RawQuery,
-		Headers:           src.Headers,
+			HeadersJSON:       src.Headers,
 			ContentType:       src.ContentType,
-		Body:              string(src.Body),
+			Body:              src.Body,
 			ResubmittedFromID: &src.ID,
-	}
+		},
-
+		targets,
-	deliveries, err := h.engine.StoreEvent(event)
+	)
 	if err != nil {
 		h.serverError(w, r, "failed to store resubmitted event", err)
 
@@ -207,11 +215,11 @@ func (h *Handlers) queueResubmit(
 		"webhook_id", webhook.ID,
 		"event_id", event.ID,
 		"resubmitted_from_id", src.ID,
-		"target_count", deliveries,
+		"target_count", len(tasks),
 	)
 
 	code := resubmitQueued
-	if deliveries == 0 {
+	if len(tasks) == 0 {
 		code = resubmitNoTargets
 	}
 

internal/handlers/event_resubmit_test.go

@@ -19,7 +19,7 @@ import (
 
 // resubmitTargetURL is a public destination, so a target configured
 // with it is one the SSRF guard would accept. Nothing in these tests
-// dispatches to it: the delivery engine is never started.
+// dispatches to it: the notifier is recorded, not run.
 const resubmitTargetURL = "http://93.184.216.34/hook"
 
 // resubmitEventHeaders and resubmitEventQuery are the stored header
@@ -134,9 +134,10 @@ func TestHandleEventResubmit_DeliversToTargetCreatedAfterTheEvent(
 		sess  *session.Session
 		db    *database.Database
 		dbMgr *database.WebhookDBManager
+		notif *recordingNotifier
 	)
 
-	app := newTestApp(t, &h, &sess, &db, &dbMgr)
+	app := newTestApp(t, &h, &sess, &db, &dbMgr, &notif)
 	app.RequireStart()
 
 	t.Cleanup(app.RequireStop)
@@ -188,6 +189,11 @@ func TestHandleEventResubmit_DeliversToTargetCreatedAfterTheEvent(
 		t, database.DeliveryStatusPending, deliveries[0].Status,
 	)
 
+	tasks := notif.Tasks()
+	require.Len(t, tasks, 1)
+	assert.Equal(t, deliveries[0].ID, tasks[0].DeliveryID)
+	assertResubmitTask(t, tasks[0], wh.ID, &fresh, tgt)
+
 	assertNoLeakedTarget(t, webhookDB)
 }
 
@@ -219,6 +225,32 @@ func assertEventCopy(
 	)
 }
 
+// assertResubmitTask proves the task handed to the delivery engine is
+// the one the receiver would build for the NEW event and this target.
+func assertResubmitTask(
+	t *testing.T,
+	task delivery.Task,
+	webhookID string,
+	fresh *database.Event,
+	target *database.Target,
+) {
+	t.Helper()
+
+	assert.Equal(t, fresh.ID, task.EventID)
+	assert.Equal(t, webhookID, task.WebhookID)
+	assert.Equal(t, fresh.EntrypointID, task.EntrypointID)
+	assert.Equal(t, target.ID, task.TargetID)
+	assert.Equal(t, target.Type, task.TargetType)
+	assert.Equal(t, fresh.Method, task.Method)
+	assert.Equal(t, fresh.RawQuery, task.RawQuery)
+	assert.Equal(t, fresh.Headers, task.Headers)
+	assert.Equal(t, fresh.ContentType, task.ContentType)
+	assert.Equal(t, 1, task.AttemptNum)
+
+	require.NotNil(t, task.Body)
+	assert.Equal(t, fresh.Body, *task.Body)
+}
+
 // TestHandleEventResubmit_IsRepeatable proves the requirement replay
 // deliberately does not meet: firing the same captured event at a
 // backend over and over must work, with no in-flight refusal, and each
@@ -231,9 +263,10 @@ func TestHandleEventResubmit_IsRepeatable(t *testing.T) {
 		sess  *session.Session
 		db    *database.Database
 		dbMgr *database.WebhookDBManager
+		notif *recordingNotifier
 	)
 
-	app := newTestApp(t, &h, &sess, &db, &dbMgr)
+	app := newTestApp(t, &h, &sess, &db, &dbMgr, &notif)
 	app.RequireStart()
 
 	t.Cleanup(app.RequireStop)
@@ -267,18 +300,30 @@ func TestHandleEventResubmit_IsRepeatable(t *testing.T) {
 	require.NoError(t, err)
 
 	events := listEvents(t, webhookDB)
-	require.Len(
+	require.Len(t, events, presses+1)
-		t, events, presses+1, "each resubmit creates its own event",
-	)
 
-	assert.Empty(
+	tasks := notif.Tasks()
-		t, listDeliveries(t, webhookDB, original.ID),
+	require.Len(t, tasks, presses)
+
+	seen := make(map[string]struct{}, presses)
+
+	for _, task := range tasks {
+		assert.Equal(t, tgt.ID, task.TargetID)
+		assert.NotEqual(
+			t, original.ID, task.EventID,
 			"each resubmit delivers its own new event",
 		)
 
-	// Every copy has its own delivery, and names the same source
+		_, dup := seen[task.EventID]
-	// event, so twenty presses stay traceable to the one captured
+		assert.False(t, dup, "each resubmit creates its own event")
-	// request.
+
+		seen[task.EventID] = struct{}{}
+
+		require.Len(t, listDeliveries(t, webhookDB, task.EventID), 1)
+	}
+
+	// Every copy names the same source event, so twenty presses stay
+	// traceable to the one captured request.
 	for _, e := range events {
 		if e.ID == original.ID {
 			continue
@@ -286,10 +331,6 @@ func TestHandleEventResubmit_IsRepeatable(t *testing.T) {
 
 		require.NotNil(t, e.ResubmittedFromID)
 		assert.Equal(t, original.ID, *e.ResubmittedFromID)
-
-		deliveries := listDeliveries(t, webhookDB, e.ID)
-		require.Len(t, deliveries, 1)
-		assert.Equal(t, tgt.ID, deliveries[0].TargetID)
 	}
 }
 
@@ -305,9 +346,10 @@ func TestHandleEventResubmit_OversizeBodySurvivesIntact(t *testing.T) {
 		sess  *session.Session
 		db    *database.Database
 		dbMgr *database.WebhookDBManager
+		notif *recordingNotifier
 	)
 
-	app := newTestApp(t, &h, &sess, &db, &dbMgr)
+	app := newTestApp(t, &h, &sess, &db, &dbMgr, &notif)
 	app.RequireStart()
 
 	t.Cleanup(app.RequireStop)
@@ -343,7 +385,22 @@ func TestHandleEventResubmit_OversizeBodySurvivesIntact(t *testing.T) {
 		t, stored, fresh.Body,
 		"the stored body must be copied byte for byte",
 	)
-	assert.Len(t, listDeliveries(t, webhookDB, fresh.ID), 1)
+
+	tasks := notif.Tasks()
+	require.Len(t, tasks, 1)
+	assert.Nil(
+		t, tasks[0].Body,
+		"a body over the inline limit is fetched from the new "+
+			"event row rather than carried on the task",
+	)
+
+	// The engine's own read of the body, against the new event id:
+	// what it would send is what was stored.
+	var loaded database.Event
+
+	require.NoError(t, webhookDB.Select("body").
+		First(&loaded, "id = ?", tasks[0].EventID).Error)
+	assert.Equal(t, stored, loaded.Body)
 }
 
 // TestHandleEventResubmit_SkipsInactiveTarget proves a deactivated
@@ -357,9 +414,10 @@ func TestHandleEventResubmit_SkipsInactiveTarget(t *testing.T) {
 		sess  *session.Session
 		db    *database.Database
 		dbMgr *database.WebhookDBManager
+		notif *recordingNotifier
 	)
 
-	app := newTestApp(t, &h, &sess, &db, &dbMgr)
+	app := newTestApp(t, &h, &sess, &db, &dbMgr, &notif)
 	app.RequireStart()
 
 	t.Cleanup(app.RequireStop)
@@ -390,19 +448,21 @@ func TestHandleEventResubmit_SkipsInactiveTarget(t *testing.T) {
 		"an inactive target is skipped, not an error",
 	)
 
+	tasks := notif.Tasks()
+	require.Len(t, tasks, 1)
+	assert.Equal(t, active.ID, tasks[0].TargetID)
+
 	webhookDB, err := dbMgr.GetDB(wh.ID)
 	require.NoError(t, err)
 
 	fresh := theOtherEvent(t, listEvents(t, webhookDB), original.ID)
-	deliveries := listDeliveries(t, webhookDB, fresh.ID)
+	require.Len(t, listDeliveries(t, webhookDB, fresh.ID), 1)
-	require.Len(t, deliveries, 1)
-	assert.Equal(t, active.ID, deliveries[0].TargetID)
 }
 
 // TestHandleEventResubmit_NoActiveTargetsStillStoresEvent proves a
 // source with nothing to deliver to behaves as the receiver does: the
-// event is stored with no delivery, and the operator is told so rather
+// event is stored, nothing is queued, and the operator is told so
-// than being shown an error.
+// rather than being shown an error.
 func TestHandleEventResubmit_NoActiveTargetsStillStoresEvent(
 	t *testing.T,
 ) {
@@ -413,9 +473,10 @@ func TestHandleEventResubmit_NoActiveTargetsStillStoresEvent(
 		sess  *session.Session
 		db    *database.Database
 		dbMgr *database.WebhookDBManager
+		notif *recordingNotifier
 	)
 
-	app := newTestApp(t, &h, &sess, &db, &dbMgr)
+	app := newTestApp(t, &h, &sess, &db, &dbMgr, &notif)
 	app.RequireStart()
 
 	t.Cleanup(app.RequireStop)
@@ -432,14 +493,12 @@ func TestHandleEventResubmit_NoActiveTargetsStillStoresEvent(
 		w.Header().Get("Location"),
 	)
 
+	assert.Empty(t, notif.Tasks())
+
 	webhookDB, err := dbMgr.GetDB(wh.ID)
 	require.NoError(t, err)
 
-	events := listEvents(t, webhookDB)
+	assert.Len(t, listEvents(t, webhookDB), 2)
-	assert.Len(t, events, 2)
-
-	fresh := theOtherEvent(t, events, original.ID)
-	assert.Empty(t, listDeliveries(t, webhookDB, fresh.ID))
 }
 
 // TestHandleEventResubmit_RefusesEventOfAnotherWebhook proves the
@@ -455,9 +514,10 @@ func TestHandleEventResubmit_RefusesEventOfAnotherWebhook(
 		sess  *session.Session
 		db    *database.Database
 		dbMgr *database.WebhookDBManager
+		notif *recordingNotifier
 	)
 
-	app := newTestApp(t, &h, &sess, &db, &dbMgr)
+	app := newTestApp(t, &h, &sess, &db, &dbMgr, &notif)
 	app.RequireStart()
 
 	t.Cleanup(app.RequireStop)
@@ -488,50 +548,11 @@ func TestHandleEventResubmit_RefusesEventOfAnotherWebhook(
 	w = postResubmit(t, h, sess, mine.ID, "not-a-uuid")
 	assert.Equal(t, http.StatusNotFound, w.Code)
 
-	for _, webhookID := range []string{theirs.ID, mine.ID} {
+	assert.Empty(
-		webhookDB, err := dbMgr.GetDB(webhookID)
+		t, notif.Tasks(),
-		require.NoError(t, err)
+		"a refused resubmit must queue nothing",
-
-		assert.Len(
-			t, listEvents(t, webhookDB), 1,
-			"a refused resubmit must store nothing",
 	)
 }
-}
-
-// TestHandleEventResubmit_FailedStoreIsAnError proves a copy the engine
-// cannot store answers with the 500 error page and leaves no new event.
-func TestHandleEventResubmit_FailedStoreIsAnError(t *testing.T) {
-	t.Parallel()
-
-	var (
-		h     *handlers.Handlers
-		sess  *session.Session
-		db    *database.Database
-		dbMgr *database.WebhookDBManager
-	)
-
-	app := newTestApp(t, &h, &sess, &db, &dbMgr)
-	app.RequireStart()
-
-	t.Cleanup(app.RequireStop)
-
-	wh := seedWebhook(t, db)
-	original := seedStoredEvent(t, dbMgr, wh.ID, `{"not":"stored"}`)
-
-	// Without its event totals table the webhook's database refuses
-	// the totals write, which comes after the event row.
-	webhookDB, err := dbMgr.GetDB(wh.ID)
-	require.NoError(t, err)
-	require.NoError(t,
-		webhookDB.Migrator().DropTable(&database.EventTotals{}))
-
-	w := postResubmit(t, h, sess, wh.ID, original.ID)
-
-	assert.Equal(t, http.StatusInternalServerError, w.Code)
-	assert.Contains(t, w.Body.String(), "500 Internal Server Error")
-	assert.Len(t, listEvents(t, webhookDB), 1)
-}
 
 // TestHandleSourceLogs_ShowsResubmitProvenance proves the event log
 // reports the relationship in both directions, which is what keeps it

internal/handlers/handlers.go

@@ -64,7 +64,7 @@ type HandlersParams struct {
 	Healthcheck     *healthcheck.Healthcheck
 	Session         *session.Session
 	Middleware      *middleware.Middleware
-	Engine          *delivery.Engine
+	Notifier        delivery.Notifier
 	Archives        archive.Archives
 	CircuitBreakers delivery.CircuitBreakers
 	SSRFGuard       *delivery.Guard
@@ -82,7 +82,7 @@ type Handlers struct {
 	dbMgr     *database.WebhookDBManager
 	session   *session.Session
 	mw        *middleware.Middleware
-	engine    *delivery.Engine
+	notifier  delivery.Notifier
 	archives  archive.Archives
 	breakers  delivery.CircuitBreakers
 	mtr       *metrics.Set
@@ -138,7 +138,7 @@ func New(
 	s.dbMgr = params.WebhookDBMgr
 	s.session = params.Session
 	s.mw = params.Middleware
-	s.engine = params.Engine
+	s.notifier = params.Notifier
 	s.archives = params.Archives
 	s.breakers = params.CircuitBreakers
 	s.mtr = params.Metrics

internal/handlers/handlers_test.go

@@ -30,6 +30,33 @@ import (
 	"sneak.berlin/go/webhooker/internal/session"
 )
 
+// recordingNotifier is a delivery.Notifier that records the tasks it
+// was handed, so a test can prove a handler queued the delivery it
+// claims to have queued — and, on the refusal paths, that it queued
+// nothing.
+type recordingNotifier struct {
+	mu    sync.Mutex
+	tasks []delivery.Task
+}
+
+func (n *recordingNotifier) Notify(tasks []delivery.Task) {
+	n.mu.Lock()
+	defer n.mu.Unlock()
+
+	n.tasks = append(n.tasks, tasks...)
+}
+
+// Tasks returns a copy of the recorded tasks.
+func (n *recordingNotifier) Tasks() []delivery.Task {
+	n.mu.Lock()
+	defer n.mu.Unlock()
+
+	out := make([]delivery.Task, len(n.tasks))
+	copy(out, n.tasks)
+
+	return out
+}
+
 // recordingArchives is an archive.Archives that records what it
 // was asked to do, so a test can prove that a deletion or rename
 // path reached the archive writers. After FailRenames, every
@@ -237,12 +264,11 @@ func newTestAppWithConfig(
 			database.NewWebhookDBManager,
 			healthcheck.New,
 			session.New,
-			archive.NewWriters,
+			func() *recordingNotifier {
-			// The engine is built on a lifecycle that is never started,
+				return &recordingNotifier{}
-			// so no worker runs: every delivery the engine creates for a
+			},
-			// handler stays pending, where the test can read it.
+			func(n *recordingNotifier) delivery.Notifier {
-			func(p delivery.EngineParams) *delivery.Engine {
+				return n
-				return delivery.New(fxtest.NewLifecycle(t), p)
 			},
 			func() *recordingArchives {
 				return &recordingArchives{}

internal/handlers/webhook.go

@@ -2,11 +2,14 @@ package handlers
 
 import (
 	"encoding/json"
+	"fmt"
 	"io"
 	"net/http"
 
 	"github.com/go-chi/chi"
+	"gorm.io/gorm"
 	"sneak.berlin/go/webhooker/internal/database"
+	"sneak.berlin/go/webhooker/internal/delivery"
 	"sneak.berlin/go/webhooker/internal/middleware"
 )
 
@@ -67,9 +70,8 @@ func (h *Handlers) HandleWebhook() http.HandlerFunc {
 	}
 }
 
-// processWebhookRequest reads the request into an event and has the
+// processWebhookRequest reads the body, serializes headers, loads
-// delivery engine store it, with a delivery for each of the webhook's
+// targets, and delivers the event.
-// active targets, and answers the sender.
 //
 // Nothing about the request itself is authenticated: the entrypoint
 // UUID in the path is the credential, and reaching here means it
@@ -91,24 +93,30 @@ func (h *Handlers) processWebhookRequest(
 		return
 	}
 
-	event := &database.Event{
+	targets, err := h.loadActiveTargets(entrypoint.WebhookID)
-		WebhookID:    entrypoint.WebhookID,
-		EntrypointID: entrypoint.ID,
-		Method:       r.Method,
-		RawQuery:     r.URL.RawQuery,
-		Headers:      string(headersJSON),
-		ContentType:  r.Header.Get("Content-Type"),
-		Body:         string(body),
-	}
-
-	deliveries, err := h.engine.StoreEvent(event)
 	if err != nil {
-		h.receiverError(w, "failed to store webhook event", err)
+		h.receiverError(w, "failed to query targets", err)
 
 		return
 	}
 
-	h.finishWebhookResponse(w, event, entrypoint, deliveries)
+	h.createAndDeliverEvent(
+		w, r, entrypoint, body, headersJSON, targets,
+	)
+}
+
+// loadActiveTargets returns all active targets for a webhook.
+func (h *Handlers) loadActiveTargets(
+	webhookID string,
+) ([]database.Target, error) {
+	var targets []database.Target
+
+	err := h.db.DB().Where(
+		"webhook_id = ? AND active = ?",
+		webhookID, true,
+	).Find(&targets).Error
+
+	return targets, err
 }
 
 // lookupEntrypoint finds an entrypoint by UUID path.
@@ -169,6 +177,28 @@ func (h *Handlers) readWebhookBody(
 	return body, true
 }
 
+// createAndDeliverEvent stores the received event, fans it out to the
+// webhook's targets, and answers the sender.
+func (h *Handlers) createAndDeliverEvent(
+	w http.ResponseWriter,
+	r *http.Request,
+	entrypoint database.Entrypoint,
+	body, headersJSON []byte,
+	targets []database.Target,
+) {
+	event, tasks, err := h.createAndFanOut(
+		requestEventSource(r, entrypoint, headersJSON, body),
+		targets,
+	)
+	if err != nil {
+		h.receiverError(w, "failed to store webhook event", err)
+
+		return
+	}
+
+	h.finishWebhookResponse(w, event, entrypoint, tasks)
+}
+
 // receiverError logs an error and answers the sender with a plain-text
 // 500. The receiver's answers are for programs, so it never sends the
 // error page the web UI uses.
@@ -182,19 +212,172 @@ func (h *Handlers) receiverError(
 	)
 }
 
-// finishWebhookResponse logs the stored event, with how many
+// eventSource carries the fields a new event is built from. The
-// deliveries it has, and writes the HTTP response.
+// receiver fills it from the live request; the resubmit handler fills
+// it from a stored event. Both then go through createAndFanOut, so an
+// event is constructed and fanned out in one place however it entered
+// the system.
+type eventSource struct {
+	WebhookID    string
+	EntrypointID string
+	Method       string
+	RawQuery     string
+	HeadersJSON  string
+	ContentType  string
+	Body         []byte
+
+	// ResubmittedFromID names the event this one copies. Only the
+	// resubmit path sets it.
+	ResubmittedFromID *string
+}
+
+// event builds the row this source stores.
+func (s eventSource) event() *database.Event {
+	return &database.Event{
+		WebhookID:         s.WebhookID,
+		EntrypointID:      s.EntrypointID,
+		Method:            s.Method,
+		RawQuery:          s.RawQuery,
+		Headers:           s.HeadersJSON,
+		Body:              string(s.Body),
+		BodyBytes:         int64(len(s.Body)),
+		ContentType:       s.ContentType,
+		ResubmittedFromID: s.ResubmittedFromID,
+	}
+}
+
+// requestEventSource describes the event a live receiver request
+// stores.
+func requestEventSource(
+	r *http.Request,
+	entrypoint database.Entrypoint,
+	headersJSON, body []byte,
+) eventSource {
+	return eventSource{
+		WebhookID:    entrypoint.WebhookID,
+		EntrypointID: entrypoint.ID,
+		Method:       r.Method,
+		RawQuery:     r.URL.RawQuery,
+		HeadersJSON:  string(headersJSON),
+		ContentType:  r.Header.Get("Content-Type"),
+		Body:         body,
+	}
+}
+
+// createAndFanOut writes the event and one pending delivery per target,
+// and adds them to the webhook's running totals, in a single
+// transaction, then hands the tasks to the delivery engine. Every
+// event is created here, received or resubmitted, so a resubmitted
+// event is retried, SSRF-guarded and circuit-broken exactly as a
+// received one is. Per-delivery replay is the one other path that
+// creates a delivery: it adds one to an existing event without
+// coming through here.
+//
+// The tasks are returned as well as queued, so a caller can report how
+// many targets the event went to.
+func (h *Handlers) createAndFanOut(
+	src eventSource,
+	targets []database.Target,
+) (*database.Event, []delivery.Task, error) {
+	webhookDB, err := h.dbMgr.GetDB(src.WebhookID)
+	if err != nil {
+		return nil, nil, fmt.Errorf(
+			"getting webhook database: %w", err,
+		)
+	}
+
+	tx := webhookDB.Begin()
+	if tx.Error != nil {
+		return nil, nil, fmt.Errorf(
+			"beginning transaction: %w", tx.Error,
+		)
+	}
+
+	event := src.event()
+
+	err = tx.Create(event).Error
+	if err != nil {
+		tx.Rollback()
+
+		return nil, nil, fmt.Errorf("creating event: %w", err)
+	}
+
+	tasks, err := buildDeliveryTasks(
+		tx, event, targets, inlineBody(src.Body),
+	)
+	if err != nil {
+		tx.Rollback()
+
+		return nil, nil, err
+	}
+
+	err = database.AddEventTotals(tx, database.EventTotals{
+		Events: 1, LastEventAt: &event.CreatedAt,
+	})
+	if err != nil {
+		tx.Rollback()
+
+		return nil, nil, err
+	}
+
+	// A resubmitted copy did not arrive on its entrypoint's URL, so it
+	// leaves the entrypoint's last event as it is.
+	if src.ResubmittedFromID == nil {
+		err = database.AddEntrypointTotals(tx, database.EntrypointTotals{
+			EntrypointID: event.EntrypointID, LastEventAt: event.CreatedAt,
+		})
+		if err != nil {
+			tx.Rollback()
+
+			return nil, nil, err
+		}
+	}
+
+	err = tx.Commit().Error
+	if err != nil {
+		return nil, nil, fmt.Errorf(
+			"committing transaction: %w", err,
+		)
+	}
+
+	// Counted here, after the commit: an event exists once it is
+	// durably stored, which is what the delivery counters are
+	// compared against on a dashboard. A resubmitted event counts
+	// too, because it produces deliveries that the delivery side
+	// counts; the resubmit counter is what separates the two.
+	h.mtr.EventReceived()
+
+	if len(tasks) > 0 {
+		h.notifier.Notify(tasks)
+	}
+
+	return event, tasks, nil
+}
+
+// inlineBody returns a pointer to body as a string if it fits
+// within the inline size limit, or nil otherwise.
+func inlineBody(body []byte) *string {
+	if len(body) < delivery.MaxInlineBodySize {
+		s := string(body)
+
+		return &s
+	}
+
+	return nil
+}
+
+// finishWebhookResponse logs the event and writes the HTTP response.
 func (h *Handlers) finishWebhookResponse(
 	w http.ResponseWriter,
 	event *database.Event,
 	entrypoint database.Entrypoint,
-	deliveries int,
+	tasks []delivery.Task,
 ) {
 	h.log.Info("webhook event created",
 		"event_id", event.ID,
 		"webhook_id", entrypoint.WebhookID,
 		"entrypoint_id", entrypoint.ID,
-		"target_count", deliveries,
+		"target_count", len(tasks),
 	)
 
 	w.WriteHeader(http.StatusOK)
@@ -206,3 +389,64 @@ func (h *Handlers) finishWebhookResponse(
 		)
 	}
 }
+
+// buildDeliveryTasks creates one pending delivery per target in the
+// transaction, adds each to its target's totals, and returns the tasks
+// for the delivery engine. The caller owns the transaction and rolls
+// it back on error.
+func buildDeliveryTasks(
+	tx *gorm.DB,
+	event *database.Event,
+	targets []database.Target,
+	bodyPtr *string,
+) ([]delivery.Task, error) {
+	tasks := make([]delivery.Task, 0, len(targets))
+
+	for i := range targets {
+		dlv := &database.Delivery{
+			EventID:  event.ID,
+			TargetID: targets[i].ID,
+			Status:   database.DeliveryStatusPending,
+		}
+
+		err := tx.Create(dlv).Error
+		if err != nil {
+			return nil, fmt.Errorf(
+				"creating delivery for target %s: %w",
+				targets[i].ID, err,
+			)
+		}
+
+		err = database.AddTargetTotals(tx, database.TargetTotals{
+			TargetID: targets[i].ID, Deliveries: 1,
+		})
+		if err != nil {
+			return nil, err
+		}
+
+		tasks = append(tasks, delivery.Task{
+			DeliveryID:      dlv.ID,
+			EventID:         event.ID,
+			WebhookID:       event.WebhookID,
+			EntrypointID:    event.EntrypointID,
+			TargetID:        targets[i].ID,
+			TargetName:      targets[i].Name,
+			TargetType:      targets[i].Type,
+			TargetURL:       targets[i].URL,
+			TargetHeaders:   targets[i].Headers,
+			TimeoutSeconds:  targets[i].TimeoutSeconds,
+			ForwardQuery:    targets[i].ForwardQuery,
+			ArchiveExpiry:   targets[i].ArchiveExpiry,
+			ArchiveRotation: targets[i].ArchiveRotation,
+			MaxAttempts:     targets[i].MaxAttempts,
+			Method:          event.Method,
+			RawQuery:        event.RawQuery,
+			Headers:         event.Headers,
+			ContentType:     event.ContentType,
+			Body:            bodyPtr,
+			AttemptNum:      1,
+		})
+	}
+
+	return tasks, nil
+}

internal/handlers/webhook_test.go

@@ -206,87 +206,3 @@ func receivedLine(t *testing.T, buf *bytes.Buffer) map[string]any {
 
 	return found[0]
 }
-
-// TestHandleWebhook_StoresTheEvent proves an accepted request is
-// answered with the receiver's 200 and body, and that the event is
-// stored with a pending delivery to the webhook's active target.
-func TestHandleWebhook_StoresTheEvent(t *testing.T) {
-	t.Parallel()
-
-	var (
-		h     *handlers.Handlers
-		db    *database.Database
-		dbMgr *database.WebhookDBManager
-	)
-
-	app := newTestApp(t, &h, &db, &dbMgr)
-	app.RequireStart()
-
-	t.Cleanup(app.RequireStop)
-
-	webhook := seedWebhook(t, db)
-	entrypoint := seedEntrypoint(t, db, webhook.ID)
-	target := seedConfiguredTarget(
-		t, db, webhook.ID, database.TargetTypeHTTP,
-		database.Target{URL: replayTargetURL},
-	)
-
-	w := httptest.NewRecorder()
-	h.HandleWebhook().ServeHTTP(w, postRequest(
-		"/h/"+entrypoint.Path, nil,
-		map[string]string{"uuid": entrypoint.Path},
-	))
-
-	require.Equal(t, http.StatusOK, w.Code)
-	assert.JSONEq(t, `{"status":"ok"}`, w.Body.String())
-
-	webhookDB, err := dbMgr.GetDB(webhook.ID)
-	require.NoError(t, err)
-
-	events := listEvents(t, webhookDB)
-	require.Len(t, events, 1)
-	assert.Equal(t, entrypoint.ID, events[0].EntrypointID)
-
-	deliveries := listDeliveries(t, webhookDB, events[0].ID)
-	require.Len(t, deliveries, 1)
-	assert.Equal(t, target.ID, deliveries[0].TargetID)
-	assert.Equal(t, database.DeliveryStatusPending, deliveries[0].Status)
-}
-
-// TestHandleWebhook_FailedStoreAnswersPlainError proves that an event
-// the engine cannot store is answered with the receiver's plain-text
-// 500, and that nothing of it is left stored.
-func TestHandleWebhook_FailedStoreAnswersPlainError(t *testing.T) {
-	t.Parallel()
-
-	var (
-		h     *handlers.Handlers
-		db    *database.Database
-		dbMgr *database.WebhookDBManager
-	)
-
-	app := newTestApp(t, &h, &db, &dbMgr)
-	app.RequireStart()
-
-	t.Cleanup(app.RequireStop)
-
-	webhook := seedWebhook(t, db)
-	entrypoint := seedEntrypoint(t, db, webhook.ID)
-
-	// Without its event totals table the webhook's database refuses
-	// the totals write, which comes after the event row.
-	webhookDB, err := dbMgr.GetDB(webhook.ID)
-	require.NoError(t, err)
-	require.NoError(t,
-		webhookDB.Migrator().DropTable(&database.EventTotals{}))
-
-	w := httptest.NewRecorder()
-	h.HandleWebhook().ServeHTTP(w, postRequest(
-		"/h/"+entrypoint.Path, nil,
-		map[string]string{"uuid": entrypoint.Path},
-	))
-
-	assert.Equal(t, http.StatusInternalServerError, w.Code)
-	assert.Equal(t, "Internal server error\n", w.Body.String())
-	assert.Empty(t, listEvents(t, webhookDB))
-}

internal/logfield/logfield_test.go

@@ -21,9 +21,8 @@ const budget = 64
 
 // batchRunes is how many consecutive code points the charge test logs
 // in one value from U+1000 up. Logging each of those on its own line
-// was too slow for the suite when it ran under the race detector;
+// is too slow for the suite under the race detector; 4,096 at a time
-// 4,096 at a time is 271 batches, each logged on two lines, so 542
+// is 271 batches, each logged on two lines, so 542 lines per handler.
-// lines per handler.
 const batchRunes = 4096
 
 // newHandlers are the two handlers internal/logger can install. Time
@@ -153,10 +152,10 @@ func TestEncodedBytes_ChargesAtLeastWhatTheHandlersEmit(t *testing.T) {
 		batches = append(batches, batch.String())
 	}
 
-	// What EncodedBytes charges for each batch. When the suite ran
+	// What EncodedBytes charges for each batch. Under -race -cover this
-	// under -race -cover this took longer than logging the batches, so
+	// takes longer than logging the batches, so it is worked out once,
-	// it is worked out once, by whichever handler finishes logging
+	// by whichever handler finishes logging first, while the other is
-	// first, while the other is still logging.
+	// still logging.
 	charged := sync.OnceValue(func() []int {
 		costs := make([]int, len(batches))
 

internal/resetpw/resetpw_test.go

@@ -131,6 +131,10 @@ func run(
 	return code, stdout.String(), stderr.String()
 }
 
+type noopNotifier struct{}
+
+func (n *noopNotifier) Notify([]delivery.Task) {}
+
 type noopArchives struct {
 	renameMu sync.Mutex
 }
@@ -186,12 +190,7 @@ func newServerApp(
 			database.NewWebhookDBManager,
 			healthcheck.New,
 			session.New,
-			archive.NewWriters,
+			func() delivery.Notifier { return &noopNotifier{} },
-			// The engine is built on a lifecycle that is never started,
-			// so no worker runs.
-			func(p delivery.EngineParams) *delivery.Engine {
-				return delivery.New(fxtest.NewLifecycle(t), p)
-			},
 			func() archive.Archives { return &noopArchives{} },
 			func() delivery.CircuitBreakers {
 				return &noopCircuitBreakers{}

internal/server/early_shutdown_test.go

@@ -10,12 +10,14 @@ import (
 	"sneak.berlin/go/webhooker/internal/server"
 )
 
-// earlyStopIterations is how many start/stop cycles the test below
+// earlyStopIterations is how many start/stop cycles the race test
 // runs. The window it aims at is the gap between the OnStart hook
 // returning and the serving goroutine reaching its first field
-// access, which is microseconds wide, so a single cycle can miss it
+// access, which is microseconds wide. The race detector reports an
-// purely on scheduling. Repetition makes hitting it reliable; the
+// unsynchronised pair whenever it observes one, but it has to observe
-// collaborators are built once, so the cycles themselves are cheap.
+// one, so a single cycle can miss purely on scheduling. Repetition
+// makes the observation reliable; the collaborators are built once,
+// so the cycles themselves are cheap.
 const earlyStopIterations = 25
 
 // TestEarlyShutdown_NoPanicAndNoRace stops the application
@@ -36,9 +38,8 @@ const earlyStopIterations = 25
 //
 // httpServer is now built in New, on the constructing goroutine, so
 // it is written before any hook exists and can never be nil.
-// sentryEnabled is atomic. This test catches the nil dereference
+// sentryEnabled is atomic. This test is what catches either one
-// coming back, as a panic. The suite runs without Go's race
+// coming back — under -race, which is how the suite runs.
-// detector, so the data race coming back would go unreported here.
 func TestEarlyShutdown_NoPanicAndNoRace(t *testing.T) {
 	t.Parallel()
 

internal/server/routes_test.go

@@ -48,6 +48,10 @@ const (
 	metricsAuthValue = "s3cret"
 )
 
+type noopNotifier struct{}
+
+func (n *noopNotifier) Notify([]delivery.Task) {}
+
 // noopArchives satisfies handlers.New's archive.Archives
 // dependency. No test here checks what gets evicted or renamed, so
 // it records nothing. Its rename lock is a plain mutex.
@@ -141,12 +145,7 @@ func newTestEnvWithConfig(
 			database.NewWebhookDBManager,
 			healthcheck.New,
 			session.New,
-			archive.NewWriters,
+			func() delivery.Notifier { return &noopNotifier{} },
-			// The engine is built on a lifecycle that is never started,
-			// so no worker runs and nothing is sent anywhere.
-			func(p delivery.EngineParams) *delivery.Engine {
-				return delivery.New(fxtest.NewLifecycle(t), p)
-			},
 			func() archive.Archives { return &noopArchives{} },
 			func() delivery.CircuitBreakers {
 				return &noopCircuitBreakers{}

internal/server/static_assets_test.go

@@ -13,6 +13,9 @@ import (
 
 // TestBaseTemplateScriptsAreServed walks every /s/ script the base
 // template loads on each page and fetches it through the real router.
+// Alpine.js is extracted from its tarball in 3p/ at build time, so the
+// file is not in the tree: this is the check that the page still gets
+// the JavaScript it asks for.
 func TestBaseTemplateScriptsAreServed(t *testing.T) {
 	t.Parallel()
 

script/assets

@@ -0,0 +1,17 @@
+#!/bin/sh
+# script/assets: extract Alpine.js from its npm package tarball, committed
+# in 3p/, to static/js/alpine.min.js, where go:embed reads it. The package
+# is @alpinejs/csp, Alpine's build for pages whose Content-Security-Policy
+# forbids eval. The extracted file is not committed. make build, make dev
+# and the Dockerfile's lint, test and build stages run this first.
+set -eu
+
+ROOT="$(cd "$(dirname "$0")/.." && pwd -P)"
+
+main() {
+    cd "$ROOT"
+    tar -xzOf 3p/alpinejs-csp-3.14.9.tgz package/dist/cdn.min.js \
+        >static/js/alpine.min.js
+}
+
+main "$@"

static/js/alpine.min.js

@@ -1,11 +0,0 @@
-(()=>{var it=!1,ot=!1,W=[],st=-1;function Wt(e){Mn(e)}function Mn(e){W.includes(e)||W.push(e),Nn()}function Gt(e){let t=W.indexOf(e);t!==-1&&t>st&&W.splice(t,1)}function Nn(){!ot&&!it&&(it=!0,queueMicrotask(kn))}function kn(){it=!1,ot=!0;for(let e=0;e<W.length;e++)W[e](),st=e;W.length=0,st=-1,ot=!1}var T,N,F,ct,at=!0;function Jt(e){at=!1,e(),at=!0}function Yt(e){T=e.reactive,F=e.release,N=t=>e.effect(t,{scheduler:r=>{at?Wt(r):r()}}),ct=e.raw}function lt(e){N=e}function Xt(e){let t=()=>{};return[n=>{let i=N(n);return e._x_effects||(e._x_effects=new Set,e._x_runEffects=()=>{e._x_effects.forEach(o=>o())}),e._x_effects.add(i),t=()=>{i!==void 0&&(e._x_effects.delete(i),F(i))},i},()=>{t()}]}function Se(e,t){let r=!0,n,i=N(()=>{let o=e();JSON.stringify(o),r?n=o:queueMicrotask(()=>{t(o,n),n=o}),r=!1});return()=>F(i)}var Zt=[],Qt=[],er=[];function tr(e){er.push(e)}function re(e,t){typeof t=="function"?(e._x_cleanups||(e._x_cleanups=[]),e._x_cleanups.push(t)):(t=e,Qt.push(t))}function Oe(e){Zt.push(e)}function Ce(e,t,r){e._x_attributeCleanups||(e._x_attributeCleanups={}),e._x_attributeCleanups[t]||(e._x_attributeCleanups[t]=[]),e._x_attributeCleanups[t].push(r)}function ut(e,t){e._x_attributeCleanups&&Object.entries(e._x_attributeCleanups).forEach(([r,n])=>{(t===void 0||t.includes(r))&&(n.forEach(i=>i()),delete e._x_attributeCleanups[r])})}function rr(e){for(e._x_effects?.forEach(Gt);e._x_cleanups?.length;)e._x_cleanups.pop()()}var ft=new MutationObserver(ht),dt=!1;function de(){ft.observe(document,{subtree:!0,childList:!0,attributes:!0,attributeOldValue:!0}),dt=!0}function pt(){Dn(),ft.disconnect(),dt=!1}var fe=[];function Dn(){let e=ft.takeRecords();fe.push(()=>e.length>0&&ht(e));let t=fe.length;queueMicrotask(()=>{if(fe.length===t)for(;fe.length>0;)fe.shift()()})}function m(e){if(!dt)return e();pt();let t=e();return de(),t}var mt=!1,Ae=[];function nr(){mt=!0}function ir(){mt=!1,ht(Ae),Ae=[]}function ht(e){if(mt){Ae=Ae.concat(e);return}let t=[],r=new Set,n=new Map,i=new Map;for(let o=0;o<e.length;o++)if(!e[o].target._x_ignoreMutationObserver&&(e[o].type==="childList"&&(e[o].removedNodes.forEach(s=>{s.nodeType===1&&s._x_marker&&r.add(s)}),e[o].addedNodes.forEach(s=>{if(s.nodeType===1){if(r.has(s)){r.delete(s);return}s._x_marker||t.push(s)}})),e[o].type==="attributes")){let s=e[o].target,a=e[o].attributeName,c=e[o].oldValue,l=()=>{n.has(s)||n.set(s,[]),n.get(s).push({name:a,value:s.getAttribute(a)})},u=()=>{i.has(s)||i.set(s,[]),i.get(s).push(a)};s.hasAttribute(a)&&c===null?l():s.hasAttribute(a)?(u(),l()):u()}i.forEach((o,s)=>{ut(s,o)}),n.forEach((o,s)=>{Zt.forEach(a=>a(s,o))});for(let o of r)t.some(s=>s.contains(o))||Qt.forEach(s=>s(o));for(let o of t)o.isConnected&&er.forEach(s=>s(o));t=null,r=null,n=null,i=null}function Te(e){return P(k(e))}function D(e,t,r){return e._x_dataStack=[t,...k(r||e)],()=>{e._x_dataStack=e._x_dataStack.filter(n=>n!==t)}}function k(e){return e._x_dataStack?e._x_dataStack:typeof ShadowRoot=="function"&&e instanceof ShadowRoot?k(e.host):e.parentNode?k(e.parentNode):[]}function P(e){return new Proxy({objects:e},Pn)}var Pn={ownKeys({objects:e}){return Array.from(new Set(e.flatMap(t=>Object.keys(t))))},has({objects:e},t){return t==Symbol.unscopables?!1:e.some(r=>Object.prototype.hasOwnProperty.call(r,t)||Reflect.has(r,t))},get({objects:e},t,r){return t=="toJSON"?In:Reflect.get(e.find(n=>Reflect.has(n,t))||{},t,r)},set({objects:e},t,r,n){let i=e.find(s=>Object.prototype.hasOwnProperty.call(s,t))||e[e.length-1],o=Object.getOwnPropertyDescriptor(i,t);return o?.set&&o?.get?o.set.call(n,r)||!0:Reflect.set(i,t,r)}};function In(){return Reflect.ownKeys(this).reduce((t,r)=>(t[r]=Reflect.get(this,r),t),{})}function Re(e){let t=n=>typeof n=="object"&&!Array.isArray(n)&&n!==null,r=(n,i="")=>{Object.entries(Object.getOwnPropertyDescriptors(n)).forEach(([o,{value:s,enumerable:a}])=>{if(a===!1||s===void 0||typeof s=="object"&&s!==null&&s.__v_skip)return;let c=i===""?o:`${i}.${o}`;typeof s=="object"&&s!==null&&s._x_interceptor?n[o]=s.initialize(e,c,o):t(s)&&s!==n&&!(s instanceof Element)&&r(s,c)})};return r(e)}function Me(e,t=()=>{}){let r={initialValue:void 0,_x_interceptor:!0,initialize(n,i,o){return e(this.initialValue,()=>$n(n,i),s=>_t(n,i,s),i,o)}};return t(r),n=>{if(typeof n=="object"&&n!==null&&n._x_interceptor){let i=r.initialize.bind(r);r.initialize=(o,s,a)=>{let c=n.initialize(o,s,a);return r.initialValue=c,i(o,s,a)}}else r.initialValue=n;return r}}function $n(e,t){return t.split(".").reduce((r,n)=>r[n],e)}function _t(e,t,r){if(typeof t=="string"&&(t=t.split(".")),t.length===1)e[t[0]]=r;else{if(t.length===0)throw error;return e[t[0]]||(e[t[0]]={}),_t(e[t[0]],t.slice(1),r)}}var or={};function y(e,t){or[e]=t}function G(e,t){let r=Ln(t);return Object.entries(or).forEach(([n,i])=>{Object.defineProperty(e,`$${n}`,{get(){return i(t,r)},enumerable:!1})}),e}function Ln(e){let[t,r]=gt(e),n={interceptor:Me,...t};return re(e,r),n}function Ne(e,t,r,...n){try{return r(...n)}catch(i){ne(i,e,t)}}function ne(e,t,Line truncated
-
-${r?'Expression: "'+r+`"
-
-`:""}`,t),setTimeout(()=>{throw e},0)}var ke=!0;function De(e){let t=ke;ke=!1;let r=e();return ke=t,r}function R(e,t,r={}){let n;return x(e,t)(i=>n=i,r),n}function x(...e){return sr(...e)}var sr=jn;function ar(e){sr=e}function jn(e,t){let r={};G(r,e);let n=[r,...k(e)],i=typeof t=="function"?yt(n,t):Bn(n,t,e);return Ne.bind(null,e,t,i)}function yt(e,t){return(r=()=>{},{scope:n={},params:i=[]}={})=>{let o=t.apply(P([n,...e]),i);ie(r,o)}}var xt={};function Fn(e,t){if(xt[e])return xt[e];let r=Object.getPrototypeOf(async function(){}).constructor,n=/^[\n\s]*if.*\(.*\)/.test(e.trim())||/^(let|const)\s/.test(e.trim())?`(async()=>{ ${e} })()`:e,o=(()=>{try{let s=new r(["__self","scope"],`with (scope) { __self.result = ${n} }; __self.finished = true; return __self.result;`);return Object.defineProperty(s,"name",{value:`[Alpine] ${e}`}),s}catch(s){return ne(s,t,e),Promise.resolve()}})();return xt[e]=o,o}function Bn(e,t,r){let n=Fn(t,r);return(i=()=>{},{scope:o={},params:s=[]}={})=>{n.result=void 0,n.finished=!1;let a=P([o,...e]);if(typeof n=="function"){let c=n(n,a).catch(l=>ne(l,r,t));n.finished?(ie(i,n.result,a,s,r),n.result=void 0):c.then(l=>{ie(i,l,a,s,r)}).catch(l=>ne(l,r,t)).finally(()=>n.result=void 0)}}}function ie(e,t,r,n,i){if(ke&&typeof t=="function"){let o=t.apply(r,n);o instanceof Promise?o.then(s=>ie(e,s,r,n)).catch(s=>ne(s,i,t)):e(o)}else typeof t=="object"&&t instanceof Promise?t.then(o=>e(o)):e(t)}var Et="x-";function C(e=""){return Et+e}function cr(e){Et=e}var Pe={};function d(e,t){return Pe[e]=t,{before(r){if(!Pe[r]){console.warn(String.raw`Cannot find directive \`${r}\`. \`${e}\` will use the default order of execution`);return}let n=J.indexOf(r);J.splice(n>=0?n:J.indexOf("DEFAULT"),0,e)}}}function lr(e){return Object.keys(Pe).includes(e)}function me(e,t,r){if(t=Array.from(t),e._x_virtualDirectives){let o=Object.entries(e._x_virtualDirectives).map(([a,c])=>({name:a,value:c})),s=vt(o);o=o.map(a=>s.find(c=>c.name===a.name)?{name:`x-bind:${a.name}`,value:`"${a.value}"`}:a),t=t.concat(o)}let n={};return t.map(dr((o,s)=>n[o]=s)).filter(mr).map(Kn(n,r)).sort(Hn).map(o=>zn(e,o))}function vt(e){return Array.from(e).map(dr()).filter(t=>!mr(t))}var bt=!1,pe=new Map,ur=Symbol();function fr(e){bt=!0;let t=Symbol();ur=t,pe.set(t,[]);let r=()=>{for(;pe.get(t).length;)pe.get(t).shift()();pe.delete(t)},n=()=>{bt=!1,r()};e(r),n()}function gt(e){let t=[],r=a=>t.push(a),[n,i]=Xt(e);return t.push(i),[{Alpine:K,effect:n,cleanup:r,evaluateLater:x.bind(x,e),evaluate:R.bind(R,e)},()=>t.forEach(a=>a())]}function zn(e,t){let r=()=>{},n=Pe[t.type]||r,[i,o]=gt(e);Ce(e,t.original,o);let s=()=>{e._x_ignore||e._x_ignoreSelf||(n.inline&&n.inline(e,t,i),n=n.bind(n,e,t,i),bt?pe.get(ur).push(n):n())};return s.runCleanups=o,s}var Ie=(e,t)=>({name:r,value:n})=>(r.startsWith(e)&&(r=r.replace(e,t)),{name:r,value:n}),$e=e=>e;function dr(e=()=>{}){return({name:t,value:r})=>{let{name:n,value:i}=pr.reduce((o,s)=>s(o),{name:t,value:r});return n!==t&&e(n,t),{name:n,value:i}}}var pr=[];function oe(e){pr.push(e)}function mr({name:e}){return hr().test(e)}var hr=()=>new RegExp(`^${Et}([^:^.]+)\\b`);function Kn(e,t){return({name:r,value:n})=>{let i=r.match(hr()),o=r.match(/:([a-zA-Z0-9\-_:]+)/),s=r.match(/\.[^.\]]+(?=[^\]]*$)/g)||[],a=t||e[r]||r;return{type:i?i[1]:null,value:o?o[1]:null,modifiers:s.map(c=>c.replace(".","")),expression:n,original:a}}}var wt="DEFAULT",J=["ignore","ref","data","id","anchor","bind","init","for","model","modelable","transition","show","if",wt,"teleport"];function Hn(e,t){let r=J.indexOf(e.type)===-1?wt:e.type,n=J.indexOf(t.type)===-1?wt:t.type;return J.indexOf(r)-J.indexOf(n)}function Y(e,t,r={}){e.dispatchEvent(new CustomEvent(t,{detail:r,bubbles:!0,composed:!0,cancelable:!0}))}function I(e,t){if(typeof ShadowRoot=="function"&&e instanceof ShadowRoot){Array.from(e.children).forEach(i=>I(i,t));return}let r=!1;if(t(e,()=>r=!0),r)return;let n=e.firstElementChild;for(;n;)I(n,t,!1),n=n.nextElementSibling}function E(e,...t){console.warn(`Alpine Warning: ${e}`,...t)}var _r=!1;function gr(){_r&&E("Alpine has already been initialized on this page. Calling Alpine.start() more than once can cause problems."),_r=!0,document.body||E("Unable to initialize. Trying to load Alpine before `<body>` is available. Did you forget to add `defer` in Alpine's `<script>` tag?"),Y(document,"alpine:init"),Y(document,"alpine:initializing"),de(),tr(t=>S(t,I)),re(t=>$(t)),Oe((t,r)=>{me(t,r).forEach(n=>n())});let e=t=>!X(t.parentElement,!0);Array.from(document.querySelectorAll(br().join(","))).filter(e).forEach(t=>{S(t)}),Y(document,"alpine:initialized"),setTimeout(()=>{qn()})}var St=[],xr=[];function yr(){return St.map(e=>e())}function br(){return St.concat(xr).map(e=>e())}function Le(e){St.push(e)}function je(e){xr.push(e)}function X(e,t=!1){return B(e,r=>{if((t?br():yr()).some(i=>r.matches(i)))return!0})}function B(e,t){if(e){if(t(e))return e;if(e._x_teleportBack&&(e=e._x_teleportBack),!!e.parentElement)return B(e.parentElement,t)}}function wr(Line truncated
-
-"${t}"
-
-Read more about the Alpine's CSP-friendly build restrictions here: https://alpinejs.dev/advanced/csp
-
-`,e)}function Mt(e,t){let r=Object.create(null),n=e.split(",");for(let i=0;i<n.length;i++)r[n[i]]=!0;return t?i=>!!r[i.toLowerCase()]:i=>!!r[i]}var mi="itemscope,allowfullscreen,formnovalidate,ismap,nomodule,novalidate,readonly";var qs=Mt(mi+",async,autofocus,autoplay,controls,default,defer,disabled,hidden,loop,open,required,reversed,scoped,seamless,checked,muted,multiple,selected");var Yr=Object.freeze({}),Us=Object.freeze([]);var hi=Object.prototype.hasOwnProperty,be=(e,t)=>hi.call(e,t),V=Array.isArray,ae=e=>Xr(e)==="[object Map]";var _i=e=>typeof e=="string",Ue=e=>typeof e=="symbol",we=e=>e!==null&&typeof e=="object";var gi=Object.prototype.toString,Xr=e=>gi.call(e),Nt=e=>Xr(e).slice(8,-1);var We=e=>_i(e)&&e!=="NaN"&&e[0]!=="-"&&""+parseInt(e,10)===e;var Ge=e=>{let t=Object.create(null);return r=>t[r]||(t[r]=e(r))},xi=/-(\w)/g,Ws=Ge(e=>e.replace(xi,(t,r)=>r?r.toUpperCase():"")),yi=/\B([A-Z])/g,Gs=Ge(e=>e.replace(yi,"-$1").toLowerCase()),kt=Ge(e=>e.charAt(0).toUpperCase()+e.slice(1)),Js=Ge(e=>e?`on${kt(e)}`:""),Dt=(e,t)=>e!==t&&(e===e||t===t);var Pt=new WeakMap,Ee=[],j,ee=Symbol("iterate"),It=Symbol("Map key iterate");function bi(e){return e&&e._isEffect===!0}function nn(e,t=Yr){bi(e)&&(e=e.raw);let r=Ei(e,t);return t.lazy||r(),r}function on(e){e.active&&(sn(e),e.options.onStop&&e.options.onStop(),e.active=!1)}var wi=0;function Ei(e,t){let r=function(){if(!r.active)return e();if(!Ee.includes(r)){sn(r);try{return Si(),Ee.push(r),j=r,e()}finally{Ee.pop(),an(),j=Ee[Ee.length-1]}}};return r.id=wi++,r.allowRecurse=!!t.allowRecurse,r._isEffect=!0,r.active=!0,r.raw=e,r.deps=[],r.options=t,r}function sn(e){let{deps:t}=e;if(t.length){for(let r=0;r<t.length;r++)t[r].delete(e);t.length=0}}var ce=!0,Lt=[];function vi(){Lt.push(ce),ce=!1}function Si(){Lt.push(ce),ce=!0}function an(){let e=Lt.pop();ce=e===void 0?!0:e}function M(e,t,r){if(!ce||j===void 0)return;let n=Pt.get(e);n||Pt.set(e,n=new Map);let i=n.get(r);i||n.set(r,i=new Set),i.has(j)||(i.add(j),j.deps.push(i),j.options.onTrack&&j.options.onTrack({effect:j,target:e,type:t,key:r}))}function U(e,t,r,n,i,o){let s=Pt.get(e);if(!s)return;let a=new Set,c=u=>{u&&u.forEach(p=>{(p!==j||p.allowRecurse)&&a.add(p)})};if(t==="clear")s.forEach(c);else if(r==="length"&&V(e))s.forEach((u,p)=>{(p==="length"||p>=n)&&c(u)});else switch(r!==void 0&&c(s.get(r)),t){case"add":V(e)?We(r)&&c(s.get("length")):(c(s.get(ee)),ae(e)&&c(s.get(It)));break;case"delete":V(e)||(c(s.get(ee)),ae(e)&&c(s.get(It)));break;case"set":ae(e)&&c(s.get(ee));break}let l=u=>{u.options.onTrigger&&u.options.onTrigger({effect:u,target:e,key:r,type:t,newValue:n,oldValue:i,oldTarget:o}),u.options.scheduler?u.options.scheduler(u):u()};a.forEach(l)}var Ai=Mt("__proto__,__v_isRef,__isVue"),cn=new Set(Object.getOwnPropertyNames(Symbol).map(e=>Symbol[e]).filter(Ue)),Oi=ln();var Ci=ln(!0);var Zr=Ti();function Ti(){let e={};return["includes","indexOf","lastIndexOf"].forEach(t=>{e[t]=function(...r){let n=_(this);for(let o=0,s=this.length;o<s;o++)M(n,"get",o+"");let i=n[t](...r);return i===-1||i===!1?n[t](...r.map(_)):i}}),["push","pop","shift","unshift","splice"].forEach(t=>{e[t]=function(...r){vi();let n=_(this)[t].apply(this,r);return an(),n}}),e}function ln(e=!1,t=!1){return function(n,i,o){if(i==="__v_isReactive")return!e;if(i==="__v_isReadonly")return e;if(i==="__v_raw"&&o===(e?t?Vi:pn:t?Hi:dn).get(n))return n;let s=V(n);if(!e&&s&&be(Zr,i))return Reflect.get(Zr,i,o);let a=Reflect.get(n,i,o);return(Ue(i)?cn.has(i):Ai(i))||(e||M(n,"get",i),t)?a:$t(a)?!s||!We(i)?a.value:a:we(a)?e?mn(a):tt(a):a}}var Ri=Mi();function Mi(e=!1){return function(r,n,i,o){let s=r[n];if(!e&&(i=_(i),s=_(s),!V(r)&&$t(s)&&!$t(i)))return s.value=i,!0;let a=V(r)&&We(n)?Number(n)<r.length:be(r,n),c=Reflect.set(r,n,i,o);return r===_(o)&&(a?Dt(i,s)&&U(r,"set",n,i,s):U(r,"add",n,i)),c}}function Ni(e,t){let r=be(e,t),n=e[t],i=Reflect.deleteProperty(e,t);return i&&r&&U(e,"delete",t,void 0,n),i}function ki(e,t){let r=Reflect.has(e,t);return(!Ue(t)||!cn.has(t))&&M(e,"has",t),r}function Di(e){return M(e,"iterate",V(e)?"length":ee),Reflect.ownKeys(e)}var Pi={get:Oi,set:Ri,deleteProperty:Ni,has:ki,ownKeys:Di},Ii={get:Ci,set(e,t){return console.warn(`Set operation on key "${String(t)}" failed: target is readonly.`,e),!0},deleteProperty(e,t){return console.warn(`Delete operation on key "${String(t)}" failed: target is readonly.`,e),!0}};var jt=e=>we(e)?tt(e):e,Ft=e=>we(e)?mn(e):e,Bt=e=>e,et=e=>Reflect.getPrototypeOf(e);function Je(e,t,r=!1,n=!1){e=e.__v_raw;let i=_(e),o=_(t);t!==o&&!r&&M(i,"get",t),!r&&M(i,"get",o);let{has:s}=et(i),a=n?Bt:r?Ft:jt;if(s.call(i,t))return a(e.get(t));if(s.call(i,o))return a(e.get(o));e!==i&&e.get(t)}function Ye(e,t=!1){let r=this.__v_raw,n=_(r),i=_(e);return e!==i&&!t&&M(n,"has",e),!t&&M(n,"has",i),e===i?r.has(e):r.has(e)||r.has(i)}function Xe(e,t=!1){return e=e.__v_raw,!t&&M(_(e),"iterate",ee),Reflect.get(e,"size",e)}function Qr(e){e=_(e);let t=_(this);return et(t).has.call(t,e)||(t.add(e),U(t,"add",e,e)),this}function en(e,t){t
Line truncated

static/static.go

@@ -6,11 +6,9 @@ import (
 )
 
 // Static holds the CSS and JavaScript files the web UI's pages load. They
-// are named one by one, so a missing file fails the build instead of
+// are named one by one so that a missing js/alpine.min.js, which make
-// leaving the pages without it.
+// assets extracts and git does not track, fails the build instead of
-//
+// leaving the pages without Alpine.js.
-// js/alpine.min.js is Alpine.js, committed unchanged. It is
-// package/dist/cdn.min.js from the npm package @alpinejs/csp 3.14.9.
 //
 //go:embed css/tailwind.css css/style.css js/app.js js/alpine.min.js
 var Static embed.FS