Store events and create deliveries in the delivery engine (closes #544)

The receiver, resubmit and replay handlers now parse the request and
make one call. Engine.StoreEvent stores an event with a pending
delivery for each active target and the totals in one transaction,
then queues them; Engine.CreateReplay does the same for one replay
delivery. The handlers take *delivery.Engine, so the Notifier
interface and its bridge are gone, and the engine's notify is no
longer exported. Archives and CircuitBreakers stay because the handler
tests fake them. The tests that read a queued task moved to the
engine's tests; the handler tests use an engine that is never started.

Model: opus-5-5

.dockerignore

@@ -72,8 +72,10 @@
 **/*.sublime-*
 
 # This repository's own host-built artifacts: the binary `make build`
-# writes.
+# writes, and the Alpine.js file `make assets` extracts from 3p/ (the
+# build extracts its own).
 /bin
+/static/js/alpine.min.js
 
 # SQLite databases, which hold the session key and webhook payloads, at
 # any depth.

.gitignore

@@ -77,3 +77,7 @@ data/
 # Temporary files
 tmp/
 temp/
+
+# Alpine.js, extracted by `make assets` from its tarball in 3p/, which is
+# what is committed.
+/static/js/alpine.min.js

.prettierignore

@@ -1,3 +1,2 @@
 node_modules/
 yarn.lock
-static/js/alpine.min.js

Dockerfile

@@ -92,6 +92,10 @@ COPY . .
 # fmt-check` is the markdown-check stage above.
 RUN if [ -n "$(gofmt -s -l .)" ]; then echo "gofmt needed on:"; gofmt -s -l .; exit 1; fi
 
+# static/static.go embeds the Alpine.js file this extracts from 3p/; without
+# it the static package does not compile and cannot be linted.
+RUN script/assets
+
 # The golangci-lint steps run with --network=none. `golangci-lint config
 # verify` is documented as fetching its JSON schema over HTTPS; this pinned
 # image resolves the schema without any network, and --network=none enforces
@@ -107,16 +111,11 @@ RUN --network=none golangci-lint run --config .golangci.yml --build-tags browser
 # Nothing is wanted from js-lint; the copy is what makes this phase run it.
 COPY --from=js-lint /src/yarn.lock /dev/null
 
-# Test phase. The tests run with cgo off and so without Go's race detector,
-# which needs cgo on Linux; sneak ruled the race detector out in
-# https://git.eeqj.de/sneak/webhooker/issues/538#issuecomment-153273.
-# `make test` (script/test) builds this stage alone; the build stage below
-# depends on it.
+# Test phase. -race needs cgo and so a C compiler, which the Debian Go image
+# ships and the alpine one does not. `make test` (script/test) builds this
+# stage alone; the build stage below depends on it.
 #
-# golang:1.26.1-bookworm (Debian-based), 2026-03-17: the builder stage's
-# image, which that stage needs for its git and make. Its test step runs
-# under the image's bash, and the tests in internal/versionscript use its
-# git, make and sh, and skip when they are missing.
+# golang:1.26.1-bookworm (Debian-based), 2026-03-17
 FROM golang:1.26.1-bookworm@sha256:4465644228bc2857a954b092167e12aa59c006a3492282a6c820bf4755fd64a4 AS test
 
 WORKDIR /src
@@ -126,11 +125,14 @@ RUN go mod download
 
 COPY . .
 
+# static/static.go embeds the Alpine.js file this extracts from 3p/.
+RUN script/assets
+
 # -timeout applies to each package on its own, so 90s has only to clear the
 # slowest one. -p 4 -parallel 8 keep the run under 2 GB of memory: at most
 # four test binaries build or run at once, each with at most eight parallel
-# tests. The defaults are one per core, so without these the memory the run
-# needs grows with the host's core count.
+# tests. Under -race every test binary and every link costs a few hundred MB,
+# so the defaults (one per core) add up to several GB on a many-core host.
 #
 # The first run has no -v: go test then prints one result line per package,
 # with its coverage, and for a package that fails, everything its tests
@@ -152,12 +154,12 @@ COPY . .
 SHELL ["/bin/bash", "-o", "pipefail", "-c"]
 RUN --mount=type=tmpfs,target=/tmp/tests,size=512m \
     export TMPDIR=/tmp/tests GOTMPDIR=/tmp; \
-    CGO_ENABLED=0 go test -cover -p 4 -parallel 8 -timeout 90s ./... 2>&1 | tee /tmp/go-test.log && exit 0; \
+    go test -race -cover -p 4 -parallel 8 -timeout 90s ./... 2>&1 | tee /tmp/go-test.log && exit 0; \
     tests="$(awk '/^--- FAIL: / { print $3 }' /tmp/go-test.log | paste -s -d '|' -)"; \
     packages="$(awk '/^FAIL\t/ { print $2 }' /tmp/go-test.log)"; \
     if [ -n "$tests" ]; then \
         echo "--- Rerunning the failed tests with -v for details ---"; \
-        CGO_ENABLED=0 go test -v -p 4 -parallel 8 -timeout 90s -run "^($tests)\$" $packages; \
+        go test -race -v -p 4 -parallel 8 -timeout 90s -run "^($tests)\$" $packages; \
     fi; \
     exit 1
 
@@ -204,8 +206,10 @@ RUN version="$(make version VERSION="$VERSION")"; \
         esac; \
     fi
 
-# The Makefile's build target builds with cgo off, so the binary is
-# statically linked and runs on the Alpine runtime image below.
+# Builds through the Makefile's build target, which runs script/assets
+# (Alpine.js, extracted from its tarball in 3p/) first. The target builds
+# with cgo off, so the binary is statically linked and runs on the Alpine
+# runtime image below as it is.
 RUN make build VERSION="$VERSION"
 
 # Runtime stage

Dockerfile.browser

@@ -15,9 +15,9 @@ RUN go mod download
 COPY . .
 
 # The test binary embeds the templates and static files, so the browser
-# stage needs nothing else. It is built with cgo off and -p 4 keeps the
-# compile's memory down, both as in the test phase of Dockerfile.
-RUN CGO_ENABLED=0 go test -c -p 4 -tags browser -o /browser.test ./internal/server
+# stage needs nothing else. -p 4 keeps the compile's memory down, as in
+# the test phase of Dockerfile.
+RUN make assets && go test -c -p 4 -tags browser -o /browser.test ./internal/server
 
 # chromedp/headless-shell:151.0.7922.109 (Debian trixie), 2026-08-11. The
 # browser is on PATH as headless-shell, where the test's browser library

Makefile

@@ -1,4 +1,4 @@
-.PHONY: bootstrap setup test test-browser lint fmt fmt-check check build run dev deps docker clean hooks css css-check version
+.PHONY: bootstrap setup assets test test-browser lint fmt fmt-check check build run dev deps docker clean hooks css css-check version
 
 # Default target
 .DEFAULT_GOAL := check
@@ -22,6 +22,9 @@ bootstrap:
 setup:
 	@script/setup
 
+assets:
+	@script/assets
+
 test:
 	@script/test
 
@@ -44,13 +47,13 @@ check:
 # REPO_POLICIES.md does: the binary carries no build paths, symbol table or
 # debug information, and on Linux, the image's included, it is statically
 # linked.
-build:
+build: assets
 	CGO_ENABLED=0 go build -trimpath -ldflags '-s -w -X main.version=$(VERSION)' -o bin/webhooker ./cmd/webhooker
 
 run: build
 	./bin/webhooker
 
-dev:
+dev: assets
 	go run ./cmd/webhooker
 
 deps:

TODO.md

@@ -1,77 +1,72 @@
 # Workflow
 
-One issue per unit of work, and one branch and one PR per issue:
+One issue per unit of work, one branch and one PR per issue:
 
-- make sure a tracked issue exists with a definition of done
-- branch from `next`, never from `main`
-- do the work and push the branch
-- open a PR with base `next`, never `main`
-- pass an independent review
-- the repository manager squash-merges it into `next`
+- ensure a tracked issue exists with a definition of done
+- branch from `next` (never from `main`)
+- do the work; open a PR based on `next` (never on `main`)
+- pass an independent review, then the manager squash-merges into `next`
+- push; nothing stays local-only
 
-The repository manager is the role that merges each issue's PR into `next`.
-Nothing stays local-only: every commit is pushed.
+`next` is the branch for the next milestone and must stay green and mergeable to
+`main` without notice. One `next` -> `main` PR accumulates the milestone;
+releases are cut from `main` separately.
 
-`next` is the branch for the next milestone. It must stay green and mergeable
-into `main` at all times. A milestone's work reaches `main` through one PR from
-`next`. The owner, @sneak, merges that PR. Cutting a release from `main` is a
-separate step after that.
-
-The tracker (https://git.eeqj.de/sneak/webhooker/issues) records what is done
-and what is under way. This file keeps no list of either.
+The tracker (https://git.eeqj.de/sneak/webhooker/issues) is the record of what
+is done and what is under way. This file keeps no list of either.
 
 # Status
 
 What is still open for 1.0.0 is in its milestone
-(https://git.eeqj.de/sneak/webhooker/milestone/9). `v1.0.0` is not tagged. Only
-@sneak tags it.
+(https://git.eeqj.de/sneak/webhooker/milestone/9). `v1.0.0` is not tagged, and
+only @sneak tags it.
 
-Delivery is at least once by design. When webhooker cannot record the outcome of
-a delivery, it sends that delivery again. The target can then receive the same
-event twice. The alternative is a lost delivery that nothing reports. The README
-says so. Such a duplicate is not a defect to re-file.
+Delivery is at-least-once by design, not by accident: a send whose result row
+does not land is attempted again, so a receiver can see a duplicate. That is
+deliberate — the alternative is a silent lost delivery — and the README says so
+under Rationale. It is not a defect to re-file.
 
 # Next Step
 
 @sneak runs webhooker in limited production, then tags `v1.0.0`. Merging `next`
-into `main` is a separate act. It waits on neither the production run nor the
-tag.
+into `main` is a separate act from tagging and waits on neither of those: `next`
+is kept mergeable at all times, which is the point of the branch.
 
 # Future Steps
 
-- Per-webhook rate limits, set in the web UI and enforced at the entrypoint.
-  Today `RECEIVER_RATE_LIMIT` is the only rate-limit setting. It is how many
-  requests per minute one client address may send to one entrypoint. The same
-  address is also capped at ten times that across all entrypoints together.
-- API key authentication for programmatic access. The `APIKey` model and its
-  `api_keys` table exist. Nothing creates a key and nothing checks one.
-- REST API v1. `/api/v1` is mounted with no routes today.
-    - CRUD for webhooks, entrypoints and targets
+- Per-webhook rate limits, set in the web UI and enforced in the receiver
+  handler, on top of the receiver's existing limits, which `RECEIVER_RATE_LIMIT`
+  sets for the whole server (see Rate Limiting in the README)
+- API key authentication for programmatic access (the `APIKey` model exists, but
+  nothing creates a key and no middleware checks one)
+- REST API v1
+    - CRUD for webhooks, entrypoints, targets
     - event viewing and filtering endpoints
     - event redelivery endpoint
     - OpenAPI specification
 - Analytics in the web UI: response times summarised per webhook or per target,
-  and figures for all webhooks together. Today the webhook page shows event,
-  delivery and failure counts and failure percentages. The target list counts
-  each target's deliveries. The event log shows every attempt's duration.
-  `/metrics` carries server-wide counts and delivery durations.
-- A remember-me option at login. Today every session survives a browser restart.
-  It lasts up to 7 days. It ends sooner after `SESSION_IDLE_TIMEOUT` (default 24
-  hours) without use.
-- Password reset over the web for a forgotten password. Today a signed-in user
-  can change their password. A lost password is reset from the command line with
-  `webhooker resetpw` (https://git.eeqj.de/sneak/webhooker/issues/208).
+  and figures for all webhooks together. The webhook page already shows event,
+  delivery and failure counts and failure percentages, the target list counts
+  each target's deliveries, the event log and each event's page show every
+  attempt's duration, and `/metrics` carries server-wide counts and delivery
+  durations
+- A remember-me option at login. Every session already survives a browser
+  restart: it lasts up to 7 days, and ends sooner after `SESSION_IDLE_TIMEOUT`
+  (default 24 hours) without activity
+- Password reset flow for a forgotten password over the web. The authenticated
+  password _change_ flow already landed, and a lost password is recoverable from
+  the console with `webhooker resetpw`
+  (https://git.eeqj.de/sneak/webhooker/issues/208)
 - Later, nice to have
     - email delivery target type
     - SNS and S3 delivery targets
-    - data transformations, such as a configurable Slack message format. The
-      Slack target sends one fixed format today.
-    - JSONL file delivery with periodic S3 upload. Today an archive target
-      writes events to SQLite files. An archive target can rotate those files
-      hourly, daily or monthly. The webhook page offers them as one gzipped JSON
-      download.
+    - data transformations (e.g. a configurable Slack message format; the Slack
+      target sends one fixed format today)
+    - JSONL file delivery with periodic S3 upload (the `database` target already
+      archives events to SQLite files, rotated hourly, daily or monthly if set,
+      and the webhook page offers them as one gzipped JSON download)
     - webhook event search, and filtering beyond the event log's failed and
       pending views
-    - multi-user with role-based access control. Today each webhook belongs to
-      one account. Only the account made at first boot exists. There is no way
-      to add another.
+    - multi-user with role-based access control (each webhook already belongs to
+      one account, but there is only the account made at first boot and no way
+      to add another)

eslint.config.mjs

@@ -5,7 +5,7 @@
 // REPO_POLICIES.md states that a linter can check: const for everything,
 // let only for a variable that is reassigned, never var.
 export default [
-    // Alpine.js, committed as published; not ours to lint.
+    // Alpine.js, extracted from 3p/ by make assets; not ours to lint.
     { ignores: ["static/js/alpine.min.js"] },
     {
         // The pages load static/js/app.js as a classic script, not a module.

internal/archive/evict_test.go

@@ -303,7 +303,7 @@ func (r *racingWrites) wait() (bool, error) {
 // contending for the writer's mutex when the eviction takes it.
 // The write that wins the mutex after the eviction must abandon
 // its work rather than reopen the archive, leaving the writer
-// permanently handle-free.
+// permanently handle-free. Run under -race.
 func TestEvictWebhook_RacingWriteDoesNotReopenHandle(
 	t *testing.T,
 ) {

internal/archive/sweeper_test.go

@@ -959,11 +959,9 @@ func TestArchiveSweep_SkipsDeletedWebhookTargets(t *testing.T) {
 	)
 }
 
-// TestArchiveSweep_ConcurrentWrites runs the sweep while writes to
-// the same target are under way. The sweep serialises against them
-// through the target's writer mutex; the suite runs without Go's race
-// detector, so a sweep that did not would fail this test only by
-// panicking, hanging or leaving no archive file.
+// TestArchiveSweep_ConcurrentWrites proves the sweep serialises
+// against writes through the target's writer mutex. Run under
+// -race, an unsynchronised sweep would be caught here.
 func TestArchiveSweep_ConcurrentWrites(t *testing.T) {
 	t.Parallel()
 

internal/database/password.go

@@ -67,10 +67,9 @@ func DefaultPasswordConfig() *PasswordConfig {
 // testArgon2Memory is the Argon2id memory cost, in KiB, that a test
 // binary hashes with: 1 MB instead of the shipped 64 MB. Every test
 // that starts a database hashes the bootstrap admin password, dozens
-// of them run in parallel, and each 64 MB hash holds at least 64 MB
-// while it runs (about 150 MB when the suite ran under the race
-// detector). VerifyPassword reads the cost from the hash it checks, so
-// verification follows.
+// of them run in parallel, and under the race detector each 64 MB hash
+// holds about 150 MB. VerifyPassword reads the cost from the hash it
+// checks, so verification follows.
 const testArgon2Memory = 1024
 
 // hashAtShippedCostInTest makes a test binary hash at the shipped

internal/delivery/engine.go

@@ -23,11 +23,10 @@ import (
 
 const (
 	// deliveryChannelSize is the buffer size for the delivery
-	// channel. The Tasks of newly stored deliveries, and of the
-	// pending deliveries the recovery routine hands back, are
-	// sent here. Workers drain this channel. Sized large enough
-	// that no task should be dropped under normal load: one that
-	// does not fit is left for the sweep.
+	// channel. The Tasks of newly stored deliveries are sent
+	// here. Workers drain this channel. Sized large enough
+	// that storing an event should never block under normal
+	// load.
 	deliveryChannelSize = 10000
 
 	// retryChannelSize is the buffer size for the retry
@@ -152,13 +151,12 @@ type EngineParams struct {
 }
 
 // Engine processes queued deliveries in the background
-// using a bounded worker pool architecture. It also stores each
-// event with its deliveries and creates replay deliveries (see
-// store.go). Beyond that it owns only the cross-target machinery:
-// the worker pool, the delivery and retry channels, the recovery
-// routine, and the persistence helpers and Scheduler that
-// individual targets rely on. Each target type owns its own
-// delivery, including retries, backoff, and circuit breaking.
+// using a bounded worker pool architecture. It owns only
+// the cross-target machinery: the worker pool, the queue
+// and retry channels, the recovery routine, and the persistence
+// helpers and Scheduler that individual targets rely on.
+// Each target type owns its own delivery, including retries,
+// backoff, and circuit breaking.
 type Engine struct {
 	database   *database.Database
 	dbManager  *database.WebhookDBManager

internal/delivery/inflight_test.go

@@ -84,7 +84,7 @@ func TestSweepLeavesAQueuedDeliveryAlone(t *testing.T) {
 	)
 	rAgePending(t, s.WebhookDB, d.ID)
 
-	// Queued exactly as StoreEvent queues it, and never dequeued:
+	// Queued exactly as the receiver queues it, and never dequeued:
 	// no workers are running in this engine.
 	s.Engine.ExportNotify([]delivery.Task{{
 		DeliveryID: d.ID,
@@ -248,9 +248,9 @@ func TestOwnershipIsReleasedAfterDelivery(t *testing.T) {
 }
 
 // TestNotifyAfterRecoveryDoesNotSendAgain is the startup race of
-// https://git.eeqj.de/sneak/webhooker/issues/299. StoreEvent has
+// https://git.eeqj.de/sneak/webhooker/issues/299. The receiver has
 // written a delivery, the recovery routine's run at start finds it
-// pending, sends it and releases it, and only then does StoreEvent's
+// pending, sends it and releases it, and only then does the receiver's
 // notify for it arrive. Nothing owns the delivery by then, so notify
 // takes it.
 func TestNotifyAfterRecoveryDoesNotSendAgain(t *testing.T) {

internal/handlers/event_resubmit.go

@@ -172,10 +172,8 @@ func loadResubmitSource(
 }
 
 // queueResubmit has the delivery engine store the copy, with a delivery
-// for each of the webhook's active targets as they stand now, through
-// the same StoreEvent the receiver uses. The one difference is that the
-// copy leaves its entrypoint's last event as it is, since it did not
-// arrive on that URL.
+// for each of the webhook's active targets as they stand now, exactly
+// as it stores a received event.
 func (h *Handlers) queueResubmit(
 	w http.ResponseWriter,
 	r *http.Request,

internal/handlers/handlers_test.go

@@ -239,8 +239,8 @@ func newTestAppWithConfig(
 			session.New,
 			archive.NewWriters,
 			// The engine is built on a lifecycle that is never started,
-			// so no worker runs: every delivery the engine creates for a
-			// handler stays pending, where the test can read it.
+			// so no worker runs: every delivery a handler creates stays
+			// pending, where the test can read it.
 			func(p delivery.EngineParams) *delivery.Engine {
 				return delivery.New(fxtest.NewLifecycle(t), p)
 			},

internal/logfield/logfield_test.go

@@ -21,9 +21,8 @@ const budget = 64
 
 // batchRunes is how many consecutive code points the charge test logs
 // in one value from U+1000 up. Logging each of those on its own line
-// was too slow for the suite when it ran under the race detector;
-// 4,096 at a time is 271 batches, each logged on two lines, so 542
-// lines per handler.
+// is too slow for the suite under the race detector; 4,096 at a time
+// is 271 batches, each logged on two lines, so 542 lines per handler.
 const batchRunes = 4096
 
 // newHandlers are the two handlers internal/logger can install. Time
@@ -153,10 +152,10 @@ func TestEncodedBytes_ChargesAtLeastWhatTheHandlersEmit(t *testing.T) {
 		batches = append(batches, batch.String())
 	}
 
-	// What EncodedBytes charges for each batch. When the suite ran
-	// under -race -cover this took longer than logging the batches, so
-	// it is worked out once, by whichever handler finishes logging
-	// first, while the other is still logging.
+	// What EncodedBytes charges for each batch. Under -race -cover this
+	// takes longer than logging the batches, so it is worked out once,
+	// by whichever handler finishes logging first, while the other is
+	// still logging.
 	charged := sync.OnceValue(func() []int {
 		costs := make([]int, len(batches))
 

internal/server/early_shutdown_test.go

@@ -10,12 +10,14 @@ import (
 	"sneak.berlin/go/webhooker/internal/server"
 )
 
-// earlyStopIterations is how many start/stop cycles the test below
+// earlyStopIterations is how many start/stop cycles the race test
 // runs. The window it aims at is the gap between the OnStart hook
 // returning and the serving goroutine reaching its first field
-// access, which is microseconds wide, so a single cycle can miss it
-// purely on scheduling. Repetition makes hitting it reliable; the
-// collaborators are built once, so the cycles themselves are cheap.
+// access, which is microseconds wide. The race detector reports an
+// unsynchronised pair whenever it observes one, but it has to observe
+// one, so a single cycle can miss purely on scheduling. Repetition
+// makes the observation reliable; the collaborators are built once,
+// so the cycles themselves are cheap.
 const earlyStopIterations = 25
 
 // TestEarlyShutdown_NoPanicAndNoRace stops the application
@@ -36,9 +38,8 @@ const earlyStopIterations = 25
 //
 // httpServer is now built in New, on the constructing goroutine, so
 // it is written before any hook exists and can never be nil.
-// sentryEnabled is atomic. This test catches the nil dereference
-// coming back, as a panic. The suite runs without Go's race
-// detector, so the data race coming back would go unreported here.
+// sentryEnabled is atomic. This test is what catches either one
+// coming back — under -race, which is how the suite runs.
 func TestEarlyShutdown_NoPanicAndNoRace(t *testing.T) {
 	t.Parallel()
 

internal/server/static_assets_test.go

@@ -13,6 +13,9 @@ import (
 
 // TestBaseTemplateScriptsAreServed walks every /s/ script the base
 // template loads on each page and fetches it through the real router.
+// Alpine.js is extracted from its tarball in 3p/ at build time, so the
+// file is not in the tree: this is the check that the page still gets
+// the JavaScript it asks for.
 func TestBaseTemplateScriptsAreServed(t *testing.T) {
 	t.Parallel()
 

script/assets

@@ -0,0 +1,17 @@
+#!/bin/sh
+# script/assets: extract Alpine.js from its npm package tarball, committed
+# in 3p/, to static/js/alpine.min.js, where go:embed reads it. The package
+# is @alpinejs/csp, Alpine's build for pages whose Content-Security-Policy
+# forbids eval. The extracted file is not committed. make build, make dev
+# and the Dockerfile's lint, test and build stages run this first.
+set -eu
+
+ROOT="$(cd "$(dirname "$0")/.." && pwd -P)"
+
+main() {
+    cd "$ROOT"
+    tar -xzOf 3p/alpinejs-csp-3.14.9.tgz package/dist/cdn.min.js \
+        >static/js/alpine.min.js
+}
+
+main "$@"

static/static.go

@@ -6,11 +6,9 @@ import (
 )
 
 // Static holds the CSS and JavaScript files the web UI's pages load. They
-// are named one by one, so a missing file fails the build instead of
-// leaving the pages without it.
-//
-// js/alpine.min.js is Alpine.js, committed unchanged. It is
-// package/dist/cdn.min.js from the npm package @alpinejs/csp 3.14.9.
+// are named one by one so that a missing js/alpine.min.js, which make
+// assets extracts and git does not track, fails the build instead of
+// leaving the pages without Alpine.js.
 //
 //go:embed css/tailwind.css css/style.css js/app.js js/alpine.min.js
 var Static embed.FS