Root background loops at context.Background() (closes #97)

The context fx hands an OnStart hook is derived with
context.WithTimeout(ctx, StartTimeout) — 15 seconds by default — and
is cancelled once the start phase ends. It is a start-phase context,
not an application-lifetime one. Two components derived their
long-lived loops from it and so stopped running roughly fifteen
seconds after boot.

Engine.start rooted the entire worker pool, restart recovery, and the
retry sweep in it. Every worker returned on ctx.Done() shortly after
startup, so the process kept receiving and persisting inbound events
while nothing at all forwarded them: deliveryCh filled up and started
logging "delivery channel full" with no consumer left. That is the
whole purpose of the application.

RetentionReaper.start had the same defect. With the default one-hour
RETENTION_SWEEP_INTERVAL the loop was cancelled forty-five minutes
before its first tick, so the reaper never ran a single sweep and
per-webhook event databases grew without bound.

Both now derive their loop context from context.Background(). Their
lifetime is bounded by OnStop, which already cancels and waits on the
WaitGroup, so shutdown is unchanged. Each hook registration moves into
a registerHooks method, the OnStart parameter is named _ so the trap
cannot be reintroduced by silencing an unused-parameter warning, and a
comment at each start explains why the hook context must not be used.
This matches the shape of the same fix applied to the archive sweeper.

The new lifecycle tests drive the genuine registered hooks with an
already-cancelled OnStart context and assert the loops still do work
afterwards — a task delivered, an expired event reaped. Reverting
either fix makes its pair of tests fail. Each component also gets a
shutdown test asserting OnStop cancels the loop and wg.Wait() returns
inside a bounded timeout, so the fix does not trade a startup bug for
a shutdown hang.

iWaitForStatus becomes iWaitForDelivered: every call site waits for
the delivered status, and the two added call sites pushed it past
unparam's threshold for reporting an always-identical argument.

internal/database/export_test.go

@@ -5,6 +5,8 @@ import (
 	"log/slog"
 	"os"
 	"time"
+
+	"go.uber.org/fx"
 )
 
 // NewTestRetentionReaper builds a RetentionReaper backed by the given
@@ -29,3 +31,26 @@ func NewTestRetentionReaper(
 func (r *RetentionReaper) ExportSweep(ctx context.Context) {
 	r.sweep(ctx)
 }
+
+// ExportRegisterHooks registers the reaper's real fx lifecycle hooks
+// on a lifecycle supplied by a test, so a test can drive the exact
+// OnStart/OnStop functions the application runs and hand OnStart the
+// kind of context fx actually supplies.
+func (r *RetentionReaper) ExportRegisterHooks(lc fx.Lifecycle) {
+	r.registerHooks(lc)
+}
+
+// ExportStart starts the reaper's background loop for tests.
+func (r *RetentionReaper) ExportStart() {
+	r.start()
+}
+
+// ExportStop stops the reaper's background loop for tests.
+func (r *RetentionReaper) ExportStop() {
+	r.stop()
+}
+
+// ExportSetInterval overrides the sweep interval for tests.
+func (r *RetentionReaper) ExportSetInterval(d time.Duration) {
+	r.interval = d
+}

internal/database/retention.go

@@ -56,9 +56,20 @@ func NewRetentionReaper(
 		interval:  params.Config.RetentionSweepInterval,
 	}
 
+	r.registerHooks(lc)
+
+	return r
+}
+
+// registerHooks wires the reaper's start and stop into the fx
+// lifecycle. The start hook's context is deliberately ignored: see
+// start for why the sweep loop must not inherit it.
+func (r *RetentionReaper) registerHooks(lc fx.Lifecycle) {
 	lc.Append(fx.Hook{
-		OnStart: func(ctx context.Context) error {
-			r.start(ctx)
+		//nolint:contextcheck // Not inheriting the hook context is
+		// the point: see start.
+		OnStart: func(_ context.Context) error {
+			r.start()
 
 			return nil
 		},
@@ -68,12 +79,20 @@ func NewRetentionReaper(
 			return nil
 		},
 	})
-
-	return r
 }
 
-func (r *RetentionReaper) start(ctx context.Context) {
-	ctx, cancel := context.WithCancel(ctx)
+// start launches the background sweep loop.
+//
+// The loop's context is derived from context.Background(), NOT from
+// the fx OnStart hook context. The hook context carries fx's start
+// timeout (15s by default) and is cancelled once the start phase
+// completes, so a loop derived from it dies 45 minutes before its
+// first tick under the default one-hour sweep interval, leaving a
+// reaper that never reaps. A long-lived goroutine must outlive the
+// startup phase, so its lifetime is bounded by OnStop instead: stop
+// cancels this context and waits on the WaitGroup.
+func (r *RetentionReaper) start() {
+	ctx, cancel := context.WithCancel(context.Background())
 	r.cancel = cancel
 
 	r.wg.Add(1)

internal/database/retention_lifecycle_test.go

@@ -0,0 +1,209 @@
+package database_test
+
+import (
+	"context"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"go.uber.org/fx"
+	"gorm.io/gorm"
+	"sneak.berlin/go/webhooker/internal/database"
+)
+
+const (
+	// reaperTestInterval is the sweep interval a lifecycle test
+	// runs the reaper at, so a loop that survives startup produces
+	// an observable sweep quickly.
+	reaperTestInterval = 10 * time.Millisecond
+
+	// reaperStopTimeout bounds how long a lifecycle test waits for
+	// the reaper's OnStop hook to return before declaring the
+	// shutdown hung.
+	reaperStopTimeout = 10 * time.Second
+
+	// reaperTestRetentionDays is the retention policy the lifecycle
+	// tests give their webhook.
+	reaperTestRetentionDays = 30
+)
+
+// recordingLifecycle is a minimal fx.Lifecycle that records the
+// hooks a component registers, so a test can invoke the real
+// OnStart/OnStop functions with a context of its choosing.
+type recordingLifecycle struct {
+	hooks []fx.Hook
+}
+
+func (l *recordingLifecycle) Append(h fx.Hook) {
+	l.hooks = append(l.hooks, h)
+}
+
+// startReaperViaHook drives the genuine fx hooks the application
+// registers for the reaper, handing OnStart a context that is
+// already done. It returns the recorded lifecycle so the caller
+// can drive OnStop too.
+func startReaperViaHook(
+	t *testing.T, r *database.RetentionReaper,
+) *recordingLifecycle {
+	t.Helper()
+
+	lc := &recordingLifecycle{}
+	r.ExportRegisterHooks(lc)
+	require.Len(t, lc.hooks, 1)
+
+	// fx hands OnStart a context carrying the application start
+	// timeout, and cancels it when the start phase ends. An
+	// already-cancelled context is that same defect taken to its
+	// limit, and unlike a plain context.Background() it actually
+	// distinguishes a correctly rooted loop from a broken one.
+	hookCtx, cancel := context.WithCancel(context.Background())
+	cancel()
+
+	require.NoError(t, lc.hooks[0].OnStart(hookCtx))
+
+	return lc
+}
+
+// eventGone reports whether an event row has been removed. It
+// takes no *testing.T because it is polled from an
+// assert.Eventually condition, which runs off the test goroutine
+// where testify assertions must not be used.
+func eventGone(db *gorm.DB, eventID string) bool {
+	var n int64
+
+	err := db.Unscoped().Model(&database.Event{}).
+		Where("id = ?", eventID).Count(&n).Error
+	if err != nil {
+		return false
+	}
+
+	return n == 0
+}
+
+// seedExpiredWebhook creates a webhook with a finite retention
+// policy plus one long-expired event chain, and returns the
+// webhook's database and the chain's event ID.
+func seedExpiredWebhook(
+	t *testing.T, env *retentionTestEnv,
+) (*gorm.DB, string) {
+	t.Helper()
+
+	webhookID := createWebhook(
+		t, env.mainDB.DB(), reaperTestRetentionDays,
+	)
+
+	db, err := env.mgr.GetDB(webhookID)
+	require.NoError(t, err)
+
+	chain := seedEventChain(
+		t, db, webhookID,
+		time.Now().Add(-365*24*time.Hour),
+	)
+
+	return db, chain.eventID
+}
+
+// TestRetentionReaper_LoopOutlivesStartHookContext is the
+// regression test for a reaper that never reaped. fx calls
+// OnStart with a context carrying the application's start timeout
+// (15s by default) and cancels it when the start phase ends, so a
+// sweep loop rooted in it is dead three quarters of an hour
+// before its first tick under the default one-hour interval, and
+// per-webhook event databases grow without bound exactly as they
+// did before retention existed.
+//
+// Driving OnStart with an already-cancelled context is that
+// defect taken to its limit: a loop that inherits the hook
+// context never ticks once, while a correctly rooted loop keeps
+// sweeping for as long as the process lives.
+func TestRetentionReaper_LoopOutlivesStartHookContext(
+	t *testing.T,
+) {
+	t.Parallel()
+
+	env := setupRetentionTest(t)
+
+	db, eventID := seedExpiredWebhook(t, env)
+
+	env.reaper.ExportSetInterval(reaperTestInterval)
+
+	lc := startReaperViaHook(t, env.reaper)
+	t.Cleanup(func() {
+		_ = lc.hooks[0].OnStop(context.Background())
+	})
+
+	assert.Eventually(
+		t,
+		func() bool { return eventGone(db, eventID) },
+		5*time.Second,
+		reaperTestInterval,
+		"the sweep loop must keep running after the start "+
+			"hook's context is done; it reaped nothing, so it "+
+			"inherited the hook context and died",
+	)
+}
+
+// TestRetentionReaper_StopHookStopsLoop proves the fix did not
+// trade a startup bug for a shutdown hang: now that the sweep
+// loop no longer observes the start hook's cancellation, OnStop
+// is the only thing that can stop it, and it must both return
+// promptly and actually leave the loop stopped.
+func TestRetentionReaper_StopHookStopsLoop(t *testing.T) {
+	t.Parallel()
+
+	env := setupRetentionTest(t)
+
+	db, eventID := seedExpiredWebhook(t, env)
+
+	env.reaper.ExportSetInterval(reaperTestInterval)
+
+	lc := startReaperViaHook(t, env.reaper)
+
+	// Let the loop prove it is running before stopping it, so a
+	// fast OnStop cannot pass by stopping something already dead.
+	require.Eventually(
+		t,
+		func() bool { return eventGone(db, eventID) },
+		5*time.Second,
+		reaperTestInterval,
+	)
+
+	var stopErr error
+
+	stopped := make(chan struct{})
+
+	go func() {
+		defer close(stopped)
+
+		// stop blocks on the loop's WaitGroup, so returning at all
+		// proves the goroutine observed the cancellation.
+		stopErr = lc.hooks[0].OnStop(context.Background())
+	}()
+
+	select {
+	case <-stopped:
+	case <-time.After(reaperStopTimeout):
+		t.Fatal(
+			"OnStop did not return: the retention reaper's " +
+				"WaitGroup is still waiting on a loop that never " +
+				"observed cancellation",
+		)
+	}
+
+	require.NoError(t, stopErr)
+
+	// With the loop gone, a newly expired chain must survive.
+	survivor := seedEventChain(
+		t, db, "stopped-webhook",
+		time.Now().Add(-365*24*time.Hour),
+	)
+
+	time.Sleep(20 * reaperTestInterval)
+
+	assert.False(
+		t,
+		eventGone(db, survivor.eventID),
+		"a stopped reaper must not sweep anything",
+	)
+}

internal/delivery/engine.go

@@ -149,18 +149,7 @@ func New(
 		Transport: NewSSRFSafeTransport(),
 	})
 
-	lc.Append(fx.Hook{
-		OnStart: func(ctx context.Context) error {
-			e.start(ctx)
-
-			return nil
-		},
-		OnStop: func(_ context.Context) error {
-			e.stop()
-
-			return nil
-		},
-	})
+	e.registerHooks(lc)
 
 	return e
 }
@@ -210,8 +199,40 @@ func (e *Engine) ScheduleRetry(
 	})
 }
 
-func (e *Engine) start(ctx context.Context) {
-	ctx, cancel := context.WithCancel(ctx)
+// registerHooks wires the engine's start and stop into the fx
+// lifecycle. The start hook's context is deliberately ignored:
+// see start for why the worker pool must not inherit it.
+func (e *Engine) registerHooks(lc fx.Lifecycle) {
+	lc.Append(fx.Hook{
+		//nolint:contextcheck // Not inheriting the hook context
+		// is the point: see start.
+		OnStart: func(_ context.Context) error {
+			e.start()
+
+			return nil
+		},
+		OnStop: func(_ context.Context) error {
+			e.stop()
+
+			return nil
+		},
+	})
+}
+
+// start launches the worker pool, restart recovery, and the
+// periodic retry sweep.
+//
+// Their context is derived from context.Background(), NOT from
+// the fx OnStart hook context. The hook context carries fx's
+// start timeout (15s by default) and is cancelled once the start
+// phase completes, so goroutines derived from it stop a few
+// seconds into the process: every worker would return and the
+// engine would silently stop delivering webhooks entirely. A
+// long-lived goroutine must outlive the startup phase, so its
+// lifetime is bounded by OnStop instead: stop cancels this
+// context and waits on the WaitGroup.
+func (e *Engine) start() {
+	ctx, cancel := context.WithCancel(context.Background())
 	e.cancel = cancel
 
 	for range e.workers {

internal/delivery/engine_integration_test.go

@@ -476,7 +476,7 @@ func TestWorkerLifecycle_StartStop(t *testing.T) {
 	t.Parallel()
 
 	s := newISetup(t)
-	s.Engine.ExportStart(context.Background())
+	s.Engine.ExportStart()
 
 	event := iSeedEvent(
 		t, s.WebhookDB, s.WebhookID,
@@ -499,21 +499,17 @@ func TestWorkerLifecycle_StartStop(t *testing.T) {
 
 	s.Engine.Notify([]delivery.Task{task})
 
-	iWaitForStatus(
-		t, s.WebhookDB, d.ID,
-		database.DeliveryStatusDelivered,
-	)
+	iWaitForDelivered(t, s.WebhookDB, d.ID)
 
 	s.Engine.ExportStop()
 }
 
-// iWaitForStatus polls until the delivery reaches the
-// expected status.
-func iWaitForStatus(
+// iWaitForDelivered polls until the delivery reaches the
+// delivered status.
+func iWaitForDelivered(
 	t *testing.T,
 	db *gorm.DB,
 	deliveryID string,
-	expected database.DeliveryStatus,
 ) {
 	t.Helper()
 
@@ -527,7 +523,7 @@ func iWaitForStatus(
 			return false
 		}
 
-		return d.Status == expected
+		return d.Status == database.DeliveryStatusDelivered
 	}, 5*time.Second, 50*time.Millisecond)
 }
 
@@ -558,7 +554,7 @@ func TestWorkerLifecycle_ProcessesRetryChannel(
 		database.DeliveryStatusRetrying,
 	)
 
-	s.Engine.ExportStart(context.Background())
+	s.Engine.ExportStart()
 
 	bodyStr := event.Body
 	cfg := iHTTPConfig(ts.URL)
@@ -569,10 +565,7 @@ func TestWorkerLifecycle_ProcessesRetryChannel(
 
 	s.Engine.ExportRetryCh() <- task
 
-	iWaitForStatus(
-		t, s.WebhookDB, d.ID,
-		database.DeliveryStatusDelivered,
-	)
+	iWaitForDelivered(t, s.WebhookDB, d.ID)
 
 	s.Engine.ExportStop()
 }

internal/delivery/engine_lifecycle_test.go

@@ -0,0 +1,199 @@
+package delivery_test
+
+import (
+	"context"
+	"testing"
+	"time"
+
+	"github.com/google/uuid"
+	"github.com/stretchr/testify/require"
+	"go.uber.org/fx"
+	"sneak.berlin/go/webhooker/internal/database"
+	"sneak.berlin/go/webhooker/internal/delivery"
+)
+
+const (
+	// hookStopTimeout bounds how long a lifecycle test waits for
+	// the engine's OnStop hook to return before declaring the
+	// shutdown hung.
+	hookStopTimeout = 10 * time.Second
+
+	// hookSettleDelay is how long startEngineViaHook waits after
+	// OnStart before the caller may enqueue work. A worker pool
+	// wrongly rooted in the already-done hook context has nothing
+	// but ctx.Done() ready in its select, so it is deterministically
+	// gone by the end of this window. Without the wait, Notify would
+	// race the pool's very first select, in which a ready ctx.Done()
+	// and a ready deliveryCh are chosen between at random and a
+	// doomed pool still delivers.
+	hookSettleDelay = 250 * time.Millisecond
+)
+
+// recordingLifecycle is a minimal fx.Lifecycle that records the
+// hooks a component registers, so a test can invoke the real
+// OnStart/OnStop functions with a context of its choosing.
+type recordingLifecycle struct {
+	hooks []fx.Hook
+}
+
+func (l *recordingLifecycle) Append(h fx.Hook) {
+	l.hooks = append(l.hooks, h)
+}
+
+// startEngineViaHook drives the genuine fx hooks the application
+// registers for the engine, handing OnStart a context that is
+// already done, and returns only once a pool that inherited that
+// context would have exited. It returns the recorded lifecycle so
+// the caller can drive OnStop too.
+//
+// Callers must not seed pending or retrying deliveries before
+// calling this: restart recovery enqueues those during startup,
+// which would put work in the queue while the pool is still
+// racing its first select.
+func startEngineViaHook(
+	t *testing.T, eng *delivery.Engine,
+) *recordingLifecycle {
+	t.Helper()
+
+	lc := &recordingLifecycle{}
+	eng.ExportRegisterHooks(lc)
+	require.Len(t, lc.hooks, 1)
+
+	// fx hands OnStart a context carrying the application start
+	// timeout, and cancels it when the start phase ends. An
+	// already-cancelled context is that same defect taken to its
+	// limit, and unlike a plain context.Background() it actually
+	// distinguishes a correctly rooted loop from a broken one.
+	hookCtx, cancel := context.WithCancel(context.Background())
+	cancel()
+
+	require.NoError(t, lc.hooks[0].OnStart(hookCtx))
+
+	time.Sleep(hookSettleDelay)
+
+	return lc
+}
+
+// seedLogTask seeds a pending delivery for a log target and
+// returns its ID together with the task that drives it. The log
+// target needs no network, so a delivery completing proves only
+// that a worker picked the task up.
+func seedLogTask(
+	t *testing.T, s iSetup,
+) (string, delivery.Task) {
+	t.Helper()
+
+	event := iSeedEvent(
+		t, s.WebhookDB, s.WebhookID,
+		`{"lifecycle":"hook-context"}`,
+	)
+	targetID := uuid.New().String()
+
+	d := iSeedDelivery(
+		t, s.WebhookDB, event.ID, targetID,
+		database.DeliveryStatusPending,
+	)
+
+	bodyStr := event.Body
+	task := iTask(
+		d, event, s.WebhookID, targetID,
+		"hook-context-test", "", 0, 1, &bodyStr,
+	)
+	task.TargetType = database.TargetTypeLog
+
+	return d.ID, task
+}
+
+// TestEngine_WorkersOutliveStartHookContext is the regression
+// test for a delivery engine that stopped delivering roughly
+// fifteen seconds after boot. fx calls OnStart with a context
+// carrying the application's start timeout (15s by default) and
+// cancels it when the start phase ends, so a worker pool rooted
+// in it exits shortly after startup: the process keeps accepting
+// and persisting events while nothing at all forwards them.
+//
+// Driving OnStart with an already-cancelled context is that
+// defect taken to its limit. A pool that inherits the hook
+// context is gone before the task is even enqueued; a correctly
+// rooted pool keeps working for as long as the process lives.
+func TestEngine_WorkersOutliveStartHookContext(t *testing.T) {
+	t.Parallel()
+
+	s := newISetup(t)
+
+	lc := startEngineViaHook(t, s.Engine)
+	t.Cleanup(func() {
+		_ = lc.hooks[0].OnStop(context.Background())
+	})
+
+	// Seeded only after the pool has settled, so restart recovery
+	// cannot enqueue it during startup.
+	deliveryID, task := seedLogTask(t, s)
+
+	s.Engine.Notify([]delivery.Task{task})
+
+	iWaitForDelivered(t, s.WebhookDB, deliveryID)
+}
+
+// TestEngine_StopHookStopsWorkers proves the fix did not trade a
+// startup bug for a shutdown hang: now that the worker pool no
+// longer observes the start hook's cancellation, OnStop is the
+// only thing that can stop it, and it must both return promptly
+// and actually leave the pool drained.
+func TestEngine_StopHookStopsWorkers(t *testing.T) {
+	t.Parallel()
+
+	s := newISetup(t)
+
+	lc := startEngineViaHook(t, s.Engine)
+
+	// Let the pool prove it is running before stopping it, so a
+	// fast OnStop cannot pass by stopping something already dead.
+	firstID, firstTask := seedLogTask(t, s)
+	s.Engine.Notify([]delivery.Task{firstTask})
+	iWaitForDelivered(t, s.WebhookDB, firstID)
+
+	var stopErr error
+
+	stopped := make(chan struct{})
+
+	go func() {
+		defer close(stopped)
+
+		// stop blocks on the workers' WaitGroup, so returning at
+		// all proves every goroutine observed the cancellation.
+		stopErr = lc.hooks[0].OnStop(context.Background())
+	}()
+
+	select {
+	case <-stopped:
+	case <-time.After(hookStopTimeout):
+		t.Fatal(
+			"OnStop did not return: the delivery engine's " +
+				"WaitGroup is still waiting on a goroutine that " +
+				"never observed cancellation",
+		)
+	}
+
+	require.NoError(t, stopErr)
+
+	// With every worker gone, a freshly notified task must sit
+	// untouched in the queue rather than being delivered.
+	secondID, secondTask := seedLogTask(t, s)
+	s.Engine.Notify([]delivery.Task{secondTask})
+
+	time.Sleep(200 * time.Millisecond)
+
+	var after database.Delivery
+
+	require.NoError(
+		t,
+		s.WebhookDB.First(&after, "id = ?", secondID).Error,
+	)
+	require.Equal(
+		t,
+		database.DeliveryStatusPending,
+		after.Status,
+		"a stopped engine must not deliver anything",
+	)
+}

internal/delivery/export_test.go

@@ -7,6 +7,7 @@ import (
 	"net/http"
 	"time"
 
+	"go.uber.org/fx"
 	"gorm.io/gorm"
 	"sneak.berlin/go/webhooker/internal/database"
 )
@@ -189,8 +190,16 @@ func (e *Engine) ExportRecoverInFlight(
 }
 
 // ExportStart exposes start for testing.
-func (e *Engine) ExportStart(ctx context.Context) {
-	e.start(ctx)
+func (e *Engine) ExportStart() {
+	e.start()
+}
+
+// ExportRegisterHooks registers the engine's real fx lifecycle
+// hooks on a lifecycle supplied by a test, so a test can drive
+// the exact OnStart/OnStop functions the application runs and
+// hand OnStart the kind of context fx actually supplies.
+func (e *Engine) ExportRegisterHooks(lc fx.Lifecycle) {
+	e.registerHooks(lc)
 }
 
 // ExportStop exposes stop for testing.