Use one recovery routine at start and on every sweep tick (closes #542)

Start-up recovery and the 60-second sweep were two parallel sets of
functions doing the same job. recoverDeliveries now does it, run once at
start and then on every tick from one goroutine. A retrying delivery is
scheduled after what is left of its backoff, now if nothing is left, so
backoffElapsed is gone. The run at start still takes every pending
delivery and opens a webhook database whose file is missing; a tick
still takes only pending deliveries untouched for 15 minutes, at most
500, and skips a missing file. A retrying delivery is taken before its
event is read. The README describes one routine.

Model: opus-5-5

internal/delivery/terminal_state_test.go

@@ -178,7 +178,7 @@ func TestRecoverSingleRetry_TargetDeleted(t *testing.T) {
 	)
 
 	s.Engine.ExportRecoverWebhookDeliveries(
-		context.Background(), s.WebhookID,
+		context.Background(), s.WebhookID, atStart,
 	)
 
 	iAssertStatus(
@@ -201,7 +201,7 @@ func TestRecoverSingleRetry_TargetDeleted(t *testing.T) {
 	)
 }
 
-func TestSweepSingleRetry_TargetDeleted(t *testing.T) {
+func TestRecoverSingleRetry_OnTick_TargetDeleted(t *testing.T) {
 	t.Parallel()
 
 	s := newISetup(t)
@@ -218,11 +218,11 @@ func TestSweepSingleRetry_TargetDeleted(t *testing.T) {
 	// Twice, because the bug was an error the sweep repeated every
 	// minute for the life of the database: the second sweep must
 	// find nothing left to do.
-	s.Engine.ExportSweepWebhookRetries(
-		context.Background(), s.WebhookID,
+	s.Engine.ExportRecoverWebhookDeliveries(
+		context.Background(), s.WebhookID, onTick,
 	)
-	s.Engine.ExportSweepWebhookRetries(
-		context.Background(), s.WebhookID,
+	s.Engine.ExportRecoverWebhookDeliveries(
+		context.Background(), s.WebhookID, onTick,
 	)
 
 	iAssertStatus(
@@ -239,10 +239,11 @@ func TestSweepSingleRetry_TargetDeleted(t *testing.T) {
 	assert.Zero(t, s.Engine.ExportInflightHeld())
 }
 
-// TestSweepSingleRetry_TargetNeverExisted covers the other half of the
-// soft-delete distinction: an id with no row at all, deleted or
-// otherwise, must not be reported as something the operator deleted.
-func TestSweepSingleRetry_TargetNeverExisted(t *testing.T) {
+// TestRecoverSingleRetry_OnTick_TargetNeverExisted covers the other
+// half of the soft-delete distinction: an id with no row at all,
+// deleted or otherwise, must not be reported as something the operator
+// deleted.
+func TestRecoverSingleRetry_OnTick_TargetNeverExisted(t *testing.T) {
 	t.Parallel()
 
 	s := newISetup(t)
@@ -264,8 +265,8 @@ func TestSweepSingleRetry_TargetNeverExisted(t *testing.T) {
 
 	iSeedFailedResult(t, s.WebhookDB, d.ID)
 
-	s.Engine.ExportSweepWebhookRetries(
-		context.Background(), s.WebhookID,
+	s.Engine.ExportRecoverWebhookDeliveries(
+		context.Background(), s.WebhookID, onTick,
 	)
 
 	iAssertStatus(
@@ -303,8 +304,8 @@ func TestFailMissingTarget_WritesNoTargetRow(
 		database.DeliveryStatusRetrying,
 	)
 
-	s.Engine.ExportSweepWebhookRetries(
-		context.Background(), s.WebhookID,
+	s.Engine.ExportRecoverWebhookDeliveries(
+		context.Background(), s.WebhookID, onTick,
 	)
 
 	iAssertStatus(
@@ -546,7 +547,7 @@ func TestRecoverSingleRetry_TargetUnreadable_LeavesDeliveryAlone(
 	require.NoError(t, sqlDB.Close())
 
 	s.Engine.ExportRecoverRetryingDeliveries(
-		s.WebhookDB, s.WebhookID,
+		context.Background(), s.WebhookDB, s.WebhookID,
 	)
 
 	iAssertStatus(
@@ -564,7 +565,7 @@ func TestRecoverSingleRetry_TargetUnreadable_LeavesDeliveryAlone(
 
 // --- 4. A pending delivery whose target is gone ---
 
-func TestRecoverPending_TargetDeleted(t *testing.T) {
+func TestRecoverPendingDeliveries_TargetDeleted(t *testing.T) {
 	t.Parallel()
 
 	s := newISetup(t)
@@ -577,7 +578,7 @@ func TestRecoverPending_TargetDeleted(t *testing.T) {
 	)
 
 	s.Engine.ExportRecoverWebhookDeliveries(
-		context.Background(), s.WebhookID,
+		context.Background(), s.WebhookID, atStart,
 	)
 
 	iAssertStatus(
@@ -600,10 +601,10 @@ func TestRecoverPending_TargetDeleted(t *testing.T) {
 	)
 }
 
-// TestRecoverPending_TargetDeleted_LeavesAnOwnedDeliveryAlone: the
-// terminal write takes ownership like every other recovery write, so a
-// delivery the engine still holds is not failed underneath its worker.
-func TestRecoverPending_TargetDeleted_LeavesAnOwnedDeliveryAlone(
+// TestRecoverPendingDeliveries_TargetDeleted_LeavesAnOwnedDeliveryAlone:
+// the terminal write takes ownership like every other recovery write, so
+// a delivery the engine still holds is not failed underneath its worker.
+func TestRecoverPendingDeliveries_TargetDeleted_LeavesAnOwnedDeliveryAlone(
 	t *testing.T,
 ) {
 	t.Parallel()
@@ -620,7 +621,7 @@ func TestRecoverPending_TargetDeleted_LeavesAnOwnedDeliveryAlone(
 	require.True(t, s.Engine.ExportRetainDelivery(deliveryID))
 
 	s.Engine.ExportRecoverWebhookDeliveries(
-		context.Background(), s.WebhookID,
+		context.Background(), s.WebhookID, atStart,
 	)
 
 	iAssertStatus(
@@ -675,18 +676,18 @@ func TestFailMissingTarget_LeavesASettledDeliveryAlone(
 	assert.Zero(t, s.Engine.ExportInflightHeld())
 }
 
-// TestSweepPending_TargetDeleted sweeps twice over a batch that also
-// holds a healthy stranded delivery. The one whose target is gone is
-// failed once and then left alone; the healthy one is queued by the
-// first sweep and not again by the second.
-func TestSweepPending_TargetDeleted(t *testing.T) {
+// TestRecoverPendingDeliveries_OnTick_TargetDeleted sweeps twice over a
+// batch that also holds a healthy stranded delivery. The one whose
+// target is gone is failed once and then left alone; the healthy one is
+// queued by the first sweep and not again by the second.
+func TestRecoverPendingDeliveries_OnTick_TargetDeleted(t *testing.T) {
 	t.Parallel()
 
 	liveTargetID := uuid.New().String()
 	s := fSweepSetup(t, liveTargetID, "still-there")
 
 	deliveryID := tSeedDeletedTarget(
-		t, s, "gone-on-pending-sweep", "http://example.com/hook",
+		t, s, "gone-while-pending-on-tick", "http://example.com/hook",
 		database.DeliveryStatusPending,
 	)
 	rAgePending(t, s.WebhookDB, deliveryID)
@@ -703,7 +704,7 @@ func TestSweepPending_TargetDeleted(t *testing.T) {
 
 	ctx := context.Background()
 
-	s.Engine.ExportSweepWebhookRetries(ctx, s.WebhookID)
+	s.Engine.ExportRecoverWebhookDeliveries(ctx, s.WebhookID, onTick)
 
 	tasks := fDrain(s.Engine)
 	require.Len(t, tasks, 1,
@@ -711,7 +712,7 @@ func TestSweepPending_TargetDeleted(t *testing.T) {
 	)
 	assert.Equal(t, healthy.ID, tasks[0].DeliveryID)
 
-	s.Engine.ExportSweepWebhookRetries(ctx, s.WebhookID)
+	s.Engine.ExportRecoverWebhookDeliveries(ctx, s.WebhookID, onTick)
 
 	assert.Empty(t, fDrain(s.Engine),
 		"the second sweep queued a delivery again",
@@ -724,7 +725,7 @@ func TestSweepPending_TargetDeleted(t *testing.T) {
 
 	last := tLastResult(t, s, deliveryID, 2)
 
-	assert.Contains(t, last.Error, "gone-on-pending-sweep")
+	assert.Contains(t, last.Error, "gone-while-pending-on-tick")
 	assert.Contains(t, last.Error, "was deleted")
 }
 
@@ -775,11 +776,11 @@ func TestSendRecoveredDeliveries_TargetMissingFromMap(
 	)
 }
 
-// TestRecoverPending_TargetUnreadable_LeavesDeliveryAlone: a failed
-// read of the main database is not a deleted target. Restart recovery
-// holds every pending delivery of the webhook in one batch, so failing
-// on this would fail all of them.
-func TestRecoverPending_TargetUnreadable_LeavesDeliveryAlone(
+// TestRecoverPendingDeliveries_TargetUnreadable_LeavesDeliveryAlone: a
+// failed read of the main database is not a deleted target. At start the
+// recovery routine holds every pending delivery of the webhook in one
+// batch, so failing on this would fail all of them.
+func TestRecoverPendingDeliveries_TargetUnreadable_LeavesDeliveryAlone(
 	t *testing.T,
 ) {
 	t.Parallel()
@@ -807,7 +808,7 @@ func TestRecoverPending_TargetUnreadable_LeavesDeliveryAlone(
 	require.NoError(t, sqlDB.Close())
 
 	s.Engine.ExportRecoverPendingDeliveries(
-		context.Background(), s.WebhookDB, s.WebhookID,
+		context.Background(), s.WebhookDB, s.WebhookID, atStart,
 	)
 
 	iAssertStatus(