Use one recovery routine at start and on every sweep tick (closes #542)

Start-up recovery and the 60-second sweep were two parallel sets of
functions doing the same job. recoverDeliveries now does it, run once at
start and then on every tick from one goroutine. A retrying delivery is
scheduled after what is left of its backoff, now if nothing is left, so
backoffElapsed is gone. The run at start still takes every pending
delivery and opens a webhook database whose file is missing; a tick
still takes only pending deliveries untouched for 15 minutes, at most
500, and skips a missing file. A retrying delivery is taken before its
event is read. The README describes one routine.

Model: opus-5-5

internal/delivery/engine_lifecycle_test.go

@@ -99,7 +99,7 @@ func requireStopHookExpires(
 // the caller can drive OnStop too.
 //
 // Callers must not seed pending or retrying deliveries before
-// calling this: restart recovery enqueues those during startup,
+// calling this: the recovery routine's run at start enqueues those,
 // which would put work in the queue while the pool is still
 // racing its first select.
 func startEngineViaHook(
@@ -178,8 +178,8 @@ func TestEngine_WorkersOutliveStartHookContext(t *testing.T) {
 		_ = lc.hooks[0].OnStop(context.Background())
 	})
 
-	// Seeded only after the pool has settled, so restart recovery
-	// cannot enqueue it during startup.
+	// Seeded only after the pool has settled, so the recovery
+	// routine's run at start cannot enqueue it.
 	deliveryID, task := seedLogTask(t, s)
 
 	s.Engine.Notify([]delivery.Task{task})
@@ -345,15 +345,15 @@ func stopDuringAttempt(
 	require.NoError(t, lc.hooks[0].OnStop(context.Background()))
 }
 
-// requireRequeued runs restart recovery on the stopped engine and
-// requires it to queue task's attempt again on ch.
+// requireRequeued runs the recovery routine as at start on the stopped
+// engine and requires it to queue task's attempt again on ch.
 func requireRequeued(
 	t *testing.T, s iSetup, ch <-chan delivery.Task, task delivery.Task,
 ) {
 	t.Helper()
 
 	s.Engine.ExportRecoverWebhookDeliveries(
-		context.Background(), s.WebhookID,
+		context.Background(), s.WebhookID, atStart,
 	)
 
 	select {
@@ -361,7 +361,7 @@ func requireRequeued(
 		assert.Equal(t, task.DeliveryID, got.DeliveryID)
 		assert.Equal(t, task.AttemptNum, got.AttemptNum)
 	case <-time.After(5 * time.Second):
-		t.Fatal("restart recovery did not queue the delivery again")
+		t.Fatal("recovery at start did not queue the delivery again")
 	}
 }
 
@@ -371,7 +371,8 @@ func requireRequeued(
 // no retries that was the delivery's only attempt, recorded as failed
 // with "context canceled", and nothing ever sent it again. The
 // delivery must instead stay pending, with no result and no count in
-// the attempt metrics, for restart recovery to send.
+// the attempt metrics, for the recovery routine to send at the next
+// start.
 func TestEngine_StopLeavesFireAndForgetAttemptPending(t *testing.T) {
 	t.Parallel()
 
@@ -399,8 +400,8 @@ func TestEngine_StopLeavesFireAndForgetAttemptPending(t *testing.T) {
 // TestEngine_StopLeavesLastRetryAttemptRetrying is the same regression
 // for a target with retries, on its last attempt: the cancelled attempt
 // used to exhaust the retries and end the delivery failed. It must stay
-// retrying with only its earlier attempt recorded, and restart recovery
-// must send that last attempt again.
+// retrying with only its earlier attempt recorded, and the recovery
+// routine at the next start must send that last attempt again.
 func TestEngine_StopLeavesLastRetryAttemptRetrying(t *testing.T) {
 	t.Parallel()