Leave a delivery cut off by a stop for the next start (closes #527)

Stopping the engine cancels the context every http and slack request is
sent with. An attempt cut off that way was recorded as failed, which for
a target with no retries, or on a last retry, ended the delivery failed
and nothing sent it again.

The shared retry core now records nothing for a failed attempt when the
engine's context is cancelled: no result, no circuit breaker outcome, no
status change. The delivery stays pending or retrying and startup
recovery sends it again. A target's own timeout is still a failed
attempt. The README's Graceful shutdown bullet, Shutdown section and the
at-least-once note say so.

Model: opus-5-5

internal/delivery/target_http.go

@@ -57,7 +57,11 @@ type httpCore struct {
 // recorded and no circuit breaker is consulted. A positive
 // maxRetries gates the attempt on the circuit breaker and
 // schedules a backed-off retry on failure.
+//
+// ctx is the context attempt sends its request with. Stopping the
+// engine cancels it; see interruptedByStop.
 func (c *httpCore) deliver(
+	ctx context.Context,
 	webhookDB *gorm.DB,
 	d *database.Delivery,
 	task *Task,
@@ -66,23 +70,57 @@ func (c *httpCore) deliver(
 	attempt func() attemptResult,
 ) {
 	if maxRetries == 0 {
-		c.fireAndForget(webhookDB, d, attempt())
+		c.fireAndForget(ctx, webhookDB, d, attempt())
 
 		return
 	}
 
 	c.withRetry(
-		webhookDB, d, task, sched, maxRetries, attempt,
+		ctx, webhookDB, d, task, sched, maxRetries, attempt,
 	)
 }
 
+// interruptedByStop reports whether an attempt failed because the
+// engine is stopping, which cancels the context its request was sent
+// with. Such an attempt is not recorded: no DeliveryResult, no circuit
+// breaker outcome, no status change. The delivery stays pending or
+// retrying, and startup recovery sends it again on the next start.
+// The request did go out, so the attempt metrics still count it.
+//
+// A target's own timeout leaves that context alone, so its attempt is
+// still recorded as the failure it is.
+func (c *httpCore) interruptedByStop(
+	ctx context.Context,
+	d *database.Delivery,
+	res attemptResult,
+) bool {
+	if res.success || ctx.Err() == nil {
+		return false
+	}
+
+	c.eng.log.Warn(
+		"delivery attempt interrupted by shutdown; not recorded, "+
+			"it is sent again on the next start",
+		"delivery_id", d.ID,
+		"event_id", d.EventID,
+		"target_id", d.TargetID,
+	)
+
+	return true
+}
+
 func (c *httpCore) fireAndForget(
+	ctx context.Context,
 	webhookDB *gorm.DB,
 	d *database.Delivery,
 	res attemptResult,
 ) {
 	c.eng.observeAttempt(d.Target.Type, res.elapsed())
 
+	if c.interruptedByStop(ctx, d, res) {
+		return
+	}
+
 	err := c.eng.recordResult(
 		webhookDB, d, 1, res.success,
 		res.statusCode, res.respBody, res.errMsg,
@@ -110,6 +148,7 @@ func (c *httpCore) fireAndForget(
 }
 
 func (c *httpCore) withRetry(
+	ctx context.Context,
 	webhookDB *gorm.DB,
 	d *database.Delivery,
 	task *Task,
@@ -133,6 +172,10 @@ func (c *httpCore) withRetry(
 
 	c.eng.observeAttempt(d.Target.Type, res.elapsed())
 
+	if c.interruptedByStop(ctx, d, res) {
+		return
+	}
+
 	err := c.eng.recordResult(
 		webhookDB, d, attemptNum, res.success,
 		res.statusCode, res.respBody, res.errMsg,
@@ -395,7 +438,7 @@ func (t *httpTarget) Deliver(
 	}
 
 	t.deliver(
-		webhookDB, d, task, sched,
+		ctx, webhookDB, d, task, sched,
 		d.Target.MaxRetries, attempt,
 	)
 }