Deploy main eacf1e7 to prod

Replaces #559, which conflicted.

Model: opus-5-5

internal/delivery/engine_lifecycle_test.go

@@ -2,14 +2,19 @@ package delivery_test
 
 import (
 	"context"
-	"fmt"
+	"io"
+	"net/http"
+	"net/http/httptest"
 	"path/filepath"
+	"sync"
 	"testing"
 	"time"
 
 	"github.com/google/uuid"
+	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 	"go.uber.org/fx"
+	"gorm.io/gorm/clause"
 	"sneak.berlin/go/webhooker/internal/database"
 	"sneak.berlin/go/webhooker/internal/delivery"
 )
@@ -24,7 +29,7 @@ const (
 	// OnStart before the caller may enqueue work. A worker pool
 	// wrongly rooted in the already-done hook context has nothing
 	// but ctx.Done() ready in its select, so it is deterministically
-	// gone by the end of this window. Without the wait, Notify would
+	// gone by the end of this window. Without the wait, notify would
 	// race the pool's very first select, in which a ready ctx.Done()
 	// and a ready deliveryCh are chosen between at random and a
 	// doomed pool still delivers.
@@ -94,7 +99,7 @@ func requireStopHookExpires(
 // the caller can drive OnStop too.
 //
 // Callers must not seed pending or retrying deliveries before
-// calling this: restart recovery enqueues those during startup,
+// calling this: the recovery routine's run at start enqueues those,
 // which would put work in the queue while the pool is still
 // racing its first select.
 func startEngineViaHook(
@@ -144,7 +149,7 @@ func seedLogTask(
 	bodyStr := event.Body
 	task := iTask(
 		d, event, s.WebhookID, targetID,
-		"hook-context-test", "", 0, 1, &bodyStr,
+		"hook-context-test", "", 1, 1, &bodyStr,
 	)
 	task.TargetType = database.TargetTypeLog
 
@@ -173,11 +178,11 @@ func TestEngine_WorkersOutliveStartHookContext(t *testing.T) {
 		_ = lc.hooks[0].OnStop(context.Background())
 	})
 
-	// Seeded only after the pool has settled, so restart recovery
-	// cannot enqueue it during startup.
+	// Seeded only after the pool has settled, so the recovery
+	// routine's run at start cannot enqueue it.
 	deliveryID, task := seedLogTask(t, s)
 
-	s.Engine.Notify([]delivery.Task{task})
+	s.Engine.ExportNotify([]delivery.Task{task})
 
 	iWaitForDelivered(t, s.WebhookDB, deliveryID)
 }
@@ -197,7 +202,7 @@ func TestEngine_StopHookStopsWorkers(t *testing.T) {
 	// Let the pool prove it is running before stopping it, so a
 	// fast OnStop cannot pass by stopping something already dead.
 	firstID, firstTask := seedLogTask(t, s)
-	s.Engine.Notify([]delivery.Task{firstTask})
+	s.Engine.ExportNotify([]delivery.Task{firstTask})
 	iWaitForDelivered(t, s.WebhookDB, firstID)
 
 	var stopErr error
@@ -227,7 +232,7 @@ func TestEngine_StopHookStopsWorkers(t *testing.T) {
 	// With every worker gone, a freshly notified task must sit
 	// untouched in the queue rather than being delivered.
 	secondID, secondTask := seedLogTask(t, s)
-	s.Engine.Notify([]delivery.Task{secondTask})
+	s.Engine.ExportNotify([]delivery.Task{secondTask})
 
 	time.Sleep(200 * time.Millisecond)
 
@@ -272,22 +277,184 @@ func TestEngine_StopHookHonoursStopTimeout(t *testing.T) {
 	requireStopHookExpires(t, lc.hooks[0], "delivery engine")
 }
 
-// deliverToArchive runs one delivery to a database target through
-// the running engine and returns the webhook's archive file path.
-// The archive writer holds the file open afterwards.
+// seedHeldDelivery gives the setup's webhook an http target with
+// maxAttempts, pointed at a receiver that keeps every request open until
+// the client gives up on it, and seeds one delivery to that target at
+// status. It returns the task that sends attempt attemptNum of the
+// delivery, and a channel that is closed when a request reaches the
+// receiver.
+func seedHeldDelivery(
+	t *testing.T,
+	s iSetup,
+	maxAttempts int,
+	status database.DeliveryStatus,
+	attemptNum int,
+) (delivery.Task, <-chan struct{}) {
+	t.Helper()
+
+	arrived := make(chan struct{})
+
+	var once sync.Once
+
+	ts := httptest.NewServer(http.HandlerFunc(
+		func(_ http.ResponseWriter, r *http.Request) {
+			// The server notices the client hanging up only once the
+			// body has been read; until then r.Context() never ends.
+			_, _ = io.Copy(io.Discard, r.Body)
+
+			once.Do(func() { close(arrived) })
+
+			<-r.Context().Done()
+		},
+	))
+	t.Cleanup(ts.Close)
+
+	targetID := uuid.New().String()
+
+	iCreateWebhook(t, s.MainDB, s.WebhookID, "hook")
+	iCreateTarget(
+		t, s.MainDB, targetID, s.WebhookID, "held",
+		database.TargetTypeHTTP, ts.URL, maxAttempts,
+	)
+
+	event := iSeedEvent(t, s.WebhookDB, s.WebhookID, `{"held":true}`)
+	d := iSeedDelivery(t, s.WebhookDB, event.ID, targetID, status)
+
+	bodyStr := event.Body
+
+	return iTask(
+		d, event, s.WebhookID, targetID, "held", ts.URL,
+		maxAttempts, attemptNum, &bodyStr,
+	), arrived
+}
+
+// stopDuringAttempt waits until the receiver holds an attempt open, then
+// stops the engine through its real stop hook.
+func stopDuringAttempt(
+	t *testing.T, lc *recordingLifecycle, arrived <-chan struct{},
+) {
+	t.Helper()
+
+	select {
+	case <-arrived:
+	case <-time.After(5 * time.Second):
+		t.Fatal("the delivery attempt never reached the receiver")
+	}
+
+	require.NoError(t, lc.hooks[0].OnStop(context.Background()))
+}
+
+// requireRequeued runs the recovery routine as at start on the stopped
+// engine and requires it to queue task's attempt again on ch.
+func requireRequeued(
+	t *testing.T, s iSetup, ch <-chan delivery.Task, task delivery.Task,
+) {
+	t.Helper()
+
+	s.Engine.ExportRecoverWebhookDeliveries(
+		context.Background(), s.WebhookID, atStart,
+	)
+
+	select {
+	case got := <-ch:
+		assert.Equal(t, task.DeliveryID, got.DeliveryID)
+		assert.Equal(t, task.AttemptNum, got.AttemptNum)
+	case <-time.After(5 * time.Second):
+		t.Fatal("recovery at start did not queue the delivery again")
+	}
+}
+
+// TestEngine_StopLeavesFireAndForgetAttemptPending is the regression
+// test for a stop that turned an attempt in flight into a terminal
+// failure. Stopping the engine cancels the request; for a target with
+// no retries that was the delivery's only attempt, recorded as failed
+// with "context canceled", and nothing ever sent it again. The
+// delivery must instead stay pending, with no result and no count in
+// the attempt metrics, for the recovery routine to send at the next
+// start.
+func TestEngine_StopLeavesFireAndForgetAttemptPending(t *testing.T) {
+	t.Parallel()
+
+	s := newISetup(t)
+	reg := mIsolate(t, s)
+	lc := startEngineViaHook(t, s.Engine)
+
+	task, arrived := seedHeldDelivery(
+		t, s, 1, database.DeliveryStatusPending, 1,
+	)
+
+	s.Engine.ExportNotify([]delivery.Task{task})
+	stopDuringAttempt(t, lc, arrived)
+
+	iAssertStatus(
+		t, s.WebhookDB, task.DeliveryID, database.DeliveryStatusPending,
+	)
+	assert.Empty(t, iResults(t, s.WebhookDB, task.DeliveryID))
+	assert.Zero(t, mCounter(t, reg, mAttempts, mTypeHTTP))
+	assert.Zero(t, mHTTPDurations(t, reg))
+
+	requireRequeued(t, s, s.Engine.ExportDeliveryCh(), task)
+}
+
+// TestEngine_StopLeavesLastRetryAttemptRetrying is the same regression
+// for a target with retries, on its last attempt: the cancelled attempt
+// used to exhaust the retries and end the delivery failed. It must stay
+// retrying with only its earlier attempt recorded, and the recovery
+// routine at the next start must send that last attempt again.
+func TestEngine_StopLeavesLastRetryAttemptRetrying(t *testing.T) {
+	t.Parallel()
+
+	const maxAttempts = 2
+
+	s := newISetup(t)
+	lc := startEngineViaHook(t, s.Engine)
+
+	task, arrived := seedHeldDelivery(
+		t, s, maxAttempts, database.DeliveryStatusRetrying, maxAttempts,
+	)
+	iSeedFailedResult(t, s.WebhookDB, task.DeliveryID)
+
+	s.Engine.ExportScheduleRetry(task, 0)
+	stopDuringAttempt(t, lc, arrived)
+
+	iAssertStatus(
+		t, s.WebhookDB, task.DeliveryID, database.DeliveryStatusRetrying,
+	)
+	assert.Len(t, iResults(t, s.WebhookDB, task.DeliveryID), 1)
+
+	requireRequeued(t, s, s.Engine.ExportRetryCh(), task)
+}
+
+// deliverToArchive gives the setup's webhook a database target,
+// runs one delivery to it through the running engine, and returns
+// the target's archive file path. The archive writer holds the file
+// open afterwards.
 func deliverToArchive(t *testing.T, s iSetup) string {
 	t.Helper()
 
+	iCreateWebhook(t, s.MainDB, s.WebhookID, "hook")
+
+	tgt := &database.Target{
+		WebhookID:   s.WebhookID,
+		Name:        "archive",
+		Type:        database.TargetTypeDatabase,
+		MaxAttempts: 1,
+	}
+	require.NoError(
+		t, s.MainDB.Omit(clause.Associations).Create(tgt).Error,
+	)
+
 	deliveryID, task := seedLogTask(t, s)
+	task.TargetID = tgt.ID
 	task.TargetType = database.TargetTypeDatabase
 
-	s.Engine.Notify([]delivery.Task{task})
+	s.Engine.ExportNotify([]delivery.Task{task})
 
 	iWaitForDelivered(t, s.WebhookDB, deliveryID)
 
 	return filepath.Join(
 		filepath.Dir(s.DBMgr.DBPath(s.WebhookID)),
-		fmt.Sprintf("archive-%s.db", s.WebhookID),
+		"archive-hook-archive-"+tgt.ID+".db",
 	)
 }
 
@@ -338,21 +505,23 @@ func TestEngine_StopHookTimeoutLeavesArchivesOpen(t *testing.T) {
 
 	lc := startEngineViaHook(t, s.Engine)
 
-	deliverToArchive(t, s)
+	path := deliverToArchive(t, s)
 
 	release := make(chan struct{})
 
 	t.Cleanup(func() {
 		close(release)
-		s.Engine.EvictWebhook(s.WebhookID)
+		s.Engine.ExportArchives().EvictWebhook(s.WebhookID)
 	})
 
 	s.Engine.ExportWedgeWorker(release)
 
 	requireStopHookExpires(t, lc.hooks[0], "delivery engine")
 
-	require.True(
-		t, s.Engine.ExportArchiveHandleOpen(s.WebhookID),
+	// Closing the last handle on an archive removes its -wal, so a
+	// -wal still there is a writer still open.
+	require.FileExists(
+		t, path+"-wal",
 		"a stop that timed out must not close archive writers",
 	)
 }