Close the two remaining delivery terminal-state gaps (closes #107)

2026-08-24 05:12:02 +02:00
parent 322d9a6d6b
commit af3703d748
5 changed files with 789 additions and 10 deletions
--- a/internal/delivery/terminal_state_test.go
+++ b/internal/delivery/terminal_state_test.go
@@ -0,0 +1,531 @@
+package delivery_test
+
+import (
+	"context"
+	"net/http"
+	"net/http/httptest"
+	"sync/atomic"
+	"testing"
+
+	"github.com/google/uuid"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"sneak.berlin/go/webhooker/internal/database"
+	"sneak.berlin/go/webhooker/internal/delivery"
+)
+
+// The two terminal-state gaps of
+// https://git.eeqj.de/sneak/webhooker/issues/107: a delivery failed
+// with nothing in its event log to say why, and a retrying delivery
+// whose target was deleted, which used to keep sending and then never
+// terminalise.
+
+// tUnknownType is a target type no build implements. It stands in for
+// a target whose type was written by a build that knew a type this one
+// does not.
+const tUnknownType = database.TargetType("pubsub")
+
+// tSeedDeletedTarget creates a target, a retrying delivery against it
+// with one recorded failed attempt, and then deletes the target the
+// way the source page does.
+//
+// It asserts the delete is soft, because that is the whole reason the
+// engine could not tell a deleted target from a target id that never
+// named a row: the surviving row is invisible to a scoped read.
+func tSeedDeletedTarget(
+	t *testing.T,
+	s iSetup,
+	name, url string,
+) string {
+	t.Helper()
+
+	targetID := uuid.New().String()
+
+	iCreateTarget(
+		t, s.MainDB, targetID, s.WebhookID, name,
+		database.TargetTypeHTTP, iHTTPConfig(url), 5,
+	)
+
+	event := iSeedEvent(
+		t, s.WebhookDB, s.WebhookID, `{"target":"deleted"}`,
+	)
+
+	d := iSeedDelivery(
+		t, s.WebhookDB, event.ID, targetID,
+		database.DeliveryStatusRetrying,
+	)
+
+	iSeedFailedResult(t, s.WebhookDB, d.ID)
+
+	require.NoError(t, s.MainDB.Delete(
+		&database.Target{}, "id = ?", targetID,
+	).Error)
+
+	var scoped, unscoped int64
+
+	require.NoError(t, s.MainDB.
+		Model(&database.Target{}).
+		Where("id = ?", targetID).
+		Count(&scoped).Error)
+
+	require.NoError(t, s.MainDB.Unscoped().
+		Model(&database.Target{}).
+		Where("id = ?", targetID).
+		Count(&unscoped).Error)
+
+	require.Zero(t, scoped,
+		"the deleted target is still visible to a scoped read",
+	)
+	require.Equal(t, int64(1), unscoped,
+		"the delete was hard, so this test proves nothing about "+
+			"the soft-delete case it exists for",
+	)
+
+	return d.ID
+}
+
+// tLastResult returns a delivery's final recorded attempt, asserting
+// the expected number of them.
+func tLastResult(
+	t *testing.T,
+	s iSetup,
+	deliveryID string,
+	want int,
+) database.DeliveryResult {
+	t.Helper()
+
+	results := iResults(t, s.WebhookDB, deliveryID)
+	require.Len(t, results, want)
+
+	return results[want-1]
+}
+
+// --- 1. A failure with nothing recorded ---
+
+func TestProcessDelivery_UnknownTargetType_RecordsWhy(
+	t *testing.T,
+) {
+	t.Parallel()
+
+	s := newISetup(t)
+
+	targetID := uuid.New().String()
+
+	event := iSeedEvent(
+		t, s.WebhookDB, s.WebhookID, `{"unknown":"type"}`,
+	)
+
+	seeded := iSeedDelivery(
+		t, s.WebhookDB, event.ID, targetID,
+		database.DeliveryStatusPending,
+	)
+
+	target := database.Target{
+		Name:   "mystery",
+		Type:   tUnknownType,
+		Config: iHTTPConfig("http://example.com/hook"),
+	}
+	target.ID = targetID
+
+	d := database.Delivery{
+		EventID:  event.ID,
+		TargetID: targetID,
+		Status:   database.DeliveryStatusPending,
+		Event:    event,
+		Target:   target,
+	}
+	d.ID = seeded.ID
+
+	body := event.Body
+	task := iTask(
+		seeded, event, s.WebhookID, targetID, "mystery",
+		target.Config, 0, 1, &body,
+	)
+	task.TargetType = tUnknownType
+
+	s.Engine.ExportProcessDelivery(
+		context.Background(), s.WebhookDB, &d, &task,
+	)
+
+	iAssertStatus(
+		t, s.WebhookDB, d.ID, database.DeliveryStatusFailed,
+	)
+
+	last := tLastResult(t, s, d.ID, 1)
+
+	assert.False(t, last.Success)
+	assert.Equal(t, 1, last.AttemptNum)
+	assert.Contains(t, last.Error, string(tUnknownType),
+		"the recorded reason does not name the offending type",
+	)
+}
+
+// --- 2. A retrying delivery whose target is gone ---
+
+func TestRecoverSingleRetry_TargetDeleted(t *testing.T) {
+	t.Parallel()
+
+	s := newISetup(t)
+
+	iCreateWebhook(
+		t, s.MainDB, s.WebhookID, "deleted-target-recovery",
+	)
+
+	deliveryID := tSeedDeletedTarget(
+		t, s, "gone-on-recovery", "http://example.com/hook",
+	)
+
+	s.Engine.ExportRecoverWebhookDeliveries(
+		context.Background(), s.WebhookID,
+	)
+
+	iAssertStatus(
+		t, s.WebhookDB, deliveryID,
+		database.DeliveryStatusFailed,
+	)
+
+	last := tLastResult(t, s, deliveryID, 2)
+
+	assert.False(t, last.Success)
+	assert.Equal(t, 2, last.AttemptNum)
+	assert.Contains(t, last.Error, "gone-on-recovery")
+	assert.Contains(t, last.Error, "was deleted")
+
+	assert.Empty(t, s.Engine.ExportRetryCh(),
+		"a delivery whose target is gone was rescheduled",
+	)
+	assert.Zero(t, s.Engine.ExportInflightHeld(),
+		"the terminal path leaked its ownership reference",
+	)
+}
+
+func TestSweepSingleRetry_TargetDeleted(t *testing.T) {
+	t.Parallel()
+
+	s := newISetup(t)
+
+	iCreateWebhook(
+		t, s.MainDB, s.WebhookID, "deleted-target-sweep",
+	)
+
+	deliveryID := tSeedDeletedTarget(
+		t, s, "gone-on-sweep", "http://example.com/hook",
+	)
+
+	// Twice, because the bug was an error the sweep repeated every
+	// minute for the life of the database: the second sweep must
+	// find nothing left to do.
+	s.Engine.ExportSweepWebhookRetries(
+		context.Background(), s.WebhookID,
+	)
+	s.Engine.ExportSweepWebhookRetries(
+		context.Background(), s.WebhookID,
+	)
+
+	iAssertStatus(
+		t, s.WebhookDB, deliveryID,
+		database.DeliveryStatusFailed,
+	)
+
+	last := tLastResult(t, s, deliveryID, 2)
+
+	assert.Contains(t, last.Error, "gone-on-sweep")
+	assert.Contains(t, last.Error, "was deleted")
+
+	assert.Empty(t, s.Engine.ExportRetryCh())
+	assert.Zero(t, s.Engine.ExportInflightHeld())
+}
+
+// TestSweepSingleRetry_TargetNeverExisted covers the other half of the
+// soft-delete distinction: an id with no row at all, deleted or
+// otherwise, must not be reported as something the operator deleted.
+func TestSweepSingleRetry_TargetNeverExisted(t *testing.T) {
+	t.Parallel()
+
+	s := newISetup(t)
+
+	iCreateWebhook(
+		t, s.MainDB, s.WebhookID, "target-never-existed",
+	)
+
+	targetID := uuid.New().String()
+
+	event := iSeedEvent(
+		t, s.WebhookDB, s.WebhookID, `{"target":"absent"}`,
+	)
+
+	d := iSeedDelivery(
+		t, s.WebhookDB, event.ID, targetID,
+		database.DeliveryStatusRetrying,
+	)
+
+	iSeedFailedResult(t, s.WebhookDB, d.ID)
+
+	s.Engine.ExportSweepWebhookRetries(
+		context.Background(), s.WebhookID,
+	)
+
+	iAssertStatus(
+		t, s.WebhookDB, d.ID, database.DeliveryStatusFailed,
+	)
+
+	last := tLastResult(t, s, d.ID, 2)
+
+	assert.Contains(t, last.Error, targetID)
+	assert.Contains(t, last.Error, "no longer exists")
+	assert.NotContains(t, last.Error, "was deleted",
+		"an id that never named a row was reported as a deletion",
+	)
+}
+
+// TestFailMissingTargetRetry_WritesNoTargetRow holds the new terminal
+// path to the same rule as the existing one: no target row, and so no
+// plaintext target config, may be written into the per-webhook event
+// database. See https://git.eeqj.de/sneak/webhooker/issues/206.
+func TestFailMissingTargetRetry_WritesNoTargetRow(
+	t *testing.T,
+) {
+	t.Parallel()
+
+	s := newISetup(t)
+
+	iCreateWebhook(
+		t, s.MainDB, s.WebhookID, "no-target-row-deleted",
+	)
+
+	hookURL := "https://hooks.slack.com/services/T00/B00/x"
+
+	deliveryID := tSeedDeletedTarget(
+		t, s, "credential-bearing", hookURL,
+	)
+
+	s.Engine.ExportSweepWebhookRetries(
+		context.Background(), s.WebhookID,
+	)
+
+	iAssertStatus(
+		t, s.WebhookDB, deliveryID,
+		database.DeliveryStatusFailed,
+	)
+
+	var configs []string
+
+	require.NoError(t, s.WebhookDB.
+		Table("targets").
+		Pluck("config", &configs).Error)
+
+	assert.Empty(t, configs,
+		"the deleted-target terminal path wrote a target row "+
+			"into the per-webhook event database",
+	)
+}
+
+// --- 3. The scheduled retry chain ---
+
+// tRetryChainSetup wires a counting sink and a retrying delivery
+// against a live target pointing at it, and returns the task a
+// scheduled retry would carry — config and all, snapshotted as
+// ScheduleRetry snapshots it.
+func tRetryChainSetup(
+	t *testing.T,
+	s iSetup,
+	name string,
+	hits *atomic.Int64,
+) (delivery.Task, string) {
+	t.Helper()
+
+	ts := httptest.NewServer(http.HandlerFunc(
+		func(w http.ResponseWriter, _ *http.Request) {
+			hits.Add(1)
+			w.WriteHeader(http.StatusOK)
+		},
+	))
+	t.Cleanup(ts.Close)
+
+	iCreateWebhook(t, s.MainDB, s.WebhookID, name)
+
+	targetID := uuid.New().String()
+	cfg := iHTTPConfig(ts.URL)
+
+	iCreateTarget(
+		t, s.MainDB, targetID, s.WebhookID, name,
+		database.TargetTypeHTTP, cfg, 5,
+	)
+
+	event := iSeedEvent(
+		t, s.WebhookDB, s.WebhookID, `{"chain":"retry"}`,
+	)
+
+	d := iSeedDelivery(
+		t, s.WebhookDB, event.ID, targetID,
+		database.DeliveryStatusRetrying,
+	)
+
+	iSeedFailedResult(t, s.WebhookDB, d.ID)
+
+	body := event.Body
+
+	return iTask(
+		d, event, s.WebhookID, targetID, name, cfg, 5, 2, &body,
+	), targetID
+}
+
+// TestProcessRetryTask_TargetDeleted_MakesNoAttempt is the half the
+// deployability audit found worse than filed: terminalising on
+// recovery and sweep alone leaves the already-scheduled timer chain
+// running, and it holds the target's configuration from before the
+// deletion, so it goes on sending to a destination that was removed.
+func TestProcessRetryTask_TargetDeleted_MakesNoAttempt(
+	t *testing.T,
+) {
+	t.Parallel()
+
+	s := newISetup(t)
+
+	var hits atomic.Int64
+
+	task, targetID := tRetryChainSetup(
+		t, s, "gone-mid-chain", &hits,
+	)
+
+	require.NoError(t, s.MainDB.Delete(
+		&database.Target{}, "id = ?", targetID,
+	).Error)
+
+	s.Engine.ExportProcessRetryTask(
+		context.Background(), &task,
+	)
+
+	assert.Zero(t, hits.Load(),
+		"a scheduled retry fired at a target the operator "+
+			"had already deleted",
+	)
+
+	iAssertStatus(
+		t, s.WebhookDB, task.DeliveryID,
+		database.DeliveryStatusFailed,
+	)
+
+	last := tLastResult(t, s, task.DeliveryID, 2)
+
+	assert.False(t, last.Success)
+	assert.Contains(t, last.Error, "was deleted")
+
+	assert.Zero(t, s.Engine.ExportInflightHeld())
+}
+
+// TestProcessRetryTask_TargetPresent_StillDelivers is the guard's
+// mutation check: a liveness check that refused every retry would pass
+// the test above and break every retry there is.
+func TestProcessRetryTask_TargetPresent_StillDelivers(
+	t *testing.T,
+) {
+	t.Parallel()
+
+	s := newISetup(t)
+
+	var hits atomic.Int64
+
+	task, _ := tRetryChainSetup(t, s, "still-there", &hits)
+
+	s.Engine.ExportProcessRetryTask(
+		context.Background(), &task,
+	)
+
+	assert.Equal(t, int64(1), hits.Load())
+
+	iAssertStatus(
+		t, s.WebhookDB, task.DeliveryID,
+		database.DeliveryStatusDelivered,
+	)
+}
+
+// TestProcessRetryTask_TargetUnreadable_StillDelivers pins the other
+// half of the guard: only a target that is confirmed gone stops a
+// retry. A main database that cannot be read is a transient fault, and
+// a guard that abandoned deliveries on one would be a worse bug than
+// the one it fixes.
+func TestProcessRetryTask_TargetUnreadable_StillDelivers(
+	t *testing.T,
+) {
+	t.Parallel()
+
+	s := newISetup(t)
+
+	var hits atomic.Int64
+
+	task, _ := tRetryChainSetup(t, s, "unreadable-main", &hits)
+
+	sqlDB, err := s.MainDB.DB()
+	require.NoError(t, err)
+	require.NoError(t, sqlDB.Close())
+
+	s.Engine.ExportProcessRetryTask(
+		context.Background(), &task,
+	)
+
+	assert.Equal(t, int64(1), hits.Load(),
+		"a retry was abandoned because the main database "+
+			"could not be read, not because its target was gone",
+	)
+
+	iAssertStatus(
+		t, s.WebhookDB, task.DeliveryID,
+		database.DeliveryStatusDelivered,
+	)
+}
+
+// TestRecoverSingleRetry_TargetUnreadable_LeavesDeliveryAlone is the
+// same rule on the recovery path. A read failure that is not
+// "record not found" must leave every retrying delivery of every
+// webhook exactly as it was.
+func TestRecoverSingleRetry_TargetUnreadable_LeavesDeliveryAlone(
+	t *testing.T,
+) {
+	t.Parallel()
+
+	s := newISetup(t)
+
+	iCreateWebhook(
+		t, s.MainDB, s.WebhookID, "unreadable-on-recovery",
+	)
+
+	targetID := uuid.New().String()
+
+	iCreateTarget(
+		t, s.MainDB, targetID, s.WebhookID, "healthy",
+		database.TargetTypeHTTP,
+		iHTTPConfig("http://example.com/hook"), 5,
+	)
+
+	event := iSeedEvent(
+		t, s.WebhookDB, s.WebhookID, `{"still":"retrying"}`,
+	)
+
+	d := iSeedDelivery(
+		t, s.WebhookDB, event.ID, targetID,
+		database.DeliveryStatusRetrying,
+	)
+
+	iSeedFailedResult(t, s.WebhookDB, d.ID)
+
+	sqlDB, err := s.MainDB.DB()
+	require.NoError(t, err)
+	require.NoError(t, sqlDB.Close())
+
+	s.Engine.ExportRecoverRetryingDeliveries(
+		s.WebhookDB, s.WebhookID,
+	)
+
+	iAssertStatus(
+		t, s.WebhookDB, d.ID,
+		database.DeliveryStatusRetrying,
+	)
+
+	assert.Len(t, iResults(t, s.WebhookDB, d.ID), 1,
+		"an unreadable main database produced a terminal "+
+			"failure row",
+	)
+
+	assert.Zero(t, s.Engine.ExportInflightHeld())
+}