Deploy main eacf1e7 to prod

Replaces #559, which conflicted.

Model: opus-5-5

internal/delivery/target_database_test.go

@@ -1,6 +1,7 @@
 package delivery_test
 
 import (
+	"context"
 	"database/sql"
 	"fmt"
 	"log/slog"
@@ -15,11 +16,54 @@ import (
 	"github.com/stretchr/testify/require"
 	"gorm.io/driver/sqlite"
 	"gorm.io/gorm"
+	"gorm.io/gorm/clause"
 	_ "modernc.org/sqlite" // Pure Go SQLite driver.
+	"sneak.berlin/go/webhooker/internal/archive"
 	"sneak.berlin/go/webhooker/internal/database"
+	"sneak.berlin/go/webhooker/internal/database/databasetest"
+	"sneak.berlin/go/webhooker/internal/db/migrations"
 	"sneak.berlin/go/webhooker/internal/delivery"
+	"sneak.berlin/go/webhooker/internal/gormlog"
 )
 
+// archiveTestWebhookName is the name of every webhook
+// seedDatabaseTarget creates. It is not safe in a file name as it
+// stands, so every archive file name made from it is made safe first.
+const archiveTestWebhookName = "Sweep Test!"
+
+// archiveEnv bundles the pieces a database target test drives: a main
+// configuration database holding webhooks and targets, a delivery
+// engine whose archive writers write into the data directory, and that
+// directory.
+type archiveEnv struct {
+	eng     *delivery.Engine
+	mainDB  *database.Database
+	dataDir string
+}
+
+func setupArchiveTest(t *testing.T) *archiveEnv {
+	t.Helper()
+
+	dataDir := t.TempDir()
+
+	mainDB, err := database.Open(dataDir, slog.New(slog.DiscardHandler))
+	require.NoError(t, err)
+
+	t.Cleanup(func() { _ = mainDB.Close() })
+
+	return &archiveEnv{
+		eng: delivery.NewTestEngineWithDB(
+			mainDB,
+			databasetest.NewWebhookDBManager(t, dataDir),
+			archiveTestLogger(),
+			&http.Client{Timeout: 5 * time.Second},
+			1,
+		),
+		mainDB:  mainDB,
+		dataDir: dataDir,
+	}
+}
+
 func archiveTestLogger() *slog.Logger {
 	return slog.New(slog.NewTextHandler(
 		os.Stderr,
@@ -27,8 +71,87 @@ func archiveTestLogger() *slog.Logger {
 	))
 }
 
+// archivePath returns where the engine keeps a database target's
+// archive file, for the names seedDatabaseTarget gave it.
+func (env *archiveEnv) archivePath(tgt *database.Target) string {
+	return filepath.Join(
+		env.dataDir,
+		archive.FileName(archiveTestWebhookName, tgt.Name, tgt.ID),
+	)
+}
+
+// seedDatabaseTarget creates a webhook with one database target,
+// which keeps its archive forever in one file, and returns the
+// target.
+func (env *archiveEnv) seedDatabaseTarget(
+	t *testing.T,
+) *database.Target {
+	t.Helper()
+
+	wh := &database.Webhook{
+		UserID: uuid.New().String(),
+		Name:   archiveTestWebhookName,
+	}
+	require.NoError(
+		t,
+		env.mainDB.DB().
+			Omit(clause.Associations).
+			Create(wh).Error,
+	)
+
+	tgt := &database.Target{
+		WebhookID:       wh.ID,
+		Name:            "Archive",
+		Type:            database.TargetTypeDatabase,
+		Active:          true,
+		ArchiveRotation: "none",
+		MaxAttempts:     1,
+	}
+	require.NoError(
+		t,
+		env.mainDB.DB().
+			Omit(clause.Associations).
+			Create(tgt).Error,
+	)
+
+	return tgt
+}
+
+// seedDatabaseTargetDelivery seeds a pending delivery of an event
+// to a database target and returns the in-memory delivery the
+// target handler is invoked with.
+func seedDatabaseTargetDelivery(
+	t *testing.T,
+	webhookDB *gorm.DB,
+	event database.Event,
+	tgt *database.Target,
+) *database.Delivery {
+	t.Helper()
+
+	dlv := seedDelivery(
+		t, webhookDB, event.ID, tgt.ID,
+		database.DeliveryStatusPending,
+	)
+
+	d := &database.Delivery{
+		EventID:  event.ID,
+		TargetID: tgt.ID,
+		Status:   database.DeliveryStatusPending,
+		Event:    event,
+		Target: database.Target{
+			Name:            tgt.Name,
+			Type:            database.TargetTypeDatabase,
+			ArchiveExpiry:   tgt.ArchiveExpiry,
+			ArchiveRotation: tgt.ArchiveRotation,
+		},
+	}
+	d.ID = dlv.ID
+
+	return d
+}
+
 // openArchiveDBForRead opens an archive file read-only so a
-// test can inspect the rows the writer persisted.
+// test can inspect the rows the database target archived.
 func openArchiveDBForRead(
 	t *testing.T, path string,
 ) *gorm.DB {
@@ -43,56 +166,70 @@ func openArchiveDBForRead(
 	t.Cleanup(func() { _ = sqlDB.Close() })
 
 	gdb, err := gorm.Open(
-		sqlite.Dialector{Conn: sqlDB}, &gorm.Config{},
+		sqlite.Dialector{Conn: sqlDB},
+		&gorm.Config{Logger: gormlog.New(slog.New(slog.DiscardHandler))},
 	)
 	require.NoError(t, err)
 
 	return gdb
 }
 
-// archiveFileSuffixes returns the archive file itself and the
-// SQLite sidecars that accompany an open database. A test that
-// asserts no archive was created has to check all of them.
-func archiveFileSuffixes() []string {
-	return []string{"", "-wal", "-shm"}
+// countArchivedRows counts the rows in an archive file without
+// asserting anything, so it is safe to poll from an
+// assert.Eventually condition (which runs off the test
+// goroutine, where testify assertions must not be used).
+func countArchivedRows(path string) (int64, error) {
+	sqlDB, err := sql.Open(
+		"sqlite", fmt.Sprintf("file:%s?mode=ro", path),
+	)
+	if err != nil {
+		return 0, err
+	}
+
+	defer func() { _ = sqlDB.Close() }()
+
+	gdb, err := gorm.Open(
+		sqlite.Dialector{Conn: sqlDB},
+		&gorm.Config{Logger: gormlog.New(slog.New(slog.DiscardHandler))},
+	)
+	if err != nil {
+		return 0, err
+	}
+
+	var count int64
+
+	err = gdb.Table("archived_events").Count(&count).Error
+	if err != nil {
+		return 0, err
+	}
+
+	return count, nil
 }
 
-// removeArchiveFiles simulates an operator moving the archive
-// away by deleting the SQLite file and its sidecar files.
-func removeArchiveFiles(t *testing.T, path string) {
-	t.Helper()
-
-	for _, suffix := range []string{
-		"", "-wal", "-shm", "-journal",
-	} {
-		err := os.Remove(path + suffix)
-		if err != nil && !os.IsNotExist(err) {
-			t.Fatalf("removing %s%s: %v", path, suffix, err)
-		}
-	}
+// archivedRow is the columns of an archive row these tests read.
+type archivedRow struct {
+	EventID   string
+	WebhookID string
+	Method    string
+	RawQuery  string
+	Body      string
 }
 
 // TestDeliverDatabase_ArchivesEvent verifies that delivering to
 // a database target marks the delivery delivered and archives
-// the full event into a separate per-webhook archive file.
+// the full event into the target's own archive file.
 func TestDeliverDatabase_ArchivesEvent(t *testing.T) {
 	t.Parallel()
 
-	dataDir := t.TempDir()
-	dbMgr := database.NewTestWebhookDBManager(dataDir)
-
-	e := delivery.NewTestEngineWithDB(
-		nil, dbMgr,
-		archiveTestLogger(),
-		&http.Client{Timeout: 5 * time.Second},
-		1,
-	)
+	env := setupArchiveTest(t)
+	tgt := env.seedDatabaseTarget(t)
 
 	webhookDB := testWebhookDB(t)
 	event := seedEvent(t, webhookDB, `{"archived":true}`)
-	d := seedDatabaseTargetDelivery(t, webhookDB, event, "")
+	event.RawQuery = eventQuery
+	d := seedDatabaseTargetDelivery(t, webhookDB, event, tgt)
 
-	e.ExportDeliverDatabase(webhookDB, d)
+	env.eng.ExportDeliverDatabase(webhookDB, d)
 
 	var updated database.Delivery
 
@@ -105,247 +242,42 @@ func TestDeliverDatabase_ArchivesEvent(t *testing.T) {
 	)
 
 	archivePath := filepath.Join(
-		dataDir,
-		fmt.Sprintf("archive-%s.db", event.WebhookID),
+		env.dataDir, "archive-sweep-test-archive-"+tgt.ID+".db",
 	)
 	assert.FileExists(t, archivePath)
 
 	rdb := openArchiveDBForRead(t, archivePath)
 
-	var rows []delivery.ExportArchivedEvent
+	var rows []archivedRow
 
-	require.NoError(t, rdb.Find(&rows).Error)
+	require.NoError(t, rdb.Table("archived_events").Find(&rows).Error)
 	require.Len(t, rows, 1)
 	assert.Equal(t, event.ID, rows[0].EventID)
 	assert.Equal(t, event.WebhookID, rows[0].WebhookID)
 	assert.Equal(t, event.Method, rows[0].Method)
+	assert.Equal(t, eventQuery, rows[0].RawQuery)
 	assert.JSONEq(t, `{"archived":true}`, rows[0].Body)
 }
 
-func TestArchiveWriter_WritesRow(t *testing.T) {
-	t.Parallel()
-
-	path := filepath.Join(t.TempDir(), "archive-wh.db")
-	w := delivery.NewExportArchiveWriter(
-		path, archiveTestLogger(), 0,
-	)
-
-	row := delivery.ExportArchivedEvent{
-		EventID:      "ev-1",
-		WebhookID:    "wh-1",
-		EntrypointID: "ep-1",
-		Method:       "POST",
-		Headers:      `{"X":"Y"}`,
-		Body:         `{"hello":"world"}`,
-		ContentType:  "application/json",
-	}
-
-	require.NoError(t, w.Write(row, 0))
-	assert.FileExists(t, path)
-
-	var got []delivery.ExportArchivedEvent
-
-	require.NoError(t, w.DB().Find(&got).Error)
-	require.Len(t, got, 1)
-	assert.Equal(t, "ev-1", got[0].EventID)
-	assert.Equal(t, "wh-1", got[0].WebhookID)
-	assert.Equal(t, "ep-1", got[0].EntrypointID)
-	assert.Equal(t, row.Method, got[0].Method)
-	assert.Equal(t, row.ContentType, got[0].ContentType)
-	assert.JSONEq(t, `{"hello":"world"}`, got[0].Body)
-	assert.False(t, got[0].ArchivedAt.IsZero())
-}
-
-func TestArchiveWriter_RecreatesAfterRemoval(
-	t *testing.T,
-) {
-	t.Parallel()
-
-	path := filepath.Join(t.TempDir(), "archive-wh.db")
-	w := delivery.NewExportArchiveWriter(
-		path, archiveTestLogger(), 0,
-	)
-
-	require.NoError(t, w.Write(
-		delivery.ExportArchivedEvent{EventID: "a"}, 0,
-	))
-	assert.FileExists(t, path)
-
-	// The operator moves the archive away while the handle is
-	// still open.
-	removeArchiveFiles(t, path)
-	require.NoFileExists(t, path)
-
-	// The next write recreates the file with a fresh schema and
-	// only the new row.
-	require.NoError(t, w.Write(
-		delivery.ExportArchivedEvent{EventID: "b"}, 0,
-	))
-	assert.FileExists(t, path)
-
-	var got []delivery.ExportArchivedEvent
-
-	require.NoError(t, w.DB().Find(&got).Error)
-	require.Len(t, got, 1)
-	assert.Equal(t, "b", got[0].EventID)
-}
-
-func TestArchiveWriter_ReopenDebounce(t *testing.T) {
-	t.Parallel()
-
-	// A generous debounce keeps the two rapid writes inside
-	// the window even on a heavily loaded test machine.
-	path := filepath.Join(t.TempDir(), "archive-wh.db")
-	w := delivery.NewExportArchiveWriter(
-		path, archiveTestLogger(), 2*time.Second,
-	)
-
-	require.NoError(t, w.Write(
-		delivery.ExportArchivedEvent{EventID: "a"}, 0,
-	))
-	require.NoError(t, w.Write(
-		delivery.ExportArchivedEvent{EventID: "b"}, 0,
-	))
-
-	// Two writes inside the debounce window trigger only the
-	// initial open — no extra close/reopen.
-	assert.Equal(t, 1, w.Reopens())
-
-	time.Sleep(2100 * time.Millisecond)
-
-	require.NoError(t, w.Write(
-		delivery.ExportArchivedEvent{EventID: "c"}, 0,
-	))
-
-	// A write after the window elapses closes and reopens once.
-	assert.Equal(t, 2, w.Reopens())
-}
-
-func TestArchiveWriter_ExpiryPrune(t *testing.T) {
-	t.Parallel()
-
-	path := filepath.Join(t.TempDir(), "archive-wh.db")
-	w := delivery.NewExportArchiveWriter(
-		path, archiveTestLogger(), 0,
-	)
-
-	require.NoError(t, w.Open(0))
-
-	old := delivery.ExportArchivedEvent{
-		EventID:    "old",
-		ArchivedAt: time.Now().Add(-2 * time.Hour),
-	}
-	fresh := delivery.ExportArchivedEvent{
-		EventID:    "fresh",
-		ArchivedAt: time.Now(),
-	}
-
-	require.NoError(t, w.DB().Create(&old).Error)
-	require.NoError(t, w.DB().Create(&fresh).Error)
-
-	// Reopening with a one-hour expiry prunes the old row.
-	require.NoError(t, w.Reopen(time.Hour))
-
-	var got []delivery.ExportArchivedEvent
-
-	require.NoError(t, w.DB().Find(&got).Error)
-	require.Len(t, got, 1)
-	assert.Equal(t, "fresh", got[0].EventID)
-}
-
-func TestParseArchiveExpiry(t *testing.T) {
-	t.Parallel()
-
-	cases := []struct {
-		name    string
-		in      string
-		want    time.Duration
-		wantErr bool
-	}{
-		{"empty config", "", 0, false},
-		{"explicit never", `{"expiry":"never"}`, 0, false},
-		{"empty expiry", `{"expiry":""}`, 0, false},
-		{"duration", `{"expiry":"1h"}`, time.Hour, false},
-		{"unparseable", `{"expiry":"nonsense"}`, 0, true},
-		{"zero duration", `{"expiry":"0s"}`, 0, true},
-		{"negative duration", `{"expiry":"-5h"}`, 0, true},
-	}
-
-	for _, tc := range cases {
-		t.Run(tc.name, func(t *testing.T) {
-			t.Parallel()
-
-			got, err := delivery.ExportParseArchiveExpiry(tc.in)
-			if tc.wantErr {
-				require.Error(t, err)
-
-				return
-			}
-
-			require.NoError(t, err)
-			assert.Equal(t, tc.want, got)
-		})
-	}
-}
-
-// seedDatabaseTargetDelivery seeds a pending delivery for a
-// database target with the given config JSON and returns the
-// in-memory delivery the target handler is invoked with.
-func seedDatabaseTargetDelivery(
-	t *testing.T,
-	webhookDB *gorm.DB,
-	event database.Event,
-	config string,
-) *database.Delivery {
-	t.Helper()
-
-	dlv := seedDelivery(
-		t, webhookDB, event.ID, uuid.New().String(),
-		database.DeliveryStatusPending,
-	)
-
-	d := &database.Delivery{
-		EventID:  event.ID,
-		TargetID: dlv.TargetID,
-		Status:   database.DeliveryStatusPending,
-		Event:    event,
-		Target: database.Target{
-			Name:   "test-db",
-			Type:   database.TargetTypeDatabase,
-			Config: config,
-		},
-	}
-	d.ID = dlv.ID
-
-	return d
-}
-
 // TestDeliverDatabase_ArchiveFailureFailsDelivery verifies that
-// an archive error (here: an unparseable expiry in the target
-// config) fails the delivery loudly: the attempt is recorded as
-// failed with the error and the delivery is marked failed, not
+// an archive error (here: an event with no webhook id to record)
+// fails the delivery loudly: the attempt is recorded as failed
+// with the error and the delivery is marked failed, not
 // delivered.
 func TestDeliverDatabase_ArchiveFailureFailsDelivery(
 	t *testing.T,
 ) {
 	t.Parallel()
 
-	dataDir := t.TempDir()
-
-	e := delivery.NewTestEngineWithDB(
-		nil, database.NewTestWebhookDBManager(dataDir),
-		archiveTestLogger(),
-		&http.Client{Timeout: 5 * time.Second},
-		1,
-	)
+	env := setupArchiveTest(t)
+	tgt := env.seedDatabaseTarget(t)
 
 	webhookDB := testWebhookDB(t)
 	event := seedEvent(t, webhookDB, `{"archived":false}`)
-	d := seedDatabaseTargetDelivery(
-		t, webhookDB, event, `{"expiry":"nonsense"}`,
-	)
+	d := seedDatabaseTargetDelivery(t, webhookDB, event, tgt)
+	d.Event.WebhookID = ""
 
-	e.ExportDeliverDatabase(webhookDB, d)
+	env.eng.ExportDeliverDatabase(webhookDB, d)
 
 	var updated database.Delivery
 
@@ -368,35 +300,96 @@ func TestDeliverDatabase_ArchiveFailureFailsDelivery(
 		"the attempt must be recorded as failed",
 	)
 	assert.Contains(t,
-		results[0].Error, "nonsense",
+		results[0].Error, "without a webhook id",
 		"the archive error must be recorded on the attempt",
 	)
 
 	assert.NoFileExists(t,
-		filepath.Join(
-			dataDir,
-			fmt.Sprintf("archive-%s.db", event.WebhookID),
-		),
-		"no archive file should exist for a failed config",
+		env.archivePath(tgt),
+		"no archive file should exist for a failed write",
 	)
 }
 
-func TestValidateArchiveExpiry(t *testing.T) {
+// seedArchivedRow creates the archive file at path holding one row, for
+// eventID, archived at archivedAt.
+func seedArchivedRow(
+	t *testing.T, path, eventID string, archivedAt time.Time,
+) {
+	t.Helper()
+
+	sqlDB, err := sql.Open(
+		"sqlite", fmt.Sprintf("file:%s?mode=rwc", path),
+	)
+	require.NoError(t, err)
+
+	defer func() { _ = sqlDB.Close() }()
+
+	gdb, err := gorm.Open(
+		sqlite.Dialector{Conn: sqlDB},
+		&gorm.Config{Logger: gormlog.New(slog.New(slog.DiscardHandler))},
+	)
+	require.NoError(t, err)
+
+	require.NoError(t, migrations.Apply(gdb, migrations.Archive))
+	require.NoError(t, gdb.Exec(
+		"INSERT INTO archived_events (event_id, archived_at) VALUES (?, ?)",
+		eventID, archivedAt,
+	).Error)
+}
+
+// TestDeliverDatabase_TaskArchiveSettings proves a task's ArchiveExpiry
+// and ArchiveRotation reach the archive through the engine. The
+// target's row has neither, so only the task can bring them: with daily
+// rotation the event goes to the file for its day, and opening that
+// file with a one-hour expiry prunes the row it already held, archived
+// two hours ago.
+func TestDeliverDatabase_TaskArchiveSettings(t *testing.T) {
 	t.Parallel()
 
-	valid := []string{"", "never", "1h", "720h", "30m"}
-	for _, in := range valid {
-		require.NoError(t,
-			delivery.ValidateArchiveExpiry(in),
-			"expiry %q should be accepted", in,
-		)
-	}
+	s := newISetup(t)
 
-	invalid := []string{"nonsense", "7d", "-5h", "0s", "0"}
-	for _, in := range invalid {
-		require.Error(t,
-			delivery.ValidateArchiveExpiry(in),
-			"expiry %q should be rejected", in,
-		)
+	webhook := database.Webhook{UserID: uuid.NewString(), Name: "settings"}
+	webhook.ID = s.WebhookID
+	require.NoError(t, s.MainDB.Create(&webhook).Error)
+
+	target := database.Target{
+		WebhookID:   s.WebhookID,
+		Name:        "archive",
+		Type:        database.TargetTypeDatabase,
+		Active:      true,
+		MaxAttempts: 1,
 	}
+	require.NoError(t, s.MainDB.Create(&target).Error)
+
+	event := iSeedEvent(t, s.WebhookDB, s.WebhookID, `{"archived":true}`)
+	d := iSeedDelivery(
+		t, s.WebhookDB, event.ID, target.ID,
+		database.DeliveryStatusPending,
+	)
+
+	task := iTask(
+		d, event, s.WebhookID, target.ID, target.Name, "", 1, 1,
+		&event.Body,
+	)
+	task.TargetType = database.TargetTypeDatabase
+	task.ArchiveExpiry = time.Hour
+	task.ArchiveRotation = "daily"
+
+	daily := target
+	daily.ArchiveRotation = task.ArchiveRotation
+	dailyFile := archive.PathAt(s.DBMgr, &webhook, &daily, event.CreatedAt)
+	seedArchivedRow(t, dailyFile, "expired", time.Now().Add(-2*time.Hour))
+
+	s.Engine.ExportProcessNewTask(context.TODO(), &task)
+
+	iAssertStatus(t, s.WebhookDB, d.ID, database.DeliveryStatusDelivered)
+
+	var rows []archivedRow
+
+	require.NoError(t, openArchiveDBForRead(t, dailyFile).
+		Table("archived_events").Find(&rows).Error)
+	require.Len(t, rows, 1)
+	assert.Equal(t, event.ID, rows[0].EventID,
+		"the event goes to its day's file and the expired row is pruned",
+	)
 }