Evict archive writers on deletion and sweep idle archives (closes #89)

The per-webhook archiveWriter registry in the database delivery target
was never evicted, so a deleted webhook's writer -- and any archive
file handle open within its debounce window -- lingered for the process
lifetime. Separately, expiry pruning ran only when an archive was
(re)opened, and reopens only happen on writes, so an archive belonging
to a webhook that stopped receiving events kept its expired rows
forever.

Eviction: a new one-method delivery.WebhookEvictor interface (kept
separate from Notifier: archiving lifecycle is not notification) is
implemented by the Engine and injected into the handlers. Deleting a
webhook, or deleting its last database target, drops the writer from
the registry and closes its handle under the writer's own mutex, so
eviction can never race an in-flight write. An evicted writer refuses
further writes rather than reopening a file nothing holds. The archive
file is deliberately left on disk: it is long-term storage an operator
may want to keep or move away, and destroying it as a side effect of
deleting a webhook would be unrecoverable.

Idle sweep: a new ArchiveSweeper, modelled on the event RetentionReaper
(fx lifecycle hooks, cancellable context, WaitGroup, ticker loop),
prunes archives whose database target declares a positive expiry. It
reuses the existing RETENTION_SWEEP_INTERVAL rather than adding a
config key. It never creates an archive -- a missing file is skipped,
and the reopen uses SQLite mode=rw so the file cannot be conjured even
if it disappears mid-sweep -- routes the prune through the per-webhook
writer so its mutex orders the sweep against concurrent writes, and
leaves the archive closed so the move-the-file-away workflow keeps
working. A failure for one webhook is logged and the sweep continues.
Archives with no expiry or the expiry "never" are untouched.

internal/delivery/archive_sweeper_test.go

@@ -0,0 +1,425 @@
+package delivery_test
+
+import (
+	"context"
+	"database/sql"
+	"fmt"
+	"net/http"
+	"path/filepath"
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/google/uuid"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"gorm.io/driver/sqlite"
+	"gorm.io/gorm"
+	"gorm.io/gorm/clause"
+	_ "modernc.org/sqlite" // Pure Go SQLite driver.
+	"sneak.berlin/go/webhooker/internal/database"
+	"sneak.berlin/go/webhooker/internal/delivery"
+)
+
+const (
+	// sweepRowOld and sweepRowNew are the event ids
+	// seedArchiveRows assigns to the first and second seeded
+	// rows.
+	sweepRowOld = "ev-0"
+	sweepRowNew = "ev-1"
+
+	// sweepConcurrentWrites is how many deliveries the
+	// concurrent write-plus-sweep test races against the sweep.
+	sweepConcurrentWrites = 20
+)
+
+// sweeperEnv bundles the pieces an archive sweep test drives:
+// a main configuration database holding webhooks and targets, a
+// delivery engine owning the archive writer registry, and the
+// data directory the archive files live in.
+type sweeperEnv struct {
+	sweeper *delivery.ArchiveSweeper
+	eng     *delivery.Engine
+	mainDB  *database.Database
+	dataDir string
+}
+
+func setupSweeperTest(t *testing.T) *sweeperEnv {
+	t.Helper()
+
+	dataDir := t.TempDir()
+	log := archiveTestLogger()
+
+	sqlDB, err := sql.Open(
+		"sqlite",
+		fmt.Sprintf(
+			"file:%s?mode=rwc",
+			filepath.Join(dataDir, "main.db"),
+		),
+	)
+	require.NoError(t, err)
+
+	t.Cleanup(func() { _ = sqlDB.Close() })
+
+	gdb, err := gorm.Open(
+		sqlite.Dialector{Conn: sqlDB}, &gorm.Config{},
+	)
+	require.NoError(t, err)
+
+	mainDB := database.NewTestDatabase(gdb)
+	require.NoError(t, mainDB.Migrate())
+
+	eng := delivery.NewTestEngineWithDB(
+		mainDB,
+		database.NewTestWebhookDBManager(dataDir),
+		log,
+		&http.Client{Timeout: 5 * time.Second},
+		1,
+	)
+
+	return &sweeperEnv{
+		sweeper: delivery.NewTestArchiveSweeper(
+			mainDB, eng, log,
+		),
+		eng:     eng,
+		mainDB:  mainDB,
+		dataDir: dataDir,
+	}
+}
+
+// archivePath returns where the engine keeps a webhook's
+// archive file.
+func (env *sweeperEnv) archivePath(webhookID string) string {
+	return filepath.Join(
+		env.dataDir, fmt.Sprintf("archive-%s.db", webhookID),
+	)
+}
+
+// seedDatabaseTarget creates a webhook with one database target
+// carrying the given target config JSON, and returns the
+// webhook id.
+func (env *sweeperEnv) seedDatabaseTarget(
+	t *testing.T, configJSON string,
+) string {
+	t.Helper()
+
+	wh := &database.Webhook{
+		UserID: uuid.New().String(),
+		Name:   "sweep-test",
+	}
+	require.NoError(
+		t,
+		env.mainDB.DB().
+			Omit(clause.Associations).
+			Create(wh).Error,
+	)
+
+	tgt := &database.Target{
+		WebhookID: wh.ID,
+		Name:      "archive",
+		Type:      database.TargetTypeDatabase,
+		Active:    true,
+		Config:    configJSON,
+	}
+	require.NoError(
+		t,
+		env.mainDB.DB().
+			Omit(clause.Associations).
+			Create(tgt).Error,
+	)
+
+	return wh.ID
+}
+
+// seedArchiveRows creates the archive file for a webhook and
+// inserts one row per supplied archived-at timestamp, returning
+// the archive path. The handle is closed before returning, so
+// the archive is idle exactly as it would be with no traffic.
+func (env *sweeperEnv) seedArchiveRows(
+	t *testing.T, webhookID string, archivedAt ...time.Time,
+) string {
+	t.Helper()
+
+	path := env.archivePath(webhookID)
+
+	sqlDB, err := sql.Open(
+		"sqlite", fmt.Sprintf("file:%s?mode=rwc", path),
+	)
+	require.NoError(t, err)
+
+	gdb, err := gorm.Open(
+		sqlite.Dialector{Conn: sqlDB}, &gorm.Config{},
+	)
+	require.NoError(t, err)
+
+	require.NoError(
+		t, gdb.AutoMigrate(&delivery.ExportArchivedEvent{}),
+	)
+
+	for i, at := range archivedAt {
+		row := delivery.ExportArchivedEvent{
+			EventID:    fmt.Sprintf("ev-%d", i),
+			WebhookID:  webhookID,
+			Method:     http.MethodPost,
+			Body:       `{"seeded":true}`,
+			ArchivedAt: at,
+		}
+		require.NoError(t, gdb.Create(&row).Error)
+	}
+
+	require.NoError(t, sqlDB.Close())
+
+	return path
+}
+
+// archivedEventIDs returns the event ids currently stored in an
+// archive file, read through a separate read-only handle.
+func archivedEventIDs(
+	t *testing.T, path string,
+) []string {
+	t.Helper()
+
+	var rows []delivery.ExportArchivedEvent
+
+	rdb := openArchiveDBForRead(t, path)
+	require.NoError(t, rdb.Order("event_id").Find(&rows).Error)
+
+	ids := make([]string, 0, len(rows))
+	for i := range rows {
+		ids = append(ids, rows[i].EventID)
+	}
+
+	return ids
+}
+
+// TestArchiveSweep_PrunesIdleArchive is the core regression
+// test for this issue: an archive that receives no further
+// writes must still lose its expired rows. Before the sweeper
+// existed, pruning only ever ran on a write-triggered reopen,
+// so an idle archive kept expired rows forever.
+func TestArchiveSweep_PrunesIdleArchive(t *testing.T) {
+	t.Parallel()
+
+	env := setupSweeperTest(t)
+
+	webhookID := env.seedDatabaseTarget(t, `{"expiry":"1h"}`)
+
+	now := time.Now()
+	path := env.seedArchiveRows(
+		t, webhookID,
+		now.Add(-48*time.Hour),
+		now.Add(-time.Minute),
+	)
+
+	require.Equal(
+		t, []string{sweepRowOld, sweepRowNew},
+		archivedEventIDs(t, path),
+	)
+
+	env.sweeper.ExportSweep(context.Background())
+
+	assert.Equal(
+		t, []string{sweepRowNew}, archivedEventIDs(t, path),
+		"the sweep should prune rows older than the expiry "+
+			"from an idle archive and keep the rest",
+	)
+}
+
+// TestArchiveSweep_LeavesArchiveClosed proves the sweep does
+// not hold the archive open afterwards, so an operator can
+// still move the file away for offline retention.
+func TestArchiveSweep_LeavesArchiveClosed(t *testing.T) {
+	t.Parallel()
+
+	env := setupSweeperTest(t)
+
+	webhookID := env.seedDatabaseTarget(t, `{"expiry":"1h"}`)
+	env.seedArchiveRows(
+		t, webhookID, time.Now().Add(-48*time.Hour),
+	)
+
+	env.sweeper.ExportSweep(context.Background())
+
+	assert.False(
+		t, env.eng.ExportArchiveHandleOpen(webhookID),
+		"an idle archive must end the sweep closed",
+	)
+}
+
+// TestArchiveSweep_NeverExpiryUntouched proves the sweep is a
+// no-op for the default retention policy, so archives with no
+// expiry (or the literal "never") behave exactly as before.
+func TestArchiveSweep_NeverExpiryUntouched(t *testing.T) {
+	t.Parallel()
+
+	for _, configJSON := range []string{
+		`{"expiry":"never"}`,
+		`{"expiry":""}`,
+		"",
+	} {
+		env := setupSweeperTest(t)
+
+		webhookID := env.seedDatabaseTarget(t, configJSON)
+		path := env.seedArchiveRows(
+			t, webhookID,
+			time.Now().Add(-10000*time.Hour),
+		)
+
+		env.sweeper.ExportSweep(context.Background())
+
+		assert.Equal(
+			t, []string{sweepRowOld}, archivedEventIDs(t, path),
+			"config %q must keep rows forever", configJSON,
+		)
+	}
+}
+
+// TestArchiveSweep_DoesNotCreateArchiveFile proves the sweep
+// never conjures an archive: a webhook with a database target
+// that has never received an event must still have no archive
+// file (nor SQLite sidecar) after a sweep.
+func TestArchiveSweep_DoesNotCreateArchiveFile(t *testing.T) {
+	t.Parallel()
+
+	env := setupSweeperTest(t)
+
+	webhookID := env.seedDatabaseTarget(t, `{"expiry":"1h"}`)
+	path := env.archivePath(webhookID)
+
+	require.NoFileExists(t, path)
+
+	env.sweeper.ExportSweep(context.Background())
+
+	for _, suffix := range []string{"", "-wal", "-shm"} {
+		assert.NoFileExists(
+			t, path+suffix,
+			"the sweep must not create an archive file",
+		)
+	}
+}
+
+// TestArchiveSweep_DoesNotCreateAfterWriterExists covers the
+// same guarantee once a writer is cached in the registry but
+// the file itself is still absent (for instance because the
+// operator moved the archive away).
+func TestArchiveSweep_DoesNotCreateAfterWriterExists(
+	t *testing.T,
+) {
+	t.Parallel()
+
+	env := setupSweeperTest(t)
+
+	webhookID := env.seedDatabaseTarget(t, `{"expiry":"1h"}`)
+
+	path, err := env.eng.ExportEnsureArchiveWriter(webhookID)
+	require.NoError(t, err)
+	require.NoFileExists(t, path)
+
+	env.sweeper.ExportSweep(context.Background())
+
+	assert.NoFileExists(t, path)
+}
+
+// TestArchiveSweep_SkipsDeletedWebhookTargets proves that the
+// sweep ignores targets soft-deleted along with their webhook,
+// so a deleted webhook's archive is never reopened.
+func TestArchiveSweep_SkipsDeletedWebhookTargets(t *testing.T) {
+	t.Parallel()
+
+	env := setupSweeperTest(t)
+
+	webhookID := env.seedDatabaseTarget(t, `{"expiry":"1h"}`)
+	path := env.seedArchiveRows(
+		t, webhookID, time.Now().Add(-48*time.Hour),
+	)
+
+	require.NoError(
+		t,
+		env.mainDB.DB().
+			Where("webhook_id = ?", webhookID).
+			Delete(&database.Target{}).Error,
+	)
+
+	env.sweeper.ExportSweep(context.Background())
+
+	assert.Equal(
+		t, []string{sweepRowOld}, archivedEventIDs(t, path),
+		"a deleted target's archive must be left alone",
+	)
+}
+
+// TestArchiveSweep_ConcurrentWrites proves the sweep serialises
+// against writes through the per-webhook writer mutex. Run
+// under -race, an unsynchronised sweep would be caught here.
+func TestArchiveSweep_ConcurrentWrites(t *testing.T) {
+	t.Parallel()
+
+	env := setupSweeperTest(t)
+
+	webhookID := env.seedDatabaseTarget(t, `{"expiry":"1h"}`)
+
+	webhookDB := testWebhookDB(t)
+
+	// The deliveries are seeded up front, on the test's own
+	// goroutine: the seed helpers assert, and testify assertions
+	// must not run off the test goroutine.
+	deliveries := make(
+		[]*database.Delivery, 0, sweepConcurrentWrites,
+	)
+
+	for range sweepConcurrentWrites {
+		event := seedEvent(t, webhookDB, `{"n":1}`)
+		event.WebhookID = webhookID
+
+		deliveries = append(
+			deliveries,
+			seedDatabaseTargetDelivery(
+				t, webhookDB, event, `{"expiry":"1h"}`,
+			),
+		)
+	}
+
+	var wg sync.WaitGroup
+
+	wg.Add(2)
+
+	go func() {
+		defer wg.Done()
+
+		for _, d := range deliveries {
+			env.eng.ExportDeliverDatabase(webhookDB, d)
+		}
+	}()
+
+	go func() {
+		defer wg.Done()
+
+		for range sweepConcurrentWrites {
+			env.sweeper.ExportSweep(context.Background())
+		}
+	}()
+
+	wg.Wait()
+
+	assert.FileExists(t, env.archivePath(webhookID))
+}
+
+// TestArchiveSweeper_StopsCleanly proves the background loop
+// exits on OnStop rather than leaking a goroutine.
+func TestArchiveSweeper_StopsCleanly(t *testing.T) {
+	t.Parallel()
+
+	env := setupSweeperTest(t)
+
+	webhookID := env.seedDatabaseTarget(t, `{"expiry":"1h"}`)
+	env.seedArchiveRows(
+		t, webhookID, time.Now().Add(-48*time.Hour),
+	)
+
+	env.sweeper.ExportSetInterval(time.Millisecond)
+	env.sweeper.ExportStart(context.Background())
+
+	// stop blocks on the loop's WaitGroup, so returning at all
+	// proves the loop observed the cancellation and exited.
+	env.sweeper.ExportStop()
+}