package delivery_test import ( "context" "database/sql" "fmt" "net/http" "path/filepath" "sync" "testing" "time" "github.com/google/uuid" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "gorm.io/driver/sqlite" "gorm.io/gorm" "gorm.io/gorm/clause" _ "modernc.org/sqlite" // Pure Go SQLite driver. "sneak.berlin/go/webhooker/internal/database" "sneak.berlin/go/webhooker/internal/delivery" ) const ( // sweepRowOld and sweepRowNew are the event ids // seedArchiveRows assigns to the first and second seeded // rows. sweepRowOld = "ev-0" sweepRowNew = "ev-1" // sweepConcurrentWrites is how many deliveries the // concurrent write-plus-sweep test races against the sweep. sweepConcurrentWrites = 20 ) // sweeperEnv bundles the pieces an archive sweep test drives: // a main configuration database holding webhooks and targets, a // delivery engine owning the archive writer registry, and the // data directory the archive files live in. type sweeperEnv struct { sweeper *delivery.ArchiveSweeper eng *delivery.Engine mainDB *database.Database dataDir string } func setupSweeperTest(t *testing.T) *sweeperEnv { t.Helper() dataDir := t.TempDir() log := archiveTestLogger() sqlDB, err := sql.Open( "sqlite", fmt.Sprintf( "file:%s?mode=rwc", filepath.Join(dataDir, "main.db"), ), ) require.NoError(t, err) t.Cleanup(func() { _ = sqlDB.Close() }) gdb, err := gorm.Open( sqlite.Dialector{Conn: sqlDB}, &gorm.Config{}, ) require.NoError(t, err) mainDB := database.NewTestDatabase(gdb) require.NoError(t, mainDB.Migrate()) eng := delivery.NewTestEngineWithDB( mainDB, database.NewTestWebhookDBManager(dataDir), log, &http.Client{Timeout: 5 * time.Second}, 1, ) return &sweeperEnv{ sweeper: delivery.NewTestArchiveSweeper( mainDB, eng, log, ), eng: eng, mainDB: mainDB, dataDir: dataDir, } } // archivePath returns where the engine keeps a webhook's // archive file. func (env *sweeperEnv) archivePath(webhookID string) string { return filepath.Join( env.dataDir, fmt.Sprintf("archive-%s.db", webhookID), ) } // seedDatabaseTarget creates a webhook with one database target // carrying the given target config JSON, and returns the // webhook id. func (env *sweeperEnv) seedDatabaseTarget( t *testing.T, configJSON string, ) string { t.Helper() wh := &database.Webhook{ UserID: uuid.New().String(), Name: "sweep-test", } require.NoError( t, env.mainDB.DB(). Omit(clause.Associations). Create(wh).Error, ) tgt := &database.Target{ WebhookID: wh.ID, Name: "archive", Type: database.TargetTypeDatabase, Active: true, Config: configJSON, } require.NoError( t, env.mainDB.DB(). Omit(clause.Associations). Create(tgt).Error, ) return wh.ID } // seedArchiveRows creates the archive file for a webhook and // inserts one row per supplied archived-at timestamp, returning // the archive path. The handle is closed before returning, so // the archive is idle exactly as it would be with no traffic. func (env *sweeperEnv) seedArchiveRows( t *testing.T, webhookID string, archivedAt ...time.Time, ) string { t.Helper() path := env.archivePath(webhookID) sqlDB, err := sql.Open( "sqlite", fmt.Sprintf("file:%s?mode=rwc", path), ) require.NoError(t, err) gdb, err := gorm.Open( sqlite.Dialector{Conn: sqlDB}, &gorm.Config{}, ) require.NoError(t, err) require.NoError( t, gdb.AutoMigrate(&delivery.ExportArchivedEvent{}), ) for i, at := range archivedAt { row := delivery.ExportArchivedEvent{ EventID: fmt.Sprintf("ev-%d", i), WebhookID: webhookID, Method: http.MethodPost, Body: `{"seeded":true}`, ArchivedAt: at, } require.NoError(t, gdb.Create(&row).Error) } require.NoError(t, sqlDB.Close()) return path } // archivedEventIDs returns the event ids currently stored in an // archive file, read through a separate read-only handle. func archivedEventIDs( t *testing.T, path string, ) []string { t.Helper() var rows []delivery.ExportArchivedEvent rdb := openArchiveDBForRead(t, path) require.NoError(t, rdb.Order("event_id").Find(&rows).Error) ids := make([]string, 0, len(rows)) for i := range rows { ids = append(ids, rows[i].EventID) } return ids } // TestArchiveSweep_PrunesIdleArchive is the core regression // test for this issue: an archive that receives no further // writes must still lose its expired rows. Before the sweeper // existed, pruning only ever ran on a write-triggered reopen, // so an idle archive kept expired rows forever. func TestArchiveSweep_PrunesIdleArchive(t *testing.T) { t.Parallel() env := setupSweeperTest(t) webhookID := env.seedDatabaseTarget(t, `{"expiry":"1h"}`) now := time.Now() path := env.seedArchiveRows( t, webhookID, now.Add(-48*time.Hour), now.Add(-time.Minute), ) require.Equal( t, []string{sweepRowOld, sweepRowNew}, archivedEventIDs(t, path), ) env.sweeper.ExportSweep(context.Background()) assert.Equal( t, []string{sweepRowNew}, archivedEventIDs(t, path), "the sweep should prune rows older than the expiry "+ "from an idle archive and keep the rest", ) } // TestArchiveSweep_LeavesArchiveClosed proves the sweep does // not hold the archive open afterwards, so an operator can // still move the file away for offline retention. func TestArchiveSweep_LeavesArchiveClosed(t *testing.T) { t.Parallel() env := setupSweeperTest(t) webhookID := env.seedDatabaseTarget(t, `{"expiry":"1h"}`) env.seedArchiveRows( t, webhookID, time.Now().Add(-48*time.Hour), ) env.sweeper.ExportSweep(context.Background()) assert.False( t, env.eng.ExportArchiveHandleOpen(webhookID), "an idle archive must end the sweep closed", ) } // TestArchiveSweep_NeverExpiryUntouched proves the sweep is a // no-op for the default retention policy, so archives with no // expiry (or the literal "never") behave exactly as before. func TestArchiveSweep_NeverExpiryUntouched(t *testing.T) { t.Parallel() for _, configJSON := range []string{ `{"expiry":"never"}`, `{"expiry":""}`, "", } { env := setupSweeperTest(t) webhookID := env.seedDatabaseTarget(t, configJSON) path := env.seedArchiveRows( t, webhookID, time.Now().Add(-10000*time.Hour), ) env.sweeper.ExportSweep(context.Background()) assert.Equal( t, []string{sweepRowOld}, archivedEventIDs(t, path), "config %q must keep rows forever", configJSON, ) } } // TestArchiveSweep_DoesNotCreateArchiveFile proves the sweep // never conjures an archive: a webhook with a database target // that has never received an event must still have no archive // file (nor SQLite sidecar) after a sweep. func TestArchiveSweep_DoesNotCreateArchiveFile(t *testing.T) { t.Parallel() env := setupSweeperTest(t) webhookID := env.seedDatabaseTarget(t, `{"expiry":"1h"}`) path := env.archivePath(webhookID) require.NoFileExists(t, path) env.sweeper.ExportSweep(context.Background()) for _, suffix := range []string{"", "-wal", "-shm"} { assert.NoFileExists( t, path+suffix, "the sweep must not create an archive file", ) } } // TestArchiveSweep_DoesNotCreateAfterWriterExists covers the // same guarantee once a writer is cached in the registry but // the file itself is still absent (for instance because the // operator moved the archive away). func TestArchiveSweep_DoesNotCreateAfterWriterExists( t *testing.T, ) { t.Parallel() env := setupSweeperTest(t) webhookID := env.seedDatabaseTarget(t, `{"expiry":"1h"}`) path, err := env.eng.ExportEnsureArchiveWriter(webhookID) require.NoError(t, err) require.NoFileExists(t, path) env.sweeper.ExportSweep(context.Background()) assert.NoFileExists(t, path) } // TestArchiveSweep_SkipsDeletedWebhookTargets proves that the // sweep ignores targets soft-deleted along with their webhook, // so a deleted webhook's archive is never reopened. func TestArchiveSweep_SkipsDeletedWebhookTargets(t *testing.T) { t.Parallel() env := setupSweeperTest(t) webhookID := env.seedDatabaseTarget(t, `{"expiry":"1h"}`) path := env.seedArchiveRows( t, webhookID, time.Now().Add(-48*time.Hour), ) require.NoError( t, env.mainDB.DB(). Where("webhook_id = ?", webhookID). Delete(&database.Target{}).Error, ) env.sweeper.ExportSweep(context.Background()) assert.Equal( t, []string{sweepRowOld}, archivedEventIDs(t, path), "a deleted target's archive must be left alone", ) } // TestArchiveSweep_ConcurrentWrites proves the sweep serialises // against writes through the per-webhook writer mutex. Run // under -race, an unsynchronised sweep would be caught here. func TestArchiveSweep_ConcurrentWrites(t *testing.T) { t.Parallel() env := setupSweeperTest(t) webhookID := env.seedDatabaseTarget(t, `{"expiry":"1h"}`) webhookDB := testWebhookDB(t) // The deliveries are seeded up front, on the test's own // goroutine: the seed helpers assert, and testify assertions // must not run off the test goroutine. deliveries := make( []*database.Delivery, 0, sweepConcurrentWrites, ) for range sweepConcurrentWrites { event := seedEvent(t, webhookDB, `{"n":1}`) event.WebhookID = webhookID deliveries = append( deliveries, seedDatabaseTargetDelivery( t, webhookDB, event, `{"expiry":"1h"}`, ), ) } var wg sync.WaitGroup wg.Add(2) go func() { defer wg.Done() for _, d := range deliveries { env.eng.ExportDeliverDatabase(webhookDB, d) } }() go func() { defer wg.Done() for range sweepConcurrentWrites { env.sweeper.ExportSweep(context.Background()) } }() wg.Wait() assert.FileExists(t, env.archivePath(webhookID)) } // TestArchiveSweeper_StopsCleanly proves the background loop // exits on OnStop rather than leaking a goroutine. func TestArchiveSweeper_StopsCleanly(t *testing.T) { t.Parallel() env := setupSweeperTest(t) webhookID := env.seedDatabaseTarget(t, `{"expiry":"1h"}`) env.seedArchiveRows( t, webhookID, time.Now().Add(-48*time.Hour), ) env.sweeper.ExportSetInterval(time.Millisecond) env.sweeper.ExportStart(context.Background()) // stop blocks on the loop's WaitGroup, so returning at all // proves the loop observed the cancellation and exited. env.sweeper.ExportStop() }