package archive_test import ( "database/sql" "fmt" "io/fs" "log/slog" "net/http" "os" "path/filepath" "strings" "testing" "time" "github.com/google/uuid" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "gorm.io/driver/sqlite" "gorm.io/gorm" _ "modernc.org/sqlite" // Pure Go SQLite driver. "sneak.berlin/go/webhooker/internal/archive" "sneak.berlin/go/webhooker/internal/database" "sneak.berlin/go/webhooker/internal/gormlog" ) const ( // testContentType is the event content type used in tests. testContentType = "application/json" // eventQuery is the query string an archived event carries. eventQuery = "a=1&b=2" ) func archiveTestLogger() *slog.Logger { return slog.New(slog.NewTextHandler( os.Stderr, &slog.HandlerOptions{Level: slog.LevelDebug}, )) } // openArchiveDBForRead opens an archive file read-only so a // test can inspect the rows the writer persisted. func openArchiveDBForRead( t *testing.T, path string, ) *gorm.DB { t.Helper() sqlDB, err := sql.Open( "sqlite", fmt.Sprintf("file:%s?mode=ro", path), ) require.NoError(t, err) t.Cleanup(func() { _ = sqlDB.Close() }) gdb, err := gorm.Open( sqlite.Dialector{Conn: sqlDB}, &gorm.Config{Logger: gormlog.New(slog.New(slog.DiscardHandler))}, ) require.NoError(t, err) return gdb } // archiveFileSuffixes returns the archive file itself and the // SQLite sidecars that accompany an open database. A test that // asserts no archive was created has to check all of them. func archiveFileSuffixes() []string { return []string{"", "-wal", "-shm"} } // removeArchiveFiles simulates an operator moving the archive // away by deleting the SQLite file and its sidecar files. func removeArchiveFiles(t *testing.T, path string) { t.Helper() for _, suffix := range []string{ "", "-wal", "-shm", "-journal", } { err := os.Remove(path + suffix) if err != nil && !os.IsNotExist(err) { t.Fatalf("removing %s%s: %v", path, suffix, err) } } } // assertStoredTime asserts that text, a timestamp as SQLite holds it, // is in the form GORM stores created_at in: the layout of // time.Time.String, with no " m=+…" monotonic clock reading after it. func assertStoredTime(t *testing.T, text string) { t.Helper() _, err := time.Parse( "2006-01-02 15:04:05.999999999 -0700 MST", text, ) assert.NoError(t, err, "stored as %q", text) } // newEvent returns an event received now with the given body, as the // webhook handler stores it. func newEvent(body string) database.Event { return database.Event{ BaseModel: database.BaseModel{ ID: uuid.New().String(), CreatedAt: time.Now(), }, WebhookID: uuid.New().String(), EntrypointID: uuid.New().String(), Method: http.MethodPost, Headers: `{"Content-Type":["application/json"]}`, Body: body, ContentType: testContentType, } } // databaseTargetDelivery returns a delivery of event to a database // target, as the database target hands it to Writers.Write. func databaseTargetDelivery( event database.Event, tgt *database.Target, ) *database.Delivery { return &database.Delivery{ EventID: event.ID, TargetID: tgt.ID, Event: event, Target: database.Target{ Name: tgt.Name, Type: database.TargetTypeDatabase, ArchiveExpiry: tgt.ArchiveExpiry, ArchiveRotation: tgt.ArchiveRotation, }, } } func TestArchiveWriter_WritesRow(t *testing.T) { t.Parallel() path := filepath.Join(t.TempDir(), "archive-wh.db") w := archive.NewExportArchiveWriter( path, archiveTestLogger(), 0, ) row := archive.ExportArchivedEvent{ EventID: "ev-1", WebhookID: "wh-1", EntrypointID: "ep-1", Method: "POST", Headers: `{"X":"Y"}`, Body: `{"hello":"world"}`, ContentType: "application/json", } require.NoError(t, w.Write(row, 0)) assert.FileExists(t, path) var got []archive.ExportArchivedEvent require.NoError(t, w.DB().Find(&got).Error) require.Len(t, got, 1) assert.Equal(t, "ev-1", got[0].EventID) assert.Equal(t, "wh-1", got[0].WebhookID) assert.Equal(t, "ep-1", got[0].EntrypointID) assert.Equal(t, row.Method, got[0].Method) assert.Equal(t, row.ContentType, got[0].ContentType) assert.JSONEq(t, `{"hello":"world"}`, got[0].Body) assert.False(t, got[0].ArchivedAt.IsZero()) var archivedAt string require.NoError(t, w.DB().Raw( "SELECT CAST(archived_at AS TEXT) FROM archived_events", ).Row().Scan(&archivedAt)) assertStoredTime(t, archivedAt) } func TestArchiveWriter_RecreatesAfterRemoval( t *testing.T, ) { t.Parallel() path := filepath.Join(t.TempDir(), "archive-wh.db") w := archive.NewExportArchiveWriter( path, archiveTestLogger(), 0, ) require.NoError(t, w.Write( archive.ExportArchivedEvent{EventID: "a"}, 0, )) assert.FileExists(t, path) // The operator moves the archive away while the handle is // still open. removeArchiveFiles(t, path) require.NoFileExists(t, path) // The next write recreates the file with a fresh schema and // only the new row. require.NoError(t, w.Write( archive.ExportArchivedEvent{EventID: "b"}, 0, )) assert.FileExists(t, path) var got []archive.ExportArchivedEvent require.NoError(t, w.DB().Find(&got).Error) require.Len(t, got, 1) assert.Equal(t, "b", got[0].EventID) } // TestArchiveHandleIsWAL closes the last gap in the durability // evidence: the main and per-webhook tiers each assert their journal // mode on a live handle, and the archive tier gets its settings from // the same code path but nothing checked the running file. func TestArchiveHandleIsWAL(t *testing.T) { t.Parallel() w := archive.NewExportArchiveWriter( filepath.Join(t.TempDir(), "archive-wal.db"), archiveTestLogger(), 0, ) require.NoError(t, w.Open(0)) var mode string row := w.DB().Raw("pragma journal_mode").Row() require.NoError(t, row.Scan(&mode)) assert.Equal(t, "wal", strings.ToLower(mode)) var busy string row = w.DB().Raw("pragma busy_timeout").Row() require.NoError(t, row.Scan(&busy)) assert.Equal(t, "10000", busy) } // TestStatArchive proves Stat finds no file before the first // write; after a write still held in the -wal, counts the -wal in the // size and takes its later time as the last write; and finds no file // again once the file has been moved away. func TestStatArchive(t *testing.T) { t.Parallel() path := filepath.Join(t.TempDir(), "archive-wh.db") _, err := archive.Stat(path) require.ErrorIs(t, err, fs.ErrNotExist) // With the clock stopped, the reopen debounce never passes, so // the handle stays open after the write. stopped := time.Now() w := archive.NewExportArchiveWriter(path, archiveTestLogger(), 0) w.SetNow(func() time.Time { return stopped }) require.NoError(t, w.Write(archive.ExportArchivedEvent{EventID: "a"}, 0)) written := time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC) earlier := written.Add(-time.Hour) require.NoError(t, os.Chtimes(path, earlier, earlier)) require.NoError(t, os.Chtimes(path+"-wal", written, written)) file, err := os.Stat(path) require.NoError(t, err) wal, err := os.Stat(path + "-wal") require.NoError(t, err) require.Positive(t, wal.Size()) got, err := archive.Stat(path) require.NoError(t, err) assert.Equal(t, 1, got.Files) assert.Equal(t, file.Size()+wal.Size(), got.Size) assert.True(t, written.Equal(got.Written), got.Written) removeArchiveFiles(t, path) _, err = archive.Stat(path) require.ErrorIs(t, err, fs.ErrNotExist) } func TestArchiveWriter_ReopenDebounce(t *testing.T) { t.Parallel() const debounce = 2 * time.Second path := filepath.Join(t.TempDir(), "archive-wh.db") w := archive.NewExportArchiveWriter( path, archiveTestLogger(), debounce, ) // The writer measures its reopen debounce on this clock, which // only the test moves, so how long the host takes between // writes cannot change the result. now := time.Now() w.SetNow(func() time.Time { return now }) require.NoError(t, w.Write( archive.ExportArchivedEvent{EventID: "a"}, 0, )) require.NoError(t, w.Write( archive.ExportArchivedEvent{EventID: "b"}, 0, )) // Two writes inside the debounce window trigger only the // initial open — no extra close/reopen. assert.Equal(t, 1, w.Reopens()) now = now.Add(debounce) require.NoError(t, w.Write( archive.ExportArchivedEvent{EventID: "c"}, 0, )) // A write after the window elapses closes and reopens once. assert.Equal(t, 2, w.Reopens()) } func TestArchiveWriter_ExpiryPrune(t *testing.T) { t.Parallel() path := filepath.Join(t.TempDir(), "archive-wh.db") w := archive.NewExportArchiveWriter( path, archiveTestLogger(), 0, ) require.NoError(t, w.Open(0)) old := archive.ExportArchivedEvent{ EventID: "old", ArchivedAt: time.Now().Add(-2 * time.Hour), } fresh := archive.ExportArchivedEvent{ EventID: "fresh", ArchivedAt: time.Now(), } require.NoError(t, w.DB().Create(&old).Error) require.NoError(t, w.DB().Create(&fresh).Error) // Reopening with a one-hour expiry prunes the old row. require.NoError(t, w.Reopen(time.Hour)) var got []archive.ExportArchivedEvent require.NoError(t, w.DB().Find(&got).Error) require.Len(t, got, 1) assert.Equal(t, "fresh", got[0].EventID) } func TestParseArchiveExpiry(t *testing.T) { t.Parallel() cases := []struct { name string in string want time.Duration wantErr bool }{ {"empty", "", 0, false}, {"explicit never", "never", 0, false}, {"hour", "1h", time.Hour, false}, {"thirty days", "720h", 720 * time.Hour, false}, {"minutes", "30m", 30 * time.Minute, false}, {"unparseable", "nonsense", 0, true}, {"days are not a Go unit", "7d", 0, true}, {"zero duration", "0s", 0, true}, {"bare zero", "0", 0, true}, {"negative duration", "-5h", 0, true}, } for _, tc := range cases { t.Run(tc.name, func(t *testing.T) { t.Parallel() got, err := archive.ParseExpiry(tc.in) if tc.wantErr { require.Error(t, err) return } require.NoError(t, err) assert.Equal(t, tc.want, got) }) } } // TestArchiveFileName pins the archive file name and the rules // that make a webhook or target name safe to put in it. func TestArchiveFileName(t *testing.T) { t.Parallel() const id = "3f2a1c9e-8d4b-4c1a-9e2f-0a1b2c3d4e5f" cases := []struct { name string webhook string target string want string }{ { "plain names", "orders", "archive", "archive-orders-archive-" + id + ".db", }, { "lowercased", "Orders", "Main Archive", "archive-orders-main-archive-" + id + ".db", }, { "a run of other characters is one dash", `a /\..b`, "c__--d", "archive-a-b-c-d-" + id + ".db", }, { "no dash at either end", " --orders!! ", "(archive)", "archive-orders-archive-" + id + ".db", }, { "path separators", "../../etc/passwd", "a/b", "archive-etc-passwd-a-b-" + id + ".db", }, { "letters outside ASCII are dropped", "Bestellungen Größe", "café", "archive-bestellungen-gr-e-caf-" + id + ".db", }, { "nothing left is unnamed", "", "!!!", "archive-unnamed-unnamed-" + id + ".db", }, { "cut to 40 characters", strings.Repeat("a", 50), "x", "archive-" + strings.Repeat("a", 40) + "-x-" + id + ".db", }, { "no dash left by the cut", strings.Repeat("a", 39) + " b", "x", "archive-" + strings.Repeat("a", 39) + "-x-" + id + ".db", }, } for _, tc := range cases { t.Run(tc.name, func(t *testing.T) { t.Parallel() assert.Equal( t, tc.want, archive.FileName(tc.webhook, tc.target, id), ) }) } } // TestWrite_EachTargetHasItsOwnArchive proves two // database targets of one webhook archive into separate files. func TestWrite_EachTargetHasItsOwnArchive(t *testing.T) { t.Parallel() env := setupArchiveTest(t) first := env.seedDatabaseTarget(t, 0, "") second := env.addDatabaseTarget(t, first.WebhookID, 0, "") event := newEvent(`{"n":1}`) for _, tgt := range []*database.Target{first, second} { require.NoError(t, env.writers.Write( databaseTargetDelivery(event, tgt), )) } require.NotEqual( t, env.archivePath(first), env.archivePath(second), ) assert.Equal( t, []string{event.ID}, archivedEventIDs(t, env.archivePath(first)), ) assert.Equal( t, []string{event.ID}, archivedEventIDs(t, env.archivePath(second)), ) } // TestRename_MovesTheFile proves a rename moves the archive, rows // and all, and that later writes go to the new name. func TestRename_MovesTheFile(t *testing.T) { t.Parallel() env := setupArchiveTest(t) tgt := env.seedDatabaseTarget(t, 0, "") oldPath := env.archivePath(tgt) first := newEvent(`{"n":1}`) require.NoError(t, env.writers.Write(databaseTargetDelivery(first, tgt))) require.FileExists(t, oldPath) require.NoError( t, env.writers.Rename(tgt.ID, "Orders", "Long Term"), ) newPath := filepath.Join( env.dataDir, "archive-orders-long-term-"+tgt.ID+".db", ) assert.NoFileExists(t, oldPath) assert.Equal(t, []string{first.ID}, archivedEventIDs(t, newPath)) second := newEvent(`{"n":2}`) require.NoError(t, env.writers.Write(databaseTargetDelivery(second, tgt))) assert.ElementsMatch( t, []string{first.ID, second.ID}, archivedEventIDs(t, newPath), ) assert.NoFileExists( t, oldPath, "a write after the rename must use the new name", ) } // TestRename_NeverReplacesAFile plants a file at the new name, once // the .db alone, once a lone -wal and once a lone -shm, and proves // each time that the rename is refused, the planted file survives, // and the archive keeps its name and its rows. func TestRename_NeverReplacesAFile(t *testing.T) { t.Parallel() for _, suffix := range archiveFileSuffixes() { t.Run("planted .db"+suffix, func(t *testing.T) { t.Parallel() env := setupArchiveTest(t) tgt := env.seedDatabaseTarget(t, 0, "") oldPath := env.archivePath(tgt) first := newEvent(`{"n":1}`) require.NoError( t, env.writers.Write(databaseTargetDelivery(first, tgt)), ) newPath := filepath.Join( env.dataDir, "archive-orders-long-term-"+tgt.ID+".db", ) plantedPath := newPath + suffix require.NoError( t, os.WriteFile(plantedPath, []byte("planted"), 0o600), ) require.ErrorIs( t, env.writers.Rename(tgt.ID, "Orders", "Long Term"), archive.ErrNameTaken, ) //nolint:gosec // reads the file the test planted under t.TempDir() planted, err := os.ReadFile(plantedPath) require.NoError(t, err) assert.Equal(t, "planted", string(planted)) second := newEvent(`{"n":2}`) require.NoError( t, env.writers.Write(databaseTargetDelivery(second, tgt)), ) assert.ElementsMatch( t, []string{first.ID, second.ID}, archivedEventIDs(t, oldPath), ) }) } } // TestRename_BeforeTheNameIsSaved covers the order the handlers // use: they rename before they save the new name, so a delivery in // between must write under the new name although the main database // still has the old one. It also shows that renaming an archive that // does not exist yet is not an error. func TestRename_BeforeTheNameIsSaved(t *testing.T) { t.Parallel() env := setupArchiveTest(t) tgt := env.seedDatabaseTarget(t, 0, "") require.NoError( t, env.writers.Rename(tgt.ID, "Orders", "Archive"), ) event := newEvent(`{"n":1}`) require.NoError(t, env.writers.Write(databaseTargetDelivery(event, tgt))) assert.FileExists( t, filepath.Join( env.dataDir, "archive-orders-archive-"+tgt.ID+".db", ), ) assert.NoFileExists(t, env.archivePath(tgt)) } // TestArchiveWriter_RenameMovesSidecars proves a rename carries // the -wal and -shm a crash can leave beside an archive no handle // has opened since. SQLite finds them by name, so a -wal left // behind would lose the transactions it holds. func TestArchiveWriter_RenameMovesSidecars(t *testing.T) { t.Parallel() dir := t.TempDir() oldPath := filepath.Join(dir, "archive-old.db") newPath := filepath.Join(dir, "archive-new.db") for _, suffix := range archiveFileSuffixes() { require.NoError( t, os.WriteFile(oldPath+suffix, []byte(suffix), 0o600), ) } w := archive.NewExportArchiveWriter( oldPath, archiveTestLogger(), 0, ) require.NoError(t, w.Rename("archive-new.db")) for _, suffix := range archiveFileSuffixes() { assert.NoFileExists(t, oldPath+suffix) assert.FileExists(t, newPath+suffix) } assert.Equal(t, newPath, w.Path()) } // TestArchiveWriter_RenameMovesBackOnFailure makes the -wal fail to // move after the .db has moved, and proves the .db is moved back, so // the archive is never split across two names. The new name is 255 // bytes, the longest a file name may be, so the .db can take it but // the -wal, four bytes longer, cannot. func TestArchiveWriter_RenameMovesBackOnFailure(t *testing.T) { t.Parallel() dir := t.TempDir() oldPath := filepath.Join(dir, "archive-old.db") newName := strings.Repeat("a", 252) + ".db" for _, suffix := range archiveFileSuffixes() { require.NoError( t, os.WriteFile(oldPath+suffix, []byte(suffix), 0o600), ) } w := archive.NewExportArchiveWriter( oldPath, archiveTestLogger(), 0, ) require.Error(t, w.Rename(newName)) for _, suffix := range archiveFileSuffixes() { assert.FileExists(t, oldPath+suffix) } assert.NoFileExists(t, filepath.Join(dir, newName)) assert.Equal(t, oldPath, w.Path()) } // TestArchiveWriter_RenameMovesBackEveryFile renames a target with a // file without a period and a file for a month, and proves that when // the month's file fails to move, the file already moved is moved back: // both files are under the old name with their rows, and nothing is // under the new name. The new name is 251 bytes, so the file without a // period, with its -wal and -shm, can take it, but the month's file, // eight bytes longer, cannot. func TestArchiveWriter_RenameMovesBackEveryFile(t *testing.T) { t.Parallel() dir := t.TempDir() oldPath := filepath.Join(dir, "archive-old.db") monthPath := filepath.Join(dir, "archive-old-2026-03.db") w := archive.NewExportArchiveWriter(oldPath, archiveTestLogger(), 0) require.NoError(t, w.WritePeriod( archive.ExportArchivedEvent{EventID: "in-none"}, 0, "", )) require.NoError(t, w.WritePeriod( archive.ExportArchivedEvent{EventID: "in-month"}, 0, "2026-03", )) require.Error(t, w.Rename(strings.Repeat("a", 248)+".db")) assert.Equal(t, []string{"in-none"}, archivedEventIDs(t, oldPath)) assert.Equal(t, []string{"in-month"}, archivedEventIDs(t, monthPath)) entries, err := os.ReadDir(dir) require.NoError(t, err) for _, entry := range entries { assert.True(t, strings.HasPrefix(entry.Name(), "archive-old"), "%s is not under the old name", entry.Name()) } assert.Equal(t, oldPath, w.Path()) } // TestRenameLock_HeldByOneEditAtATime holds the rename lock, as an edit // does from loading the stored names until it has saved them. While it // is held, the edit's rename and an archive write still finish, since // neither takes it, and the event ends up under the new name whichever // ran first. A second edit waits for the lock until the first lets go. func TestRenameLock_HeldByOneEditAtATime(t *testing.T) { t.Parallel() env := setupArchiveTest(t) tgt := env.seedDatabaseTarget(t, 0, "") event := newEvent(`{"n":1}`) env.writers.RenameLock().Lock() finished := make(chan error, 2) go func() { finished <- env.writers.Rename(tgt.ID, "Orders", "Long Term") }() go func() { finished <- env.writers.Write(databaseTargetDelivery(event, tgt)) }() for range 2 { select { case err := <-finished: require.NoError(t, err) case <-time.After(10 * time.Second): t.Fatal("a rename or a write waited for the rename lock") } } newPath := filepath.Join( env.dataDir, "archive-orders-long-term-"+tgt.ID+".db", ) assert.Equal(t, []string{event.ID}, archivedEventIDs(t, newPath)) secondEdit := make(chan struct{}) go func() { env.writers.RenameLock().Lock() close(secondEdit) env.writers.RenameLock().Unlock() }() select { case <-secondEdit: t.Fatal("a second edit took the rename lock while it was held") case <-time.After(100 * time.Millisecond): } env.writers.RenameLock().Unlock() select { case <-secondEdit: case <-time.After(10 * time.Second): t.Fatal("a second edit did not get the rename lock once released") } }