diff --git a/README.md b/README.md index a7706e3..28b8606 100644 --- a/README.md +++ b/README.md @@ -1050,7 +1050,8 @@ Archive databases are the one exception the service is built for: the archive writer closes and reopens its handle around writes (debounced to at most one reopen per second), so an operator can move an `archive-….db` away for offline retention while the service runs, -and it is recreated on the next write. See +and it is recreated on the next write. The webhook page names each +`database` target's archive file. See [Database Architecture](#database-architecture). That is a move-the-file-away workflow, not a substitute for the backup procedures above. @@ -2033,6 +2034,14 @@ Because each `database` target has its own archive file, a target's webhook with different expiries keep two archives, each pruned on its own schedule. +The webhook page shows, for each `database` target, its archive file's +name, its size on disk and when it was last written. The size counts +the `.db` and its `-wal` together, and the last write is the later of +their two modification times, since a write lands in the `-wal` first. +Both are read from the files' metadata; the archive is never opened. +Before the first write, and after the file has been moved away, the page +shows `not created yet` beside the name. + Each `database` target on the webhook page has a **Download** button, which returns its archive as one gzipped JSON file, `archive-{webhook_name}-{target_name}-{YYYYMMDDTHHMMSSZ}.json.gz`, the diff --git a/internal/delivery/target_database_archive.go b/internal/delivery/target_database_archive.go index 4730602..c58664c 100644 --- a/internal/delivery/target_database_archive.go +++ b/internal/delivery/target_database_archive.go @@ -515,6 +515,47 @@ func (w *archiveWriter) prune(expiry time.Duration) { } } +// ArchiveFileInfo is what the metadata of a database target's archive +// file says about it. +type ArchiveFileInfo struct { + // Size is the bytes on disk of the file and its -wal together. + Size int64 + + // Written is when the file or its -wal was last modified, whichever + // is later: a write lands in the -wal first. + Written time.Time +} + +// StatArchive reads the metadata of the archive file at path and of +// its -wal, without opening the archive. With no file at path, which is +// so before the first write and after the operator moved it away, the +// error wraps fs.ErrNotExist. +func StatArchive(path string) (ArchiveFileInfo, error) { + file, err := os.Stat(path) + if err != nil { + return ArchiveFileInfo{}, err + } + + info := ArchiveFileInfo{Size: file.Size(), Written: file.ModTime()} + + wal, err := os.Stat(path + "-wal") + if errors.Is(err, fs.ErrNotExist) { + return info, nil + } + + if err != nil { + return ArchiveFileInfo{}, err + } + + info.Size += wal.Size() + + if wal.ModTime().After(info.Written) { + info.Written = wal.ModTime() + } + + return info, nil +} + // fileExists reports whether a path currently exists. func fileExists(path string) bool { _, err := os.Stat(path) diff --git a/internal/delivery/target_database_test.go b/internal/delivery/target_database_test.go index 4a12dc7..516e579 100644 --- a/internal/delivery/target_database_test.go +++ b/internal/delivery/target_database_test.go @@ -3,6 +3,7 @@ package delivery_test import ( "database/sql" "fmt" + "io/fs" "log/slog" "os" "path/filepath" @@ -183,6 +184,49 @@ func TestArchiveWriter_RecreatesAfterRemoval( assert.Equal(t, "b", got[0].EventID) } +// TestStatArchive proves StatArchive finds no file before the first +// write; after a write still held in the -wal, counts the -wal in the +// size and takes its later time as the last write; and finds no file +// again once the file has been moved away. +func TestStatArchive(t *testing.T) { + t.Parallel() + + path := filepath.Join(t.TempDir(), "archive-wh.db") + + _, err := delivery.StatArchive(path) + require.ErrorIs(t, err, fs.ErrNotExist) + + // With the clock stopped, the reopen debounce never passes, so + // the handle stays open after the write. + stopped := time.Now() + w := delivery.NewExportArchiveWriter(path, archiveTestLogger(), 0) + w.SetNow(func() time.Time { return stopped }) + + require.NoError(t, w.Write(delivery.ExportArchivedEvent{EventID: "a"}, 0)) + + written := time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC) + earlier := written.Add(-time.Hour) + require.NoError(t, os.Chtimes(path, earlier, earlier)) + require.NoError(t, os.Chtimes(path+"-wal", written, written)) + + file, err := os.Stat(path) + require.NoError(t, err) + + wal, err := os.Stat(path + "-wal") + require.NoError(t, err) + require.Positive(t, wal.Size()) + + got, err := delivery.StatArchive(path) + require.NoError(t, err) + assert.Equal(t, file.Size()+wal.Size(), got.Size) + assert.True(t, written.Equal(got.Written), got.Written) + + removeArchiveFiles(t, path) + + _, err = delivery.StatArchive(path) + require.ErrorIs(t, err, fs.ErrNotExist) +} + func TestArchiveWriter_ReopenDebounce(t *testing.T) { t.Parallel() diff --git a/internal/handlers/source_management.go b/internal/handlers/source_management.go index 377eac8..bfa9eb7 100644 --- a/internal/handlers/source_management.go +++ b/internal/handlers/source_management.go @@ -521,7 +521,7 @@ func (h *Handlers) renderSourceDetail( // target's stored config blob holds a credential, and it // must never reach a template. "Entrypoints": NewEntrypointViews(entrypoints), - "Targets": delivery.NewTargetViews(targets), + "Targets": h.targetRows(&webhook, targets), "Events": events, "BaseURL": baseURL, "Stats": h.loadWebhookStats(webhook.ID, entrypoints, targets), diff --git a/internal/handlers/target_list.go b/internal/handlers/target_list.go new file mode 100644 index 0000000..579e079 --- /dev/null +++ b/internal/handlers/target_list.go @@ -0,0 +1,90 @@ +package handlers + +import ( + "errors" + "io/fs" + "path/filepath" + "time" + + "github.com/dustin/go-humanize" + "sneak.berlin/go/webhooker/internal/database" + "sneak.berlin/go/webhooker/internal/delivery" +) + +// TargetRowView is one row of the target list on a webhook's page. +type TargetRowView struct { + delivery.TargetView + + // Archive is a database target's archive file, and nil for a target + // of any other type. + Archive *ArchiveFileView +} + +// ArchiveFileView is what a database target's row shows about its +// archive file. +type ArchiveFileView struct { + Name string + + // Note stands in for the size and the last write when there are + // none to show, and is empty when there are. + Note string + + // Size is the size on disk. Written is how long ago the file was + // last written, and WrittenUTC the full time the page shows on + // hover. + Size string + Written string + WrittenUTC string +} + +// targetRows projects a webhook's targets for the target list on its +// page. +func (h *Handlers) targetRows( + webhook *database.Webhook, targets []database.Target, +) []TargetRowView { + views := delivery.NewTargetViews(targets) + rows := make([]TargetRowView, len(views)) + + // NewTargetViews returns one view per target, in order. + for i := range views { + rows[i].TargetView = views[i] + + if targets[i].Type == database.TargetTypeDatabase { + rows[i].Archive = h.archiveFileView(webhook, &targets[i]) + } + } + + return rows +} + +// archiveFileView describes a database target's archive file from the +// file's metadata alone; the archive is never opened. The file is found +// by the name the archive writer uses, so it follows a rename of the +// webhook or the target. +func (h *Handlers) archiveFileView( + webhook *database.Webhook, target *database.Target, +) *ArchiveFileView { + path := delivery.ArchivePath(h.dbMgr, webhook, target) + view := &ArchiveFileView{Name: filepath.Base(path)} + + file, err := delivery.StatArchive(path) + + switch { + case errors.Is(err, fs.ErrNotExist): + view.Note = "not created yet" + case err != nil: + h.log.Error( + "failed to read archive file metadata", + "target_id", target.ID, + "error", err, + ) + + view.Note = "could not be read" + default: + view.Size = humanize.Bytes(uint64(file.Size)) //nolint:gosec // never negative + view.Written = humanize.Time(file.Written) + view.WrittenUTC = file.Written.UTC().Format(time.DateTime) + " UTC" + } + + return view +} diff --git a/internal/handlers/target_list_test.go b/internal/handlers/target_list_test.go new file mode 100644 index 0000000..75e46d6 --- /dev/null +++ b/internal/handlers/target_list_test.go @@ -0,0 +1,71 @@ +package handlers_test + +import ( + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "sneak.berlin/go/webhooker/internal/database" + "sneak.berlin/go/webhooker/internal/delivery" + "sneak.berlin/go/webhooker/internal/handlers" + "sneak.berlin/go/webhooker/internal/session" +) + +// TestHandleSourceDetail_ShowsArchiveFile proves a database target's +// row names its archive file and says "not created yet" before the +// first write, adds the file's size and last write once it has one row, +// and says "not created yet" again once the file has been moved away. +// A target of another type shows no archive file. +func TestHandleSourceDetail_ShowsArchiveFile(t *testing.T) { + t.Parallel() + + var ( + h *handlers.Handlers + sess *session.Session + db *database.Database + dbMgr *database.WebhookDBManager + ) + + app := newTestApp(t, &h, &sess, &db, &dbMgr) + app.RequireStart() + + t.Cleanup(app.RequireStop) + + wh := seedWebhook(t, db) + archive := seedTarget(t, db, wh.ID, database.TargetTypeDatabase) + seedTarget(t, db, wh.ID, database.TargetTypeLog) + + path := delivery.ArchivePath(dbMgr, wh, archive) + + body := renderSourceDetailPage(t, h, sess, wh.ID) + assert.Equal(t, 1, strings.Count(body, "Archive File:")) + assert.Contains(t, body, filepath.Base(path)) + assert.Contains(t, body, "not created yet") + assert.NotContains(t, body, "Archive Size:") + + seedArchive(t, path, 1, 100) + + file, err := os.Stat(path) + require.NoError(t, err) + + body = renderSourceDetailPage(t, h, sess, wh.ID) + assert.Contains(t, body, filepath.Base(path)) + assert.NotContains(t, body, "not created yet") + assert.Regexp(t, + `Archive Size:\s*[1-9][0-9.]* [kM]?B`, body, + ) + assert.Contains(t, body, + `title="`+file.ModTime().UTC().Format(time.DateTime)+` UTC"`, + ) + + require.NoError(t, os.Rename(path, filepath.Join(t.TempDir(), "moved.db"))) + + body = renderSourceDetailPage(t, h, sess, wh.ID) + assert.Contains(t, body, filepath.Base(path)) + assert.Contains(t, body, "not created yet") + assert.NotContains(t, body, "Archive Size:") +} diff --git a/templates/source_detail.html b/templates/source_detail.html index b37cda6..98045a0 100644 --- a/templates/source_detail.html +++ b/templates/source_detail.html @@ -179,6 +179,23 @@ {{.Value}} {{end}} + {{with .Archive}} +
+ Archive File: + {{.Name}} + {{with .Note}}({{.}}){{end}} +
+ {{if .Size}} +
+ Archive Size: + {{.Size}} +
+
+ Last Written: + {{.Written}} +
+ {{end}} + {{end}} {{else}}
No targets configured.