From 2967c475a156997aa5ae1d42dd301582c60db8ba Mon Sep 17 00:00:00 2001 From: clawbot <35+clawbot@noreply.example.org> Date: Fri, 2 Oct 2026 21:22:33 +0200 Subject: [PATCH] Show a database target's archive file, size and last write in the target list (closes #397) For a database target, the target list showed only its expiry, so the archive file the README's backup and move-away advice depend on could only be found from a shell on the host. Each database target now shows its archive file's name, its size on disk (the file and its -wal together) and when it was last written, relative with the full UTC time on hover, all from the files' metadata without opening the archive. Before the first write, and after the file has been moved away, it shows the name and "not created yet". Tests cover all three states. Model: opus-5-5 --- README.md | 11 ++- internal/delivery/target_database_archive.go | 41 +++++++++ internal/delivery/target_database_test.go | 44 ++++++++++ internal/handlers/source_management.go | 2 +- internal/handlers/target_list.go | 90 ++++++++++++++++++++ internal/handlers/target_list_test.go | 71 +++++++++++++++ templates/source_detail.html | 17 ++++ 7 files changed, 274 insertions(+), 2 deletions(-) create mode 100644 internal/handlers/target_list.go create mode 100644 internal/handlers/target_list_test.go diff --git a/README.md b/README.md index a7706e3..28b8606 100644 --- a/README.md +++ b/README.md @@ -1050,7 +1050,8 @@ Archive databases are the one exception the service is built for: the archive writer closes and reopens its handle around writes (debounced to at most one reopen per second), so an operator can move an `archive-….db` away for offline retention while the service runs, -and it is recreated on the next write. See +and it is recreated on the next write. The webhook page names each +`database` target's archive file. See [Database Architecture](#database-architecture). That is a move-the-file-away workflow, not a substitute for the backup procedures above. @@ -2033,6 +2034,14 @@ Because each `database` target has its own archive file, a target's webhook with different expiries keep two archives, each pruned on its own schedule. +The webhook page shows, for each `database` target, its archive file's +name, its size on disk and when it was last written. The size counts +the `.db` and its `-wal` together, and the last write is the later of +their two modification times, since a write lands in the `-wal` first. +Both are read from the files' metadata; the archive is never opened. +Before the first write, and after the file has been moved away, the page +shows `not created yet` beside the name. + Each `database` target on the webhook page has a **Download** button, which returns its archive as one gzipped JSON file, `archive-{webhook_name}-{target_name}-{YYYYMMDDTHHMMSSZ}.json.gz`, the diff --git a/internal/delivery/target_database_archive.go b/internal/delivery/target_database_archive.go index 4730602..c58664c 100644 --- a/internal/delivery/target_database_archive.go +++ b/internal/delivery/target_database_archive.go @@ -515,6 +515,47 @@ func (w *archiveWriter) prune(expiry time.Duration) { } } +// ArchiveFileInfo is what the metadata of a database target's archive +// file says about it. +type ArchiveFileInfo struct { + // Size is the bytes on disk of the file and its -wal together. + Size int64 + + // Written is when the file or its -wal was last modified, whichever + // is later: a write lands in the -wal first. + Written time.Time +} + +// StatArchive reads the metadata of the archive file at path and of +// its -wal, without opening the archive. With no file at path, which is +// so before the first write and after the operator moved it away, the +// error wraps fs.ErrNotExist. +func StatArchive(path string) (ArchiveFileInfo, error) { + file, err := os.Stat(path) + if err != nil { + return ArchiveFileInfo{}, err + } + + info := ArchiveFileInfo{Size: file.Size(), Written: file.ModTime()} + + wal, err := os.Stat(path + "-wal") + if errors.Is(err, fs.ErrNotExist) { + return info, nil + } + + if err != nil { + return ArchiveFileInfo{}, err + } + + info.Size += wal.Size() + + if wal.ModTime().After(info.Written) { + info.Written = wal.ModTime() + } + + return info, nil +} + // fileExists reports whether a path currently exists. func fileExists(path string) bool { _, err := os.Stat(path) diff --git a/internal/delivery/target_database_test.go b/internal/delivery/target_database_test.go index 4a12dc7..516e579 100644 --- a/internal/delivery/target_database_test.go +++ b/internal/delivery/target_database_test.go @@ -3,6 +3,7 @@ package delivery_test import ( "database/sql" "fmt" + "io/fs" "log/slog" "os" "path/filepath" @@ -183,6 +184,49 @@ func TestArchiveWriter_RecreatesAfterRemoval( assert.Equal(t, "b", got[0].EventID) } +// TestStatArchive proves StatArchive finds no file before the first +// write; after a write still held in the -wal, counts the -wal in the +// size and takes its later time as the last write; and finds no file +// again once the file has been moved away. +func TestStatArchive(t *testing.T) { + t.Parallel() + + path := filepath.Join(t.TempDir(), "archive-wh.db") + + _, err := delivery.StatArchive(path) + require.ErrorIs(t, err, fs.ErrNotExist) + + // With the clock stopped, the reopen debounce never passes, so + // the handle stays open after the write. + stopped := time.Now() + w := delivery.NewExportArchiveWriter(path, archiveTestLogger(), 0) + w.SetNow(func() time.Time { return stopped }) + + require.NoError(t, w.Write(delivery.ExportArchivedEvent{EventID: "a"}, 0)) + + written := time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC) + earlier := written.Add(-time.Hour) + require.NoError(t, os.Chtimes(path, earlier, earlier)) + require.NoError(t, os.Chtimes(path+"-wal", written, written)) + + file, err := os.Stat(path) + require.NoError(t, err) + + wal, err := os.Stat(path + "-wal") + require.NoError(t, err) + require.Positive(t, wal.Size()) + + got, err := delivery.StatArchive(path) + require.NoError(t, err) + assert.Equal(t, file.Size()+wal.Size(), got.Size) + assert.True(t, written.Equal(got.Written), got.Written) + + removeArchiveFiles(t, path) + + _, err = delivery.StatArchive(path) + require.ErrorIs(t, err, fs.ErrNotExist) +} + func TestArchiveWriter_ReopenDebounce(t *testing.T) { t.Parallel() diff --git a/internal/handlers/source_management.go b/internal/handlers/source_management.go index 377eac8..bfa9eb7 100644 --- a/internal/handlers/source_management.go +++ b/internal/handlers/source_management.go @@ -521,7 +521,7 @@ func (h *Handlers) renderSourceDetail( // target's stored config blob holds a credential, and it // must never reach a template. "Entrypoints": NewEntrypointViews(entrypoints), - "Targets": delivery.NewTargetViews(targets), + "Targets": h.targetRows(&webhook, targets), "Events": events, "BaseURL": baseURL, "Stats": h.loadWebhookStats(webhook.ID, entrypoints, targets), diff --git a/internal/handlers/target_list.go b/internal/handlers/target_list.go new file mode 100644 index 0000000..579e079 --- /dev/null +++ b/internal/handlers/target_list.go @@ -0,0 +1,90 @@ +package handlers + +import ( + "errors" + "io/fs" + "path/filepath" + "time" + + "github.com/dustin/go-humanize" + "sneak.berlin/go/webhooker/internal/database" + "sneak.berlin/go/webhooker/internal/delivery" +) + +// TargetRowView is one row of the target list on a webhook's page. +type TargetRowView struct { + delivery.TargetView + + // Archive is a database target's archive file, and nil for a target + // of any other type. + Archive *ArchiveFileView +} + +// ArchiveFileView is what a database target's row shows about its +// archive file. +type ArchiveFileView struct { + Name string + + // Note stands in for the size and the last write when there are + // none to show, and is empty when there are. + Note string + + // Size is the size on disk. Written is how long ago the file was + // last written, and WrittenUTC the full time the page shows on + // hover. + Size string + Written string + WrittenUTC string +} + +// targetRows projects a webhook's targets for the target list on its +// page. +func (h *Handlers) targetRows( + webhook *database.Webhook, targets []database.Target, +) []TargetRowView { + views := delivery.NewTargetViews(targets) + rows := make([]TargetRowView, len(views)) + + // NewTargetViews returns one view per target, in order. + for i := range views { + rows[i].TargetView = views[i] + + if targets[i].Type == database.TargetTypeDatabase { + rows[i].Archive = h.archiveFileView(webhook, &targets[i]) + } + } + + return rows +} + +// archiveFileView describes a database target's archive file from the +// file's metadata alone; the archive is never opened. The file is found +// by the name the archive writer uses, so it follows a rename of the +// webhook or the target. +func (h *Handlers) archiveFileView( + webhook *database.Webhook, target *database.Target, +) *ArchiveFileView { + path := delivery.ArchivePath(h.dbMgr, webhook, target) + view := &ArchiveFileView{Name: filepath.Base(path)} + + file, err := delivery.StatArchive(path) + + switch { + case errors.Is(err, fs.ErrNotExist): + view.Note = "not created yet" + case err != nil: + h.log.Error( + "failed to read archive file metadata", + "target_id", target.ID, + "error", err, + ) + + view.Note = "could not be read" + default: + view.Size = humanize.Bytes(uint64(file.Size)) //nolint:gosec // never negative + view.Written = humanize.Time(file.Written) + view.WrittenUTC = file.Written.UTC().Format(time.DateTime) + " UTC" + } + + return view +} diff --git a/internal/handlers/target_list_test.go b/internal/handlers/target_list_test.go new file mode 100644 index 0000000..75e46d6 --- /dev/null +++ b/internal/handlers/target_list_test.go @@ -0,0 +1,71 @@ +package handlers_test + +import ( + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "sneak.berlin/go/webhooker/internal/database" + "sneak.berlin/go/webhooker/internal/delivery" + "sneak.berlin/go/webhooker/internal/handlers" + "sneak.berlin/go/webhooker/internal/session" +) + +// TestHandleSourceDetail_ShowsArchiveFile proves a database target's +// row names its archive file and says "not created yet" before the +// first write, adds the file's size and last write once it has one row, +// and says "not created yet" again once the file has been moved away. +// A target of another type shows no archive file. +func TestHandleSourceDetail_ShowsArchiveFile(t *testing.T) { + t.Parallel() + + var ( + h *handlers.Handlers + sess *session.Session + db *database.Database + dbMgr *database.WebhookDBManager + ) + + app := newTestApp(t, &h, &sess, &db, &dbMgr) + app.RequireStart() + + t.Cleanup(app.RequireStop) + + wh := seedWebhook(t, db) + archive := seedTarget(t, db, wh.ID, database.TargetTypeDatabase) + seedTarget(t, db, wh.ID, database.TargetTypeLog) + + path := delivery.ArchivePath(dbMgr, wh, archive) + + body := renderSourceDetailPage(t, h, sess, wh.ID) + assert.Equal(t, 1, strings.Count(body, "Archive File:")) + assert.Contains(t, body, filepath.Base(path)) + assert.Contains(t, body, "not created yet") + assert.NotContains(t, body, "Archive Size:") + + seedArchive(t, path, 1, 100) + + file, err := os.Stat(path) + require.NoError(t, err) + + body = renderSourceDetailPage(t, h, sess, wh.ID) + assert.Contains(t, body, filepath.Base(path)) + assert.NotContains(t, body, "not created yet") + assert.Regexp(t, + `Archive Size:\s*[1-9][0-9.]* [kM]?B`, body, + ) + assert.Contains(t, body, + `title="`+file.ModTime().UTC().Format(time.DateTime)+` UTC"`, + ) + + require.NoError(t, os.Rename(path, filepath.Join(t.TempDir(), "moved.db"))) + + body = renderSourceDetailPage(t, h, sess, wh.ID) + assert.Contains(t, body, filepath.Base(path)) + assert.Contains(t, body, "not created yet") + assert.NotContains(t, body, "Archive Size:") +} diff --git a/templates/source_detail.html b/templates/source_detail.html index b37cda6..98045a0 100644 --- a/templates/source_detail.html +++ b/templates/source_detail.html @@ -179,6 +179,23 @@ {{.Value}} {{end}} + {{with .Archive}} +
+ Archive File: + {{.Name}} + {{with .Note}}({{.}}){{end}} +
+ {{if .Size}} +
+ Archive Size: + {{.Size}} +
+
+ Last Written: + {{.Written}} +
+ {{end}} + {{end}} {{else}}
No targets configured.