Show a database target's archive file, size and last write in the target list (closes #397)
check / check (push) Waiting to run
check / check (push) Waiting to run
For a database target, the target list showed only its expiry, so the archive file the README's backup and move-away advice depend on could only be found from a shell on the host. Each database target now shows its archive file's name, its size on disk (the file and its -wal together) and when it was last written, relative with the full UTC time on hover, all from the files' metadata without opening the archive. Before the first write, and after the file has been moved away, it shows the name and "not created yet". Tests cover all three states. Model: opus-5-5
This commit was merged in pull request #469.
This commit is contained in:
@@ -1050,7 +1050,8 @@ Archive databases are the one exception the service is built for: the
|
||||
archive writer closes and reopens its handle around writes (debounced
|
||||
to at most one reopen per second), so an operator can move an
|
||||
`archive-….db` away for offline retention while the service runs,
|
||||
and it is recreated on the next write. See
|
||||
and it is recreated on the next write. The webhook page names each
|
||||
`database` target's archive file. See
|
||||
[Database Architecture](#database-architecture). That is a
|
||||
move-the-file-away workflow, not a substitute for the backup procedures
|
||||
above.
|
||||
@@ -2033,6 +2034,14 @@ Because each `database` target has its own archive file, a target's
|
||||
webhook with different expiries keep two archives, each pruned on its
|
||||
own schedule.
|
||||
|
||||
The webhook page shows, for each `database` target, its archive file's
|
||||
name, its size on disk and when it was last written. The size counts
|
||||
the `.db` and its `-wal` together, and the last write is the later of
|
||||
their two modification times, since a write lands in the `-wal` first.
|
||||
Both are read from the files' metadata; the archive is never opened.
|
||||
Before the first write, and after the file has been moved away, the page
|
||||
shows `not created yet` beside the name.
|
||||
|
||||
Each `database` target on the webhook page has a **Download** button,
|
||||
which returns its archive as one gzipped JSON file,
|
||||
`archive-{webhook_name}-{target_name}-{YYYYMMDDTHHMMSSZ}.json.gz`, the
|
||||
|
||||
@@ -515,6 +515,47 @@ func (w *archiveWriter) prune(expiry time.Duration) {
|
||||
}
|
||||
}
|
||||
|
||||
// ArchiveFileInfo is what the metadata of a database target's archive
|
||||
// file says about it.
|
||||
type ArchiveFileInfo struct {
|
||||
// Size is the bytes on disk of the file and its -wal together.
|
||||
Size int64
|
||||
|
||||
// Written is when the file or its -wal was last modified, whichever
|
||||
// is later: a write lands in the -wal first.
|
||||
Written time.Time
|
||||
}
|
||||
|
||||
// StatArchive reads the metadata of the archive file at path and of
|
||||
// its -wal, without opening the archive. With no file at path, which is
|
||||
// so before the first write and after the operator moved it away, the
|
||||
// error wraps fs.ErrNotExist.
|
||||
func StatArchive(path string) (ArchiveFileInfo, error) {
|
||||
file, err := os.Stat(path)
|
||||
if err != nil {
|
||||
return ArchiveFileInfo{}, err
|
||||
}
|
||||
|
||||
info := ArchiveFileInfo{Size: file.Size(), Written: file.ModTime()}
|
||||
|
||||
wal, err := os.Stat(path + "-wal")
|
||||
if errors.Is(err, fs.ErrNotExist) {
|
||||
return info, nil
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
return ArchiveFileInfo{}, err
|
||||
}
|
||||
|
||||
info.Size += wal.Size()
|
||||
|
||||
if wal.ModTime().After(info.Written) {
|
||||
info.Written = wal.ModTime()
|
||||
}
|
||||
|
||||
return info, nil
|
||||
}
|
||||
|
||||
// fileExists reports whether a path currently exists.
|
||||
func fileExists(path string) bool {
|
||||
_, err := os.Stat(path)
|
||||
|
||||
@@ -3,6 +3,7 @@ package delivery_test
|
||||
import (
|
||||
"database/sql"
|
||||
"fmt"
|
||||
"io/fs"
|
||||
"log/slog"
|
||||
"os"
|
||||
"path/filepath"
|
||||
@@ -183,6 +184,49 @@ func TestArchiveWriter_RecreatesAfterRemoval(
|
||||
assert.Equal(t, "b", got[0].EventID)
|
||||
}
|
||||
|
||||
// TestStatArchive proves StatArchive finds no file before the first
|
||||
// write; after a write still held in the -wal, counts the -wal in the
|
||||
// size and takes its later time as the last write; and finds no file
|
||||
// again once the file has been moved away.
|
||||
func TestStatArchive(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
path := filepath.Join(t.TempDir(), "archive-wh.db")
|
||||
|
||||
_, err := delivery.StatArchive(path)
|
||||
require.ErrorIs(t, err, fs.ErrNotExist)
|
||||
|
||||
// With the clock stopped, the reopen debounce never passes, so
|
||||
// the handle stays open after the write.
|
||||
stopped := time.Now()
|
||||
w := delivery.NewExportArchiveWriter(path, archiveTestLogger(), 0)
|
||||
w.SetNow(func() time.Time { return stopped })
|
||||
|
||||
require.NoError(t, w.Write(delivery.ExportArchivedEvent{EventID: "a"}, 0))
|
||||
|
||||
written := time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC)
|
||||
earlier := written.Add(-time.Hour)
|
||||
require.NoError(t, os.Chtimes(path, earlier, earlier))
|
||||
require.NoError(t, os.Chtimes(path+"-wal", written, written))
|
||||
|
||||
file, err := os.Stat(path)
|
||||
require.NoError(t, err)
|
||||
|
||||
wal, err := os.Stat(path + "-wal")
|
||||
require.NoError(t, err)
|
||||
require.Positive(t, wal.Size())
|
||||
|
||||
got, err := delivery.StatArchive(path)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, file.Size()+wal.Size(), got.Size)
|
||||
assert.True(t, written.Equal(got.Written), got.Written)
|
||||
|
||||
removeArchiveFiles(t, path)
|
||||
|
||||
_, err = delivery.StatArchive(path)
|
||||
require.ErrorIs(t, err, fs.ErrNotExist)
|
||||
}
|
||||
|
||||
func TestArchiveWriter_ReopenDebounce(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
|
||||
@@ -521,7 +521,7 @@ func (h *Handlers) renderSourceDetail(
|
||||
// target's stored config blob holds a credential, and it
|
||||
// must never reach a template.
|
||||
"Entrypoints": NewEntrypointViews(entrypoints),
|
||||
"Targets": delivery.NewTargetViews(targets),
|
||||
"Targets": h.targetRows(&webhook, targets),
|
||||
"Events": events,
|
||||
"BaseURL": baseURL,
|
||||
"Stats": h.loadWebhookStats(webhook.ID, entrypoints, targets),
|
||||
|
||||
@@ -0,0 +1,90 @@
|
||||
package handlers
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"io/fs"
|
||||
"path/filepath"
|
||||
"time"
|
||||
|
||||
"github.com/dustin/go-humanize"
|
||||
"sneak.berlin/go/webhooker/internal/database"
|
||||
"sneak.berlin/go/webhooker/internal/delivery"
|
||||
)
|
||||
|
||||
// TargetRowView is one row of the target list on a webhook's page.
|
||||
type TargetRowView struct {
|
||||
delivery.TargetView
|
||||
|
||||
// Archive is a database target's archive file, and nil for a target
|
||||
// of any other type.
|
||||
Archive *ArchiveFileView
|
||||
}
|
||||
|
||||
// ArchiveFileView is what a database target's row shows about its
|
||||
// archive file.
|
||||
type ArchiveFileView struct {
|
||||
Name string
|
||||
|
||||
// Note stands in for the size and the last write when there are
|
||||
// none to show, and is empty when there are.
|
||||
Note string
|
||||
|
||||
// Size is the size on disk. Written is how long ago the file was
|
||||
// last written, and WrittenUTC the full time the page shows on
|
||||
// hover.
|
||||
Size string
|
||||
Written string
|
||||
WrittenUTC string
|
||||
}
|
||||
|
||||
// targetRows projects a webhook's targets for the target list on its
|
||||
// page.
|
||||
func (h *Handlers) targetRows(
|
||||
webhook *database.Webhook, targets []database.Target,
|
||||
) []TargetRowView {
|
||||
views := delivery.NewTargetViews(targets)
|
||||
rows := make([]TargetRowView, len(views))
|
||||
|
||||
// NewTargetViews returns one view per target, in order.
|
||||
for i := range views {
|
||||
rows[i].TargetView = views[i]
|
||||
|
||||
if targets[i].Type == database.TargetTypeDatabase {
|
||||
rows[i].Archive = h.archiveFileView(webhook, &targets[i])
|
||||
}
|
||||
}
|
||||
|
||||
return rows
|
||||
}
|
||||
|
||||
// archiveFileView describes a database target's archive file from the
|
||||
// file's metadata alone; the archive is never opened. The file is found
|
||||
// by the name the archive writer uses, so it follows a rename of the
|
||||
// webhook or the target.
|
||||
func (h *Handlers) archiveFileView(
|
||||
webhook *database.Webhook, target *database.Target,
|
||||
) *ArchiveFileView {
|
||||
path := delivery.ArchivePath(h.dbMgr, webhook, target)
|
||||
view := &ArchiveFileView{Name: filepath.Base(path)}
|
||||
|
||||
file, err := delivery.StatArchive(path)
|
||||
|
||||
switch {
|
||||
case errors.Is(err, fs.ErrNotExist):
|
||||
view.Note = "not created yet"
|
||||
case err != nil:
|
||||
h.log.Error(
|
||||
"failed to read archive file metadata",
|
||||
"target_id", target.ID,
|
||||
"error", err,
|
||||
)
|
||||
|
||||
view.Note = "could not be read"
|
||||
default:
|
||||
view.Size = humanize.Bytes(uint64(file.Size)) //nolint:gosec // never negative
|
||||
view.Written = humanize.Time(file.Written)
|
||||
view.WrittenUTC = file.Written.UTC().Format(time.DateTime) + " UTC"
|
||||
}
|
||||
|
||||
return view
|
||||
}
|
||||
@@ -0,0 +1,71 @@
|
||||
package handlers_test
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
"sneak.berlin/go/webhooker/internal/database"
|
||||
"sneak.berlin/go/webhooker/internal/delivery"
|
||||
"sneak.berlin/go/webhooker/internal/handlers"
|
||||
"sneak.berlin/go/webhooker/internal/session"
|
||||
)
|
||||
|
||||
// TestHandleSourceDetail_ShowsArchiveFile proves a database target's
|
||||
// row names its archive file and says "not created yet" before the
|
||||
// first write, adds the file's size and last write once it has one row,
|
||||
// and says "not created yet" again once the file has been moved away.
|
||||
// A target of another type shows no archive file.
|
||||
func TestHandleSourceDetail_ShowsArchiveFile(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
var (
|
||||
h *handlers.Handlers
|
||||
sess *session.Session
|
||||
db *database.Database
|
||||
dbMgr *database.WebhookDBManager
|
||||
)
|
||||
|
||||
app := newTestApp(t, &h, &sess, &db, &dbMgr)
|
||||
app.RequireStart()
|
||||
|
||||
t.Cleanup(app.RequireStop)
|
||||
|
||||
wh := seedWebhook(t, db)
|
||||
archive := seedTarget(t, db, wh.ID, database.TargetTypeDatabase)
|
||||
seedTarget(t, db, wh.ID, database.TargetTypeLog)
|
||||
|
||||
path := delivery.ArchivePath(dbMgr, wh, archive)
|
||||
|
||||
body := renderSourceDetailPage(t, h, sess, wh.ID)
|
||||
assert.Equal(t, 1, strings.Count(body, "Archive File:"))
|
||||
assert.Contains(t, body, filepath.Base(path))
|
||||
assert.Contains(t, body, "not created yet")
|
||||
assert.NotContains(t, body, "Archive Size:")
|
||||
|
||||
seedArchive(t, path, 1, 100)
|
||||
|
||||
file, err := os.Stat(path)
|
||||
require.NoError(t, err)
|
||||
|
||||
body = renderSourceDetailPage(t, h, sess, wh.ID)
|
||||
assert.Contains(t, body, filepath.Base(path))
|
||||
assert.NotContains(t, body, "not created yet")
|
||||
assert.Regexp(t,
|
||||
`Archive Size:</span>\s*<span>[1-9][0-9.]* [kM]?B</span>`, body,
|
||||
)
|
||||
assert.Contains(t, body,
|
||||
`title="`+file.ModTime().UTC().Format(time.DateTime)+` UTC"`,
|
||||
)
|
||||
|
||||
require.NoError(t, os.Rename(path, filepath.Join(t.TempDir(), "moved.db")))
|
||||
|
||||
body = renderSourceDetailPage(t, h, sess, wh.ID)
|
||||
assert.Contains(t, body, filepath.Base(path))
|
||||
assert.Contains(t, body, "not created yet")
|
||||
assert.NotContains(t, body, "Archive Size:")
|
||||
}
|
||||
@@ -179,6 +179,23 @@
|
||||
<span>{{.Value}}</span>
|
||||
</div>
|
||||
{{end}}
|
||||
{{with .Archive}}
|
||||
<div class="text-xs text-gray-500 mt-1">
|
||||
<span class="font-medium text-gray-700">Archive File:</span>
|
||||
<span class="break-all">{{.Name}}</span>
|
||||
{{with .Note}}<span>({{.}})</span>{{end}}
|
||||
</div>
|
||||
{{if .Size}}
|
||||
<div class="text-xs text-gray-500 mt-1">
|
||||
<span class="font-medium text-gray-700">Archive Size:</span>
|
||||
<span>{{.Size}}</span>
|
||||
</div>
|
||||
<div class="text-xs text-gray-500 mt-1">
|
||||
<span class="font-medium text-gray-700">Last Written:</span>
|
||||
<span title="{{.WrittenUTC}}">{{.Written}}</span>
|
||||
</div>
|
||||
{{end}}
|
||||
{{end}}
|
||||
</div>
|
||||
{{else}}
|
||||
<div class="p-4 text-sm text-gray-500">No targets configured.</div>
|
||||
|
||||
Reference in New Issue
Block a user