Rotate a database target's archive monthly, daily or hourly (closes #379)
check / check (push) Successful in 3m31s

A database target's rotation (none, monthly, daily or hourly) puts the
UTC period of each event's receive time in its archive file name, so
each file holds exactly its period's events. It is on the new webhook
page, the add target form and the target edit form, and shown in the
target list.

Renames move every one of a target's files and move them back if one
fails. The sweep prunes every file, one at a time under the target's
lock, and deletes a rotated file it leaves empty. Download lists the
files, then opens one at a time, oldest first, finding each again under
the target's current names, and gives each row its period. The target
list names the current file and totals the size of all of them.

Model: opus-5-5
This commit is contained in:
2026-10-03 00:47:31 +00:00
parent bcdd4791ec
commit e55134532c
36 changed files with 2120 additions and 406 deletions
+59 -28
View File
@@ -4,6 +4,7 @@ import (
"errors"
"fmt"
"io/fs"
"os"
"path/filepath"
"time"
@@ -21,8 +22,8 @@ type TargetRowView struct {
// and is nil when the webhook's event database could not be read.
Deliveries *TargetDeliveries
// Archive is a database target's archive file, and nil for a target
// of any other type.
// Archive is a database target's archive files, and nil for a
// target of any other type.
Archive *ArchiveFileView
// Paused is set while the target's circuit breaker is turning its
@@ -117,17 +118,23 @@ type TargetDeliveries struct {
}
// ArchiveFileView is what a database target's row shows about its
// archive file.
// archive files.
type ArchiveFileView struct {
// Name is the name of the file an event received now goes to.
Name string
// Note stands in for the size and the last write when there are
// none to show, and is empty when there are.
// Note says that file does not exist yet, or that the files could
// not be read, and is empty otherwise.
Note string
// Size is the size on disk. Written is how long ago the file was
// last written, and WrittenUTC the full time the page shows on
// hover.
// Files counts the target's archive files, and is 0 when there are
// none, or they could not be read, and so no size or last write to
// show.
Files int
// Size is the size on disk of all the files together. Written is
// how long ago the latest of them was last written, and WrittenUTC
// the full time the page shows on hover.
Size string
Written string
WrittenUTC string
@@ -140,6 +147,7 @@ func (h *Handlers) targetRows(
) []TargetRowView {
views := delivery.NewTargetViews(targets)
rows := make([]TargetRowView, len(views))
now := time.Now()
deliveries, err := h.loadTargetDeliveries(webhook.ID)
if err != nil {
@@ -160,7 +168,7 @@ func (h *Handlers) targetRows(
}
if targets[i].Type == database.TargetTypeDatabase {
rows[i].Archive = h.archiveFileView(webhook, &targets[i])
rows[i].Archive = h.archiveFileView(webhook, &targets[i], now)
}
rows[i].Paused = h.pausedView(targets[i].ID)
@@ -227,34 +235,57 @@ func readTargetDeliveries(
return byTarget, nil
}
// archiveFileView describes a database target's archive file from the
// file's metadata alone; the archive is never opened. The file is found
// by the name the archive writer uses, so it follows a rename of the
// webhook or the target.
// archiveFileView describes a database target's archive files from
// their metadata alone; the archive is never opened. It names the file
// an event received at now goes to. The files are found by the name
// the archive writer uses, so they follow a rename of the webhook or
// the target.
func (h *Handlers) archiveFileView(
webhook *database.Webhook, target *database.Target,
webhook *database.Webhook, target *database.Target, now time.Time,
) *ArchiveFileView {
path := delivery.ArchivePath(h.dbMgr, webhook, target)
view := &ArchiveFileView{Name: filepath.Base(path)}
current, err := delivery.ArchivePathAt(h.dbMgr, webhook, target, now)
if err != nil {
return h.archiveUnreadable(&ArchiveFileView{}, target, err)
}
file, err := delivery.StatArchive(path)
view := &ArchiveFileView{Name: filepath.Base(current)}
_, statErr := os.Stat(current)
if errors.Is(statErr, fs.ErrNotExist) {
view.Note = "not created yet"
}
files, err := delivery.StatArchive(
delivery.ArchivePath(h.dbMgr, webhook, target),
)
switch {
case errors.Is(err, fs.ErrNotExist):
view.Note = "not created yet"
// No files: no size or last write to show.
case err != nil:
h.log.Error(
"failed to read archive file metadata",
"target_id", target.ID,
"error", err,
)
view.Note = "could not be read"
return h.archiveUnreadable(view, target, err)
default:
view.Size = humanize.Bytes(uint64(file.Size)) //nolint:gosec // never negative
view.Written = humanize.Time(file.Written)
view.WrittenUTC = file.Written.UTC().Format(time.DateTime) + " UTC"
view.Files = files.Files
view.Size = humanize.Bytes(uint64(files.Size)) //nolint:gosec // never negative
view.Written = humanize.Time(files.Written)
view.WrittenUTC = files.Written.UTC().Format(time.DateTime) + " UTC"
}
return view
}
// archiveUnreadable logs why a database target's archive files could
// not be described, and returns view saying so.
func (h *Handlers) archiveUnreadable(
view *ArchiveFileView, target *database.Target, err error,
) *ArchiveFileView {
h.log.Error(
"failed to read archive file metadata",
"target_id", target.ID,
"error", err,
)
view.Note = "could not be read"
return view
}