Rotate a database target's archive monthly, daily or hourly (closes #379)

A database target's rotation (none, monthly, daily or hourly) puts the
UTC period of each event's receive time in its archive file name, so
each file holds exactly its period's events. It is on the new webhook
page, the add target form and the target edit form, and shown in the
target list.

Renames move every one of a target's files, the sweep prunes every file
and deletes a rotated file it leaves empty, Download exports every file
oldest first with each row's period, and the target list names the
current file and totals the size of all of them.

Model: opus-5-5

internal/delivery/target_database_archive.go

@@ -85,6 +85,10 @@ type databaseTargetConfig struct {
 	// archived rows are pruned, or "never" (the default) to
 	// keep them forever.
 	Expiry string `json:"expiry"`
+
+	// Rotation is none (the default), monthly, daily or hourly: see
+	// archivePeriod.
+	Rotation string `json:"rotation"`
 }
 
 // archivedEvent is one fully captured webhook event stored in a
@@ -178,7 +182,7 @@ func ValidateArchiveExpiry(expiry string) error {
 	return nil
 }
 
-// archiveWriter owns one database target's archive SQLite file.
+// archiveWriter owns one database target's archive SQLite files.
 // It serialises writes, and after each write closes and reopens
 // the file (debounced to at most once per debounce window) so
 // an operator can move the file away for offline archiving. The
@@ -186,8 +190,16 @@ func ValidateArchiveExpiry(expiry string) error {
 // file is opened create-if-missing and its schema is migrated
 // on every open.
 type archiveWriter struct {
-	mu         sync.Mutex
-	path       string
+	mu sync.Mutex
+
+	// path is the target's archive file as ArchivePath names it. A
+	// target that rotates writes to the files archivePeriodPath names
+	// for path and a period instead.
+	path string
+
+	// current is the file db is open on.
+	current string
+
 	log        *slog.Logger
 	debounce   time.Duration
 	db         *gorm.DB
@@ -236,12 +248,14 @@ func newArchiveWriter(
 	}
 }
 
-// write appends the event as a row, then applies the debounced
-// close/reopen. It recreates the archive file if it was moved
-// or removed since the last open. A positive expiry prunes rows
-// older than it on each (re)open.
+// write appends the event as a row to the archive file for period
+// (see archivePeriodPath), then applies the debounced close/reopen.
+// When period names a different file from the one open, the open one
+// is closed first. It recreates the archive file if it was moved or
+// removed since the last open. A positive expiry prunes rows older
+// than it on each (re)open.
 func (w *archiveWriter) write(
-	row archivedEvent, expiry time.Duration,
+	row archivedEvent, expiry time.Duration, period string,
 ) error {
 	w.mu.Lock()
 	defer w.mu.Unlock()
@@ -252,8 +266,10 @@ func (w *archiveWriter) write(
 		)
 	}
 
-	if w.db == nil || !fileExists(w.path) {
-		err := w.reopen(expiry)
+	file := archivePeriodPath(w.path, period)
+
+	if w.db == nil || w.current != file || !fileExists(file) {
+		err := w.reopen(file, expiry)
 		if err != nil {
 			return err
 		}
@@ -264,41 +280,41 @@ func (w *archiveWriter) write(
 	err := w.db.Create(&row).Error
 	if err != nil {
 		return fmt.Errorf(
-			"archiving event to %s: %w", w.path, err,
+			"archiving event to %s: %w", file, err,
 		)
 	}
 
 	if w.now().Sub(w.lastReopen) >= w.debounce {
-		return w.reopen(expiry)
+		return w.reopen(file, expiry)
 	}
 
 	return nil
 }
 
-// open opens (creating if missing) the archive file, migrates
+// open opens (creating if missing) an archive file, migrates
 // its schema, records the reopen time, and prunes expired rows
 // when expiry is positive.
-func (w *archiveWriter) open(expiry time.Duration) error {
-	return w.openMode(archiveModeCreate, expiry)
+func (w *archiveWriter) open(file string, expiry time.Duration) error {
+	return w.openMode(file, archiveModeCreate, expiry)
 }
 
-// openMode opens the archive file with the given SQLite URI
+// openMode opens an archive file with the given SQLite URI
 // mode, migrates its schema, records the reopen time, and
 // prunes expired rows when expiry is positive. The write path
 // passes archiveModeCreate so a missing file is recreated; the
 // idle sweep passes archiveModeExisting so a missing file is an
 // error rather than a newly conjured empty archive.
 func (w *archiveWriter) openMode(
-	mode string, expiry time.Duration,
+	file, mode string, expiry time.Duration,
 ) error {
 	// Opened through database.OpenSQLite so an archive file carries
 	// the same WAL journaling, busy timeout, immediate-transaction
 	// locking, and pool bounds as every other database file. See
 	// internal/database/sqlite_open.go.
-	sqlDB, err := database.OpenSQLite(w.path, mode)
+	sqlDB, err := database.OpenSQLite(file, mode)
 	if err != nil {
 		return fmt.Errorf(
-			"opening archive database %s: %w", w.path, err,
+			"opening archive database %s: %w", file, err,
 		)
 	}
 
@@ -314,7 +330,7 @@ func (w *archiveWriter) openMode(
 
 		return fmt.Errorf(
 			"connecting to archive database %s: %w",
-			w.path, err,
+			file, err,
 		)
 	}
 
@@ -323,11 +339,12 @@ func (w *archiveWriter) openMode(
 		_ = sqlDB.Close()
 
 		return fmt.Errorf(
-			"migrating archive database %s: %w", w.path, err,
+			"migrating archive database %s: %w", file, err,
 		)
 	}
 
 	w.db = gdb
+	w.current = file
 	w.lastReopen = w.now()
 	w.reopens++
 
@@ -338,12 +355,12 @@ func (w *archiveWriter) openMode(
 	return nil
 }
 
-// reopen closes any open handle and opens the file afresh. The
+// reopen closes any open handle and opens file afresh. The
 // fresh open recreates the file if it was moved away.
-func (w *archiveWriter) reopen(expiry time.Duration) error {
+func (w *archiveWriter) reopen(file string, expiry time.Duration) error {
 	w.close()
 
-	return w.open(expiry)
+	return w.open(file, expiry)
 }
 
 // close closes the underlying handle, if any.
@@ -360,16 +377,17 @@ func (w *archiveWriter) close() {
 	w.db = nil
 }
 
-// sweepExpired prunes an archive that may have gone idle, with
-// no write to trigger the usual on-reopen prune. It takes the
-// writer's own mutex for the whole operation, so a sweep is
-// ordered against concurrent writes rather than reaching around
-// them to the file.
+// sweepExpired prunes the target's archive files, which may have
+// gone idle, with no write to trigger the usual on-reopen prune. It
+// takes the writer's own mutex for the whole operation, so a sweep
+// is ordered against concurrent writes rather than reaching around
+// them to the files.
 //
-// It never creates the archive file: a missing file is skipped,
-// and the reopen uses archiveModeExisting so SQLite itself
-// refuses to create one if the file disappears between the
-// check and the open.
+// It never creates an archive file: it prunes only the files
+// archiveFiles finds, and opens each with archiveModeExisting so
+// SQLite itself refuses to create one if the file disappears
+// between the listing and the open. A file named for a period that
+// the prune leaves empty is deleted.
 //
 // The archive is left CLOSED afterwards. An idle archive holding
 // no handle is what keeps the operator's move-the-file-away
@@ -385,34 +403,84 @@ func (w *archiveWriter) sweepExpired(expiry time.Duration) error {
 		)
 	}
 
-	if !fileExists(w.path) {
-		return nil
-	}
-
-	// Drop any live handle first so the prune runs against a
-	// freshly opened file, matching the write path's semantics.
-	w.close()
-
-	err := w.openMode(archiveModeExisting, expiry)
+	files, err := archiveFiles(w.path)
 	if err != nil {
 		return err
 	}
 
+	var errs []error
+
+	for _, file := range files {
+		err = w.sweepFile(file, expiry)
+		if err != nil {
+			errs = append(errs, err)
+		}
+	}
+
+	return errors.Join(errs...)
+}
+
+// sweepFile prunes one of the target's archive files for
+// sweepExpired, which holds w.mu, and deletes the file, with its -wal
+// and -shm, when it is named for a period and the prune leaves it
+// empty.
+func (w *archiveWriter) sweepFile(
+	file archiveFile, expiry time.Duration,
+) error {
+	// Drop any live handle first so the prune runs against a
+	// freshly opened file, matching the write path's semantics.
 	w.close()
 
+	err := w.openMode(file.path, archiveModeExisting, expiry)
+	if err != nil {
+		return err
+	}
+
+	if file.period == "" {
+		w.close()
+
+		return nil
+	}
+
+	var rows int64
+
+	err = w.db.Model(&archivedEvent{}).Count(&rows).Error
+
+	w.close()
+
+	if err != nil {
+		return fmt.Errorf(
+			"counting rows in archive %s: %w", file.path, err,
+		)
+	}
+
+	if rows > 0 {
+		return nil
+	}
+
+	for _, suffix := range []string{"", "-wal", "-shm"} {
+		err = os.Remove(file.path + suffix)
+		if err != nil && !errors.Is(err, fs.ErrNotExist) {
+			return fmt.Errorf("deleting empty archive file: %w", err)
+		}
+	}
+
+	w.log.Info("deleted empty archive file", "path", file.path)
+
 	return nil
 }
 
-// rename gives the archive file a new name in the same directory,
-// and the writer uses the file under that name from now on. The
-// handle is closed first, which folds the -wal into the .db; any
-// -wal or -shm still beside the file (left by a crash) is moved with
-// it, because SQLite finds them by name. A missing file is not an
-// error: the operator may have moved it away, and the next write
-// creates it under the new name.
+// rename gives every one of the target's archive files the new
+// name, keeping the period in the name of each (see
+// archivePeriodPath), and the writer uses the files under that name
+// from now on. The handle is closed first, which folds the -wal into
+// the .db; any -wal or -shm still beside a file (left by a crash) is
+// moved with it, because SQLite finds them by name. A target with no
+// files is not an error: the operator may have moved them away, and
+// the next write creates its file under the new name.
 //
-// If a file already has the new name, nothing is moved and the
-// error is ErrArchiveNameTaken. If one file fails to move, those
+// If a file already has one of the new names, nothing is moved and
+// the error is ErrArchiveNameTaken. If one file fails to move, those
 // already moved are moved back before the error is returned, so the
 // archive is never split across two names.
 func (w *archiveWriter) rename(name string) error {
@@ -430,38 +498,53 @@ func (w *archiveWriter) rename(name string) error {
 		return nil
 	}
 
-	suffixes := []string{"", "-wal", "-shm"}
+	files, err := archiveFiles(w.path)
+	if err != nil {
+		return err
+	}
 
-	for _, suffix := range suffixes {
-		if fileExists(path + suffix) {
+	// from[i] moves to to[i].
+	var from, to []string
+
+	for _, file := range files {
+		renamed := archivePeriodPath(path, file.period)
+
+		for _, suffix := range []string{"", "-wal", "-shm"} {
+			from = append(from, file.path+suffix)
+			to = append(to, renamed+suffix)
+		}
+	}
+
+	for _, taken := range to {
+		if fileExists(taken) {
 			return fmt.Errorf(
-				"%w: %s", ErrArchiveNameTaken, name+suffix,
+				"%w: %s", ErrArchiveNameTaken, filepath.Base(taken),
 			)
 		}
 	}
 
 	w.close()
 
-	for i, suffix := range suffixes {
-		err := os.Rename(w.path+suffix, path+suffix)
+	for i := range from {
+		err = os.Rename(from[i], to[i])
 		if err == nil || errors.Is(err, fs.ErrNotExist) {
 			continue
 		}
 
-		for _, moved := range suffixes[:i] {
-			backErr := os.Rename(path+moved, w.path+moved)
+		for j := range i {
+			backErr := os.Rename(to[j], from[j])
 			if backErr != nil && !errors.Is(backErr, fs.ErrNotExist) {
 				w.log.Error(
 					"failed to move archive file back",
-					"from", path+moved,
-					"to", w.path+moved,
+					"from", to[j],
+					"to", from[j],
 					"error", backErr,
 				)
 			}
 		}
 
 		return fmt.Errorf(
-			"renaming archive %s to %s: %w", w.path+suffix, path+suffix, err,
+			"renaming archive %s to %s: %w", from[i], to[i], err,
 		)
 	}
 
@@ -499,7 +582,7 @@ func (w *archiveWriter) prune(expiry time.Duration) {
 	if res.Error != nil {
 		w.log.Error(
 			"failed to prune expired archive rows",
-			"path", w.path,
+			"path", w.current,
 			"error", res.Error,
 		)
 
@@ -509,48 +592,76 @@ func (w *archiveWriter) prune(expiry time.Duration) {
 	if res.RowsAffected > 0 {
 		w.log.Info(
 			"pruned expired archive rows",
-			"path", w.path,
+			"path", w.current,
 			"rows_deleted", res.RowsAffected,
 		)
 	}
 }
 
 // ArchiveFileInfo is what the metadata of a database target's archive
-// file says about it.
+// files says about them.
 type ArchiveFileInfo struct {
-	// Size is the bytes on disk of the file and its -wal together.
+	// Files counts the files.
+	Files int
+
+	// Size is the bytes on disk of the files and their -wal together.
 	Size int64
 
-	// Written is when the file or its -wal was last modified, whichever
-	// is later: a write lands in the -wal first.
+	// Written is when a file or a -wal was last modified, whichever is
+	// latest: a write lands in the -wal first.
 	Written time.Time
 }
 
-// StatArchive reads the metadata of the archive file at path and of
-// its -wal, without opening the archive. With no file at path, which is
-// so before the first write and after the operator moved it away, the
+// StatArchive reads the metadata of a database target's archive
+// files, given the path ArchivePath gives it (see archiveFiles), and
+// of their -wal, without opening them. With no files, which is so
+// before the first write and after the operator moved them away, the
 // error wraps fs.ErrNotExist.
 func StatArchive(path string) (ArchiveFileInfo, error) {
-	file, err := os.Stat(path)
+	files, err := archiveFiles(path)
 	if err != nil {
 		return ArchiveFileInfo{}, err
 	}
 
-	info := ArchiveFileInfo{Size: file.Size(), Written: file.ModTime()}
+	var info ArchiveFileInfo
 
-	wal, err := os.Stat(path + "-wal")
-	if errors.Is(err, fs.ErrNotExist) {
-		return info, nil
+	for _, file := range files {
+		db, err := os.Stat(file.path)
+		if errors.Is(err, fs.ErrNotExist) {
+			continue
+		}
+
+		if err != nil {
+			return ArchiveFileInfo{}, err
+		}
+
+		info.Files++
+		info.Size += db.Size()
+
+		if db.ModTime().After(info.Written) {
+			info.Written = db.ModTime()
+		}
+
+		wal, err := os.Stat(file.path + "-wal")
+		if errors.Is(err, fs.ErrNotExist) {
+			continue
+		}
+
+		if err != nil {
+			return ArchiveFileInfo{}, err
+		}
+
+		info.Size += wal.Size()
+
+		if wal.ModTime().After(info.Written) {
+			info.Written = wal.ModTime()
+		}
 	}
 
-	if err != nil {
-		return ArchiveFileInfo{}, err
-	}
-
-	info.Size += wal.Size()
-
-	if wal.ModTime().After(info.Written) {
-		info.Written = wal.ModTime()
+	if info.Files == 0 {
+		return ArchiveFileInfo{}, fmt.Errorf(
+			"no archive file for %s: %w", path, fs.ErrNotExist,
+		)
 	}
 
 	return info, nil