Rotate a database target's archive monthly, daily or hourly (closes #379)

A database target's rotation (none, monthly, daily or hourly) puts the
UTC period of each event's receive time in its archive file name, so
each file holds exactly its period's events. It is on the new webhook
page, the add target form and the target edit form, and shown in the
target list.

Renames move every one of a target's files, the sweep prunes every file
and deletes a rotated file it leaves empty, Download exports every file
oldest first with each row's period, and the target list names the
current file and totals the size of all of them.

Model: opus-5-5

internal/delivery/target_database_export.go

@@ -6,6 +6,7 @@ import (
 	"database/sql"
 	"encoding/base64"
 	"encoding/json"
+	"errors"
 	"fmt"
 	"io"
 	"log/slog"
@@ -50,21 +51,29 @@ func ArchiveExportFileName(
 		at.UTC().Format("20060102T150405Z") + ".json.gz"
 }
 
-// ArchiveExport is a database target's archive opened for download.
-// It reads the file on its own connection, inside one read-only
-// transaction, so it writes out the archive as it stood when
+// ArchiveExport is a database target's archive opened for download:
+// every one of its files, each read on its own connection inside one
+// read-only transaction, so it writes out the archive as it stood when
 // OpenArchiveExport returned.
 //
 // Archives are in WAL mode, where a reader works from a snapshot and
 // never blocks a writer: archive writes go on while an export is open,
-// and the export does not see them. SQLite cannot checkpoint the -wal
-// past an open snapshot, so the -wal grows until the export is closed.
+// and the export does not see them. SQLite cannot checkpoint a -wal
+// past an open snapshot, so a file's -wal grows until the export has
+// written out that file.
 type ArchiveExport struct {
+	files []*exportFile
+}
+
+// exportFile is one archive file opened for an export.
+type exportFile struct {
 	db *sql.DB
 	tx *gorm.DB
 
-	// empty is true when there is nothing to read: no file, or a file
-	// without the archive's table yet.
+	// period is the period in the file's name, "" for none.
+	period string
+
+	// empty is true for a file without the archive's table yet.
 	empty bool
 }
 
@@ -74,26 +83,49 @@ type exportedName struct {
 	Name string `json:"name"`
 }
 
-// OpenArchiveExport opens the archive file at path for export and
-// takes the snapshot the export reads. It never creates the file: with
-// no file at path, the export has no rows.
+// OpenArchiveExport opens every one of a database target's archive
+// files for export, given the path ArchivePath gives it (see
+// archiveFiles), and takes the snapshots the export reads. It never
+// creates a file: with no files, the export has no rows.
 //
-// Once it has returned, the file is open, so a rename or a move of it
-// does not affect the export, which reads the same file under its new
-// name.
+// Once it has returned, the files are open, so a rename or a move of
+// them does not affect the export, which reads the same files under
+// their new names.
 //
-// The transaction lasts as long as ctx does, so ctx must last for the
+// The transactions last as long as ctx does, so ctx must last for the
 // whole export.
 func OpenArchiveExport(
 	ctx context.Context, path string, log *slog.Logger,
 ) (*ArchiveExport, error) {
-	if !fileExists(path) {
-		return &ArchiveExport{empty: true}, nil
+	files, err := archiveFiles(path)
+	if err != nil {
+		return nil, err
 	}
 
-	db, err := database.OpenSQLite(path, archiveModeExisting)
+	x := &ArchiveExport{}
+
+	for _, file := range files {
+		f, err := openExportFile(ctx, file, log)
+		if err != nil {
+			_ = x.Close()
+
+			return nil, err
+		}
+
+		x.files = append(x.files, f)
+	}
+
+	return x, nil
+}
+
+// openExportFile opens one archive file for an export and takes its
+// snapshot.
+func openExportFile(
+	ctx context.Context, file archiveFile, log *slog.Logger,
+) (*exportFile, error) {
+	db, err := database.OpenSQLite(file.path, archiveModeExisting)
 	if err != nil {
-		return nil, fmt.Errorf("opening archive %s: %w", path, err)
+		return nil, fmt.Errorf("opening archive %s: %w", file.path, err)
 	}
 
 	gdb, err := gorm.Open(
@@ -106,7 +138,7 @@ func OpenArchiveExport(
 	if err != nil {
 		_ = db.Close()
 
-		return nil, fmt.Errorf("opening archive %s: %w", path, err)
+		return nil, fmt.Errorf("opening archive %s: %w", file.path, err)
 	}
 
 	// ReadOnly makes the driver begin a deferred transaction in place
@@ -116,7 +148,9 @@ func OpenArchiveExport(
 	if tx.Error != nil {
 		_ = db.Close()
 
-		return nil, fmt.Errorf("reading archive %s: %w", path, tx.Error)
+		return nil, fmt.Errorf(
+			"reading archive %s: %w", file.path, tx.Error,
+		)
 	}
 
 	// The transaction's first read is what takes the snapshot.
@@ -127,22 +161,27 @@ func OpenArchiveExport(
 		_ = tx.Rollback()
 		_ = db.Close()
 
-		return nil, fmt.Errorf("reading archive %s: %w", path, err)
+		return nil, fmt.Errorf("reading archive %s: %w", file.path, err)
 	}
 
-	return &ArchiveExport{db: db, tx: tx, empty: tables == 0}, nil
+	return &exportFile{
+		db: db, tx: tx, period: file.period, empty: tables == 0,
+	}, nil
 }
 
 // WriteGzipJSON writes the export to w as one gzipped JSON object:
 // webhook and target, each an id and a name; exported_at; and
-// archived_events, one object per archived row, keyed by column name.
-// A body that is not valid UTF-8 cannot be a JSON string, so it is
-// written in base64, with "body_encoding": "base64" beside it.
+// archived_events, one object per archived row, keyed by column name,
+// the files in the order archiveFiles lists them. A row from a file
+// named for a period has "period" beside its columns. A body that is
+// not valid UTF-8 cannot be a JSON string, so it is written in base64,
+// with "body_encoding": "base64" beside it.
 //
 // Each row is written out before the next is read, so neither the
-// archive nor its JSON is ever held in memory whole. After an error
-// the gzip stream is left unfinished, so what was written does not
-// decompress as a whole file.
+// archive nor its JSON is ever held in memory whole, and each file is
+// closed once its rows are written. After an error the gzip stream is
+// left unfinished, so what was written does not decompress as a whole
+// file.
 func (x *ArchiveExport) WriteGzipJSON(
 	ctx context.Context,
 	w io.Writer,
@@ -169,15 +208,30 @@ func (x *ArchiveExport) WriteGzipJSON(
 	return zw.Close()
 }
 
-// Close ends the export's transaction and closes its connection.
+// Close ends the transactions of the files still open and closes
+// their connections.
 func (x *ArchiveExport) Close() error {
-	if x.db == nil {
+	errs := make([]error, 0, len(x.files))
+
+	for _, f := range x.files {
+		errs = append(errs, f.close())
+	}
+
+	return errors.Join(errs...)
+}
+
+// close ends the file's transaction and closes its connection, once.
+func (f *exportFile) close() error {
+	if f.db == nil {
 		return nil
 	}
 
-	_ = x.tx.Rollback()
+	_ = f.tx.Rollback()
 
-	return x.db.Close()
+	err := f.db.Close()
+	f.db = nil
+
+	return err
 }
 
 // writeJSON writes head with archived_events added as its last key,
@@ -207,46 +261,72 @@ func (x *ArchiveExport) writeJSON(
 	return err
 }
 
-// writeRows writes each archived row to w, oldest first, one per line,
-// separated by commas.
+// writeRows writes the archived rows of each file to w, one per line,
+// separated by commas, and closes each file once its rows are written.
 func (x *ArchiveExport) writeRows(ctx context.Context, w io.Writer) error {
-	if x.empty {
-		return nil
+	sep := "\n"
+
+	for _, f := range x.files {
+		var err error
+
+		sep, err = f.writeRows(ctx, w, sep)
+		if err != nil {
+			return err
+		}
+
+		err = f.close()
+		if err != nil {
+			return err
+		}
 	}
 
-	rows, err := x.tx.WithContext(ctx).
+	return nil
+}
+
+// writeRows writes the file's archived rows to w, oldest first, the
+// first after sep and each other after ",\n". It returns what goes
+// before the next row: sep again when the file had no rows.
+func (f *exportFile) writeRows(
+	ctx context.Context, w io.Writer, sep string,
+) (string, error) {
+	if f.empty {
+		return sep, nil
+	}
+
+	rows, err := f.tx.WithContext(ctx).
 		Model(&archivedEvent{}).Order("id").Rows()
 	if err != nil {
-		return err
+		return "", err
 	}
 
 	defer func() { _ = rows.Close() }()
 
-	for sep := "\n"; rows.Next(); sep = ",\n" {
+	for ; rows.Next(); sep = ",\n" {
 		var ev archivedEvent
 
-		err = x.tx.ScanRows(rows, &ev)
+		err = f.tx.ScanRows(rows, &ev)
 		if err != nil {
-			return err
+			return "", err
 		}
 
 		_, err = io.WriteString(w, sep)
 		if err != nil {
-			return err
+			return "", err
 		}
 
-		err = writeRow(w, &ev)
+		err = writeRow(w, &ev, f.period)
 		if err != nil {
-			return err
+			return "", err
 		}
 	}
 
-	return rows.Err()
+	return sep, rows.Err()
 }
 
 // writeRow writes an archived row to w as a JSON object keyed by
-// column name, its body in base64 when it is not valid UTF-8.
-func writeRow(w io.Writer, ev *archivedEvent) error {
+// column name, its body in base64 when it is not valid UTF-8, with
+// the period of its file beside them unless that is "".
+func writeRow(w io.Writer, ev *archivedEvent, period string) error {
 	row := map[string]any{
 		"id":            ev.ID,
 		"event_id":      ev.EventID,
@@ -264,6 +344,10 @@ func writeRow(w io.Writer, ev *archivedEvent) error {
 		row["body_encoding"] = "base64"
 	}
 
+	if period != "" {
+		row["period"] = period
+	}
+
 	line, err := json.Marshal(row)
 	if err != nil {
 		return err