Add a Download button that exports a database target's archive as gzipped JSON (closes #374)

Each database target on the webhook page links to
/hook/ID/targets/TARGETID/download, which streams the target's archive
as archive-WEBHOOK-TARGET-YYYYMMDDTHHMMSSZ.json.gz: the webhook, the
target, exported_at, and archived_events, one object per row keyed by
column, a body that is not valid UTF-8 in base64 with body_encoding.
The export reads one row at a time on its own connection inside a
read-only transaction: one snapshot, and no write lock. The handler
holds the rename lock only while it reads the names and opens the file.
A missing archive exports empty and is not created.

Model: opus-5-5

internal/delivery/target_database.go

@@ -3,7 +3,6 @@ package delivery
 import (
 	"context"
 	"fmt"
-	"path/filepath"
 	"strings"
 	"sync"
 	"time"
@@ -277,10 +276,9 @@ func (t *databaseTarget) releaseSweepWriter(
 }
 
 // newWriter builds the writer for a database target's archive. The
-// file lives beside the webhook's event database in the data
-// directory and is named for the webhook and the target as the main
-// database has them now; from then on only rename changes the name
-// the writer uses. It does not touch the archive file.
+// file is the one ArchivePath gives for the webhook and the target as
+// the main database names them now; from then on only rename changes
+// the name the writer uses. It does not touch the archive file.
 func (t *databaseTarget) newWriter(
 	targetID string,
 ) (*archiveWriter, error) {
@@ -299,12 +297,10 @@ func (t *databaseTarget) newWriter(
 		)
 	}
 
-	dir := filepath.Dir(t.eng.dbManager.DBPath(target.WebhookID))
-	name := ArchiveFileName(
-		target.Webhook.Name, target.Name, target.ID,
+	w := newArchiveWriter(
+		ArchivePath(t.eng.dbManager, &target.Webhook, &target),
+		t.eng.log,
 	)
-
-	w := newArchiveWriter(filepath.Join(dir, name), t.eng.log)
 	w.webhookID = target.WebhookID
 
 	return w, nil

internal/delivery/target_database_export.go

@@ -0,0 +1,270 @@
+package delivery
+
+import (
+	"compress/gzip"
+	"context"
+	"database/sql"
+	"encoding/base64"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"path/filepath"
+	"time"
+	"unicode/utf8"
+
+	"sneak.berlin/go/webhooker/internal/database"
+)
+
+// archiveTableQuery counts the archive's table: 0 when the archive
+// writer has created the file but not yet the table in it.
+const archiveTableQuery = "SELECT count(*) FROM sqlite_master " +
+	"WHERE type = 'table' AND name = 'archived_events'"
+
+// archiveNextRowQuery reads every column of the first archived row
+// after a given id. An export reads the archive a row at a time this
+// way rather than through one cursor, because the Scan check in
+// internal/gormlog accepts only a Scan straight on QueryRowContext's
+// result.
+const archiveNextRowQuery = "SELECT id, event_id, webhook_id, " +
+	"entrypoint_id, method, headers, body, content_type, archived_at " +
+	"FROM archived_events WHERE id > ? ORDER BY id LIMIT 1"
+
+// ArchivePath returns where a database target's archive file is: in
+// the data directory, beside the webhook's event database, under the
+// name ArchiveFileName gives it.
+func ArchivePath(
+	dbMgr *database.WebhookDBManager,
+	webhook *database.Webhook,
+	target *database.Target,
+) string {
+	return filepath.Join(
+		filepath.Dir(dbMgr.DBPath(webhook.ID)),
+		ArchiveFileName(webhook.Name, target.Name, target.ID),
+	)
+}
+
+// ArchiveExportFileName returns the name a database target's archive
+// downloads under:
+// archive-WEBHOOKNAME-TARGETNAME-YYYYMMDDTHHMMSSZ.json.gz, the names
+// made safe as in ArchiveFileName and the time in UTC.
+func ArchiveExportFileName(
+	webhookName, targetName string, at time.Time,
+) string {
+	return "archive-" + archiveNamePart(webhookName) + "-" +
+		archiveNamePart(targetName) + "-" +
+		at.UTC().Format("20060102T150405Z") + ".json.gz"
+}
+
+// ArchiveExport is a database target's archive opened for download.
+// It reads the file on its own connection, inside one read-only
+// transaction, so it writes out the archive as it stood when
+// OpenArchiveExport returned.
+//
+// Archives are in WAL mode, where a reader works from a snapshot and
+// never blocks a writer: archive writes go on while an export is open,
+// and the export does not see them. SQLite cannot checkpoint the -wal
+// past an open snapshot, so the -wal grows until the export is closed.
+type ArchiveExport struct {
+	db *sql.DB
+	tx *sql.Tx
+
+	// empty is true when there is nothing to read: no file, or a file
+	// without the archive's table yet.
+	empty bool
+}
+
+// exportedName is how an export names its webhook and its target.
+type exportedName struct {
+	ID   string `json:"id"`
+	Name string `json:"name"`
+}
+
+// OpenArchiveExport opens the archive file at path for export and
+// takes the snapshot the export reads. It never creates the file: with
+// no file at path, the export has no rows.
+//
+// Once it has returned, the file is open, so a rename or a move of it
+// does not affect the export, which reads the same file under its new
+// name.
+func OpenArchiveExport(
+	ctx context.Context, path string,
+) (*ArchiveExport, error) {
+	if !fileExists(path) {
+		return &ArchiveExport{empty: true}, nil
+	}
+
+	db, err := database.OpenSQLite(path, archiveModeExisting)
+	if err != nil {
+		return nil, fmt.Errorf("opening archive %s: %w", path, err)
+	}
+
+	// ReadOnly makes the driver begin a deferred transaction in place
+	// of the BEGIN IMMEDIATE the connection string asks for, so the
+	// export never takes the archive's write lock.
+	tx, err := db.BeginTx(ctx, &sql.TxOptions{ReadOnly: true})
+	if err != nil {
+		_ = db.Close()
+
+		return nil, fmt.Errorf("reading archive %s: %w", path, err)
+	}
+
+	// The transaction's first read is what takes the snapshot.
+	var tables int
+
+	err = tx.QueryRowContext(ctx, archiveTableQuery).Scan(&tables)
+	if err != nil {
+		_ = tx.Rollback()
+		_ = db.Close()
+
+		return nil, fmt.Errorf("reading archive %s: %w", path, err)
+	}
+
+	return &ArchiveExport{db: db, tx: tx, empty: tables == 0}, nil
+}
+
+// WriteGzipJSON writes the export to w as one gzipped JSON object:
+// webhook and target, each an id and a name; exported_at; and
+// archived_events, one object per archived row, keyed by column name.
+// A body that is not valid UTF-8 cannot be a JSON string, so it is
+// written in base64, with "body_encoding": "base64" beside it.
+//
+// Each row is written out before the next is read, so neither the
+// archive nor its JSON is ever held in memory whole. After an error
+// the gzip stream is left unfinished, so what was written does not
+// decompress as a whole file.
+func (x *ArchiveExport) WriteGzipJSON(
+	ctx context.Context,
+	w io.Writer,
+	webhook *database.Webhook,
+	target *database.Target,
+	exportedAt time.Time,
+) error {
+	head, err := json.Marshal(map[string]any{
+		"webhook":     exportedName{ID: webhook.ID, Name: webhook.Name},
+		"target":      exportedName{ID: target.ID, Name: target.Name},
+		"exported_at": exportedAt.UTC(),
+	})
+	if err != nil {
+		return fmt.Errorf("encoding archive export: %w", err)
+	}
+
+	zw := gzip.NewWriter(w)
+
+	err = x.writeJSON(ctx, zw, head)
+	if err != nil {
+		return fmt.Errorf("writing archive export: %w", err)
+	}
+
+	return zw.Close()
+}
+
+// Close ends the export's transaction and closes its connection.
+func (x *ArchiveExport) Close() error {
+	if x.db == nil {
+		return nil
+	}
+
+	_ = x.tx.Rollback()
+
+	return x.db.Close()
+}
+
+// writeJSON writes head with archived_events added as its last key,
+// the rows going into it one at a time.
+func (x *ArchiveExport) writeJSON(
+	ctx context.Context, w io.Writer, head []byte,
+) error {
+	// head goes out without its closing brace, so that
+	// archived_events can follow it.
+	_, err := w.Write(head[:len(head)-1])
+	if err != nil {
+		return err
+	}
+
+	_, err = io.WriteString(w, `,"archived_events":[`)
+	if err != nil {
+		return err
+	}
+
+	err = x.writeRows(ctx, w)
+	if err != nil {
+		return err
+	}
+
+	_, err = io.WriteString(w, "\n]}\n")
+
+	return err
+}
+
+// writeRows writes each archived row to w, oldest first, one per line,
+// separated by commas.
+func (x *ArchiveExport) writeRows(ctx context.Context, w io.Writer) error {
+	if x.empty {
+		return nil
+	}
+
+	next, err := x.tx.PrepareContext(ctx, archiveNextRowQuery)
+	if err != nil {
+		return err
+	}
+
+	defer func() { _ = next.Close() }()
+
+	var ev archivedEvent
+
+	for sep := "\n"; ; sep = ",\n" {
+		err = next.QueryRowContext(ctx, ev.ID).Scan(
+			&ev.ID, &ev.EventID, &ev.WebhookID, &ev.EntrypointID,
+			&ev.Method, &ev.Headers, &ev.Body, &ev.ContentType,
+			&ev.ArchivedAt,
+		)
+		if errors.Is(err, sql.ErrNoRows) {
+			return nil
+		}
+
+		if err != nil {
+			return err
+		}
+
+		_, err = io.WriteString(w, sep)
+		if err != nil {
+			return err
+		}
+
+		err = writeRow(w, &ev)
+		if err != nil {
+			return err
+		}
+	}
+}
+
+// writeRow writes an archived row to w as a JSON object keyed by
+// column name, its body in base64 when it is not valid UTF-8.
+func writeRow(w io.Writer, ev *archivedEvent) error {
+	row := map[string]any{
+		"id":            ev.ID,
+		"event_id":      ev.EventID,
+		"webhook_id":    ev.WebhookID,
+		"entrypoint_id": ev.EntrypointID,
+		"method":        ev.Method,
+		"headers":       ev.Headers,
+		"body":          ev.Body,
+		"content_type":  ev.ContentType,
+		"archived_at":   ev.ArchivedAt.UTC(),
+	}
+
+	if !utf8.ValidString(ev.Body) {
+		row["body"] = base64.StdEncoding.EncodeToString([]byte(ev.Body))
+		row["body_encoding"] = "base64"
+	}
+
+	line, err := json.Marshal(row)
+	if err != nil {
+		return err
+	}
+
+	_, err = w.Write(line)
+
+	return err
+}

internal/delivery/target_database_export_test.go

@@ -0,0 +1,373 @@
+package delivery_test
+
+import (
+	"bytes"
+	"compress/gzip"
+	"encoding/base64"
+	"encoding/json"
+	"fmt"
+	"io"
+	"os"
+	"path/filepath"
+	"runtime"
+	"runtime/debug"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"sneak.berlin/go/webhooker/internal/database"
+	"sneak.berlin/go/webhooker/internal/delivery"
+)
+
+// The webhook and the target the export tests' archives belong to.
+const (
+	exportWebhookID   = "wh-export"
+	exportWebhookName = "Orders (EU)"
+	exportTargetID    = "tgt-export"
+	exportTargetName  = "Long-term archive"
+)
+
+const (
+	// binaryBody is a body that is not valid UTF-8.
+	binaryBody = "\xff\xfe\x00\x01binary\x80"
+
+	// openedEventID is the event the snapshot tests archive before
+	// they open the export.
+	openedEventID = "opened"
+)
+
+// writeExportTo writes export to w as the archive of the export tests'
+// webhook and target, exported at 2026-10-02T12:03:04Z.
+func writeExportTo(
+	t *testing.T, export *delivery.ArchiveExport, w io.Writer,
+) error {
+	t.Helper()
+
+	return export.WriteGzipJSON(
+		t.Context(), w,
+		&database.Webhook{
+			BaseModel: database.BaseModel{ID: exportWebhookID},
+			Name:      exportWebhookName,
+		},
+		&database.Target{
+			BaseModel: database.BaseModel{ID: exportTargetID},
+			Name:      exportTargetName,
+		},
+		time.Date(2026, 10, 2, 12, 3, 4, 0, time.UTC),
+	)
+}
+
+// exportArchive runs a whole export of the archive at path and returns
+// its JSON, decompressed and parsed.
+func exportArchive(t *testing.T, path string) map[string]any {
+	t.Helper()
+
+	export, err := delivery.OpenArchiveExport(t.Context(), path)
+	require.NoError(t, err)
+
+	defer func() { require.NoError(t, export.Close()) }()
+
+	return writeExport(t, export)
+}
+
+// writeExport writes an opened export and returns its JSON,
+// decompressed and parsed. Reading to the end makes the gzip reader
+// check that the stream was finished.
+func writeExport(
+	t *testing.T, export *delivery.ArchiveExport,
+) map[string]any {
+	t.Helper()
+
+	var buf bytes.Buffer
+
+	require.NoError(t, writeExportTo(t, export, &buf))
+
+	zr, err := gzip.NewReader(&buf)
+	require.NoError(t, err)
+
+	raw, err := io.ReadAll(zr)
+	require.NoError(t, err)
+
+	var got map[string]any
+
+	require.NoError(t, json.Unmarshal(raw, &got))
+
+	return got
+}
+
+// exportedEvents returns an export's archived_events.
+func exportedEvents(t *testing.T, got map[string]any) []map[string]any {
+	t.Helper()
+
+	list, ok := got["archived_events"].([]any)
+	require.True(t, ok, "archived_events must be an array: %v", got)
+
+	events := make([]map[string]any, len(list))
+
+	for i, v := range list {
+		events[i], ok = v.(map[string]any)
+		require.True(t, ok, "an archived event must be an object: %v", v)
+	}
+
+	return events
+}
+
+// exportedEventIDs returns the event_id of each of an export's
+// archived_events.
+func exportedEventIDs(t *testing.T, got map[string]any) []string {
+	t.Helper()
+
+	events := exportedEvents(t, got)
+	ids := make([]string, 0, len(events))
+
+	for _, ev := range events {
+		ids = append(ids, fmt.Sprint(ev["event_id"]))
+	}
+
+	return ids
+}
+
+// TestArchiveExport_MatchesStoredRows proves an export holds the
+// webhook, the target, the time, and every column of every stored
+// row: a body that is valid UTF-8 as a string, and one that is not in
+// base64, marked as such.
+func TestArchiveExport_MatchesStoredRows(t *testing.T) {
+	t.Parallel()
+
+	path := filepath.Join(t.TempDir(), "archive.db")
+	w := delivery.NewExportArchiveWriter(path, archiveTestLogger(), 0)
+	bodies := []string{`{"order":1}`, "plain text", "", binaryBody}
+
+	for i, body := range bodies {
+		require.NoError(t, w.Write(delivery.ExportArchivedEvent{
+			EventID:      fmt.Sprintf("ev-%d", i),
+			WebhookID:    exportWebhookID,
+			EntrypointID: "ep-1",
+			Method:       "POST",
+			Headers:      `{"X-Test":["yes"]}`,
+			Body:         body,
+			ContentType:  testContentType,
+		}, 0))
+	}
+
+	var stored []delivery.ExportArchivedEvent
+
+	require.NoError(t, openArchiveDBForRead(t, path).
+		Order("id").Find(&stored).Error)
+
+	got := exportArchive(t, path)
+
+	assert.Equal(t,
+		map[string]any{"id": exportWebhookID, "name": exportWebhookName},
+		got["webhook"],
+	)
+	assert.Equal(t,
+		map[string]any{"id": exportTargetID, "name": exportTargetName},
+		got["target"],
+	)
+	assert.Equal(t, "2026-10-02T12:03:04Z", got["exported_at"])
+
+	events := exportedEvents(t, got)
+	require.Len(t, events, len(bodies))
+
+	for i, row := range stored {
+		assertExportedRow(t, row, events[i])
+	}
+}
+
+// assertExportedRow checks that ev, from an export, holds every column
+// of the stored row.
+func assertExportedRow(
+	t *testing.T, row delivery.ExportArchivedEvent, ev map[string]any,
+) {
+	t.Helper()
+
+	archivedAt, err := time.Parse(
+		time.RFC3339Nano, fmt.Sprint(ev["archived_at"]),
+	)
+	require.NoError(t, err)
+	assert.True(t, archivedAt.Equal(row.ArchivedAt))
+
+	assert.EqualValues(t, row.ID, ev["id"])
+	assert.Equal(t, row.EventID, ev["event_id"])
+	assert.Equal(t, row.WebhookID, ev["webhook_id"])
+	assert.Equal(t, row.EntrypointID, ev["entrypoint_id"])
+	assert.Equal(t, row.Method, ev["method"])
+	assert.Equal(t, row.Headers, ev["headers"])
+	assert.Equal(t, row.ContentType, ev["content_type"])
+
+	if row.Body != binaryBody {
+		assert.Equal(t, row.Body, ev["body"])
+		assert.Len(t, ev, 9, "the nine columns and nothing else: %v", ev)
+
+		return
+	}
+
+	body, err := base64.StdEncoding.DecodeString(fmt.Sprint(ev["body"]))
+	require.NoError(t, err)
+	assert.Equal(t, binaryBody, string(body))
+	assert.Equal(t, "base64", ev["body_encoding"])
+	assert.Len(t, ev, 10, "the nine columns and body_encoding: %v", ev)
+}
+
+// TestArchiveExport_Empty proves an archive with nothing in it exports
+// as an empty archived_events: no file, which the export must not
+// create; a file the archive writer has not yet put its table in; and
+// a table with no rows.
+func TestArchiveExport_Empty(t *testing.T) {
+	t.Parallel()
+
+	dir := t.TempDir()
+	missing := filepath.Join(dir, "missing.db")
+	noTable := filepath.Join(dir, "no-table.db")
+	noRows := filepath.Join(dir, "no-rows.db")
+
+	require.NoError(t, os.WriteFile(noTable, nil, 0o600))
+	require.NoError(t,
+		delivery.NewExportArchiveWriter(noRows, archiveTestLogger(), 0).
+			Open(0),
+	)
+
+	for _, path := range []string{missing, noTable, noRows} {
+		assert.Empty(t, exportedEvents(t, exportArchive(t, path)), path)
+	}
+
+	for _, suffix := range archiveFileSuffixes() {
+		assert.NoFileExists(t, missing+suffix)
+	}
+}
+
+// TestArchiveExport_ReadsOneSnapshot proves an export writes the
+// archive as it was when it was opened, and holds up no archive
+// write: a row written while the export is open is stored, and is not
+// in the export. A write held up for the whole busy timeout would
+// fail.
+func TestArchiveExport_ReadsOneSnapshot(t *testing.T) {
+	t.Parallel()
+
+	path := filepath.Join(t.TempDir(), "archive.db")
+	w := delivery.NewExportArchiveWriter(path, archiveTestLogger(), 0)
+
+	require.NoError(t, w.Write(delivery.ExportArchivedEvent{EventID: openedEventID}, 0))
+
+	export, err := delivery.OpenArchiveExport(t.Context(), path)
+	require.NoError(t, err)
+
+	defer func() { require.NoError(t, export.Close()) }()
+
+	require.NoError(t, w.Write(delivery.ExportArchivedEvent{EventID: "during"}, 0))
+
+	assert.Equal(t,
+		[]string{openedEventID}, exportedEventIDs(t, writeExport(t, export)),
+	)
+
+	var stored int64
+
+	require.NoError(t, openArchiveDBForRead(t, path).
+		Model(&delivery.ExportArchivedEvent{}).Count(&stored).Error)
+	assert.Equal(t, int64(2), stored)
+}
+
+// TestArchiveExport_SurvivesRename proves that renaming the archive
+// while an export of it is open, as renaming its webhook or target
+// does, leaves the export reading the same file.
+func TestArchiveExport_SurvivesRename(t *testing.T) {
+	t.Parallel()
+
+	path := filepath.Join(t.TempDir(), "archive-old.db")
+	w := delivery.NewExportArchiveWriter(path, archiveTestLogger(), 0)
+
+	require.NoError(t, w.Write(delivery.ExportArchivedEvent{EventID: openedEventID}, 0))
+
+	export, err := delivery.OpenArchiveExport(t.Context(), path)
+	require.NoError(t, err)
+
+	defer func() { require.NoError(t, export.Close()) }()
+
+	require.NoError(t, w.Rename("archive-new.db"))
+	require.NoError(t, w.Write(delivery.ExportArchivedEvent{EventID: "after"}, 0))
+	require.NoFileExists(t, path)
+
+	assert.Equal(t,
+		[]string{openedEventID}, exportedEventIDs(t, writeExport(t, export)),
+	)
+}
+
+// heapPeak is an io.Writer that discards what it is given and records
+// the largest heap it saw at a write.
+type heapPeak struct {
+	max uint64
+}
+
+func (p *heapPeak) Write(b []byte) (int, error) {
+	var m runtime.MemStats
+
+	runtime.ReadMemStats(&m)
+	p.max = max(p.max, m.HeapAlloc)
+
+	return len(b), nil
+}
+
+// TestArchiveExport_Streams proves an export holds neither the archive
+// nor its JSON in memory whole: exporting a 16 MiB archive grows the
+// heap by less than half of that. An export that read every row before
+// writing, or built the JSON before writing it, would hold all 16 MiB
+// at a write.
+//
+//nolint:paralleltest // It measures the heap, which tests share.
+func TestArchiveExport_Streams(t *testing.T) {
+	const (
+		rows     = 64
+		bodySize = 256 << 10
+		limit    = rows * bodySize / 2
+	)
+
+	path := filepath.Join(t.TempDir(), "archive.db")
+	w := delivery.NewExportArchiveWriter(path, archiveTestLogger(), 0)
+	body := strings.Repeat("x", bodySize)
+
+	for range rows {
+		require.NoError(t, w.Write(delivery.ExportArchivedEvent{Body: body}, 0))
+	}
+
+	export, err := delivery.OpenArchiveExport(t.Context(), path)
+	require.NoError(t, err)
+
+	defer func() { require.NoError(t, export.Close()) }()
+
+	// A low GC target collects garbage soon after it is made, so the
+	// heap at each write is close to what the export is holding.
+	defer debug.SetGCPercent(debug.SetGCPercent(10))
+
+	runtime.GC()
+
+	var start runtime.MemStats
+
+	runtime.ReadMemStats(&start)
+
+	peak := &heapPeak{}
+
+	require.NoError(t, writeExportTo(t, export, peak))
+	assert.Less(t, peak.max, start.HeapAlloc+limit,
+		"heap at the start %d, at its peak %d", start.HeapAlloc, peak.max,
+	)
+}
+
+// TestArchiveExportFileName proves the download is named for the
+// webhook and the target, with the names made safe as for the archive
+// file, and the export time in UTC.
+func TestArchiveExportFileName(t *testing.T) {
+	t.Parallel()
+
+	cest := time.FixedZone("CEST", int((2 * time.Hour).Seconds()))
+
+	assert.Equal(t,
+		"archive-orders-eu-long-term-archive-20261002T120304Z.json.gz",
+		delivery.ArchiveExportFileName(
+			exportWebhookName, exportTargetName,
+			time.Date(2026, 10, 2, 14, 3, 4, 0, cest),
+		),
+	)
+}