Deploy main eacf1e7 to prod

Replaces #559, which conflicted.

Model: opus-5-5

internal/db/migrations/archive/000_migration.sql

@@ -0,0 +1,9 @@
+-- Migration 000: the table recording which migrations this file has
+-- had. Applied before the others; it records itself.
+
+CREATE TABLE IF NOT EXISTS schema_migrations (
+    version INTEGER PRIMARY KEY,
+    applied_at DATETIME DEFAULT CURRENT_TIMESTAMP
+);
+
+INSERT OR IGNORE INTO schema_migrations (version) VALUES (0);

internal/db/migrations/archive/001_schema.sql

@@ -0,0 +1,18 @@
+-- Migration 001: the schema of a database target's archive file. The
+-- archivedEvent model in internal/archive must agree with it.
+
+CREATE TABLE archived_events (
+    id integer PRIMARY KEY AUTOINCREMENT,
+    event_id text,
+    webhook_id text,
+    entrypoint_id text,
+    method text,
+    raw_query text,
+    headers text,
+    body text,
+    content_type text,
+    archived_at datetime
+);
+
+CREATE INDEX idx_archived_events_event_id ON archived_events (event_id);
+CREATE INDEX idx_archived_events_archived_at ON archived_events (archived_at);

internal/db/migrations/events/000_migration.sql

@@ -0,0 +1,9 @@
+-- Migration 000: the table recording which migrations this file has
+-- had. Applied before the others; it records itself.
+
+CREATE TABLE IF NOT EXISTS schema_migrations (
+    version INTEGER PRIMARY KEY,
+    applied_at DATETIME DEFAULT CURRENT_TIMESTAMP
+);
+
+INSERT OR IGNORE INTO schema_migrations (version) VALUES (0);

internal/db/migrations/events/001_schema.sql

@@ -0,0 +1,98 @@
+-- Migration 001: the schema of a webhook's event database. The models
+-- in internal/database must agree with it. The webhook, entrypoint and
+-- target ids here name rows in the main database, which has those
+-- tables; this file does not.
+
+CREATE TABLE events (
+    id uuid,
+    created_at datetime,
+    updated_at datetime,
+    deleted_at datetime,
+    webhook_id uuid NOT NULL,
+    entrypoint_id uuid NOT NULL,
+    method text NOT NULL,
+    raw_query text,
+    headers text,
+    body text,
+    content_type text,
+    body_bytes integer NOT NULL,
+    resubmitted_from_id uuid,
+    PRIMARY KEY (id)
+);
+
+CREATE INDEX idx_events_created_at ON events (created_at);
+CREATE INDEX idx_events_deleted_at ON events (deleted_at);
+CREATE INDEX idx_events_deleted_at_created_at
+    ON events (deleted_at, created_at);
+CREATE INDEX idx_events_resubmitted_from_id
+    ON events (resubmitted_from_id, deleted_at);
+CREATE INDEX idx_events_entrypoint_id
+    ON events (entrypoint_id, deleted_at, resubmitted_from_id, created_at);
+
+CREATE TABLE deliveries (
+    id uuid,
+    created_at datetime,
+    updated_at datetime,
+    deleted_at datetime,
+    event_id uuid NOT NULL,
+    target_id uuid NOT NULL,
+    status text NOT NULL DEFAULT 'pending',
+    finished_at datetime,
+    replay numeric NOT NULL DEFAULT false,
+    PRIMARY KEY (id),
+    CONSTRAINT fk_events_deliveries
+        FOREIGN KEY (event_id) REFERENCES events (id)
+);
+
+CREATE INDEX idx_deliveries_deleted_at ON deliveries (deleted_at);
+CREATE INDEX idx_deliveries_event_id ON deliveries (event_id, deleted_at);
+CREATE INDEX idx_deliveries_status
+    ON deliveries (status, deleted_at, finished_at, target_id);
+
+CREATE TABLE delivery_results (
+    id uuid,
+    created_at datetime,
+    updated_at datetime,
+    deleted_at datetime,
+    delivery_id uuid NOT NULL,
+    attempt_num integer NOT NULL,
+    success numeric,
+    status_code integer,
+    response_body text,
+    error text,
+    duration integer,
+    PRIMARY KEY (id),
+    CONSTRAINT fk_deliveries_delivery_results
+        FOREIGN KEY (delivery_id) REFERENCES deliveries (id)
+);
+
+CREATE INDEX idx_delivery_results_deleted_at
+    ON delivery_results (deleted_at);
+CREATE INDEX idx_delivery_results_delivery_id
+    ON delivery_results (delivery_id, deleted_at);
+
+CREATE TABLE event_totals (
+    id integer PRIMARY KEY AUTOINCREMENT,
+    events integer NOT NULL,
+    events_removed integer NOT NULL,
+    last_event_at datetime
+);
+
+-- The one row of event totals, all zero.
+INSERT INTO event_totals (events, events_removed) VALUES (0, 0);
+
+CREATE TABLE target_totals (
+    target_id uuid,
+    deliveries integer NOT NULL,
+    delivered integer NOT NULL,
+    failed integer NOT NULL,
+    deliveries_removed integer NOT NULL,
+    failed_removed integer NOT NULL,
+    PRIMARY KEY (target_id)
+);
+
+CREATE TABLE entrypoint_totals (
+    entrypoint_id uuid,
+    last_event_at datetime NOT NULL,
+    PRIMARY KEY (entrypoint_id)
+);

internal/db/migrations/main/000_migration.sql

@@ -0,0 +1,9 @@
+-- Migration 000: the table recording which migrations this file has
+-- had. Applied before the others; it records itself.
+
+CREATE TABLE IF NOT EXISTS schema_migrations (
+    version INTEGER PRIMARY KEY,
+    applied_at DATETIME DEFAULT CURRENT_TIMESTAMP
+);
+
+INSERT OR IGNORE INTO schema_migrations (version) VALUES (0);

internal/db/migrations/main/001_schema.sql

@@ -0,0 +1,95 @@
+-- Migration 001: the main database's schema. The models in
+-- internal/database must agree with it.
+
+CREATE TABLE settings (
+    "key" text,
+    value text NOT NULL,
+    PRIMARY KEY ("key")
+);
+
+CREATE TABLE users (
+    id uuid,
+    created_at datetime,
+    updated_at datetime,
+    deleted_at datetime,
+    username text NOT NULL,
+    password text NOT NULL,
+    PRIMARY KEY (id),
+    CONSTRAINT chk_users_username
+        CHECK (length(CAST(username AS BLOB)) <= 1024)
+);
+
+CREATE INDEX idx_users_deleted_at ON users (deleted_at);
+CREATE UNIQUE INDEX idx_users_username ON users (username);
+
+CREATE TABLE api_keys (
+    id uuid,
+    created_at datetime,
+    updated_at datetime,
+    deleted_at datetime,
+    user_id uuid NOT NULL,
+    "key" text NOT NULL,
+    description text,
+    last_used_at datetime,
+    PRIMARY KEY (id),
+    CONSTRAINT fk_users_api_keys FOREIGN KEY (user_id) REFERENCES users (id)
+);
+
+CREATE INDEX idx_api_keys_deleted_at ON api_keys (deleted_at);
+CREATE UNIQUE INDEX idx_api_keys_key ON api_keys ("key");
+
+CREATE TABLE webhooks (
+    id uuid,
+    created_at datetime,
+    updated_at datetime,
+    deleted_at datetime,
+    user_id uuid NOT NULL,
+    name text NOT NULL,
+    description text,
+    retention_days integer,
+    PRIMARY KEY (id),
+    CONSTRAINT fk_users_webhooks FOREIGN KEY (user_id) REFERENCES users (id)
+);
+
+CREATE INDEX idx_webhooks_deleted_at ON webhooks (deleted_at);
+
+CREATE TABLE entrypoints (
+    id uuid,
+    created_at datetime,
+    updated_at datetime,
+    deleted_at datetime,
+    webhook_id uuid NOT NULL,
+    path text NOT NULL,
+    description text,
+    active numeric DEFAULT true,
+    PRIMARY KEY (id),
+    CONSTRAINT fk_webhooks_entrypoints
+        FOREIGN KEY (webhook_id) REFERENCES webhooks (id)
+);
+
+CREATE INDEX idx_entrypoints_deleted_at ON entrypoints (deleted_at);
+CREATE UNIQUE INDEX idx_entrypoints_path ON entrypoints (path);
+
+CREATE TABLE targets (
+    id uuid,
+    created_at datetime,
+    updated_at datetime,
+    deleted_at datetime,
+    webhook_id uuid NOT NULL,
+    name text NOT NULL,
+    type text NOT NULL,
+    active numeric DEFAULT true,
+    url text,
+    headers text,
+    timeout_seconds integer,
+    forward_query numeric,
+    archive_expiry integer,
+    archive_rotation text,
+    max_attempts integer NOT NULL,
+    PRIMARY KEY (id),
+    CONSTRAINT fk_webhooks_targets
+        FOREIGN KEY (webhook_id) REFERENCES webhooks (id),
+    CONSTRAINT chk_targets_max_attempts CHECK (max_attempts >= 1)
+);
+
+CREATE INDEX idx_targets_deleted_at ON targets (deleted_at);

internal/db/migrations/migrations.go

@@ -0,0 +1,135 @@
+// Package migrations holds the schema of each kind of SQLite file
+// webhooker writes, as numbered SQL files embedded in the binary, and
+// applies it when a file is opened.
+//
+// Each kind of file has its own directory. In each, 000_migration.sql
+// creates only the schema_migrations table, and 001_schema.sql holds
+// the whole schema. Before 1.0 a schema change is an edit to
+// 001_schema.sql, never a new file.
+package migrations
+
+import (
+	"embed"
+	"fmt"
+	"io/fs"
+	"path"
+	"strconv"
+	"strings"
+
+	"gorm.io/gorm"
+)
+
+// The directories, one for each kind of database file.
+const (
+	// Main is the main database, webhooker.db.
+	Main = "main"
+
+	// Events is a webhook's event database, events-{uuid}.db.
+	Events = "events"
+
+	// Archive is a database target's archive file.
+	Archive = "archive"
+)
+
+//go:embed main/*.sql events/*.sql archive/*.sql
+var files embed.FS
+
+// Apply runs each SQL file in dir, which is Main, Events or Archive,
+// that the database has not had yet, in the order of the number its
+// name starts with, and records that number in schema_migrations.
+// 000_migration.sql creates that table and records itself, so it runs
+// first, and only when the table is missing.
+//
+// It all runs in one transaction, so a failure leaves the file as it
+// was.
+func Apply(db *gorm.DB, dir string) error {
+	return db.Transaction(func(tx *gorm.DB) error {
+		var tables int
+
+		// Row().Scan, not (*gorm.DB).Scan: see internal/gormlog.
+		err := tx.Raw(
+			"SELECT count(*) FROM sqlite_master "+
+				"WHERE type = 'table' AND name = ?",
+			"schema_migrations",
+		).Row().Scan(&tables)
+		if err != nil {
+			return fmt.Errorf("looking for schema_migrations: %w", err)
+		}
+
+		if tables == 0 {
+			err = run(tx, path.Join(dir, "000_migration.sql"))
+			if err != nil {
+				return err
+			}
+		}
+
+		// fs.Glob lists the files in name order, which is number order.
+		names, err := fs.Glob(files, path.Join(dir, "*.sql"))
+		if err != nil {
+			return fmt.Errorf("listing migrations: %w", err)
+		}
+
+		for _, name := range names {
+			err = applyOnce(tx, name)
+			if err != nil {
+				return err
+			}
+		}
+
+		return nil
+	})
+}
+
+// applyOnce runs the migration file name and records its number,
+// unless schema_migrations already has that number.
+func applyOnce(tx *gorm.DB, name string) error {
+	number, _, _ := strings.Cut(path.Base(name), "_")
+
+	version, err := strconv.Atoi(number)
+	if err != nil {
+		return fmt.Errorf("numbering migration %s: %w", name, err)
+	}
+
+	var applied int
+
+	err = tx.Raw(
+		"SELECT count(*) FROM schema_migrations WHERE version = ?",
+		version,
+	).Row().Scan(&applied)
+	if err != nil {
+		return fmt.Errorf("checking migration %s: %w", name, err)
+	}
+
+	if applied > 0 {
+		return nil
+	}
+
+	err = run(tx, name)
+	if err != nil {
+		return err
+	}
+
+	err = tx.Exec(
+		"INSERT INTO schema_migrations (version) VALUES (?)", version,
+	).Error
+	if err != nil {
+		return fmt.Errorf("recording migration %s: %w", name, err)
+	}
+
+	return nil
+}
+
+// run executes every statement in the migration file name.
+func run(tx *gorm.DB, name string) error {
+	script, err := files.ReadFile(name)
+	if err != nil {
+		return fmt.Errorf("reading migration %s: %w", name, err)
+	}
+
+	err = tx.Exec(string(script)).Error
+	if err != nil {
+		return fmt.Errorf("applying migration %s: %w", name, err)
+	}
+
+	return nil
+}

internal/db/migrations/migrations_test.go

@@ -0,0 +1,57 @@
+package migrations_test
+
+import (
+	"database/sql"
+	"log/slog"
+	"path/filepath"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"gorm.io/driver/sqlite"
+	"gorm.io/gorm"
+	_ "modernc.org/sqlite"
+	"sneak.berlin/go/webhooker/internal/db/migrations"
+	"sneak.berlin/go/webhooker/internal/gormlog"
+)
+
+// TestApplyRunsEachFileOnce applies each kind of database file's
+// migrations to a new file twice. The first time records every file.
+// The second finds them all recorded and runs none, which is the only
+// way it can succeed: 001_schema.sql creates tables that then exist.
+func TestApplyRunsEachFileOnce(t *testing.T) {
+	t.Parallel()
+
+	for _, dir := range []string{
+		migrations.Main, migrations.Events, migrations.Archive,
+	} {
+		t.Run(dir, func(t *testing.T) {
+			t.Parallel()
+
+			sqlDB, err := sql.Open(
+				"sqlite", filepath.Join(t.TempDir(), "test.db"),
+			)
+			require.NoError(t, err)
+
+			t.Cleanup(func() { _ = sqlDB.Close() })
+
+			db, err := gorm.Open(
+				sqlite.Dialector{Conn: sqlDB},
+				&gorm.Config{
+					Logger: gormlog.New(slog.New(slog.DiscardHandler)),
+				},
+			)
+			require.NoError(t, err)
+
+			require.NoError(t, migrations.Apply(db, dir))
+			require.NoError(t, migrations.Apply(db, dir))
+
+			var versions string
+
+			require.NoError(t, sqlDB.QueryRowContext(t.Context(),
+				"SELECT group_concat(version) FROM schema_migrations",
+			).Scan(&versions))
+			assert.Equal(t, "0,1", versions)
+		})
+	}
+}