Build every database's schema from embedded SQL files (closes #56)

The main database, each webhook's event database and each archive file
get their schema from SQL files under internal/db/migrations/, one
directory per kind. Opening a file runs, in one transaction, each file
that schema_migrations does not list. GORM stays for queries.
AutoMigrate is gone, tests included.

An event database holds only its own tables now, so the sweep that
removed target rows copied into it is deleted. The association guard
stays: without it a delivery write fails on the missing targets table.

AutoMigrate also had GORM parse every model before a handle was shared.
That is now done on its own: GORM races when two goroutines first use
models that refer to each other.

Model: opus-5-5

internal/db/migrations/archive/000_migration.sql

@@ -0,0 +1,9 @@
+-- Migration 000: the table recording which migrations this file has
+-- had. Applied before the others; it records itself.
+
+CREATE TABLE IF NOT EXISTS schema_migrations (
+    version INTEGER PRIMARY KEY,
+    applied_at DATETIME DEFAULT CURRENT_TIMESTAMP
+);
+
+INSERT OR IGNORE INTO schema_migrations (version) VALUES (0);

internal/db/migrations/archive/001_schema.sql

@@ -0,0 +1,18 @@
+-- Migration 001: the schema of a database target's archive file. The
+-- archivedEvent model in internal/delivery must agree with it.
+
+CREATE TABLE archived_events (
+    id integer PRIMARY KEY AUTOINCREMENT,
+    event_id text,
+    webhook_id text,
+    entrypoint_id text,
+    method text,
+    raw_query text,
+    headers text,
+    body text,
+    content_type text,
+    archived_at datetime
+);
+
+CREATE INDEX idx_archived_events_event_id ON archived_events (event_id);
+CREATE INDEX idx_archived_events_archived_at ON archived_events (archived_at);

internal/db/migrations/events/000_migration.sql

@@ -0,0 +1,9 @@
+-- Migration 000: the table recording which migrations this file has
+-- had. Applied before the others; it records itself.
+
+CREATE TABLE IF NOT EXISTS schema_migrations (
+    version INTEGER PRIMARY KEY,
+    applied_at DATETIME DEFAULT CURRENT_TIMESTAMP
+);
+
+INSERT OR IGNORE INTO schema_migrations (version) VALUES (0);

internal/db/migrations/events/001_schema.sql

@@ -0,0 +1,98 @@
+-- Migration 001: the schema of a webhook's event database. The models
+-- in internal/database must agree with it. The webhook, entrypoint and
+-- target ids here name rows in the main database, which has those
+-- tables; this file does not.
+
+CREATE TABLE events (
+    id uuid,
+    created_at datetime,
+    updated_at datetime,
+    deleted_at datetime,
+    webhook_id uuid NOT NULL,
+    entrypoint_id uuid NOT NULL,
+    method text NOT NULL,
+    raw_query text,
+    headers text,
+    body text,
+    content_type text,
+    body_bytes integer NOT NULL,
+    resubmitted_from_id uuid,
+    PRIMARY KEY (id)
+);
+
+CREATE INDEX idx_events_created_at ON events (created_at);
+CREATE INDEX idx_events_deleted_at ON events (deleted_at);
+CREATE INDEX idx_events_deleted_at_created_at
+    ON events (deleted_at, created_at);
+CREATE INDEX idx_events_resubmitted_from_id
+    ON events (resubmitted_from_id, deleted_at);
+CREATE INDEX idx_events_entrypoint_id
+    ON events (entrypoint_id, deleted_at, resubmitted_from_id, created_at);
+
+CREATE TABLE deliveries (
+    id uuid,
+    created_at datetime,
+    updated_at datetime,
+    deleted_at datetime,
+    event_id uuid NOT NULL,
+    target_id uuid NOT NULL,
+    status text NOT NULL DEFAULT 'pending',
+    finished_at datetime,
+    replay numeric NOT NULL DEFAULT false,
+    PRIMARY KEY (id),
+    CONSTRAINT fk_events_deliveries
+        FOREIGN KEY (event_id) REFERENCES events (id)
+);
+
+CREATE INDEX idx_deliveries_deleted_at ON deliveries (deleted_at);
+CREATE INDEX idx_deliveries_event_id ON deliveries (event_id, deleted_at);
+CREATE INDEX idx_deliveries_status
+    ON deliveries (status, deleted_at, finished_at, target_id);
+
+CREATE TABLE delivery_results (
+    id uuid,
+    created_at datetime,
+    updated_at datetime,
+    deleted_at datetime,
+    delivery_id uuid NOT NULL,
+    attempt_num integer NOT NULL,
+    success numeric,
+    status_code integer,
+    response_body text,
+    error text,
+    duration integer,
+    PRIMARY KEY (id),
+    CONSTRAINT fk_deliveries_delivery_results
+        FOREIGN KEY (delivery_id) REFERENCES deliveries (id)
+);
+
+CREATE INDEX idx_delivery_results_deleted_at
+    ON delivery_results (deleted_at);
+CREATE INDEX idx_delivery_results_delivery_id
+    ON delivery_results (delivery_id, deleted_at);
+
+CREATE TABLE event_totals (
+    id integer PRIMARY KEY AUTOINCREMENT,
+    events integer NOT NULL,
+    events_removed integer NOT NULL,
+    last_event_at datetime
+);
+
+-- The one row of event totals, all zero.
+INSERT INTO event_totals (events, events_removed) VALUES (0, 0);
+
+CREATE TABLE target_totals (
+    target_id uuid,
+    deliveries integer NOT NULL,
+    delivered integer NOT NULL,
+    failed integer NOT NULL,
+    deliveries_removed integer NOT NULL,
+    failed_removed integer NOT NULL,
+    PRIMARY KEY (target_id)
+);
+
+CREATE TABLE entrypoint_totals (
+    entrypoint_id uuid,
+    last_event_at datetime NOT NULL,
+    PRIMARY KEY (entrypoint_id)
+);

internal/db/migrations/main/000_migration.sql

@@ -0,0 +1,9 @@
+-- Migration 000: the table recording which migrations this file has
+-- had. Applied before the others; it records itself.
+
+CREATE TABLE IF NOT EXISTS schema_migrations (
+    version INTEGER PRIMARY KEY,
+    applied_at DATETIME DEFAULT CURRENT_TIMESTAMP
+);
+
+INSERT OR IGNORE INTO schema_migrations (version) VALUES (0);

internal/db/migrations/main/001_schema.sql

@@ -0,0 +1,89 @@
+-- Migration 001: the main database's schema. The models in
+-- internal/database must agree with it.
+
+CREATE TABLE settings (
+    "key" text,
+    value text NOT NULL,
+    PRIMARY KEY ("key")
+);
+
+CREATE TABLE users (
+    id uuid,
+    created_at datetime,
+    updated_at datetime,
+    deleted_at datetime,
+    username text NOT NULL,
+    password text NOT NULL,
+    PRIMARY KEY (id),
+    CONSTRAINT chk_users_username
+        CHECK (length(CAST(username AS BLOB)) <= 1024)
+);
+
+CREATE INDEX idx_users_deleted_at ON users (deleted_at);
+CREATE UNIQUE INDEX idx_users_username ON users (username);
+
+CREATE TABLE api_keys (
+    id uuid,
+    created_at datetime,
+    updated_at datetime,
+    deleted_at datetime,
+    user_id uuid NOT NULL,
+    "key" text NOT NULL,
+    description text,
+    last_used_at datetime,
+    PRIMARY KEY (id),
+    CONSTRAINT fk_users_api_keys FOREIGN KEY (user_id) REFERENCES users (id)
+);
+
+CREATE INDEX idx_api_keys_deleted_at ON api_keys (deleted_at);
+CREATE UNIQUE INDEX idx_api_keys_key ON api_keys ("key");
+
+CREATE TABLE webhooks (
+    id uuid,
+    created_at datetime,
+    updated_at datetime,
+    deleted_at datetime,
+    user_id uuid NOT NULL,
+    name text NOT NULL,
+    description text,
+    retention_days integer DEFAULT 30,
+    PRIMARY KEY (id),
+    CONSTRAINT fk_users_webhooks FOREIGN KEY (user_id) REFERENCES users (id)
+);
+
+CREATE INDEX idx_webhooks_deleted_at ON webhooks (deleted_at);
+
+CREATE TABLE entrypoints (
+    id uuid,
+    created_at datetime,
+    updated_at datetime,
+    deleted_at datetime,
+    webhook_id uuid NOT NULL,
+    path text NOT NULL,
+    description text,
+    active numeric DEFAULT true,
+    PRIMARY KEY (id),
+    CONSTRAINT fk_webhooks_entrypoints
+        FOREIGN KEY (webhook_id) REFERENCES webhooks (id)
+);
+
+CREATE INDEX idx_entrypoints_deleted_at ON entrypoints (deleted_at);
+CREATE UNIQUE INDEX idx_entrypoints_path ON entrypoints (path);
+
+CREATE TABLE targets (
+    id uuid,
+    created_at datetime,
+    updated_at datetime,
+    deleted_at datetime,
+    webhook_id uuid NOT NULL,
+    name text NOT NULL,
+    type text NOT NULL,
+    active numeric DEFAULT true,
+    config text,
+    max_retries integer,
+    PRIMARY KEY (id),
+    CONSTRAINT fk_webhooks_targets
+        FOREIGN KEY (webhook_id) REFERENCES webhooks (id)
+);
+
+CREATE INDEX idx_targets_deleted_at ON targets (deleted_at);

internal/db/migrations/migrations.go

@@ -0,0 +1,135 @@
+// Package migrations holds the schema of each kind of SQLite file
+// webhooker writes, as numbered SQL files embedded in the binary, and
+// applies it when a file is opened.
+//
+// Each kind of file has its own directory. In each, 000_migration.sql
+// creates only the schema_migrations table, and 001_schema.sql holds
+// the whole schema. Before 1.0 a schema change is an edit to
+// 001_schema.sql, never a new file.
+package migrations
+
+import (
+	"embed"
+	"fmt"
+	"io/fs"
+	"path"
+	"strconv"
+	"strings"
+
+	"gorm.io/gorm"
+)
+
+// The directories, one for each kind of database file.
+const (
+	// Main is the main database, webhooker.db.
+	Main = "main"
+
+	// Events is a webhook's event database, events-{uuid}.db.
+	Events = "events"
+
+	// Archive is a database target's archive file.
+	Archive = "archive"
+)
+
+//go:embed main/*.sql events/*.sql archive/*.sql
+var files embed.FS
+
+// Apply runs each SQL file in dir, which is Main, Events or Archive,
+// that the database has not had yet, in the order of the number its
+// name starts with, and records that number in schema_migrations.
+// 000_migration.sql creates that table and records itself, so it runs
+// first, and only when the table is missing.
+//
+// It all runs in one transaction, so a failure leaves the file as it
+// was.
+func Apply(db *gorm.DB, dir string) error {
+	return db.Transaction(func(tx *gorm.DB) error {
+		var tables int
+
+		// Row().Scan, not (*gorm.DB).Scan: see internal/gormlog.
+		err := tx.Raw(
+			"SELECT count(*) FROM sqlite_master "+
+				"WHERE type = 'table' AND name = ?",
+			"schema_migrations",
+		).Row().Scan(&tables)
+		if err != nil {
+			return fmt.Errorf("looking for schema_migrations: %w", err)
+		}
+
+		if tables == 0 {
+			err = run(tx, path.Join(dir, "000_migration.sql"))
+			if err != nil {
+				return err
+			}
+		}
+
+		// fs.Glob lists the files in name order, which is number order.
+		names, err := fs.Glob(files, path.Join(dir, "*.sql"))
+		if err != nil {
+			return fmt.Errorf("listing migrations: %w", err)
+		}
+
+		for _, name := range names {
+			err = applyOnce(tx, name)
+			if err != nil {
+				return err
+			}
+		}
+
+		return nil
+	})
+}
+
+// applyOnce runs the migration file name and records its number,
+// unless schema_migrations already has that number.
+func applyOnce(tx *gorm.DB, name string) error {
+	number, _, _ := strings.Cut(path.Base(name), "_")
+
+	version, err := strconv.Atoi(number)
+	if err != nil {
+		return fmt.Errorf("numbering migration %s: %w", name, err)
+	}
+
+	var applied int
+
+	err = tx.Raw(
+		"SELECT count(*) FROM schema_migrations WHERE version = ?",
+		version,
+	).Row().Scan(&applied)
+	if err != nil {
+		return fmt.Errorf("checking migration %s: %w", name, err)
+	}
+
+	if applied > 0 {
+		return nil
+	}
+
+	err = run(tx, name)
+	if err != nil {
+		return err
+	}
+
+	err = tx.Exec(
+		"INSERT INTO schema_migrations (version) VALUES (?)", version,
+	).Error
+	if err != nil {
+		return fmt.Errorf("recording migration %s: %w", name, err)
+	}
+
+	return nil
+}
+
+// run executes every statement in the migration file name.
+func run(tx *gorm.DB, name string) error {
+	script, err := files.ReadFile(name)
+	if err != nil {
+		return fmt.Errorf("reading migration %s: %w", name, err)
+	}
+
+	err = tx.Exec(string(script)).Error
+	if err != nil {
+		return fmt.Errorf("applying migration %s: %w", name, err)
+	}
+
+	return nil
+}

internal/db/migrations/migrations_test.go

@@ -0,0 +1,57 @@
+package migrations_test
+
+import (
+	"database/sql"
+	"log/slog"
+	"path/filepath"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"gorm.io/driver/sqlite"
+	"gorm.io/gorm"
+	_ "modernc.org/sqlite"
+	"sneak.berlin/go/webhooker/internal/db/migrations"
+	"sneak.berlin/go/webhooker/internal/gormlog"
+)
+
+// TestApplyRunsEachFileOnce applies each kind of database file's
+// migrations to a new file twice. The first time records every file.
+// The second finds them all recorded and runs none, which is the only
+// way it can succeed: 001_schema.sql creates tables that then exist.
+func TestApplyRunsEachFileOnce(t *testing.T) {
+	t.Parallel()
+
+	for _, dir := range []string{
+		migrations.Main, migrations.Events, migrations.Archive,
+	} {
+		t.Run(dir, func(t *testing.T) {
+			t.Parallel()
+
+			sqlDB, err := sql.Open(
+				"sqlite", filepath.Join(t.TempDir(), "test.db"),
+			)
+			require.NoError(t, err)
+
+			t.Cleanup(func() { _ = sqlDB.Close() })
+
+			db, err := gorm.Open(
+				sqlite.Dialector{Conn: sqlDB},
+				&gorm.Config{
+					Logger: gormlog.New(slog.New(slog.DiscardHandler)),
+				},
+			)
+			require.NoError(t, err)
+
+			require.NoError(t, migrations.Apply(db, dir))
+			require.NoError(t, migrations.Apply(db, dir))
+
+			var versions string
+
+			require.NoError(t, sqlDB.QueryRowContext(t.Context(),
+				"SELECT group_concat(version) FROM schema_migrations",
+			).Scan(&versions))
+			assert.Equal(t, "0,1", versions)
+		})
+	}
+}