diff --git a/internal/database/database.go b/internal/database/database.go index 06d611d..0cc7c4e 100644 --- a/internal/database/database.go +++ b/internal/database/database.go @@ -6,24 +6,32 @@ // multiple source files. Blobs are content-addressed, meaning their filename // is derived from their SHA256 hash after compression and encryption. // -// The database does not support migrations. If the schema changes, delete -// the local database and perform a full backup to recreate it. +// Schema is managed via numbered SQL migrations embedded in the schema/ +// directory. Migration 000.sql bootstraps the schema_migrations tracking +// table; subsequent migrations (001, 002, …) are applied in order. package database import ( "context" "database/sql" - _ "embed" + "embed" "fmt" "os" + "path/filepath" + "sort" + "strconv" "strings" "git.eeqj.de/sneak/vaultik/internal/log" _ "modernc.org/sqlite" ) -//go:embed schema.sql -var schemaSQL string +//go:embed schema/*.sql +var schemaFS embed.FS + +// bootstrapVersion is the migration that creates the schema_migrations +// table itself. It is applied before the normal migration loop. +const bootstrapVersion = 0 // DB represents the Vaultik local index database connection. // It uses SQLite to track file metadata, content-defined chunks, and blob associations. @@ -35,6 +43,46 @@ type DB struct { path string } +// ParseMigrationVersion extracts the numeric version prefix from a migration +// filename. Filenames must follow the pattern ".sql" or +// "_.sql", where version is a zero-padded numeric +// string (e.g. "001", "002"). Returns the version as an integer and an +// error if the filename does not match the expected pattern. +func ParseMigrationVersion(filename string) (int, error) { + name := strings.TrimSuffix(filename, filepath.Ext(filename)) + if name == "" { + return 0, fmt.Errorf("invalid migration filename %q: empty name", filename) + } + + // Split on underscore to separate version from description. + // If there's no underscore, the entire stem is the version. + versionStr := name + if idx := strings.IndexByte(name, '_'); idx >= 0 { + versionStr = name[:idx] + } + + if versionStr == "" { + return 0, fmt.Errorf("invalid migration filename %q: empty version prefix", filename) + } + + // Validate the version is purely numeric. + for _, ch := range versionStr { + if ch < '0' || ch > '9' { + return 0, fmt.Errorf( + "invalid migration filename %q: version %q contains non-numeric character %q", + filename, versionStr, string(ch), + ) + } + } + + version, err := strconv.Atoi(versionStr) + if err != nil { + return 0, fmt.Errorf("invalid migration filename %q: %w", filename, err) + } + + return version, nil +} + // New creates a new database connection at the specified path. // It creates the schema if needed and configures SQLite with WAL mode for // better concurrency. SQLite handles crash recovery automatically when @@ -72,9 +120,9 @@ func New(ctx context.Context, path string) (*DB, error) { } db := &DB{conn: conn, path: path} - if err := db.createSchema(ctx); err != nil { + if err := applyMigrations(ctx, conn); err != nil { _ = conn.Close() - return nil, fmt.Errorf("creating schema: %w", err) + return nil, fmt.Errorf("applying migrations: %w", err) } return db, nil } @@ -125,9 +173,9 @@ func New(ctx context.Context, path string) (*DB, error) { } db := &DB{conn: conn, path: path} - if err := db.createSchema(ctx); err != nil { + if err := applyMigrations(ctx, conn); err != nil { _ = conn.Close() - return nil, fmt.Errorf("creating schema: %w", err) + return nil, fmt.Errorf("applying migrations: %w", err) } log.Debug("Database connection established successfully", "path", path) @@ -198,9 +246,120 @@ func (db *DB) QueryRowWithLog( return db.conn.QueryRowContext(ctx, query, args...) } -func (db *DB) createSchema(ctx context.Context) error { - _, err := db.conn.ExecContext(ctx, schemaSQL) - return err +// collectMigrations reads the embedded schema directory and returns +// migration filenames sorted lexicographically. +func collectMigrations() ([]string, error) { + entries, err := schemaFS.ReadDir("schema") + if err != nil { + return nil, fmt.Errorf("failed to read schema directory: %w", err) + } + + var migrations []string + + for _, entry := range entries { + if !entry.IsDir() && strings.HasSuffix(entry.Name(), ".sql") { + migrations = append(migrations, entry.Name()) + } + } + + sort.Strings(migrations) + + return migrations, nil +} + +// bootstrapMigrationsTable ensures the schema_migrations table exists +// by applying 000.sql if the table is missing. +func bootstrapMigrationsTable(ctx context.Context, db *sql.DB) error { + var tableExists int + + err := db.QueryRowContext(ctx, + "SELECT COUNT(*) FROM sqlite_master WHERE type='table' AND name='schema_migrations'", + ).Scan(&tableExists) + if err != nil { + return fmt.Errorf("failed to check for migrations table: %w", err) + } + + if tableExists > 0 { + return nil + } + + content, err := schemaFS.ReadFile("schema/000.sql") + if err != nil { + return fmt.Errorf("failed to read bootstrap migration 000.sql: %w", err) + } + + log.Info("applying bootstrap migration", "version", bootstrapVersion) + + _, err = db.ExecContext(ctx, string(content)) + if err != nil { + return fmt.Errorf("failed to apply bootstrap migration: %w", err) + } + + return nil +} + +// applyMigrations applies all pending migrations to db. It first bootstraps +// the schema_migrations table via 000.sql, then iterates through remaining +// migration files in order. +func applyMigrations(ctx context.Context, db *sql.DB) error { + if err := bootstrapMigrationsTable(ctx, db); err != nil { + return err + } + + migrations, err := collectMigrations() + if err != nil { + return err + } + + for _, migration := range migrations { + version, parseErr := ParseMigrationVersion(migration) + if parseErr != nil { + return parseErr + } + + // Check if already applied. + var count int + + err := db.QueryRowContext(ctx, + "SELECT COUNT(*) FROM schema_migrations WHERE version = ?", + version, + ).Scan(&count) + if err != nil { + return fmt.Errorf("failed to check migration status: %w", err) + } + + if count > 0 { + log.Debug("migration already applied", "version", version) + + continue + } + + // Read and apply migration. + content, readErr := schemaFS.ReadFile(filepath.Join("schema", migration)) + if readErr != nil { + return fmt.Errorf("failed to read migration %s: %w", migration, readErr) + } + + log.Info("applying migration", "version", version) + + _, execErr := db.ExecContext(ctx, string(content)) + if execErr != nil { + return fmt.Errorf("failed to apply migration %s: %w", migration, execErr) + } + + // Record migration as applied. + _, recErr := db.ExecContext(ctx, + "INSERT INTO schema_migrations (version) VALUES (?)", + version, + ) + if recErr != nil { + return fmt.Errorf("failed to record migration %s: %w", migration, recErr) + } + + log.Info("migration applied successfully", "version", version) + } + + return nil } // NewTestDB creates an in-memory SQLite database for testing purposes. diff --git a/internal/database/database_test.go b/internal/database/database_test.go index 65457d1..6d763a3 100644 --- a/internal/database/database_test.go +++ b/internal/database/database_test.go @@ -2,6 +2,7 @@ package database import ( "context" + "database/sql" "fmt" "path/filepath" "testing" @@ -26,9 +27,10 @@ func TestDatabase(t *testing.T) { t.Fatal("database connection is nil") } - // Test schema creation (already done in New) + // Test schema creation (already done in New via migrations) // Verify tables exist tables := []string{ + "schema_migrations", "files", "file_chunks", "chunks", "blobs", "blob_chunks", "chunk_files", "snapshots", } @@ -99,3 +101,139 @@ func TestDatabaseConcurrentAccess(t *testing.T) { t.Errorf("expected 10 chunks, got %d", count) } } + +func TestParseMigrationVersion(t *testing.T) { + tests := []struct { + name string + filename string + wantVer int + wantError bool + }{ + {name: "valid 000.sql", filename: "000.sql", wantVer: 0, wantError: false}, + {name: "valid 001.sql", filename: "001.sql", wantVer: 1, wantError: false}, + {name: "valid 099.sql", filename: "099.sql", wantVer: 99, wantError: false}, + {name: "valid with description", filename: "001_initial_schema.sql", wantVer: 1, wantError: false}, + {name: "valid large version", filename: "123_big_migration.sql", wantVer: 123, wantError: false}, + {name: "invalid alpha version", filename: "abc.sql", wantVer: 0, wantError: true}, + {name: "invalid mixed chars", filename: "12a.sql", wantVer: 0, wantError: true}, + {name: "invalid no extension", filename: "schema.sql", wantVer: 0, wantError: true}, + {name: "empty string", filename: "", wantVer: 0, wantError: true}, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + got, err := ParseMigrationVersion(tc.filename) + if tc.wantError { + if err == nil { + t.Errorf("ParseMigrationVersion(%q) = %d, nil; want error", tc.filename, got) + } + return + } + if err != nil { + t.Errorf("ParseMigrationVersion(%q) unexpected error: %v", tc.filename, err) + return + } + if got != tc.wantVer { + t.Errorf("ParseMigrationVersion(%q) = %d; want %d", tc.filename, got, tc.wantVer) + } + }) + } +} + +func TestApplyMigrations_Idempotent(t *testing.T) { + ctx := context.Background() + + conn, err := sql.Open("sqlite", ":memory:?_foreign_keys=ON") + if err != nil { + t.Fatalf("failed to open database: %v", err) + } + defer func() { + if err := conn.Close(); err != nil { + t.Errorf("failed to close database: %v", err) + } + }() + + conn.SetMaxOpenConns(1) + conn.SetMaxIdleConns(1) + + // First run: apply all migrations. + if err := applyMigrations(ctx, conn); err != nil { + t.Fatalf("first applyMigrations failed: %v", err) + } + + // Count rows in schema_migrations after first run. + var countBefore int + if err := conn.QueryRowContext(ctx, "SELECT COUNT(*) FROM schema_migrations").Scan(&countBefore); err != nil { + t.Fatalf("failed to count schema_migrations after first run: %v", err) + } + + // Second run: must be a no-op. + if err := applyMigrations(ctx, conn); err != nil { + t.Fatalf("second applyMigrations failed: %v", err) + } + + // Count rows in schema_migrations after second run — must be unchanged. + var countAfter int + if err := conn.QueryRowContext(ctx, "SELECT COUNT(*) FROM schema_migrations").Scan(&countAfter); err != nil { + t.Fatalf("failed to count schema_migrations after second run: %v", err) + } + + if countBefore != countAfter { + t.Errorf("schema_migrations row count changed: before=%d, after=%d", countBefore, countAfter) + } +} + +func TestBootstrapMigrationsTable_FreshDatabase(t *testing.T) { + ctx := context.Background() + + conn, err := sql.Open("sqlite", ":memory:?_foreign_keys=ON") + if err != nil { + t.Fatalf("failed to open database: %v", err) + } + defer func() { + if err := conn.Close(); err != nil { + t.Errorf("failed to close database: %v", err) + } + }() + + conn.SetMaxOpenConns(1) + conn.SetMaxIdleConns(1) + + // Verify schema_migrations does NOT exist yet. + var tableBefore int + if err := conn.QueryRowContext(ctx, + "SELECT COUNT(*) FROM sqlite_master WHERE type='table' AND name='schema_migrations'", + ).Scan(&tableBefore); err != nil { + t.Fatalf("failed to check for table before bootstrap: %v", err) + } + if tableBefore != 0 { + t.Fatal("schema_migrations table should not exist before bootstrap") + } + + // Run bootstrap. + if err := bootstrapMigrationsTable(ctx, conn); err != nil { + t.Fatalf("bootstrapMigrationsTable failed: %v", err) + } + + // Verify schema_migrations now exists. + var tableAfter int + if err := conn.QueryRowContext(ctx, + "SELECT COUNT(*) FROM sqlite_master WHERE type='table' AND name='schema_migrations'", + ).Scan(&tableAfter); err != nil { + t.Fatalf("failed to check for table after bootstrap: %v", err) + } + if tableAfter != 1 { + t.Fatalf("schema_migrations table should exist after bootstrap, got count=%d", tableAfter) + } + + // Verify version 0 row exists. + var version int + if err := conn.QueryRowContext(ctx, + "SELECT version FROM schema_migrations WHERE version = 0", + ).Scan(&version); err != nil { + t.Fatalf("version 0 row not found in schema_migrations: %v", err) + } + if version != 0 { + t.Errorf("expected version 0, got %d", version) + } +} diff --git a/internal/database/schema/000.sql b/internal/database/schema/000.sql new file mode 100644 index 0000000..e06a2da --- /dev/null +++ b/internal/database/schema/000.sql @@ -0,0 +1,9 @@ +-- Migration 000: Schema migrations tracking table +-- Applied as a bootstrap step before the normal migration loop. + +CREATE TABLE IF NOT EXISTS schema_migrations ( + version INTEGER PRIMARY KEY, + applied_at DATETIME DEFAULT CURRENT_TIMESTAMP +); + +INSERT OR IGNORE INTO schema_migrations (version) VALUES (0); diff --git a/internal/database/schema.sql b/internal/database/schema/001.sql similarity index 96% rename from internal/database/schema.sql rename to internal/database/schema/001.sql index cdc8533..5f54565 100644 --- a/internal/database/schema.sql +++ b/internal/database/schema/001.sql @@ -1,6 +1,5 @@ --- Vaultik Database Schema --- Note: This database does not support migrations. If the schema changes, --- delete the local database and perform a full backup to recreate it. +-- Migration 001: Initial Vaultik schema +-- All core tables for tracking files, chunks, blobs, snapshots, and uploads. -- Files table: stores metadata about files in the filesystem CREATE TABLE IF NOT EXISTS files ( @@ -133,4 +132,4 @@ CREATE TABLE IF NOT EXISTS uploads ( ); -- Index for efficient snapshot lookups -CREATE INDEX IF NOT EXISTS idx_uploads_snapshot_id ON uploads(snapshot_id); \ No newline at end of file +CREATE INDEX IF NOT EXISTS idx_uploads_snapshot_id ON uploads(snapshot_id); diff --git a/internal/database/schema/008_uploads.sql b/internal/database/schema/008_uploads.sql deleted file mode 100644 index 49b5add..0000000 --- a/internal/database/schema/008_uploads.sql +++ /dev/null @@ -1,11 +0,0 @@ --- Track blob upload metrics -CREATE TABLE IF NOT EXISTS uploads ( - blob_hash TEXT PRIMARY KEY, - uploaded_at TIMESTAMP NOT NULL, - size INTEGER NOT NULL, - duration_ms INTEGER NOT NULL, - FOREIGN KEY (blob_hash) REFERENCES blobs(blob_hash) -); - -CREATE INDEX idx_uploads_uploaded_at ON uploads(uploaded_at); -CREATE INDEX idx_uploads_duration ON uploads(duration_ms); \ No newline at end of file