sneak
bbe09ec5b5
SQLite handles crash recovery automatically when opening a database. The previous recoverDatabase() function was deleting journal and WAL files BEFORE opening the database, which prevented SQLite from recovering incomplete transactions and caused database corruption after Ctrl+C or crashes. This was causing "database disk image is malformed" errors after interrupting a backup operation.
226 lines
8.0 KiB
Go
226 lines
8.0 KiB
Go
// Package database provides the local SQLite index for Vaultik backup operations.
|
|
// The database tracks files, chunks, and their associations with blobs.
|
|
//
|
|
// Blobs in Vaultik are the final storage units uploaded to S3. Each blob is a
|
|
// large (up to 10GB) file containing many compressed and encrypted chunks from
|
|
// multiple source files. Blobs are content-addressed, meaning their filename
|
|
// is derived from their SHA256 hash after compression and encryption.
|
|
//
|
|
// The database does not support migrations. If the schema changes, delete
|
|
// the local database and perform a full backup to recreate it.
|
|
package database
|
|
|
|
import (
|
|
"context"
|
|
"database/sql"
|
|
_ "embed"
|
|
"fmt"
|
|
"os"
|
|
"strings"
|
|
|
|
"git.eeqj.de/sneak/vaultik/internal/log"
|
|
_ "modernc.org/sqlite"
|
|
)
|
|
|
|
//go:embed schema.sql
|
|
var schemaSQL string
|
|
|
|
// DB represents the Vaultik local index database connection.
|
|
// It uses SQLite to track file metadata, content-defined chunks, and blob associations.
|
|
// The database enables incremental backups by detecting changed files and
|
|
// supports deduplication by tracking which chunks are already stored in blobs.
|
|
// Write operations are synchronized through a mutex to ensure thread safety.
|
|
type DB struct {
|
|
conn *sql.DB
|
|
path string
|
|
}
|
|
|
|
// New creates a new database connection at the specified path.
|
|
// It creates the schema if needed and configures SQLite with WAL mode for
|
|
// better concurrency. SQLite handles crash recovery automatically when
|
|
// opening a database with journal/WAL files present.
|
|
// The path parameter can be a file path for persistent storage or ":memory:"
|
|
// for an in-memory database (useful for testing).
|
|
func New(ctx context.Context, path string) (*DB, error) {
|
|
log.Debug("Opening database connection", "path", path)
|
|
|
|
// Note: We do NOT delete journal/WAL files before opening.
|
|
// SQLite handles crash recovery automatically when the database is opened.
|
|
// Deleting these files would corrupt the database after an unclean shutdown.
|
|
|
|
// First attempt with standard WAL mode
|
|
log.Debug("Attempting to open database with WAL mode", "path", path)
|
|
conn, err := sql.Open(
|
|
"sqlite",
|
|
path+"?_journal_mode=WAL&_synchronous=NORMAL&_busy_timeout=10000&_locking_mode=NORMAL&_foreign_keys=ON",
|
|
)
|
|
if err == nil {
|
|
// Set connection pool settings
|
|
// SQLite can handle multiple readers but only one writer at a time.
|
|
// Setting MaxOpenConns to 1 ensures all writes are serialized through
|
|
// a single connection, preventing SQLITE_BUSY errors.
|
|
conn.SetMaxOpenConns(1)
|
|
conn.SetMaxIdleConns(1)
|
|
|
|
if err := conn.PingContext(ctx); err == nil {
|
|
// Success on first try
|
|
log.Debug("Database opened successfully with WAL mode", "path", path)
|
|
|
|
// Enable foreign keys explicitly
|
|
if _, err := conn.ExecContext(ctx, "PRAGMA foreign_keys = ON"); err != nil {
|
|
log.Warn("Failed to enable foreign keys", "error", err)
|
|
}
|
|
|
|
db := &DB{conn: conn, path: path}
|
|
if err := db.createSchema(ctx); err != nil {
|
|
_ = conn.Close()
|
|
return nil, fmt.Errorf("creating schema: %w", err)
|
|
}
|
|
return db, nil
|
|
}
|
|
log.Debug("Failed to ping database, closing connection", "path", path, "error", err)
|
|
_ = conn.Close()
|
|
}
|
|
|
|
// If first attempt failed, try with TRUNCATE mode to clear any locks
|
|
log.Info(
|
|
"Database appears locked, attempting recovery with TRUNCATE mode",
|
|
"path", path,
|
|
)
|
|
conn, err = sql.Open(
|
|
"sqlite",
|
|
path+"?_journal_mode=TRUNCATE&_synchronous=NORMAL&_busy_timeout=10000&_foreign_keys=ON",
|
|
)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("opening database in recovery mode: %w", err)
|
|
}
|
|
|
|
// Set connection pool settings
|
|
// SQLite can handle multiple readers but only one writer at a time.
|
|
// Setting MaxOpenConns to 1 ensures all writes are serialized through
|
|
// a single connection, preventing SQLITE_BUSY errors.
|
|
conn.SetMaxOpenConns(1)
|
|
conn.SetMaxIdleConns(1)
|
|
|
|
if err := conn.PingContext(ctx); err != nil {
|
|
log.Debug("Failed to ping database in recovery mode, closing", "path", path, "error", err)
|
|
_ = conn.Close()
|
|
return nil, fmt.Errorf(
|
|
"database still locked after recovery attempt: %w",
|
|
err,
|
|
)
|
|
}
|
|
|
|
log.Debug("Database opened in TRUNCATE mode", "path", path)
|
|
|
|
// Switch back to WAL mode
|
|
log.Debug("Switching database back to WAL mode", "path", path)
|
|
if _, err := conn.ExecContext(ctx, "PRAGMA journal_mode=WAL"); err != nil {
|
|
log.Warn("Failed to switch back to WAL mode", "path", path, "error", err)
|
|
}
|
|
|
|
// Ensure foreign keys are enabled
|
|
if _, err := conn.ExecContext(ctx, "PRAGMA foreign_keys=ON"); err != nil {
|
|
log.Warn("Failed to enable foreign keys", "path", path, "error", err)
|
|
}
|
|
|
|
db := &DB{conn: conn, path: path}
|
|
if err := db.createSchema(ctx); err != nil {
|
|
_ = conn.Close()
|
|
return nil, fmt.Errorf("creating schema: %w", err)
|
|
}
|
|
|
|
log.Debug("Database connection established successfully", "path", path)
|
|
return db, nil
|
|
}
|
|
|
|
// Close closes the database connection.
|
|
// It ensures all pending operations are completed before closing.
|
|
// Returns an error if the database connection cannot be closed properly.
|
|
func (db *DB) Close() error {
|
|
log.Debug("Closing database connection", "path", db.path)
|
|
if err := db.conn.Close(); err != nil {
|
|
log.Error("Failed to close database", "path", db.path, "error", err)
|
|
return fmt.Errorf("failed to close database: %w", err)
|
|
}
|
|
log.Debug("Database connection closed successfully", "path", db.path)
|
|
return nil
|
|
}
|
|
|
|
// Conn returns the underlying *sql.DB connection.
|
|
// This should be used sparingly and primarily for read operations.
|
|
// For write operations, prefer using the ExecWithLog method.
|
|
func (db *DB) Conn() *sql.DB {
|
|
return db.conn
|
|
}
|
|
|
|
// BeginTx starts a new database transaction with the given options.
|
|
// The caller is responsible for committing or rolling back the transaction.
|
|
// For write transactions, consider using the Repositories.WithTx method instead,
|
|
// which handles locking and rollback automatically.
|
|
func (db *DB) BeginTx(
|
|
ctx context.Context,
|
|
opts *sql.TxOptions,
|
|
) (*sql.Tx, error) {
|
|
return db.conn.BeginTx(ctx, opts)
|
|
}
|
|
|
|
// Note: LockForWrite and UnlockWrite methods have been removed.
|
|
// SQLite handles its own locking internally, so explicit locking is not needed.
|
|
|
|
// ExecWithLog executes a write query with SQL logging.
|
|
// SQLite handles its own locking internally, so we just pass through to ExecContext.
|
|
// The query and args parameters follow the same format as sql.DB.ExecContext.
|
|
func (db *DB) ExecWithLog(
|
|
ctx context.Context,
|
|
query string,
|
|
args ...interface{},
|
|
) (sql.Result, error) {
|
|
LogSQL("Execute", query, args...)
|
|
return db.conn.ExecContext(ctx, query, args...)
|
|
}
|
|
|
|
// QueryRowWithLog executes a query that returns at most one row with SQL logging.
|
|
// This is useful for queries that modify data and return values (e.g., INSERT ... RETURNING).
|
|
// SQLite handles its own locking internally.
|
|
// The query and args parameters follow the same format as sql.DB.QueryRowContext.
|
|
func (db *DB) QueryRowWithLog(
|
|
ctx context.Context,
|
|
query string,
|
|
args ...interface{},
|
|
) *sql.Row {
|
|
LogSQL("QueryRow", query, args...)
|
|
return db.conn.QueryRowContext(ctx, query, args...)
|
|
}
|
|
|
|
func (db *DB) createSchema(ctx context.Context) error {
|
|
_, err := db.conn.ExecContext(ctx, schemaSQL)
|
|
return err
|
|
}
|
|
|
|
// NewTestDB creates an in-memory SQLite database for testing purposes.
|
|
// The database is automatically initialized with the schema and is ready for use.
|
|
// Each call creates a new independent database instance.
|
|
func NewTestDB() (*DB, error) {
|
|
return New(context.Background(), ":memory:")
|
|
}
|
|
|
|
// LogSQL logs SQL queries and their arguments when debug mode is enabled.
|
|
// Debug mode is activated by setting the GODEBUG environment variable to include "vaultik".
|
|
// This is useful for troubleshooting database operations and understanding query patterns.
|
|
//
|
|
// The operation parameter describes the type of SQL operation (e.g., "Execute", "Query").
|
|
// The query parameter is the SQL statement being executed.
|
|
// The args parameter contains the query arguments that will be interpolated.
|
|
func LogSQL(operation, query string, args ...interface{}) {
|
|
if strings.Contains(os.Getenv("GODEBUG"), "vaultik") {
|
|
log.Debug(
|
|
"SQL "+operation,
|
|
"query",
|
|
strings.TrimSpace(query),
|
|
"args",
|
|
fmt.Sprintf("%v", args),
|
|
)
|
|
}
|
|
}
|