Run the local index in WAL mode with a busy timeout (closes #217)
check / check (pull_request) Successful in 5m4s

The connection settings were passed as `_journal_mode=`-style
parameters, which the SQLite driver drops without an error, so the
index ran in rollback-journal mode with no busy timeout. `snapshot
list` or `info` reading during a backup could make the backup's next
write fail with "database is locked". Both open paths now pass
`_pragma=` parameters; foreign keys moved there too.

With WAL on, rows committed to the open index can still be in the
-wal file, which a copy of the main file misses. The metadata export
now copies the index with VACUUM INTO, into an empty 0600 file.

The retry after a failed open no longer claims a TRUNCATE recovery; it
retries with the same settings.

Model: opus-5-5
This commit is contained in:
2026-10-06 08:49:03 +00:00
parent 4a167e153a
commit 7a39442eac
7 changed files with 252 additions and 91 deletions
+33 -38
View File
@@ -386,12 +386,12 @@ func (sm *SnapshotManager) prepareExportDB(
ctx context.Context, dbPath, snapshotID, tempDir string,
) ([]byte, string, error) {
// Step 1: Copy database to temp file
// The main database should be closed at this point
// The main database is still open here, so it is copied through SQLite
tempDBPath := filepath.Join(tempDir, "snapshot.db")
log.Debug("Copying database to temporary location",
"source", dbPath, "destination", tempDBPath)
err := sm.copyFile(dbPath, tempDBPath)
err := sm.copyDatabase(ctx, dbPath, tempDBPath)
if err != nil {
return nil, "", fmt.Errorf("copying database: %w", err)
}
@@ -648,9 +648,10 @@ func (sm *SnapshotManager) collectCleanupStats(
//
// VACUUM runs through the modernc.org/sqlite driver, on a freshly opened
// connection with no transaction in flight (VACUUM cannot run inside one).
// The database opens in WAL mode, so VACUUM's rewrite lands in the WAL; the
// checkpoint on Close flushes it into the main file, which is the file we
// then compress and upload.
// database.New opens the file in WAL mode, so VACUUM's rewrite lands in the
// -wal file. This is the only connection to the file, so closing it
// checkpoints the rewrite into the main file and removes the -wal file; the
// main file is the one compressFile then compresses and uploads.
func (sm *SnapshotManager) vacuumDatabase(ctx context.Context, dbPath string) error {
log.Debug("Running VACUUM on database", "path", dbPath)
@@ -744,26 +745,15 @@ func (sm *SnapshotManager) compressFile(inputPath, outputPath string) error {
// user; it holds the same private index data as the local index file.
const exportCopyPerm = 0o600
// copyFile copies a file from src to dst. The destination is the exported
// snapshot database, so it is created owner-only rather than with the
// umask-dependent default.
func (sm *SnapshotManager) copyFile(src, dst string) error {
log.Debug("Opening source file for copy", "path", src)
sourceFile, err := sm.fs.Open(src)
if err != nil {
return err
}
defer func() {
log.Debug("Closing source file", "path", src)
err := sourceFile.Close()
if err != nil {
log.Debug("Failed to close source file", "path", src, "error", err)
}
}()
// copyDatabase copies the database at src to dst with VACUUM INTO. It reads
// through SQLite, so the copy holds rows committed to src that are still in
// its -wal file, which a copy of the file alone would miss. The destination
// is the exported snapshot database, so it is created empty and owner-only
// first rather than with the umask-dependent default: VACUUM INTO writes
// into an existing empty file and keeps its mode.
func (sm *SnapshotManager) copyDatabase(
ctx context.Context, src, dst string,
) error {
log.Debug("Creating destination file", "path", dst)
destFile, err := sm.fs.OpenFile(
@@ -773,23 +763,28 @@ func (sm *SnapshotManager) copyFile(src, dst string) error {
return err
}
defer func() {
log.Debug("Closing destination file", "path", dst)
err := destFile.Close()
if err != nil {
log.Debug("Failed to close destination file", "path", dst, "error", err)
}
}()
log.Debug("Copying file data")
n, err := io.Copy(destFile, sourceFile)
err = destFile.Close()
if err != nil {
return err
}
log.Debug("File copy complete", "bytes_copied", n)
db, err := database.New(ctx, src)
if err != nil {
return fmt.Errorf("opening database to copy: %w", err)
}
defer func() {
cerr := db.Close()
if cerr != nil {
log.Debug("Failed to close database after copy",
"path", src, "error", cerr)
}
}()
_, err = db.ExecWithLog(ctx, "VACUUM INTO ?", dst)
if err != nil {
return fmt.Errorf("running VACUUM INTO: %w", err)
}
return nil
}