Run the local index in WAL mode with a busy timeout (closes #217)
The connection settings were passed as `_journal_mode=`-style parameters, which the SQLite driver drops without an error, so the index ran in rollback-journal mode with no busy timeout. `snapshot list` or `info` reading during a backup could make the backup's next write fail with "database is locked". Both open paths now pass `_pragma=` parameters; foreign keys moved there too. With WAL on, rows committed to the open index can still be in the -wal file, which a copy of the main file misses. The metadata export now copies the index with VACUUM INTO, into an empty 0600 file. The retry after a failed open no longer claims a TRUNCATE recovery; it retries with the same settings. Model: opus-5-5
This commit was merged in pull request #242.
This commit is contained in:
@@ -386,12 +386,12 @@ func (sm *SnapshotManager) prepareExportDB(
|
||||
ctx context.Context, dbPath, snapshotID, tempDir string,
|
||||
) ([]byte, string, error) {
|
||||
// Step 1: Copy database to temp file
|
||||
// The main database should be closed at this point
|
||||
// The main database is still open here, so it is copied through SQLite
|
||||
tempDBPath := filepath.Join(tempDir, "snapshot.db")
|
||||
log.Debug("Copying database to temporary location",
|
||||
"source", dbPath, "destination", tempDBPath)
|
||||
|
||||
err := sm.copyFile(dbPath, tempDBPath)
|
||||
err := sm.copyDatabase(ctx, dbPath, tempDBPath)
|
||||
if err != nil {
|
||||
return nil, "", fmt.Errorf("copying database: %w", err)
|
||||
}
|
||||
@@ -648,9 +648,10 @@ func (sm *SnapshotManager) collectCleanupStats(
|
||||
//
|
||||
// VACUUM runs through the modernc.org/sqlite driver, on a freshly opened
|
||||
// connection with no transaction in flight (VACUUM cannot run inside one).
|
||||
// The database opens in WAL mode, so VACUUM's rewrite lands in the WAL; the
|
||||
// checkpoint on Close flushes it into the main file, which is the file we
|
||||
// then compress and upload.
|
||||
// database.New opens the file in WAL mode, so VACUUM's rewrite lands in the
|
||||
// -wal file. This is the only connection to the file, so closing it
|
||||
// checkpoints the rewrite into the main file and removes the -wal file; the
|
||||
// main file is the one compressFile then compresses and uploads.
|
||||
func (sm *SnapshotManager) vacuumDatabase(ctx context.Context, dbPath string) error {
|
||||
log.Debug("Running VACUUM on database", "path", dbPath)
|
||||
|
||||
@@ -744,26 +745,15 @@ func (sm *SnapshotManager) compressFile(inputPath, outputPath string) error {
|
||||
// user; it holds the same private index data as the local index file.
|
||||
const exportCopyPerm = 0o600
|
||||
|
||||
// copyFile copies a file from src to dst. The destination is the exported
|
||||
// snapshot database, so it is created owner-only rather than with the
|
||||
// umask-dependent default.
|
||||
func (sm *SnapshotManager) copyFile(src, dst string) error {
|
||||
log.Debug("Opening source file for copy", "path", src)
|
||||
|
||||
sourceFile, err := sm.fs.Open(src)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
defer func() {
|
||||
log.Debug("Closing source file", "path", src)
|
||||
|
||||
err := sourceFile.Close()
|
||||
if err != nil {
|
||||
log.Debug("Failed to close source file", "path", src, "error", err)
|
||||
}
|
||||
}()
|
||||
|
||||
// copyDatabase copies the database at src to dst with VACUUM INTO. It reads
|
||||
// through SQLite, so the copy holds rows committed to src that are still in
|
||||
// its -wal file, which a copy of the file alone would miss. The destination
|
||||
// is the exported snapshot database, so it is created empty and owner-only
|
||||
// first rather than with the umask-dependent default: VACUUM INTO writes
|
||||
// into an existing empty file and keeps its mode.
|
||||
func (sm *SnapshotManager) copyDatabase(
|
||||
ctx context.Context, src, dst string,
|
||||
) error {
|
||||
log.Debug("Creating destination file", "path", dst)
|
||||
|
||||
destFile, err := sm.fs.OpenFile(
|
||||
@@ -773,23 +763,28 @@ func (sm *SnapshotManager) copyFile(src, dst string) error {
|
||||
return err
|
||||
}
|
||||
|
||||
defer func() {
|
||||
log.Debug("Closing destination file", "path", dst)
|
||||
|
||||
err := destFile.Close()
|
||||
if err != nil {
|
||||
log.Debug("Failed to close destination file", "path", dst, "error", err)
|
||||
}
|
||||
}()
|
||||
|
||||
log.Debug("Copying file data")
|
||||
|
||||
n, err := io.Copy(destFile, sourceFile)
|
||||
err = destFile.Close()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
log.Debug("File copy complete", "bytes_copied", n)
|
||||
db, err := database.New(ctx, src)
|
||||
if err != nil {
|
||||
return fmt.Errorf("opening database to copy: %w", err)
|
||||
}
|
||||
|
||||
defer func() {
|
||||
cerr := db.Close()
|
||||
if cerr != nil {
|
||||
log.Debug("Failed to close database after copy",
|
||||
"path", src, "error", cerr)
|
||||
}
|
||||
}()
|
||||
|
||||
_, err = db.ExecWithLog(ctx, "VACUUM INTO ?", dst)
|
||||
if err != nil {
|
||||
return fmt.Errorf("running VACUUM INTO: %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user