Skip vaultik's own index and PID file in a backup (closes #421)

The default config backs up the home directory, which holds vaultik's local index and PID file, so a backup stored a torn copy of the index it was writing. The scan now skips the index, its SQLite side files and the PID file whatever the config says, including when index_path or its directory is a symlink. database purge removes the same list of index files.

Model: opus-5-5

internal/snapshot/index_and_lock_files_test.go

@@ -0,0 +1,65 @@
+package snapshot_test
+
+import (
+	"context"
+	"testing"
+
+	"github.com/spf13/afero"
+	"github.com/stretchr/testify/require"
+	"sneak.berlin/go/vaultik/internal/database"
+	"sneak.berlin/go/vaultik/internal/snapshot"
+)
+
+// A backup of the directory that holds vaultik's index and PID file
+// stores neither, with no exclude pattern set, and still stores the
+// files beside them. See https://git.eeqj.de/sneak/vaultik/issues/421.
+func TestScanSkipsIndexAndLockFiles(t *testing.T) {
+	t.Parallel()
+
+	const dataDir = "/home/user/.local/share/vaultik"
+
+	indexAndLockFiles := append(database.IndexFiles(dataDir+"/index.sqlite"),
+		dataDir+"/vaultik.pid")
+
+	fs := afero.NewMemMapFs()
+	for _, p := range append(indexAndLockFiles, dataDir+"/notes.txt") {
+		require.NoError(t, afero.WriteFile(fs, p, []byte("data"), 0o600))
+	}
+
+	db, err := database.NewTestDB()
+	require.NoError(t, err)
+
+	defer func() { _ = db.Close() }()
+
+	repos := database.NewRepositories(db)
+
+	scanner := snapshot.NewScanner(snapshot.ScannerConfig{
+		FS:               fs,
+		ChunkSize:        64 * 1024,
+		Repositories:     repos,
+		MaxBlobSize:      1024 * 1024,
+		CompressionLevel: 3,
+		AgeRecipients: []string{
+			"age1ql3z7hjy54pw3hyww5ayyfg7zqgvc7w3j2elw8zmrj2kg5sfn9aqmcac8p"},
+		IndexAndLockFiles: indexAndLockFiles,
+	})
+	require.NotNil(t, scanner)
+
+	ctx := context.Background()
+	createSnapshotRecord(ctx, t, repos, "test-snapshot")
+
+	result, err := scanner.Scan(ctx, "/home/user", "test-snapshot")
+	require.NoError(t, err)
+
+	require.Equal(t, 1, result.FilesScanned)
+
+	stored, err := repos.Files.GetByPath(ctx, dataDir+"/notes.txt")
+	require.NoError(t, err)
+	require.NotNil(t, stored, "a file beside the index is stored")
+
+	for _, p := range indexAndLockFiles {
+		f, err := repos.Files.GetByPath(ctx, p)
+		require.NoError(t, err)
+		require.Nil(t, f, "%s is not stored", p)
+	}
+}

internal/snapshot/module.go

@@ -2,11 +2,13 @@ package snapshot
 
 import (
 	"os"
+	"path/filepath"
 
 	"github.com/spf13/afero"
 	"go.uber.org/fx"
 	"sneak.berlin/go/vaultik/internal/config"
 	"sneak.berlin/go/vaultik/internal/database"
+	"sneak.berlin/go/vaultik/internal/pidlock"
 	"sneak.berlin/go/vaultik/internal/storage"
 	"sneak.berlin/go/vaultik/internal/ui"
 )
@@ -62,9 +64,52 @@ func provideScannerFactory(
 			EnableProgress:     params.EnableProgress,
 			UI:                 params.UI,
 			Exclude:            excludes,
+			IndexAndLockFiles:  indexAndLockFiles(cfg.IndexPath, pidlock.Dir()),
 			SkipErrors:         params.SkipErrors,
 			PreviousSnapshotID: params.PreviousSnapshotID,
 			DeviceNumber:       params.DeviceNumber,
 		})
 	}
 }
+
+// indexAndLockFiles returns the paths of the index at indexPath, its side
+// files and the PID file in lockDir. Symlinks in their directories are
+// resolved, as they are in the snapshot's source paths, so the scan finds
+// them under the same names. If indexPath is a symlink, the file it points
+// to and that file's side files are returned too.
+func indexAndLockFiles(indexPath, lockDir string) []string {
+	paths := append(database.IndexFiles(indexPath), pidlock.Path(lockDir))
+	for i, p := range paths {
+		paths[i] = resolveParentDir(p)
+	}
+
+	// SQLite writes the file a symlinked index points to and keeps the
+	// side files beside that file. The database package creates the index
+	// before the scanner is built, so the link resolves.
+	index := resolveParentDir(indexPath)
+
+	target, err := filepath.EvalSymlinks(index)
+	if err == nil && target != index {
+		paths = append(paths, database.IndexFiles(target)...)
+	}
+
+	return paths
+}
+
+// resolveParentDir returns the absolute form of path with the symlinks in
+// its parent directory resolved. If the parent directory cannot be
+// resolved, it returns the absolute form unresolved. If path has no
+// absolute form, it returns path.
+func resolveParentDir(path string) string {
+	abs, err := filepath.Abs(path)
+	if err != nil {
+		return path
+	}
+
+	dir, err := filepath.EvalSymlinks(filepath.Dir(abs))
+	if err != nil {
+		return abs
+	}
+
+	return filepath.Join(dir, filepath.Base(abs))
+}

internal/snapshot/module_test.go

@@ -0,0 +1,32 @@
+//nolint:testpackage // exercises the unexported indexAndLockFiles helper
+package snapshot
+
+import (
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+// The scan walks source paths with symlinks resolved, so vaultik's own
+// files are named the same way, or a symlinked home would back them up.
+func TestIndexAndLockFilesResolvesSymlinkedDirectory(t *testing.T) {
+	t.Parallel()
+
+	realDir, err := filepath.EvalSymlinks(t.TempDir())
+	require.NoError(t, err)
+
+	link := filepath.Join(t.TempDir(), "link")
+	require.NoError(t, os.Symlink(realDir, link))
+
+	paths := indexAndLockFiles(filepath.Join(link, "index.sqlite"), link)
+
+	require.Equal(t, []string{
+		filepath.Join(realDir, "index.sqlite"),
+		filepath.Join(realDir, "index.sqlite-wal"),
+		filepath.Join(realDir, "index.sqlite-shm"),
+		filepath.Join(realDir, "index.sqlite-journal"),
+		filepath.Join(realDir, "vaultik.pid"),
+	}, paths)
+}

internal/snapshot/scanner.go

@@ -67,6 +67,9 @@ type Scanner struct {
 	currentSourcePath string
 	exclude           []string          // Glob patterns for files/directories to exclude
 	compiledExclude   []compiledPattern // Compiled glob patterns
+	// indexAndLockFiles holds the paths of vaultik's own working files,
+	// which the scan treats as excluded (ScannerConfig.IndexAndLockFiles).
+	indexAndLockFiles map[string]struct{}
 	progress          *ProgressReporter
 	// skipErrors skips files that cannot be opened or read (logged loudly);
 	// packer, database, encryption, and upload errors still abort the run.
@@ -133,6 +136,11 @@ type ScannerConfig struct {
 	EnableProgress   bool       // Enable the live progress reporter (ETAs, throughput)
 	UI               *ui.Writer // Where user-facing scanner messages go; nil = discard
 	Exclude          []string   // Glob patterns for files/directories to exclude
+	// IndexAndLockFiles are the absolute paths of vaultik's own working
+	// files: the index, its side files and the PID file. The scan treats
+	// them as excluded, whatever Exclude says, since a copy taken while
+	// they are being written is torn.
+	IndexAndLockFiles []string
 	// SkipErrors skips files that cannot be opened or read (log loudly but
 	// continue); packer, database, encryption, and upload errors still abort.
 	SkipErrors bool
@@ -226,6 +234,11 @@ func NewScanner(cfg ScannerConfig) *Scanner {
 		deviceNumber = deviceNumberFromStat
 	}
 
+	indexAndLockFiles := make(map[string]struct{}, len(cfg.IndexAndLockFiles))
+	for _, p := range cfg.IndexAndLockFiles {
+		indexAndLockFiles[filepath.Clean(p)] = struct{}{}
+	}
+
 	return &Scanner{
 		fs:                 cfg.FS,
 		chunker:            chunker.NewChunker(cfg.ChunkSize),
@@ -237,6 +250,7 @@ func NewScanner(cfg ScannerConfig) *Scanner {
 		ageRecipient:       strings.Join(cfg.AgeRecipients, ","),
 		exclude:            cfg.Exclude,
 		compiledExclude:    compiledExclude,
+		indexAndLockFiles:  indexAndLockFiles,
 		progress:           progress,
 		skipErrors:         cfg.SkipErrors,
 		previousSnapshotID: cfg.PreviousSnapshotID,
@@ -2252,8 +2266,9 @@ func (s *Scanner) detectDeletedFiles(
 
 // excludedOrInExcludedDir reports whether filePath, which is at or
 // beneath rootPath, or any directory between rootPath and it, matches an
-// exclude pattern. A pattern such as */.local/share/Trash matches the
-// directory but not the files inside it, which the walk skips too.
+// exclude pattern or is one of vaultik's own files. A pattern such as
+// */.local/share/Trash matches the directory but not the files inside it,
+// which the walk skips too.
 func (s *Scanner) excludedOrInExcludedDir(filePath, rootPath string) bool {
 	for p := filePath; len(p) > len(rootPath); p = filepath.Dir(p) {
 		if s.shouldExclude(p, rootPath) {
@@ -2377,9 +2392,14 @@ func doubleStarVariants(pattern string) []string {
 }
 
 // shouldExclude checks if a path should be excluded based on exclude patterns
+// or because it is one of vaultik's own working files.
 // filePath is the full path to the file
 // rootPath is the root of the backup source directory
 func (s *Scanner) shouldExclude(filePath, rootPath string) bool {
+	if _, ok := s.indexAndLockFiles[filePath]; ok {
+		return true
+	}
+
 	if len(s.compiledExclude) == 0 {
 		return false
 	}