Skip vaultik's own index and PID file in a backup (closes #421)

A backup of a directory that holds the local index stored a torn copy
of it, its -wal and -shm side files and vaultik.pid, all being written
while they were read. The scan now treats these files as excluded,
whatever the config says. Their paths come from index_path and the lock
directory, with symlinks in their directories resolved the way source
paths are. pidlock now owns the lock directory and the PID file name,
so the CLI and the scanner use the same path.

Model: opus-5-5

internal/cli/app.go

@@ -7,11 +7,9 @@ import (
 	"context"
 	"errors"
 	"fmt"
-	"path/filepath"
 	"sync"
 	"time"
 
-	"github.com/adrg/xdg"
 	"github.com/spf13/cobra"
 	"go.uber.org/dig"
 	"go.uber.org/fx"
@@ -351,8 +349,7 @@ func runVaultikApp(
 // only one runs at a time; a read-only command runs without it and is not
 // blocked while a mutator holds the lock (opts.Mode).
 func RunWithApp(ctx context.Context, opts AppOptions) error {
-	release, err := acquireLockIfMutating(opts.Mode,
+	release, err := acquireLockIfMutating(opts.Mode, pidlock.Dir())
-		filepath.Join(xdg.DataHome, "vaultik"))
 	if err != nil {
 		return err
 	}

internal/database/module.go

@@ -20,6 +20,17 @@ const indexDirPerm = 0o700
 // backed-up path and chunk hash and must stay private.
 const indexFilePerm = 0o600
 
+// IndexFiles returns the path of the index at indexPath and the paths of
+// the side files SQLite keeps beside it while it writes.
+func IndexFiles(indexPath string) []string {
+	return []string{
+		indexPath,
+		indexPath + "-wal",
+		indexPath + "-shm",
+		indexPath + "-journal",
+	}
+}
+
 // ensureIndexFileMode makes the index file owner-only before the SQLite
 // driver opens it: it creates the file 0600 if absent, or chmods an existing
 // one to 0600. Doing this first matters because SQLite creates its -wal and

internal/pidlock/pidlock.go

@@ -11,6 +11,7 @@ import (
 	"strconv"
 	"strings"
 
+	"github.com/adrg/xdg"
 	"golang.org/x/sys/unix"
 )
 
@@ -29,8 +30,20 @@ const (
 	lockDirPerm = 0o700
 	// pidFilePerm is the mode for the PID file (owner-only).
 	pidFilePerm = 0o600
+	// pidFileName is the name of the PID file in the lock directory.
+	pidFileName = "vaultik.pid"
 )
 
+// Dir returns the directory that holds vaultik's PID file.
+func Dir() string {
+	return filepath.Join(xdg.DataHome, "vaultik")
+}
+
+// Path returns the path of the PID file that Acquire uses in lockDir.
+func Path(lockDir string) string {
+	return filepath.Join(lockDir, pidFileName)
+}
+
 // Acquire attempts to acquire a PID lock in the specified directory.
 // If another process holds the lock, it returns ErrAlreadyRunning with
 // that process's PID. On success, it writes the current PID to the lock
@@ -42,7 +55,7 @@ func Acquire(lockDir string) (*Lock, error) {
 		return nil, fmt.Errorf("creating lock directory: %w", err)
 	}
 
-	lockPath := filepath.Join(lockDir, "vaultik.pid")
+	lockPath := Path(lockDir)
 
 	// No O_TRUNC: the file may hold the PID of the process that has the
 	// lock, which the error below reports.

internal/snapshot/module.go

@@ -2,11 +2,13 @@ package snapshot
 
 import (
 	"os"
+	"path/filepath"
 
 	"github.com/spf13/afero"
 	"go.uber.org/fx"
 	"sneak.berlin/go/vaultik/internal/config"
 	"sneak.berlin/go/vaultik/internal/database"
+	"sneak.berlin/go/vaultik/internal/pidlock"
 	"sneak.berlin/go/vaultik/internal/storage"
 	"sneak.berlin/go/vaultik/internal/ui"
 )
@@ -62,9 +64,39 @@ func provideScannerFactory(
 			EnableProgress:     params.EnableProgress,
 			UI:                 params.UI,
 			Exclude:            excludes,
+			OwnFiles:           ownFiles(cfg.IndexPath, pidlock.Dir()),
 			SkipErrors:         params.SkipErrors,
 			PreviousSnapshotID: params.PreviousSnapshotID,
 			DeviceNumber:       params.DeviceNumber,
 		})
 	}
 }
+
+// ownFiles returns the paths of the index at indexPath, its side files
+// and the PID file in lockDir. Symlinks in their directories are resolved,
+// as they are in the snapshot's source paths, so the scan finds them under
+// the same names.
+func ownFiles(indexPath, lockDir string) []string {
+	paths := append(database.IndexFiles(indexPath), pidlock.Path(lockDir))
+	for i, p := range paths {
+		paths[i] = resolveDir(p)
+	}
+
+	return paths
+}
+
+// resolveDir returns the absolute form of path with the symlinks in its
+// directory resolved. It returns path unchanged when that fails.
+func resolveDir(path string) string {
+	abs, err := filepath.Abs(path)
+	if err != nil {
+		return path
+	}
+
+	dir, err := filepath.EvalSymlinks(filepath.Dir(abs))
+	if err != nil {
+		return abs
+	}
+
+	return filepath.Join(dir, filepath.Base(abs))
+}

internal/snapshot/module_test.go

@@ -0,0 +1,32 @@
+//nolint:testpackage // exercises the unexported ownFiles helper
+package snapshot
+
+import (
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+// The scan walks source paths with symlinks resolved, so vaultik's own
+// files are named the same way, or a symlinked home would back them up.
+func TestOwnFilesResolvesSymlinkedDirectory(t *testing.T) {
+	t.Parallel()
+
+	realDir, err := filepath.EvalSymlinks(t.TempDir())
+	require.NoError(t, err)
+
+	link := filepath.Join(t.TempDir(), "link")
+	require.NoError(t, os.Symlink(realDir, link))
+
+	paths := ownFiles(filepath.Join(link, "index.sqlite"), link)
+
+	require.Equal(t, []string{
+		filepath.Join(realDir, "index.sqlite"),
+		filepath.Join(realDir, "index.sqlite-wal"),
+		filepath.Join(realDir, "index.sqlite-shm"),
+		filepath.Join(realDir, "index.sqlite-journal"),
+		filepath.Join(realDir, "vaultik.pid"),
+	}, paths)
+}

internal/snapshot/own_files_test.go

@@ -0,0 +1,65 @@
+package snapshot_test
+
+import (
+	"context"
+	"testing"
+
+	"github.com/spf13/afero"
+	"github.com/stretchr/testify/require"
+	"sneak.berlin/go/vaultik/internal/database"
+	"sneak.berlin/go/vaultik/internal/snapshot"
+)
+
+// A backup of the directory that holds vaultik's index and PID file
+// stores neither, with no exclude pattern set, and still stores the
+// files beside them. See https://git.eeqj.de/sneak/vaultik/issues/421.
+func TestScanSkipsOwnFiles(t *testing.T) {
+	t.Parallel()
+
+	const dataDir = "/home/user/.local/share/vaultik"
+
+	ownFiles := append(database.IndexFiles(dataDir+"/index.sqlite"),
+		dataDir+"/vaultik.pid")
+
+	fs := afero.NewMemMapFs()
+	for _, p := range append(ownFiles, dataDir+"/notes.txt") {
+		require.NoError(t, afero.WriteFile(fs, p, []byte("data"), 0o600))
+	}
+
+	db, err := database.NewTestDB()
+	require.NoError(t, err)
+
+	defer func() { _ = db.Close() }()
+
+	repos := database.NewRepositories(db)
+
+	scanner := snapshot.NewScanner(snapshot.ScannerConfig{
+		FS:               fs,
+		ChunkSize:        64 * 1024,
+		Repositories:     repos,
+		MaxBlobSize:      1024 * 1024,
+		CompressionLevel: 3,
+		AgeRecipients: []string{
+			"age1ql3z7hjy54pw3hyww5ayyfg7zqgvc7w3j2elw8zmrj2kg5sfn9aqmcac8p"},
+		OwnFiles: ownFiles,
+	})
+	require.NotNil(t, scanner)
+
+	ctx := context.Background()
+	createSnapshotRecord(ctx, t, repos, "test-snapshot")
+
+	result, err := scanner.Scan(ctx, "/home/user", "test-snapshot")
+	require.NoError(t, err)
+
+	require.Equal(t, 1, result.FilesScanned)
+
+	stored, err := repos.Files.GetByPath(ctx, dataDir+"/notes.txt")
+	require.NoError(t, err)
+	require.NotNil(t, stored, "a file beside the index is stored")
+
+	for _, p := range ownFiles {
+		f, err := repos.Files.GetByPath(ctx, p)
+		require.NoError(t, err)
+		require.Nil(t, f, "%s is not stored", p)
+	}
+}

internal/snapshot/scanner.go

@@ -67,7 +67,10 @@ type Scanner struct {
 	currentSourcePath string
 	exclude           []string          // Glob patterns for files/directories to exclude
 	compiledExclude   []compiledPattern // Compiled glob patterns
-	progress          *ProgressReporter
+	// ownFiles holds the paths of vaultik's own working files, which the
+	// scan treats as excluded (ScannerConfig.OwnFiles).
+	ownFiles map[string]struct{}
+	progress *ProgressReporter
 	// skipErrors skips files that cannot be opened or read (logged loudly);
 	// packer, database, encryption, and upload errors still abort the run.
 	skipErrors bool
@@ -133,6 +136,11 @@ type ScannerConfig struct {
 	EnableProgress   bool       // Enable the live progress reporter (ETAs, throughput)
 	UI               *ui.Writer // Where user-facing scanner messages go; nil = discard
 	Exclude          []string   // Glob patterns for files/directories to exclude
+	// OwnFiles are the absolute paths of vaultik's own working files: the
+	// index, its side files and the PID file. The scan treats them as
+	// excluded, whatever Exclude says, since a copy taken while they are
+	// being written is torn.
+	OwnFiles []string
 	// SkipErrors skips files that cannot be opened or read (log loudly but
 	// continue); packer, database, encryption, and upload errors still abort.
 	SkipErrors bool
@@ -226,6 +234,11 @@ func NewScanner(cfg ScannerConfig) *Scanner {
 		deviceNumber = deviceNumberFromStat
 	}
 
+	ownFiles := make(map[string]struct{}, len(cfg.OwnFiles))
+	for _, p := range cfg.OwnFiles {
+		ownFiles[filepath.Clean(p)] = struct{}{}
+	}
+
 	return &Scanner{
 		fs:                 cfg.FS,
 		chunker:            chunker.NewChunker(cfg.ChunkSize),
@@ -237,6 +250,7 @@ func NewScanner(cfg ScannerConfig) *Scanner {
 		ageRecipient:       strings.Join(cfg.AgeRecipients, ","),
 		exclude:            cfg.Exclude,
 		compiledExclude:    compiledExclude,
+		ownFiles:           ownFiles,
 		progress:           progress,
 		skipErrors:         cfg.SkipErrors,
 		previousSnapshotID: cfg.PreviousSnapshotID,
@@ -2373,9 +2387,14 @@ func doubleStarVariants(pattern string) []string {
 }
 
 // shouldExclude checks if a path should be excluded based on exclude patterns
+// or because it is one of vaultik's own working files.
 // filePath is the full path to the file
 // rootPath is the root of the backup source directory
 func (s *Scanner) shouldExclude(filePath, rootPath string) bool {
+	if _, own := s.ownFiles[filePath]; own {
+		return true
+	}
+
 	if len(s.compiledExclude) == 0 {
 		return false
 	}