Do not prune when a backup misses an earlier mount point (closes #395)

snapshot create --prune purged the older snapshots holding a volume's
files when the volume, mounted below a configured path, was not
mounted: its empty mount point read as deleted files.

The scan now records in the local index each directory at or below a
configured path that is on a different device from its parent
directory. Under --prune, when an earlier snapshot of that name
recorded such a directory and this run does not find it on a different
device, the run skips the prune, keeps every older snapshot, names the
directory and exits non-zero, as for an empty configured path. For a
volume retired, or a directory no longer backed up, on purpose, the
older snapshots are removed with snapshot purge.

Model: opus-5-5

internal/snapshot/device_number_test.go

@@ -0,0 +1,40 @@
+//nolint:testpackage // exercises the unexported deviceNumberFromStat
+package snapshot
+
+import (
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// The device number a real backup reads is the same for a directory and
+// a file in it, and differs between the temp directory and /dev.
+// Without it no mount point is ever found. See
+// https://git.eeqj.de/sneak/vaultik/issues/395.
+func TestDeviceNumberFromStat(t *testing.T) {
+	t.Parallel()
+
+	dir := t.TempDir()
+	file := filepath.Join(dir, "file.txt")
+	require.NoError(t, os.WriteFile(file, nil, 0o600))
+
+	deviceOf := func(path string) uint64 {
+		t.Helper()
+
+		info, err := os.Stat(path)
+		require.NoError(t, err)
+
+		device, ok := deviceNumberFromStat(info)
+		require.True(t, ok, "no device number read for %s", path)
+
+		return device
+	}
+
+	assert.Equal(t, deviceOf(dir), deviceOf(file))
+
+	// /dev is a different filesystem from the temp dir on Linux, macOS and in Docker.
+	assert.NotEqual(t, deviceOf(dir), deviceOf("/dev"))
+}

internal/snapshot/module.go

@@ -1,6 +1,8 @@
 package snapshot
 
 import (
+	"os"
+
 	"github.com/spf13/afero"
 	"go.uber.org/fx"
 	"sneak.berlin/go/vaultik/internal/config"
@@ -19,6 +21,9 @@ type ScannerParams struct {
 	// PreviousSnapshotID is the most recent completed snapshot of the
 	// same name, "" for none.
 	PreviousSnapshotID string
+	// DeviceNumber is ScannerConfig.DeviceNumber; nil reads it from the
+	// file's stat.
+	DeviceNumber func(info os.FileInfo) (uint64, bool)
 }
 
 // Module exports backup functionality as an fx module.
@@ -59,6 +64,7 @@ func provideScannerFactory(
 			Exclude:            excludes,
 			SkipErrors:         params.SkipErrors,
 			PreviousSnapshotID: params.PreviousSnapshotID,
+			DeviceNumber:       params.DeviceNumber,
 		})
 	}
 }

internal/snapshot/mount_point_test.go

@@ -0,0 +1,97 @@
+package snapshot_test
+
+import (
+	"context"
+	"os"
+	"path/filepath"
+	"slices"
+	"testing"
+
+	"github.com/spf13/afero"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"sneak.berlin/go/vaultik/internal/database"
+	"sneak.berlin/go/vaultik/internal/snapshot"
+)
+
+// A scan records with the snapshot each directory on a different device
+// from its parent directory, as a mount point is, the scanned path
+// included. A filesystem that gives no device numbers, as the in-memory
+// one, records none. See https://git.eeqj.de/sneak/vaultik/issues/395.
+func TestScanRecordsMountPoints(t *testing.T) {
+	t.Parallel()
+
+	const (
+		scannedPath = "/data/source"
+		snapshotID  = "mount-points"
+	)
+
+	tests := []struct {
+		name string
+		// onVolume names the directories on the volume's device; every
+		// other directory is on another one. nil leaves the device
+		// numbers to the in-memory filesystem, which has none.
+		onVolume []string
+		want     []string
+	}{
+		{
+			name:     "volume below the scanned path",
+			onVolume: []string{"nas"},
+			want:     []string{scannedPath + "/nas"},
+		},
+		{
+			name:     "volume at the scanned path",
+			onVolume: []string{"source", "nas"},
+			want:     []string{scannedPath},
+		},
+		{name: "no device numbers", want: nil},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			t.Parallel()
+
+			ctx := context.Background()
+			fs := afero.NewMemMapFs()
+
+			for _, path := range []string{
+				scannedPath + "/notes.txt", scannedPath + "/nas/a.txt",
+			} {
+				require.NoError(t, fs.MkdirAll(filepath.Dir(path), 0o755))
+				require.NoError(t, afero.WriteFile(fs, path, []byte(path), 0o644))
+			}
+
+			db, err := database.NewTestDB()
+			require.NoError(t, err)
+			t.Cleanup(func() { _ = db.Close() })
+
+			repos := database.NewRepositories(db)
+			createTestSnapshotRecord(ctx, t, repos, snapshotID)
+
+			cfg := snapshot.ScannerConfig{
+				FS:               fs,
+				ChunkSize:        int64(1024 * 16),
+				Repositories:     repos,
+				MaxBlobSize:      int64(1024 * 1024),
+				CompressionLevel: 3,
+				AgeRecipients:    []string{testAgePublicKey},
+			}
+			if tt.onVolume != nil {
+				cfg.DeviceNumber = func(info os.FileInfo) (uint64, bool) {
+					if slices.Contains(tt.onVolume, info.Name()) {
+						return 2, true
+					}
+
+					return 1, true
+				}
+			}
+
+			_, err = snapshot.NewScanner(cfg).Scan(ctx, scannedPath, snapshotID)
+			require.NoError(t, err)
+
+			got, err := repos.Snapshots.GetMountPoints(ctx, snapshotID)
+			require.NoError(t, err)
+			assert.Equal(t, tt.want, got)
+		})
+	}
+}

internal/snapshot/scanner.go

@@ -73,6 +73,9 @@ type Scanner struct {
 	skipErrors bool
 	// ui is the user-facing output; never nil (defaults to a discarding writer).
 	ui *ui.Writer
+	// deviceNumber returns the number of the device that holds a file;
+	// never nil (defaults to deviceNumberFromStat).
+	deviceNumber func(info os.FileInfo) (uint64, bool)
 
 	// In-memory cache of known chunk hashes for fast existence checks
 	knownChunks   map[string]struct{}
@@ -136,6 +139,10 @@ type ScannerConfig struct {
 	// PreviousSnapshotID is the most recent completed snapshot of the
 	// same name, "" for none.
 	PreviousSnapshotID string
+	// DeviceNumber returns the number of the device that holds a file,
+	// and false when there is none to read. nil reads it from the file's
+	// stat; tests replace it, since the in-memory test filesystem has none.
+	DeviceNumber func(info os.FileInfo) (uint64, bool)
 }
 
 // ScanResult contains the results of a scan operation. Files and bytes
@@ -214,6 +221,11 @@ func NewScanner(cfg ScannerConfig) *Scanner {
 		uiw = ui.NewWithColor(io.Discard, false)
 	}
 
+	deviceNumber := cfg.DeviceNumber
+	if deviceNumber == nil {
+		deviceNumber = deviceNumberFromStat
+	}
+
 	return &Scanner{
 		fs:                 cfg.FS,
 		chunker:            chunker.NewChunker(cfg.ChunkSize),
@@ -229,6 +241,7 @@ func NewScanner(cfg ScannerConfig) *Scanner {
 		skipErrors:         cfg.SkipErrors,
 		previousSnapshotID: cfg.PreviousSnapshotID,
 		ui:                 uiw,
+		deviceNumber:       deviceNumber,
 		pendingChunkHashes: make(map[string]struct{}),
 	}
 }
@@ -1290,9 +1303,9 @@ func (s *Scanner) buildDirectoryEntry(path string, info os.FileInfo) *database.F
 	}
 }
 
-// recordDirectories records every directory the walk found. A directory
-// has no data to back up, so it is not among the files counted and
-// backed up.
+// recordDirectories records every directory the walk found, and its
+// mount points (see recordMountPoints). A directory has no data to back
+// up, so it is not among the files counted and backed up.
 func (s *Scanner) recordDirectories(
 	ctx context.Context, directories []*FileToProcess,
 ) error {
@@ -1303,7 +1316,97 @@ func (s *Scanner) recordDirectories(
 		}
 	}
 
-	return nil
+	return s.recordMountPoints(ctx, directories)
+}
+
+// recordMountPoints records with the snapshot each directory the walk
+// found on a different device from its parent directory, as a mount
+// point is, the scanned path included.
+func (s *Scanner) recordMountPoints(
+	ctx context.Context, directories []*FileToProcess,
+) error {
+	mountPoints, err := s.findMountPoints(directories)
+	if err != nil {
+		return err
+	}
+
+	return s.repos.WithTx(ctx, func(txCtx context.Context, tx *sql.Tx) error {
+		for _, dir := range mountPoints {
+			log.Info("Found a mount point", "path", dir)
+
+			err := s.repos.Snapshots.AddMountPoint(txCtx, tx, s.snapshotID, dir)
+			if err != nil {
+				return fmt.Errorf("recording mount point %s: %w", dir, err)
+			}
+		}
+
+		return nil
+	})
+}
+
+// findMountPoints returns the directories, among those the walk found,
+// that are on a different device from their parent directory. The
+// scanned path is compared with the directory that holds it, which the
+// walk did not reach.
+func (s *Scanner) findMountPoints(
+	directories []*FileToProcess,
+) ([]string, error) {
+	walked := make(map[string]os.FileInfo, len(directories))
+	for _, dir := range directories {
+		walked[dir.Path] = dir.FileInfo
+	}
+
+	var mountPoints []string
+
+	for _, dir := range directories {
+		parentPath := filepath.Dir(dir.Path)
+
+		parent, ok := walked[parentPath]
+		if !ok {
+			var err error
+
+			parent, err = s.fs.Stat(parentPath)
+			if err != nil {
+				return nil, fmt.Errorf(
+					"looking up the parent of %s: %w", dir.Path, err)
+			}
+		}
+
+		if s.onOtherDevice(dir.FileInfo, parent) {
+			mountPoints = append(mountPoints, dir.Path)
+		}
+	}
+
+	return mountPoints, nil
+}
+
+// onOtherDevice reports whether dir is on a different device from
+// parent. A file whose device number cannot be read counts as on the
+// same device as its parent.
+func (s *Scanner) onOtherDevice(dir, parent os.FileInfo) bool {
+	dirDevice, ok := s.deviceNumber(dir)
+	if !ok {
+		return false
+	}
+
+	parentDevice, ok := s.deviceNumber(parent)
+	if !ok {
+		return false
+	}
+
+	return dirDevice != parentDevice
+}
+
+// deviceNumberFromStat returns the number of the device that holds the
+// file info describes, and false when info carries no stat, as from the
+// in-memory test filesystem.
+func deviceNumberFromStat(info os.FileInfo) (uint64, bool) {
+	stat, ok := info.Sys().(*syscall.Stat_t)
+	if !ok {
+		return 0, false
+	}
+
+	return uint64(stat.Dev), true //nolint:unconvert // Dev is an int32 on darwin
 }
 
 // recordNonRegularFile writes a symlink or directory entry to the database

internal/snapshot/snapshot.go

@@ -518,8 +518,8 @@ type CleanupStats struct {
 //
 // The cleanup is performed in a specific order to maintain referential integrity:
 //  1. Delete other snapshots
-//  2. Delete orphaned snapshot associations (snapshot_files, snapshot_blobs)
-//     for deleted snapshots
+//  2. Delete orphaned snapshot associations (snapshot_files, snapshot_blobs,
+//     snapshot_mount_points) for deleted snapshots
 //  3. Delete orphaned files (not in the current snapshot)
 //  4. Delete orphaned chunk-to-file mappings (references to deleted files)
 //  5. Delete orphaned blobs (not in the current snapshot)
@@ -948,8 +948,8 @@ func (sm *SnapshotManager) deleteOtherSnapshots(
 	return nil
 }
 
-// deleteOrphanedSnapshotAssociations deletes snapshot_files and
-// snapshot_blobs for deleted snapshots
+// deleteOrphanedSnapshotAssociations deletes snapshot_files,
+// snapshot_blobs and snapshot_mount_points for deleted snapshots
 func (sm *SnapshotManager) deleteOrphanedSnapshotAssociations(
 	ctx context.Context, tx *sql.Tx, currentSnapshotID string,
 ) error {
@@ -983,6 +983,19 @@ func (sm *SnapshotManager) deleteOrphanedSnapshotAssociations(
 	log.Debug("[Temp DB Cleanup] Deleted snapshot_blobs associations",
 		"count", rowsAffected)
 
+	// Delete the mount points other snapshots recorded
+	query := "DELETE FROM snapshot_mount_points WHERE snapshot_id != ?"
+	database.LogSQL("Execute", query, currentSnapshotID)
+
+	result, err = tx.ExecContext(ctx, query, currentSnapshotID)
+	if err != nil {
+		return fmt.Errorf("deleting orphaned snapshot_mount_points: %w", err)
+	}
+
+	rowsAffected, _ = result.RowsAffected()
+	log.Debug("[Temp DB Cleanup] Deleted snapshot_mount_points rows",
+		"count", rowsAffected)
+
 	return nil
 }
 

internal/snapshot/snapshot_test.go

@@ -7,12 +7,14 @@ import (
 	"database/sql"
 	"io"
 	"path/filepath"
+	"slices"
 	"testing"
 
 	"github.com/spf13/afero"
 	"sneak.berlin/go/vaultik/internal/config"
 	"sneak.berlin/go/vaultik/internal/database"
 	"sneak.berlin/go/vaultik/internal/log"
+	"sneak.berlin/go/vaultik/internal/types"
 )
 
 const (
@@ -387,6 +389,77 @@ func TestCleanSnapshotDBNonExistentSnapshot(t *testing.T) {
 	}
 }
 
+// The exported metadata database keeps only the mount points that the
+// exported snapshot recorded. See
+// https://git.eeqj.de/sneak/vaultik/issues/395.
+func TestCleanSnapshotDBKeepsOnlyTheSnapshotsMountPoints(t *testing.T) {
+	log.Initialize(log.Config{})
+	t.Parallel()
+
+	ctx := context.Background()
+	dbPath := filepath.Join(t.TempDir(), "export.db")
+
+	db, err := database.New(ctx, dbPath)
+	if err != nil {
+		t.Fatalf("failed to create database: %v", err)
+	}
+
+	repos := database.NewRepositories(db)
+
+	for _, id := range []types.SnapshotID{"exported", "other"} {
+		err = repos.Snapshots.Create(ctx, nil, &database.Snapshot{ID: id})
+		if err != nil {
+			t.Fatalf("failed to create snapshot: %v", err)
+		}
+
+		err = repos.Snapshots.AddMountPoint(ctx, nil, id.String(), "/data/"+id.String())
+		if err != nil {
+			t.Fatalf("failed to add mount point: %v", err)
+		}
+	}
+
+	err = db.Close()
+	if err != nil {
+		t.Fatalf("failed to close database: %v", err)
+	}
+
+	sm := &SnapshotManager{
+		config: &config.Config{
+			CompressionLevel: 3,
+			AgeRecipients:    []string{testAgeRecipient},
+		},
+		fs: afero.NewOsFs(),
+	}
+
+	_, err = sm.cleanSnapshotDB(ctx, dbPath, "exported")
+	if err != nil {
+		t.Fatalf("failed to clean snapshot database: %v", err)
+	}
+
+	cleaned, err := database.New(ctx, dbPath)
+	if err != nil {
+		t.Fatalf("failed to open cleaned database: %v", err)
+	}
+
+	defer func() { _ = cleaned.Close() }()
+
+	cleanedRepos := database.NewRepositories(cleaned)
+
+	for id, want := range map[string][]string{
+		"exported": {"/data/exported"},
+		"other":    nil,
+	} {
+		got, err := cleanedRepos.Snapshots.GetMountPoints(ctx, id)
+		if err != nil {
+			t.Fatalf("failed to read mount points: %v", err)
+		}
+
+		if !slices.Equal(got, want) {
+			t.Errorf("mount points of %s: got %v, want %v", id, got, want)
+		}
+	}
+}
+
 // Two creates of one snapshot name back to back start within one second,
 // the resolution of the timestamp in a snapshot ID. See
 // https://git.eeqj.de/sneak/vaultik/issues/270.