Do not prune when a backup misses an earlier mount point (closes #395)

snapshot create --prune purged the older snapshots holding a volume's
files when the volume, mounted below a configured path, was not
mounted: its empty mount point read as deleted files.

The scan now records in the local index each directory at or below a
configured path that the walk found an entry in and that, once listed,
is on a different device from its parent; listing mounts an
automounted volume. Under --prune, when an earlier snapshot of that
name recorded such a directory and this run does not, the run skips
the prune, keeps every older snapshot and exits non-zero, naming the
directory. For a volume retired on purpose, snapshot purge removes the
older snapshots.

Model: opus-5-5

internal/snapshot/device_number_test.go

@@ -0,0 +1,40 @@
+//nolint:testpackage // exercises the unexported deviceNumberFromStat
+package snapshot
+
+import (
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// The device number a real backup reads is the same for a directory and
+// a file in it, and differs between the temp directory and /dev.
+// Without it no mount point is ever found. See
+// https://git.eeqj.de/sneak/vaultik/issues/395.
+func TestDeviceNumberFromStat(t *testing.T) {
+	t.Parallel()
+
+	dir := t.TempDir()
+	file := filepath.Join(dir, "file.txt")
+	require.NoError(t, os.WriteFile(file, nil, 0o600))
+
+	deviceOf := func(path string) uint64 {
+		t.Helper()
+
+		info, err := os.Stat(path)
+		require.NoError(t, err)
+
+		device, ok := deviceNumberFromStat(info)
+		require.True(t, ok, "no device number read for %s", path)
+
+		return device
+	}
+
+	assert.Equal(t, deviceOf(dir), deviceOf(file))
+
+	// /dev is a different filesystem from the temp dir on Linux, macOS and in Docker.
+	assert.NotEqual(t, deviceOf(dir), deviceOf("/dev"))
+}

internal/snapshot/module.go

@@ -1,6 +1,8 @@
 package snapshot
 
 import (
+	"os"
+
 	"github.com/spf13/afero"
 	"go.uber.org/fx"
 	"sneak.berlin/go/vaultik/internal/config"
@@ -19,6 +21,9 @@ type ScannerParams struct {
 	// PreviousSnapshotID is the most recent completed snapshot of the
 	// same name, "" for none.
 	PreviousSnapshotID string
+	// DeviceNumber is ScannerConfig.DeviceNumber; nil reads it from the
+	// file's stat.
+	DeviceNumber func(info os.FileInfo) (uint64, bool)
 }
 
 // Module exports backup functionality as an fx module.
@@ -59,6 +64,7 @@ func provideScannerFactory(
 			Exclude:            excludes,
 			SkipErrors:         params.SkipErrors,
 			PreviousSnapshotID: params.PreviousSnapshotID,
+			DeviceNumber:       params.DeviceNumber,
 		})
 	}
 }

internal/snapshot/mount_point_test.go

@@ -0,0 +1,245 @@
+package snapshot_test
+
+import (
+	"context"
+	"os"
+	"path/filepath"
+	"slices"
+	"strings"
+	"syscall"
+	"testing"
+
+	"github.com/spf13/afero"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"sneak.berlin/go/vaultik/internal/database"
+	"sneak.berlin/go/vaultik/internal/snapshot"
+)
+
+// A scan records with the snapshot each directory on a different device
+// from its parent directory that the walk found an entry in, as a mount
+// point is, the scanned path included. A filesystem that gives no device
+// numbers, as the in-memory one, records none. See
+// https://git.eeqj.de/sneak/vaultik/issues/395.
+func TestScanRecordsMountPoints(t *testing.T) {
+	t.Parallel()
+
+	const (
+		scannedPath = "/data/source"
+		snapshotID  = "mount-points"
+	)
+
+	tests := []struct {
+		name string
+		// onVolume names the directories on the volume's device; every
+		// other directory is on another one. nil leaves the device
+		// numbers to the in-memory filesystem, which has none.
+		onVolume []string
+		want     []string
+	}{
+		{
+			name:     "volume below the scanned path",
+			onVolume: []string{"nas"},
+			want:     []string{scannedPath + "/nas"},
+		},
+		{
+			name:     "volume at the scanned path",
+			onVolume: []string{"source", "nas"},
+			want:     []string{scannedPath},
+		},
+		{name: "no device numbers", want: nil},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			t.Parallel()
+
+			ctx := context.Background()
+			fs := afero.NewMemMapFs()
+
+			for _, path := range []string{
+				scannedPath + "/notes.txt", scannedPath + "/nas/a.txt",
+			} {
+				require.NoError(t, fs.MkdirAll(filepath.Dir(path), 0o755))
+				require.NoError(t, afero.WriteFile(fs, path, []byte(path), 0o644))
+			}
+
+			db, err := database.NewTestDB()
+			require.NoError(t, err)
+			t.Cleanup(func() { _ = db.Close() })
+
+			repos := database.NewRepositories(db)
+			createTestSnapshotRecord(ctx, t, repos, snapshotID)
+
+			cfg := snapshot.ScannerConfig{
+				FS:               fs,
+				ChunkSize:        int64(1024 * 16),
+				Repositories:     repos,
+				MaxBlobSize:      int64(1024 * 1024),
+				CompressionLevel: 3,
+				AgeRecipients:    []string{testAgePublicKey},
+			}
+			if tt.onVolume != nil {
+				cfg.DeviceNumber = func(info os.FileInfo) (uint64, bool) {
+					if slices.Contains(tt.onVolume, info.Name()) {
+						return 2, true
+					}
+
+					return 1, true
+				}
+			}
+
+			_, err = snapshot.NewScanner(cfg).Scan(ctx, scannedPath, snapshotID)
+			require.NoError(t, err)
+
+			got, err := repos.Snapshots.GetMountPoints(ctx, snapshotID)
+			require.NoError(t, err)
+			assert.Equal(t, tt.want, got)
+		})
+	}
+}
+
+// automountFs is an in-memory filesystem with a volume at mountPoint
+// that is mounted on first access: opening mountPoint to list it mounts
+// the volume or, if it cannot be mounted, fails with "no such file or
+// directory", as with the autofs daemon. Stat records the device an
+// entry is on at that moment, as a real stat does: unmountedDevice for
+// mountPoint until the volume is mounted; the volume's, 2, for
+// mountPoint once it is and for everything in it; 1 for everything else.
+type automountFs struct {
+	afero.Fs
+
+	mountPoint      string
+	unmountedDevice uint64
+	canMount        bool
+	mounted         bool
+}
+
+//nolint:ireturn // afero.Fs.Open is defined to return the interface.
+func (fs *automountFs) Open(name string) (afero.File, error) {
+	if name == fs.mountPoint && !fs.canMount {
+		return nil, &os.PathError{Op: "open", Path: name, Err: syscall.ENOENT}
+	}
+
+	if name == fs.mountPoint {
+		fs.mounted = true
+	}
+
+	return fs.Fs.Open(name)
+}
+
+func (fs *automountFs) Stat(name string) (os.FileInfo, error) {
+	info, err := fs.Fs.Stat(name)
+	if err != nil {
+		return nil, err
+	}
+
+	device := uint64(1)
+
+	switch {
+	case name == fs.mountPoint && !fs.mounted:
+		device = fs.unmountedDevice
+	case name == fs.mountPoint || strings.HasPrefix(name, fs.mountPoint+"/"):
+		device = 2
+	}
+
+	return statWithDevice{FileInfo: info, device: device}, nil
+}
+
+// statWithDevice is an entry automountFs.Stat looked up, with the device
+// it was on then.
+type statWithDevice struct {
+	os.FileInfo
+
+	device uint64
+}
+
+// deviceAtLookup returns the device automountFs.Stat recorded for info.
+func deviceAtLookup(info os.FileInfo) (uint64, bool) {
+	stat, ok := info.(statWithDevice)
+
+	return stat.device, ok
+}
+
+// A volume mounted on first access is mounted when the walk lists its
+// mount point. Until then the mount point is on the automounter's own
+// device (a systemd automount) or on its parent's (an autofs map that
+// shows its mount points before mounting them). Either way only the
+// mount point is recorded, by the backup that mounts the volume and by a
+// later one that finds it mounted. Listing an automount point whose
+// volume cannot be mounted fails; it is not recorded, because the walk
+// finds no entry in it. See https://git.eeqj.de/sneak/vaultik/issues/395.
+func TestScanRecordsOnlyTheMountPointOfAnAutomountedVolume(t *testing.T) {
+	t.Parallel()
+
+	const scannedPath = "/data/source"
+
+	mountPoint := scannedPath + "/nas"
+
+	tests := []struct {
+		name            string
+		unmountedDevice uint64
+		canMount        bool
+		want            []string
+	}{
+		{
+			name:            "on the automounter's device until listed",
+			unmountedDevice: 3, canMount: true, want: []string{mountPoint},
+		},
+		{
+			name:            "on its parent's device until listed",
+			unmountedDevice: 1, canMount: true, want: []string{mountPoint},
+		},
+		{
+			name:            "volume that cannot be mounted",
+			unmountedDevice: 1, canMount: false, want: nil,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			t.Parallel()
+
+			ctx := context.Background()
+			fs := &automountFs{
+				Fs: afero.NewMemMapFs(), mountPoint: mountPoint,
+				unmountedDevice: tt.unmountedDevice, canMount: tt.canMount,
+			}
+
+			require.NoError(t, fs.MkdirAll(mountPoint+"/photos", 0o755))
+
+			for _, path := range []string{
+				scannedPath + "/notes.txt", mountPoint + "/photos/a.jpg",
+			} {
+				require.NoError(t, afero.WriteFile(fs, path, []byte(path), 0o644))
+			}
+
+			db, err := database.NewTestDB()
+			require.NoError(t, err)
+			t.Cleanup(func() { _ = db.Close() })
+
+			repos := database.NewRepositories(db)
+
+			for _, snapshotID := range []string{"first", "later"} {
+				createTestSnapshotRecord(ctx, t, repos, snapshotID)
+
+				scanner := snapshot.NewScanner(snapshot.ScannerConfig{
+					FS:               fs,
+					ChunkSize:        int64(1024 * 16),
+					Repositories:     repos,
+					MaxBlobSize:      int64(1024 * 1024),
+					CompressionLevel: 3,
+					AgeRecipients:    []string{testAgePublicKey},
+					DeviceNumber:     deviceAtLookup,
+				})
+
+				_, err = scanner.Scan(ctx, scannedPath, snapshotID)
+				require.NoError(t, err)
+
+				got, err := repos.Snapshots.GetMountPoints(ctx, snapshotID)
+				require.NoError(t, err)
+				assert.Equal(t, tt.want, got, "mount points of the %s backup", snapshotID)
+			}
+		})
+	}
+}

internal/snapshot/scanner.go

@@ -73,6 +73,9 @@ type Scanner struct {
 	skipErrors bool
 	// ui is the user-facing output; never nil (defaults to a discarding writer).
 	ui *ui.Writer
+	// deviceNumber returns the number of the device that holds a file;
+	// never nil (defaults to deviceNumberFromStat).
+	deviceNumber func(info os.FileInfo) (uint64, bool)
 
 	// In-memory cache of known chunk hashes for fast existence checks
 	knownChunks   map[string]struct{}
@@ -136,6 +139,10 @@ type ScannerConfig struct {
 	// PreviousSnapshotID is the most recent completed snapshot of the
 	// same name, "" for none.
 	PreviousSnapshotID string
+	// DeviceNumber returns the number of the device that holds a file,
+	// and false when there is none to read. nil reads it from the file's
+	// stat; tests replace it, since the in-memory test filesystem has none.
+	DeviceNumber func(info os.FileInfo) (uint64, bool)
 }
 
 // ScanResult contains the results of a scan operation. Files and bytes
@@ -214,6 +221,11 @@ func NewScanner(cfg ScannerConfig) *Scanner {
 		uiw = ui.NewWithColor(io.Discard, false)
 	}
 
+	deviceNumber := cfg.DeviceNumber
+	if deviceNumber == nil {
+		deviceNumber = deviceNumberFromStat
+	}
+
 	return &Scanner{
 		fs:                 cfg.FS,
 		chunker:            chunker.NewChunker(cfg.ChunkSize),
@@ -229,6 +241,7 @@ func NewScanner(cfg ScannerConfig) *Scanner {
 		skipErrors:         cfg.SkipErrors,
 		previousSnapshotID: cfg.PreviousSnapshotID,
 		ui:                 uiw,
+		deviceNumber:       deviceNumber,
 		pendingChunkHashes: make(map[string]struct{}),
 	}
 }
@@ -297,7 +310,7 @@ func (s *Scanner) Scan(
 		}
 	}
 
-	err = s.recordDirectories(ctx, scanResult.Directories)
+	err = s.recordDirectories(ctx, scanResult)
 	if err != nil {
 		return nil, err
 	}
@@ -875,6 +888,11 @@ type ScanPhaseResult struct {
 	FilesToProcess   []*FileToProcess
 	UnchangedFileIDs []types.FileID   // IDs of unchanged files to associate with snapshot
 	Directories      []*FileToProcess // Every directory found, to record
+	// MountPoints are the directories, the scanned path included, that
+	// the walk found an entry in and, once it had listed them, on a
+	// different device from their parent directory (see
+	// recordIfMountPoint).
+	MountPoints []string
 }
 
 // scanPhase performs the initial directory scan to identify files to process
@@ -894,7 +912,7 @@ func (s *Scanner) scanPhase(
 	// estimate leaves them out.
 	estimatedTotal := countNonDirectories(knownFiles)
 
-	collector := &scanCollector{}
+	collector := &scanCollector{dirsWithEntries: make(map[string]struct{})}
 
 	// Set up periodic status output
 	startTime := time.Now()
@@ -939,6 +957,8 @@ func (s *Scanner) scanPhase(
 			result.EntriesFound++
 		}
 
+		s.recordIfMountPoint(filePath, path, collector)
+
 		// Handle symlinks and directories
 		handled, err := s.recordSpecialEntry(
 			filePath, info, existingFiles, knownFiles, collector, result)
@@ -973,6 +993,7 @@ func (s *Scanner) scanPhase(
 		FilesToProcess:   collector.filesToProcess,
 		UnchangedFileIDs: collector.unchangedFileIDs,
 		Directories:      collector.directories,
+		MountPoints:      collector.mountPoints,
 	}, nil
 }
 
@@ -1099,6 +1120,29 @@ type scanCollector struct {
 	unchangedFileIDs []types.FileID // Just IDs - no new records needed
 	directories      []*FileToProcess
 	filesScanned     int64
+	// dirsWithEntries holds each directory the walk has found an entry in.
+	dirsWithEntries map[string]struct{}
+	mountPoints     []string
+}
+
+// firstEntryIn notes that the walk found an entry in directory dir and
+// reports whether it is the first one.
+func (c *scanCollector) firstEntryIn(dir string) bool {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+
+	_, seen := c.dirsWithEntries[dir]
+	c.dirsWithEntries[dir] = struct{}{}
+
+	return !seen
+}
+
+// addMountPoint sets a mount point aside to be recorded.
+func (c *scanCollector) addMountPoint(dir string) {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+
+	c.mountPoints = append(c.mountPoints, dir)
 }
 
 // addDirectory sets a directory aside to be recorded. It is not counted
@@ -1290,20 +1334,100 @@ func (s *Scanner) buildDirectoryEntry(path string, info os.FileInfo) *database.F
 	}
 }
 
-// recordDirectories records every directory the walk found. A directory
-// has no data to back up, so it is not among the files counted and
-// backed up.
+// recordDirectories records every directory the walk found, and its
+// mount points (see recordMountPoints). A directory has no data to back
+// up, so it is not among the files counted and backed up.
 func (s *Scanner) recordDirectories(
-	ctx context.Context, directories []*FileToProcess,
+	ctx context.Context, scanned *ScanPhaseResult,
 ) error {
-	for _, dir := range directories {
+	for _, dir := range scanned.Directories {
 		err := s.recordNonRegularFile(ctx, dir)
 		if err != nil {
 			return fmt.Errorf("recording directory %s: %w", dir.Path, err)
 		}
 	}
 
-	return nil
+	return s.recordMountPoints(ctx, scanned.MountPoints)
+}
+
+// recordMountPoints records with the snapshot the mount points the walk
+// found (see ScanPhaseResult.MountPoints).
+func (s *Scanner) recordMountPoints(
+	ctx context.Context, mountPoints []string,
+) error {
+	return s.repos.WithTx(ctx, func(txCtx context.Context, tx *sql.Tx) error {
+		for _, dir := range mountPoints {
+			log.Info("Found a mount point", "path", dir)
+
+			err := s.repos.Snapshots.AddMountPoint(txCtx, tx, s.snapshotID, dir)
+			if err != nil {
+				return fmt.Errorf("recording mount point %s: %w", dir, err)
+			}
+		}
+
+		return nil
+	})
+}
+
+// recordIfMountPoint sets the directory that holds entry, a path the
+// walk found, aside to be recorded as a mount point if it is on a
+// different device from its parent directory. The walk calls it for each
+// entry it finds, and it decides on the first entry in a directory, so a
+// directory the walk found no entry in is never recorded. By then the
+// walk has listed the directory, which mounts a volume mounted there on
+// first access, so the directory and its parent are looked up again:
+// before the listing, the directory can be on the automounter's own
+// device (a systemd automount) or on its parent's (an autofs map that
+// shows its mount points before mounting them). The scanned path is in
+// no listing the walk made, so it is skipped as an entry. A directory
+// that cannot be looked up then, or whose parent cannot, is not
+// recorded, so this snapshot does not stop the prune of a later backup
+// that misses its volume.
+func (s *Scanner) recordIfMountPoint(
+	entry, scannedPath string, collector *scanCollector,
+) {
+	dir := filepath.Dir(entry)
+
+	if entry == scannedPath || !collector.firstEntryIn(dir) {
+		return
+	}
+
+	dirDevice, ok := s.deviceNumberAt(dir)
+	if !ok {
+		return
+	}
+
+	parentDevice, ok := s.deviceNumberAt(filepath.Dir(dir))
+	if ok && dirDevice != parentDevice {
+		collector.addMountPoint(dir)
+	}
+}
+
+// deviceNumberAt looks path up now and returns the number of the device
+// that holds it, and false when the lookup fails (logged) or gives no
+// device number.
+func (s *Scanner) deviceNumberAt(path string) (uint64, bool) {
+	info, err := s.fs.Stat(path)
+	if err != nil {
+		log.Warn("Could not look up a directory to check for a mount point",
+			"path", path, "error", err)
+
+		return 0, false
+	}
+
+	return s.deviceNumber(info)
+}
+
+// deviceNumberFromStat returns the number of the device that holds the
+// file info describes, and false when info carries no stat, as from the
+// in-memory test filesystem.
+func deviceNumberFromStat(info os.FileInfo) (uint64, bool) {
+	stat, ok := info.Sys().(*syscall.Stat_t)
+	if !ok {
+		return 0, false
+	}
+
+	return uint64(stat.Dev), true //nolint:unconvert // Dev is an int32 on darwin
 }
 
 // recordNonRegularFile writes a symlink or directory entry to the database

internal/snapshot/snapshot.go

@@ -518,8 +518,8 @@ type CleanupStats struct {
 //
 // The cleanup is performed in a specific order to maintain referential integrity:
 //  1. Delete other snapshots
-//  2. Delete orphaned snapshot associations (snapshot_files, snapshot_blobs)
-//     for deleted snapshots
+//  2. Delete orphaned snapshot associations (snapshot_files, snapshot_blobs,
+//     snapshot_mount_points) for deleted snapshots
 //  3. Delete orphaned files (not in the current snapshot)
 //  4. Delete orphaned chunk-to-file mappings (references to deleted files)
 //  5. Delete orphaned blobs (not in the current snapshot)
@@ -948,8 +948,8 @@ func (sm *SnapshotManager) deleteOtherSnapshots(
 	return nil
 }
 
-// deleteOrphanedSnapshotAssociations deletes snapshot_files and
-// snapshot_blobs for deleted snapshots
+// deleteOrphanedSnapshotAssociations deletes snapshot_files,
+// snapshot_blobs and snapshot_mount_points for deleted snapshots
 func (sm *SnapshotManager) deleteOrphanedSnapshotAssociations(
 	ctx context.Context, tx *sql.Tx, currentSnapshotID string,
 ) error {
@@ -983,6 +983,19 @@ func (sm *SnapshotManager) deleteOrphanedSnapshotAssociations(
 	log.Debug("[Temp DB Cleanup] Deleted snapshot_blobs associations",
 		"count", rowsAffected)
 
+	// Delete the mount points other snapshots recorded
+	query := "DELETE FROM snapshot_mount_points WHERE snapshot_id != ?"
+	database.LogSQL("Execute", query, currentSnapshotID)
+
+	result, err = tx.ExecContext(ctx, query, currentSnapshotID)
+	if err != nil {
+		return fmt.Errorf("deleting orphaned snapshot_mount_points: %w", err)
+	}
+
+	rowsAffected, _ = result.RowsAffected()
+	log.Debug("[Temp DB Cleanup] Deleted snapshot_mount_points rows",
+		"count", rowsAffected)
+
 	return nil
 }
 

internal/snapshot/snapshot_test.go

@@ -7,12 +7,14 @@ import (
 	"database/sql"
 	"io"
 	"path/filepath"
+	"slices"
 	"testing"
 
 	"github.com/spf13/afero"
 	"sneak.berlin/go/vaultik/internal/config"
 	"sneak.berlin/go/vaultik/internal/database"
 	"sneak.berlin/go/vaultik/internal/log"
+	"sneak.berlin/go/vaultik/internal/types"
 )
 
 const (
@@ -387,6 +389,77 @@ func TestCleanSnapshotDBNonExistentSnapshot(t *testing.T) {
 	}
 }
 
+// The exported metadata database keeps only the mount points that the
+// exported snapshot recorded. See
+// https://git.eeqj.de/sneak/vaultik/issues/395.
+func TestCleanSnapshotDBKeepsOnlyTheSnapshotsMountPoints(t *testing.T) {
+	log.Initialize(log.Config{})
+	t.Parallel()
+
+	ctx := context.Background()
+	dbPath := filepath.Join(t.TempDir(), "export.db")
+
+	db, err := database.New(ctx, dbPath)
+	if err != nil {
+		t.Fatalf("failed to create database: %v", err)
+	}
+
+	repos := database.NewRepositories(db)
+
+	for _, id := range []types.SnapshotID{"exported", "other"} {
+		err = repos.Snapshots.Create(ctx, nil, &database.Snapshot{ID: id})
+		if err != nil {
+			t.Fatalf("failed to create snapshot: %v", err)
+		}
+
+		err = repos.Snapshots.AddMountPoint(ctx, nil, id.String(), "/data/"+id.String())
+		if err != nil {
+			t.Fatalf("failed to add mount point: %v", err)
+		}
+	}
+
+	err = db.Close()
+	if err != nil {
+		t.Fatalf("failed to close database: %v", err)
+	}
+
+	sm := &SnapshotManager{
+		config: &config.Config{
+			CompressionLevel: 3,
+			AgeRecipients:    []string{testAgeRecipient},
+		},
+		fs: afero.NewOsFs(),
+	}
+
+	_, err = sm.cleanSnapshotDB(ctx, dbPath, "exported")
+	if err != nil {
+		t.Fatalf("failed to clean snapshot database: %v", err)
+	}
+
+	cleaned, err := database.New(ctx, dbPath)
+	if err != nil {
+		t.Fatalf("failed to open cleaned database: %v", err)
+	}
+
+	defer func() { _ = cleaned.Close() }()
+
+	cleanedRepos := database.NewRepositories(cleaned)
+
+	for id, want := range map[string][]string{
+		"exported": {"/data/exported"},
+		"other":    nil,
+	} {
+		got, err := cleanedRepos.Snapshots.GetMountPoints(ctx, id)
+		if err != nil {
+			t.Fatalf("failed to read mount points: %v", err)
+		}
+
+		if !slices.Equal(got, want) {
+			t.Errorf("mount points of %s: got %v, want %v", id, got, want)
+		}
+	}
+}
+
 // Two creates of one snapshot name back to back start within one second,
 // the resolution of the timestamp in a snapshot ID. See
 // https://git.eeqj.de/sneak/vaultik/issues/270.