Do not prune when a backup misses an earlier mount point (closes #395)

snapshot create --prune purged the older snapshots holding a volume's
files when the volume, mounted below a configured path, was not
mounted: its empty mount point read as deleted files.

The scan now records in the local index each directory at or below a
configured path that the walk found an entry in and that, once listed,
is on a different device from its parent; listing mounts an
automounted volume. Under --prune, when an earlier snapshot of that
name recorded such a directory and this run does not, the run skips
the prune, keeps every older snapshot and exits non-zero, naming the
directory. For a volume retired on purpose, snapshot purge removes the
older snapshots.

Model: opus-5-5

internal/vaultik/fault_injection_test.go

@@ -679,6 +679,7 @@ func faultScannerFactory(
 			Exclude:            params.Exclude,
 			SkipErrors:         params.SkipErrors,
 			PreviousSnapshotID: params.PreviousSnapshotID,
+			DeviceNumber:       params.DeviceNumber,
 		})
 	}
 }

internal/vaultik/snapshot.go

@@ -7,6 +7,7 @@ import (
 	"os"
 	"path/filepath"
 	"regexp"
+	"slices"
 	"sort"
 	"strconv"
 	"strings"
@@ -52,6 +53,17 @@ var ErrFilesUnreadable = errors.New("file(s) could not be read")
 // returns ErrFilesUnreadable instead.
 var ErrSourcePathEmpty = errors.New("source path(s) were empty")
 
+// ErrVolumeNotMounted is returned when --prune skipped the post-backup
+// prune because a directory that an earlier snapshot of the same name
+// found as a mount point was not found as one in this run: the volume
+// mounted there is not mounted, the run found no entry in it, or the
+// directory is no longer backed up. The snapshots are complete and
+// kept, so the caller can report it apart from a failed backup. A run
+// that also skipped unreadable files or found a configured path empty
+// returns ErrFilesUnreadable or ErrSourcePathEmpty instead.
+var ErrVolumeNotMounted = errors.New(
+	"mount point(s) an earlier snapshot recorded were not found")
+
 // listRecentLimit caps how many snapshot rows are fetched from the
 // local index for listing and purge operations.
 const listRecentLimit = 10000
@@ -142,7 +154,7 @@ func (v *Vaultik) CreateSnapshot(opts *SnapshotCreateOptions) error {
 		return err
 	}
 
-	filesUnreadable, emptyPaths, err := v.createNamedSnapshots(
+	filesUnreadable, emptyPaths, unmountedPaths, err := v.createNamedSnapshots(
 		opts, hostname, snapshotNames)
 	if err != nil {
 		return err
@@ -155,8 +167,10 @@ func (v *Vaultik) CreateSnapshot(opts *SnapshotCreateOptions) error {
 	}
 
 	switch {
-	case opts.Prune && (filesUnreadable > 0 || len(emptyPaths) > 0):
-		return v.skipPostBackupPrune(filesUnreadable, emptyPaths)
+	case opts.Prune && (filesUnreadable > 0 || len(emptyPaths) > 0 ||
+		len(unmountedPaths) > 0):
+		return v.skipPostBackupPrune(
+			filesUnreadable, emptyPaths, unmountedPaths)
 	case opts.Prune:
 		err = v.runPostBackupPrune(snapshotNames, opts.KeepNewerThan)
 		if err != nil {
@@ -169,20 +183,22 @@ func (v *Vaultik) CreateSnapshot(opts *SnapshotCreateOptions) error {
 
 // createNamedSnapshots creates each named snapshot in turn. It returns
 // the number of entries --skip-errors skipped because it could not read
-// or list them, and each configured path in which the scan found
-// nothing, named with its snapshot.
+// or list them, each configured path in which the scan found nothing,
+// and, under --prune, each directory where an earlier snapshot found a
+// volume mounted and this run did not, the paths named with their
+// snapshot.
 func (v *Vaultik) createNamedSnapshots(
 	opts *SnapshotCreateOptions, hostname string, snapshotNames []string,
-) (int, []string, error) {
+) (int, []string, []string, error) {
 	filesUnreadable := 0
 
-	var emptyPaths []string
+	var emptyPaths, unmountedPaths []string
 
 	for snapIdx, snapName := range snapshotNames {
 		stats, err := v.createNamedSnapshot(
 			opts, hostname, snapName, snapIdx+1, len(snapshotNames))
 		if err != nil {
-			return 0, nil, err
+			return 0, nil, nil, err
 		}
 
 		filesUnreadable += stats.totalFilesUnreadable
@@ -190,21 +206,27 @@ func (v *Vaultik) createNamedSnapshots(
 		for _, dir := range stats.emptyPaths {
 			emptyPaths = append(emptyPaths, dir+" of snapshot "+snapName)
 		}
+
+		for _, dir := range stats.unmountedPaths {
+			unmountedPaths = append(unmountedPaths,
+				dir+" of snapshot "+snapName)
+		}
 	}
 
-	return filesUnreadable, emptyPaths, nil
+	return filesUnreadable, emptyPaths, unmountedPaths, nil
 }
 
 // skipPostBackupPrune says why the post-backup prune does not run, naming
 // every cause, and returns the error the run ends with. An older snapshot
 // may be the only one that still holds a file this run could not read,
-// or what an empty path held. The previous snapshot is no guide: after
-// one run with an empty path it is empty too.
+// or what an empty path or a volume that is not mounted held. The
+// previous snapshot is no guide: after one such run it lacks them too.
 func (v *Vaultik) skipPostBackupPrune(
-	filesUnreadable int, emptyPaths []string,
+	filesUnreadable int, emptyPaths, unmountedPaths []string,
 ) error {
 	log.Warn("Skipping post-backup prune",
-		"files_unreadable", filesUnreadable, "empty_paths", emptyPaths)
+		"files_unreadable", filesUnreadable, "empty_paths", emptyPaths,
+		"unmounted_paths", unmountedPaths)
 
 	if len(emptyPaths) > 0 {
 		v.UI.Warningf("Skipped the post-backup prune because nothing was "+
@@ -212,6 +234,17 @@ func (v *Vaultik) skipPostBackupPrune(
 			strings.Join(emptyPaths, ", "))
 	}
 
+	if len(unmountedPaths) > 0 {
+		v.UI.Warningf("Skipped the post-backup prune because this backup "+
+			"did not find a mount point at %s, where an earlier snapshot "+
+			"found one: the volume is not mounted, this backup found no "+
+			"entry in it, or the directory is no longer backed up. Older "+
+			"snapshots are kept. Mount the volume again, or, if that is on "+
+			"purpose, remove the older snapshots by hand with "+
+			"'vaultik snapshot purge'.",
+			strings.Join(unmountedPaths, ", "))
+	}
+
 	if filesUnreadable > 0 {
 		v.UI.Warningf("Skipped the post-backup prune because %d file(s) "+
 			"could not be read; older snapshots are kept.", filesUnreadable)
@@ -219,7 +252,11 @@ func (v *Vaultik) skipPostBackupPrune(
 		return fmt.Errorf("%d %w", filesUnreadable, ErrFilesUnreadable)
 	}
 
-	return fmt.Errorf("%d %w", len(emptyPaths), ErrSourcePathEmpty)
+	if len(emptyPaths) > 0 {
+		return fmt.Errorf("%d %w", len(emptyPaths), ErrSourcePathEmpty)
+	}
+
+	return fmt.Errorf("%d %w", len(unmountedPaths), ErrVolumeNotMounted)
 }
 
 // finishCreateSnapshot ends a run whose snapshots are all complete and
@@ -329,6 +366,10 @@ type snapshotStats struct {
 	// The configured paths in which the scan found nothing.
 	emptyPaths []string
 
+	// Under --prune, the directories where an earlier snapshot of the
+	// same name found a volume mounted and this one did not.
+	unmountedPaths []string
+
 	// The sizes of all blobs the snapshot references, set by
 	// finalizeSnapshotMetadata once snapshot_blobs is populated.
 	blobSize             int64
@@ -394,9 +435,57 @@ func (v *Vaultik) createNamedSnapshot(
 
 	v.printSnapshotSummary(snapshotID, snapshotStartTime, stats)
 
+	if opts.Prune {
+		stats.unmountedPaths, err = v.findUnmountedPaths(snapName, snapshotID)
+		if err != nil {
+			return nil, err
+		}
+	}
+
 	return stats, nil
 }
 
+// findUnmountedPaths returns each directory that an earlier snapshot
+// named snapName in the local index recorded as a mount point and
+// snapshot snapshotID did not, as when the volume mounted there is not
+// mounted.
+func (v *Vaultik) findUnmountedPaths(
+	snapName, snapshotID string,
+) ([]string, error) {
+	found, err := v.Repositories.Snapshots.GetMountPoints(v.ctx, snapshotID)
+	if err != nil {
+		return nil, err
+	}
+
+	snapshots, err := v.Repositories.Snapshots.ListRecent(v.ctx, listRecentLimit)
+	if err != nil {
+		return nil, fmt.Errorf("listing snapshots: %w", err)
+	}
+
+	var unmounted []string
+
+	for _, s := range snapshots {
+		if s.ID.String() == snapshotID ||
+			parseSnapshotName(s.ID.String(), s.Hostname.String()) != snapName {
+			continue
+		}
+
+		recorded, err := v.Repositories.Snapshots.GetMountPoints(
+			v.ctx, s.ID.String())
+		if err != nil {
+			return nil, err
+		}
+
+		for _, dir := range recorded {
+			if !slices.Contains(found, dir) && !slices.Contains(unmounted, dir) {
+				unmounted = append(unmounted, dir)
+			}
+		}
+	}
+
+	return unmounted, nil
+}
+
 // previousSnapshotID returns the ID of the most recent completed snapshot
 // named snapName in the local index, or "" when there is none.
 func (v *Vaultik) previousSnapshotID(snapName string) (string, error) {

internal/vaultik/snapshot_prune_test.go

@@ -4,12 +4,15 @@ import (
 	"context"
 	"os"
 	"path/filepath"
+	"syscall"
 	"testing"
 
+	"github.com/spf13/afero"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 	"sneak.berlin/go/vaultik/internal/config"
 	"sneak.berlin/go/vaultik/internal/log"
+	"sneak.berlin/go/vaultik/internal/snapshot"
 	"sneak.berlin/go/vaultik/internal/vaultik"
 )
 
@@ -259,3 +262,152 @@ func TestBackupWithNoEmptySourcePathPrunes(t *testing.T) {
 		})
 	}
 }
+
+// automountFs is the real filesystem with a volume at volumeDir that is
+// mounted on first access, as by an autofs map that shows its mount
+// points before mounting them: listing volumeDir mounts the volume or,
+// while cannotMount, fails with "no such file or directory".
+type automountFs struct {
+	afero.OsFs
+
+	volumeDir   string
+	cannotMount bool
+	mounted     bool
+}
+
+//nolint:ireturn // afero.Fs.Open is defined to return the interface.
+func (f *automountFs) Open(name string) (afero.File, error) {
+	if name == f.volumeDir && f.cannotMount {
+		return nil, &os.PathError{Op: "open", Path: name, Err: syscall.ENOENT}
+	}
+
+	if name == f.volumeDir {
+		f.mounted = true
+	}
+
+	return f.OsFs.Open(name)
+}
+
+// A backup under --prune in which a volume that an earlier snapshot found
+// mounted below the configured path is missing does not prune: run after
+// run, the snapshot that holds the volume's files is kept with everything
+// it stored, and the run fails. Once the volume is back, the prune runs.
+// The volume is mounted on first access and every backup starts with it
+// not mounted, so its directory is on its parent's device until the walk
+// lists it. While the volume is missing, its directory is left empty, or
+// listing it fails with "no such file or directory", as when an autofs
+// volume cannot be mounted, which the walk counts as a directory removed
+// during the backup. See https://git.eeqj.de/sneak/vaultik/issues/395.
+func TestBackupWithVolumeNotMountedDoesNotPrune(t *testing.T) {
+	log.Initialize(log.Config{})
+	t.Parallel()
+
+	tests := []struct {
+		name         string
+		listingFails bool
+	}{
+		{name: "directory left empty", listingFails: false},
+		{name: "listing fails", listingFails: true},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			t.Parallel()
+
+			backUpWithVolumeNotMounted(t, tt.listingFails)
+		})
+	}
+}
+
+// backUpWithVolumeNotMounted runs the backups of
+// TestBackupWithVolumeNotMountedDoesNotPrune. While the volume is
+// missing, listing its directory fails if listingFails; otherwise the
+// directory is empty.
+func backUpWithVolumeNotMounted(t *testing.T, listingFails bool) {
+	t.Helper()
+
+	// volume is the directory, below the configured path, where the
+	// volume is mounted.
+	const (
+		snapshotName = "data"
+		volume       = "nas"
+	)
+
+	ctx := context.Background()
+	env := newDeletedCountEnv(t, map[string][]byte{
+		volume + "/a.txt": []byte("file a"),
+		"notes.txt":       []byte("notes"),
+	})
+	env.setSnapshot(snapshotName, "")
+	prune := &vaultik.SnapshotCreateOptions{
+		Prune: true, Snapshots: []string{snapshotName},
+	}
+
+	// The volume's directory is on the volume's device, 2, while the
+	// volume is mounted. Everything else is on device 1.
+	volumeDir := filepath.Join(env.src, volume)
+	fs := &automountFs{volumeDir: volumeDir}
+	newScanner := env.v.ScannerFactory
+	env.v.ScannerFactory = func(params snapshot.ScannerParams) *snapshot.Scanner {
+		fs.mounted = false
+		params.Fs = fs
+		params.DeviceNumber = func(info os.FileInfo) (uint64, bool) {
+			if info.Name() == volume && fs.mounted {
+				return 2, true
+			}
+
+			return 1, true
+		}
+
+		return newScanner(params)
+	}
+
+	env.backUp(t, snapshotName)
+
+	firstID := localSnapshotID(ctx, t, env.v.Repositories, snapshotName)
+	stored, err := env.v.Storage.List(ctx, "")
+	require.NoError(t, err)
+
+	// The volume goes missing.
+	fs.cannotMount = listingFails
+
+	if !listingFails {
+		require.NoError(t, os.RemoveAll(volumeDir))
+		require.NoError(t, os.Mkdir(volumeDir, 0o750))
+	}
+
+	for range 2 {
+		env.out.Reset()
+
+		err := env.v.CreateSnapshot(prune)
+		require.ErrorIs(t, err, vaultik.ErrVolumeNotMounted)
+		assert.Contains(t, env.out.String(),
+			"did not find a mount point at "+volumeDir+" of snapshot "+snapshotName)
+		assert.Contains(t, env.out.String(), "'vaultik snapshot purge'")
+		assert.NotContains(t, env.out.String(), "Running post-backup prune.")
+	}
+
+	snapshots, err := env.v.Repositories.Snapshots.ListRecent(
+		ctx, listRecentTestLimit)
+	require.NoError(t, err)
+	require.Len(t, snapshots, 3)
+	assert.Equal(t, firstID, snapshots[2].ID.String(),
+		"the snapshot that holds the volume's files is kept")
+
+	for _, key := range stored {
+		_, err := env.v.Storage.Stat(ctx, key)
+		require.NoError(t, err, "%s is kept", key)
+	}
+
+	// The volume is back.
+	fs.cannotMount = false
+
+	require.NoError(t, os.WriteFile(
+		filepath.Join(volumeDir, "a.txt"), []byte("file a"), 0o600))
+	require.NoError(t, env.v.CreateSnapshot(prune))
+
+	snapshots, err = env.v.Repositories.Snapshots.ListRecent(
+		ctx, listRecentTestLimit)
+	require.NoError(t, err)
+	require.Len(t, snapshots, 1, "the older snapshots are pruned")
+}