Count a file as deleted only if the previous snapshot of its name had it (closes #304)

A backup counted as deleted every file in the local index under the
source path that the walk did not find. That took in files the
snapshot excludes but another snapshot backs up, and files removed long
ago, which every later backup, and the first backup of another name
over the same path, reported as deleted again.

The backup now looks up the most recent completed snapshot of the same
name and counts only the files it lists that the walk did not find. A
file that is excluded, or inside an excluded directory, is not counted.

The snapshots row stores no deleted figure; only the backup's output
changes.

Model: opus-5-5

internal/vaultik/fault_injection_test.go

@@ -661,17 +661,18 @@ func faultScannerFactory(
 ) snapshot.ScannerFactory {
 	return func(params snapshot.ScannerParams) *snapshot.Scanner {
 		return snapshot.NewScanner(snapshot.ScannerConfig{
-			FS:               params.Fs,
-			Storage:          storer,
-			ChunkSize:        faultChunkSize,
-			MaxBlobSize:      faultMaxBlobSize,
-			CompressionLevel: cfg.CompressionLevel,
-			AgeRecipients:    cfg.AgeRecipients,
-			Repositories:     repos,
-			EnableProgress:   params.EnableProgress,
-			UI:               params.UI,
-			Exclude:          params.Exclude,
-			SkipErrors:       params.SkipErrors,
+			FS:                 params.Fs,
+			Storage:            storer,
+			ChunkSize:          faultChunkSize,
+			MaxBlobSize:        faultMaxBlobSize,
+			CompressionLevel:   cfg.CompressionLevel,
+			AgeRecipients:      cfg.AgeRecipients,
+			Repositories:       repos,
+			EnableProgress:     params.EnableProgress,
+			UI:                 params.UI,
+			Exclude:            params.Exclude,
+			SkipErrors:         params.SkipErrors,
+			PreviousSnapshotID: params.PreviousSnapshotID,
 		})
 	}
 }

internal/vaultik/snapshot.go

@@ -242,12 +242,18 @@ func (v *Vaultik) createNamedSnapshot(
 		return err
 	}
 
+	previousSnapshotID, err := v.previousSnapshotID(snapName)
+	if err != nil {
+		return err
+	}
+
 	scanner := v.ScannerFactory(snapshot.ScannerParams{
-		EnableProgress: !opts.Cron,
-		UI:             v.UI,
-		Fs:             v.Fs,
-		Exclude:        v.Config.GetExcludes(snapName),
-		SkipErrors:     opts.SkipErrors,
+		EnableProgress:     !opts.Cron,
+		UI:                 v.UI,
+		Fs:                 v.Fs,
+		Exclude:            v.Config.GetExcludes(snapName),
+		SkipErrors:         opts.SkipErrors,
+		PreviousSnapshotID: previousSnapshotID,
 	})
 
 	snapshotID, err := v.SnapshotManager.CreateSnapshotWithName(
@@ -282,6 +288,24 @@ func (v *Vaultik) createNamedSnapshot(
 	return nil
 }
 
+// previousSnapshotID returns the ID of the most recent completed snapshot
+// named snapName in the local index, or "" when there is none.
+func (v *Vaultik) previousSnapshotID(snapName string) (string, error) {
+	snapshots, err := v.Repositories.Snapshots.ListRecent(v.ctx, listRecentLimit)
+	if err != nil {
+		return "", fmt.Errorf("listing snapshots: %w", err)
+	}
+
+	for _, s := range snapshots {
+		if s.CompletedAt != nil &&
+			parseSnapshotName(s.ID.String(), s.Hostname.String()) == snapName {
+			return s.ID.String(), nil
+		}
+	}
+
+	return "", nil
+}
+
 // resolveSnapshotPaths resolves source directories to absolute paths
 // with symlink resolution.
 func (v *Vaultik) resolveSnapshotPaths(snapName string) ([]string, error) {

internal/vaultik/snapshot_deleted_count_test.go

@@ -0,0 +1,196 @@
+package vaultik_test
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+	"os"
+	"path/filepath"
+	"testing"
+	"time"
+
+	"github.com/spf13/afero"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"sneak.berlin/go/vaultik/internal/config"
+	"sneak.berlin/go/vaultik/internal/database"
+	"sneak.berlin/go/vaultik/internal/log"
+	"sneak.berlin/go/vaultik/internal/storage"
+	"sneak.berlin/go/vaultik/internal/ui"
+	"sneak.berlin/go/vaultik/internal/vaultik"
+)
+
+// These tests cover https://git.eeqj.de/sneak/vaultik/issues/304: a
+// backup counts a file as deleted only when the previous snapshot of
+// the same name had it and it is gone, and never counts an excluded
+// file.
+
+// deletedCountEnv is a backup setup over the source directory src whose
+// user-facing output is kept in out. Each test configures its snapshots
+// with setSnapshot.
+type deletedCountEnv struct {
+	v   *vaultik.Vaultik
+	out *bytes.Buffer
+	src string
+}
+
+// newDeletedCountEnv writes files, keyed by path relative to src.
+func newDeletedCountEnv(t *testing.T, files map[string][]byte) *deletedCountEnv {
+	t.Helper()
+
+	ctx := context.Background()
+	fs := afero.NewOsFs()
+	tempDir := t.TempDir()
+	srcDir := filepath.Join(tempDir, "src")
+
+	for rel, content := range files {
+		path := filepath.Join(srcDir, rel)
+		require.NoError(t, fs.MkdirAll(filepath.Dir(path), 0o755))
+		require.NoError(t, afero.WriteFile(fs, path, content, 0o644))
+	}
+
+	cfg := faultTestConfig()
+	cfg.IndexPath = filepath.Join(tempDir, "index.sqlite")
+	cfg.Snapshots = map[string]config.SnapshotConfig{}
+
+	store, err := storage.NewFileStorer(filepath.Join(tempDir, "remote"))
+	require.NoError(t, err)
+
+	db, err := database.New(ctx, cfg.IndexPath)
+	require.NoError(t, err)
+	t.Cleanup(func() { _ = db.Close() })
+
+	repos := database.NewRepositories(db)
+	out := &bytes.Buffer{}
+	v := newBackupVaultik(ctx, cfg, store, repos, db, fs)
+	v.UI = ui.NewWithColor(out, false)
+
+	return &deletedCountEnv{v: v, out: out, src: srcDir}
+}
+
+// setSnapshot configures the snapshot name to back up the path rel
+// beneath src, with the given excludes.
+func (e *deletedCountEnv) setSnapshot(name, rel string, exclude ...string) {
+	e.v.Config.Snapshots[name] = config.SnapshotConfig{
+		Paths:   []string{filepath.Join(e.src, rel)},
+		Exclude: exclude,
+	}
+}
+
+// backUp runs a backup of the named snapshot and returns its output.
+func (e *deletedCountEnv) backUp(t *testing.T, name string) string {
+	t.Helper()
+
+	e.out.Reset()
+	require.NoError(t, e.v.CreateSnapshot(&vaultik.SnapshotCreateOptions{
+		Snapshots: []string{name},
+	}))
+
+	return e.out.String()
+}
+
+// A file this snapshot excludes is not counted as deleted: neither one
+// that another snapshot backs up, nor one that the previous snapshot of
+// this name had before the exclude was added.
+func TestBackupDoesNotCountExcludedFileAsDeleted(t *testing.T) {
+	log.Initialize(log.Config{})
+	t.Parallel()
+
+	env := newDeletedCountEnv(t, map[string][]byte{
+		"top.txt":       []byte("top"),
+		"cache/tmp.bin": []byte("cache"),
+		"photos/a.jpg":  []byte("photo a"),
+		"photos/b.jpg":  []byte("photo b"),
+		"photos/c.jpg":  []byte("photo c"),
+	})
+	env.setSnapshot("photos", "photos")
+	env.setSnapshot("home", "", "/photos")
+
+	env.backUp(t, "photos")
+	assert.Contains(t, env.backUp(t, "home"), filesLine(2, 2, 0))
+
+	env.setSnapshot("home", "", "/photos", "/cache")
+	assert.Contains(t, env.backUp(t, "home"), filesLine(1, 0, 1))
+}
+
+// A file inside a directory that an exclude added after a backup
+// matches is not counted as deleted, though the pattern matches only the
+// directory.
+func TestBackupDoesNotCountFileInExcludedDirectoryAsDeleted(t *testing.T) {
+	log.Initialize(log.Config{})
+	t.Parallel()
+
+	env := newDeletedCountEnv(t, map[string][]byte{
+		"top.txt":                         []byte("top"),
+		"user/.local/share/Trash/old.txt": []byte("old"),
+	})
+	env.setSnapshot("home", "")
+
+	assert.Contains(t, env.backUp(t, "home"), filesLine(2, 2, 0))
+
+	env.setSnapshot("home", "", "*/.local/share/Trash")
+	assert.Contains(t, env.backUp(t, "home"), filesLine(1, 0, 1))
+}
+
+// A backup counts only files that the previous snapshot of its own name
+// had, not one that a later snapshot of another name over the same path
+// had and that is since removed.
+func TestBackupIgnoresSnapshotOfAnotherName(t *testing.T) {
+	log.Initialize(log.Config{})
+	t.Parallel()
+
+	env := newDeletedCountEnv(t, map[string][]byte{
+		"kept.txt": []byte("kept"),
+	})
+	env.setSnapshot("home", "")
+	env.setSnapshot("other", "")
+
+	env.backUp(t, "home")
+
+	// Snapshot start times are kept in whole seconds: wait so that other's
+	// snapshot is strictly the most recent.
+	time.Sleep(time.Second)
+
+	extra := filepath.Join(env.src, "extra.txt")
+	require.NoError(t, os.WriteFile(extra, []byte("extra"), 0o600))
+	env.backUp(t, "other")
+	require.NoError(t, os.Remove(extra))
+
+	assert.Contains(t, env.backUp(t, "home"), filesLine(1, 0, 1))
+}
+
+// A file removed from disk is counted as deleted by the next backup of
+// the snapshot that had it, and not by any later backup of that
+// snapshot, nor by the first backup of another name over the same path.
+func TestBackupCountsDeletedFileOnce(t *testing.T) {
+	log.Initialize(log.Config{})
+	t.Parallel()
+
+	aContent := bytes.Repeat([]byte("a"), 1000)
+	bContent := bytes.Repeat([]byte("b"), 2000)
+	keptContent := bytes.Repeat([]byte("k"), 500)
+
+	env := newDeletedCountEnv(t, map[string][]byte{
+		"a.txt":    aContent,
+		"b.txt":    bContent,
+		"kept.txt": keptContent,
+	})
+	env.setSnapshot("home", "")
+	env.setSnapshot("other", "")
+
+	assert.Contains(t, env.backUp(t, "home"), filesLine(3, 3, 0))
+
+	require.NoError(t, os.Remove(filepath.Join(env.src, "a.txt")))
+	require.NoError(t, os.Remove(filepath.Join(env.src, "b.txt")))
+
+	summary := env.backUp(t, "home")
+	assert.Contains(t, summary,
+		"Files: 1 examined, 0 backed up, 1 unchanged, 2 deleted.")
+	assert.Contains(t, summary,
+		fmt.Sprintf("Data: %s total (%s backed up), %s deleted.",
+			env.v.UI.Size(int64(len(keptContent))), env.v.UI.Size(0),
+			env.v.UI.Size(int64(len(aContent)+len(bContent)))))
+
+	assert.Contains(t, env.backUp(t, "home"), filesLine(1, 0, 1))
+	assert.Contains(t, env.backUp(t, "other"), filesLine(1, 0, 1))
+}