Count a file as deleted only if the previous snapshot of its name had it (closes #304)

A backup counted as deleted every file in the local index under the
source path that the walk did not find. That took in files the
snapshot excludes but another snapshot backs up, and files removed long
ago, which every later backup, and the first backup of another name
over the same path, reported as deleted again.

The backup now counts only the files that the most recent completed
snapshot of the same name lists, the walk did not find, and are gone
from disk. A file that is excluded, or inside an excluded directory, is
not counted, nor is one the walk could not reach under --skip-errors.

The snapshots row stores no deleted figure; only the backup's output
changes.

Model: opus-5-5

internal/snapshot/exclude_test.go

@@ -578,3 +578,54 @@ func TestExcludePatterns_DoubleStarFileAtEveryDepth(t *testing.T) {
 	require.Equal(t, 1, result.FilesScanned,
 		"**/ file pattern should match at the top level and nested")
 }
+
+func TestExcludePatterns_AnchoredDoubleStarAtEveryDepth(t *testing.T) {
+	t.Parallel()
+
+	// As in gitignore, /**/ also matches directly under the source dir
+	fs := setupAnchoredTestFS(t)
+
+	scanner, repos, cleanup := createTestScanner(t, fs, []string{"/**/projectname"})
+	defer cleanup()
+
+	ctx := context.Background()
+	createSnapshotRecord(ctx, t, repos, "test-snapshot")
+
+	result, err := scanner.Scan(ctx, "/backup", "test-snapshot")
+	require.NoError(t, err)
+
+	// /**/projectname should exclude BOTH:
+	// - /backup/projectname/file.txt
+	// - /backup/otherproject/projectname/file.txt
+	// Total files: 4, excluded: 2
+	require.Equal(t, 2, result.FilesScanned,
+		"/**/ pattern should match at the top level and nested")
+}
+
+func TestExcludePatterns_DoubleStarInMiddleMatchesZeroDirectories(t *testing.T) {
+	t.Parallel()
+
+	// As in gitignore, a/**/b matches a/b as well as a/c/b. The last part
+	// is a glob: with a plain name the glob library already matched a/b.
+	fs := &afero.MemMapFs{}
+	files := []string{"/backup/a/x.log", "/backup/a/c/y.log", "/backup/a/c/kept.txt"}
+
+	for _, path := range files {
+		require.NoError(t, fs.MkdirAll(filepath.Dir(path), 0755))
+		require.NoError(t, afero.WriteFile(fs, path, []byte(path), 0644))
+	}
+
+	scanner, repos, cleanup := createTestScanner(t, fs, []string{"a/**/*.log"})
+	defer cleanup()
+
+	ctx := context.Background()
+	createSnapshotRecord(ctx, t, repos, "test-snapshot")
+
+	result, err := scanner.Scan(ctx, "/backup", "test-snapshot")
+	require.NoError(t, err)
+
+	// a/**/*.log should exclude /backup/a/x.log and /backup/a/c/y.log
+	// Total files: 3, excluded: 2
+	require.Equal(t, 1, result.FilesScanned,
+		"a/**/*.log should match a/x.log as well as a/c/y.log")
+}

internal/snapshot/scanner.go

@@ -2088,33 +2088,60 @@ func compileExcludePatterns(patterns []string) []compiledPattern {
 		pattern = strings.TrimSuffix(pattern, "/")
 
 		// A leading "**/" demands a "/" before the name and so misses an
-		// entry directly under the source path. An unanchored pattern
-		// already matches at every depth, so drop the prefix: as in
-		// gitignore, "**/x" then also matches a top-level "x".
-		if !anchored {
-			pattern = strings.TrimPrefix(pattern, "**/")
+		// entry directly under the source path. It means "at every depth",
+		// which is what an unanchored pattern already does, so drop the
+		// prefix and match unanchored: as in gitignore, "**/x" and
+		// "/**/x" then also match a top-level "x".
+		if rest, found := strings.CutPrefix(pattern, "**/"); found {
+			pattern = rest
+			anchored = false
 		}
 
-		// Compile the glob pattern
-		// For patterns without path separators, we need to match them as components
-		// e.g., ".git" should match ".git" anywhere in the path
-		g, err := glob.Compile(pattern, '/')
-		if err != nil {
-			log.Warn("Invalid exclude pattern, skipping", "pattern", p, "error", err)
+		// Likewise "/**/" inside a pattern demands two slashes, so
+		// "a/**/*.log" misses "a/x.log". As in gitignore it stands for
+		// zero or more directories, so also compile the pattern with it
+		// made a single "/". The glob library's {a,b} is no help: with a
+		// "**" inside it in mid-pattern, it misses paths it should match.
+		for _, variant := range doubleStarVariants(pattern) {
+			// For patterns without path separators, we need to match them as components
+			// e.g., ".git" should match ".git" anywhere in the path
+			g, err := glob.Compile(variant, '/')
+			if err != nil {
+				log.Warn("Invalid exclude pattern, skipping", "pattern", p, "error", err)
 
-			continue
+				break
+			}
+
+			compiled = append(compiled, compiledPattern{
+				pattern:  g,
+				anchored: anchored,
+				original: p,
+			})
 		}
-
-		compiled = append(compiled, compiledPattern{
-			pattern:  g,
-			anchored: anchored,
-			original: p,
-		})
 	}
 
 	return compiled
 }
 
+// doubleStarVariants returns pattern once for every way of keeping each
+// "/**/" in it or making it a single "/".
+func doubleStarVariants(pattern string) []string {
+	const doubleStar = "/**/"
+
+	before, after, found := strings.Cut(pattern, doubleStar)
+	if !found {
+		return []string{pattern}
+	}
+
+	var variants []string
+
+	for _, rest := range doubleStarVariants(after) {
+		variants = append(variants, before+doubleStar+rest, before+"/"+rest)
+	}
+
+	return variants
+}
+
 // shouldExclude checks if a path should be excluded based on exclude patterns
 // filePath is the full path to the file
 // rootPath is the root of the backup source directory

internal/storage/rclone.go

@@ -7,10 +7,12 @@ import (
 	"errors"
 	"fmt"
 	"io"
+	"os"
 	"strings"
 	"time"
 
 	"github.com/rclone/rclone/fs"
+	"github.com/rclone/rclone/fs/config"
 	"github.com/rclone/rclone/fs/config/configfile"
 	"github.com/rclone/rclone/fs/operations"
 
@@ -33,6 +35,15 @@ type RcloneStorer struct {
 // The remote parameter is the rclone remote name (as configured via `rclone config`).
 // The path parameter is the path within the remote.
 func NewRcloneStorer(ctx context.Context, remote, path string) (*RcloneStorer, error) {
+	// rclone reads RCLONE_CONFIG only as the value of its --config flag,
+	// which vaultik does not have, so it is applied here as that flag would.
+	if configPath, ok := os.LookupEnv("RCLONE_CONFIG"); ok {
+		err := config.SetConfigPath(configPath)
+		if err != nil {
+			return nil, fmt.Errorf("using RCLONE_CONFIG %q: %w", configPath, err)
+		}
+	}
+
 	// Install the default config file handler
 	configfile.Install()
 

internal/storage/rclone_test.go

@@ -11,6 +11,7 @@ import (
 	"testing"
 
 	"github.com/rclone/rclone/fs"
+	"github.com/rclone/rclone/fs/config"
 	"github.com/rclone/rclone/fs/config/configmap"
 	"sneak.berlin/go/vaultik/internal/storage"
 )
@@ -390,3 +391,41 @@ func TestNewRcloneStorerUnknownRemote(t *testing.T) {
 		t.Errorf("NewRcloneStorer error = %v, want ErrRemoteNotFound", err)
 	}
 }
+
+// TestNewRcloneStorerReadsRcloneConfigEnv checks that a remote defined only
+// in the file RCLONE_CONFIG names is found and written to, as rclone itself
+// would.
+func TestNewRcloneStorerReadsRcloneConfigEnv(t *testing.T) {
+	// rclone keeps the config file path in a process-global variable, which
+	// NewRcloneStorer leaves pointing at this test's file.
+	previous := config.GetConfigPath()
+
+	t.Cleanup(func() { _ = config.SetConfigPath(previous) })
+
+	confFile := filepath.Join(t.TempDir(), "rclone.conf")
+
+	err := os.WriteFile(confFile, []byte("[loc]\ntype = local\n"), 0o600)
+	if err != nil {
+		t.Fatalf("writing rclone config: %v", err)
+	}
+
+	t.Setenv("RCLONE_CONFIG", confFile)
+
+	dir := t.TempDir()
+	ctx := context.Background()
+
+	s, err := storage.NewRcloneStorer(ctx, "loc", dir)
+	if err != nil {
+		t.Fatalf("NewRcloneStorer: %v", err)
+	}
+
+	err = s.Put(ctx, testBlobKey, strings.NewReader("blob-bytes"))
+	if err != nil {
+		t.Fatalf("Put: %v", err)
+	}
+
+	_, err = os.Stat(filepath.Join(dir, testBlobKey))
+	if err != nil {
+		t.Errorf("object is not in the remote's directory: %v", err)
+	}
+}