Count a file a backup could not store as failed (closes #280)

A file that phase 1 of a backup counted and phase 2 could not open,
because it was unreadable under --skip-errors or removed in between,
was added to the unchanged count while its size stayed in
BytesScanned. The summary showed it as unchanged with its bytes backed
up, and the snapshots row's file_count and total_size included it.

The scanner now counts such a file in FilesFailed and takes its size
out of BytesScanned. The summary's files line adds "N failed", and
file_count leaves the file out.

A directory phase 2 cannot record is not counted as failed, since
phase 1 counts no directories; that case has no test.

Model: opus-5-5

internal/vaultik/snapshot.go

@@ -185,6 +185,7 @@ type snapshotStats struct {
 	totalBlobs         int
 	totalBytesSkipped  int64
 	totalFilesSkipped  int
+	totalFilesFailed   int
 	totalFilesDeleted  int
 	totalBytesDeleted  int64
 	totalBytesUploaded int64
@@ -315,6 +316,7 @@ func (v *Vaultik) scanAllDirectories(
 		stats.totalChunks += result.ChunksCreated
 		stats.totalBlobs += result.BlobsCreated
 		stats.totalFilesSkipped += result.FilesSkipped
+		stats.totalFilesFailed += result.FilesFailed
 		stats.totalBytesSkipped += result.BytesSkipped
 		stats.totalFilesDeleted += result.FilesDeleted
 		stats.totalBytesDeleted += result.BytesDeleted
@@ -326,6 +328,7 @@ func (v *Vaultik) scanAllDirectories(
 			"path", dir,
 			"files", result.FilesScanned,
 			"files_skipped", result.FilesSkipped,
+			"files_failed", result.FilesFailed,
 			"bytes", result.BytesScanned,
 			"bytes_skipped", result.BytesSkipped,
 			"chunks", result.ChunksCreated,
@@ -359,9 +362,11 @@ func (v *Vaultik) finalizeSnapshotMetadata(
 		return fmt.Errorf("getting snapshot blob sizes: %w", err)
 	}
 
+	// file_count and total_size leave out the files that could not be
+	// stored; stats.totalBytes already does.
 	extStats := snapshot.ExtendedBackupStats{
 		BackupStats: snapshot.BackupStats{
-			FilesScanned:  stats.totalFiles,
+			FilesScanned:  stats.totalFiles - stats.totalFilesFailed,
 			TotalSize:     stats.totalBytes + stats.totalBytesSkipped,
 			ChunksCreated: stats.totalChunks,
 			BlobsCreated:  stats.totalBlobs,
@@ -409,7 +414,8 @@ func (v *Vaultik) printSnapshotSummary(
 	snapshotID string, startTime time.Time, stats *snapshotStats,
 ) {
 	snapshotDuration := time.Since(startTime)
-	totalFilesChanged := stats.totalFiles - stats.totalFilesSkipped
+	totalFilesChanged := stats.totalFiles - stats.totalFilesSkipped -
+		stats.totalFilesFailed
 	totalBytesAll := stats.totalBytes + stats.totalBytesSkipped
 
 	var compressionRatio float64
@@ -426,6 +432,10 @@ func (v *Vaultik) printSnapshotSummary(
 		v.UI.Count(stats.totalFiles),
 		v.UI.Count(totalFilesChanged),
 		v.UI.Count(stats.totalFilesSkipped))
+	if stats.totalFilesFailed > 0 {
+		filesMsg += fmt.Sprintf(", %s failed", v.UI.Count(stats.totalFilesFailed))
+	}
+
 	if stats.totalFilesDeleted > 0 {
 		filesMsg += fmt.Sprintf(", %s deleted", v.UI.Count(stats.totalFilesDeleted))
 	}

internal/vaultik/snapshot_failed_file_test.go

@@ -0,0 +1,123 @@
+package vaultik_test
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/spf13/afero"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"sneak.berlin/go/vaultik/internal/config"
+	"sneak.berlin/go/vaultik/internal/database"
+	"sneak.berlin/go/vaultik/internal/log"
+	"sneak.berlin/go/vaultik/internal/storage"
+	"sneak.berlin/go/vaultik/internal/ui"
+	"sneak.berlin/go/vaultik/internal/vaultik"
+)
+
+// openFailFs is the real filesystem, except that opening path fails
+// with err. Phase 1 of a backup only lstats a file, so it still counts
+// path; phase 2 is the first to open it.
+type openFailFs struct {
+	afero.OsFs
+
+	path string
+	err  error
+}
+
+//nolint:ireturn // afero.Fs.Open is defined to return the interface.
+func (f *openFailFs) Open(name string) (afero.File, error) {
+	if name == f.path {
+		return nil, &os.PathError{Op: "open", Path: name, Err: f.err}
+	}
+
+	return f.OsFs.Open(name)
+}
+
+// A file that phase 2 cannot open is reported as failed, not as
+// unchanged, and neither the summary's data total nor the snapshots row
+// counts it. See https://git.eeqj.de/sneak/vaultik/issues/280.
+func TestSnapshotSummaryCountsFileNotStoredAsFailed(t *testing.T) {
+	log.Initialize(log.Config{})
+	t.Parallel()
+
+	tests := []struct {
+		name       string
+		openErr    error
+		skipErrors bool
+	}{
+		// What a normal user gets opening a file with mode 000.
+		{name: "unopenable under skip-errors",
+			openErr: os.ErrPermission, skipErrors: true},
+		// What opening a file removed after phase 1 gives.
+		{name: "removed between the phases",
+			openErr: os.ErrNotExist, skipErrors: false},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			t.Parallel()
+
+			ctx := context.Background()
+
+			// The scan walks the source path with symlinks resolved, so
+			// failedPath must be spelled the same way to match.
+			tempDir, err := filepath.EvalSymlinks(t.TempDir())
+			require.NoError(t, err)
+
+			srcDir := filepath.Join(tempDir, "src")
+			failedPath := filepath.Join(srcDir, "failed.txt")
+			storedContent := []byte("this file is backed up")
+			storedSize := int64(len(storedContent))
+
+			fs := &openFailFs{path: failedPath, err: tt.openErr}
+			require.NoError(t, fs.MkdirAll(srcDir, 0o755))
+			require.NoError(t, afero.WriteFile(fs,
+				filepath.Join(srcDir, "stored.txt"), storedContent, 0o644))
+			require.NoError(t, afero.WriteFile(fs,
+				failedPath, []byte("this file cannot be opened"), 0o644))
+
+			cfg := faultTestConfig()
+			cfg.IndexPath = filepath.Join(tempDir, "index.sqlite")
+			cfg.Snapshots = map[string]config.SnapshotConfig{
+				"src": {Paths: []string{srcDir}},
+			}
+
+			store, err := storage.NewFileStorer(filepath.Join(tempDir, "remote"))
+			require.NoError(t, err)
+
+			db, err := database.New(ctx, cfg.IndexPath)
+			require.NoError(t, err)
+			t.Cleanup(func() { _ = db.Close() })
+
+			repos := database.NewRepositories(db)
+			out := &bytes.Buffer{}
+			v := newBackupVaultik(ctx, cfg, store, repos, db, fs)
+			v.UI = ui.NewWithColor(out, false)
+
+			require.NoError(t, v.CreateSnapshot(&vaultik.SnapshotCreateOptions{
+				SkipErrors: tt.skipErrors,
+				Snapshots:  []string{"src"},
+			}))
+
+			summary := out.String()
+			assert.Contains(t, summary,
+				"Files: 2 examined, 1 backed up, 0 unchanged, 1 failed.")
+			assert.Contains(t, summary,
+				fmt.Sprintf("Data: %s total (%s backed up).",
+					v.UI.Size(storedSize), v.UI.Size(storedSize)))
+
+			snap, err := repos.Snapshots.GetByID(ctx,
+				localSnapshotID(ctx, t, repos, "src"))
+			require.NoError(t, err)
+			require.NotNil(t, snap)
+
+			assert.Equal(t, int64(1), snap.FileCount)
+			assert.Equal(t, storedSize, snap.TotalSize)
+		})
+	}
+}