Stop counting directories and unchanged symlinks as files to back up (closes #305)

Every backup queued every directory and symlink for the second phase.
The line ending the walk counted directories, and their stat sizes, as
files to back up while its enumerated count left them out, and a
symlink was counted as backed up on every run, its target length as
data.

The walk now sets directories aside: they are recorded before the files
are backed up and are never counted as files or data, and the walk's
progress estimate leaves out the directories the local index knows. A
symlink is checked against its record as a regular file is, its target
included, so an unchanged one counts as unchanged. A test comment that
said directories count toward the backup's total size is corrected.

Judgement call: every directory is still written to the local index on
every backup, as before; no report shows it.

Model: opus-5-5

ARCHITECTURE.md

@@ -359,7 +359,7 @@ CreateSnapshot(opts)
 
 ## Deduplication Strategy
 
-1. **File-level**: Files unchanged since last backup are skipped (metadata comparison: size, mtime, mode, uid, gid), unless the file lists a chunk that no uploaded blob holds; such a file is re-chunked
+1. **File-level**: Files and symlinks unchanged since last backup are skipped (metadata comparison: size, mtime, mode, uid, gid, and a symlink's target), unless the file lists a chunk that no uploaded blob holds; such a file is re-chunked
 
 2. **Chunk-level**: Chunks are content-addressed by SHA256 hash. If a chunk hash already exists in the database, the chunk data is not re-uploaded.
 

README.md

@@ -182,7 +182,7 @@ vaultik version
 * `--verbose`, `-v`: Enable verbose output (on stderr — see below)
 * `--debug`: Enable debug output (on stderr — see below)
 * `--quiet`, `-q`: Suppress non-error output (also suppresses startup banner)
-* `--skip-errors`: Skip files that cannot be read when creating a snapshot, or that cannot be restored when restoring, instead of aborting. Packing and storage errors while creating a snapshot (which would leave a chunk recorded but not stored) still abort the run. A run that skipped anything still finishes but exits non-zero: `snapshot create` keeps the snapshot and counts each file or directory it skipped as failed. A file removed during the backup is also counted as failed but does not change the exit status.
+* `--skip-errors`: Skip files that cannot be read when creating a snapshot, or that cannot be restored when restoring, instead of aborting. Packing and storage errors while creating a snapshot (which would leave a chunk recorded but not stored) still abort the run.
 
 ### locking
 

internal/cli/entry_snapshot_create_skip_errors_test.go

@@ -1,30 +0,0 @@
-package cli //nolint:testpackage // shares runEntry and the hermetic config helpers
-
-import (
-	"testing"
-
-	"github.com/stretchr/testify/assert"
-)
-
-// TestEntrySnapshotCreateSkipErrorsExitsNonZeroWhenItSkips backs up,
-// under --skip-errors, a source directory that does not exist, which the
-// walk skips. The command exits non-zero and reports the backup as
-// completed, not as a failed snapshot creation. See
-// https://git.eeqj.de/sneak/vaultik/issues/310.
-//
-//nolint:paralleltest // replaces os.Args, os.Stdout, os.Stderr and the xdg globals
-func TestEntrySnapshotCreateSkipErrorsExitsNonZeroWhenItSkips(t *testing.T) {
-	// writeBackupConfig would create the source directory.
-	configPath := writeHermeticPruneConfig(t, false)
-
-	code, stdout, stderr := runEntry(t,
-		flagConfig, configPath, "--skip-errors", cmdSnapshot, cmdCreate)
-
-	assert.Equal(t, 1, code)
-	assert.Contains(t, stdout,
-		"Files: 1 examined, 0 backed up, 0 unchanged, 1 failed.")
-	assert.NotContains(t, stdout, "Finished successfully.")
-	assert.Contains(t, stderr,
-		"Backup completed, but 1 file(s) could not be read")
-	assert.NotContains(t, stderr, "Snapshot creation failed")
-}

internal/cli/snapshot.go

@@ -115,13 +115,6 @@ on Linux), then /etc/vaultik/config.yml.`,
 					return
 				}
 
-				if errors.Is(err, vaultik.ErrFilesUnreadable) {
-					log.Error("Backup completed with unreadable files", "error", err)
-					ReportErrorf("Backup completed, but %v", err)
-
-					return
-				}
-
 				log.Error("Snapshot creation failed", "error", err)
 				ReportErrorf("Snapshot creation failed: %v", err)
 			})

internal/snapshot/progress_test.go

@@ -78,8 +78,7 @@ func TestProgressPercentWithSecondPathSmaller(t *testing.T) {
 
 	stats := progress.GetStats()
 
-	// Directories count toward the total size but produce no chunks, so
-	// the percentage ends just under 100%.
+	// With both paths counted, the percentage must not exceed 100%.
 	percent := float64(stats.BytesProcessed.Load()) /
 		float64(stats.TotalSize.Load()) * 100
 	if percent > 100 {

internal/snapshot/scanner.go

@@ -141,12 +141,10 @@ type ScannerConfig struct {
 // ScanResult contains the results of a scan operation. Files and bytes
 // are counted per file: BytesScanned is the size of the new and changed
 // files, BytesSkipped that of the unchanged ones. FilesFailed counts the
-// new and changed files that phase 2 could not store, each entry removed
-// after the walk listed its directory and, under --skip-errors, each
-// entry the walk could not read, such as a directory it could not list;
-// FilesScanned includes them and BytesScanned does not. ChunksCreated
-// counts the chunks that were not already stored, and NewChunkSize is
-// their total size.
+// new and changed files that phase 2 could not store; FilesScanned
+// includes them and BytesScanned does not. ChunksCreated counts the
+// chunks that were not already stored, and NewChunkSize is their total
+// size.
 type ScanResult struct {
 	FilesScanned   int
 	FilesSkipped   int
@@ -163,11 +161,6 @@ type ScanResult struct {
 	UploadDuration time.Duration
 	StartTime      time.Time
 	EndTime        time.Time
-
-	// FilesUnreadable is the part of FilesFailed that --skip-errors
-	// skipped because it could not be read or listed; the rest are files
-	// removed after the walk listed them.
-	FilesUnreadable int
 }
 
 // NewScanner creates a new scanner instance
@@ -291,6 +284,11 @@ func (s *Scanner) Scan(
 		}
 	}
 
+	err = s.recordDirectories(ctx, scanResult.Directories)
+	if err != nil {
+		return nil, err
+	}
+
 	// Summarize scan phase results and update progress
 	s.summarizeScanPhase(result, filesToProcess)
 
@@ -862,14 +860,15 @@ func (s *Scanner) batchInsertFileData(
 // ScanPhaseResult contains the results of the scan phase
 type ScanPhaseResult struct {
 	FilesToProcess   []*FileToProcess
-	UnchangedFileIDs []types.FileID // IDs of unchanged files to associate with snapshot
+	UnchangedFileIDs []types.FileID   // IDs of unchanged files to associate with snapshot
+	Directories      []*FileToProcess // Every directory found, to record
 }
 
 // scanPhase performs the initial directory scan to identify files to process
 // It uses the pre-loaded knownFiles map for fast change detection without DB
 // queries. It also populates existingFiles map for deletion detection.
-// Returns files needing processing and IDs of unchanged files for snapshot
-// association.
+// Returns files needing processing, IDs of unchanged files for snapshot
+// association, and the directories found.
 func (s *Scanner) scanPhase(
 	ctx context.Context,
 	path string,
@@ -877,8 +876,10 @@ func (s *Scanner) scanPhase(
 	existingFiles map[string]struct{},
 	knownFiles map[string]*database.File,
 ) (*ScanPhaseResult, error) {
-	// Use known file count as estimate for progress (accurate for subsequent backups)
-	estimatedTotal := int64(len(knownFiles))
+	// Use known file count as estimate for progress (accurate for
+	// subsequent backups). The walk does not count directories, so the
+	// estimate leaves them out.
+	estimatedTotal := countNonDirectories(knownFiles)
 
 	collector := &scanCollector{}
 
@@ -889,16 +890,8 @@ func (s *Scanner) scanPhase(
 	log.Debug("Starting directory walk", "path", path)
 
 	walkFn := func(filePath string, info os.FileInfo, err error) error {
-		// The source path is in no directory listing, so its absence is
-		// an error rather than a removal.
-		if errors.Is(err, os.ErrNotExist) && filePath != path {
-			countRemovedDuringWalk(filePath, existingFiles, result)
-
-			return nil
-		}
-
 		if err != nil {
-			return s.handleWalkError(filePath, err, result)
+			return s.handleWalkError(filePath, err)
 		}
 
 		// Check context cancellation
@@ -919,7 +912,7 @@ func (s *Scanner) scanPhase(
 
 		// Handle symlinks and directories
 		handled, err := s.recordSpecialEntry(
-			filePath, info, existingFiles, collector, result)
+			filePath, info, existingFiles, knownFiles, collector, result)
 		if handled {
 			return err
 		}
@@ -933,9 +926,10 @@ func (s *Scanner) scanPhase(
 		existingFiles[filePath] = struct{}{}
 
 		// Check file against in-memory map (no DB query!)
-		file, needsProcessing := s.checkFileInMemory(filePath, info, knownFiles)
+		file := s.buildRegularFileEntry(filePath, info)
+		needsProcessing := s.checkFileInMemory(file, knownFiles)
 
-		filesScanned, changedCount := collector.recordRegularFile(
+		filesScanned, changedCount := collector.recordFile(
 			filePath, info, file, needsProcessing)
 
 		// Update result stats
@@ -959,27 +953,45 @@ func (s *Scanner) scanPhase(
 	return &ScanPhaseResult{
 		FilesToProcess:   collector.filesToProcess,
 		UnchangedFileIDs: collector.unchangedFileIDs,
+		Directories:      collector.directories,
 	}, nil
 }
 
+// countNonDirectories returns how many of files are not directories.
+func countNonDirectories(files map[string]*database.File) int64 {
+	var count int64
+
+	for _, f := range files {
+		if f.Mode&uint32(os.ModeDir) == 0 {
+			count++
+		}
+	}
+
+	return count
+}
+
 // recordSpecialEntry records symlinks and directories (which have no
-// data to chunk) and reports whether it handled the entry. For a symlink
-// whose target cannot be read it returns handleWalkError's result.
+// data to chunk) and reports whether it handled the entry. A symlink is
+// counted and checked for changes as a regular file is; a directory is
+// set aside to be recorded and is not counted. For a symlink whose
+// target cannot be read it returns handleWalkError's result.
 func (s *Scanner) recordSpecialEntry(
 	filePath string, info os.FileInfo,
 	existingFiles map[string]struct{},
+	knownFiles map[string]*database.File,
 	collector *scanCollector, result *ScanResult,
 ) (bool, error) {
 	// Handle symlinks
 	if info.Mode()&os.ModeSymlink != 0 {
 		file, err := s.buildSymlinkEntry(filePath, info)
 		if err != nil {
-			return true, s.handleWalkError(filePath, err, result)
+			return true, s.handleWalkError(filePath, err)
 		}
 
 		existingFiles[filePath] = struct{}{}
-		collector.addToProcess(filePath, info, file)
-		s.updateScanEntryStats(result, true, info)
+		needsProcessing := s.checkFileInMemory(file, knownFiles)
+		collector.recordFile(filePath, info, file, needsProcessing)
+		s.updateScanEntryStats(result, needsProcessing, info)
 
 		return true, nil
 	}
@@ -989,7 +1001,7 @@ func (s *Scanner) recordSpecialEntry(
 	if info.IsDir() {
 		file := s.buildDirectoryEntry(filePath, info)
 		existingFiles[filePath] = struct{}{}
-		collector.addToProcess(filePath, info, file)
+		collector.addDirectory(filePath, info, file)
 
 		return true, nil
 	}
@@ -997,38 +1009,15 @@ func (s *Scanner) recordSpecialEntry(
 	return false, nil
 }
 
-// countRemovedDuringWalk counts an entry that was in its directory's
-// listing but was gone when the walk looked it up, like a file removed
-// before phase 2 reads it: as examined and failed, and not as an error,
-// with or without --skip-errors.
-func countRemovedDuringWalk(
-	filePath string, existingFiles map[string]struct{}, result *ScanResult,
-) {
-	log.Warn("File was deleted during the directory walk, skipping",
-		"path", filePath)
-
-	// The walk found it, so it is not also counted as deleted.
-	existingFiles[filePath] = struct{}{}
-	result.FilesScanned++
-	result.FilesFailed++
-}
-
 // handleWalkError deals with a filesystem error surfaced by the walk:
-// under --skip-errors the entry is skipped with a UI notice and counted
-// as examined, failed and unreadable, otherwise the error is fatal.
-func (s *Scanner) handleWalkError(
-	filePath string, err error, result *ScanResult,
-) error {
+// skipped (with a UI notice) under --skip-errors, fatal otherwise.
+func (s *Scanner) handleWalkError(filePath string, err error) error {
 	if s.skipErrors {
 		log.Error("Failed to access file (skipping due to --skip-errors)",
 			"path", filePath, "error", err)
 		s.ui.Errorf("Failed to access %s: %v. Skipping (--skip-errors).",
 			s.ui.Path(filePath), err)
 
-		result.FilesScanned++
-		result.FilesFailed++
-		result.FilesUnreadable++
-
 		return nil // Continue scanning
 	}
 
@@ -1042,28 +1031,29 @@ type scanCollector struct {
 	mu               sync.Mutex
 	filesToProcess   []*FileToProcess
 	unchangedFileIDs []types.FileID // Just IDs - no new records needed
+	directories      []*FileToProcess
 	filesScanned     int64
 }
 
-// addToProcess queues a file entry for processing.
-func (c *scanCollector) addToProcess(
+// addDirectory sets a directory aside to be recorded. It is not counted
+// as a file.
+func (c *scanCollector) addDirectory(
 	path string, info os.FileInfo, file *database.File,
 ) {
 	c.mu.Lock()
 	defer c.mu.Unlock()
 
-	c.filesToProcess = append(c.filesToProcess, &FileToProcess{
+	c.directories = append(c.directories, &FileToProcess{
 		Path:     path,
 		FileInfo: info,
 		File:     file,
 	})
-	c.filesScanned++
 }
 
-// recordRegularFile records a regular file as changed (to process) or
-// unchanged (snapshot association only) and returns the running scanned
-// and changed counts.
-func (c *scanCollector) recordRegularFile(
+// recordFile records a regular file or symlink as changed (to process)
+// or unchanged (snapshot association only) and returns the running
+// scanned and changed counts.
+func (c *scanCollector) recordFile(
 	path string, info os.FileInfo, file *database.File, needsProcessing bool,
 ) (int64, int) {
 	c.mu.Lock()
@@ -1166,7 +1156,28 @@ func (s *Scanner) printScanProgressLine(
 	}
 }
 
-// buildSymlinkEntry creates a File record for a symlink.
+// buildRegularFileEntry creates a File record for a regular file, with
+// no ID: checkFileInMemory sets it.
+func (s *Scanner) buildRegularFileEntry(path string, info os.FileInfo) *database.File {
+	var uid, gid uint32
+	if stat, ok := info.Sys().(*syscall.Stat_t); ok {
+		uid = stat.Uid
+		gid = stat.Gid
+	}
+
+	return &database.File{
+		Path:       types.FilePath(path),
+		SourcePath: types.SourcePath(s.currentSourcePath),
+		MTime:      info.ModTime(),
+		Size:       info.Size(),
+		Mode:       uint32(info.Mode()),
+		UID:        uid,
+		GID:        gid,
+	}
+}
+
+// buildSymlinkEntry creates a File record for a symlink, with no ID:
+// checkFileInMemory sets it.
 func (s *Scanner) buildSymlinkEntry(
 	path string, info os.FileInfo,
 ) (*database.File, error) {
@@ -1182,7 +1193,6 @@ func (s *Scanner) buildSymlinkEntry(
 	}
 
 	return &database.File{
-		ID:         types.NewFileID(),
 		Path:       types.FilePath(path),
 		SourcePath: types.SourcePath(s.currentSourcePath),
 		MTime:      info.ModTime(),
@@ -1214,6 +1224,22 @@ func (s *Scanner) buildDirectoryEntry(path string, info os.FileInfo) *database.F
 	}
 }
 
+// recordDirectories records every directory the walk found. A directory
+// has no data to back up, so it is not among the files counted and
+// backed up.
+func (s *Scanner) recordDirectories(
+	ctx context.Context, directories []*FileToProcess,
+) error {
+	for _, dir := range directories {
+		err := s.recordNonRegularFile(ctx, dir)
+		if err != nil {
+			return fmt.Errorf("recording directory %s: %w", dir.Path, err)
+		}
+	}
+
+	return nil
+}
+
 // recordNonRegularFile writes a symlink or directory entry to the database
 // and associates it with the current snapshot. No chunking is performed.
 func (s *Scanner) recordNonRegularFile(ctx context.Context, ftp *FileToProcess) error {
@@ -1227,62 +1253,36 @@ func (s *Scanner) recordNonRegularFile(ctx context.Context, ftp *FileToProcess)
 	})
 }
 
-// checkFileInMemory checks if a file needs processing using the in-memory map
+// checkFileInMemory reports whether a regular file or symlink needs
+// processing, and gives file the ID of its known record, or a new ID for
+// a new path.
 // No database access is performed - this is purely CPU/memory work
 func (s *Scanner) checkFileInMemory(
-	path string, info os.FileInfo, knownFiles map[string]*database.File,
-) (*database.File, bool) {
-	var uid, gid uint32
-	if stat, ok := info.Sys().(*syscall.Stat_t); ok {
-		uid = stat.Uid
-		gid = stat.Gid
-	}
-
-	// Check against in-memory map first to get existing ID if available
-	existingFile, exists := knownFiles[path]
-
-	// Create file record with ID set upfront
-	// For new files, generate UUID immediately so it's available for chunk associations
-	// For existing files, reuse the existing ID
-	var fileID types.FileID
-	if exists {
-		fileID = existingFile.ID
-	} else {
-		fileID = types.NewFileID()
-	}
-
-	file := &database.File{
-		ID:         fileID,
-		Path:       types.FilePath(path),
-		SourcePath: types.SourcePath(s.currentSourcePath),
-		MTime:      info.ModTime(),
-		Size:       info.Size(),
-		Mode:       uint32(info.Mode()),
-		UID:        uid,
-		GID:        gid,
-	}
+	file *database.File, knownFiles map[string]*database.File,
+) bool {
+	existingFile, exists := knownFiles[file.Path.String()]
 
 	// New file - needs processing
 	if !exists {
-		return file, true
+		file.ID = types.NewFileID()
+
+		return true
 	}
 
+	file.ID = existingFile.ID
+
 	// No uploaded blob holds one of its chunks (see loadFilesToRechunk)
-	if _, rechunk := s.filesToRechunk[fileID]; rechunk {
-		return file, true
+	if _, rechunk := s.filesToRechunk[file.ID]; rechunk {
+		return true
 	}
 
 	// Check if file has changed
-	if existingFile.Size != file.Size ||
+	return existingFile.Size != file.Size ||
 		!existingFile.MTime.Equal(file.MTime) ||
 		existingFile.Mode != file.Mode ||
 		existingFile.UID != file.UID ||
-		existingFile.GID != file.GID {
-		return file, true
-	}
-
-	// File unchanged
-	return file, false
+		existingFile.GID != file.GID ||
+		existingFile.LinkTarget != file.LinkTarget
 }
 
 // batchAddFilesToSnapshot adds existing file IDs to the snapshot association table
@@ -1409,17 +1409,16 @@ func (s *Scanner) processPhase(
 	return s.finalizeProcessPhase(ctx, result)
 }
 
-// processFileWithErrorHandling records a directory or symlink, or wraps
+// processFileWithErrorHandling records a symlink, or wraps
 // processFileStreaming for a regular file with error recovery for deleted
 // files and skip-errors mode. Returns (skipped, error).
 func (s *Scanner) processFileWithErrorHandling(
 	ctx context.Context, fileToProcess *FileToProcess, result *ScanResult,
 ) (bool, error) {
-	// A directory or symlink has no data to open or read; it is only
-	// recorded in the local index. An error recording it stops the run
-	// even under --skip-errors, or the snapshot would complete without it.
-	mode := os.FileMode(fileToProcess.File.Mode)
-	if mode&os.ModeSymlink != 0 || mode.IsDir() {
+	// A symlink has no data to open or read; it is only recorded in the
+	// local index. An error recording it stops the run even under
+	// --skip-errors, or the snapshot would complete without it.
+	if os.FileMode(fileToProcess.File.Mode)&os.ModeSymlink != 0 {
 		err := s.recordNonRegularFile(ctx, fileToProcess)
 		if err != nil {
 			return false, fmt.Errorf("processing file %s: %w", fileToProcess.Path, err)
@@ -1460,7 +1459,6 @@ func (s *Scanner) processFileWithErrorHandling(
 				s.ui.Path(fileToProcess.Path), err)
 
 			countFailedFile(fileToProcess, result)
-			result.FilesUnreadable++
 
 			return true, nil
 		}

internal/snapshot/scanner_estimate_test.go

@@ -0,0 +1,28 @@
+//nolint:testpackage // exercises the unexported countNonDirectories helper
+package snapshot
+
+import (
+	"os"
+	"testing"
+
+	"sneak.berlin/go/vaultik/internal/database"
+)
+
+// The walk counts regular files and symlinks, not directories, so the
+// estimate of its count made from the local index's records leaves the
+// directories out. See https://git.eeqj.de/sneak/vaultik/issues/305.
+func TestCountNonDirectories(t *testing.T) {
+	t.Parallel()
+
+	known := map[string]*database.File{
+		"/src":       {Mode: uint32(os.ModeDir | 0o755)},
+		"/src/sub":   {Mode: uint32(os.ModeDir | 0o755)},
+		"/src/a.txt": {Mode: 0o644},
+		"/src/link":  {Mode: uint32(os.ModeSymlink | 0o777), LinkTarget: "a.txt"},
+	}
+
+	got := countNonDirectories(known)
+	if got != 2 {
+		t.Errorf("countNonDirectories = %d, want 2", got)
+	}
+}

internal/vaultik/snapshot.go

@@ -39,11 +39,6 @@ var (
 // caller can report it apart from a failed backup.
 var ErrPostBackupPruneFailed = errors.New("post-backup prune failed")
 
-// ErrFilesUnreadable is returned when --skip-errors skipped files or
-// directories it could not read or list. The snapshot is complete and
-// kept, so the caller can report it apart from a failed backup.
-var ErrFilesUnreadable = errors.New("file(s) could not be read")
-
 // listRecentLimit caps how many snapshot rows are fetched from the
 // local index for listing and purge operations.
 const listRecentLimit = 10000
@@ -135,16 +130,12 @@ func (v *Vaultik) CreateSnapshot(opts *SnapshotCreateOptions) error {
 	}
 
 	// Process each named snapshot
-	filesUnreadable := 0
-
 	for snapIdx, snapName := range snapshotNames {
-		unreadable, err := v.createNamedSnapshot(
+		err = v.createNamedSnapshot(
 			opts, hostname, snapName, snapIdx+1, len(snapshotNames))
 		if err != nil {
 			return err
 		}
-
-		filesUnreadable += unreadable
 	}
 
 	// Print overall summary if multiple snapshots
@@ -160,19 +151,6 @@ func (v *Vaultik) CreateSnapshot(opts *SnapshotCreateOptions) error {
 		}
 	}
 
-	return v.finishCreateSnapshot(filesUnreadable)
-}
-
-// finishCreateSnapshot ends a run whose snapshots are all complete and
-// kept. Like finishRestore, it returns an error when --skip-errors
-// skipped anything it could not read or list; otherwise it prints the
-// closing line. A file removed during the run is counted as failed in
-// the summary but is not an error, with or without --skip-errors.
-func (v *Vaultik) finishCreateSnapshot(filesUnreadable int) error {
-	if filesUnreadable > 0 {
-		return fmt.Errorf("%d %w", filesUnreadable, ErrFilesUnreadable)
-	}
-
 	// Terminus must obey the --cron invariant: silent on total
 	// success only. UI.Complete is dropped in cron/quiet mode (that's
 	// the success path), but if any warnings fired during the run we
@@ -262,22 +240,16 @@ type snapshotStats struct {
 	totalBlobsUploaded int
 	uploadDuration     time.Duration
 
-	// The part of totalFilesFailed that --skip-errors skipped because it
-	// could not be read or listed.
-	totalFilesUnreadable int
-
 	// The sizes of all blobs the snapshot references, set by
 	// finalizeSnapshotMetadata once snapshot_blobs is populated.
 	blobSize             int64
 	blobUncompressedSize int64
 }
 
-// createNamedSnapshot creates a single named snapshot and returns the
-// number of entries --skip-errors skipped because it could not read or
-// list them.
+// createNamedSnapshot creates a single named snapshot
 func (v *Vaultik) createNamedSnapshot(
 	opts *SnapshotCreateOptions, hostname, snapName string, idx, total int,
-) (int, error) {
+) error {
 	snapshotStartTime := time.Now()
 
 	if total > 1 {
@@ -286,12 +258,12 @@ func (v *Vaultik) createNamedSnapshot(
 
 	resolvedDirs, err := v.resolveSnapshotPaths(snapName)
 	if err != nil {
-		return 0, err
+		return err
 	}
 
 	previousSnapshotID, err := v.previousSnapshotID(snapName)
 	if err != nil {
-		return 0, err
+		return err
 	}
 
 	scanner := v.ScannerFactory(snapshot.ScannerParams{
@@ -306,7 +278,7 @@ func (v *Vaultik) createNamedSnapshot(
 	snapshotID, err := v.SnapshotManager.CreateSnapshotWithName(
 		v.ctx, hostname, snapName, v.Globals.Version, v.Globals.Commit)
 	if err != nil {
-		return 0, fmt.Errorf("creating snapshot: %w", err)
+		return fmt.Errorf("creating snapshot: %w", err)
 	}
 
 	log.Info("Beginning snapshot", "snapshot_id", snapshotID, "name", snapName)
@@ -314,12 +286,12 @@ func (v *Vaultik) createNamedSnapshot(
 
 	stats, err := v.scanAllDirectories(scanner, resolvedDirs, snapshotID)
 	if err != nil {
-		return 0, err
+		return err
 	}
 
 	err = v.finalizeSnapshotMetadata(snapshotID, stats)
 	if err != nil {
-		return 0, err
+		return err
 	}
 
 	log.Info("Snapshot complete",
@@ -332,7 +304,7 @@ func (v *Vaultik) createNamedSnapshot(
 
 	v.printSnapshotSummary(snapshotID, snapshotStartTime, stats)
 
-	return stats.totalFilesUnreadable, nil
+	return nil
 }
 
 // previousSnapshotID returns the ID of the most recent completed snapshot
@@ -418,7 +390,6 @@ func (v *Vaultik) scanAllDirectories(
 		stats.totalBlobs += result.BlobsCreated
 		stats.totalFilesSkipped += result.FilesSkipped
 		stats.totalFilesFailed += result.FilesFailed
-		stats.totalFilesUnreadable += result.FilesUnreadable
 		stats.totalBytesSkipped += result.BytesSkipped
 		stats.totalFilesDeleted += result.FilesDeleted
 		stats.totalBytesDeleted += result.BytesDeleted

internal/vaultik/snapshot_deleted_count_test.go

@@ -178,13 +178,11 @@ func TestBackupDoesNotCountFileInUnopenableDirectoryAsDeleted(t *testing.T) {
 	}
 
 	env.out.Reset()
-	err := env.v.CreateSnapshot(&vaultik.SnapshotCreateOptions{
+	require.NoError(t, env.v.CreateSnapshot(&vaultik.SnapshotCreateOptions{
 		SkipErrors: true,
 		Snapshots:  []string{"work"},
-	})
-	require.ErrorIs(t, err, vaultik.ErrFilesUnreadable)
-	assert.Contains(t, env.out.String(),
-		"Files: 2 examined, 0 backed up, 1 unchanged, 1 failed.")
+	}))
+	assert.Contains(t, env.out.String(), filesLine(1, 0, 1))
 }
 
 // A backup counts only files that the previous snapshot of its own name

internal/vaultik/snapshot_failed_file_test.go

@@ -49,19 +49,13 @@ func TestSnapshotSummaryCountsFileNotStoredAsFailed(t *testing.T) {
 		name       string
 		openErr    error
 		skipErrors bool
-		wantErr    error // nil for none
 	}{
 		// What a normal user gets opening a file with mode 000.
 		{name: "unopenable under skip-errors",
-			openErr: os.ErrPermission, skipErrors: true,
-			wantErr: vaultik.ErrFilesUnreadable},
+			openErr: os.ErrPermission, skipErrors: true},
 		// What opening a file removed after phase 1 gives.
 		{name: "removed between the phases",
 			openErr: os.ErrNotExist, skipErrors: false},
-		// A removal is not an error, so --skip-errors does not make the
-		// run fail. See https://git.eeqj.de/sneak/vaultik/issues/310.
-		{name: "removed between the phases under skip-errors",
-			openErr: os.ErrNotExist, skipErrors: true},
 	}
 
 	for _, tt := range tests {
@@ -105,11 +99,10 @@ func TestSnapshotSummaryCountsFileNotStoredAsFailed(t *testing.T) {
 			v := newBackupVaultik(ctx, cfg, store, repos, db, fs)
 			v.UI = ui.NewWithColor(out, false)
 
-			err = v.CreateSnapshot(&vaultik.SnapshotCreateOptions{
+			require.NoError(t, v.CreateSnapshot(&vaultik.SnapshotCreateOptions{
 				SkipErrors: tt.skipErrors,
 				Snapshots:  []string{"src"},
-			})
-			require.ErrorIs(t, err, tt.wantErr)
+			}))
 
 			summary := out.String()
 			assert.Contains(t, summary,

internal/vaultik/snapshot_skip_errors_test.go

@@ -3,7 +3,6 @@ package vaultik_test
 import (
 	"context"
 	"fmt"
-	"os"
 	"path/filepath"
 	"testing"
 
@@ -74,120 +73,3 @@ func TestSkipErrorsBackupStopsOnIndexErrorRecordingDirectory(t *testing.T) {
 	require.Len(t, snapshots, 1)
 	assert.Nil(t, snapshots[0].CompletedAt)
 }
-
-// A backup under --skip-errors that skips a file it cannot open, or a
-// directory it cannot list, keeps the snapshot, counts what it skipped
-// as failed and returns an error instead of finishing successfully. See
-// https://git.eeqj.de/sneak/vaultik/issues/310.
-func TestSkipErrorsBackupKeepsSnapshotAndFailsWhenItSkips(t *testing.T) {
-	log.Initialize(log.Config{})
-	t.Parallel()
-
-	tests := []struct {
-		name    string
-		skipped string // relative to the source directory
-	}{
-		{name: "file it cannot open", skipped: "skipdir/file.txt"},
-		{name: "directory it cannot list", skipped: "skipdir"},
-	}
-
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			t.Parallel()
-
-			env := newDeletedCountEnv(t, map[string][]byte{
-				"stored.txt":       []byte("stored"),
-				"skipdir/file.txt": []byte("in skipdir"),
-			})
-			env.setSnapshot("work", "")
-
-			// A mode does not lock out root, who runs the tests, so
-			// opening the skipped path is made to fail instead.
-			env.v.Fs = &openFailFs{
-				path: filepath.Join(env.src, tt.skipped),
-				err:  os.ErrPermission,
-			}
-
-			err := env.v.CreateSnapshot(&vaultik.SnapshotCreateOptions{
-				SkipErrors: true,
-				Snapshots:  []string{"work"},
-			})
-			require.ErrorIs(t, err, vaultik.ErrFilesUnreadable)
-
-			assert.Contains(t, env.out.String(),
-				"Files: 2 examined, 1 backed up, 0 unchanged, 1 failed.")
-			assert.NotContains(t, env.out.String(), "Finished successfully.")
-
-			snapshots, err := env.v.Repositories.Snapshots.ListRecent(
-				context.Background(), listRecentTestLimit)
-			require.NoError(t, err)
-			require.Len(t, snapshots, 1)
-			assert.NotNil(t, snapshots[0].CompletedAt, "the snapshot is kept")
-		})
-	}
-}
-
-// removeOnLookupFs is the real filesystem, except that path is removed
-// just before it is looked up. The walk looks an entry up after listing
-// its directory, so path is listed but gone when the walk reaches it.
-type removeOnLookupFs struct {
-	afero.OsFs
-
-	path string
-}
-
-func (f *removeOnLookupFs) LstatIfPossible(
-	name string,
-) (os.FileInfo, bool, error) {
-	if name == f.path {
-		_ = os.Remove(name)
-	}
-
-	return f.OsFs.LstatIfPossible(name)
-}
-
-// A file removed before the walk reaches it is counted once, as failed,
-// like a file removed before its data is read, and the backup finishes
-// successfully, with or without --skip-errors. See
-// https://git.eeqj.de/sneak/vaultik/issues/310.
-func TestBackupCountsFileRemovedBeforeWalkReachesItAsFailed(t *testing.T) {
-	log.Initialize(log.Config{})
-	t.Parallel()
-
-	tests := []struct {
-		name       string
-		skipErrors bool
-	}{
-		{name: "under skip-errors", skipErrors: true},
-		{name: "without skip-errors", skipErrors: false},
-	}
-
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			t.Parallel()
-
-			env := newDeletedCountEnv(t, map[string][]byte{
-				"stays.txt":   []byte("stays"),
-				"removed.txt": []byte("removed"),
-			})
-			env.setSnapshot("walk", "")
-
-			// The previous snapshot has the file, so a removal the walk
-			// also counted as deleted would show.
-			env.backUp(t, "walk")
-
-			env.v.Fs = &removeOnLookupFs{
-				path: filepath.Join(env.src, "removed.txt"),
-			}
-
-			env.out.Reset()
-			require.NoError(t, env.v.CreateSnapshot(&vaultik.SnapshotCreateOptions{
-				SkipErrors: tt.skipErrors,
-				Snapshots:  []string{"walk"},
-			}))
-			assert.Contains(t, env.out.String(),
-				"Files: 2 examined, 0 backed up, 1 unchanged, 1 failed.")
-			assert.Contains(t, env.out.String(), "Finished successfully.")
-		})
-	}
-}

internal/vaultik/snapshot_summary_test.go

@@ -13,6 +13,7 @@ import (
 	"github.com/spf13/afero"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
+	"golang.org/x/sys/unix"
 	"sneak.berlin/go/vaultik/internal/config"
 	"sneak.berlin/go/vaultik/internal/database"
 	"sneak.berlin/go/vaultik/internal/log"
@@ -188,6 +189,42 @@ func (e *summaryEnv) dataLine(total, backedUp int64) string {
 		e.v.UI.Size(total), e.v.UI.Size(backedUp))
 }
 
+// enumeratedLine returns the line printed when the walk of a source path
+// ends.
+func (e *summaryEnv) enumeratedLine(
+	files int, total int64, toBackUp int, toBackUpSize int64,
+) string {
+	return fmt.Sprintf(
+		"Enumerated %d snapshot source files (%s total), %d to back up (%s).",
+		files, e.v.UI.Size(total), toBackUp, e.v.UI.Size(toBackUpSize))
+}
+
+// snapshotPaths returns the paths the snapshot lists, sorted.
+func (e *summaryEnv) snapshotPaths(t *testing.T, snapshotID string) []string {
+	t.Helper()
+
+	rows, err := e.db.Conn().QueryContext(context.Background(), `
+		SELECT f.path FROM snapshot_files sf JOIN files f ON f.id = sf.file_id
+		WHERE sf.snapshot_id = ? ORDER BY f.path`, snapshotID)
+	require.NoError(t, err)
+
+	defer func() { _ = rows.Close() }()
+
+	var paths []string
+
+	for rows.Next() {
+		var path string
+
+		require.NoError(t, rows.Scan(&path))
+
+		paths = append(paths, path)
+	}
+
+	require.NoError(t, rows.Err())
+
+	return paths
+}
+
 // A first backup stores copy.bin's chunks while backing up a.bin, so
 // copy.bin's chunks are deduplicated within the run. Each file and byte
 // is still counted once.
@@ -248,6 +285,85 @@ func TestSnapshotSummaryIncrementalRunWithDeduplicatedChunks(t *testing.T) {
 	assert.Equal(t, uncompressed, snap.BlobUncompressedSize)
 }
 
+// A backup of a tree with directories and a symlink, none of which
+// changed since the last backup, backs up nothing, and the snapshot
+// still lists every one of them. Both sides of the line that ends the
+// walk count the same entries, the files and the symlink, and no
+// directory's size. See https://git.eeqj.de/sneak/vaultik/issues/305.
+func TestSnapshotSummaryUnchangedRunWithDirectoriesAndSymlink(t *testing.T) {
+	log.Initialize(log.Config{})
+	t.Parallel()
+
+	env := newSummaryEnv(t)
+
+	const linkTarget = "a.bin"
+
+	require.NoError(t, os.Symlink(linkTarget,
+		filepath.Join(filepath.Dir(env.aPath), "link")))
+
+	// A symlink's size is the length of its target.
+	total := env.totalSize + int64(len(linkTarget))
+
+	first := env.backUp(t, "first", false)
+	assert.Contains(t, first, env.enumeratedLine(4, total, 4, total))
+	assert.Contains(t, first, filesLine(4, 4, 0))
+
+	second := env.backUp(t, "second", false)
+	assert.Contains(t, second, env.enumeratedLine(4, total, 0, 0))
+	assert.Contains(t, second, filesLine(4, 0, 4))
+	assert.Contains(t, second, env.dataLine(total, 0))
+
+	assert.Equal(t,
+		env.snapshotPaths(t, env.snapshot(t, "first").ID.String()),
+		env.snapshotPaths(t, env.snapshot(t, "second").ID.String()))
+}
+
+// A directory whose mtime changed and a symlink whose target changed are
+// recorded again by the next backup, which counts only the symlink as
+// backed up.
+func TestSnapshotSummaryChangedDirectoryAndSymlink(t *testing.T) {
+	log.Initialize(log.Config{})
+	t.Parallel()
+
+	env := newSummaryEnv(t)
+
+	// The local index keeps paths with symlinks resolved.
+	dir, err := filepath.EvalSymlinks(filepath.Dir(env.aPath))
+	require.NoError(t, err)
+
+	// The new link gets the old one's mtime, and a target, which need not
+	// exist, of the old target's length, a symlink's size: only the
+	// target differs.
+	linkTime := unix.NsecToTimeval(
+		time.Date(2029, time.March, 4, 5, 6, 7, 0, time.UTC).UnixNano())
+	link := filepath.Join(dir, "link")
+	require.NoError(t, os.Symlink("a.bin", link))
+	require.NoError(t, unix.Lutimes(link, []unix.Timeval{linkTime, linkTime}))
+
+	env.backUp(t, "first", false)
+
+	require.NoError(t, os.Remove(link))
+	require.NoError(t, os.Symlink("b.bin", link))
+	require.NoError(t, unix.Lutimes(link, []unix.Timeval{linkTime, linkTime}))
+
+	mtime := time.Date(2030, time.January, 2, 3, 4, 5, 0, time.UTC)
+	require.NoError(t, os.Chtimes(dir, mtime, mtime))
+
+	assert.Contains(t, env.backUp(t, "second", false), filesLine(4, 1, 3))
+
+	ctx := context.Background()
+
+	dirRow, err := env.repos.Files.GetByPath(ctx, dir)
+	require.NoError(t, err)
+	require.NotNil(t, dirRow)
+	assert.Equal(t, mtime.Unix(), dirRow.MTime.Unix())
+
+	linkRow, err := env.repos.Files.GetByPath(ctx, link)
+	require.NoError(t, err)
+	require.NotNil(t, linkRow)
+	assert.Equal(t, "b.bin", linkRow.LinkTarget.String())
+}
+
 // Under --cron the progress reporter is off; the upload figures must
 // still reach the summary and the snapshots row. The snapshot has two
 // paths, each backed up by its own scan.