Stop counting directories and unchanged symlinks as files to back up (closes #305)

Every backup queued every directory and symlink for the second phase.
The line ending the walk counted directories, and their stat sizes, as
files to back up while its enumerated count left them out, and a
symlink was counted as backed up on every run, its target length as
data.

The walk now sets directories aside: they are recorded before the files
are backed up and never counted as files or data, and the walk's
progress estimate leaves out the directories the local index knows. A
symlink is checked against its record like a regular file, its target
included, so an unchanged one counts as unchanged.

Judgement call: every directory is still written to the local index on
every backup, as before; no report shows it.

Model: opus-5-5

internal/snapshot/scanner.go

@@ -284,6 +284,11 @@ func (s *Scanner) Scan(
 		}
 	}
 
+	err = s.recordDirectories(ctx, scanResult.Directories)
+	if err != nil {
+		return nil, err
+	}
+
 	// Summarize scan phase results and update progress
 	s.summarizeScanPhase(result, filesToProcess)
 
@@ -855,14 +860,15 @@ func (s *Scanner) batchInsertFileData(
 // ScanPhaseResult contains the results of the scan phase
 type ScanPhaseResult struct {
 	FilesToProcess   []*FileToProcess
-	UnchangedFileIDs []types.FileID // IDs of unchanged files to associate with snapshot
+	UnchangedFileIDs []types.FileID   // IDs of unchanged files to associate with snapshot
+	Directories      []*FileToProcess // Every directory found, to record
 }
 
 // scanPhase performs the initial directory scan to identify files to process
 // It uses the pre-loaded knownFiles map for fast change detection without DB
 // queries. It also populates existingFiles map for deletion detection.
-// Returns files needing processing and IDs of unchanged files for snapshot
-// association.
+// Returns files needing processing, IDs of unchanged files for snapshot
+// association, and the directories found.
 func (s *Scanner) scanPhase(
 	ctx context.Context,
 	path string,
@@ -870,8 +876,10 @@ func (s *Scanner) scanPhase(
 	existingFiles map[string]struct{},
 	knownFiles map[string]*database.File,
 ) (*ScanPhaseResult, error) {
-	// Use known file count as estimate for progress (accurate for subsequent backups)
-	estimatedTotal := int64(len(knownFiles))
+	// Use known file count as estimate for progress (accurate for
+	// subsequent backups). The walk does not count directories, so the
+	// estimate leaves them out.
+	estimatedTotal := countNonDirectories(knownFiles)
 
 	collector := &scanCollector{}
 
@@ -904,7 +912,7 @@ func (s *Scanner) scanPhase(
 
 		// Handle symlinks and directories
 		handled, err := s.recordSpecialEntry(
-			filePath, info, existingFiles, collector, result)
+			filePath, info, existingFiles, knownFiles, collector, result)
 		if handled {
 			return err
 		}
@@ -918,9 +926,10 @@ func (s *Scanner) scanPhase(
 		existingFiles[filePath] = struct{}{}
 
 		// Check file against in-memory map (no DB query!)
-		file, needsProcessing := s.checkFileInMemory(filePath, info, knownFiles)
+		file := s.buildRegularFileEntry(filePath, info)
+		needsProcessing := s.checkFileInMemory(file, knownFiles)
 
-		filesScanned, changedCount := collector.recordRegularFile(
+		filesScanned, changedCount := collector.recordFile(
 			filePath, info, file, needsProcessing)
 
 		// Update result stats
@@ -944,15 +953,32 @@ func (s *Scanner) scanPhase(
 	return &ScanPhaseResult{
 		FilesToProcess:   collector.filesToProcess,
 		UnchangedFileIDs: collector.unchangedFileIDs,
+		Directories:      collector.directories,
 	}, nil
 }
 
+// countNonDirectories returns how many of files are not directories.
+func countNonDirectories(files map[string]*database.File) int64 {
+	var count int64
+
+	for _, f := range files {
+		if f.Mode&uint32(os.ModeDir) == 0 {
+			count++
+		}
+	}
+
+	return count
+}
+
 // recordSpecialEntry records symlinks and directories (which have no
-// data to chunk) and reports whether it handled the entry. For a symlink
-// whose target cannot be read it returns handleWalkError's result.
+// data to chunk) and reports whether it handled the entry. A symlink is
+// counted and checked for changes as a regular file is; a directory is
+// set aside to be recorded and is not counted. For a symlink whose
+// target cannot be read it returns handleWalkError's result.
 func (s *Scanner) recordSpecialEntry(
 	filePath string, info os.FileInfo,
 	existingFiles map[string]struct{},
+	knownFiles map[string]*database.File,
 	collector *scanCollector, result *ScanResult,
 ) (bool, error) {
 	// Handle symlinks
@@ -963,8 +989,9 @@ func (s *Scanner) recordSpecialEntry(
 		}
 
 		existingFiles[filePath] = struct{}{}
-		collector.addToProcess(filePath, info, file)
-		s.updateScanEntryStats(result, true, info)
+		needsProcessing := s.checkFileInMemory(file, knownFiles)
+		collector.recordFile(filePath, info, file, needsProcessing)
+		s.updateScanEntryStats(result, needsProcessing, info)
 
 		return true, nil
 	}
@@ -974,7 +1001,7 @@ func (s *Scanner) recordSpecialEntry(
 	if info.IsDir() {
 		file := s.buildDirectoryEntry(filePath, info)
 		existingFiles[filePath] = struct{}{}
-		collector.addToProcess(filePath, info, file)
+		collector.addDirectory(filePath, info, file)
 
 		return true, nil
 	}
@@ -1004,28 +1031,29 @@ type scanCollector struct {
 	mu               sync.Mutex
 	filesToProcess   []*FileToProcess
 	unchangedFileIDs []types.FileID // Just IDs - no new records needed
+	directories      []*FileToProcess
 	filesScanned     int64
 }
 
-// addToProcess queues a file entry for processing.
-func (c *scanCollector) addToProcess(
+// addDirectory sets a directory aside to be recorded. It is not counted
+// as a file.
+func (c *scanCollector) addDirectory(
 	path string, info os.FileInfo, file *database.File,
 ) {
 	c.mu.Lock()
 	defer c.mu.Unlock()
 
-	c.filesToProcess = append(c.filesToProcess, &FileToProcess{
+	c.directories = append(c.directories, &FileToProcess{
 		Path:     path,
 		FileInfo: info,
 		File:     file,
 	})
-	c.filesScanned++
 }
 
-// recordRegularFile records a regular file as changed (to process) or
-// unchanged (snapshot association only) and returns the running scanned
-// and changed counts.
-func (c *scanCollector) recordRegularFile(
+// recordFile records a regular file or symlink as changed (to process)
+// or unchanged (snapshot association only) and returns the running
+// scanned and changed counts.
+func (c *scanCollector) recordFile(
 	path string, info os.FileInfo, file *database.File, needsProcessing bool,
 ) (int64, int) {
 	c.mu.Lock()
@@ -1128,7 +1156,28 @@ func (s *Scanner) printScanProgressLine(
 	}
 }
 
-// buildSymlinkEntry creates a File record for a symlink.
+// buildRegularFileEntry creates a File record for a regular file, with
+// no ID: checkFileInMemory sets it.
+func (s *Scanner) buildRegularFileEntry(path string, info os.FileInfo) *database.File {
+	var uid, gid uint32
+	if stat, ok := info.Sys().(*syscall.Stat_t); ok {
+		uid = stat.Uid
+		gid = stat.Gid
+	}
+
+	return &database.File{
+		Path:       types.FilePath(path),
+		SourcePath: types.SourcePath(s.currentSourcePath),
+		MTime:      info.ModTime(),
+		Size:       info.Size(),
+		Mode:       uint32(info.Mode()),
+		UID:        uid,
+		GID:        gid,
+	}
+}
+
+// buildSymlinkEntry creates a File record for a symlink, with no ID:
+// checkFileInMemory sets it.
 func (s *Scanner) buildSymlinkEntry(
 	path string, info os.FileInfo,
 ) (*database.File, error) {
@@ -1144,7 +1193,6 @@ func (s *Scanner) buildSymlinkEntry(
 	}
 
 	return &database.File{
-		ID:         types.NewFileID(),
 		Path:       types.FilePath(path),
 		SourcePath: types.SourcePath(s.currentSourcePath),
 		MTime:      info.ModTime(),
@@ -1176,6 +1224,22 @@ func (s *Scanner) buildDirectoryEntry(path string, info os.FileInfo) *database.F
 	}
 }
 
+// recordDirectories records every directory the walk found. A directory
+// has no data to back up, so it is not among the files counted and
+// backed up.
+func (s *Scanner) recordDirectories(
+	ctx context.Context, directories []*FileToProcess,
+) error {
+	for _, dir := range directories {
+		err := s.recordNonRegularFile(ctx, dir)
+		if err != nil {
+			return fmt.Errorf("recording directory %s: %w", dir.Path, err)
+		}
+	}
+
+	return nil
+}
+
 // recordNonRegularFile writes a symlink or directory entry to the database
 // and associates it with the current snapshot. No chunking is performed.
 func (s *Scanner) recordNonRegularFile(ctx context.Context, ftp *FileToProcess) error {
@@ -1189,62 +1253,36 @@ func (s *Scanner) recordNonRegularFile(ctx context.Context, ftp *FileToProcess)
 	})
 }
 
-// checkFileInMemory checks if a file needs processing using the in-memory map
+// checkFileInMemory reports whether a regular file or symlink needs
+// processing, and gives file the ID of its known record, or a new ID for
+// a new path.
 // No database access is performed - this is purely CPU/memory work
 func (s *Scanner) checkFileInMemory(
-	path string, info os.FileInfo, knownFiles map[string]*database.File,
-) (*database.File, bool) {
-	var uid, gid uint32
-	if stat, ok := info.Sys().(*syscall.Stat_t); ok {
-		uid = stat.Uid
-		gid = stat.Gid
-	}
-
-	// Check against in-memory map first to get existing ID if available
-	existingFile, exists := knownFiles[path]
-
-	// Create file record with ID set upfront
-	// For new files, generate UUID immediately so it's available for chunk associations
-	// For existing files, reuse the existing ID
-	var fileID types.FileID
-	if exists {
-		fileID = existingFile.ID
-	} else {
-		fileID = types.NewFileID()
-	}
-
-	file := &database.File{
-		ID:         fileID,
-		Path:       types.FilePath(path),
-		SourcePath: types.SourcePath(s.currentSourcePath),
-		MTime:      info.ModTime(),
-		Size:       info.Size(),
-		Mode:       uint32(info.Mode()),
-		UID:        uid,
-		GID:        gid,
-	}
+	file *database.File, knownFiles map[string]*database.File,
+) bool {
+	existingFile, exists := knownFiles[file.Path.String()]
 
 	// New file - needs processing
 	if !exists {
-		return file, true
+		file.ID = types.NewFileID()
+
+		return true
 	}
 
+	file.ID = existingFile.ID
+
 	// No uploaded blob holds one of its chunks (see loadFilesToRechunk)
-	if _, rechunk := s.filesToRechunk[fileID]; rechunk {
-		return file, true
+	if _, rechunk := s.filesToRechunk[file.ID]; rechunk {
+		return true
 	}
 
 	// Check if file has changed
-	if existingFile.Size != file.Size ||
+	return existingFile.Size != file.Size ||
 		!existingFile.MTime.Equal(file.MTime) ||
 		existingFile.Mode != file.Mode ||
 		existingFile.UID != file.UID ||
-		existingFile.GID != file.GID {
-		return file, true
-	}
-
-	// File unchanged
-	return file, false
+		existingFile.GID != file.GID ||
+		existingFile.LinkTarget != file.LinkTarget
 }
 
 // batchAddFilesToSnapshot adds existing file IDs to the snapshot association table
@@ -1371,17 +1409,16 @@ func (s *Scanner) processPhase(
 	return s.finalizeProcessPhase(ctx, result)
 }
 
-// processFileWithErrorHandling records a directory or symlink, or wraps
+// processFileWithErrorHandling records a symlink, or wraps
 // processFileStreaming for a regular file with error recovery for deleted
 // files and skip-errors mode. Returns (skipped, error).
 func (s *Scanner) processFileWithErrorHandling(
 	ctx context.Context, fileToProcess *FileToProcess, result *ScanResult,
 ) (bool, error) {
-	// A directory or symlink has no data to open or read; it is only
-	// recorded in the local index. An error recording it stops the run
-	// even under --skip-errors, or the snapshot would complete without it.
-	mode := os.FileMode(fileToProcess.File.Mode)
-	if mode&os.ModeSymlink != 0 || mode.IsDir() {
+	// A symlink has no data to open or read; it is only recorded in the
+	// local index. An error recording it stops the run even under
+	// --skip-errors, or the snapshot would complete without it.
+	if os.FileMode(fileToProcess.File.Mode)&os.ModeSymlink != 0 {
 		err := s.recordNonRegularFile(ctx, fileToProcess)
 		if err != nil {
 			return false, fmt.Errorf("processing file %s: %w", fileToProcess.Path, err)