Check every group member and show the content phase from its start

The content phase now checks every record sharing a size, head and
tail with lstat, including those that already have a content hash, and
reads those without one only when at least two pass. Only a missing
file is passed over silently; any other lstat error is warned about and
counted as skipped. The phase shows a running count while it queries
and checks, then its bar. The README states once when content is empty.
New tests cover these and hard links.

Model: opus-5-5

scan.go

@@ -6,6 +6,7 @@ import (
 	"crypto/sha256"
 	"database/sql"
 	"encoding/hex"
+	"errors"
 	"fmt"
 	"io"
 	"io/fs"
@@ -578,7 +579,7 @@ func (s *scanState) updatePhase(ctx context.Context) error {
 // its empty content, so it is never grouped, and a later scan tries
 // again.
 func (s *scanState) contentPhase(ctx context.Context, workers int) error {
-	toRead, recs, err := contentCandidates(ctx, s.db)
+	toRead, recs, err := s.contentCandidates(ctx)
 	if err != nil {
 		return err
 	}
@@ -602,47 +603,71 @@ func (s *scanState) contentPhase(ctx context.Context, workers int) error {
 	return applyChanges(ctx, s.db, s.batch, nil, nil)
 }
 
-// contentCandidates returns the files the content phase reads, and the
-// records of the files that passed the check, by path. Every record
-// contentCandidatesSQL returns has its file checked with lstat: a file
-// that is gone, is no longer a regular file, or has changed by the
-// walk's rule keeps its record as it is and is not a duplicate. The
-// files of a group that pass are read only if the group still has at
-// least minGroupSize members, counting its records that already have a
-// content hash, so a group whose other members are all stale costs no
-// reads.
-func contentCandidates(ctx context.Context,
-	db *sql.DB,
+// contentCandidates returns the files the content phase reads, and
+// their records by path. Every record contentCandidatesSQL returns has
+// its file checked with lstat, whether or not it already has a content
+// hash: a file that is gone, is no longer a regular file, or has
+// changed by the walk's rule keeps its record as it is and is not a
+// duplicate, and any other lstat error is warned about and counted as
+// skipped. The files of a group that pass and have no content hash are
+// read only if at least minGroupSize of the group's files pass, so a
+// group whose other members are all stale costs no reads. Only the
+// records to be read are kept.
+func (s *scanState) contentCandidates(
+	ctx context.Context,
 ) ([]fileRec, map[string]scanRec, error) {
+	// The query and the checks take real time on a large database;
+	// without a display the scan looks hung before the reads begin.
+	prog := newProgress("content", -1)
+	defer prog.finish()
+
 	var (
 		toRead []fileRec
-		passed []fileRec // the current group's files that passed the check
 		first  scanRec   // the current group's first record
-		hashed int       // the current group's records with a content hash
+		passed int       // the current group's files that passed the check
+		unread []fileRec // those of them without a content hash
 	)
 
 	recs := make(map[string]scanRec)
 
-	// endGroup queues the current group's files that passed the check,
-	// if the group still has at least minGroupSize members.
+	// endGroup queues the current group's files to read if at least
+	// minGroupSize of its files passed, and drops their records if not.
 	endGroup := func() {
-		if len(passed)+hashed >= minGroupSize {
-			toRead = append(toRead, passed...)
+		if passed >= minGroupSize {
+			toRead = append(toRead, unread...)
+		} else {
+			for _, f := range unread {
+				delete(recs, f.path)
+			}
 		}
 
-		passed = nil
+		passed, unread = 0, nil
 	}
 
-	err := loadContentCandidates(ctx, db, func(r scanRec, groupHashed int) {
+	err := loadContentCandidates(ctx, s.db, func(r scanRec, hashed bool) {
+		prog.increment()
+
 		if r.size != first.size || r.head != first.head || r.tail != first.tail {
 			endGroup()
 
-			first, hashed = r, groupHashed
+			first = r
 		}
 
-		f, ok := unchangedFile(r)
-		if ok {
-			passed = append(passed, f)
+		f, ok, err := unchangedFile(r)
+		if err != nil {
+			s.st.skipped++
+
+			prog.warnf("content %s: %v", r.path, err)
+		}
+
+		if !ok {
+			return
+		}
+
+		passed++
+
+		if !hashed {
+			unread = append(unread, f)
 			recs[r.path] = r
 		}
 	})
@@ -657,20 +682,28 @@ func contentCandidates(ctx context.Context,
 
 // unchangedFile lstats the file r names and returns it for reading if
 // it is still the regular file r records: the same size, and an mtime
-// no newer than recorded (the walk's change rule). Otherwise it
-// reports false.
-func unchangedFile(r scanRec) (fileRec, bool) {
+// no newer than recorded (the walk's change rule). A file that is gone
+// or has changed reports false; any other lstat error is returned.
+func unchangedFile(r scanRec) (fileRec, bool, error) {
 	fi, err := os.Lstat(r.path)
-	if err != nil || !fi.Mode().IsRegular() || fi.Size() != r.size ||
+	if errors.Is(err, fs.ErrNotExist) {
+		return fileRec{}, false, nil
+	}
+
+	if err != nil {
+		return fileRec{}, false, err
+	}
+
+	if !fi.Mode().IsRegular() || fi.Size() != r.size ||
 		fi.ModTime().Unix() > r.mtime {
-		return fileRec{}, false
+		return fileRec{}, false, nil
 	}
 
 	dev, ino := inodeOfInfo(fi)
 
 	return fileRec{
 		path: r.path, size: r.size, mtime: r.mtime, dev: dev, ino: ino,
-	}, true
+	}, true, nil
 }
 
 // underAnyRoot reports whether path is any of the roots or lies under