Check every group member and show the content phase from its start
check / check (push) Successful in 57s

The content phase now checks every record sharing a size, head and
tail with lstat, including those that already have a content hash, and
reads those without one only when at least two pass. Only a missing
file is passed over silently; any other lstat error is warned about and
counted as skipped. The phase shows a running count while it queries
and checks, then its bar. The README states once when content is empty.
New tests cover these and hard links.

Model: opus-5-5
This commit is contained in:
2026-09-23 13:09:03 +00:00
parent 89fc9e4595
commit fff4409a24
6 changed files with 243 additions and 81 deletions
+130
View File
@@ -524,6 +524,58 @@ func TestScanContentStalePartners(t *testing.T) {
}
}
// TestScanContentHashedStalePartners checks that stored matches outside
// the operand that already have a content hash are checked like any
// other: once one has vanished and the other has changed, a copy of
// them scanned in another tree has no match left, so it is not read and
// is not reported as their duplicate.
func TestScanContentHashedStalePartners(t *testing.T) {
t.Parallel()
db := openTestDB(t)
dirA := t.TempDir()
stored := []string{
sparseFile(t, dirA, "changed", headTailMin),
sparseFile(t, dirA, "gone", headTailMin),
}
// The two stored files match, so this scan gives both a content
// hash.
syncTree(t, db, dirA)
err := os.Remove(stored[1])
if err != nil {
t.Fatal(err)
}
future := time.Now().Add(time.Hour)
err = os.Chtimes(stored[0], future, future)
if err != nil {
t.Fatal(err)
}
b := sparseFile(t, t.TempDir(), "copy", headTailMin)
st := syncTree(t, db, filepath.Dir(b))
if st != (scanStats{added: 1}) {
t.Errorf("stats = %+v, want 1 added and nothing skipped", st)
}
recs := dbRecords(t, db)
if r := recordByPath(t, recs, b); r.content != "" {
t.Errorf("copy: content = %q, want none: its only matches are stale",
r.content)
}
// The stored records lie outside the operand and are left as they
// are, so they still group with each other, but not with the copy.
groups := collectDupeGroups(recs)
if len(groups) != 1 || !slices.Equal(groups[0].paths, stored) {
t.Errorf("groups = %+v, want only the stored pair %q", groups, stored)
}
}
// TestScanContentReadFailure checks that a failed content read is
// counted as skipped and leaves the record without a content hash, and
// that a later scan tries the read again.
@@ -574,6 +626,84 @@ func TestScanContentReadFailure(t *testing.T) {
}
}
// TestScanContentCheckError checks that a stored file the content phase
// cannot lstat, for a reason other than its being gone, is counted as
// skipped and does not count as a match.
func TestScanContentCheckError(t *testing.T) {
t.Parallel()
db := openTestDB(t)
sub := filepath.Join(t.TempDir(), "sub")
err := os.Mkdir(sub, 0o700)
if err != nil {
t.Fatal(err)
}
sparseFileWithoutMatch(t, sub, "a", headTailMin)
syncTree(t, db, sub)
// Without search permission on its directory, the stored file's
// lstat fails with permission denied.
err = os.Chmod(sub, 0)
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() {
//nolint:gosec // removing the directory needs its search bit back
_ = os.Chmod(sub, 0o700)
})
b := sparseFile(t, t.TempDir(), "b", headTailMin)
st := syncTree(t, db, filepath.Dir(b))
if st != (scanStats{added: 1, skipped: 1}) {
t.Fatalf("stats = %+v, want 1 added 1 skipped", st)
}
if r := recordByPath(t, dbRecords(t, db), b); r.content != "" {
t.Errorf("b: content = %q, want none: its only match could not be "+
"checked", r.content)
}
}
// TestScanContentHardlinks checks that the content phase stores the
// content hash of a hard-linked file on every one of its links.
func TestScanContentHardlinks(t *testing.T) {
t.Parallel()
dir := t.TempDir()
db := openTestDB(t)
a := sparseFile(t, dir, "a", headTailMin)
b := filepath.Join(dir, "b")
err := os.Link(a, b)
if err != nil {
t.Fatal(err)
}
c := sparseFile(t, dir, "copy", headTailMin)
st := syncTree(t, db, dir)
if st != (scanStats{added: 3}) {
t.Fatalf("stats = %+v, want 3 added", st)
}
recs := dbRecords(t, db)
want := recordByPath(t, recs, c).content
if want == "" {
t.Fatal("the copy has no content hash")
}
for _, p := range []string{a, b} {
if got := recordByPath(t, recs, p).content; got != want {
t.Errorf("%s: content = %q, want %q", p, got, want)
}
}
}
// collectWalk runs a walk over roots and returns the emitted records
// and the number of warning events.
func collectWalk(t *testing.T, roots []string, oneFS bool,