Check every group member and show the content phase from its start
check / check (push) Successful in 57s
check / check (push) Successful in 57s
The content phase now checks every record sharing a size, head and tail with lstat, including those that already have a content hash, and reads those without one only when at least two pass. Only a missing file is passed over silently; any other lstat error is warned about and counted as skipped. The phase shows a running count while it queries and checks, then its bar. The README states once when content is empty. New tests cover these and hard links. Model: opus-5-5
This commit is contained in:
+130
@@ -524,6 +524,58 @@ func TestScanContentStalePartners(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// TestScanContentHashedStalePartners checks that stored matches outside
|
||||
// the operand that already have a content hash are checked like any
|
||||
// other: once one has vanished and the other has changed, a copy of
|
||||
// them scanned in another tree has no match left, so it is not read and
|
||||
// is not reported as their duplicate.
|
||||
func TestScanContentHashedStalePartners(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
db := openTestDB(t)
|
||||
dirA := t.TempDir()
|
||||
stored := []string{
|
||||
sparseFile(t, dirA, "changed", headTailMin),
|
||||
sparseFile(t, dirA, "gone", headTailMin),
|
||||
}
|
||||
|
||||
// The two stored files match, so this scan gives both a content
|
||||
// hash.
|
||||
syncTree(t, db, dirA)
|
||||
|
||||
err := os.Remove(stored[1])
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
future := time.Now().Add(time.Hour)
|
||||
|
||||
err = os.Chtimes(stored[0], future, future)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
b := sparseFile(t, t.TempDir(), "copy", headTailMin)
|
||||
|
||||
st := syncTree(t, db, filepath.Dir(b))
|
||||
if st != (scanStats{added: 1}) {
|
||||
t.Errorf("stats = %+v, want 1 added and nothing skipped", st)
|
||||
}
|
||||
|
||||
recs := dbRecords(t, db)
|
||||
if r := recordByPath(t, recs, b); r.content != "" {
|
||||
t.Errorf("copy: content = %q, want none: its only matches are stale",
|
||||
r.content)
|
||||
}
|
||||
|
||||
// The stored records lie outside the operand and are left as they
|
||||
// are, so they still group with each other, but not with the copy.
|
||||
groups := collectDupeGroups(recs)
|
||||
if len(groups) != 1 || !slices.Equal(groups[0].paths, stored) {
|
||||
t.Errorf("groups = %+v, want only the stored pair %q", groups, stored)
|
||||
}
|
||||
}
|
||||
|
||||
// TestScanContentReadFailure checks that a failed content read is
|
||||
// counted as skipped and leaves the record without a content hash, and
|
||||
// that a later scan tries the read again.
|
||||
@@ -574,6 +626,84 @@ func TestScanContentReadFailure(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// TestScanContentCheckError checks that a stored file the content phase
|
||||
// cannot lstat, for a reason other than its being gone, is counted as
|
||||
// skipped and does not count as a match.
|
||||
func TestScanContentCheckError(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
db := openTestDB(t)
|
||||
sub := filepath.Join(t.TempDir(), "sub")
|
||||
|
||||
err := os.Mkdir(sub, 0o700)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
sparseFileWithoutMatch(t, sub, "a", headTailMin)
|
||||
syncTree(t, db, sub)
|
||||
|
||||
// Without search permission on its directory, the stored file's
|
||||
// lstat fails with permission denied.
|
||||
err = os.Chmod(sub, 0)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
t.Cleanup(func() {
|
||||
//nolint:gosec // removing the directory needs its search bit back
|
||||
_ = os.Chmod(sub, 0o700)
|
||||
})
|
||||
|
||||
b := sparseFile(t, t.TempDir(), "b", headTailMin)
|
||||
|
||||
st := syncTree(t, db, filepath.Dir(b))
|
||||
if st != (scanStats{added: 1, skipped: 1}) {
|
||||
t.Fatalf("stats = %+v, want 1 added 1 skipped", st)
|
||||
}
|
||||
|
||||
if r := recordByPath(t, dbRecords(t, db), b); r.content != "" {
|
||||
t.Errorf("b: content = %q, want none: its only match could not be "+
|
||||
"checked", r.content)
|
||||
}
|
||||
}
|
||||
|
||||
// TestScanContentHardlinks checks that the content phase stores the
|
||||
// content hash of a hard-linked file on every one of its links.
|
||||
func TestScanContentHardlinks(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
dir := t.TempDir()
|
||||
db := openTestDB(t)
|
||||
a := sparseFile(t, dir, "a", headTailMin)
|
||||
b := filepath.Join(dir, "b")
|
||||
|
||||
err := os.Link(a, b)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
c := sparseFile(t, dir, "copy", headTailMin)
|
||||
|
||||
st := syncTree(t, db, dir)
|
||||
if st != (scanStats{added: 3}) {
|
||||
t.Fatalf("stats = %+v, want 3 added", st)
|
||||
}
|
||||
|
||||
recs := dbRecords(t, db)
|
||||
|
||||
want := recordByPath(t, recs, c).content
|
||||
if want == "" {
|
||||
t.Fatal("the copy has no content hash")
|
||||
}
|
||||
|
||||
for _, p := range []string{a, b} {
|
||||
if got := recordByPath(t, recs, p).content; got != want {
|
||||
t.Errorf("%s: content = %q, want %q", p, got, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// collectWalk runs a walk over roots and returns the emitted records
|
||||
// and the number of warning events.
|
||||
func collectWalk(t *testing.T, roots []string, oneFS bool,
|
||||
|
||||
Reference in New Issue
Block a user