Compare commits
2
Commits
ac2f4416fa
..
next
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
f52bbe5d95 | ||
|
|
30dae35c51 |
@@ -407,9 +407,9 @@ operands:
|
|||||||
is what makes a daily rescan cheap.
|
is what makes a daily rescan cheap.
|
||||||
- When such a file's size on disk differs from its recorded `disk_size`, `scan`
|
- When such a file's size on disk differs from its recorded `disk_size`, `scan`
|
||||||
updates `disk_size` alone. It still does not read the file, and counts it as
|
updates `disk_size` alone. It still does not read the file, and counts it as
|
||||||
unchanged. ZFS, for example, counts a new file's blocks only seconds after the
|
unchanged. ZFS, for example, counts a new file's blocks up to seconds after
|
||||||
write. A block is the unit in which a filesystem allocates disk space. A file
|
the write. A block is the unit in which a filesystem allocates disk space. A
|
||||||
scanned within those seconds gets its real `disk_size` at the next scan.
|
file scanned within those seconds gets its real `disk_size` at the next scan.
|
||||||
- `scan` hashes a file again when its size differs or its mtime is newer. The
|
- `scan` hashes a file again when its size differs or its mtime is newer. The
|
||||||
mtime is compared to the nanosecond, as finely as the filesystem records it. A
|
mtime is compared to the nanosecond, as finely as the filesystem records it. A
|
||||||
same-size rewrite counts as a change whenever the filesystem gives it a later
|
same-size rewrite counts as a change whenever the filesystem gives it a later
|
||||||
|
|||||||
@@ -29,12 +29,18 @@
|
|||||||
Newest first. An entry ends with its date and its issue or branch, where those
|
Newest first. An entry ends with its date and its issue or branch, where those
|
||||||
are known.
|
are known.
|
||||||
|
|
||||||
|
- `README.md` and a comment in `scan.go` no longer say ZFS counts a new file's
|
||||||
|
blocks only seconds after the write. The count can land in under a second.
|
||||||
|
Both now say up to seconds after the write, as the comment on `diskSizeOf` in
|
||||||
|
`scan_test.go` does (2026-11-03, https://git.eeqj.de/sneak/sfdupes/issues/152)
|
||||||
|
|
||||||
- The three disk size tests in `scan_test.go` no longer fail at random on ZFS.
|
- The three disk size tests in `scan_test.go` no longer fail at random on ZFS.
|
||||||
`unix.Sync` can return before ZFS has counted a file's blocks, and one ZFS
|
`unix.Sync` can return before ZFS has counted a file's blocks, and one ZFS
|
||||||
commit can take a file through several sizes. The test that needs the size to
|
commit can take a file through several sizes. The test that needs the size to
|
||||||
change waits for it, for at most 30 seconds. Each test reads the size before
|
change waits for it, for at most 30 seconds. Each test reads the size before
|
||||||
and after its scan. It fails only on a `disk_size` outside those two reads
|
and after its scan. The rewrite test rescans until the two reads match, then
|
||||||
(2026-11-02, https://git.eeqj.de/sneak/sfdupes/issues/150)
|
compares exactly. The other two fail only on a `disk_size` outside the two
|
||||||
|
reads (2026-11-02, https://git.eeqj.de/sneak/sfdupes/issues/150)
|
||||||
|
|
||||||
- `TODO.md` rewritten from scratch by the same rules as `README.md`, sentence by
|
- `TODO.md` rewritten from scratch by the same rules as `README.md`, sentence by
|
||||||
sentence. Every completed step is kept, with its date and its issue or branch.
|
sentence. Every completed step is kept, with its date and its issue or branch.
|
||||||
|
|||||||
@@ -554,7 +554,7 @@ func (s *scanState) walkPhase(events <-chan walkEvent, roots []string) {
|
|||||||
|
|
||||||
// The space a file takes on disk can change while its size and
|
// The space a file takes on disk can change while its size and
|
||||||
// mtime stay the same. ZFS, for one, counts a new file's blocks
|
// mtime stay the same. ZFS, for one, counts a new file's blocks
|
||||||
// only seconds after the write.
|
// up to seconds after the write.
|
||||||
if old.diskSize != ev.rec.diskSize {
|
if old.diskSize != ev.rec.diskSize {
|
||||||
s.diskSizes = append(s.diskSizes, ev.rec)
|
s.diskSizes = append(s.diskSizes, ev.rec)
|
||||||
}
|
}
|
||||||
|
|||||||
+35
-14
@@ -50,8 +50,9 @@ func writeFile(t *testing.T, dir, name string, data []byte) string {
|
|||||||
// but a unix.Sync call often takes long enough that the blocks are
|
// but a unix.Sync call often takes long enough that the blocks are
|
||||||
// counted when it returns. A test that needs the value to change waits
|
// counted when it returns. A test that needs the value to change waits
|
||||||
// for the change. A test that compares it with a scan's disk_size reads
|
// for the change. A test that compares it with a scan's disk_size reads
|
||||||
// it before and after the scan, and fails only on a disk_size outside
|
// it before and after the scan. The rewrite test rescans until the two
|
||||||
// those two reads.
|
// reads match, then compares exactly. The other two tests fail only on
|
||||||
|
// a disk_size outside the two reads.
|
||||||
func diskSizeOf(t *testing.T, path string) int64 {
|
func diskSizeOf(t *testing.T, path string) int64 {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
|
|
||||||
@@ -1514,22 +1515,42 @@ func TestSyncScanRewriteUpdatesDiskSize(t *testing.T) {
|
|||||||
unix.Sync()
|
unix.Sync()
|
||||||
}
|
}
|
||||||
|
|
||||||
// ZFS may still be counting, so the size may change during the rescan.
|
// ZFS may still be counting, so the size may change during a rescan.
|
||||||
// A rescan that keeps the old size still fails, because the wait made
|
// Only a rescan with the same size on both sides of it can be compared
|
||||||
// before differ from it.
|
// exactly. A rescan that keeps the old size still fails, because the
|
||||||
before := diskSizeOf(t, p)
|
// wait made before differ from it.
|
||||||
|
for {
|
||||||
|
before := diskSizeOf(t, p)
|
||||||
|
|
||||||
st := syncTree(t, db, dir)
|
st := syncTree(t, db, dir)
|
||||||
|
|
||||||
after := diskSizeOf(t, p)
|
after := diskSizeOf(t, p)
|
||||||
|
|
||||||
if st != (scanStats{walked: 1, updated: 1}) {
|
if st != (scanStats{walked: 1, updated: 1}) {
|
||||||
t.Fatalf("rescan stats = %+v, want 1 updated", st)
|
t.Fatalf("rescan stats = %+v, want 1 updated", st)
|
||||||
}
|
}
|
||||||
|
|
||||||
got := recordByPath(t, dbRecords(t, db), p).diskSize
|
if before == after {
|
||||||
if got < before || got > after {
|
got := recordByPath(t, dbRecords(t, db), p).diskSize
|
||||||
t.Errorf("disk_size = %d, want %d to %d", got, before, after)
|
if got != after {
|
||||||
|
t.Errorf("disk_size = %d, want %d", got, after)
|
||||||
|
}
|
||||||
|
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
if time.Now().After(deadline) {
|
||||||
|
t.Fatalf("the size on disk was still changing 30 seconds after "+
|
||||||
|
"the write into its hole: %d, then %d", before, after)
|
||||||
|
}
|
||||||
|
|
||||||
|
// A later mtime makes the next rescan read the file again.
|
||||||
|
later = later.Add(time.Second)
|
||||||
|
|
||||||
|
err = os.Chtimes(p, later, later)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user