sneak/sfdupes
1
0
Fork 0
Code Issues 4 Pull Requests 3 Actions 9 Packages Projects Releases Wiki Activity

Hash every file's head, tail and content in the hash phase (closes #132)
check / check (push) Canceled after 0s
Details

Browse Source 
Every file the walk records now gets three hashes in the hash phase:
head and tail, of its first and last 64 KiB, and content. The content
hash covers the whole file under 100 MiB, and 1 MiB at each gigabyte
from 100 MiB up, so a large file is never read in full. One 100 MiB
constant replaces the 10 MiB and 50 MiB thresholds. The size census,
the records stored without hashes and the content phase are gone, so a
scan has four phases and a rescan reads only new and changed files.
report and trees no longer special-case a missing content hash. README
"Duplicate detection" and "Rationale" are rewritten.

Judgement call: info drops its two hash-count lines, which would now
always equal the record count.

Model: opus-5-5
This commit is contained in:
clawbot
2026-10-25 23:37:09 +00:00
parent f4ddb58ae1
commit 62b9cfc7ae
16 changed files with 530 additions and 1767 deletions
Download Patch File Download Diff File Expand all files Collapse all files
progress_test.go
+37 -46
View File
@@ -23,7 +23,7 @@ const testRedraw = time.Millisecond
const testDeadline = 5 * time.Second
// hashStalledOn is where the hash phase says it waits once it has
// stalled, as readRuns gives it.
// stalled, as hashPhase gives it.
const hashStalledOn = "disk/db"
// testClock is a clock a test moves by hand, for a display that reads
@@ -117,38 +117,38 @@ func TestProgressLine(t *testing.T) {
label: scanPhase("load"), total: -1,
waitingFor: countingRecords,
},
want: "[1/5] load: counting the records, elapsed 10s, eta unknown",
want: "[1/4] load: counting the records, elapsed 10s, eta unknown",
}, {
p: &progress{
label: scanPhase("hash"), unit: scanUnit, total: 100, count: 20,
},
want: "[3/5] hash: [20/100] 20%, 2 files/s, elapsed 10s, eta 40s",
want: "[3/4] hash: [20/100] 20%, 2 files/s, elapsed 10s, eta 40s",
}, {
p: &progress{label: scanPhase("hash"), unit: scanUnit, total: 100},
want: "[3/5] hash: [0/100] 0%, 0 files/s, elapsed 10s, eta unknown",
want: "[3/4] hash: [0/100] 0%, 0 files/s, elapsed 10s, eta unknown",
}, {
p: &progress{label: scanPhase("update"), unit: scanUnit, total: 0},
want: "[4/5] update: [0/0] 100%, 0 files/s, elapsed 10s, eta 0s",
want: "[4/4] update: [0/0] 100%, 0 files/s, elapsed 10s, eta 0s",
}, {
p: &progress{
label: scanPhase("walk"), unit: scanUnit, total: 100,
estimate: true, count: 20,
},
want: "[2/5] walk: [20/~100] 20%, 2 files/s, elapsed 10s, eta 40s",
want: "[2/4] walk: [20/~100] 20%, 2 files/s, elapsed 10s, eta 40s",
}, {
// Past the estimate the walk cannot tell how long it has left.
p: &progress{
label: scanPhase("walk"), unit: scanUnit, total: 100,
estimate: true, count: 120,
},
want: "[2/5] walk: [120/~100] 120%, 12 files/s, elapsed 10s, " +
want: "[2/4] walk: [120/~100] 120%, 12 files/s, elapsed 10s, " +
"eta unknown",
}, {
// A first scan's walk has no estimate.
p: &progress{
label: scanPhase("walk"), unit: scanUnit, total: -1, count: 20,
},
want: "[2/5] walk: 20 files, 2 files/s, elapsed 10s, eta unknown",
want: "[2/4] walk: 20 files, 2 files/s, elapsed 10s, eta unknown",
}, {
// The rate is in what the phase counts.
p: &progress{
@@ -184,9 +184,9 @@ func TestLineFitsRoom(t *testing.T) {
start: ago, rateStart: ago, lastItem: now,
drawn: []drawnCount{{at: ago}},
}
content := &progress{
label: scanPhase("content"), total: -1,
waitingFor: "searching the database", start: ago, rateStart: ago,
load := &progress{
label: scanPhase("load"), total: -1,
waitingFor: countingRecords, start: ago, rateStart: ago,
}
started := now.Add(-50 * time.Second)
@@ -204,27 +204,27 @@ func TestLineFitsRoom(t *testing.T) {
want string
}{{
p: hash, room: 57, percent: true,
want: "[3/5] hash: [20/100] 20%, 2 files/s, elapsed 10s, eta 40s",
want: "[3/4] hash: [20/100] 20%, 2 files/s, elapsed 10s, eta 40s",
}, {
p: hash, room: 56, percent: true,
want: "[3/5] hash: [20/100] 20%, elapsed 10s, eta 40s",
want: "[3/4] hash: [20/100] 20%, elapsed 10s, eta 40s",
}, {
p: hash, room: 45, percent: true,
want: "[3/5] hash: [20/100] 20%, elapsed 10s, eta 40",
want: "[3/4] hash: [20/100] 20%, elapsed 10s, eta 40",
}, {
// Before the library's bar, which shows the percent.
p: hash, room: 57, percent: false,
want: "[3/5] hash: [20/100], 2 files/s, elapsed 10s, eta 40s",
want: "[3/4] hash: [20/100], 2 files/s, elapsed 10s, eta 40s",
}, {
p: content, room: 40, percent: true,
want: "[5/5] content: elapsed 10s, eta unknown",
p: load, room: 40, percent: true,
want: "[1/4] load: elapsed 10s, eta unknown",
}, {
p: stalled, room: 65, percent: true,
want: "[3/5] hash: [20/100] 20%, stalled disk/db, elapsed 50s, " +
want: "[3/4] hash: [20/100] 20%, stalled disk/db, elapsed 50s, " +
"eta 3m20s",
}, {
p: stalled, room: 64, percent: true,
want: "[3/5] hash: [20/100] 20%, elapsed 50s, eta 3m20s",
want: "[3/4] hash: [20/100] 20%, elapsed 50s, eta 3m20s",
}}
for _, tc := range cases {
@@ -255,14 +255,14 @@ func TestSetTotalStartsRate(t *testing.T) {
clock.Add(5 * time.Second)
want := "[1/5] load: [0/100] 0%, 0 files/s, elapsed 25s, eta unknown"
want := "[1/4] load: [0/100] 0%, 0 files/s, elapsed 25s, eta unknown"
if got := p.line(clock.Now(), math.MaxInt, true); got != want {
t.Errorf("line = %q, want %q", got, want)
}
clock.Add(5 * time.Second)
want = "[1/5] load: [0/100] 0%, stalled db, elapsed 30s, eta unknown"
want = "[1/4] load: [0/100] 0%, stalled db, elapsed 30s, eta unknown"
if got := p.line(clock.Now(), math.MaxInt, true); got != want {
t.Errorf("line = %q, want %q", got, want)
}
@@ -272,7 +272,7 @@ func TestSetTotalStartsRate(t *testing.T) {
p.increment()
}
want = "[1/5] load: [20/100] 20%, 2 files/s, elapsed 30s, eta 40s"
want = "[1/4] load: [20/100] 20%, 2 files/s, elapsed 30s, eta 40s"
if got := p.line(clock.Now(), math.MaxInt, true); got != want {
t.Errorf("line = %q, want %q", got, want)
}
@@ -316,9 +316,9 @@ func TestPlainProgressPrintsAtStartAndEnd(t *testing.T) {
lines := strings.Split(strings.TrimSuffix(stderr(), "\n"), "\n")
want := "[3/5] hash: [0/1000] 0%, 0 files/s, elapsed 0s, eta unknown"
want := "[3/4] hash: [0/1000] 0%, 0 files/s, elapsed 0s, eta unknown"
if len(lines) != 2 || lines[0] != want ||
!strings.HasPrefix(lines[1], "[3/5] hash: [1000/1000] 100%, ") {
!strings.HasPrefix(lines[1], "[3/4] hash: [1000/1000] 100%, ") {
t.Errorf("stderr lines = %q, want %q, then the line at 1000",
lines, want)
}
@@ -336,7 +336,7 @@ func TestPlainProgressPrintsWhileWaiting(t *testing.T) {
}, nil, testRedraw, time.Now)
awaitOutput(t, stderr,
"[1/5] load: counting the records, elapsed 0s, eta unknown\n", 3)
"[1/4] load: counting the records, elapsed 0s, eta unknown\n", 3)
p.finish()
}
@@ -366,15 +366,15 @@ func TestPlainProgressStalledProducer(t *testing.T) {
clock.Add(5 * time.Second)
awaitOutput(t, stderr,
"[3/5] hash: [20/100] 20%, 4 files/s, elapsed 5s, eta 20s\n", 3)
"[3/4] hash: [20/100] 20%, 4 files/s, elapsed 5s, eta 20s\n", 3)
clock.Add(time.Second)
awaitOutput(t, stderr,
"[3/5] hash: [20/100] 20%, 0 files/s, elapsed 6s, eta 24s\n", 3)
"[3/4] hash: [20/100] 20%, 0 files/s, elapsed 6s, eta 24s\n", 3)
clock.Add(4 * time.Second)
awaitOutput(t, stderr,
"[3/5] hash: [20/100] 20%, stalled disk/db, elapsed 10s, eta 40s\n",
"[3/4] hash: [20/100] 20%, stalled disk/db, elapsed 10s, eta 40s\n",
3)
}
@@ -410,10 +410,10 @@ func TestPlainProgressRateAfterStall(t *testing.T) {
p.finish()
want := []string{
"[3/5] hash: [0/1000] 0%, 0 files/s, elapsed 0s, eta unknown",
"[3/5] hash: [0/1000] 0%, 0 files/s, elapsed 5s, eta unknown",
"[3/5] hash: [0/1000] 0%, stalled disk/db, elapsed 10s, eta unknown",
"[3/5] hash: [100/1000] 10%, 20 files/s, elapsed 15s, eta 2m15s",
"[3/4] hash: [0/1000] 0%, 0 files/s, elapsed 0s, eta unknown",
"[3/4] hash: [0/1000] 0%, 0 files/s, elapsed 5s, eta unknown",
"[3/4] hash: [0/1000] 0%, stalled disk/db, elapsed 10s, eta unknown",
"[3/4] hash: [100/1000] 10%, 20 files/s, elapsed 15s, eta 2m15s",
}
lines := strings.Split(strings.TrimSuffix(stderr(), "\n"), "\n")
@@ -457,8 +457,8 @@ func TestProgressRateAfterStallRedrawn(t *testing.T) {
redraw(rateWindow + redrawInterval)
p.finish()
want := "[3/5] hash: [50/1000] 5%, 10 files/s, elapsed 15s, eta 4m45s\n" +
"[3/5] hash: [50/1000] 5%, 0 files/s, elapsed 15s, eta 4m47s\n"
want := "[3/4] hash: [50/1000] 5%, 10 files/s, elapsed 15s, eta 4m45s\n" +
"[3/4] hash: [50/1000] 5%, 0 files/s, elapsed 15s, eta 4m47s\n"
if !strings.Contains(stderr(), want) {
t.Errorf("stderr = %q, want the lines %q", stderr(), want)
}
@@ -570,19 +570,13 @@ func TestTerminalLinesKeepETA(t *testing.T) {
label: scanPhase("walk"), unit: scanUnit, total: 1200000,
estimate: true,
},
want: "[2/5] walk: [1234567/~1200000] 103%, elapsed 2h30m0s, " +
want: "[2/4] walk: [1234567/~1200000] 103%, elapsed 2h30m0s, " +
"eta unknown",
}, {
params: progressParams{
label: scanPhase("hash"), unit: scanUnit, total: 9876543,
},
want: "[3/5] hash: [1234567/9876543] 12%, elapsed 2h30m0s, " +
"eta 17h30m0s",
}, {
params: progressParams{
label: scanPhase("content"), unit: scanUnit, total: 9876543,
},
want: "[5/5] content: [1234567/9876543] 12%, elapsed 2h30m0s, " +
want: "[3/4] hash: [1234567/9876543] 12%, elapsed 2h30m0s, " +
"eta 17h30m0s",
}, {
params: progressParams{
@@ -643,9 +637,6 @@ func TestTerminalStalledLines(t *testing.T) {
{label: scanPhase("walk"), estimate: true, stalledOn: "disk"},
{label: scanPhase("hash"), stalledOn: hashStalledOn},
{label: scanPhase("update"), stalledOn: "db"},
// The content phase's checks and reads, then its last write.
{label: scanPhase("content"), stalledOn: "disk/db"},
{label: scanPhase("content"), stalledOn: "db"},
{label: treesPhase("read"), stalledOn: "db"},
{label: treesPhase("group"), stalledOn: "cpu"},
{label: treesPhase("write"), stalledOn: "out"},
@@ -693,7 +684,7 @@ func TestTerminalLinesFitWidth(t *testing.T) {
for _, width := range []int{120, 100, 80, 72, 60, 50, 40} {
for _, total := range []int64{9876543, 20000} {
params := progressParams{
label: scanPhase("content"), unit: scanUnit, total: total,
label: scanPhase("update"), unit: scanUnit, total: total,
}
out := drawLate(stderr, params, width, total/8)