sneak/sfdupes
1
0
Fork 0
Code Issues 4 Pull Requests 3 Actions 5 Packages Projects Releases Wiki Activity

Store each hash as its raw 32 bytes in a BLOB column (closes #134)
check / check (push) Canceled after 0s
Details

Browse Source 
The head, tail and content columns of the files table are BLOB NOT
NULL and hold the raw 32-byte SHA-256, half the size of the hex text,
in the table and in the files_signature index. The hash phase, the
records, report and trees carry each hash as a [sha256.Size]byte and
compare it with ==; nothing is hex-encoded. Reading a record whose
hash column is not 32 bytes is an error naming its path.

In the tests, testHash is a stand-in hash: a string's bytes padded
with zeros, so fixtures keep the sort order their strings give.

Judgement call: a hash of the wrong length is refused on reading,
not copied silently.

Model: opus-5-5
This commit is contained in:
clawbot
2026-10-26 01:14:23 +00:00
parent f7a746f912
commit 894603059b
12 changed files with 310 additions and 157 deletions
Download Patch File Download Diff File Expand all files Collapse all files
report_test.go
+62 -38
View File
@@ -2,6 +2,7 @@ package main
import (
"bytes"
"crypto/sha256"
"database/sql"
"errors"
"fmt"
@@ -18,14 +19,27 @@ import (
// awkwardDir is a directory name holding every byte the reports escape.
const awkwardDir = "/d/\tone\ntwo\rthree\\four"
// testHash returns a stand-in hash for a test record: the bytes of s,
// then zeros. Stand-ins sort as their strings do, so a test can choose
// the order in which report reads its records.
func testHash(s string) [sha256.Size]byte {
var h [sha256.Size]byte
copy(h[:], s)
return h
}
// awkwardPairRecs is a duplicate pair in sibling directories /d/A and
// awkwardDir. A raw tab sorts before "A" but its escaped form `\t`
// sorts after it, so awkwardDir coming first shows that sorting uses
// the raw path.
func awkwardPairRecs() []scanRec {
h, tl, c := testHash("h"), testHash("t"), testHash("c")
return []scanRec{
{size: 5, head: "h", tail: "t", content: "c", path: "/d/A/f"},
{size: 5, head: "h", tail: "t", content: "c", path: awkwardDir + "/f"},
{size: 5, head: h, tail: tl, content: c, path: "/d/A/f"},
{size: 5, head: h, tail: tl, content: c, path: awkwardDir + "/f"},
}
}
@@ -143,8 +157,8 @@ func TestReportStdoutFailsWhileReading(t *testing.T) {
recs := make([]scanRec, 2*ioBufSize/len(dir))
for i := range recs {
recs[i] = scanRec{
size: int64(i/2) + 1, head: "h", tail: "t", content: "c",
path: fmt.Sprintf("%s/%d", dir, i),
size: int64(i/2) + 1, head: testHash("h"), tail: testHash("t"),
content: testHash("c"), path: fmt.Sprintf("%s/%d", dir, i),
}
}
@@ -277,12 +291,14 @@ func TestReadReportOnTerminal(t *testing.T) {
func TestRunReportsIgnoreInsertionOrder(t *testing.T) {
// README §Constraints: identical database contents give identical
// output, whatever order the records were inserted in.
a, b := testHash("a"), testHash("b")
recs := append(smokeTreeRecs(), awkwardPairRecs()...)
recs = append(recs,
scanRec{size: 50, head: "b", tail: "b", content: "b", path: "/y/2"},
scanRec{size: 50, head: "b", tail: "b", content: "b", path: "/y/1"},
scanRec{size: 50, head: "a", tail: "a", content: "a", path: "/x/2"},
scanRec{size: 50, head: "a", tail: "a", content: "a", path: "/x/1"},
scanRec{size: 50, head: b, tail: b, content: b, path: "/y/2"},
scanRec{size: 50, head: b, tail: b, content: b, path: "/y/1"},
scanRec{size: 50, head: a, tail: a, content: a, path: "/x/2"},
scanRec{size: 50, head: a, tail: a, content: a, path: "/x/1"},
)
reversed := slices.Clone(recs)
@@ -385,16 +401,19 @@ func TestWarnfEscapes(t *testing.T) {
func TestDupeGroups(t *testing.T) {
t.Parallel()
h, tl, c := testHash("h"), testHash("t"), testHash("c")
bigH, bigT, bigC := testHash("H"), testHash("T"), testHash("C")
u := testHash("u")
recs := []scanRec{
{size: 100, head: "h", tail: "t", content: "c", path: "/z/b"},
{size: 100, head: "h", tail: "t", content: "c", path: "/z/a"},
{size: 100, head: "h", tail: "t", content: "c", path: "/z/c"},
{size: 4000, head: "H", tail: "T", content: "C", path: "/big/2"},
{size: 4000, head: "H", tail: "T", content: "C", path: "/big/1"},
{size: 100, head: h, tail: tl, content: c, path: "/z/b"},
{size: 100, head: h, tail: tl, content: c, path: "/z/a"},
{size: 100, head: h, tail: tl, content: c, path: "/z/c"},
{size: 4000, head: bigH, tail: bigT, content: bigC, path: "/big/2"},
{size: 4000, head: bigH, tail: bigT, content: bigC, path: "/big/1"},
// Same size as the /z group but a different head hash.
{size: 100, head: "other", tail: "t", content: "c", path: "/z/d"},
{size: 100, head: testHash("other"), tail: tl, content: c, path: "/z/d"},
// A singleton signature must not form a group.
{size: 7, head: "u", tail: "u", content: "u", path: "/lonely"},
{size: 7, head: u, tail: u, content: u, path: "/lonely"},
}
groups := dupeGroupsOf(t, recs)
@@ -420,10 +439,11 @@ func TestDupeGroupsContentSeparates(t *testing.T) {
// Same size, head, and tail: /b differs from the others only in its
// content hash, so it joins no group, while /a and /c group.
h, tl := testHash("h"), testHash("t")
recs := []scanRec{
{size: 100, head: "h", tail: "t", content: "c1", path: "/a"},
{size: 100, head: "h", tail: "t", content: "c2", path: "/b"},
{size: 100, head: "h", tail: "t", content: "c1", path: "/c"},
{size: 100, head: h, tail: tl, content: testHash("c1"), path: "/a"},
{size: 100, head: h, tail: tl, content: testHash("c2"), path: "/b"},
{size: 100, head: h, tail: tl, content: testHash("c1"), path: "/c"},
}
groups := dupeGroupsOf(t, recs)
@@ -442,14 +462,15 @@ func TestDupeGroupsMtimeExcluded(t *testing.T) {
// mtime is informational only; records differing only in mtime
// still group together.
h, tl, c := testHash("h"), testHash("t"), testHash("c")
recs := []scanRec{
{
size: 9, mtime: time.Unix(100, 0), head: "h", tail: "t",
content: "c", path: "/m/1",
size: 9, mtime: time.Unix(100, 0), head: h, tail: tl,
content: c, path: "/m/1",
},
{
size: 9, mtime: time.Unix(200, 0), head: "h", tail: "t",
content: "c", path: "/m/2",
size: 9, mtime: time.Unix(200, 0), head: h, tail: tl,
content: c, path: "/m/2",
},
}
@@ -464,11 +485,12 @@ func TestDupeGroupsTieBreak(t *testing.T) {
// The hashes sort opposite to the first paths, so ordering the
// groups by hash instead of by first path fails this test.
a, b := testHash("a"), testHash("b")
recs := []scanRec{
{size: 50, head: "a", tail: "a", content: "a", path: "/beta/2"},
{size: 50, head: "a", tail: "a", content: "a", path: "/beta/1"},
{size: 50, head: "b", tail: "b", content: "b", path: "/alpha/2"},
{size: 50, head: "b", tail: "b", content: "b", path: "/alpha/1"},
{size: 50, head: a, tail: a, content: a, path: "/beta/2"},
{size: 50, head: a, tail: a, content: a, path: "/beta/1"},
{size: 50, head: b, tail: b, content: b, path: "/alpha/2"},
{size: 50, head: b, tail: b, content: b, path: "/alpha/1"},
}
groups := dupeGroupsOf(t, recs)
@@ -490,15 +512,16 @@ func TestDupeGroupsOrderedByFirstPath(t *testing.T) {
// /z, /k, which is neither the order of their first paths nor its
// reverse, and must write them ordered by first path. The group of
// size 7 comes after them all, though its first path sorts first.
a, b, c, d := testHash("a"), testHash("b"), testHash("c"), testHash("d")
recs := []scanRec{
{size: 50, head: "a", tail: "a", content: "a", path: "/k/2"},
{size: 50, head: "a", tail: "a", content: "a", path: "/k/1"},
{size: 50, head: "b", tail: "b", content: "b", path: "/z/1"},
{size: 50, head: "b", tail: "b", content: "b", path: "/z/2"},
{size: 50, head: "c", tail: "c", content: "c", path: "/a/2"},
{size: 50, head: "c", tail: "c", content: "c", path: "/a/1"},
{size: 7, head: "d", tail: "d", content: "d", path: "/0/2"},
{size: 7, head: "d", tail: "d", content: "d", path: "/0/1"},
{size: 50, head: a, tail: a, content: a, path: "/k/2"},
{size: 50, head: a, tail: a, content: a, path: "/k/1"},
{size: 50, head: b, tail: b, content: b, path: "/z/1"},
{size: 50, head: b, tail: b, content: b, path: "/z/2"},
{size: 50, head: c, tail: c, content: c, path: "/a/2"},
{size: 50, head: c, tail: c, content: c, path: "/a/1"},
{size: 7, head: d, tail: d, content: d, path: "/0/2"},
{size: 7, head: d, tail: d, content: d, path: "/0/1"},
}
groups := dupeGroupsOf(t, recs)
@@ -517,11 +540,12 @@ func TestDupeGroupsOrderedByFirstPath(t *testing.T) {
func TestDupeGroupsDeterministic(t *testing.T) {
t.Parallel()
a, b := testHash("a"), testHash("b")
recs := []scanRec{
{size: 1, head: "a", tail: "a", content: "a", path: "/p/1"},
{size: 1, head: "a", tail: "a", content: "a", path: "/p/2"},
{size: 2, head: "b", tail: "b", content: "b", path: "/q/1"},
{size: 2, head: "b", tail: "b", content: "b", path: "/q/2"},
{size: 1, head: a, tail: a, content: a, path: "/p/1"},
{size: 1, head: a, tail: a, content: a, path: "/p/2"},
{size: 2, head: b, tail: b, content: b, path: "/q/1"},
{size: 2, head: b, tail: b, content: b, path: "/q/2"},
}
forward := dupeGroupsOf(t, recs)