From 68881f04f212c45004d86d2dc09b67f1304817a1 Mon Sep 17 00:00:00 2001 From: clawbot <35+clawbot@noreply.example.org> Date: Mon, 26 Oct 2026 03:15:38 +0100 Subject: [PATCH] Store each hash as its raw 32 bytes in a BLOB column (closes #134) The head, tail and content columns of the files table are BLOB NOT NULL and hold the raw 32-byte SHA-256, half the size of the hex text, in the table and in the files_signature index. The hash phase, the records, report and trees carry each hash as a [sha256.Size]byte and compare it with ==; nothing is hex-encoded. Reading a record whose hash column is not 32 bytes is an error naming its path. In the tests, testHash is a stand-in hash: a string's bytes padded with zeros, so fixtures keep the sort order their strings give. Judgement call: a hash of the wrong length is refused on reading, not copied silently. Model: opus-5-5 --- README.md | 6 +-- TODO.md | 6 +++ cancel_test.go | 9 ++++- db.go | 59 ++++++++++++++++++++++------- db_test.go | 79 ++++++++++++++++++++++++++++++++++---- info_test.go | 11 ++++-- report.go | 7 ++-- report_test.go | 100 ++++++++++++++++++++++++++++++------------------- scan.go | 71 ++++++++++++++++++----------------- scan_test.go | 99 ++++++++++++++++++++++++++++++++++-------------- trees.go | 6 ++- trees_test.go | 39 +++++++++---------- 12 files changed, 335 insertions(+), 157 deletions(-) diff --git a/README.md b/README.md index e7b3b9c..53af2c3 100644 --- a/README.md +++ b/README.md @@ -388,9 +388,9 @@ All four subcommands operate on a single SQLite database file: size INTEGER NOT NULL, -- bytes, from lstat mtime INTEGER NOT NULL, -- whole Unix seconds of the mtime, from lstat mtime_nsec INTEGER NOT NULL, -- nanoseconds within that second, 0 to 999999999 - head TEXT NOT NULL, -- lowercase-hex SHA-256 of the first 64 KiB - tail TEXT NOT NULL, -- lowercase-hex SHA-256 of the last 64 KiB - content TEXT NOT NULL -- lowercase-hex SHA-256; whole file under 100 MiB, else samples + head BLOB NOT NULL, -- raw 32-byte SHA-256 of the first 64 KiB + tail BLOB NOT NULL, -- raw 32-byte SHA-256 of the last 64 KiB + content BLOB NOT NULL -- raw 32-byte SHA-256; whole file under 100 MiB, else samples ) WITHOUT ROWID; CREATE INDEX files_signature ON files (size, head, tail, content); ``` diff --git a/TODO.md b/TODO.md index eb1d682..9b2ead0 100644 --- a/TODO.md +++ b/TODO.md @@ -29,6 +29,12 @@ # Completed Steps +- the `head`, `tail` and `content` columns are BLOBs, each holding the raw 32 + bytes of a SHA-256, half the size of the hex text they held. The hashes stay + raw from the hash phase through the database to `report` and `trees`. Reading + a record whose hash is not 32 bytes is an error (2026-10-26, + https://git.eeqj.de/sneak/sfdupes/issues/134) + - the hash phase gives every file the walk records three hashes: `head` and `tail`, the SHA-256 of its first and last 64 KiB, and `content`, the SHA-256 of the whole file under 100 MiB and of 1 MiB at each gigabyte from 100 MiB up. diff --git a/cancel_test.go b/cancel_test.go index a202dda..577c54d 100644 --- a/cancel_test.go +++ b/cancel_test.go @@ -2,6 +2,7 @@ package main import ( "context" + "crypto/sha256" "database/sql" "errors" "fmt" @@ -762,7 +763,9 @@ func TestHashWorkerDropsQueuedRuns(t *testing.T) { var hashed atomic.Bool - hash := func(path string, size int64) (string, string, string, error) { + hash := func(path string, size int64) ( + [sha256.Size]byte, [sha256.Size]byte, [sha256.Size]byte, error, + ) { hashed.Store(true) return hashSignature(path, size) @@ -802,7 +805,9 @@ func TestHashWorkerAbandonsBlockedSend(t *testing.T) { // The scan is cancelled while the worker hashes, so the worker has // already passed the check that drops queued runs. - hash := func(path string, size int64) (string, string, string, error) { + hash := func(path string, size int64) ( + [sha256.Size]byte, [sha256.Size]byte, [sha256.Size]byte, error, + ) { cancel() return hashSignature(path, size) diff --git a/db.go b/db.go index 0d180d8..c7835b4 100644 --- a/db.go +++ b/db.go @@ -2,6 +2,7 @@ package main import ( "context" + "crypto/sha256" "database/sql" "errors" "fmt" @@ -48,16 +49,17 @@ const lockFilePerm = 0o600 // createTableSQL is the schema applied to a fresh database. Paths are // BLOBs because Unix paths are raw bytes, not guaranteed UTF-8. mtime // holds whole Unix seconds and mtime_nsec the nanoseconds within that -// second. +// second. head, tail and content each hold the raw 32 bytes of a +// SHA-256. const createTableSQL = ` CREATE TABLE files ( path BLOB PRIMARY KEY, size INTEGER NOT NULL, mtime INTEGER NOT NULL, mtime_nsec INTEGER NOT NULL, - head TEXT NOT NULL, - tail TEXT NOT NULL, - content TEXT NOT NULL + head BLOB NOT NULL, + tail BLOB NOT NULL, + content BLOB NOT NULL ) WITHOUT ROWID ` @@ -92,6 +94,10 @@ var errSchemaVersion = errors.New("unsupported database schema version") // database. var errScanRunning = errors.New("another scan is running") +// errHashSize reports a record whose head, tail or content is not the +// 32 bytes of a SHA-256. +var errHashSize = errors.New("hash is not 32 bytes") + // databasePath resolves the database location: SFDUPES_DATABASE when // set and non-empty, the compiled-in default otherwise. It returns the // path, with the working directory, its symlinks resolved, put in front @@ -492,19 +498,24 @@ func loadFileRows(ctx context.Context, db *sql.DB, total func(n int64), for rows.Next() { var ( - path []byte - sec, nsec int64 - r scanRec + path, head, tail, content []byte + sec, nsec int64 + r scanRec ) - err = rows.Scan(&path, &r.size, &sec, &nsec, &r.head, &r.tail, - &r.content) + err = rows.Scan(&path, &r.size, &sec, &nsec, &head, &tail, &content) if err != nil { return fmt.Errorf("read record: %w", err) } r.path = string(path) r.mtime = time.Unix(sec, nsec) + + err = r.setHashes(head, tail, content) + if err != nil { + return fmt.Errorf("read record %s: %w", escapePath(r.path), err) + } + fn(r) } @@ -561,17 +572,22 @@ func loadSignatureRows(ctx context.Context, db *sql.DB, for rows.Next() { var ( - path []byte - r scanRec + path, head, tail, content []byte + r scanRec ) - err = rows.Scan(&r.size, &r.head, &r.tail, &r.content, &path) + err = rows.Scan(&r.size, &head, &tail, &content, &path) if err != nil { return fmt.Errorf("read record: %w", err) } r.path = string(path) + err = r.setHashes(head, tail, content) + if err != nil { + return fmt.Errorf("read record %s: %w", escapePath(r.path), err) + } + err = fn(r) if err != nil { return err @@ -586,6 +602,22 @@ func loadSignatureRows(ctx context.Context, db *sql.DB, return nil } +// setHashes sets r's head, tail and content to the values read from +// those columns. Each must be the raw 32 bytes of a SHA-256. +func (r *scanRec) setHashes(head, tail, content []byte) error { + for _, b := range [][]byte{head, tail, content} { + if len(b) != sha256.Size { + return fmt.Errorf("%w: found %d bytes", errHashSize, len(b)) + } + } + + r.head = [sha256.Size]byte(head) + r.tail = [sha256.Size]byte(tail) + r.content = [sha256.Size]byte(content) + + return nil +} + // countRecords returns the number of records in the database. func countRecords(ctx context.Context, db *sql.DB) (int64, error) { var n int64 @@ -756,7 +788,8 @@ func execUpserts(ctx context.Context, tx *sql.Tx, upserts []scanRec, for _, r := range upserts { _, err = st.ExecContext(ctx, []byte(r.path), r.size, - r.mtime.Unix(), r.mtime.Nanosecond(), r.head, r.tail, r.content) + r.mtime.Unix(), r.mtime.Nanosecond(), + r.head[:], r.tail[:], r.content[:]) if err != nil { return fmt.Errorf("upsert %s: %w", r.path, err) } diff --git a/db_test.go b/db_test.go index 55e2c01..9268048 100644 --- a/db_test.go +++ b/db_test.go @@ -2,6 +2,7 @@ package main import ( "context" + "crypto/sha256" "database/sql" "errors" "fmt" @@ -473,6 +474,67 @@ func TestLoadRowsCountAgreeDuringScan(t *testing.T) { } } +// TestLoadRowsRefuseWrongSizeHash stores a record whose head, tail or +// content is not the 32 bytes of a SHA-256: either one byte, or the 64 +// bytes of a SHA-256 written as hex text. loadFileRows and +// loadSignatureRows must each fail on it with an error naming its path. +func TestLoadRowsRefuseWrongSizeHash(t *testing.T) { + t.Parallel() + + hexText := strings.Repeat("ab", sha256.Size) + + cases := []struct { + name string + update string + value any + }{ + {"one-byte head", "UPDATE files SET head = ?", []byte{0}}, + {"hex-text head", "UPDATE files SET head = ?", hexText}, + {"one-byte tail", "UPDATE files SET tail = ?", []byte{0}}, + {"hex-text tail", "UPDATE files SET tail = ?", hexText}, + {"one-byte content", "UPDATE files SET content = ?", []byte{0}}, + {"hex-text content", "UPDATE files SET content = ?", hexText}, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + t.Parallel() + + db := openTestDB(t) + + err := applyChanges(t.Context(), db, + []scanRec{{path: "/a/f"}}, nil, nil) + if err != nil { + t.Fatal(err) + } + + _, err = db.ExecContext(t.Context(), c.update, c.value) + if err != nil { + t.Fatal(err) + } + + loads := map[string]func() error{ + "loadFileRows": func() error { + return loadFileRows(t.Context(), db, func(int64) {}, + func(scanRec) {}) + }, + "loadSignatureRows": func() error { + return loadSignatureRows(t.Context(), db, func(int64) {}, + func(scanRec) error { return nil }) + }, + } + + for name, load := range loads { + err := load() + if !errors.Is(err, errHashSize) || + !strings.Contains(err.Error(), "/a/f") { + t.Errorf("%s: err = %v, want %v naming /a/f", + name, err, errHashSize) + } + } + }) + } +} + // TestSignatureRowsNeedNoSort checks that SQLite reads signatureRowsSQL // from the files_signature index alone, with nothing to sort: the // first record comes at once, so report's read phase counts each @@ -527,12 +589,13 @@ func TestApplyChangesRoundTrip(t *testing.T) { // written. recs := []scanRec{ { - size: 2, mtime: time.Unix(20, 999_999_999), head: "h2", tail: "t2", - content: "c2", path: "/a/tab\tnew\nline", + size: 2, mtime: time.Unix(20, 999_999_999), head: testHash("h2"), + tail: testHash("t2"), content: testHash("c2"), + path: "/a/tab\tnew\nline", }, { - size: 1, mtime: time.Unix(10, 0), head: "h1", tail: "t1", - content: "c1", path: "/a/x", + size: 1, mtime: time.Unix(10, 0), head: testHash("h1"), + tail: testHash("t1"), content: testHash("c1"), path: "/a/x", }, } @@ -553,8 +616,8 @@ func TestApplyChangesRoundTrip(t *testing.T) { // An upsert for an existing path updates in place; a delete // removes exactly its path. upd := scanRec{ - size: 3, mtime: time.Unix(30, 0), head: "h3", tail: "t3", content: "c3", - path: "/a/x", + size: 3, mtime: time.Unix(30, 0), head: testHash("h3"), + tail: testHash("t3"), content: testHash("c3"), path: "/a/x", } err = applyChanges(t.Context(), db, []scanRec{upd}, @@ -581,8 +644,8 @@ func TestApplyChangesBatching(t *testing.T) { recs := make([]scanRec, 0, n) for i := range n { recs = append(recs, scanRec{ - size: int64(i), mtime: time.Unix(1, 0), head: "h", tail: "t", - path: fmt.Sprintf("/batch/%07d", i), + size: int64(i), mtime: time.Unix(1, 0), head: testHash("h"), + tail: testHash("t"), path: fmt.Sprintf("/batch/%07d", i), }) } diff --git a/info_test.go b/info_test.go index cff5420..c927f41 100644 --- a/info_test.go +++ b/info_test.go @@ -23,11 +23,14 @@ const infoCountLine = "[2/2] count: counting the records, elapsed 0s, " + // infoRecs are four records: a duplicate pair, a file of 20 MiB and // another small file. func infoRecs() []scanRec { + h, tl, c, u := testHash("h"), testHash("t"), testHash("c"), testHash("u") + bigH, bigT, bigC := testHash("bh"), testHash("bt"), testHash("bc") + return []scanRec{ - {size: 5, head: "h", tail: "t", content: "c", path: "/d/a"}, - {size: 5, head: "h", tail: "t", content: "c", path: "/d/b"}, - {size: 20 << 20, head: "bh", tail: "bt", content: "bc", path: "/d/big"}, - {size: 7, head: "u", tail: "u", content: "u", path: "/d/unique"}, + {size: 5, head: h, tail: tl, content: c, path: "/d/a"}, + {size: 5, head: h, tail: tl, content: c, path: "/d/b"}, + {size: 20 << 20, head: bigH, tail: bigT, content: bigC, path: "/d/big"}, + {size: 7, head: u, tail: u, content: u, path: "/d/unique"}, } } diff --git a/report.go b/report.go index 81a5a9f..5db1217 100644 --- a/report.go +++ b/report.go @@ -2,6 +2,7 @@ package main import ( "context" + "crypto/sha256" "database/sql" "fmt" "io" @@ -24,9 +25,9 @@ const minGroupSize = 2 type scanRec struct { size int64 mtime time.Time - head string - tail string - content string + head [sha256.Size]byte + tail [sha256.Size]byte + content [sha256.Size]byte path string } diff --git a/report_test.go b/report_test.go index b9b7d35..df6d50f 100644 --- a/report_test.go +++ b/report_test.go @@ -2,6 +2,7 @@ package main import ( "bytes" + "crypto/sha256" "database/sql" "errors" "fmt" @@ -18,14 +19,27 @@ import ( // awkwardDir is a directory name holding every byte the reports escape. const awkwardDir = "/d/\tone\ntwo\rthree\\four" +// testHash returns a stand-in hash for a test record: the bytes of s, +// then zeros. Stand-ins sort as their strings do, so a test can choose +// the order in which report reads its records. +func testHash(s string) [sha256.Size]byte { + var h [sha256.Size]byte + + copy(h[:], s) + + return h +} + // awkwardPairRecs is a duplicate pair in sibling directories /d/A and // awkwardDir. A raw tab sorts before "A" but its escaped form `\t` // sorts after it, so awkwardDir coming first shows that sorting uses // the raw path. func awkwardPairRecs() []scanRec { + h, tl, c := testHash("h"), testHash("t"), testHash("c") + return []scanRec{ - {size: 5, head: "h", tail: "t", content: "c", path: "/d/A/f"}, - {size: 5, head: "h", tail: "t", content: "c", path: awkwardDir + "/f"}, + {size: 5, head: h, tail: tl, content: c, path: "/d/A/f"}, + {size: 5, head: h, tail: tl, content: c, path: awkwardDir + "/f"}, } } @@ -143,8 +157,8 @@ func TestReportStdoutFailsWhileReading(t *testing.T) { recs := make([]scanRec, 2*ioBufSize/len(dir)) for i := range recs { recs[i] = scanRec{ - size: int64(i/2) + 1, head: "h", tail: "t", content: "c", - path: fmt.Sprintf("%s/%d", dir, i), + size: int64(i/2) + 1, head: testHash("h"), tail: testHash("t"), + content: testHash("c"), path: fmt.Sprintf("%s/%d", dir, i), } } @@ -277,12 +291,14 @@ func TestReadReportOnTerminal(t *testing.T) { func TestRunReportsIgnoreInsertionOrder(t *testing.T) { // README §Constraints: identical database contents give identical // output, whatever order the records were inserted in. + a, b := testHash("a"), testHash("b") + recs := append(smokeTreeRecs(), awkwardPairRecs()...) recs = append(recs, - scanRec{size: 50, head: "b", tail: "b", content: "b", path: "/y/2"}, - scanRec{size: 50, head: "b", tail: "b", content: "b", path: "/y/1"}, - scanRec{size: 50, head: "a", tail: "a", content: "a", path: "/x/2"}, - scanRec{size: 50, head: "a", tail: "a", content: "a", path: "/x/1"}, + scanRec{size: 50, head: b, tail: b, content: b, path: "/y/2"}, + scanRec{size: 50, head: b, tail: b, content: b, path: "/y/1"}, + scanRec{size: 50, head: a, tail: a, content: a, path: "/x/2"}, + scanRec{size: 50, head: a, tail: a, content: a, path: "/x/1"}, ) reversed := slices.Clone(recs) @@ -385,16 +401,19 @@ func TestWarnfEscapes(t *testing.T) { func TestDupeGroups(t *testing.T) { t.Parallel() + h, tl, c := testHash("h"), testHash("t"), testHash("c") + bigH, bigT, bigC := testHash("H"), testHash("T"), testHash("C") + u := testHash("u") recs := []scanRec{ - {size: 100, head: "h", tail: "t", content: "c", path: "/z/b"}, - {size: 100, head: "h", tail: "t", content: "c", path: "/z/a"}, - {size: 100, head: "h", tail: "t", content: "c", path: "/z/c"}, - {size: 4000, head: "H", tail: "T", content: "C", path: "/big/2"}, - {size: 4000, head: "H", tail: "T", content: "C", path: "/big/1"}, + {size: 100, head: h, tail: tl, content: c, path: "/z/b"}, + {size: 100, head: h, tail: tl, content: c, path: "/z/a"}, + {size: 100, head: h, tail: tl, content: c, path: "/z/c"}, + {size: 4000, head: bigH, tail: bigT, content: bigC, path: "/big/2"}, + {size: 4000, head: bigH, tail: bigT, content: bigC, path: "/big/1"}, // Same size as the /z group but a different head hash. - {size: 100, head: "other", tail: "t", content: "c", path: "/z/d"}, + {size: 100, head: testHash("other"), tail: tl, content: c, path: "/z/d"}, // A singleton signature must not form a group. - {size: 7, head: "u", tail: "u", content: "u", path: "/lonely"}, + {size: 7, head: u, tail: u, content: u, path: "/lonely"}, } groups := dupeGroupsOf(t, recs) @@ -420,10 +439,11 @@ func TestDupeGroupsContentSeparates(t *testing.T) { // Same size, head, and tail: /b differs from the others only in its // content hash, so it joins no group, while /a and /c group. + h, tl := testHash("h"), testHash("t") recs := []scanRec{ - {size: 100, head: "h", tail: "t", content: "c1", path: "/a"}, - {size: 100, head: "h", tail: "t", content: "c2", path: "/b"}, - {size: 100, head: "h", tail: "t", content: "c1", path: "/c"}, + {size: 100, head: h, tail: tl, content: testHash("c1"), path: "/a"}, + {size: 100, head: h, tail: tl, content: testHash("c2"), path: "/b"}, + {size: 100, head: h, tail: tl, content: testHash("c1"), path: "/c"}, } groups := dupeGroupsOf(t, recs) @@ -442,14 +462,15 @@ func TestDupeGroupsMtimeExcluded(t *testing.T) { // mtime is informational only; records differing only in mtime // still group together. + h, tl, c := testHash("h"), testHash("t"), testHash("c") recs := []scanRec{ { - size: 9, mtime: time.Unix(100, 0), head: "h", tail: "t", - content: "c", path: "/m/1", + size: 9, mtime: time.Unix(100, 0), head: h, tail: tl, + content: c, path: "/m/1", }, { - size: 9, mtime: time.Unix(200, 0), head: "h", tail: "t", - content: "c", path: "/m/2", + size: 9, mtime: time.Unix(200, 0), head: h, tail: tl, + content: c, path: "/m/2", }, } @@ -464,11 +485,12 @@ func TestDupeGroupsTieBreak(t *testing.T) { // The hashes sort opposite to the first paths, so ordering the // groups by hash instead of by first path fails this test. + a, b := testHash("a"), testHash("b") recs := []scanRec{ - {size: 50, head: "a", tail: "a", content: "a", path: "/beta/2"}, - {size: 50, head: "a", tail: "a", content: "a", path: "/beta/1"}, - {size: 50, head: "b", tail: "b", content: "b", path: "/alpha/2"}, - {size: 50, head: "b", tail: "b", content: "b", path: "/alpha/1"}, + {size: 50, head: a, tail: a, content: a, path: "/beta/2"}, + {size: 50, head: a, tail: a, content: a, path: "/beta/1"}, + {size: 50, head: b, tail: b, content: b, path: "/alpha/2"}, + {size: 50, head: b, tail: b, content: b, path: "/alpha/1"}, } groups := dupeGroupsOf(t, recs) @@ -490,15 +512,16 @@ func TestDupeGroupsOrderedByFirstPath(t *testing.T) { // /z, /k, which is neither the order of their first paths nor its // reverse, and must write them ordered by first path. The group of // size 7 comes after them all, though its first path sorts first. + a, b, c, d := testHash("a"), testHash("b"), testHash("c"), testHash("d") recs := []scanRec{ - {size: 50, head: "a", tail: "a", content: "a", path: "/k/2"}, - {size: 50, head: "a", tail: "a", content: "a", path: "/k/1"}, - {size: 50, head: "b", tail: "b", content: "b", path: "/z/1"}, - {size: 50, head: "b", tail: "b", content: "b", path: "/z/2"}, - {size: 50, head: "c", tail: "c", content: "c", path: "/a/2"}, - {size: 50, head: "c", tail: "c", content: "c", path: "/a/1"}, - {size: 7, head: "d", tail: "d", content: "d", path: "/0/2"}, - {size: 7, head: "d", tail: "d", content: "d", path: "/0/1"}, + {size: 50, head: a, tail: a, content: a, path: "/k/2"}, + {size: 50, head: a, tail: a, content: a, path: "/k/1"}, + {size: 50, head: b, tail: b, content: b, path: "/z/1"}, + {size: 50, head: b, tail: b, content: b, path: "/z/2"}, + {size: 50, head: c, tail: c, content: c, path: "/a/2"}, + {size: 50, head: c, tail: c, content: c, path: "/a/1"}, + {size: 7, head: d, tail: d, content: d, path: "/0/2"}, + {size: 7, head: d, tail: d, content: d, path: "/0/1"}, } groups := dupeGroupsOf(t, recs) @@ -517,11 +540,12 @@ func TestDupeGroupsOrderedByFirstPath(t *testing.T) { func TestDupeGroupsDeterministic(t *testing.T) { t.Parallel() + a, b := testHash("a"), testHash("b") recs := []scanRec{ - {size: 1, head: "a", tail: "a", content: "a", path: "/p/1"}, - {size: 1, head: "a", tail: "a", content: "a", path: "/p/2"}, - {size: 2, head: "b", tail: "b", content: "b", path: "/q/1"}, - {size: 2, head: "b", tail: "b", content: "b", path: "/q/2"}, + {size: 1, head: a, tail: a, content: a, path: "/p/1"}, + {size: 1, head: a, tail: a, content: a, path: "/p/2"}, + {size: 2, head: b, tail: b, content: b, path: "/q/1"}, + {size: 2, head: b, tail: b, content: b, path: "/q/2"}, } forward := dupeGroupsOf(t, recs) diff --git a/scan.go b/scan.go index 3069cc8..1162e6e 100644 --- a/scan.go +++ b/scan.go @@ -5,7 +5,6 @@ import ( "context" "crypto/sha256" "database/sql" - "encoding/hex" "errors" "fmt" "io" @@ -1199,9 +1198,9 @@ func inodeOfInfo(fi fs.FileInfo) (uint64, uint64) { // hash phase. type hashResult struct { run []fileRec - head string - tail string - content string + head [sha256.Size]byte + tail [sha256.Size]byte + content [sha256.Size]byte err error } @@ -1221,7 +1220,8 @@ type hashPool struct { // hash each run's first path with hash (all paths in a run are hard // links to the same inode) and write one result per run. func startHashPool(ctx context.Context, runs [][]fileRec, workers int, - hash func(path string, size int64) (string, string, string, error), + hash func(path string, size int64) ( + [sha256.Size]byte, [sha256.Size]byte, [sha256.Size]byte, error), ) *hashPool { ctx, cancel := context.WithCancel(ctx) @@ -1286,7 +1286,8 @@ func feedHashJobs(ctx context.Context, runs [][]fileRec, // the hash of only delays that. func hashWorker(ctx context.Context, jobs <-chan []fileRec, results chan<- hashResult, - hash func(path string, size int64) (string, string, string, error), + hash func(path string, size int64) ( + [sha256.Size]byte, [sha256.Size]byte, [sha256.Size]byte, error), ) { for run := range jobs { if ctx.Err() != nil { @@ -1337,37 +1338,38 @@ func openRegularFile(path string) (*os.File, error) { return f, nil } -// emptyHash is the lowercase-hex SHA-256 of the empty input: the head, -// tail, and content hash of every zero-length file. -const emptyHash = "e3b0c44298fc1c149afbf4c8996fb924" + - "27ae41e4649b934ca495991b7852b855" - // hashSignature computes the three hashes the hash phase records for a // file: head and tail (see hashEnds) and content (see hashContent). // With the file size they form its duplicate signature, and two files // are duplicates only when all four agree. size is the value recorded // when the file was statted; a zero-length file has constant hashes and -// is never opened. -func hashSignature(path string, size int64) (string, string, string, error) { +// is never opened. On an error every hash is left zero. +func hashSignature(path string, size int64) ( + [sha256.Size]byte, [sha256.Size]byte, [sha256.Size]byte, error, +) { + var zero [sha256.Size]byte + if size == 0 { - return emptyHash, emptyHash, emptyHash, nil + empty := sha256.Sum256(nil) + + return empty, empty, empty, nil } f, err := openRegularFile(path) if err != nil { - return "", "", "", err + return zero, zero, zero, err } defer func() { _ = f.Close() }() head, tail, err := hashEnds(f, size) if err != nil { - return "", "", "", err + return zero, zero, zero, err } content, err := hashContent(f, size) if err != nil { - return "", "", "", err + return zero, zero, zero, err } return head, tail, content, nil @@ -1377,31 +1379,32 @@ func hashSignature(path string, size int64) (string, string, string, error) { // bytes of f. In a file no larger than headTailWindow, each window is // the whole file. Reading fewer bytes than asked means the file shrank // between the stat and the hash, and is an error. -func hashEnds(f *os.File, size int64) (string, string, error) { +func hashEnds(f *os.File, size int64) ( + [sha256.Size]byte, [sha256.Size]byte, error, +) { + var zero [sha256.Size]byte + buf := make([]byte, min(int64(headTailWindow), size)) _, err := f.ReadAt(buf, 0) if err != nil { - return "", "", err + return zero, zero, err } - h := sha256.Sum256(buf) - head := hex.EncodeToString(h[:]) + head := sha256.Sum256(buf) _, err = f.ReadAt(buf, size-int64(len(buf))) if err != nil { - return "", "", err + return zero, zero, err } - t := sha256.Sum256(buf) - - return head, hex.EncodeToString(t[:]), nil + return head, sha256.Sum256(buf), nil } // hashContent returns the content hash of f: the SHA-256 of the whole // file when it is smaller than wholeFileMax, or of sampled windows when // it is that size or larger. -func hashContent(f *os.File, size int64) (string, error) { +func hashContent(f *os.File, size int64) ([sha256.Size]byte, error) { if size >= int64(wholeFileMax) { return hashSamples(f, size) } @@ -1414,20 +1417,20 @@ func hashContent(f *os.File, size int64) (string, error) { // reads. Reading fewer than size bytes means the file shrank between // the stat and the hash; that is an error rather than a hash of content // that no longer matches the recorded size. -func hashWhole(f *os.File, size int64) (string, error) { +func hashWhole(f *os.File, size int64) ([sha256.Size]byte, error) { h := sha256.New() n, err := io.Copy(h, io.NewSectionReader(f, 0, size)) if err != nil { - return "", err + return [sha256.Size]byte{}, err } if n != size { - return "", fmt.Errorf("read %d of %d bytes: %w", n, size, - io.ErrUnexpectedEOF) + return [sha256.Size]byte{}, fmt.Errorf("read %d of %d bytes: %w", + n, size, io.ErrUnexpectedEOF) } - return hex.EncodeToString(h.Sum(nil)), nil + return [sha256.Size]byte(h.Sum(nil)), nil } // hashSamples feeds sampleWindow bytes at each gigabyte-aligned offset @@ -1437,7 +1440,7 @@ func hashWhole(f *os.File, size int64) (string, error) { // without every byte being read. Because size is part of the signature, // only files of one size are compared, so their sample boundaries // always align. -func hashSamples(f *os.File, size int64) (string, error) { +func hashSamples(f *os.File, size int64) ([sha256.Size]byte, error) { h := sha256.New() buf := make([]byte, sampleWindow) @@ -1446,11 +1449,11 @@ func hashSamples(f *os.File, size int64) (string, error) { _, err := f.ReadAt(buf[:n], off) if err != nil { - return "", err + return [sha256.Size]byte{}, err } h.Write(buf[:n]) } - return hex.EncodeToString(h.Sum(nil)), nil + return [sha256.Size]byte(h.Sum(nil)), nil } diff --git a/scan_test.go b/scan_test.go index fb687e1..5f14906 100644 --- a/scan_test.go +++ b/scan_test.go @@ -5,7 +5,6 @@ import ( "context" "crypto/sha256" "database/sql" - "encoding/hex" "errors" "fmt" "io" @@ -43,13 +42,6 @@ func writeFile(t *testing.T, dir, name string, data []byte) string { return p } -// hexSum returns the lowercase-hex SHA-256 of data. -func hexSum(data []byte) string { - s := sha256.Sum256(data) - - return hex.EncodeToString(s[:]) -} - // pattern returns n bytes of deterministic content seeded by tag. func pattern(tag byte, n int) []byte { data := make([]byte, n) @@ -62,7 +54,9 @@ func pattern(tag byte, n int) []byte { // sig returns a file's full signature (head, tail, content), failing the // test on any error. -func sig(t *testing.T, path string, size int64) (string, string, string) { +func sig(t *testing.T, path string, size int64) ( + [sha256.Size]byte, [sha256.Size]byte, [sha256.Size]byte, +) { t.Helper() head, tail, content, err := hashSignature(path, size) @@ -100,12 +94,12 @@ func TestHashSignature(t *testing.T) { head, tail, content := sig(t, p, int64(len(c.data))) window := min(len(c.data), headTailWindow) - wantHead := hexSum(c.data[:window]) - wantTail := hexSum(c.data[len(c.data)-window:]) - wantContent := hexSum(c.data) + wantHead := sha256.Sum256(c.data[:window]) + wantTail := sha256.Sum256(c.data[len(c.data)-window:]) + wantContent := sha256.Sum256(c.data) if head != wantHead || tail != wantTail || content != wantContent { - t.Errorf("head=%s tail=%s content=%s, want %s %s %s", + t.Errorf("head=%x tail=%x content=%x, want %x %x %x", head, tail, content, wantHead, wantTail, wantContent) } }) @@ -170,22 +164,25 @@ func TestHashSignatureErrors(t *testing.T) { dir := t.TempDir() - // A missing file: an error, and every hash left empty. + var zero [sha256.Size]byte + + // A missing file: an error, and every hash left zero. head, tail, content, err := hashSignature(filepath.Join(dir, "missing"), 1) if err == nil { t.Error("no error for a missing file") } - if head != "" || tail != "" || content != "" { - t.Errorf("missing file returned hashes: %q %q %q", head, tail, content) + if head != zero || tail != zero || content != zero { + t.Errorf("missing file returned hashes: %x %x %x", head, tail, content) } // A zero-length file has constant hashes and is never opened: even // a missing path succeeds. + empty := sha256.Sum256(nil) + head, tail, content, err = hashSignature(filepath.Join(dir, "missing"), 0) - if err != nil || - head != emptyHash || tail != emptyHash || content != emptyHash { - t.Errorf("empty: head=%q tail=%q content=%q err=%v, "+ + if err != nil || head != empty || tail != empty || content != empty { + t.Errorf("empty: head=%x tail=%x content=%x err=%v, "+ "want constant hashes", head, tail, content, err) } @@ -198,8 +195,8 @@ func TestHashSignatureErrors(t *testing.T) { t.Error("no error when the stat size exceeds the file size") } - if head != "" || tail != "" || content != "" { - t.Errorf("shrunk file returned hashes: %q %q %q", head, tail, content) + if head != zero || tail != zero || content != zero { + t.Errorf("shrunk file returned hashes: %x %x %x", head, tail, content) } } @@ -1211,8 +1208,10 @@ func TestSyncScanUnchangedReuse(t *testing.T) { } } + sentinel := sha256.Sum256([]byte("sentinel")) + _, err := db.ExecContext(context.Background(), - "UPDATE files SET head = 'sentinel' WHERE path = ?", []byte(a)) + "UPDATE files SET head = ? WHERE path = ?", sentinel[:], []byte(a)) if err != nil { t.Fatal(err) } @@ -1223,9 +1222,9 @@ func TestSyncScanUnchangedReuse(t *testing.T) { st) } - if r := recordByPath(t, dbRecords(t, db), a); r.head != "sentinel" { - t.Fatalf("head = %q, want sentinel (file must not be re-read)", - r.head) + if r := recordByPath(t, dbRecords(t, db), a); r.head != sentinel { + t.Fatalf("head = %x, want the sentinel %x (file must not be re-read)", + r.head, sentinel) } } @@ -1265,13 +1264,56 @@ func assertWholeFileHashed(t *testing.T, db *sql.DB, path string, t.Helper() r := recordByPath(t, dbRecords(t, db), path) - if want := hexSum(data); r.head != want || r.tail != want || + if want := sha256.Sum256(data); r.head != want || r.tail != want || r.content != want { - t.Fatalf("head, tail, content = %q, %q, %q, want %q for each", + t.Fatalf("head, tail, content = %x, %x, %x, want %x for each", r.head, r.tail, r.content, want) } } +// TestScanStoresRawHashes checks what the database holds for a file +// larger than both end windows, whose three hashes all differ: head, +// tail and content are each a BLOB of the 32 bytes of the SHA-256 the +// scan computed. +func TestScanStoresRawHashes(t *testing.T) { + t.Parallel() + + dir := t.TempDir() + db := openTestDB(t) + data := pattern(3, 3*headTailWindow) + p := writeFile(t, dir, "f.bin", data) + + syncTree(t, db, dir) + + var ( + blobs bool + head, tail, content []byte + ) + + err := db.QueryRowContext(t.Context(), + "SELECT typeof(head) = 'blob' AND typeof(tail) = 'blob' AND "+ + "typeof(content) = 'blob', head, tail, content "+ + "FROM files WHERE path = ?", []byte(p)). + Scan(&blobs, &head, &tail, &content) + if err != nil { + t.Fatal(err) + } + + if !blobs { + t.Error("head, tail and content are not all BLOBs") + } + + wantHead := sha256.Sum256(data[:headTailWindow]) + wantTail := sha256.Sum256(data[len(data)-headTailWindow:]) + wantContent := sha256.Sum256(data) + + if !bytes.Equal(head, wantHead[:]) || !bytes.Equal(tail, wantTail[:]) || + !bytes.Equal(content, wantContent[:]) { + t.Errorf("head, tail, content = %x, %x, %x, want %x, %x, %x", + head, tail, content, wantHead, wantTail, wantContent) + } +} + // TestSyncScanSameSecondRewrite rewrites a file in place at the same // size with an mtime later in the same second as the recorded one: the // next scan must notice the change and re-hash the file. The operand is @@ -1567,7 +1609,8 @@ func TestScanHardlinksReadOnce(t *testing.T) { ra := recordByPath(t, recs, a) rb := recordByPath(t, recs, b) - if ra.head == "" || ra.head != rb.head || ra.tail != rb.tail { + if ra.head == [sha256.Size]byte{} || ra.head != rb.head || + ra.tail != rb.tail { t.Fatalf("hardlink hashes differ: %+v vs %+v", ra, rb) } diff --git a/trees.go b/trees.go index 21bd6fc..1d2e689 100644 --- a/trees.go +++ b/trees.go @@ -287,7 +287,8 @@ func (b *treeBuilder) finish() (*treeNode, []*treeNode) { // excluded. func fileEntry(name string, r scanRec) string { return "f\x00" + name + "\x00" + strconv.FormatInt(r.size, 10) + - "\x00" + r.head + "\x00" + r.tail + "\x00" + r.content + "\x00" + string(r.head[:]) + "\x00" + string(r.tail[:]) + + "\x00" + string(r.content[:]) } // collectTreeGroups groups directories by digest, counting each one on @@ -337,7 +338,8 @@ func collectTreeGroups(allDirs []*treeNode, super *treeNode, // digest is the SHA-256 of its child entries — files serialized with // name and signature, subdirectories with name and recursive digest — // sorted byte-lexicographically. Filenames cannot contain NUL or "/", -// so NUL delimiters are unambiguous. +// and every hash and digest is 32 bytes, so NUL delimiters are +// unambiguous. func (n *treeNode) computeDigest() { slices.Sort(n.entries) diff --git a/trees_test.go b/trees_test.go index 9249aa0..e800d26 100644 --- a/trees_test.go +++ b/trees_test.go @@ -10,16 +10,6 @@ import ( "time" ) -// Signature hashes shared by the smoke-test records. -const ( - f1Head = "f1h" - f1Tail = "f1t" - f1Content = "f1c" - f2Head = "f2h" - f2Tail = "f2t" - f2Content = "f2c" -) - // treesOpenLine is the plain line of trees' open phase when it opens // the database at once; it prints it as the phase starts and as it ends. const treesOpenLine = "[1/4] open: opening the database, elapsed 0s, " + @@ -28,6 +18,9 @@ const treesOpenLine = "[1/4] open: opening the database, elapsed 0s, " + // smokeTreeRecs mirrors the README smoke-test tree layout: /d/t1 and // /d/t2 are identical, /d/t3 differs from them only by one filename. func smokeTreeRecs() []scanRec { + f1Head, f1Tail, f1Content := testHash("f1h"), testHash("f1t"), testHash("f1c") + f2Head, f2Tail, f2Content := testHash("f2h"), testHash("f2t"), testHash("f2c") + return []scanRec{ {size: 3000, head: f1Head, tail: f1Tail, content: f1Content, path: "/d/t1/f1"}, {size: 100, head: f2Head, tail: f2Tail, content: f2Content, path: "/d/t1/sub/f2"}, @@ -153,13 +146,14 @@ func TestTreeNamesSortingBeforeSlash(t *testing.T) { for _, top := range []string{"/a", "/c"} { for _, p := range []string{"/b", "/b-x/f", "/b.txt", "/b/f"} { - content := "c" + content := testHash("c") if p == "/b-x/f" { - content = "other" + content = testHash("other") } recs = append(recs, scanRec{ - size: 1, head: "h", tail: "t", content: content, path: top + p, + size: 1, head: testHash("h"), tail: testHash("t"), + content: content, path: top + p, }) } } @@ -332,11 +326,10 @@ func TestTreeDigests(t *testing.T) { func TestTreeDigestContentSensitivity(t *testing.T) { t.Parallel() - const sharedTail = "same" - + same, c := testHash("same"), testHash("c") recs := []scanRec{ - {size: 10, head: sharedTail, tail: sharedTail, content: "c", path: "/r/a/f"}, - {size: 10, head: "DIFF", tail: sharedTail, content: "c", path: "/r/b/f"}, + {size: 10, head: same, tail: same, content: c, path: "/r/a/f"}, + {size: 10, head: testHash("DIFF"), tail: same, content: c, path: "/r/b/f"}, } _, dirs := treeOf(t, recs) @@ -397,9 +390,10 @@ func TestCollectTreeGroupsSiblings(t *testing.T) { // Identical sibling dirs share a parent, so their group cannot be // implied by a parent group and must be reported. + h, tl, c := testHash("h"), testHash("t"), testHash("c") recs := []scanRec{ - {size: 10, head: "h", tail: "t", content: "c", path: "/p/x1/f"}, - {size: 10, head: "h", tail: "t", content: "c", path: "/p/x2/f"}, + {size: 10, head: h, tail: tl, content: c, path: "/p/x1/f"}, + {size: 10, head: h, tail: tl, content: c, path: "/p/x2/f"}, } super, dirs := treeOf(t, recs) @@ -418,10 +412,11 @@ func TestCollectTreeGroupsDifferingParents(t *testing.T) { // /p/a and /q/b contain identical x subtrees, but /p/a has an // extra file, so the parents' digests differ and the x group must // be reported. + h, tl, c, e := testHash("h"), testHash("t"), testHash("c"), testHash("e") recs := []scanRec{ - {size: 10, head: "h", tail: "t", content: "c", path: "/p/a/x/f"}, - {size: 99, head: "e", tail: "e", content: "e", path: "/p/a/extra"}, - {size: 10, head: "h", tail: "t", content: "c", path: "/q/b/x/f"}, + {size: 10, head: h, tail: tl, content: c, path: "/p/a/x/f"}, + {size: 99, head: e, tail: e, content: e, path: "/p/a/extra"}, + {size: 10, head: h, tail: tl, content: c, path: "/q/b/x/f"}, } super, dirs := treeOf(t, recs)