Compute the content hash only when head and tail match (closes #61)
check / check (push) Successful in 49s

A file of 10 MiB or more now gets only its 64 KiB head and tail in the
hash phase, so its content is read only when it can be a duplicate. A
new content phase after the update phase finds every group of records,
anywhere in the database, that share size, head and tail and include
one without a content hash. It checks every member with lstat and, when
at least two pass, reads those without a content hash through the
existing worker pool; a stale file does not count as a match. report
and trees leave out records without a content hash. The README, help
text and TODO entry describe the gate; the schema stays at version 1.

Lint suppressed: gosec on the file open in hashContentOnly, as in
hashSignature, and on one chmod in a test.

Model: opus-5-5
This commit was merged in pull request #65.
This commit is contained in:
2026-09-23 16:06:09 +02:00
parent 09a39ddf37
commit c737490a53
12 changed files with 930 additions and 220 deletions
+56
View File
@@ -278,6 +278,62 @@ func loadFileMeta(ctx context.Context, db *sql.DB,
return nil
}
// contentCandidatesSQL selects every record of at least headTailMin
// bytes whose size, head, and tail equal another record's, in each
// group (the records sharing a size, head, and tail) where at least one
// record has no content hash, with whether each record has one. SQLite
// does the grouping, so no other record's hashes are loaded into
// memory; the rows come ordered by size, head, and tail, so each
// group's rows arrive together.
const contentCandidatesSQL = `
SELECT f.path, f.size, f.mtime, f.head, f.tail, f.content <> ''
FROM files AS f
JOIN (
SELECT size, head, tail
FROM files
WHERE size >= ? AND head <> ''
GROUP BY size, head, tail
HAVING COUNT(*) > 1 AND SUM(content = '') > 0
) AS g USING (size, head, tail)
ORDER BY size, head, tail
`
// loadContentCandidates streams the rows of contentCandidatesSQL to fn:
// each record, without its content hash, and whether it has one.
func loadContentCandidates(ctx context.Context, db *sql.DB,
fn func(r scanRec, hashed bool),
) error {
rows, err := db.QueryContext(ctx, contentCandidatesSQL, headTailMin)
if err != nil {
return fmt.Errorf("read records: %w", err)
}
defer func() { _ = rows.Close() }()
for rows.Next() {
var (
path []byte
r scanRec
hashed int64
)
err = rows.Scan(&path, &r.size, &r.mtime, &r.head, &r.tail, &hashed)
if err != nil {
return fmt.Errorf("read record: %w", err)
}
r.path = string(path)
fn(r, hashed != 0)
}
err = rows.Err()
if err != nil {
return fmt.Errorf("read records: %w", err)
}
return nil
}
// updateBatchSize is the number of record changes committed per
// transaction during the update pass. The filesystem is authoritative
// and the database an eventually-consistent reflection of it, so