Compute the content hash only when head and tail match (closes #61)
check / check (push) Successful in 49s
check / check (push) Successful in 49s
A file of 10 MiB or more now gets only its 64 KiB head and tail in the hash phase, so its content is read only when it can be a duplicate. A new content phase after the update phase finds every group of records, anywhere in the database, that share size, head and tail and include one without a content hash. It checks every member with lstat and, when at least two pass, reads those without a content hash through the existing worker pool; a stale file does not count as a match. report and trees leave out records without a content hash. The README, help text and TODO entry describe the gate; the schema stays at version 1. Lint suppressed: gosec on the file open in hashContentOnly, as in hashSignature, and on one chmod in a test. Model: opus-5-5
This commit was merged in pull request #65.
This commit is contained in:
@@ -278,6 +278,62 @@ func loadFileMeta(ctx context.Context, db *sql.DB,
|
||||
return nil
|
||||
}
|
||||
|
||||
// contentCandidatesSQL selects every record of at least headTailMin
|
||||
// bytes whose size, head, and tail equal another record's, in each
|
||||
// group (the records sharing a size, head, and tail) where at least one
|
||||
// record has no content hash, with whether each record has one. SQLite
|
||||
// does the grouping, so no other record's hashes are loaded into
|
||||
// memory; the rows come ordered by size, head, and tail, so each
|
||||
// group's rows arrive together.
|
||||
const contentCandidatesSQL = `
|
||||
SELECT f.path, f.size, f.mtime, f.head, f.tail, f.content <> ''
|
||||
FROM files AS f
|
||||
JOIN (
|
||||
SELECT size, head, tail
|
||||
FROM files
|
||||
WHERE size >= ? AND head <> ''
|
||||
GROUP BY size, head, tail
|
||||
HAVING COUNT(*) > 1 AND SUM(content = '') > 0
|
||||
) AS g USING (size, head, tail)
|
||||
ORDER BY size, head, tail
|
||||
`
|
||||
|
||||
// loadContentCandidates streams the rows of contentCandidatesSQL to fn:
|
||||
// each record, without its content hash, and whether it has one.
|
||||
func loadContentCandidates(ctx context.Context, db *sql.DB,
|
||||
fn func(r scanRec, hashed bool),
|
||||
) error {
|
||||
rows, err := db.QueryContext(ctx, contentCandidatesSQL, headTailMin)
|
||||
if err != nil {
|
||||
return fmt.Errorf("read records: %w", err)
|
||||
}
|
||||
|
||||
defer func() { _ = rows.Close() }()
|
||||
|
||||
for rows.Next() {
|
||||
var (
|
||||
path []byte
|
||||
r scanRec
|
||||
hashed int64
|
||||
)
|
||||
|
||||
err = rows.Scan(&path, &r.size, &r.mtime, &r.head, &r.tail, &hashed)
|
||||
if err != nil {
|
||||
return fmt.Errorf("read record: %w", err)
|
||||
}
|
||||
|
||||
r.path = string(path)
|
||||
fn(r, hashed != 0)
|
||||
}
|
||||
|
||||
err = rows.Err()
|
||||
if err != nil {
|
||||
return fmt.Errorf("read records: %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// updateBatchSize is the number of record changes committed per
|
||||
// transaction during the update pass. The filesystem is authoritative
|
||||
// and the database an eventually-consistent reflection of it, so
|
||||
|
||||
Reference in New Issue
Block a user