Check every group member and show the content phase from its start
check / check (push) Successful in 57s

The content phase now checks every record sharing a size, head and
tail with lstat, including those that already have a content hash, and
reads those without one only when at least two pass. Only a missing
file is passed over silently; any other lstat error is warned about and
counted as skipped. The phase shows a running count while it queries
and checks, then its bar. The README states once when content is empty.
New tests cover these and hard links.

Model: opus-5-5
This commit is contained in:
2026-09-23 13:09:03 +00:00
parent 89fc9e4595
commit fff4409a24
6 changed files with 243 additions and 81 deletions
+13 -15
View File
@@ -279,31 +279,29 @@ func loadFileMeta(ctx context.Context, db *sql.DB,
}
// contentCandidatesSQL selects every record of at least headTailMin
// bytes that has no content hash but whose size, head, and tail equal
// another record's, with the number of records in its group (the
// records sharing that size, head, and tail) that already have a
// content hash. SQLite does the grouping, so no other record's hashes
// are loaded into memory; the rows come ordered by size, head, and
// tail, so each group's rows arrive together.
// bytes whose size, head, and tail equal another record's, in each
// group (the records sharing a size, head, and tail) where at least one
// record has no content hash, with whether each record has one. SQLite
// does the grouping, so no other record's hashes are loaded into
// memory; the rows come ordered by size, head, and tail, so each
// group's rows arrive together.
const contentCandidatesSQL = `
SELECT f.path, f.size, f.mtime, f.head, f.tail, g.hashed
SELECT f.path, f.size, f.mtime, f.head, f.tail, f.content <> ''
FROM files AS f
JOIN (
SELECT size, head, tail, SUM(content <> '') AS hashed
SELECT size, head, tail
FROM files
WHERE size >= ? AND head <> ''
GROUP BY size, head, tail
HAVING COUNT(*) > 1
HAVING COUNT(*) > 1 AND SUM(content = '') > 0
) AS g USING (size, head, tail)
WHERE f.content = ''
ORDER BY size, head, tail
`
// loadContentCandidates streams the rows of contentCandidatesSQL to fn:
// each record, without its content hash, and the number of records in
// its group that already have one.
// each record, without its content hash, and whether it has one.
func loadContentCandidates(ctx context.Context, db *sql.DB,
fn func(r scanRec, hashed int),
fn func(r scanRec, hashed bool),
) error {
rows, err := db.QueryContext(ctx, contentCandidatesSQL, headTailMin)
if err != nil {
@@ -316,7 +314,7 @@ func loadContentCandidates(ctx context.Context, db *sql.DB,
var (
path []byte
r scanRec
hashed int
hashed int64
)
err = rows.Scan(&path, &r.size, &r.mtime, &r.head, &r.tail, &hashed)
@@ -325,7 +323,7 @@ func loadContentCandidates(ctx context.Context, db *sql.DB,
}
r.path = string(path)
fn(r, hashed)
fn(r, hashed != 0)
}
err = rows.Err()