Check every group member and show the content phase from its start
check / check (push) Successful in 57s
check / check (push) Successful in 57s
The content phase now checks every record sharing a size, head and tail with lstat, including those that already have a content hash, and reads those without one only when at least two pass. Only a missing file is passed over silently; any other lstat error is warned about and counted as skipped. The phase shows a running count while it queries and checks, then its bar. The README states once when content is empty. New tests cover these and hard links. Model: opus-5-5
This commit is contained in:
@@ -279,31 +279,29 @@ func loadFileMeta(ctx context.Context, db *sql.DB,
|
||||
}
|
||||
|
||||
// contentCandidatesSQL selects every record of at least headTailMin
|
||||
// bytes that has no content hash but whose size, head, and tail equal
|
||||
// another record's, with the number of records in its group (the
|
||||
// records sharing that size, head, and tail) that already have a
|
||||
// content hash. SQLite does the grouping, so no other record's hashes
|
||||
// are loaded into memory; the rows come ordered by size, head, and
|
||||
// tail, so each group's rows arrive together.
|
||||
// bytes whose size, head, and tail equal another record's, in each
|
||||
// group (the records sharing a size, head, and tail) where at least one
|
||||
// record has no content hash, with whether each record has one. SQLite
|
||||
// does the grouping, so no other record's hashes are loaded into
|
||||
// memory; the rows come ordered by size, head, and tail, so each
|
||||
// group's rows arrive together.
|
||||
const contentCandidatesSQL = `
|
||||
SELECT f.path, f.size, f.mtime, f.head, f.tail, g.hashed
|
||||
SELECT f.path, f.size, f.mtime, f.head, f.tail, f.content <> ''
|
||||
FROM files AS f
|
||||
JOIN (
|
||||
SELECT size, head, tail, SUM(content <> '') AS hashed
|
||||
SELECT size, head, tail
|
||||
FROM files
|
||||
WHERE size >= ? AND head <> ''
|
||||
GROUP BY size, head, tail
|
||||
HAVING COUNT(*) > 1
|
||||
HAVING COUNT(*) > 1 AND SUM(content = '') > 0
|
||||
) AS g USING (size, head, tail)
|
||||
WHERE f.content = ''
|
||||
ORDER BY size, head, tail
|
||||
`
|
||||
|
||||
// loadContentCandidates streams the rows of contentCandidatesSQL to fn:
|
||||
// each record, without its content hash, and the number of records in
|
||||
// its group that already have one.
|
||||
// each record, without its content hash, and whether it has one.
|
||||
func loadContentCandidates(ctx context.Context, db *sql.DB,
|
||||
fn func(r scanRec, hashed int),
|
||||
fn func(r scanRec, hashed bool),
|
||||
) error {
|
||||
rows, err := db.QueryContext(ctx, contentCandidatesSQL, headTailMin)
|
||||
if err != nil {
|
||||
@@ -316,7 +314,7 @@ func loadContentCandidates(ctx context.Context, db *sql.DB,
|
||||
var (
|
||||
path []byte
|
||||
r scanRec
|
||||
hashed int
|
||||
hashed int64
|
||||
)
|
||||
|
||||
err = rows.Scan(&path, &r.size, &r.mtime, &r.head, &r.tail, &hashed)
|
||||
@@ -325,7 +323,7 @@ func loadContentCandidates(ctx context.Context, db *sql.DB,
|
||||
}
|
||||
|
||||
r.path = string(path)
|
||||
fn(r, hashed)
|
||||
fn(r, hashed != 0)
|
||||
}
|
||||
|
||||
err = rows.Err()
|
||||
|
||||
Reference in New Issue
Block a user