scan: show each phase as N of M, with progress, elapsed time and ETA, also while it waits (closes #112)
check / check (push) Canceled after 0s
check / check (push) Canceled after 0s
Every scan display names its phase and its place among the five, as "[2/5] walk", with its progress, elapsed time and ETA. load counts the records first; the walk estimates its total from the records under the operands; the content phase adds up the sizes of the matching groups and says it is searching until the first record returns. Each display redraws every 100 ms on a terminal and every 5 seconds in plain mode. On a terminal no line reaches the last column: a line too wide drops its rate, then its bar. After the terminal is made narrower than the widest line drawn, the display makes a new library object, whose erase fits. Judgement call: load and the content checks draw the bar too, once their totals are known. Judgement call: the bar is 10 columns wide, not the width the line leaves. Model: opus-5-5
This commit was merged in pull request #120.
This commit is contained in:
@@ -514,6 +514,18 @@ func loadDupeRows(ctx context.Context, db *sql.DB,
|
||||
return records, nil
|
||||
}
|
||||
|
||||
// countRecords returns the number of records in the database.
|
||||
func countRecords(ctx context.Context, db *sql.DB) (int64, error) {
|
||||
var n int64
|
||||
|
||||
err := db.QueryRowContext(ctx, "SELECT COUNT(*) FROM files").Scan(&n)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("count records: %w", err)
|
||||
}
|
||||
|
||||
return n, nil
|
||||
}
|
||||
|
||||
// loadFileMeta streams every record's path, size, mtime, and whether
|
||||
// it carries hashes to fn. Scan change detection needs no hash
|
||||
// values, and skipping the hash columns keeps the scan's in-memory
|
||||
@@ -552,26 +564,46 @@ func loadFileMeta(ctx context.Context, db *sql.DB,
|
||||
return nil
|
||||
}
|
||||
|
||||
// contentCandidatesSQL selects every record of at least headTailMin
|
||||
// bytes whose size, head, and tail equal another record's, in each
|
||||
// group (the records sharing a size, head, and tail) where at least one
|
||||
// record has no content hash, with whether each record has one. SQLite
|
||||
// does the grouping, so no other record's hashes are loaded into
|
||||
// memory; the rows come ordered by size, head, and tail, so each
|
||||
// group's rows arrive together.
|
||||
// contentGroupsSQL selects each group (the records sharing a size,
|
||||
// head, and tail) of records of at least headTailMin bytes that has more
|
||||
// than one record and at least one record without a content hash, with
|
||||
// the number of its records.
|
||||
const contentGroupsSQL = `
|
||||
SELECT size, head, tail, COUNT(*) AS records
|
||||
FROM files
|
||||
WHERE size >= ? AND head <> ''
|
||||
GROUP BY size, head, tail
|
||||
HAVING COUNT(*) > 1 AND SUM(content = '') > 0
|
||||
`
|
||||
|
||||
// contentCandidatesSQL selects every record of the groups
|
||||
// contentGroupsSQL selects, with whether each record has a content
|
||||
// hash. SQLite does the grouping, so no other record's hashes are
|
||||
// loaded into memory; the rows come ordered by size, head, and tail, so
|
||||
// each group's rows arrive together.
|
||||
const contentCandidatesSQL = `
|
||||
SELECT f.path, f.size, f.mtime, f.mtime_nsec, f.head, f.tail, f.content <> ''
|
||||
FROM files AS f
|
||||
JOIN (
|
||||
SELECT size, head, tail
|
||||
FROM files
|
||||
WHERE size >= ? AND head <> ''
|
||||
GROUP BY size, head, tail
|
||||
HAVING COUNT(*) > 1 AND SUM(content = '') > 0
|
||||
) AS g USING (size, head, tail)
|
||||
JOIN (` + contentGroupsSQL + `) AS g USING (size, head, tail)
|
||||
ORDER BY size, head, tail
|
||||
`
|
||||
|
||||
// countContentCandidates returns the number of rows of
|
||||
// contentCandidatesSQL, adding up the records of the groups
|
||||
// contentGroupsSQL selects, so it neither joins nor sorts.
|
||||
func countContentCandidates(ctx context.Context, db *sql.DB) (int64, error) {
|
||||
var n int64
|
||||
|
||||
err := db.QueryRowContext(ctx,
|
||||
"SELECT COALESCE(SUM(records), 0) FROM ("+contentGroupsSQL+")",
|
||||
headTailMin).Scan(&n)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("count content candidates: %w", err)
|
||||
}
|
||||
|
||||
return n, nil
|
||||
}
|
||||
|
||||
// loadContentCandidates streams the rows of contentCandidatesSQL to fn:
|
||||
// each record, without its content hash, and whether it has one.
|
||||
func loadContentCandidates(ctx context.Context, db *sql.DB,
|
||||
|
||||
Reference in New Issue
Block a user