scan: show each phase as N of M, with progress, elapsed time and ETA, also while it waits (closes #112)
check / check (push) Canceled after 0s
check / check (push) Canceled after 0s
Every scan display now starts with its phase and its place among the five, as "[2/5] walk", and shows its progress, rate, elapsed time and ETA. On a terminal hash, update and the content reads keep the library's bar, which fills what the rest of the line leaves; load, the walk and the content checks are a spinner whose text is cut to the terminal's width. load counts the records first. The content phase adds up the sizes of the matching groups, without the join and sort, and says it is searching until the first record returns. The walk's total is estimated from the records under the operands. Each display is drawn from its own goroutine: several times a second on a terminal, every 5 seconds in plain mode. Judgement call: load and the content checks stay spinners once their totals are known. Model: opus-5-5
This commit is contained in:
@@ -514,6 +514,18 @@ func loadDupeRows(ctx context.Context, db *sql.DB,
|
||||
return records, nil
|
||||
}
|
||||
|
||||
// countRecords returns the number of records in the database.
|
||||
func countRecords(ctx context.Context, db *sql.DB) (int64, error) {
|
||||
var n int64
|
||||
|
||||
err := db.QueryRowContext(ctx, "SELECT COUNT(*) FROM files").Scan(&n)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("count records: %w", err)
|
||||
}
|
||||
|
||||
return n, nil
|
||||
}
|
||||
|
||||
// loadFileMeta streams every record's path, size, mtime, and whether
|
||||
// it carries hashes to fn. Scan change detection needs no hash
|
||||
// values, and skipping the hash columns keeps the scan's in-memory
|
||||
@@ -552,26 +564,46 @@ func loadFileMeta(ctx context.Context, db *sql.DB,
|
||||
return nil
|
||||
}
|
||||
|
||||
// contentCandidatesSQL selects every record of at least headTailMin
|
||||
// bytes whose size, head, and tail equal another record's, in each
|
||||
// group (the records sharing a size, head, and tail) where at least one
|
||||
// record has no content hash, with whether each record has one. SQLite
|
||||
// does the grouping, so no other record's hashes are loaded into
|
||||
// memory; the rows come ordered by size, head, and tail, so each
|
||||
// group's rows arrive together.
|
||||
// contentGroupsSQL selects each group (the records sharing a size,
|
||||
// head, and tail) of records of at least headTailMin bytes that has more
|
||||
// than one record and at least one record without a content hash, with
|
||||
// the number of its records.
|
||||
const contentGroupsSQL = `
|
||||
SELECT size, head, tail, COUNT(*) AS records
|
||||
FROM files
|
||||
WHERE size >= ? AND head <> ''
|
||||
GROUP BY size, head, tail
|
||||
HAVING COUNT(*) > 1 AND SUM(content = '') > 0
|
||||
`
|
||||
|
||||
// contentCandidatesSQL selects every record of the groups
|
||||
// contentGroupsSQL selects, with whether each record has a content
|
||||
// hash. SQLite does the grouping, so no other record's hashes are
|
||||
// loaded into memory; the rows come ordered by size, head, and tail, so
|
||||
// each group's rows arrive together.
|
||||
const contentCandidatesSQL = `
|
||||
SELECT f.path, f.size, f.mtime, f.mtime_nsec, f.head, f.tail, f.content <> ''
|
||||
FROM files AS f
|
||||
JOIN (
|
||||
SELECT size, head, tail
|
||||
FROM files
|
||||
WHERE size >= ? AND head <> ''
|
||||
GROUP BY size, head, tail
|
||||
HAVING COUNT(*) > 1 AND SUM(content = '') > 0
|
||||
) AS g USING (size, head, tail)
|
||||
JOIN (` + contentGroupsSQL + `) AS g USING (size, head, tail)
|
||||
ORDER BY size, head, tail
|
||||
`
|
||||
|
||||
// countContentCandidates returns the number of rows of
|
||||
// contentCandidatesSQL, adding up the records of the groups
|
||||
// contentGroupsSQL selects, so it neither joins nor sorts.
|
||||
func countContentCandidates(ctx context.Context, db *sql.DB) (int64, error) {
|
||||
var n int64
|
||||
|
||||
err := db.QueryRowContext(ctx,
|
||||
"SELECT COALESCE(SUM(records), 0) FROM ("+contentGroupsSQL+")",
|
||||
headTailMin).Scan(&n)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("count content candidates: %w", err)
|
||||
}
|
||||
|
||||
return n, nil
|
||||
}
|
||||
|
||||
// loadContentCandidates streams the rows of contentCandidatesSQL to fn:
|
||||
// each record, without its content hash, and whether it has one.
|
||||
func loadContentCandidates(ctx context.Context, db *sql.DB,
|
||||
|
||||
Reference in New Issue
Block a user