Compare commits
1
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
207bd918c2 |
@@ -33,14 +33,14 @@ export SFDUPES_DATABASE="$HOME/.local/share/sfdupes/db.sqlite"
|
||||
```
|
||||
|
||||
`scan` walks one or more filesystem trees and maintains one database record per
|
||||
regular file (path, size, size on disk, mtime, head hash, tail hash, content
|
||||
hash). The database persists between runs. A rescan hashes only the files that
|
||||
are new or changed, and removes the records of files that no longer exist.
|
||||
`report` reads the database and prints the file-level duplicates report. `trees`
|
||||
reads the same database and prints the duplicate-tree report. `info` prints the
|
||||
database's details: its path, the sizes of its files and the number of its
|
||||
records. A missing/invalid subcommand — or a `scan` invocation with no `PATH`
|
||||
operand — prints a usage message and exits 2.
|
||||
regular file (path, size, mtime, head hash, tail hash, content hash). The
|
||||
database persists between runs. A rescan hashes only the files that are new or
|
||||
changed, and removes the records of files that no longer exist. `report` reads
|
||||
the database and prints the file-level duplicates report. `trees` reads the same
|
||||
database and prints the duplicate-tree report. `info` prints the database's
|
||||
details: its path, the sizes of its files and the number of its records. A
|
||||
missing/invalid subcommand — or a `scan` invocation with no `PATH` operand —
|
||||
prints a usage message and exits 2.
|
||||
|
||||
The database defaults to `/var/lib/sfdupes/db.sqlite` and can be placed anywhere
|
||||
by setting `SFDUPES_DATABASE`. The intended deployment is a daily `sfdupes scan`
|
||||
@@ -386,7 +386,6 @@ All four subcommands operate on a single SQLite database file:
|
||||
CREATE TABLE files (
|
||||
path BLOB PRIMARY KEY, -- absolute path, raw bytes
|
||||
size INTEGER NOT NULL, -- bytes, from lstat
|
||||
disk_size INTEGER NOT NULL, -- bytes allocated on disk (st_blocks × 512), from lstat
|
||||
mtime INTEGER NOT NULL, -- whole Unix seconds of the mtime, from lstat
|
||||
mtime_nsec INTEGER NOT NULL, -- nanoseconds within that second, 0 to 999999999
|
||||
head BLOB NOT NULL, -- raw 32-byte SHA-256 of the first 64 KiB
|
||||
@@ -394,20 +393,6 @@ All four subcommands operate on a single SQLite database file:
|
||||
content BLOB NOT NULL -- raw 32-byte SHA-256; whole file under 100 MiB, else samples
|
||||
) WITHOUT ROWID;
|
||||
CREATE INDEX files_signature ON files (size, head, tail, content);
|
||||
CREATE TABLE totals (
|
||||
records INTEGER NOT NULL, -- number of records in files
|
||||
size INTEGER NOT NULL -- sizes of all records in files added up
|
||||
);
|
||||
INSERT INTO totals (records, size) VALUES (0, 0);
|
||||
CREATE TRIGGER totals_after_insert AFTER INSERT ON files BEGIN
|
||||
UPDATE totals SET records = records + 1, size = size + NEW.size;
|
||||
END;
|
||||
CREATE TRIGGER totals_after_delete AFTER DELETE ON files BEGIN
|
||||
UPDATE totals SET records = records - 1, size = size - OLD.size;
|
||||
END;
|
||||
CREATE TRIGGER totals_after_update AFTER UPDATE OF size ON files BEGIN
|
||||
UPDATE totals SET size = size + NEW.size - OLD.size;
|
||||
END;
|
||||
```
|
||||
|
||||
Paths are stored as BLOBs because Unix paths are raw bytes, not guaranteed
|
||||
@@ -421,17 +406,6 @@ All four subcommands operate on a single SQLite database file:
|
||||
`files_signature` index lets `report` read the records in signature order
|
||||
without sorting the table.
|
||||
|
||||
`totals` holds one row: the number of records in `files` and the sum of
|
||||
their sizes. Three triggers on `files` keep that row current. Each runs in
|
||||
the transaction of the change to `files` that fires it, so the row and the
|
||||
records always agree. `info` reads both numbers from this row. `scan`,
|
||||
`report` and `trees` read the number of records from it as the total of
|
||||
their progress displays. Counting the records in `files` would read every
|
||||
record, which takes minutes on a database of tens of millions. The triggers
|
||||
make each write to `files` slower, so a first scan of many small files takes
|
||||
about two fifths longer. On large files, where reading them takes most of
|
||||
the time, the difference is small.
|
||||
|
||||
### Duplicate detection
|
||||
|
||||
Two files are duplicates when they have the same size and the same three hashes.
|
||||
@@ -553,7 +527,7 @@ hash phase:
|
||||
files; per-directory parallelism is what makes the walk tractable on large
|
||||
or busy pools. The walk compares each file with its record as it goes, and
|
||||
an unchanged file needs nothing more. Every new or changed file is carried
|
||||
to the hash phase as a (path, size, size on disk, mtime) record.
|
||||
to the hash phase as a (path, size, mtime) record.
|
||||
3. **hash** — the worker pool hashes every carried file as described in
|
||||
"Duplicate detection" above, which gives the file its `head`, `tail` and
|
||||
`content`. Zero-length files have constant hashes and are never opened.
|
||||
@@ -714,18 +688,14 @@ Processing:
|
||||
database state.
|
||||
|
||||
`report` runs in two phases, each with its own progress display on stderr (see
|
||||
"Progress"):
|
||||
|
||||
- `open` opens the database.
|
||||
- `read` gets the number of records from `totals`, then reads the records in
|
||||
signature order, largest size first.
|
||||
|
||||
Signature order, largest size first, is the `files_signature` index read
|
||||
backwards, so SQLite sorts nothing and returns the first record at once. The
|
||||
records sharing a signature come together, and so do the records of one size, so
|
||||
`report` holds only the duplicate groups of the size it is reading, and writes
|
||||
them, ordered by `first` path, once it reaches a smaller size. Its memory holds
|
||||
at most the duplicate paths of one size at a time, never every record.
|
||||
"Progress"): `open` opens the database; `read` counts the records, then reads
|
||||
them in signature order, largest size first. That is the order of the
|
||||
`files_signature` index read backwards, so SQLite sorts nothing and returns the
|
||||
first record at once. The records sharing a signature come together, and so do
|
||||
the records of one size, so `report` holds only the duplicate groups of the size
|
||||
it is reading, and writes them, ordered by `first` path, once it reaches a
|
||||
smaller size. Its memory holds at most the duplicate paths of one size at a
|
||||
time, never every record.
|
||||
|
||||
#### Report output format
|
||||
|
||||
@@ -799,37 +769,22 @@ Processing:
|
||||
ascending. Output must be fully deterministic for a given input.
|
||||
|
||||
`trees` runs in four phases, each with its own progress display on stderr (see
|
||||
"Progress"):
|
||||
|
||||
- `open` opens the database.
|
||||
- `read` gets the number of records from `totals`, then reads the records in
|
||||
path order, building the hierarchy and computing the digests as it goes.
|
||||
- `group` groups the directories by digest and picks the maximal groups.
|
||||
- `write` writes the groups to stdout.
|
||||
"Progress"): `open` opens the database; `read` counts the records, then reads
|
||||
them in path order, building the hierarchy and computing the digests as it goes;
|
||||
`group` groups the directories by digest and picks the maximal groups; `write`
|
||||
writes the groups to stdout.
|
||||
|
||||
#### Trees output format
|
||||
|
||||
TSV on stdout: a header line, then one row per duplicate tree (N-1 rows for a
|
||||
group of N). After the `first` and `dupe` paths come three numbers:
|
||||
|
||||
- `files`, the number of regular files in one copy of the tree, counted
|
||||
recursively;
|
||||
- `size`, the bytes of one copy, its files' sizes added up;
|
||||
- `disk_size`, the bytes the `dupe` copy takes on disk, its files' `disk_size`
|
||||
added up.
|
||||
group of N). `files` is the recursive regular-file count of one copy of the
|
||||
tree; `size` is the recursive total byte size of one copy:
|
||||
|
||||
```
|
||||
first dupe files size disk_size
|
||||
/srv/a/project /srv/backup/project 3417 104857600 61865984
|
||||
first dupe files size
|
||||
/srv/a/project /srv/backup/project 3417 104857600
|
||||
```
|
||||
|
||||
A file's `disk_size` is the space its blocks take on disk. It is less than the
|
||||
file's size on a compressed or sparse file, and a little more on most other
|
||||
files. Two copies of one tree can therefore differ in `disk_size`. When a file
|
||||
in one copy is a hard link to one in the other, both copies count its blocks in
|
||||
full. Removing it from one copy then frees nothing, as the known limitation
|
||||
under "`trees` mode" says.
|
||||
|
||||
The `first` and `dupe` paths are escaped as described under "Report output
|
||||
format". The root directory's path is `/`.
|
||||
|
||||
@@ -864,19 +819,15 @@ size of recorded files 150000000000000 (136.4 TiB)
|
||||
database is in WAL mode, the newest changes may be in the `-wal` file alone.
|
||||
- `schema version` is the database's `PRAGMA user_version`. A database with
|
||||
another version than this build's is refused, as by `report`.
|
||||
- `records` is the number of records. `size of recorded files` is the sum of the
|
||||
sizes of all records, in bytes, then in human units.
|
||||
- `records` is the number of records. `size of recorded files` adds up the sizes
|
||||
of all records, in bytes, then in human units.
|
||||
|
||||
`info` runs in two phases, each with its own progress display on stderr (see
|
||||
"Progress"):
|
||||
|
||||
- `open` opens the database.
|
||||
- `count` reads the schema version and the row of `totals` (see "Database").
|
||||
Both reads run in one read-only transaction, so the numbers agree with each
|
||||
other while a scan is committing.
|
||||
|
||||
`info` writes its lines once the `count` display has ended, so on a terminal
|
||||
they come after the displays.
|
||||
"Progress"): `open` opens the database; `count` reads the schema version and
|
||||
counts the records with one query, a single pass over the table, both in one
|
||||
read-only transaction, so the numbers agree with each other while a scan is
|
||||
committing. `info` writes its lines once the `count` display has ended, so on a
|
||||
terminal they come after the displays.
|
||||
|
||||
### Progress
|
||||
|
||||
@@ -892,9 +843,8 @@ progress, its elapsed time and its time remaining.
|
||||
A scan has four phases (`[1/4] load`, `[2/4] walk`, `[3/4] hash`,
|
||||
`[4/4] update`):
|
||||
|
||||
- `load` gets the number of records from `totals` first, then reads the records
|
||||
against that number. Until it has the number, the display says it is counting
|
||||
the records.
|
||||
- `load` counts the records first, then reads them against that count. While the
|
||||
count runs, the display says it is counting the records.
|
||||
- `walk` has no exact total. Its estimate is the number of records the last scan
|
||||
left under the operands, known once `load` is done and shown with a `~`. The
|
||||
time remaining is unknown once the walk passes the estimate, and when there
|
||||
@@ -907,9 +857,8 @@ A scan has four phases (`[1/4] load`, `[2/4] walk`, `[3/4] hash`,
|
||||
|
||||
- `open` has no total. While it opens the database, waiting on a busy one
|
||||
included, the display says it is opening the database.
|
||||
- `read` gets the number of records from `totals` first, then reads the records
|
||||
against that number. Until it has the number, the display says it is counting
|
||||
the records.
|
||||
- `read` counts the records first, then reads them against that count. While the
|
||||
count runs, the display says it is counting the records.
|
||||
- `group` has the directories `read` found as its total, and `write` the
|
||||
duplicate-tree groups.
|
||||
|
||||
@@ -917,16 +866,17 @@ A scan has four phases (`[1/4] load`, `[2/4] walk`, `[3/4] hash`,
|
||||
|
||||
- `open` has no total. While it opens the database, waiting on a busy one
|
||||
included, the display says it is opening the database.
|
||||
- `read` gets the number of records from `totals` first, then reads the records
|
||||
against that number, writing the rows to stdout as it goes. Until it has the
|
||||
number, the display says it is counting the records.
|
||||
- `read` counts the records first, then reads them against that count, writing
|
||||
the rows to stdout as it goes. While the count runs, the display says it is
|
||||
counting the records.
|
||||
|
||||
`info` has two phases (`[1/2] open`, `[2/2] count`, see "`info` mode"):
|
||||
|
||||
- `open` has no total. While it opens the database, waiting on a busy one
|
||||
included, the display says it is opening the database.
|
||||
- `count` has no total either, because it reads one row: the display says it is
|
||||
counting the records until it has the row, and its time remaining is unknown.
|
||||
- `count` has no total either, because one query counts everything: the display
|
||||
says it is counting the records until the query returns, and its time
|
||||
remaining is unknown.
|
||||
|
||||
Each display is one line holding:
|
||||
|
||||
@@ -1213,9 +1163,9 @@ All of the following, run in this directory, must pass:
|
||||
ordered by size descending.
|
||||
|
||||
Expected from `trees`: exactly one row — `first` `$d/t1`, `dupe` `$d/t2`, 2
|
||||
files, 3100 bytes, and a `disk_size` that depends on the filesystem.
|
||||
`$d/t1/sub` vs `$d/t2/sub` is suppressed as non-maximal (implied by the
|
||||
`t1`/`t2` group), and `t3` appears nowhere (its file set differs by name).
|
||||
files, 3100 bytes. `$d/t1/sub` vs `$d/t2/sub` is suppressed as non-maximal
|
||||
(implied by the `t1`/`t2` group), and `t3` appears nowhere (its file set
|
||||
differs by name).
|
||||
|
||||
Expected from the second `report` (after the modify/delete rescan):
|
||||
`one.bin` has left its group (its content changed), so
|
||||
|
||||
@@ -47,15 +47,14 @@ const dbFilePerm = 0o644
|
||||
const lockFilePerm = 0o600
|
||||
|
||||
// createTableSQL is the schema applied to a fresh database. Paths are
|
||||
// BLOBs because Unix paths are raw bytes, not guaranteed UTF-8.
|
||||
// disk_size holds the bytes the file takes on disk. mtime holds whole
|
||||
// Unix seconds and mtime_nsec the nanoseconds within that second. head,
|
||||
// tail and content each hold the raw 32 bytes of a SHA-256.
|
||||
// BLOBs because Unix paths are raw bytes, not guaranteed UTF-8. mtime
|
||||
// holds whole Unix seconds and mtime_nsec the nanoseconds within that
|
||||
// second. head, tail and content each hold the raw 32 bytes of a
|
||||
// SHA-256.
|
||||
const createTableSQL = `
|
||||
CREATE TABLE files (
|
||||
path BLOB PRIMARY KEY,
|
||||
size INTEGER NOT NULL,
|
||||
disk_size INTEGER NOT NULL,
|
||||
mtime INTEGER NOT NULL,
|
||||
mtime_nsec INTEGER NOT NULL,
|
||||
head BLOB NOT NULL,
|
||||
@@ -70,59 +69,14 @@ const createIndexSQL = `
|
||||
CREATE INDEX files_signature ON files (size, head, tail, content)
|
||||
`
|
||||
|
||||
// createTotalsSQL creates the totals table, which holds one row: the
|
||||
// number of records and the sizes of all records added up. Counting
|
||||
// them over files would read every record. The three triggers below
|
||||
// keep the row current, in the transaction of each change to files.
|
||||
const createTotalsSQL = `
|
||||
CREATE TABLE totals (
|
||||
records INTEGER NOT NULL,
|
||||
size INTEGER NOT NULL
|
||||
)
|
||||
`
|
||||
|
||||
// insertTotalsSQL adds the one row of totals, for an empty files table.
|
||||
const insertTotalsSQL = `
|
||||
INSERT INTO totals (records, size) VALUES (0, 0)
|
||||
`
|
||||
|
||||
// totalsAfterInsertSQL counts each inserted record into totals.
|
||||
const totalsAfterInsertSQL = `
|
||||
CREATE TRIGGER totals_after_insert AFTER INSERT ON files BEGIN
|
||||
UPDATE totals SET records = records + 1, size = size + NEW.size;
|
||||
END
|
||||
`
|
||||
|
||||
// totalsAfterDeleteSQL takes each deleted record out of totals.
|
||||
const totalsAfterDeleteSQL = `
|
||||
CREATE TRIGGER totals_after_delete AFTER DELETE ON files BEGIN
|
||||
UPDATE totals SET records = records - 1, size = size - OLD.size;
|
||||
END
|
||||
`
|
||||
|
||||
// totalsAfterUpdateSQL replaces a record's old size in totals with its
|
||||
// new one. scan's upsert of an existing record is such an update.
|
||||
const totalsAfterUpdateSQL = `
|
||||
CREATE TRIGGER totals_after_update AFTER UPDATE OF size ON files BEGIN
|
||||
UPDATE totals SET size = size + NEW.size - OLD.size;
|
||||
END
|
||||
`
|
||||
|
||||
// recordCountSQL reads the number of records from totals.
|
||||
const recordCountSQL = `
|
||||
SELECT records FROM totals
|
||||
`
|
||||
|
||||
// upsertSQL inserts one file record, replacing any existing record for
|
||||
// the same path.
|
||||
const upsertSQL = `
|
||||
INSERT INTO files (
|
||||
path, size, disk_size, mtime, mtime_nsec, head, tail, content
|
||||
)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?)
|
||||
INSERT INTO files (path, size, mtime, mtime_nsec, head, tail, content)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?)
|
||||
ON CONFLICT (path) DO UPDATE SET
|
||||
size = excluded.size, disk_size = excluded.disk_size,
|
||||
mtime = excluded.mtime, mtime_nsec = excluded.mtime_nsec,
|
||||
size = excluded.size, mtime = excluded.mtime,
|
||||
mtime_nsec = excluded.mtime_nsec,
|
||||
head = excluded.head, tail = excluded.tail,
|
||||
content = excluded.content
|
||||
`
|
||||
@@ -196,14 +150,9 @@ func createDatabaseDirectory(path string) error {
|
||||
// scanParams are the connection parameters for scan: read-write, with
|
||||
// WAL journaling and a busy timeout, so a report can run while a cron
|
||||
// scan is in progress. closeScanDatabase leaves WAL mode again.
|
||||
//
|
||||
// Each write to files fires a trigger, so SQLite keeps a statement
|
||||
// journal for it. temp_store(MEMORY) keeps that journal in memory,
|
||||
// which costs far less than a temporary file.
|
||||
const scanParams = "_pragma=busy_timeout(10000)" +
|
||||
"&_pragma=journal_mode(WAL)" +
|
||||
"&_pragma=synchronous(NORMAL)" +
|
||||
"&_pragma=temp_store(MEMORY)"
|
||||
"&_pragma=synchronous(NORMAL)"
|
||||
|
||||
// reportParams are the connection parameters for report, trees and
|
||||
// info: read-only, with the same busy timeout. They set no journal
|
||||
@@ -477,14 +426,14 @@ func createSchema(ctx context.Context, db *sql.DB) error {
|
||||
|
||||
defer func() { _ = tx.Rollback() }()
|
||||
|
||||
for _, stmt := range []string{
|
||||
createTableSQL, createIndexSQL, createTotalsSQL, insertTotalsSQL,
|
||||
totalsAfterInsertSQL, totalsAfterDeleteSQL, totalsAfterUpdateSQL,
|
||||
} {
|
||||
_, err = tx.ExecContext(ctx, stmt)
|
||||
if err != nil {
|
||||
return fmt.Errorf("create schema: %w", err)
|
||||
}
|
||||
_, err = tx.ExecContext(ctx, createTableSQL)
|
||||
if err != nil {
|
||||
return fmt.Errorf("create schema: %w", err)
|
||||
}
|
||||
|
||||
_, err = tx.ExecContext(ctx, createIndexSQL)
|
||||
if err != nil {
|
||||
return fmt.Errorf("create schema: %w", err)
|
||||
}
|
||||
|
||||
_, err = tx.ExecContext(ctx,
|
||||
@@ -531,7 +480,7 @@ func loadFileRows(ctx context.Context, db *sql.DB, total func(n int64),
|
||||
|
||||
var n int64
|
||||
|
||||
err = tx.QueryRowContext(ctx, recordCountSQL).Scan(&n)
|
||||
err = tx.QueryRowContext(ctx, "SELECT COUNT(*) FROM files").Scan(&n)
|
||||
if err != nil {
|
||||
return fmt.Errorf("count records: %w", err)
|
||||
}
|
||||
@@ -539,8 +488,8 @@ func loadFileRows(ctx context.Context, db *sql.DB, total func(n int64),
|
||||
total(n)
|
||||
|
||||
rows, err := tx.QueryContext(ctx,
|
||||
"SELECT path, size, disk_size, mtime, mtime_nsec, head, tail, "+
|
||||
"content FROM files ORDER BY path")
|
||||
"SELECT path, size, mtime, mtime_nsec, head, tail, content "+
|
||||
"FROM files ORDER BY path")
|
||||
if err != nil {
|
||||
return fmt.Errorf("read records: %w", err)
|
||||
}
|
||||
@@ -554,8 +503,7 @@ func loadFileRows(ctx context.Context, db *sql.DB, total func(n int64),
|
||||
r scanRec
|
||||
)
|
||||
|
||||
err = rows.Scan(&path, &r.size, &r.diskSize, &sec, &nsec,
|
||||
&head, &tail, &content)
|
||||
err = rows.Scan(&path, &r.size, &sec, &nsec, &head, &tail, &content)
|
||||
if err != nil {
|
||||
return fmt.Errorf("read record: %w", err)
|
||||
}
|
||||
@@ -590,11 +538,10 @@ ORDER BY size DESC, head DESC, tail DESC, content DESC
|
||||
`
|
||||
|
||||
// loadSignatureRows gives total the number of records, then streams
|
||||
// every record's signature and path to fn, without its mtime or its
|
||||
// disk_size, in the order of signatureRowsSQL. The count and the
|
||||
// records are read in one transaction, so they agree while a scan is
|
||||
// committing. An error from fn stops the reading and is returned as it
|
||||
// is.
|
||||
// every record's signature and path to fn, without its mtime, in the
|
||||
// order of signatureRowsSQL. The count and the records are read in one
|
||||
// transaction, so they agree while a scan is committing. An error from
|
||||
// fn stops the reading and is returned as it is.
|
||||
func loadSignatureRows(ctx context.Context, db *sql.DB,
|
||||
total func(n int64), fn func(r scanRec) error,
|
||||
) error {
|
||||
@@ -609,7 +556,7 @@ func loadSignatureRows(ctx context.Context, db *sql.DB,
|
||||
|
||||
var n int64
|
||||
|
||||
err = tx.QueryRowContext(ctx, recordCountSQL).Scan(&n)
|
||||
err = tx.QueryRowContext(ctx, "SELECT COUNT(*) FROM files").Scan(&n)
|
||||
if err != nil {
|
||||
return fmt.Errorf("count records: %w", err)
|
||||
}
|
||||
@@ -675,7 +622,7 @@ func (r *scanRec) setHashes(head, tail, content []byte) error {
|
||||
func countRecords(ctx context.Context, db *sql.DB) (int64, error) {
|
||||
var n int64
|
||||
|
||||
err := db.QueryRowContext(ctx, recordCountSQL).Scan(&n)
|
||||
err := db.QueryRowContext(ctx, "SELECT COUNT(*) FROM files").Scan(&n)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("count records: %w", err)
|
||||
}
|
||||
@@ -683,10 +630,12 @@ func countRecords(ctx context.Context, db *sql.DB) (int64, error) {
|
||||
return n, nil
|
||||
}
|
||||
|
||||
// databaseCountsSQL reads the number of records and the sizes of all
|
||||
// records added up from totals.
|
||||
// databaseCountsSQL counts the records and adds up their sizes, in one
|
||||
// pass over the table. A SUM over no rows is NULL, so on an empty table
|
||||
// the SUM falls back to 0.
|
||||
const databaseCountsSQL = `
|
||||
SELECT records, size FROM totals
|
||||
SELECT COUNT(*), COALESCE(SUM(size), 0)
|
||||
FROM files
|
||||
`
|
||||
|
||||
// databaseCounts is what info reads from the database: its schema
|
||||
@@ -838,7 +787,7 @@ func execUpserts(ctx context.Context, tx *sql.Tx, upserts []scanRec,
|
||||
defer func() { _ = st.Close() }()
|
||||
|
||||
for _, r := range upserts {
|
||||
_, err = st.ExecContext(ctx, []byte(r.path), r.size, r.diskSize,
|
||||
_, err = st.ExecContext(ctx, []byte(r.path), r.size,
|
||||
r.mtime.Unix(), r.mtime.Nanosecond(),
|
||||
r.head[:], r.tail[:], r.content[:])
|
||||
if err != nil {
|
||||
|
||||
+10
-157
@@ -535,12 +535,17 @@ func TestLoadRowsRefuseWrongSizeHash(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// queryPlan returns the detail of each line of SQLite's EXPLAIN QUERY
|
||||
// PLAN for query.
|
||||
func queryPlan(t *testing.T, db *sql.DB, query string) []string {
|
||||
t.Helper()
|
||||
// TestSignatureRowsNeedNoSort checks that SQLite reads signatureRowsSQL
|
||||
// from the files_signature index alone, with nothing to sort: the
|
||||
// first record comes at once, so report's read phase counts each
|
||||
// record as it comes rather than waiting on SQLite for all of them.
|
||||
func TestSignatureRowsNeedNoSort(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
rows, err := db.QueryContext(t.Context(), "EXPLAIN QUERY PLAN "+query)
|
||||
db := openTestDB(t)
|
||||
|
||||
rows, err := db.QueryContext(t.Context(),
|
||||
"EXPLAIN QUERY PLAN "+signatureRowsSQL)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -568,164 +573,12 @@ func queryPlan(t *testing.T, db *sql.DB, query string) []string {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
return plan
|
||||
}
|
||||
|
||||
// TestSignatureRowsNeedNoSort checks that SQLite reads signatureRowsSQL
|
||||
// from the files_signature index alone, with nothing to sort: the
|
||||
// first record comes at once, so report's read phase counts each
|
||||
// record as it comes rather than waiting on SQLite for all of them.
|
||||
func TestSignatureRowsNeedNoSort(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
db := openTestDB(t)
|
||||
|
||||
plan := queryPlan(t, db, signatureRowsSQL)
|
||||
|
||||
want := []string{"SCAN files USING COVERING INDEX files_signature"}
|
||||
if !slices.Equal(plan, want) {
|
||||
t.Errorf("query plan = %q, want %q", plan, want)
|
||||
}
|
||||
}
|
||||
|
||||
// TestCountsReadOnlyTotals checks that SQLite answers the queries for
|
||||
// the number of records and their total size from the one row of
|
||||
// totals. A plan that reads files or its index reads every record.
|
||||
func TestCountsReadOnlyTotals(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
db := openTestDB(t)
|
||||
|
||||
for _, query := range []string{recordCountSQL, databaseCountsSQL} {
|
||||
plan := queryPlan(t, db, query)
|
||||
|
||||
want := []string{"SCAN totals"}
|
||||
if !slices.Equal(plan, want) {
|
||||
t.Errorf("query plan of %q = %q, want %q", query, plan, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestCountsComeFromTotals sets the row of totals to numbers that files
|
||||
// does not hold, then checks that scan's load phase, the read phases of
|
||||
// trees and report, and info each report the numbers from totals.
|
||||
func TestCountsComeFromTotals(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
// files is empty, so a count over files gives 0 records of 0 bytes.
|
||||
const records, size = 7, 9000
|
||||
|
||||
db := openTestDB(t)
|
||||
|
||||
_, err := db.ExecContext(t.Context(),
|
||||
"UPDATE totals SET records = ?, size = ?", records, size)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
scanTotal, err := countRecords(t.Context(), db)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
var treesTotal, reportTotal int64
|
||||
|
||||
err = loadFileRows(t.Context(), db,
|
||||
func(n int64) { treesTotal = n }, func(scanRec) {})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
err = loadSignatureRows(t.Context(), db,
|
||||
func(n int64) { reportTotal = n },
|
||||
func(scanRec) error { return nil })
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
info, err := readDatabaseCounts(t.Context(), db)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
for name, got := range map[string]int64{
|
||||
"scan's load": scanTotal,
|
||||
"trees' read": treesTotal,
|
||||
"report's read": reportTotal,
|
||||
"info": info.records,
|
||||
} {
|
||||
if got != records {
|
||||
t.Errorf("%s: %d records, want %d from totals",
|
||||
name, got, records)
|
||||
}
|
||||
}
|
||||
|
||||
if info.size != size {
|
||||
t.Errorf("info: size %d, want %d from totals", info.size, size)
|
||||
}
|
||||
}
|
||||
|
||||
// assertTotalsMatchFiles fails unless the row of totals holds the
|
||||
// number of records and the sizes of all records added up, as counted
|
||||
// over files.
|
||||
func assertTotalsMatchFiles(t *testing.T, db *sql.DB) {
|
||||
t.Helper()
|
||||
|
||||
got, err := readDatabaseCounts(t.Context(), db)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
var records, size int64
|
||||
|
||||
err = db.QueryRowContext(t.Context(),
|
||||
"SELECT COUNT(*), COALESCE(SUM(size), 0) FROM files").
|
||||
Scan(&records, &size)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
if got.records != records || got.size != size {
|
||||
t.Errorf("totals = %d records of %d bytes, files holds %d of %d",
|
||||
got.records, got.size, records, size)
|
||||
}
|
||||
}
|
||||
|
||||
// TestTotalsMatchFiles scans the smoke tree, rescans it after two files
|
||||
// change size, and rescans it after two files are removed. After each
|
||||
// scan the row of totals must agree with files.
|
||||
func TestTotalsMatchFiles(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
dir := buildSmokeTree(t)
|
||||
db := openTestDB(t)
|
||||
|
||||
syncTree(t, db, dir)
|
||||
assertTotalsMatchFiles(t, db)
|
||||
|
||||
writeFile(t, dir, "a/one.bin", pattern(10, 2500))
|
||||
writeFile(t, dir, "t1/f1", pattern(30, 1000))
|
||||
|
||||
if st := syncTree(t, db, dir); st.updated != 2 {
|
||||
t.Fatalf("rescan stats = %+v, want 2 updated", st)
|
||||
}
|
||||
|
||||
assertTotalsMatchFiles(t, db)
|
||||
|
||||
for _, name := range []string{"a/unique.bin", "t2/sub/f2"} {
|
||||
err := os.Remove(filepath.Join(dir, name))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
if st := syncTree(t, db, dir); st.removed != 2 {
|
||||
t.Fatalf("rescan stats = %+v, want 2 removed", st)
|
||||
}
|
||||
|
||||
assertTotalsMatchFiles(t, db)
|
||||
}
|
||||
|
||||
func TestApplyChangesRoundTrip(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
|
||||
@@ -26,11 +26,11 @@ func infoPhase(name string) string {
|
||||
// it. Any database problem, including a missing database, is fatal.
|
||||
//
|
||||
// The work runs in two phases, each under a progress display of its
|
||||
// own on stderr: open (open the database) and count (read the number of
|
||||
// records and their total size from the totals table). Each display
|
||||
// ends before the next phase starts, and before an error is returned
|
||||
// to be printed. The details are written once the last display has
|
||||
// ended, so no display is drawn while they are written.
|
||||
// own on stderr: open (open the database) and count (count the records,
|
||||
// with one query). Each display ends before the next phase starts, and
|
||||
// before an error is returned to be printed. The details are written
|
||||
// once the last display has ended, so no display is drawn while they
|
||||
// are written.
|
||||
func runInfo(ctx context.Context, stdout io.Writer) error {
|
||||
dbPath, source, err := databasePath()
|
||||
if err != nil {
|
||||
@@ -56,7 +56,7 @@ func runInfo(ctx context.Context, stdout io.Writer) error {
|
||||
|
||||
defer func() { _ = db.Close() }()
|
||||
|
||||
// One row holds both counts, so the phase never has a total.
|
||||
// One query counts everything, so the phase never has a total.
|
||||
prog = newProgress(progressParams{
|
||||
label: infoPhase("count"), total: -1,
|
||||
waitingFor: countingRecords,
|
||||
|
||||
+11
-15
@@ -180,9 +180,9 @@ func capture(t *testing.T, std **os.File) func() string {
|
||||
}
|
||||
|
||||
// brokenDatabase writes a database that opens cleanly and passes the
|
||||
// schema-version check but has no tables, so the first query fails
|
||||
// with the database already open: a fatal error on a path that owns an
|
||||
// open database. It closes the database the way scan does.
|
||||
// schema-version check but has no files table, so the first query
|
||||
// fails with the database already open: a fatal error on a path that
|
||||
// owns an open database. It closes the database the way scan does.
|
||||
func brokenDatabase(t *testing.T) string {
|
||||
t.Helper()
|
||||
|
||||
@@ -265,9 +265,8 @@ func TestRunFatalAfterOpenClosesDatabase(t *testing.T) {
|
||||
assertFatalOutput(t, stderr.String(), stdout())
|
||||
|
||||
// Proof that the failure happened after the open: only a
|
||||
// query against the opened database can report this. Each
|
||||
// subcommand's first query reads totals.
|
||||
if !strings.Contains(stderr.String(), "no such table: totals") {
|
||||
// query against the opened database can report this.
|
||||
if !strings.Contains(stderr.String(), "no such table: files") {
|
||||
t.Errorf("stderr = %q, want the failure to come from a "+
|
||||
"query on the open database", stderr.String())
|
||||
}
|
||||
@@ -1141,9 +1140,8 @@ func TestRunTreesSucceeds(t *testing.T) {
|
||||
|
||||
// The two directories holding the duplicate pair are duplicate
|
||||
// trees of each other.
|
||||
want := "first\tdupe\tfiles\tsize\tdisk_size\n" +
|
||||
filepath.Dir(dupes[0]) + "\t" + filepath.Dir(dupes[1]) + "\t1\t300\t" +
|
||||
strconv.FormatInt(diskSizeOf(t, dupes[1]), 10) + "\n"
|
||||
want := "first\tdupe\tfiles\tsize\n" +
|
||||
filepath.Dir(dupes[0]) + "\t" + filepath.Dir(dupes[1]) + "\t1\t300\n"
|
||||
if got := stdout.String(); got != want {
|
||||
t.Errorf("stdout = %q, want %q", got, want)
|
||||
}
|
||||
@@ -1177,10 +1175,9 @@ func TestRunReportsNeedOnlyReadAccess(t *testing.T) {
|
||||
cases := map[string]string{
|
||||
cmdReport: "first\tdupe\tsize\n" +
|
||||
dupes[0] + "\t" + dupes[1] + "\t300\n",
|
||||
cmdTrees: "first\tdupe\tfiles\tsize\tdisk_size\n" +
|
||||
cmdTrees: "first\tdupe\tfiles\tsize\n" +
|
||||
filepath.Dir(dupes[0]) + "\t" + filepath.Dir(dupes[1]) +
|
||||
"\t1\t300\t" + strconv.FormatInt(diskSizeOf(t, dupes[1]), 10) +
|
||||
"\n",
|
||||
"\t1\t300\n",
|
||||
}
|
||||
|
||||
for name, want := range cases {
|
||||
@@ -1215,9 +1212,8 @@ func assertRunsUseDatabase(t *testing.T, dir, name string) {
|
||||
t.Errorf("report stdout = %q, want %q", got, want)
|
||||
}
|
||||
|
||||
want = "first\tdupe\tfiles\tsize\tdisk_size\n" +
|
||||
filepath.Dir(dupes[0]) + "\t" + filepath.Dir(dupes[1]) + "\t1\t300\t" +
|
||||
strconv.FormatInt(diskSizeOf(t, dupes[1]), 10) + "\n"
|
||||
want = "first\tdupe\tfiles\tsize\n" +
|
||||
filepath.Dir(dupes[0]) + "\t" + filepath.Dir(dupes[1]) + "\t1\t300\n"
|
||||
if got := runStdout(t, cmdTrees); got != want {
|
||||
t.Errorf("trees stdout = %q, want %q", got, want)
|
||||
}
|
||||
|
||||
+1
-1
@@ -60,7 +60,7 @@ const fallbackWidth = 80
|
||||
const spinnerMargin = 5
|
||||
|
||||
// countingRecords is what the display of a phase that reads the records
|
||||
// says until it has the number of records, its total.
|
||||
// says while it counts them, before it has its total.
|
||||
const countingRecords = "counting the records"
|
||||
|
||||
// openPhase is the name of the phase in which report, trees and info
|
||||
|
||||
@@ -21,16 +21,14 @@ const minGroupSize = 2
|
||||
|
||||
// scanRec is one file record from the database. The signature (size,
|
||||
// head, tail, content) is the duplicate key; mtime is informational
|
||||
// only and used by scan for change detection. diskSize is the bytes the
|
||||
// file takes on disk, which trees adds up.
|
||||
// only and used by scan for change detection.
|
||||
type scanRec struct {
|
||||
size int64
|
||||
diskSize int64
|
||||
mtime time.Time
|
||||
head [sha256.Size]byte
|
||||
tail [sha256.Size]byte
|
||||
content [sha256.Size]byte
|
||||
path string
|
||||
size int64
|
||||
mtime time.Time
|
||||
head [sha256.Size]byte
|
||||
tail [sha256.Size]byte
|
||||
content [sha256.Size]byte
|
||||
path string
|
||||
}
|
||||
|
||||
// reportPhase names one phase of report with its place among all of
|
||||
@@ -47,11 +45,11 @@ func reportPhase(name string) string {
|
||||
// problem, including a missing database, is fatal.
|
||||
//
|
||||
// The work runs in two phases, each under a progress display of its
|
||||
// own on stderr: open (open the database) and read (read the number of
|
||||
// records from the totals table, then read the records in signature
|
||||
// order, writing each size's duplicate groups to stdout once that size
|
||||
// is read; see readReport). Each display ends before the next phase
|
||||
// starts, and before an error is returned to be printed.
|
||||
// own on stderr: open (open the database) and read (count the records,
|
||||
// then read them in signature order, writing each size's duplicate
|
||||
// groups to stdout once that size is read; see readReport). Each
|
||||
// display ends before the next phase starts, and before an error is
|
||||
// returned to be printed.
|
||||
func runReport(ctx context.Context, stdout io.Writer) error {
|
||||
dbPath, source, err := databasePath()
|
||||
if err != nil {
|
||||
|
||||
+2
-2
@@ -38,8 +38,8 @@ func awkwardPairRecs() []scanRec {
|
||||
h, tl, c := testHash("h"), testHash("t"), testHash("c")
|
||||
|
||||
return []scanRec{
|
||||
{size: 5, diskSize: 4096, head: h, tail: tl, content: c, path: "/d/A/f"},
|
||||
{size: 5, diskSize: 4096, head: h, tail: tl, content: c, path: awkwardDir + "/f"},
|
||||
{size: 5, head: h, tail: tl, content: c, path: "/d/A/f"},
|
||||
{size: 5, head: h, tail: tl, content: c, path: awkwardDir + "/f"},
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -52,17 +52,15 @@ const workQueueDepth = 1024
|
||||
// has already printed its line, so run prints nothing more.
|
||||
var errInterrupted = errors.New("scan interrupted")
|
||||
|
||||
// fileRec carries one statted file between the scan phases. diskSize is
|
||||
// the bytes the file takes on disk. dev and ino identify the underlying
|
||||
// inode so hard-linked paths can share one read; both are zero when the
|
||||
// platform exposes no inode.
|
||||
// fileRec carries one statted file between the scan phases. dev and
|
||||
// ino identify the underlying inode so hard-linked paths can share
|
||||
// one read; both are zero when the platform exposes no inode.
|
||||
type fileRec struct {
|
||||
path string
|
||||
size int64
|
||||
diskSize int64
|
||||
mtime time.Time
|
||||
dev uint64
|
||||
ino uint64
|
||||
path string
|
||||
size int64
|
||||
mtime time.Time
|
||||
dev uint64
|
||||
ino uint64
|
||||
}
|
||||
|
||||
// fileMeta is the in-memory index entry for one existing database
|
||||
@@ -457,8 +455,8 @@ func (s *scanState) walkableRoots(roots []string, rules walkRules) []string {
|
||||
// included, so its display counts against the number of records.
|
||||
func (s *scanState) loadIndex(ctx context.Context, roots []string) error {
|
||||
// Indexing tens of millions of records takes real time; without a
|
||||
// display the scan looks hung before the walk begins. The number of
|
||||
// records, read first, gives the display its total.
|
||||
// display the scan looks hung before the walk begins. Counting them
|
||||
// first gives the display its total.
|
||||
prog := newProgress(progressParams{
|
||||
label: scanPhase("load"), unit: scanUnit, total: -1,
|
||||
waitingFor: countingRecords, stalledOn: "db",
|
||||
@@ -697,13 +695,12 @@ func (s *scanState) recordRun(ctx context.Context, r hashResult) error {
|
||||
s.resolve(rec.path)
|
||||
|
||||
s.batch = append(s.batch, scanRec{
|
||||
size: rec.size,
|
||||
diskSize: rec.diskSize,
|
||||
mtime: rec.mtime,
|
||||
head: r.head,
|
||||
tail: r.tail,
|
||||
content: r.content,
|
||||
path: rec.path,
|
||||
size: rec.size,
|
||||
mtime: rec.mtime,
|
||||
head: r.head,
|
||||
tail: r.tail,
|
||||
content: r.content,
|
||||
path: rec.path,
|
||||
})
|
||||
}
|
||||
|
||||
@@ -964,12 +961,11 @@ func seedRoot(ctx context.Context, root string, rules walkRules,
|
||||
|
||||
dev, ino := inodeOfInfo(fi)
|
||||
rec := fileRec{
|
||||
path: root,
|
||||
size: fi.Size(),
|
||||
diskSize: diskSizeOfInfo(fi),
|
||||
mtime: fi.ModTime(),
|
||||
dev: dev,
|
||||
ino: ino,
|
||||
path: root,
|
||||
size: fi.Size(),
|
||||
mtime: fi.ModTime(),
|
||||
dev: dev,
|
||||
ino: ino,
|
||||
}
|
||||
|
||||
if !isDatabaseFile(rec, rules) {
|
||||
@@ -1149,12 +1145,11 @@ func emitFile(ctx context.Context, p string, e fs.DirEntry,
|
||||
|
||||
dev, ino := inodeOfInfo(info)
|
||||
rec := fileRec{
|
||||
path: p,
|
||||
size: info.Size(),
|
||||
diskSize: diskSizeOfInfo(info),
|
||||
mtime: info.ModTime(),
|
||||
dev: dev,
|
||||
ino: ino,
|
||||
path: p,
|
||||
size: info.Size(),
|
||||
mtime: info.ModTime(),
|
||||
dev: dev,
|
||||
ino: ino,
|
||||
}
|
||||
|
||||
if !isDatabaseFile(rec, rules) {
|
||||
@@ -1246,23 +1241,6 @@ func inodeOfInfo(fi fs.FileInfo) (uint64, uint64) {
|
||||
return statDev(st), st.Ino
|
||||
}
|
||||
|
||||
// statBlockSize is the unit of st_blocks in an lstat result on Linux and
|
||||
// macOS, whatever the filesystem's own block size.
|
||||
const statBlockSize = 512
|
||||
|
||||
// diskSizeOfInfo returns the bytes a file takes on disk: its allocated
|
||||
// blocks, st_blocks × 512. On a compressed or sparse file it is less
|
||||
// than the file's size. It is 0 when the platform exposes no block
|
||||
// count.
|
||||
func diskSizeOfInfo(fi fs.FileInfo) int64 {
|
||||
st, ok := fi.Sys().(*syscall.Stat_t)
|
||||
if !ok {
|
||||
return 0
|
||||
}
|
||||
|
||||
return st.Blocks * statBlockSize
|
||||
}
|
||||
|
||||
// hashResult carries the hashes computed for one inode run (or the
|
||||
// error that prevented computing them) from the pool's workers to the
|
||||
// hash phase. Its run leaves out the paths found removed.
|
||||
|
||||
+3
-58
@@ -42,21 +42,6 @@ func writeFile(t *testing.T, dir, name string, data []byte) string {
|
||||
return p
|
||||
}
|
||||
|
||||
// diskSizeOf returns the bytes the file at path takes on disk, from its
|
||||
// own lstat: st_blocks counts 512-byte units.
|
||||
func diskSizeOf(t *testing.T, path string) int64 {
|
||||
t.Helper()
|
||||
|
||||
var st syscall.Stat_t
|
||||
|
||||
err := syscall.Lstat(path, &st)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
return st.Blocks * 512
|
||||
}
|
||||
|
||||
// pattern returns n bytes of deterministic content seeded by tag.
|
||||
func pattern(tag byte, n int) []byte {
|
||||
data := make([]byte, n)
|
||||
@@ -1243,9 +1228,9 @@ func TestScanPipeline(t *testing.T) {
|
||||
// TestScanProgressNamesEachPhase checks the plain progress lines of a
|
||||
// first scan, and of a rescan of one operand holding two of the
|
||||
// database's records. Each phase's first line names it with its place
|
||||
// among the four: load gets the number of records before reading them,
|
||||
// and the walk estimates its total from the records under its operand,
|
||||
// or says its time remaining is unknown when there are none.
|
||||
// among the four: load counts the records before reading them, and the
|
||||
// walk estimates its total from the records under its operand, or says
|
||||
// its time remaining is unknown when there are none.
|
||||
//
|
||||
//nolint:paralleltest // captureStderr replaces the process-wide os.Stderr
|
||||
func TestScanProgressNamesEachPhase(t *testing.T) {
|
||||
@@ -1431,46 +1416,6 @@ func TestScanStoresRawHashes(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// TestScanStoresDiskSize scans a sparse file, which takes less space on
|
||||
// disk than its length, and checks the disk_size stored for it against
|
||||
// the file's own lstat.
|
||||
func TestScanStoresDiskSize(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
dir := t.TempDir()
|
||||
db := openTestDB(t)
|
||||
p := writeFile(t, dir, "sparse", []byte("x"))
|
||||
|
||||
// Truncate makes the file longer without giving it blocks for the
|
||||
// new bytes.
|
||||
const length = 1024 * 1024
|
||||
|
||||
err := os.Truncate(p, length)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
syncTree(t, db, dir)
|
||||
|
||||
want := diskSizeOf(t, p)
|
||||
if want >= length {
|
||||
t.Fatalf("the file takes %d bytes on disk, want fewer than its "+
|
||||
"length %d", want, length)
|
||||
}
|
||||
|
||||
var got int64
|
||||
|
||||
err = db.QueryRowContext(t.Context(),
|
||||
"SELECT disk_size FROM files WHERE path = ?", []byte(p)).Scan(&got)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
if got != want {
|
||||
t.Errorf("disk_size = %d, want %d", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
// TestSyncScanSameSecondRewrite rewrites a file in place at the same
|
||||
// size with an mtime later in the same second as the recorded one: the
|
||||
// next scan must notice the change and re-hash the file. The operand is
|
||||
|
||||
@@ -22,10 +22,6 @@ type treeNode struct {
|
||||
digest [sha256.Size]byte
|
||||
fileCount int64
|
||||
totalSize int64
|
||||
// diskSize is the bytes the directory's files take on disk. Two
|
||||
// copies of one tree have the same totalSize, but their diskSize
|
||||
// can differ.
|
||||
diskSize int64
|
||||
}
|
||||
|
||||
// treesPhase names one phase of trees with its place among all of them,
|
||||
@@ -44,11 +40,11 @@ func treesPhase(name string) string {
|
||||
// database problem, including a missing database, is fatal.
|
||||
//
|
||||
// The work runs in four phases, each under a progress display of its
|
||||
// own on stderr: open (open the database), read (read the number of
|
||||
// records from the totals table, then read the records into the
|
||||
// hierarchy), group (group the directories by digest) and write (write
|
||||
// the groups to stdout). Each display ends before the next phase
|
||||
// starts, and before an error is returned to be printed.
|
||||
// own on stderr: open (open the database), read (count the records,
|
||||
// then read them into the hierarchy), group (group the directories by
|
||||
// digest) and write (write the groups to stdout). Each display ends
|
||||
// before the next phase starts, and before an error is returned to be
|
||||
// printed.
|
||||
func runTrees(ctx context.Context, stdout io.Writer) error {
|
||||
dbPath, source, err := databasePath()
|
||||
if err != nil {
|
||||
@@ -143,16 +139,15 @@ func readTree(ctx context.Context, db *sql.DB,
|
||||
|
||||
// writeTreeGroups writes dupes to stdout as TSV, a header line and then
|
||||
// a row for every member of each group but its first, counting each
|
||||
// group on prog once its rows are written. The disk_size column is the
|
||||
// dupe's own. stdout may be the terminal prog is drawn on, so the lines
|
||||
// go through prog's writer. It returns the number of rows (dupe trees)
|
||||
// and the bytes they could reclaim.
|
||||
// group on prog once its rows are written. stdout may be the terminal
|
||||
// prog is drawn on, so the lines go through prog's writer. It returns
|
||||
// the number of rows (dupe trees) and the bytes they could reclaim.
|
||||
func writeTreeGroups(stdout io.Writer, dupes [][]*treeNode,
|
||||
prog *progress,
|
||||
) (int, int64, error) {
|
||||
out := prog.writer(stdout)
|
||||
|
||||
_, err := fmt.Fprintln(out, "first\tdupe\tfiles\tsize\tdisk_size")
|
||||
_, err := fmt.Fprintln(out, "first\tdupe\tfiles\tsize")
|
||||
if err != nil {
|
||||
return 0, 0, fmt.Errorf("write stdout: %w", err)
|
||||
}
|
||||
@@ -164,9 +159,9 @@ func writeTreeGroups(stdout io.Writer, dupes [][]*treeNode,
|
||||
for _, g := range dupes {
|
||||
first := g[0]
|
||||
for _, n := range g[1:] {
|
||||
_, err = fmt.Fprintf(out, "%s\t%s\t%d\t%d\t%d\n",
|
||||
_, err = fmt.Fprintf(out, "%s\t%s\t%d\t%d\n",
|
||||
escapePath(first.path), escapePath(n.path),
|
||||
first.fileCount, first.totalSize, n.diskSize)
|
||||
first.fileCount, first.totalSize)
|
||||
if err != nil {
|
||||
return 0, 0, fmt.Errorf("write stdout: %w", err)
|
||||
}
|
||||
@@ -237,7 +232,6 @@ func (b *treeBuilder) add(r scanRec) {
|
||||
dir.entries = append(dir.entries, fileEntry(name, r))
|
||||
dir.fileCount++
|
||||
dir.totalSize += r.size
|
||||
dir.diskSize += r.diskSize
|
||||
}
|
||||
|
||||
// openDir starts the directory called name inside the innermost open
|
||||
@@ -276,7 +270,6 @@ func (b *treeBuilder) closeTo(n int) {
|
||||
"d\x00"+name+"\x00"+string(dir.digest[:]))
|
||||
dir.parent.fileCount += dir.fileCount
|
||||
dir.parent.totalSize += dir.totalSize
|
||||
dir.parent.diskSize += dir.diskSize
|
||||
b.dirs = append(b.dirs, dir)
|
||||
}
|
||||
}
|
||||
|
||||
+11
-40
@@ -17,24 +17,18 @@ const treesOpenLine = "[1/4] open: opening the database, elapsed 0s, " +
|
||||
|
||||
// smokeTreeRecs mirrors the README smoke-test tree layout: /d/t1 and
|
||||
// /d/t2 are identical, /d/t3 differs from them only by one filename.
|
||||
// Each file takes one 4 KiB block on disk.
|
||||
func smokeTreeRecs() []scanRec {
|
||||
f1Head, f1Tail, f1Content := testHash("f1h"), testHash("f1t"), testHash("f1c")
|
||||
f2Head, f2Tail, f2Content := testHash("f2h"), testHash("f2t"), testHash("f2c")
|
||||
|
||||
return []scanRec{
|
||||
{size: 3000, diskSize: 4096, head: f1Head, tail: f1Tail,
|
||||
content: f1Content, path: "/d/t1/f1"},
|
||||
{size: 100, diskSize: 4096, head: f2Head, tail: f2Tail,
|
||||
content: f2Content, path: "/d/t1/sub/f2"},
|
||||
{size: 3000, diskSize: 4096, head: f1Head, tail: f1Tail,
|
||||
content: f1Content, path: "/d/t2/f1"},
|
||||
{size: 100, diskSize: 4096, head: f2Head, tail: f2Tail,
|
||||
content: f2Content, path: "/d/t2/sub/f2"},
|
||||
{size: 3000, diskSize: 4096, head: f1Head, tail: f1Tail,
|
||||
content: f1Content, path: "/d/t3/f1"},
|
||||
{size: 100, diskSize: 4096, head: f2Head, tail: f2Tail,
|
||||
content: f2Content, path: "/d/t3/sub/f2renamed"},
|
||||
{size: 3000, head: f1Head, tail: f1Tail, content: f1Content, path: "/d/t1/f1"},
|
||||
{size: 100, head: f2Head, tail: f2Tail, content: f2Content, path: "/d/t1/sub/f2"},
|
||||
{size: 3000, head: f1Head, tail: f1Tail, content: f1Content, path: "/d/t2/f1"},
|
||||
{size: 100, head: f2Head, tail: f2Tail, content: f2Content, path: "/d/t2/sub/f2"},
|
||||
{size: 3000, head: f1Head, tail: f1Tail, content: f1Content, path: "/d/t3/f1"},
|
||||
{size: 100, head: f2Head, tail: f2Tail, content: f2Content,
|
||||
path: "/d/t3/sub/f2renamed"},
|
||||
}
|
||||
}
|
||||
|
||||
@@ -204,33 +198,13 @@ func TestRunTreesEscapesPaths(t *testing.T) {
|
||||
code, exitOK, stderr.String())
|
||||
}
|
||||
|
||||
want := "first\tdupe\tfiles\tsize\tdisk_size\n" +
|
||||
`/d/\tone\ntwo\rthree\\four` + "\t/d/A\t1\t5\t4096\n"
|
||||
want := "first\tdupe\tfiles\tsize\n" +
|
||||
`/d/\tone\ntwo\rthree\\four` + "\t/d/A\t1\t5\n"
|
||||
if got := stdout.String(); got != want {
|
||||
t.Errorf("stdout = %q, want %q", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
// TestRunTreesDiskSize runs trees on two copies of a tree whose files
|
||||
// take different space on disk. The row's disk_size is the sum over the
|
||||
// dupe copy's files, those in its subdirectory included.
|
||||
func TestRunTreesDiskSize(t *testing.T) {
|
||||
f, g := testHash("f"), testHash("g")
|
||||
recs := []scanRec{
|
||||
{size: 10, diskSize: 4096, head: f, tail: f, content: f, path: "/a/f"},
|
||||
{size: 20, diskSize: 4096, head: g, tail: g, content: g, path: "/a/sub/g"},
|
||||
{size: 10, diskSize: 512, head: f, tail: f, content: f, path: "/b/f"},
|
||||
{size: 20, diskSize: 1024, head: g, tail: g, content: g, path: "/b/sub/g"},
|
||||
}
|
||||
|
||||
t.Setenv(databaseEnv, seedDatabase(t, recs))
|
||||
|
||||
want := "first\tdupe\tfiles\tsize\tdisk_size\n/a\t/b\t2\t30\t1536\n"
|
||||
if got := runStdout(t, cmdTrees); got != want {
|
||||
t.Errorf("stdout = %q, want %q", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
// TestRunTreesProgressNamesEachPhase runs trees on the smoke-test
|
||||
// records with stderr not a terminal, and checks every line it prints
|
||||
// there. After the line naming the database, each phase prints a line
|
||||
@@ -252,8 +226,7 @@ func TestRunTreesProgressNamesEachPhase(t *testing.T) {
|
||||
code, exitOK, stderr())
|
||||
}
|
||||
|
||||
report := "first\tdupe\tfiles\tsize\tdisk_size\n" +
|
||||
"/d/t1\t/d/t2\t2\t3100\t8192\n"
|
||||
report := "first\tdupe\tfiles\tsize\n/d/t1\t/d/t2\t2\t3100\n"
|
||||
if got := stdout.String(); got != report {
|
||||
t.Errorf("stdout = %q, want %q", got, report)
|
||||
}
|
||||
@@ -318,9 +291,7 @@ func TestWriteTreeGroupsOnTerminal(t *testing.T) {
|
||||
shown = append(shown, line[strings.LastIndex(line, "\r")+1:])
|
||||
}
|
||||
|
||||
want := []string{
|
||||
"first\tdupe\tfiles\tsize\tdisk_size", "/d/t1\t/d/t2\t2\t3100\t8192",
|
||||
}
|
||||
want := []string{"first\tdupe\tfiles\tsize", "/d/t1\t/d/t2\t2\t3100"}
|
||||
if len(shown) < len(want) || !slices.Equal(shown[:len(want)], want) {
|
||||
t.Errorf("terminal shows lines %q, want %q first", shown, want)
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user