Compare commits
1
Commits
next
..
070cf5b598
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
070cf5b598 |
@@ -33,14 +33,14 @@ export SFDUPES_DATABASE="$HOME/.local/share/sfdupes/db.sqlite"
|
|||||||
```
|
```
|
||||||
|
|
||||||
`scan` walks one or more filesystem trees and maintains one database record per
|
`scan` walks one or more filesystem trees and maintains one database record per
|
||||||
regular file (path, size, size on disk, mtime, head hash, tail hash, content
|
regular file (path, size, mtime, head hash, tail hash, content hash). The
|
||||||
hash). The database persists between runs. A rescan hashes only the files that
|
database persists between runs. A rescan hashes only the files that are new or
|
||||||
are new or changed, and removes the records of files that no longer exist.
|
changed, and removes the records of files that no longer exist. `report` reads
|
||||||
`report` reads the database and prints the file-level duplicates report. `trees`
|
the database and prints the file-level duplicates report. `trees` reads the same
|
||||||
reads the same database and prints the duplicate-tree report. `info` prints the
|
database and prints the duplicate-tree report. `info` prints the database's
|
||||||
database's details: its path, the sizes of its files and the number of its
|
details: its path, the sizes of its files and the number of its records. A
|
||||||
records. A missing/invalid subcommand — or a `scan` invocation with no `PATH`
|
missing/invalid subcommand — or a `scan` invocation with no `PATH` operand —
|
||||||
operand — prints a usage message and exits 2.
|
prints a usage message and exits 2.
|
||||||
|
|
||||||
The database defaults to `/var/lib/sfdupes/db.sqlite` and can be placed anywhere
|
The database defaults to `/var/lib/sfdupes/db.sqlite` and can be placed anywhere
|
||||||
by setting `SFDUPES_DATABASE`. The intended deployment is a daily `sfdupes scan`
|
by setting `SFDUPES_DATABASE`. The intended deployment is a daily `sfdupes scan`
|
||||||
@@ -226,12 +226,12 @@ Goals, in order:
|
|||||||
its ends and at 1 MiB per gigabyte (see "Duplicate detection" below).
|
its ends and at 1 MiB per gigabyte (see "Duplicate detection" below).
|
||||||
Therefore no file of 100 MiB or more is read in full. Scale target: tens of
|
Therefore no file of 100 MiB or more is read in full. Scale target: tens of
|
||||||
millions of files, ~150 TB filesystem, possibly slow or busy disks (ZFS pool
|
millions of files, ~150 TB filesystem, possibly slow or busy disks (ZFS pool
|
||||||
under resilver). Holding one small record (path, size, size on disk, mtime)
|
under resilver). Holding one small record (path, size, mtime) per file in
|
||||||
per file in memory during a scan is acceptable; holding every file's hashes
|
memory during a scan is acceptable; holding every file's hashes is not (they
|
||||||
is not (they stay in the database). The reporting commands do not hold every
|
stay in the database). The reporting commands do not hold every file's
|
||||||
file's hashes either: `report` reads the records in signature order and
|
hashes either: `report` reads the records in signature order and holds the
|
||||||
holds the duplicate paths of one size at a time, so its memory grows with
|
duplicate paths of one size at a time, so its memory grows with the
|
||||||
the duplicates of the size that has the most of them, and `trees` reads the
|
duplicates of the size that has the most of them, and `trees` reads the
|
||||||
records in path order and keeps each directory's path, digest and totals,
|
records in path order and keeps each directory's path, digest and totals,
|
||||||
plus the hashes of only the files in the directories holding the record
|
plus the hashes of only the files in the directories holding the record
|
||||||
being read, so its memory grows with the number of directories and with the
|
being read, so its memory grows with the number of directories and with the
|
||||||
@@ -368,8 +368,7 @@ All four subcommands operate on a single SQLite database file:
|
|||||||
- While it loads the existing records (`load`) and walks: no records. A
|
- While it loads the existing records (`load`) and walks: no records. A
|
||||||
first scan's database holds none, however long its walk has run.
|
first scan's database holds none, however long its walk has run.
|
||||||
- During the hash phase: the first records, committed 10,000 at a time.
|
- During the hash phase: the first records, committed 10,000 at a time.
|
||||||
- In the update phase: the rest of the hashed records, the new `disk_size`
|
- In the update phase: the rest of the hashed records, and the deletions.
|
||||||
of unchanged files whose size on disk changed, and the deletions.
|
|
||||||
- At the end, as it closes the database: the `-wal` and `-shm` files go
|
- At the end, as it closes the database: the `-wal` and `-shm` files go
|
||||||
away, unless a report still has the database open (see above).
|
away, unless a report still has the database open (see above).
|
||||||
- `report`, `trees` and `info` open the database read-only and need only read
|
- `report`, `trees` and `info` open the database read-only and need only read
|
||||||
@@ -387,7 +386,6 @@ All four subcommands operate on a single SQLite database file:
|
|||||||
CREATE TABLE files (
|
CREATE TABLE files (
|
||||||
path BLOB PRIMARY KEY, -- absolute path, raw bytes
|
path BLOB PRIMARY KEY, -- absolute path, raw bytes
|
||||||
size INTEGER NOT NULL, -- bytes, from lstat
|
size INTEGER NOT NULL, -- bytes, from lstat
|
||||||
disk_size INTEGER NOT NULL, -- bytes allocated on disk (st_blocks × 512), from lstat
|
|
||||||
mtime INTEGER NOT NULL, -- whole Unix seconds of the mtime, from lstat
|
mtime INTEGER NOT NULL, -- whole Unix seconds of the mtime, from lstat
|
||||||
mtime_nsec INTEGER NOT NULL, -- nanoseconds within that second, 0 to 999999999
|
mtime_nsec INTEGER NOT NULL, -- nanoseconds within that second, 0 to 999999999
|
||||||
head BLOB NOT NULL, -- raw 32-byte SHA-256 of the first 64 KiB
|
head BLOB NOT NULL, -- raw 32-byte SHA-256 of the first 64 KiB
|
||||||
@@ -395,20 +393,6 @@ All four subcommands operate on a single SQLite database file:
|
|||||||
content BLOB NOT NULL -- raw 32-byte SHA-256; whole file under 100 MiB, else samples
|
content BLOB NOT NULL -- raw 32-byte SHA-256; whole file under 100 MiB, else samples
|
||||||
) WITHOUT ROWID;
|
) WITHOUT ROWID;
|
||||||
CREATE INDEX files_signature ON files (size, head, tail, content);
|
CREATE INDEX files_signature ON files (size, head, tail, content);
|
||||||
CREATE TABLE totals (
|
|
||||||
records INTEGER NOT NULL, -- number of records in files
|
|
||||||
size INTEGER NOT NULL -- sizes of all records in files added up
|
|
||||||
);
|
|
||||||
INSERT INTO totals (records, size) VALUES (0, 0);
|
|
||||||
CREATE TRIGGER totals_after_insert AFTER INSERT ON files BEGIN
|
|
||||||
UPDATE totals SET records = records + 1, size = size + NEW.size;
|
|
||||||
END;
|
|
||||||
CREATE TRIGGER totals_after_delete AFTER DELETE ON files BEGIN
|
|
||||||
UPDATE totals SET records = records - 1, size = size - OLD.size;
|
|
||||||
END;
|
|
||||||
CREATE TRIGGER totals_after_update AFTER UPDATE OF size ON files BEGIN
|
|
||||||
UPDATE totals SET size = size + NEW.size - OLD.size;
|
|
||||||
END;
|
|
||||||
```
|
```
|
||||||
|
|
||||||
Paths are stored as BLOBs because Unix paths are raw bytes, not guaranteed
|
Paths are stored as BLOBs because Unix paths are raw bytes, not guaranteed
|
||||||
@@ -422,17 +406,6 @@ All four subcommands operate on a single SQLite database file:
|
|||||||
`files_signature` index lets `report` read the records in signature order
|
`files_signature` index lets `report` read the records in signature order
|
||||||
without sorting the table.
|
without sorting the table.
|
||||||
|
|
||||||
`totals` holds one row: the number of records in `files` and the sum of
|
|
||||||
their sizes. Three triggers on `files` keep that row current. Each runs in
|
|
||||||
the transaction of the change to `files` that fires it, so the row and the
|
|
||||||
records always agree. `info` reads both numbers from this row. `scan`,
|
|
||||||
`report` and `trees` read the number of records from it as the total of
|
|
||||||
their progress displays. Counting the records in `files` would read every
|
|
||||||
record, which takes minutes on a database of tens of millions. The triggers
|
|
||||||
make each write to `files` slower, so a first scan of many small files takes
|
|
||||||
about two fifths longer. On large files, where reading them takes most of
|
|
||||||
the time, the difference is small.
|
|
||||||
|
|
||||||
### Duplicate detection
|
### Duplicate detection
|
||||||
|
|
||||||
Two files are duplicates when they have the same size and the same three hashes.
|
Two files are duplicates when they have the same size and the same three hashes.
|
||||||
@@ -488,26 +461,17 @@ exclude, is not on a persistent filesystem (see "Rules for the walk"), or, with
|
|||||||
`-x`, is on a different filesystem than that operand, because that operand's
|
`-x`, is on a different filesystem than that operand, because that operand's
|
||||||
walk does not enter such a directory and so does not reach it.
|
walk does not enter such a directory and so does not reach it.
|
||||||
|
|
||||||
An operand is not scanned when it is:
|
An operand that is a symlink (never followed, not even as an operand), socket,
|
||||||
|
FIFO, or device node, or a directory named `.zfs`, is not scanned. `scan` prints
|
||||||
- a symlink, which `scan` never follows, not even as an operand;
|
a one-line warning naming the path and what it is, counts it as skipped, and
|
||||||
- a socket, FIFO or device node;
|
drops it from the scanned operands before reading the database. Another operand
|
||||||
- at or inside a directory named `.zfs`.
|
beneath it is still scanned. The records stored beneath it are not deleted: they
|
||||||
|
are treated like any other record outside the scanned operands. If it lies under
|
||||||
An operand reached through a symlink into a `.zfs` directory is not scanned
|
another operand, they are under that operand instead, and are treated like any
|
||||||
either.
|
other record there. This is not an error: a scan whose every operand is dropped
|
||||||
|
walks nothing and exits 0. An operand that is not on a persistent filesystem
|
||||||
This holds even when the operand lies under another operand. `scan` prints a
|
(see "Rules for the walk") is dropped the same way. The line `scan` prints for
|
||||||
one-line warning naming the path and what it is, counts it as skipped, and drops
|
it is not a warning, though, and it is not counted as skipped.
|
||||||
it from the scanned operands before reading the database. An operand beneath a
|
|
||||||
symlink operand is still scanned. The records stored beneath the dropped operand
|
|
||||||
are not deleted: they are treated like any other record outside the scanned
|
|
||||||
operands. If it lies under another operand, they are under that operand instead,
|
|
||||||
and are treated like any other record there. This is not an error: a scan whose
|
|
||||||
every operand is dropped walks nothing and exits 0. An operand that is not on a
|
|
||||||
persistent filesystem (see "Rules for the walk") is dropped the same way. The
|
|
||||||
line `scan` prints for it is not a warning, though, and it is not counted as
|
|
||||||
skipped.
|
|
||||||
|
|
||||||
`scan` synchronizes the database with the filesystem state under the scanned
|
`scan` synchronizes the database with the filesystem state under the scanned
|
||||||
operands:
|
operands:
|
||||||
@@ -516,11 +480,6 @@ operands:
|
|||||||
- A file already in the database is **skipped without reading its contents**
|
- A file already in the database is **skipped without reading its contents**
|
||||||
when its lstat size equals the recorded size and its lstat mtime is not newer
|
when its lstat size equals the recorded size and its lstat mtime is not newer
|
||||||
than the recorded mtime. This is what makes a daily rescan cheap.
|
than the recorded mtime. This is what makes a daily rescan cheap.
|
||||||
- When such a file's size on disk differs from the recorded `disk_size`, the
|
|
||||||
scan updates `disk_size` alone. It still does not read the file, and counts it
|
|
||||||
as unchanged. ZFS, for example, counts a new file's blocks only seconds after
|
|
||||||
the write. A file scanned within those seconds gets its real `disk_size` at
|
|
||||||
the next scan.
|
|
||||||
- A file whose mtime is newer than recorded, or whose size differs, is hashed
|
- A file whose mtime is newer than recorded, or whose size differs, is hashed
|
||||||
again. Change detection compares the mtime to the nanosecond, as finely as the
|
again. Change detection compares the mtime to the nanosecond, as finely as the
|
||||||
filesystem records it, so a same-size rewrite counts as a change whenever the
|
filesystem records it, so a same-size rewrite counts as a change whenever the
|
||||||
@@ -548,8 +507,8 @@ inside the walk and hash phases, and batched database writes begin during the
|
|||||||
hash phase:
|
hash phase:
|
||||||
|
|
||||||
1. **load** — read the existing records. Each record under the operands is kept
|
1. **load** — read the existing records. Each record under the operands is kept
|
||||||
in memory, by path, with its size, its size on disk and its mtime, for the
|
in memory, by path, with its size and its mtime, for the walk to compare
|
||||||
walk to compare each file with.
|
each file with.
|
||||||
2. **walk + stat** — enumerate the trees under all `PATH` operands concurrently
|
2. **walk + stat** — enumerate the trees under all `PATH` operands concurrently
|
||||||
with the walk worker pool: every operand seeds the shared queue, and each
|
with the walk worker pool: every operand seeds the shared queue, and each
|
||||||
worker reads one directory at a time, handing discovered subdirectories back
|
worker reads one directory at a time, handing discovered subdirectories back
|
||||||
@@ -557,10 +516,9 @@ hash phase:
|
|||||||
(while the directory's metadata is still hot). Sequential directory
|
(while the directory's metadata is still hot). Sequential directory
|
||||||
enumeration is metadata-latency-bound and takes hours at tens of millions of
|
enumeration is metadata-latency-bound and takes hours at tens of millions of
|
||||||
files; per-directory parallelism is what makes the walk tractable on large
|
files; per-directory parallelism is what makes the walk tractable on large
|
||||||
or busy pools. The walk compares each file with its record as it goes. An
|
or busy pools. The walk compares each file with its record as it goes, and
|
||||||
unchanged file is not read again. When its size on disk has changed, the
|
an unchanged file needs nothing more. Every new or changed file is carried
|
||||||
update phase writes its new `disk_size`. Every new or changed file is
|
to the hash phase as a (path, size, mtime) record.
|
||||||
carried to the hash phase as a (path, size, size on disk, mtime) record.
|
|
||||||
3. **hash** — the worker pool hashes every carried file as described in
|
3. **hash** — the worker pool hashes every carried file as described in
|
||||||
"Duplicate detection" above, which gives the file its `head`, `tail` and
|
"Duplicate detection" above, which gives the file its `head`, `tail` and
|
||||||
`content`. Zero-length files have constant hashes and are never opened.
|
`content`. Zero-length files have constant hashes and are never opened.
|
||||||
@@ -572,8 +530,7 @@ hash phase:
|
|||||||
committed in batched transactions **while hashing runs**. A scan interrupted
|
committed in batched transactions **while hashing runs**. A scan interrupted
|
||||||
after hours therefore keeps everything hashed so far, and the next scan
|
after hours therefore keeps everything hashed so far, and the next scan
|
||||||
resumes cheaply, skipping records already written.
|
resumes cheaply, skipping records already written.
|
||||||
4. **update** — commit the final partial batch of hashed records, and the new
|
4. **update** — commit the final partial batch of hashed records. Then delete
|
||||||
`disk_size` of each unchanged file whose size on disk changed. Then delete
|
|
||||||
the records the scan did not verify: those of vanished files, and of paths
|
the records the scan did not verify: those of vanished files, and of paths
|
||||||
that failed to stat or hash.
|
that failed to stat or hash.
|
||||||
|
|
||||||
@@ -722,18 +679,14 @@ Processing:
|
|||||||
database state.
|
database state.
|
||||||
|
|
||||||
`report` runs in two phases, each with its own progress display on stderr (see
|
`report` runs in two phases, each with its own progress display on stderr (see
|
||||||
"Progress"):
|
"Progress"): `open` opens the database; `read` counts the records, then reads
|
||||||
|
them in signature order, largest size first. That is the order of the
|
||||||
- `open` opens the database.
|
`files_signature` index read backwards, so SQLite sorts nothing and returns the
|
||||||
- `read` gets the number of records from `totals`, then reads the records in
|
first record at once. The records sharing a signature come together, and so do
|
||||||
signature order, largest size first.
|
the records of one size, so `report` holds only the duplicate groups of the size
|
||||||
|
it is reading, and writes them, ordered by `first` path, once it reaches a
|
||||||
Signature order, largest size first, is the `files_signature` index read
|
smaller size. Its memory holds at most the duplicate paths of one size at a
|
||||||
backwards, so SQLite sorts nothing and returns the first record at once. The
|
time, never every record.
|
||||||
records sharing a signature come together, and so do the records of one size, so
|
|
||||||
`report` holds only the duplicate groups of the size it is reading, and writes
|
|
||||||
them, ordered by `first` path, once it reaches a smaller size. Its memory holds
|
|
||||||
at most the duplicate paths of one size at a time, never every record.
|
|
||||||
|
|
||||||
#### Report output format
|
#### Report output format
|
||||||
|
|
||||||
@@ -807,37 +760,22 @@ Processing:
|
|||||||
ascending. Output must be fully deterministic for a given input.
|
ascending. Output must be fully deterministic for a given input.
|
||||||
|
|
||||||
`trees` runs in four phases, each with its own progress display on stderr (see
|
`trees` runs in four phases, each with its own progress display on stderr (see
|
||||||
"Progress"):
|
"Progress"): `open` opens the database; `read` counts the records, then reads
|
||||||
|
them in path order, building the hierarchy and computing the digests as it goes;
|
||||||
- `open` opens the database.
|
`group` groups the directories by digest and picks the maximal groups; `write`
|
||||||
- `read` gets the number of records from `totals`, then reads the records in
|
writes the groups to stdout.
|
||||||
path order, building the hierarchy and computing the digests as it goes.
|
|
||||||
- `group` groups the directories by digest and picks the maximal groups.
|
|
||||||
- `write` writes the groups to stdout.
|
|
||||||
|
|
||||||
#### Trees output format
|
#### Trees output format
|
||||||
|
|
||||||
TSV on stdout: a header line, then one row per duplicate tree (N-1 rows for a
|
TSV on stdout: a header line, then one row per duplicate tree (N-1 rows for a
|
||||||
group of N). After the `first` and `dupe` paths come three numbers:
|
group of N). `files` is the recursive regular-file count of one copy of the
|
||||||
|
tree; `size` is the recursive total byte size of one copy:
|
||||||
- `files`, the number of regular files in one copy of the tree, counted
|
|
||||||
recursively;
|
|
||||||
- `size`, the bytes of one copy, its files' sizes added up;
|
|
||||||
- `disk_size`, the bytes the `dupe` copy takes on disk, its files' `disk_size`
|
|
||||||
added up.
|
|
||||||
|
|
||||||
```
|
```
|
||||||
first dupe files size disk_size
|
first dupe files size
|
||||||
/srv/a/project /srv/backup/project 3417 104857600 61865984
|
/srv/a/project /srv/backup/project 3417 104857600
|
||||||
```
|
```
|
||||||
|
|
||||||
A file's `disk_size` is the space its blocks take on disk. It can be more than
|
|
||||||
the file's size, because space is allocated in whole blocks. It is less than the
|
|
||||||
file's size on a compressed or sparse file, so two copies of one tree can differ
|
|
||||||
in `disk_size`. When a file in one copy is a hard link to one in the other, both
|
|
||||||
copies count its blocks in full. Removing it from one copy then frees nothing,
|
|
||||||
as the known limitation under "`trees` mode" says.
|
|
||||||
|
|
||||||
The `first` and `dupe` paths are escaped as described under "Report output
|
The `first` and `dupe` paths are escaped as described under "Report output
|
||||||
format". The root directory's path is `/`.
|
format". The root directory's path is `/`.
|
||||||
|
|
||||||
@@ -872,19 +810,15 @@ size of recorded files 150000000000000 (136.4 TiB)
|
|||||||
database is in WAL mode, the newest changes may be in the `-wal` file alone.
|
database is in WAL mode, the newest changes may be in the `-wal` file alone.
|
||||||
- `schema version` is the database's `PRAGMA user_version`. A database with
|
- `schema version` is the database's `PRAGMA user_version`. A database with
|
||||||
another version than this build's is refused, as by `report`.
|
another version than this build's is refused, as by `report`.
|
||||||
- `records` is the number of records. `size of recorded files` is the sum of the
|
- `records` is the number of records. `size of recorded files` adds up the sizes
|
||||||
sizes of all records, in bytes, then in human units.
|
of all records, in bytes, then in human units.
|
||||||
|
|
||||||
`info` runs in two phases, each with its own progress display on stderr (see
|
`info` runs in two phases, each with its own progress display on stderr (see
|
||||||
"Progress"):
|
"Progress"): `open` opens the database; `count` reads the schema version and
|
||||||
|
counts the records with one query, a single pass over the table, both in one
|
||||||
- `open` opens the database.
|
read-only transaction, so the numbers agree with each other while a scan is
|
||||||
- `count` reads the schema version and the row of `totals` (see "Database").
|
committing. `info` writes its lines once the `count` display has ended, so on a
|
||||||
Both reads run in one read-only transaction, so the numbers agree with each
|
terminal they come after the displays.
|
||||||
other while a scan is committing.
|
|
||||||
|
|
||||||
`info` writes its lines once the `count` display has ended, so on a terminal
|
|
||||||
they come after the displays.
|
|
||||||
|
|
||||||
### Progress
|
### Progress
|
||||||
|
|
||||||
@@ -900,9 +834,8 @@ progress, its elapsed time and its time remaining.
|
|||||||
A scan has four phases (`[1/4] load`, `[2/4] walk`, `[3/4] hash`,
|
A scan has four phases (`[1/4] load`, `[2/4] walk`, `[3/4] hash`,
|
||||||
`[4/4] update`):
|
`[4/4] update`):
|
||||||
|
|
||||||
- `load` gets the number of records from `totals` first, then reads the records
|
- `load` counts the records first, then reads them against that count. While the
|
||||||
against that number. Until it has the number, the display says it is counting
|
count runs, the display says it is counting the records.
|
||||||
the records.
|
|
||||||
- `walk` has no exact total. Its estimate is the number of records the last scan
|
- `walk` has no exact total. Its estimate is the number of records the last scan
|
||||||
left under the operands, known once `load` is done and shown with a `~`. The
|
left under the operands, known once `load` is done and shown with a `~`. The
|
||||||
time remaining is unknown once the walk passes the estimate, and when there
|
time remaining is unknown once the walk passes the estimate, and when there
|
||||||
@@ -915,9 +848,8 @@ A scan has four phases (`[1/4] load`, `[2/4] walk`, `[3/4] hash`,
|
|||||||
|
|
||||||
- `open` has no total. While it opens the database, waiting on a busy one
|
- `open` has no total. While it opens the database, waiting on a busy one
|
||||||
included, the display says it is opening the database.
|
included, the display says it is opening the database.
|
||||||
- `read` gets the number of records from `totals` first, then reads the records
|
- `read` counts the records first, then reads them against that count. While the
|
||||||
against that number. Until it has the number, the display says it is counting
|
count runs, the display says it is counting the records.
|
||||||
the records.
|
|
||||||
- `group` has the directories `read` found as its total, and `write` the
|
- `group` has the directories `read` found as its total, and `write` the
|
||||||
duplicate-tree groups.
|
duplicate-tree groups.
|
||||||
|
|
||||||
@@ -925,16 +857,17 @@ A scan has four phases (`[1/4] load`, `[2/4] walk`, `[3/4] hash`,
|
|||||||
|
|
||||||
- `open` has no total. While it opens the database, waiting on a busy one
|
- `open` has no total. While it opens the database, waiting on a busy one
|
||||||
included, the display says it is opening the database.
|
included, the display says it is opening the database.
|
||||||
- `read` gets the number of records from `totals` first, then reads the records
|
- `read` counts the records first, then reads them against that count, writing
|
||||||
against that number, writing the rows to stdout as it goes. Until it has the
|
the rows to stdout as it goes. While the count runs, the display says it is
|
||||||
number, the display says it is counting the records.
|
counting the records.
|
||||||
|
|
||||||
`info` has two phases (`[1/2] open`, `[2/2] count`, see "`info` mode"):
|
`info` has two phases (`[1/2] open`, `[2/2] count`, see "`info` mode"):
|
||||||
|
|
||||||
- `open` has no total. While it opens the database, waiting on a busy one
|
- `open` has no total. While it opens the database, waiting on a busy one
|
||||||
included, the display says it is opening the database.
|
included, the display says it is opening the database.
|
||||||
- `count` has no total either, because it reads one row: the display says it is
|
- `count` has no total either, because one query counts everything: the display
|
||||||
counting the records until it has the row, and its time remaining is unknown.
|
says it is counting the records until the query returns, and its time
|
||||||
|
remaining is unknown.
|
||||||
|
|
||||||
Each display is one line holding:
|
Each display is one line holding:
|
||||||
|
|
||||||
@@ -1221,9 +1154,9 @@ All of the following, run in this directory, must pass:
|
|||||||
ordered by size descending.
|
ordered by size descending.
|
||||||
|
|
||||||
Expected from `trees`: exactly one row — `first` `$d/t1`, `dupe` `$d/t2`, 2
|
Expected from `trees`: exactly one row — `first` `$d/t1`, `dupe` `$d/t2`, 2
|
||||||
files, 3100 bytes, and a `disk_size` that depends on the filesystem.
|
files, 3100 bytes. `$d/t1/sub` vs `$d/t2/sub` is suppressed as non-maximal
|
||||||
`$d/t1/sub` vs `$d/t2/sub` is suppressed as non-maximal (implied by the
|
(implied by the `t1`/`t2` group), and `t3` appears nowhere (its file set
|
||||||
`t1`/`t2` group), and `t3` appears nowhere (its file set differs by name).
|
differs by name).
|
||||||
|
|
||||||
Expected from the second `report` (after the modify/delete rescan):
|
Expected from the second `report` (after the modify/delete rescan):
|
||||||
`one.bin` has left its group (its content changed), so
|
`one.bin` has left its group (its content changed), so
|
||||||
|
|||||||
@@ -1,55 +1,46 @@
|
|||||||
# Workflow
|
# Workflow
|
||||||
|
|
||||||
- take an issue from the `1.0.0` milestone on the tracker; work not yet on the
|
- Take an issue from the `1.0.0` milestone on the tracker. Work that is not on
|
||||||
tracker gets filed as an issue first
|
the tracker yet is filed as an issue first.
|
||||||
- branch from `next`
|
- Branch from `next`.
|
||||||
- do the work, with tests, in small focused commits
|
- Do the work, with tests, in small focused commits.
|
||||||
- record it at the top of Completed Steps (`TODO.md` changes in the same commit
|
- Record the step at the top of Completed Steps, in the same commit as the work.
|
||||||
as the work)
|
- Push the branch and open a PR against `next`. Its title ends with
|
||||||
- push the branch and open a PR against `next` whose title ends with
|
` (closes #N)`.
|
||||||
` (closes #N)`
|
- An independent review gates each merge to `next`. Every finding is addressed
|
||||||
- an independent review gates each merge to `next`; every finding is addressed
|
or rebutted on the PR.
|
||||||
or explicitly rebutted on the PR
|
- Only the owner merges `next` into `main`.
|
||||||
- only the owner merges `next` to `main`
|
|
||||||
|
|
||||||
# Status
|
# Status
|
||||||
|
|
||||||
- pre-1.0
|
- Pre-1.0.
|
||||||
- the Gitea tracker is authoritative for the pre-1.0 backlog: the open issues
|
- The tracker is authoritative for the backlog. The open issues under the
|
||||||
under the `1.0.0` milestone are what remains before the tag, and this file
|
`1.0.0` milestone are what remains before the tag. This file records history
|
||||||
records history and process, not the queue
|
and process, not the queue.
|
||||||
|
|
||||||
# Next Step
|
# Next Step
|
||||||
|
|
||||||
- take the next issue from the `1.0.0` milestone on the tracker:
|
- Take the next issue from the `1.0.0` milestone:
|
||||||
https://git.eeqj.de/sneak/sfdupes/milestone/17 — the milestone is the source
|
https://git.eeqj.de/sneak/sfdupes/milestone/17. The issues are not restated
|
||||||
of truth for what is left before 1.0.0. Individual issues are deliberately not
|
here, because a copy drifts as soon as the tracker moves.
|
||||||
restated here; a copy in this file drifts out of date the moment the tracker
|
|
||||||
moves
|
|
||||||
|
|
||||||
# Completed Steps
|
# Completed Steps
|
||||||
|
|
||||||
- each record holds `disk_size`, the bytes the file takes on disk: its allocated
|
Newest first. Each entry ends with its date and its issue or branch.
|
||||||
blocks, `st_blocks` × 512, from the lstat the walk already does. `trees`
|
|
||||||
prints a `disk_size` column after `size`, the sum over the `dupe` copy's
|
|
||||||
files; on a compressed or sparse file it differs from `size`. A rescan updates
|
|
||||||
the `disk_size` of an unchanged file whose size on disk changed, without
|
|
||||||
reading the file (2026-10-27, https://git.eeqj.de/sneak/sfdupes/issues/147)
|
|
||||||
|
|
||||||
- the database keeps the number of records and the sum of their sizes in the one
|
- `TODO.md` rewritten from scratch by the same rules as `README.md`: short
|
||||||
row of a `totals` table, which three triggers on `files` keep current. `info`
|
sentences, lists as bullets, nothing a newcomer reads twice. Every completed
|
||||||
reads that row, and so do `scan`'s `load` phase and the `read` phases of
|
step is kept, with its date and issue, in fewer words. The policy-compliance
|
||||||
`report` and `trees` for their totals; no count reads every record any more.
|
checklist of 2026-07-22 is folded into its step (2026-10-26,
|
||||||
The triggers make each write to `files` slower. `scan` keeps SQLite's
|
https://git.eeqj.de/sneak/sfdupes/issues/135)
|
||||||
temporary data in memory, which wins back part of that time (2026-10-27,
|
|
||||||
https://git.eeqj.de/sneak/sfdupes/issues/146)
|
|
||||||
|
|
||||||
- an operand at or inside a directory named `.zfs` is never scanned, whether
|
- `README.md` rewritten from scratch, sentence by sentence, as a newcomer reads
|
||||||
given alone or under another operand. `scan` warns about it, counts it as
|
it. It leads with what the program is and how to use it. A reference section
|
||||||
skipped and does not walk it (2026-10-26,
|
covers the settings, the database, the output formats and the exit codes;
|
||||||
https://git.eeqj.de/sneak/sfdupes/issues/142)
|
development and testing come last. Every statement was checked against the
|
||||||
|
code on `next` (2026-10-26, https://git.eeqj.de/sneak/sfdupes/issues/135)
|
||||||
|
|
||||||
- a file or directory removed while the scan runs is not an error, in the walk
|
- A file or directory removed while the scan runs is not an error, in the walk
|
||||||
or in the hash phase. The scan prints no line for it and does not count it as
|
or in the hash phase. The scan prints no line for it and does not count it as
|
||||||
skipped. When the first of several hard links to a file is gone, the hash
|
skipped. When the first of several hard links to a file is gone, the hash
|
||||||
phase reads the next one instead. On a terminal every progress display starts
|
phase reads the next one instead. On a terminal every progress display starts
|
||||||
@@ -59,218 +50,222 @@
|
|||||||
terminal now loses its rate before its bar. The time remaining is labelled
|
terminal now loses its rate before its bar. The time remaining is labelled
|
||||||
`remain`, not `eta` (2026-10-26, https://git.eeqj.de/sneak/sfdupes/issues/133)
|
`remain`, not `eta` (2026-10-26, https://git.eeqj.de/sneak/sfdupes/issues/133)
|
||||||
|
|
||||||
- the `head`, `tail` and `content` columns are BLOBs, each holding the raw 32
|
- The `head`, `tail` and `content` columns are BLOBs, each holding the raw 32
|
||||||
bytes of a SHA-256, half the size of the hex text they held. The hashes stay
|
bytes of a SHA-256, half the size of the hex text they held. The hashes stay
|
||||||
raw from the hash phase through the database to `report` and `trees`. Reading
|
raw from the hash phase through the database to `report` and `trees`. Reading
|
||||||
a record whose hash is not 32 bytes is an error (2026-10-26,
|
a record whose hash is not 32 bytes is an error (2026-10-26,
|
||||||
https://git.eeqj.de/sneak/sfdupes/issues/134)
|
https://git.eeqj.de/sneak/sfdupes/issues/134)
|
||||||
|
|
||||||
- the hash phase gives every file the walk records three hashes: `head` and
|
- Every file the walk records gets three hashes in the hash phase. `head` and
|
||||||
`tail`, the SHA-256 of its first and last 64 KiB, and `content`, the SHA-256
|
`tail` are the SHA-256 of its first and last 64 KiB. `content` is the SHA-256
|
||||||
of the whole file under 100 MiB and of 1 MiB at each gigabyte from 100 MiB up.
|
of the whole file under 100 MiB, and of 1 MiB at each gigabyte from 100 MiB
|
||||||
The 10 MiB and 50 MiB thresholds, the size census, the records stored without
|
up. The 10 MiB and 50 MiB thresholds, the size census, the records without
|
||||||
hashes and the content phase are gone, so a scan has four phases and a rescan
|
hashes and the content phase are gone. A scan has four phases, and a rescan
|
||||||
reads only new and changed files. `info` no longer counts the records with
|
reads only new and changed files. `info` no longer counts the records with
|
||||||
hashes, since every record has them (2026-10-25,
|
hashes, since every record has them (2026-10-25,
|
||||||
https://git.eeqj.de/sneak/sfdupes/issues/132)
|
https://git.eeqj.de/sneak/sfdupes/issues/132)
|
||||||
|
|
||||||
- re-vendor `REPO_POLICIES.md` and the four scripts that pass `CACHEBUST` from
|
- `REPO_POLICIES.md` and the four scripts that pass `CACHEBUST` re-vendored from
|
||||||
`sneak/prompts` `next` at `0653579`: in the `lint`, `test` and `builder`
|
`sneak/prompts` `next` at `0653579`. The module download in the `lint`, `test`
|
||||||
stages of the `Dockerfile` the module download, and in `builder` the
|
and `builder` stages of the `Dockerfile`, and the package install in
|
||||||
system-package install, now come before `ARG CACHEBUST`, so Docker keeps them
|
`builder`, now come before `ARG CACHEBUST`. Docker keeps them between builds.
|
||||||
until the base image, the install line, `go.mod` or `go.sum` changes, and
|
Every step that lints, tests or compiles still comes after it.
|
||||||
every step that lints, tests or compiles still comes after it;
|
|
||||||
`golangci-lint run` also keeps golangci-lint's cache. README says the same
|
`golangci-lint run` also keeps golangci-lint's cache. README says the same
|
||||||
(2026-10-18, https://git.eeqj.de/sneak/sfdupes/issues/109)
|
(2026-10-18, https://git.eeqj.de/sneak/sfdupes/issues/109)
|
||||||
|
|
||||||
- releases until 1.0 are `v0.0.N` tags on `main`'s tip, so
|
- Release rule set: until 1.0, each tip of `main` that passes the build gets the
|
||||||
`go install sneak.berlin/go/sfdupes@latest` installs `main`'s tip once it has
|
next `v0.0.N` tag, so `go install sneak.berlin/go/sfdupes@latest` installs it.
|
||||||
passed the build; README "Install" says so, says what `@main` and `v0.0.1`
|
`go.mod` retracts `v0.0.1`, the early build without a database. When the
|
||||||
give, and how to check an installed binary. `go.mod` retracts `v0.0.1`, the
|
linker set no version, `sfdupes --version` prints the version Go recorded in
|
||||||
early build without a database. When the linker set no version,
|
the binary: the tag, or the pseudo-version of a `go install` build. It prints
|
||||||
`sfdupes --version` prints the main module's version that Go recorded in the
|
`dev` only when neither is known. README "Install" documented the rule, what
|
||||||
binary, the tag or the pseudo-version of a `go install` build, and `dev` only
|
`@main` and `v0.0.1` give, and how to check an installed binary (2026-10-17,
|
||||||
when neither is known (2026-10-17,
|
|
||||||
https://git.eeqj.de/sneak/sfdupes/issues/116)
|
https://git.eeqj.de/sneak/sfdupes/issues/116)
|
||||||
|
|
||||||
- `scan` creates the database's directory where the database path leads: the
|
- `scan` creates the database's directory where the database path leads. The
|
||||||
path with its last element cut off as text, never cleaned, so a `..` after a
|
directory is the path with its last element cut off as text, never cleaned. A
|
||||||
symlinked directory is followed through the symlink, as when the file is
|
`..` after a symlinked directory therefore goes through the symlink, as it
|
||||||
opened. With `SFDUPES_DATABASE=/srv/link/../new/db.sqlite` and `/srv/link` a
|
does when the file is opened. With
|
||||||
symlink to `/data/x`, it creates `/data/new` and the database in it, where it
|
`SFDUPES_DATABASE=/srv/link/../new/db.sqlite` and `/srv/link` a symlink to
|
||||||
used to create `/srv/new` and then fail (2026-10-17,
|
`/data/x`, it now creates `/data/new` and the database in it. It used to
|
||||||
|
create `/srv/new` and then fail (2026-10-17,
|
||||||
https://git.eeqj.de/sneak/sfdupes/issues/119)
|
https://git.eeqj.de/sneak/sfdupes/issues/119)
|
||||||
|
|
||||||
- `scan` never records its own database: once the database and its `.lock` file
|
- `scan` never records its own database. Once the database and its `.lock` file
|
||||||
are open, it notes the device and inode of the database file, its `-wal` and
|
are open, it notes the device and inode of the database file, its `-wal` and
|
||||||
`-shm` files and its `.lock` file, and the walk passes over a file with one of
|
`-shm` files and the `.lock` file. The walk passes over a file with one of
|
||||||
them wherever the operands reach it, through a symlinked directory included,
|
them wherever it reaches it, through a symlinked directory included, printing
|
||||||
printing no line and counting nothing. A record an earlier scan left for one
|
no line and counting nothing. A record an earlier scan left for one of them is
|
||||||
of them is deleted (2026-10-17, https://git.eeqj.de/sneak/sfdupes/issues/115)
|
deleted (2026-10-17, https://git.eeqj.de/sneak/sfdupes/issues/115)
|
||||||
|
|
||||||
- `scan` never records or reads anything that is not on a persistent filesystem
|
- `scan` never records or reads anything on a filesystem that is not persistent:
|
||||||
(`tmpfs`, `proc`, `sysfs` and the other kernel or memory-backed types),
|
`tmpfs`, `proc`, `sysfs` and the other kernel or memory-backed types. It
|
||||||
matched by the device ID of each such filesystem, listed as the scan starts
|
matches them by device ID, listed as the scan starts from
|
||||||
from `/proc/self/mountinfo` on Linux and the mounted-filesystem list on macOS,
|
`/proc/self/mountinfo` on Linux and from the mount list on macOS. No flag
|
||||||
however the walk or the content phase reaches it; no flag turns that off.
|
turns that off. Below an operand, the walk does not enter the default excludes
|
||||||
Below an operand, its walk does not enter the default excludes either, so
|
either, so `sfdupes scan /` skips the kernel's view of the running system and
|
||||||
`sfdupes scan /` skips the kernel's view of the running system and the
|
the temporary directories. On Linux those are `/proc`, `/sys`, `/dev`, `/run`,
|
||||||
temporary directories: on Linux `/proc`, `/sys`, `/dev`, `/run`, `/tmp` and
|
`/tmp` and `/var/tmp`. On macOS they are `/dev`, `/private/tmp`,
|
||||||
`/var/tmp`, on macOS `/dev`, `/private/tmp`, `/private/var/tmp`,
|
`/private/var/tmp`, `/private/var/vm`, `/System/Volumes/VM` and
|
||||||
`/private/var/vm`, `/System/Volumes/VM` and `/System/Volumes/Data`, and a
|
`/System/Volumes/Data`. A Windows list exists in the code but is not used. An
|
||||||
Windows list, not used yet. An operand at or inside a default exclude is still
|
operand at or inside a default exclude is still walked. The scan prints a line
|
||||||
walked. The scan prints a line for each path it leaves out, without counting
|
for each path it leaves out, without counting it as skipped, and the records
|
||||||
it as skipped, and the records there count as outside the scanned operands, as
|
there count as outside the scanned operands, as for `-x`.
|
||||||
for `-x`. `--no-default-excludes` turns off the default excludes only, and
|
`--no-default-excludes` turns off the default excludes only, and `scan --help`
|
||||||
`scan --help` lists them (2026-10-17,
|
lists them (2026-10-17, https://git.eeqj.de/sneak/sfdupes/issues/124)
|
||||||
https://git.eeqj.de/sneak/sfdupes/issues/124)
|
|
||||||
|
|
||||||
- the rate a progress display shows covers the items completed over the last 5
|
- The rate a progress display shows covers the items completed over the last 5
|
||||||
seconds, or since the line before when stderr is not a terminal, so it falls
|
seconds, or since the line before when stderr is not a terminal. It falls to 0
|
||||||
to 0 while nothing completes, and a burst after a wait no longer reads as a
|
while nothing completes, and a burst after a wait no longer reads as a steady
|
||||||
steady rate of over 1,000 files a second; the ETA still comes from the phase's
|
rate. The time remaining still comes from the phase's average. Once no item
|
||||||
average. Once no item has completed for 10 seconds, the word `stalled` and
|
has completed for 10 seconds, the word `stalled` and where the phase waits
|
||||||
where the phase waits take the rate's place, as `stalled disk/db`: `disk`,
|
take the rate's place, as `stalled disk/db`. The places are `disk`, `db`,
|
||||||
`db`, `out` for stdout or `cpu`, or two of them for a phase that may wait on
|
`out` for stdout and `cpu`, or two of them for a phase that may wait on
|
||||||
either, each phase giving its word where it is defined. A line too wide for
|
either. A line too wide for the terminal loses its bar first, then its rate or
|
||||||
the terminal loses its bar first, then its rate or those words, so the phase,
|
those words, so the phase, progress, elapsed time and time remaining stay
|
||||||
progress, elapsed time and ETA stay whole. On an 80-column terminal, at counts
|
whole. The content phase's last partial batch is written under a display of
|
||||||
like those of README's example, the content phase's checks and reads lose
|
its own (2026-10-16, https://git.eeqj.de/sneak/sfdupes/issues/125)
|
||||||
those words at most times in minutes, and every other phase keeps them
|
|
||||||
whenever the elapsed time and the ETA are both in minutes. The content phase's
|
|
||||||
last partial batch is now written under a display of its own (2026-10-16,
|
|
||||||
https://git.eeqj.de/sneak/sfdupes/issues/125)
|
|
||||||
|
|
||||||
- new subcommand `info` prints the database's details on stdout, one per line as
|
- New subcommand `info` prints the database's details on stdout, one per line as
|
||||||
a name, a tab and the value: its path and where it came from, the sizes of the
|
a name, a tab and the value (2026-10-16,
|
||||||
database file and of its `-wal` file when there is one, when the database file
|
https://git.eeqj.de/sneak/sfdupes/issues/117):
|
||||||
was last changed, the schema version, the number of records, how many have
|
- the path and where it came from;
|
||||||
head and tail hashes and how many a content hash, and the sizes of all records
|
- the sizes of the database file and of its `-wal` file, when there is one;
|
||||||
added up. It opens the database as `report` does, and shows two phases on
|
- when the database file was last changed;
|
||||||
stderr, `[1/2] open` and `[2/2] count`; the schema version and the counts come
|
- the schema version, the record counts and the sizes of all records added
|
||||||
from one read-only transaction, the counts from one query (2026-10-16,
|
up.
|
||||||
https://git.eeqj.de/sneak/sfdupes/issues/117)
|
|
||||||
|
It opens the database as `report` does and shows two phases on stderr,
|
||||||
|
`[1/2] open` and `[2/2] count`. The schema version and the counts come from
|
||||||
|
one read-only transaction, the counts from one query.
|
||||||
|
|
||||||
- `report` shows its work on stderr in two phases, `[1/2] open` and
|
- `report` shows its work on stderr in two phases, `[1/2] open` and
|
||||||
`[2/2] read`, with the progress display `scan` and `trees` use, from the
|
`[2/2] read`, with the progress display `scan` and `trees` use. The display
|
||||||
moment it starts until the summary line: `open` says it is opening the
|
runs from the moment `report` starts until the summary line. `open` says it is
|
||||||
database, also while it waits on a busy one; `read` counts the records, then
|
opening the database, also while it waits on a busy one. `read` counts the
|
||||||
reads them against that count in signature order, which SQLite takes from the
|
records, then reads them against that count in signature order. SQLite takes
|
||||||
`files_signature` index without sorting, so the first record comes at once.
|
that order from the `files_signature` index without sorting, so the first
|
||||||
`report` forms the duplicate groups itself, holding only those of the size it
|
record comes at once. `report` forms the duplicate groups itself, holding only
|
||||||
is reading, and writes them ordered by first path before it reads the next
|
those of the size it is reading, and writes them ordered by first path before
|
||||||
size. stdout and the summary line are unchanged, and SQLite no longer needs a
|
it reads the next size. stdout and the summary line are unchanged, and SQLite
|
||||||
temporary file to sort in (2026-10-16,
|
no longer needs a temporary file to sort in (2026-10-16,
|
||||||
https://git.eeqj.de/sneak/sfdupes/issues/114)
|
https://git.eeqj.de/sneak/sfdupes/issues/114)
|
||||||
|
|
||||||
- `trees` shows its work on stderr in four phases, `[1/4] open`, `[2/4] read`,
|
- `trees` shows its work on stderr in four phases, `[1/4] open`, `[2/4] read`,
|
||||||
`[3/4] group` and `[4/4] write`, each with the progress display `scan` uses,
|
`[3/4] group` and `[4/4] write`, each with the progress display `scan` uses.
|
||||||
from the moment it starts until the summary line: `open` says it is opening
|
The display runs from the moment `trees` starts until the summary line. `open`
|
||||||
the database, also while it waits on a busy one; `read` counts the records,
|
says it is opening the database, also while it waits on a busy one. `read`
|
||||||
then reads them against that count; `group` counts the directories it groups,
|
counts the records, then reads them against that count. `group` counts the
|
||||||
and `write` the groups it writes. A display's rate names what its phase
|
directories it groups, and `write` the groups it writes. A display's rate
|
||||||
counts: files, records, directories or groups. stdout and the summary line are
|
names what its phase counts: files, records, directories or groups. stdout and
|
||||||
unchanged. When stdout and stderr are the same file, as one terminal is, none
|
the summary line are unchanged. When stdout and stderr are the same file, as
|
||||||
of stdout's lines shares a line with the display; when they are not, the
|
one terminal is, no line of stdout shares a line with the display. When they
|
||||||
display keeps being drawn while stdout waits on a slow reader (2026-10-16,
|
are not, the display keeps being drawn while stdout waits on a slow reader
|
||||||
https://git.eeqj.de/sneak/sfdupes/issues/113)
|
(2026-10-16, https://git.eeqj.de/sneak/sfdupes/issues/113)
|
||||||
|
|
||||||
- each `scan` phase's progress display names the phase and its place among the
|
- Each `scan` phase's progress display names the phase and its place among the
|
||||||
five, as `[2/5] walk`, from the moment it starts, with its progress, elapsed
|
five, as `[2/5] walk`, from the moment it starts, with its progress, elapsed
|
||||||
time and ETA: `load` counts the records first, the walk's total is estimated
|
time and ETA. `load` counts the records first. The walk's total is estimated
|
||||||
from the records under the operands, and the content phase counts the records
|
from the records under the operands. The content phase counts the records it
|
||||||
it checks from SQLite's grouping alone, then says it is searching the database
|
checks from SQLite's grouping alone, then says it is searching the database
|
||||||
until the first one is returned. On a terminal a phase with an exact total
|
until the first one is returned. On a terminal a phase with an exact total
|
||||||
draws its line followed by the library's bar, and any other a spinner followed
|
draws its line followed by the library's bar, and any other a spinner followed
|
||||||
by the line; every display is drawn again several times a second. No line
|
by the line. Every display is drawn again several times a second. No line
|
||||||
reaches the terminal's last column, however narrow, also after the terminal is
|
reaches the terminal's last column, however narrow, also after the terminal is
|
||||||
made narrower during a phase: a line too wide leaves out its rate, then its
|
made narrower during a phase. A line too wide leaves out its rate, then its
|
||||||
bar, keeping the phase, progress, elapsed time and ETA whole, and is cut short
|
bar, keeping the phase, progress, elapsed time and ETA whole. It is cut short
|
||||||
only when those alone do not fit, as below about 70 columns late in a large
|
only when those alone do not fit, as below about 70 columns late in a large
|
||||||
scan. In plain mode the line is printed every 5 seconds whether or not an item
|
scan. In plain mode the line is printed every 5 seconds whether or not an item
|
||||||
completed (2026-10-16, https://git.eeqj.de/sneak/sfdupes/issues/112)
|
completed (2026-10-16, https://git.eeqj.de/sneak/sfdupes/issues/112)
|
||||||
|
|
||||||
- `scan`, `report` and `trees` first print a line on stderr naming the database
|
- `scan`, `report` and `trees` first print a line on stderr naming the database
|
||||||
they use, as an absolute path, and whether that path is the default or comes
|
they use, as an absolute path, and whether that path is the default or comes
|
||||||
from `SFDUPES_DATABASE`; `scan` says when the database is new. README
|
from `SFDUPES_DATABASE`. `scan` says when the database is new. README
|
||||||
"Database" says when `scan` writes each file (2026-10-16,
|
"Database" says when `scan` writes each file (2026-10-16,
|
||||||
https://git.eeqj.de/sneak/sfdupes/issues/111)
|
https://git.eeqj.de/sneak/sfdupes/issues/111)
|
||||||
|
|
||||||
- re-vendor the canonical files and model scripts from `sneak/prompts` `next` at
|
- The canonical files and model scripts re-vendored from `sneak/prompts` `next`
|
||||||
`e3f79c4`: golangci-lint v2.14.0; lint and test are phases of the `Dockerfile`
|
at `e3f79c4` (2026-10-15, https://git.eeqj.de/sneak/sfdupes/issues/95):
|
||||||
that write no image, and `make test` runs the suite under the race detector,
|
- golangci-lint v2.14.0;
|
||||||
so `Dockerfile.lint`, `script/verify-lint-image-pin` and `make test-race` are
|
- lint and test are phases of the `Dockerfile` that write no image, and
|
||||||
gone; every `docker build` in `script/` passes a new `CACHEBUST` value, which
|
`make test` runs the suite under the race detector. `Dockerfile.lint`,
|
||||||
every stage of the `Dockerfile` declares first, so every `RUN` step runs on
|
`script/verify-lint-image-pin` and `make test-race` are gone;
|
||||||
every build, while the steps that compile keep Go's build cache and the tests
|
- every `docker build` in `script/` passes a new `CACHEBUST` value, which
|
||||||
run with `-count=1`; prettier runs on the host, from the node and yarn
|
every stage of the `Dockerfile` declares first, so every `RUN` step runs
|
||||||
`script/bootstrap` installs, so the `prettier` and `markdown` stages are gone
|
on every build. The steps that compile keep Go's build cache, and the
|
||||||
and the build stage installs `git` and `make` itself; a new push cancels the
|
tests run with `-count=1`;
|
||||||
workflow's older run on the same branch, and a run stops after 20 minutes;
|
- prettier runs on the host, from the node and yarn `script/bootstrap`
|
||||||
`.claude/settings.json` is deleted (2026-10-15,
|
installs, so the `prettier` and `markdown` stages are gone and the build
|
||||||
https://git.eeqj.de/sneak/sfdupes/issues/95). Since
|
stage installs `git` and `make` itself;
|
||||||
https://git.eeqj.de/sneak/sfdupes/issues/109 the module download in `lint`,
|
- a new push cancels the workflow's older run on the same branch, and a run
|
||||||
`test` and the build stage, and the install of `git` and `make` in the build
|
stops after 20 minutes;
|
||||||
stage, come before `ARG CACHEBUST`, and Docker keeps them between builds;
|
- `.claude/settings.json` is deleted.
|
||||||
every step that lints, tests or compiles still runs on every build
|
|
||||||
|
|
||||||
- with `-x`, an operand under another operand is walked on its own instead of
|
Since https://git.eeqj.de/sneak/sfdupes/issues/109 the module download in
|
||||||
being dropped when a directory from it (from its directory, for a regular
|
`lint`, `test` and the build stage, and the install of `git` and `make` in
|
||||||
file) up to but not including that operand is on a different filesystem than
|
the build stage, come before `ARG CACHEBUST`. Docker keeps them between
|
||||||
that operand, whose walk then cannot reach it, and the records of files gone
|
builds, and every step that lints, tests or compiles still runs on every
|
||||||
from below it are deleted (2026-10-14,
|
build.
|
||||||
|
|
||||||
|
- With `-x`, an operand under another operand is no longer dropped when the
|
||||||
|
outer operand's walk cannot reach it. That is the case when a directory from
|
||||||
|
the inner operand up to, but not including, the outer one is on a different
|
||||||
|
filesystem than the outer one. The inner operand is then walked on its own.
|
||||||
|
For a regular-file operand the directories checked start at its directory. The
|
||||||
|
records of files gone from below it are deleted (2026-10-14,
|
||||||
https://git.eeqj.de/sneak/sfdupes/issues/106)
|
https://git.eeqj.de/sneak/sfdupes/issues/106)
|
||||||
|
|
||||||
- with `-x`, `scan` keeps the records below a directory on another filesystem
|
- With `-x`, `scan` keeps the records below a directory on another filesystem
|
||||||
under its operand, and counts their sizes in the size census, instead of
|
under its operand, and counts their sizes in the size census, instead of
|
||||||
deleting them (2026-10-14, https://git.eeqj.de/sneak/sfdupes/issues/99)
|
deleting them (2026-10-14, https://git.eeqj.de/sneak/sfdupes/issues/99)
|
||||||
|
|
||||||
- the hash and content phases open each file without blocking and check that it
|
- The hash and content phases open each file without blocking and check that it
|
||||||
is still a regular file, so one replaced by a FIFO or device node after it was
|
is still a regular file. A file replaced by a FIFO or device node after it was
|
||||||
statted is warned about and skipped instead of hanging the scan (2026-10-14,
|
statted is warned about and skipped instead of hanging the scan (2026-10-14,
|
||||||
https://git.eeqj.de/sneak/sfdupes/issues/100)
|
https://git.eeqj.de/sneak/sfdupes/issues/100)
|
||||||
|
|
||||||
- the scan lock file is named from the database's path with symlinks resolved,
|
- The scan lock file is named from the database's path with symlinks resolved,
|
||||||
after `scan` creates the database file if it is missing, so a scan through a
|
after `scan` creates the database file if it is missing. A scan through a
|
||||||
symlink to the database fails while another scan holds the lock (2026-10-14,
|
symlink to the database therefore fails while another scan holds the lock
|
||||||
https://git.eeqj.de/sneak/sfdupes/issues/101)
|
(2026-10-14, https://git.eeqj.de/sneak/sfdupes/issues/101)
|
||||||
|
|
||||||
- on a terminal, a warning for the last file of a phase no longer erases the
|
- On a terminal, a warning for the last file of a phase no longer erases the
|
||||||
phase's finished bar: the bar is drawn again below the warning (2026-10-14,
|
phase's finished bar. The bar is drawn again below the warning (2026-10-14,
|
||||||
https://git.eeqj.de/sneak/sfdupes/issues/102)
|
https://git.eeqj.de/sneak/sfdupes/issues/102)
|
||||||
|
|
||||||
- `scan` records mtime to the nanosecond, as whole seconds in `mtime` plus
|
- `scan` records the mtime to the nanosecond, as whole seconds in `mtime` plus
|
||||||
`mtime_nsec`, and compares it at that resolution, so a same-size rewrite
|
`mtime_nsec`, and compares it at that resolution. A same-size rewrite within
|
||||||
within the same second is re-hashed (2026-10-07,
|
the same second is hashed again (2026-10-07,
|
||||||
https://git.eeqj.de/sneak/sfdupes/issues/12)
|
https://git.eeqj.de/sneak/sfdupes/issues/12)
|
||||||
|
|
||||||
- cut the narration from `TODO.md` Completed Steps and from the comments in
|
- The narration was cut from `TODO.md` Completed Steps and from the comments in
|
||||||
`script/`, `Dockerfile` and `Dockerfile.lint` (gone since
|
`script/`, `Dockerfile` and `Dockerfile.lint` (gone since
|
||||||
https://git.eeqj.de/sneak/sfdupes/issues/95); §Workflow now branches from and
|
https://git.eeqj.de/sneak/sfdupes/issues/95). The Workflow section now
|
||||||
merges to `next` (2026-10-04, https://git.eeqj.de/sneak/sfdupes/issues/49)
|
branches from and merges to `next` (2026-10-04,
|
||||||
|
https://git.eeqj.de/sneak/sfdupes/issues/49)
|
||||||
|
|
||||||
- `make test-race` ran the test suite under the race detector in a cgo-enabled
|
- `make test-race` ran the test suite under the race detector in a cgo-enabled
|
||||||
container, outside `make check` (2026-10-04,
|
container, outside `make check`. Since
|
||||||
https://git.eeqj.de/sneak/sfdupes/issues/18). Since
|
|
||||||
https://git.eeqj.de/sneak/sfdupes/issues/95 `make test` itself runs the suite
|
https://git.eeqj.de/sneak/sfdupes/issues/95 `make test` itself runs the suite
|
||||||
under the race detector, in the `Dockerfile`'s Debian-based `test` phase, and
|
under the race detector, in the `Dockerfile`'s Debian-based `test` phase, and
|
||||||
`make test-race` is gone
|
`make test-race` is gone (2026-10-04,
|
||||||
|
https://git.eeqj.de/sneak/sfdupes/issues/18)
|
||||||
|
|
||||||
- a bare `docker build .` failed, naming `script/cibuild` and `script/docker`,
|
- A bare `docker build .` failed, naming `script/cibuild` and `script/docker`,
|
||||||
rather than serve the gates from cache (2026-10-04,
|
rather than serve the gates from cache. Since
|
||||||
https://git.eeqj.de/sneak/sfdupes/issues/39). Since
|
|
||||||
https://git.eeqj.de/sneak/sfdupes/issues/95 a bare build succeeds, as
|
https://git.eeqj.de/sneak/sfdupes/issues/95 a bare build succeeds, as
|
||||||
`REPO_POLICIES.md` requires, and may serve the gates from cache; the builds in
|
`REPO_POLICIES.md` requires, and may serve the gates from cache. The builds in
|
||||||
`script/` pass a new `CACHEBUST` value, so theirs always run
|
`script/` pass a new `CACHEBUST` value, so theirs always run (2026-10-04,
|
||||||
|
https://git.eeqj.de/sneak/sfdupes/issues/39)
|
||||||
|
|
||||||
- `make fmt` and `make fmt-check` run prettier over all Markdown, and CI checks
|
- `make fmt` and `make fmt-check` run prettier over all Markdown, and CI checks
|
||||||
it; all Markdown reformatted (2026-10-04,
|
it. All Markdown was reformatted (2026-10-04,
|
||||||
https://git.eeqj.de/sneak/sfdupes/issues/19)
|
https://git.eeqj.de/sneak/sfdupes/issues/19)
|
||||||
|
|
||||||
- `script/lint` writes no image, so a run no longer leaves an untagged one
|
- `script/lint` writes no image, so a run no longer leaves an untagged one
|
||||||
behind (2026-10-04, https://git.eeqj.de/sneak/sfdupes/issues/48)
|
behind (2026-10-04, https://git.eeqj.de/sneak/sfdupes/issues/48)
|
||||||
|
|
||||||
- tests cover a missing database, `scan` keeping stdout empty, its skip warning,
|
- Tests cover a missing database, `scan` keeping stdout empty, its skip warning,
|
||||||
the `report` and `trees` summary lines, and every subcommand going through
|
the `report` and `trees` summary lines, and every subcommand going through
|
||||||
`runE` (2026-10-04, https://git.eeqj.de/sneak/sfdupes/issues/16)
|
`runE` (2026-10-04, https://git.eeqj.de/sneak/sfdupes/issues/16)
|
||||||
|
|
||||||
@@ -278,296 +273,284 @@
|
|||||||
`gomodguard_v2`, so lint no longer prints a deprecation warning (2026-10-04,
|
`gomodguard_v2`, so lint no longer prints a deprecation warning (2026-10-04,
|
||||||
https://git.eeqj.de/sneak/sfdupes/issues/26)
|
https://git.eeqj.de/sneak/sfdupes/issues/26)
|
||||||
|
|
||||||
- a test fails when either `hashWorker` cancellation check in `scan.go` is
|
- A test fails when either `hashWorker` cancellation check in `scan.go` is
|
||||||
removed (2026-10-04, https://git.eeqj.de/sneak/sfdupes/issues/83)
|
removed (2026-10-04, https://git.eeqj.de/sneak/sfdupes/issues/83)
|
||||||
|
|
||||||
- a database path holding `?`, `#` or `%` opens exactly the file it names
|
- A database path holding `?`, `#` or `%` opens exactly the file it names
|
||||||
(2026-10-04, https://git.eeqj.de/sneak/sfdupes/issues/55)
|
(2026-10-04, https://git.eeqj.de/sneak/sfdupes/issues/55)
|
||||||
|
|
||||||
- `scan` rejects `--workers` below 1 as a usage error instead of running
|
- `scan` rejects `--workers` below 1 as a usage error instead of running
|
||||||
single-threaded (2026-10-04, https://git.eeqj.de/sneak/sfdupes/issues/10)
|
single-threaded (2026-10-04, https://git.eeqj.de/sneak/sfdupes/issues/10)
|
||||||
|
|
||||||
- a test fails when either walk cancellation check in `scan.go` is removed
|
- A test fails when either walk cancellation check in `scan.go` is removed
|
||||||
(2026-10-04, https://git.eeqj.de/sneak/sfdupes/issues/81)
|
(2026-10-04, https://git.eeqj.de/sneak/sfdupes/issues/81)
|
||||||
|
|
||||||
- test that `scan` refuses a database with another schema version (2026-10-04,
|
- A test checks that `scan` refuses a database with another schema version
|
||||||
https://git.eeqj.de/sneak/sfdupes/issues/64)
|
(2026-10-04, https://git.eeqj.de/sneak/sfdupes/issues/64)
|
||||||
|
|
||||||
- correct four inaccurate comments in `cancel_test.go` and rename
|
- Four inaccurate comments in `cancel_test.go` corrected, and
|
||||||
`walkCancelInFlightDirs` to `walkCancelInFlightFiles` (2026-10-04,
|
`walkCancelInFlightDirs` renamed to `walkCancelInFlightFiles` (2026-10-04,
|
||||||
https://git.eeqj.de/sneak/sfdupes/issues/33)
|
https://git.eeqj.de/sneak/sfdupes/issues/33)
|
||||||
|
|
||||||
- test the `-x` filesystem-boundary rules in `subdirJob` (2026-10-04,
|
- The `-x` filesystem-boundary rules in `subdirJob` are tested (2026-10-04,
|
||||||
https://git.eeqj.de/sneak/sfdupes/issues/17)
|
https://git.eeqj.de/sneak/sfdupes/issues/17)
|
||||||
|
|
||||||
- `scan` creates the schema in one transaction; a version-0 database with a
|
- `scan` creates the schema in one transaction. A version-0 database with a
|
||||||
`files` table is refused with a clear schema-version error (2026-10-04,
|
`files` table is refused with a clear schema-version error (2026-10-04,
|
||||||
https://git.eeqj.de/sneak/sfdupes/issues/11)
|
https://git.eeqj.de/sneak/sfdupes/issues/11)
|
||||||
|
|
||||||
- README documents install, Docker, a daily cron scan and how to read and check
|
- README documents install, Docker, a daily cron scan and how to read and check
|
||||||
the reports (2026-10-04, https://git.eeqj.de/sneak/sfdupes/issues/54)
|
the reports (2026-10-04, https://git.eeqj.de/sneak/sfdupes/issues/54)
|
||||||
|
|
||||||
- the `Dockerfile` build stage kept the Go module cache out of `builder`'s home
|
- The `Dockerfile` build stage kept the Go module cache out of `builder`'s home
|
||||||
and copied the sources with `--chown`, so no `chown -R` walked them
|
and copied the sources with `--chown`, so no `chown -R` walked them. Since
|
||||||
(2026-10-04, https://git.eeqj.de/sneak/sfdupes/issues/43). Since
|
|
||||||
https://git.eeqj.de/sneak/sfdupes/issues/95 there is no `builder` user and
|
https://git.eeqj.de/sneak/sfdupes/issues/95 there is no `builder` user and
|
||||||
nothing changes owner: the build stage only compiles, as root, and the tests
|
nothing changes owner: the build stage only compiles, as root, and the tests
|
||||||
run as `nobody` in the `test` phase
|
run as `nobody` in the `test` phase (2026-10-04,
|
||||||
|
https://git.eeqj.de/sneak/sfdupes/issues/43)
|
||||||
|
|
||||||
- `--version` prints `sfdupes VERSION` to stdout; README documents it and
|
- `--version` prints `sfdupes VERSION` to stdout. README documents it and
|
||||||
`--help` (2026-10-04, https://git.eeqj.de/sneak/sfdupes/issues/15)
|
`--help` (2026-10-04, https://git.eeqj.de/sneak/sfdupes/issues/15)
|
||||||
|
|
||||||
- `scan` stops cleanly on `SIGINT` or `SIGTERM`: commits what it has hashed,
|
- `scan` stops cleanly on `SIGINT` or `SIGTERM`: it commits what it has hashed,
|
||||||
deletes nothing more, exits 1 (2026-10-04,
|
deletes nothing more, and exits 1 (2026-10-04,
|
||||||
https://git.eeqj.de/sneak/sfdupes/issues/5)
|
https://git.eeqj.de/sneak/sfdupes/issues/5)
|
||||||
|
|
||||||
- `report` and `trees` stream the records instead of holding them all in memory;
|
- `report` and `trees` stream the records instead of holding them all in memory.
|
||||||
the schema gains the `files_signature` index (2026-10-04,
|
The schema gains the `files_signature` index (2026-10-04,
|
||||||
https://git.eeqj.de/sneak/sfdupes/issues/14)
|
https://git.eeqj.de/sneak/sfdupes/issues/14)
|
||||||
|
|
||||||
- progress prints at once on a non-terminal, uses a real terminal test, and
|
- Progress prints at once on a non-terminal, uses a real terminal test, and
|
||||||
prints warnings through a spinner instead of racing its redraw (2026-10-03,
|
prints warnings through a spinner instead of racing its redraw (2026-10-03,
|
||||||
https://git.eeqj.de/sneak/sfdupes/issues/13)
|
https://git.eeqj.de/sneak/sfdupes/issues/13)
|
||||||
|
|
||||||
- warn about and skip symlink, socket, FIFO, device and `.zfs` operands, keeping
|
- Symlink, socket, FIFO, device and `.zfs` operands are warned about and
|
||||||
the records beneath them (2026-10-03,
|
skipped, and the records beneath them are kept (2026-10-03,
|
||||||
https://git.eeqj.de/sneak/sfdupes/issues/9)
|
https://git.eeqj.de/sneak/sfdupes/issues/9)
|
||||||
|
|
||||||
- `scan` holds a lock on a lock file beside the database for its whole run, so a
|
- `scan` holds a lock on a lock file beside the database for its whole run, so a
|
||||||
second `scan` fails at once with exit 1 (2026-10-03,
|
second `scan` fails at once with exit 1 (2026-10-03,
|
||||||
https://git.eeqj.de/sneak/sfdupes/issues/53)
|
https://git.eeqj.de/sneak/sfdupes/issues/53)
|
||||||
|
|
||||||
- test stdout write failures in `report` and `trees`; README states that
|
- Tests cover stdout write failures in `report` and `trees`. README states that
|
||||||
`| head` ends sfdupes by `SIGPIPE` and `>&-` writes to `/dev/null`
|
`| head` ends sfdupes by `SIGPIPE` and that `>&-` writes to `/dev/null`
|
||||||
(2026-10-03, https://git.eeqj.de/sneak/sfdupes/issues/30)
|
(2026-10-03, https://git.eeqj.de/sneak/sfdupes/issues/30)
|
||||||
|
|
||||||
- `report` and `trees` open the database read-only, and `scan` leaves it out of
|
- `report` and `trees` open the database read-only, and `scan` leaves it out of
|
||||||
WAL mode, so reading needs only read access (2026-10-03, closes
|
WAL mode, so reading needs only read access (2026-10-03,
|
||||||
https://git.eeqj.de/sneak/sfdupes/issues/8)
|
https://git.eeqj.de/sneak/sfdupes/issues/8)
|
||||||
|
|
||||||
- escape tabs, newlines, carriage returns and backslashes in report, trees and
|
- Tabs, newlines, carriage returns and backslashes are escaped in report, trees
|
||||||
warning paths; the root directory's path is `/` (2026-10-03,
|
and warning paths. The root directory's path is `/` (2026-10-03,
|
||||||
https://git.eeqj.de/sneak/sfdupes/issues/7)
|
https://git.eeqj.de/sneak/sfdupes/issues/7)
|
||||||
|
|
||||||
- stamp the git tag or short commit in a plain `docker build .` instead of `dev`
|
- A plain `docker build .` stamps the git tag or short commit instead of `dev`.
|
||||||
(2026-10-02, branch `next`, closes
|
`.dockerignore` sends `.git` without `.git/config`. The build stage stamps the
|
||||||
https://git.eeqj.de/sneak/sfdupes/issues/67): `.dockerignore` sends `.git`
|
`VERSION` build argument, else `git describe --tags --always`. It fails if the
|
||||||
without `.git/config`; the build stage stamps the `VERSION` build argument,
|
context carries `.git` and the version is still empty, `dev` or `unknown`. CI
|
||||||
else `git describe --tags --always`, and fails if the context carries `.git`
|
checks out the full history (`fetch-depth: 0`), so it stamps the same value as
|
||||||
and the version is still empty, `dev` or `unknown`. CI checks out the full
|
`make build` (2026-10-02, branch `next`,
|
||||||
history (`fetch-depth: 0`) so it stamps the same value as `make build`.
|
https://git.eeqj.de/sneak/sfdupes/issues/67)
|
||||||
|
|
||||||
- replace the 1 KiB end-window sampling with the head/tail plus content-hash
|
- The 1 KiB end-window sampling was replaced with the head/tail plus
|
||||||
ladder (2026-09-22, branch `next`, closes
|
content-hash ladder, and README "Duplicate detection" documented every rung. A
|
||||||
https://git.eeqj.de/sneak/sfdupes/issues/61); README "Duplicate detection"
|
file under 10 MiB was hashed in full, and its `head`, `tail` and `content` all
|
||||||
documents every rung. A file under 10 MiB is hashed in full, and its `head`,
|
held that hash. A larger file got only its 64 KiB `head` and `tail` in the
|
||||||
`tail` and `content` all hold that hash. A larger file gets only its 64 KiB
|
hash phase. The content phase, after the update phase, read it for `content`
|
||||||
`head` and `tail` in the hash phase; the content phase, after the update
|
only when its size, `head` and `tail` matched another record's. `content` was
|
||||||
phase, reads it for `content` (the whole file below 50 MiB, gigabyte-spaced 1
|
the whole file below 50 MiB, and gigabyte-spaced 1 MiB samples at or above. A
|
||||||
MiB samples at or above) only when its size, `head` and `tail` match another
|
file gone or changed since its record was written was never read. `report` and
|
||||||
record's from this scan or an earlier one, and never reads a file gone or
|
`trees` left out any record without a `content` hash. The `content` column is
|
||||||
changed since its record was written. `report` and `trees` leave out any
|
part of the version 1 schema. Superseded by
|
||||||
record without a `content` hash. The `content` column is part of the version 1
|
https://git.eeqj.de/sneak/sfdupes/issues/132 (2026-09-22, branch `next`,
|
||||||
schema.
|
https://git.eeqj.de/sneak/sfdupes/issues/61)
|
||||||
|
|
||||||
- remove the dead `files.dat` references from `Makefile`, `.gitignore` and
|
- The dead `files.dat` references removed from `Makefile`, `.gitignore` and
|
||||||
`.dockerignore` (2026-09-21, branch `next`, closes
|
`.dockerignore` (2026-09-21, branch `next`,
|
||||||
https://git.eeqj.de/sneak/sfdupes/issues/22)
|
https://git.eeqj.de/sneak/sfdupes/issues/22)
|
||||||
|
|
||||||
- fix the lint-image pin comments and `FROM` form in `Dockerfile` and
|
- The lint-image pin comments and `FROM` form in `Dockerfile` and
|
||||||
`Dockerfile.lint` (2026-08-10, branch `next`, closes
|
`Dockerfile.lint` fixed. Both pins became the policy
|
||||||
https://git.eeqj.de/sneak/sfdupes/issues/25): both pins became the policy
|
|
||||||
`# image:vX.Y.Z, YYYY-MM-DD` comment over a bare `FROM image@sha256:...`,
|
`# image:vX.Y.Z, YYYY-MM-DD` comment over a bare `FROM image@sha256:...`,
|
||||||
without the false `(Debian-based)` note or the tag. Since
|
without the false `(Debian-based)` note or the tag. Since
|
||||||
https://git.eeqj.de/sneak/sfdupes/issues/95 the `Dockerfile`'s `lint` phase
|
https://git.eeqj.de/sneak/sfdupes/issues/95 the `Dockerfile`'s `lint` phase
|
||||||
holds the only golangci-lint pin, in that form, so `Dockerfile.lint` and
|
holds the only golangci-lint pin, in that form. `Dockerfile.lint` and
|
||||||
`script/verify-lint-image-pin`, which compared the two pins, are gone.
|
`script/verify-lint-image-pin`, which compared the two pins, are gone
|
||||||
|
(2026-08-10, branch `next`, https://git.eeqj.de/sneak/sfdupes/issues/25)
|
||||||
|
|
||||||
- run all linting in Docker via `Dockerfile.lint` and `script/lint` (2026-08-10,
|
- All linting runs in Docker, via `Dockerfile.lint` and `script/lint`. Per the
|
||||||
branch `next`, closes https://git.eeqj.de/sneak/sfdupes/issues/46): per the
|
|
||||||
owner ruling the linter is never installed on a host, and
|
owner ruling the linter is never installed on a host, and
|
||||||
`golangci-lint config verify` runs before `golangci-lint run`.
|
`golangci-lint config verify` runs before `golangci-lint run`.
|
||||||
`script/bootstrap` stopped installing or pinning the linter, and
|
`script/bootstrap` stopped installing or pinning the linter, and
|
||||||
`script/verify-linter-pin` was retired. Since
|
`script/verify-linter-pin` was retired. Since
|
||||||
https://git.eeqj.de/sneak/sfdupes/issues/95 both commands run in the
|
https://git.eeqj.de/sneak/sfdupes/issues/95 both commands run in the
|
||||||
`Dockerfile`'s `lint` phase, which `script/lint` builds alone and the build
|
`Dockerfile`'s `lint` phase, which `script/lint` builds alone and the build
|
||||||
stage depends on through `COPY --from=lint /src/go.sum /dev/null`;
|
stage depends on through `COPY --from=lint /src/go.sum /dev/null`.
|
||||||
`Dockerfile.lint` and `script/verify-lint-image-pin` are gone. Nothing inside
|
`Dockerfile.lint` and `script/verify-lint-image-pin` are gone. Nothing inside
|
||||||
an image build may run docker, so the phase calls `golangci-lint` directly.
|
an image build may run docker, so the phase calls `golangci-lint` directly
|
||||||
|
(2026-08-10, branch `next`, https://git.eeqj.de/sneak/sfdupes/issues/46)
|
||||||
|
|
||||||
- install the Docker build stage's prerequisites by running `script/bootstrap`
|
- The Docker build stage's prerequisites were installed by running
|
||||||
instead of `apk add --no-cache make` inline (2026-08-09, branch
|
`script/bootstrap` instead of `apk add --no-cache make` inline.
|
||||||
`dockerfile-bootstrap`, closes https://git.eeqj.de/sneak/sfdupes/issues/42),
|
`script/verify-linter-pin` failed the build unless the linter copied from the
|
||||||
with `script/verify-linter-pin` failing the build unless the linter copied
|
lint stage was the version `script/bootstrap` pinned. Builds then took up to
|
||||||
from the lint stage was the version `script/bootstrap` pinned. Builds then
|
5m14s cold, mostly in a `chown -R` of the module cache, filed as
|
||||||
took up to 5m14s cold, mostly in a `chown -R` of the module cache, filed as
|
|
||||||
https://git.eeqj.de/sneak/sfdupes/issues/43. Since
|
https://git.eeqj.de/sneak/sfdupes/issues/43. Since
|
||||||
https://git.eeqj.de/sneak/sfdupes/issues/95 the build stage installs `git` and
|
https://git.eeqj.de/sneak/sfdupes/issues/95 the build stage installs `git` and
|
||||||
`make` with `apk add --no-cache` and only compiles; the `lint` and `test`
|
`make` with `apk add --no-cache` and only compiles; the `lint` and `test`
|
||||||
phases are the gates
|
phases are the gates (2026-08-09, branch `dockerfile-bootstrap`,
|
||||||
- bust the Docker layer cache for the gate steps, so `script/cibuild` and
|
https://git.eeqj.de/sneak/sfdupes/issues/42)
|
||||||
`script/docker` cannot report a green they did not earn (2026-08-09, branch
|
|
||||||
`cibuild-cache-bust`, closes https://git.eeqj.de/sneak/sfdupes/issues/32): the
|
- The Docker layer cache is busted for the gate steps, so `script/cibuild` and
|
||||||
`Dockerfile` copies the tree before its gates, so on an unchanged tree Docker
|
`script/docker` cannot report a green they did not earn. The `Dockerfile`
|
||||||
served them from cache and the build exited 0 having run nothing. The fix was
|
copies the tree before its gates, so on an unchanged tree Docker served them
|
||||||
a `CHECK_EPOCH` build argument; since
|
from cache and the build exited 0 having run nothing. The fix was a
|
||||||
|
`CHECK_EPOCH` build argument. Since
|
||||||
https://git.eeqj.de/sneak/sfdupes/issues/95 every `docker build` in `script/`
|
https://git.eeqj.de/sneak/sfdupes/issues/95 every `docker build` in `script/`
|
||||||
passes a new `CACHEBUST` value instead, which every stage of the `Dockerfile`
|
passes a new `CACHEBUST` value instead, which every stage of the `Dockerfile`
|
||||||
declares before every step that lints, tests or compiles. Run as root, the
|
declares before every step that lints, tests or compiles. Run as root, the
|
||||||
tests fail `TestScanHardlinkRunFailsTogether`, because root reads through the
|
tests fail `TestScanHardlinkRunFailsTogether`, because root reads through the
|
||||||
`chmod(0)` the test relies on, so the `test` phase runs them as `nobody`
|
`chmod(0)` the test relies on, so the `test` phase runs them as `nobody`
|
||||||
- check the installed golangci-lint version in `script/bootstrap` instead of
|
(2026-08-09, branch `cibuild-cache-bust`,
|
||||||
only its presence (2026-08-09, branch `bootstrap-version-check`, closes
|
https://git.eeqj.de/sneak/sfdupes/issues/32)
|
||||||
https://git.eeqj.de/sneak/sfdupes/issues/24): the version lives only in
|
|
||||||
`GOLANGCI_LINT_VERSION`, with the `go install` module ref derived from it, and
|
|
||||||
any installed version that is not the pin — older, newer, absent or
|
|
||||||
unparseable — is reinstalled. `go install` writes into `GOBIN` (or
|
|
||||||
`GOPATH/bin`) while `make lint` runs the first `golangci-lint` on `PATH`, so
|
|
||||||
bootstrap re-reads the effective version after installing and, on a mismatch,
|
|
||||||
prints both paths and both versions and exits non-zero; it does not reorder
|
|
||||||
`PATH` or delete anyone's binary. The `--version` call keeps its stderr and is
|
|
||||||
bounded by `timeout(1)` where that exists. `git`, `make` and `go` keep
|
|
||||||
presence-only checks. Verified by bootstrapping this host from v2.10.1 to
|
|
||||||
v2.12.2 and again to a no-op, and by stub runs of the script under `dash`
|
|
||||||
covering a thirteen-input version-parse matrix, a shadowed install that must
|
|
||||||
exit non-zero, an install destination not on `PATH`, `GOBIN` set, and a wedged
|
|
||||||
binary that must hit the timeout; `make check` and `make lint` are clean at
|
|
||||||
v2.12.2, so v2.10.1 was not hiding any findings on `main`
|
|
||||||
- unwind the hash worker pool on the error path (2026-08-09, branch
|
|
||||||
`hash-pool-cleanup`, closes https://git.eeqj.de/sneak/sfdupes/issues/6): the
|
|
||||||
pool is now an owned, context-aware `hashPool`: every blocking send in the
|
|
||||||
feeder and the workers selects on `ctx.Done()`, `jobs` is closed on every path
|
|
||||||
out, and `hashPhase` defers `pool.stop()`, which cancels and then drains
|
|
||||||
`results` until the last goroutine has exited — draining is what frees a
|
|
||||||
worker already parked on a send. `ctx` is threaded from `cmd.Context()`
|
|
||||||
through `runScan`, `syncScan`, both worker pools and the whole database layer,
|
|
||||||
as the first parameter everywhere. The walk pool gets the same treatment plus
|
|
||||||
a `ctx.Err()` guard after the walk: a cancelled walk yields a partial size
|
|
||||||
census, and every file it never reached looks vanished to the update phase.
|
|
||||||
That phase's own `BeginTx` also fails on the cancelled context before deleting
|
|
||||||
anything, but the guard is the barrier that still holds once an interrupted
|
|
||||||
scan may commit what it has. Tests drive `run(scan)` against a database whose
|
|
||||||
insert trigger aborts and assert that the scan fails instead of hanging and
|
|
||||||
that `runtime.NumGoroutine()` polls back to its pre-scan baseline; others
|
|
||||||
cancel a scan part-way through the walk, deterministically, by counting its
|
|
||||||
own consultations of `ctx.Done()`, and assert that it stops at the guard
|
|
||||||
holding a partial census and a still-populated record index, with every record
|
|
||||||
intact. Direct tests of `sendEvent`, the walk workers, `dispatchDirs`,
|
|
||||||
`feedHashJobs`, `hashWorker` and `hashPhase` cover the remaining cancellation
|
|
||||||
branches of both pools
|
|
||||||
- guarantee the database is closed on every fatal exit path (2026-08-09, branch
|
|
||||||
`db-close-on-fatal`, closes https://git.eeqj.de/sneak/sfdupes/issues/4):
|
|
||||||
`fatalf` and its `os.Exit(1)` are gone, so the deferred `db.Close()` — and
|
|
||||||
with it the SQLite WAL checkpoint — now actually runs when a subcommand fails;
|
|
||||||
`runScan`, `runReport`, `runTrees`, `loadRecords` and `resolveRoots` return
|
|
||||||
errors instead. The single exit point is `run` in `main.go`: it maps a
|
|
||||||
`fatalError` (anything a subcommand returned) to exit 1 and cobra's own
|
|
||||||
argument and flag errors to exit 2, which keeps a runtime failure from being
|
|
||||||
reported as a usage error or printing the usage text. New `main_test.go`
|
|
||||||
drives the CLI in-process and asserts the exit codes from README §Error
|
|
||||||
handling plus the stdout/stderr split, including that a fatal error raised
|
|
||||||
after the database is open leaves no `-wal`/`-shm` sidecar behind for `scan`,
|
|
||||||
`report` or `trees`
|
|
||||||
- update golangci-lint to v2.12.2 with the canonical config (2026-08-09, branch
|
|
||||||
`golangci-v2.12.2`, merged as `38a01bd`, closes
|
|
||||||
https://git.eeqj.de/sneak/sfdupes/issues/3): bumped the pinned linter in the
|
|
||||||
`Dockerfile` lint stage and `script/bootstrap` from v2.12.1 to v2.12.2, and
|
|
||||||
replaced `.golangci.yml` with the canonical file — the linter settings (`lll`,
|
|
||||||
`funlen`, `cyclop`, `dupl` thresholds) now live under `linters.settings` per
|
|
||||||
the v2 schema, so they are actually applied; no new lint findings surfaced
|
|
||||||
- convert Makefile targets to scripts-to-rule-them-all `script/` entrypoints
|
|
||||||
like the other managed repos (2026-07-26, commit `3abeacf`, closes
|
|
||||||
https://git.eeqj.de/sneak/sfdupes/issues/1): all 12 `script/` entrypoints
|
|
||||||
exist (`bootstrap`, `setup`, `projectname`, `test`, `lint`, `fmt`,
|
|
||||||
`fmt-check`, `check`, `docker`, `cibuild`, `precommit`, `install-precommit`)
|
|
||||||
and every Makefile target is now a thin shim over them
|
|
||||||
- make the binary the default Make target (2026-07-24, branch
|
|
||||||
`make-default-target`): plain `make` now builds `sfdupes` (previously it ran
|
|
||||||
`check` plus `build`); `make build` remains as an alias
|
|
||||||
- scan-wide phases, concurrent operands, batched updates (2026-07-24, branch
|
|
||||||
`scan-wide-phases`): all operands seed the shared walk pool and every pass
|
|
||||||
runs once over the whole scan, so totals and ETAs are scan-global; the
|
|
||||||
per-operand walk/hash/update cycles and their stderr announcements are gone;
|
|
||||||
the update pass commits in batched transactions — the filesystem is
|
|
||||||
authoritative and the database an eventually-consistent reflection, so
|
|
||||||
scan-level atomicity is not required
|
|
||||||
- split the stat pass back out of the walk (2026-07-24, branch
|
|
||||||
`parallel-phases`): phases are strictly sequential again — walk, stat, hash,
|
|
||||||
update per operand — with parallelism only inside each phase; the walk
|
|
||||||
enumerates paths with per-directory workers and the stat pass lstats them with
|
|
||||||
per-file workers, restoring the exact total/ETA stat bar
|
|
||||||
- announce each operand on stderr before its passes (2026-07-24, branch
|
|
||||||
`scan-operand-progress`): with per-operand walk/hash/update cycles, a
|
|
||||||
multi-operand run (e.g. `scan /srv/*`) showed pass totals that looked like the
|
|
||||||
whole run's
|
|
||||||
- parallel walk (2026-07-24, branch `parallel-walk`): the walk pass was a single
|
|
||||||
goroutine and took hours at ~20M files on a busy pool (observed: 22M files in
|
|
||||||
4h on a ZFS server); it is now a per-directory worker-pool traversal that
|
|
||||||
records size/mtime during the walk (folding away the separate stat pass,
|
|
||||||
halving metadata I/O), and each `PATH` operand commits in its own transaction
|
|
||||||
so an interrupted scan keeps completed operands
|
|
||||||
|
|
||||||
- persistent scan database (2026-07-24, branch `persistent-database`): `scan`
|
- `script/bootstrap` checked the installed golangci-lint version instead of only
|
||||||
now maintains a SQLite database (`modernc.org/sqlite`, pure Go, cgo stays
|
its presence. The version lived only in `GOLANGCI_LINT_VERSION`, with the
|
||||||
disabled) keyed by absolute path that survives between runs — a rescan hashes
|
`go install` module ref derived from it, and any installed version that was
|
||||||
only new or changed files (by mtime/size), deletes records for files vanished
|
not the pin was reinstalled. `go install` writes into `GOBIN` (or
|
||||||
from under the scanned operands, and leaves records outside them untouched, so
|
`GOPATH/bin`) while `make lint` ran the first `golangci-lint` on `PATH`, so
|
||||||
`scan` can be cronned daily; `report` and `trees` read the database (no
|
bootstrap re-read the effective version after installing. On a mismatch it
|
||||||
positional arguments) instead of a scan stream. Database at
|
printed both paths and both versions and exited non-zero; it did not reorder
|
||||||
`/var/lib/sfdupes/db.sqlite`, overridable via `SFDUPES_DATABASE`; WAL
|
`PATH` or delete anyone's binary. The `--version` call kept its stderr and was
|
||||||
|
bounded by `timeout(1)` where that exists. `git`, `make` and `go` kept
|
||||||
|
presence-only checks. Superseded by
|
||||||
|
https://git.eeqj.de/sneak/sfdupes/issues/46: the linter is no longer installed
|
||||||
|
on a host (2026-08-09, branch `bootstrap-version-check`,
|
||||||
|
https://git.eeqj.de/sneak/sfdupes/issues/24)
|
||||||
|
|
||||||
|
- The hash worker pool unwinds on the error path. The pool is an owned,
|
||||||
|
context-aware `hashPool`. Every blocking send in the feeder and the workers
|
||||||
|
selects on `ctx.Done()`, and `jobs` is closed on every path out. `hashPhase`
|
||||||
|
defers `pool.stop()`, which cancels and then drains `results` until the last
|
||||||
|
goroutine has exited; draining is what frees a worker parked on a send. `ctx`
|
||||||
|
is threaded from `cmd.Context()` through `runScan`, `syncScan`, both worker
|
||||||
|
pools and the whole database layer, as the first parameter everywhere. The
|
||||||
|
walk pool gets the same treatment, plus a `ctx.Err()` guard after the walk. A
|
||||||
|
cancelled walk yields a partial size census, and every file it never reached
|
||||||
|
looks vanished to the update phase. The guard is the barrier that still holds
|
||||||
|
once an interrupted scan may commit what it has. Tests drive `run(scan)`
|
||||||
|
against a database whose insert trigger aborts, and assert that the scan fails
|
||||||
|
instead of hanging and that the goroutine count returns to its pre-scan
|
||||||
|
baseline. Other tests cancel a scan part-way through the walk and assert that
|
||||||
|
it stops at the guard with every record intact (2026-08-09, branch
|
||||||
|
`hash-pool-cleanup`, https://git.eeqj.de/sneak/sfdupes/issues/6)
|
||||||
|
|
||||||
|
- The database is closed on every fatal exit path. `fatalf` and its `os.Exit(1)`
|
||||||
|
are gone, so the deferred `db.Close()`, and with it the SQLite WAL checkpoint,
|
||||||
|
runs when a subcommand fails. `runScan`, `runReport`, `runTrees`,
|
||||||
|
`loadRecords` and `resolveRoots` return errors instead. The single exit point
|
||||||
|
is `run` in `main.go`. It maps a `fatalError` to exit 1 and cobra's own
|
||||||
|
argument and flag errors to exit 2, so a runtime failure is never reported as
|
||||||
|
a usage error. `main_test.go` drives the CLI in-process and asserts the exit
|
||||||
|
codes from README "Error handling" and the stdout/stderr split. That includes
|
||||||
|
a fatal error after the database is open leaving no `-wal` or `-shm` file
|
||||||
|
behind (2026-08-09, branch `db-close-on-fatal`,
|
||||||
|
https://git.eeqj.de/sneak/sfdupes/issues/4)
|
||||||
|
|
||||||
|
- golangci-lint updated to v2.12.2 with the canonical config, merged as
|
||||||
|
`38a01bd`. The pinned linter in the `Dockerfile` lint stage and
|
||||||
|
`script/bootstrap` went from v2.12.1 to v2.12.2, and `.golangci.yml` was
|
||||||
|
replaced with the canonical file. The linter settings (`lll`, `funlen`,
|
||||||
|
`cyclop`, `dupl` thresholds) now live under `linters.settings`, per the v2
|
||||||
|
schema, so they are applied. No new lint findings surfaced (2026-08-09, branch
|
||||||
|
`golangci-v2.12.2`, https://git.eeqj.de/sneak/sfdupes/issues/3)
|
||||||
|
|
||||||
|
- Makefile targets converted to scripts-to-rule-them-all `script/` entrypoints,
|
||||||
|
like the other managed repos. All 12 entrypoints exist, and every Makefile
|
||||||
|
target is a thin shim over them (2026-07-26, commit `3abeacf`,
|
||||||
|
https://git.eeqj.de/sneak/sfdupes/issues/1)
|
||||||
|
|
||||||
|
- The binary is the default Make target: plain `make` builds `sfdupes`, where it
|
||||||
|
used to run `check` plus `build`. `make build` remains as an alias
|
||||||
|
(2026-07-24, branch `make-default-target`)
|
||||||
|
|
||||||
|
- Scan-wide phases, concurrent operands and batched updates. All operands seed
|
||||||
|
the shared walk pool, and every pass runs once over the whole scan, so totals
|
||||||
|
and ETAs are scan-global. The per-operand walk, hash and update cycles and
|
||||||
|
their stderr announcements are gone. The update pass commits in batched
|
||||||
|
transactions. The filesystem is authoritative and the database an
|
||||||
|
eventually-consistent reflection of it, so scan-level atomicity is not
|
||||||
|
required (2026-07-24, branch `scan-wide-phases`)
|
||||||
|
|
||||||
|
- The stat pass split back out of the walk. Phases are strictly sequential
|
||||||
|
again, walk, stat, hash and update per operand, with parallelism only inside
|
||||||
|
each phase. The walk enumerates paths with per-directory workers, and the stat
|
||||||
|
pass lstats them with per-file workers, restoring the exact total and ETA of
|
||||||
|
the stat bar (2026-07-24, branch `parallel-phases`)
|
||||||
|
|
||||||
|
- Each operand is announced on stderr before its passes. With per-operand walk,
|
||||||
|
hash and update cycles, a multi-operand run such as `scan /srv/*` showed pass
|
||||||
|
totals that looked like the whole run's (2026-07-24, branch
|
||||||
|
`scan-operand-progress`)
|
||||||
|
|
||||||
|
- Parallel walk. The walk pass was a single goroutine and took hours at about 20
|
||||||
|
million files on a busy pool: 22 million files in 4 hours on a ZFS server. It
|
||||||
|
is now a per-directory worker-pool traversal that records size and mtime
|
||||||
|
during the walk, folding away the separate stat pass and halving metadata I/O.
|
||||||
|
Each `PATH` operand commits in its own transaction, so an interrupted scan
|
||||||
|
keeps completed operands (2026-07-24, branch `parallel-walk`)
|
||||||
|
|
||||||
|
- Persistent scan database. `scan` maintains a SQLite database
|
||||||
|
(`modernc.org/sqlite`, pure Go, cgo stays disabled), keyed by absolute path,
|
||||||
|
that survives between runs. A rescan hashes only new or changed files, by
|
||||||
|
mtime and size. It deletes the records of files vanished from under the
|
||||||
|
scanned operands and leaves records outside them untouched, so `scan` can be
|
||||||
|
cronned daily. `report` and `trees` read the database, with no positional
|
||||||
|
arguments, instead of a scan stream. The database is at
|
||||||
|
`/var/lib/sfdupes/db.sqlite`, overridable via `SFDUPES_DATABASE`. WAL
|
||||||
journaling plus a single-transaction update keep a report run during a scan
|
journaling plus a single-transaction update keep a report run during a scan
|
||||||
safe
|
safe (2026-07-24, branch `persistent-database`)
|
||||||
- add the `origin` remote (`git@git.eeqj.de:sneak/sfdupes.git`), tag `v0.0.1`,
|
|
||||||
and push `main` plus tags (2026-07-23)
|
- The `origin` remote (`git@git.eeqj.de:sneak/sfdupes.git`) added, `v0.0.1`
|
||||||
- `scan` CLI rework (2026-07-23, branch `scan-required-paths`): required
|
tagged, and `main` plus tags pushed (2026-07-23)
|
||||||
`PATH...` operands via cobra flags replacing the `/srv` `-root` default; new
|
|
||||||
`-x`/`--one-file-system` flag (GNU convention) to stop at filesystem
|
- `scan` CLI rework. Required `PATH...` operands via cobra replace the `/srv`
|
||||||
boundaries, which are crossed by default
|
`-root` default. The new `-x`/`--one-file-system` flag (GNU convention) stops
|
||||||
- bring the repo into full policy compliance (2026-07-23, branch
|
at filesystem boundaries, which are crossed by default (2026-07-23, branch
|
||||||
`repo-policy-compliance`; checklist below)
|
`scan-required-paths`)
|
||||||
- `git init` with README-only first commit; code baseline committed on `main`
|
|
||||||
(2026-07-22)
|
- The repo brought into compliance with `REPO_POLICIES.md` (2026-07-06), the
|
||||||
- implement `scan`, `report`, and `trees` subcommands (pre-git history)
|
existing repo checklist and the Go styleguide, audited 2026-07-22 (2026-07-23,
|
||||||
|
branch `repo-policy-compliance`). Added or fixed:
|
||||||
|
- `.gitignore`, with OS, editor, Go and secrets patterns; the compiled
|
||||||
|
`sfdupes` binary and `files.dat` had been sitting untracked in the tree;
|
||||||
|
- `.editorconfig`;
|
||||||
|
- `LICENSE` (MIT) and the README License section;
|
||||||
|
- `REPO_POLICIES.md` in the repo root;
|
||||||
|
- the canonical `.golangci.yml`, with the 150 findings it raised fixed;
|
||||||
|
- the `Makefile` targets `test`, `lint`, `fmt`, `fmt-check`, `docker` and
|
||||||
|
`hooks`, and a `check` target that no longer depends on `build`;
|
||||||
|
- a test suite covering parsing, grouping, digests, suppression, hashing and
|
||||||
|
the scan pipeline, at 64% coverage;
|
||||||
|
- a multistage `Dockerfile` with hash-pinned images, and `.dockerignore`;
|
||||||
|
- `.gitea/workflows/check.yml`, running `docker build .` on push with the
|
||||||
|
checkout action pinned by commit SHA;
|
||||||
|
- the README sections the policy requires;
|
||||||
|
- the stale README non-goal "no git repository setup and no CI" removed;
|
||||||
|
- the pre-commit hook installed.
|
||||||
|
|
||||||
|
- `git init` with a README-only first commit; the code baseline committed on
|
||||||
|
`main` (2026-07-22)
|
||||||
|
|
||||||
|
- `scan`, `report` and `trees` subcommands implemented (pre-git history)
|
||||||
|
|
||||||
# Future Steps
|
# Future Steps
|
||||||
|
|
||||||
- possible later features (explicitly out of scope per README): full-content
|
- Possible later features, out of scope per README: full-content verification of
|
||||||
verification of candidates, removal-script helpers
|
candidates, and helper scripts for removal.
|
||||||
|
|
||||||
# Repo Policy Compliance
|
# Accepted Divergences
|
||||||
|
|
||||||
Audited 2026-07-22 against `REPO_POLICIES.md` (2026-07-06), the existing repo
|
- Flat single-package layout, with the `.go` files in the repo root. That is
|
||||||
checklist, and the Go styleguide. Code is already gofmt-clean, so no standalone
|
fine for a small single-binary tool per the Go styleguide, and the tracker
|
||||||
formatting commit is needed.
|
audit agrees.
|
||||||
|
|
||||||
- [x] `.gitignore` missing — the compiled `sfdupes` binary and `files.dat` sit
|
|
||||||
untracked in the tree; needs OS/editor/Go artifacts plus secrets patterns
|
|
||||||
- [x] `.editorconfig` missing
|
|
||||||
- [x] `LICENSE` missing and README has no License section (MIT assumed from
|
|
||||||
house convention — user to confirm)
|
|
||||||
- [x] `REPO_POLICIES.md` missing from repo root
|
|
||||||
- [x] `.golangci.yml` missing (install canonical copy); code must then pass
|
|
||||||
`make lint` (150 findings fixed; `make lint` is clean)
|
|
||||||
- [x] `Makefile` lacks required targets `test`, `lint`, `fmt`, `fmt-check`,
|
|
||||||
`docker`, `hooks`; `check` currently depends on `build`, which writes the
|
|
||||||
binary (`make check` must not modify files)
|
|
||||||
- [x] no tests — `go test ./...` has nothing to run; policy requires real tests
|
|
||||||
with a 30-second timeout and the conditional `-v` rerun pattern (suite
|
|
||||||
covers parsing, grouping, digests, suppression, hashing, and the scan
|
|
||||||
pipeline; 64% coverage)
|
|
||||||
- [x] `Dockerfile` missing — Go multistage with hash-pinned images: fail-fast
|
|
||||||
lint stage, build stage running `make check`
|
|
||||||
- [x] `.dockerignore` missing
|
|
||||||
- [x] `.gitea/workflows/check.yml` missing (`docker build .` on push, checkout
|
|
||||||
action pinned by commit SHA)
|
|
||||||
- [x] README lacks required sections: Description first line
|
|
||||||
(name/purpose/category/license/author), Getting Started, Rationale, TODO,
|
|
||||||
License, Author
|
|
||||||
- [x] README non-goal "no git repository setup and no CI" is stale now that the
|
|
||||||
repo is under git with CI
|
|
||||||
- [x] pre-commit hook not installed (`make hooks` once the target exists)
|
|
||||||
|
|
||||||
Accepted divergences (no action):
|
|
||||||
|
|
||||||
- flat single-package layout with `.go` files in the repo root — fine for a
|
|
||||||
small single-binary tool per the Go styleguide; the tracker audit agrees
|
|
||||||
|
|||||||
@@ -47,15 +47,14 @@ const dbFilePerm = 0o644
|
|||||||
const lockFilePerm = 0o600
|
const lockFilePerm = 0o600
|
||||||
|
|
||||||
// createTableSQL is the schema applied to a fresh database. Paths are
|
// createTableSQL is the schema applied to a fresh database. Paths are
|
||||||
// BLOBs because Unix paths are raw bytes, not guaranteed UTF-8.
|
// BLOBs because Unix paths are raw bytes, not guaranteed UTF-8. mtime
|
||||||
// disk_size holds the bytes the file takes on disk. mtime holds whole
|
// holds whole Unix seconds and mtime_nsec the nanoseconds within that
|
||||||
// Unix seconds and mtime_nsec the nanoseconds within that second. head,
|
// second. head, tail and content each hold the raw 32 bytes of a
|
||||||
// tail and content each hold the raw 32 bytes of a SHA-256.
|
// SHA-256.
|
||||||
const createTableSQL = `
|
const createTableSQL = `
|
||||||
CREATE TABLE files (
|
CREATE TABLE files (
|
||||||
path BLOB PRIMARY KEY,
|
path BLOB PRIMARY KEY,
|
||||||
size INTEGER NOT NULL,
|
size INTEGER NOT NULL,
|
||||||
disk_size INTEGER NOT NULL,
|
|
||||||
mtime INTEGER NOT NULL,
|
mtime INTEGER NOT NULL,
|
||||||
mtime_nsec INTEGER NOT NULL,
|
mtime_nsec INTEGER NOT NULL,
|
||||||
head BLOB NOT NULL,
|
head BLOB NOT NULL,
|
||||||
@@ -70,59 +69,14 @@ const createIndexSQL = `
|
|||||||
CREATE INDEX files_signature ON files (size, head, tail, content)
|
CREATE INDEX files_signature ON files (size, head, tail, content)
|
||||||
`
|
`
|
||||||
|
|
||||||
// createTotalsSQL creates the totals table, which holds one row: the
|
|
||||||
// number of records and the sizes of all records added up. Counting
|
|
||||||
// them over files would read every record. The three triggers below
|
|
||||||
// keep the row current, in the transaction of each change to files.
|
|
||||||
const createTotalsSQL = `
|
|
||||||
CREATE TABLE totals (
|
|
||||||
records INTEGER NOT NULL,
|
|
||||||
size INTEGER NOT NULL
|
|
||||||
)
|
|
||||||
`
|
|
||||||
|
|
||||||
// insertTotalsSQL adds the one row of totals, for an empty files table.
|
|
||||||
const insertTotalsSQL = `
|
|
||||||
INSERT INTO totals (records, size) VALUES (0, 0)
|
|
||||||
`
|
|
||||||
|
|
||||||
// totalsAfterInsertSQL counts each inserted record into totals.
|
|
||||||
const totalsAfterInsertSQL = `
|
|
||||||
CREATE TRIGGER totals_after_insert AFTER INSERT ON files BEGIN
|
|
||||||
UPDATE totals SET records = records + 1, size = size + NEW.size;
|
|
||||||
END
|
|
||||||
`
|
|
||||||
|
|
||||||
// totalsAfterDeleteSQL takes each deleted record out of totals.
|
|
||||||
const totalsAfterDeleteSQL = `
|
|
||||||
CREATE TRIGGER totals_after_delete AFTER DELETE ON files BEGIN
|
|
||||||
UPDATE totals SET records = records - 1, size = size - OLD.size;
|
|
||||||
END
|
|
||||||
`
|
|
||||||
|
|
||||||
// totalsAfterUpdateSQL replaces a record's old size in totals with its
|
|
||||||
// new one. scan's upsert of an existing record is such an update.
|
|
||||||
const totalsAfterUpdateSQL = `
|
|
||||||
CREATE TRIGGER totals_after_update AFTER UPDATE OF size ON files BEGIN
|
|
||||||
UPDATE totals SET size = size + NEW.size - OLD.size;
|
|
||||||
END
|
|
||||||
`
|
|
||||||
|
|
||||||
// recordCountSQL reads the number of records from totals.
|
|
||||||
const recordCountSQL = `
|
|
||||||
SELECT records FROM totals
|
|
||||||
`
|
|
||||||
|
|
||||||
// upsertSQL inserts one file record, replacing any existing record for
|
// upsertSQL inserts one file record, replacing any existing record for
|
||||||
// the same path.
|
// the same path.
|
||||||
const upsertSQL = `
|
const upsertSQL = `
|
||||||
INSERT INTO files (
|
INSERT INTO files (path, size, mtime, mtime_nsec, head, tail, content)
|
||||||
path, size, disk_size, mtime, mtime_nsec, head, tail, content
|
VALUES (?, ?, ?, ?, ?, ?, ?)
|
||||||
)
|
|
||||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?)
|
|
||||||
ON CONFLICT (path) DO UPDATE SET
|
ON CONFLICT (path) DO UPDATE SET
|
||||||
size = excluded.size, disk_size = excluded.disk_size,
|
size = excluded.size, mtime = excluded.mtime,
|
||||||
mtime = excluded.mtime, mtime_nsec = excluded.mtime_nsec,
|
mtime_nsec = excluded.mtime_nsec,
|
||||||
head = excluded.head, tail = excluded.tail,
|
head = excluded.head, tail = excluded.tail,
|
||||||
content = excluded.content
|
content = excluded.content
|
||||||
`
|
`
|
||||||
@@ -196,14 +150,9 @@ func createDatabaseDirectory(path string) error {
|
|||||||
// scanParams are the connection parameters for scan: read-write, with
|
// scanParams are the connection parameters for scan: read-write, with
|
||||||
// WAL journaling and a busy timeout, so a report can run while a cron
|
// WAL journaling and a busy timeout, so a report can run while a cron
|
||||||
// scan is in progress. closeScanDatabase leaves WAL mode again.
|
// scan is in progress. closeScanDatabase leaves WAL mode again.
|
||||||
//
|
|
||||||
// Each write to files fires a trigger, so SQLite keeps a statement
|
|
||||||
// journal for it. temp_store(MEMORY) keeps that journal in memory,
|
|
||||||
// which costs far less than a temporary file.
|
|
||||||
const scanParams = "_pragma=busy_timeout(10000)" +
|
const scanParams = "_pragma=busy_timeout(10000)" +
|
||||||
"&_pragma=journal_mode(WAL)" +
|
"&_pragma=journal_mode(WAL)" +
|
||||||
"&_pragma=synchronous(NORMAL)" +
|
"&_pragma=synchronous(NORMAL)"
|
||||||
"&_pragma=temp_store(MEMORY)"
|
|
||||||
|
|
||||||
// reportParams are the connection parameters for report, trees and
|
// reportParams are the connection parameters for report, trees and
|
||||||
// info: read-only, with the same busy timeout. They set no journal
|
// info: read-only, with the same busy timeout. They set no journal
|
||||||
@@ -477,14 +426,14 @@ func createSchema(ctx context.Context, db *sql.DB) error {
|
|||||||
|
|
||||||
defer func() { _ = tx.Rollback() }()
|
defer func() { _ = tx.Rollback() }()
|
||||||
|
|
||||||
for _, stmt := range []string{
|
_, err = tx.ExecContext(ctx, createTableSQL)
|
||||||
createTableSQL, createIndexSQL, createTotalsSQL, insertTotalsSQL,
|
if err != nil {
|
||||||
totalsAfterInsertSQL, totalsAfterDeleteSQL, totalsAfterUpdateSQL,
|
return fmt.Errorf("create schema: %w", err)
|
||||||
} {
|
}
|
||||||
_, err = tx.ExecContext(ctx, stmt)
|
|
||||||
if err != nil {
|
_, err = tx.ExecContext(ctx, createIndexSQL)
|
||||||
return fmt.Errorf("create schema: %w", err)
|
if err != nil {
|
||||||
}
|
return fmt.Errorf("create schema: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
_, err = tx.ExecContext(ctx,
|
_, err = tx.ExecContext(ctx,
|
||||||
@@ -531,7 +480,7 @@ func loadFileRows(ctx context.Context, db *sql.DB, total func(n int64),
|
|||||||
|
|
||||||
var n int64
|
var n int64
|
||||||
|
|
||||||
err = tx.QueryRowContext(ctx, recordCountSQL).Scan(&n)
|
err = tx.QueryRowContext(ctx, "SELECT COUNT(*) FROM files").Scan(&n)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("count records: %w", err)
|
return fmt.Errorf("count records: %w", err)
|
||||||
}
|
}
|
||||||
@@ -539,8 +488,8 @@ func loadFileRows(ctx context.Context, db *sql.DB, total func(n int64),
|
|||||||
total(n)
|
total(n)
|
||||||
|
|
||||||
rows, err := tx.QueryContext(ctx,
|
rows, err := tx.QueryContext(ctx,
|
||||||
"SELECT path, size, disk_size, mtime, mtime_nsec, head, tail, "+
|
"SELECT path, size, mtime, mtime_nsec, head, tail, content "+
|
||||||
"content FROM files ORDER BY path")
|
"FROM files ORDER BY path")
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("read records: %w", err)
|
return fmt.Errorf("read records: %w", err)
|
||||||
}
|
}
|
||||||
@@ -554,8 +503,7 @@ func loadFileRows(ctx context.Context, db *sql.DB, total func(n int64),
|
|||||||
r scanRec
|
r scanRec
|
||||||
)
|
)
|
||||||
|
|
||||||
err = rows.Scan(&path, &r.size, &r.diskSize, &sec, &nsec,
|
err = rows.Scan(&path, &r.size, &sec, &nsec, &head, &tail, &content)
|
||||||
&head, &tail, &content)
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("read record: %w", err)
|
return fmt.Errorf("read record: %w", err)
|
||||||
}
|
}
|
||||||
@@ -590,11 +538,10 @@ ORDER BY size DESC, head DESC, tail DESC, content DESC
|
|||||||
`
|
`
|
||||||
|
|
||||||
// loadSignatureRows gives total the number of records, then streams
|
// loadSignatureRows gives total the number of records, then streams
|
||||||
// every record's signature and path to fn, without its mtime or its
|
// every record's signature and path to fn, without its mtime, in the
|
||||||
// disk_size, in the order of signatureRowsSQL. The count and the
|
// order of signatureRowsSQL. The count and the records are read in one
|
||||||
// records are read in one transaction, so they agree while a scan is
|
// transaction, so they agree while a scan is committing. An error from
|
||||||
// committing. An error from fn stops the reading and is returned as it
|
// fn stops the reading and is returned as it is.
|
||||||
// is.
|
|
||||||
func loadSignatureRows(ctx context.Context, db *sql.DB,
|
func loadSignatureRows(ctx context.Context, db *sql.DB,
|
||||||
total func(n int64), fn func(r scanRec) error,
|
total func(n int64), fn func(r scanRec) error,
|
||||||
) error {
|
) error {
|
||||||
@@ -609,7 +556,7 @@ func loadSignatureRows(ctx context.Context, db *sql.DB,
|
|||||||
|
|
||||||
var n int64
|
var n int64
|
||||||
|
|
||||||
err = tx.QueryRowContext(ctx, recordCountSQL).Scan(&n)
|
err = tx.QueryRowContext(ctx, "SELECT COUNT(*) FROM files").Scan(&n)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("count records: %w", err)
|
return fmt.Errorf("count records: %w", err)
|
||||||
}
|
}
|
||||||
@@ -675,7 +622,7 @@ func (r *scanRec) setHashes(head, tail, content []byte) error {
|
|||||||
func countRecords(ctx context.Context, db *sql.DB) (int64, error) {
|
func countRecords(ctx context.Context, db *sql.DB) (int64, error) {
|
||||||
var n int64
|
var n int64
|
||||||
|
|
||||||
err := db.QueryRowContext(ctx, recordCountSQL).Scan(&n)
|
err := db.QueryRowContext(ctx, "SELECT COUNT(*) FROM files").Scan(&n)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return 0, fmt.Errorf("count records: %w", err)
|
return 0, fmt.Errorf("count records: %w", err)
|
||||||
}
|
}
|
||||||
@@ -683,10 +630,12 @@ func countRecords(ctx context.Context, db *sql.DB) (int64, error) {
|
|||||||
return n, nil
|
return n, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// databaseCountsSQL reads the number of records and the sizes of all
|
// databaseCountsSQL counts the records and adds up their sizes, in one
|
||||||
// records added up from totals.
|
// pass over the table. A SUM over no rows is NULL, so on an empty table
|
||||||
|
// the SUM falls back to 0.
|
||||||
const databaseCountsSQL = `
|
const databaseCountsSQL = `
|
||||||
SELECT records, size FROM totals
|
SELECT COUNT(*), COALESCE(SUM(size), 0)
|
||||||
|
FROM files
|
||||||
`
|
`
|
||||||
|
|
||||||
// databaseCounts is what info reads from the database: its schema
|
// databaseCounts is what info reads from the database: its schema
|
||||||
@@ -729,15 +678,15 @@ func readDatabaseCounts(ctx context.Context,
|
|||||||
return c, nil
|
return c, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// loadFileMeta streams every record's path, size, disk_size and mtime
|
// loadFileMeta streams every record's path, size and mtime to fn. Scan
|
||||||
// to fn. Scan change detection needs no hash values, and skipping the
|
// change detection needs no hash values, and skipping the hash columns
|
||||||
// hash columns keeps the scan's in-memory index small on
|
// keeps the scan's in-memory index small on multi-million-file
|
||||||
// multi-million-file databases.
|
// databases.
|
||||||
func loadFileMeta(ctx context.Context, db *sql.DB,
|
func loadFileMeta(ctx context.Context, db *sql.DB,
|
||||||
fn func(path string, m fileMeta),
|
fn func(path string, size int64, mtime time.Time),
|
||||||
) error {
|
) error {
|
||||||
rows, err := db.QueryContext(ctx,
|
rows, err := db.QueryContext(ctx,
|
||||||
"SELECT path, size, disk_size, mtime, mtime_nsec FROM files")
|
"SELECT path, size, mtime, mtime_nsec FROM files")
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("read records: %w", err)
|
return fmt.Errorf("read records: %w", err)
|
||||||
}
|
}
|
||||||
@@ -746,19 +695,16 @@ func loadFileMeta(ctx context.Context, db *sql.DB,
|
|||||||
|
|
||||||
for rows.Next() {
|
for rows.Next() {
|
||||||
var (
|
var (
|
||||||
path []byte
|
path []byte
|
||||||
m fileMeta
|
size, sec, nsec int64
|
||||||
sec, nsec int64
|
|
||||||
)
|
)
|
||||||
|
|
||||||
err = rows.Scan(&path, &m.size, &m.diskSize, &sec, &nsec)
|
err = rows.Scan(&path, &size, &sec, &nsec)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("read record: %w", err)
|
return fmt.Errorf("read record: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
m.mtime = time.Unix(sec, nsec)
|
fn(string(path), size, time.Unix(sec, nsec))
|
||||||
|
|
||||||
fn(string(path), m)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
err = rows.Err()
|
err = rows.Err()
|
||||||
@@ -776,21 +722,21 @@ func loadFileMeta(ctx context.Context, db *sql.DB,
|
|||||||
// WAL small and let concurrent reports observe progress.
|
// WAL small and let concurrent reports observe progress.
|
||||||
const updateBatchSize = 10000
|
const updateBatchSize = 10000
|
||||||
|
|
||||||
// applyChanges writes upserts for new and changed files and deletes for
|
// applyChanges writes one scan's database changes — upserts for new and
|
||||||
// vanished ones, in batched transactions. Progress is rendered on prog
|
// changed files, deletes for vanished ones — in batched transactions.
|
||||||
// (one increment per change).
|
// Progress is rendered on prog (one increment per change).
|
||||||
func applyChanges(ctx context.Context, db *sql.DB, upserts []scanRec,
|
func applyChanges(ctx context.Context, db *sql.DB, upserts []scanRec,
|
||||||
deletes []string, prog *progress,
|
deletes []string, prog *progress,
|
||||||
) error {
|
) error {
|
||||||
for batch := range slices.Chunk(upserts, updateBatchSize) {
|
for batch := range slices.Chunk(upserts, updateBatchSize) {
|
||||||
err := applyBatch(ctx, db, batch, nil, nil, prog)
|
err := applyBatch(ctx, db, batch, nil, prog)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
for batch := range slices.Chunk(deletes, updateBatchSize) {
|
for batch := range slices.Chunk(deletes, updateBatchSize) {
|
||||||
err := applyBatch(ctx, db, nil, nil, batch, prog)
|
err := applyBatch(ctx, db, nil, batch, prog)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
@@ -799,26 +745,10 @@ func applyChanges(ctx context.Context, db *sql.DB, upserts []scanRec,
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// applyDiskSizes writes the disk_size of each of recs into its record,
|
// applyBatch commits one batch of upserts and deletes in a single
|
||||||
// in batched transactions, and leaves the rest of the record alone.
|
// transaction.
|
||||||
// Progress is rendered on prog (one increment per record).
|
|
||||||
func applyDiskSizes(ctx context.Context, db *sql.DB, recs []fileRec,
|
|
||||||
prog *progress,
|
|
||||||
) error {
|
|
||||||
for batch := range slices.Chunk(recs, updateBatchSize) {
|
|
||||||
err := applyBatch(ctx, db, nil, batch, nil, prog)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// applyBatch commits one batch of upserts, disk_size updates and
|
|
||||||
// deletes in a single transaction.
|
|
||||||
func applyBatch(ctx context.Context, db *sql.DB, upserts []scanRec,
|
func applyBatch(ctx context.Context, db *sql.DB, upserts []scanRec,
|
||||||
diskSizes []fileRec, deletes []string, prog *progress,
|
deletes []string, prog *progress,
|
||||||
) error {
|
) error {
|
||||||
tx, err := db.BeginTx(ctx, nil)
|
tx, err := db.BeginTx(ctx, nil)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -832,11 +762,6 @@ func applyBatch(ctx context.Context, db *sql.DB, upserts []scanRec,
|
|||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
err = execDiskSizes(ctx, tx, diskSizes, prog)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
err = execDeletes(ctx, tx, deletes, prog)
|
err = execDeletes(ctx, tx, deletes, prog)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
@@ -862,7 +787,7 @@ func execUpserts(ctx context.Context, tx *sql.Tx, upserts []scanRec,
|
|||||||
defer func() { _ = st.Close() }()
|
defer func() { _ = st.Close() }()
|
||||||
|
|
||||||
for _, r := range upserts {
|
for _, r := range upserts {
|
||||||
_, err = st.ExecContext(ctx, []byte(r.path), r.size, r.diskSize,
|
_, err = st.ExecContext(ctx, []byte(r.path), r.size,
|
||||||
r.mtime.Unix(), r.mtime.Nanosecond(),
|
r.mtime.Unix(), r.mtime.Nanosecond(),
|
||||||
r.head[:], r.tail[:], r.content[:])
|
r.head[:], r.tail[:], r.content[:])
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -875,31 +800,6 @@ func execUpserts(ctx context.Context, tx *sql.Tx, upserts []scanRec,
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// execDiskSizes sets the disk_size of the record of each of recs. It
|
|
||||||
// does not touch size, so the totals triggers do not fire.
|
|
||||||
func execDiskSizes(ctx context.Context, tx *sql.Tx, recs []fileRec,
|
|
||||||
prog *progress,
|
|
||||||
) error {
|
|
||||||
st, err := tx.PrepareContext(ctx,
|
|
||||||
"UPDATE files SET disk_size = ? WHERE path = ?")
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("prepare disk_size update: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
defer func() { _ = st.Close() }()
|
|
||||||
|
|
||||||
for _, r := range recs {
|
|
||||||
_, err = st.ExecContext(ctx, r.diskSize, []byte(r.path))
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("update disk_size of %s: %w", r.path, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
prog.increment()
|
|
||||||
}
|
|
||||||
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// execDeletes removes the records for paths no longer present.
|
// execDeletes removes the records for paths no longer present.
|
||||||
func execDeletes(ctx context.Context, tx *sql.Tx, deletes []string,
|
func execDeletes(ctx context.Context, tx *sql.Tx, deletes []string,
|
||||||
prog *progress,
|
prog *progress,
|
||||||
|
|||||||
+10
-157
@@ -535,12 +535,17 @@ func TestLoadRowsRefuseWrongSizeHash(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// queryPlan returns the detail of each line of SQLite's EXPLAIN QUERY
|
// TestSignatureRowsNeedNoSort checks that SQLite reads signatureRowsSQL
|
||||||
// PLAN for query.
|
// from the files_signature index alone, with nothing to sort: the
|
||||||
func queryPlan(t *testing.T, db *sql.DB, query string) []string {
|
// first record comes at once, so report's read phase counts each
|
||||||
t.Helper()
|
// record as it comes rather than waiting on SQLite for all of them.
|
||||||
|
func TestSignatureRowsNeedNoSort(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
rows, err := db.QueryContext(t.Context(), "EXPLAIN QUERY PLAN "+query)
|
db := openTestDB(t)
|
||||||
|
|
||||||
|
rows, err := db.QueryContext(t.Context(),
|
||||||
|
"EXPLAIN QUERY PLAN "+signatureRowsSQL)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
@@ -568,164 +573,12 @@ func queryPlan(t *testing.T, db *sql.DB, query string) []string {
|
|||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
|
|
||||||
return plan
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestSignatureRowsNeedNoSort checks that SQLite reads signatureRowsSQL
|
|
||||||
// from the files_signature index alone, with nothing to sort: the
|
|
||||||
// first record comes at once, so report's read phase counts each
|
|
||||||
// record as it comes rather than waiting on SQLite for all of them.
|
|
||||||
func TestSignatureRowsNeedNoSort(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
db := openTestDB(t)
|
|
||||||
|
|
||||||
plan := queryPlan(t, db, signatureRowsSQL)
|
|
||||||
|
|
||||||
want := []string{"SCAN files USING COVERING INDEX files_signature"}
|
want := []string{"SCAN files USING COVERING INDEX files_signature"}
|
||||||
if !slices.Equal(plan, want) {
|
if !slices.Equal(plan, want) {
|
||||||
t.Errorf("query plan = %q, want %q", plan, want)
|
t.Errorf("query plan = %q, want %q", plan, want)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// TestCountsReadOnlyTotals checks that SQLite answers the queries for
|
|
||||||
// the number of records and their total size from the one row of
|
|
||||||
// totals. A plan that reads files or its index reads every record.
|
|
||||||
func TestCountsReadOnlyTotals(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
db := openTestDB(t)
|
|
||||||
|
|
||||||
for _, query := range []string{recordCountSQL, databaseCountsSQL} {
|
|
||||||
plan := queryPlan(t, db, query)
|
|
||||||
|
|
||||||
want := []string{"SCAN totals"}
|
|
||||||
if !slices.Equal(plan, want) {
|
|
||||||
t.Errorf("query plan of %q = %q, want %q", query, plan, want)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestCountsComeFromTotals sets the row of totals to numbers that files
|
|
||||||
// does not hold, then checks that scan's load phase, the read phases of
|
|
||||||
// trees and report, and info each report the numbers from totals.
|
|
||||||
func TestCountsComeFromTotals(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
// files is empty, so a count over files gives 0 records of 0 bytes.
|
|
||||||
const records, size = 7, 9000
|
|
||||||
|
|
||||||
db := openTestDB(t)
|
|
||||||
|
|
||||||
_, err := db.ExecContext(t.Context(),
|
|
||||||
"UPDATE totals SET records = ?, size = ?", records, size)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
|
|
||||||
scanTotal, err := countRecords(t.Context(), db)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
|
|
||||||
var treesTotal, reportTotal int64
|
|
||||||
|
|
||||||
err = loadFileRows(t.Context(), db,
|
|
||||||
func(n int64) { treesTotal = n }, func(scanRec) {})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
|
|
||||||
err = loadSignatureRows(t.Context(), db,
|
|
||||||
func(n int64) { reportTotal = n },
|
|
||||||
func(scanRec) error { return nil })
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
|
|
||||||
info, err := readDatabaseCounts(t.Context(), db)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
|
|
||||||
for name, got := range map[string]int64{
|
|
||||||
"scan's load": scanTotal,
|
|
||||||
"trees' read": treesTotal,
|
|
||||||
"report's read": reportTotal,
|
|
||||||
"info": info.records,
|
|
||||||
} {
|
|
||||||
if got != records {
|
|
||||||
t.Errorf("%s: %d records, want %d from totals",
|
|
||||||
name, got, records)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if info.size != size {
|
|
||||||
t.Errorf("info: size %d, want %d from totals", info.size, size)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// assertTotalsMatchFiles fails unless the row of totals holds the
|
|
||||||
// number of records and the sizes of all records added up, as counted
|
|
||||||
// over files.
|
|
||||||
func assertTotalsMatchFiles(t *testing.T, db *sql.DB) {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
got, err := readDatabaseCounts(t.Context(), db)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
|
|
||||||
var records, size int64
|
|
||||||
|
|
||||||
err = db.QueryRowContext(t.Context(),
|
|
||||||
"SELECT COUNT(*), COALESCE(SUM(size), 0) FROM files").
|
|
||||||
Scan(&records, &size)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
|
|
||||||
if got.records != records || got.size != size {
|
|
||||||
t.Errorf("totals = %d records of %d bytes, files holds %d of %d",
|
|
||||||
got.records, got.size, records, size)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestTotalsMatchFiles scans the smoke tree, rescans it after two files
|
|
||||||
// change size, and rescans it after two files are removed. After each
|
|
||||||
// scan the row of totals must agree with files.
|
|
||||||
func TestTotalsMatchFiles(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
dir := buildSmokeTree(t)
|
|
||||||
db := openTestDB(t)
|
|
||||||
|
|
||||||
syncTree(t, db, dir)
|
|
||||||
assertTotalsMatchFiles(t, db)
|
|
||||||
|
|
||||||
writeFile(t, dir, "a/one.bin", pattern(10, 2500))
|
|
||||||
writeFile(t, dir, "t1/f1", pattern(30, 1000))
|
|
||||||
|
|
||||||
if st := syncTree(t, db, dir); st.updated != 2 {
|
|
||||||
t.Fatalf("rescan stats = %+v, want 2 updated", st)
|
|
||||||
}
|
|
||||||
|
|
||||||
assertTotalsMatchFiles(t, db)
|
|
||||||
|
|
||||||
for _, name := range []string{"a/unique.bin", "t2/sub/f2"} {
|
|
||||||
err := os.Remove(filepath.Join(dir, name))
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if st := syncTree(t, db, dir); st.removed != 2 {
|
|
||||||
t.Fatalf("rescan stats = %+v, want 2 removed", st)
|
|
||||||
}
|
|
||||||
|
|
||||||
assertTotalsMatchFiles(t, db)
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestApplyChangesRoundTrip(t *testing.T) {
|
func TestApplyChangesRoundTrip(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
|
|||||||
@@ -26,11 +26,11 @@ func infoPhase(name string) string {
|
|||||||
// it. Any database problem, including a missing database, is fatal.
|
// it. Any database problem, including a missing database, is fatal.
|
||||||
//
|
//
|
||||||
// The work runs in two phases, each under a progress display of its
|
// The work runs in two phases, each under a progress display of its
|
||||||
// own on stderr: open (open the database) and count (read the number of
|
// own on stderr: open (open the database) and count (count the records,
|
||||||
// records and their total size from the totals table). Each display
|
// with one query). Each display ends before the next phase starts, and
|
||||||
// ends before the next phase starts, and before an error is returned
|
// before an error is returned to be printed. The details are written
|
||||||
// to be printed. The details are written once the last display has
|
// once the last display has ended, so no display is drawn while they
|
||||||
// ended, so no display is drawn while they are written.
|
// are written.
|
||||||
func runInfo(ctx context.Context, stdout io.Writer) error {
|
func runInfo(ctx context.Context, stdout io.Writer) error {
|
||||||
dbPath, source, err := databasePath()
|
dbPath, source, err := databasePath()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -56,7 +56,7 @@ func runInfo(ctx context.Context, stdout io.Writer) error {
|
|||||||
|
|
||||||
defer func() { _ = db.Close() }()
|
defer func() { _ = db.Close() }()
|
||||||
|
|
||||||
// One row holds both counts, so the phase never has a total.
|
// One query counts everything, so the phase never has a total.
|
||||||
prog = newProgress(progressParams{
|
prog = newProgress(progressParams{
|
||||||
label: infoPhase("count"), total: -1,
|
label: infoPhase("count"), total: -1,
|
||||||
waitingFor: countingRecords,
|
waitingFor: countingRecords,
|
||||||
|
|||||||
+22
-157
@@ -180,9 +180,9 @@ func capture(t *testing.T, std **os.File) func() string {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// brokenDatabase writes a database that opens cleanly and passes the
|
// brokenDatabase writes a database that opens cleanly and passes the
|
||||||
// schema-version check but has no tables, so the first query fails
|
// schema-version check but has no files table, so the first query
|
||||||
// with the database already open: a fatal error on a path that owns an
|
// fails with the database already open: a fatal error on a path that
|
||||||
// open database. It closes the database the way scan does.
|
// owns an open database. It closes the database the way scan does.
|
||||||
func brokenDatabase(t *testing.T) string {
|
func brokenDatabase(t *testing.T) string {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
|
|
||||||
@@ -265,9 +265,8 @@ func TestRunFatalAfterOpenClosesDatabase(t *testing.T) {
|
|||||||
assertFatalOutput(t, stderr.String(), stdout())
|
assertFatalOutput(t, stderr.String(), stdout())
|
||||||
|
|
||||||
// Proof that the failure happened after the open: only a
|
// Proof that the failure happened after the open: only a
|
||||||
// query against the opened database can report this. Each
|
// query against the opened database can report this.
|
||||||
// subcommand's first query reads totals.
|
if !strings.Contains(stderr.String(), "no such table: files") {
|
||||||
if !strings.Contains(stderr.String(), "no such table: totals") {
|
|
||||||
t.Errorf("stderr = %q, want the failure to come from a "+
|
t.Errorf("stderr = %q, want the failure to come from a "+
|
||||||
"query on the open database", stderr.String())
|
"query on the open database", stderr.String())
|
||||||
}
|
}
|
||||||
@@ -818,23 +817,6 @@ func scanFixture(t *testing.T) []string {
|
|||||||
return dupes
|
return dupes
|
||||||
}
|
}
|
||||||
|
|
||||||
// storedDiskSize returns the disk_size of the record for path in the
|
|
||||||
// database at dbPath. A test of a trees run compares with this rather
|
|
||||||
// than with a new lstat, which on ZFS can differ from the scan's (see
|
|
||||||
// diskSizeOf).
|
|
||||||
func storedDiskSize(t *testing.T, dbPath, path string) int64 {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
db, err := openReportDatabase(t.Context(), dbPath)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
|
|
||||||
defer func() { _ = db.Close() }()
|
|
||||||
|
|
||||||
return recordByPath(t, dbRecords(t, db), path).diskSize
|
|
||||||
}
|
|
||||||
|
|
||||||
// scanOK runs scan over operands, fails the test unless it exits 0 with
|
// scanOK runs scan over operands, fails the test unless it exits 0 with
|
||||||
// nothing on stdout, and returns everything it printed to stderr.
|
// nothing on stdout, and returns everything it printed to stderr.
|
||||||
func scanOK(t *testing.T, operands ...string) string {
|
func scanOK(t *testing.T, operands ...string) string {
|
||||||
@@ -946,21 +928,24 @@ func TestRunScanWalksOperandUnderSymlinkOperand(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestRunScanSkipsZFSOperand(t *testing.T) {
|
func TestRunScanSkipsZFSOperand(t *testing.T) {
|
||||||
zfs := filepath.Join(t.TempDir(), ".zfs")
|
path := testDBPath(t)
|
||||||
f := writeFile(t, zfs, "snapshot/hourly/f", pattern(1, 10))
|
|
||||||
|
|
||||||
// No scan records a file inside a .zfs directory, so the record
|
|
||||||
// beneath the operand is written straight into the database.
|
|
||||||
path := seedDatabase(t, []scanRec{{size: 10, path: f}})
|
|
||||||
t.Setenv(databaseEnv, path)
|
t.Setenv(databaseEnv, path)
|
||||||
|
|
||||||
|
zfs := filepath.Join(t.TempDir(), ".zfs")
|
||||||
|
snapshot := filepath.Join(zfs, "snapshot", "hourly")
|
||||||
|
f := writeFile(t, snapshot, "f", pattern(1, 10))
|
||||||
|
|
||||||
|
// An operand beneath a .zfs directory is walked, because it is not
|
||||||
|
// itself named .zfs.
|
||||||
|
scanOK(t, snapshot)
|
||||||
|
|
||||||
assertOperandSkipped(t, path, zfs, ".zfs directory", f)
|
assertOperandSkipped(t, path, zfs, ".zfs directory", f)
|
||||||
}
|
}
|
||||||
|
|
||||||
// assertOperandSkipped scans operand alone and checks that it is skipped
|
// assertOperandSkipped scans operand alone and checks that it is skipped
|
||||||
// as kind: a warning naming it, one skip in the summary, exit 0, and the
|
// as kind: a warning naming it, one skip in the summary, exit 0, and the
|
||||||
// record for kept, stored beneath operand before this scan, still in the
|
// record for kept, which an earlier scan stored beneath operand, still
|
||||||
// database at dbPath.
|
// in the database at dbPath.
|
||||||
func assertOperandSkipped(t *testing.T, dbPath, operand, kind,
|
func assertOperandSkipped(t *testing.T, dbPath, operand, kind,
|
||||||
kept string,
|
kept string,
|
||||||
) {
|
) {
|
||||||
@@ -989,121 +974,6 @@ func assertOperandSkipped(t *testing.T, dbPath, operand, kind,
|
|||||||
recordByPath(t, dbRecords(t, db), kept)
|
recordByPath(t, dbRecords(t, db), kept)
|
||||||
}
|
}
|
||||||
|
|
||||||
// TestRunScanSkipsOperandInsideZFS scans a directory inside a .zfs
|
|
||||||
// directory, first alone and then under another operand. Each time it
|
|
||||||
// is warned about and counted as skipped, and nothing in it is walked.
|
|
||||||
// The other operand is scanned as usual.
|
|
||||||
func TestRunScanSkipsOperandInsideZFS(t *testing.T) {
|
|
||||||
path := testDBPath(t)
|
|
||||||
t.Setenv(databaseEnv, path)
|
|
||||||
|
|
||||||
dir := t.TempDir()
|
|
||||||
hourly := filepath.Join(dir, ".zfs", "snapshot", "hourly")
|
|
||||||
writeFile(t, hourly, "f", pattern(1, 10))
|
|
||||||
f := writeFile(t, dir, "f", pattern(2, 10))
|
|
||||||
|
|
||||||
warning := "walk " + hourly + ": skipping operand inside a .zfs directory\n"
|
|
||||||
|
|
||||||
for _, tc := range []struct {
|
|
||||||
operands []string
|
|
||||||
summary string
|
|
||||||
}{
|
|
||||||
{
|
|
||||||
[]string{hourly},
|
|
||||||
"scan: 0 files seen (0 added, 0 updated, 0 removed, " +
|
|
||||||
"0 unchanged), 1 skipped\n",
|
|
||||||
},
|
|
||||||
{
|
|
||||||
[]string{dir, hourly},
|
|
||||||
"scan: 1 files seen (1 added, 0 updated, 0 removed, " +
|
|
||||||
"0 unchanged), 1 skipped\n",
|
|
||||||
},
|
|
||||||
} {
|
|
||||||
stderr := scanOK(t, tc.operands...)
|
|
||||||
if !strings.Contains(stderr, warning) ||
|
|
||||||
!strings.Contains(stderr, tc.summary) {
|
|
||||||
t.Errorf("scan %q: stderr = %q, want %q and %q", tc.operands,
|
|
||||||
stderr, warning, tc.summary)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
db, err := openDB(path, reportParams)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
|
|
||||||
t.Cleanup(func() { _ = db.Close() })
|
|
||||||
|
|
||||||
if got := recordPaths(dbRecords(t, db)); !slices.Equal(got, []string{f}) {
|
|
||||||
t.Errorf("records = %q, want %q", got, []string{f})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestRunScanSkipsOperandThroughSymlinkIntoZFS scans snaps/daily, where
|
|
||||||
// snaps is a symlink to pool/.zfs/snapshot. The operand is warned about
|
|
||||||
// by the path given, counted as skipped, and not walked.
|
|
||||||
func TestRunScanSkipsOperandThroughSymlinkIntoZFS(t *testing.T) {
|
|
||||||
t.Setenv(databaseEnv, testDBPath(t))
|
|
||||||
|
|
||||||
dir := t.TempDir()
|
|
||||||
writeFile(t, dir, "pool/.zfs/snapshot/daily/f", pattern(1, 10))
|
|
||||||
|
|
||||||
snaps := filepath.Join(dir, "snaps")
|
|
||||||
|
|
||||||
err := os.Symlink(filepath.Join(dir, "pool", ".zfs", "snapshot"), snaps)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
|
|
||||||
daily := filepath.Join(snaps, "daily")
|
|
||||||
stderr := scanOK(t, daily)
|
|
||||||
|
|
||||||
warning := "walk " + daily + ": skipping operand inside a .zfs directory\n"
|
|
||||||
summary := "scan: 0 files seen (0 added, 0 updated, 0 removed, " +
|
|
||||||
"0 unchanged), 1 skipped\n"
|
|
||||||
|
|
||||||
if !strings.Contains(stderr, warning) ||
|
|
||||||
!strings.Contains(stderr, summary) {
|
|
||||||
t.Errorf("stderr = %q, want %q and %q", stderr, warning, summary)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestRunScanWalksOperandThroughSymlink scans data/daily, where data is
|
|
||||||
// a symlink to a directory outside any .zfs directory. The operand is
|
|
||||||
// scanned as usual, and its file is recorded by the path given.
|
|
||||||
func TestRunScanWalksOperandThroughSymlink(t *testing.T) {
|
|
||||||
path := testDBPath(t)
|
|
||||||
t.Setenv(databaseEnv, path)
|
|
||||||
|
|
||||||
dir := t.TempDir()
|
|
||||||
writeFile(t, dir, "pool/data/daily/f", pattern(1, 10))
|
|
||||||
|
|
||||||
data := filepath.Join(dir, "data")
|
|
||||||
|
|
||||||
err := os.Symlink(filepath.Join(dir, "pool", "data"), data)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
|
|
||||||
daily := filepath.Join(data, "daily")
|
|
||||||
stderr := scanOK(t, daily)
|
|
||||||
|
|
||||||
summary := "scan: 1 files seen (1 added, 0 updated, 0 removed, " +
|
|
||||||
"0 unchanged), 0 skipped\n"
|
|
||||||
if !strings.Contains(stderr, summary) {
|
|
||||||
t.Errorf("stderr = %q, want %q", stderr, summary)
|
|
||||||
}
|
|
||||||
|
|
||||||
db, err := openDB(path, reportParams)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
|
|
||||||
t.Cleanup(func() { _ = db.Close() })
|
|
||||||
|
|
||||||
recordByPath(t, dbRecords(t, db), filepath.Join(daily, "f"))
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestRunReportSucceeds(t *testing.T) {
|
func TestRunReportSucceeds(t *testing.T) {
|
||||||
path := testDBPath(t)
|
path := testDBPath(t)
|
||||||
t.Setenv(databaseEnv, path)
|
t.Setenv(databaseEnv, path)
|
||||||
@@ -1158,9 +1028,8 @@ func TestRunTreesSucceeds(t *testing.T) {
|
|||||||
|
|
||||||
// The two directories holding the duplicate pair are duplicate
|
// The two directories holding the duplicate pair are duplicate
|
||||||
// trees of each other.
|
// trees of each other.
|
||||||
want := "first\tdupe\tfiles\tsize\tdisk_size\n" +
|
want := "first\tdupe\tfiles\tsize\n" +
|
||||||
filepath.Dir(dupes[0]) + "\t" + filepath.Dir(dupes[1]) + "\t1\t300\t" +
|
filepath.Dir(dupes[0]) + "\t" + filepath.Dir(dupes[1]) + "\t1\t300\n"
|
||||||
strconv.FormatInt(storedDiskSize(t, path, dupes[1]), 10) + "\n"
|
|
||||||
if got := stdout.String(); got != want {
|
if got := stdout.String(); got != want {
|
||||||
t.Errorf("stdout = %q, want %q", got, want)
|
t.Errorf("stdout = %q, want %q", got, want)
|
||||||
}
|
}
|
||||||
@@ -1194,10 +1063,9 @@ func TestRunReportsNeedOnlyReadAccess(t *testing.T) {
|
|||||||
cases := map[string]string{
|
cases := map[string]string{
|
||||||
cmdReport: "first\tdupe\tsize\n" +
|
cmdReport: "first\tdupe\tsize\n" +
|
||||||
dupes[0] + "\t" + dupes[1] + "\t300\n",
|
dupes[0] + "\t" + dupes[1] + "\t300\n",
|
||||||
cmdTrees: "first\tdupe\tfiles\tsize\tdisk_size\n" +
|
cmdTrees: "first\tdupe\tfiles\tsize\n" +
|
||||||
filepath.Dir(dupes[0]) + "\t" + filepath.Dir(dupes[1]) +
|
filepath.Dir(dupes[0]) + "\t" + filepath.Dir(dupes[1]) +
|
||||||
"\t1\t300\t" +
|
"\t1\t300\n",
|
||||||
strconv.FormatInt(storedDiskSize(t, path, dupes[1]), 10) + "\n",
|
|
||||||
}
|
}
|
||||||
|
|
||||||
for name, want := range cases {
|
for name, want := range cases {
|
||||||
@@ -1232,11 +1100,8 @@ func assertRunsUseDatabase(t *testing.T, dir, name string) {
|
|||||||
t.Errorf("report stdout = %q, want %q", got, want)
|
t.Errorf("report stdout = %q, want %q", got, want)
|
||||||
}
|
}
|
||||||
|
|
||||||
diskSize := storedDiskSize(t, filepath.Join(dir, name), dupes[1])
|
want = "first\tdupe\tfiles\tsize\n" +
|
||||||
|
filepath.Dir(dupes[0]) + "\t" + filepath.Dir(dupes[1]) + "\t1\t300\n"
|
||||||
want = "first\tdupe\tfiles\tsize\tdisk_size\n" +
|
|
||||||
filepath.Dir(dupes[0]) + "\t" + filepath.Dir(dupes[1]) + "\t1\t300\t" +
|
|
||||||
strconv.FormatInt(diskSize, 10) + "\n"
|
|
||||||
if got := runStdout(t, cmdTrees); got != want {
|
if got := runStdout(t, cmdTrees); got != want {
|
||||||
t.Errorf("trees stdout = %q, want %q", got, want)
|
t.Errorf("trees stdout = %q, want %q", got, want)
|
||||||
}
|
}
|
||||||
|
|||||||
+1
-1
@@ -60,7 +60,7 @@ const fallbackWidth = 80
|
|||||||
const spinnerMargin = 5
|
const spinnerMargin = 5
|
||||||
|
|
||||||
// countingRecords is what the display of a phase that reads the records
|
// countingRecords is what the display of a phase that reads the records
|
||||||
// says until it has the number of records, its total.
|
// says while it counts them, before it has its total.
|
||||||
const countingRecords = "counting the records"
|
const countingRecords = "counting the records"
|
||||||
|
|
||||||
// openPhase is the name of the phase in which report, trees and info
|
// openPhase is the name of the phase in which report, trees and info
|
||||||
|
|||||||
@@ -21,16 +21,14 @@ const minGroupSize = 2
|
|||||||
|
|
||||||
// scanRec is one file record from the database. The signature (size,
|
// scanRec is one file record from the database. The signature (size,
|
||||||
// head, tail, content) is the duplicate key; mtime is informational
|
// head, tail, content) is the duplicate key; mtime is informational
|
||||||
// only and used by scan for change detection. diskSize is the bytes the
|
// only and used by scan for change detection.
|
||||||
// file takes on disk, which trees adds up.
|
|
||||||
type scanRec struct {
|
type scanRec struct {
|
||||||
size int64
|
size int64
|
||||||
diskSize int64
|
mtime time.Time
|
||||||
mtime time.Time
|
head [sha256.Size]byte
|
||||||
head [sha256.Size]byte
|
tail [sha256.Size]byte
|
||||||
tail [sha256.Size]byte
|
content [sha256.Size]byte
|
||||||
content [sha256.Size]byte
|
path string
|
||||||
path string
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// reportPhase names one phase of report with its place among all of
|
// reportPhase names one phase of report with its place among all of
|
||||||
@@ -47,11 +45,11 @@ func reportPhase(name string) string {
|
|||||||
// problem, including a missing database, is fatal.
|
// problem, including a missing database, is fatal.
|
||||||
//
|
//
|
||||||
// The work runs in two phases, each under a progress display of its
|
// The work runs in two phases, each under a progress display of its
|
||||||
// own on stderr: open (open the database) and read (read the number of
|
// own on stderr: open (open the database) and read (count the records,
|
||||||
// records from the totals table, then read the records in signature
|
// then read them in signature order, writing each size's duplicate
|
||||||
// order, writing each size's duplicate groups to stdout once that size
|
// groups to stdout once that size is read; see readReport). Each
|
||||||
// is read; see readReport). Each display ends before the next phase
|
// display ends before the next phase starts, and before an error is
|
||||||
// starts, and before an error is returned to be printed.
|
// returned to be printed.
|
||||||
func runReport(ctx context.Context, stdout io.Writer) error {
|
func runReport(ctx context.Context, stdout io.Writer) error {
|
||||||
dbPath, source, err := databasePath()
|
dbPath, source, err := databasePath()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
|||||||
+2
-2
@@ -38,8 +38,8 @@ func awkwardPairRecs() []scanRec {
|
|||||||
h, tl, c := testHash("h"), testHash("t"), testHash("c")
|
h, tl, c := testHash("h"), testHash("t"), testHash("c")
|
||||||
|
|
||||||
return []scanRec{
|
return []scanRec{
|
||||||
{size: 5, diskSize: 4096, head: h, tail: tl, content: c, path: "/d/A/f"},
|
{size: 5, head: h, tail: tl, content: c, path: "/d/A/f"},
|
||||||
{size: 5, diskSize: 4096, head: h, tail: tl, content: c, path: awkwardDir + "/f"},
|
{size: 5, head: h, tail: tl, content: c, path: awkwardDir + "/f"},
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -52,26 +52,23 @@ const workQueueDepth = 1024
|
|||||||
// has already printed its line, so run prints nothing more.
|
// has already printed its line, so run prints nothing more.
|
||||||
var errInterrupted = errors.New("scan interrupted")
|
var errInterrupted = errors.New("scan interrupted")
|
||||||
|
|
||||||
// fileRec carries one statted file between the scan phases. diskSize is
|
// fileRec carries one statted file between the scan phases. dev and
|
||||||
// the bytes the file takes on disk. dev and ino identify the underlying
|
// ino identify the underlying inode so hard-linked paths can share
|
||||||
// inode so hard-linked paths can share one read; both are zero when the
|
// one read; both are zero when the platform exposes no inode.
|
||||||
// platform exposes no inode.
|
|
||||||
type fileRec struct {
|
type fileRec struct {
|
||||||
path string
|
path string
|
||||||
size int64
|
size int64
|
||||||
diskSize int64
|
mtime time.Time
|
||||||
mtime time.Time
|
dev uint64
|
||||||
dev uint64
|
ino uint64
|
||||||
ino uint64
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// fileMeta is the in-memory index entry for one existing database
|
// fileMeta is the in-memory index entry for one existing database
|
||||||
// record: what the walk compares the file with. Hashes stay on disk; at
|
// record: just enough for change detection. Hashes stay on disk; at
|
||||||
// tens of millions of records they would dominate the scan's memory.
|
// tens of millions of records they would dominate the scan's memory.
|
||||||
type fileMeta struct {
|
type fileMeta struct {
|
||||||
size int64
|
size int64
|
||||||
diskSize int64
|
mtime time.Time
|
||||||
mtime time.Time
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// scanUnit is what every scan phase counts, as its progress display
|
// scanUnit is what every scan phase counts, as its progress display
|
||||||
@@ -342,12 +339,11 @@ type scanStats struct {
|
|||||||
// whatever remains after the walk and hash phases is deleted by the
|
// whatever remains after the walk and hash phases is deleted by the
|
||||||
// update phase.
|
// update phase.
|
||||||
type scanState struct {
|
type scanState struct {
|
||||||
db *sql.DB
|
db *sql.DB
|
||||||
existing map[string]fileMeta
|
existing map[string]fileMeta
|
||||||
toHash []fileRec // new and changed files: must be read
|
toHash []fileRec // new and changed files: must be read
|
||||||
diskSizes []fileRec // unchanged files whose disk_size changed
|
batch []scanRec
|
||||||
batch []scanRec
|
st scanStats
|
||||||
st scanStats
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// syncScan synchronizes the database with the filesystem under roots;
|
// syncScan synchronizes the database with the filesystem under roots;
|
||||||
@@ -387,8 +383,7 @@ func (s *scanState) runPhases(ctx context.Context, roots []string,
|
|||||||
) error {
|
) error {
|
||||||
// Types and filesystems are checked before pruning so that an
|
// Types and filesystems are checked before pruning so that an
|
||||||
// operand under a dropped one is still scanned, not dropped as lying
|
// operand under a dropped one is still scanned, not dropped as lying
|
||||||
// under it, and so that an operand inside a .zfs directory gets its
|
// under it.
|
||||||
// warning even when it lies under another operand.
|
|
||||||
roots = pruneRoots(s.walkableRoots(roots, rules), oneFS, rules,
|
roots = pruneRoots(s.walkableRoots(roots, rules), oneFS, rules,
|
||||||
deviceOfPath)
|
deviceOfPath)
|
||||||
|
|
||||||
@@ -416,15 +411,15 @@ func (s *scanState) runPhases(ctx context.Context, roots []string,
|
|||||||
}
|
}
|
||||||
|
|
||||||
// walkableRoots returns the operands the walk can start from: regular
|
// walkableRoots returns the operands the walk can start from: regular
|
||||||
// files, and directories not named .zfs, none of them inside a
|
// files, and directories not named .zfs. Every other operand is warned
|
||||||
// directory named .zfs. Every other operand is warned about, counted as
|
// about, counted as skipped, and dropped. A dropped operand is no
|
||||||
// skipped, and dropped. A dropped operand is no longer a root, so the
|
// longer a root, so the records stored beneath it are treated as if it
|
||||||
// records stored beneath it are treated as if it had not been given:
|
// had not been given: those under no other root count as outside the
|
||||||
// those under no other root count as outside the roots and are not
|
// roots and are not deleted as unverified. An operand on a filesystem
|
||||||
// deleted as unverified. An operand on a filesystem in rules that is not
|
// in rules that is not persistent is dropped the same way, with a line
|
||||||
// persistent is dropped the same way, with a line saying so that is not
|
// saying so that is not a warning, and is not counted as skipped. An
|
||||||
// a warning, and is not counted as skipped. An operand that fails lstat
|
// operand that fails lstat here is kept, and the walk warns about it
|
||||||
// here is kept, and the walk warns about it unless it no longer exists.
|
// unless it no longer exists.
|
||||||
func (s *scanState) walkableRoots(roots []string, rules walkRules) []string {
|
func (s *scanState) walkableRoots(roots []string, rules walkRules) []string {
|
||||||
kept := make([]string, 0, len(roots))
|
kept := make([]string, 0, len(roots))
|
||||||
|
|
||||||
@@ -459,8 +454,8 @@ func (s *scanState) walkableRoots(roots []string, rules walkRules) []string {
|
|||||||
// included, so its display counts against the number of records.
|
// included, so its display counts against the number of records.
|
||||||
func (s *scanState) loadIndex(ctx context.Context, roots []string) error {
|
func (s *scanState) loadIndex(ctx context.Context, roots []string) error {
|
||||||
// Indexing tens of millions of records takes real time; without a
|
// Indexing tens of millions of records takes real time; without a
|
||||||
// display the scan looks hung before the walk begins. The number of
|
// display the scan looks hung before the walk begins. Counting them
|
||||||
// records, read first, gives the display its total.
|
// first gives the display its total.
|
||||||
prog := newProgress(progressParams{
|
prog := newProgress(progressParams{
|
||||||
label: scanPhase("load"), unit: scanUnit, total: -1,
|
label: scanPhase("load"), unit: scanUnit, total: -1,
|
||||||
waitingFor: countingRecords, stalledOn: "db",
|
waitingFor: countingRecords, stalledOn: "db",
|
||||||
@@ -476,24 +471,24 @@ func (s *scanState) loadIndex(ctx context.Context, roots []string) error {
|
|||||||
|
|
||||||
s.existing = make(map[string]fileMeta)
|
s.existing = make(map[string]fileMeta)
|
||||||
|
|
||||||
return loadFileMeta(ctx, s.db, func(path string, m fileMeta) {
|
return loadFileMeta(ctx, s.db,
|
||||||
prog.increment()
|
func(path string, size int64, mtime time.Time) {
|
||||||
|
prog.increment()
|
||||||
|
|
||||||
if underAnyRoot(path, roots) {
|
if underAnyRoot(path, roots) {
|
||||||
s.existing[path] = m
|
s.existing[path] = fileMeta{size: size, mtime: mtime}
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
// walkPhase drains the walk and queues every new or changed file for
|
// walkPhase drains the walk and queues every new or changed file for
|
||||||
// the hash phase. An unchanged file is not read, and its record counts
|
// the hash phase. An unchanged file needs nothing more, and its record
|
||||||
// as verified. When its disk_size differs from the record's, it is
|
// counts as verified. A record at or below a path the walk left out,
|
||||||
// queued for the update phase, which writes the new disk_size. A record
|
// because -x stopped at it, it is a default exclude, or it is not on a
|
||||||
// at or below a path the walk left out, because -x stopped at it, it is
|
// persistent filesystem, counts as outside the roots: it is not
|
||||||
// a default exclude, or it is not on a persistent filesystem, counts as
|
// deleted. That does not hold below a root at or inside that path,
|
||||||
// outside the roots: it is not deleted. That does not hold below a root
|
// since that root's own walk verifies the records below it; nor is a
|
||||||
// at or inside that path, since that root's own walk verifies the
|
// line printed for such a root.
|
||||||
// records below it; nor is a line printed for such a root.
|
|
||||||
func (s *scanState) walkPhase(events <-chan walkEvent, roots []string) {
|
func (s *scanState) walkPhase(events <-chan walkEvent, roots []string) {
|
||||||
// The records the last scan left under the roots estimate how many
|
// The records the last scan left under the roots estimate how many
|
||||||
// files the walk will find. A first scan has none, and no estimate.
|
// files the walk will find. A first scan has none, and no estimate.
|
||||||
@@ -551,13 +546,6 @@ func (s *scanState) walkPhase(events <-chan walkEvent, roots []string) {
|
|||||||
delete(s.existing, ev.rec.path)
|
delete(s.existing, ev.rec.path)
|
||||||
|
|
||||||
s.st.unchanged++
|
s.st.unchanged++
|
||||||
|
|
||||||
// The space a file takes on disk can change while its size and
|
|
||||||
// mtime stay the same. ZFS, for one, counts a new file's blocks
|
|
||||||
// only seconds after the write.
|
|
||||||
if old.diskSize != ev.rec.diskSize {
|
|
||||||
s.diskSizes = append(s.diskSizes, ev.rec)
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
prog.finish()
|
prog.finish()
|
||||||
@@ -706,13 +694,12 @@ func (s *scanState) recordRun(ctx context.Context, r hashResult) error {
|
|||||||
s.resolve(rec.path)
|
s.resolve(rec.path)
|
||||||
|
|
||||||
s.batch = append(s.batch, scanRec{
|
s.batch = append(s.batch, scanRec{
|
||||||
size: rec.size,
|
size: rec.size,
|
||||||
diskSize: rec.diskSize,
|
mtime: rec.mtime,
|
||||||
mtime: rec.mtime,
|
head: r.head,
|
||||||
head: r.head,
|
tail: r.tail,
|
||||||
tail: r.tail,
|
content: r.content,
|
||||||
content: r.content,
|
path: rec.path,
|
||||||
path: rec.path,
|
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -728,7 +715,7 @@ func (s *scanState) commitFullBatch(ctx context.Context) error {
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
err := applyBatch(ctx, s.db, s.batch, nil, nil, nil)
|
err := applyBatch(ctx, s.db, s.batch, nil, nil)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
@@ -739,10 +726,9 @@ func (s *scanState) commitFullBatch(ctx context.Context) error {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// updatePhase writes the scan's tail under one progress display: the
|
// updatePhase writes the scan's tail under one progress display: the
|
||||||
// final partial batch of hashed records, the new disk_size of every
|
// final partial batch of hashed records, and deletions for every record
|
||||||
// unchanged file whose disk_size changed, and deletions for every
|
// the scan did not verify (vanished files, plus paths that failed to
|
||||||
// record the scan did not verify (vanished files, plus paths that
|
// stat or hash).
|
||||||
// failed to stat or hash).
|
|
||||||
func (s *scanState) updatePhase(ctx context.Context) error {
|
func (s *scanState) updatePhase(ctx context.Context) error {
|
||||||
deletes := make([]string, 0, len(s.existing))
|
deletes := make([]string, 0, len(s.existing))
|
||||||
for path := range s.existing {
|
for path := range s.existing {
|
||||||
@@ -754,7 +740,7 @@ func (s *scanState) updatePhase(ctx context.Context) error {
|
|||||||
|
|
||||||
s.st.removed = len(deletes)
|
s.st.removed = len(deletes)
|
||||||
|
|
||||||
total := len(s.batch) + len(s.diskSizes) + len(deletes)
|
total := len(s.batch) + len(deletes)
|
||||||
prog := newProgress(progressParams{
|
prog := newProgress(progressParams{
|
||||||
label: scanPhase("update"), unit: scanUnit, total: int64(total),
|
label: scanPhase("update"), unit: scanUnit, total: int64(total),
|
||||||
stalledOn: "db",
|
stalledOn: "db",
|
||||||
@@ -769,11 +755,6 @@ func (s *scanState) updatePhase(ctx context.Context) error {
|
|||||||
|
|
||||||
s.batch = nil
|
s.batch = nil
|
||||||
|
|
||||||
err = applyDiskSizes(ctx, s.db, s.diskSizes, prog)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
return applyChanges(ctx, s.db, nil, deletes, prog)
|
return applyChanges(ctx, s.db, nil, deletes, prog)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -893,24 +874,9 @@ func sendEvent(ctx context.Context, events chan<- walkEvent,
|
|||||||
|
|
||||||
// operandWarning returns the one-line warning for an operand the walk
|
// operandWarning returns the one-line warning for an operand the walk
|
||||||
// does not start from, naming the path and what it is, or "" for one it
|
// does not start from, naming the path and what it is, or "" for one it
|
||||||
// does: a regular file, or a directory not named .zfs, that is not
|
// does: a regular file, or a directory not named .zfs. Symlinks are
|
||||||
// inside a directory named .zfs, neither as given nor with its symlinks
|
// never followed, including as operands.
|
||||||
// resolved. Symlinks are never followed, including as operands.
|
|
||||||
func operandWarning(root string, fi fs.FileInfo) string {
|
func operandWarning(root string, fi fs.FileInfo) string {
|
||||||
// The walk never enters a .zfs directory, so it starts from nothing
|
|
||||||
// inside one either, not even through a symlink. An operand whose
|
|
||||||
// symlinks cannot be resolved is checked as given.
|
|
||||||
resolved, err := filepath.EvalSymlinks(root)
|
|
||||||
if err != nil {
|
|
||||||
resolved = root
|
|
||||||
}
|
|
||||||
|
|
||||||
if slices.Contains(strings.Split(filepath.Dir(root), "/"), ".zfs") ||
|
|
||||||
slices.Contains(strings.Split(filepath.Dir(resolved), "/"), ".zfs") {
|
|
||||||
return fmt.Sprintf("walk %s: skipping operand inside a .zfs directory",
|
|
||||||
root)
|
|
||||||
}
|
|
||||||
|
|
||||||
var kind string
|
var kind string
|
||||||
|
|
||||||
switch mode := fi.Mode(); {
|
switch mode := fi.Mode(); {
|
||||||
@@ -979,12 +945,11 @@ func seedRoot(ctx context.Context, root string, rules walkRules,
|
|||||||
|
|
||||||
dev, ino := inodeOfInfo(fi)
|
dev, ino := inodeOfInfo(fi)
|
||||||
rec := fileRec{
|
rec := fileRec{
|
||||||
path: root,
|
path: root,
|
||||||
size: fi.Size(),
|
size: fi.Size(),
|
||||||
diskSize: diskSizeOfInfo(fi),
|
mtime: fi.ModTime(),
|
||||||
mtime: fi.ModTime(),
|
dev: dev,
|
||||||
dev: dev,
|
ino: ino,
|
||||||
ino: ino,
|
|
||||||
}
|
}
|
||||||
|
|
||||||
if !isDatabaseFile(rec, rules) {
|
if !isDatabaseFile(rec, rules) {
|
||||||
@@ -1164,12 +1129,11 @@ func emitFile(ctx context.Context, p string, e fs.DirEntry,
|
|||||||
|
|
||||||
dev, ino := inodeOfInfo(info)
|
dev, ino := inodeOfInfo(info)
|
||||||
rec := fileRec{
|
rec := fileRec{
|
||||||
path: p,
|
path: p,
|
||||||
size: info.Size(),
|
size: info.Size(),
|
||||||
diskSize: diskSizeOfInfo(info),
|
mtime: info.ModTime(),
|
||||||
mtime: info.ModTime(),
|
dev: dev,
|
||||||
dev: dev,
|
ino: ino,
|
||||||
ino: ino,
|
|
||||||
}
|
}
|
||||||
|
|
||||||
if !isDatabaseFile(rec, rules) {
|
if !isDatabaseFile(rec, rules) {
|
||||||
@@ -1261,23 +1225,6 @@ func inodeOfInfo(fi fs.FileInfo) (uint64, uint64) {
|
|||||||
return statDev(st), st.Ino
|
return statDev(st), st.Ino
|
||||||
}
|
}
|
||||||
|
|
||||||
// statBlockSize is the unit of st_blocks in an lstat result on Linux and
|
|
||||||
// macOS, whatever the filesystem's own block size.
|
|
||||||
const statBlockSize = 512
|
|
||||||
|
|
||||||
// diskSizeOfInfo returns the bytes a file takes on disk: its allocated
|
|
||||||
// blocks, st_blocks × 512. On a compressed or sparse file it is less
|
|
||||||
// than the file's size. It is 0 when the platform exposes no block
|
|
||||||
// count.
|
|
||||||
func diskSizeOfInfo(fi fs.FileInfo) int64 {
|
|
||||||
st, ok := fi.Sys().(*syscall.Stat_t)
|
|
||||||
if !ok {
|
|
||||||
return 0
|
|
||||||
}
|
|
||||||
|
|
||||||
return st.Blocks * statBlockSize
|
|
||||||
}
|
|
||||||
|
|
||||||
// hashResult carries the hashes computed for one inode run (or the
|
// hashResult carries the hashes computed for one inode run (or the
|
||||||
// error that prevented computing them) from the pool's workers to the
|
// error that prevented computing them) from the pool's workers to the
|
||||||
// hash phase. Its run leaves out the paths found removed.
|
// hash phase. Its run leaves out the paths found removed.
|
||||||
|
|||||||
+3
-131
@@ -42,24 +42,6 @@ func writeFile(t *testing.T, dir, name string, data []byte) string {
|
|||||||
return p
|
return p
|
||||||
}
|
}
|
||||||
|
|
||||||
// diskSizeOf returns the bytes the file at path takes on disk, from its
|
|
||||||
// own lstat: st_blocks counts 512-byte units. ZFS counts a new file's
|
|
||||||
// blocks only seconds after the write, and fsync does not hasten that.
|
|
||||||
// A test therefore calls unix.Sync, which waits for it, before the scan
|
|
||||||
// whose disk_size it compares with this.
|
|
||||||
func diskSizeOf(t *testing.T, path string) int64 {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
var st syscall.Stat_t
|
|
||||||
|
|
||||||
err := syscall.Lstat(path, &st)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
|
|
||||||
return st.Blocks * 512
|
|
||||||
}
|
|
||||||
|
|
||||||
// pattern returns n bytes of deterministic content seeded by tag.
|
// pattern returns n bytes of deterministic content seeded by tag.
|
||||||
func pattern(tag byte, n int) []byte {
|
func pattern(tag byte, n int) []byte {
|
||||||
data := make([]byte, n)
|
data := make([]byte, n)
|
||||||
@@ -1246,9 +1228,9 @@ func TestScanPipeline(t *testing.T) {
|
|||||||
// TestScanProgressNamesEachPhase checks the plain progress lines of a
|
// TestScanProgressNamesEachPhase checks the plain progress lines of a
|
||||||
// first scan, and of a rescan of one operand holding two of the
|
// first scan, and of a rescan of one operand holding two of the
|
||||||
// database's records. Each phase's first line names it with its place
|
// database's records. Each phase's first line names it with its place
|
||||||
// among the four: load gets the number of records before reading them,
|
// among the four: load counts the records before reading them, and the
|
||||||
// and the walk estimates its total from the records under its operand,
|
// walk estimates its total from the records under its operand, or says
|
||||||
// or says its time remaining is unknown when there are none.
|
// its time remaining is unknown when there are none.
|
||||||
//
|
//
|
||||||
//nolint:paralleltest // captureStderr replaces the process-wide os.Stderr
|
//nolint:paralleltest // captureStderr replaces the process-wide os.Stderr
|
||||||
func TestScanProgressNamesEachPhase(t *testing.T) {
|
func TestScanProgressNamesEachPhase(t *testing.T) {
|
||||||
@@ -1434,116 +1416,6 @@ func TestScanStoresRawHashes(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// TestScanStoresDiskSize scans a sparse file, which takes less space on
|
|
||||||
// disk than its length, once through its directory and once as the
|
|
||||||
// operand itself. Each scan stores the disk_size of the file's lstat.
|
|
||||||
func TestScanStoresDiskSize(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
const length = 1024 * 1024
|
|
||||||
|
|
||||||
p := sparseFile(t, t.TempDir(), "sparse", length)
|
|
||||||
pokeAt(t, p, 0, []byte("x"))
|
|
||||||
|
|
||||||
unix.Sync() // see diskSizeOf
|
|
||||||
|
|
||||||
want := diskSizeOf(t, p)
|
|
||||||
if want == 0 || want >= length {
|
|
||||||
t.Fatalf("the file takes %d bytes on disk, want more than none "+
|
|
||||||
"and fewer than its length %d", want, length)
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, operand := range []string{filepath.Dir(p), p} {
|
|
||||||
db := openTestDB(t)
|
|
||||||
syncTree(t, db, operand)
|
|
||||||
|
|
||||||
got := recordByPath(t, dbRecords(t, db), p).diskSize
|
|
||||||
if got != want {
|
|
||||||
t.Errorf("scan of %s: disk_size = %d, want %d",
|
|
||||||
operand, got, want)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestSyncScanRewriteUpdatesDiskSize writes into the hole of a scanned
|
|
||||||
// sparse file and moves its mtime on. The rescan hashes the file again
|
|
||||||
// and rewrites its record with the new disk_size.
|
|
||||||
func TestSyncScanRewriteUpdatesDiskSize(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
dir := t.TempDir()
|
|
||||||
db := openTestDB(t)
|
|
||||||
p := sparseFile(t, dir, "sparse", 1024*1024)
|
|
||||||
pokeAt(t, p, 0, []byte("x"))
|
|
||||||
|
|
||||||
syncTree(t, db, dir)
|
|
||||||
|
|
||||||
before := recordByPath(t, dbRecords(t, db), p).diskSize
|
|
||||||
|
|
||||||
pokeAt(t, p, 512*1024, pattern(7, 64*1024))
|
|
||||||
|
|
||||||
// The write may leave the mtime the scan recorded; a later one makes
|
|
||||||
// the rescan read the file again.
|
|
||||||
later := time.Now().Add(time.Hour)
|
|
||||||
|
|
||||||
err := os.Chtimes(p, later, later)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
|
|
||||||
unix.Sync() // see diskSizeOf
|
|
||||||
|
|
||||||
want := diskSizeOf(t, p)
|
|
||||||
if want == before {
|
|
||||||
t.Fatalf("the file still takes %d bytes on disk after the write "+
|
|
||||||
"into its hole", want)
|
|
||||||
}
|
|
||||||
|
|
||||||
st := syncTree(t, db, dir)
|
|
||||||
if st != (scanStats{walked: 1, updated: 1}) {
|
|
||||||
t.Fatalf("rescan stats = %+v, want 1 updated", st)
|
|
||||||
}
|
|
||||||
|
|
||||||
got := recordByPath(t, dbRecords(t, db), p).diskSize
|
|
||||||
if got != want {
|
|
||||||
t.Errorf("disk_size = %d, want %d", got, want)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestSyncScanUnchangedUpdatesDiskSize gives a file's record a
|
|
||||||
// disk_size the file does not have, as when ZFS had not yet counted the
|
|
||||||
// file's blocks at the last scan. The rescan writes the file's own
|
|
||||||
// disk_size into the record without reading the file again.
|
|
||||||
func TestSyncScanUnchangedUpdatesDiskSize(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
dir := t.TempDir()
|
|
||||||
db := openTestDB(t)
|
|
||||||
p := writeFile(t, dir, "a.bin", pattern(1, 500))
|
|
||||||
|
|
||||||
syncTree(t, db, dir)
|
|
||||||
|
|
||||||
// lstat counts 512-byte units, so no file takes 1 byte on disk.
|
|
||||||
_, err := db.ExecContext(t.Context(),
|
|
||||||
"UPDATE files SET disk_size = 1 WHERE path = ?", []byte(p))
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
|
|
||||||
unix.Sync() // see diskSizeOf
|
|
||||||
|
|
||||||
// A file read again would count as updated.
|
|
||||||
st := syncTree(t, db, dir)
|
|
||||||
if st != (scanStats{walked: 1, unchanged: 1}) {
|
|
||||||
t.Fatalf("rescan stats = %+v, want 1 unchanged", st)
|
|
||||||
}
|
|
||||||
|
|
||||||
got := recordByPath(t, dbRecords(t, db), p).diskSize
|
|
||||||
if want := diskSizeOf(t, p); got != want {
|
|
||||||
t.Errorf("disk_size = %d, want %d", got, want)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestSyncScanSameSecondRewrite rewrites a file in place at the same
|
// TestSyncScanSameSecondRewrite rewrites a file in place at the same
|
||||||
// size with an mtime later in the same second as the recorded one: the
|
// size with an mtime later in the same second as the recorded one: the
|
||||||
// next scan must notice the change and re-hash the file. The operand is
|
// next scan must notice the change and re-hash the file. The operand is
|
||||||
|
|||||||
@@ -22,10 +22,6 @@ type treeNode struct {
|
|||||||
digest [sha256.Size]byte
|
digest [sha256.Size]byte
|
||||||
fileCount int64
|
fileCount int64
|
||||||
totalSize int64
|
totalSize int64
|
||||||
// diskSize is the bytes the directory's files take on disk. Two
|
|
||||||
// copies of one tree have the same totalSize, but their diskSize
|
|
||||||
// can differ.
|
|
||||||
diskSize int64
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// treesPhase names one phase of trees with its place among all of them,
|
// treesPhase names one phase of trees with its place among all of them,
|
||||||
@@ -44,11 +40,11 @@ func treesPhase(name string) string {
|
|||||||
// database problem, including a missing database, is fatal.
|
// database problem, including a missing database, is fatal.
|
||||||
//
|
//
|
||||||
// The work runs in four phases, each under a progress display of its
|
// The work runs in four phases, each under a progress display of its
|
||||||
// own on stderr: open (open the database), read (read the number of
|
// own on stderr: open (open the database), read (count the records,
|
||||||
// records from the totals table, then read the records into the
|
// then read them into the hierarchy), group (group the directories by
|
||||||
// hierarchy), group (group the directories by digest) and write (write
|
// digest) and write (write the groups to stdout). Each display ends
|
||||||
// the groups to stdout). Each display ends before the next phase
|
// before the next phase starts, and before an error is returned to be
|
||||||
// starts, and before an error is returned to be printed.
|
// printed.
|
||||||
func runTrees(ctx context.Context, stdout io.Writer) error {
|
func runTrees(ctx context.Context, stdout io.Writer) error {
|
||||||
dbPath, source, err := databasePath()
|
dbPath, source, err := databasePath()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -143,16 +139,15 @@ func readTree(ctx context.Context, db *sql.DB,
|
|||||||
|
|
||||||
// writeTreeGroups writes dupes to stdout as TSV, a header line and then
|
// writeTreeGroups writes dupes to stdout as TSV, a header line and then
|
||||||
// a row for every member of each group but its first, counting each
|
// a row for every member of each group but its first, counting each
|
||||||
// group on prog once its rows are written. The disk_size column is the
|
// group on prog once its rows are written. stdout may be the terminal
|
||||||
// dupe's own. stdout may be the terminal prog is drawn on, so the lines
|
// prog is drawn on, so the lines go through prog's writer. It returns
|
||||||
// go through prog's writer. It returns the number of rows (dupe trees)
|
// the number of rows (dupe trees) and the bytes they could reclaim.
|
||||||
// and the bytes they could reclaim.
|
|
||||||
func writeTreeGroups(stdout io.Writer, dupes [][]*treeNode,
|
func writeTreeGroups(stdout io.Writer, dupes [][]*treeNode,
|
||||||
prog *progress,
|
prog *progress,
|
||||||
) (int, int64, error) {
|
) (int, int64, error) {
|
||||||
out := prog.writer(stdout)
|
out := prog.writer(stdout)
|
||||||
|
|
||||||
_, err := fmt.Fprintln(out, "first\tdupe\tfiles\tsize\tdisk_size")
|
_, err := fmt.Fprintln(out, "first\tdupe\tfiles\tsize")
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return 0, 0, fmt.Errorf("write stdout: %w", err)
|
return 0, 0, fmt.Errorf("write stdout: %w", err)
|
||||||
}
|
}
|
||||||
@@ -164,9 +159,9 @@ func writeTreeGroups(stdout io.Writer, dupes [][]*treeNode,
|
|||||||
for _, g := range dupes {
|
for _, g := range dupes {
|
||||||
first := g[0]
|
first := g[0]
|
||||||
for _, n := range g[1:] {
|
for _, n := range g[1:] {
|
||||||
_, err = fmt.Fprintf(out, "%s\t%s\t%d\t%d\t%d\n",
|
_, err = fmt.Fprintf(out, "%s\t%s\t%d\t%d\n",
|
||||||
escapePath(first.path), escapePath(n.path),
|
escapePath(first.path), escapePath(n.path),
|
||||||
first.fileCount, first.totalSize, n.diskSize)
|
first.fileCount, first.totalSize)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return 0, 0, fmt.Errorf("write stdout: %w", err)
|
return 0, 0, fmt.Errorf("write stdout: %w", err)
|
||||||
}
|
}
|
||||||
@@ -237,7 +232,6 @@ func (b *treeBuilder) add(r scanRec) {
|
|||||||
dir.entries = append(dir.entries, fileEntry(name, r))
|
dir.entries = append(dir.entries, fileEntry(name, r))
|
||||||
dir.fileCount++
|
dir.fileCount++
|
||||||
dir.totalSize += r.size
|
dir.totalSize += r.size
|
||||||
dir.diskSize += r.diskSize
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// openDir starts the directory called name inside the innermost open
|
// openDir starts the directory called name inside the innermost open
|
||||||
@@ -276,7 +270,6 @@ func (b *treeBuilder) closeTo(n int) {
|
|||||||
"d\x00"+name+"\x00"+string(dir.digest[:]))
|
"d\x00"+name+"\x00"+string(dir.digest[:]))
|
||||||
dir.parent.fileCount += dir.fileCount
|
dir.parent.fileCount += dir.fileCount
|
||||||
dir.parent.totalSize += dir.totalSize
|
dir.parent.totalSize += dir.totalSize
|
||||||
dir.parent.diskSize += dir.diskSize
|
|
||||||
b.dirs = append(b.dirs, dir)
|
b.dirs = append(b.dirs, dir)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
+11
-40
@@ -17,24 +17,18 @@ const treesOpenLine = "[1/4] open: opening the database, elapsed 0s, " +
|
|||||||
|
|
||||||
// smokeTreeRecs mirrors the README smoke-test tree layout: /d/t1 and
|
// smokeTreeRecs mirrors the README smoke-test tree layout: /d/t1 and
|
||||||
// /d/t2 are identical, /d/t3 differs from them only by one filename.
|
// /d/t2 are identical, /d/t3 differs from them only by one filename.
|
||||||
// Each file takes one 4 KiB block on disk.
|
|
||||||
func smokeTreeRecs() []scanRec {
|
func smokeTreeRecs() []scanRec {
|
||||||
f1Head, f1Tail, f1Content := testHash("f1h"), testHash("f1t"), testHash("f1c")
|
f1Head, f1Tail, f1Content := testHash("f1h"), testHash("f1t"), testHash("f1c")
|
||||||
f2Head, f2Tail, f2Content := testHash("f2h"), testHash("f2t"), testHash("f2c")
|
f2Head, f2Tail, f2Content := testHash("f2h"), testHash("f2t"), testHash("f2c")
|
||||||
|
|
||||||
return []scanRec{
|
return []scanRec{
|
||||||
{size: 3000, diskSize: 4096, head: f1Head, tail: f1Tail,
|
{size: 3000, head: f1Head, tail: f1Tail, content: f1Content, path: "/d/t1/f1"},
|
||||||
content: f1Content, path: "/d/t1/f1"},
|
{size: 100, head: f2Head, tail: f2Tail, content: f2Content, path: "/d/t1/sub/f2"},
|
||||||
{size: 100, diskSize: 4096, head: f2Head, tail: f2Tail,
|
{size: 3000, head: f1Head, tail: f1Tail, content: f1Content, path: "/d/t2/f1"},
|
||||||
content: f2Content, path: "/d/t1/sub/f2"},
|
{size: 100, head: f2Head, tail: f2Tail, content: f2Content, path: "/d/t2/sub/f2"},
|
||||||
{size: 3000, diskSize: 4096, head: f1Head, tail: f1Tail,
|
{size: 3000, head: f1Head, tail: f1Tail, content: f1Content, path: "/d/t3/f1"},
|
||||||
content: f1Content, path: "/d/t2/f1"},
|
{size: 100, head: f2Head, tail: f2Tail, content: f2Content,
|
||||||
{size: 100, diskSize: 4096, head: f2Head, tail: f2Tail,
|
path: "/d/t3/sub/f2renamed"},
|
||||||
content: f2Content, path: "/d/t2/sub/f2"},
|
|
||||||
{size: 3000, diskSize: 4096, head: f1Head, tail: f1Tail,
|
|
||||||
content: f1Content, path: "/d/t3/f1"},
|
|
||||||
{size: 100, diskSize: 4096, head: f2Head, tail: f2Tail,
|
|
||||||
content: f2Content, path: "/d/t3/sub/f2renamed"},
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -204,33 +198,13 @@ func TestRunTreesEscapesPaths(t *testing.T) {
|
|||||||
code, exitOK, stderr.String())
|
code, exitOK, stderr.String())
|
||||||
}
|
}
|
||||||
|
|
||||||
want := "first\tdupe\tfiles\tsize\tdisk_size\n" +
|
want := "first\tdupe\tfiles\tsize\n" +
|
||||||
`/d/\tone\ntwo\rthree\\four` + "\t/d/A\t1\t5\t4096\n"
|
`/d/\tone\ntwo\rthree\\four` + "\t/d/A\t1\t5\n"
|
||||||
if got := stdout.String(); got != want {
|
if got := stdout.String(); got != want {
|
||||||
t.Errorf("stdout = %q, want %q", got, want)
|
t.Errorf("stdout = %q, want %q", got, want)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// TestRunTreesDiskSize runs trees on two copies of a tree whose files
|
|
||||||
// take different space on disk. The row's disk_size is the sum over the
|
|
||||||
// dupe copy's files, those in its subdirectory included.
|
|
||||||
func TestRunTreesDiskSize(t *testing.T) {
|
|
||||||
f, g := testHash("f"), testHash("g")
|
|
||||||
recs := []scanRec{
|
|
||||||
{size: 10, diskSize: 4096, head: f, tail: f, content: f, path: "/a/f"},
|
|
||||||
{size: 20, diskSize: 4096, head: g, tail: g, content: g, path: "/a/sub/g"},
|
|
||||||
{size: 10, diskSize: 512, head: f, tail: f, content: f, path: "/b/f"},
|
|
||||||
{size: 20, diskSize: 1024, head: g, tail: g, content: g, path: "/b/sub/g"},
|
|
||||||
}
|
|
||||||
|
|
||||||
t.Setenv(databaseEnv, seedDatabase(t, recs))
|
|
||||||
|
|
||||||
want := "first\tdupe\tfiles\tsize\tdisk_size\n/a\t/b\t2\t30\t1536\n"
|
|
||||||
if got := runStdout(t, cmdTrees); got != want {
|
|
||||||
t.Errorf("stdout = %q, want %q", got, want)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestRunTreesProgressNamesEachPhase runs trees on the smoke-test
|
// TestRunTreesProgressNamesEachPhase runs trees on the smoke-test
|
||||||
// records with stderr not a terminal, and checks every line it prints
|
// records with stderr not a terminal, and checks every line it prints
|
||||||
// there. After the line naming the database, each phase prints a line
|
// there. After the line naming the database, each phase prints a line
|
||||||
@@ -252,8 +226,7 @@ func TestRunTreesProgressNamesEachPhase(t *testing.T) {
|
|||||||
code, exitOK, stderr())
|
code, exitOK, stderr())
|
||||||
}
|
}
|
||||||
|
|
||||||
report := "first\tdupe\tfiles\tsize\tdisk_size\n" +
|
report := "first\tdupe\tfiles\tsize\n/d/t1\t/d/t2\t2\t3100\n"
|
||||||
"/d/t1\t/d/t2\t2\t3100\t8192\n"
|
|
||||||
if got := stdout.String(); got != report {
|
if got := stdout.String(); got != report {
|
||||||
t.Errorf("stdout = %q, want %q", got, report)
|
t.Errorf("stdout = %q, want %q", got, report)
|
||||||
}
|
}
|
||||||
@@ -318,9 +291,7 @@ func TestWriteTreeGroupsOnTerminal(t *testing.T) {
|
|||||||
shown = append(shown, line[strings.LastIndex(line, "\r")+1:])
|
shown = append(shown, line[strings.LastIndex(line, "\r")+1:])
|
||||||
}
|
}
|
||||||
|
|
||||||
want := []string{
|
want := []string{"first\tdupe\tfiles\tsize", "/d/t1\t/d/t2\t2\t3100"}
|
||||||
"first\tdupe\tfiles\tsize\tdisk_size", "/d/t1\t/d/t2\t2\t3100\t8192",
|
|
||||||
}
|
|
||||||
if len(shown) < len(want) || !slices.Equal(shown[:len(want)], want) {
|
if len(shown) < len(want) || !slices.Equal(shown[:len(want)], want) {
|
||||||
t.Errorf("terminal shows lines %q, want %q first", shown, want)
|
t.Errorf("terminal shows lines %q, want %q first", shown, want)
|
||||||
}
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user