Compare commits
1
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
dfbfdd0601 |
@@ -29,20 +29,11 @@
|
|||||||
|
|
||||||
# Completed Steps
|
# Completed Steps
|
||||||
|
|
||||||
- `README.md` rewritten from scratch, sentence by sentence, as a newcomer reads
|
|
||||||
it. It leads with what the program is and how to use it. A reference section
|
|
||||||
covers every setting and every subcommand. Development and testing come last.
|
|
||||||
Every statement was checked against the code on `next`. Five comments in
|
|
||||||
`main_test.go` cite "Settings", where their rules now live. One comment each
|
|
||||||
in `scan_test.go`, `trees_test.go` and `exclude_test.go` cites what the README
|
|
||||||
now says (2026-11-02, https://git.eeqj.de/sneak/sfdupes/issues/135)
|
|
||||||
|
|
||||||
- each record holds `disk_size`, the bytes the file takes on disk: its allocated
|
- each record holds `disk_size`, the bytes the file takes on disk: its allocated
|
||||||
blocks, `st_blocks` × 512, from the lstat the walk already does. `trees`
|
blocks, `st_blocks` × 512, from the lstat the walk already does. `trees`
|
||||||
prints a `disk_size` column after `size`, the sum over the `dupe` copy's
|
prints a `disk_size` column after `size`, the sum over the `dupe` copy's
|
||||||
files; on a compressed or sparse file it differs from `size`. A rescan updates
|
files; on a compressed or sparse file it differs from `size` (2026-10-27,
|
||||||
the `disk_size` of an unchanged file whose size on disk changed, without
|
https://git.eeqj.de/sneak/sfdupes/issues/147)
|
||||||
reading the file (2026-10-27, https://git.eeqj.de/sneak/sfdupes/issues/147)
|
|
||||||
|
|
||||||
- the database keeps the number of records and the sum of their sizes in the one
|
- the database keeps the number of records and the sum of their sizes in the one
|
||||||
row of a `totals` table, which three triggers on `files` keep current. `info`
|
row of a `totals` table, which three triggers on `files` keep current. `info`
|
||||||
|
|||||||
@@ -729,15 +729,15 @@ func readDatabaseCounts(ctx context.Context,
|
|||||||
return c, nil
|
return c, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// loadFileMeta streams every record's path, size, disk_size and mtime
|
// loadFileMeta streams every record's path, size and mtime to fn. Scan
|
||||||
// to fn. Scan change detection needs no hash values, and skipping the
|
// change detection needs no hash values, and skipping the hash columns
|
||||||
// hash columns keeps the scan's in-memory index small on
|
// keeps the scan's in-memory index small on multi-million-file
|
||||||
// multi-million-file databases.
|
// databases.
|
||||||
func loadFileMeta(ctx context.Context, db *sql.DB,
|
func loadFileMeta(ctx context.Context, db *sql.DB,
|
||||||
fn func(path string, m fileMeta),
|
fn func(path string, size int64, mtime time.Time),
|
||||||
) error {
|
) error {
|
||||||
rows, err := db.QueryContext(ctx,
|
rows, err := db.QueryContext(ctx,
|
||||||
"SELECT path, size, disk_size, mtime, mtime_nsec FROM files")
|
"SELECT path, size, mtime, mtime_nsec FROM files")
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("read records: %w", err)
|
return fmt.Errorf("read records: %w", err)
|
||||||
}
|
}
|
||||||
@@ -747,18 +747,15 @@ func loadFileMeta(ctx context.Context, db *sql.DB,
|
|||||||
for rows.Next() {
|
for rows.Next() {
|
||||||
var (
|
var (
|
||||||
path []byte
|
path []byte
|
||||||
m fileMeta
|
size, sec, nsec int64
|
||||||
sec, nsec int64
|
|
||||||
)
|
)
|
||||||
|
|
||||||
err = rows.Scan(&path, &m.size, &m.diskSize, &sec, &nsec)
|
err = rows.Scan(&path, &size, &sec, &nsec)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("read record: %w", err)
|
return fmt.Errorf("read record: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
m.mtime = time.Unix(sec, nsec)
|
fn(string(path), size, time.Unix(sec, nsec))
|
||||||
|
|
||||||
fn(string(path), m)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
err = rows.Err()
|
err = rows.Err()
|
||||||
@@ -776,21 +773,21 @@ func loadFileMeta(ctx context.Context, db *sql.DB,
|
|||||||
// WAL small and let concurrent reports observe progress.
|
// WAL small and let concurrent reports observe progress.
|
||||||
const updateBatchSize = 10000
|
const updateBatchSize = 10000
|
||||||
|
|
||||||
// applyChanges writes upserts for new and changed files and deletes for
|
// applyChanges writes one scan's database changes — upserts for new and
|
||||||
// vanished ones, in batched transactions. Progress is rendered on prog
|
// changed files, deletes for vanished ones — in batched transactions.
|
||||||
// (one increment per change).
|
// Progress is rendered on prog (one increment per change).
|
||||||
func applyChanges(ctx context.Context, db *sql.DB, upserts []scanRec,
|
func applyChanges(ctx context.Context, db *sql.DB, upserts []scanRec,
|
||||||
deletes []string, prog *progress,
|
deletes []string, prog *progress,
|
||||||
) error {
|
) error {
|
||||||
for batch := range slices.Chunk(upserts, updateBatchSize) {
|
for batch := range slices.Chunk(upserts, updateBatchSize) {
|
||||||
err := applyBatch(ctx, db, batch, nil, nil, prog)
|
err := applyBatch(ctx, db, batch, nil, prog)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
for batch := range slices.Chunk(deletes, updateBatchSize) {
|
for batch := range slices.Chunk(deletes, updateBatchSize) {
|
||||||
err := applyBatch(ctx, db, nil, nil, batch, prog)
|
err := applyBatch(ctx, db, nil, batch, prog)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
@@ -799,26 +796,10 @@ func applyChanges(ctx context.Context, db *sql.DB, upserts []scanRec,
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// applyDiskSizes writes the disk_size of each of recs into its record,
|
// applyBatch commits one batch of upserts and deletes in a single
|
||||||
// in batched transactions, and leaves the rest of the record alone.
|
// transaction.
|
||||||
// Progress is rendered on prog (one increment per record).
|
|
||||||
func applyDiskSizes(ctx context.Context, db *sql.DB, recs []fileRec,
|
|
||||||
prog *progress,
|
|
||||||
) error {
|
|
||||||
for batch := range slices.Chunk(recs, updateBatchSize) {
|
|
||||||
err := applyBatch(ctx, db, nil, batch, nil, prog)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// applyBatch commits one batch of upserts, disk_size updates and
|
|
||||||
// deletes in a single transaction.
|
|
||||||
func applyBatch(ctx context.Context, db *sql.DB, upserts []scanRec,
|
func applyBatch(ctx context.Context, db *sql.DB, upserts []scanRec,
|
||||||
diskSizes []fileRec, deletes []string, prog *progress,
|
deletes []string, prog *progress,
|
||||||
) error {
|
) error {
|
||||||
tx, err := db.BeginTx(ctx, nil)
|
tx, err := db.BeginTx(ctx, nil)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -832,11 +813,6 @@ func applyBatch(ctx context.Context, db *sql.DB, upserts []scanRec,
|
|||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
err = execDiskSizes(ctx, tx, diskSizes, prog)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
err = execDeletes(ctx, tx, deletes, prog)
|
err = execDeletes(ctx, tx, deletes, prog)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
@@ -875,31 +851,6 @@ func execUpserts(ctx context.Context, tx *sql.Tx, upserts []scanRec,
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// execDiskSizes sets the disk_size of the record of each of recs. It
|
|
||||||
// does not touch size, so the totals triggers do not fire.
|
|
||||||
func execDiskSizes(ctx context.Context, tx *sql.Tx, recs []fileRec,
|
|
||||||
prog *progress,
|
|
||||||
) error {
|
|
||||||
st, err := tx.PrepareContext(ctx,
|
|
||||||
"UPDATE files SET disk_size = ? WHERE path = ?")
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("prepare disk_size update: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
defer func() { _ = st.Close() }()
|
|
||||||
|
|
||||||
for _, r := range recs {
|
|
||||||
_, err = st.ExecContext(ctx, r.diskSize, []byte(r.path))
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("update disk_size of %s: %w", r.path, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
prog.increment()
|
|
||||||
}
|
|
||||||
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// execDeletes removes the records for paths no longer present.
|
// execDeletes removes the records for paths no longer present.
|
||||||
func execDeletes(ctx context.Context, tx *sql.Tx, deletes []string,
|
func execDeletes(ctx context.Context, tx *sql.Tx, deletes []string,
|
||||||
prog *progress,
|
prog *progress,
|
||||||
|
|||||||
+1
-3
@@ -17,9 +17,7 @@ import (
|
|||||||
func TestDefaultExcludes(t *testing.T) {
|
func TestDefaultExcludes(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
// README "Rules for the walk": the Linux and macOS lists, and none
|
// README "Rules for the walk": each system's list.
|
||||||
// on any other system. The Windows list exists in the code only: the
|
|
||||||
// binary does not build for Windows, so the README leaves it out.
|
|
||||||
cases := map[string][]string{
|
cases := map[string][]string{
|
||||||
goosLinux: {"/proc", "/sys", "/dev", "/run", "/tmp", "/var/tmp"},
|
goosLinux: {"/proc", "/sys", "/dev", "/run", "/tmp", "/var/tmp"},
|
||||||
goosDarwin: {
|
goosDarwin: {
|
||||||
|
|||||||
+19
-37
@@ -394,7 +394,7 @@ func TestRunNamesRelativeDatabase(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestRunUsesDatabaseThroughSymlinkedDirectory(t *testing.T) {
|
func TestRunUsesDatabaseThroughSymlinkedDirectory(t *testing.T) {
|
||||||
// README §Settings: the path names the file exactly. With link a
|
// README §Database: the path names the file exactly. With link a
|
||||||
// symlink to the directory real/sub, link/../db.sqlite names
|
// symlink to the directory real/sub, link/../db.sqlite names
|
||||||
// real/db.sqlite, not db.sqlite beside link. Every subcommand must
|
// real/db.sqlite, not db.sqlite beside link. Every subcommand must
|
||||||
// use that file, relative or absolute, and its first line must name
|
// use that file, relative or absolute, and its first line must name
|
||||||
@@ -649,7 +649,7 @@ func TestRunUsageErrors(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestRunScanRejectsWorkersBelowOne(t *testing.T) {
|
func TestRunScanRejectsWorkersBelowOne(t *testing.T) {
|
||||||
// README §Settings: --workers below 1 is a usage error reported in
|
// README §scan mode: --workers below 1 is a usage error reported in
|
||||||
// one line on stderr, before the scan opens the database.
|
// one line on stderr, before the scan opens the database.
|
||||||
for _, workers := range []string{"0", "-1"} {
|
for _, workers := range []string{"0", "-1"} {
|
||||||
t.Run(workers, func(t *testing.T) {
|
t.Run(workers, func(t *testing.T) {
|
||||||
@@ -687,8 +687,8 @@ func TestRunScanRejectsWorkersBelowOne(t *testing.T) {
|
|||||||
func TestRunHelp(t *testing.T) {
|
func TestRunHelp(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
// README §Settings: help goes to stderr, exits 0, and leaves stdout
|
// README §Subcommands: help goes to stderr, exits 0, and leaves
|
||||||
// empty.
|
// stdout empty.
|
||||||
cases := [][]string{{"--help"}, {"-h"}, {cmdScan, "--help"}}
|
cases := [][]string{{"--help"}, {"-h"}, {cmdScan, "--help"}}
|
||||||
|
|
||||||
for _, args := range cases {
|
for _, args := range cases {
|
||||||
@@ -714,8 +714,8 @@ func TestRunHelp(t *testing.T) {
|
|||||||
func TestRunVersion(t *testing.T) {
|
func TestRunVersion(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
// README §Settings: the version is one line on stdout, with nothing
|
// README §Subcommands: the version is one line on stdout, with
|
||||||
// on stderr, and exits 0.
|
// nothing on stderr, and exits 0.
|
||||||
info, _ := debug.ReadBuildInfo()
|
info, _ := debug.ReadBuildInfo()
|
||||||
|
|
||||||
for _, arg := range []string{"--version", "-v"} {
|
for _, arg := range []string{"--version", "-v"} {
|
||||||
@@ -774,8 +774,8 @@ func TestBuildVersion(t *testing.T) {
|
|||||||
func TestRunVersionWriteFailureIsFatal(t *testing.T) {
|
func TestRunVersionWriteFailureIsFatal(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
// README §Error handling: a failed write to stdout is a fatal error,
|
// README §Error handling: a stdout write failure exits 1, reported
|
||||||
// exit 1, reported on stderr starting with "sfdupes:".
|
// in one line on stderr.
|
||||||
var stderr bytes.Buffer
|
var stderr bytes.Buffer
|
||||||
|
|
||||||
code := run([]string{"--version"}, failingWriter{}, &stderr)
|
code := run([]string{"--version"}, failingWriter{}, &stderr)
|
||||||
@@ -818,23 +818,6 @@ func scanFixture(t *testing.T) []string {
|
|||||||
return dupes
|
return dupes
|
||||||
}
|
}
|
||||||
|
|
||||||
// storedDiskSize returns the disk_size of the record for path in the
|
|
||||||
// database at dbPath. A test of a trees run compares with this rather
|
|
||||||
// than with a new lstat, which on ZFS can differ from the scan's (see
|
|
||||||
// diskSizeOf).
|
|
||||||
func storedDiskSize(t *testing.T, dbPath, path string) int64 {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
db, err := openReportDatabase(t.Context(), dbPath)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
|
|
||||||
defer func() { _ = db.Close() }()
|
|
||||||
|
|
||||||
return recordByPath(t, dbRecords(t, db), path).diskSize
|
|
||||||
}
|
|
||||||
|
|
||||||
// scanOK runs scan over operands, fails the test unless it exits 0 with
|
// scanOK runs scan over operands, fails the test unless it exits 0 with
|
||||||
// nothing on stdout, and returns everything it printed to stderr.
|
// nothing on stdout, and returns everything it printed to stderr.
|
||||||
func scanOK(t *testing.T, operands ...string) string {
|
func scanOK(t *testing.T, operands ...string) string {
|
||||||
@@ -1160,7 +1143,7 @@ func TestRunTreesSucceeds(t *testing.T) {
|
|||||||
// trees of each other.
|
// trees of each other.
|
||||||
want := "first\tdupe\tfiles\tsize\tdisk_size\n" +
|
want := "first\tdupe\tfiles\tsize\tdisk_size\n" +
|
||||||
filepath.Dir(dupes[0]) + "\t" + filepath.Dir(dupes[1]) + "\t1\t300\t" +
|
filepath.Dir(dupes[0]) + "\t" + filepath.Dir(dupes[1]) + "\t1\t300\t" +
|
||||||
strconv.FormatInt(storedDiskSize(t, path, dupes[1]), 10) + "\n"
|
strconv.FormatInt(diskSizeOf(t, dupes[1]), 10) + "\n"
|
||||||
if got := stdout.String(); got != want {
|
if got := stdout.String(); got != want {
|
||||||
t.Errorf("stdout = %q, want %q", got, want)
|
t.Errorf("stdout = %q, want %q", got, want)
|
||||||
}
|
}
|
||||||
@@ -1196,8 +1179,8 @@ func TestRunReportsNeedOnlyReadAccess(t *testing.T) {
|
|||||||
dupes[0] + "\t" + dupes[1] + "\t300\n",
|
dupes[0] + "\t" + dupes[1] + "\t300\n",
|
||||||
cmdTrees: "first\tdupe\tfiles\tsize\tdisk_size\n" +
|
cmdTrees: "first\tdupe\tfiles\tsize\tdisk_size\n" +
|
||||||
filepath.Dir(dupes[0]) + "\t" + filepath.Dir(dupes[1]) +
|
filepath.Dir(dupes[0]) + "\t" + filepath.Dir(dupes[1]) +
|
||||||
"\t1\t300\t" +
|
"\t1\t300\t" + strconv.FormatInt(diskSizeOf(t, dupes[1]), 10) +
|
||||||
strconv.FormatInt(storedDiskSize(t, path, dupes[1]), 10) + "\n",
|
"\n",
|
||||||
}
|
}
|
||||||
|
|
||||||
for name, want := range cases {
|
for name, want := range cases {
|
||||||
@@ -1232,11 +1215,9 @@ func assertRunsUseDatabase(t *testing.T, dir, name string) {
|
|||||||
t.Errorf("report stdout = %q, want %q", got, want)
|
t.Errorf("report stdout = %q, want %q", got, want)
|
||||||
}
|
}
|
||||||
|
|
||||||
diskSize := storedDiskSize(t, filepath.Join(dir, name), dupes[1])
|
|
||||||
|
|
||||||
want = "first\tdupe\tfiles\tsize\tdisk_size\n" +
|
want = "first\tdupe\tfiles\tsize\tdisk_size\n" +
|
||||||
filepath.Dir(dupes[0]) + "\t" + filepath.Dir(dupes[1]) + "\t1\t300\t" +
|
filepath.Dir(dupes[0]) + "\t" + filepath.Dir(dupes[1]) + "\t1\t300\t" +
|
||||||
strconv.FormatInt(diskSize, 10) + "\n"
|
strconv.FormatInt(diskSizeOf(t, dupes[1]), 10) + "\n"
|
||||||
if got := runStdout(t, cmdTrees); got != want {
|
if got := runStdout(t, cmdTrees); got != want {
|
||||||
t.Errorf("trees stdout = %q, want %q", got, want)
|
t.Errorf("trees stdout = %q, want %q", got, want)
|
||||||
}
|
}
|
||||||
@@ -1257,7 +1238,7 @@ func assertRunsUseDatabase(t *testing.T, dir, name string) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestRunDatabasePathUsedAsGiven(t *testing.T) {
|
func TestRunDatabasePathUsedAsGiven(t *testing.T) {
|
||||||
// README §Settings: the path names the database file exactly. In
|
// README §Database: the path names the database file exactly. In
|
||||||
// SQLite's connection string an unescaped ? or # would end the file
|
// SQLite's connection string an unescaped ? or # would end the file
|
||||||
// name and % would start an escape, and a path starting with //
|
// name and % would start an escape, and a path starting with //
|
||||||
// could be read as a host name. %25 is a valid escape, so unescaped
|
// could be read as a host name. %25 is a valid escape, so unescaped
|
||||||
@@ -1302,8 +1283,9 @@ func holdScanLock(t *testing.T, path string) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestRunSecondScanFails(t *testing.T) {
|
func TestRunSecondScanFails(t *testing.T) {
|
||||||
// README §Database: a second scan that finds the lock held fails
|
// README §Database: while one scan holds the lock, a second scan
|
||||||
// at once, naming the lock file, without opening the database.
|
// fails at once, naming the lock file, without opening the
|
||||||
|
// database.
|
||||||
path := testDBPath(t)
|
path := testDBPath(t)
|
||||||
t.Setenv(databaseEnv, path)
|
t.Setenv(databaseEnv, path)
|
||||||
|
|
||||||
@@ -1362,7 +1344,7 @@ func assertSecondScanFails(t *testing.T, path string) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestRunSecondScanThroughSymlinkFails(t *testing.T) {
|
func TestRunSecondScanThroughSymlinkFails(t *testing.T) {
|
||||||
// README §Database: a path through a symlink names the same lock
|
// README §Database: every path to one database names the same lock
|
||||||
// file, so while a scan holds it through one path, a scan through
|
// file, so while a scan holds it through one path, a scan through
|
||||||
// another fails, naming the lock file beside the database's real
|
// another fails, naming the lock file beside the database's real
|
||||||
// path.
|
// path.
|
||||||
@@ -1467,8 +1449,8 @@ func TestRunReportsDuringScan(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestRunStdoutClosedIsFatal(t *testing.T) {
|
func TestRunStdoutClosedIsFatal(t *testing.T) {
|
||||||
// README §Error handling: a failed write to stdout is a fatal error,
|
// README §Error handling: a stdout write failure exits 1, reported
|
||||||
// exit 1, reported on stderr starting with "sfdupes:".
|
// in one line on stderr.
|
||||||
for _, name := range []string{cmdReport, cmdTrees, cmdInfo} {
|
for _, name := range []string{cmdReport, cmdTrees, cmdInfo} {
|
||||||
t.Run(name, func(t *testing.T) {
|
t.Run(name, func(t *testing.T) {
|
||||||
t.Setenv(databaseEnv, testDBPath(t))
|
t.Setenv(databaseEnv, testDBPath(t))
|
||||||
|
|||||||
@@ -66,11 +66,10 @@ type fileRec struct {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// fileMeta is the in-memory index entry for one existing database
|
// fileMeta is the in-memory index entry for one existing database
|
||||||
// record: what the walk compares the file with. Hashes stay on disk; at
|
// record: just enough for change detection. Hashes stay on disk; at
|
||||||
// tens of millions of records they would dominate the scan's memory.
|
// tens of millions of records they would dominate the scan's memory.
|
||||||
type fileMeta struct {
|
type fileMeta struct {
|
||||||
size int64
|
size int64
|
||||||
diskSize int64
|
|
||||||
mtime time.Time
|
mtime time.Time
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -345,7 +344,6 @@ type scanState struct {
|
|||||||
db *sql.DB
|
db *sql.DB
|
||||||
existing map[string]fileMeta
|
existing map[string]fileMeta
|
||||||
toHash []fileRec // new and changed files: must be read
|
toHash []fileRec // new and changed files: must be read
|
||||||
diskSizes []fileRec // unchanged files whose disk_size changed
|
|
||||||
batch []scanRec
|
batch []scanRec
|
||||||
st scanStats
|
st scanStats
|
||||||
}
|
}
|
||||||
@@ -476,24 +474,24 @@ func (s *scanState) loadIndex(ctx context.Context, roots []string) error {
|
|||||||
|
|
||||||
s.existing = make(map[string]fileMeta)
|
s.existing = make(map[string]fileMeta)
|
||||||
|
|
||||||
return loadFileMeta(ctx, s.db, func(path string, m fileMeta) {
|
return loadFileMeta(ctx, s.db,
|
||||||
|
func(path string, size int64, mtime time.Time) {
|
||||||
prog.increment()
|
prog.increment()
|
||||||
|
|
||||||
if underAnyRoot(path, roots) {
|
if underAnyRoot(path, roots) {
|
||||||
s.existing[path] = m
|
s.existing[path] = fileMeta{size: size, mtime: mtime}
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
// walkPhase drains the walk and queues every new or changed file for
|
// walkPhase drains the walk and queues every new or changed file for
|
||||||
// the hash phase. An unchanged file is not read, and its record counts
|
// the hash phase. An unchanged file needs nothing more, and its record
|
||||||
// as verified. When its disk_size differs from the record's, it is
|
// counts as verified. A record at or below a path the walk left out,
|
||||||
// queued for the update phase, which writes the new disk_size. A record
|
// because -x stopped at it, it is a default exclude, or it is not on a
|
||||||
// at or below a path the walk left out, because -x stopped at it, it is
|
// persistent filesystem, counts as outside the roots: it is not
|
||||||
// a default exclude, or it is not on a persistent filesystem, counts as
|
// deleted. That does not hold below a root at or inside that path,
|
||||||
// outside the roots: it is not deleted. That does not hold below a root
|
// since that root's own walk verifies the records below it; nor is a
|
||||||
// at or inside that path, since that root's own walk verifies the
|
// line printed for such a root.
|
||||||
// records below it; nor is a line printed for such a root.
|
|
||||||
func (s *scanState) walkPhase(events <-chan walkEvent, roots []string) {
|
func (s *scanState) walkPhase(events <-chan walkEvent, roots []string) {
|
||||||
// The records the last scan left under the roots estimate how many
|
// The records the last scan left under the roots estimate how many
|
||||||
// files the walk will find. A first scan has none, and no estimate.
|
// files the walk will find. A first scan has none, and no estimate.
|
||||||
@@ -551,13 +549,6 @@ func (s *scanState) walkPhase(events <-chan walkEvent, roots []string) {
|
|||||||
delete(s.existing, ev.rec.path)
|
delete(s.existing, ev.rec.path)
|
||||||
|
|
||||||
s.st.unchanged++
|
s.st.unchanged++
|
||||||
|
|
||||||
// The space a file takes on disk can change while its size and
|
|
||||||
// mtime stay the same. ZFS, for one, counts a new file's blocks
|
|
||||||
// only seconds after the write.
|
|
||||||
if old.diskSize != ev.rec.diskSize {
|
|
||||||
s.diskSizes = append(s.diskSizes, ev.rec)
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
prog.finish()
|
prog.finish()
|
||||||
@@ -728,7 +719,7 @@ func (s *scanState) commitFullBatch(ctx context.Context) error {
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
err := applyBatch(ctx, s.db, s.batch, nil, nil, nil)
|
err := applyBatch(ctx, s.db, s.batch, nil, nil)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
@@ -739,10 +730,9 @@ func (s *scanState) commitFullBatch(ctx context.Context) error {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// updatePhase writes the scan's tail under one progress display: the
|
// updatePhase writes the scan's tail under one progress display: the
|
||||||
// final partial batch of hashed records, the new disk_size of every
|
// final partial batch of hashed records, and deletions for every record
|
||||||
// unchanged file whose disk_size changed, and deletions for every
|
// the scan did not verify (vanished files, plus paths that failed to
|
||||||
// record the scan did not verify (vanished files, plus paths that
|
// stat or hash).
|
||||||
// failed to stat or hash).
|
|
||||||
func (s *scanState) updatePhase(ctx context.Context) error {
|
func (s *scanState) updatePhase(ctx context.Context) error {
|
||||||
deletes := make([]string, 0, len(s.existing))
|
deletes := make([]string, 0, len(s.existing))
|
||||||
for path := range s.existing {
|
for path := range s.existing {
|
||||||
@@ -754,7 +744,7 @@ func (s *scanState) updatePhase(ctx context.Context) error {
|
|||||||
|
|
||||||
s.st.removed = len(deletes)
|
s.st.removed = len(deletes)
|
||||||
|
|
||||||
total := len(s.batch) + len(s.diskSizes) + len(deletes)
|
total := len(s.batch) + len(deletes)
|
||||||
prog := newProgress(progressParams{
|
prog := newProgress(progressParams{
|
||||||
label: scanPhase("update"), unit: scanUnit, total: int64(total),
|
label: scanPhase("update"), unit: scanUnit, total: int64(total),
|
||||||
stalledOn: "db",
|
stalledOn: "db",
|
||||||
@@ -769,11 +759,6 @@ func (s *scanState) updatePhase(ctx context.Context) error {
|
|||||||
|
|
||||||
s.batch = nil
|
s.batch = nil
|
||||||
|
|
||||||
err = applyDiskSizes(ctx, s.db, s.diskSizes, prog)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
return applyChanges(ctx, s.db, nil, deletes, prog)
|
return applyChanges(ctx, s.db, nil, deletes, prog)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
+23
-97
@@ -43,10 +43,7 @@ func writeFile(t *testing.T, dir, name string, data []byte) string {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// diskSizeOf returns the bytes the file at path takes on disk, from its
|
// diskSizeOf returns the bytes the file at path takes on disk, from its
|
||||||
// own lstat: st_blocks counts 512-byte units. ZFS counts a new file's
|
// own lstat: st_blocks counts 512-byte units.
|
||||||
// blocks only seconds after the write, and fsync does not hasten that.
|
|
||||||
// A test therefore calls unix.Sync, which waits for it, before the scan
|
|
||||||
// whose disk_size it compares with this.
|
|
||||||
func diskSizeOf(t *testing.T, path string) int64 {
|
func diskSizeOf(t *testing.T, path string) int64 {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
|
|
||||||
@@ -1082,9 +1079,8 @@ func TestScanOneFilesystemWalksNestedFileOperand(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// buildSmokeTree recreates the tree of the test under README
|
// buildSmokeTree recreates the README smoke-test filesystem layout
|
||||||
// "Definition of done" with deterministic content and returns the tree
|
// with deterministic content and returns the tree root.
|
||||||
// root.
|
|
||||||
func buildSmokeTree(t *testing.T) string {
|
func buildSmokeTree(t *testing.T) string {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
|
|
||||||
@@ -1436,111 +1432,41 @@ func TestScanStoresRawHashes(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// TestScanStoresDiskSize scans a sparse file, which takes less space on
|
// TestScanStoresDiskSize scans a sparse file, which takes less space on
|
||||||
// disk than its length, once through its directory and once as the
|
// disk than its length, and checks the disk_size stored for it against
|
||||||
// operand itself. Each scan stores the disk_size of the file's lstat.
|
// the file's own lstat.
|
||||||
func TestScanStoresDiskSize(t *testing.T) {
|
func TestScanStoresDiskSize(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
|
dir := t.TempDir()
|
||||||
|
db := openTestDB(t)
|
||||||
|
p := writeFile(t, dir, "sparse", []byte("x"))
|
||||||
|
|
||||||
|
// Truncate makes the file longer without giving it blocks for the
|
||||||
|
// new bytes.
|
||||||
const length = 1024 * 1024
|
const length = 1024 * 1024
|
||||||
|
|
||||||
p := sparseFile(t, t.TempDir(), "sparse", length)
|
err := os.Truncate(p, length)
|
||||||
pokeAt(t, p, 0, []byte("x"))
|
|
||||||
|
|
||||||
unix.Sync() // see diskSizeOf
|
|
||||||
|
|
||||||
want := diskSizeOf(t, p)
|
|
||||||
if want == 0 || want >= length {
|
|
||||||
t.Fatalf("the file takes %d bytes on disk, want more than none "+
|
|
||||||
"and fewer than its length %d", want, length)
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, operand := range []string{filepath.Dir(p), p} {
|
|
||||||
db := openTestDB(t)
|
|
||||||
syncTree(t, db, operand)
|
|
||||||
|
|
||||||
got := recordByPath(t, dbRecords(t, db), p).diskSize
|
|
||||||
if got != want {
|
|
||||||
t.Errorf("scan of %s: disk_size = %d, want %d",
|
|
||||||
operand, got, want)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestSyncScanRewriteUpdatesDiskSize writes into the hole of a scanned
|
|
||||||
// sparse file and moves its mtime on. The rescan hashes the file again
|
|
||||||
// and rewrites its record with the new disk_size.
|
|
||||||
func TestSyncScanRewriteUpdatesDiskSize(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
dir := t.TempDir()
|
|
||||||
db := openTestDB(t)
|
|
||||||
p := sparseFile(t, dir, "sparse", 1024*1024)
|
|
||||||
pokeAt(t, p, 0, []byte("x"))
|
|
||||||
|
|
||||||
syncTree(t, db, dir)
|
|
||||||
|
|
||||||
before := recordByPath(t, dbRecords(t, db), p).diskSize
|
|
||||||
|
|
||||||
pokeAt(t, p, 512*1024, pattern(7, 64*1024))
|
|
||||||
|
|
||||||
// The write may leave the mtime the scan recorded; a later one makes
|
|
||||||
// the rescan read the file again.
|
|
||||||
later := time.Now().Add(time.Hour)
|
|
||||||
|
|
||||||
err := os.Chtimes(p, later, later)
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
|
|
||||||
unix.Sync() // see diskSizeOf
|
|
||||||
|
|
||||||
want := diskSizeOf(t, p)
|
|
||||||
if want == before {
|
|
||||||
t.Fatalf("the file still takes %d bytes on disk after the write "+
|
|
||||||
"into its hole", want)
|
|
||||||
}
|
|
||||||
|
|
||||||
st := syncTree(t, db, dir)
|
|
||||||
if st != (scanStats{walked: 1, updated: 1}) {
|
|
||||||
t.Fatalf("rescan stats = %+v, want 1 updated", st)
|
|
||||||
}
|
|
||||||
|
|
||||||
got := recordByPath(t, dbRecords(t, db), p).diskSize
|
|
||||||
if got != want {
|
|
||||||
t.Errorf("disk_size = %d, want %d", got, want)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestSyncScanUnchangedUpdatesDiskSize gives a file's record a
|
|
||||||
// disk_size the file does not have, as when ZFS had not yet counted the
|
|
||||||
// file's blocks at the last scan. The rescan writes the file's own
|
|
||||||
// disk_size into the record without reading the file again.
|
|
||||||
func TestSyncScanUnchangedUpdatesDiskSize(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
dir := t.TempDir()
|
|
||||||
db := openTestDB(t)
|
|
||||||
p := writeFile(t, dir, "a.bin", pattern(1, 500))
|
|
||||||
|
|
||||||
syncTree(t, db, dir)
|
syncTree(t, db, dir)
|
||||||
|
|
||||||
// lstat counts 512-byte units, so no file takes 1 byte on disk.
|
want := diskSizeOf(t, p)
|
||||||
_, err := db.ExecContext(t.Context(),
|
if want >= length {
|
||||||
"UPDATE files SET disk_size = 1 WHERE path = ?", []byte(p))
|
t.Fatalf("the file takes %d bytes on disk, want fewer than its "+
|
||||||
|
"length %d", want, length)
|
||||||
|
}
|
||||||
|
|
||||||
|
var got int64
|
||||||
|
|
||||||
|
err = db.QueryRowContext(t.Context(),
|
||||||
|
"SELECT disk_size FROM files WHERE path = ?", []byte(p)).Scan(&got)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
|
|
||||||
unix.Sync() // see diskSizeOf
|
if got != want {
|
||||||
|
|
||||||
// A file read again would count as updated.
|
|
||||||
st := syncTree(t, db, dir)
|
|
||||||
if st != (scanStats{walked: 1, unchanged: 1}) {
|
|
||||||
t.Fatalf("rescan stats = %+v, want 1 unchanged", st)
|
|
||||||
}
|
|
||||||
|
|
||||||
got := recordByPath(t, dbRecords(t, db), p).diskSize
|
|
||||||
if want := diskSizeOf(t, p); got != want {
|
|
||||||
t.Errorf("disk_size = %d, want %d", got, want)
|
t.Errorf("disk_size = %d, want %d", got, want)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
+3
-3
@@ -15,9 +15,9 @@ import (
|
|||||||
const treesOpenLine = "[1/4] open: opening the database, elapsed 0s, " +
|
const treesOpenLine = "[1/4] open: opening the database, elapsed 0s, " +
|
||||||
"remain unknown"
|
"remain unknown"
|
||||||
|
|
||||||
// smokeTreeRecs mirrors the tree of the test under README "Definition
|
// smokeTreeRecs mirrors the README smoke-test tree layout: /d/t1 and
|
||||||
// of done": /d/t1 and /d/t2 are identical, /d/t3 differs from them only
|
// /d/t2 are identical, /d/t3 differs from them only by one filename.
|
||||||
// by one filename. Each file takes one 4 KiB block on disk.
|
// Each file takes one 4 KiB block on disk.
|
||||||
func smokeTreeRecs() []scanRec {
|
func smokeTreeRecs() []scanRec {
|
||||||
f1Head, f1Tail, f1Content := testHash("f1h"), testHash("f1t"), testHash("f1c")
|
f1Head, f1Tail, f1Content := testHash("f1h"), testHash("f1t"), testHash("f1c")
|
||||||
f2Head, f2Tail, f2Content := testHash("f2h"), testHash("f2t"), testHash("f2c")
|
f2Head, f2Tail, f2Content := testHash("f2h"), testHash("f2t"), testHash("f2c")
|
||||||
|
|||||||
Reference in New Issue
Block a user