Compare commits
1
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
dfbfdd0601 |
@@ -146,44 +146,33 @@ var errHashSize = errors.New("hash is not 32 bytes")
|
||||
|
||||
// databasePath resolves the database location: SFDUPES_DATABASE when
|
||||
// set and non-empty, the compiled-in default otherwise. It returns the
|
||||
// path, made absolute by absolutePath, and where the path came from,
|
||||
// "default" or "from SFDUPES_DATABASE", for the line each subcommand
|
||||
// prints first on stderr.
|
||||
// path, with the working directory, its symlinks resolved, put in front
|
||||
// of it when it is relative, and where the path came from, "default" or
|
||||
// "from SFDUPES_DATABASE", for the line each subcommand prints first on
|
||||
// stderr.
|
||||
func databasePath() (string, string, error) {
|
||||
path, source := defaultDatabasePath, "default"
|
||||
if p := os.Getenv(databaseEnv); p != "" {
|
||||
path, source = p, "from "+databaseEnv
|
||||
}
|
||||
|
||||
abs, err := absolutePath(path)
|
||||
if filepath.IsAbs(path) {
|
||||
return path, source, nil
|
||||
}
|
||||
|
||||
// The working directory as the kernel reports it, with no symlink
|
||||
// in it, as README "Database" says. os.Getwd may return PWD
|
||||
// instead, which can name it through a symlink.
|
||||
wd, err := unix.Getwd()
|
||||
if err != nil {
|
||||
return "", "", fmt.Errorf("resolve database path %s: %w", path, err)
|
||||
}
|
||||
|
||||
return abs, source, nil
|
||||
}
|
||||
|
||||
// absolutePath returns path unchanged when it is absolute, and
|
||||
// otherwise the working directory, a "/" and path. The working
|
||||
// directory is the one the kernel reports, with no symlink in it;
|
||||
// os.Getwd may return PWD instead, which can name it through a symlink.
|
||||
// The result is not cleaned, as filepath.Abs and filepath.Join would
|
||||
// clean it. Cleaning drops each ".." together with the directory name
|
||||
// before it. When that name is a symlink, the cleaned path names
|
||||
// another file than the one the system reaches by following the symlink
|
||||
// first.
|
||||
func absolutePath(path string) (string, error) {
|
||||
if filepath.IsAbs(path) {
|
||||
return path, nil
|
||||
}
|
||||
|
||||
wd, err := unix.Getwd()
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
// Not filepath.Abs or filepath.Join: they clean the path, which
|
||||
// drops each ".." together with the directory name before it. When
|
||||
// that directory is a symlink, the cleaned path names another file.
|
||||
// The working directory ends in "/" only when it is the root.
|
||||
return strings.TrimSuffix(wd, "/") + "/" + path, nil
|
||||
return strings.TrimSuffix(wd, "/") + "/" + path, source, nil
|
||||
}
|
||||
|
||||
// createDatabaseDirectory creates the directory the database at path
|
||||
@@ -740,15 +729,15 @@ func readDatabaseCounts(ctx context.Context,
|
||||
return c, nil
|
||||
}
|
||||
|
||||
// loadFileMeta streams every record's path, size, disk_size and mtime
|
||||
// to fn. Scan change detection needs no hash values, and skipping the
|
||||
// hash columns keeps the scan's in-memory index small on
|
||||
// multi-million-file databases.
|
||||
// loadFileMeta streams every record's path, size and mtime to fn. Scan
|
||||
// change detection needs no hash values, and skipping the hash columns
|
||||
// keeps the scan's in-memory index small on multi-million-file
|
||||
// databases.
|
||||
func loadFileMeta(ctx context.Context, db *sql.DB,
|
||||
fn func(path string, m fileMeta),
|
||||
fn func(path string, size int64, mtime time.Time),
|
||||
) error {
|
||||
rows, err := db.QueryContext(ctx,
|
||||
"SELECT path, size, disk_size, mtime, mtime_nsec FROM files")
|
||||
"SELECT path, size, mtime, mtime_nsec FROM files")
|
||||
if err != nil {
|
||||
return fmt.Errorf("read records: %w", err)
|
||||
}
|
||||
@@ -757,19 +746,16 @@ func loadFileMeta(ctx context.Context, db *sql.DB,
|
||||
|
||||
for rows.Next() {
|
||||
var (
|
||||
path []byte
|
||||
m fileMeta
|
||||
sec, nsec int64
|
||||
path []byte
|
||||
size, sec, nsec int64
|
||||
)
|
||||
|
||||
err = rows.Scan(&path, &m.size, &m.diskSize, &sec, &nsec)
|
||||
err = rows.Scan(&path, &size, &sec, &nsec)
|
||||
if err != nil {
|
||||
return fmt.Errorf("read record: %w", err)
|
||||
}
|
||||
|
||||
m.mtime = time.Unix(sec, nsec)
|
||||
|
||||
fn(string(path), m)
|
||||
fn(string(path), size, time.Unix(sec, nsec))
|
||||
}
|
||||
|
||||
err = rows.Err()
|
||||
@@ -787,21 +773,21 @@ func loadFileMeta(ctx context.Context, db *sql.DB,
|
||||
// WAL small and let concurrent reports observe progress.
|
||||
const updateBatchSize = 10000
|
||||
|
||||
// applyChanges writes upserts for new and changed files and deletes for
|
||||
// vanished ones, in batched transactions. Progress is rendered on prog
|
||||
// (one increment per change).
|
||||
// applyChanges writes one scan's database changes — upserts for new and
|
||||
// changed files, deletes for vanished ones — in batched transactions.
|
||||
// Progress is rendered on prog (one increment per change).
|
||||
func applyChanges(ctx context.Context, db *sql.DB, upserts []scanRec,
|
||||
deletes []string, prog *progress,
|
||||
) error {
|
||||
for batch := range slices.Chunk(upserts, updateBatchSize) {
|
||||
err := applyBatch(ctx, db, batch, nil, nil, prog)
|
||||
err := applyBatch(ctx, db, batch, nil, prog)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
for batch := range slices.Chunk(deletes, updateBatchSize) {
|
||||
err := applyBatch(ctx, db, nil, nil, batch, prog)
|
||||
err := applyBatch(ctx, db, nil, batch, prog)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -810,26 +796,10 @@ func applyChanges(ctx context.Context, db *sql.DB, upserts []scanRec,
|
||||
return nil
|
||||
}
|
||||
|
||||
// applyDiskSizes writes the disk_size of each of recs into its record,
|
||||
// in batched transactions, and leaves the rest of the record alone.
|
||||
// Progress is rendered on prog (one increment per record).
|
||||
func applyDiskSizes(ctx context.Context, db *sql.DB, recs []fileRec,
|
||||
prog *progress,
|
||||
) error {
|
||||
for batch := range slices.Chunk(recs, updateBatchSize) {
|
||||
err := applyBatch(ctx, db, nil, batch, nil, prog)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// applyBatch commits one batch of upserts, disk_size updates and
|
||||
// deletes in a single transaction.
|
||||
// applyBatch commits one batch of upserts and deletes in a single
|
||||
// transaction.
|
||||
func applyBatch(ctx context.Context, db *sql.DB, upserts []scanRec,
|
||||
diskSizes []fileRec, deletes []string, prog *progress,
|
||||
deletes []string, prog *progress,
|
||||
) error {
|
||||
tx, err := db.BeginTx(ctx, nil)
|
||||
if err != nil {
|
||||
@@ -843,11 +813,6 @@ func applyBatch(ctx context.Context, db *sql.DB, upserts []scanRec,
|
||||
return err
|
||||
}
|
||||
|
||||
err = execDiskSizes(ctx, tx, diskSizes, prog)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
err = execDeletes(ctx, tx, deletes, prog)
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -886,31 +851,6 @@ func execUpserts(ctx context.Context, tx *sql.Tx, upserts []scanRec,
|
||||
return nil
|
||||
}
|
||||
|
||||
// execDiskSizes sets the disk_size of the record of each of recs. It
|
||||
// does not touch size, so the totals triggers do not fire.
|
||||
func execDiskSizes(ctx context.Context, tx *sql.Tx, recs []fileRec,
|
||||
prog *progress,
|
||||
) error {
|
||||
st, err := tx.PrepareContext(ctx,
|
||||
"UPDATE files SET disk_size = ? WHERE path = ?")
|
||||
if err != nil {
|
||||
return fmt.Errorf("prepare disk_size update: %w", err)
|
||||
}
|
||||
|
||||
defer func() { _ = st.Close() }()
|
||||
|
||||
for _, r := range recs {
|
||||
_, err = st.ExecContext(ctx, r.diskSize, []byte(r.path))
|
||||
if err != nil {
|
||||
return fmt.Errorf("update disk_size of %s: %w", r.path, err)
|
||||
}
|
||||
|
||||
prog.increment()
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// execDeletes removes the records for paths no longer present.
|
||||
func execDeletes(ctx context.Context, tx *sql.Tx, deletes []string,
|
||||
prog *progress,
|
||||
|
||||
+1
-3
@@ -17,9 +17,7 @@ import (
|
||||
func TestDefaultExcludes(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
// README "Rules for the walk": the Linux and macOS lists, and none
|
||||
// on any other system. The Windows list exists in the code only: the
|
||||
// binary does not build for Windows, so the README leaves it out.
|
||||
// README "Rules for the walk": each system's list.
|
||||
cases := map[string][]string{
|
||||
goosLinux: {"/proc", "/sys", "/dev", "/run", "/tmp", "/var/tmp"},
|
||||
goosDarwin: {
|
||||
|
||||
+75
-285
@@ -14,8 +14,6 @@ import (
|
||||
"strconv"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"golang.org/x/sys/unix"
|
||||
)
|
||||
|
||||
// usageMarker is the first line of cobra's usage text, which a usage
|
||||
@@ -396,7 +394,7 @@ func TestRunNamesRelativeDatabase(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestRunUsesDatabaseThroughSymlinkedDirectory(t *testing.T) {
|
||||
// README §Settings: the path names the file exactly. With link a
|
||||
// README §Database: the path names the file exactly. With link a
|
||||
// symlink to the directory real/sub, link/../db.sqlite names
|
||||
// real/db.sqlite, not db.sqlite beside link. Every subcommand must
|
||||
// use that file, relative or absolute, and its first line must name
|
||||
@@ -651,7 +649,7 @@ func TestRunUsageErrors(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestRunScanRejectsWorkersBelowOne(t *testing.T) {
|
||||
// README §Settings: --workers below 1 is a usage error reported in
|
||||
// README §scan mode: --workers below 1 is a usage error reported in
|
||||
// one line on stderr, before the scan opens the database.
|
||||
for _, workers := range []string{"0", "-1"} {
|
||||
t.Run(workers, func(t *testing.T) {
|
||||
@@ -689,8 +687,8 @@ func TestRunScanRejectsWorkersBelowOne(t *testing.T) {
|
||||
func TestRunHelp(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
// README §Settings: help goes to stderr, exits 0, and leaves stdout
|
||||
// empty.
|
||||
// README §Subcommands: help goes to stderr, exits 0, and leaves
|
||||
// stdout empty.
|
||||
cases := [][]string{{"--help"}, {"-h"}, {cmdScan, "--help"}}
|
||||
|
||||
for _, args := range cases {
|
||||
@@ -716,8 +714,8 @@ func TestRunHelp(t *testing.T) {
|
||||
func TestRunVersion(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
// README §Settings: the version is one line on stdout, with nothing
|
||||
// on stderr, and exits 0.
|
||||
// README §Subcommands: the version is one line on stdout, with
|
||||
// nothing on stderr, and exits 0.
|
||||
info, _ := debug.ReadBuildInfo()
|
||||
|
||||
for _, arg := range []string{"--version", "-v"} {
|
||||
@@ -776,8 +774,8 @@ func TestBuildVersion(t *testing.T) {
|
||||
func TestRunVersionWriteFailureIsFatal(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
// README §Error handling: a failed write to stdout is a fatal error,
|
||||
// exit 1, reported on stderr starting with "sfdupes:".
|
||||
// README §Error handling: a stdout write failure exits 1, reported
|
||||
// in one line on stderr.
|
||||
var stderr bytes.Buffer
|
||||
|
||||
code := run([]string{"--version"}, failingWriter{}, &stderr)
|
||||
@@ -820,23 +818,6 @@ func scanFixture(t *testing.T) []string {
|
||||
return dupes
|
||||
}
|
||||
|
||||
// storedDiskSize returns the disk_size of the record for path in the
|
||||
// database at dbPath. A test of a trees run compares with this rather
|
||||
// than with a new lstat, which on ZFS can differ from the scan's (see
|
||||
// diskSizeOf).
|
||||
func storedDiskSize(t *testing.T, dbPath, path string) int64 {
|
||||
t.Helper()
|
||||
|
||||
db, err := openReportDatabase(t.Context(), dbPath)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
defer func() { _ = db.Close() }()
|
||||
|
||||
return recordByPath(t, dbRecords(t, db), path).diskSize
|
||||
}
|
||||
|
||||
// scanOK runs scan over operands, fails the test unless it exits 0 with
|
||||
// nothing on stdout, and returns everything it printed to stderr.
|
||||
func scanOK(t *testing.T, operands ...string) string {
|
||||
@@ -897,262 +878,54 @@ func TestRunScanSucceedsDespiteWarnings(t *testing.T) {
|
||||
assertNoSidecars(t, path)
|
||||
}
|
||||
|
||||
// realTempDir returns a new temporary directory by its real path, so
|
||||
// tests hold where the temporary directory is reached through a
|
||||
// symlink.
|
||||
func realTempDir(t *testing.T) string {
|
||||
t.Helper()
|
||||
|
||||
dir, err := filepath.EvalSymlinks(t.TempDir())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
return dir
|
||||
}
|
||||
|
||||
// symlink creates link pointing at target.
|
||||
func symlink(t *testing.T, target, link string) {
|
||||
t.Helper()
|
||||
|
||||
err := os.Symlink(target, link)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
// assertSummary checks that stderr holds the scan summary line want.
|
||||
func assertSummary(t *testing.T, stderr, want string) {
|
||||
t.Helper()
|
||||
|
||||
if !strings.Contains(stderr, want) {
|
||||
t.Errorf("stderr = %q, want %q", stderr, want)
|
||||
}
|
||||
}
|
||||
|
||||
// TestRunScanFollowsSymlinkOperand scans link, a symlink to target.
|
||||
// The operand is scanned at its real path, and its file is recorded
|
||||
// there.
|
||||
func TestRunScanFollowsSymlinkOperand(t *testing.T) {
|
||||
func TestRunScanSkipsSymlinkOperand(t *testing.T) {
|
||||
path := testDBPath(t)
|
||||
t.Setenv(databaseEnv, path)
|
||||
|
||||
dir := realTempDir(t)
|
||||
f := writeFile(t, dir, "target/sub/f", pattern(1, 10))
|
||||
dir := t.TempDir()
|
||||
writeFile(t, dir, "target/sub/f", pattern(1, 10))
|
||||
|
||||
link := filepath.Join(dir, "link")
|
||||
symlink(t, filepath.Join(dir, "target"), link)
|
||||
|
||||
assertSummary(t, scanOK(t, link), "scan: 1 files seen (1 added, "+
|
||||
"0 updated, 0 removed, 0 unchanged), 0 skipped\n")
|
||||
|
||||
if got := storedPaths(t, path); !slices.Equal(got, []string{f}) {
|
||||
t.Errorf("records = %q, want %q", got, []string{f})
|
||||
err := os.Symlink(filepath.Join(dir, "target"), link)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// Scanning a directory through the symlink stores a record beneath
|
||||
// the symlink's own path for a file beneath its target.
|
||||
scanOK(t, filepath.Join(link, "sub"))
|
||||
|
||||
assertOperandSkipped(t, path, link, "symlink",
|
||||
filepath.Join(link, "sub", "f"))
|
||||
}
|
||||
|
||||
// TestRunScanWalksOperandUnderSymlinkOperand scans link and link/sub,
|
||||
// with link a symlink to target. Both resolve under target, so
|
||||
// target/sub is walked once, as part of target.
|
||||
func TestRunScanWalksOperandUnderSymlinkOperand(t *testing.T) {
|
||||
path := testDBPath(t)
|
||||
t.Setenv(databaseEnv, path)
|
||||
|
||||
dir := realTempDir(t)
|
||||
f := writeFile(t, dir, "target/sub/f", pattern(1, 10))
|
||||
dir := t.TempDir()
|
||||
writeFile(t, dir, "target/sub/f", pattern(1, 10))
|
||||
|
||||
link := filepath.Join(dir, "link")
|
||||
symlink(t, filepath.Join(dir, "target"), link)
|
||||
|
||||
assertSummary(t, scanOK(t, link, filepath.Join(link, "sub")),
|
||||
"scan: 1 files seen (1 added, 0 updated, 0 removed, "+
|
||||
"0 unchanged), 0 skipped\n")
|
||||
|
||||
if got := storedPaths(t, path); !slices.Equal(got, []string{f}) {
|
||||
t.Errorf("records = %q, want %q", got, []string{f})
|
||||
}
|
||||
}
|
||||
|
||||
// TestRunScanWalksSymlinkedOperandBelowOperand is the case of
|
||||
// https://git.eeqj.de/sneak/sfdupes/issues/144: scan dir dir/link/sub,
|
||||
// with dir/link a symlink to a directory outside dir. The walk of dir
|
||||
// does not follow link, so dir/link/sub is walked on its own at its
|
||||
// real path, and the record an earlier scan stored there survives.
|
||||
func TestRunScanWalksSymlinkedOperandBelowOperand(t *testing.T) {
|
||||
path := testDBPath(t)
|
||||
t.Setenv(databaseEnv, path)
|
||||
|
||||
dir := realTempDir(t)
|
||||
a := writeFile(t, dir, "a", pattern(1, 10))
|
||||
|
||||
outside := realTempDir(t)
|
||||
f := writeFile(t, outside, "sub/f", pattern(2, 10))
|
||||
|
||||
symlink(t, outside, filepath.Join(dir, "link"))
|
||||
|
||||
scanOK(t, filepath.Join(outside, "sub"))
|
||||
|
||||
stderr := scanOK(t, dir, filepath.Join(dir, "link", "sub"))
|
||||
assertSummary(t, stderr, "scan: 2 files seen (1 added, 0 updated, "+
|
||||
"0 removed, 1 unchanged), 0 skipped\n")
|
||||
|
||||
want := []string{a, f}
|
||||
slices.Sort(want)
|
||||
|
||||
if got := storedPaths(t, path); !slices.Equal(got, want) {
|
||||
t.Errorf("records = %q, want %q", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
// TestRunScanDropsSymlinkOperandInsideOperand scans dir and link, with
|
||||
// link, outside dir, a symlink to dir/real. link resolves inside dir,
|
||||
// so it is dropped as covered by dir, and its file is recorded once.
|
||||
func TestRunScanDropsSymlinkOperandInsideOperand(t *testing.T) {
|
||||
path := testDBPath(t)
|
||||
t.Setenv(databaseEnv, path)
|
||||
|
||||
dir := realTempDir(t)
|
||||
f := writeFile(t, dir, "real/f", pattern(1, 10))
|
||||
|
||||
link := filepath.Join(realTempDir(t), "link")
|
||||
symlink(t, filepath.Join(dir, "real"), link)
|
||||
|
||||
assertSummary(t, scanOK(t, dir, link), "scan: 1 files seen (1 added, "+
|
||||
"0 updated, 0 removed, 0 unchanged), 0 skipped\n")
|
||||
|
||||
if got := storedPaths(t, path); !slices.Equal(got, []string{f}) {
|
||||
t.Errorf("records = %q, want %q", got, []string{f})
|
||||
}
|
||||
}
|
||||
|
||||
// TestRunScanDanglingSymlinkOperandIsFatal scans a valid directory and
|
||||
// a symlink whose target does not exist. The scan fails with exit 1 and
|
||||
// an error naming the symlink as given, before it walks anything, so no
|
||||
// database is created.
|
||||
func TestRunScanDanglingSymlinkOperandIsFatal(t *testing.T) {
|
||||
path := testDBPath(t)
|
||||
t.Setenv(databaseEnv, path)
|
||||
|
||||
dir := realTempDir(t)
|
||||
writeFile(t, dir, "f", pattern(1, 10))
|
||||
|
||||
dangling := filepath.Join(realTempDir(t), "dangling")
|
||||
symlink(t, filepath.Join(dir, "missing"), dangling)
|
||||
|
||||
stdout := captureStdout(t)
|
||||
|
||||
var stderr bytes.Buffer
|
||||
|
||||
code := run([]string{cmdScan, dir, dangling}, os.Stdout, &stderr)
|
||||
if code != exitFatal {
|
||||
t.Errorf("run(scan %s %s) = %d, want %d", dir, dangling, code,
|
||||
exitFatal)
|
||||
}
|
||||
|
||||
assertFatalOutput(t, stderr.String(), stdout())
|
||||
|
||||
want := "sfdupes: resolve " + dangling + ": "
|
||||
if !strings.Contains(stderr.String(), want) {
|
||||
t.Errorf("stderr = %q, want %q", stderr.String(), want)
|
||||
}
|
||||
|
||||
_, err := os.Stat(path)
|
||||
if !errors.Is(err, fs.ErrNotExist) {
|
||||
t.Errorf("database: stat error %v, want it missing", err)
|
||||
}
|
||||
}
|
||||
|
||||
// TestRunScanFromSymlinkedWorkingDirectoryRecordsRealPath runs scan .
|
||||
// after a cd through home/proj, a symlink to data/proj. The file is
|
||||
// recorded under data/proj, its real path.
|
||||
func TestRunScanFromSymlinkedWorkingDirectoryRecordsRealPath(t *testing.T) {
|
||||
path := testDBPath(t)
|
||||
t.Setenv(databaseEnv, path)
|
||||
|
||||
dir := realTempDir(t)
|
||||
f := writeFile(t, dir, "data/proj/f", pattern(1, 10))
|
||||
|
||||
err := os.Mkdir(filepath.Join(dir, "home"), 0o750)
|
||||
err := os.Symlink(filepath.Join(dir, "target"), link)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
symlink(t, filepath.Join(dir, "data", "proj"),
|
||||
filepath.Join(dir, "home", "proj"))
|
||||
// link is dropped as a symlink, but link/sub must still be scanned,
|
||||
// not dropped as lying under link.
|
||||
scanOK(t, link, filepath.Join(link, "sub"))
|
||||
|
||||
// t.Chdir also sets PWD to the path through the symlink, as a
|
||||
// shell does after cd.
|
||||
t.Chdir(filepath.Join(dir, "home", "proj"))
|
||||
|
||||
scanOK(t, ".")
|
||||
|
||||
if got := storedPaths(t, path); !slices.Equal(got, []string{f}) {
|
||||
t.Errorf("records = %q, want %q", got, []string{f})
|
||||
}
|
||||
}
|
||||
|
||||
// TestRunScanFollowsSymlinkBeforeDotDot scans a path with a ".." after
|
||||
// home/proj, a symlink to data/proj: home/proj/.. from dir, and .. after
|
||||
// a cd through home/proj. As with realpath, the ".." counts from
|
||||
// data/proj, so data is scanned and home is not.
|
||||
func TestRunScanFollowsSymlinkBeforeDotDot(t *testing.T) {
|
||||
for _, tc := range []struct{ wd, operand string }{
|
||||
{"", "home/proj/.."},
|
||||
{"home/proj", ".."},
|
||||
} {
|
||||
t.Run(tc.operand, func(t *testing.T) {
|
||||
path := testDBPath(t)
|
||||
t.Setenv(databaseEnv, path)
|
||||
|
||||
dir := realTempDir(t)
|
||||
f := writeFile(t, dir, "data/proj/f", pattern(1, 10))
|
||||
g := writeFile(t, dir, "data/g", pattern(2, 10))
|
||||
writeFile(t, dir, "home/h", pattern(3, 10))
|
||||
symlink(t, filepath.Join(dir, "data", "proj"),
|
||||
filepath.Join(dir, "home", "proj"))
|
||||
|
||||
// t.Chdir also sets PWD to the path given, as a shell does
|
||||
// after cd.
|
||||
t.Chdir(filepath.Join(dir, tc.wd))
|
||||
|
||||
scanOK(t, tc.operand)
|
||||
|
||||
want := []string{f, g}
|
||||
slices.Sort(want)
|
||||
|
||||
if got := storedPaths(t, path); !slices.Equal(got, want) {
|
||||
t.Errorf("records = %q, want %q", got, want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestRunScanSkipsSymlinkToFIFOOperand scans link, a symlink to a FIFO.
|
||||
// The scan warns about the FIFO by its real path, counts it as skipped,
|
||||
// and exits 0.
|
||||
func TestRunScanSkipsSymlinkToFIFOOperand(t *testing.T) {
|
||||
t.Setenv(databaseEnv, testDBPath(t))
|
||||
|
||||
dir := realTempDir(t)
|
||||
fifo := filepath.Join(dir, "fifo")
|
||||
|
||||
err := unix.Mkfifo(fifo, 0o600)
|
||||
db, err := openDB(path, reportParams)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
link := filepath.Join(dir, "link")
|
||||
symlink(t, fifo, link)
|
||||
t.Cleanup(func() { _ = db.Close() })
|
||||
|
||||
stderr := scanOK(t, link)
|
||||
|
||||
warning := "walk " + fifo + ": skipping FIFO operand\n"
|
||||
if !strings.Contains(stderr, warning) {
|
||||
t.Errorf("stderr = %q, want %q", stderr, warning)
|
||||
}
|
||||
|
||||
assertSummary(t, stderr, "scan: 0 files seen (0 added, 0 updated, "+
|
||||
"0 removed, 0 unchanged), 1 skipped\n")
|
||||
recordByPath(t, dbRecords(t, db), filepath.Join(link, "sub", "f"))
|
||||
}
|
||||
|
||||
func TestRunScanSkipsZFSOperand(t *testing.T) {
|
||||
@@ -1251,19 +1024,23 @@ func TestRunScanSkipsOperandInsideZFS(t *testing.T) {
|
||||
|
||||
// TestRunScanSkipsOperandThroughSymlinkIntoZFS scans snaps/daily, where
|
||||
// snaps is a symlink to pool/.zfs/snapshot. The operand is warned about
|
||||
// by its real path, counted as skipped, and not walked.
|
||||
// by the path given, counted as skipped, and not walked.
|
||||
func TestRunScanSkipsOperandThroughSymlinkIntoZFS(t *testing.T) {
|
||||
t.Setenv(databaseEnv, testDBPath(t))
|
||||
|
||||
dir := realTempDir(t)
|
||||
dir := t.TempDir()
|
||||
writeFile(t, dir, "pool/.zfs/snapshot/daily/f", pattern(1, 10))
|
||||
|
||||
snaps := filepath.Join(dir, "snaps")
|
||||
symlink(t, filepath.Join(dir, "pool", ".zfs", "snapshot"), snaps)
|
||||
|
||||
stderr := scanOK(t, filepath.Join(snaps, "daily"))
|
||||
err := os.Symlink(filepath.Join(dir, "pool", ".zfs", "snapshot"), snaps)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
daily := filepath.Join(snaps, "daily")
|
||||
stderr := scanOK(t, daily)
|
||||
|
||||
daily := filepath.Join(dir, "pool", ".zfs", "snapshot", "daily")
|
||||
warning := "walk " + daily + ": skipping operand inside a .zfs directory\n"
|
||||
summary := "scan: 0 files seen (0 added, 0 updated, 0 removed, " +
|
||||
"0 unchanged), 1 skipped\n"
|
||||
@@ -1275,25 +1052,39 @@ func TestRunScanSkipsOperandThroughSymlinkIntoZFS(t *testing.T) {
|
||||
}
|
||||
|
||||
// TestRunScanWalksOperandThroughSymlink scans data/daily, where data is
|
||||
// a symlink to pool/data. The operand is scanned as usual, and its file
|
||||
// is recorded by its real path.
|
||||
// a symlink to a directory outside any .zfs directory. The operand is
|
||||
// scanned as usual, and its file is recorded by the path given.
|
||||
func TestRunScanWalksOperandThroughSymlink(t *testing.T) {
|
||||
path := testDBPath(t)
|
||||
t.Setenv(databaseEnv, path)
|
||||
|
||||
dir := realTempDir(t)
|
||||
f := writeFile(t, dir, "pool/data/daily/f", pattern(1, 10))
|
||||
dir := t.TempDir()
|
||||
writeFile(t, dir, "pool/data/daily/f", pattern(1, 10))
|
||||
|
||||
data := filepath.Join(dir, "data")
|
||||
symlink(t, filepath.Join(dir, "pool", "data"), data)
|
||||
|
||||
assertSummary(t, scanOK(t, filepath.Join(data, "daily")),
|
||||
"scan: 1 files seen (1 added, 0 updated, 0 removed, "+
|
||||
"0 unchanged), 0 skipped\n")
|
||||
|
||||
if got := storedPaths(t, path); !slices.Equal(got, []string{f}) {
|
||||
t.Errorf("records = %q, want %q", got, []string{f})
|
||||
err := os.Symlink(filepath.Join(dir, "pool", "data"), data)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
daily := filepath.Join(data, "daily")
|
||||
stderr := scanOK(t, daily)
|
||||
|
||||
summary := "scan: 1 files seen (1 added, 0 updated, 0 removed, " +
|
||||
"0 unchanged), 0 skipped\n"
|
||||
if !strings.Contains(stderr, summary) {
|
||||
t.Errorf("stderr = %q, want %q", stderr, summary)
|
||||
}
|
||||
|
||||
db, err := openDB(path, reportParams)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
t.Cleanup(func() { _ = db.Close() })
|
||||
|
||||
recordByPath(t, dbRecords(t, db), filepath.Join(daily, "f"))
|
||||
}
|
||||
|
||||
func TestRunReportSucceeds(t *testing.T) {
|
||||
@@ -1352,7 +1143,7 @@ func TestRunTreesSucceeds(t *testing.T) {
|
||||
// trees of each other.
|
||||
want := "first\tdupe\tfiles\tsize\tdisk_size\n" +
|
||||
filepath.Dir(dupes[0]) + "\t" + filepath.Dir(dupes[1]) + "\t1\t300\t" +
|
||||
strconv.FormatInt(storedDiskSize(t, path, dupes[1]), 10) + "\n"
|
||||
strconv.FormatInt(diskSizeOf(t, dupes[1]), 10) + "\n"
|
||||
if got := stdout.String(); got != want {
|
||||
t.Errorf("stdout = %q, want %q", got, want)
|
||||
}
|
||||
@@ -1388,8 +1179,8 @@ func TestRunReportsNeedOnlyReadAccess(t *testing.T) {
|
||||
dupes[0] + "\t" + dupes[1] + "\t300\n",
|
||||
cmdTrees: "first\tdupe\tfiles\tsize\tdisk_size\n" +
|
||||
filepath.Dir(dupes[0]) + "\t" + filepath.Dir(dupes[1]) +
|
||||
"\t1\t300\t" +
|
||||
strconv.FormatInt(storedDiskSize(t, path, dupes[1]), 10) + "\n",
|
||||
"\t1\t300\t" + strconv.FormatInt(diskSizeOf(t, dupes[1]), 10) +
|
||||
"\n",
|
||||
}
|
||||
|
||||
for name, want := range cases {
|
||||
@@ -1424,11 +1215,9 @@ func assertRunsUseDatabase(t *testing.T, dir, name string) {
|
||||
t.Errorf("report stdout = %q, want %q", got, want)
|
||||
}
|
||||
|
||||
diskSize := storedDiskSize(t, filepath.Join(dir, name), dupes[1])
|
||||
|
||||
want = "first\tdupe\tfiles\tsize\tdisk_size\n" +
|
||||
filepath.Dir(dupes[0]) + "\t" + filepath.Dir(dupes[1]) + "\t1\t300\t" +
|
||||
strconv.FormatInt(diskSize, 10) + "\n"
|
||||
strconv.FormatInt(diskSizeOf(t, dupes[1]), 10) + "\n"
|
||||
if got := runStdout(t, cmdTrees); got != want {
|
||||
t.Errorf("trees stdout = %q, want %q", got, want)
|
||||
}
|
||||
@@ -1449,7 +1238,7 @@ func assertRunsUseDatabase(t *testing.T, dir, name string) {
|
||||
}
|
||||
|
||||
func TestRunDatabasePathUsedAsGiven(t *testing.T) {
|
||||
// README §Settings: the path names the database file exactly. In
|
||||
// README §Database: the path names the database file exactly. In
|
||||
// SQLite's connection string an unescaped ? or # would end the file
|
||||
// name and % would start an escape, and a path starting with //
|
||||
// could be read as a host name. %25 is a valid escape, so unescaped
|
||||
@@ -1494,8 +1283,9 @@ func holdScanLock(t *testing.T, path string) {
|
||||
}
|
||||
|
||||
func TestRunSecondScanFails(t *testing.T) {
|
||||
// README §Database: a second scan that finds the lock held fails
|
||||
// at once, naming the lock file, without opening the database.
|
||||
// README §Database: while one scan holds the lock, a second scan
|
||||
// fails at once, naming the lock file, without opening the
|
||||
// database.
|
||||
path := testDBPath(t)
|
||||
t.Setenv(databaseEnv, path)
|
||||
|
||||
@@ -1554,7 +1344,7 @@ func assertSecondScanFails(t *testing.T, path string) {
|
||||
}
|
||||
|
||||
func TestRunSecondScanThroughSymlinkFails(t *testing.T) {
|
||||
// README §Database: a path through a symlink names the same lock
|
||||
// README §Database: every path to one database names the same lock
|
||||
// file, so while a scan holds it through one path, a scan through
|
||||
// another fails, naming the lock file beside the database's real
|
||||
// path.
|
||||
@@ -1659,8 +1449,8 @@ func TestRunReportsDuringScan(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestRunStdoutClosedIsFatal(t *testing.T) {
|
||||
// README §Error handling: a failed write to stdout is a fatal error,
|
||||
// exit 1, reported on stderr starting with "sfdupes:".
|
||||
// README §Error handling: a stdout write failure exits 1, reported
|
||||
// in one line on stderr.
|
||||
for _, name := range []string{cmdReport, cmdTrees, cmdInfo} {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
t.Setenv(databaseEnv, testDBPath(t))
|
||||
|
||||
@@ -66,12 +66,11 @@ type fileRec struct {
|
||||
}
|
||||
|
||||
// fileMeta is the in-memory index entry for one existing database
|
||||
// record: what the walk compares the file with. Hashes stay on disk; at
|
||||
// record: just enough for change detection. Hashes stay on disk; at
|
||||
// tens of millions of records they would dominate the scan's memory.
|
||||
type fileMeta struct {
|
||||
size int64
|
||||
diskSize int64
|
||||
mtime time.Time
|
||||
size int64
|
||||
mtime time.Time
|
||||
}
|
||||
|
||||
// scanUnit is what every scan phase counts, as its progress display
|
||||
@@ -209,30 +208,29 @@ func interrupted(walked int) error {
|
||||
return errInterrupted
|
||||
}
|
||||
|
||||
// resolveRoots converts each PATH operand to its real path: absolute,
|
||||
// with every symlink in it followed. Database records are keyed by
|
||||
// that path, so scan results depend neither on the working directory
|
||||
// nor on the symlinks an operand was given through. An operand that
|
||||
// cannot be resolved, because it or a symlink's target does not exist
|
||||
// or cannot be read, is a fatal error before any scanning.
|
||||
// resolveRoots converts each PATH operand to an absolute, lexically
|
||||
// cleaned path (symlinks are not resolved) and verifies that it
|
||||
// exists. Database records are keyed by absolute path, so scan results
|
||||
// must not depend on the working directory.
|
||||
func resolveRoots(roots []string) ([]string, error) {
|
||||
resolved := make([]string, 0, len(roots))
|
||||
abs := make([]string, 0, len(roots))
|
||||
|
||||
for _, root := range roots {
|
||||
a, err := absolutePath(root)
|
||||
a, err := filepath.Abs(root)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("resolve %s: %w", root, err)
|
||||
}
|
||||
|
||||
r, err := filepath.EvalSymlinks(a)
|
||||
// A nonexistent operand is a fatal error before any scanning.
|
||||
_, err = os.Lstat(a)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("resolve %s: %w", root, err)
|
||||
return nil, err
|
||||
}
|
||||
|
||||
resolved = append(resolved, r)
|
||||
abs = append(abs, a)
|
||||
}
|
||||
|
||||
return resolved, nil
|
||||
return abs, nil
|
||||
}
|
||||
|
||||
// pruneRoots drops operands already covered by another operand:
|
||||
@@ -343,12 +341,11 @@ type scanStats struct {
|
||||
// whatever remains after the walk and hash phases is deleted by the
|
||||
// update phase.
|
||||
type scanState struct {
|
||||
db *sql.DB
|
||||
existing map[string]fileMeta
|
||||
toHash []fileRec // new and changed files: must be read
|
||||
diskSizes []fileRec // unchanged files whose disk_size changed
|
||||
batch []scanRec
|
||||
st scanStats
|
||||
db *sql.DB
|
||||
existing map[string]fileMeta
|
||||
toHash []fileRec // new and changed files: must be read
|
||||
batch []scanRec
|
||||
st scanStats
|
||||
}
|
||||
|
||||
// syncScan synchronizes the database with the filesystem under roots;
|
||||
@@ -418,16 +415,14 @@ func (s *scanState) runPhases(ctx context.Context, roots []string,
|
||||
|
||||
// walkableRoots returns the operands the walk can start from: regular
|
||||
// files, and directories not named .zfs, none of them inside a
|
||||
// directory named .zfs. The operands are real paths from resolveRoots,
|
||||
// so these checks apply to what a symlink operand points at. Every
|
||||
// other operand is warned about, counted as skipped, and dropped. A
|
||||
// dropped operand is no longer a root, so the records stored beneath it
|
||||
// are treated as if it had not been given: those under no other root
|
||||
// count as outside the roots and are not deleted as unverified. An
|
||||
// operand on a filesystem in rules that is not persistent is dropped
|
||||
// the same way, with a line saying so that is not a warning, and is not
|
||||
// counted as skipped. An operand that fails lstat here is kept, and the
|
||||
// walk warns about it unless it no longer exists.
|
||||
// directory named .zfs. Every other operand is warned about, counted as
|
||||
// skipped, and dropped. A dropped operand is no longer a root, so the
|
||||
// records stored beneath it are treated as if it had not been given:
|
||||
// those under no other root count as outside the roots and are not
|
||||
// deleted as unverified. An operand on a filesystem in rules that is not
|
||||
// persistent is dropped the same way, with a line saying so that is not
|
||||
// a warning, and is not counted as skipped. An operand that fails lstat
|
||||
// here is kept, and the walk warns about it unless it no longer exists.
|
||||
func (s *scanState) walkableRoots(roots []string, rules walkRules) []string {
|
||||
kept := make([]string, 0, len(roots))
|
||||
|
||||
@@ -479,24 +474,24 @@ func (s *scanState) loadIndex(ctx context.Context, roots []string) error {
|
||||
|
||||
s.existing = make(map[string]fileMeta)
|
||||
|
||||
return loadFileMeta(ctx, s.db, func(path string, m fileMeta) {
|
||||
prog.increment()
|
||||
return loadFileMeta(ctx, s.db,
|
||||
func(path string, size int64, mtime time.Time) {
|
||||
prog.increment()
|
||||
|
||||
if underAnyRoot(path, roots) {
|
||||
s.existing[path] = m
|
||||
}
|
||||
})
|
||||
if underAnyRoot(path, roots) {
|
||||
s.existing[path] = fileMeta{size: size, mtime: mtime}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// walkPhase drains the walk and queues every new or changed file for
|
||||
// the hash phase. An unchanged file is not read, and its record counts
|
||||
// as verified. When its disk_size differs from the record's, it is
|
||||
// queued for the update phase, which writes the new disk_size. A record
|
||||
// at or below a path the walk left out, because -x stopped at it, it is
|
||||
// a default exclude, or it is not on a persistent filesystem, counts as
|
||||
// outside the roots: it is not deleted. That does not hold below a root
|
||||
// at or inside that path, since that root's own walk verifies the
|
||||
// records below it; nor is a line printed for such a root.
|
||||
// the hash phase. An unchanged file needs nothing more, and its record
|
||||
// counts as verified. A record at or below a path the walk left out,
|
||||
// because -x stopped at it, it is a default exclude, or it is not on a
|
||||
// persistent filesystem, counts as outside the roots: it is not
|
||||
// deleted. That does not hold below a root at or inside that path,
|
||||
// since that root's own walk verifies the records below it; nor is a
|
||||
// line printed for such a root.
|
||||
func (s *scanState) walkPhase(events <-chan walkEvent, roots []string) {
|
||||
// The records the last scan left under the roots estimate how many
|
||||
// files the walk will find. A first scan has none, and no estimate.
|
||||
@@ -554,13 +549,6 @@ func (s *scanState) walkPhase(events <-chan walkEvent, roots []string) {
|
||||
delete(s.existing, ev.rec.path)
|
||||
|
||||
s.st.unchanged++
|
||||
|
||||
// The space a file takes on disk can change while its size and
|
||||
// mtime stay the same. ZFS, for one, counts a new file's blocks
|
||||
// up to seconds after the write.
|
||||
if old.diskSize != ev.rec.diskSize {
|
||||
s.diskSizes = append(s.diskSizes, ev.rec)
|
||||
}
|
||||
}
|
||||
|
||||
prog.finish()
|
||||
@@ -731,7 +719,7 @@ func (s *scanState) commitFullBatch(ctx context.Context) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
err := applyBatch(ctx, s.db, s.batch, nil, nil, nil)
|
||||
err := applyBatch(ctx, s.db, s.batch, nil, nil)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -742,10 +730,9 @@ func (s *scanState) commitFullBatch(ctx context.Context) error {
|
||||
}
|
||||
|
||||
// updatePhase writes the scan's tail under one progress display: the
|
||||
// final partial batch of hashed records, the new disk_size of every
|
||||
// unchanged file whose disk_size changed, and deletions for every
|
||||
// record the scan did not verify (vanished files, plus paths that
|
||||
// failed to stat or hash).
|
||||
// final partial batch of hashed records, and deletions for every record
|
||||
// the scan did not verify (vanished files, plus paths that failed to
|
||||
// stat or hash).
|
||||
func (s *scanState) updatePhase(ctx context.Context) error {
|
||||
deletes := make([]string, 0, len(s.existing))
|
||||
for path := range s.existing {
|
||||
@@ -757,7 +744,7 @@ func (s *scanState) updatePhase(ctx context.Context) error {
|
||||
|
||||
s.st.removed = len(deletes)
|
||||
|
||||
total := len(s.batch) + len(s.diskSizes) + len(deletes)
|
||||
total := len(s.batch) + len(deletes)
|
||||
prog := newProgress(progressParams{
|
||||
label: scanPhase("update"), unit: scanUnit, total: int64(total),
|
||||
stalledOn: "db",
|
||||
@@ -772,11 +759,6 @@ func (s *scanState) updatePhase(ctx context.Context) error {
|
||||
|
||||
s.batch = nil
|
||||
|
||||
err = applyDiskSizes(ctx, s.db, s.diskSizes, prog)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return applyChanges(ctx, s.db, nil, deletes, prog)
|
||||
}
|
||||
|
||||
@@ -897,13 +879,19 @@ func sendEvent(ctx context.Context, events chan<- walkEvent,
|
||||
// operandWarning returns the one-line warning for an operand the walk
|
||||
// does not start from, naming the path and what it is, or "" for one it
|
||||
// does: a regular file, or a directory not named .zfs, that is not
|
||||
// inside a directory named .zfs. resolveRoots has already followed the
|
||||
// operand's symlinks, so a symlink here is an operand replaced by one
|
||||
// after it was resolved, and is not followed.
|
||||
// inside a directory named .zfs, neither as given nor with its symlinks
|
||||
// resolved. Symlinks are never followed, including as operands.
|
||||
func operandWarning(root string, fi fs.FileInfo) string {
|
||||
// The walk never enters a .zfs directory, so it starts from nothing
|
||||
// inside one either.
|
||||
if slices.Contains(strings.Split(filepath.Dir(root), "/"), ".zfs") {
|
||||
// inside one either, not even through a symlink. An operand whose
|
||||
// symlinks cannot be resolved is checked as given.
|
||||
resolved, err := filepath.EvalSymlinks(root)
|
||||
if err != nil {
|
||||
resolved = root
|
||||
}
|
||||
|
||||
if slices.Contains(strings.Split(filepath.Dir(root), "/"), ".zfs") ||
|
||||
slices.Contains(strings.Split(filepath.Dir(resolved), "/"), ".zfs") {
|
||||
return fmt.Sprintf("walk %s: skipping operand inside a .zfs directory",
|
||||
root)
|
||||
}
|
||||
|
||||
+52
-182
@@ -43,16 +43,7 @@ func writeFile(t *testing.T, dir, name string, data []byte) string {
|
||||
}
|
||||
|
||||
// diskSizeOf returns the bytes the file at path takes on disk, from its
|
||||
// own lstat: st_blocks counts 512-byte units. ZFS counts a write's
|
||||
// blocks only when it commits the write, up to seconds later, and one
|
||||
// commit can take the file through several growing sizes. So this value
|
||||
// can change at any moment. Neither fsync nor unix.Sync hastens that,
|
||||
// but a unix.Sync call often takes long enough that the blocks are
|
||||
// counted when it returns. A test that needs the value to change waits
|
||||
// for the change. A test that compares it with a scan's disk_size reads
|
||||
// it before and after the scan. The rewrite test rescans until the two
|
||||
// reads match, then compares exactly. The other two tests fail only on
|
||||
// a disk_size outside the two reads.
|
||||
// own lstat: st_blocks counts 512-byte units.
|
||||
func diskSizeOf(t *testing.T, path string) int64 {
|
||||
t.Helper()
|
||||
|
||||
@@ -1088,9 +1079,8 @@ func TestScanOneFilesystemWalksNestedFileOperand(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// buildSmokeTree recreates the tree of the test under README
|
||||
// "Definition of done" with deterministic content and returns the tree
|
||||
// root.
|
||||
// buildSmokeTree recreates the README smoke-test filesystem layout
|
||||
// with deterministic content and returns the tree root.
|
||||
func buildSmokeTree(t *testing.T) string {
|
||||
t.Helper()
|
||||
|
||||
@@ -1442,155 +1432,42 @@ func TestScanStoresRawHashes(t *testing.T) {
|
||||
}
|
||||
|
||||
// TestScanStoresDiskSize scans a sparse file, which takes less space on
|
||||
// disk than its length, once through its directory and once as the
|
||||
// operand itself. Each scan stores the disk_size of the file's lstat.
|
||||
// disk than its length, and checks the disk_size stored for it against
|
||||
// the file's own lstat.
|
||||
func TestScanStoresDiskSize(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
dir := t.TempDir()
|
||||
db := openTestDB(t)
|
||||
p := writeFile(t, dir, "sparse", []byte("x"))
|
||||
|
||||
// Truncate makes the file longer without giving it blocks for the
|
||||
// new bytes.
|
||||
const length = 1024 * 1024
|
||||
|
||||
p := sparseFile(t, t.TempDir(), "sparse", length)
|
||||
pokeAt(t, p, 0, []byte("x"))
|
||||
|
||||
unix.Sync() // see diskSizeOf
|
||||
|
||||
size := diskSizeOf(t, p)
|
||||
if size == 0 || size >= length {
|
||||
t.Fatalf("the file takes %d bytes on disk, want more than none "+
|
||||
"and fewer than its length %d", size, length)
|
||||
}
|
||||
|
||||
for _, operand := range []string{filepath.Dir(p), p} {
|
||||
db := openTestDB(t)
|
||||
|
||||
// ZFS may count the blocks during the scan; see diskSizeOf.
|
||||
before := diskSizeOf(t, p)
|
||||
|
||||
syncTree(t, db, operand)
|
||||
|
||||
after := diskSizeOf(t, p)
|
||||
|
||||
got := recordByPath(t, dbRecords(t, db), p).diskSize
|
||||
if got < before || got > after {
|
||||
t.Errorf("scan of %s: disk_size = %d, want %d to %d",
|
||||
operand, got, before, after)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestSyncScanRewriteUpdatesDiskSize writes into the hole of a scanned
|
||||
// sparse file and moves its mtime on. The rescan hashes the file again
|
||||
// and rewrites its record with the new disk_size.
|
||||
func TestSyncScanRewriteUpdatesDiskSize(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
dir := t.TempDir()
|
||||
db := openTestDB(t)
|
||||
p := sparseFile(t, dir, "sparse", 1024*1024)
|
||||
pokeAt(t, p, 0, []byte("x"))
|
||||
|
||||
syncTree(t, db, dir)
|
||||
|
||||
recorded := recordByPath(t, dbRecords(t, db), p).diskSize
|
||||
|
||||
pokeAt(t, p, 512*1024, pattern(7, 64*1024))
|
||||
|
||||
// The write may leave the mtime the scan recorded; a later one makes
|
||||
// the rescan read the file again.
|
||||
later := time.Now().Add(time.Hour)
|
||||
|
||||
err := os.Chtimes(p, later, later)
|
||||
err := os.Truncate(p, length)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// Wait for ZFS to count the write; see diskSizeOf.
|
||||
deadline := time.Now().Add(30 * time.Second)
|
||||
for diskSizeOf(t, p) == recorded {
|
||||
if time.Now().After(deadline) {
|
||||
t.Fatalf("the file still takes %d bytes on disk 30 seconds "+
|
||||
"after the write into its hole", recorded)
|
||||
}
|
||||
|
||||
unix.Sync()
|
||||
}
|
||||
|
||||
// ZFS may still be counting, so the size may change during a rescan.
|
||||
// Only a rescan with the same size on both sides of it can be compared
|
||||
// exactly. A rescan that keeps the old size still fails, because the
|
||||
// wait made before differ from it.
|
||||
for {
|
||||
before := diskSizeOf(t, p)
|
||||
|
||||
st := syncTree(t, db, dir)
|
||||
|
||||
after := diskSizeOf(t, p)
|
||||
|
||||
if st != (scanStats{walked: 1, updated: 1}) {
|
||||
t.Fatalf("rescan stats = %+v, want 1 updated", st)
|
||||
}
|
||||
|
||||
if before == after {
|
||||
got := recordByPath(t, dbRecords(t, db), p).diskSize
|
||||
if got != after {
|
||||
t.Errorf("disk_size = %d, want %d", got, after)
|
||||
}
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
if time.Now().After(deadline) {
|
||||
t.Fatalf("the size on disk was still changing 30 seconds after "+
|
||||
"the write into its hole: %d, then %d", before, after)
|
||||
}
|
||||
|
||||
// A later mtime makes the next rescan read the file again.
|
||||
later = later.Add(time.Second)
|
||||
|
||||
err = os.Chtimes(p, later, later)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestSyncScanUnchangedUpdatesDiskSize gives a file's record a
|
||||
// disk_size the file does not have, as when ZFS had not yet counted the
|
||||
// file's blocks at the last scan. The rescan writes the file's own
|
||||
// disk_size into the record without reading the file again.
|
||||
func TestSyncScanUnchangedUpdatesDiskSize(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
dir := t.TempDir()
|
||||
db := openTestDB(t)
|
||||
p := writeFile(t, dir, "a.bin", pattern(1, 500))
|
||||
|
||||
syncTree(t, db, dir)
|
||||
|
||||
// lstat counts 512-byte units, so no file takes 1 byte on disk.
|
||||
_, err := db.ExecContext(t.Context(),
|
||||
"UPDATE files SET disk_size = 1 WHERE path = ?", []byte(p))
|
||||
want := diskSizeOf(t, p)
|
||||
if want >= length {
|
||||
t.Fatalf("the file takes %d bytes on disk, want fewer than its "+
|
||||
"length %d", want, length)
|
||||
}
|
||||
|
||||
var got int64
|
||||
|
||||
err = db.QueryRowContext(t.Context(),
|
||||
"SELECT disk_size FROM files WHERE path = ?", []byte(p)).Scan(&got)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
unix.Sync() // see diskSizeOf
|
||||
|
||||
// ZFS may count the blocks during the rescan; see diskSizeOf.
|
||||
before := diskSizeOf(t, p)
|
||||
|
||||
// A file read again would count as updated.
|
||||
st := syncTree(t, db, dir)
|
||||
|
||||
after := diskSizeOf(t, p)
|
||||
|
||||
if st != (scanStats{walked: 1, unchanged: 1}) {
|
||||
t.Fatalf("rescan stats = %+v, want 1 unchanged", st)
|
||||
}
|
||||
|
||||
got := recordByPath(t, dbRecords(t, db), p).diskSize
|
||||
if got < before || got > after {
|
||||
t.Errorf("disk_size = %d, want %d to %d", got, before, after)
|
||||
if got != want {
|
||||
t.Errorf("disk_size = %d, want %d", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2310,24 +2187,33 @@ func TestUnderRoot(t *testing.T) {
|
||||
}
|
||||
|
||||
// TestRunScanPassesOverDatabase scans a tree that holds the scan's own
|
||||
// database, and in one case also a hard link to the database file. The
|
||||
// walk knows the database's files by device ID and inode number, so it
|
||||
// passes over the database file, its -wal and -shm files, its .lock
|
||||
// file and the hard link with no line and no count. The record an
|
||||
// earlier scan left for the .lock file is deleted. A rescan of the
|
||||
// unchanged tree updates nothing, and report lists none of the
|
||||
// database's files.
|
||||
// database, reached directly and through a symlinked directory. The
|
||||
// walk passes over the database file, its -wal and -shm files and its
|
||||
// .lock file with no line and no count, and the record an earlier scan
|
||||
// left for the .lock file is deleted. A rescan of the unchanged tree
|
||||
// updates nothing, and report lists none of the database's files.
|
||||
func TestRunScanPassesOverDatabase(t *testing.T) {
|
||||
for _, viaHardLink := range []bool{false, true} {
|
||||
t.Run(fmt.Sprintf("via hard link %v", viaHardLink), func(t *testing.T) {
|
||||
tree := realTempDir(t)
|
||||
hardLink := filepath.Join(tree, "hardlink")
|
||||
for _, viaSymlink := range []bool{false, true} {
|
||||
t.Run(fmt.Sprintf("via symlink %v", viaSymlink), func(t *testing.T) {
|
||||
base := t.TempDir()
|
||||
tree := filepath.Join(base, "real", "tree")
|
||||
operand := tree
|
||||
|
||||
writeFile(t, tree, "a", pattern(1, 100))
|
||||
writeFile(t, tree, "b", pattern(1, 100))
|
||||
writeFile(t, tree, "empty1", nil)
|
||||
writeFile(t, tree, "empty2", nil)
|
||||
|
||||
if viaSymlink {
|
||||
err := os.Symlink(filepath.Join(base, "real"),
|
||||
filepath.Join(base, "link"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
operand = filepath.Join(base, "link", "tree")
|
||||
}
|
||||
|
||||
dbPath := filepath.Join(tree, "db.sqlite")
|
||||
t.Setenv(databaseEnv, dbPath)
|
||||
|
||||
@@ -2337,7 +2223,7 @@ func TestRunScanPassesOverDatabase(t *testing.T) {
|
||||
}
|
||||
|
||||
err = applyChanges(t.Context(), db, []scanRec{
|
||||
{path: filepath.Join(tree, "db.sqlite.lock")},
|
||||
{path: filepath.Join(operand, "db.sqlite.lock")},
|
||||
}, nil, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
@@ -2345,29 +2231,16 @@ func TestRunScanPassesOverDatabase(t *testing.T) {
|
||||
|
||||
closeScanDatabase(t.Context(), db, dbPath)
|
||||
|
||||
if viaHardLink {
|
||||
err = os.Link(dbPath, hardLink)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
stderr := assertScanSummary(t,
|
||||
assertScanSummary(t,
|
||||
"4 files seen (4 added, 0 updated, 1 removed, 0 unchanged)",
|
||||
tree)
|
||||
stderr += assertScanSummary(t,
|
||||
operand)
|
||||
assertScanSummary(t,
|
||||
"4 files seen (0 added, 0 updated, 0 removed, 4 unchanged)",
|
||||
tree)
|
||||
|
||||
if strings.Contains(stderr, hardLink) {
|
||||
t.Errorf("scan stderr = %q, want no line naming %s",
|
||||
stderr, hardLink)
|
||||
}
|
||||
operand)
|
||||
|
||||
report := runStdout(t, cmdReport)
|
||||
if strings.Contains(report, "db.sqlite") ||
|
||||
strings.Contains(report, hardLink) ||
|
||||
!strings.Contains(report, filepath.Join(tree, "empty2")) {
|
||||
!strings.Contains(report, filepath.Join(operand, "empty2")) {
|
||||
t.Errorf("report = %q, want the empty files and none of "+
|
||||
"the database's files", report)
|
||||
}
|
||||
@@ -2392,9 +2265,8 @@ func TestRunScanPassesOverDatabaseOperands(t *testing.T) {
|
||||
|
||||
// assertScanSummary scans operands and fails the test unless the
|
||||
// summary line gives counts with nothing skipped, and the line naming
|
||||
// the database is the only one that names one of its files. It returns
|
||||
// everything the scan printed to stderr.
|
||||
func assertScanSummary(t *testing.T, counts string, operands ...string) string {
|
||||
// the database is the only one that names one of its files.
|
||||
func assertScanSummary(t *testing.T, counts string, operands ...string) {
|
||||
t.Helper()
|
||||
|
||||
stderr := scanOK(t, operands...)
|
||||
@@ -2408,6 +2280,4 @@ func assertScanSummary(t *testing.T, counts string, operands ...string) string {
|
||||
t.Errorf("scan %q: stderr = %q, want only the line naming the "+
|
||||
"database to name one of its files", operands, stderr)
|
||||
}
|
||||
|
||||
return stderr
|
||||
}
|
||||
|
||||
+3
-3
@@ -15,9 +15,9 @@ import (
|
||||
const treesOpenLine = "[1/4] open: opening the database, elapsed 0s, " +
|
||||
"remain unknown"
|
||||
|
||||
// smokeTreeRecs mirrors the tree of the test under README "Definition
|
||||
// of done": /d/t1 and /d/t2 are identical, /d/t3 differs from them only
|
||||
// by one filename. Each file takes one 4 KiB block on disk.
|
||||
// smokeTreeRecs mirrors the README smoke-test tree layout: /d/t1 and
|
||||
// /d/t2 are identical, /d/t3 differs from them only by one filename.
|
||||
// Each file takes one 4 KiB block on disk.
|
||||
func smokeTreeRecs() []scanRec {
|
||||
f1Head, f1Tail, f1Content := testHash("f1h"), testHash("f1t"), testHash("f1c")
|
||||
f2Head, f2Tail, f2Content := testHash("f2h"), testHash("f2t"), testHash("f2c")
|
||||
|
||||
Reference in New Issue
Block a user