Compare commits
1
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
6ccb9d7899 |
No files matched your search
@@ -274,18 +274,16 @@ All three subcommands operate on a single SQLite database file:
|
|||||||
by appending `.lock` to the database's path with symlinks resolved
|
by appending `.lock` to the database's path with symlinks resolved
|
||||||
(`/var/lib/sfdupes/db.sqlite.lock` by default), taken before it walks the
|
(`/var/lib/sfdupes/db.sqlite.lock` by default), taken before it walks the
|
||||||
filesystem or opens the database. Every path to one database, through a
|
filesystem or opens the database. Every path to one database, through a
|
||||||
symlink to the file or to a directory above it, names the same lock file. Just
|
symlink to the file or to a directory above it, names the same lock file.
|
||||||
before it takes the lock, `scan` creates the database file, empty, if it is
|
Before the database exists, only its directory's path is resolved. A second
|
||||||
missing (through a symlink at the database path, it creates the symlink's
|
`scan` against the same database does not wait: it fails at once with a
|
||||||
target), so the lock file is named the same way before the first scan sets up
|
one-line error naming the lock file and exits 1, without walking anything or
|
||||||
the database as after. A second `scan` against the same database does not
|
opening the database, and the running scan carries on. The lock file is
|
||||||
wait: it fails at once with a one-line error naming the lock file and exits 1,
|
created on first use, open to its owner only, and left in place: a leftover
|
||||||
without walking anything or opening the database, and the running scan carries
|
file blocks nothing, because the lock ends with the process holding it however
|
||||||
on. The lock file is created on first use, open to its owner only, and left in
|
it ends, a fatal error or an interrupt included, and deleting the file while a
|
||||||
place: a leftover file blocks nothing, because the lock ends with the process
|
scan runs would let a second scan start. `report` and `trees` never take the
|
||||||
holding it however it ends, a fatal error or an interrupt included, and
|
lock, so they run during a scan.
|
||||||
deleting the file while a scan runs would let a second scan start. `report`
|
|
||||||
and `trees` never take the lock, so they run during a scan.
|
|
||||||
- While `scan` runs, the database is in WAL journal mode with a busy timeout, so
|
- While `scan` runs, the database is in WAL journal mode with a busy timeout, so
|
||||||
running a report while a cron `scan` is in progress is safe. The filesystem is
|
running a report while a cron `scan` is in progress is safe. The filesystem is
|
||||||
authoritative; the database is an eventually-consistent reflection of it.
|
authoritative; the database is an eventually-consistent reflection of it.
|
||||||
|
|||||||
@@ -30,13 +30,8 @@
|
|||||||
# Completed Steps
|
# Completed Steps
|
||||||
|
|
||||||
- the scan lock file is named from the database's path with symlinks resolved,
|
- the scan lock file is named from the database's path with symlinks resolved,
|
||||||
after `scan` creates the database file if it is missing, so a scan through a
|
so a scan through a symlink to the database fails while another scan holds the
|
||||||
symlink to the database fails while another scan holds the lock (2026-10-14,
|
lock (2026-10-14, https://git.eeqj.de/sneak/sfdupes/issues/101)
|
||||||
https://git.eeqj.de/sneak/sfdupes/issues/101)
|
|
||||||
|
|
||||||
- on a terminal, a warning for the last file of a phase no longer erases the
|
|
||||||
phase's finished bar: the bar is drawn again below the warning (2026-10-14,
|
|
||||||
https://git.eeqj.de/sneak/sfdupes/issues/102)
|
|
||||||
|
|
||||||
- `scan` records mtime to the nanosecond, as whole seconds in `mtime` plus
|
- `scan` records mtime to the nanosecond, as whole seconds in `mtime` plus
|
||||||
`mtime_nsec`, and compares it at that resolution, so a same-size rewrite
|
`mtime_nsec`, and compares it at that resolution, so a same-size rewrite
|
||||||
|
|||||||
@@ -35,10 +35,6 @@ const schemaVersion = 1
|
|||||||
// scan.
|
// scan.
|
||||||
const dbDirPerm = 0o755
|
const dbDirPerm = 0o755
|
||||||
|
|
||||||
// dbFilePerm is the mode for a database file created by scan, the mode
|
|
||||||
// SQLite gives a database file it creates.
|
|
||||||
const dbFilePerm = 0o644
|
|
||||||
|
|
||||||
// lockFilePerm is the mode for the scan lock file. Anyone who can open
|
// lockFilePerm is the mode for the scan lock file. Anyone who can open
|
||||||
// the file can hold the lock and keep every scan from running, so it
|
// the file can hold the lock and keep every scan from running, so it
|
||||||
// is open to its owner only.
|
// is open to its owner only.
|
||||||
@@ -145,31 +141,28 @@ func openDB(path, params string) (*sql.DB, error) {
|
|||||||
// lockScanDatabase takes the lock that keeps a second scan off the
|
// lockScanDatabase takes the lock that keeps a second scan off the
|
||||||
// database at path: an exclusive flock(2) on the file beside it named
|
// database at path: an exclusive flock(2) on the file beside it named
|
||||||
// by the database's path with symlinks resolved and ".lock" appended,
|
// by the database's path with symlinks resolved and ".lock" appended,
|
||||||
// so every path to one database names the same lock file. If missing,
|
// so every path to one database names the same lock file. The lock
|
||||||
// the database's parent directory and an empty database file are
|
// file is created along with the database's parent directory if
|
||||||
// created first, so the lock file is named the same way before the
|
// missing. A lock held by another scan fails at once instead of
|
||||||
// database is set up as after. A lock held by another scan fails at
|
// waiting. The lock lasts until the returned file is closed or the
|
||||||
// once instead of waiting. The lock lasts until the returned file is
|
// process ends. The file is never deleted: a scan that deleted it
|
||||||
// closed or the process ends. The lock file is never deleted: a scan
|
// would let the next scan lock a new file while another still holds
|
||||||
// that deleted it would let the next scan lock a new file while
|
// the old one.
|
||||||
// another still holds the old one.
|
|
||||||
func lockScanDatabase(path string) (*os.File, error) {
|
func lockScanDatabase(path string) (*os.File, error) {
|
||||||
err := os.MkdirAll(filepath.Dir(path), dbDirPerm)
|
err := os.MkdirAll(filepath.Dir(path), dbDirPerm)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("create database directory: %w", err)
|
return nil, fmt.Errorf("create database directory: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Like SQLite, this follows a symlink at path and creates its
|
resolved, err := filepath.EvalSymlinks(path)
|
||||||
// target. SQLite opens an empty file as a new database.
|
if errors.Is(err, fs.ErrNotExist) {
|
||||||
//nolint:gosec // the operator chooses the database path
|
// No database yet: resolve its directory, which exists now.
|
||||||
dbFile, err := os.OpenFile(path, os.O_RDWR|os.O_CREATE, dbFilePerm)
|
var dir string
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("create database: %w", err)
|
dir, err = filepath.EvalSymlinks(filepath.Dir(path))
|
||||||
|
resolved = filepath.Join(dir, filepath.Base(path))
|
||||||
}
|
}
|
||||||
|
|
||||||
_ = dbFile.Close()
|
|
||||||
|
|
||||||
resolved, err := filepath.EvalSymlinks(path)
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("resolve database path: %w", err)
|
return nil, fmt.Errorf("resolve database path: %w", err)
|
||||||
}
|
}
|
||||||
|
|||||||
+12
-43
@@ -822,7 +822,7 @@ func holdScanLock(t *testing.T, path string) {
|
|||||||
|
|
||||||
func TestRunSecondScanFails(t *testing.T) {
|
func TestRunSecondScanFails(t *testing.T) {
|
||||||
// README §Database: while one scan holds the lock, a second scan
|
// README §Database: while one scan holds the lock, a second scan
|
||||||
// fails at once, naming the lock file, without opening the
|
// fails at once, naming the lock file, without creating the
|
||||||
// database.
|
// database.
|
||||||
path := testDBPath(t)
|
path := testDBPath(t)
|
||||||
t.Setenv(databaseEnv, path)
|
t.Setenv(databaseEnv, path)
|
||||||
@@ -848,25 +848,21 @@ func TestRunSecondScanFails(t *testing.T) {
|
|||||||
t.Errorf("stdout = %q, want nothing (data only)", got)
|
t.Errorf("stdout = %q, want nothing (data only)", got)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Taking the lock created the database file, empty; opening it as
|
_, err := os.Stat(path)
|
||||||
// a database would have written to it.
|
if !errors.Is(err, fs.ErrNotExist) {
|
||||||
info, err := os.Stat(path)
|
t.Errorf("stat %s = %v, want the database not created", path, err)
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
|
|
||||||
if info.Size() != 0 {
|
|
||||||
t.Errorf("database size = %d, want 0 (not opened)", info.Size())
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// assertSecondScanFails runs a scan against the database
|
// assertSecondScanFails holds the lock on the database at path and
|
||||||
// SFDUPES_DATABASE names while the test holds the lock on it. It fails
|
// runs a scan against the database SFDUPES_DATABASE names, another
|
||||||
// the test unless that scan fails at once, naming the lock file beside
|
// path to the same database. It fails the test unless that scan fails
|
||||||
// path, the database's real path.
|
// at once, naming the lock file beside path.
|
||||||
func assertSecondScanFails(t *testing.T, path string) {
|
func assertSecondScanFails(t *testing.T, path string) {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
|
|
||||||
|
holdScanLock(t, path)
|
||||||
|
|
||||||
var stderr bytes.Buffer
|
var stderr bytes.Buffer
|
||||||
|
|
||||||
code := run([]string{cmdScan, t.TempDir()}, io.Discard, &stderr)
|
code := run([]string{cmdScan, t.TempDir()}, io.Discard, &stderr)
|
||||||
@@ -883,9 +879,8 @@ func assertSecondScanFails(t *testing.T, path string) {
|
|||||||
|
|
||||||
func TestRunSecondScanThroughSymlinkFails(t *testing.T) {
|
func TestRunSecondScanThroughSymlinkFails(t *testing.T) {
|
||||||
// README §Database: every path to one database names the same lock
|
// README §Database: every path to one database names the same lock
|
||||||
// file, so while a scan holds it through one path, a scan through
|
// file, so while a scan holds it through the database's real path, a
|
||||||
// another fails, naming the lock file beside the database's real
|
// scan through a symlink fails, naming the real path's lock file.
|
||||||
// path.
|
|
||||||
t.Run("symlink to the database", func(t *testing.T) {
|
t.Run("symlink to the database", func(t *testing.T) {
|
||||||
path := testDBPath(t)
|
path := testDBPath(t)
|
||||||
t.Setenv(databaseEnv, path)
|
t.Setenv(databaseEnv, path)
|
||||||
@@ -899,7 +894,6 @@ func TestRunSecondScanThroughSymlinkFails(t *testing.T) {
|
|||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
|
|
||||||
holdScanLock(t, path)
|
|
||||||
t.Setenv(databaseEnv, link)
|
t.Setenv(databaseEnv, link)
|
||||||
assertSecondScanFails(t, path)
|
assertSecondScanFails(t, path)
|
||||||
})
|
})
|
||||||
@@ -913,34 +907,9 @@ func TestRunSecondScanThroughSymlinkFails(t *testing.T) {
|
|||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
|
|
||||||
holdScanLock(t, path)
|
|
||||||
t.Setenv(databaseEnv, filepath.Join(link, filepath.Base(path)))
|
t.Setenv(databaseEnv, filepath.Join(link, filepath.Base(path)))
|
||||||
assertSecondScanFails(t, path)
|
assertSecondScanFails(t, path)
|
||||||
})
|
})
|
||||||
|
|
||||||
t.Run("symlink whose target does not exist yet", func(t *testing.T) {
|
|
||||||
path := testDBPath(t)
|
|
||||||
link := filepath.Join(t.TempDir(), "link.sqlite")
|
|
||||||
|
|
||||||
err := os.Symlink(path, link)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
|
|
||||||
// The first scan takes the lock through the link before the
|
|
||||||
// database exists, then creates the database through it.
|
|
||||||
holdScanLock(t, link)
|
|
||||||
|
|
||||||
db, err := openScanDatabase(t.Context(), link)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
|
|
||||||
closeScanDatabase(t.Context(), db, link)
|
|
||||||
|
|
||||||
t.Setenv(databaseEnv, link)
|
|
||||||
assertSecondScanFails(t, path)
|
|
||||||
})
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestRunScanReleasesLock(t *testing.T) {
|
func TestRunScanReleasesLock(t *testing.T) {
|
||||||
|
|||||||
@@ -139,13 +139,6 @@ func (p *progress) warnf(format string, args ...any) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
fmt.Fprintln(os.Stderr, msg)
|
fmt.Fprintln(os.Stderr, msg)
|
||||||
|
|
||||||
// The library never draws a finished bar again, so the warning for
|
|
||||||
// a pass's last item would erase it for good: draw its last frame
|
|
||||||
// again below the warning.
|
|
||||||
if p.bar != nil && p.bar.IsFinished() {
|
|
||||||
fmt.Fprint(os.Stderr, p.bar.String())
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// finish terminates the pass's display. A bar whose pass stopped short
|
// finish terminates the pass's display. A bar whose pass stopped short
|
||||||
|
|||||||
@@ -187,25 +187,3 @@ func TestBarStoppedShortKeepsCount(t *testing.T) {
|
|||||||
t.Errorf("terminal shows %q, want a count of 2 of 10", shown)
|
t.Errorf("terminal shows %q, want a count of 2 of 10", shown)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// TestWarningAfterLastItemKeepsBar checks that a warning printed after
|
|
||||||
// a pass's last item, as for a failed read of the last file, leaves the
|
|
||||||
// pass's finished bar on the terminal below the warning.
|
|
||||||
//
|
|
||||||
//nolint:paralleltest // captureStderr replaces the process-wide os.Stderr
|
|
||||||
func TestWarningAfterLastItemKeepsBar(t *testing.T) {
|
|
||||||
stderr := captureStderr(t)
|
|
||||||
p := &progress{
|
|
||||||
label: "hash", total: 2, start: time.Now(),
|
|
||||||
bar: newBar("hash", 2),
|
|
||||||
}
|
|
||||||
|
|
||||||
p.increment()
|
|
||||||
p.increment()
|
|
||||||
p.warnf("warning")
|
|
||||||
p.finish()
|
|
||||||
|
|
||||||
if shown := lastFrame(stderr()); !strings.Contains(shown, "(2/2,") {
|
|
||||||
t.Errorf("terminal shows %q, want a count of 2 of 2", shown)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
Reference in new issue
Block a user