Compare commits
3
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
6d1cf717f2 | ||
|
|
7ff0dbb6e1 | ||
|
|
bccc14ffef |
@@ -329,7 +329,12 @@ All three subcommands operate on a single SQLite database file:
|
||||
`-shm` files beside it, which SQLite creates with the database
|
||||
file's permissions.
|
||||
- Schema (`PRAGMA user_version` is the schema version, currently 1; a
|
||||
database with any other version is a fatal error):
|
||||
database with any other version is a fatal error. `scan` creates
|
||||
the schema and sets the version in one transaction, so a first scan
|
||||
stopped while doing so leaves an empty database the next scan sets
|
||||
up. A database at version 0 that already has a `files` table was
|
||||
therefore not made by sfdupes; every subcommand refuses it with an
|
||||
error telling the user to remove the file and rescan):
|
||||
|
||||
```sql
|
||||
CREATE TABLE files (
|
||||
|
||||
@@ -33,6 +33,13 @@
|
||||
`walkCancelInFlightDirs` to `walkCancelInFlightFiles` (2026-10-04,
|
||||
https://git.eeqj.de/sneak/sfdupes/issues/33)
|
||||
|
||||
- test the `-x` filesystem-boundary rules in `subdirJob` (2026-10-04,
|
||||
https://git.eeqj.de/sneak/sfdupes/issues/17)
|
||||
|
||||
- `scan` creates the schema in one transaction; a version-0 database with a
|
||||
`files` table is refused with a clear schema-version error (2026-10-04,
|
||||
https://git.eeqj.de/sneak/sfdupes/issues/11)
|
||||
|
||||
- README documents install, Docker, a daily cron scan and how to read
|
||||
and check the reports (2026-10-04,
|
||||
https://git.eeqj.de/sneak/sfdupes/issues/54)
|
||||
|
||||
+16
-14
@@ -26,12 +26,13 @@ import (
|
||||
// and scan_test.go is already over 1,600 lines. That is the deliberate
|
||||
// exception the convention otherwise expects to be stated.
|
||||
|
||||
// poolUnwind bounds how long a goroutine is given to leave a pool after
|
||||
// its context is cancelled. The tests that use it turn a pool that
|
||||
// ignored its cancellation — and so parks forever — into a failed
|
||||
// assertion within this bound instead of a hang. Not every test in this
|
||||
// file has that property: a few catch their regression only as the test
|
||||
// binary's own timeout, and each of those says so.
|
||||
// poolUnwind bounds how long a test waits for a cancellation to take
|
||||
// effect: for a goroutine to return or a channel to close once its
|
||||
// context is cancelled, or for a signal to cancel the scan's context.
|
||||
// Only a failing run waits this long, and the bound is what makes that
|
||||
// failure an assertion instead of a hang. A call made without it, as
|
||||
// most of this file's scans are, has no bound: a regression that parks
|
||||
// it is caught only as the test binary's own timeout.
|
||||
const poolUnwind = 2 * time.Second
|
||||
|
||||
// walkClock is a context whose cancellation is driven by the scan's
|
||||
@@ -702,14 +703,15 @@ func TestFeedHashJobsClosesJobsWhenCancelled(t *testing.T) {
|
||||
// so the pool tears down. The queued run names a file that does not
|
||||
// exist, so a worker that hashed it anyway would produce a result.
|
||||
//
|
||||
// hashWorker has a second cancellation exit: the send of a completed
|
||||
// result on the results channel. That branch is reachable from the
|
||||
// production scan path, not dead code — pool.stop() cancels the context
|
||||
// before it starts draining results, so a worker parked on that send
|
||||
// leaves through this case. It is exercised by
|
||||
// TestScanHashWriteFailureUnwindsPool, which strands every worker on a
|
||||
// full results channel until stop() unwinds the pool. So both of
|
||||
// hashWorker's cancellation branches have a test.
|
||||
// hashWorker's other cancellation exit, abandoning the send of a
|
||||
// result, is reachable from the scan: stop cancels the pool before it
|
||||
// drains results, so a worker waiting on that send can leave through
|
||||
// it. TestScanHashWriteFailureUnwindsPool reaches it in some runs only,
|
||||
// depending on whether the workers have filled the results channel when
|
||||
// stop runs, and no test fails without it, since stop's drain frees a
|
||||
// waiting worker anyway. This test, for its part, catches a removed
|
||||
// drop check in some runs only: a worker that hashes the run anyway
|
||||
// then picks at random between sending the result and leaving.
|
||||
func TestHashWorkerDropsQueuedRuns(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
|
||||
@@ -219,6 +219,12 @@ func openReportDatabase(ctx context.Context,
|
||||
}
|
||||
|
||||
v, err := userVersion(ctx, db)
|
||||
if err == nil && v == 0 {
|
||||
// An empty database passes this check and fails the version
|
||||
// check below.
|
||||
err = checkUnversioned(ctx, db)
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
_ = db.Close()
|
||||
|
||||
@@ -245,6 +251,11 @@ func initSchema(ctx context.Context, db *sql.DB) error {
|
||||
|
||||
switch v {
|
||||
case 0:
|
||||
err = checkUnversioned(ctx, db)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return createSchema(ctx, db)
|
||||
case schemaVersion:
|
||||
return nil
|
||||
@@ -254,25 +265,65 @@ func initSchema(ctx context.Context, db *sql.DB) error {
|
||||
}
|
||||
}
|
||||
|
||||
// checkUnversioned checks a database at user_version 0 before it is
|
||||
// taken for an empty one. createSchema creates the files table and
|
||||
// sets the version together, so a files table at version 0 was made by
|
||||
// something else. Adopting it could corrupt unrelated data, so that is
|
||||
// a schema-version error telling the operator to remove the file and
|
||||
// rescan.
|
||||
func checkUnversioned(ctx context.Context, db *sql.DB) error {
|
||||
var name string
|
||||
|
||||
err := db.QueryRowContext(ctx,
|
||||
"SELECT name FROM sqlite_master "+
|
||||
"WHERE type = 'table' AND name = 'files'").Scan(&name)
|
||||
|
||||
switch {
|
||||
case err == nil:
|
||||
return fmt.Errorf(
|
||||
"has a files table but no schema version; "+
|
||||
"remove the file and rescan: %w", errSchemaVersion)
|
||||
case errors.Is(err, sql.ErrNoRows):
|
||||
return nil
|
||||
default:
|
||||
return fmt.Errorf("check for files table: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
// createSchema applies the schema to a fresh database and stamps the
|
||||
// schema version.
|
||||
// schema version in one transaction, so a creation stopped partway, by
|
||||
// an interrupt or an error, leaves an empty database the next scan
|
||||
// sets up, never a files table at version 0, which checkUnversioned
|
||||
// refuses.
|
||||
func createSchema(ctx context.Context, db *sql.DB) error {
|
||||
_, err := db.ExecContext(ctx, createTableSQL)
|
||||
tx, err := db.BeginTx(ctx, nil)
|
||||
if err != nil {
|
||||
return fmt.Errorf("create schema: %w", err)
|
||||
}
|
||||
|
||||
_, err = db.ExecContext(ctx, createIndexSQL)
|
||||
defer func() { _ = tx.Rollback() }()
|
||||
|
||||
_, err = tx.ExecContext(ctx, createTableSQL)
|
||||
if err != nil {
|
||||
return fmt.Errorf("create schema: %w", err)
|
||||
}
|
||||
|
||||
_, err = db.ExecContext(ctx,
|
||||
_, err = tx.ExecContext(ctx, createIndexSQL)
|
||||
if err != nil {
|
||||
return fmt.Errorf("create schema: %w", err)
|
||||
}
|
||||
|
||||
_, err = tx.ExecContext(ctx,
|
||||
"PRAGMA user_version = "+strconv.Itoa(schemaVersion))
|
||||
if err != nil {
|
||||
return fmt.Errorf("set schema version: %w", err)
|
||||
}
|
||||
|
||||
err = tx.Commit()
|
||||
if err != nil {
|
||||
return fmt.Errorf("create schema: %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
|
||||
+71
@@ -8,6 +8,7 @@ import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"slices"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
@@ -77,6 +78,76 @@ func TestOpenScanDatabaseCreates(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestOpenDatabaseUnversionedForeign(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
// A database that has a files table but user_version 0, written by
|
||||
// some other tool. report, trees and scan must refuse it with the
|
||||
// schema-version error, not adopt it and not emit a raw SQLite
|
||||
// "table files already exists".
|
||||
path := testDBPath(t)
|
||||
|
||||
db, err := sql.Open("sqlite", path)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
_, err = db.ExecContext(t.Context(), "CREATE TABLE files (x INTEGER)")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
_ = db.Close()
|
||||
|
||||
_, err = openReportDatabase(t.Context(), path)
|
||||
if !errors.Is(err, errSchemaVersion) ||
|
||||
!strings.Contains(err.Error(), "remove the file and rescan") {
|
||||
t.Fatalf("report: err = %v, want errSchemaVersion telling the "+
|
||||
"operator to remove the file and rescan", err)
|
||||
}
|
||||
|
||||
_, err = openScanDatabase(t.Context(), path)
|
||||
if !errors.Is(err, errSchemaVersion) ||
|
||||
!strings.Contains(err.Error(), "remove the file and rescan") {
|
||||
t.Fatalf("scan: err = %v, want errSchemaVersion telling the "+
|
||||
"operator to remove the file and rescan", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSchemaCreationStoppedPartway(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
// A first scan stopped while creating the schema must leave a
|
||||
// database the next scan accepts. max_page_count(2) leaves room for
|
||||
// the files table but not its index, so schema creation fails right
|
||||
// after CREATE TABLE, a point an interrupt could also stop it at.
|
||||
path := testDBPath(t)
|
||||
|
||||
db, err := openDB(path, scanParams+"&_pragma=max_page_count(2)")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
err = initSchema(t.Context(), db)
|
||||
_ = db.Close()
|
||||
|
||||
if err == nil {
|
||||
t.Fatal("initSchema with no room for the index succeeded")
|
||||
}
|
||||
|
||||
db, err = openScanDatabase(t.Context(), path)
|
||||
if err != nil {
|
||||
t.Fatalf("next scan: %v", err)
|
||||
}
|
||||
|
||||
defer func() { _ = db.Close() }()
|
||||
|
||||
v, err := userVersion(t.Context(), db)
|
||||
if err != nil || v != schemaVersion {
|
||||
t.Fatalf("userVersion = %d, %v; want %d, nil", v, err, schemaVersion)
|
||||
}
|
||||
}
|
||||
|
||||
func TestOpenReportDatabaseMissing(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
|
||||
+155
@@ -6,8 +6,10 @@ import (
|
||||
"crypto/sha256"
|
||||
"database/sql"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/fs"
|
||||
"os"
|
||||
"os/signal"
|
||||
"path/filepath"
|
||||
@@ -912,6 +914,159 @@ func TestDeviceOfInfo(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// dirEntryFor returns the fs.DirEntry for name within dir, obtained via
|
||||
// the same os.ReadDir the walk uses, so it carries a real Info().
|
||||
func dirEntryFor(t *testing.T, dir, name string) fs.DirEntry {
|
||||
t.Helper()
|
||||
|
||||
entries, err := os.ReadDir(dir)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
for _, e := range entries {
|
||||
if e.Name() == name {
|
||||
return e
|
||||
}
|
||||
}
|
||||
|
||||
t.Fatalf("entry %q not found in %q", name, dir)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// callSubdirJob runs subdirJob against e under parent, collecting any
|
||||
// warning events it emits (subdirJob emits at most one).
|
||||
func callSubdirJob(t *testing.T, p string, e fs.DirEntry,
|
||||
parent dirJob, oneFS bool,
|
||||
) (dirJob, bool, []walkEvent) {
|
||||
t.Helper()
|
||||
|
||||
events := make(chan walkEvent, 1)
|
||||
job, ok := subdirJob(t.Context(), p, e, parent, oneFS, events)
|
||||
close(events)
|
||||
|
||||
var evs []walkEvent
|
||||
for ev := range events {
|
||||
evs = append(evs, ev)
|
||||
}
|
||||
|
||||
return job, ok, evs
|
||||
}
|
||||
|
||||
// TestSubdirJobOneFilesystem exercises the -x boundary check in
|
||||
// subdirJob directly, so no second real filesystem is needed. The
|
||||
// subdirectory's real device is compared against a fabricated operand
|
||||
// device.
|
||||
func TestSubdirJobOneFilesystem(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
dir := t.TempDir()
|
||||
|
||||
sub := filepath.Join(dir, "sub")
|
||||
|
||||
err := os.Mkdir(sub, 0o750)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
info, err := os.Lstat(sub)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
dev, ok := deviceOfInfo(info)
|
||||
if !ok {
|
||||
t.Skip("platform exposes no device id")
|
||||
}
|
||||
|
||||
// A device the subdirectory is not on, standing in for an operand
|
||||
// rooted on a different filesystem.
|
||||
otherDev := dev + 1
|
||||
e := dirEntryFor(t, dir, "sub")
|
||||
|
||||
cases := []struct {
|
||||
name string
|
||||
oneFS bool
|
||||
parent dirJob
|
||||
wantOK bool
|
||||
}{
|
||||
// -x on, subdirectory on a different device than its operand:
|
||||
// descent is refused.
|
||||
{"reject across boundary", true,
|
||||
dirJob{rootDev: otherDev, rootDevOK: true}, false},
|
||||
// -x on but the operand's own device is unknown: the boundary
|
||||
// check is bypassed and descent proceeds.
|
||||
{"bypass when root device unknown", true,
|
||||
dirJob{rootDev: otherDev, rootDevOK: false}, true},
|
||||
// Default (no -x): boundaries are crossed even onto a different
|
||||
// device.
|
||||
{"cross by default", false,
|
||||
dirJob{rootDev: otherDev, rootDevOK: true}, true},
|
||||
}
|
||||
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
job, ok, evs := callSubdirJob(t, sub, e, tc.parent, tc.oneFS)
|
||||
if ok != tc.wantOK {
|
||||
t.Fatalf("accepted = %v, want %v", ok, tc.wantOK)
|
||||
}
|
||||
|
||||
if len(evs) != 0 {
|
||||
t.Fatalf("unexpected events: %+v", evs)
|
||||
}
|
||||
|
||||
// The accepted job must carry the operand's device down, or -x
|
||||
// stops checking below the first level.
|
||||
want := dirJob{
|
||||
path: sub,
|
||||
rootDev: tc.parent.rootDev,
|
||||
rootDevOK: tc.parent.rootDevOK,
|
||||
}
|
||||
if ok && job != want {
|
||||
t.Fatalf("job = %+v, want %+v", job, want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// errInfoUnavailable is returned by errDirEntry.Info().
|
||||
var errInfoUnavailable = errors.New("info unavailable")
|
||||
|
||||
// errDirEntry is a directory entry whose Info() always fails, driving
|
||||
// subdirJob's stat-error branch deterministically.
|
||||
type errDirEntry struct{ name string }
|
||||
|
||||
func (e errDirEntry) Name() string { return e.name }
|
||||
func (errDirEntry) IsDir() bool { return true }
|
||||
func (errDirEntry) Type() fs.FileMode {
|
||||
return fs.ModeDir
|
||||
}
|
||||
|
||||
func (errDirEntry) Info() (fs.FileInfo, error) {
|
||||
return nil, errInfoUnavailable
|
||||
}
|
||||
|
||||
// TestSubdirJobStatError asserts that when a subdirectory's Info()
|
||||
// fails under -x, subdirJob warns and refuses descent.
|
||||
func TestSubdirJobStatError(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
p := "/does/not/matter/sub"
|
||||
|
||||
_, ok, evs := callSubdirJob(t, p, errDirEntry{name: "sub"},
|
||||
dirJob{rootDev: 1, rootDevOK: true}, true)
|
||||
if ok {
|
||||
t.Fatal("descent accepted after stat error, want refused")
|
||||
}
|
||||
|
||||
if len(evs) != 1 || !evs[0].fail || !strings.Contains(evs[0].warn, p) {
|
||||
t.Fatalf("want one warning naming the path, got %+v", evs)
|
||||
}
|
||||
}
|
||||
|
||||
// buildSmokeTree recreates the README smoke-test filesystem layout
|
||||
// with deterministic content and returns the tree root.
|
||||
func buildSmokeTree(t *testing.T) string {
|
||||
|
||||
Reference in New Issue
Block a user