Implement persistent SQLite scan database
All checks were successful
check / check (push) Successful in 3s

scan now synchronizes a database that survives between runs
(SFDUPES_DATABASE, default /var/lib/sfdupes/db.sqlite) instead of
emitting a stream: operands are resolved to absolute paths, unchanged
files (same size, mtime not newer than recorded) are never re-read, new
and changed files are hashed, and records under the scanned operands
that were not verified this run are deleted; records outside the
operands are untouched. All changes commit in a single transaction, and
WAL journaling with a busy timeout keeps a report run during a cron
scan safe.

report and trees read the database (no positional arguments); the
NUL-terminated stream format, its parser, and the malformed-record
handling are gone. The driver is modernc.org/sqlite (pure Go), so
builds keep cgo disabled.
This commit is contained in:
2026-07-24 03:08:49 +07:00
parent e0d578a707
commit d8fbcb32c2
12 changed files with 1229 additions and 379 deletions

180
db_test.go Normal file
View File

@@ -0,0 +1,180 @@
package main
import (
"context"
"database/sql"
"errors"
"path/filepath"
"slices"
"strings"
"testing"
)
// testDBPath returns a database path inside a fresh temp dir.
func testDBPath(t *testing.T) string {
t.Helper()
return filepath.Join(t.TempDir(), "db.sqlite")
}
// openTestDB creates a fresh scan database in a temp dir.
func openTestDB(t *testing.T) *sql.DB {
t.Helper()
db, err := openScanDatabase(testDBPath(t))
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() { _ = db.Close() })
return db
}
func TestDatabasePath(t *testing.T) {
t.Setenv(databaseEnv, "")
if got := databasePath(); got != defaultDatabasePath {
t.Errorf("databasePath() = %q, want %q", got, defaultDatabasePath)
}
t.Setenv(databaseEnv, "/custom/place.sqlite")
if got := databasePath(); got != "/custom/place.sqlite" {
t.Errorf("databasePath() = %q, want the env override", got)
}
}
func TestOpenScanDatabaseCreates(t *testing.T) {
t.Parallel()
// The parent directory does not exist yet; scan must create it.
path := filepath.Join(t.TempDir(), "nested", "dir", "db.sqlite")
db, err := openScanDatabase(path)
if err != nil {
t.Fatalf("openScanDatabase: %v", err)
}
v, err := userVersion(db)
if err != nil || v != schemaVersion {
t.Fatalf("userVersion = %d, %v; want %d, nil", v, err, schemaVersion)
}
_ = db.Close()
// Reopening an existing database must succeed and find the schema.
db, err = openScanDatabase(path)
if err != nil {
t.Fatalf("reopen: %v", err)
}
defer func() { _ = db.Close() }()
recs, err := loadFileRows(db)
if err != nil || len(recs) != 0 {
t.Fatalf("loadFileRows = %v, %v; want empty, nil", recs, err)
}
}
func TestOpenReportDatabaseMissing(t *testing.T) {
t.Parallel()
_, err := openReportDatabase(testDBPath(t))
if !errors.Is(err, errNoDatabase) {
t.Fatalf("err = %v, want errNoDatabase", err)
}
}
func TestOpenReportDatabaseVersionMismatch(t *testing.T) {
t.Parallel()
path := testDBPath(t)
db, err := openScanDatabase(path)
if err != nil {
t.Fatal(err)
}
_, err = db.ExecContext(context.Background(), "PRAGMA user_version = 99")
if err != nil {
t.Fatal(err)
}
_ = db.Close()
_, err = openReportDatabase(path)
if !errors.Is(err, errSchemaVersion) {
t.Fatalf("err = %v, want errSchemaVersion", err)
}
}
func TestOpenReportDatabaseOK(t *testing.T) {
t.Parallel()
path := testDBPath(t)
db, err := openScanDatabase(path)
if err != nil {
t.Fatal(err)
}
_ = db.Close()
db, err = openReportDatabase(path)
if err != nil {
t.Fatalf("openReportDatabase: %v", err)
}
_ = db.Close()
}
func TestApplyChangesRoundTrip(t *testing.T) {
t.Parallel()
db := openTestDB(t)
// Paths may contain tabs and newlines; the database must store
// them byte-exactly.
recs := []scanRec{
{size: 2, mtime: 20, head: "h2", tail: "t2", path: "/a/tab\tnew\nline"},
{size: 1, mtime: 10, head: "h1", tail: "t1", path: "/a/x"},
}
err := applyChanges(db, recs, nil, newProgress("update", 2))
if err != nil {
t.Fatalf("applyChanges: %v", err)
}
got, err := loadFileRows(db)
if err != nil {
t.Fatal(err)
}
slices.SortFunc(got, func(a, b scanRec) int {
return strings.Compare(a.path, b.path)
})
if !slices.Equal(got, recs) {
t.Fatalf("rows = %+v, want %+v", got, recs)
}
// An upsert for an existing path updates in place; a delete
// removes exactly its path.
upd := scanRec{size: 3, mtime: 30, head: "h3", tail: "t3", path: "/a/x"}
err = applyChanges(db, []scanRec{upd},
[]string{"/a/tab\tnew\nline"}, newProgress("update", 2))
if err != nil {
t.Fatalf("applyChanges: %v", err)
}
got, err = loadFileRows(db)
if err != nil {
t.Fatal(err)
}
if len(got) != 1 || got[0] != upd {
t.Fatalf("rows = %+v, want just %+v", got, upd)
}
}