Implement persistent SQLite scan database
All checks were successful
check / check (push) Successful in 3s

scan now synchronizes a database that survives between runs
(SFDUPES_DATABASE, default /var/lib/sfdupes/db.sqlite) instead of
emitting a stream: operands are resolved to absolute paths, unchanged
files (same size, mtime not newer than recorded) are never re-read, new
and changed files are hashed, and records under the scanned operands
that were not verified this run are deleted; records outside the
operands are untouched. All changes commit in a single transaction, and
WAL journaling with a busy timeout keeps a report run during a cron
scan safe.

report and trees read the database (no positional arguments); the
NUL-terminated stream format, its parser, and the malformed-record
handling are gone. The driver is modernc.org/sqlite (pure Go), so
builds keep cgo disabled.
This commit is contained in:
2026-07-24 03:08:49 +07:00
parent e0d578a707
commit d8fbcb32c2
12 changed files with 1229 additions and 379 deletions

View File

@@ -1,92 +1,10 @@
package main
import (
"bufio"
"fmt"
"slices"
"strings"
"testing"
)
// mkRecord serializes one scan record in the on-the-wire format.
func mkRecord(size, mtime int64, head, tail, path string) string {
return fmt.Sprintf("%d\t%d\t%s\t%s\t%s\x00",
size, mtime, head, tail, path)
}
func TestSplitNULScanner(t *testing.T) {
t.Parallel()
sc := bufio.NewScanner(strings.NewReader("a\x00bb\x00\x00tail"))
sc.Split(splitNUL)
var got []string
for sc.Scan() {
got = append(got, sc.Text())
}
err := sc.Err()
if err != nil {
t.Fatalf("scanner error: %v", err)
}
want := []string{"a", "bb", "", "tail"}
if !slices.Equal(got, want) {
t.Fatalf("tokens = %q, want %q", got, want)
}
}
func TestParseScanStream(t *testing.T) {
t.Parallel()
stream := mkRecord(10, 1, "h1", "t1", "/a/x") +
"garbage-without-tabs\x00" +
"notanumber\t1\th\tt\t/a/bad\x00" +
mkRecord(20, 2, "h2", "t2", "/a/tab\tin\tname") +
"30\t3\th3\tt3\t/trailing/no-nul"
recs, malformed := parseScanStream(strings.NewReader(stream), "test")
if malformed != 2 {
t.Errorf("malformed = %d, want 2", malformed)
}
want := []scanRec{
{size: 10, head: "h1", tail: "t1", path: "/a/x"},
{size: 20, head: "h2", tail: "t2", path: "/a/tab\tin\tname"},
{size: 30, head: "h3", tail: "t3", path: "/trailing/no-nul"},
}
if !slices.Equal(recs, want) {
t.Fatalf("recs = %+v, want %+v", recs, want)
}
}
func TestParseScanStreamPathWithNewline(t *testing.T) {
t.Parallel()
in := strings.NewReader(mkRecord(5, 9, "h", "t", "/a/new\nline"))
recs, malformed := parseScanStream(in, "test")
if malformed != 0 || len(recs) != 1 {
t.Fatalf("got %d recs, %d malformed, want 1, 0",
len(recs), malformed)
}
if recs[0].path != "/a/new\nline" {
t.Fatalf("path = %q, want %q", recs[0].path, "/a/new\nline")
}
}
func TestParseScanStreamEmpty(t *testing.T) {
t.Parallel()
recs, malformed := parseScanStream(strings.NewReader(""), "test")
if len(recs) != 0 || malformed != 0 {
t.Fatalf("got %d recs, %d malformed, want 0, 0",
len(recs), malformed)
}
}
func TestCollectDupeGroups(t *testing.T) {
t.Parallel()
@@ -120,6 +38,22 @@ func TestCollectDupeGroups(t *testing.T) {
}
}
func TestCollectDupeGroupsMtimeExcluded(t *testing.T) {
t.Parallel()
// mtime is informational only; records differing only in mtime
// still group together.
recs := []scanRec{
{size: 9, mtime: 100, head: "h", tail: "t", path: "/m/1"},
{size: 9, mtime: 200, head: "h", tail: "t", path: "/m/2"},
}
groups := collectDupeGroups(recs)
if len(groups) != 1 {
t.Fatalf("len(groups) = %d, want 1", len(groups))
}
}
func TestCollectDupeGroupsTieBreak(t *testing.T) {
t.Parallel()
@@ -190,15 +124,3 @@ func TestHumanBytes(t *testing.T) {
}
}
}
func TestMalformedNote(t *testing.T) {
t.Parallel()
if got := malformedNote(0); got != "" {
t.Errorf("malformedNote(0) = %q, want empty", got)
}
if got := malformedNote(3); got != " (3 malformed, skipped)" {
t.Errorf("malformedNote(3) = %q", got)
}
}