Implement persistent SQLite scan database
All checks were successful
check / check (push) Successful in 3s
All checks were successful
check / check (push) Successful in 3s
scan now synchronizes a database that survives between runs (SFDUPES_DATABASE, default /var/lib/sfdupes/db.sqlite) instead of emitting a stream: operands are resolved to absolute paths, unchanged files (same size, mtime not newer than recorded) are never re-read, new and changed files are hashed, and records under the scanned operands that were not verified this run are deleted; records outside the operands are untouched. All changes commit in a single transaction, and WAL journaling with a busy timeout keeps a report run during a cron scan safe. report and trees read the database (no positional arguments); the NUL-terminated stream format, its parser, and the malformed-record handling are gone. The driver is modernc.org/sqlite (pure Go), so builds keep cgo disabled.
This commit is contained in:
110
report_test.go
110
report_test.go
@@ -1,92 +1,10 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"fmt"
|
||||
"slices"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// mkRecord serializes one scan record in the on-the-wire format.
|
||||
func mkRecord(size, mtime int64, head, tail, path string) string {
|
||||
return fmt.Sprintf("%d\t%d\t%s\t%s\t%s\x00",
|
||||
size, mtime, head, tail, path)
|
||||
}
|
||||
|
||||
func TestSplitNULScanner(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
sc := bufio.NewScanner(strings.NewReader("a\x00bb\x00\x00tail"))
|
||||
sc.Split(splitNUL)
|
||||
|
||||
var got []string
|
||||
for sc.Scan() {
|
||||
got = append(got, sc.Text())
|
||||
}
|
||||
|
||||
err := sc.Err()
|
||||
if err != nil {
|
||||
t.Fatalf("scanner error: %v", err)
|
||||
}
|
||||
|
||||
want := []string{"a", "bb", "", "tail"}
|
||||
if !slices.Equal(got, want) {
|
||||
t.Fatalf("tokens = %q, want %q", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseScanStream(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
stream := mkRecord(10, 1, "h1", "t1", "/a/x") +
|
||||
"garbage-without-tabs\x00" +
|
||||
"notanumber\t1\th\tt\t/a/bad\x00" +
|
||||
mkRecord(20, 2, "h2", "t2", "/a/tab\tin\tname") +
|
||||
"30\t3\th3\tt3\t/trailing/no-nul"
|
||||
|
||||
recs, malformed := parseScanStream(strings.NewReader(stream), "test")
|
||||
|
||||
if malformed != 2 {
|
||||
t.Errorf("malformed = %d, want 2", malformed)
|
||||
}
|
||||
|
||||
want := []scanRec{
|
||||
{size: 10, head: "h1", tail: "t1", path: "/a/x"},
|
||||
{size: 20, head: "h2", tail: "t2", path: "/a/tab\tin\tname"},
|
||||
{size: 30, head: "h3", tail: "t3", path: "/trailing/no-nul"},
|
||||
}
|
||||
if !slices.Equal(recs, want) {
|
||||
t.Fatalf("recs = %+v, want %+v", recs, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseScanStreamPathWithNewline(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
in := strings.NewReader(mkRecord(5, 9, "h", "t", "/a/new\nline"))
|
||||
|
||||
recs, malformed := parseScanStream(in, "test")
|
||||
if malformed != 0 || len(recs) != 1 {
|
||||
t.Fatalf("got %d recs, %d malformed, want 1, 0",
|
||||
len(recs), malformed)
|
||||
}
|
||||
|
||||
if recs[0].path != "/a/new\nline" {
|
||||
t.Fatalf("path = %q, want %q", recs[0].path, "/a/new\nline")
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseScanStreamEmpty(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
recs, malformed := parseScanStream(strings.NewReader(""), "test")
|
||||
if len(recs) != 0 || malformed != 0 {
|
||||
t.Fatalf("got %d recs, %d malformed, want 0, 0",
|
||||
len(recs), malformed)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCollectDupeGroups(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
@@ -120,6 +38,22 @@ func TestCollectDupeGroups(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestCollectDupeGroupsMtimeExcluded(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
// mtime is informational only; records differing only in mtime
|
||||
// still group together.
|
||||
recs := []scanRec{
|
||||
{size: 9, mtime: 100, head: "h", tail: "t", path: "/m/1"},
|
||||
{size: 9, mtime: 200, head: "h", tail: "t", path: "/m/2"},
|
||||
}
|
||||
|
||||
groups := collectDupeGroups(recs)
|
||||
if len(groups) != 1 {
|
||||
t.Fatalf("len(groups) = %d, want 1", len(groups))
|
||||
}
|
||||
}
|
||||
|
||||
func TestCollectDupeGroupsTieBreak(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
@@ -190,15 +124,3 @@ func TestHumanBytes(t *testing.T) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestMalformedNote(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
if got := malformedNote(0); got != "" {
|
||||
t.Errorf("malformedNote(0) = %q, want empty", got)
|
||||
}
|
||||
|
||||
if got := malformedNote(3); got != " (3 malformed, skipped)" {
|
||||
t.Errorf("malformedNote(3) = %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user