sfdupes/db_test.go

package main

import (
	"context"
	"crypto/sha256"
	"database/sql"
	"errors"
	"fmt"
	"io/fs"
	"os"
	"path/filepath"
	"slices"
	"strings"
	"testing"
	"time"
)

// testDBPath returns a database path inside a fresh temp dir, with
// symlinks resolved as the scan lock resolves them, so the lock file
// is this path plus ".lock" even where the temp dir is reached through
// a symlink.
func testDBPath(t *testing.T) string {
	t.Helper()

	dir, err := filepath.EvalSymlinks(t.TempDir())
	if err != nil {
		t.Fatal(err)
	}

	return filepath.Join(dir, "db.sqlite")
}

// openTestDB creates a fresh scan database in a temp dir.
func openTestDB(t *testing.T) *sql.DB {
	t.Helper()

	db, err := openScanDatabase(t.Context(), testDBPath(t))
	if err != nil {
		t.Fatal(err)
	}

	t.Cleanup(func() { _ = db.Close() })

	return db
}

func TestDatabasePath(t *testing.T) {
	// README §Database: SFDUPES_DATABASE when set and non-empty, the
	// default otherwise, with the working directory, its symlinks
	// resolved, put in front of a relative path. Nothing is cleaned, so
	// a ".." stays.
	dir, err := filepath.EvalSymlinks(t.TempDir())
	if err != nil {
		t.Fatal(err)
	}

	t.Chdir(dir)

	fromEnv := "from " + databaseEnv
	cases := []struct {
		env, path, source string
	}{
		{"", defaultDatabasePath, "default"},
		{"/custom/place.sqlite", "/custom/place.sqlite", fromEnv},
		{"/custom/link/../place.sqlite", "/custom/link/../place.sqlite",
			fromEnv},
		{"sub/place.sqlite", dir + "/sub/place.sqlite", fromEnv},
		{"link/../place.sqlite", dir + "/link/../place.sqlite", fromEnv},
	}

	for _, tc := range cases {
		t.Setenv(databaseEnv, tc.env)

		path, source, err := databasePath()
		if err != nil || path != tc.path || source != tc.source {
			t.Errorf("databasePath() with %s=%q = %q, %q, %v; want %q, %q",
				databaseEnv, tc.env, path, source, err, tc.path, tc.source)
		}
	}

	// In the root directory the path gets one slash in front, not two.
	t.Chdir("/")
	t.Setenv(databaseEnv, "place.sqlite")

	path, _, err := databasePath()
	if err != nil || path != "/place.sqlite" {
		t.Errorf("databasePath() in / = %q, %v; want %q",
			path, err, "/place.sqlite")
	}
}

func TestOpenScanDatabaseCreates(t *testing.T) {
	t.Parallel()

	// The parent directory does not exist yet; scan must create it.
	path := filepath.Join(t.TempDir(), "nested", "dir", "db.sqlite")

	db, err := openScanDatabase(t.Context(), path)
	if err != nil {
		t.Fatalf("openScanDatabase: %v", err)
	}

	v, err := userVersion(t.Context(), db)
	if err != nil || v != schemaVersion {
		t.Fatalf("userVersion = %d, %v; want %d, nil", v, err, schemaVersion)
	}

	_ = db.Close()

	// Reopening an existing database must succeed and find the schema.
	db, err = openScanDatabase(t.Context(), path)
	if err != nil {
		t.Fatalf("reopen: %v", err)
	}

	defer func() { _ = db.Close() }()

	if recs := dbRecords(t, db); len(recs) != 0 {
		t.Fatalf("records = %v, want none", recs)
	}
}

// TestScanCreatesDatabaseDirectoryAfterSymlink has a database path go
// through a symlink to a directory, then "..", to a missing directory.
// With srv/link a symlink to data/x, srv/link/../new/db.sqlite leads to
// data/new/db.sqlite. lockScanDatabase and openScanDatabase, the two
// places scan creates the database's directory, must each create the
// database there, and nothing at srv/new, where the path leads once
// cleaned as text.
func TestScanCreatesDatabaseDirectoryAfterSymlink(t *testing.T) {
	t.Parallel()

	creates := map[string]func(ctx context.Context, path string) error{
		"lockScanDatabase": func(_ context.Context, path string) error {
			lock, err := lockScanDatabase(path)
			if err == nil {
				_ = lock.Close()
			}

			return err
		},
		"openScanDatabase": func(ctx context.Context, path string) error {
			db, err := openScanDatabase(ctx, path)
			if err == nil {
				_ = db.Close()
			}

			return err
		},
	}

	for name, create := range creates {
		t.Run(name, func(t *testing.T) {
			t.Parallel()

			dir, err := filepath.EvalSymlinks(t.TempDir())
			if err != nil {
				t.Fatal(err)
			}

			err = os.MkdirAll(filepath.Join(dir, "data", "x"), 0o750)
			if err != nil {
				t.Fatal(err)
			}

			err = os.Mkdir(filepath.Join(dir, "srv"), 0o750)
			if err != nil {
				t.Fatal(err)
			}

			err = os.Symlink(filepath.Join(dir, "data", "x"),
				filepath.Join(dir, "srv", "link"))
			if err != nil {
				t.Fatal(err)
			}

			err = create(t.Context(), dir+"/srv/link/../new/db.sqlite")
			if err != nil {
				t.Fatalf("%s: %v", name, err)
			}

			_, err = os.Stat(filepath.Join(dir, "data", "new", "db.sqlite"))
			if err != nil {
				t.Errorf("database the path leads to: %v", err)
			}

			_, err = os.Lstat(filepath.Join(dir, "srv", "new"))
			if !errors.Is(err, fs.ErrNotExist) {
				t.Errorf("srv/new: stat error %v, want it missing", err)
			}
		})
	}
}

// TestDatabaseFiles opens a database through a symlink to its file:
// databaseFiles finds the database file and its -wal and -shm files
// beside the symlink's target, where SQLite keeps them, and leaves out
// the .lock file, which only a scan creates.
func TestDatabaseFiles(t *testing.T) {
	t.Parallel()

	path := testDBPath(t)
	link := filepath.Join(t.TempDir(), "link.sqlite")

	err := os.Symlink(path, link)
	if err != nil {
		t.Fatal(err)
	}

	db, err := openScanDatabase(t.Context(), link)
	if err != nil {
		t.Fatal(err)
	}

	defer func() { _ = db.Close() }()

	files, err := databaseFiles(link)
	if err != nil {
		t.Fatal(err)
	}

	got := make([]string, 0, len(files))
	for _, f := range files {
		got = append(got, f.path)
	}

	want := []string{path, path + "-wal", path + "-shm"}
	if !slices.Equal(got, want) {
		t.Errorf("databaseFiles(%s) = %q, want %q", link, got, want)
	}
}

func TestOpenDatabaseUnversionedForeign(t *testing.T) {
	t.Parallel()

	// A database that has a files table but user_version 0, written by
	// some other tool. report, trees and scan must refuse it with the
	// schema-version error, not adopt it and not emit a raw SQLite
	// "table files already exists".
	path := testDBPath(t)

	db, err := sql.Open("sqlite", path)
	if err != nil {
		t.Fatal(err)
	}

	_, err = db.ExecContext(t.Context(), "CREATE TABLE files (x INTEGER)")
	if err != nil {
		t.Fatal(err)
	}

	_ = db.Close()

	_, err = openReportDatabase(t.Context(), path)
	if !errors.Is(err, errSchemaVersion) ||
		!strings.Contains(err.Error(), "remove the file and rescan") {
		t.Fatalf("report: err = %v, want errSchemaVersion telling the "+
			"operator to remove the file and rescan", err)
	}

	_, err = openScanDatabase(t.Context(), path)
	if !errors.Is(err, errSchemaVersion) ||
		!strings.Contains(err.Error(), "remove the file and rescan") {
		t.Fatalf("scan: err = %v, want errSchemaVersion telling the "+
			"operator to remove the file and rescan", err)
	}
}

func TestSchemaCreationStoppedPartway(t *testing.T) {
	t.Parallel()

	// A first scan stopped while creating the schema must leave a
	// database the next scan accepts. max_page_count(2) leaves room for
	// the files table but not its index, so schema creation fails right
	// after CREATE TABLE, a point an interrupt could also stop it at.
	path := testDBPath(t)

	db, err := openDB(path, scanParams+"&_pragma=max_page_count(2)")
	if err != nil {
		t.Fatal(err)
	}

	err = initSchema(t.Context(), db)
	_ = db.Close()

	if err == nil {
		t.Fatal("initSchema with no room for the index succeeded")
	}

	db, err = openScanDatabase(t.Context(), path)
	if err != nil {
		t.Fatalf("next scan: %v", err)
	}

	defer func() { _ = db.Close() }()

	v, err := userVersion(t.Context(), db)
	if err != nil || v != schemaVersion {
		t.Fatalf("userVersion = %d, %v; want %d, nil", v, err, schemaVersion)
	}
}

func TestOpenReportDatabaseMissing(t *testing.T) {
	t.Parallel()

	_, err := openReportDatabase(t.Context(), testDBPath(t))
	if !errors.Is(err, errNoDatabase) {
		t.Fatalf("err = %v, want errNoDatabase", err)
	}
}

func TestOpenDatabaseVersionMismatch(t *testing.T) {
	t.Parallel()

	// A database stamped with a schema version other than 0 and
	// schemaVersion. report, trees and scan must all refuse it.
	path := testDBPath(t)

	db, err := openScanDatabase(t.Context(), path)
	if err != nil {
		t.Fatal(err)
	}

	_, err = db.ExecContext(context.Background(), "PRAGMA user_version = 99")
	if err != nil {
		t.Fatal(err)
	}

	_ = db.Close()

	_, err = openReportDatabase(t.Context(), path)
	if !errors.Is(err, errSchemaVersion) {
		t.Fatalf("report: err = %v, want errSchemaVersion", err)
	}

	_, err = openScanDatabase(t.Context(), path)
	if !errors.Is(err, errSchemaVersion) {
		t.Fatalf("scan: err = %v, want errSchemaVersion", err)
	}
}

func TestOpenReportDatabaseOK(t *testing.T) {
	t.Parallel()

	path := testDBPath(t)

	db, err := openScanDatabase(t.Context(), path)
	if err != nil {
		t.Fatal(err)
	}

	_ = db.Close()

	db, err = openReportDatabase(t.Context(), path)
	if err != nil {
		t.Fatalf("openReportDatabase: %v", err)
	}

	_ = db.Close()
}

func TestCloseScanDatabaseWhileReportOpen(t *testing.T) {
	t.Parallel()

	// A report holding the database open stops scan from taking it out
	// of WAL mode. The -wal and -shm files must then stay beside it, so
	// that a later report still needs only read access.
	path := testDBPath(t)

	scanDB, err := openScanDatabase(t.Context(), path)
	if err != nil {
		t.Fatal(err)
	}

	reportDB, err := openReportDatabase(t.Context(), path)
	if err != nil {
		t.Fatal(err)
	}

	closeScanDatabase(t.Context(), scanDB, path)

	_ = reportDB.Close()

	_, err = os.Stat(path + "-wal")
	if err != nil {
		t.Fatalf("no -wal left: the switch out of WAL mode was not "+
			"stopped: %v", err)
	}

	makeReadOnly(t, path)

	reportDB, err = openReportDatabase(t.Context(), path)
	if err != nil {
		t.Fatalf("openReportDatabase: %v", err)
	}

	defer func() { _ = reportDB.Close() }()

	err = loadFileRows(t.Context(), reportDB, func(int64) {},
		func(scanRec) {})
	if err != nil {
		t.Fatalf("loadFileRows: %v", err)
	}
}

// TestLoadRowsCountAgreeDuringScan has a scan commit records after
// loadFileRows, as trees calls it, or loadSignatureRows, as report
// calls it, has counted the records and before it reads them, as when
// trees or report runs during a scan: each must read exactly as many
// records as it counted.
func TestLoadRowsCountAgreeDuringScan(t *testing.T) {
	t.Parallel()

	loads := map[string]func(context.Context, *sql.DB, func(int64),
		func(scanRec)) error{
		"loadFileRows": loadFileRows,
		"loadSignatureRows": func(ctx context.Context, db *sql.DB,
			total func(int64), fn func(scanRec),
		) error {
			return loadSignatureRows(ctx, db, total, func(r scanRec) error {
				fn(r)

				return nil
			})
		},
	}

	for name, load := range loads {
		t.Run(name, func(t *testing.T) {
			t.Parallel()

			path := testDBPath(t)

			scanDB, err := openScanDatabase(t.Context(), path)
			if err != nil {
				t.Fatal(err)
			}

			defer func() { _ = scanDB.Close() }()

			recs := smokeTreeRecs()

			err = applyChanges(t.Context(), scanDB, recs[:3], nil, nil)
			if err != nil {
				t.Fatal(err)
			}

			reportDB, err := openReportDatabase(t.Context(), path)
			if err != nil {
				t.Fatal(err)
			}

			defer func() { _ = reportDB.Close() }()

			var counted, read int64

			err = load(t.Context(), reportDB, func(n int64) {
				counted = n

				err := applyChanges(t.Context(), scanDB, recs[3:], nil, nil)
				if err != nil {
					t.Error(err)
				}
			}, func(scanRec) { read++ })
			if err != nil {
				t.Fatal(err)
			}

			if counted != 3 || read != counted {
				t.Errorf("counted %d records and read %d, want 3 and 3",
					counted, read)
			}
		})
	}
}

// TestLoadRowsRefuseWrongSizeHash stores a record whose head, tail or
// content is not the 32 bytes of a SHA-256: either one byte, or the 64
// bytes of a SHA-256 written as hex text. loadFileRows and
// loadSignatureRows must each fail on it with an error naming its path.
func TestLoadRowsRefuseWrongSizeHash(t *testing.T) {
	t.Parallel()

	hexText := strings.Repeat("ab", sha256.Size)

	cases := []struct {
		name   string
		update string
		value  any
	}{
		{"one-byte head", "UPDATE files SET head = ?", []byte{0}},
		{"hex-text head", "UPDATE files SET head = ?", hexText},
		{"one-byte tail", "UPDATE files SET tail = ?", []byte{0}},
		{"hex-text tail", "UPDATE files SET tail = ?", hexText},
		{"one-byte content", "UPDATE files SET content = ?", []byte{0}},
		{"hex-text content", "UPDATE files SET content = ?", hexText},
	}
	for _, c := range cases {
		t.Run(c.name, func(t *testing.T) {
			t.Parallel()

			db := openTestDB(t)

			err := applyChanges(t.Context(), db,
				[]scanRec{{path: "/a/f"}}, nil, nil)
			if err != nil {
				t.Fatal(err)
			}

			_, err = db.ExecContext(t.Context(), c.update, c.value)
			if err != nil {
				t.Fatal(err)
			}

			loads := map[string]func() error{
				"loadFileRows": func() error {
					return loadFileRows(t.Context(), db, func(int64) {},
						func(scanRec) {})
				},
				"loadSignatureRows": func() error {
					return loadSignatureRows(t.Context(), db, func(int64) {},
						func(scanRec) error { return nil })
				},
			}

			for name, load := range loads {
				err := load()
				if !errors.Is(err, errHashSize) ||
					!strings.Contains(err.Error(), "/a/f") {
					t.Errorf("%s: err = %v, want %v naming /a/f",
						name, err, errHashSize)
				}
			}
		})
	}
}

// queryPlan returns the detail of each line of SQLite's EXPLAIN QUERY
// PLAN for query.
func queryPlan(t *testing.T, db *sql.DB, query string) []string {
	t.Helper()

	rows, err := db.QueryContext(t.Context(), "EXPLAIN QUERY PLAN "+query)
	if err != nil {
		t.Fatal(err)
	}

	defer func() { _ = rows.Close() }()

	var plan []string

	for rows.Next() {
		var (
			id, parent, unused int
			detail             string
		)

		err = rows.Scan(&id, &parent, &unused, &detail)
		if err != nil {
			t.Fatal(err)
		}

		plan = append(plan, detail)
	}

	err = rows.Err()
	if err != nil {
		t.Fatal(err)
	}

	return plan
}

// TestSignatureRowsNeedNoSort checks that SQLite reads signatureRowsSQL
// from the files_signature index alone, with nothing to sort: the
// first record comes at once, so report's read phase counts each
// record as it comes rather than waiting on SQLite for all of them.
func TestSignatureRowsNeedNoSort(t *testing.T) {
	t.Parallel()

	db := openTestDB(t)

	plan := queryPlan(t, db, signatureRowsSQL)

	want := []string{"SCAN files USING COVERING INDEX files_signature"}
	if !slices.Equal(plan, want) {
		t.Errorf("query plan = %q, want %q", plan, want)
	}
}

// TestCountsReadOnlyTotals checks that SQLite answers the queries for
// the number of records and their total size from the one row of
// totals. A plan that reads files or its index reads every record.
func TestCountsReadOnlyTotals(t *testing.T) {
	t.Parallel()

	db := openTestDB(t)

	for _, query := range []string{recordCountSQL, databaseCountsSQL} {
		plan := queryPlan(t, db, query)

		want := []string{"SCAN totals"}
		if !slices.Equal(plan, want) {
			t.Errorf("query plan of %q = %q, want %q", query, plan, want)
		}
	}
}

// TestCountsComeFromTotals sets the row of totals to numbers that files
// does not hold, then checks that scan's load phase, the read phases of
// trees and report, and info each report the numbers from totals.
func TestCountsComeFromTotals(t *testing.T) {
	t.Parallel()

	// files is empty, so a count over files gives 0 records of 0 bytes.
	const records, size = 7, 9000

	db := openTestDB(t)

	_, err := db.ExecContext(t.Context(),
		"UPDATE totals SET records = ?, size = ?", records, size)
	if err != nil {
		t.Fatal(err)
	}

	scanTotal, err := countRecords(t.Context(), db)
	if err != nil {
		t.Fatal(err)
	}

	var treesTotal, reportTotal int64

	err = loadFileRows(t.Context(), db,
		func(n int64) { treesTotal = n }, func(scanRec) {})
	if err != nil {
		t.Fatal(err)
	}

	err = loadSignatureRows(t.Context(), db,
		func(n int64) { reportTotal = n },
		func(scanRec) error { return nil })
	if err != nil {
		t.Fatal(err)
	}

	info, err := readDatabaseCounts(t.Context(), db)
	if err != nil {
		t.Fatal(err)
	}

	for name, got := range map[string]int64{
		"scan's load":   scanTotal,
		"trees' read":   treesTotal,
		"report's read": reportTotal,
		"info":          info.records,
	} {
		if got != records {
			t.Errorf("%s: %d records, want %d from totals",
				name, got, records)
		}
	}

	if info.size != size {
		t.Errorf("info: size %d, want %d from totals", info.size, size)
	}
}

// assertTotalsMatchFiles fails unless the row of totals holds the
// number of records and the sizes of all records added up, as counted
// over files.
func assertTotalsMatchFiles(t *testing.T, db *sql.DB) {
	t.Helper()

	got, err := readDatabaseCounts(t.Context(), db)
	if err != nil {
		t.Fatal(err)
	}

	var records, size int64

	err = db.QueryRowContext(t.Context(),
		"SELECT COUNT(*), COALESCE(SUM(size), 0) FROM files").
		Scan(&records, &size)
	if err != nil {
		t.Fatal(err)
	}

	if got.records != records || got.size != size {
		t.Errorf("totals = %d records of %d bytes, files holds %d of %d",
			got.records, got.size, records, size)
	}
}

// TestTotalsMatchFiles scans the smoke tree, rescans it after two files
// change size, and rescans it after two files are removed. After each
// scan the row of totals must agree with files.
func TestTotalsMatchFiles(t *testing.T) {
	t.Parallel()

	dir := buildSmokeTree(t)
	db := openTestDB(t)

	syncTree(t, db, dir)
	assertTotalsMatchFiles(t, db)

	writeFile(t, dir, "a/one.bin", pattern(10, 2500))
	writeFile(t, dir, "t1/f1", pattern(30, 1000))

	if st := syncTree(t, db, dir); st.updated != 2 {
		t.Fatalf("rescan stats = %+v, want 2 updated", st)
	}

	assertTotalsMatchFiles(t, db)

	for _, name := range []string{"a/unique.bin", "t2/sub/f2"} {
		err := os.Remove(filepath.Join(dir, name))
		if err != nil {
			t.Fatal(err)
		}
	}

	if st := syncTree(t, db, dir); st.removed != 2 {
		t.Fatalf("rescan stats = %+v, want 2 removed", st)
	}

	assertTotalsMatchFiles(t, db)
}

func TestApplyChangesRoundTrip(t *testing.T) {
	t.Parallel()

	db := openTestDB(t)

	// Paths may contain tabs and newlines; the database must store
	// them byte-exactly. Every hash, content included, comes back as
	// written.
	recs := []scanRec{
		{
			size: 2, mtime: time.Unix(20, 999_999_999), head: testHash("h2"),
			tail: testHash("t2"), content: testHash("c2"),
			path: "/a/tab\tnew\nline",
		},
		{
			size: 1, mtime: time.Unix(10, 0), head: testHash("h1"),
			tail: testHash("t1"), content: testHash("c1"), path: "/a/x",
		},
	}

	prog := newProgress(progressParams{label: "update", total: 4})
	defer prog.finish()

	err := applyChanges(t.Context(), db, recs, nil, prog)
	if err != nil {
		t.Fatalf("applyChanges: %v", err)
	}

	// The records come back in path order, which is the order of recs.
	got := dbRecords(t, db)
	if !slices.Equal(got, recs) {
		t.Fatalf("rows = %+v, want %+v", got, recs)
	}

	// An upsert for an existing path updates in place; a delete
	// removes exactly its path.
	upd := scanRec{
		size: 3, mtime: time.Unix(30, 0), head: testHash("h3"),
		tail: testHash("t3"), content: testHash("c3"), path: "/a/x",
	}

	err = applyChanges(t.Context(), db, []scanRec{upd},
		[]string{"/a/tab\tnew\nline"}, prog)
	if err != nil {
		t.Fatalf("applyChanges: %v", err)
	}

	got = dbRecords(t, db)
	if len(got) != 1 || got[0] != upd {
		t.Fatalf("rows = %+v, want just %+v", got, upd)
	}
}

func TestApplyChangesBatching(t *testing.T) {
	t.Parallel()

	db := openTestDB(t)

	// One more change than the batch size, so the update spans two
	// transactions.
	n := updateBatchSize + 1

	recs := make([]scanRec, 0, n)
	for i := range n {
		recs = append(recs, scanRec{
			size: int64(i), mtime: time.Unix(1, 0), head: testHash("h"),
			tail: testHash("t"), path: fmt.Sprintf("/batch/%07d", i),
		})
	}

	prog := newProgress(progressParams{label: "update", total: int64(2 * n)})
	defer prog.finish()

	err := applyChanges(t.Context(), db, recs, nil, prog)
	if err != nil {
		t.Fatalf("applyChanges: %v", err)
	}

	if got := dbRecords(t, db); len(got) != n {
		t.Fatalf("records = %d, want %d", len(got), n)
	}

	deletes := make([]string, 0, n)
	for _, r := range recs {
		deletes = append(deletes, r.path)
	}

	err = applyChanges(t.Context(), db, nil, deletes, prog)
	if err != nil {
		t.Fatalf("applyChanges deletes: %v", err)
	}

	if got := dbRecords(t, db); len(got) != 0 {
		t.Fatalf("records = %d, want 0", len(got))
	}
}