sfdupes/db_test.go

package main

import (
	"context"
	"database/sql"
	"errors"
	"fmt"
	"os"
	"path/filepath"
	"slices"
	"strings"
	"testing"
	"time"
)

// testDBPath returns a database path inside a fresh temp dir, with
// symlinks resolved as the scan lock resolves them, so the lock file
// is this path plus ".lock" even where the temp dir is reached through
// a symlink.
func testDBPath(t *testing.T) string {
	t.Helper()

	dir, err := filepath.EvalSymlinks(t.TempDir())
	if err != nil {
		t.Fatal(err)
	}

	return filepath.Join(dir, "db.sqlite")
}

// openTestDB creates a fresh scan database in a temp dir.
func openTestDB(t *testing.T) *sql.DB {
	t.Helper()

	db, err := openScanDatabase(t.Context(), testDBPath(t))
	if err != nil {
		t.Fatal(err)
	}

	t.Cleanup(func() { _ = db.Close() })

	return db
}

func TestDatabasePath(t *testing.T) {
	// README §Database: SFDUPES_DATABASE when set and non-empty, the
	// default otherwise, with the working directory, its symlinks
	// resolved, put in front of a relative path. Nothing is cleaned, so
	// a ".." stays.
	dir, err := filepath.EvalSymlinks(t.TempDir())
	if err != nil {
		t.Fatal(err)
	}

	t.Chdir(dir)

	fromEnv := "from " + databaseEnv
	cases := []struct {
		env, path, source string
	}{
		{"", defaultDatabasePath, "default"},
		{"/custom/place.sqlite", "/custom/place.sqlite", fromEnv},
		{"/custom/link/../place.sqlite", "/custom/link/../place.sqlite",
			fromEnv},
		{"sub/place.sqlite", dir + "/sub/place.sqlite", fromEnv},
		{"link/../place.sqlite", dir + "/link/../place.sqlite", fromEnv},
	}

	for _, tc := range cases {
		t.Setenv(databaseEnv, tc.env)

		path, source, err := databasePath()
		if err != nil || path != tc.path || source != tc.source {
			t.Errorf("databasePath() with %s=%q = %q, %q, %v; want %q, %q",
				databaseEnv, tc.env, path, source, err, tc.path, tc.source)
		}
	}

	// In the root directory the path gets one slash in front, not two.
	t.Chdir("/")
	t.Setenv(databaseEnv, "place.sqlite")

	path, _, err := databasePath()
	if err != nil || path != "/place.sqlite" {
		t.Errorf("databasePath() in / = %q, %v; want %q",
			path, err, "/place.sqlite")
	}
}

func TestOpenScanDatabaseCreates(t *testing.T) {
	t.Parallel()

	// The parent directory does not exist yet; scan must create it.
	path := filepath.Join(t.TempDir(), "nested", "dir", "db.sqlite")

	db, err := openScanDatabase(t.Context(), path)
	if err != nil {
		t.Fatalf("openScanDatabase: %v", err)
	}

	v, err := userVersion(t.Context(), db)
	if err != nil || v != schemaVersion {
		t.Fatalf("userVersion = %d, %v; want %d, nil", v, err, schemaVersion)
	}

	_ = db.Close()

	// Reopening an existing database must succeed and find the schema.
	db, err = openScanDatabase(t.Context(), path)
	if err != nil {
		t.Fatalf("reopen: %v", err)
	}

	defer func() { _ = db.Close() }()

	if recs := dbRecords(t, db); len(recs) != 0 {
		t.Fatalf("records = %v, want none", recs)
	}
}

func TestOpenDatabaseUnversionedForeign(t *testing.T) {
	t.Parallel()

	// A database that has a files table but user_version 0, written by
	// some other tool. report, trees and scan must refuse it with the
	// schema-version error, not adopt it and not emit a raw SQLite
	// "table files already exists".
	path := testDBPath(t)

	db, err := sql.Open("sqlite", path)
	if err != nil {
		t.Fatal(err)
	}

	_, err = db.ExecContext(t.Context(), "CREATE TABLE files (x INTEGER)")
	if err != nil {
		t.Fatal(err)
	}

	_ = db.Close()

	_, err = openReportDatabase(t.Context(), path)
	if !errors.Is(err, errSchemaVersion) ||
		!strings.Contains(err.Error(), "remove the file and rescan") {
		t.Fatalf("report: err = %v, want errSchemaVersion telling the "+
			"operator to remove the file and rescan", err)
	}

	_, err = openScanDatabase(t.Context(), path)
	if !errors.Is(err, errSchemaVersion) ||
		!strings.Contains(err.Error(), "remove the file and rescan") {
		t.Fatalf("scan: err = %v, want errSchemaVersion telling the "+
			"operator to remove the file and rescan", err)
	}
}

func TestSchemaCreationStoppedPartway(t *testing.T) {
	t.Parallel()

	// A first scan stopped while creating the schema must leave a
	// database the next scan accepts. max_page_count(2) leaves room for
	// the files table but not its index, so schema creation fails right
	// after CREATE TABLE, a point an interrupt could also stop it at.
	path := testDBPath(t)

	db, err := openDB(path, scanParams+"&_pragma=max_page_count(2)")
	if err != nil {
		t.Fatal(err)
	}

	err = initSchema(t.Context(), db)
	_ = db.Close()

	if err == nil {
		t.Fatal("initSchema with no room for the index succeeded")
	}

	db, err = openScanDatabase(t.Context(), path)
	if err != nil {
		t.Fatalf("next scan: %v", err)
	}

	defer func() { _ = db.Close() }()

	v, err := userVersion(t.Context(), db)
	if err != nil || v != schemaVersion {
		t.Fatalf("userVersion = %d, %v; want %d, nil", v, err, schemaVersion)
	}
}

func TestOpenReportDatabaseMissing(t *testing.T) {
	t.Parallel()

	_, err := openReportDatabase(t.Context(), testDBPath(t))
	if !errors.Is(err, errNoDatabase) {
		t.Fatalf("err = %v, want errNoDatabase", err)
	}
}

func TestOpenDatabaseVersionMismatch(t *testing.T) {
	t.Parallel()

	// A database stamped with a schema version other than 0 and
	// schemaVersion. report, trees and scan must all refuse it.
	path := testDBPath(t)

	db, err := openScanDatabase(t.Context(), path)
	if err != nil {
		t.Fatal(err)
	}

	_, err = db.ExecContext(context.Background(), "PRAGMA user_version = 99")
	if err != nil {
		t.Fatal(err)
	}

	_ = db.Close()

	_, err = openReportDatabase(t.Context(), path)
	if !errors.Is(err, errSchemaVersion) {
		t.Fatalf("report: err = %v, want errSchemaVersion", err)
	}

	_, err = openScanDatabase(t.Context(), path)
	if !errors.Is(err, errSchemaVersion) {
		t.Fatalf("scan: err = %v, want errSchemaVersion", err)
	}
}

func TestOpenReportDatabaseOK(t *testing.T) {
	t.Parallel()

	path := testDBPath(t)

	db, err := openScanDatabase(t.Context(), path)
	if err != nil {
		t.Fatal(err)
	}

	_ = db.Close()

	db, err = openReportDatabase(t.Context(), path)
	if err != nil {
		t.Fatalf("openReportDatabase: %v", err)
	}

	_ = db.Close()
}

func TestCloseScanDatabaseWhileReportOpen(t *testing.T) {
	t.Parallel()

	// A report holding the database open stops scan from taking it out
	// of WAL mode. The -wal and -shm files must then stay beside it, so
	// that a later report still needs only read access.
	path := testDBPath(t)

	scanDB, err := openScanDatabase(t.Context(), path)
	if err != nil {
		t.Fatal(err)
	}

	reportDB, err := openReportDatabase(t.Context(), path)
	if err != nil {
		t.Fatal(err)
	}

	closeScanDatabase(t.Context(), scanDB, path)

	_ = reportDB.Close()

	_, err = os.Stat(path + "-wal")
	if err != nil {
		t.Fatalf("no -wal left: the switch out of WAL mode was not "+
			"stopped: %v", err)
	}

	makeReadOnly(t, path)

	reportDB, err = openReportDatabase(t.Context(), path)
	if err != nil {
		t.Fatalf("openReportDatabase: %v", err)
	}

	defer func() { _ = reportDB.Close() }()

	err = loadFileRows(t.Context(), reportDB, func(int64) {},
		func(scanRec) {})
	if err != nil {
		t.Fatalf("loadFileRows: %v", err)
	}
}

// TestLoadRowsCountAgreeDuringScan has a scan commit records after
// loadFileRows, as trees calls it, or loadSignatureRows, as report
// calls it, has counted the records and before it reads them, as when
// trees or report runs during a scan: each must read exactly as many
// records as it counted.
func TestLoadRowsCountAgreeDuringScan(t *testing.T) {
	t.Parallel()

	loads := map[string]func(context.Context, *sql.DB, func(int64),
		func(scanRec)) error{
		"loadFileRows": loadFileRows,
		"loadSignatureRows": func(ctx context.Context, db *sql.DB,
			total func(int64), fn func(scanRec),
		) error {
			return loadSignatureRows(ctx, db, total, func(r scanRec) error {
				fn(r)

				return nil
			})
		},
	}

	for name, load := range loads {
		t.Run(name, func(t *testing.T) {
			t.Parallel()

			path := testDBPath(t)

			scanDB, err := openScanDatabase(t.Context(), path)
			if err != nil {
				t.Fatal(err)
			}

			defer func() { _ = scanDB.Close() }()

			recs := smokeTreeRecs()

			err = applyChanges(t.Context(), scanDB, recs[:3], nil, nil)
			if err != nil {
				t.Fatal(err)
			}

			reportDB, err := openReportDatabase(t.Context(), path)
			if err != nil {
				t.Fatal(err)
			}

			defer func() { _ = reportDB.Close() }()

			var counted, read int64

			err = load(t.Context(), reportDB, func(n int64) {
				counted = n

				err := applyChanges(t.Context(), scanDB, recs[3:], nil, nil)
				if err != nil {
					t.Error(err)
				}
			}, func(scanRec) { read++ })
			if err != nil {
				t.Fatal(err)
			}

			if counted != 3 || read != counted {
				t.Errorf("counted %d records and read %d, want 3 and 3",
					counted, read)
			}
		})
	}
}

// TestSignatureRowsNeedNoSort checks that SQLite reads signatureRowsSQL
// from the files_signature index alone, with nothing to sort: the
// first record comes at once, so report's read phase counts each
// record as it comes rather than waiting on SQLite for all of them.
func TestSignatureRowsNeedNoSort(t *testing.T) {
	t.Parallel()

	db := openTestDB(t)

	rows, err := db.QueryContext(t.Context(),
		"EXPLAIN QUERY PLAN "+signatureRowsSQL)
	if err != nil {
		t.Fatal(err)
	}

	defer func() { _ = rows.Close() }()

	var plan []string

	for rows.Next() {
		var (
			id, parent, unused int
			detail             string
		)

		err = rows.Scan(&id, &parent, &unused, &detail)
		if err != nil {
			t.Fatal(err)
		}

		plan = append(plan, detail)
	}

	err = rows.Err()
	if err != nil {
		t.Fatal(err)
	}

	want := []string{"SCAN files USING COVERING INDEX files_signature"}
	if !slices.Equal(plan, want) {
		t.Errorf("query plan = %q, want %q", plan, want)
	}
}

func TestApplyChangesRoundTrip(t *testing.T) {
	t.Parallel()

	db := openTestDB(t)

	// Paths may contain tabs and newlines; the database must store
	// them byte-exactly. Every hash, content included, comes back as
	// written.
	recs := []scanRec{
		{
			size: 2, mtime: time.Unix(20, 999_999_999), head: "h2", tail: "t2",
			content: "c2", path: "/a/tab\tnew\nline",
		},
		{
			size: 1, mtime: time.Unix(10, 0), head: "h1", tail: "t1",
			content: "c1", path: "/a/x",
		},
	}

	prog := newProgress(progressParams{label: "update", total: 4})
	defer prog.finish()

	err := applyChanges(t.Context(), db, recs, nil, prog)
	if err != nil {
		t.Fatalf("applyChanges: %v", err)
	}

	// The records come back in path order, which is the order of recs.
	got := dbRecords(t, db)
	if !slices.Equal(got, recs) {
		t.Fatalf("rows = %+v, want %+v", got, recs)
	}

	// An upsert for an existing path updates in place; a delete
	// removes exactly its path.
	upd := scanRec{
		size: 3, mtime: time.Unix(30, 0), head: "h3", tail: "t3", content: "c3",
		path: "/a/x",
	}

	err = applyChanges(t.Context(), db, []scanRec{upd},
		[]string{"/a/tab\tnew\nline"}, prog)
	if err != nil {
		t.Fatalf("applyChanges: %v", err)
	}

	got = dbRecords(t, db)
	if len(got) != 1 || got[0] != upd {
		t.Fatalf("rows = %+v, want just %+v", got, upd)
	}
}

func TestApplyChangesBatching(t *testing.T) {
	t.Parallel()

	db := openTestDB(t)

	// One more change than the batch size, so the update spans two
	// transactions.
	n := updateBatchSize + 1

	recs := make([]scanRec, 0, n)
	for i := range n {
		recs = append(recs, scanRec{
			size: int64(i), mtime: time.Unix(1, 0), head: "h", tail: "t",
			path: fmt.Sprintf("/batch/%07d", i),
		})
	}

	prog := newProgress(progressParams{label: "update", total: int64(2 * n)})
	defer prog.finish()

	err := applyChanges(t.Context(), db, recs, nil, prog)
	if err != nil {
		t.Fatalf("applyChanges: %v", err)
	}

	if got := dbRecords(t, db); len(got) != n {
		t.Fatalf("records = %d, want %d", len(got), n)
	}

	deletes := make([]string, 0, n)
	for _, r := range recs {
		deletes = append(deletes, r.path)
	}

	err = applyChanges(t.Context(), db, nil, deletes, prog)
	if err != nil {
		t.Fatalf("applyChanges deletes: %v", err)
	}

	if got := dbRecords(t, db); len(got) != 0 {
		t.Fatalf("records = %d, want 0", len(got))
	}
}

// TestCountContentCandidates checks that countContentCandidates counts
// the records the content phase's search returns: those of each group
// of two or more records of headTailMin or more with a head, where at
// least one has no content hash.
func TestCountContentCandidates(t *testing.T) {
	t.Parallel()

	db := openTestDB(t)

	// Each group's records: size, head, and content, one per record.
	groups := []struct {
		size     int64
		head     string
		contents []string
	}{
		{headTailMin, "h", []string{"", "c", "c"}}, // returned
		{headTailMin + 1, "h", []string{"", ""}},   // returned
		{headTailMin + 2, "h", []string{"c", "c"}}, // all hashed
		{headTailMin + 3, "h", []string{""}},       // one record
		{headTailMin - 1, "h", []string{"", ""}},   // too small
		{headTailMin + 4, "", []string{"", ""}},    // no head
	}

	var recs []scanRec

	for _, g := range groups {
		for _, content := range g.contents {
			recs = append(recs, scanRec{
				size: g.size, mtime: time.Unix(1, 0), head: g.head,
				tail: "t", content: content,
				path: fmt.Sprintf("/r/%d", len(recs)),
			})
		}
	}

	err := applyChanges(t.Context(), db, recs, nil, nil)
	if err != nil {
		t.Fatalf("applyChanges: %v", err)
	}

	got, err := countContentCandidates(t.Context(), db)
	if err != nil {
		t.Fatalf("countContentCandidates: %v", err)
	}

	var returned int64

	err = loadContentCandidates(t.Context(), db,
		func(scanRec, bool) { returned++ })
	if err != nil {
		t.Fatalf("loadContentCandidates: %v", err)
	}

	if got != 5 || returned != 5 {
		t.Errorf("count = %d, records returned = %d, want 5 and 5",
			got, returned)
	}
}