Hash every file's head, tail and content in the hash phase (closes #132)

Every file the walk records now gets three hashes in the hash phase:
head and tail, of its first and last 64 KiB, and content. The content
hash covers the whole file under 100 MiB, and 1 MiB at each gigabyte
from 100 MiB up, so a large file is never read in full. One 100 MiB
constant replaces the 10 MiB and 50 MiB thresholds. The size census,
the records stored without hashes and the content phase are gone, so a
scan has four phases and a rescan reads only new and changed files.
report and trees no longer special-case a missing content hash. README
"Duplicate detection" and "Rationale" are rewritten.

Judgement call: info drops its two hash-count lines, which would now
always equal the record count.

Model: opus-5-5

TODO.md

@@ -29,35 +29,6 @@
 
 # Completed Steps
 
-- `README.md` rewritten from scratch, sentence by sentence, as a newcomer reads
-  it. It leads with what the program is and how to use it. A reference section
-  covers every setting and every subcommand. Development and testing come last.
-  Every statement was checked against the code on `next`. Five comments in
-  `main_test.go` cite "Settings", where their rules now live. One comment each
-  in `scan_test.go`, `trees_test.go` and `exclude_test.go` cites what the README
-  now says (2026-10-26, https://git.eeqj.de/sneak/sfdupes/issues/135)
-
-- an operand at or inside a directory named `.zfs` is never scanned, whether
-  given alone or under another operand. `scan` warns about it, counts it as
-  skipped and does not walk it (2026-10-26,
-  https://git.eeqj.de/sneak/sfdupes/issues/142)
-
-- a file or directory removed while the scan runs is not an error, in the walk
-  or in the hash phase. The scan prints no line for it and does not count it as
-  skipped. When the first of several hard links to a file is gone, the hash
-  phase reads the next one instead. On a terminal every progress display starts
-  with the spinner, the hash phase's included. A phase with an exact total ends
-  its line with its percent and a 10-column bar at the terminal's right edge,
-  which no longer moves as the line's length changes. A line too wide for the
-  terminal now loses its rate before its bar. The time remaining is labelled
-  `remain`, not `eta` (2026-10-26, https://git.eeqj.de/sneak/sfdupes/issues/133)
-
-- the `head`, `tail` and `content` columns are BLOBs, each holding the raw 32
-  bytes of a SHA-256, half the size of the hex text they held. The hashes stay
-  raw from the hash phase through the database to `report` and `trees`. Reading
-  a record whose hash is not 32 bytes is an error (2026-10-26,
-  https://git.eeqj.de/sneak/sfdupes/issues/134)
-
 - the hash phase gives every file the walk records three hashes: `head` and
   `tail`, the SHA-256 of its first and last 64 KiB, and `content`, the SHA-256
   of the whole file under 100 MiB and of 1 MiB at each gigabyte from 100 MiB up.

cancel_test.go

@@ -2,7 +2,6 @@ package main
 
 import (
 	"context"
-	"crypto/sha256"
 	"database/sql"
 	"errors"
 	"fmt"
@@ -763,9 +762,7 @@ func TestHashWorkerDropsQueuedRuns(t *testing.T) {
 
 	var hashed atomic.Bool
 
-	hash := func(path string, size int64) (
-		[sha256.Size]byte, [sha256.Size]byte, [sha256.Size]byte, error,
-	) {
+	hash := func(path string, size int64) (string, string, string, error) {
 		hashed.Store(true)
 
 		return hashSignature(path, size)
@@ -805,9 +802,7 @@ func TestHashWorkerAbandonsBlockedSend(t *testing.T) {
 
 	// The scan is cancelled while the worker hashes, so the worker has
 	// already passed the check that drops queued runs.
-	hash := func(path string, size int64) (
-		[sha256.Size]byte, [sha256.Size]byte, [sha256.Size]byte, error,
-	) {
+	hash := func(path string, size int64) (string, string, string, error) {
 		cancel()
 
 		return hashSignature(path, size)

db.go

@@ -2,7 +2,6 @@ package main
 
 import (
 	"context"
-	"crypto/sha256"
 	"database/sql"
 	"errors"
 	"fmt"
@@ -49,17 +48,16 @@ const lockFilePerm = 0o600
 // createTableSQL is the schema applied to a fresh database. Paths are
 // BLOBs because Unix paths are raw bytes, not guaranteed UTF-8. mtime
 // holds whole Unix seconds and mtime_nsec the nanoseconds within that
-// second. head, tail and content each hold the raw 32 bytes of a
-// SHA-256.
+// second.
 const createTableSQL = `
 CREATE TABLE files (
     path       BLOB PRIMARY KEY,
     size       INTEGER NOT NULL,
     mtime      INTEGER NOT NULL,
     mtime_nsec INTEGER NOT NULL,
-    head       BLOB NOT NULL,
-    tail       BLOB NOT NULL,
-    content    BLOB NOT NULL
+    head       TEXT NOT NULL,
+    tail       TEXT NOT NULL,
+    content    TEXT NOT NULL
 ) WITHOUT ROWID
 `
 
@@ -94,10 +92,6 @@ var errSchemaVersion = errors.New("unsupported database schema version")
 // database.
 var errScanRunning = errors.New("another scan is running")
 
-// errHashSize reports a record whose head, tail or content is not the
-// 32 bytes of a SHA-256.
-var errHashSize = errors.New("hash is not 32 bytes")
-
 // databasePath resolves the database location: SFDUPES_DATABASE when
 // set and non-empty, the compiled-in default otherwise. It returns the
 // path, with the working directory, its symlinks resolved, put in front
@@ -498,24 +492,19 @@ func loadFileRows(ctx context.Context, db *sql.DB, total func(n int64),
 
 	for rows.Next() {
 		var (
-			path, head, tail, content []byte
-			sec, nsec                 int64
-			r                         scanRec
+			path      []byte
+			sec, nsec int64
+			r         scanRec
 		)
 
-		err = rows.Scan(&path, &r.size, &sec, &nsec, &head, &tail, &content)
+		err = rows.Scan(&path, &r.size, &sec, &nsec, &r.head, &r.tail,
+			&r.content)
 		if err != nil {
 			return fmt.Errorf("read record: %w", err)
 		}
 
 		r.path = string(path)
 		r.mtime = time.Unix(sec, nsec)
-
-		err = r.setHashes(head, tail, content)
-		if err != nil {
-			return fmt.Errorf("read record %s: %w", escapePath(r.path), err)
-		}
-
 		fn(r)
 	}
 
@@ -572,22 +561,17 @@ func loadSignatureRows(ctx context.Context, db *sql.DB,
 
 	for rows.Next() {
 		var (
-			path, head, tail, content []byte
-			r                         scanRec
+			path []byte
+			r    scanRec
 		)
 
-		err = rows.Scan(&r.size, &head, &tail, &content, &path)
+		err = rows.Scan(&r.size, &r.head, &r.tail, &r.content, &path)
 		if err != nil {
 			return fmt.Errorf("read record: %w", err)
 		}
 
 		r.path = string(path)
 
-		err = r.setHashes(head, tail, content)
-		if err != nil {
-			return fmt.Errorf("read record %s: %w", escapePath(r.path), err)
-		}
-
 		err = fn(r)
 		if err != nil {
 			return err
@@ -602,22 +586,6 @@ func loadSignatureRows(ctx context.Context, db *sql.DB,
 	return nil
 }
 
-// setHashes sets r's head, tail and content to the values read from
-// those columns. Each must be the raw 32 bytes of a SHA-256.
-func (r *scanRec) setHashes(head, tail, content []byte) error {
-	for _, b := range [][]byte{head, tail, content} {
-		if len(b) != sha256.Size {
-			return fmt.Errorf("%w: found %d bytes", errHashSize, len(b))
-		}
-	}
-
-	r.head = [sha256.Size]byte(head)
-	r.tail = [sha256.Size]byte(tail)
-	r.content = [sha256.Size]byte(content)
-
-	return nil
-}
-
 // countRecords returns the number of records in the database.
 func countRecords(ctx context.Context, db *sql.DB) (int64, error) {
 	var n int64
@@ -788,8 +756,7 @@ func execUpserts(ctx context.Context, tx *sql.Tx, upserts []scanRec,
 
 	for _, r := range upserts {
 		_, err = st.ExecContext(ctx, []byte(r.path), r.size,
-			r.mtime.Unix(), r.mtime.Nanosecond(),
-			r.head[:], r.tail[:], r.content[:])
+			r.mtime.Unix(), r.mtime.Nanosecond(), r.head, r.tail, r.content)
 		if err != nil {
 			return fmt.Errorf("upsert %s: %w", r.path, err)
 		}

db_test.go

@@ -2,7 +2,6 @@ package main
 
 import (
 	"context"
-	"crypto/sha256"
 	"database/sql"
 	"errors"
 	"fmt"
@@ -474,67 +473,6 @@ func TestLoadRowsCountAgreeDuringScan(t *testing.T) {
 	}
 }
 
-// TestLoadRowsRefuseWrongSizeHash stores a record whose head, tail or
-// content is not the 32 bytes of a SHA-256: either one byte, or the 64
-// bytes of a SHA-256 written as hex text. loadFileRows and
-// loadSignatureRows must each fail on it with an error naming its path.
-func TestLoadRowsRefuseWrongSizeHash(t *testing.T) {
-	t.Parallel()
-
-	hexText := strings.Repeat("ab", sha256.Size)
-
-	cases := []struct {
-		name   string
-		update string
-		value  any
-	}{
-		{"one-byte head", "UPDATE files SET head = ?", []byte{0}},
-		{"hex-text head", "UPDATE files SET head = ?", hexText},
-		{"one-byte tail", "UPDATE files SET tail = ?", []byte{0}},
-		{"hex-text tail", "UPDATE files SET tail = ?", hexText},
-		{"one-byte content", "UPDATE files SET content = ?", []byte{0}},
-		{"hex-text content", "UPDATE files SET content = ?", hexText},
-	}
-	for _, c := range cases {
-		t.Run(c.name, func(t *testing.T) {
-			t.Parallel()
-
-			db := openTestDB(t)
-
-			err := applyChanges(t.Context(), db,
-				[]scanRec{{path: "/a/f"}}, nil, nil)
-			if err != nil {
-				t.Fatal(err)
-			}
-
-			_, err = db.ExecContext(t.Context(), c.update, c.value)
-			if err != nil {
-				t.Fatal(err)
-			}
-
-			loads := map[string]func() error{
-				"loadFileRows": func() error {
-					return loadFileRows(t.Context(), db, func(int64) {},
-						func(scanRec) {})
-				},
-				"loadSignatureRows": func() error {
-					return loadSignatureRows(t.Context(), db, func(int64) {},
-						func(scanRec) error { return nil })
-				},
-			}
-
-			for name, load := range loads {
-				err := load()
-				if !errors.Is(err, errHashSize) ||
-					!strings.Contains(err.Error(), "/a/f") {
-					t.Errorf("%s: err = %v, want %v naming /a/f",
-						name, err, errHashSize)
-				}
-			}
-		})
-	}
-}
-
 // TestSignatureRowsNeedNoSort checks that SQLite reads signatureRowsSQL
 // from the files_signature index alone, with nothing to sort: the
 // first record comes at once, so report's read phase counts each
@@ -589,13 +527,12 @@ func TestApplyChangesRoundTrip(t *testing.T) {
 	// written.
 	recs := []scanRec{
 		{
-			size: 2, mtime: time.Unix(20, 999_999_999), head: testHash("h2"),
-			tail: testHash("t2"), content: testHash("c2"),
-			path: "/a/tab\tnew\nline",
+			size: 2, mtime: time.Unix(20, 999_999_999), head: "h2", tail: "t2",
+			content: "c2", path: "/a/tab\tnew\nline",
 		},
 		{
-			size: 1, mtime: time.Unix(10, 0), head: testHash("h1"),
-			tail: testHash("t1"), content: testHash("c1"), path: "/a/x",
+			size: 1, mtime: time.Unix(10, 0), head: "h1", tail: "t1",
+			content: "c1", path: "/a/x",
 		},
 	}
 
@@ -616,8 +553,8 @@ func TestApplyChangesRoundTrip(t *testing.T) {
 	// An upsert for an existing path updates in place; a delete
 	// removes exactly its path.
 	upd := scanRec{
-		size: 3, mtime: time.Unix(30, 0), head: testHash("h3"),
-		tail: testHash("t3"), content: testHash("c3"), path: "/a/x",
+		size: 3, mtime: time.Unix(30, 0), head: "h3", tail: "t3", content: "c3",
+		path: "/a/x",
 	}
 
 	err = applyChanges(t.Context(), db, []scanRec{upd},
@@ -644,8 +581,8 @@ func TestApplyChangesBatching(t *testing.T) {
 	recs := make([]scanRec, 0, n)
 	for i := range n {
 		recs = append(recs, scanRec{
-			size: int64(i), mtime: time.Unix(1, 0), head: testHash("h"),
-			tail: testHash("t"), path: fmt.Sprintf("/batch/%07d", i),
+			size: int64(i), mtime: time.Unix(1, 0), head: "h", tail: "t",
+			path: fmt.Sprintf("/batch/%07d", i),
 		})
 	}
 

exclude_test.go

@@ -17,9 +17,7 @@ import (
 func TestDefaultExcludes(t *testing.T) {
 	t.Parallel()
 
-	// README "Rules for the walk": the Linux and macOS lists, and none
-	// on any other system. The Windows list exists in the code only: the
-	// binary does not build for Windows, so the README leaves it out.
+	// README "Rules for the walk": each system's list.
 	cases := map[string][]string{
 		goosLinux: {"/proc", "/sys", "/dev", "/run", "/tmp", "/var/tmp"},
 		goosDarwin: {

info_test.go

@@ -12,25 +12,22 @@ import (
 // infoOpenLine is the plain line of info's open phase when it opens the
 // database at once; it prints it as the phase starts and as it ends.
 const infoOpenLine = "[1/2] open: opening the database, elapsed 0s, " +
-	"remain unknown"
+	"eta unknown"
 
 // infoCountLine is the plain line of info's count phase when the count
 // takes under a second; it prints it as the phase starts and as it
 // ends.
 const infoCountLine = "[2/2] count: counting the records, elapsed 0s, " +
-	"remain unknown"
+	"eta unknown"
 
 // infoRecs are four records: a duplicate pair, a file of 20 MiB and
 // another small file.
 func infoRecs() []scanRec {
-	h, tl, c, u := testHash("h"), testHash("t"), testHash("c"), testHash("u")
-	bigH, bigT, bigC := testHash("bh"), testHash("bt"), testHash("bc")
-
 	return []scanRec{
-		{size: 5, head: h, tail: tl, content: c, path: "/d/a"},
-		{size: 5, head: h, tail: tl, content: c, path: "/d/b"},
-		{size: 20 << 20, head: bigH, tail: bigT, content: bigC, path: "/d/big"},
-		{size: 7, head: u, tail: u, content: u, path: "/d/unique"},
+		{size: 5, head: "h", tail: "t", content: "c", path: "/d/a"},
+		{size: 5, head: "h", tail: "t", content: "c", path: "/d/b"},
+		{size: 20 << 20, head: "bh", tail: "bt", content: "bc", path: "/d/big"},
+		{size: 7, head: "u", tail: "u", content: "u", path: "/d/unique"},
 	}
 }
 

main_test.go

@@ -393,7 +393,7 @@ func TestRunNamesRelativeDatabase(t *testing.T) {
 }
 
 func TestRunUsesDatabaseThroughSymlinkedDirectory(t *testing.T) {
-	// README §Settings: the path names the file exactly. With link a
+	// README §Database: the path names the file exactly. With link a
 	// symlink to the directory real/sub, link/../db.sqlite names
 	// real/db.sqlite, not db.sqlite beside link. Every subcommand must
 	// use that file, relative or absolute, and its first line must name
@@ -648,7 +648,7 @@ func TestRunUsageErrors(t *testing.T) {
 }
 
 func TestRunScanRejectsWorkersBelowOne(t *testing.T) {
-	// README §Settings: --workers below 1 is a usage error reported in
+	// README §scan mode: --workers below 1 is a usage error reported in
 	// one line on stderr, before the scan opens the database.
 	for _, workers := range []string{"0", "-1"} {
 		t.Run(workers, func(t *testing.T) {
@@ -686,8 +686,8 @@ func TestRunScanRejectsWorkersBelowOne(t *testing.T) {
 func TestRunHelp(t *testing.T) {
 	t.Parallel()
 
-	// README §Settings: help goes to stderr, exits 0, and leaves stdout
-	// empty.
+	// README §Subcommands: help goes to stderr, exits 0, and leaves
+	// stdout empty.
 	cases := [][]string{{"--help"}, {"-h"}, {cmdScan, "--help"}}
 
 	for _, args := range cases {
@@ -713,8 +713,8 @@ func TestRunHelp(t *testing.T) {
 func TestRunVersion(t *testing.T) {
 	t.Parallel()
 
-	// README §Settings: the version is one line on stdout, with nothing
-	// on stderr, and exits 0.
+	// README §Subcommands: the version is one line on stdout, with
+	// nothing on stderr, and exits 0.
 	info, _ := debug.ReadBuildInfo()
 
 	for _, arg := range []string{"--version", "-v"} {
@@ -773,8 +773,8 @@ func TestBuildVersion(t *testing.T) {
 func TestRunVersionWriteFailureIsFatal(t *testing.T) {
 	t.Parallel()
 
-	// README §Error handling: a failed write to stdout is a fatal error,
-	// exit 1, reported on stderr starting with "sfdupes:".
+	// README §Error handling: a stdout write failure exits 1, reported
+	// in one line on stderr.
 	var stderr bytes.Buffer
 
 	code := run([]string{"--version"}, failingWriter{}, &stderr)
@@ -928,21 +928,24 @@ func TestRunScanWalksOperandUnderSymlinkOperand(t *testing.T) {
 }
 
 func TestRunScanSkipsZFSOperand(t *testing.T) {
-	zfs := filepath.Join(t.TempDir(), ".zfs")
-	f := writeFile(t, zfs, "snapshot/hourly/f", pattern(1, 10))
-
-	// No scan records a file inside a .zfs directory, so the record
-	// beneath the operand is written straight into the database.
-	path := seedDatabase(t, []scanRec{{size: 10, path: f}})
+	path := testDBPath(t)
 	t.Setenv(databaseEnv, path)
 
+	zfs := filepath.Join(t.TempDir(), ".zfs")
+	snapshot := filepath.Join(zfs, "snapshot", "hourly")
+	f := writeFile(t, snapshot, "f", pattern(1, 10))
+
+	// An operand beneath a .zfs directory is walked, because it is not
+	// itself named .zfs.
+	scanOK(t, snapshot)
+
 	assertOperandSkipped(t, path, zfs, ".zfs directory", f)
 }
 
 // assertOperandSkipped scans operand alone and checks that it is skipped
 // as kind: a warning naming it, one skip in the summary, exit 0, and the
-// record for kept, stored beneath operand before this scan, still in the
-// database at dbPath.
+// record for kept, which an earlier scan stored beneath operand, still
+// in the database at dbPath.
 func assertOperandSkipped(t *testing.T, dbPath, operand, kind,
 	kept string,
 ) {
@@ -971,121 +974,6 @@ func assertOperandSkipped(t *testing.T, dbPath, operand, kind,
 	recordByPath(t, dbRecords(t, db), kept)
 }
 
-// TestRunScanSkipsOperandInsideZFS scans a directory inside a .zfs
-// directory, first alone and then under another operand. Each time it
-// is warned about and counted as skipped, and nothing in it is walked.
-// The other operand is scanned as usual.
-func TestRunScanSkipsOperandInsideZFS(t *testing.T) {
-	path := testDBPath(t)
-	t.Setenv(databaseEnv, path)
-
-	dir := t.TempDir()
-	hourly := filepath.Join(dir, ".zfs", "snapshot", "hourly")
-	writeFile(t, hourly, "f", pattern(1, 10))
-	f := writeFile(t, dir, "f", pattern(2, 10))
-
-	warning := "walk " + hourly + ": skipping operand inside a .zfs directory\n"
-
-	for _, tc := range []struct {
-		operands []string
-		summary  string
-	}{
-		{
-			[]string{hourly},
-			"scan: 0 files seen (0 added, 0 updated, 0 removed, " +
-				"0 unchanged), 1 skipped\n",
-		},
-		{
-			[]string{dir, hourly},
-			"scan: 1 files seen (1 added, 0 updated, 0 removed, " +
-				"0 unchanged), 1 skipped\n",
-		},
-	} {
-		stderr := scanOK(t, tc.operands...)
-		if !strings.Contains(stderr, warning) ||
-			!strings.Contains(stderr, tc.summary) {
-			t.Errorf("scan %q: stderr = %q, want %q and %q", tc.operands,
-				stderr, warning, tc.summary)
-		}
-	}
-
-	db, err := openDB(path, reportParams)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	t.Cleanup(func() { _ = db.Close() })
-
-	if got := recordPaths(dbRecords(t, db)); !slices.Equal(got, []string{f}) {
-		t.Errorf("records = %q, want %q", got, []string{f})
-	}
-}
-
-// TestRunScanSkipsOperandThroughSymlinkIntoZFS scans snaps/daily, where
-// snaps is a symlink to pool/.zfs/snapshot. The operand is warned about
-// by the path given, counted as skipped, and not walked.
-func TestRunScanSkipsOperandThroughSymlinkIntoZFS(t *testing.T) {
-	t.Setenv(databaseEnv, testDBPath(t))
-
-	dir := t.TempDir()
-	writeFile(t, dir, "pool/.zfs/snapshot/daily/f", pattern(1, 10))
-
-	snaps := filepath.Join(dir, "snaps")
-
-	err := os.Symlink(filepath.Join(dir, "pool", ".zfs", "snapshot"), snaps)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	daily := filepath.Join(snaps, "daily")
-	stderr := scanOK(t, daily)
-
-	warning := "walk " + daily + ": skipping operand inside a .zfs directory\n"
-	summary := "scan: 0 files seen (0 added, 0 updated, 0 removed, " +
-		"0 unchanged), 1 skipped\n"
-
-	if !strings.Contains(stderr, warning) ||
-		!strings.Contains(stderr, summary) {
-		t.Errorf("stderr = %q, want %q and %q", stderr, warning, summary)
-	}
-}
-
-// TestRunScanWalksOperandThroughSymlink scans data/daily, where data is
-// a symlink to a directory outside any .zfs directory. The operand is
-// scanned as usual, and its file is recorded by the path given.
-func TestRunScanWalksOperandThroughSymlink(t *testing.T) {
-	path := testDBPath(t)
-	t.Setenv(databaseEnv, path)
-
-	dir := t.TempDir()
-	writeFile(t, dir, "pool/data/daily/f", pattern(1, 10))
-
-	data := filepath.Join(dir, "data")
-
-	err := os.Symlink(filepath.Join(dir, "pool", "data"), data)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	daily := filepath.Join(data, "daily")
-	stderr := scanOK(t, daily)
-
-	summary := "scan: 1 files seen (1 added, 0 updated, 0 removed, " +
-		"0 unchanged), 0 skipped\n"
-	if !strings.Contains(stderr, summary) {
-		t.Errorf("stderr = %q, want %q", stderr, summary)
-	}
-
-	db, err := openDB(path, reportParams)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	t.Cleanup(func() { _ = db.Close() })
-
-	recordByPath(t, dbRecords(t, db), filepath.Join(daily, "f"))
-}
-
 func TestRunReportSucceeds(t *testing.T) {
 	path := testDBPath(t)
 	t.Setenv(databaseEnv, path)
@@ -1234,7 +1122,7 @@ func assertRunsUseDatabase(t *testing.T, dir, name string) {
 }
 
 func TestRunDatabasePathUsedAsGiven(t *testing.T) {
-	// README §Settings: the path names the database file exactly. In
+	// README §Database: the path names the database file exactly. In
 	// SQLite's connection string an unescaped ? or # would end the file
 	// name and % would start an escape, and a path starting with //
 	// could be read as a host name. %25 is a valid escape, so unescaped
@@ -1279,8 +1167,9 @@ func holdScanLock(t *testing.T, path string) {
 }
 
 func TestRunSecondScanFails(t *testing.T) {
-	// README §Database: a second scan that finds the lock held fails
-	// at once, naming the lock file, without opening the database.
+	// README §Database: while one scan holds the lock, a second scan
+	// fails at once, naming the lock file, without opening the
+	// database.
 	path := testDBPath(t)
 	t.Setenv(databaseEnv, path)
 
@@ -1339,7 +1228,7 @@ func assertSecondScanFails(t *testing.T, path string) {
 }
 
 func TestRunSecondScanThroughSymlinkFails(t *testing.T) {
-	// README §Database: a path through a symlink names the same lock
+	// README §Database: every path to one database names the same lock
 	// file, so while a scan holds it through one path, a scan through
 	// another fails, naming the lock file beside the database's real
 	// path.
@@ -1444,8 +1333,8 @@ func TestRunReportsDuringScan(t *testing.T) {
 }
 
 func TestRunStdoutClosedIsFatal(t *testing.T) {
-	// README §Error handling: a failed write to stdout is a fatal error,
-	// exit 1, reported on stderr starting with "sfdupes:".
+	// README §Error handling: a stdout write failure exits 1, reported
+	// in one line on stderr.
 	for _, name := range []string{cmdReport, cmdTrees, cmdInfo} {
 		t.Run(name, func(t *testing.T) {
 			t.Setenv(databaseEnv, testDBPath(t))

progress.go

@@ -9,7 +9,6 @@ import (
 	"strings"
 	"sync"
 	"time"
-	"unicode/utf8"
 
 	"github.com/schollz/progressbar/v3"
 	"golang.org/x/term"
@@ -35,17 +34,10 @@ const stallTime = 10 * time.Second
 // spinnerType selects the progressbar library's spinner style.
 const spinnerType = 14
 
-// barWidth is the width of a phase's bar between its two ends, as in
-// the example line in README "Progress".
+// barWidth is the width of the progressbar library's bar between its
+// two ends, as in the example line in README "Progress".
 const barWidth = 10
 
-// barCell is one column of the filled part of a bar.
-const barCell = "█"
-
-// barColumns is the width of what percentBar returns: the percent in
-// four columns and its sign, a space, and the bar with its two ends.
-const barColumns = barWidth + 8
-
 // percentScale converts a fraction to a percentage.
 const percentScale = 100
 
@@ -53,12 +45,17 @@ const percentScale = 100
 // read, as the progressbar library takes it.
 const fallbackWidth = 80
 
-// spinnerMargin is the columns of a line besides its text: the spinner
-// and the three spaces the library draws around the text, and the
-// terminal's last column, which every line stays clear of so that no
-// terminal wraps it.
+// spinnerMargin is the columns of a spinner's line besides its text:
+// the spinner and the three spaces the library draws around the text,
+// and the terminal's last column, which every line stays clear of so
+// that no terminal wraps it.
 const spinnerMargin = 5
 
+// barMargin is the columns of a bar's line besides its text: the
+// percent and the bar the library draws after the text, the spaces
+// around them, and the terminal's last column.
+const barMargin = barWidth + 11
+
 // countingRecords is what the display of a phase that reads the records
 // says while it counts them, before it has its total.
 const countingRecords = "counting the records"
@@ -112,13 +109,13 @@ type drawnCount struct {
 }
 
 // progress renders one phase's progress on stderr. On a TTY the
-// progressbar library draws its spinner and then line, fitted to the
-// terminal's width. A phase with an exact total ends the line with its
-// percent and bar, at the terminal's right edge, when the terminal has
-// room for them. It is drawn again every redrawInterval, so its elapsed
-// time keeps moving while the phase waits for its next item. When
-// stderr is not a TTY it emits no ANSI redraws: it prints line as the
-// phase starts, every plainInterval while it runs, and as it ends.
+// progressbar library draws line, fitted to the terminal's width: after
+// the library's spinner, or, for a phase with an exact total, before the
+// library's bar when the terminal has room for the bar beside it. It is
+// drawn again every redrawInterval, so its elapsed time keeps moving
+// while the phase waits for its next item. When stderr is not a TTY it
+// emits no ANSI redraws: it prints line as the phase starts, every
+// plainInterval while it runs, and as it ends.
 //
 // The draws come from a goroutine of their own, so mu guards the fields
 // below it. Every method must be called from the main goroutine. A nil
@@ -134,6 +131,7 @@ type progress struct {
 
 	mu         sync.Mutex
 	bar        *progressbar.ProgressBar // what the library draws; nil until then
+	withBar    bool                     // bar is the library's bar, not its spinner
 	widest     int                      // the widest line bar has drawn, in columns
 	label      string
 	unit       string
@@ -143,7 +141,7 @@ type progress struct {
 	stalledOn  string
 	count      int64
 	start      time.Time // when the phase started, for the elapsed time
-	rateStart  time.Time // when the count started, for the time remaining
+	rateStart  time.Time // when the count started, for the ETA
 	lastItem   time.Time // when the last item was completed, or the count started
 	// drawn holds the count at each draw over the last rateWindow, oldest
 	// first; the rate counts from the first. It keeps the draw before the
@@ -217,34 +215,55 @@ func (p *progress) draw() {
 	}
 
 	if p.width == nil {
-		fmt.Fprintln(os.Stderr, p.line(now, math.MaxInt, false))
+		fmt.Fprintln(os.Stderr, p.line(now, math.MaxInt, true))
 
 		return
 	}
 
 	width := p.width()
-	text := p.line(now, width-spinnerMargin, true)
 
-	// Before each draw the library erases its line with as many spaces as
-	// the widest line it has drawn. A display gets a new one when the
-	// terminal has been made too narrow for that erase, which would wrap
-	// and leave a line behind at every draw.
-	if p.bar == nil || p.widest >= width {
-		p.newBar(width)
+	// A phase with an exact total gets the library's bar when its whole
+	// line, without the percent the bar shows, fits beside the bar: the
+	// bar gives way before any part of the line. A total of 0 leaves the
+	// bar nothing to fill.
+	text, withBar := "", false
+	if p.total > 0 && !p.estimate {
+		text = p.line(now, math.MaxInt, false)
+		withBar = len(text) <= width-barMargin
+	}
+
+	margin := barMargin
+	if !withBar {
+		margin = spinnerMargin
+		text = p.line(now, width-margin, true)
+	}
+
+	// The library makes a bar or a spinner, and before each draw erases
+	// its line with as many spaces as the widest line it has drawn. A
+	// display gets a new one when it turns from one to the other, as its
+	// line grows or the terminal is resized, and when the terminal has
+	// been made too narrow for that erase, which would wrap and leave a
+	// line behind at every draw.
+	if p.bar == nil || p.withBar != withBar || p.widest >= width {
+		p.newBar(withBar, width)
+	}
+
+	if withBar {
+		_ = p.bar.Set64(p.count)
 	}
 
 	p.bar.Describe(text)
 
-	// The library drew the spinner, the text and the spaces around it,
-	// less the terminal's last column, which spinnerMargin keeps clear. A
-	// bar's cells take one column each but more than one byte.
-	p.widest = max(p.widest, utf8.RuneCountInString(text)+spinnerMargin-1)
+	// The library drew the text and its margin, less the terminal's last
+	// column, which the margin keeps clear.
+	p.widest = max(p.widest, len(text)+margin-1)
 }
 
-// newBar has the library make what draw draws: its spinner, followed by
-// the text draw gives it and nothing of the library's own. p.mu must be
-// held.
-func (p *progress) newBar(width int) {
+// newBar has the library make what draw draws: a bar for the phase's
+// total when withBar is set, and a spinner when not. Either shows the
+// text draw gives it, and nothing of its own besides the spinner, or
+// the bar and its percent. p.mu must be held.
+func (p *progress) newBar(withBar bool, width int) {
 	if p.bar != nil {
 		// Erase what was drawn before, short of the last column of a
 		// terminal that may have been made narrower since: the library's
@@ -253,36 +272,29 @@ func (p *progress) newBar(width int) {
 		fmt.Fprint(os.Stderr, "\r"+blank+"\r")
 	}
 
-	// A total of -1 gives the library's spinner.
-	p.bar = progressbar.NewOptions64(-1,
+	total := int64(-1) // the library's spinner
+	if withBar {
+		total = p.total
+	}
+
+	p.bar = progressbar.NewOptions64(total,
 		progressbar.OptionSetWriter(os.Stderr),
+		progressbar.OptionSetWidth(barWidth),
 		progressbar.OptionSpinnerType(spinnerType),
 		// The spinner turns at each draw, not on a timer of the
 		// library's own.
 		progressbar.OptionSetSpinnerChangeInterval(0),
-		// The text carries the elapsed time and the time remaining.
+		// The text carries the elapsed time and the ETA.
 		progressbar.OptionSetElapsedTime(false),
 		progressbar.OptionSetPredictTime(false),
 	)
-	p.widest = 0
-}
-
-// percentBar returns the percent and the bar that end the text of a
-// phase with an exact total, as "  25% |██        |": the bar's
-// barWidth columns fill in proportion to the items done. A count past
-// the total, as when a scan adds records while trees reads them, shows
-// as 100%. p.mu must be held.
-func (p *progress) percentBar() string {
-	done := min(float64(p.count)/float64(p.total), 1)
-	filled := strings.Repeat(barCell, int(done*barWidth))
-
-	return fmt.Sprintf("%4.0f%% |%-*s|", done*percentScale, barWidth, filled)
+	p.withBar, p.widest = withBar, 0
 }
 
 // setTotal gives a phase that started without a total its total, once
-// it is known. The rate, the time remaining and the time with no item
-// completed count from then on: the time spent waiting for the total
-// says nothing about how fast items go.
+// it is known. The rate, the ETA and the time with no item completed
+// count from then on: the time spent waiting for the total says nothing
+// about how fast items go.
 func (p *progress) setTotal(total int64) {
 	if p == nil {
 		return
@@ -426,26 +438,22 @@ func (p *progress) finish() {
 // line formats the phase's progress as of now in at most room columns,
 // as
 //
-//	[3/4] hash: [1234/5000] 25%, 300 files/s, elapsed 4s, remain 13s
+//	[3/4] hash: [1234/5000] 25%, 300 files/s, elapsed 4s, eta 13s
 //
-// where "files" is the phase's unit. A phase with no total shows the
-// items done instead, or what it waits for. The rate is that since the
-// first draw in drawn, and the time remaining comes from the rate since
-// the count started. Once no item has been completed for stallTime, the
-// rate, which would read 0, gives way to the word "stalled" and where
-// the phase waits, as "stalled disk/db".
+// where "files" is the phase's unit, without the percent when percent
+// is false, as before the library's bar, which shows its own. A phase
+// with no total shows the items done instead, or what it waits for. The
+// rate is that since the first draw in drawn, and the ETA comes from
+// the rate since the count started. Once no item has been completed for
+// stallTime, the rate, which would read 0, gives way to the word
+// "stalled" and where the phase waits, as "stalled disk/db".
 //
-// When bar is true, as on a terminal, a phase with an exact total ends
-// the line with its percent and bar, in place of the percent after its
-// count. A total of 0 leaves the bar nothing to fill.
-//
-// The phase and its place, the progress, the elapsed time and the time
-// remaining are kept whole. A line wider than room first leaves out the
-// rate, the words of a stalled phase, or what the phase waits for. Next
-// it leaves out the bar. Only if it is still wider is it cut at room.
-// Every part but the bar is ASCII, so its length is its width in
-// columns. p.mu must be held.
-func (p *progress) line(now time.Time, room int, bar bool) string {
+// The phase and its place, the progress, the elapsed time and the ETA
+// are kept whole. A line wider than room leaves out the rate, the words
+// of a stalled phase, or what the phase waits for; only if it is still
+// wider is it cut at room. Every part is ASCII, so its length is its
+// width in columns. p.mu must be held.
+func (p *progress) line(now time.Time, room int, percent bool) string {
 	elapsed := now.Sub(p.start).Round(time.Second)
 
 	// head and tail hold the parts kept whole; extra, between them, is
@@ -454,41 +462,21 @@ func (p *progress) line(now time.Time, room int, bar bool) string {
 
 	if p.total < 0 && p.waitingFor != "" {
 		head, extra = p.label+": ", p.waitingFor+", "
-		tail = fmt.Sprintf("elapsed %s, remain unknown", elapsed)
+		tail = fmt.Sprintf("elapsed %s, eta unknown", elapsed)
 	} else {
 		since := p.drawn[0]
 		rate := perSecond(p.count-since.count, now.Sub(since.at))
 		average := perSecond(p.count, now.Sub(p.rateStart))
 
-		head = p.label + ": " + p.itemsDone(true) + ", "
+		head = p.label + ": " + p.itemsDone(percent) + ", "
 		extra = fmt.Sprintf("%.0f %s/s, ", rate, p.unit)
-		tail = fmt.Sprintf("elapsed %s, remain %s", elapsed, p.remain(average))
+		tail = fmt.Sprintf("elapsed %s, eta %s", elapsed, p.eta(average))
 
 		if now.Sub(p.lastItem) >= stallTime {
 			extra = "stalled " + p.stalledOn + ", "
 		}
 	}
 
-	if bar && p.total > 0 && !p.estimate {
-		// Beside the bar, which shows the percent, the count shows none.
-		// The line is padded to the room the bar leaves it, so the bar
-		// stays at room's right edge however long the line is.
-		beside := room - barColumns
-		barHead := p.label + ": " + p.itemsDone(false) + ", "
-
-		switch {
-		case len(barHead+extra+tail) <= beside:
-			return fmt.Sprintf("%-*s%s", beside, barHead+extra+tail,
-				p.percentBar())
-		case len(barHead+tail) <= beside:
-			return fmt.Sprintf("%-*s%s", beside, barHead+tail, p.percentBar())
-		}
-
-		// The bar gives way. The rate, or the words of a stalled phase,
-		// gave way before it and stay out.
-		extra = ""
-	}
-
 	full, kept := head+extra+tail, head+tail
 
 	switch {
@@ -538,10 +526,10 @@ func (p *progress) itemsDone(percent bool) string {
 	return fmt.Sprintf("%s %.0f%%", done, pct)
 }
 
-// remain returns the time the phase has left at rate items a second, or
+// eta returns the time the phase has left at rate items a second, or
 // "unknown": with no total, before the first item, and once the count
 // passes an estimate. p.mu must be held.
-func (p *progress) remain(rate float64) string {
+func (p *progress) eta(rate float64) string {
 	switch {
 	case p.total < 0, p.estimate && p.count > p.total:
 		return "unknown"
@@ -551,8 +539,8 @@ func (p *progress) remain(rate float64) string {
 		return "unknown"
 	}
 
-	left := time.Duration(
+	remaining := time.Duration(
 		float64(p.total-p.count) / rate * float64(time.Second))
 
-	return left.Round(time.Second).String()
+	return remaining.Round(time.Second).String()
 }

progress_test.go

@@ -117,24 +117,24 @@ func TestProgressLine(t *testing.T) {
 			label: scanPhase("load"), total: -1,
 			waitingFor: countingRecords,
 		},
-		want: "[1/4] load: counting the records, elapsed 10s, remain unknown",
+		want: "[1/4] load: counting the records, elapsed 10s, eta unknown",
 	}, {
 		p: &progress{
 			label: scanPhase("hash"), unit: scanUnit, total: 100, count: 20,
 		},
-		want: "[3/4] hash: [20/100] 20%, 2 files/s, elapsed 10s, remain 40s",
+		want: "[3/4] hash: [20/100] 20%, 2 files/s, elapsed 10s, eta 40s",
 	}, {
 		p:    &progress{label: scanPhase("hash"), unit: scanUnit, total: 100},
-		want: "[3/4] hash: [0/100] 0%, 0 files/s, elapsed 10s, remain unknown",
+		want: "[3/4] hash: [0/100] 0%, 0 files/s, elapsed 10s, eta unknown",
 	}, {
 		p:    &progress{label: scanPhase("update"), unit: scanUnit, total: 0},
-		want: "[4/4] update: [0/0] 100%, 0 files/s, elapsed 10s, remain 0s",
+		want: "[4/4] update: [0/0] 100%, 0 files/s, elapsed 10s, eta 0s",
 	}, {
 		p: &progress{
 			label: scanPhase("walk"), unit: scanUnit, total: 100,
 			estimate: true, count: 20,
 		},
-		want: "[2/4] walk: [20/~100] 20%, 2 files/s, elapsed 10s, remain 40s",
+		want: "[2/4] walk: [20/~100] 20%, 2 files/s, elapsed 10s, eta 40s",
 	}, {
 		// Past the estimate the walk cannot tell how long it has left.
 		p: &progress{
@@ -142,13 +142,13 @@ func TestProgressLine(t *testing.T) {
 			estimate: true, count: 120,
 		},
 		want: "[2/4] walk: [120/~100] 120%, 12 files/s, elapsed 10s, " +
-			"remain unknown",
+			"eta unknown",
 	}, {
 		// A first scan's walk has no estimate.
 		p: &progress{
 			label: scanPhase("walk"), unit: scanUnit, total: -1, count: 20,
 		},
-		want: "[2/4] walk: 20 files, 2 files/s, elapsed 10s, remain unknown",
+		want: "[2/4] walk: 20 files, 2 files/s, elapsed 10s, eta unknown",
 	}, {
 		// The rate is in what the phase counts.
 		p: &progress{
@@ -156,14 +156,14 @@ func TestProgressLine(t *testing.T) {
 			count: 20,
 		},
 		want: "[3/4] group: [20/100] 20%, 2 directories/s, elapsed 10s, " +
-			"remain 40s",
+			"eta 40s",
 	}}
 
 	for _, tc := range cases {
 		tc.p.start, tc.p.rateStart, tc.p.lastItem = ago, ago, now
 		tc.p.drawn = []drawnCount{{at: ago}}
 
-		if got := tc.p.line(now, math.MaxInt, false); got != tc.want {
+		if got := tc.p.line(now, math.MaxInt, true); got != tc.want {
 			t.Errorf("line = %q, want %q", got, tc.want)
 		}
 	}
@@ -198,49 +198,48 @@ func TestLineFitsRoom(t *testing.T) {
 	}
 
 	cases := []struct {
-		p    *progress
-		room int
-		bar  bool
-		want string
+		p       *progress
+		room    int
+		percent bool
+		want    string
 	}{{
-		p: hash, room: 60,
-		want: "[3/4] hash: [20/100] 20%, 2 files/s, elapsed 10s, remain 40s",
+		p: hash, room: 57, percent: true,
+		want: "[3/4] hash: [20/100] 20%, 2 files/s, elapsed 10s, eta 40s",
 	}, {
-		p: hash, room: 59,
-		want: "[3/4] hash: [20/100] 20%, elapsed 10s, remain 40s",
+		p: hash, room: 56, percent: true,
+		want: "[3/4] hash: [20/100] 20%, elapsed 10s, eta 40s",
 	}, {
-		p: hash, room: 48,
-		want: "[3/4] hash: [20/100] 20%, elapsed 10s, remain 40",
+		p: hash, room: 45, percent: true,
+		want: "[3/4] hash: [20/100] 20%, elapsed 10s, eta 40",
 	}, {
-		// The bar shows the percent.
-		p: hash, room: 74, bar: true,
-		want: "[3/4] hash: [20/100], 2 files/s, elapsed 10s, remain 40s" +
-			"  20% |██        |",
+		// Before the library's bar, which shows the percent.
+		p: hash, room: 57, percent: false,
+		want: "[3/4] hash: [20/100], 2 files/s, elapsed 10s, eta 40s",
 	}, {
-		p: load, room: 43,
-		want: "[1/4] load: elapsed 10s, remain unknown",
+		p: load, room: 40, percent: true,
+		want: "[1/4] load: elapsed 10s, eta unknown",
 	}, {
-		p: stalled, room: 68,
+		p: stalled, room: 65, percent: true,
 		want: "[3/4] hash: [20/100] 20%, stalled disk/db, elapsed 50s, " +
-			"remain 3m20s",
+			"eta 3m20s",
 	}, {
-		p: stalled, room: 67,
-		want: "[3/4] hash: [20/100] 20%, elapsed 50s, remain 3m20s",
+		p: stalled, room: 64, percent: true,
+		want: "[3/4] hash: [20/100] 20%, elapsed 50s, eta 3m20s",
 	}}
 
 	for _, tc := range cases {
-		if got := tc.p.line(now, tc.room, tc.bar); got != tc.want {
-			t.Errorf("line(%d, %t) = %q, want %q", tc.room, tc.bar, got,
-				tc.want)
+		if got := tc.p.line(now, tc.room, tc.percent); got != tc.want {
+			t.Errorf("line(%d, %t) = %q, want %q",
+				tc.room, tc.percent, got, tc.want)
 		}
 	}
 }
 
 // TestSetTotalStartsRate checks that setTotal starts a phase's rate,
-// its time remaining and its time with no record read from when the
-// total becomes known, not from when the phase started waiting for it:
-// after 20 seconds of waiting for the total, the phase has not stalled 5
-// seconds after it, and has 10 seconds after it with no record read.
+// its ETA and its time with no record read from when the total becomes
+// known, not from when the phase started waiting for it: after 20
+// seconds of waiting for the total, the phase has not stalled 5 seconds
+// after it, and has 10 seconds after it with no record read.
 func TestSetTotalStartsRate(t *testing.T) {
 	t.Parallel()
 
@@ -256,15 +255,15 @@ func TestSetTotalStartsRate(t *testing.T) {
 
 	clock.Add(5 * time.Second)
 
-	want := "[1/4] load: [0/100] 0%, 0 files/s, elapsed 25s, remain unknown"
-	if got := p.line(clock.Now(), math.MaxInt, false); got != want {
+	want := "[1/4] load: [0/100] 0%, 0 files/s, elapsed 25s, eta unknown"
+	if got := p.line(clock.Now(), math.MaxInt, true); got != want {
 		t.Errorf("line = %q, want %q", got, want)
 	}
 
 	clock.Add(5 * time.Second)
 
-	want = "[1/4] load: [0/100] 0%, stalled db, elapsed 30s, remain unknown"
-	if got := p.line(clock.Now(), math.MaxInt, false); got != want {
+	want = "[1/4] load: [0/100] 0%, stalled db, elapsed 30s, eta unknown"
+	if got := p.line(clock.Now(), math.MaxInt, true); got != want {
 		t.Errorf("line = %q, want %q", got, want)
 	}
 
@@ -273,8 +272,8 @@ func TestSetTotalStartsRate(t *testing.T) {
 		p.increment()
 	}
 
-	want = "[1/4] load: [20/100] 20%, 2 files/s, elapsed 30s, remain 40s"
-	if got := p.line(clock.Now(), math.MaxInt, false); got != want {
+	want = "[1/4] load: [20/100] 20%, 2 files/s, elapsed 30s, eta 40s"
+	if got := p.line(clock.Now(), math.MaxInt, true); got != want {
 		t.Errorf("line = %q, want %q", got, want)
 	}
 }
@@ -317,7 +316,7 @@ func TestPlainProgressPrintsAtStartAndEnd(t *testing.T) {
 
 	lines := strings.Split(strings.TrimSuffix(stderr(), "\n"), "\n")
 
-	want := "[3/4] hash: [0/1000] 0%, 0 files/s, elapsed 0s, remain unknown"
+	want := "[3/4] hash: [0/1000] 0%, 0 files/s, elapsed 0s, eta unknown"
 	if len(lines) != 2 || lines[0] != want ||
 		!strings.HasPrefix(lines[1], "[3/4] hash: [1000/1000] 100%, ") {
 		t.Errorf("stderr lines = %q, want %q, then the line at 1000",
@@ -337,7 +336,7 @@ func TestPlainProgressPrintsWhileWaiting(t *testing.T) {
 	}, nil, testRedraw, time.Now)
 
 	awaitOutput(t, stderr,
-		"[1/4] load: counting the records, elapsed 0s, remain unknown\n", 3)
+		"[1/4] load: counting the records, elapsed 0s, eta unknown\n", 3)
 	p.finish()
 }
 
@@ -367,23 +366,23 @@ func TestPlainProgressStalledProducer(t *testing.T) {
 
 	clock.Add(5 * time.Second)
 	awaitOutput(t, stderr,
-		"[3/4] hash: [20/100] 20%, 4 files/s, elapsed 5s, remain 20s\n", 3)
+		"[3/4] hash: [20/100] 20%, 4 files/s, elapsed 5s, eta 20s\n", 3)
 
 	clock.Add(time.Second)
 	awaitOutput(t, stderr,
-		"[3/4] hash: [20/100] 20%, 0 files/s, elapsed 6s, remain 24s\n", 3)
+		"[3/4] hash: [20/100] 20%, 0 files/s, elapsed 6s, eta 24s\n", 3)
 
 	clock.Add(4 * time.Second)
 	awaitOutput(t, stderr,
-		"[3/4] hash: [20/100] 20%, stalled disk/db, elapsed 10s, remain 40s\n",
+		"[3/4] hash: [20/100] 20%, stalled disk/db, elapsed 10s, eta 40s\n",
 		3)
 }
 
 // TestPlainProgressRateAfterStall draws a phase's plain line by hand, a
 // little over plainInterval after the line before, as a late tick
 // does, through a stall and a burst of files after it: the rate covers
-// the files read since the line before, while the time remaining comes
-// from the phase's average.
+// the files read since the line before, while the ETA comes from the
+// phase's average.
 //
 //nolint:paralleltest // captureStderr replaces the process-wide os.Stderr
 func TestPlainProgressRateAfterStall(t *testing.T) {
@@ -411,10 +410,10 @@ func TestPlainProgressRateAfterStall(t *testing.T) {
 	p.finish()
 
 	want := []string{
-		"[3/4] hash: [0/1000] 0%, 0 files/s, elapsed 0s, remain unknown",
-		"[3/4] hash: [0/1000] 0%, 0 files/s, elapsed 5s, remain unknown",
-		"[3/4] hash: [0/1000] 0%, stalled disk/db, elapsed 10s, remain unknown",
-		"[3/4] hash: [100/1000] 10%, 20 files/s, elapsed 15s, remain 2m15s",
+		"[3/4] hash: [0/1000] 0%, 0 files/s, elapsed 0s, eta unknown",
+		"[3/4] hash: [0/1000] 0%, 0 files/s, elapsed 5s, eta unknown",
+		"[3/4] hash: [0/1000] 0%, stalled disk/db, elapsed 10s, eta unknown",
+		"[3/4] hash: [100/1000] 10%, 20 files/s, elapsed 15s, eta 2m15s",
 	}
 
 	lines := strings.Split(strings.TrimSuffix(stderr(), "\n"), "\n")
@@ -427,8 +426,8 @@ func TestPlainProgressRateAfterStall(t *testing.T) {
 // redrawInterval, as a terminal does, through a stall of stallTime and
 // a burst of files after it: the rate covers the files read over the
 // last rateWindow, and falls to 0 once the burst is older than that,
-// while the time remaining comes from the phase's average. Plain lines
-// show each draw.
+// while the ETA comes from the phase's average. Plain lines show each
+// draw.
 //
 //nolint:paralleltest // captureStderr replaces the process-wide os.Stderr
 func TestProgressRateAfterStallRedrawn(t *testing.T) {
@@ -458,8 +457,8 @@ func TestProgressRateAfterStallRedrawn(t *testing.T) {
 	redraw(rateWindow + redrawInterval)
 	p.finish()
 
-	want := "[3/4] hash: [50/1000] 5%, 10 files/s, elapsed 15s, remain 4m45s\n" +
-		"[3/4] hash: [50/1000] 5%, 0 files/s, elapsed 15s, remain 4m47s\n"
+	want := "[3/4] hash: [50/1000] 5%, 10 files/s, elapsed 15s, eta 4m45s\n" +
+		"[3/4] hash: [50/1000] 5%, 0 files/s, elapsed 15s, eta 4m47s\n"
 	if !strings.Contains(stderr(), want) {
 		t.Errorf("stderr = %q, want the lines %q", stderr(), want)
 	}
@@ -481,10 +480,10 @@ func lastFrame(out string) string {
 }
 
 // TestTerminalRedrawsWhileWaiting checks that the terminal display of a
-// phase, with a bar or without, keeps being drawn while the phase waits
-// for its next item, with the elapsed time as of now, and shows every
-// item completed before the wait. The terminal is 120 columns wide, so
-// the hash phase's display has room for its bar.
+// phase, spinner or bar, keeps being drawn while the phase waits for its
+// next item, with the elapsed time as of now, and shows every item
+// completed before the wait. stderr is not a terminal here, so the
+// displays take it to be fallbackWidth wide.
 //
 //nolint:paralleltest // captureStderr replaces the process-wide os.Stderr
 func TestTerminalRedrawsWhileWaiting(t *testing.T) {
@@ -507,8 +506,7 @@ func TestTerminalRedrawsWhileWaiting(t *testing.T) {
 	for _, tc := range cases {
 		before := len(stderr())
 		since := func() string { return stderr()[before:] }
-		p := startProgress(tc.params, func() int { return 120 }, testRedraw,
-			time.Now)
+		p := startProgress(tc.params, terminalWidth, testRedraw, time.Now)
 
 		for range 50 {
 			p.increment()
@@ -554,13 +552,13 @@ func drawLate(stderr func() string, params progressParams, width int,
 	return stderr()[before:]
 }
 
-// TestTerminalLinesKeepTimeRemaining checks the line each display shows
-// on an 80-column terminal at a large scan's counts and times, and the
-// line of trees' read phase at the same counts: the rate is left out,
-// so that the elapsed time and the time remaining stay whole.
+// TestTerminalLinesKeepETA checks the line each display shows on an
+// 80-column terminal at a large scan's counts and times, and the line
+// of trees' read phase at the same counts: the rate is left out, so
+// that the elapsed time and the ETA stay whole.
 //
 //nolint:paralleltest // captureStderr replaces the process-wide os.Stderr
-func TestTerminalLinesKeepTimeRemaining(t *testing.T) {
+func TestTerminalLinesKeepETA(t *testing.T) {
 	stderr := captureStderr(t)
 
 	cases := []struct {
@@ -573,19 +571,19 @@ func TestTerminalLinesKeepTimeRemaining(t *testing.T) {
 			estimate: true,
 		},
 		want: "[2/4] walk: [1234567/~1200000] 103%, elapsed 2h30m0s, " +
-			"remain unknown",
+			"eta unknown",
 	}, {
 		params: progressParams{
 			label: scanPhase("hash"), unit: scanUnit, total: 9876543,
 		},
 		want: "[3/4] hash: [1234567/9876543] 12%, elapsed 2h30m0s, " +
-			"remain 17h30m0s",
+			"eta 17h30m0s",
 	}, {
 		params: progressParams{
 			label: treesPhase("read"), unit: "records", total: 9876543,
 		},
 		want: "[2/4] read: [1234567/9876543] 12%, elapsed 2h30m0s, " +
-			"remain 17h30m0s",
+			"eta 17h30m0s",
 	}}
 
 	for _, tc := range cases {
@@ -596,32 +594,30 @@ func TestTerminalLinesKeepTimeRemaining(t *testing.T) {
 	}
 }
 
-// TestTerminalStalledLines draws every phase's line once the phase has
-// stalled, its last item done 42 seconds before. At an ordinary scan's
-// counts and times, 12345 of 45678 items done four and a half minutes
-// in, on a terminal 100 columns wide, the line shows where the phase
-// waits whole, beside the phase, progress, elapsed time and time
-// remaining. At a large scan's, 1234567 of 9876543 two and a half hours
-// in, on an 80-column terminal, those four still show whole, as the
-// words give way first where they do not fit.
+// TestTerminalStalledLines draws every phase's line on an 80-column
+// terminal once the phase has stalled, its last item done 42 seconds
+// before. At an ordinary scan's counts and times, 12345 of 45678 items
+// done four and a half minutes in, the line shows where the phase waits
+// whole, beside the phase, progress, elapsed time and ETA. At a large
+// scan's, 1234567 of 9876543 two and a half hours in, those four still
+// show whole, as the words give way first where they do not fit.
 //
 //nolint:paralleltest // captureStderr replaces the process-wide os.Stderr
 func TestTerminalStalledLines(t *testing.T) {
 	stderr := captureStderr(t)
 
-	// drawStalled draws the display params describes on a terminal width
-	// columns wide as it starts and once more as it ends, with count of
-	// total items done elapsed into the phase, and returns what the
-	// terminal shows.
-	drawStalled := func(params progressParams, width int, total,
-		count int64, elapsed time.Duration,
+	// drawStalled draws the display params describes as it starts and
+	// once more as it ends, with count of total items done elapsed into
+	// the phase, and returns what the terminal shows.
+	drawStalled := func(params progressParams, total, count int64,
+		elapsed time.Duration,
 	) string {
 		before := len(stderr())
 		clock := &testClock{now: time.Now()}
 		params.total = total
 
 		// No timed draws come between the two.
-		p := startProgress(params, func() int { return width }, time.Hour,
+		p := startProgress(params, func() int { return 80 }, time.Hour,
 			clock.Now)
 
 		clock.Add(elapsed - 42*time.Second)
@@ -653,22 +649,19 @@ func TestTerminalStalledLines(t *testing.T) {
 			approx = "~"
 		}
 
-		// The percent is in the line, or beside the bar.
-		shown := drawStalled(params, 100, 45678, 12345,
+		shown := drawStalled(params, 45678, 12345,
 			4*time.Minute+30*time.Second)
-		for _, want := range []string{
-			params.label + ": [12345/" + approx + "45678]",
-			", stalled " + params.stalledOn + ", elapsed 4m30s, remain 12m9s",
-		} {
-			if !strings.Contains(shown, want) {
-				t.Errorf("terminal shows %q, want %q", shown, want)
-			}
+		want := params.label + ": [12345/" + approx + "45678] 27%, stalled " +
+			params.stalledOn + ", elapsed 4m30s, eta 12m9s"
+
+		if !strings.Contains(shown, want) {
+			t.Errorf("terminal shows %q, want %q", shown, want)
 		}
 
-		shown = drawStalled(params, 80, 9876543, 1234567, 150*time.Minute)
+		shown = drawStalled(params, 9876543, 1234567, 150*time.Minute)
 		for _, want := range []string{
 			params.label + ": [1234567/" + approx + "9876543] 12%, ",
-			", elapsed 2h30m0s, remain 17h30m0s",
+			", elapsed 2h30m0s, eta 17h30m0s",
 		} {
 			if !strings.Contains(shown, want) {
 				t.Errorf("terminal shows %q, want %q", shown, want)
@@ -713,164 +706,13 @@ func TestTerminalLinesFitWidth(t *testing.T) {
 	}
 }
 
-// TestTerminalLinesStartWithSpinner draws a display with no total and
-// one with a bar, two and a half hours into the phase, on a terminal 120
-// columns wide. Each line must start with the library's spinner in the
-// first column, then a space and the phase's label.
-//
-//nolint:paralleltest // captureStderr replaces the process-wide os.Stderr
-func TestTerminalLinesStartWithSpinner(t *testing.T) {
-	stderr := captureStderr(t)
-
-	for _, params := range []progressParams{
-		{label: scanPhase("walk"), unit: scanUnit, total: -1},
-		{label: scanPhase("hash"), unit: scanUnit, total: 9876543},
-	} {
-		shown := lastFrame(drawLate(stderr, params, 120, 1234567))
-		spinner, rest, _ := strings.Cut(shown, " ")
-
-		if utf8.RuneCountInString(spinner) != 1 ||
-			!strings.HasPrefix(rest, params.label+": ") {
-			t.Errorf("terminal shows %q, want the spinner, a space and %q",
-				shown, params.label)
-		}
-
-		if params.total > 0 && !strings.Contains(shown, "% |") {
-			t.Errorf("terminal shows %q, want a bar", shown)
-		}
-	}
-}
-
-// TestTerminalBarStaysAtRightEdge draws the hash phase's display two
-// and a half hours in, at two counts whose lines differ in length, on
-// terminals 110 and 120 columns wide. The bar is partly filled at the
-// first count and full at the second. It must start the same number of
-// columns from the terminal's right edge each time, and end the same
-// number of columns from it: a line that grows or shrinks, as its time
-// remaining does, never moves the bar, and nor do the bar's filled
-// cells.
-//
-//nolint:paralleltest // captureStderr replaces the process-wide os.Stderr
-func TestTerminalBarStaysAtRightEdge(t *testing.T) {
-	stderr := captureStderr(t)
-	params := progressParams{
-		label: scanPhase("hash"), unit: scanUnit, total: 9876543,
-	}
-
-	widths := []int{110, 120}
-	counts := []int64{1234567, 9876543}
-	startFromEdge := make([]int, 0, len(widths)*len(counts))
-	endFromEdge := make([]int, 0, len(widths)*len(counts))
-
-	for _, width := range widths {
-		for _, count := range counts {
-			shown := lastFrame(drawLate(stderr, params, width, count))
-
-			beforeBar, _, found := strings.Cut(shown, "% |")
-			if !found {
-				t.Fatalf("terminal %d wide shows %q, want a bar", width, shown)
-			}
-
-			// The bar ends with the last "|".
-			throughBar := shown[:strings.LastIndex(shown, "|")+1]
-
-			startFromEdge = append(startFromEdge,
-				width-utf8.RuneCountInString(beforeBar))
-			endFromEdge = append(endFromEdge,
-				width-utf8.RuneCountInString(throughBar))
-		}
-	}
-
-	if slices.Min(startFromEdge) != slices.Max(startFromEdge) {
-		t.Errorf("bar starts %v columns from the right edge, want the same "+
-			"every time", startFromEdge)
-	}
-
-	if slices.Min(endFromEdge) != slices.Max(endFromEdge) {
-		t.Errorf("bar ends %v columns from the right edge, want the same "+
-			"every time", endFromEdge)
-	}
-}
-
-// TestTerminalRateGivesWayBeforeBar draws the hash phase's display two
-// and a half hours in. On a terminal 90 columns wide the line fits
-// beside the bar without its rate, but not with it: the bar is drawn
-// and the rate is not. On a terminal 85 columns wide the line does not
-// fit beside the bar even without its rate: neither is drawn, though
-// the rate would fit where the bar was.
-//
-//nolint:paralleltest // captureStderr replaces the process-wide os.Stderr
-func TestTerminalRateGivesWayBeforeBar(t *testing.T) {
-	stderr := captureStderr(t)
-	params := progressParams{
-		label: scanPhase("hash"), unit: scanUnit, total: 9876543,
-	}
-
-	shown := lastFrame(drawLate(stderr, params, 90, 1234567))
-	if !strings.Contains(shown, "% |") || strings.Contains(shown, "files/s") {
-		t.Errorf("terminal 90 wide shows %q, want the bar and no rate", shown)
-	}
-
-	shown = lastFrame(drawLate(stderr, params, 85, 1234567))
-	if strings.Contains(shown, "% |") || strings.Contains(shown, "files/s") {
-		t.Errorf("terminal 85 wide shows %q, want neither bar nor rate", shown)
-	}
-}
-
-// TestTerminalSpinnerTurnsBesideBar draws the hash phase's display, its
-// bar partly filled, several times on a terminal 120 columns wide. The
-// spinner must turn at every draw. A filled cell of the bar takes one
-// column but three bytes. A line measured in bytes would seem too wide
-// for the terminal, and every draw would start the display over, its
-// spinner included.
-//
-//nolint:paralleltest // captureStderr replaces the process-wide os.Stderr
-func TestTerminalSpinnerTurnsBesideBar(t *testing.T) {
-	stderr := captureStderr(t)
-	params := progressParams{
-		label: scanPhase("hash"), unit: scanUnit, total: 9876543,
-	}
-
-	// No timed draws: the test draws.
-	p := startProgress(params, func() int { return 120 }, time.Hour,
-		time.Now)
-
-	p.mu.Lock()
-	p.count = 1234567
-	p.mu.Unlock()
-
-	const draws = 4
-
-	spinners := make([]string, 0, draws)
-
-	for range draws {
-		before := len(stderr())
-
-		p.draw()
-
-		spinner, _, _ := strings.Cut(lastFrame(stderr()[before:]), " ")
-		spinners = append(spinners, spinner)
-	}
-
-	p.finish()
-
-	for i := 1; i < len(spinners); i++ {
-		if spinners[i] == spinners[i-1] {
-			t.Errorf("spinners at successive draws are %q, want each to "+
-				"differ from the one before", spinners)
-
-			break
-		}
-	}
-}
-
 // TestTerminalNarrowedRedrawsInPlace draws a large scan's display, two
 // and a half hours into the phase, on a terminal 120 columns wide, then
 // makes the terminal narrower than that line and draws it again a few
 // times. Nothing written after that, the erase before each redraw
 // included, may reach the narrower terminal's last column: it would
-// wrap, and every redraw would leave a line behind. At the narrower
-// width the hash phase's bar gives way.
+// wrap, and every redraw would leave a line behind. At these widths the
+// spinner stays a spinner and the bar gives way to the spinner.
 //
 //nolint:paralleltest // captureStderr replaces the process-wide os.Stderr
 func TestTerminalNarrowedRedrawsInPlace(t *testing.T) {
@@ -888,7 +730,7 @@ func TestTerminalNarrowedRedrawsInPlace(t *testing.T) {
 			progressParams{
 				label: scanPhase("hash"), unit: scanUnit, total: 9876543,
 			},
-			80,
+			90,
 		},
 	}
 
@@ -928,10 +770,10 @@ func TestTerminalNarrowedRedrawsInPlace(t *testing.T) {
 	}
 }
 
-// TestProgressWarningsOnOwnLines drives the terminal displays, with a
-// bar and without, through a run of warnings with no items between
-// them, as when the walk meets many unreadable paths, for many redraws:
-// every warning must land on a line of its own, never inside a redraw.
+// TestProgressWarningsOnOwnLines drives the terminal displays, spinner
+// and bar, through a run of warnings with no items between them, as when
+// the walk meets many unreadable paths, for many redraws: every warning
+// must land on a line of its own, never inside a redraw.
 //
 //nolint:paralleltest // captureStderr replaces the process-wide os.Stderr
 func TestProgressWarningsOnOwnLines(t *testing.T) {
@@ -977,12 +819,11 @@ func TestProgressWarningsOnOwnLines(t *testing.T) {
 	}
 }
 
-// TestProgressWriterRowsOnOwnLines drives the terminal displays, with a
-// bar and without, through a run of rows written through the display's
+// TestProgressWriterRowsOnOwnLines drives the terminal displays,
+// spinner and bar, through a run of rows written through the display's
 // writer to the same terminal, as stdout is in a plain "sfdupes trees",
 // each written out at once, for many redraws: the terminal must show
-// every row whole, in order, on a line of its own. The terminal is 120
-// columns wide, so the write phase's display has room for its bar.
+// every row whole, in order, on a line of its own.
 //
 //nolint:paralleltest // captureStderr replaces the process-wide os.Stderr
 func TestProgressWriterRowsOnOwnLines(t *testing.T) {
@@ -999,8 +840,7 @@ func TestProgressWriterRowsOnOwnLines(t *testing.T) {
 	var want []string
 
 	for _, params := range cases {
-		p := startProgress(params, func() int { return 120 }, testRedraw,
-			time.Now)
+		p := startProgress(params, terminalWidth, testRedraw, time.Now)
 		out := p.writer(os.Stderr)
 
 		// No pause between rows: one is garbled only if a redraw lands

report.go

@@ -2,7 +2,6 @@ package main
 
 import (
 	"context"
-	"crypto/sha256"
 	"database/sql"
 	"fmt"
 	"io"
@@ -25,9 +24,9 @@ const minGroupSize = 2
 type scanRec struct {
 	size    int64
 	mtime   time.Time
-	head    [sha256.Size]byte
-	tail    [sha256.Size]byte
-	content [sha256.Size]byte
+	head    string
+	tail    string
+	content string
 	path    string
 }
 

report_test.go

@@ -2,7 +2,6 @@ package main
 
 import (
 	"bytes"
-	"crypto/sha256"
 	"database/sql"
 	"errors"
 	"fmt"
@@ -19,27 +18,14 @@ import (
 // awkwardDir is a directory name holding every byte the reports escape.
 const awkwardDir = "/d/\tone\ntwo\rthree\\four"
 
-// testHash returns a stand-in hash for a test record: the bytes of s,
-// then zeros. Stand-ins sort as their strings do, so a test can choose
-// the order in which report reads its records.
-func testHash(s string) [sha256.Size]byte {
-	var h [sha256.Size]byte
-
-	copy(h[:], s)
-
-	return h
-}
-
 // awkwardPairRecs is a duplicate pair in sibling directories /d/A and
 // awkwardDir. A raw tab sorts before "A" but its escaped form `\t`
 // sorts after it, so awkwardDir coming first shows that sorting uses
 // the raw path.
 func awkwardPairRecs() []scanRec {
-	h, tl, c := testHash("h"), testHash("t"), testHash("c")
-
 	return []scanRec{
-		{size: 5, head: h, tail: tl, content: c, path: "/d/A/f"},
-		{size: 5, head: h, tail: tl, content: c, path: awkwardDir + "/f"},
+		{size: 5, head: "h", tail: "t", content: "c", path: "/d/A/f"},
+		{size: 5, head: "h", tail: "t", content: "c", path: awkwardDir + "/f"},
 	}
 }
 
@@ -157,8 +143,8 @@ func TestReportStdoutFailsWhileReading(t *testing.T) {
 	recs := make([]scanRec, 2*ioBufSize/len(dir))
 	for i := range recs {
 		recs[i] = scanRec{
-			size: int64(i/2) + 1, head: testHash("h"), tail: testHash("t"),
-			content: testHash("c"), path: fmt.Sprintf("%s/%d", dir, i),
+			size: int64(i/2) + 1, head: "h", tail: "t", content: "c",
+			path: fmt.Sprintf("%s/%d", dir, i),
 		}
 	}
 
@@ -187,7 +173,7 @@ func TestReportStdoutFailsWhileReading(t *testing.T) {
 // reportOpenLine is the plain line of report's open phase when it opens
 // the database at once; it prints it as the phase starts and as it ends.
 const reportOpenLine = "[1/2] open: opening the database, elapsed 0s, " +
-	"remain unknown"
+	"eta unknown"
 
 // smokeReport is report's stdout for the smoke-test records.
 const smokeReport = "first\tdupe\tsize\n" +
@@ -225,7 +211,7 @@ func TestRunReportProgressNamesEachPhase(t *testing.T) {
 		"report: database " + path + " (from " + databaseEnv + ")",
 		reportOpenLine,
 		reportOpenLine,
-		"[2/2] read: counting the records, elapsed 0s, remain unknown",
+		"[2/2] read: counting the records, elapsed 0s, eta unknown",
 		"[2/2] read: [6/6] 100%, ",
 		"report: 6 records read, 2 duplicate groups, 4 dupe files, " +
 			"6.1 KiB reclaimable",
@@ -291,14 +277,12 @@ func TestReadReportOnTerminal(t *testing.T) {
 func TestRunReportsIgnoreInsertionOrder(t *testing.T) {
 	// README §Constraints: identical database contents give identical
 	// output, whatever order the records were inserted in.
-	a, b := testHash("a"), testHash("b")
-
 	recs := append(smokeTreeRecs(), awkwardPairRecs()...)
 	recs = append(recs,
-		scanRec{size: 50, head: b, tail: b, content: b, path: "/y/2"},
-		scanRec{size: 50, head: b, tail: b, content: b, path: "/y/1"},
-		scanRec{size: 50, head: a, tail: a, content: a, path: "/x/2"},
-		scanRec{size: 50, head: a, tail: a, content: a, path: "/x/1"},
+		scanRec{size: 50, head: "b", tail: "b", content: "b", path: "/y/2"},
+		scanRec{size: 50, head: "b", tail: "b", content: "b", path: "/y/1"},
+		scanRec{size: 50, head: "a", tail: "a", content: "a", path: "/x/2"},
+		scanRec{size: 50, head: "a", tail: "a", content: "a", path: "/x/1"},
 	)
 
 	reversed := slices.Clone(recs)
@@ -401,19 +385,16 @@ func TestWarnfEscapes(t *testing.T) {
 func TestDupeGroups(t *testing.T) {
 	t.Parallel()
 
-	h, tl, c := testHash("h"), testHash("t"), testHash("c")
-	bigH, bigT, bigC := testHash("H"), testHash("T"), testHash("C")
-	u := testHash("u")
 	recs := []scanRec{
-		{size: 100, head: h, tail: tl, content: c, path: "/z/b"},
-		{size: 100, head: h, tail: tl, content: c, path: "/z/a"},
-		{size: 100, head: h, tail: tl, content: c, path: "/z/c"},
-		{size: 4000, head: bigH, tail: bigT, content: bigC, path: "/big/2"},
-		{size: 4000, head: bigH, tail: bigT, content: bigC, path: "/big/1"},
+		{size: 100, head: "h", tail: "t", content: "c", path: "/z/b"},
+		{size: 100, head: "h", tail: "t", content: "c", path: "/z/a"},
+		{size: 100, head: "h", tail: "t", content: "c", path: "/z/c"},
+		{size: 4000, head: "H", tail: "T", content: "C", path: "/big/2"},
+		{size: 4000, head: "H", tail: "T", content: "C", path: "/big/1"},
 		// Same size as the /z group but a different head hash.
-		{size: 100, head: testHash("other"), tail: tl, content: c, path: "/z/d"},
+		{size: 100, head: "other", tail: "t", content: "c", path: "/z/d"},
 		// A singleton signature must not form a group.
-		{size: 7, head: u, tail: u, content: u, path: "/lonely"},
+		{size: 7, head: "u", tail: "u", content: "u", path: "/lonely"},
 	}
 
 	groups := dupeGroupsOf(t, recs)
@@ -439,11 +420,10 @@ func TestDupeGroupsContentSeparates(t *testing.T) {
 
 	// Same size, head, and tail: /b differs from the others only in its
 	// content hash, so it joins no group, while /a and /c group.
-	h, tl := testHash("h"), testHash("t")
 	recs := []scanRec{
-		{size: 100, head: h, tail: tl, content: testHash("c1"), path: "/a"},
-		{size: 100, head: h, tail: tl, content: testHash("c2"), path: "/b"},
-		{size: 100, head: h, tail: tl, content: testHash("c1"), path: "/c"},
+		{size: 100, head: "h", tail: "t", content: "c1", path: "/a"},
+		{size: 100, head: "h", tail: "t", content: "c2", path: "/b"},
+		{size: 100, head: "h", tail: "t", content: "c1", path: "/c"},
 	}
 
 	groups := dupeGroupsOf(t, recs)
@@ -462,15 +442,14 @@ func TestDupeGroupsMtimeExcluded(t *testing.T) {
 
 	// mtime is informational only; records differing only in mtime
 	// still group together.
-	h, tl, c := testHash("h"), testHash("t"), testHash("c")
 	recs := []scanRec{
 		{
-			size: 9, mtime: time.Unix(100, 0), head: h, tail: tl,
-			content: c, path: "/m/1",
+			size: 9, mtime: time.Unix(100, 0), head: "h", tail: "t",
+			content: "c", path: "/m/1",
 		},
 		{
-			size: 9, mtime: time.Unix(200, 0), head: h, tail: tl,
-			content: c, path: "/m/2",
+			size: 9, mtime: time.Unix(200, 0), head: "h", tail: "t",
+			content: "c", path: "/m/2",
 		},
 	}
 
@@ -485,12 +464,11 @@ func TestDupeGroupsTieBreak(t *testing.T) {
 
 	// The hashes sort opposite to the first paths, so ordering the
 	// groups by hash instead of by first path fails this test.
-	a, b := testHash("a"), testHash("b")
 	recs := []scanRec{
-		{size: 50, head: a, tail: a, content: a, path: "/beta/2"},
-		{size: 50, head: a, tail: a, content: a, path: "/beta/1"},
-		{size: 50, head: b, tail: b, content: b, path: "/alpha/2"},
-		{size: 50, head: b, tail: b, content: b, path: "/alpha/1"},
+		{size: 50, head: "a", tail: "a", content: "a", path: "/beta/2"},
+		{size: 50, head: "a", tail: "a", content: "a", path: "/beta/1"},
+		{size: 50, head: "b", tail: "b", content: "b", path: "/alpha/2"},
+		{size: 50, head: "b", tail: "b", content: "b", path: "/alpha/1"},
 	}
 
 	groups := dupeGroupsOf(t, recs)
@@ -512,16 +490,15 @@ func TestDupeGroupsOrderedByFirstPath(t *testing.T) {
 	// /z, /k, which is neither the order of their first paths nor its
 	// reverse, and must write them ordered by first path. The group of
 	// size 7 comes after them all, though its first path sorts first.
-	a, b, c, d := testHash("a"), testHash("b"), testHash("c"), testHash("d")
 	recs := []scanRec{
-		{size: 50, head: a, tail: a, content: a, path: "/k/2"},
-		{size: 50, head: a, tail: a, content: a, path: "/k/1"},
-		{size: 50, head: b, tail: b, content: b, path: "/z/1"},
-		{size: 50, head: b, tail: b, content: b, path: "/z/2"},
-		{size: 50, head: c, tail: c, content: c, path: "/a/2"},
-		{size: 50, head: c, tail: c, content: c, path: "/a/1"},
-		{size: 7, head: d, tail: d, content: d, path: "/0/2"},
-		{size: 7, head: d, tail: d, content: d, path: "/0/1"},
+		{size: 50, head: "a", tail: "a", content: "a", path: "/k/2"},
+		{size: 50, head: "a", tail: "a", content: "a", path: "/k/1"},
+		{size: 50, head: "b", tail: "b", content: "b", path: "/z/1"},
+		{size: 50, head: "b", tail: "b", content: "b", path: "/z/2"},
+		{size: 50, head: "c", tail: "c", content: "c", path: "/a/2"},
+		{size: 50, head: "c", tail: "c", content: "c", path: "/a/1"},
+		{size: 7, head: "d", tail: "d", content: "d", path: "/0/2"},
+		{size: 7, head: "d", tail: "d", content: "d", path: "/0/1"},
 	}
 
 	groups := dupeGroupsOf(t, recs)
@@ -540,12 +517,11 @@ func TestDupeGroupsOrderedByFirstPath(t *testing.T) {
 func TestDupeGroupsDeterministic(t *testing.T) {
 	t.Parallel()
 
-	a, b := testHash("a"), testHash("b")
 	recs := []scanRec{
-		{size: 1, head: a, tail: a, content: a, path: "/p/1"},
-		{size: 1, head: a, tail: a, content: a, path: "/p/2"},
-		{size: 2, head: b, tail: b, content: b, path: "/q/1"},
-		{size: 2, head: b, tail: b, content: b, path: "/q/2"},
+		{size: 1, head: "a", tail: "a", content: "a", path: "/p/1"},
+		{size: 1, head: "a", tail: "a", content: "a", path: "/p/2"},
+		{size: 2, head: "b", tail: "b", content: "b", path: "/q/1"},
+		{size: 2, head: "b", tail: "b", content: "b", path: "/q/2"},
 	}
 
 	forward := dupeGroupsOf(t, recs)

scan.go

@@ -5,6 +5,7 @@ import (
 	"context"
 	"crypto/sha256"
 	"database/sql"
+	"encoding/hex"
 	"errors"
 	"fmt"
 	"io"
@@ -383,8 +384,7 @@ func (s *scanState) runPhases(ctx context.Context, roots []string,
 ) error {
 	// Types and filesystems are checked before pruning so that an
 	// operand under a dropped one is still scanned, not dropped as lying
-	// under it, and so that an operand inside a .zfs directory gets its
-	// warning even when it lies under another operand.
+	// under it.
 	roots = pruneRoots(s.walkableRoots(roots, rules), oneFS, rules,
 		deviceOfPath)
 
@@ -412,15 +412,14 @@ func (s *scanState) runPhases(ctx context.Context, roots []string,
 }
 
 // walkableRoots returns the operands the walk can start from: regular
-// files, and directories not named .zfs, none of them inside a
-// directory named .zfs. Every other operand is warned about, counted as
-// skipped, and dropped. A dropped operand is no longer a root, so the
-// records stored beneath it are treated as if it had not been given:
-// those under no other root count as outside the roots and are not
-// deleted as unverified. An operand on a filesystem in rules that is not
-// persistent is dropped the same way, with a line saying so that is not
-// a warning, and is not counted as skipped. An operand that fails lstat
-// here is kept, and the walk warns about it unless it no longer exists.
+// files, and directories not named .zfs. Every other operand is warned
+// about, counted as skipped, and dropped. A dropped operand is no
+// longer a root, so the records stored beneath it are treated as if it
+// had not been given: those under no other root count as outside the
+// roots and are not deleted as unverified. An operand on a filesystem
+// in rules that is not persistent is dropped the same way, with a line
+// saying so that is not a warning, and is not counted as skipped. An
+// operand that fails lstat here is kept, and the walk warns about it.
 func (s *scanState) walkableRoots(roots []string, rules walkRules) []string {
 	kept := make([]string, 0, len(roots))
 
@@ -631,9 +630,8 @@ func isDatabaseFile(f fileRec, rules walkRules) bool {
 // (an interrupted scan resumes cheaply: the next run skips everything
 // already recorded). A run that fails to hash is warned about and
 // counted as skipped; stale records for its paths, if any, are deleted
-// by the update phase. A run whose file no longer exists gets no record
-// and no warning, and is not counted as skipped. The total counts actual
-// reads, so the display's time remaining is meaningful.
+// by the update phase. The total counts actual reads, so the display
+// shows a real ETA.
 //
 // Returning early — a failed database write, or a cancelled scan — must
 // not strand the pool: the feeder would park forever on a full jobs
@@ -665,11 +663,6 @@ func (s *scanState) hashPhase(ctx context.Context, workers int) error {
 
 		prog.increment()
 
-		// A file removed since the walk is no error: it gets no record.
-		if errors.Is(r.err, fs.ErrNotExist) {
-			continue
-		}
-
 		if r.err != nil {
 			s.st.skipped += len(r.run)
 
@@ -837,12 +830,10 @@ type walkEvent struct {
 // startWalk seeds every root into the shared walk worker pool and
 // returns the event stream: one record per regular file, one warning
 // event per per-path error, and one event per path -x or rules kept the
-// walk from entering or recording. A path that no longer exists when
-// the walk reaches it is no error, and gets no event. The channel is
-// closed when the walk completes, and also when ctx is cancelled —
-// every goroutine in the pool abandons its blocking send in that case,
-// so the consumer sees a truncated but properly terminated stream
-// instead of a stalled one.
+// walk from entering or recording. The channel is closed when the walk
+// completes, and also when ctx is cancelled — every goroutine in the
+// pool abandons its blocking send in that case, so the consumer sees a
+// truncated but properly terminated stream instead of a stalled one.
 func startWalk(ctx context.Context, roots []string, oneFS bool,
 	rules walkRules, workers int,
 ) <-chan walkEvent {
@@ -875,24 +866,9 @@ func sendEvent(ctx context.Context, events chan<- walkEvent,
 
 // operandWarning returns the one-line warning for an operand the walk
 // does not start from, naming the path and what it is, or "" for one it
-// does: a regular file, or a directory not named .zfs, that is not
-// inside a directory named .zfs, neither as given nor with its symlinks
-// resolved. Symlinks are never followed, including as operands.
+// does: a regular file, or a directory not named .zfs. Symlinks are
+// never followed, including as operands.
 func operandWarning(root string, fi fs.FileInfo) string {
-	// The walk never enters a .zfs directory, so it starts from nothing
-	// inside one either, not even through a symlink. An operand whose
-	// symlinks cannot be resolved is checked as given.
-	resolved, err := filepath.EvalSymlinks(root)
-	if err != nil {
-		resolved = root
-	}
-
-	if slices.Contains(strings.Split(filepath.Dir(root), "/"), ".zfs") ||
-		slices.Contains(strings.Split(filepath.Dir(resolved), "/"), ".zfs") {
-		return fmt.Sprintf("walk %s: skipping operand inside a .zfs directory",
-			root)
-	}
-
 	var kind string
 
 	switch mode := fi.Mode(); {
@@ -924,19 +900,13 @@ func operandWarning(root string, fi fs.FileInfo) string {
 // one of the scan's own database files in rules, and a directory
 // operand becomes an initial job. walkableRoots has already dropped
 // every other operand. One that has changed into something else since
-// is warned about and skipped here, and one removed since is skipped
-// without a warning. Either is still a root, so the records stored
-// beneath it, other than those under another root, are deleted as
-// unverified.
+// is warned about and skipped here; it is still a root, so the records
+// stored beneath it, other than those under another root, are deleted
+// as unverified.
 func seedRoot(ctx context.Context, root string, rules walkRules,
 	events chan<- walkEvent,
 ) []dirJob {
 	fi, err := os.Lstat(root)
-	if errors.Is(err, fs.ErrNotExist) {
-		// Removed since the scan started: no error, and no files.
-		return nil
-	}
-
 	if err != nil {
 		sendEvent(ctx, events, walkEvent{
 			warn: fmt.Sprintf("walk %s: %v", root, err),
@@ -1054,17 +1024,11 @@ func dispatchDirs(ctx context.Context, initial []dirJob,
 
 // walkOneDir reads one directory, emitting an event per regular-file
 // entry and a warning event per unreadable one, and returns the
-// subdirectories to descend into. A directory or entry removed since
-// the walk found it gets no event.
+// subdirectories to descend into.
 func walkOneDir(ctx context.Context, job dirJob, oneFS bool,
 	rules walkRules, events chan<- walkEvent,
 ) []dirJob {
 	entries, err := os.ReadDir(job.path)
-	if errors.Is(err, fs.ErrNotExist) {
-		// Removed since its parent was read: no error, and no files.
-		return nil
-	}
-
 	if err != nil {
 		sendEvent(ctx, events, walkEvent{
 			warn: fmt.Sprintf("walk %s: %v", job.path, err),
@@ -1107,22 +1071,16 @@ func walkOneDir(ctx context.Context, job dirJob, oneFS bool,
 
 // emitFile stats one regular directory entry and emits its record.
 // The lstat happens here in the walk worker, while the directory's
-// metadata is still hot; a path that fails to stat between the
-// directory read and the lstat is warned about and skipped, unless it
-// no longer exists, and one that has stopped being a regular file is
-// skipped. A file on a filesystem in rules that is not persistent, as
-// through a bind mount of one file, is not recorded: it gets an event
-// naming it instead. One of the scan's own database files in rules gets
-// no event at all.
+// metadata is still hot; a path that fails to stat (or stops being a
+// regular file) between the directory read and the lstat is warned
+// about and skipped. A file on a filesystem in rules that is not
+// persistent, as through a bind mount of one file, is not recorded: it
+// gets an event naming it instead. One of the scan's own database files
+// in rules gets no event at all.
 func emitFile(ctx context.Context, p string, e fs.DirEntry,
 	rules walkRules, events chan<- walkEvent,
 ) {
 	info, err := e.Info()
-	if errors.Is(err, fs.ErrNotExist) {
-		// Removed since its directory was read: no error.
-		return
-	}
-
 	if err != nil {
 		sendEvent(ctx, events, walkEvent{
 			warn: fmt.Sprintf("stat %s: %v", p, err),
@@ -1174,11 +1132,6 @@ func subdirJob(ctx context.Context, p string, e fs.DirEntry,
 	}
 
 	info, err := e.Info()
-	if errors.Is(err, fs.ErrNotExist) {
-		// Removed since its parent was read: no error, and no files.
-		return dirJob{}, false
-	}
-
 	if err != nil {
 		sendEvent(ctx, events, walkEvent{
 			warn: fmt.Sprintf("walk %s: %v", p, err),
@@ -1243,12 +1196,12 @@ func inodeOfInfo(fi fs.FileInfo) (uint64, uint64) {
 
 // hashResult carries the hashes computed for one inode run (or the
 // error that prevented computing them) from the pool's workers to the
-// hash phase. Its run leaves out the paths found removed.
+// hash phase.
 type hashResult struct {
 	run     []fileRec
-	head    [sha256.Size]byte
-	tail    [sha256.Size]byte
-	content [sha256.Size]byte
+	head    string
+	tail    string
+	content string
 	err     error
 }
 
@@ -1268,8 +1221,7 @@ type hashPool struct {
 // hash each run's first path with hash (all paths in a run are hard
 // links to the same inode) and write one result per run.
 func startHashPool(ctx context.Context, runs [][]fileRec, workers int,
-	hash func(path string, size int64) (
-		[sha256.Size]byte, [sha256.Size]byte, [sha256.Size]byte, error),
+	hash func(path string, size int64) (string, string, string, error),
 ) *hashPool {
 	ctx, cancel := context.WithCancel(ctx)
 
@@ -1334,8 +1286,7 @@ func feedHashJobs(ctx context.Context, runs [][]fileRec,
 // the hash of only delays that.
 func hashWorker(ctx context.Context, jobs <-chan []fileRec,
 	results chan<- hashResult,
-	hash func(path string, size int64) (
-		[sha256.Size]byte, [sha256.Size]byte, [sha256.Size]byte, error),
+	hash func(path string, size int64) (string, string, string, error),
 ) {
 	for run := range jobs {
 		if ctx.Err() != nil {
@@ -1344,13 +1295,6 @@ func hashWorker(ctx context.Context, jobs <-chan []fileRec,
 
 		head, tail, content, err := hash(run[0].path, run[0].size)
 
-		// A path removed since the walk is dropped from its run. The
-		// next path is a hard link to the same file, and is read instead.
-		for errors.Is(err, fs.ErrNotExist) && len(run) > 1 {
-			run = run[1:]
-			head, tail, content, err = hash(run[0].path, run[0].size)
-		}
-
 		select {
 		case results <- hashResult{
 			run: run, head: head, tail: tail, content: content, err: err,
@@ -1393,38 +1337,37 @@ func openRegularFile(path string) (*os.File, error) {
 	return f, nil
 }
 
+// emptyHash is the lowercase-hex SHA-256 of the empty input: the head,
+// tail, and content hash of every zero-length file.
+const emptyHash = "e3b0c44298fc1c149afbf4c8996fb924" +
+	"27ae41e4649b934ca495991b7852b855"
+
 // hashSignature computes the three hashes the hash phase records for a
 // file: head and tail (see hashEnds) and content (see hashContent).
 // With the file size they form its duplicate signature, and two files
 // are duplicates only when all four agree. size is the value recorded
 // when the file was statted; a zero-length file has constant hashes and
-// is never opened. On an error every hash is left zero.
-func hashSignature(path string, size int64) (
-	[sha256.Size]byte, [sha256.Size]byte, [sha256.Size]byte, error,
-) {
-	var zero [sha256.Size]byte
-
+// is never opened.
+func hashSignature(path string, size int64) (string, string, string, error) {
 	if size == 0 {
-		empty := sha256.Sum256(nil)
-
-		return empty, empty, empty, nil
+		return emptyHash, emptyHash, emptyHash, nil
 	}
 
 	f, err := openRegularFile(path)
 	if err != nil {
-		return zero, zero, zero, err
+		return "", "", "", err
 	}
 
 	defer func() { _ = f.Close() }()
 
 	head, tail, err := hashEnds(f, size)
 	if err != nil {
-		return zero, zero, zero, err
+		return "", "", "", err
 	}
 
 	content, err := hashContent(f, size)
 	if err != nil {
-		return zero, zero, zero, err
+		return "", "", "", err
 	}
 
 	return head, tail, content, nil
@@ -1434,32 +1377,31 @@ func hashSignature(path string, size int64) (
 // bytes of f. In a file no larger than headTailWindow, each window is
 // the whole file. Reading fewer bytes than asked means the file shrank
 // between the stat and the hash, and is an error.
-func hashEnds(f *os.File, size int64) (
-	[sha256.Size]byte, [sha256.Size]byte, error,
-) {
-	var zero [sha256.Size]byte
-
+func hashEnds(f *os.File, size int64) (string, string, error) {
 	buf := make([]byte, min(int64(headTailWindow), size))
 
 	_, err := f.ReadAt(buf, 0)
 	if err != nil {
-		return zero, zero, err
+		return "", "", err
 	}
 
-	head := sha256.Sum256(buf)
+	h := sha256.Sum256(buf)
+	head := hex.EncodeToString(h[:])
 
 	_, err = f.ReadAt(buf, size-int64(len(buf)))
 	if err != nil {
-		return zero, zero, err
+		return "", "", err
 	}
 
-	return head, sha256.Sum256(buf), nil
+	t := sha256.Sum256(buf)
+
+	return head, hex.EncodeToString(t[:]), nil
 }
 
 // hashContent returns the content hash of f: the SHA-256 of the whole
 // file when it is smaller than wholeFileMax, or of sampled windows when
 // it is that size or larger.
-func hashContent(f *os.File, size int64) ([sha256.Size]byte, error) {
+func hashContent(f *os.File, size int64) (string, error) {
 	if size >= int64(wholeFileMax) {
 		return hashSamples(f, size)
 	}
@@ -1472,20 +1414,20 @@ func hashContent(f *os.File, size int64) ([sha256.Size]byte, error) {
 // reads. Reading fewer than size bytes means the file shrank between
 // the stat and the hash; that is an error rather than a hash of content
 // that no longer matches the recorded size.
-func hashWhole(f *os.File, size int64) ([sha256.Size]byte, error) {
+func hashWhole(f *os.File, size int64) (string, error) {
 	h := sha256.New()
 
 	n, err := io.Copy(h, io.NewSectionReader(f, 0, size))
 	if err != nil {
-		return [sha256.Size]byte{}, err
+		return "", err
 	}
 
 	if n != size {
-		return [sha256.Size]byte{}, fmt.Errorf("read %d of %d bytes: %w",
-			n, size, io.ErrUnexpectedEOF)
+		return "", fmt.Errorf("read %d of %d bytes: %w", n, size,
+			io.ErrUnexpectedEOF)
 	}
 
-	return [sha256.Size]byte(h.Sum(nil)), nil
+	return hex.EncodeToString(h.Sum(nil)), nil
 }
 
 // hashSamples feeds sampleWindow bytes at each gigabyte-aligned offset
@@ -1495,7 +1437,7 @@ func hashWhole(f *os.File, size int64) ([sha256.Size]byte, error) {
 // without every byte being read. Because size is part of the signature,
 // only files of one size are compared, so their sample boundaries
 // always align.
-func hashSamples(f *os.File, size int64) ([sha256.Size]byte, error) {
+func hashSamples(f *os.File, size int64) (string, error) {
 	h := sha256.New()
 	buf := make([]byte, sampleWindow)
 
@@ -1504,11 +1446,11 @@ func hashSamples(f *os.File, size int64) ([sha256.Size]byte, error) {
 
 		_, err := f.ReadAt(buf[:n], off)
 		if err != nil {
-			return [sha256.Size]byte{}, err
+			return "", err
 		}
 
 		h.Write(buf[:n])
 	}
 
-	return [sha256.Size]byte(h.Sum(nil)), nil
+	return hex.EncodeToString(h.Sum(nil)), nil
 }

scan_test.go

@@ -5,6 +5,7 @@ import (
 	"context"
 	"crypto/sha256"
 	"database/sql"
+	"encoding/hex"
 	"errors"
 	"fmt"
 	"io"
@@ -42,6 +43,13 @@ func writeFile(t *testing.T, dir, name string, data []byte) string {
 	return p
 }
 
+// hexSum returns the lowercase-hex SHA-256 of data.
+func hexSum(data []byte) string {
+	s := sha256.Sum256(data)
+
+	return hex.EncodeToString(s[:])
+}
+
 // pattern returns n bytes of deterministic content seeded by tag.
 func pattern(tag byte, n int) []byte {
 	data := make([]byte, n)
@@ -54,9 +62,7 @@ func pattern(tag byte, n int) []byte {
 
 // sig returns a file's full signature (head, tail, content), failing the
 // test on any error.
-func sig(t *testing.T, path string, size int64) (
-	[sha256.Size]byte, [sha256.Size]byte, [sha256.Size]byte,
-) {
+func sig(t *testing.T, path string, size int64) (string, string, string) {
 	t.Helper()
 
 	head, tail, content, err := hashSignature(path, size)
@@ -94,12 +100,12 @@ func TestHashSignature(t *testing.T) {
 			head, tail, content := sig(t, p, int64(len(c.data)))
 
 			window := min(len(c.data), headTailWindow)
-			wantHead := sha256.Sum256(c.data[:window])
-			wantTail := sha256.Sum256(c.data[len(c.data)-window:])
-			wantContent := sha256.Sum256(c.data)
+			wantHead := hexSum(c.data[:window])
+			wantTail := hexSum(c.data[len(c.data)-window:])
+			wantContent := hexSum(c.data)
 
 			if head != wantHead || tail != wantTail || content != wantContent {
-				t.Errorf("head=%x tail=%x content=%x, want %x %x %x",
+				t.Errorf("head=%s tail=%s content=%s, want %s %s %s",
 					head, tail, content, wantHead, wantTail, wantContent)
 			}
 		})
@@ -164,25 +170,22 @@ func TestHashSignatureErrors(t *testing.T) {
 
 	dir := t.TempDir()
 
-	var zero [sha256.Size]byte
-
-	// A missing file: an error, and every hash left zero.
+	// A missing file: an error, and every hash left empty.
 	head, tail, content, err := hashSignature(filepath.Join(dir, "missing"), 1)
 	if err == nil {
 		t.Error("no error for a missing file")
 	}
 
-	if head != zero || tail != zero || content != zero {
-		t.Errorf("missing file returned hashes: %x %x %x", head, tail, content)
+	if head != "" || tail != "" || content != "" {
+		t.Errorf("missing file returned hashes: %q %q %q", head, tail, content)
 	}
 
 	// A zero-length file has constant hashes and is never opened: even
 	// a missing path succeeds.
-	empty := sha256.Sum256(nil)
-
 	head, tail, content, err = hashSignature(filepath.Join(dir, "missing"), 0)
-	if err != nil || head != empty || tail != empty || content != empty {
-		t.Errorf("empty: head=%x tail=%x content=%x err=%v, "+
+	if err != nil ||
+		head != emptyHash || tail != emptyHash || content != emptyHash {
+		t.Errorf("empty: head=%q tail=%q content=%q err=%v, "+
 			"want constant hashes", head, tail, content, err)
 	}
 
@@ -195,8 +198,8 @@ func TestHashSignatureErrors(t *testing.T) {
 		t.Error("no error when the stat size exceeds the file size")
 	}
 
-	if head != zero || tail != zero || content != zero {
-		t.Errorf("shrunk file returned hashes: %x %x %x", head, tail, content)
+	if head != "" || tail != "" || content != "" {
+		t.Errorf("shrunk file returned hashes: %q %q %q", head, tail, content)
 	}
 }
 
@@ -683,108 +686,6 @@ func TestSubdirJobStatError(t *testing.T) {
 	}
 }
 
-// TestWalkPassesOverRemovedPaths removes a file and a directory after
-// their parent was read, as when they are deleted while a scan runs.
-// The walk then reaches them, the directory also as one to read and as
-// an operand. None of that is an error: the walk sends no event.
-func TestWalkPassesOverRemovedPaths(t *testing.T) {
-	t.Parallel()
-
-	dir := t.TempDir()
-	file := writeFile(t, dir, "file", []byte("x"))
-	writeFile(t, dir, "sub/f", []byte("y"))
-	sub := filepath.Join(dir, "sub")
-	fileEntry := dirEntryFor(t, dir, "file")
-	subEntry := dirEntryFor(t, dir, "sub")
-
-	for _, p := range []string{file, sub} {
-		err := os.RemoveAll(p)
-		if err != nil {
-			t.Fatal(err)
-		}
-	}
-
-	ctx := t.Context()
-	events := make(chan walkEvent, 4)
-
-	emitFile(ctx, file, fileEntry, walkRules{}, events)
-
-	_, descend := subdirJob(ctx, sub, subEntry, dirJob{path: dir}, false,
-		walkRules{}, events)
-	subs := walkOneDir(ctx, dirJob{path: sub}, false, walkRules{}, events)
-	jobs := seedRoot(ctx, sub, walkRules{}, events)
-
-	close(events)
-
-	for ev := range events {
-		t.Errorf("event %+v for a removed path, want none", ev)
-	}
-
-	if descend || len(subs) != 0 || len(jobs) != 0 {
-		t.Errorf("descend = %t, subdirectories %+v, jobs %+v, want none",
-			descend, subs, jobs)
-	}
-}
-
-// TestScanWarnsAboutUnreadablePaths checks that a permission error is
-// still warned about and counted as skipped, unlike a removed path. The
-// walk reaches an unreadable directory, and a file in a directory that
-// can be read but not searched. Another such file is an operand.
-//
-//nolint:paralleltest // captureStderr replaces the process-wide os.Stderr
-func TestScanWarnsAboutUnreadablePaths(t *testing.T) {
-	if os.Geteuid() == 0 {
-		t.Skip("root ignores file permissions")
-	}
-
-	stderr := captureStderr(t)
-	dir := t.TempDir()
-	tree := filepath.Join(dir, "tree")
-	walked := writeFile(t, tree, "unsearchable/f", []byte("x"))
-	operand := writeFile(t, dir, "unsearchable/f", []byte("y"))
-	unreadable := filepath.Join(tree, "unreadable")
-
-	err := os.Mkdir(unreadable, 0o750)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	modes := map[string]fs.FileMode{
-		unreadable:            0,
-		filepath.Dir(walked):  0o400,
-		filepath.Dir(operand): 0o400,
-	}
-
-	for p, mode := range modes {
-		err = os.Chmod(p, mode)
-		if err != nil {
-			t.Fatal(err)
-		}
-
-		// Runs before t.TempDir's own cleanup, which must delete the files.
-		t.Cleanup(func() {
-			//nolint:gosec // removing the files needs the search bit back
-			_ = os.Chmod(p, 0o700)
-		})
-	}
-
-	st := syncTree(t, openTestDB(t), tree, operand)
-	if st != (scanStats{skipped: 3}) {
-		t.Errorf("stats = %+v, want 3 skipped and nothing else", st)
-	}
-
-	got := stderr()
-	for _, warning := range []string{
-		"walk " + unreadable + ": open " + unreadable + ": permission denied\n",
-		"stat " + walked + ": lstat " + walked + ": permission denied\n",
-		"walk " + operand + ": lstat " + operand + ": permission denied\n",
-	} {
-		if !strings.Contains(got, warning) {
-			t.Errorf("stderr = %q, want %q", got, warning)
-		}
-	}
-}
-
 // syncOneFilesystem scans roots into db with -x and returns the scan
 // stats. Every device pruneRoots looks up, and the device each
 // operand's walk compares its subdirectories with, comes from devices
@@ -1064,9 +965,8 @@ func TestScanOneFilesystemWalksNestedFileOperand(t *testing.T) {
 	}
 }
 
-// buildSmokeTree recreates the tree of the test under README
-// "Definition of done" with deterministic content and returns the tree
-// root.
+// buildSmokeTree recreates the README smoke-test filesystem layout
+// with deterministic content and returns the tree root.
 func buildSmokeTree(t *testing.T) string {
 	t.Helper()
 
@@ -1231,7 +1131,7 @@ func TestScanPipeline(t *testing.T) {
 // database's records. Each phase's first line names it with its place
 // among the four: load counts the records before reading them, and the
 // walk estimates its total from the records under its operand, or says
-// its time remaining is unknown when there are none.
+// its ETA is unknown when there are none.
 //
 //nolint:paralleltest // captureStderr replaces the process-wide os.Stderr
 func TestScanProgressNamesEachPhase(t *testing.T) {
@@ -1246,14 +1146,14 @@ func TestScanProgressNamesEachPhase(t *testing.T) {
 	syncTree(t, db, filepath.Join(dir, "a"))
 
 	rescan := stderr()[len(first):]
-	load := "[1/4] load: counting the records, elapsed 0s, remain unknown"
+	load := "[1/4] load: counting the records, elapsed 0s, eta unknown"
 	cases := map[string][]string{
 		first: {
-			load, "[2/4] walk: 0 files, 0 files/s, elapsed 0s, remain unknown",
+			load, "[2/4] walk: 0 files, 0 files/s, elapsed 0s, eta unknown",
 			"[3/4] hash: [0/", "[4/4] update: [0/",
 		},
 		rescan: {
-			load, "[2/4] walk: [0/~2] 0%, 0 files/s, elapsed 0s, remain unknown",
+			load, "[2/4] walk: [0/~2] 0%, 0 files/s, elapsed 0s, eta unknown",
 			"[3/4] hash: [0/", "[4/4] update: [0/",
 		},
 	}
@@ -1311,10 +1211,8 @@ func TestSyncScanUnchangedReuse(t *testing.T) {
 		}
 	}
 
-	sentinel := sha256.Sum256([]byte("sentinel"))
-
 	_, err := db.ExecContext(context.Background(),
-		"UPDATE files SET head = ? WHERE path = ?", sentinel[:], []byte(a))
+		"UPDATE files SET head = 'sentinel' WHERE path = ?", []byte(a))
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -1325,9 +1223,9 @@ func TestSyncScanUnchangedReuse(t *testing.T) {
 			st)
 	}
 
-	if r := recordByPath(t, dbRecords(t, db), a); r.head != sentinel {
-		t.Fatalf("head = %x, want the sentinel %x (file must not be re-read)",
-			r.head, sentinel)
+	if r := recordByPath(t, dbRecords(t, db), a); r.head != "sentinel" {
+		t.Fatalf("head = %q, want sentinel (file must not be re-read)",
+			r.head)
 	}
 }
 
@@ -1367,56 +1265,13 @@ func assertWholeFileHashed(t *testing.T, db *sql.DB, path string,
 	t.Helper()
 
 	r := recordByPath(t, dbRecords(t, db), path)
-	if want := sha256.Sum256(data); r.head != want || r.tail != want ||
+	if want := hexSum(data); r.head != want || r.tail != want ||
 		r.content != want {
-		t.Fatalf("head, tail, content = %x, %x, %x, want %x for each",
+		t.Fatalf("head, tail, content = %q, %q, %q, want %q for each",
 			r.head, r.tail, r.content, want)
 	}
 }
 
-// TestScanStoresRawHashes checks what the database holds for a file
-// larger than both end windows, whose three hashes all differ: head,
-// tail and content are each a BLOB of the 32 bytes of the SHA-256 the
-// scan computed.
-func TestScanStoresRawHashes(t *testing.T) {
-	t.Parallel()
-
-	dir := t.TempDir()
-	db := openTestDB(t)
-	data := pattern(3, 3*headTailWindow)
-	p := writeFile(t, dir, "f.bin", data)
-
-	syncTree(t, db, dir)
-
-	var (
-		blobs               bool
-		head, tail, content []byte
-	)
-
-	err := db.QueryRowContext(t.Context(),
-		"SELECT typeof(head) = 'blob' AND typeof(tail) = 'blob' AND "+
-			"typeof(content) = 'blob', head, tail, content "+
-			"FROM files WHERE path = ?", []byte(p)).
-		Scan(&blobs, &head, &tail, &content)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	if !blobs {
-		t.Error("head, tail and content are not all BLOBs")
-	}
-
-	wantHead := sha256.Sum256(data[:headTailWindow])
-	wantTail := sha256.Sum256(data[len(data)-headTailWindow:])
-	wantContent := sha256.Sum256(data)
-
-	if !bytes.Equal(head, wantHead[:]) || !bytes.Equal(tail, wantTail[:]) ||
-		!bytes.Equal(content, wantContent[:]) {
-		t.Errorf("head, tail, content = %x, %x, %x, want %x, %x, %x",
-			head, tail, content, wantHead, wantTail, wantContent)
-	}
-}
-
 // TestSyncScanSameSecondRewrite rewrites a file in place at the same
 // size with an mtime later in the same second as the recorded one: the
 // next scan must notice the change and re-hash the file. The operand is
@@ -1712,8 +1567,7 @@ func TestScanHardlinksReadOnce(t *testing.T) {
 	ra := recordByPath(t, recs, a)
 	rb := recordByPath(t, recs, b)
 
-	if ra.head == [sha256.Size]byte{} || ra.head != rb.head ||
-		ra.tail != rb.tail {
+	if ra.head == "" || ra.head != rb.head || ra.tail != rb.tail {
 		t.Fatalf("hardlink hashes differ: %+v vs %+v", ra, rb)
 	}
 
@@ -1788,55 +1642,6 @@ func TestHashPhaseSkipsFileReplacedByFIFO(t *testing.T) {
 	}
 }
 
-// TestHashPhasePassesOverRemovedFiles removes two walked files before
-// the hash phase reads them: one with no other path, and the first of
-// two hard links to one file. Neither is an error. The phase prints
-// nothing but its progress, counts nothing as skipped, and records only
-// the hard link still there.
-//
-//nolint:paralleltest // captureStderr replaces the process-wide os.Stderr
-func TestHashPhasePassesOverRemovedFiles(t *testing.T) {
-	stderr := captureStderr(t)
-	dir := t.TempDir()
-	alone := writeFile(t, dir, "alone.bin", pattern(1, 300))
-	a := writeFile(t, dir, "a.bin", pattern(2, 300))
-	b := filepath.Join(dir, "b.bin")
-
-	err := os.Link(a, b)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	walked, _ := collectWalk(t, []string{dir}, false, 1)
-
-	// a.bin sorts before b.bin, so the run of the two reads it first.
-	for _, p := range []string{alone, a} {
-		err = os.Remove(p)
-		if err != nil {
-			t.Fatal(err)
-		}
-	}
-
-	s := &scanState{db: openTestDB(t), toHash: walked}
-
-	err = s.hashPhase(t.Context(), 1)
-	if err != nil {
-		t.Fatalf("hashPhase = %v, want nil", err)
-	}
-
-	if s.st.skipped != 0 || len(s.batch) != 1 || s.batch[0].path != b {
-		t.Errorf("skipped = %d, records %+v, want none skipped and a "+
-			"record for %s alone", s.st.skipped, s.batch, b)
-	}
-
-	for line := range strings.Lines(stderr()) {
-		if !strings.HasPrefix(line, scanPhase("hash")+": ") {
-			t.Errorf("stderr holds %q, want the hash phase's progress alone",
-				line)
-		}
-	}
-}
-
 // injectedWriteFailure is the message the injected database trigger
 // aborts with, so the test can recognize its own failure in the error
 // the scan reports.

trees.go

@@ -287,8 +287,7 @@ func (b *treeBuilder) finish() (*treeNode, []*treeNode) {
 // excluded.
 func fileEntry(name string, r scanRec) string {
 	return "f\x00" + name + "\x00" + strconv.FormatInt(r.size, 10) +
-		"\x00" + string(r.head[:]) + "\x00" + string(r.tail[:]) +
-		"\x00" + string(r.content[:])
+		"\x00" + r.head + "\x00" + r.tail + "\x00" + r.content
 }
 
 // collectTreeGroups groups directories by digest, counting each one on
@@ -338,8 +337,7 @@ func collectTreeGroups(allDirs []*treeNode, super *treeNode,
 // digest is the SHA-256 of its child entries — files serialized with
 // name and signature, subdirectories with name and recursive digest —
 // sorted byte-lexicographically. Filenames cannot contain NUL or "/",
-// and every hash and digest is 32 bytes, so NUL delimiters are
-// unambiguous.
+// so NUL delimiters are unambiguous.
 func (n *treeNode) computeDigest() {
 	slices.Sort(n.entries)
 

trees_test.go

@@ -10,18 +10,24 @@ import (
 	"time"
 )
 
+// Signature hashes shared by the smoke-test records.
+const (
+	f1Head    = "f1h"
+	f1Tail    = "f1t"
+	f1Content = "f1c"
+	f2Head    = "f2h"
+	f2Tail    = "f2t"
+	f2Content = "f2c"
+)
+
 // treesOpenLine is the plain line of trees' open phase when it opens
 // the database at once; it prints it as the phase starts and as it ends.
 const treesOpenLine = "[1/4] open: opening the database, elapsed 0s, " +
-	"remain unknown"
+	"eta unknown"
 
-// smokeTreeRecs mirrors the tree of the test under README "Definition
-// of done": /d/t1 and /d/t2 are identical, /d/t3 differs from them only
-// by one filename.
+// smokeTreeRecs mirrors the README smoke-test tree layout: /d/t1 and
+// /d/t2 are identical, /d/t3 differs from them only by one filename.
 func smokeTreeRecs() []scanRec {
-	f1Head, f1Tail, f1Content := testHash("f1h"), testHash("f1t"), testHash("f1c")
-	f2Head, f2Tail, f2Content := testHash("f2h"), testHash("f2t"), testHash("f2c")
-
 	return []scanRec{
 		{size: 3000, head: f1Head, tail: f1Tail, content: f1Content, path: "/d/t1/f1"},
 		{size: 100, head: f2Head, tail: f2Tail, content: f2Content, path: "/d/t1/sub/f2"},
@@ -147,14 +153,13 @@ func TestTreeNamesSortingBeforeSlash(t *testing.T) {
 
 	for _, top := range []string{"/a", "/c"} {
 		for _, p := range []string{"/b", "/b-x/f", "/b.txt", "/b/f"} {
-			content := testHash("c")
+			content := "c"
 			if p == "/b-x/f" {
-				content = testHash("other")
+				content = "other"
 			}
 
 			recs = append(recs, scanRec{
-				size: 1, head: testHash("h"), tail: testHash("t"),
-				content: content, path: top + p,
+				size: 1, head: "h", tail: "t", content: content, path: top + p,
 			})
 		}
 	}
@@ -237,11 +242,11 @@ func TestRunTreesProgressNamesEachPhase(t *testing.T) {
 		"trees: database " + path + " (from " + databaseEnv + ")",
 		treesOpenLine,
 		treesOpenLine,
-		"[2/4] read: counting the records, elapsed 0s, remain unknown",
+		"[2/4] read: counting the records, elapsed 0s, eta unknown",
 		"[2/4] read: [6/6] 100%, ",
-		"[3/4] group: [0/8] 0%, 0 directories/s, elapsed 0s, remain unknown",
+		"[3/4] group: [0/8] 0%, 0 directories/s, elapsed 0s, eta unknown",
 		"[3/4] group: [8/8] 100%, ",
-		"[4/4] write: [0/1] 0%, 0 groups/s, elapsed 0s, remain unknown",
+		"[4/4] write: [0/1] 0%, 0 groups/s, elapsed 0s, eta unknown",
 		"[4/4] write: [1/1] 100%, ",
 		"trees: 6 records read, 1 duplicate tree groups, 1 dupe trees, ",
 	}
@@ -327,10 +332,11 @@ func TestTreeDigests(t *testing.T) {
 func TestTreeDigestContentSensitivity(t *testing.T) {
 	t.Parallel()
 
-	same, c := testHash("same"), testHash("c")
+	const sharedTail = "same"
+
 	recs := []scanRec{
-		{size: 10, head: same, tail: same, content: c, path: "/r/a/f"},
-		{size: 10, head: testHash("DIFF"), tail: same, content: c, path: "/r/b/f"},
+		{size: 10, head: sharedTail, tail: sharedTail, content: "c", path: "/r/a/f"},
+		{size: 10, head: "DIFF", tail: sharedTail, content: "c", path: "/r/b/f"},
 	}
 
 	_, dirs := treeOf(t, recs)
@@ -391,10 +397,9 @@ func TestCollectTreeGroupsSiblings(t *testing.T) {
 
 	// Identical sibling dirs share a parent, so their group cannot be
 	// implied by a parent group and must be reported.
-	h, tl, c := testHash("h"), testHash("t"), testHash("c")
 	recs := []scanRec{
-		{size: 10, head: h, tail: tl, content: c, path: "/p/x1/f"},
-		{size: 10, head: h, tail: tl, content: c, path: "/p/x2/f"},
+		{size: 10, head: "h", tail: "t", content: "c", path: "/p/x1/f"},
+		{size: 10, head: "h", tail: "t", content: "c", path: "/p/x2/f"},
 	}
 
 	super, dirs := treeOf(t, recs)
@@ -413,11 +418,10 @@ func TestCollectTreeGroupsDifferingParents(t *testing.T) {
 	// /p/a and /q/b contain identical x subtrees, but /p/a has an
 	// extra file, so the parents' digests differ and the x group must
 	// be reported.
-	h, tl, c, e := testHash("h"), testHash("t"), testHash("c"), testHash("e")
 	recs := []scanRec{
-		{size: 10, head: h, tail: tl, content: c, path: "/p/a/x/f"},
-		{size: 99, head: e, tail: e, content: e, path: "/p/a/extra"},
-		{size: 10, head: h, tail: tl, content: c, path: "/q/b/x/f"},
+		{size: 10, head: "h", tail: "t", content: "c", path: "/p/a/x/f"},
+		{size: 99, head: "e", tail: "e", content: "e", path: "/p/a/extra"},
+		{size: 10, head: "h", tail: "t", content: "c", path: "/q/b/x/f"},
 	}
 
 	super, dirs := treeOf(t, recs)