Store mtime to the nanosecond so a same-second rewrite is re-hashed (closes #12)

scan recorded mtime in whole seconds, so a file rewritten in place at
the same size within the same second as its recorded mtime was classed
unchanged and kept its old hashes. The files table keeps mtime as whole
Unix seconds and gains mtime_nsec, the nanoseconds within that second.
scan holds the mtime as a time.Time and decides "newer" with After, so
any time a filesystem can record, one after 2262 included, compares in
the right order. The walk, a file given as an operand, and the content
phase's recheck all move over. PRAGMA user_version stays 1, per the
owner's ruling. README states what both columns hold.

Model: opus-5-5

README.md

@@ -314,7 +314,8 @@ All three subcommands operate on a single SQLite database file:
     CREATE TABLE files (
         path       BLOB PRIMARY KEY,  -- absolute path, raw bytes
         size       INTEGER NOT NULL,  -- bytes, from lstat
-        mtime   INTEGER NOT NULL,  -- Unix seconds, from lstat
+        mtime      INTEGER NOT NULL,  -- whole Unix seconds of the mtime, from lstat
+        mtime_nsec INTEGER NOT NULL,  -- nanoseconds within that second, 0 to 999999999
         head       TEXT NOT NULL,     -- lowercase-hex SHA-256; first 64 KiB, or whole file under 10 MiB
         tail       TEXT NOT NULL,     -- lowercase-hex SHA-256; last 64 KiB, or whole file under 10 MiB
         content    TEXT NOT NULL      -- lowercase-hex SHA-256, whole file or samples
@@ -323,18 +324,22 @@ All three subcommands operate on a single SQLite database file:
     ```
 
     Paths are stored as BLOBs because Unix paths are raw bytes, not guaranteed
-    UTF-8. `mtime` is used only for change detection; it is not part of the
-    duplicate key. For a file under 10 MiB `head`, `tail`, and `content` all
-    hold the whole-file hash (that range is hashed in full, with no end
-    windows); for a larger file `head` and `tail` hold the first- and last-64
-    KiB hashes and `content` the whole-file or sampled hash. All three are empty
-    strings when the file has never been hashed because its size was unique as
-    of the last scan that covered it. For a file of 10 MiB or more, `content`
-    stays empty until the content phase of a scan (see "`scan` mode" below) has
-    read the file. A record with an empty `content` is never part of a duplicate
-    group, though it still defines the file for tree reconstruction. The
-    `files_signature` index lets SQLite group the records by signature for
-    `report` without sorting the whole table.
+    UTF-8. `mtime` and `mtime_nsec` hold the file's mtime to the nanosecond:
+    `mtime` the whole Unix seconds, rounded down, and `mtime_nsec` the
+    nanoseconds past that second. Split this way they hold any mtime a
+    filesystem can record, one before 1678 or after 2262 included, which a
+    single 64-bit count of nanoseconds cannot. They are used only for change
+    detection and are not part of the duplicate key. For a file under 10 MiB
+    `head`, `tail`, and `content` all hold the whole-file hash (that range is
+    hashed in full, with no end windows); for a larger file `head` and `tail`
+    hold the first- and last-64 KiB hashes and `content` the whole-file or
+    sampled hash. All three are empty strings when the file has never been
+    hashed because its size was unique as of the last scan that covered it. For
+    a file of 10 MiB or more, `content` stays empty until the content phase of a
+    scan (see "`scan` mode" below) has read the file. A record with an empty
+    `content` is never part of a duplicate group, though it still defines the
+    file for tree reconstruction. The `files_signature` index lets SQLite group
+    the records by signature for `report` without sorting the whole table.
 
 ### Duplicate detection
 
@@ -421,6 +426,9 @@ operands:
   once its size, `head`, and `tail` match another record's.
 - A file whose mtime is newer than recorded, or whose size differs, is processed
   as if new: re-hashed, or recorded without hashes, per the shared-size rule.
+  Change detection compares the mtime to the nanosecond, as finely as the
+  filesystem records it, so a same-size rewrite later in the same second as the
+  recorded mtime still counts as a change.
 - A database record whose path lies under one of the scanned operands but was
   not successfully processed this run is deleted. This removes records for
   deleted files. It also removes records for paths that failed to stat or hash

TODO.md

@@ -29,6 +29,11 @@
 
 # Completed Steps
 
+- `scan` records mtime to the nanosecond, as whole seconds in `mtime` plus
+  `mtime_nsec`, and compares it at that resolution, so a same-size rewrite
+  within the same second is re-hashed (2026-10-07,
+  https://git.eeqj.de/sneak/sfdupes/issues/12)
+
 - cut the narration from `TODO.md` Completed Steps and from the comments in
   `script/` and both Dockerfiles; §Workflow now branches from and merges to
   `next` (2026-10-04, https://git.eeqj.de/sneak/sfdupes/issues/49)

db.go

@@ -11,6 +11,7 @@ import (
 	"path/filepath"
 	"slices"
 	"strconv"
+	"time"
 
 	"golang.org/x/sys/unix"
 	// The pure-Go SQLite driver, registered as "sqlite"; keeps cgo
@@ -40,12 +41,15 @@ const dbDirPerm = 0o755
 const lockFilePerm = 0o600
 
 // createTableSQL is the schema applied to a fresh database. Paths are
-// BLOBs because Unix paths are raw bytes, not guaranteed UTF-8.
+// BLOBs because Unix paths are raw bytes, not guaranteed UTF-8. mtime
+// holds whole Unix seconds and mtime_nsec the nanoseconds within that
+// second.
 const createTableSQL = `
 CREATE TABLE files (
     path       BLOB PRIMARY KEY,
     size       INTEGER NOT NULL,
     mtime      INTEGER NOT NULL,
+    mtime_nsec INTEGER NOT NULL,
     head       TEXT NOT NULL,
     tail       TEXT NOT NULL,
     content    TEXT NOT NULL
@@ -61,10 +65,11 @@ CREATE INDEX files_signature ON files (size, head, tail, content)
 // upsertSQL inserts one file record, replacing any existing record for
 // the same path.
 const upsertSQL = `
-INSERT INTO files (path, size, mtime, head, tail, content)
-VALUES (?, ?, ?, ?, ?, ?)
+INSERT INTO files (path, size, mtime, mtime_nsec, head, tail, content)
+VALUES (?, ?, ?, ?, ?, ?, ?)
 ON CONFLICT (path) DO UPDATE SET
     size = excluded.size, mtime = excluded.mtime,
+    mtime_nsec = excluded.mtime_nsec,
     head = excluded.head, tail = excluded.tail,
     content = excluded.content
 `
@@ -356,8 +361,8 @@ func userVersion(ctx context.Context, db *sql.DB) (int, error) {
 // which is the order of the primary key, so SQLite does not sort.
 func loadFileRows(ctx context.Context, db *sql.DB, fn func(r scanRec)) error {
 	rows, err := db.QueryContext(ctx,
-		"SELECT path, size, mtime, head, tail, content FROM files "+
-			"ORDER BY path")
+		"SELECT path, size, mtime, mtime_nsec, head, tail, content "+
+			"FROM files ORDER BY path")
 	if err != nil {
 		return fmt.Errorf("read records: %w", err)
 	}
@@ -367,16 +372,18 @@ func loadFileRows(ctx context.Context, db *sql.DB, fn func(r scanRec)) error {
 	for rows.Next() {
 		var (
 			path      []byte
+			sec, nsec int64
 			r         scanRec
 		)
 
-		err = rows.Scan(&path, &r.size, &r.mtime, &r.head, &r.tail,
+		err = rows.Scan(&path, &r.size, &sec, &nsec, &r.head, &r.tail,
 			&r.content)
 		if err != nil {
 			return fmt.Errorf("read record: %w", err)
 		}
 
 		r.path = string(path)
+		r.mtime = time.Unix(sec, nsec)
 		fn(r)
 	}
 
@@ -466,10 +473,10 @@ func loadDupeRows(ctx context.Context, db *sql.DB,
 // values, and skipping the hash columns keeps the scan's in-memory
 // index small on multi-million-file databases.
 func loadFileMeta(ctx context.Context, db *sql.DB,
-	fn func(path string, size, mtime int64, hashed bool),
+	fn func(path string, size int64, mtime time.Time, hashed bool),
 ) error {
 	rows, err := db.QueryContext(ctx,
-		"SELECT path, size, mtime, head <> '' FROM files")
+		"SELECT path, size, mtime, mtime_nsec, head <> '' FROM files")
 	if err != nil {
 		return fmt.Errorf("read records: %w", err)
 	}
@@ -479,16 +486,16 @@ func loadFileMeta(ctx context.Context, db *sql.DB,
 	for rows.Next() {
 		var (
 			path            []byte
-			size, mtime int64
+			size, sec, nsec int64
 			hashed          int64
 		)
 
-		err = rows.Scan(&path, &size, &mtime, &hashed)
+		err = rows.Scan(&path, &size, &sec, &nsec, &hashed)
 		if err != nil {
 			return fmt.Errorf("read record: %w", err)
 		}
 
-		fn(string(path), size, mtime, hashed != 0)
+		fn(string(path), size, time.Unix(sec, nsec), hashed != 0)
 	}
 
 	err = rows.Err()
@@ -507,7 +514,7 @@ func loadFileMeta(ctx context.Context, db *sql.DB,
 // memory; the rows come ordered by size, head, and tail, so each
 // group's rows arrive together.
 const contentCandidatesSQL = `
-SELECT f.path, f.size, f.mtime, f.head, f.tail, f.content <> ''
+SELECT f.path, f.size, f.mtime, f.mtime_nsec, f.head, f.tail, f.content <> ''
 FROM files AS f
 JOIN (
     SELECT size, head, tail
@@ -534,16 +541,19 @@ func loadContentCandidates(ctx context.Context, db *sql.DB,
 	for rows.Next() {
 		var (
 			path      []byte
+			sec, nsec int64
 			r         scanRec
 			hashed    int64
 		)
 
-		err = rows.Scan(&path, &r.size, &r.mtime, &r.head, &r.tail, &hashed)
+		err = rows.Scan(&path, &r.size, &sec, &nsec, &r.head, &r.tail,
+			&hashed)
 		if err != nil {
 			return fmt.Errorf("read record: %w", err)
 		}
 
 		r.path = string(path)
+		r.mtime = time.Unix(sec, nsec)
 		fn(r, hashed != 0)
 	}
 
@@ -627,8 +637,8 @@ func execUpserts(ctx context.Context, tx *sql.Tx, upserts []scanRec,
 	defer func() { _ = st.Close() }()
 
 	for _, r := range upserts {
-		_, err = st.ExecContext(ctx,
-			[]byte(r.path), r.size, r.mtime, r.head, r.tail, r.content)
+		_, err = st.ExecContext(ctx, []byte(r.path), r.size,
+			r.mtime.Unix(), r.mtime.Nanosecond(), r.head, r.tail, r.content)
 		if err != nil {
 			return fmt.Errorf("upsert %s: %w", r.path, err)
 		}

db_test.go

@@ -10,6 +10,7 @@ import (
 	"slices"
 	"strings"
 	"testing"
+	"time"
 )
 
 // testDBPath returns a database path inside a fresh temp dir.
@@ -260,10 +261,13 @@ func TestApplyChangesRoundTrip(t *testing.T) {
 	// written.
 	recs := []scanRec{
 		{
-			size: 2, mtime: 20, head: "h2", tail: "t2", content: "c2",
-			path: "/a/tab\tnew\nline",
+			size: 2, mtime: time.Unix(20, 999_999_999), head: "h2", tail: "t2",
+			content: "c2", path: "/a/tab\tnew\nline",
+		},
+		{
+			size: 1, mtime: time.Unix(10, 0), head: "h1", tail: "t1",
+			content: "c1", path: "/a/x",
 		},
-		{size: 1, mtime: 10, head: "h1", tail: "t1", content: "c1", path: "/a/x"},
 	}
 
 	err := applyChanges(t.Context(), db, recs, nil,
@@ -281,7 +285,8 @@ func TestApplyChangesRoundTrip(t *testing.T) {
 	// An upsert for an existing path updates in place; a delete
 	// removes exactly its path.
 	upd := scanRec{
-		size: 3, mtime: 30, head: "h3", tail: "t3", content: "c3", path: "/a/x",
+		size: 3, mtime: time.Unix(30, 0), head: "h3", tail: "t3", content: "c3",
+		path: "/a/x",
 	}
 
 	err = applyChanges(t.Context(), db, []scanRec{upd},
@@ -308,7 +313,7 @@ func TestApplyChangesBatching(t *testing.T) {
 	recs := make([]scanRec, 0, n)
 	for i := range n {
 		recs = append(recs, scanRec{
-			size: int64(i), mtime: 1, head: "h", tail: "t",
+			size: int64(i), mtime: time.Unix(1, 0), head: "h", tail: "t",
 			path: fmt.Sprintf("/batch/%07d", i),
 		})
 	}

report.go

@@ -7,6 +7,7 @@ import (
 	"io"
 	"os"
 	"strings"
+	"time"
 )
 
 // ioBufSize is the buffer size for the buffered stdout writers.
@@ -21,7 +22,7 @@ const minGroupSize = 2
 // only and used by scan for change detection.
 type scanRec struct {
 	size    int64
-	mtime   int64
+	mtime   time.Time
 	head    string
 	tail    string
 	content string

report_test.go

@@ -11,6 +11,7 @@ import (
 	"slices"
 	"strings"
 	"testing"
+	"time"
 )
 
 // awkwardDir is a directory name holding every byte the reports escape.
@@ -313,8 +314,14 @@ func TestDupeGroupsMtimeExcluded(t *testing.T) {
 	// mtime is informational only; records differing only in mtime
 	// still group together.
 	recs := []scanRec{
-		{size: 9, mtime: 100, head: "h", tail: "t", content: "c", path: "/m/1"},
-		{size: 9, mtime: 200, head: "h", tail: "t", content: "c", path: "/m/2"},
+		{
+			size: 9, mtime: time.Unix(100, 0), head: "h", tail: "t",
+			content: "c", path: "/m/1",
+		},
+		{
+			size: 9, mtime: time.Unix(200, 0), head: "h", tail: "t",
+			content: "c", path: "/m/2",
+		},
 	}
 
 	groups := dupeGroupsOf(t, recs)

scan.go

@@ -17,6 +17,7 @@ import (
 	"strings"
 	"sync"
 	"syscall"
+	"time"
 )
 
 // The duplicate ladder (see hashSignature and README "Duplicate
@@ -68,7 +69,7 @@ var errInterrupted = errors.New("scan interrupted")
 type fileRec struct {
 	path  string
 	size  int64
-	mtime int64
+	mtime time.Time
 	dev   uint64
 	ino   uint64
 }
@@ -79,7 +80,7 @@ type fileRec struct {
 // they would dominate the scan's memory.
 type fileMeta struct {
 	size   int64
-	mtime  int64
+	mtime  time.Time
 	hashed bool
 }
 
@@ -369,7 +370,7 @@ func (s *scanState) loadIndex(ctx context.Context, roots []string) error {
 	s.existing = make(map[string]fileMeta)
 
 	return loadFileMeta(ctx, s.db,
-		func(path string, size, mtime int64, hashed bool) {
+		func(path string, size int64, mtime time.Time, hashed bool) {
 			prog.increment()
 
 			if underAnyRoot(path, roots) {
@@ -415,7 +416,7 @@ func (s *scanState) walkPhase(
 		old, ok := s.existing[ev.rec.path]
 
 		switch {
-		case !ok || old.size != ev.rec.size || old.mtime < ev.rec.mtime:
+		case !ok || old.size != ev.rec.size || ev.rec.mtime.After(old.mtime):
 			changed = append(changed, ev.rec)
 		case old.hashed:
 			delete(s.existing, ev.rec.path)
@@ -807,7 +808,7 @@ func unchangedFile(r scanRec) (fileRec, bool, error) {
 	}
 
 	if !fi.Mode().IsRegular() || fi.Size() != r.size ||
-		fi.ModTime().Unix() > r.mtime {
+		fi.ModTime().After(r.mtime) {
 		return fileRec{}, false, nil
 	}
 
@@ -966,7 +967,7 @@ func seedRoot(ctx context.Context, root string,
 	sendEvent(ctx, events, walkEvent{rec: fileRec{
 		path:  root,
 		size:  fi.Size(),
-		mtime: fi.ModTime().Unix(),
+		mtime: fi.ModTime(),
 		dev:   dev,
 		ino:   ino,
 	}})
@@ -1124,7 +1125,7 @@ func emitFile(ctx context.Context, p string, e fs.DirEntry,
 	sendEvent(ctx, events, walkEvent{rec: fileRec{
 		path:  p,
 		size:  info.Size(),
-		mtime: info.ModTime().Unix(),
+		mtime: info.ModTime(),
 		dev:   dev,
 		ino:   ino,
 	}})

scan_test.go

@@ -783,8 +783,8 @@ func TestWalk(t *testing.T) {
 			t.Errorf("%s: size = %d, want 1..3", r.path, r.size)
 		}
 
-		if r.mtime <= 0 {
-			t.Errorf("%s: mtime = %d, want positive", r.path, r.mtime)
+		if r.mtime.Unix() <= 0 {
+			t.Errorf("%s: mtime = %v, want after 1970", r.path, r.mtime)
 		}
 	}
 }
@@ -1285,11 +1285,151 @@ func TestSyncScanMtimeBump(t *testing.T) {
 		t.Fatalf("mtime-bump stats = %+v, want 1 updated", st)
 	}
 
-	if r := recordByPath(t, dbRecords(t, db), a); r.mtime != future.Unix() {
-		t.Fatalf("mtime = %d, want %d", r.mtime, future.Unix())
+	if r := recordByPath(t, dbRecords(t, db), a); !r.mtime.Equal(future) {
+		t.Fatalf("mtime = %v, want %v", r.mtime, future)
 	}
 }
 
+// assertWholeFileHashed fails unless the record for path holds the
+// whole-file hash of data as its head, tail, and content.
+func assertWholeFileHashed(t *testing.T, db *sql.DB, path string,
+	data []byte,
+) {
+	t.Helper()
+
+	r := recordByPath(t, dbRecords(t, db), path)
+	if want := hexSum(data); r.head != want || r.tail != want ||
+		r.content != want {
+		t.Fatalf("head, tail, content = %q, %q, %q, want %q for each",
+			r.head, r.tail, r.content, want)
+	}
+}
+
+// TestSyncScanSameSecondRewrite rewrites a file in place at the same
+// size with an mtime later in the same second as the recorded one: the
+// next scan must notice the change and re-hash the file.
+func TestSyncScanSameSecondRewrite(t *testing.T) {
+	t.Parallel()
+
+	dir := t.TempDir()
+	db := openTestDB(t)
+	a := writeFile(t, dir, "a.bin", pattern(1, 500))
+
+	// b.bin shares the size of a.bin, so a.bin is hashed.
+	writeFile(t, dir, "b.bin", pattern(2, 500))
+
+	first := time.Date(2026, 1, 2, 3, 4, 5, 100_000_000, time.UTC)
+
+	err := os.Chtimes(a, first, first)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	syncTree(t, db, dir)
+
+	rewritten := pattern(3, 500)
+	writeFile(t, dir, "a.bin", rewritten)
+
+	later := first.Add(500 * time.Millisecond)
+
+	err = os.Chtimes(a, later, later)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	st := syncTree(t, db, dir)
+	if st != (scanStats{walked: 2, updated: 1, unchanged: 1}) {
+		t.Fatalf("rescan stats = %+v, want 1 updated 1 unchanged", st)
+	}
+
+	assertWholeFileHashed(t, db, a, rewritten)
+}
+
+// TestSyncScanOperandSameSecondRewrite is TestSyncScanSameSecondRewrite
+// for files given to scan as operands, which scan stats without reading
+// their directory.
+func TestSyncScanOperandSameSecondRewrite(t *testing.T) {
+	t.Parallel()
+
+	dir := t.TempDir()
+	db := openTestDB(t)
+	a := writeFile(t, dir, "a.bin", pattern(1, 500))
+
+	// b.bin shares the size of a.bin, so a.bin is hashed.
+	b := writeFile(t, dir, "b.bin", pattern(2, 500))
+
+	first := time.Date(2026, 1, 2, 3, 4, 5, 100_000_000, time.UTC)
+
+	err := os.Chtimes(a, first, first)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	syncTree(t, db, a, b)
+
+	rewritten := pattern(3, 500)
+	writeFile(t, dir, "a.bin", rewritten)
+
+	later := first.Add(500 * time.Millisecond)
+
+	err = os.Chtimes(a, later, later)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	st := syncTree(t, db, a, b)
+	if st != (scanStats{walked: 2, updated: 1, unchanged: 1}) {
+		t.Fatalf("rescan stats = %+v, want 1 updated 1 unchanged", st)
+	}
+
+	assertWholeFileHashed(t, db, a, rewritten)
+}
+
+// TestSyncScanRewriteAfter2262 rewrites a file in place at the same
+// size and sets its mtime after 2262, a time too late to count in
+// nanoseconds in an int64: the next scan must still re-hash it.
+func TestSyncScanRewriteAfter2262(t *testing.T) {
+	t.Parallel()
+
+	dir := t.TempDir()
+	db := openTestDB(t)
+	a := writeFile(t, dir, "a.bin", pattern(1, 500))
+
+	// b.bin shares the size of a.bin, so a.bin is hashed.
+	writeFile(t, dir, "b.bin", pattern(2, 500))
+
+	syncTree(t, db, dir)
+
+	rewritten := pattern(3, 500)
+	writeFile(t, dir, "a.bin", rewritten)
+
+	// os.Chtimes cannot set this time: it converts through UnixNano.
+	late := time.Date(2300, 1, 2, 3, 4, 5, 0, time.UTC)
+	ts := syscall.Timespec{Sec: late.Unix()}
+
+	err := syscall.UtimesNano(a, []syscall.Timespec{ts, ts})
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	fi, err := os.Lstat(a)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if !fi.ModTime().Equal(late) {
+		t.Skipf("the filesystem stored mtime %v instead of %v",
+			fi.ModTime().UTC(), late)
+	}
+
+	st := syncTree(t, db, dir)
+	if st != (scanStats{walked: 2, updated: 1, unchanged: 1}) {
+		t.Fatalf("rescan stats = %+v, want 1 updated 1 unchanged", st)
+	}
+
+	assertWholeFileHashed(t, db, a, rewritten)
+}
+
 func TestSyncScanAddRemove(t *testing.T) {
 	t.Parallel()
 
@@ -1335,9 +1475,7 @@ func TestSyncScanSizeChange(t *testing.T) {
 
 	writeFile(t, dir, "f", pattern(1, 200))
 
-	mt := time.Unix(old.mtime, 0)
-
-	err := os.Chtimes(p, mt, mt)
+	err := os.Chtimes(p, old.mtime, old.mtime)
 	if err != nil {
 		t.Fatal(err)
 	}