Store mtime to the nanosecond so a same-second rewrite is re-hashed (closes #12)
check / check (push) Waiting to run

scan recorded mtime in whole seconds, so a file rewritten in place at
the same size within the same second as its recorded mtime was classed
unchanged and kept its old hashes. The files table keeps mtime as whole
Unix seconds and gains mtime_nsec, the nanoseconds within that second.
scan holds the mtime as a time.Time and decides "newer" by comparing
Unix() and then Nanosecond(), so any time a filesystem can record
compares in the right order; After would misorder one too late for a
time.Time to hold without wrapping. The walk, a file given as an
operand, and the content phase's recheck all move over. PRAGMA
user_version stays 1, per the owner's ruling. README states what both
columns hold.

Model: opus-5-5
This commit was merged in pull request #96.
This commit is contained in:
2026-10-08 02:51:53 +02:00
parent 0064eba542
commit 5900feb515
8 changed files with 335 additions and 68 deletions
+227 -7
View File
@@ -20,6 +20,8 @@ import (
"syscall"
"testing"
"time"
"golang.org/x/sys/unix"
)
// writeFile creates a file with the given content and returns its path.
@@ -581,6 +583,65 @@ func TestScanContentHashedStalePartners(t *testing.T) {
}
}
// TestScanContentSameSecondRewrite is TestScanContentStalePartners for
// a stored file rewritten in place at the same size with an mtime later
// in the same second than recorded: the file counts as changed, so
// neither it nor its match inside the operand is read.
func TestScanContentSameSecondRewrite(t *testing.T) {
t.Parallel()
db := openTestDB(t)
dirA := t.TempDir()
changed := sparseFileWithoutMatch(t, dirA, "changed", headTailMin)
first := time.Date(2026, 1, 2, 3, 4, 5, 100_000_000, time.UTC)
err := os.Chtimes(changed, first, first)
if err != nil {
t.Fatal(err)
}
syncTree(t, db, dirA)
before := dbRecords(t, db)
// Rewrite one byte in place, keeping the size.
pokeAt(t, changed, headTailMin/2, []byte{1})
later := first.Add(500 * time.Millisecond)
err = os.Chtimes(changed, later, later)
if err != nil {
t.Fatal(err)
}
dirB := t.TempDir()
sparseFile(t, dirB, "changed-copy", headTailMin)
st := syncTree(t, db, dirB)
if st != (scanStats{walked: 1, added: 1}) {
t.Errorf("stats = %+v, want 1 added and nothing skipped", st)
}
recs := dbRecords(t, db)
for _, r := range recs {
if r.content != "" {
t.Errorf("%s: content = %q, want none: its only match is stale",
r.path, r.content)
}
}
for _, old := range before {
if r := recordByPath(t, recs, old.path); r != old {
t.Errorf("record = %+v, want it left as %+v", r, old)
}
}
if groups := dupeGroups(t, db); len(groups) != 0 {
t.Errorf("groups = %+v, want none", groups)
}
}
// TestScanContentReadFailure checks that a failed content read is
// counted as skipped and leaves the record without a content hash, and
// that a later scan tries the read again.
@@ -783,8 +844,8 @@ func TestWalk(t *testing.T) {
t.Errorf("%s: size = %d, want 1..3", r.path, r.size)
}
if r.mtime <= 0 {
t.Errorf("%s: mtime = %d, want positive", r.path, r.mtime)
if r.mtime.Unix() <= 0 {
t.Errorf("%s: mtime = %v, want after 1970", r.path, r.mtime)
}
}
}
@@ -1285,11 +1346,172 @@ func TestSyncScanMtimeBump(t *testing.T) {
t.Fatalf("mtime-bump stats = %+v, want 1 updated", st)
}
if r := recordByPath(t, dbRecords(t, db), a); r.mtime != future.Unix() {
t.Fatalf("mtime = %d, want %d", r.mtime, future.Unix())
if r := recordByPath(t, dbRecords(t, db), a); !r.mtime.Equal(future) {
t.Fatalf("mtime = %v, want %v", r.mtime, future)
}
}
// assertWholeFileHashed fails unless the record for path holds the
// whole-file hash of data as its head, tail, and content.
func assertWholeFileHashed(t *testing.T, db *sql.DB, path string,
data []byte,
) {
t.Helper()
r := recordByPath(t, dbRecords(t, db), path)
if want := hexSum(data); r.head != want || r.tail != want ||
r.content != want {
t.Fatalf("head, tail, content = %q, %q, %q, want %q for each",
r.head, r.tail, r.content, want)
}
}
// TestSyncScanSameSecondRewrite rewrites a file in place at the same
// size with an mtime later in the same second as the recorded one: the
// next scan must notice the change and re-hash the file.
func TestSyncScanSameSecondRewrite(t *testing.T) {
t.Parallel()
dir := t.TempDir()
db := openTestDB(t)
a := writeFile(t, dir, "a.bin", pattern(1, 500))
// b.bin shares the size of a.bin, so a.bin is hashed.
writeFile(t, dir, "b.bin", pattern(2, 500))
first := time.Date(2026, 1, 2, 3, 4, 5, 100_000_000, time.UTC)
err := os.Chtimes(a, first, first)
if err != nil {
t.Fatal(err)
}
syncTree(t, db, dir)
rewritten := pattern(3, 500)
writeFile(t, dir, "a.bin", rewritten)
later := first.Add(500 * time.Millisecond)
err = os.Chtimes(a, later, later)
if err != nil {
t.Fatal(err)
}
st := syncTree(t, db, dir)
if st != (scanStats{walked: 2, updated: 1, unchanged: 1}) {
t.Fatalf("rescan stats = %+v, want 1 updated 1 unchanged", st)
}
assertWholeFileHashed(t, db, a, rewritten)
}
// TestSyncScanOperandSameSecondRewrite is TestSyncScanSameSecondRewrite
// for files given to scan as operands, which scan stats without reading
// their directory.
func TestSyncScanOperandSameSecondRewrite(t *testing.T) {
t.Parallel()
dir := t.TempDir()
db := openTestDB(t)
a := writeFile(t, dir, "a.bin", pattern(1, 500))
// b.bin shares the size of a.bin, so a.bin is hashed.
b := writeFile(t, dir, "b.bin", pattern(2, 500))
first := time.Date(2026, 1, 2, 3, 4, 5, 100_000_000, time.UTC)
err := os.Chtimes(a, first, first)
if err != nil {
t.Fatal(err)
}
syncTree(t, db, a, b)
rewritten := pattern(3, 500)
writeFile(t, dir, "a.bin", rewritten)
later := first.Add(500 * time.Millisecond)
err = os.Chtimes(a, later, later)
if err != nil {
t.Fatal(err)
}
st := syncTree(t, db, a, b)
if st != (scanStats{walked: 2, updated: 1, unchanged: 1}) {
t.Fatalf("rescan stats = %+v, want 1 updated 1 unchanged", st)
}
assertWholeFileHashed(t, db, a, rewritten)
}
// TestSyncScanRewriteAfter2262 runs assertLateRewriteRehashed with an
// mtime after 2262, a time too late to count in nanoseconds in an int64.
func TestSyncScanRewriteAfter2262(t *testing.T) {
t.Parallel()
assertLateRewriteRehashed(t, time.Date(2300, 1, 2, 3, 4, 5, 0, time.UTC))
}
// TestSyncScanRewritePastTimeLimit runs assertLateRewriteRehashed with an
// mtime one second past the latest a time.Time holds without wrapping it
// to a time far in the past.
func TestSyncScanRewritePastTimeLimit(t *testing.T) {
t.Parallel()
assertLateRewriteRehashed(t, time.Unix(9223371974719179008, 0))
}
// assertLateRewriteRehashed scans a directory, rewrites a file in it in
// place at the same size, sets its mtime to late, and fails unless the
// next scan re-hashes the file. It skips where late does not fit the
// platform's timespec or the filesystem does not store it.
func assertLateRewriteRehashed(t *testing.T, late time.Time) {
t.Helper()
dir := t.TempDir()
db := openTestDB(t)
a := writeFile(t, dir, "a.bin", pattern(1, 500))
// b.bin shares the size of a.bin, so a.bin is hashed.
writeFile(t, dir, "b.bin", pattern(2, 500))
syncTree(t, db, dir)
rewritten := pattern(3, 500)
writeFile(t, dir, "a.bin", rewritten)
// os.Chtimes cannot set such a time: it converts through UnixNano.
ts, err := unix.TimeToTimespec(late)
if err != nil {
t.Skipf("an mtime %d seconds after 1970 does not fit this platform's "+
"timespec: %v", late.Unix(), err)
}
err = unix.UtimesNano(a, []unix.Timespec{ts, ts})
if err != nil {
t.Fatal(err)
}
fi, err := os.Lstat(a)
if err != nil {
t.Fatal(err)
}
if fi.ModTime().Unix() != late.Unix() {
t.Skipf("the filesystem stored the mtime as %d seconds after 1970, "+
"not %d", fi.ModTime().Unix(), late.Unix())
}
st := syncTree(t, db, dir)
if st != (scanStats{walked: 2, updated: 1, unchanged: 1}) {
t.Fatalf("rescan stats = %+v, want 1 updated 1 unchanged", st)
}
assertWholeFileHashed(t, db, a, rewritten)
}
func TestSyncScanAddRemove(t *testing.T) {
t.Parallel()
@@ -1335,9 +1557,7 @@ func TestSyncScanSizeChange(t *testing.T) {
writeFile(t, dir, "f", pattern(1, 200))
mt := time.Unix(old.mtime, 0)
err := os.Chtimes(p, mt, mt)
err := os.Chtimes(p, old.mtime, old.mtime)
if err != nil {
t.Fatal(err)
}