From d27e2760320b3c6d8b694f17d7f2f17d805bec55 Mon Sep 17 00:00:00 2001 From: sneak Date: Wed, 7 Oct 2026 01:11:33 +0000 Subject: [PATCH] Store and compare file mtimes in nanoseconds (closes #226) The files table held mtime in whole seconds and the scanner compared whole seconds. A file rewritten with its size unchanged and a new mtime in the same second as the indexed one was treated as unchanged, and every later snapshot restored the old content. mtime is now stored as nanoseconds since the Unix epoch and compared at that precision. A local index written before this change holds seconds in mtime: its next backup re-chunks every file once, and a restore of a snapshot made before it sets mtimes near 1970. An mtime outside the years 1678 to 2262 does not fit in int64 nanoseconds; it is stored wrong and restored wrong. Model: opus-5-5 --- TODO.md | 10 +++ docs/DATAMODEL.md | 2 +- internal/database/files.go | 16 ++-- .../database/repository_edge_cases_test.go | 3 +- internal/database/schema/001.sql | 2 +- internal/snapshot/scanner.go | 2 +- internal/vaultik/same_second_rewrite_test.go | 73 +++++++++++++++++++ 7 files changed, 95 insertions(+), 13 deletions(-) create mode 100644 internal/vaultik/same_second_rewrite_test.go diff --git a/TODO.md b/TODO.md index ec17c26..684e325 100644 --- a/TODO.md +++ b/TODO.md @@ -22,6 +22,16 @@ the tag exists and is exercised; what is left is merging `next` to # Completed Steps +- 2026-10-07: Made a backup notice a file rewritten with its size + unchanged and a new mtime in the same second as the one in the index + ([issue #226](https://git.eeqj.de/sneak/vaultik/issues/226)). The + `files` table held mtime in whole seconds and the scanner compared + whole seconds, so every later snapshot kept the old content. `mtime` + now holds nanoseconds since the Unix epoch and is compared at that + precision. A local index written before the change holds seconds + there, so its next backup re-chunks every file once; chunks it already + stored are not uploaded again. + - 2026-10-06: Made the backup summary and the `snapshots` row count each file, byte and upload once ([issue #225](https://git.eeqj.de/sneak/vaultik/issues/225)). The diff --git a/docs/DATAMODEL.md b/docs/DATAMODEL.md index a1967a3..5851247 100644 --- a/docs/DATAMODEL.md +++ b/docs/DATAMODEL.md @@ -36,7 +36,7 @@ Stores metadata about files in the filesystem being backed up. **Columns:** - `id` (TEXT PRIMARY KEY) - UUID for the file record - `path` (TEXT NOT NULL UNIQUE) - Absolute file path -- `mtime` (INTEGER NOT NULL) - Modification time as Unix timestamp +- `mtime` (INTEGER NOT NULL) - Modification time in nanoseconds since the Unix epoch - `size` (INTEGER NOT NULL) - File size in bytes - `mode` (INTEGER NOT NULL) - Unix file permissions and type - `uid` (INTEGER NOT NULL) - User ID of file owner diff --git a/internal/database/files.go b/internal/database/files.go index 02bd26c..c7b3044 100644 --- a/internal/database/files.go +++ b/internal/database/files.go @@ -54,16 +54,16 @@ func (r *FileRepository) Create(ctx context.Context, tx *sql.Tx, file *File) err if tx != nil { LogSQL("Execute", query, file.ID.String(), file.Path.String(), file.SourcePath.String(), - file.MTime.Unix(), file.Size, file.Mode, file.UID, file.GID, + file.MTime.UnixNano(), file.Size, file.Mode, file.UID, file.GID, file.LinkTarget.String()) err = tx.QueryRowContext(ctx, query, file.ID.String(), file.Path.String(), file.SourcePath.String(), - file.MTime.Unix(), file.Size, file.Mode, file.UID, file.GID, + file.MTime.UnixNano(), file.Size, file.Mode, file.UID, file.GID, file.LinkTarget.String()).Scan(&idStr) } else { err = r.db.QueryRowWithLog(ctx, query, file.ID.String(), file.Path.String(), file.SourcePath.String(), - file.MTime.Unix(), file.Size, file.Mode, file.UID, file.GID, + file.MTime.UnixNano(), file.Size, file.Mode, file.UID, file.GID, file.LinkTarget.String()).Scan(&idStr) } @@ -164,7 +164,7 @@ func (r *FileRepository) ListModifiedSince( ORDER BY path ` - rows, err := r.db.conn.QueryContext(ctx, query, since.Unix()) + rows, err := r.db.conn.QueryContext(ctx, query, since.UnixNano()) if err != nil { return nil, fmt.Errorf("querying files: %w", err) } @@ -392,7 +392,7 @@ func (r *FileRepository) CreateBatch( args = append(args, f.ID.String(), f.Path.String(), f.SourcePath.String(), - f.MTime.Unix(), f.Size, f.Mode, f.UID, f.GID, + f.MTime.UnixNano(), f.Size, f.Mode, f.UID, f.GID, f.LinkTarget.String()) } @@ -460,7 +460,7 @@ func (r *FileRepository) scanFileFrom(row fileRowScanner) (*File, error) { var ( file File idStr, pathStr, sourcePathStr string - mtimeUnix int64 + mtimeUnixNano int64 linkTarget sql.NullString ) @@ -468,7 +468,7 @@ func (r *FileRepository) scanFileFrom(row fileRowScanner) (*File, error) { &idStr, &pathStr, &sourcePathStr, - &mtimeUnix, + &mtimeUnixNano, &file.Size, &file.Mode, &file.UID, @@ -487,7 +487,7 @@ func (r *FileRepository) scanFileFrom(row fileRowScanner) (*File, error) { file.Path = types.FilePath(pathStr) file.SourcePath = types.SourcePath(sourcePathStr) - file.MTime = time.Unix(mtimeUnix, 0).UTC() + file.MTime = time.Unix(0, mtimeUnixNano).UTC() if linkTarget.Valid { file.LinkTarget = types.FilePath(linkTarget.String) } diff --git a/internal/database/repository_edge_cases_test.go b/internal/database/repository_edge_cases_test.go index a4cf9f7..59f85ca 100644 --- a/internal/database/repository_edge_cases_test.go +++ b/internal/database/repository_edge_cases_test.go @@ -606,8 +606,7 @@ func TestTimezoneHandling(t *testing.T) { t.Skip("timezone not available") } - // Use Truncate to remove sub-second precision since we store as Unix timestamps - nyTime := time.Now().In(loc).Truncate(time.Second) + nyTime := time.Now().In(loc) file := &File{ Path: "/timezone-test.txt", MTime: nyTime, diff --git a/internal/database/schema/001.sql b/internal/database/schema/001.sql index 944c3a6..d99dc50 100644 --- a/internal/database/schema/001.sql +++ b/internal/database/schema/001.sql @@ -6,7 +6,7 @@ CREATE TABLE IF NOT EXISTS files ( id TEXT PRIMARY KEY, -- UUID path TEXT NOT NULL UNIQUE, source_path TEXT NOT NULL DEFAULT '', -- The source directory this file came from (for restore path stripping) - mtime INTEGER NOT NULL, + mtime INTEGER NOT NULL, -- nanoseconds since the Unix epoch size INTEGER NOT NULL, mode INTEGER NOT NULL, uid INTEGER NOT NULL, diff --git a/internal/snapshot/scanner.go b/internal/snapshot/scanner.go index 6676f6e..52daa85 100644 --- a/internal/snapshot/scanner.go +++ b/internal/snapshot/scanner.go @@ -1221,7 +1221,7 @@ func (s *Scanner) checkFileInMemory( // Check if file has changed if existingFile.Size != file.Size || - existingFile.MTime.Unix() != file.MTime.Unix() || + existingFile.MTime.UnixNano() != file.MTime.UnixNano() || existingFile.Mode != file.Mode || existingFile.UID != file.UID || existingFile.GID != file.GID { diff --git a/internal/vaultik/same_second_rewrite_test.go b/internal/vaultik/same_second_rewrite_test.go new file mode 100644 index 0000000..71ca958 --- /dev/null +++ b/internal/vaultik/same_second_rewrite_test.go @@ -0,0 +1,73 @@ +package vaultik_test + +import ( + "bytes" + "context" + "path/filepath" + "testing" + "time" + + "github.com/spf13/afero" + "github.com/stretchr/testify/require" + "sneak.berlin/go/vaultik/internal/database" + "sneak.berlin/go/vaultik/internal/log" + "sneak.berlin/go/vaultik/internal/storage" + "sneak.berlin/go/vaultik/internal/vaultik" +) + +// A file rewritten with its size unchanged and a new mtime in the same +// second as the mtime the index holds must still be backed up. See +// https://git.eeqj.de/sneak/vaultik/issues/226. +// +//nolint:paralleltest // installs the global logger via log.Initialize +func TestBackupOfSameSecondRewriteRestoresNewContent(t *testing.T) { + log.Initialize(log.Config{}) + + fs := afero.NewOsFs() + tempDir := t.TempDir() + dataDir := filepath.Join(tempDir, "src") + storeDir := filepath.Join(tempDir, "remote") + restoreDir := filepath.Join(tempDir, "restored") + dbPath := filepath.Join(tempDir, "index.sqlite") + rewrittenPath := filepath.Join(dataDir, "small.txt") + + ctx := context.Background() + files := writeFaultSourceTree(t, fs, dataDir) + cfg := changedFileConfig(dataDir, dbPath) + + firstMTime := time.Date(2026, time.January, 2, 3, 4, 5, 0, time.UTC). + Add(100 * time.Millisecond) + secondMTime := firstMTime.Add(800 * time.Millisecond) + + require.NoError(t, fs.Chtimes(rewrittenPath, firstMTime, firstMTime)) + + store, err := storage.NewFileStorer(storeDir) + require.NoError(t, err) + + db, err := database.New(ctx, dbPath) + require.NoError(t, err) + + repos := database.NewRepositories(db) + v := newBackupVaultik(ctx, cfg, store, repos, db, fs) + + require.NoError(t, backUp(v, "first")) + + // Upper-casing ASCII text keeps its size. + files[rewrittenPath] = bytes.ToUpper(files[rewrittenPath]) + require.NoError(t, afero.WriteFile(fs, rewrittenPath, files[rewrittenPath], 0o644)) + require.NoError(t, fs.Chtimes(rewrittenPath, secondMTime, secondMTime)) + + require.NoError(t, backUp(v, "second")) + + id := localSnapshotID(ctx, t, repos, "second") + require.NoError(t, db.Close()) + + reader := newReaderVaultik(ctx, cfg, store, nil, fs) + require.NoError(t, reader.Restore(&vaultik.RestoreOptions{ + SnapshotID: id, + TargetDir: restoreDir, + Verify: true, + })) + + assertRestoredTree(t, fs, restoreDir, files) +}