diff --git a/TODO.md b/TODO.md index ec17c26..684e325 100644 --- a/TODO.md +++ b/TODO.md @@ -22,6 +22,16 @@ the tag exists and is exercised; what is left is merging `next` to # Completed Steps +- 2026-10-07: Made a backup notice a file rewritten with its size + unchanged and a new mtime in the same second as the one in the index + ([issue #226](https://git.eeqj.de/sneak/vaultik/issues/226)). The + `files` table held mtime in whole seconds and the scanner compared + whole seconds, so every later snapshot kept the old content. `mtime` + now holds nanoseconds since the Unix epoch and is compared at that + precision. A local index written before the change holds seconds + there, so its next backup re-chunks every file once; chunks it already + stored are not uploaded again. + - 2026-10-06: Made the backup summary and the `snapshots` row count each file, byte and upload once ([issue #225](https://git.eeqj.de/sneak/vaultik/issues/225)). The diff --git a/docs/DATAMODEL.md b/docs/DATAMODEL.md index a1967a3..5851247 100644 --- a/docs/DATAMODEL.md +++ b/docs/DATAMODEL.md @@ -36,7 +36,7 @@ Stores metadata about files in the filesystem being backed up. **Columns:** - `id` (TEXT PRIMARY KEY) - UUID for the file record - `path` (TEXT NOT NULL UNIQUE) - Absolute file path -- `mtime` (INTEGER NOT NULL) - Modification time as Unix timestamp +- `mtime` (INTEGER NOT NULL) - Modification time in nanoseconds since the Unix epoch - `size` (INTEGER NOT NULL) - File size in bytes - `mode` (INTEGER NOT NULL) - Unix file permissions and type - `uid` (INTEGER NOT NULL) - User ID of file owner diff --git a/internal/database/files.go b/internal/database/files.go index 02bd26c..c7b3044 100644 --- a/internal/database/files.go +++ b/internal/database/files.go @@ -54,16 +54,16 @@ func (r *FileRepository) Create(ctx context.Context, tx *sql.Tx, file *File) err if tx != nil { LogSQL("Execute", query, file.ID.String(), file.Path.String(), file.SourcePath.String(), - file.MTime.Unix(), file.Size, file.Mode, file.UID, file.GID, + file.MTime.UnixNano(), file.Size, file.Mode, file.UID, file.GID, file.LinkTarget.String()) err = tx.QueryRowContext(ctx, query, file.ID.String(), file.Path.String(), file.SourcePath.String(), - file.MTime.Unix(), file.Size, file.Mode, file.UID, file.GID, + file.MTime.UnixNano(), file.Size, file.Mode, file.UID, file.GID, file.LinkTarget.String()).Scan(&idStr) } else { err = r.db.QueryRowWithLog(ctx, query, file.ID.String(), file.Path.String(), file.SourcePath.String(), - file.MTime.Unix(), file.Size, file.Mode, file.UID, file.GID, + file.MTime.UnixNano(), file.Size, file.Mode, file.UID, file.GID, file.LinkTarget.String()).Scan(&idStr) } @@ -164,7 +164,7 @@ func (r *FileRepository) ListModifiedSince( ORDER BY path ` - rows, err := r.db.conn.QueryContext(ctx, query, since.Unix()) + rows, err := r.db.conn.QueryContext(ctx, query, since.UnixNano()) if err != nil { return nil, fmt.Errorf("querying files: %w", err) } @@ -392,7 +392,7 @@ func (r *FileRepository) CreateBatch( args = append(args, f.ID.String(), f.Path.String(), f.SourcePath.String(), - f.MTime.Unix(), f.Size, f.Mode, f.UID, f.GID, + f.MTime.UnixNano(), f.Size, f.Mode, f.UID, f.GID, f.LinkTarget.String()) } @@ -460,7 +460,7 @@ func (r *FileRepository) scanFileFrom(row fileRowScanner) (*File, error) { var ( file File idStr, pathStr, sourcePathStr string - mtimeUnix int64 + mtimeUnixNano int64 linkTarget sql.NullString ) @@ -468,7 +468,7 @@ func (r *FileRepository) scanFileFrom(row fileRowScanner) (*File, error) { &idStr, &pathStr, &sourcePathStr, - &mtimeUnix, + &mtimeUnixNano, &file.Size, &file.Mode, &file.UID, @@ -487,7 +487,7 @@ func (r *FileRepository) scanFileFrom(row fileRowScanner) (*File, error) { file.Path = types.FilePath(pathStr) file.SourcePath = types.SourcePath(sourcePathStr) - file.MTime = time.Unix(mtimeUnix, 0).UTC() + file.MTime = time.Unix(0, mtimeUnixNano).UTC() if linkTarget.Valid { file.LinkTarget = types.FilePath(linkTarget.String) } diff --git a/internal/database/repository_edge_cases_test.go b/internal/database/repository_edge_cases_test.go index a4cf9f7..59f85ca 100644 --- a/internal/database/repository_edge_cases_test.go +++ b/internal/database/repository_edge_cases_test.go @@ -606,8 +606,7 @@ func TestTimezoneHandling(t *testing.T) { t.Skip("timezone not available") } - // Use Truncate to remove sub-second precision since we store as Unix timestamps - nyTime := time.Now().In(loc).Truncate(time.Second) + nyTime := time.Now().In(loc) file := &File{ Path: "/timezone-test.txt", MTime: nyTime, diff --git a/internal/database/schema/001.sql b/internal/database/schema/001.sql index 944c3a6..d99dc50 100644 --- a/internal/database/schema/001.sql +++ b/internal/database/schema/001.sql @@ -6,7 +6,7 @@ CREATE TABLE IF NOT EXISTS files ( id TEXT PRIMARY KEY, -- UUID path TEXT NOT NULL UNIQUE, source_path TEXT NOT NULL DEFAULT '', -- The source directory this file came from (for restore path stripping) - mtime INTEGER NOT NULL, + mtime INTEGER NOT NULL, -- nanoseconds since the Unix epoch size INTEGER NOT NULL, mode INTEGER NOT NULL, uid INTEGER NOT NULL, diff --git a/internal/snapshot/scanner.go b/internal/snapshot/scanner.go index 6676f6e..52daa85 100644 --- a/internal/snapshot/scanner.go +++ b/internal/snapshot/scanner.go @@ -1221,7 +1221,7 @@ func (s *Scanner) checkFileInMemory( // Check if file has changed if existingFile.Size != file.Size || - existingFile.MTime.Unix() != file.MTime.Unix() || + existingFile.MTime.UnixNano() != file.MTime.UnixNano() || existingFile.Mode != file.Mode || existingFile.UID != file.UID || existingFile.GID != file.GID { diff --git a/internal/vaultik/same_second_rewrite_test.go b/internal/vaultik/same_second_rewrite_test.go new file mode 100644 index 0000000..71ca958 --- /dev/null +++ b/internal/vaultik/same_second_rewrite_test.go @@ -0,0 +1,73 @@ +package vaultik_test + +import ( + "bytes" + "context" + "path/filepath" + "testing" + "time" + + "github.com/spf13/afero" + "github.com/stretchr/testify/require" + "sneak.berlin/go/vaultik/internal/database" + "sneak.berlin/go/vaultik/internal/log" + "sneak.berlin/go/vaultik/internal/storage" + "sneak.berlin/go/vaultik/internal/vaultik" +) + +// A file rewritten with its size unchanged and a new mtime in the same +// second as the mtime the index holds must still be backed up. See +// https://git.eeqj.de/sneak/vaultik/issues/226. +// +//nolint:paralleltest // installs the global logger via log.Initialize +func TestBackupOfSameSecondRewriteRestoresNewContent(t *testing.T) { + log.Initialize(log.Config{}) + + fs := afero.NewOsFs() + tempDir := t.TempDir() + dataDir := filepath.Join(tempDir, "src") + storeDir := filepath.Join(tempDir, "remote") + restoreDir := filepath.Join(tempDir, "restored") + dbPath := filepath.Join(tempDir, "index.sqlite") + rewrittenPath := filepath.Join(dataDir, "small.txt") + + ctx := context.Background() + files := writeFaultSourceTree(t, fs, dataDir) + cfg := changedFileConfig(dataDir, dbPath) + + firstMTime := time.Date(2026, time.January, 2, 3, 4, 5, 0, time.UTC). + Add(100 * time.Millisecond) + secondMTime := firstMTime.Add(800 * time.Millisecond) + + require.NoError(t, fs.Chtimes(rewrittenPath, firstMTime, firstMTime)) + + store, err := storage.NewFileStorer(storeDir) + require.NoError(t, err) + + db, err := database.New(ctx, dbPath) + require.NoError(t, err) + + repos := database.NewRepositories(db) + v := newBackupVaultik(ctx, cfg, store, repos, db, fs) + + require.NoError(t, backUp(v, "first")) + + // Upper-casing ASCII text keeps its size. + files[rewrittenPath] = bytes.ToUpper(files[rewrittenPath]) + require.NoError(t, afero.WriteFile(fs, rewrittenPath, files[rewrittenPath], 0o644)) + require.NoError(t, fs.Chtimes(rewrittenPath, secondMTime, secondMTime)) + + require.NoError(t, backUp(v, "second")) + + id := localSnapshotID(ctx, t, repos, "second") + require.NoError(t, db.Close()) + + reader := newReaderVaultik(ctx, cfg, store, nil, fs) + require.NoError(t, reader.Restore(&vaultik.RestoreOptions{ + SnapshotID: id, + TargetDir: restoreDir, + Verify: true, + })) + + assertRestoredTree(t, fs, restoreDir, files) +}