Compare commits

..
1 Commits
Author SHA1 Message Date
sneak c113e120d6 Leave remote info orphan figures unknown when a manifest is unreadable (closes #228)
check / check (push) Canceled after 0s
When a manifest could not be read, remote info skipped it, counted that
snapshot's blobs as orphaned and advised running prune. The orphan
figures are now unknown in that case: the report says how many
manifests could not be read and gives no prune advice, and --json gives
orphaned_blob_count and orphaned_blob_size as null and lists the remote
keys in unreadable_manifests.

Directory names under metadata/ were used unchecked and printed raw, so
control characters in one reached the terminal. A name that is not a
remote key (64 lowercase hex characters) is now skipped with a warning,
as the snapshot listing already does.

The closing log line of remote info now carries the unreadable manifest
count in place of the orphan count.

Model: opus-5-5
2026-10-07 04:36:04 +00:00
11 changed files with 52 additions and 298 deletions
+3 -6
View File
@@ -390,12 +390,9 @@ recipients, and local database statistics.
**`remote info`**: Show storage backend type and location plus detailed **`remote info`**: Show storage backend type and location plus detailed
remote storage inventory: per-snapshot metadata sizes, blob counts, and remote storage inventory: per-snapshot metadata sizes, blob counts, and
orphaned blob detection. A name under `metadata/` that is not a remote orphaned blob detection. If a manifest cannot be read, the orphaned blob
key is skipped with a warning and is not printed. If a manifest cannot be figures are reported as unknown; `--json` gives them as `null` and lists
read, or a name was skipped, the orphaned blob figures are reported as the remote key of each unreadable manifest in `unreadable_manifests`.
unknown; `--json` gives them as `null`, lists the remote key of each
unreadable manifest in `unreadable_manifests` and counts the skipped
names in `skipped_metadata_names`.
* `--json`: Output as JSON * `--json`: Output as JSON
**`remote nuke`**: Delete every snapshot's metadata and every blob from the **`remote nuke`**: Delete every snapshot's metadata and every blob from the
+1 -13
View File
@@ -32,19 +32,7 @@ the tag exists and is exercised; what is left is merging `next` to
advice, and `--json` gives them as `null` with the unreadable remote advice, and `--json` gives them as `null` with the unreadable remote
keys in `unreadable_manifests`. A name under `metadata/` that is not keys in `unreadable_manifests`. A name under `metadata/` that is not
64 lowercase hex characters is now skipped with a warning instead of 64 lowercase hex characters is now skipped with a warning instead of
being printed, control characters included. Its manifest is then not being printed, control characters included.
read either, so a skipped name also leaves the orphan figures unknown,
and `--json` counts such names in `skipped_metadata_names`.
- 2026-10-07: Made a backup notice a file rewritten with its size
unchanged and a new mtime in the same second as the one in the index
([issue #226](https://git.eeqj.de/sneak/vaultik/issues/226)). The
`files` table held mtime in whole seconds and the scanner compared
whole seconds, so every later snapshot kept the old content. A new
`mtime_nsec` column holds the nanoseconds within the second that
`mtime` holds, and the scanner compares the full mtime. A local index
created before the change lacks the column and is rebuilt with
`vaultik database delete` and a full backup.
- 2026-10-07: Made taking the process-wide lock atomic - 2026-10-07: Made taking the process-wide lock atomic
([issue #227](https://git.eeqj.de/sneak/vaultik/issues/227)). The lock ([issue #227](https://git.eeqj.de/sneak/vaultik/issues/227)). The lock
+1 -2
View File
@@ -36,8 +36,7 @@ Stores metadata about files in the filesystem being backed up.
**Columns:** **Columns:**
- `id` (TEXT PRIMARY KEY) - UUID for the file record - `id` (TEXT PRIMARY KEY) - UUID for the file record
- `path` (TEXT NOT NULL UNIQUE) - Absolute file path - `path` (TEXT NOT NULL UNIQUE) - Absolute file path
- `mtime` (INTEGER NOT NULL) - Modification time, whole seconds since the Unix epoch - `mtime` (INTEGER NOT NULL) - Modification time as Unix timestamp
- `mtime_nsec` (INTEGER NOT NULL) - Nanoseconds within that second, 0 to 999999999
- `size` (INTEGER NOT NULL) - File size in bytes - `size` (INTEGER NOT NULL) - File size in bytes
- `mode` (INTEGER NOT NULL) - Unix file permissions and type - `mode` (INTEGER NOT NULL) - Unix file permissions and type
- `uid` (INTEGER NOT NULL) - User ID of file owner - `uid` (INTEGER NOT NULL) - User ID of file owner
+19 -28
View File
@@ -33,13 +33,11 @@ func (r *FileRepository) Create(ctx context.Context, tx *sql.Tx, file *File) err
} }
query := ` query := `
INSERT INTO files INSERT INTO files (id, path, source_path, mtime, size, mode, uid, gid, link_target)
(id, path, source_path, mtime, mtime_nsec, size, mode, uid, gid, link_target) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
ON CONFLICT(path) DO UPDATE SET ON CONFLICT(path) DO UPDATE SET
source_path = excluded.source_path, source_path = excluded.source_path,
mtime = excluded.mtime, mtime = excluded.mtime,
mtime_nsec = excluded.mtime_nsec,
size = excluded.size, size = excluded.size,
mode = excluded.mode, mode = excluded.mode,
uid = excluded.uid, uid = excluded.uid,
@@ -56,19 +54,16 @@ func (r *FileRepository) Create(ctx context.Context, tx *sql.Tx, file *File) err
if tx != nil { if tx != nil {
LogSQL("Execute", query, LogSQL("Execute", query,
file.ID.String(), file.Path.String(), file.SourcePath.String(), file.ID.String(), file.Path.String(), file.SourcePath.String(),
file.MTime.Unix(), file.MTime.Nanosecond(), file.MTime.Unix(), file.Size, file.Mode, file.UID, file.GID,
file.Size, file.Mode, file.UID, file.GID,
file.LinkTarget.String()) file.LinkTarget.String())
err = tx.QueryRowContext(ctx, query, err = tx.QueryRowContext(ctx, query,
file.ID.String(), file.Path.String(), file.SourcePath.String(), file.ID.String(), file.Path.String(), file.SourcePath.String(),
file.MTime.Unix(), file.MTime.Nanosecond(), file.MTime.Unix(), file.Size, file.Mode, file.UID, file.GID,
file.Size, file.Mode, file.UID, file.GID,
file.LinkTarget.String()).Scan(&idStr) file.LinkTarget.String()).Scan(&idStr)
} else { } else {
err = r.db.QueryRowWithLog(ctx, query, err = r.db.QueryRowWithLog(ctx, query,
file.ID.String(), file.Path.String(), file.SourcePath.String(), file.ID.String(), file.Path.String(), file.SourcePath.String(),
file.MTime.Unix(), file.MTime.Nanosecond(), file.MTime.Unix(), file.Size, file.Mode, file.UID, file.GID,
file.Size, file.Mode, file.UID, file.GID,
file.LinkTarget.String()).Scan(&idStr) file.LinkTarget.String()).Scan(&idStr)
} }
@@ -89,7 +84,7 @@ func (r *FileRepository) Create(ctx context.Context, tx *sql.Tx, file *File) err
// in the index. // in the index.
func (r *FileRepository) GetByPath(ctx context.Context, path string) (*File, error) { func (r *FileRepository) GetByPath(ctx context.Context, path string) (*File, error) {
query := ` query := `
SELECT id, path, source_path, mtime, mtime_nsec, size, mode, uid, gid, link_target SELECT id, path, source_path, mtime, size, mode, uid, gid, link_target
FROM files FROM files
WHERE path = ? WHERE path = ?
` `
@@ -109,7 +104,7 @@ func (r *FileRepository) GetByPath(ctx context.Context, path string) (*File, err
// GetByID retrieves a file by its UUID // GetByID retrieves a file by its UUID
func (r *FileRepository) GetByID(ctx context.Context, id types.FileID) (*File, error) { func (r *FileRepository) GetByID(ctx context.Context, id types.FileID) (*File, error) {
query := ` query := `
SELECT id, path, source_path, mtime, mtime_nsec, size, mode, uid, gid, link_target SELECT id, path, source_path, mtime, size, mode, uid, gid, link_target
FROM files FROM files
WHERE id = ? WHERE id = ?
` `
@@ -132,7 +127,7 @@ func (r *FileRepository) GetByPathTx(
ctx context.Context, tx *sql.Tx, path string, ctx context.Context, tx *sql.Tx, path string,
) (*File, error) { ) (*File, error) {
query := ` query := `
SELECT id, path, source_path, mtime, mtime_nsec, size, mode, uid, gid, link_target SELECT id, path, source_path, mtime, size, mode, uid, gid, link_target
FROM files FROM files
WHERE path = ? WHERE path = ?
` `
@@ -163,14 +158,13 @@ func (r *FileRepository) ListModifiedSince(
ctx context.Context, since time.Time, ctx context.Context, since time.Time,
) ([]*File, error) { ) ([]*File, error) {
query := ` query := `
SELECT id, path, source_path, mtime, mtime_nsec, size, mode, uid, gid, link_target SELECT id, path, source_path, mtime, size, mode, uid, gid, link_target
FROM files FROM files
WHERE (mtime, mtime_nsec) >= (?, ?) WHERE mtime >= ?
ORDER BY path ORDER BY path
` `
rows, err := r.db.conn.QueryContext(ctx, query, rows, err := r.db.conn.QueryContext(ctx, query, since.Unix())
since.Unix(), since.Nanosecond())
if err != nil { if err != nil {
return nil, fmt.Errorf("querying files: %w", err) return nil, fmt.Errorf("querying files: %w", err)
} }
@@ -245,7 +239,7 @@ func (r *FileRepository) ListUnderPath(
// LIKE would ignore ASCII case and treat _ and % in path as wildcards. // LIKE would ignore ASCII case and treat _ and % in path as wildcards.
query := ` query := `
SELECT id, path, source_path, mtime, mtime_nsec, size, mode, uid, gid, link_target SELECT id, path, source_path, mtime, size, mode, uid, gid, link_target
FROM files FROM files
WHERE path = ? OR substr(path, 1, length(?)) = ? WHERE path = ? OR substr(path, 1, length(?)) = ?
ORDER BY path ORDER BY path
@@ -330,7 +324,7 @@ func (r *FileRepository) ListIDsWithChunksNotInUploadedBlobs(
// ListAll returns all files in the database // ListAll returns all files in the database
func (r *FileRepository) ListAll(ctx context.Context) ([]*File, error) { func (r *FileRepository) ListAll(ctx context.Context) ([]*File, error) {
query := ` query := `
SELECT id, path, source_path, mtime, mtime_nsec, size, mode, uid, gid, link_target SELECT id, path, source_path, mtime, size, mode, uid, gid, link_target
FROM files FROM files
ORDER BY path ORDER BY path
` `
@@ -371,7 +365,7 @@ func (r *FileRepository) CreateBatch(
} }
// Each files row binds this many SQL variables. // Each files row binds this many SQL variables.
const fileCols = 10 const fileCols = 9
// Batch at 100 rows to be safe with SQLite's variable limit. // Batch at 100 rows to be safe with SQLite's variable limit.
const batchSize = 100 const batchSize = 100
@@ -382,7 +376,7 @@ func (r *FileRepository) CreateBatch(
batch := files[i:end] batch := files[i:end]
query := `INSERT INTO files query := `INSERT INTO files
(id, path, source_path, mtime, mtime_nsec, size, mode, uid, gid, link_target) (id, path, source_path, mtime, size, mode, uid, gid, link_target)
VALUES ` VALUES `
args := make([]any, 0, len(batch)*fileCols) args := make([]any, 0, len(batch)*fileCols)
@@ -394,12 +388,11 @@ func (r *FileRepository) CreateBatch(
querySb325.WriteString(", ") querySb325.WriteString(", ")
} }
querySb325.WriteString("(?, ?, ?, ?, ?, ?, ?, ?, ?, ?)") querySb325.WriteString("(?, ?, ?, ?, ?, ?, ?, ?, ?)")
args = append(args, args = append(args,
f.ID.String(), f.Path.String(), f.SourcePath.String(), f.ID.String(), f.Path.String(), f.SourcePath.String(),
f.MTime.Unix(), f.MTime.Nanosecond(), f.MTime.Unix(), f.Size, f.Mode, f.UID, f.GID,
f.Size, f.Mode, f.UID, f.GID,
f.LinkTarget.String()) f.LinkTarget.String())
} }
@@ -408,7 +401,6 @@ func (r *FileRepository) CreateBatch(
query += ` ON CONFLICT(path) DO UPDATE SET query += ` ON CONFLICT(path) DO UPDATE SET
source_path = excluded.source_path, source_path = excluded.source_path,
mtime = excluded.mtime, mtime = excluded.mtime,
mtime_nsec = excluded.mtime_nsec,
size = excluded.size, size = excluded.size,
mode = excluded.mode, mode = excluded.mode,
uid = excluded.uid, uid = excluded.uid,
@@ -468,7 +460,7 @@ func (r *FileRepository) scanFileFrom(row fileRowScanner) (*File, error) {
var ( var (
file File file File
idStr, pathStr, sourcePathStr string idStr, pathStr, sourcePathStr string
mtimeUnix, mtimeNsec int64 mtimeUnix int64
linkTarget sql.NullString linkTarget sql.NullString
) )
@@ -477,7 +469,6 @@ func (r *FileRepository) scanFileFrom(row fileRowScanner) (*File, error) {
&pathStr, &pathStr,
&sourcePathStr, &sourcePathStr,
&mtimeUnix, &mtimeUnix,
&mtimeNsec,
&file.Size, &file.Size,
&file.Mode, &file.Mode,
&file.UID, &file.UID,
@@ -496,7 +487,7 @@ func (r *FileRepository) scanFileFrom(row fileRowScanner) (*File, error) {
file.Path = types.FilePath(pathStr) file.Path = types.FilePath(pathStr)
file.SourcePath = types.SourcePath(sourcePathStr) file.SourcePath = types.SourcePath(sourcePathStr)
file.MTime = time.Unix(mtimeUnix, mtimeNsec).UTC() file.MTime = time.Unix(mtimeUnix, 0).UTC()
if linkTarget.Valid { if linkTarget.Valid {
file.LinkTarget = types.FilePath(linkTarget.String) file.LinkTarget = types.FilePath(linkTarget.String)
} }
-109
View File
@@ -252,115 +252,6 @@ func TestFileRepositorySymlink(t *testing.T) {
} }
} }
// An mtime after 2262 or before 1678 does not fit in int64 nanoseconds
// since the epoch, and must still come back from the database unchanged.
func TestFileRepositoryMTimeOutsideInt64NanosecondRange(t *testing.T) {
t.Parallel()
db, cleanup := setupTestDB(t)
defer cleanup()
ctx := context.Background()
repo := database.NewFileRepository(db)
mtimes := []time.Time{
time.Date(2300, time.January, 1, 0, 0, 0, 123456789, time.UTC),
time.Date(1601, time.January, 1, 0, 0, 0, 987654321, time.UTC),
}
for _, mtime := range mtimes {
created := &database.File{
Path: types.FilePath("/created-" + mtime.Format(time.RFC3339Nano)),
MTime: mtime,
}
err := repo.Create(ctx, nil, created)
if err != nil {
t.Fatalf("failed to create file: %v", err)
}
batched := &database.File{
ID: types.NewFileID(),
Path: types.FilePath("/batched-" + mtime.Format(time.RFC3339Nano)),
MTime: mtime,
}
err = repo.CreateBatch(ctx, nil, []*database.File{batched})
if err != nil {
t.Fatalf("failed to batch create file: %v", err)
}
for _, path := range []types.FilePath{created.Path, batched.Path} {
retrieved, err := repo.GetByPath(ctx, path.String())
if err != nil {
t.Fatalf("failed to get file: %v", err)
}
if !retrieved.MTime.Equal(mtime) {
t.Errorf("%s: mtime got %v, want %v",
path, retrieved.MTime, mtime)
}
}
}
}
// A file already in the index and rewritten within the same second must get
// its new nanoseconds stored, through both Create and CreateBatch.
func TestFileRepositoryUpsertMTimeInSameSecond(t *testing.T) {
t.Parallel()
db, cleanup := setupTestDB(t)
defer cleanup()
ctx := context.Background()
repo := database.NewFileRepository(db)
indexed := time.Date(2026, time.October, 7, 12, 0, 0, 100000000, time.UTC)
rewritten := indexed.Add(800 * time.Millisecond)
tests := []struct {
name string
upsert func(file *database.File) error
}{
{"Create", func(file *database.File) error {
return repo.Create(ctx, nil, file)
}},
{"CreateBatch", func(file *database.File) error {
return repo.CreateBatch(ctx, nil, []*database.File{file})
}},
}
for _, tt := range tests {
file := &database.File{
ID: types.NewFileID(),
Path: types.FilePath("/" + tt.name),
MTime: indexed,
}
err := tt.upsert(file)
if err != nil {
t.Fatalf("%s: failed to create file: %v", tt.name, err)
}
file.MTime = rewritten
err = tt.upsert(file)
if err != nil {
t.Fatalf("%s: failed to update file: %v", tt.name, err)
}
retrieved, err := repo.GetByPath(ctx, file.Path.String())
if err != nil {
t.Fatalf("%s: failed to get file: %v", tt.name, err)
}
if !retrieved.MTime.Equal(rewritten) {
t.Errorf("%s: mtime got %v, want %v",
tt.name, retrieved.MTime, rewritten)
}
}
}
func TestFileRepositoryTransaction(t *testing.T) { func TestFileRepositoryTransaction(t *testing.T) {
t.Parallel() t.Parallel()
@@ -606,7 +606,8 @@ func TestTimezoneHandling(t *testing.T) {
t.Skip("timezone not available") t.Skip("timezone not available")
} }
nyTime := time.Now().In(loc) // Use Truncate to remove sub-second precision since we store as Unix timestamps
nyTime := time.Now().In(loc).Truncate(time.Second)
file := &File{ file := &File{
Path: "/timezone-test.txt", Path: "/timezone-test.txt",
MTime: nyTime, MTime: nyTime,
+1 -2
View File
@@ -6,8 +6,7 @@ CREATE TABLE IF NOT EXISTS files (
id TEXT PRIMARY KEY, -- UUID id TEXT PRIMARY KEY, -- UUID
path TEXT NOT NULL UNIQUE, path TEXT NOT NULL UNIQUE,
source_path TEXT NOT NULL DEFAULT '', -- The source directory this file came from (for restore path stripping) source_path TEXT NOT NULL DEFAULT '', -- The source directory this file came from (for restore path stripping)
mtime INTEGER NOT NULL, -- whole seconds since the Unix epoch mtime INTEGER NOT NULL,
mtime_nsec INTEGER NOT NULL, -- nanoseconds within that second, 0 to 999999999
size INTEGER NOT NULL, size INTEGER NOT NULL,
mode INTEGER NOT NULL, mode INTEGER NOT NULL,
uid INTEGER NOT NULL, uid INTEGER NOT NULL,
+1 -1
View File
@@ -1221,7 +1221,7 @@ func (s *Scanner) checkFileInMemory(
// Check if file has changed // Check if file has changed
if existingFile.Size != file.Size || if existingFile.Size != file.Size ||
!existingFile.MTime.Equal(file.MTime) || existingFile.MTime.Unix() != file.MTime.Unix() ||
existingFile.Mode != file.Mode || existingFile.Mode != file.Mode ||
existingFile.UID != file.UID || existingFile.UID != file.UID ||
existingFile.GID != file.GID { existingFile.GID != file.GID {
+15 -28
View File
@@ -207,18 +207,13 @@ type RemoteInfoResult struct {
ReferencedBlobSize int64 `json:"referenced_blob_size"` ReferencedBlobSize int64 `json:"referenced_blob_size"`
// Orphaned blobs. Both stay nil (null in the JSON) when a manifest // Orphaned blobs. Both stay nil (null in the JSON) when a manifest
// could not be read or a name under metadata/ was skipped, since // could not be read, since that snapshot's blobs would be counted
// that snapshot's blobs would be counted as orphaned. // as orphaned.
OrphanedBlobCount *int `json:"orphaned_blob_count"` OrphanedBlobCount *int `json:"orphaned_blob_count"`
OrphanedBlobSize *int64 `json:"orphaned_blob_size"` OrphanedBlobSize *int64 `json:"orphaned_blob_size"`
// Remote key of each snapshot whose manifest could not be read // Remote key of each snapshot whose manifest could not be read
UnreadableManifests []string `json:"unreadable_manifests,omitempty"` UnreadableManifests []string `json:"unreadable_manifests,omitempty"`
// Number of names under metadata/ skipped because they are not
// remote keys. The names themselves are not reported: they come
// from the destination store and may hold control characters.
SkippedMetadataNames int `json:"skipped_metadata_names,omitempty"`
} }
// RemoteInfo displays information about remote storage // RemoteInfo displays information about remote storage
@@ -244,13 +239,11 @@ func (v *Vaultik) RemoteInfo(jsonOutput bool) error {
v.stdoutf("Scanning snapshot metadata...\n") v.stdoutf("Scanning snapshot metadata...\n")
} }
snapshotMetadata, snapshotIDs, skippedNames, err := v.collectSnapshotMetadata() snapshotMetadata, snapshotIDs, err := v.collectSnapshotMetadata()
if err != nil { if err != nil {
return err return err
} }
result.SkippedMetadataNames = skippedNames
if showText { if showText {
v.stdoutf("Downloading %d manifest(s)...\n", len(snapshotIDs)) v.stdoutf("Downloading %d manifest(s)...\n", len(snapshotIDs))
} }
@@ -287,18 +280,16 @@ func (v *Vaultik) RemoteInfo(jsonOutput bool) error {
} }
// collectSnapshotMetadata scans remote metadata and returns // collectSnapshotMetadata scans remote metadata and returns
// per-snapshot info, sorted IDs and the number of names it skipped // per-snapshot info and sorted IDs.
// because they are not remote keys.
func (v *Vaultik) collectSnapshotMetadata() ( func (v *Vaultik) collectSnapshotMetadata() (
map[string]*SnapshotMetadataInfo, []string, int, error, map[string]*SnapshotMetadataInfo, []string, error,
) { ) {
snapshotMetadata := make(map[string]*SnapshotMetadataInfo) snapshotMetadata := make(map[string]*SnapshotMetadataInfo)
skippedNames := make(map[string]bool)
metadataCh := v.Storage.ListStream(v.ctx, "metadata/") metadataCh := v.Storage.ListStream(v.ctx, "metadata/")
for obj := range metadataCh { for obj := range metadataCh {
if obj.Err != nil { if obj.Err != nil {
return nil, nil, 0, fmt.Errorf("listing metadata: %w", obj.Err) return nil, nil, fmt.Errorf("listing metadata: %w", obj.Err)
} }
parts := strings.Split(obj.Key, "/") parts := strings.Split(obj.Key, "/")
@@ -315,8 +306,6 @@ func (v *Vaultik) collectSnapshotMetadata() (
log.Warn("Skipping non-conforming key under metadata/", log.Warn("Skipping non-conforming key under metadata/",
"key", obj.Key) "key", obj.Key)
skippedNames[snapshotID] = true
continue continue
} }
@@ -343,7 +332,7 @@ func (v *Vaultik) collectSnapshotMetadata() (
sort.Strings(snapshotIDs) sort.Strings(snapshotIDs)
return snapshotMetadata, snapshotIDs, len(skippedNames), nil return snapshotMetadata, snapshotIDs, nil
} }
// collectReferencedBlobsFromManifests downloads manifests and returns // collectReferencedBlobsFromManifests downloads manifests and returns
@@ -411,9 +400,9 @@ func (v *Vaultik) populateRemoteInfoResult(
} }
// scanRemoteBlobStorage lists all blobs on remote and computes orphan // scanRemoteBlobStorage lists all blobs on remote and computes orphan
// stats when every manifest was read and no name under metadata/ was // stats when every manifest was read. showText is true only when the
// skipped. showText is true only when the human report is being // human report is being printed (not --json, not --quiet), gating the
// printed (not --json, not --quiet), gating the progress line. // progress line.
func (v *Vaultik) scanRemoteBlobStorage( func (v *Vaultik) scanRemoteBlobStorage(
result *RemoteInfoResult, referencedBlobs map[string]int64, showText bool, result *RemoteInfoResult, referencedBlobs map[string]int64, showText bool,
) error { ) error {
@@ -440,10 +429,9 @@ func (v *Vaultik) scanRemoteBlobStorage(
result.TotalBlobSize += obj.Size result.TotalBlobSize += obj.Size
} }
// A blob named only by a manifest that could not be read, or by one // A blob named only by a manifest that could not be read would be
// under a skipped name, would be counted as orphaned, so the orphan // counted as orphaned, so the orphan figures stay unknown.
// figures stay unknown. if len(result.UnreadableManifests) > 0 {
if len(result.UnreadableManifests) > 0 || result.SkippedMetadataNames > 0 {
return nil return nil
} }
@@ -517,9 +505,8 @@ func (v *Vaultik) printRemoteInfoTable(result *RemoteInfoResult) {
if result.OrphanedBlobCount == nil { if result.OrphanedBlobCount == nil {
v.stdoutf("Orphaned (unreferenced): unknown "+ v.stdoutf("Orphaned (unreferenced): unknown "+
"(%d manifest(s) could not be read, "+ "(%d manifest(s) could not be read)\n",
"%d name(s) under metadata/ skipped)\n", len(result.UnreadableManifests))
len(result.UnreadableManifests), result.SkippedMetadataNames)
return return
} }
+9 -35
View File
@@ -13,11 +13,6 @@ import (
"sneak.berlin/go/vaultik/internal/snapshot" "sneak.berlin/go/vaultik/internal/snapshot"
) )
// testBlobHashB is a blob that the manifest written by addRemote does
// not reference.
const testBlobHashB = "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" +
"bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"
// TestRemoteInfo_UnreadableManifestLeavesOrphansUnknown checks that a // TestRemoteInfo_UnreadableManifestLeavesOrphansUnknown checks that a
// manifest remote info cannot read makes the orphan figures unknown. A // manifest remote info cannot read makes the orphan figures unknown. A
// blob referenced only by that snapshot would otherwise be counted as // blob referenced only by that snapshot would otherwise be counted as
@@ -26,6 +21,9 @@ func TestRemoteInfo_UnreadableManifestLeavesOrphansUnknown(t *testing.T) {
log.Initialize(log.Config{}) log.Initialize(log.Config{})
t.Parallel() t.Parallel()
const testBlobHashB = "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" +
"bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"
env := newListEnv(t) env := newListEnv(t)
// The readable manifest references blob A only. // The readable manifest references blob A only.
@@ -52,8 +50,8 @@ func TestRemoteInfo_UnreadableManifestLeavesOrphansUnknown(t *testing.T) {
require.NoError(t, env.v.RemoteInfo(false)) require.NoError(t, env.v.RemoteInfo(false))
text := env.stdout.String() text := env.stdout.String()
assert.Contains(t, text, "Orphaned (unreferenced): unknown "+ assert.Contains(t, text,
"(1 manifest(s) could not be read, 0 name(s) under metadata/ skipped)") "Orphaned (unreferenced): unknown (1 manifest(s) could not be read)")
assert.NotContains(t, text, "vaultik prune") assert.NotContains(t, text, "vaultik prune")
env.stdout.Reset() env.stdout.Reset()
@@ -70,45 +68,21 @@ func TestRemoteInfo_UnreadableManifestLeavesOrphansUnknown(t *testing.T) {
// TestRemoteInfo_SkipsNonConformingMetadataName checks that a directory // TestRemoteInfo_SkipsNonConformingMetadataName checks that a directory
// under metadata/ whose name is not a remote key is left out of the // under metadata/ whose name is not a remote key is left out of the
// report, and that the orphan figures are then unknown. The name comes // report. The name comes from the destination store; printed raw, its
// from the destination store; printed raw, its control characters would // control characters would reach the terminal.
// reach the terminal. Its manifest is not read, so a blob only it
// references would otherwise be counted as orphaned.
func TestRemoteInfo_SkipsNonConformingMetadataName(t *testing.T) { func TestRemoteInfo_SkipsNonConformingMetadataName(t *testing.T) {
log.Initialize(log.Config{}) log.Initialize(log.Config{})
t.Parallel() t.Parallel()
env := newListEnv(t) env := newListEnv(t)
env.addRemote(t, listRemoteID, time.Date(2026, 3, 2, 0, 0, 0, 0, time.UTC)) env.addRemote(t, listRemoteID, time.Date(2026, 3, 2, 0, 0, 0, 0, time.UTC))
addBlob(t, env.store.testStorer, testBlobHashA)
addBlob(t, env.store.testStorer, testBlobHashB)
require.NoError(t, env.store.Put(context.Background(), require.NoError(t, env.store.Put(context.Background(),
"metadata/\x1b[31mred/manifest.json.zst", "metadata/\x1b[31mred/manifest.json.zst",
bytes.NewReader([]byte("not a valid manifest")))) bytes.NewReader([]byte("not a valid manifest"))))
require.NoError(t, env.v.RemoteInfo(false)) require.NoError(t, env.v.RemoteInfo(false))
text := env.stdout.String()
assert.NotContains(t, text, "\x1b")
assert.NotContains(t, text, "31mred")
assert.Contains(t, text, "Total (1 snapshots)")
assert.Contains(t, text, "Orphaned (unreferenced): unknown "+
"(0 manifest(s) could not be read, 1 name(s) under metadata/ skipped)")
assert.NotContains(t, text, "vaultik prune")
env.stdout.Reset()
require.NoError(t, env.v.RemoteInfo(true))
out := env.stdout.String() out := env.stdout.String()
assert.NotContains(t, out, "31mred") assert.NotContains(t, out, "\x1b")
assert.Contains(t, out, "Total (1 snapshots)")
var doc map[string]any
require.NoError(t, json.Unmarshal([]byte(out), &doc))
assert.Contains(t, doc, "orphaned_blob_count")
assert.Nil(t, doc["orphaned_blob_count"])
assert.Contains(t, doc, "orphaned_blob_size")
assert.Nil(t, doc["orphaned_blob_size"])
assert.InDelta(t, 1, doc["skipped_metadata_names"], 0)
assert.NotContains(t, doc, "unreadable_manifests")
} }
@@ -1,73 +0,0 @@
package vaultik_test
import (
"bytes"
"context"
"path/filepath"
"testing"
"time"
"github.com/spf13/afero"
"github.com/stretchr/testify/require"
"sneak.berlin/go/vaultik/internal/database"
"sneak.berlin/go/vaultik/internal/log"
"sneak.berlin/go/vaultik/internal/storage"
"sneak.berlin/go/vaultik/internal/vaultik"
)
// A file rewritten with its size unchanged and a new mtime in the same
// second as the mtime the index holds must still be backed up. See
// https://git.eeqj.de/sneak/vaultik/issues/226.
//
//nolint:paralleltest // installs the global logger via log.Initialize
func TestBackupOfSameSecondRewriteRestoresNewContent(t *testing.T) {
log.Initialize(log.Config{})
fs := afero.NewOsFs()
tempDir := t.TempDir()
dataDir := filepath.Join(tempDir, "src")
storeDir := filepath.Join(tempDir, "remote")
restoreDir := filepath.Join(tempDir, "restored")
dbPath := filepath.Join(tempDir, "index.sqlite")
rewrittenPath := filepath.Join(dataDir, "small.txt")
ctx := context.Background()
files := writeFaultSourceTree(t, fs, dataDir)
cfg := changedFileConfig(dataDir, dbPath)
firstMTime := time.Date(2026, time.January, 2, 3, 4, 5, 0, time.UTC).
Add(100 * time.Millisecond)
secondMTime := firstMTime.Add(800 * time.Millisecond)
require.NoError(t, fs.Chtimes(rewrittenPath, firstMTime, firstMTime))
store, err := storage.NewFileStorer(storeDir)
require.NoError(t, err)
db, err := database.New(ctx, dbPath)
require.NoError(t, err)
repos := database.NewRepositories(db)
v := newBackupVaultik(ctx, cfg, store, repos, db, fs)
require.NoError(t, backUp(v, "first"))
// Upper-casing ASCII text keeps its size.
files[rewrittenPath] = bytes.ToUpper(files[rewrittenPath])
require.NoError(t, afero.WriteFile(fs, rewrittenPath, files[rewrittenPath], 0o644))
require.NoError(t, fs.Chtimes(rewrittenPath, secondMTime, secondMTime))
require.NoError(t, backUp(v, "second"))
id := localSnapshotID(ctx, t, repos, "second")
require.NoError(t, db.Close())
reader := newReaderVaultik(ctx, cfg, store, nil, fs)
require.NoError(t, reader.Restore(&vaultik.RestoreOptions{
SnapshotID: id,
TargetDir: restoreDir,
Verify: true,
}))
assertRestoredTree(t, fs, restoreDir, files)
}