Compare commits
2
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
167adf8eb1 | ||
|
|
b57ce2277d |
@@ -390,9 +390,12 @@ recipients, and local database statistics.
|
|||||||
|
|
||||||
**`remote info`**: Show storage backend type and location plus detailed
|
**`remote info`**: Show storage backend type and location plus detailed
|
||||||
remote storage inventory: per-snapshot metadata sizes, blob counts, and
|
remote storage inventory: per-snapshot metadata sizes, blob counts, and
|
||||||
orphaned blob detection. If a manifest cannot be read, the orphaned blob
|
orphaned blob detection. A name under `metadata/` that is not a remote
|
||||||
figures are reported as unknown; `--json` gives them as `null` and lists
|
key is skipped with a warning and is not printed. If a manifest cannot be
|
||||||
the remote key of each unreadable manifest in `unreadable_manifests`.
|
read, or a name was skipped, the orphaned blob figures are reported as
|
||||||
|
unknown; `--json` gives them as `null`, lists the remote key of each
|
||||||
|
unreadable manifest in `unreadable_manifests` and counts the skipped
|
||||||
|
names in `skipped_metadata_names`.
|
||||||
* `--json`: Output as JSON
|
* `--json`: Output as JSON
|
||||||
|
|
||||||
**`remote nuke`**: Delete every snapshot's metadata and every blob from the
|
**`remote nuke`**: Delete every snapshot's metadata and every blob from the
|
||||||
|
|||||||
@@ -32,7 +32,19 @@ the tag exists and is exercised; what is left is merging `next` to
|
|||||||
advice, and `--json` gives them as `null` with the unreadable remote
|
advice, and `--json` gives them as `null` with the unreadable remote
|
||||||
keys in `unreadable_manifests`. A name under `metadata/` that is not
|
keys in `unreadable_manifests`. A name under `metadata/` that is not
|
||||||
64 lowercase hex characters is now skipped with a warning instead of
|
64 lowercase hex characters is now skipped with a warning instead of
|
||||||
being printed, control characters included.
|
being printed, control characters included. Its manifest is then not
|
||||||
|
read either, so a skipped name also leaves the orphan figures unknown,
|
||||||
|
and `--json` counts such names in `skipped_metadata_names`.
|
||||||
|
|
||||||
|
- 2026-10-07: Made a backup notice a file rewritten with its size
|
||||||
|
unchanged and a new mtime in the same second as the one in the index
|
||||||
|
([issue #226](https://git.eeqj.de/sneak/vaultik/issues/226)). The
|
||||||
|
`files` table held mtime in whole seconds and the scanner compared
|
||||||
|
whole seconds, so every later snapshot kept the old content. A new
|
||||||
|
`mtime_nsec` column holds the nanoseconds within the second that
|
||||||
|
`mtime` holds, and the scanner compares the full mtime. A local index
|
||||||
|
created before the change lacks the column and is rebuilt with
|
||||||
|
`vaultik database delete` and a full backup.
|
||||||
|
|
||||||
- 2026-10-07: Made taking the process-wide lock atomic
|
- 2026-10-07: Made taking the process-wide lock atomic
|
||||||
([issue #227](https://git.eeqj.de/sneak/vaultik/issues/227)). The lock
|
([issue #227](https://git.eeqj.de/sneak/vaultik/issues/227)). The lock
|
||||||
|
|||||||
+2
-1
@@ -36,7 +36,8 @@ Stores metadata about files in the filesystem being backed up.
|
|||||||
**Columns:**
|
**Columns:**
|
||||||
- `id` (TEXT PRIMARY KEY) - UUID for the file record
|
- `id` (TEXT PRIMARY KEY) - UUID for the file record
|
||||||
- `path` (TEXT NOT NULL UNIQUE) - Absolute file path
|
- `path` (TEXT NOT NULL UNIQUE) - Absolute file path
|
||||||
- `mtime` (INTEGER NOT NULL) - Modification time as Unix timestamp
|
- `mtime` (INTEGER NOT NULL) - Modification time, whole seconds since the Unix epoch
|
||||||
|
- `mtime_nsec` (INTEGER NOT NULL) - Nanoseconds within that second, 0 to 999999999
|
||||||
- `size` (INTEGER NOT NULL) - File size in bytes
|
- `size` (INTEGER NOT NULL) - File size in bytes
|
||||||
- `mode` (INTEGER NOT NULL) - Unix file permissions and type
|
- `mode` (INTEGER NOT NULL) - Unix file permissions and type
|
||||||
- `uid` (INTEGER NOT NULL) - User ID of file owner
|
- `uid` (INTEGER NOT NULL) - User ID of file owner
|
||||||
|
|||||||
+28
-19
@@ -33,11 +33,13 @@ func (r *FileRepository) Create(ctx context.Context, tx *sql.Tx, file *File) err
|
|||||||
}
|
}
|
||||||
|
|
||||||
query := `
|
query := `
|
||||||
INSERT INTO files (id, path, source_path, mtime, size, mode, uid, gid, link_target)
|
INSERT INTO files
|
||||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
|
(id, path, source_path, mtime, mtime_nsec, size, mode, uid, gid, link_target)
|
||||||
|
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||||
ON CONFLICT(path) DO UPDATE SET
|
ON CONFLICT(path) DO UPDATE SET
|
||||||
source_path = excluded.source_path,
|
source_path = excluded.source_path,
|
||||||
mtime = excluded.mtime,
|
mtime = excluded.mtime,
|
||||||
|
mtime_nsec = excluded.mtime_nsec,
|
||||||
size = excluded.size,
|
size = excluded.size,
|
||||||
mode = excluded.mode,
|
mode = excluded.mode,
|
||||||
uid = excluded.uid,
|
uid = excluded.uid,
|
||||||
@@ -54,16 +56,19 @@ func (r *FileRepository) Create(ctx context.Context, tx *sql.Tx, file *File) err
|
|||||||
if tx != nil {
|
if tx != nil {
|
||||||
LogSQL("Execute", query,
|
LogSQL("Execute", query,
|
||||||
file.ID.String(), file.Path.String(), file.SourcePath.String(),
|
file.ID.String(), file.Path.String(), file.SourcePath.String(),
|
||||||
file.MTime.Unix(), file.Size, file.Mode, file.UID, file.GID,
|
file.MTime.Unix(), file.MTime.Nanosecond(),
|
||||||
|
file.Size, file.Mode, file.UID, file.GID,
|
||||||
file.LinkTarget.String())
|
file.LinkTarget.String())
|
||||||
err = tx.QueryRowContext(ctx, query,
|
err = tx.QueryRowContext(ctx, query,
|
||||||
file.ID.String(), file.Path.String(), file.SourcePath.String(),
|
file.ID.String(), file.Path.String(), file.SourcePath.String(),
|
||||||
file.MTime.Unix(), file.Size, file.Mode, file.UID, file.GID,
|
file.MTime.Unix(), file.MTime.Nanosecond(),
|
||||||
|
file.Size, file.Mode, file.UID, file.GID,
|
||||||
file.LinkTarget.String()).Scan(&idStr)
|
file.LinkTarget.String()).Scan(&idStr)
|
||||||
} else {
|
} else {
|
||||||
err = r.db.QueryRowWithLog(ctx, query,
|
err = r.db.QueryRowWithLog(ctx, query,
|
||||||
file.ID.String(), file.Path.String(), file.SourcePath.String(),
|
file.ID.String(), file.Path.String(), file.SourcePath.String(),
|
||||||
file.MTime.Unix(), file.Size, file.Mode, file.UID, file.GID,
|
file.MTime.Unix(), file.MTime.Nanosecond(),
|
||||||
|
file.Size, file.Mode, file.UID, file.GID,
|
||||||
file.LinkTarget.String()).Scan(&idStr)
|
file.LinkTarget.String()).Scan(&idStr)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -84,7 +89,7 @@ func (r *FileRepository) Create(ctx context.Context, tx *sql.Tx, file *File) err
|
|||||||
// in the index.
|
// in the index.
|
||||||
func (r *FileRepository) GetByPath(ctx context.Context, path string) (*File, error) {
|
func (r *FileRepository) GetByPath(ctx context.Context, path string) (*File, error) {
|
||||||
query := `
|
query := `
|
||||||
SELECT id, path, source_path, mtime, size, mode, uid, gid, link_target
|
SELECT id, path, source_path, mtime, mtime_nsec, size, mode, uid, gid, link_target
|
||||||
FROM files
|
FROM files
|
||||||
WHERE path = ?
|
WHERE path = ?
|
||||||
`
|
`
|
||||||
@@ -104,7 +109,7 @@ func (r *FileRepository) GetByPath(ctx context.Context, path string) (*File, err
|
|||||||
// GetByID retrieves a file by its UUID
|
// GetByID retrieves a file by its UUID
|
||||||
func (r *FileRepository) GetByID(ctx context.Context, id types.FileID) (*File, error) {
|
func (r *FileRepository) GetByID(ctx context.Context, id types.FileID) (*File, error) {
|
||||||
query := `
|
query := `
|
||||||
SELECT id, path, source_path, mtime, size, mode, uid, gid, link_target
|
SELECT id, path, source_path, mtime, mtime_nsec, size, mode, uid, gid, link_target
|
||||||
FROM files
|
FROM files
|
||||||
WHERE id = ?
|
WHERE id = ?
|
||||||
`
|
`
|
||||||
@@ -127,7 +132,7 @@ func (r *FileRepository) GetByPathTx(
|
|||||||
ctx context.Context, tx *sql.Tx, path string,
|
ctx context.Context, tx *sql.Tx, path string,
|
||||||
) (*File, error) {
|
) (*File, error) {
|
||||||
query := `
|
query := `
|
||||||
SELECT id, path, source_path, mtime, size, mode, uid, gid, link_target
|
SELECT id, path, source_path, mtime, mtime_nsec, size, mode, uid, gid, link_target
|
||||||
FROM files
|
FROM files
|
||||||
WHERE path = ?
|
WHERE path = ?
|
||||||
`
|
`
|
||||||
@@ -158,13 +163,14 @@ func (r *FileRepository) ListModifiedSince(
|
|||||||
ctx context.Context, since time.Time,
|
ctx context.Context, since time.Time,
|
||||||
) ([]*File, error) {
|
) ([]*File, error) {
|
||||||
query := `
|
query := `
|
||||||
SELECT id, path, source_path, mtime, size, mode, uid, gid, link_target
|
SELECT id, path, source_path, mtime, mtime_nsec, size, mode, uid, gid, link_target
|
||||||
FROM files
|
FROM files
|
||||||
WHERE mtime >= ?
|
WHERE (mtime, mtime_nsec) >= (?, ?)
|
||||||
ORDER BY path
|
ORDER BY path
|
||||||
`
|
`
|
||||||
|
|
||||||
rows, err := r.db.conn.QueryContext(ctx, query, since.Unix())
|
rows, err := r.db.conn.QueryContext(ctx, query,
|
||||||
|
since.Unix(), since.Nanosecond())
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("querying files: %w", err)
|
return nil, fmt.Errorf("querying files: %w", err)
|
||||||
}
|
}
|
||||||
@@ -239,7 +245,7 @@ func (r *FileRepository) ListUnderPath(
|
|||||||
|
|
||||||
// LIKE would ignore ASCII case and treat _ and % in path as wildcards.
|
// LIKE would ignore ASCII case and treat _ and % in path as wildcards.
|
||||||
query := `
|
query := `
|
||||||
SELECT id, path, source_path, mtime, size, mode, uid, gid, link_target
|
SELECT id, path, source_path, mtime, mtime_nsec, size, mode, uid, gid, link_target
|
||||||
FROM files
|
FROM files
|
||||||
WHERE path = ? OR substr(path, 1, length(?)) = ?
|
WHERE path = ? OR substr(path, 1, length(?)) = ?
|
||||||
ORDER BY path
|
ORDER BY path
|
||||||
@@ -324,7 +330,7 @@ func (r *FileRepository) ListIDsWithChunksNotInUploadedBlobs(
|
|||||||
// ListAll returns all files in the database
|
// ListAll returns all files in the database
|
||||||
func (r *FileRepository) ListAll(ctx context.Context) ([]*File, error) {
|
func (r *FileRepository) ListAll(ctx context.Context) ([]*File, error) {
|
||||||
query := `
|
query := `
|
||||||
SELECT id, path, source_path, mtime, size, mode, uid, gid, link_target
|
SELECT id, path, source_path, mtime, mtime_nsec, size, mode, uid, gid, link_target
|
||||||
FROM files
|
FROM files
|
||||||
ORDER BY path
|
ORDER BY path
|
||||||
`
|
`
|
||||||
@@ -365,7 +371,7 @@ func (r *FileRepository) CreateBatch(
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Each files row binds this many SQL variables.
|
// Each files row binds this many SQL variables.
|
||||||
const fileCols = 9
|
const fileCols = 10
|
||||||
|
|
||||||
// Batch at 100 rows to be safe with SQLite's variable limit.
|
// Batch at 100 rows to be safe with SQLite's variable limit.
|
||||||
const batchSize = 100
|
const batchSize = 100
|
||||||
@@ -376,7 +382,7 @@ func (r *FileRepository) CreateBatch(
|
|||||||
batch := files[i:end]
|
batch := files[i:end]
|
||||||
|
|
||||||
query := `INSERT INTO files
|
query := `INSERT INTO files
|
||||||
(id, path, source_path, mtime, size, mode, uid, gid, link_target)
|
(id, path, source_path, mtime, mtime_nsec, size, mode, uid, gid, link_target)
|
||||||
VALUES `
|
VALUES `
|
||||||
|
|
||||||
args := make([]any, 0, len(batch)*fileCols)
|
args := make([]any, 0, len(batch)*fileCols)
|
||||||
@@ -388,11 +394,12 @@ func (r *FileRepository) CreateBatch(
|
|||||||
querySb325.WriteString(", ")
|
querySb325.WriteString(", ")
|
||||||
}
|
}
|
||||||
|
|
||||||
querySb325.WriteString("(?, ?, ?, ?, ?, ?, ?, ?, ?)")
|
querySb325.WriteString("(?, ?, ?, ?, ?, ?, ?, ?, ?, ?)")
|
||||||
|
|
||||||
args = append(args,
|
args = append(args,
|
||||||
f.ID.String(), f.Path.String(), f.SourcePath.String(),
|
f.ID.String(), f.Path.String(), f.SourcePath.String(),
|
||||||
f.MTime.Unix(), f.Size, f.Mode, f.UID, f.GID,
|
f.MTime.Unix(), f.MTime.Nanosecond(),
|
||||||
|
f.Size, f.Mode, f.UID, f.GID,
|
||||||
f.LinkTarget.String())
|
f.LinkTarget.String())
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -401,6 +408,7 @@ func (r *FileRepository) CreateBatch(
|
|||||||
query += ` ON CONFLICT(path) DO UPDATE SET
|
query += ` ON CONFLICT(path) DO UPDATE SET
|
||||||
source_path = excluded.source_path,
|
source_path = excluded.source_path,
|
||||||
mtime = excluded.mtime,
|
mtime = excluded.mtime,
|
||||||
|
mtime_nsec = excluded.mtime_nsec,
|
||||||
size = excluded.size,
|
size = excluded.size,
|
||||||
mode = excluded.mode,
|
mode = excluded.mode,
|
||||||
uid = excluded.uid,
|
uid = excluded.uid,
|
||||||
@@ -460,7 +468,7 @@ func (r *FileRepository) scanFileFrom(row fileRowScanner) (*File, error) {
|
|||||||
var (
|
var (
|
||||||
file File
|
file File
|
||||||
idStr, pathStr, sourcePathStr string
|
idStr, pathStr, sourcePathStr string
|
||||||
mtimeUnix int64
|
mtimeUnix, mtimeNsec int64
|
||||||
linkTarget sql.NullString
|
linkTarget sql.NullString
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -469,6 +477,7 @@ func (r *FileRepository) scanFileFrom(row fileRowScanner) (*File, error) {
|
|||||||
&pathStr,
|
&pathStr,
|
||||||
&sourcePathStr,
|
&sourcePathStr,
|
||||||
&mtimeUnix,
|
&mtimeUnix,
|
||||||
|
&mtimeNsec,
|
||||||
&file.Size,
|
&file.Size,
|
||||||
&file.Mode,
|
&file.Mode,
|
||||||
&file.UID,
|
&file.UID,
|
||||||
@@ -487,7 +496,7 @@ func (r *FileRepository) scanFileFrom(row fileRowScanner) (*File, error) {
|
|||||||
file.Path = types.FilePath(pathStr)
|
file.Path = types.FilePath(pathStr)
|
||||||
file.SourcePath = types.SourcePath(sourcePathStr)
|
file.SourcePath = types.SourcePath(sourcePathStr)
|
||||||
|
|
||||||
file.MTime = time.Unix(mtimeUnix, 0).UTC()
|
file.MTime = time.Unix(mtimeUnix, mtimeNsec).UTC()
|
||||||
if linkTarget.Valid {
|
if linkTarget.Valid {
|
||||||
file.LinkTarget = types.FilePath(linkTarget.String)
|
file.LinkTarget = types.FilePath(linkTarget.String)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -252,6 +252,115 @@ func TestFileRepositorySymlink(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// An mtime after 2262 or before 1678 does not fit in int64 nanoseconds
|
||||||
|
// since the epoch, and must still come back from the database unchanged.
|
||||||
|
func TestFileRepositoryMTimeOutsideInt64NanosecondRange(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
db, cleanup := setupTestDB(t)
|
||||||
|
defer cleanup()
|
||||||
|
|
||||||
|
ctx := context.Background()
|
||||||
|
repo := database.NewFileRepository(db)
|
||||||
|
|
||||||
|
mtimes := []time.Time{
|
||||||
|
time.Date(2300, time.January, 1, 0, 0, 0, 123456789, time.UTC),
|
||||||
|
time.Date(1601, time.January, 1, 0, 0, 0, 987654321, time.UTC),
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, mtime := range mtimes {
|
||||||
|
created := &database.File{
|
||||||
|
Path: types.FilePath("/created-" + mtime.Format(time.RFC3339Nano)),
|
||||||
|
MTime: mtime,
|
||||||
|
}
|
||||||
|
|
||||||
|
err := repo.Create(ctx, nil, created)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("failed to create file: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
batched := &database.File{
|
||||||
|
ID: types.NewFileID(),
|
||||||
|
Path: types.FilePath("/batched-" + mtime.Format(time.RFC3339Nano)),
|
||||||
|
MTime: mtime,
|
||||||
|
}
|
||||||
|
|
||||||
|
err = repo.CreateBatch(ctx, nil, []*database.File{batched})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("failed to batch create file: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, path := range []types.FilePath{created.Path, batched.Path} {
|
||||||
|
retrieved, err := repo.GetByPath(ctx, path.String())
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("failed to get file: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if !retrieved.MTime.Equal(mtime) {
|
||||||
|
t.Errorf("%s: mtime got %v, want %v",
|
||||||
|
path, retrieved.MTime, mtime)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A file already in the index and rewritten within the same second must get
|
||||||
|
// its new nanoseconds stored, through both Create and CreateBatch.
|
||||||
|
func TestFileRepositoryUpsertMTimeInSameSecond(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
db, cleanup := setupTestDB(t)
|
||||||
|
defer cleanup()
|
||||||
|
|
||||||
|
ctx := context.Background()
|
||||||
|
repo := database.NewFileRepository(db)
|
||||||
|
|
||||||
|
indexed := time.Date(2026, time.October, 7, 12, 0, 0, 100000000, time.UTC)
|
||||||
|
rewritten := indexed.Add(800 * time.Millisecond)
|
||||||
|
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
upsert func(file *database.File) error
|
||||||
|
}{
|
||||||
|
{"Create", func(file *database.File) error {
|
||||||
|
return repo.Create(ctx, nil, file)
|
||||||
|
}},
|
||||||
|
{"CreateBatch", func(file *database.File) error {
|
||||||
|
return repo.CreateBatch(ctx, nil, []*database.File{file})
|
||||||
|
}},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tt := range tests {
|
||||||
|
file := &database.File{
|
||||||
|
ID: types.NewFileID(),
|
||||||
|
Path: types.FilePath("/" + tt.name),
|
||||||
|
MTime: indexed,
|
||||||
|
}
|
||||||
|
|
||||||
|
err := tt.upsert(file)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("%s: failed to create file: %v", tt.name, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
file.MTime = rewritten
|
||||||
|
|
||||||
|
err = tt.upsert(file)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("%s: failed to update file: %v", tt.name, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
retrieved, err := repo.GetByPath(ctx, file.Path.String())
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("%s: failed to get file: %v", tt.name, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if !retrieved.MTime.Equal(rewritten) {
|
||||||
|
t.Errorf("%s: mtime got %v, want %v",
|
||||||
|
tt.name, retrieved.MTime, rewritten)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestFileRepositoryTransaction(t *testing.T) {
|
func TestFileRepositoryTransaction(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
|
|||||||
@@ -606,8 +606,7 @@ func TestTimezoneHandling(t *testing.T) {
|
|||||||
t.Skip("timezone not available")
|
t.Skip("timezone not available")
|
||||||
}
|
}
|
||||||
|
|
||||||
// Use Truncate to remove sub-second precision since we store as Unix timestamps
|
nyTime := time.Now().In(loc)
|
||||||
nyTime := time.Now().In(loc).Truncate(time.Second)
|
|
||||||
file := &File{
|
file := &File{
|
||||||
Path: "/timezone-test.txt",
|
Path: "/timezone-test.txt",
|
||||||
MTime: nyTime,
|
MTime: nyTime,
|
||||||
|
|||||||
@@ -6,7 +6,8 @@ CREATE TABLE IF NOT EXISTS files (
|
|||||||
id TEXT PRIMARY KEY, -- UUID
|
id TEXT PRIMARY KEY, -- UUID
|
||||||
path TEXT NOT NULL UNIQUE,
|
path TEXT NOT NULL UNIQUE,
|
||||||
source_path TEXT NOT NULL DEFAULT '', -- The source directory this file came from (for restore path stripping)
|
source_path TEXT NOT NULL DEFAULT '', -- The source directory this file came from (for restore path stripping)
|
||||||
mtime INTEGER NOT NULL,
|
mtime INTEGER NOT NULL, -- whole seconds since the Unix epoch
|
||||||
|
mtime_nsec INTEGER NOT NULL, -- nanoseconds within that second, 0 to 999999999
|
||||||
size INTEGER NOT NULL,
|
size INTEGER NOT NULL,
|
||||||
mode INTEGER NOT NULL,
|
mode INTEGER NOT NULL,
|
||||||
uid INTEGER NOT NULL,
|
uid INTEGER NOT NULL,
|
||||||
|
|||||||
@@ -1221,7 +1221,7 @@ func (s *Scanner) checkFileInMemory(
|
|||||||
|
|
||||||
// Check if file has changed
|
// Check if file has changed
|
||||||
if existingFile.Size != file.Size ||
|
if existingFile.Size != file.Size ||
|
||||||
existingFile.MTime.Unix() != file.MTime.Unix() ||
|
!existingFile.MTime.Equal(file.MTime) ||
|
||||||
existingFile.Mode != file.Mode ||
|
existingFile.Mode != file.Mode ||
|
||||||
existingFile.UID != file.UID ||
|
existingFile.UID != file.UID ||
|
||||||
existingFile.GID != file.GID {
|
existingFile.GID != file.GID {
|
||||||
|
|||||||
+28
-15
@@ -207,13 +207,18 @@ type RemoteInfoResult struct {
|
|||||||
ReferencedBlobSize int64 `json:"referenced_blob_size"`
|
ReferencedBlobSize int64 `json:"referenced_blob_size"`
|
||||||
|
|
||||||
// Orphaned blobs. Both stay nil (null in the JSON) when a manifest
|
// Orphaned blobs. Both stay nil (null in the JSON) when a manifest
|
||||||
// could not be read, since that snapshot's blobs would be counted
|
// could not be read or a name under metadata/ was skipped, since
|
||||||
// as orphaned.
|
// that snapshot's blobs would be counted as orphaned.
|
||||||
OrphanedBlobCount *int `json:"orphaned_blob_count"`
|
OrphanedBlobCount *int `json:"orphaned_blob_count"`
|
||||||
OrphanedBlobSize *int64 `json:"orphaned_blob_size"`
|
OrphanedBlobSize *int64 `json:"orphaned_blob_size"`
|
||||||
|
|
||||||
// Remote key of each snapshot whose manifest could not be read
|
// Remote key of each snapshot whose manifest could not be read
|
||||||
UnreadableManifests []string `json:"unreadable_manifests,omitempty"`
|
UnreadableManifests []string `json:"unreadable_manifests,omitempty"`
|
||||||
|
|
||||||
|
// Number of names under metadata/ skipped because they are not
|
||||||
|
// remote keys. The names themselves are not reported: they come
|
||||||
|
// from the destination store and may hold control characters.
|
||||||
|
SkippedMetadataNames int `json:"skipped_metadata_names,omitempty"`
|
||||||
}
|
}
|
||||||
|
|
||||||
// RemoteInfo displays information about remote storage
|
// RemoteInfo displays information about remote storage
|
||||||
@@ -239,11 +244,13 @@ func (v *Vaultik) RemoteInfo(jsonOutput bool) error {
|
|||||||
v.stdoutf("Scanning snapshot metadata...\n")
|
v.stdoutf("Scanning snapshot metadata...\n")
|
||||||
}
|
}
|
||||||
|
|
||||||
snapshotMetadata, snapshotIDs, err := v.collectSnapshotMetadata()
|
snapshotMetadata, snapshotIDs, skippedNames, err := v.collectSnapshotMetadata()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
|
result.SkippedMetadataNames = skippedNames
|
||||||
|
|
||||||
if showText {
|
if showText {
|
||||||
v.stdoutf("Downloading %d manifest(s)...\n", len(snapshotIDs))
|
v.stdoutf("Downloading %d manifest(s)...\n", len(snapshotIDs))
|
||||||
}
|
}
|
||||||
@@ -280,16 +287,18 @@ func (v *Vaultik) RemoteInfo(jsonOutput bool) error {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// collectSnapshotMetadata scans remote metadata and returns
|
// collectSnapshotMetadata scans remote metadata and returns
|
||||||
// per-snapshot info and sorted IDs.
|
// per-snapshot info, sorted IDs and the number of names it skipped
|
||||||
|
// because they are not remote keys.
|
||||||
func (v *Vaultik) collectSnapshotMetadata() (
|
func (v *Vaultik) collectSnapshotMetadata() (
|
||||||
map[string]*SnapshotMetadataInfo, []string, error,
|
map[string]*SnapshotMetadataInfo, []string, int, error,
|
||||||
) {
|
) {
|
||||||
snapshotMetadata := make(map[string]*SnapshotMetadataInfo)
|
snapshotMetadata := make(map[string]*SnapshotMetadataInfo)
|
||||||
|
skippedNames := make(map[string]bool)
|
||||||
|
|
||||||
metadataCh := v.Storage.ListStream(v.ctx, "metadata/")
|
metadataCh := v.Storage.ListStream(v.ctx, "metadata/")
|
||||||
for obj := range metadataCh {
|
for obj := range metadataCh {
|
||||||
if obj.Err != nil {
|
if obj.Err != nil {
|
||||||
return nil, nil, fmt.Errorf("listing metadata: %w", obj.Err)
|
return nil, nil, 0, fmt.Errorf("listing metadata: %w", obj.Err)
|
||||||
}
|
}
|
||||||
|
|
||||||
parts := strings.Split(obj.Key, "/")
|
parts := strings.Split(obj.Key, "/")
|
||||||
@@ -306,6 +315,8 @@ func (v *Vaultik) collectSnapshotMetadata() (
|
|||||||
log.Warn("Skipping non-conforming key under metadata/",
|
log.Warn("Skipping non-conforming key under metadata/",
|
||||||
"key", obj.Key)
|
"key", obj.Key)
|
||||||
|
|
||||||
|
skippedNames[snapshotID] = true
|
||||||
|
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -332,7 +343,7 @@ func (v *Vaultik) collectSnapshotMetadata() (
|
|||||||
|
|
||||||
sort.Strings(snapshotIDs)
|
sort.Strings(snapshotIDs)
|
||||||
|
|
||||||
return snapshotMetadata, snapshotIDs, nil
|
return snapshotMetadata, snapshotIDs, len(skippedNames), nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// collectReferencedBlobsFromManifests downloads manifests and returns
|
// collectReferencedBlobsFromManifests downloads manifests and returns
|
||||||
@@ -400,9 +411,9 @@ func (v *Vaultik) populateRemoteInfoResult(
|
|||||||
}
|
}
|
||||||
|
|
||||||
// scanRemoteBlobStorage lists all blobs on remote and computes orphan
|
// scanRemoteBlobStorage lists all blobs on remote and computes orphan
|
||||||
// stats when every manifest was read. showText is true only when the
|
// stats when every manifest was read and no name under metadata/ was
|
||||||
// human report is being printed (not --json, not --quiet), gating the
|
// skipped. showText is true only when the human report is being
|
||||||
// progress line.
|
// printed (not --json, not --quiet), gating the progress line.
|
||||||
func (v *Vaultik) scanRemoteBlobStorage(
|
func (v *Vaultik) scanRemoteBlobStorage(
|
||||||
result *RemoteInfoResult, referencedBlobs map[string]int64, showText bool,
|
result *RemoteInfoResult, referencedBlobs map[string]int64, showText bool,
|
||||||
) error {
|
) error {
|
||||||
@@ -429,9 +440,10 @@ func (v *Vaultik) scanRemoteBlobStorage(
|
|||||||
result.TotalBlobSize += obj.Size
|
result.TotalBlobSize += obj.Size
|
||||||
}
|
}
|
||||||
|
|
||||||
// A blob named only by a manifest that could not be read would be
|
// A blob named only by a manifest that could not be read, or by one
|
||||||
// counted as orphaned, so the orphan figures stay unknown.
|
// under a skipped name, would be counted as orphaned, so the orphan
|
||||||
if len(result.UnreadableManifests) > 0 {
|
// figures stay unknown.
|
||||||
|
if len(result.UnreadableManifests) > 0 || result.SkippedMetadataNames > 0 {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -505,8 +517,9 @@ func (v *Vaultik) printRemoteInfoTable(result *RemoteInfoResult) {
|
|||||||
|
|
||||||
if result.OrphanedBlobCount == nil {
|
if result.OrphanedBlobCount == nil {
|
||||||
v.stdoutf("Orphaned (unreferenced): unknown "+
|
v.stdoutf("Orphaned (unreferenced): unknown "+
|
||||||
"(%d manifest(s) could not be read)\n",
|
"(%d manifest(s) could not be read, "+
|
||||||
len(result.UnreadableManifests))
|
"%d name(s) under metadata/ skipped)\n",
|
||||||
|
len(result.UnreadableManifests), result.SkippedMetadataNames)
|
||||||
|
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -13,6 +13,11 @@ import (
|
|||||||
"sneak.berlin/go/vaultik/internal/snapshot"
|
"sneak.berlin/go/vaultik/internal/snapshot"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
// testBlobHashB is a blob that the manifest written by addRemote does
|
||||||
|
// not reference.
|
||||||
|
const testBlobHashB = "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" +
|
||||||
|
"bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"
|
||||||
|
|
||||||
// TestRemoteInfo_UnreadableManifestLeavesOrphansUnknown checks that a
|
// TestRemoteInfo_UnreadableManifestLeavesOrphansUnknown checks that a
|
||||||
// manifest remote info cannot read makes the orphan figures unknown. A
|
// manifest remote info cannot read makes the orphan figures unknown. A
|
||||||
// blob referenced only by that snapshot would otherwise be counted as
|
// blob referenced only by that snapshot would otherwise be counted as
|
||||||
@@ -21,9 +26,6 @@ func TestRemoteInfo_UnreadableManifestLeavesOrphansUnknown(t *testing.T) {
|
|||||||
log.Initialize(log.Config{})
|
log.Initialize(log.Config{})
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
const testBlobHashB = "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" +
|
|
||||||
"bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"
|
|
||||||
|
|
||||||
env := newListEnv(t)
|
env := newListEnv(t)
|
||||||
|
|
||||||
// The readable manifest references blob A only.
|
// The readable manifest references blob A only.
|
||||||
@@ -50,8 +52,8 @@ func TestRemoteInfo_UnreadableManifestLeavesOrphansUnknown(t *testing.T) {
|
|||||||
require.NoError(t, env.v.RemoteInfo(false))
|
require.NoError(t, env.v.RemoteInfo(false))
|
||||||
|
|
||||||
text := env.stdout.String()
|
text := env.stdout.String()
|
||||||
assert.Contains(t, text,
|
assert.Contains(t, text, "Orphaned (unreferenced): unknown "+
|
||||||
"Orphaned (unreferenced): unknown (1 manifest(s) could not be read)")
|
"(1 manifest(s) could not be read, 0 name(s) under metadata/ skipped)")
|
||||||
assert.NotContains(t, text, "vaultik prune")
|
assert.NotContains(t, text, "vaultik prune")
|
||||||
|
|
||||||
env.stdout.Reset()
|
env.stdout.Reset()
|
||||||
@@ -68,21 +70,45 @@ func TestRemoteInfo_UnreadableManifestLeavesOrphansUnknown(t *testing.T) {
|
|||||||
|
|
||||||
// TestRemoteInfo_SkipsNonConformingMetadataName checks that a directory
|
// TestRemoteInfo_SkipsNonConformingMetadataName checks that a directory
|
||||||
// under metadata/ whose name is not a remote key is left out of the
|
// under metadata/ whose name is not a remote key is left out of the
|
||||||
// report. The name comes from the destination store; printed raw, its
|
// report, and that the orphan figures are then unknown. The name comes
|
||||||
// control characters would reach the terminal.
|
// from the destination store; printed raw, its control characters would
|
||||||
|
// reach the terminal. Its manifest is not read, so a blob only it
|
||||||
|
// references would otherwise be counted as orphaned.
|
||||||
func TestRemoteInfo_SkipsNonConformingMetadataName(t *testing.T) {
|
func TestRemoteInfo_SkipsNonConformingMetadataName(t *testing.T) {
|
||||||
log.Initialize(log.Config{})
|
log.Initialize(log.Config{})
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
env := newListEnv(t)
|
env := newListEnv(t)
|
||||||
env.addRemote(t, listRemoteID, time.Date(2026, 3, 2, 0, 0, 0, 0, time.UTC))
|
env.addRemote(t, listRemoteID, time.Date(2026, 3, 2, 0, 0, 0, 0, time.UTC))
|
||||||
|
addBlob(t, env.store.testStorer, testBlobHashA)
|
||||||
|
addBlob(t, env.store.testStorer, testBlobHashB)
|
||||||
require.NoError(t, env.store.Put(context.Background(),
|
require.NoError(t, env.store.Put(context.Background(),
|
||||||
"metadata/\x1b[31mred/manifest.json.zst",
|
"metadata/\x1b[31mred/manifest.json.zst",
|
||||||
bytes.NewReader([]byte("not a valid manifest"))))
|
bytes.NewReader([]byte("not a valid manifest"))))
|
||||||
|
|
||||||
require.NoError(t, env.v.RemoteInfo(false))
|
require.NoError(t, env.v.RemoteInfo(false))
|
||||||
|
|
||||||
|
text := env.stdout.String()
|
||||||
|
assert.NotContains(t, text, "\x1b")
|
||||||
|
assert.NotContains(t, text, "31mred")
|
||||||
|
assert.Contains(t, text, "Total (1 snapshots)")
|
||||||
|
assert.Contains(t, text, "Orphaned (unreferenced): unknown "+
|
||||||
|
"(0 manifest(s) could not be read, 1 name(s) under metadata/ skipped)")
|
||||||
|
assert.NotContains(t, text, "vaultik prune")
|
||||||
|
|
||||||
|
env.stdout.Reset()
|
||||||
|
require.NoError(t, env.v.RemoteInfo(true))
|
||||||
|
|
||||||
out := env.stdout.String()
|
out := env.stdout.String()
|
||||||
assert.NotContains(t, out, "\x1b")
|
assert.NotContains(t, out, "31mred")
|
||||||
assert.Contains(t, out, "Total (1 snapshots)")
|
|
||||||
|
var doc map[string]any
|
||||||
|
|
||||||
|
require.NoError(t, json.Unmarshal([]byte(out), &doc))
|
||||||
|
assert.Contains(t, doc, "orphaned_blob_count")
|
||||||
|
assert.Nil(t, doc["orphaned_blob_count"])
|
||||||
|
assert.Contains(t, doc, "orphaned_blob_size")
|
||||||
|
assert.Nil(t, doc["orphaned_blob_size"])
|
||||||
|
assert.InDelta(t, 1, doc["skipped_metadata_names"], 0)
|
||||||
|
assert.NotContains(t, doc, "unreadable_manifests")
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,73 @@
|
|||||||
|
package vaultik_test
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"context"
|
||||||
|
"path/filepath"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/spf13/afero"
|
||||||
|
"github.com/stretchr/testify/require"
|
||||||
|
"sneak.berlin/go/vaultik/internal/database"
|
||||||
|
"sneak.berlin/go/vaultik/internal/log"
|
||||||
|
"sneak.berlin/go/vaultik/internal/storage"
|
||||||
|
"sneak.berlin/go/vaultik/internal/vaultik"
|
||||||
|
)
|
||||||
|
|
||||||
|
// A file rewritten with its size unchanged and a new mtime in the same
|
||||||
|
// second as the mtime the index holds must still be backed up. See
|
||||||
|
// https://git.eeqj.de/sneak/vaultik/issues/226.
|
||||||
|
//
|
||||||
|
//nolint:paralleltest // installs the global logger via log.Initialize
|
||||||
|
func TestBackupOfSameSecondRewriteRestoresNewContent(t *testing.T) {
|
||||||
|
log.Initialize(log.Config{})
|
||||||
|
|
||||||
|
fs := afero.NewOsFs()
|
||||||
|
tempDir := t.TempDir()
|
||||||
|
dataDir := filepath.Join(tempDir, "src")
|
||||||
|
storeDir := filepath.Join(tempDir, "remote")
|
||||||
|
restoreDir := filepath.Join(tempDir, "restored")
|
||||||
|
dbPath := filepath.Join(tempDir, "index.sqlite")
|
||||||
|
rewrittenPath := filepath.Join(dataDir, "small.txt")
|
||||||
|
|
||||||
|
ctx := context.Background()
|
||||||
|
files := writeFaultSourceTree(t, fs, dataDir)
|
||||||
|
cfg := changedFileConfig(dataDir, dbPath)
|
||||||
|
|
||||||
|
firstMTime := time.Date(2026, time.January, 2, 3, 4, 5, 0, time.UTC).
|
||||||
|
Add(100 * time.Millisecond)
|
||||||
|
secondMTime := firstMTime.Add(800 * time.Millisecond)
|
||||||
|
|
||||||
|
require.NoError(t, fs.Chtimes(rewrittenPath, firstMTime, firstMTime))
|
||||||
|
|
||||||
|
store, err := storage.NewFileStorer(storeDir)
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
db, err := database.New(ctx, dbPath)
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
repos := database.NewRepositories(db)
|
||||||
|
v := newBackupVaultik(ctx, cfg, store, repos, db, fs)
|
||||||
|
|
||||||
|
require.NoError(t, backUp(v, "first"))
|
||||||
|
|
||||||
|
// Upper-casing ASCII text keeps its size.
|
||||||
|
files[rewrittenPath] = bytes.ToUpper(files[rewrittenPath])
|
||||||
|
require.NoError(t, afero.WriteFile(fs, rewrittenPath, files[rewrittenPath], 0o644))
|
||||||
|
require.NoError(t, fs.Chtimes(rewrittenPath, secondMTime, secondMTime))
|
||||||
|
|
||||||
|
require.NoError(t, backUp(v, "second"))
|
||||||
|
|
||||||
|
id := localSnapshotID(ctx, t, repos, "second")
|
||||||
|
require.NoError(t, db.Close())
|
||||||
|
|
||||||
|
reader := newReaderVaultik(ctx, cfg, store, nil, fs)
|
||||||
|
require.NoError(t, reader.Restore(&vaultik.RestoreOptions{
|
||||||
|
SnapshotID: id,
|
||||||
|
TargetDir: restoreDir,
|
||||||
|
Verify: true,
|
||||||
|
}))
|
||||||
|
|
||||||
|
assertRestoredTree(t, fs, restoreDir, files)
|
||||||
|
}
|
||||||
Reference in New Issue
Block a user