Compare commits
3
Commits
5fbb18bf0a
...
2e597f45ab
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
2e597f45ab | ||
|
|
a0de8bfdb7 | ||
|
|
291c39174f |
+14
-1
@@ -311,6 +311,8 @@ Repository interfaces:
|
||||
|
||||
```
|
||||
CreateSnapshot(opts)
|
||||
│
|
||||
├─► markDestination() // Refuse a destination without the marker, then write it
|
||||
│
|
||||
├─► PruneDatabase() // Critical: avoid dedup errors
|
||||
│
|
||||
@@ -357,7 +359,7 @@ CreateSnapshot(opts)
|
||||
|
||||
## Deduplication Strategy
|
||||
|
||||
1. **File-level**: Files unchanged since last backup are skipped (metadata comparison: size, mtime, mode, uid, gid), unless the file lists a chunk that no uploaded blob holds; such a file is re-chunked
|
||||
1. **File-level**: Files and symlinks unchanged since last backup are skipped (metadata comparison: size, mtime, mode, uid, gid, and a symlink's target), unless the file lists a chunk that no uploaded blob holds; such a file is re-chunked
|
||||
|
||||
2. **Chunk-level**: Chunks are content-addressed by SHA256 hash. If a chunk hash already exists in the database, the chunk data is not re-uploaded.
|
||||
|
||||
@@ -367,6 +369,8 @@ CreateSnapshot(opts)
|
||||
|
||||
```
|
||||
bucket/
|
||||
├── vaultik-destination.txt # Marker written by every backup
|
||||
│
|
||||
├── blobs/
|
||||
│ └── {hash[0:2]}/
|
||||
│ └── {hash[2:4]}/
|
||||
@@ -378,6 +382,15 @@ bucket/
|
||||
└── manifest.json.zst # Blob list (for pruning/verification)
|
||||
```
|
||||
|
||||
A destination without `vaultik-destination.txt` is treated as not present
|
||||
while the local index holds snapshots stored there (`checkDestinationMarker`
|
||||
in `internal/vaultik/destination_marker.go`): `snapshot create` refuses to
|
||||
write to it, `prune` and `snapshot purge` fail, and `snapshot list` and
|
||||
`snapshot remove` behave as for a destination that cannot be listed. This
|
||||
keeps those commands from taking an unplugged volume's empty mount point for
|
||||
a destination whose snapshots are all gone. The `file://` backend never
|
||||
creates its destination directory, so a write to a missing one fails.
|
||||
|
||||
The `{remote-key}` directory name is a one-way double SHA-256 hash of the human
|
||||
snapshot ID, so the human ID (hostname, snapshot name, timestamp) is never
|
||||
written to the store as a directory name. See
|
||||
|
||||
@@ -24,6 +24,10 @@ grep 'public key' vaultik_backup_private_key.txt
|
||||
vaultik config set age_recipients.0 age1YOUR_PUBLIC_KEY_HERE
|
||||
vaultik config set storage_url "file:///Volumes/usbstick/mybackup"
|
||||
|
||||
# create the destination directory; vaultik never creates it, so a backup
|
||||
# to a volume that is not plugged in fails instead of filling the disk
|
||||
mkdir /Volumes/usbstick/mybackup
|
||||
|
||||
# macOS only: grant your terminal app Full Disk Access first
|
||||
# (System Settings → Privacy & Security → Full Disk Access), otherwise
|
||||
# the backup will abort with a permission error on protected directories
|
||||
@@ -298,6 +302,10 @@ value where a list belongs or a word where a number belongs), is refused.
|
||||
this duration instead of only the latest (e.g. `4w`, `30d`, `6mo`, `1y`).
|
||||
Without `--prune`, or with a value that is not a valid duration, the
|
||||
command fails before the backup starts.
|
||||
* Fails before it writes anything when a `file://` destination directory
|
||||
does not exist, or when the local index holds snapshots and the
|
||||
destination has no `vaultik-destination.txt` (see
|
||||
[remote storage layout](#remote-storage-layout))
|
||||
|
||||
**`snapshot list`**: Show every snapshot known to this host — the union
|
||||
of the local index and the backup destination store — with timestamps
|
||||
@@ -331,7 +339,10 @@ gives: revert the configuration, or run `vaultik database delete`.
|
||||
|
||||
If the destination store cannot be listed (unmounted volume,
|
||||
permission denied, network down), the command warns, falls back to the
|
||||
local index alone, and still exits zero.
|
||||
local index alone, and still exits zero. It does the same when the
|
||||
destination has no `vaultik-destination.txt` while the local index
|
||||
holds snapshots stored there, as when a volume mounted at the
|
||||
destination is unplugged and leaves its empty mount point behind.
|
||||
* `--json`: Output in JSON format. Each entry carries `locally_tracked`
|
||||
(whether the snapshot is in the local index), `remote_key` (the full
|
||||
64-character storage key), and `remote_present` (whether it was seen
|
||||
@@ -373,13 +384,14 @@ snapshot in the local index or on the destination store is an error;
|
||||
with `--local-only`, the snapshot must be in the local index. Local row
|
||||
cleanup (files, chunks, blobs the snapshot was the last referrer for)
|
||||
runs automatically. If removing the snapshot's metadata from the
|
||||
destination store fails (for example because the store is unreachable),
|
||||
the local-DB removal still completes and a warning is emitted; run
|
||||
`vaultik snapshot remove <snapshot-id>` again once the store is reachable
|
||||
to remove the snapshot's metadata from it (`vaultik prune` does not). For
|
||||
a snapshot that is only on the destination store, that failure is an
|
||||
error. To wipe everything on the destination in one go, use `vaultik
|
||||
remote nuke --force`.
|
||||
destination store fails (for example because the store is unreachable,
|
||||
or the local index holds snapshots and the destination has no
|
||||
`vaultik-destination.txt`), the local-DB removal still completes and a
|
||||
warning is emitted; run `vaultik snapshot remove <snapshot-id>` again
|
||||
once the store is reachable to remove the snapshot's metadata from it
|
||||
(`vaultik prune` does not). For a snapshot that is only on the
|
||||
destination store, that failure is an error. To wipe everything on the
|
||||
destination in one go, use `vaultik remote nuke --force`.
|
||||
* `--local-only`: Skip remote cleanup; only touch the local index
|
||||
* `--dry-run`: Show what would be deleted without deleting
|
||||
* `--force`: Skip confirmation prompt
|
||||
@@ -410,8 +422,12 @@ manifest on the destination store to compute the still-referenced blob
|
||||
set and delete any blob not in that set. Step (3) reads all remote
|
||||
manifests — network cost scales with the number of snapshots. `snapshot
|
||||
create --prune` runs the same cleanup automatically; this is the
|
||||
manual entry point for the same work. If a blob cannot be deleted, the
|
||||
command (and `snapshot create --prune`) exits non-zero.
|
||||
manual entry point for the same work. If the destination store cannot
|
||||
be listed, or the local index holds snapshots and the destination has
|
||||
no `vaultik-destination.txt` (an unplugged volume's empty mount point),
|
||||
prune fails and deletes nothing. `snapshot purge` does the same. If a
|
||||
blob cannot be deleted, the command (and `snapshot create --prune`)
|
||||
exits non-zero.
|
||||
* `--force`: Skip confirmation prompt
|
||||
* `--json`: Output stats as JSON. Also skips the confirmation prompt, as
|
||||
`--force` does. A blob that cannot be deleted is counted in
|
||||
@@ -435,11 +451,12 @@ since `prune` never deletes it.
|
||||
* `--json`: Output as JSON
|
||||
|
||||
**`remote nuke`**: Delete every object under `metadata/` and `blobs/` on
|
||||
the backup destination store, whatever its name, leaving the bucket prefix
|
||||
empty. Destructive and irreversible. This is the single supported way to
|
||||
wipe the entire destination store. If a listing afterwards still finds
|
||||
anything there, the command names each object left and exits non-zero;
|
||||
running it again deletes what is left.
|
||||
the backup destination store, whatever its name, and then
|
||||
`vaultik-destination.txt`, leaving the bucket prefix empty. Destructive
|
||||
and irreversible. This is the single supported way to wipe the entire
|
||||
destination store. If a listing afterwards still finds anything under
|
||||
`metadata/` or `blobs/`, the command names each object left and exits
|
||||
non-zero; running it again deletes what is left.
|
||||
* `--force`: Required to confirm destruction.
|
||||
|
||||
**`database delete`**: Delete the local SQLite state database file
|
||||
@@ -464,7 +481,12 @@ object store. Credentials are read from `s3.access_key_id` and
|
||||
a local or mounted filesystem. Useful for testing or backing up to a NAS. The
|
||||
path must be absolute; a relative one is refused when the config loads.
|
||||
Percent-escapes in the path are decoded, so `my%20stick` and `my stick` name
|
||||
the same directory.
|
||||
the same directory. The directory must exist before the first backup: vaultik
|
||||
never creates it. A directory on a removable volume is gone while the volume is
|
||||
not mounted, so a backup then fails instead of writing to the disk under the
|
||||
mount point. The mount point itself stays behind when the volume is unplugged:
|
||||
there only `vaultik-destination.txt` stops such a backup, and only while the
|
||||
local index holds snapshots.
|
||||
|
||||
**Rclone** (`rclone://remote/path`): Uses rclone's 70+ supported cloud
|
||||
providers. Requires rclone to be configured separately (`rclone config`).
|
||||
@@ -495,6 +517,7 @@ For an implementation-level view of the internals — the data model, the
|
||||
|
||||
```
|
||||
<bucket>/<prefix>/
|
||||
├── vaultik-destination.txt # Marker written by every backup
|
||||
├── blobs/
|
||||
│ └── <aa>/<bb>/<full_blob_hash>
|
||||
└── metadata/
|
||||
@@ -503,6 +526,15 @@ For an implementation-level view of the internals — the data model, the
|
||||
└── manifest.json.zst # Unencrypted blob list (for pruning)
|
||||
```
|
||||
|
||||
* `vaultik-destination.txt` holds one fixed line of text, with no host
|
||||
name, path or time. A destination without it is treated as not
|
||||
present while the local index holds snapshots stored there:
|
||||
`snapshot create` refuses to write to it, `prune` and `snapshot purge`
|
||||
fail, and `snapshot list` and `snapshot remove` behave as for a
|
||||
destination that cannot be listed. This keeps an unplugged volume's empty mount point
|
||||
from being taken for a destination whose snapshots are all gone. Mount
|
||||
the volume if it is removable; if the destination was emptied on
|
||||
purpose, run `vaultik database delete` and back up again.
|
||||
* Blobs are two-level directory sharded using the first 4 hex chars of the blob hash
|
||||
* `db.zst.age` is a binary SQLite database (zstd compressed, age encrypted)
|
||||
containing all file metadata, chunk mappings, and relationships for the snapshot
|
||||
|
||||
@@ -4,7 +4,7 @@ This document describes the structure and organization of data stored in the S3
|
||||
|
||||
## Overview
|
||||
|
||||
Vaultik stores all backup data in an S3-compatible object store. The repository consists of two main components:
|
||||
Vaultik stores all backup data in an S3-compatible object store. The repository consists of two main components, plus a marker file:
|
||||
1. **Blobs** - The actual backup data (content-addressed, encrypted)
|
||||
2. **Metadata** - Snapshot information and manifests (partially encrypted)
|
||||
|
||||
@@ -12,6 +12,7 @@ Vaultik stores all backup data in an S3-compatible object store. The repository
|
||||
|
||||
```
|
||||
<bucket>/<prefix>/
|
||||
├── vaultik-destination.txt
|
||||
├── blobs/
|
||||
│ └── <hash[0:2]>/
|
||||
│ └── <hash[2:4]>/
|
||||
@@ -24,6 +25,10 @@ Vaultik stores all backup data in an S3-compatible object store. The repository
|
||||
|
||||
The metadata subdirectory is named with the **remote key**, a one-way hash of the snapshot ID, not with the human-readable snapshot ID itself. See [Remote Key Derivation](#remote-key-derivation).
|
||||
|
||||
## Marker File (`vaultik-destination.txt`)
|
||||
|
||||
Every backup writes this file at the root before it uploads anything. It holds one fixed line of plain text and names no host, path or time. A destination without it is treated as not present while the local index holds snapshots stored there: `snapshot create` refuses to write to it, and `prune`, `snapshot purge`, `snapshot remove` and `snapshot list` behave as for a destination that cannot be listed. An unplugged volume leaves an empty directory at its mount point, and without the marker that directory would read as a destination whose snapshots are all gone. `remote nuke` deletes the marker last.
|
||||
|
||||
## Blobs Directory (`blobs/`)
|
||||
|
||||
### Structure
|
||||
|
||||
@@ -101,7 +101,8 @@ func TestEntryPruneJSONStdoutIsExactlyOneDocument(t *testing.T) {
|
||||
// writeHermeticPruneConfig builds a config over a temp directory with an
|
||||
// empty destination directory and, if seedStale is set, creates the
|
||||
// index database up front with one snapshot record that has no
|
||||
// counterpart on the destination store. Returns the config path.
|
||||
// counterpart on the destination store, and gives the destination the
|
||||
// marker a backup writes. Returns the config path.
|
||||
func writeHermeticPruneConfig(t *testing.T, seedStale bool) string {
|
||||
t.Helper()
|
||||
|
||||
@@ -131,6 +132,11 @@ func writeHermeticPruneConfig(t *testing.T, seedStale bool) string {
|
||||
|
||||
if seedStale {
|
||||
seedStaleSnapshotRecord(t, indexPath)
|
||||
|
||||
// While the local index holds a snapshot, prune fails on a
|
||||
// destination without the marker a backup writes.
|
||||
require.NoError(t, os.WriteFile(
|
||||
filepath.Join(storeDir, "vaultik-destination.txt"), nil, 0o600))
|
||||
}
|
||||
|
||||
return configPath
|
||||
|
||||
@@ -41,6 +41,11 @@ func TestEntrySnapshotPurgeKeepsMetadataWhenLocalIndexDeleteFails(t *testing.T)
|
||||
// too, so the purge selects it rather than dropping it as stale.
|
||||
seedStaleSnapshotRecord(t, indexPath)
|
||||
|
||||
// While the local index holds a snapshot, purge fails on a
|
||||
// destination without the marker a backup writes.
|
||||
require.NoError(t, os.WriteFile(
|
||||
filepath.Join(dir, "store", "vaultik-destination.txt"), nil, 0o600))
|
||||
|
||||
manifestDir := filepath.Join(dir, "store", "metadata",
|
||||
snapshot.RemoteSnapshotKey(stalePruneSnapshotID))
|
||||
require.NoError(t, os.MkdirAll(manifestDir, 0o750))
|
||||
@@ -59,10 +64,12 @@ func TestEntrySnapshotPurgeKeepsMetadataWhenLocalIndexDeleteFails(t *testing.T)
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, db.Close())
|
||||
|
||||
code, _, _ := runEntry(t, flagConfig, configPath,
|
||||
code, _, stderr := runEntry(t, flagConfig, configPath,
|
||||
cmdSnapshot, cmdPurge, flagOlderThan, "30d", flagForce)
|
||||
|
||||
assert.Equal(t, 1, code)
|
||||
assert.Contains(t, stderr, "simulated index error",
|
||||
"the purge must fail at the local-index delete")
|
||||
assert.FileExists(t, manifestPath,
|
||||
"the snapshot is still in the local index, so its metadata must stay")
|
||||
}
|
||||
|
||||
+108
-71
@@ -284,6 +284,11 @@ func (s *Scanner) Scan(
|
||||
}
|
||||
}
|
||||
|
||||
err = s.recordDirectories(ctx, scanResult.Directories)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Summarize scan phase results and update progress
|
||||
s.summarizeScanPhase(result, filesToProcess)
|
||||
|
||||
@@ -855,14 +860,15 @@ func (s *Scanner) batchInsertFileData(
|
||||
// ScanPhaseResult contains the results of the scan phase
|
||||
type ScanPhaseResult struct {
|
||||
FilesToProcess []*FileToProcess
|
||||
UnchangedFileIDs []types.FileID // IDs of unchanged files to associate with snapshot
|
||||
UnchangedFileIDs []types.FileID // IDs of unchanged files to associate with snapshot
|
||||
Directories []*FileToProcess // Every directory found, to record
|
||||
}
|
||||
|
||||
// scanPhase performs the initial directory scan to identify files to process
|
||||
// It uses the pre-loaded knownFiles map for fast change detection without DB
|
||||
// queries. It also populates existingFiles map for deletion detection.
|
||||
// Returns files needing processing and IDs of unchanged files for snapshot
|
||||
// association.
|
||||
// Returns files needing processing, IDs of unchanged files for snapshot
|
||||
// association, and the directories found.
|
||||
func (s *Scanner) scanPhase(
|
||||
ctx context.Context,
|
||||
path string,
|
||||
@@ -870,8 +876,10 @@ func (s *Scanner) scanPhase(
|
||||
existingFiles map[string]struct{},
|
||||
knownFiles map[string]*database.File,
|
||||
) (*ScanPhaseResult, error) {
|
||||
// Use known file count as estimate for progress (accurate for subsequent backups)
|
||||
estimatedTotal := int64(len(knownFiles))
|
||||
// Use known file count as estimate for progress (accurate for
|
||||
// subsequent backups). The walk does not count directories, so the
|
||||
// estimate leaves them out.
|
||||
estimatedTotal := countNonDirectories(knownFiles)
|
||||
|
||||
collector := &scanCollector{}
|
||||
|
||||
@@ -904,7 +912,7 @@ func (s *Scanner) scanPhase(
|
||||
|
||||
// Handle symlinks and directories
|
||||
handled, err := s.recordSpecialEntry(
|
||||
filePath, info, existingFiles, collector, result)
|
||||
filePath, info, existingFiles, knownFiles, collector, result)
|
||||
if handled {
|
||||
return err
|
||||
}
|
||||
@@ -918,9 +926,10 @@ func (s *Scanner) scanPhase(
|
||||
existingFiles[filePath] = struct{}{}
|
||||
|
||||
// Check file against in-memory map (no DB query!)
|
||||
file, needsProcessing := s.checkFileInMemory(filePath, info, knownFiles)
|
||||
file := s.buildRegularFileEntry(filePath, info)
|
||||
needsProcessing := s.checkFileInMemory(file, knownFiles)
|
||||
|
||||
filesScanned, changedCount := collector.recordRegularFile(
|
||||
filesScanned, changedCount := collector.recordFile(
|
||||
filePath, info, file, needsProcessing)
|
||||
|
||||
// Update result stats
|
||||
@@ -944,15 +953,32 @@ func (s *Scanner) scanPhase(
|
||||
return &ScanPhaseResult{
|
||||
FilesToProcess: collector.filesToProcess,
|
||||
UnchangedFileIDs: collector.unchangedFileIDs,
|
||||
Directories: collector.directories,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// countNonDirectories returns how many of files are not directories.
|
||||
func countNonDirectories(files map[string]*database.File) int64 {
|
||||
var count int64
|
||||
|
||||
for _, f := range files {
|
||||
if f.Mode&uint32(os.ModeDir) == 0 {
|
||||
count++
|
||||
}
|
||||
}
|
||||
|
||||
return count
|
||||
}
|
||||
|
||||
// recordSpecialEntry records symlinks and directories (which have no
|
||||
// data to chunk) and reports whether it handled the entry. For a symlink
|
||||
// whose target cannot be read it returns handleWalkError's result.
|
||||
// data to chunk) and reports whether it handled the entry. A symlink is
|
||||
// counted and checked for changes as a regular file is; a directory is
|
||||
// set aside to be recorded and is not counted. For a symlink whose
|
||||
// target cannot be read it returns handleWalkError's result.
|
||||
func (s *Scanner) recordSpecialEntry(
|
||||
filePath string, info os.FileInfo,
|
||||
existingFiles map[string]struct{},
|
||||
knownFiles map[string]*database.File,
|
||||
collector *scanCollector, result *ScanResult,
|
||||
) (bool, error) {
|
||||
// Handle symlinks
|
||||
@@ -963,8 +989,9 @@ func (s *Scanner) recordSpecialEntry(
|
||||
}
|
||||
|
||||
existingFiles[filePath] = struct{}{}
|
||||
collector.addToProcess(filePath, info, file)
|
||||
s.updateScanEntryStats(result, true, info)
|
||||
needsProcessing := s.checkFileInMemory(file, knownFiles)
|
||||
collector.recordFile(filePath, info, file, needsProcessing)
|
||||
s.updateScanEntryStats(result, needsProcessing, info)
|
||||
|
||||
return true, nil
|
||||
}
|
||||
@@ -974,7 +1001,7 @@ func (s *Scanner) recordSpecialEntry(
|
||||
if info.IsDir() {
|
||||
file := s.buildDirectoryEntry(filePath, info)
|
||||
existingFiles[filePath] = struct{}{}
|
||||
collector.addToProcess(filePath, info, file)
|
||||
collector.addDirectory(filePath, info, file)
|
||||
|
||||
return true, nil
|
||||
}
|
||||
@@ -1004,28 +1031,29 @@ type scanCollector struct {
|
||||
mu sync.Mutex
|
||||
filesToProcess []*FileToProcess
|
||||
unchangedFileIDs []types.FileID // Just IDs - no new records needed
|
||||
directories []*FileToProcess
|
||||
filesScanned int64
|
||||
}
|
||||
|
||||
// addToProcess queues a file entry for processing.
|
||||
func (c *scanCollector) addToProcess(
|
||||
// addDirectory sets a directory aside to be recorded. It is not counted
|
||||
// as a file.
|
||||
func (c *scanCollector) addDirectory(
|
||||
path string, info os.FileInfo, file *database.File,
|
||||
) {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
|
||||
c.filesToProcess = append(c.filesToProcess, &FileToProcess{
|
||||
c.directories = append(c.directories, &FileToProcess{
|
||||
Path: path,
|
||||
FileInfo: info,
|
||||
File: file,
|
||||
})
|
||||
c.filesScanned++
|
||||
}
|
||||
|
||||
// recordRegularFile records a regular file as changed (to process) or
|
||||
// unchanged (snapshot association only) and returns the running scanned
|
||||
// and changed counts.
|
||||
func (c *scanCollector) recordRegularFile(
|
||||
// recordFile records a regular file or symlink as changed (to process)
|
||||
// or unchanged (snapshot association only) and returns the running
|
||||
// scanned and changed counts.
|
||||
func (c *scanCollector) recordFile(
|
||||
path string, info os.FileInfo, file *database.File, needsProcessing bool,
|
||||
) (int64, int) {
|
||||
c.mu.Lock()
|
||||
@@ -1128,7 +1156,28 @@ func (s *Scanner) printScanProgressLine(
|
||||
}
|
||||
}
|
||||
|
||||
// buildSymlinkEntry creates a File record for a symlink.
|
||||
// buildRegularFileEntry creates a File record for a regular file, with
|
||||
// no ID: checkFileInMemory sets it.
|
||||
func (s *Scanner) buildRegularFileEntry(path string, info os.FileInfo) *database.File {
|
||||
var uid, gid uint32
|
||||
if stat, ok := info.Sys().(*syscall.Stat_t); ok {
|
||||
uid = stat.Uid
|
||||
gid = stat.Gid
|
||||
}
|
||||
|
||||
return &database.File{
|
||||
Path: types.FilePath(path),
|
||||
SourcePath: types.SourcePath(s.currentSourcePath),
|
||||
MTime: info.ModTime(),
|
||||
Size: info.Size(),
|
||||
Mode: uint32(info.Mode()),
|
||||
UID: uid,
|
||||
GID: gid,
|
||||
}
|
||||
}
|
||||
|
||||
// buildSymlinkEntry creates a File record for a symlink, with no ID:
|
||||
// checkFileInMemory sets it.
|
||||
func (s *Scanner) buildSymlinkEntry(
|
||||
path string, info os.FileInfo,
|
||||
) (*database.File, error) {
|
||||
@@ -1144,7 +1193,6 @@ func (s *Scanner) buildSymlinkEntry(
|
||||
}
|
||||
|
||||
return &database.File{
|
||||
ID: types.NewFileID(),
|
||||
Path: types.FilePath(path),
|
||||
SourcePath: types.SourcePath(s.currentSourcePath),
|
||||
MTime: info.ModTime(),
|
||||
@@ -1176,6 +1224,22 @@ func (s *Scanner) buildDirectoryEntry(path string, info os.FileInfo) *database.F
|
||||
}
|
||||
}
|
||||
|
||||
// recordDirectories records every directory the walk found. A directory
|
||||
// has no data to back up, so it is not among the files counted and
|
||||
// backed up.
|
||||
func (s *Scanner) recordDirectories(
|
||||
ctx context.Context, directories []*FileToProcess,
|
||||
) error {
|
||||
for _, dir := range directories {
|
||||
err := s.recordNonRegularFile(ctx, dir)
|
||||
if err != nil {
|
||||
return fmt.Errorf("recording directory %s: %w", dir.Path, err)
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// recordNonRegularFile writes a symlink or directory entry to the database
|
||||
// and associates it with the current snapshot. No chunking is performed.
|
||||
func (s *Scanner) recordNonRegularFile(ctx context.Context, ftp *FileToProcess) error {
|
||||
@@ -1189,62 +1253,36 @@ func (s *Scanner) recordNonRegularFile(ctx context.Context, ftp *FileToProcess)
|
||||
})
|
||||
}
|
||||
|
||||
// checkFileInMemory checks if a file needs processing using the in-memory map
|
||||
// checkFileInMemory reports whether a regular file or symlink needs
|
||||
// processing, and gives file the ID of its known record, or a new ID for
|
||||
// a new path.
|
||||
// No database access is performed - this is purely CPU/memory work
|
||||
func (s *Scanner) checkFileInMemory(
|
||||
path string, info os.FileInfo, knownFiles map[string]*database.File,
|
||||
) (*database.File, bool) {
|
||||
var uid, gid uint32
|
||||
if stat, ok := info.Sys().(*syscall.Stat_t); ok {
|
||||
uid = stat.Uid
|
||||
gid = stat.Gid
|
||||
}
|
||||
|
||||
// Check against in-memory map first to get existing ID if available
|
||||
existingFile, exists := knownFiles[path]
|
||||
|
||||
// Create file record with ID set upfront
|
||||
// For new files, generate UUID immediately so it's available for chunk associations
|
||||
// For existing files, reuse the existing ID
|
||||
var fileID types.FileID
|
||||
if exists {
|
||||
fileID = existingFile.ID
|
||||
} else {
|
||||
fileID = types.NewFileID()
|
||||
}
|
||||
|
||||
file := &database.File{
|
||||
ID: fileID,
|
||||
Path: types.FilePath(path),
|
||||
SourcePath: types.SourcePath(s.currentSourcePath),
|
||||
MTime: info.ModTime(),
|
||||
Size: info.Size(),
|
||||
Mode: uint32(info.Mode()),
|
||||
UID: uid,
|
||||
GID: gid,
|
||||
}
|
||||
file *database.File, knownFiles map[string]*database.File,
|
||||
) bool {
|
||||
existingFile, exists := knownFiles[file.Path.String()]
|
||||
|
||||
// New file - needs processing
|
||||
if !exists {
|
||||
return file, true
|
||||
file.ID = types.NewFileID()
|
||||
|
||||
return true
|
||||
}
|
||||
|
||||
file.ID = existingFile.ID
|
||||
|
||||
// No uploaded blob holds one of its chunks (see loadFilesToRechunk)
|
||||
if _, rechunk := s.filesToRechunk[fileID]; rechunk {
|
||||
return file, true
|
||||
if _, rechunk := s.filesToRechunk[file.ID]; rechunk {
|
||||
return true
|
||||
}
|
||||
|
||||
// Check if file has changed
|
||||
if existingFile.Size != file.Size ||
|
||||
return existingFile.Size != file.Size ||
|
||||
!existingFile.MTime.Equal(file.MTime) ||
|
||||
existingFile.Mode != file.Mode ||
|
||||
existingFile.UID != file.UID ||
|
||||
existingFile.GID != file.GID {
|
||||
return file, true
|
||||
}
|
||||
|
||||
// File unchanged
|
||||
return file, false
|
||||
existingFile.GID != file.GID ||
|
||||
existingFile.LinkTarget != file.LinkTarget
|
||||
}
|
||||
|
||||
// batchAddFilesToSnapshot adds existing file IDs to the snapshot association table
|
||||
@@ -1371,17 +1409,16 @@ func (s *Scanner) processPhase(
|
||||
return s.finalizeProcessPhase(ctx, result)
|
||||
}
|
||||
|
||||
// processFileWithErrorHandling records a directory or symlink, or wraps
|
||||
// processFileWithErrorHandling records a symlink, or wraps
|
||||
// processFileStreaming for a regular file with error recovery for deleted
|
||||
// files and skip-errors mode. Returns (skipped, error).
|
||||
func (s *Scanner) processFileWithErrorHandling(
|
||||
ctx context.Context, fileToProcess *FileToProcess, result *ScanResult,
|
||||
) (bool, error) {
|
||||
// A directory or symlink has no data to open or read; it is only
|
||||
// recorded in the local index. An error recording it stops the run
|
||||
// even under --skip-errors, or the snapshot would complete without it.
|
||||
mode := os.FileMode(fileToProcess.File.Mode)
|
||||
if mode&os.ModeSymlink != 0 || mode.IsDir() {
|
||||
// A symlink has no data to open or read; it is only recorded in the
|
||||
// local index. An error recording it stops the run even under
|
||||
// --skip-errors, or the snapshot would complete without it.
|
||||
if os.FileMode(fileToProcess.File.Mode)&os.ModeSymlink != 0 {
|
||||
err := s.recordNonRegularFile(ctx, fileToProcess)
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("processing file %s: %w", fileToProcess.Path, err)
|
||||
|
||||
@@ -0,0 +1,28 @@
|
||||
//nolint:testpackage // exercises the unexported countNonDirectories helper
|
||||
package snapshot
|
||||
|
||||
import (
|
||||
"os"
|
||||
"testing"
|
||||
|
||||
"sneak.berlin/go/vaultik/internal/database"
|
||||
)
|
||||
|
||||
// The walk counts regular files and symlinks, not directories, so the
|
||||
// estimate of its count made from the local index's records leaves the
|
||||
// directories out. See https://git.eeqj.de/sneak/vaultik/issues/305.
|
||||
func TestCountNonDirectories(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
known := map[string]*database.File{
|
||||
"/src": {Mode: uint32(os.ModeDir | 0o755)},
|
||||
"/src/sub": {Mode: uint32(os.ModeDir | 0o755)},
|
||||
"/src/a.txt": {Mode: 0o644},
|
||||
"/src/link": {Mode: uint32(os.ModeSymlink | 0o777), LinkTarget: "a.txt"},
|
||||
}
|
||||
|
||||
got := countNonDirectories(known)
|
||||
if got != 2 {
|
||||
t.Errorf("countNonDirectories = %d, want 2", got)
|
||||
}
|
||||
}
|
||||
@@ -21,9 +21,13 @@ type FileStorer struct {
|
||||
// NewFileStorer creates a new filesystem storage backend.
|
||||
//
|
||||
// Construction is intentionally cheap and does not touch the filesystem.
|
||||
// The basePath is recorded; the directory is created lazily on first
|
||||
// write. Write operations (Put/PutWithProgress) call MkdirAll for the
|
||||
// per-blob parent directory, which also covers basePath on first use.
|
||||
// The backend never creates basePath: the user does, so that when
|
||||
// basePath is a directory on an unplugged volume, a write fails instead
|
||||
// of landing on the disk under the mount point. This does not hold when
|
||||
// basePath is the mount point itself, which stays behind when its volume
|
||||
// is unplugged.
|
||||
// Put and PutWithProgress fail on a missing basePath and create the
|
||||
// directories below it as needed.
|
||||
// Get and Stat report a key under a missing basePath as ErrNotFound.
|
||||
// List and ListStream fail on a missing basePath, because listing it as
|
||||
// an empty store would make `prune` drop every local snapshot record.
|
||||
@@ -299,7 +303,19 @@ func (f *FileStorer) writeAtomic(
|
||||
path := f.fullPath(key)
|
||||
dir := filepath.Dir(path)
|
||||
|
||||
err := f.fs.MkdirAll(dir, storageDirPerm)
|
||||
// Checked on every write, so a destination that disappears part-way
|
||||
// through a backup fails it rather than being recreated by MkdirAll.
|
||||
_, err := f.fs.Stat(f.basePath)
|
||||
if os.IsNotExist(err) {
|
||||
return fmt.Errorf("destination directory does not exist; "+
|
||||
"create it, or mount the volume it is on: %w", err)
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
return fmt.Errorf("checking destination directory: %w", err)
|
||||
}
|
||||
|
||||
err = f.fs.MkdirAll(dir, storageDirPerm)
|
||||
if err != nil {
|
||||
return fmt.Errorf("creating directories: %w", err)
|
||||
}
|
||||
|
||||
@@ -292,3 +292,43 @@ func TestFileStorerDeletePartialUploadsThroughSymlinkedDirectory(t *testing.T) {
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestFileStorerPutMissingDestination checks that a write fails when the
|
||||
// destination directory has gone, even after earlier writes to it
|
||||
// succeeded, and that it does not create the directory again.
|
||||
func TestFileStorerPutMissingDestination(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
ctx := context.Background()
|
||||
base := filepath.Join(t.TempDir(), "usbstick")
|
||||
|
||||
err := os.Mkdir(base, 0o750)
|
||||
if err != nil {
|
||||
t.Fatalf("Mkdir: %v", err)
|
||||
}
|
||||
|
||||
s, err := storage.NewFileStorer(base)
|
||||
if err != nil {
|
||||
t.Fatalf("NewFileStorer: %v", err)
|
||||
}
|
||||
|
||||
err = s.Put(ctx, "blobs/aa/bb/first", strings.NewReader("first"))
|
||||
if err != nil {
|
||||
t.Fatalf("first Put: %v", err)
|
||||
}
|
||||
|
||||
err = os.Rename(base, base+"-unplugged")
|
||||
if err != nil {
|
||||
t.Fatalf("Rename: %v", err)
|
||||
}
|
||||
|
||||
err = s.Put(ctx, "blobs/aa/bb/second", strings.NewReader("second"))
|
||||
if !errors.Is(err, fs.ErrNotExist) {
|
||||
t.Errorf("Put = %v, want a not-exist error", err)
|
||||
}
|
||||
|
||||
_, err = os.Stat(base)
|
||||
if !errors.Is(err, fs.ErrNotExist) {
|
||||
t.Errorf("Put created the destination directory again (Stat = %v)", err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -41,7 +41,8 @@ type Info struct {
|
||||
// directory for filesystem).
|
||||
type Storer interface {
|
||||
// Put stores data at the specified key.
|
||||
// Parent directories are created automatically for filesystem backends.
|
||||
// The filesystem backend fails when its destination directory does
|
||||
// not exist, and creates the directories below it as needed.
|
||||
Put(ctx context.Context, key string, data io.Reader) error
|
||||
|
||||
// PutWithProgress stores data with progress reporting.
|
||||
|
||||
@@ -137,7 +137,7 @@ func TestBackupAfterRemovingNewestSnapshotRestoresChangedFile(t *testing.T) {
|
||||
fs := afero.NewOsFs()
|
||||
tempDir := t.TempDir()
|
||||
dataDir := filepath.Join(tempDir, "src")
|
||||
storeDir := filepath.Join(tempDir, "remote")
|
||||
storeDir := t.TempDir()
|
||||
restoreDir := filepath.Join(tempDir, "restored")
|
||||
dbPath := filepath.Join(tempDir, "index.sqlite")
|
||||
changedPath := filepath.Join(dataDir, "appended.bin")
|
||||
@@ -184,7 +184,7 @@ func TestBackupAfterInterruptedRunRestoresChangedFile(t *testing.T) {
|
||||
fs := afero.NewOsFs()
|
||||
tempDir := t.TempDir()
|
||||
dataDir := filepath.Join(tempDir, "src")
|
||||
storeDir := filepath.Join(tempDir, "remote")
|
||||
storeDir := t.TempDir()
|
||||
restoreDir := filepath.Join(tempDir, "restored")
|
||||
dbPath := filepath.Join(tempDir, "index.sqlite")
|
||||
changedPath := filepath.Join(dataDir, "appended.bin")
|
||||
|
||||
@@ -35,7 +35,7 @@ func TestDeepVerifyAcceptsHealthyAndRejectsCorruptBlob(t *testing.T) {
|
||||
tempDir := t.TempDir()
|
||||
|
||||
dataDir := filepath.Join(tempDir, "source")
|
||||
storeDir := filepath.Join(tempDir, "remote")
|
||||
storeDir := t.TempDir()
|
||||
dbPath := filepath.Join(tempDir, "index.sqlite")
|
||||
|
||||
chunkSize := int64(64 * 1024)
|
||||
@@ -94,7 +94,7 @@ func TestDeepVerifyJSONCountsBlobs(t *testing.T) {
|
||||
|
||||
fs := afero.NewOsFs()
|
||||
ctx := context.Background()
|
||||
storeDir := filepath.Join(t.TempDir(), "remote")
|
||||
storeDir := t.TempDir()
|
||||
|
||||
cfg, storer, snapshotID, keys := backUpSeveralBlobs(ctx, t, storeDir)
|
||||
|
||||
@@ -151,7 +151,7 @@ func TestDeepVerifyJSONCountsWrongSizeBlob(t *testing.T) {
|
||||
|
||||
fs := afero.NewOsFs()
|
||||
ctx := context.Background()
|
||||
storeDir := filepath.Join(t.TempDir(), "remote")
|
||||
storeDir := t.TempDir()
|
||||
|
||||
cfg, storer, snapshotID, keys := backUpSeveralBlobs(ctx, t, storeDir)
|
||||
|
||||
@@ -183,7 +183,7 @@ func TestDeepVerifyStopsOnBlobThatCannotBeChecked(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
ctx := context.Background()
|
||||
storeDir := filepath.Join(t.TempDir(), "remote")
|
||||
storeDir := t.TempDir()
|
||||
|
||||
cfg, storer, snapshotID, keys := backUpSeveralBlobs(ctx, t, storeDir)
|
||||
failing := statFailingStorer{Storer: storer, failKeys: []string{keys[0]}}
|
||||
|
||||
@@ -0,0 +1,90 @@
|
||||
package vaultik
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"sneak.berlin/go/vaultik/internal/storage"
|
||||
)
|
||||
|
||||
// destinationMarkerKey is the object every backup writes at the root of
|
||||
// the backup destination store. While the local index holds a snapshot
|
||||
// stored there, a destination without it is treated as not present: an
|
||||
// unplugged volume leaves an empty directory at its mount point, which
|
||||
// would otherwise read as a store whose snapshots are all gone.
|
||||
const destinationMarkerKey = "vaultik-destination.txt"
|
||||
|
||||
// destinationMarkerText is the whole content of the marker. It names no
|
||||
// host, path or time.
|
||||
const destinationMarkerText = "This is a vaultik backup destination store.\n"
|
||||
|
||||
// errDestinationNotPresent is returned when the local index holds a
|
||||
// snapshot stored on the backup destination store and the store has no
|
||||
// marker.
|
||||
var errDestinationNotPresent = errors.New("backup destination store is not present")
|
||||
|
||||
// checkDestinationMarker returns errDestinationNotPresent when the local
|
||||
// index holds a snapshot stored on the backup destination store and the
|
||||
// store has no marker. With no snapshot in the local index, as on a
|
||||
// first backup or on another machine, there is nothing to check; nor
|
||||
// when the local index is bound to a different destination than the
|
||||
// configured one, since its snapshots are stored there.
|
||||
func (v *Vaultik) checkDestinationMarker() error {
|
||||
ids, err := v.localSnapshotIDs()
|
||||
if err != nil {
|
||||
return fmt.Errorf("listing local snapshots: %w", err)
|
||||
}
|
||||
|
||||
if len(ids) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Config is nil only in tests, which then record no binding (see
|
||||
// EnsureStorageBinding).
|
||||
if v.Config != nil {
|
||||
stored, configured, err := v.storageBinding()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if stored != "" && stored != configured {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
_, err = v.Storage.Stat(v.ctx, destinationMarkerKey)
|
||||
if errors.Is(err, storage.ErrNotFound) {
|
||||
return fmt.Errorf("%w: %s has no %s, but the local index holds "+
|
||||
"snapshots stored there; mount the volume if it is removable, "+
|
||||
"or if the destination was emptied on purpose, run "+
|
||||
"'vaultik database delete' and back up again",
|
||||
errDestinationNotPresent, v.Storage.Info().Location,
|
||||
destinationMarkerKey)
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
return fmt.Errorf("checking for %s: %w", destinationMarkerKey, err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// markDestination checks the backup destination store with
|
||||
// checkDestinationMarker and then writes the marker, so a first backup
|
||||
// marks its destination and a later one writes the same text again.
|
||||
func (v *Vaultik) markDestination() error {
|
||||
err := v.checkDestinationMarker()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
err = v.Storage.Put(v.ctx, destinationMarkerKey,
|
||||
strings.NewReader(destinationMarkerText))
|
||||
if err != nil {
|
||||
return fmt.Errorf("writing %s to the backup destination store: %w",
|
||||
destinationMarkerKey, err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,278 @@
|
||||
package vaultik_test
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
"sneak.berlin/go/vaultik/internal/database"
|
||||
"sneak.berlin/go/vaultik/internal/log"
|
||||
"sneak.berlin/go/vaultik/internal/storage"
|
||||
"sneak.berlin/go/vaultik/internal/vaultik"
|
||||
)
|
||||
|
||||
// These tests cover https://git.eeqj.de/sneak/vaultik/issues/239: a
|
||||
// file:// destination at a volume's mount point leaves an empty directory
|
||||
// behind when the volume is unplugged. While the local index holds
|
||||
// snapshots, a destination without vaultik-destination.txt is not present,
|
||||
// so no command may write to it or conclude from it that the local
|
||||
// snapshots are gone.
|
||||
|
||||
// markerFile is the name of the file a backup writes at the root of the
|
||||
// destination.
|
||||
const markerFile = "vaultik-destination.txt"
|
||||
|
||||
// backUpThenUnplugMountPoint backs up to a file:// destination, then
|
||||
// moves the destination directory away and leaves an empty directory in
|
||||
// its place, as unplugging a volume mounted there would. It returns the
|
||||
// path of that empty directory.
|
||||
func backUpThenUnplugMountPoint(
|
||||
ctx context.Context, t *testing.T,
|
||||
) (*vaultik.Vaultik, *database.Repositories, *bytes.Buffer, string) {
|
||||
t.Helper()
|
||||
|
||||
storeDir := filepath.Join(t.TempDir(), "mnt")
|
||||
v, repos, out := backUpToFileDestination(ctx, t, storeDir)
|
||||
|
||||
require.NoError(t, os.Rename(storeDir, storeDir+"-unplugged"))
|
||||
require.NoError(t, os.Mkdir(storeDir, 0o750))
|
||||
|
||||
return v, repos, out, storeDir
|
||||
}
|
||||
|
||||
// assertUntouched checks that the local index still holds its one
|
||||
// snapshot and that nothing was written to the empty mount point.
|
||||
func assertUntouched(
|
||||
ctx context.Context, t *testing.T,
|
||||
repos *database.Repositories, storeDir string,
|
||||
) {
|
||||
t.Helper()
|
||||
|
||||
snapshots, err := repos.Snapshots.ListRecent(ctx, listRecentTestLimit)
|
||||
require.NoError(t, err)
|
||||
assert.Len(t, snapshots, 1, "no local snapshot record may be deleted")
|
||||
|
||||
entries, err := os.ReadDir(storeDir)
|
||||
require.NoError(t, err)
|
||||
assert.Empty(t, entries, "nothing may be written to the empty mount point")
|
||||
}
|
||||
|
||||
// TestFirstBackupWritesMarker checks that a backup writes the marker
|
||||
// file at the root of the destination.
|
||||
func TestFirstBackupWritesMarker(t *testing.T) {
|
||||
log.Initialize(log.Config{})
|
||||
t.Parallel()
|
||||
|
||||
storeDir := t.TempDir()
|
||||
backUpToFileDestination(context.Background(), t, storeDir)
|
||||
|
||||
assert.FileExists(t, filepath.Join(storeDir, markerFile))
|
||||
}
|
||||
|
||||
// TestBackupRefusesDestinationWithoutMarker checks that snapshot create
|
||||
// --prune writes nothing and drops no local snapshot record when the
|
||||
// destination has no marker.
|
||||
func TestBackupRefusesDestinationWithoutMarker(t *testing.T) {
|
||||
log.Initialize(log.Config{})
|
||||
t.Parallel()
|
||||
|
||||
ctx := context.Background()
|
||||
v, repos, _, storeDir := backUpThenUnplugMountPoint(ctx, t)
|
||||
|
||||
err := v.CreateSnapshot(&vaultik.SnapshotCreateOptions{
|
||||
Cron: true,
|
||||
Prune: true,
|
||||
})
|
||||
require.ErrorContains(t, err, markerFile)
|
||||
|
||||
assertUntouched(ctx, t, repos, storeDir)
|
||||
}
|
||||
|
||||
// TestBackupChecksMarkerBeforeDroppingIncompleteSnapshots checks that
|
||||
// snapshot create writes nothing when the local index holds only an
|
||||
// incomplete snapshot and the destination has no marker. The backup drops
|
||||
// incomplete snapshots before it scans; checked after that, the local
|
||||
// index would hold none and nothing would be checked.
|
||||
func TestBackupChecksMarkerBeforeDroppingIncompleteSnapshots(t *testing.T) {
|
||||
log.Initialize(log.Config{})
|
||||
t.Parallel()
|
||||
|
||||
ctx := context.Background()
|
||||
storeDir := t.TempDir()
|
||||
v, repos := newFileDestinationVaultik(ctx, t, storeDir)
|
||||
|
||||
require.NoError(t, repos.Snapshots.Create(ctx, nil, &database.Snapshot{
|
||||
ID: "host_first_2026-01-02T03:04:05Z",
|
||||
Hostname: "host",
|
||||
StartedAt: time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC),
|
||||
}))
|
||||
|
||||
err := backUp(v, "first")
|
||||
require.ErrorContains(t, err, markerFile)
|
||||
|
||||
assertUntouched(ctx, t, repos, storeDir)
|
||||
}
|
||||
|
||||
// TestPruneRefusesDestinationWithoutMarker checks that prune fails and
|
||||
// deletes no local snapshot record when the destination has no marker.
|
||||
func TestPruneRefusesDestinationWithoutMarker(t *testing.T) {
|
||||
log.Initialize(log.Config{})
|
||||
t.Parallel()
|
||||
|
||||
ctx := context.Background()
|
||||
v, repos, _, storeDir := backUpThenUnplugMountPoint(ctx, t)
|
||||
|
||||
err := v.Prune(&vaultik.PruneOptions{Force: true})
|
||||
require.ErrorContains(t, err, markerFile)
|
||||
|
||||
assertUntouched(ctx, t, repos, storeDir)
|
||||
}
|
||||
|
||||
// TestPurgeRefusesDestinationWithoutMarker checks that snapshot purge
|
||||
// fails and deletes no local snapshot record when the destination has no
|
||||
// marker.
|
||||
func TestPurgeRefusesDestinationWithoutMarker(t *testing.T) {
|
||||
log.Initialize(log.Config{})
|
||||
t.Parallel()
|
||||
|
||||
ctx := context.Background()
|
||||
v, repos, _, storeDir := backUpThenUnplugMountPoint(ctx, t)
|
||||
|
||||
err := v.PurgeSnapshotsWithOptions(&vaultik.SnapshotPurgeOptions{
|
||||
KeepLatest: true,
|
||||
Force: true,
|
||||
})
|
||||
require.ErrorContains(t, err, markerFile)
|
||||
|
||||
assertUntouched(ctx, t, repos, storeDir)
|
||||
}
|
||||
|
||||
// TestRemoveSnapshotWarnsWhenDestinationHasNoMarker checks that snapshot
|
||||
// remove warns that the metadata could not be removed from a destination
|
||||
// without a marker, instead of reporting that it was.
|
||||
func TestRemoveSnapshotWarnsWhenDestinationHasNoMarker(t *testing.T) {
|
||||
log.Initialize(log.Config{})
|
||||
t.Parallel()
|
||||
|
||||
ctx := context.Background()
|
||||
v, repos, out, _ := backUpThenUnplugMountPoint(ctx, t)
|
||||
|
||||
result, err := v.RemoveSnapshot(localSnapshotID(ctx, t, repos, "first"),
|
||||
&vaultik.RemoveOptions{Force: true})
|
||||
require.NoError(t, err)
|
||||
|
||||
assert.False(t, result.RemoteRemoved)
|
||||
assert.Contains(t, out.String(),
|
||||
"Could not remove snapshot metadata from remote")
|
||||
assert.Contains(t, out.String(), markerFile)
|
||||
assert.NotContains(t, out.String(),
|
||||
"Removed snapshot metadata from remote storage")
|
||||
}
|
||||
|
||||
// TestRemoveSnapshotOnlyOnDestinationFailsWithoutMarker checks that
|
||||
// snapshot remove, for a snapshot that is only on the destination, fails
|
||||
// saying the destination is not present when it has no marker, instead
|
||||
// of saying the snapshot does not exist.
|
||||
func TestRemoveSnapshotOnlyOnDestinationFailsWithoutMarker(t *testing.T) {
|
||||
log.Initialize(log.Config{})
|
||||
t.Parallel()
|
||||
|
||||
ctx := context.Background()
|
||||
storeDir := filepath.Join(t.TempDir(), "mnt")
|
||||
v, repos, _ := backUpToFileDestination(ctx, t, storeDir)
|
||||
require.NoError(t, backUp(v, "second"))
|
||||
|
||||
id := localSnapshotID(ctx, t, repos, "first")
|
||||
_, err := v.RemoveSnapshot(id,
|
||||
&vaultik.RemoveOptions{Force: true, LocalOnly: true})
|
||||
require.NoError(t, err)
|
||||
|
||||
// Unplug the volume: an empty directory stays at its mount point.
|
||||
require.NoError(t, os.Rename(storeDir, storeDir+"-unplugged"))
|
||||
require.NoError(t, os.Mkdir(storeDir, 0o750))
|
||||
|
||||
_, err = v.RemoveSnapshot(id, &vaultik.RemoveOptions{Force: true})
|
||||
require.ErrorContains(t, err, markerFile)
|
||||
require.ErrorContains(t, err, "mount the volume")
|
||||
|
||||
assertUntouched(ctx, t, repos, storeDir)
|
||||
}
|
||||
|
||||
// TestListSnapshotsWarnsWhenDestinationHasNoMarker checks that snapshot
|
||||
// list warns and shows the local index alone when the destination has no
|
||||
// marker, without reporting the local snapshot as missing from it.
|
||||
func TestListSnapshotsWarnsWhenDestinationHasNoMarker(t *testing.T) {
|
||||
log.Initialize(log.Config{})
|
||||
t.Parallel()
|
||||
|
||||
ctx := context.Background()
|
||||
v, repos, out, _ := backUpThenUnplugMountPoint(ctx, t)
|
||||
id := localSnapshotID(ctx, t, repos, "first")
|
||||
|
||||
require.NoError(t, v.ListSnapshots(false))
|
||||
|
||||
assert.Contains(t, out.String(), "Could not list backup destination store")
|
||||
assert.Contains(t, out.String(), markerFile)
|
||||
assert.Contains(t, out.String(), id)
|
||||
assert.NotContains(t, out.String(), "not found in backup destination store")
|
||||
}
|
||||
|
||||
// TestListSnapshotsAfterStorageURLChangeToEmptyDirectory checks that
|
||||
// snapshot list, after storage_url is changed to an empty directory,
|
||||
// warns only that the local index is bound to a different destination:
|
||||
// the local snapshots are stored at the old one, so the new one's lack
|
||||
// of a marker says nothing. With --json the local row is not on the new
|
||||
// destination, so remote_present is false, not unknown.
|
||||
func TestListSnapshotsAfterStorageURLChangeToEmptyDirectory(t *testing.T) {
|
||||
log.Initialize(log.Config{})
|
||||
t.Parallel()
|
||||
|
||||
ctx := context.Background()
|
||||
v, repos, out := backUpToFileDestination(ctx, t, t.TempDir())
|
||||
id := localSnapshotID(ctx, t, repos, "first")
|
||||
|
||||
newDir := t.TempDir()
|
||||
store, err := storage.NewFileStorer(newDir)
|
||||
require.NoError(t, err)
|
||||
|
||||
v.Storage = store
|
||||
v.Config.StorageURL = "file://" + newDir
|
||||
|
||||
require.NoError(t, v.ListSnapshots(false))
|
||||
|
||||
assert.Contains(t, out.String(), id)
|
||||
assert.Contains(t, out.String(),
|
||||
"local index is bound to a different backup destination")
|
||||
assert.NotContains(t, out.String(), "Could not list backup destination store")
|
||||
assert.NotContains(t, out.String(), markerFile)
|
||||
|
||||
out.Reset()
|
||||
require.NoError(t, v.ListSnapshots(true))
|
||||
|
||||
rows := decodeListJSON(t, out.String())
|
||||
require.Len(t, rows, 1)
|
||||
require.NotNil(t, rows[0].RemotePresent)
|
||||
assert.False(t, *rows[0].RemotePresent)
|
||||
}
|
||||
|
||||
// TestBackupAfterNukeRemoteSucceeds checks that remote nuke leaves the
|
||||
// destination ready for a new first backup, which writes the marker
|
||||
// again.
|
||||
func TestBackupAfterNukeRemoteSucceeds(t *testing.T) {
|
||||
log.Initialize(log.Config{})
|
||||
t.Parallel()
|
||||
|
||||
storeDir := t.TempDir()
|
||||
v, _, _ := backUpToFileDestination(context.Background(), t, storeDir)
|
||||
|
||||
require.NoError(t, v.NukeRemote(true))
|
||||
assert.NoFileExists(t, filepath.Join(storeDir, markerFile))
|
||||
|
||||
require.NoError(t, backUp(v, "second"))
|
||||
assert.FileExists(t, filepath.Join(storeDir, markerFile))
|
||||
}
|
||||
@@ -192,7 +192,7 @@ func assertRestoreRejectsDamagedBlob(
|
||||
fs := afero.NewOsFs()
|
||||
tempDir := t.TempDir()
|
||||
dataDir := filepath.Join(tempDir, "src")
|
||||
storeDir := filepath.Join(tempDir, "remote")
|
||||
storeDir := t.TempDir()
|
||||
restoreDir := filepath.Join(tempDir, "restored")
|
||||
dbPath := filepath.Join(tempDir, "index.sqlite")
|
||||
|
||||
@@ -238,7 +238,7 @@ func TestDeepVerifyCatchesLyingBackend(t *testing.T) {
|
||||
fs := afero.NewOsFs()
|
||||
tempDir := t.TempDir()
|
||||
dataDir := filepath.Join(tempDir, "src")
|
||||
storeDir := filepath.Join(tempDir, "remote")
|
||||
storeDir := t.TempDir()
|
||||
dbPath := filepath.Join(tempDir, "index.sqlite")
|
||||
|
||||
ctx := context.Background()
|
||||
@@ -290,7 +290,7 @@ func TestInterruptedBlobUploadRecordsNoUploadedBlob(t *testing.T) {
|
||||
fs := afero.NewOsFs()
|
||||
tempDir := t.TempDir()
|
||||
dataDir := filepath.Join(tempDir, "src")
|
||||
storeDir := filepath.Join(tempDir, "remote")
|
||||
storeDir := t.TempDir()
|
||||
dbPath := filepath.Join(tempDir, "index.sqlite")
|
||||
|
||||
ctx := context.Background()
|
||||
@@ -298,10 +298,6 @@ func TestInterruptedBlobUploadRecordsNoUploadedBlob(t *testing.T) {
|
||||
|
||||
writeFaultSourceTree(t, fs, dataDir)
|
||||
|
||||
// No upload succeeds, so nothing creates the destination directory.
|
||||
// It must exist for the listing below to show that no blob survived.
|
||||
require.NoError(t, os.Mkdir(storeDir, 0o750))
|
||||
|
||||
inner, err := storage.NewFileStorer(storeDir)
|
||||
require.NoError(t, err)
|
||||
|
||||
@@ -365,7 +361,7 @@ func TestBackupRetryAfterInterruptedUploadIsRestorable(t *testing.T) {
|
||||
fs := afero.NewOsFs()
|
||||
tempDir := t.TempDir()
|
||||
dataDir := filepath.Join(tempDir, "src")
|
||||
storeDir := filepath.Join(tempDir, "remote")
|
||||
storeDir := t.TempDir()
|
||||
restoreDir := filepath.Join(tempDir, "restored")
|
||||
dbPath := filepath.Join(tempDir, "index.sqlite")
|
||||
|
||||
@@ -429,7 +425,7 @@ func TestBackupSurvivesMetadataExportInterruption(t *testing.T) {
|
||||
fs := afero.NewOsFs()
|
||||
tempDir := t.TempDir()
|
||||
dataDir := filepath.Join(tempDir, "src")
|
||||
storeDir := filepath.Join(tempDir, "remote")
|
||||
storeDir := t.TempDir()
|
||||
dbPath := filepath.Join(tempDir, "index.sqlite")
|
||||
|
||||
ctx := context.Background()
|
||||
@@ -445,6 +441,10 @@ func TestBackupSurvivesMetadataExportInterruption(t *testing.T) {
|
||||
|
||||
repos := database.NewRepositories(db)
|
||||
|
||||
// snapshot create writes the marker; this backup runs its steps one
|
||||
// by one, so the marker is written here.
|
||||
require.NoError(t, inner.Put(ctx, markerFile, strings.NewReader("")))
|
||||
|
||||
// Back up and complete with a working backend.
|
||||
sm := newFaultSnapshotManager(fs, inner, cfg, repos)
|
||||
scanner := newFaultScanner(fs, inner, cfg, repos)
|
||||
@@ -493,8 +493,14 @@ func TestBackupSurvivesMetadataExportInterruption(t *testing.T) {
|
||||
"shallow verify must report the missing manifest")
|
||||
|
||||
// snapshot list must not crash on the partial snapshot.
|
||||
var listOut strings.Builder
|
||||
|
||||
reader.UI = ui.NewWithColor(&listOut, false)
|
||||
require.NoError(t, reader.ListSnapshots(false),
|
||||
"snapshot list must tolerate a partially-exported snapshot")
|
||||
assert.NotContains(t, listOut.String(),
|
||||
"Could not list backup destination store",
|
||||
"snapshot list must read the partial snapshot from the destination")
|
||||
}
|
||||
|
||||
// Scenario 2, repair: the process dies during the metadata export of a
|
||||
@@ -511,7 +517,7 @@ func TestBackupCompletesOnlyAfterMetadataExport(t *testing.T) {
|
||||
fs := afero.NewOsFs()
|
||||
tempDir := t.TempDir()
|
||||
dataDir := filepath.Join(tempDir, "src")
|
||||
storeDir := filepath.Join(tempDir, "remote")
|
||||
storeDir := t.TempDir()
|
||||
restoreDir := filepath.Join(tempDir, "restored")
|
||||
dbPath := filepath.Join(tempDir, "index.sqlite")
|
||||
|
||||
@@ -687,7 +693,7 @@ func TestRestoreReportsDiskFull(t *testing.T) {
|
||||
osFS := afero.NewOsFs()
|
||||
tempDir := t.TempDir()
|
||||
dataDir := filepath.Join(tempDir, "src")
|
||||
storeDir := filepath.Join(tempDir, "remote")
|
||||
storeDir := t.TempDir()
|
||||
restoreDir := filepath.Join(tempDir, "restored")
|
||||
dbPath := filepath.Join(tempDir, "index.sqlite")
|
||||
|
||||
|
||||
@@ -771,7 +771,7 @@ func TestEndToEndFileStorage(t *testing.T) {
|
||||
tempDir := t.TempDir()
|
||||
|
||||
dataDir := filepath.Join(tempDir, "source")
|
||||
storeDir := filepath.Join(tempDir, "remote")
|
||||
storeDir := t.TempDir()
|
||||
restoreDir := filepath.Join(tempDir, "restored")
|
||||
dbPath := filepath.Join(tempDir, "index.sqlite")
|
||||
|
||||
@@ -964,7 +964,7 @@ func TestDedupOnlySnapshotRestores(t *testing.T) {
|
||||
tempDir := t.TempDir()
|
||||
|
||||
dataDir := filepath.Join(tempDir, "source")
|
||||
storeDir := filepath.Join(tempDir, "remote")
|
||||
storeDir := t.TempDir()
|
||||
restoreDir := filepath.Join(tempDir, "restored")
|
||||
dbPath := filepath.Join(tempDir, "index.sqlite")
|
||||
|
||||
|
||||
@@ -24,13 +24,11 @@ import (
|
||||
// that is not plugged in, cannot be listed. It is not an empty store, so
|
||||
// no command may conclude from it that the local snapshots are gone.
|
||||
|
||||
// backUpToFileDestination backs up the snapshot named "first" to a
|
||||
// file:// destination at storeDir, which need not exist yet. Everything
|
||||
// the returned Vaultik prints after the backup goes to the returned
|
||||
// buffer.
|
||||
func backUpToFileDestination(
|
||||
// newFileDestinationVaultik returns a Vaultik that backs up the
|
||||
// snapshot named "first" to a file:// destination at storeDir.
|
||||
func newFileDestinationVaultik(
|
||||
ctx context.Context, t *testing.T, storeDir string,
|
||||
) (*vaultik.Vaultik, *database.Repositories, *bytes.Buffer) {
|
||||
) (*vaultik.Vaultik, *database.Repositories) {
|
||||
t.Helper()
|
||||
|
||||
osFs := afero.NewOsFs()
|
||||
@@ -49,8 +47,22 @@ func backUpToFileDestination(
|
||||
|
||||
repos := database.NewRepositories(db)
|
||||
cfg := changedFileConfig(dataDir, dbPath)
|
||||
v := newBackupVaultik(ctx, cfg, store, repos, db, osFs)
|
||||
|
||||
return newBackupVaultik(ctx, cfg, store, repos, db, osFs), repos
|
||||
}
|
||||
|
||||
// backUpToFileDestination creates the directory storeDir, as a user must
|
||||
// before the first backup, and backs up the snapshot named "first" to a
|
||||
// file:// destination there. Everything the returned Vaultik prints
|
||||
// after the backup goes to the returned buffer.
|
||||
func backUpToFileDestination(
|
||||
ctx context.Context, t *testing.T, storeDir string,
|
||||
) (*vaultik.Vaultik, *database.Repositories, *bytes.Buffer) {
|
||||
t.Helper()
|
||||
|
||||
require.NoError(t, os.MkdirAll(storeDir, 0o750))
|
||||
|
||||
v, repos := newFileDestinationVaultik(ctx, t, storeDir)
|
||||
require.NoError(t, backUp(v, "first"))
|
||||
|
||||
out := &bytes.Buffer{}
|
||||
@@ -75,21 +87,28 @@ func backUpThenUnplug(
|
||||
return v, repos, out
|
||||
}
|
||||
|
||||
// TestFirstBackupCreatesDestinationDirectory checks that a first backup
|
||||
// to a destination directory that does not exist yet creates it, and
|
||||
// that the destination can be listed afterwards.
|
||||
func TestFirstBackupCreatesDestinationDirectory(t *testing.T) {
|
||||
// TestBackupFailsWhenDestinationDirectoryMissing checks that a backup to
|
||||
// a file:// destination whose directory does not exist fails before it
|
||||
// records a snapshot, and does not create the directory.
|
||||
func TestBackupFailsWhenDestinationDirectoryMissing(t *testing.T) {
|
||||
log.Initialize(log.Config{})
|
||||
t.Parallel()
|
||||
|
||||
ctx := context.Background()
|
||||
storeDir := filepath.Join(t.TempDir(), "volume", "backup")
|
||||
v, _, out := backUpToFileDestination(ctx, t, storeDir)
|
||||
volume := filepath.Join(t.TempDir(), "volume")
|
||||
v, repos := newFileDestinationVaultik(ctx, t, filepath.Join(volume, "backup"))
|
||||
|
||||
require.NoError(t, v.ListSnapshots(false))
|
||||
err := backUp(v, "first")
|
||||
require.ErrorIs(t, err, fs.ErrNotExist)
|
||||
require.ErrorContains(t, err, "create it, or mount the volume it is on")
|
||||
|
||||
assert.NotContains(t, out.String(), "Could not list backup destination store")
|
||||
assert.NotContains(t, out.String(), "not found in backup destination store")
|
||||
_, err = os.Stat(volume)
|
||||
require.ErrorIs(t, err, fs.ErrNotExist,
|
||||
"the backup must not create the destination directory")
|
||||
|
||||
snapshots, err := repos.Snapshots.ListRecent(ctx, listRecentTestLimit)
|
||||
require.NoError(t, err)
|
||||
assert.Empty(t, snapshots)
|
||||
}
|
||||
|
||||
// TestListSnapshotsWarnsWhenDestinationMissing checks that snapshot list
|
||||
|
||||
@@ -39,7 +39,7 @@ func TestNewChunkSizeOfChangedFileStaysFixed(t *testing.T) {
|
||||
|
||||
appendRandomBytes(t, fs, files, changedPath, appendedFileSize)
|
||||
|
||||
store, err := storage.NewFileStorer(filepath.Join(tempDir, "remote"))
|
||||
store, err := storage.NewFileStorer(t.TempDir())
|
||||
require.NoError(t, err)
|
||||
|
||||
db, err := database.New(ctx, dbPath)
|
||||
|
||||
@@ -42,9 +42,10 @@ const blobKeyParts = 4
|
||||
|
||||
// NukeRemote deletes every object under metadata/ and blobs/ in remote
|
||||
// storage, whatever its name, along with any object an upload cut off
|
||||
// part-way left under a temporary `.partial` name. It returns
|
||||
// successfully only when a listing afterwards finds nothing left there,
|
||||
// and the next backup then starts from scratch.
|
||||
// part-way left under a temporary `.partial` name, and last the
|
||||
// destination marker. It returns successfully only when a listing
|
||||
// afterwards finds nothing left under metadata/ and blobs/, and the next
|
||||
// backup then starts from scratch.
|
||||
//
|
||||
// Refuses to run unless force is true. The caller is responsible for
|
||||
// confirming with the user.
|
||||
@@ -105,6 +106,13 @@ func (v *Vaultik) emptyDestinationStore() error {
|
||||
return fmt.Errorf("%d %w", len(left), errObjectsLeft)
|
||||
}
|
||||
|
||||
// Last, once the local snapshot records are gone: the next backup
|
||||
// then has none and writes the marker again.
|
||||
err = v.Storage.Delete(v.ctx, destinationMarkerKey)
|
||||
if err != nil {
|
||||
return fmt.Errorf("deleting %s: %w", destinationMarkerKey, err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
|
||||
@@ -35,6 +35,9 @@ func setupConsistencyTest(
|
||||
repos := database.NewRepositories(db)
|
||||
mockStorage := NewMockStorer()
|
||||
|
||||
// A backup wrote to this destination, so it carries the marker.
|
||||
require.NoError(t, mockStorage.Put(ctx, markerFile, strings.NewReader("")))
|
||||
|
||||
for _, id := range snapshotIDs {
|
||||
parts := strings.Split(id, "_")
|
||||
startedAt, err := time.Parse(time.RFC3339, parts[len(parts)-1])
|
||||
|
||||
@@ -45,6 +45,9 @@ func setupPurgeTest(
|
||||
repos := database.NewRepositories(db)
|
||||
mockStorage := NewMockStorer()
|
||||
|
||||
// A backup wrote to this destination, so it carries the marker.
|
||||
require.NoError(t, mockStorage.Put(ctx, markerFile, strings.NewReader("")))
|
||||
|
||||
// Insert each snapshot into the DB and create remote metadata stubs.
|
||||
// Use timestamps parsed from snapshot IDs for realistic ordering.
|
||||
for _, id := range snapshotIDs {
|
||||
|
||||
@@ -96,6 +96,10 @@ func TestRemoveSnapshot_QuietSuppressesReport(t *testing.T) {
|
||||
store := newTestStorer()
|
||||
addManifest(t, store, "snapshot-001", nil)
|
||||
|
||||
// A backup wrote to this destination, so it carries the marker.
|
||||
require.NoError(t, store.Put(context.Background(), markerFile,
|
||||
bytes.NewReader(nil)))
|
||||
|
||||
tv := vaultik.NewForTesting(store)
|
||||
tv.Repositories = newLocalIndex(t, "snapshot-001")
|
||||
tv.UI.SetQuiet(true)
|
||||
|
||||
@@ -304,6 +304,10 @@ func TestRemoveSnapshot_DefaultRemovesMetadataNotBlobs(t *testing.T) {
|
||||
addBlob(t, store, blobUnique)
|
||||
addBlob(t, store, blobShared)
|
||||
|
||||
// A backup wrote to this destination, so it carries the marker.
|
||||
require.NoError(t, store.Put(context.Background(), markerFile,
|
||||
bytes.NewReader(nil)))
|
||||
|
||||
tv := vaultik.NewForTesting(store)
|
||||
tv.Repositories = newLocalIndex(t, "snapshot-001", "snapshot-002")
|
||||
|
||||
@@ -373,6 +377,10 @@ func TestRemoveSnapshot_UnknownIDIsError(t *testing.T) {
|
||||
store := newTestStorer()
|
||||
addManifest(t, store, "snapshot-001", nil)
|
||||
|
||||
// A backup wrote to this destination, so it carries the marker.
|
||||
require.NoError(t, store.Put(context.Background(), markerFile,
|
||||
bytes.NewReader(nil)))
|
||||
|
||||
tv := vaultik.NewForTesting(store)
|
||||
tv.Repositories = newLocalIndex(t, "snapshot-001")
|
||||
|
||||
@@ -393,7 +401,12 @@ func TestRemoveSnapshot_NoMetadataOnDestination(t *testing.T) {
|
||||
log.Initialize(log.Config{})
|
||||
t.Parallel()
|
||||
|
||||
tv := vaultik.NewForTesting(newTestStorer())
|
||||
// A backup wrote to this destination, so it carries the marker.
|
||||
store := newTestStorer()
|
||||
require.NoError(t, store.Put(context.Background(), markerFile,
|
||||
bytes.NewReader(nil)))
|
||||
|
||||
tv := vaultik.NewForTesting(store)
|
||||
tv.Repositories = newLocalIndex(t, "snapshot-001")
|
||||
|
||||
result, err := tv.RemoveSnapshot("snapshot-001",
|
||||
|
||||
@@ -12,6 +12,7 @@ import (
|
||||
"path/filepath"
|
||||
"slices"
|
||||
"strings"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
"filippo.io/age"
|
||||
@@ -219,9 +220,7 @@ func (v *Vaultik) finishRestore(
|
||||
)
|
||||
|
||||
if os.Geteuid() != 0 {
|
||||
v.UI.Warningf("Restore did not preserve file ownership: chown(2) " +
|
||||
"requires root. Re-run as root (e.g. with sudo) if you need " +
|
||||
"original UID/GID preserved.")
|
||||
v.warnIfOwnershipNotPreserved(files, opts.TargetDir)
|
||||
}
|
||||
|
||||
if result.FilesFailed > 0 {
|
||||
@@ -247,6 +246,43 @@ func (v *Vaultik) finishRestore(
|
||||
return unmatchedErr
|
||||
}
|
||||
|
||||
// warnIfOwnershipNotPreserved warns once if some entry restored under
|
||||
// targetDir has another owner or group on disk than the snapshot
|
||||
// records. It is meant for a restore by a normal user, which changes no
|
||||
// owner or group: a new entry is owned by that user, and its group is
|
||||
// that user's, or the group of the directory it is created in on macOS
|
||||
// and in a setgid directory on Linux. An entry that is not on disk, or
|
||||
// whose filesystem reports no owner, is skipped.
|
||||
func (v *Vaultik) warnIfOwnershipNotPreserved(
|
||||
files []*database.File, targetDir string,
|
||||
) {
|
||||
for _, file := range files {
|
||||
targetPath, err := containedRestorePath(
|
||||
v.Fs, targetDir, file.Path.String())
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
|
||||
info, err := lstatIfPossible(v.Fs, targetPath)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
|
||||
owner, ok := info.Sys().(*syscall.Stat_t)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
|
||||
if owner.Uid != file.UID || owner.Gid != file.GID {
|
||||
v.UI.Warningf("Restore did not preserve file ownership: chown(2) " +
|
||||
"requires root. Re-run as root (e.g. with sudo) if you need " +
|
||||
"original UID/GID preserved.")
|
||||
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// reportUnmatchedPaths warns, by name, about each requested path that
|
||||
// matched nothing in the snapshot, and returns an error counting them,
|
||||
// or nil when there are none.
|
||||
@@ -965,8 +1001,9 @@ type restoreSession struct {
|
||||
// root can chown a file to an arbitrary UID/GID — non-root chown
|
||||
// always fails with EPERM. Attempting it anyway produces N
|
||||
// guaranteed-failed syscalls + N noisy debug lines, so we skip
|
||||
// the call entirely as non-root and emit one warning at the end
|
||||
// of the restore explaining that ownership was not preserved.
|
||||
// the call entirely as non-root; warnIfOwnershipNotPreserved warns
|
||||
// once at the end of the restore if that left any owner or group
|
||||
// other than recorded.
|
||||
runningAsRoot bool
|
||||
// directories holds every restored directory, for
|
||||
// applyDirectoryMetadata to finish after the restore loop.
|
||||
|
||||
@@ -42,7 +42,7 @@ func TestRestoreOnAnotherMachine(t *testing.T) {
|
||||
tempDir := t.TempDir()
|
||||
|
||||
dataDir := filepath.Join(tempDir, "source")
|
||||
storeDir := filepath.Join(tempDir, "remote")
|
||||
storeDir := t.TempDir()
|
||||
restoreDir := filepath.Join(tempDir, "restored")
|
||||
dbPath := filepath.Join(tempDir, "index.sqlite")
|
||||
|
||||
|
||||
@@ -65,7 +65,7 @@ func TestRestoreCleansTempDirOnInterrupt(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
|
||||
dataDir := filepath.Join(root, "source")
|
||||
storeDir := filepath.Join(root, "remote")
|
||||
storeDir := t.TempDir()
|
||||
restoreDir := filepath.Join(root, "restored")
|
||||
dbPath := filepath.Join(root, "index.sqlite")
|
||||
|
||||
|
||||
@@ -201,7 +201,7 @@ func TestRestoreLocalityAndReadAt(t *testing.T) {
|
||||
tempDir := t.TempDir()
|
||||
|
||||
dataDir := filepath.Join(tempDir, "source")
|
||||
storeDir := filepath.Join(tempDir, "remote")
|
||||
storeDir := t.TempDir()
|
||||
restoreDir := filepath.Join(tempDir, "restored")
|
||||
dbPath := filepath.Join(tempDir, "index.sqlite")
|
||||
|
||||
|
||||
@@ -0,0 +1,131 @@
|
||||
package vaultik //nolint:testpackage // calls the unexported ownership check
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"os"
|
||||
"strings"
|
||||
"syscall"
|
||||
"testing"
|
||||
|
||||
"github.com/spf13/afero"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
"sneak.berlin/go/vaultik/internal/database"
|
||||
"sneak.berlin/go/vaultik/internal/log"
|
||||
"sneak.berlin/go/vaultik/internal/ui"
|
||||
)
|
||||
|
||||
// These tests cover https://git.eeqj.de/sneak/vaultik/issues/316: a
|
||||
// restore by a normal user warns that ownership was not preserved only
|
||||
// when some restored entry has another owner or group on disk than the
|
||||
// snapshot records.
|
||||
|
||||
const ownershipWarning = "did not preserve file ownership"
|
||||
|
||||
// TestOwnershipWarningSilentWhenEntriesMatch checks that a restore whose
|
||||
// entries all end up with their recorded owner and group prints no
|
||||
// warning.
|
||||
func TestOwnershipWarningSilentWhenEntriesMatch(t *testing.T) {
|
||||
log.Initialize(log.Config{})
|
||||
t.Parallel()
|
||||
|
||||
targetDir := t.TempDir()
|
||||
owner := ownerOf(t, targetDir)
|
||||
|
||||
out := restoreAsNormalUser(t, targetDir, []*database.File{
|
||||
{Path: "/a", Mode: plainFileMode, UID: owner.Uid, GID: owner.Gid},
|
||||
{Path: "/b", Mode: plainFileMode, UID: owner.Uid, GID: owner.Gid},
|
||||
})
|
||||
|
||||
assert.Empty(t, out)
|
||||
}
|
||||
|
||||
// TestOwnershipWarningWhenOwnerDiffers checks that the warning is printed
|
||||
// when an entry recorded with another owner is restored owned by the
|
||||
// restoring user, even though its group matches the recorded one.
|
||||
func TestOwnershipWarningWhenOwnerDiffers(t *testing.T) {
|
||||
log.Initialize(log.Config{})
|
||||
t.Parallel()
|
||||
|
||||
targetDir := t.TempDir()
|
||||
owner := ownerOf(t, targetDir)
|
||||
|
||||
out := restoreAsNormalUser(t, targetDir, []*database.File{
|
||||
{Path: "/a", Mode: plainFileMode, UID: otherOwnerID, GID: owner.Gid},
|
||||
})
|
||||
|
||||
assert.Equal(t, 1, strings.Count(out, ownershipWarning), out)
|
||||
}
|
||||
|
||||
// TestOwnershipWarningWhenDirectoryGivesItsGroup checks that the warning
|
||||
// is printed, once, when entries recorded with the restoring user's
|
||||
// owner and group take another group from the setgid directory they are
|
||||
// restored into, as every new entry takes its directory's group on
|
||||
// macOS.
|
||||
func TestOwnershipWarningWhenDirectoryGivesItsGroup(t *testing.T) {
|
||||
log.Initialize(log.Config{})
|
||||
t.Parallel()
|
||||
|
||||
if os.Geteuid() != 0 {
|
||||
t.Skip("giving a directory to another group needs root")
|
||||
}
|
||||
|
||||
targetDir := t.TempDir()
|
||||
user := ownerOf(t, targetDir)
|
||||
|
||||
require.NoError(t, os.Chown(targetDir, -1, int(otherOwnerID)))
|
||||
require.NoError(t, os.Chmod(targetDir, os.ModeSetgid|writableTestMode))
|
||||
|
||||
out := restoreAsNormalUser(t, targetDir, []*database.File{
|
||||
{Path: "/a", Mode: plainFileMode, UID: user.Uid, GID: user.Gid},
|
||||
{Path: "/b", Mode: plainFileMode, UID: user.Uid, GID: user.Gid},
|
||||
})
|
||||
|
||||
assert.Equal(t, 1, strings.Count(out, ownershipWarning), out)
|
||||
}
|
||||
|
||||
// ownerOf returns the owner and group path has on disk.
|
||||
func ownerOf(t *testing.T, path string) *syscall.Stat_t {
|
||||
t.Helper()
|
||||
|
||||
info, err := os.Lstat(path)
|
||||
require.NoError(t, err)
|
||||
|
||||
owner, ok := info.Sys().(*syscall.Stat_t)
|
||||
require.True(t, ok)
|
||||
|
||||
return owner
|
||||
}
|
||||
|
||||
// restoreAsNormalUser restores rows, all empty regular files, into
|
||||
// targetDir without changing any owner or group, as restore does for a
|
||||
// normal user, and returns what warnIfOwnershipNotPreserved prints.
|
||||
func restoreAsNormalUser(
|
||||
t *testing.T, targetDir string, rows []*database.File,
|
||||
) string {
|
||||
t.Helper()
|
||||
|
||||
ctx := context.Background()
|
||||
rows, repos := makeFiles(ctx, t, rows)
|
||||
|
||||
var out bytes.Buffer
|
||||
|
||||
v := newContainmentVaultik(ctx, afero.NewOsFs())
|
||||
v.UI = ui.NewWithColor(&out, false)
|
||||
|
||||
session := &restoreSession{
|
||||
v: v,
|
||||
ctx: ctx,
|
||||
repos: repos,
|
||||
opts: &RestoreOptions{TargetDir: targetDir},
|
||||
result: &RestoreResult{},
|
||||
}
|
||||
for _, row := range rows {
|
||||
require.NoError(t, session.restoreFile(row))
|
||||
}
|
||||
|
||||
v.warnIfOwnershipNotPreserved(rows, targetDir)
|
||||
|
||||
return out.String()
|
||||
}
|
||||
@@ -107,7 +107,7 @@ func backupOneFile(
|
||||
require.NoError(t, afero.WriteFile(fs, srcPath, content, mode))
|
||||
require.NoError(t, fs.Chmod(srcPath, mode))
|
||||
|
||||
storeDir := filepath.Join(tempDir, "remote")
|
||||
storeDir := t.TempDir()
|
||||
dbPath := filepath.Join(tempDir, "index.sqlite")
|
||||
|
||||
storer, err := storage.NewFileStorer(storeDir)
|
||||
|
||||
@@ -348,7 +348,7 @@ func backupSingleChunkFiles(
|
||||
require.NoError(t, afero.WriteFile(fs, path, files[path], 0o644))
|
||||
}
|
||||
|
||||
storer, err := storage.NewFileStorer(filepath.Join(tempDir, "remote"))
|
||||
storer, err := storage.NewFileStorer(t.TempDir())
|
||||
require.NoError(t, err)
|
||||
|
||||
db, err := database.New(ctx, dbPath)
|
||||
|
||||
@@ -29,7 +29,7 @@ func TestRestoreAndDeepVerifyRejectSwappedDatabase(t *testing.T) {
|
||||
|
||||
fs := afero.NewOsFs()
|
||||
tempDir := t.TempDir()
|
||||
storeDir := filepath.Join(tempDir, "remote")
|
||||
storeDir := t.TempDir()
|
||||
|
||||
chunkSize := int64(64 * 1024)
|
||||
|
||||
@@ -100,7 +100,7 @@ func TestDeepVerifyRejectsSwappedDatabaseWithEmptyManifest(t *testing.T) {
|
||||
|
||||
fs := afero.NewOsFs()
|
||||
tempDir := t.TempDir()
|
||||
storeDir := filepath.Join(tempDir, "remote")
|
||||
storeDir := t.TempDir()
|
||||
|
||||
chunkSize := int64(64 * 1024)
|
||||
|
||||
|
||||
@@ -186,7 +186,7 @@ func TestRestoreSweeperEvictsBlobs(t *testing.T) {
|
||||
tempDir := t.TempDir()
|
||||
|
||||
dataDir := filepath.Join(tempDir, "source")
|
||||
storeDir := filepath.Join(tempDir, "remote")
|
||||
storeDir := t.TempDir()
|
||||
restoreDir := filepath.Join(tempDir, "restored")
|
||||
dbPath := filepath.Join(tempDir, "index.sqlite")
|
||||
|
||||
|
||||
@@ -34,7 +34,7 @@ func TestRestoreFreesBlobsNoPendingFileNeeds(t *testing.T) {
|
||||
dataDir := filepath.Join(tempDir, "source")
|
||||
originalsDir := filepath.Join(dataDir, "originals")
|
||||
copiesDir := filepath.Join(dataDir, "copies")
|
||||
storeDir := filepath.Join(tempDir, "remote")
|
||||
storeDir := t.TempDir()
|
||||
dbPath := filepath.Join(tempDir, "index.sqlite")
|
||||
|
||||
originals := writeFilesWithCopies(t, fs, originalsDir, copiesDir, fileBytes)
|
||||
|
||||
@@ -33,7 +33,7 @@ func TestRestoreRejectsTruncatedMetadataDB(t *testing.T) {
|
||||
tempDir := t.TempDir()
|
||||
|
||||
dataDir := filepath.Join(tempDir, "source")
|
||||
storeDir := filepath.Join(tempDir, "remote")
|
||||
storeDir := t.TempDir()
|
||||
restoreDir := filepath.Join(tempDir, "restored")
|
||||
dbPath := filepath.Join(tempDir, "index.sqlite")
|
||||
|
||||
|
||||
@@ -32,7 +32,7 @@ func TestRestoreNamesUnmatchedPathAndRestoresTheMatchingOne(t *testing.T) {
|
||||
fs := afero.NewOsFs()
|
||||
tempDir := t.TempDir()
|
||||
dataDir := filepath.Join(tempDir, "src")
|
||||
storeDir := filepath.Join(tempDir, "remote")
|
||||
storeDir := t.TempDir()
|
||||
restoreDir := filepath.Join(tempDir, "restored")
|
||||
dbPath := filepath.Join(tempDir, "index.sqlite")
|
||||
|
||||
@@ -89,7 +89,7 @@ func TestRestoreNamesEveryPathWhenNothingMatches(t *testing.T) {
|
||||
tempDir := t.TempDir()
|
||||
dataDir := filepath.Join(tempDir, "src")
|
||||
linkDir := filepath.Join(tempDir, "srclink")
|
||||
storeDir := filepath.Join(tempDir, "remote")
|
||||
storeDir := t.TempDir()
|
||||
restoreDir := filepath.Join(tempDir, "restored")
|
||||
dbPath := filepath.Join(tempDir, "index.sqlite")
|
||||
|
||||
@@ -152,7 +152,7 @@ func TestRestoreNamesUnmatchedPathWhenADamagedBlobStopsIt(t *testing.T) {
|
||||
fs := afero.NewOsFs()
|
||||
tempDir := t.TempDir()
|
||||
dataDir := filepath.Join(tempDir, "src")
|
||||
storeDir := filepath.Join(tempDir, "remote")
|
||||
storeDir := t.TempDir()
|
||||
restoreDir := filepath.Join(tempDir, "restored")
|
||||
dbPath := filepath.Join(tempDir, "index.sqlite")
|
||||
|
||||
|
||||
@@ -28,7 +28,7 @@ func TestVerifyPassesWhenBackupReusesStoredBlob(t *testing.T) {
|
||||
tempDir := t.TempDir()
|
||||
|
||||
dataDir := filepath.Join(tempDir, "source")
|
||||
storeDir := filepath.Join(tempDir, "remote")
|
||||
storeDir := t.TempDir()
|
||||
secondDBPath := filepath.Join(tempDir, "second.sqlite")
|
||||
|
||||
chunkSize := int64(64 * 1024)
|
||||
|
||||
@@ -25,7 +25,7 @@ func TestBackupOfSameSecondRewriteRestoresNewContent(t *testing.T) {
|
||||
fs := afero.NewOsFs()
|
||||
tempDir := t.TempDir()
|
||||
dataDir := filepath.Join(tempDir, "src")
|
||||
storeDir := filepath.Join(tempDir, "remote")
|
||||
storeDir := t.TempDir()
|
||||
restoreDir := filepath.Join(tempDir, "restored")
|
||||
dbPath := filepath.Join(tempDir, "index.sqlite")
|
||||
rewrittenPath := filepath.Join(dataDir, "small.txt")
|
||||
|
||||
@@ -36,7 +36,7 @@ func TestShallowVerifyDetectsWrongBlobSize(t *testing.T) {
|
||||
tempDir := t.TempDir()
|
||||
|
||||
dataDir := filepath.Join(tempDir, "source")
|
||||
storeDir := filepath.Join(tempDir, "remote")
|
||||
storeDir := t.TempDir()
|
||||
dbPath := filepath.Join(tempDir, "index.sqlite")
|
||||
|
||||
chunkSize := int64(32 * 1024)
|
||||
@@ -116,7 +116,7 @@ func TestShallowVerifyDetectsMissingDatabase(t *testing.T) {
|
||||
tempDir := t.TempDir()
|
||||
|
||||
dataDir := filepath.Join(tempDir, "source")
|
||||
storeDir := filepath.Join(tempDir, "remote")
|
||||
storeDir := t.TempDir()
|
||||
dbPath := filepath.Join(tempDir, "index.sqlite")
|
||||
|
||||
chunkSize := int64(32 * 1024)
|
||||
@@ -199,7 +199,7 @@ func TestShallowVerifyReportsStatErrorsApartFromMissing(t *testing.T) {
|
||||
tempDir := t.TempDir()
|
||||
|
||||
dataDir := filepath.Join(tempDir, "source")
|
||||
storeDir := filepath.Join(tempDir, "remote")
|
||||
storeDir := t.TempDir()
|
||||
dbPath := filepath.Join(tempDir, "index.sqlite")
|
||||
|
||||
chunkSize := int64(32 * 1024)
|
||||
@@ -288,7 +288,7 @@ func TestShallowVerifyQuietFailsOnStatErrorAlone(t *testing.T) {
|
||||
tempDir := t.TempDir()
|
||||
|
||||
dataDir := filepath.Join(tempDir, "source")
|
||||
storeDir := filepath.Join(tempDir, "remote")
|
||||
storeDir := t.TempDir()
|
||||
dbPath := filepath.Join(tempDir, "index.sqlite")
|
||||
|
||||
chunkSize := int64(32 * 1024)
|
||||
|
||||
@@ -95,6 +95,13 @@ func (v *Vaultik) CreateSnapshot(opts *SnapshotCreateOptions) error {
|
||||
return err
|
||||
}
|
||||
|
||||
// Before PruneDatabase: it drops incomplete snapshots, which could
|
||||
// leave the local index without one and so skip the marker check.
|
||||
err = v.markDestination()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Clean up incomplete snapshots FIRST, before any scanning.
|
||||
// This is critical for data safety; PruneDatabase below does it.
|
||||
hostname := v.Config.Hostname
|
||||
@@ -117,21 +124,9 @@ func (v *Vaultik) CreateSnapshot(opts *SnapshotCreateOptions) error {
|
||||
return fmt.Errorf("prune database: %w", err)
|
||||
}
|
||||
|
||||
// Determine which snapshots to process
|
||||
snapshotNames := opts.Snapshots
|
||||
if len(snapshotNames) == 0 {
|
||||
snapshotNames = v.Config.SnapshotNames()
|
||||
} else {
|
||||
// Validate requested snapshot names exist
|
||||
for _, name := range snapshotNames {
|
||||
if _, ok := v.Config.Snapshots[name]; !ok {
|
||||
return fmt.Errorf("%w: %q", errSnapshotNotInConfig, name)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if len(snapshotNames) == 0 {
|
||||
return errNoSnapshotsInConfig
|
||||
snapshotNames, err := v.snapshotNamesToCreate(opts.Snapshots)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Process each named snapshot
|
||||
@@ -171,6 +166,28 @@ func (v *Vaultik) CreateSnapshot(opts *SnapshotCreateOptions) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// snapshotNamesToCreate returns the requested snapshot names, each checked
|
||||
// against the config, or every configured name when none was requested.
|
||||
func (v *Vaultik) snapshotNamesToCreate(requested []string) ([]string, error) {
|
||||
snapshotNames := requested
|
||||
if len(snapshotNames) == 0 {
|
||||
snapshotNames = v.Config.SnapshotNames()
|
||||
} else {
|
||||
// Validate requested snapshot names exist
|
||||
for _, name := range snapshotNames {
|
||||
if _, ok := v.Config.Snapshots[name]; !ok {
|
||||
return nil, fmt.Errorf("%w: %q", errSnapshotNotInConfig, name)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if len(snapshotNames) == 0 {
|
||||
return nil, errNoSnapshotsInConfig
|
||||
}
|
||||
|
||||
return snapshotNames, nil
|
||||
}
|
||||
|
||||
// runPostBackupPrune drops older snapshots of the given names and removes
|
||||
// orphan blobs from remote storage. If keepNewerThan is set (e.g. "4w"),
|
||||
// snapshots newer than that duration are kept. Otherwise only the latest
|
||||
@@ -1281,10 +1298,18 @@ func (v *Vaultik) RemoveSnapshot(
|
||||
return v.removeSnapshotDryRun(result, snapshotID, opts)
|
||||
}
|
||||
|
||||
if !opts.Force && !opts.JSON && !v.confirmRemoveSnapshot(snapshotID, opts) {
|
||||
if !v.confirmRemoveSnapshot(snapshotID, opts) {
|
||||
return result, nil
|
||||
}
|
||||
|
||||
// Checked before the local record is deleted: it may be the last
|
||||
// snapshot in the local index, and the check applies only while the
|
||||
// local index holds one.
|
||||
var markerErr error
|
||||
if !opts.LocalOnly {
|
||||
markerErr = v.checkDestinationMarker()
|
||||
}
|
||||
|
||||
if inLocalIndex {
|
||||
log.Info("Removing snapshot from local database", "snapshot_id", snapshotID)
|
||||
|
||||
@@ -1295,7 +1320,7 @@ func (v *Vaultik) RemoveSnapshot(
|
||||
}
|
||||
|
||||
result.RemoteRemoved, err = v.removeSnapshotRemote(
|
||||
snapshotID, inLocalIndex, opts)
|
||||
snapshotID, inLocalIndex, markerErr, opts)
|
||||
if err != nil {
|
||||
return result, err
|
||||
}
|
||||
@@ -1332,7 +1357,9 @@ func (v *Vaultik) RemoveSnapshot(
|
||||
// remove: no row and, unless LocalOnly is set, no metadata on the
|
||||
// destination store either. The destination store is consulted only
|
||||
// when the local index has no row, so a snapshot in the local index can
|
||||
// still be removed from it while the destination store is unreachable.
|
||||
// still be removed from it while the destination store is unreachable,
|
||||
// and only after checkDestinationMarker passes, so an unplugged volume's
|
||||
// empty mount point is not taken for a store without the snapshot.
|
||||
func (v *Vaultik) findSnapshotToRemove(
|
||||
snapshotID string, opts *RemoveOptions,
|
||||
) (bool, error) {
|
||||
@@ -1352,6 +1379,11 @@ func (v *Vaultik) findSnapshotToRemove(
|
||||
errSnapshotNotFound, snapshotID)
|
||||
}
|
||||
|
||||
err := v.checkDestinationMarker()
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
|
||||
objects, err := v.remoteSnapshotObjects(snapshot.RemoteSnapshotKey(snapshotID))
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("looking up snapshot on destination store: %w", err)
|
||||
@@ -1390,11 +1422,15 @@ func (v *Vaultik) removeSnapshotDryRun(
|
||||
}
|
||||
|
||||
// confirmRemoveSnapshot prompts for confirmation before removing a
|
||||
// snapshot. Returns false (treating EOF/read errors as "no") unless the
|
||||
// user answers "y". The prompt and its answer bypass the UI: they are an
|
||||
// interactive exchange the operator must see, so --quiet does not
|
||||
// silence them.
|
||||
// snapshot, unless --force or --json is set. Returns false (treating
|
||||
// EOF/read errors as "no") unless the user answers "y". The prompt and
|
||||
// its answer bypass the UI: they are an interactive exchange the
|
||||
// operator must see, so --quiet does not silence them.
|
||||
func (v *Vaultik) confirmRemoveSnapshot(snapshotID string, opts *RemoveOptions) bool {
|
||||
if opts.Force || opts.JSON {
|
||||
return true
|
||||
}
|
||||
|
||||
if opts.LocalOnly {
|
||||
v.stdoutf("Remove snapshot '%s' from local database "+
|
||||
"(remote untouched)? [y/N] ", snapshotID)
|
||||
@@ -1424,15 +1460,17 @@ func (v *Vaultik) confirmRemoveSnapshot(snapshotID string, opts *RemoveOptions)
|
||||
// removeSnapshotRemote strips the snapshot's metadata from the
|
||||
// destination store. It returns true only when the delete finished
|
||||
// without an error and removed at least one object; under LocalOnly it
|
||||
// does nothing. When the snapshot is only on the destination store, a
|
||||
// failure is returned as an error, because that removal is all the
|
||||
// command does. When the snapshot was in the local index, its removal
|
||||
// there has already happened, so a failure is only a warning telling the
|
||||
// user to run `vaultik snapshot remove` for the snapshot again once the
|
||||
// destination store is reachable (`vaultik prune` never removes snapshot
|
||||
// metadata).
|
||||
// does nothing. markerErr is the result of checkDestinationMarker; when
|
||||
// it is set, nothing is deleted and it is the failure. When the snapshot
|
||||
// is only on the destination store, a failure is returned as an error,
|
||||
// because that removal is all the command does. When the snapshot was in
|
||||
// the local index, its removal there has already happened, so a failure
|
||||
// is only a warning telling the user to run `vaultik snapshot remove`
|
||||
// for the snapshot again once the destination store is reachable
|
||||
// (`vaultik prune` never removes snapshot metadata).
|
||||
func (v *Vaultik) removeSnapshotRemote(
|
||||
snapshotID string, inLocalIndex bool, opts *RemoveOptions,
|
||||
snapshotID string, inLocalIndex bool, markerErr error,
|
||||
opts *RemoveOptions,
|
||||
) (bool, error) {
|
||||
if opts.LocalOnly {
|
||||
return false, nil
|
||||
@@ -1441,9 +1479,14 @@ func (v *Vaultik) removeSnapshotRemote(
|
||||
log.Info("Removing snapshot metadata from remote storage",
|
||||
"snapshot_id", snapshotID)
|
||||
|
||||
remoteKey := snapshot.RemoteSnapshotKey(snapshotID)
|
||||
var deleted int
|
||||
|
||||
err := markerErr
|
||||
if err == nil {
|
||||
deleted, err = v.deleteRemoteSnapshotByKey(
|
||||
snapshot.RemoteSnapshotKey(snapshotID))
|
||||
}
|
||||
|
||||
deleted, err := v.deleteRemoteSnapshotByKey(remoteKey)
|
||||
if err != nil && !inLocalIndex {
|
||||
return false, fmt.Errorf(
|
||||
"removing snapshot metadata from destination store: %w", err)
|
||||
@@ -1552,8 +1595,8 @@ func (v *Vaultik) localSnapshotIDs() ([]string, error) {
|
||||
// listAllRemoteSnapshotKeys collects the hashed remote keys
|
||||
// (subdirectories under metadata/) currently present in the
|
||||
// destination store. Returns (nil, err) when the store cannot be
|
||||
// listed; callers must treat that as "no remote info available," not
|
||||
// fatal.
|
||||
// listed, or is not present by checkDestinationMarker; callers must
|
||||
// treat that as "no remote info available," not fatal.
|
||||
func (v *Vaultik) listAllRemoteSnapshotKeys() ([]string, error) {
|
||||
log.Info("Listing all remote snapshots")
|
||||
|
||||
@@ -1602,6 +1645,11 @@ func (v *Vaultik) listAllRemoteSnapshotKeys() ([]string, error) {
|
||||
}
|
||||
}
|
||||
|
||||
err := v.checkDestinationMarker()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return keys, nil
|
||||
}
|
||||
|
||||
|
||||
@@ -53,7 +53,7 @@ func newDeletedCountEnv(t *testing.T, files map[string][]byte) *deletedCountEnv
|
||||
cfg.IndexPath = filepath.Join(tempDir, "index.sqlite")
|
||||
cfg.Snapshots = map[string]config.SnapshotConfig{}
|
||||
|
||||
store, err := storage.NewFileStorer(filepath.Join(tempDir, "remote"))
|
||||
store, err := storage.NewFileStorer(t.TempDir())
|
||||
require.NoError(t, err)
|
||||
|
||||
db, err := database.New(ctx, cfg.IndexPath)
|
||||
|
||||
@@ -87,7 +87,7 @@ func TestSnapshotSummaryCountsFileNotStoredAsFailed(t *testing.T) {
|
||||
"src": {Paths: []string{srcDir}},
|
||||
}
|
||||
|
||||
store, err := storage.NewFileStorer(filepath.Join(tempDir, "remote"))
|
||||
store, err := storage.NewFileStorer(t.TempDir())
|
||||
require.NoError(t, err)
|
||||
|
||||
db, err := database.New(ctx, cfg.IndexPath)
|
||||
|
||||
@@ -122,6 +122,9 @@ func newListEnv(t *testing.T) *listEnv {
|
||||
store := newObservingStorer()
|
||||
stdout := &bytes.Buffer{}
|
||||
|
||||
// A backup wrote to this destination, so it carries the marker.
|
||||
require.NoError(t, store.Put(ctx, markerFile, strings.NewReader("")))
|
||||
|
||||
v := &vaultik.Vaultik{
|
||||
Config: &config.Config{
|
||||
AgeSecretKey: "",
|
||||
|
||||
@@ -45,7 +45,7 @@ func TestSkipErrorsBackupStopsOnIndexErrorRecordingDirectory(t *testing.T) {
|
||||
"tree": {Paths: []string{srcDir}},
|
||||
}
|
||||
|
||||
store, err := storage.NewFileStorer(filepath.Join(tempDir, "remote"))
|
||||
store, err := storage.NewFileStorer(t.TempDir())
|
||||
require.NoError(t, err)
|
||||
|
||||
db, err := database.New(ctx, cfg.IndexPath)
|
||||
|
||||
@@ -13,6 +13,7 @@ import (
|
||||
"github.com/spf13/afero"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
"golang.org/x/sys/unix"
|
||||
"sneak.berlin/go/vaultik/internal/config"
|
||||
"sneak.berlin/go/vaultik/internal/database"
|
||||
"sneak.berlin/go/vaultik/internal/log"
|
||||
@@ -84,7 +85,7 @@ func newSummaryEnv(t *testing.T) *summaryEnv {
|
||||
"split": {Paths: []string{dirOne, dirTwo}},
|
||||
}
|
||||
|
||||
inner, err := storage.NewFileStorer(filepath.Join(tempDir, "remote"))
|
||||
inner, err := storage.NewFileStorer(t.TempDir())
|
||||
require.NoError(t, err)
|
||||
|
||||
store := faultstore.New(inner)
|
||||
@@ -188,6 +189,42 @@ func (e *summaryEnv) dataLine(total, backedUp int64) string {
|
||||
e.v.UI.Size(total), e.v.UI.Size(backedUp))
|
||||
}
|
||||
|
||||
// enumeratedLine returns the line printed when the walk of a source path
|
||||
// ends.
|
||||
func (e *summaryEnv) enumeratedLine(
|
||||
files int, total int64, toBackUp int, toBackUpSize int64,
|
||||
) string {
|
||||
return fmt.Sprintf(
|
||||
"Enumerated %d snapshot source files (%s total), %d to back up (%s).",
|
||||
files, e.v.UI.Size(total), toBackUp, e.v.UI.Size(toBackUpSize))
|
||||
}
|
||||
|
||||
// snapshotPaths returns the paths the snapshot lists, sorted.
|
||||
func (e *summaryEnv) snapshotPaths(t *testing.T, snapshotID string) []string {
|
||||
t.Helper()
|
||||
|
||||
rows, err := e.db.Conn().QueryContext(context.Background(), `
|
||||
SELECT f.path FROM snapshot_files sf JOIN files f ON f.id = sf.file_id
|
||||
WHERE sf.snapshot_id = ? ORDER BY f.path`, snapshotID)
|
||||
require.NoError(t, err)
|
||||
|
||||
defer func() { _ = rows.Close() }()
|
||||
|
||||
var paths []string
|
||||
|
||||
for rows.Next() {
|
||||
var path string
|
||||
|
||||
require.NoError(t, rows.Scan(&path))
|
||||
|
||||
paths = append(paths, path)
|
||||
}
|
||||
|
||||
require.NoError(t, rows.Err())
|
||||
|
||||
return paths
|
||||
}
|
||||
|
||||
// A first backup stores copy.bin's chunks while backing up a.bin, so
|
||||
// copy.bin's chunks are deduplicated within the run. Each file and byte
|
||||
// is still counted once.
|
||||
@@ -248,6 +285,85 @@ func TestSnapshotSummaryIncrementalRunWithDeduplicatedChunks(t *testing.T) {
|
||||
assert.Equal(t, uncompressed, snap.BlobUncompressedSize)
|
||||
}
|
||||
|
||||
// A backup of a tree with directories and a symlink, none of which
|
||||
// changed since the last backup, backs up nothing, and the snapshot
|
||||
// still lists every one of them. Both sides of the line that ends the
|
||||
// walk count the same entries, the files and the symlink, and no
|
||||
// directory's size. See https://git.eeqj.de/sneak/vaultik/issues/305.
|
||||
func TestSnapshotSummaryUnchangedRunWithDirectoriesAndSymlink(t *testing.T) {
|
||||
log.Initialize(log.Config{})
|
||||
t.Parallel()
|
||||
|
||||
env := newSummaryEnv(t)
|
||||
|
||||
const linkTarget = "a.bin"
|
||||
|
||||
require.NoError(t, os.Symlink(linkTarget,
|
||||
filepath.Join(filepath.Dir(env.aPath), "link")))
|
||||
|
||||
// A symlink's size is the length of its target.
|
||||
total := env.totalSize + int64(len(linkTarget))
|
||||
|
||||
first := env.backUp(t, "first", false)
|
||||
assert.Contains(t, first, env.enumeratedLine(4, total, 4, total))
|
||||
assert.Contains(t, first, filesLine(4, 4, 0))
|
||||
|
||||
second := env.backUp(t, "second", false)
|
||||
assert.Contains(t, second, env.enumeratedLine(4, total, 0, 0))
|
||||
assert.Contains(t, second, filesLine(4, 0, 4))
|
||||
assert.Contains(t, second, env.dataLine(total, 0))
|
||||
|
||||
assert.Equal(t,
|
||||
env.snapshotPaths(t, env.snapshot(t, "first").ID.String()),
|
||||
env.snapshotPaths(t, env.snapshot(t, "second").ID.String()))
|
||||
}
|
||||
|
||||
// A directory whose mtime changed and a symlink whose target changed are
|
||||
// recorded again by the next backup, which counts only the symlink as
|
||||
// backed up.
|
||||
func TestSnapshotSummaryChangedDirectoryAndSymlink(t *testing.T) {
|
||||
log.Initialize(log.Config{})
|
||||
t.Parallel()
|
||||
|
||||
env := newSummaryEnv(t)
|
||||
|
||||
// The local index keeps paths with symlinks resolved.
|
||||
dir, err := filepath.EvalSymlinks(filepath.Dir(env.aPath))
|
||||
require.NoError(t, err)
|
||||
|
||||
// The new link gets the old one's mtime, and a target, which need not
|
||||
// exist, of the old target's length, a symlink's size: only the
|
||||
// target differs.
|
||||
linkTime := unix.NsecToTimeval(
|
||||
time.Date(2029, time.March, 4, 5, 6, 7, 0, time.UTC).UnixNano())
|
||||
link := filepath.Join(dir, "link")
|
||||
require.NoError(t, os.Symlink("a.bin", link))
|
||||
require.NoError(t, unix.Lutimes(link, []unix.Timeval{linkTime, linkTime}))
|
||||
|
||||
env.backUp(t, "first", false)
|
||||
|
||||
require.NoError(t, os.Remove(link))
|
||||
require.NoError(t, os.Symlink("b.bin", link))
|
||||
require.NoError(t, unix.Lutimes(link, []unix.Timeval{linkTime, linkTime}))
|
||||
|
||||
mtime := time.Date(2030, time.January, 2, 3, 4, 5, 0, time.UTC)
|
||||
require.NoError(t, os.Chtimes(dir, mtime, mtime))
|
||||
|
||||
assert.Contains(t, env.backUp(t, "second", false), filesLine(4, 1, 3))
|
||||
|
||||
ctx := context.Background()
|
||||
|
||||
dirRow, err := env.repos.Files.GetByPath(ctx, dir)
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, dirRow)
|
||||
assert.Equal(t, mtime.Unix(), dirRow.MTime.Unix())
|
||||
|
||||
linkRow, err := env.repos.Files.GetByPath(ctx, link)
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, linkRow)
|
||||
assert.Equal(t, "b.bin", linkRow.LinkTarget.String())
|
||||
}
|
||||
|
||||
// Under --cron the progress reporter is off; the upload figures must
|
||||
// still reach the summary and the snapshots row. The snapshot has two
|
||||
// paths, each backed up by its own scan.
|
||||
|
||||
@@ -28,7 +28,7 @@ func TestBackupWithoutCronOfTwoPathSnapshotRestoresBothPaths(t *testing.T) {
|
||||
tempDir := t.TempDir()
|
||||
firstDir := filepath.Join(tempDir, "first")
|
||||
secondDir := filepath.Join(tempDir, "second")
|
||||
storeDir := filepath.Join(tempDir, "remote")
|
||||
storeDir := t.TempDir()
|
||||
restoreDir := filepath.Join(tempDir, "restored")
|
||||
dbPath := filepath.Join(tempDir, "index.sqlite")
|
||||
|
||||
|
||||
@@ -36,7 +36,7 @@ func TestUncompressedSizeOfChangedFileStaysFixed(t *testing.T) {
|
||||
|
||||
appendRandomBytes(t, fs, files, changedPath, appendedFileSize)
|
||||
|
||||
store, err := storage.NewFileStorer(filepath.Join(tempDir, "remote"))
|
||||
store, err := storage.NewFileStorer(t.TempDir())
|
||||
require.NoError(t, err)
|
||||
|
||||
db, err := database.New(ctx, dbPath)
|
||||
|
||||
@@ -28,7 +28,7 @@ func TestVerify_QuietSuppressesReport(t *testing.T) {
|
||||
tempDir := t.TempDir()
|
||||
|
||||
dataDir := filepath.Join(tempDir, "source")
|
||||
storeDir := filepath.Join(tempDir, "remote")
|
||||
storeDir := t.TempDir()
|
||||
dbPath := filepath.Join(tempDir, "index.sqlite")
|
||||
|
||||
chunkSize := int64(32 * 1024)
|
||||
|
||||
Reference in New Issue
Block a user