Compare commits
1
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
fd5e077fa0 |
@@ -325,7 +325,13 @@ value where a list belongs or a word where a number belongs), is refused.
|
|||||||
`--skip-errors` skipped a file or directory it could not read or list, the
|
`--skip-errors` skipped a file or directory it could not read or list, the
|
||||||
prune does not run and every older snapshot is kept, since one of them may
|
prune does not run and every older snapshot is kept, since one of them may
|
||||||
be the only one that holds what was skipped; the command says so and exits
|
be the only one that holds what was skipped; the command says so and exits
|
||||||
non-zero. A file removed during the backup does not stop the prune.
|
non-zero. When the backup found nothing in a configured path, as in the
|
||||||
|
mount point of a volume that is not mounted, the prune does not run
|
||||||
|
either: every older snapshot is kept, and the command names the path and
|
||||||
|
exits non-zero, on every run until the path holds something again. Take a
|
||||||
|
path that is empty on purpose out of the config, or remove the older
|
||||||
|
snapshots by hand with `snapshot purge`. A file removed during the backup
|
||||||
|
does not stop the prune.
|
||||||
* `--keep-newer-than <duration>`: With `--prune`, keep snapshots newer than
|
* `--keep-newer-than <duration>`: With `--prune`, keep snapshots newer than
|
||||||
this duration instead of only the latest (e.g. `4w`, `30d`, `6mo`, `1y`).
|
this duration instead of only the latest (e.g. `4w`, `30d`, `6mo`, `1y`).
|
||||||
Without `--prune`, or with a value that is not a valid duration, the
|
Without `--prune`, or with a value that is not a valid duration, the
|
||||||
@@ -334,6 +340,8 @@ value where a list belongs or a word where a number belongs), is refused.
|
|||||||
does not exist, or when the local index holds snapshots and the
|
does not exist, or when the local index holds snapshots and the
|
||||||
destination has no `vaultik-destination.txt` (see
|
destination has no `vaultik-destination.txt` (see
|
||||||
[remote storage layout](#remote-storage-layout))
|
[remote storage layout](#remote-storage-layout))
|
||||||
|
* Fails, before anything is pruned, when a configured path is a symlink
|
||||||
|
whose target does not exist, as one into a volume that is not mounted
|
||||||
|
|
||||||
**`snapshot list`**: Show every snapshot known to this host — the union
|
**`snapshot list`**: Show every snapshot known to this host — the union
|
||||||
of the local index and the backup destination store — with timestamps
|
of the local index and the backup destination store — with timestamps
|
||||||
|
|||||||
@@ -24,6 +24,14 @@ owner's; a `v*` tag runs the release build in
|
|||||||
|
|
||||||
# Completed Steps
|
# Completed Steps
|
||||||
|
|
||||||
|
- 2026-10-14: Stopped `snapshot create --prune` pruning after a backup
|
||||||
|
that found nothing in a configured path
|
||||||
|
([issue #388](https://git.eeqj.de/sneak/vaultik/issues/388)), as in
|
||||||
|
the mount point of a volume that is not mounted. Such a prune could
|
||||||
|
delete every snapshot still holding that path's files. A configured
|
||||||
|
path that is a symlink whose target is gone, as one into such a
|
||||||
|
volume, now fails the backup, with or without `--prune`.
|
||||||
|
|
||||||
- 2026-10-14: Stopped `snapshot create --prune` pruning after a backup
|
- 2026-10-14: Stopped `snapshot create --prune` pruning after a backup
|
||||||
that skipped files it could not read
|
that skipped files it could not read
|
||||||
([issue #382](https://git.eeqj.de/sneak/vaultik/issues/382)). Such a
|
([issue #382](https://git.eeqj.de/sneak/vaultik/issues/382)). Such a
|
||||||
|
|||||||
@@ -87,6 +87,36 @@ func TestEntrySnapshotCreateReportsPostBackupPruneFailure(t *testing.T) {
|
|||||||
"the new snapshot is on the store beside the planted one")
|
"the new snapshot is on the store beside the planted one")
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// TestEntrySnapshotCreatePruneExitsNonZeroWhenSourceIsEmpty backs up,
|
||||||
|
// empties the source directory, as a volume that is not mounted leaves
|
||||||
|
// its mount point, and runs `snapshot create --prune`. The command exits
|
||||||
|
// non-zero and reports the backup as completed, not as a failed snapshot
|
||||||
|
// creation. See https://git.eeqj.de/sneak/vaultik/issues/388.
|
||||||
|
//
|
||||||
|
//nolint:paralleltest // replaces os.Args, os.Stdout, os.Stderr and the xdg globals
|
||||||
|
func TestEntrySnapshotCreatePruneExitsNonZeroWhenSourceIsEmpty(t *testing.T) {
|
||||||
|
configPath, storeDir := writeBackupConfig(t)
|
||||||
|
|
||||||
|
code, _, _ := runEntry(t, flagConfig, configPath, cmdSnapshot, cmdCreate)
|
||||||
|
require.Equal(t, 0, code)
|
||||||
|
|
||||||
|
// writeBackupConfig puts the source directory beside the config file.
|
||||||
|
require.NoError(t, os.Remove(
|
||||||
|
filepath.Join(filepath.Dir(configPath), "source", "file.txt")))
|
||||||
|
|
||||||
|
code, _, stderr := runEntry(t,
|
||||||
|
flagConfig, configPath, cmdSnapshot, cmdCreate, flagPrune)
|
||||||
|
|
||||||
|
assert.Equal(t, 1, code)
|
||||||
|
assert.Contains(t, stderr,
|
||||||
|
"Backup completed, but 1 source path(s) were empty")
|
||||||
|
assert.NotContains(t, stderr, "Snapshot creation failed")
|
||||||
|
|
||||||
|
snapshots, err := os.ReadDir(filepath.Join(storeDir, "metadata"))
|
||||||
|
require.NoError(t, err)
|
||||||
|
assert.Len(t, snapshots, 2, "the older snapshot is kept")
|
||||||
|
}
|
||||||
|
|
||||||
// writeBackupConfig builds a hermetic config with one small file in its
|
// writeBackupConfig builds a hermetic config with one small file in its
|
||||||
// source directory and an empty destination store directory. Returns the
|
// source directory and an empty destination store directory. Returns the
|
||||||
// config path and the store directory.
|
// config path and the store directory.
|
||||||
|
|||||||
@@ -115,8 +115,9 @@ on Linux), then /etc/vaultik/config.yml.`,
|
|||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
if errors.Is(err, vaultik.ErrFilesUnreadable) {
|
if errors.Is(err, vaultik.ErrFilesUnreadable) ||
|
||||||
log.Error("Backup completed with unreadable files", "error", err)
|
errors.Is(err, vaultik.ErrSourcePathEmpty) {
|
||||||
|
log.Error("Backup completed with an error", "error", err)
|
||||||
ReportErrorf("Backup completed, but %v", err)
|
ReportErrorf("Backup completed, but %v", err)
|
||||||
|
|
||||||
return
|
return
|
||||||
@@ -133,7 +134,8 @@ on Linux), then /etc/vaultik/config.yml.`,
|
|||||||
cmd.Flags().BoolVar(&opts.Prune, "prune", false,
|
cmd.Flags().BoolVar(&opts.Prune, "prune", false,
|
||||||
"After backup, drop older snapshots of the same name and remove "+
|
"After backup, drop older snapshots of the same name and remove "+
|
||||||
"orphaned blobs older than prune_grace_period (not after a "+
|
"orphaned blobs older than prune_grace_period (not after a "+
|
||||||
"backup that skipped unreadable files)")
|
"backup that skipped unreadable files or found a configured "+
|
||||||
|
"path empty)")
|
||||||
cmd.Flags().StringVar(&opts.KeepNewerThan, "keep-newer-than", "",
|
cmd.Flags().StringVar(&opts.KeepNewerThan, "keep-newer-than", "",
|
||||||
"With --prune: keep snapshots newer than this duration "+
|
"With --prune: keep snapshots newer than this duration "+
|
||||||
"(e.g. 30d, 4w, 6mo, 1y; m is minutes, mo is months) "+
|
"(e.g. 30d, 4w, 6mo, 1y; m is minutes, mo is months) "+
|
||||||
|
|||||||
@@ -168,6 +168,12 @@ type ScanResult struct {
|
|||||||
// skipped because it could not be read or listed; the rest are files
|
// skipped because it could not be read or listed; the rest are files
|
||||||
// removed after the walk listed them.
|
// removed after the walk listed them.
|
||||||
FilesUnreadable int
|
FilesUnreadable int
|
||||||
|
|
||||||
|
// EntriesFound counts the entries the walk found and could look up
|
||||||
|
// that no exclude pattern matches, leaving out the scanned path
|
||||||
|
// itself when it is a directory. It is zero for an empty directory,
|
||||||
|
// such as the mount point of a volume that is not mounted.
|
||||||
|
EntriesFound int
|
||||||
}
|
}
|
||||||
|
|
||||||
// NewScanner creates a new scanner instance
|
// NewScanner creates a new scanner instance
|
||||||
@@ -928,6 +934,11 @@ func (s *Scanner) scanPhase(
|
|||||||
default:
|
default:
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// The scanned directory itself is not an entry found in it.
|
||||||
|
if filePath != path || !info.IsDir() {
|
||||||
|
result.EntriesFound++
|
||||||
|
}
|
||||||
|
|
||||||
// Handle symlinks and directories
|
// Handle symlinks and directories
|
||||||
handled, err := s.recordSpecialEntry(
|
handled, err := s.recordSpecialEntry(
|
||||||
filePath, info, existingFiles, knownFiles, collector, result)
|
filePath, info, existingFiles, knownFiles, collector, result)
|
||||||
@@ -940,18 +951,8 @@ func (s *Scanner) scanPhase(
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// Track this file as existing (for deletion detection)
|
filesScanned, changedCount := s.recordRegularFile(
|
||||||
existingFiles[filePath] = struct{}{}
|
filePath, info, existingFiles, knownFiles, collector, result)
|
||||||
|
|
||||||
// Check file against in-memory map (no DB query!)
|
|
||||||
file := s.buildRegularFileEntry(filePath, info)
|
|
||||||
needsProcessing := s.checkFileInMemory(file, knownFiles)
|
|
||||||
|
|
||||||
filesScanned, changedCount := collector.recordFile(
|
|
||||||
filePath, info, file, needsProcessing)
|
|
||||||
|
|
||||||
// Update result stats
|
|
||||||
s.updateScanEntryStats(result, needsProcessing, info)
|
|
||||||
|
|
||||||
// Output periodic status
|
// Output periodic status
|
||||||
if time.Since(lastStatusTime) >= scanStatusInterval {
|
if time.Since(lastStatusTime) >= scanStatusInterval {
|
||||||
@@ -1027,6 +1028,30 @@ func (s *Scanner) recordSpecialEntry(
|
|||||||
return false, nil
|
return false, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// recordRegularFile records a regular file as recordSpecialEntry does a
|
||||||
|
// symlink and returns the running scanned and changed counts.
|
||||||
|
func (s *Scanner) recordRegularFile(
|
||||||
|
filePath string, info os.FileInfo,
|
||||||
|
existingFiles map[string]struct{},
|
||||||
|
knownFiles map[string]*database.File,
|
||||||
|
collector *scanCollector, result *ScanResult,
|
||||||
|
) (int64, int) {
|
||||||
|
// Track this file as existing (for deletion detection)
|
||||||
|
existingFiles[filePath] = struct{}{}
|
||||||
|
|
||||||
|
// Check file against in-memory map (no DB query!)
|
||||||
|
file := s.buildRegularFileEntry(filePath, info)
|
||||||
|
needsProcessing := s.checkFileInMemory(file, knownFiles)
|
||||||
|
|
||||||
|
filesScanned, changedCount := collector.recordFile(
|
||||||
|
filePath, info, file, needsProcessing)
|
||||||
|
|
||||||
|
// Update result stats
|
||||||
|
s.updateScanEntryStats(result, needsProcessing, info)
|
||||||
|
|
||||||
|
return filesScanned, changedCount
|
||||||
|
}
|
||||||
|
|
||||||
// countRemovedDuringWalk counts an entry that was in its directory's
|
// countRemovedDuringWalk counts an entry that was in its directory's
|
||||||
// listing but was gone when the walk looked it up, like a file removed
|
// listing but was gone when the walk looked it up, like a file removed
|
||||||
// before phase 2 reads it: as examined and failed, and not as an error,
|
// before phase 2 reads it: as examined and failed, and not as an error,
|
||||||
|
|||||||
@@ -44,6 +44,14 @@ var ErrPostBackupPruneFailed = errors.New("post-backup prune failed")
|
|||||||
// kept, so the caller can report it apart from a failed backup.
|
// kept, so the caller can report it apart from a failed backup.
|
||||||
var ErrFilesUnreadable = errors.New("file(s) could not be read")
|
var ErrFilesUnreadable = errors.New("file(s) could not be read")
|
||||||
|
|
||||||
|
// ErrSourcePathEmpty is returned when --prune skipped the post-backup
|
||||||
|
// prune because the scan found nothing in a configured path of a
|
||||||
|
// snapshot, as in the mount point of a volume that is not mounted. The
|
||||||
|
// snapshots are complete and kept, so the caller can report it apart
|
||||||
|
// from a failed backup. A run that also skipped unreadable files
|
||||||
|
// returns ErrFilesUnreadable instead.
|
||||||
|
var ErrSourcePathEmpty = errors.New("source path(s) were empty")
|
||||||
|
|
||||||
// listRecentLimit caps how many snapshot rows are fetched from the
|
// listRecentLimit caps how many snapshot rows are fetched from the
|
||||||
// local index for listing and purge operations.
|
// local index for listing and purge operations.
|
||||||
const listRecentLimit = 10000
|
const listRecentLimit = 10000
|
||||||
@@ -134,17 +142,10 @@ func (v *Vaultik) CreateSnapshot(opts *SnapshotCreateOptions) error {
|
|||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
// Process each named snapshot
|
filesUnreadable, emptyPaths, err := v.createNamedSnapshots(
|
||||||
filesUnreadable := 0
|
opts, hostname, snapshotNames)
|
||||||
|
if err != nil {
|
||||||
for snapIdx, snapName := range snapshotNames {
|
return err
|
||||||
unreadable, err := v.createNamedSnapshot(
|
|
||||||
opts, hostname, snapName, snapIdx+1, len(snapshotNames))
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
filesUnreadable += unreadable
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// Print overall summary if multiple snapshots
|
// Print overall summary if multiple snapshots
|
||||||
@@ -154,13 +155,8 @@ func (v *Vaultik) CreateSnapshot(opts *SnapshotCreateOptions) error {
|
|||||||
}
|
}
|
||||||
|
|
||||||
switch {
|
switch {
|
||||||
case opts.Prune && filesUnreadable > 0:
|
case opts.Prune && (filesUnreadable > 0 || len(emptyPaths) > 0):
|
||||||
// An older snapshot may be the only one that still holds a file
|
return v.skipPostBackupPrune(filesUnreadable, emptyPaths)
|
||||||
// this run could not read.
|
|
||||||
log.Warn("Skipping post-backup prune",
|
|
||||||
"files_unreadable", filesUnreadable)
|
|
||||||
v.UI.Warningf("Skipped the post-backup prune because %d file(s) "+
|
|
||||||
"could not be read; older snapshots are kept.", filesUnreadable)
|
|
||||||
case opts.Prune:
|
case opts.Prune:
|
||||||
err = v.runPostBackupPrune(snapshotNames, opts.KeepNewerThan)
|
err = v.runPostBackupPrune(snapshotNames, opts.KeepNewerThan)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -171,6 +167,61 @@ func (v *Vaultik) CreateSnapshot(opts *SnapshotCreateOptions) error {
|
|||||||
return v.finishCreateSnapshot(filesUnreadable)
|
return v.finishCreateSnapshot(filesUnreadable)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// createNamedSnapshots creates each named snapshot in turn. It returns
|
||||||
|
// the number of entries --skip-errors skipped because it could not read
|
||||||
|
// or list them, and each configured path in which the scan found
|
||||||
|
// nothing, named with its snapshot.
|
||||||
|
func (v *Vaultik) createNamedSnapshots(
|
||||||
|
opts *SnapshotCreateOptions, hostname string, snapshotNames []string,
|
||||||
|
) (int, []string, error) {
|
||||||
|
filesUnreadable := 0
|
||||||
|
|
||||||
|
var emptyPaths []string
|
||||||
|
|
||||||
|
for snapIdx, snapName := range snapshotNames {
|
||||||
|
stats, err := v.createNamedSnapshot(
|
||||||
|
opts, hostname, snapName, snapIdx+1, len(snapshotNames))
|
||||||
|
if err != nil {
|
||||||
|
return 0, nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
filesUnreadable += stats.totalFilesUnreadable
|
||||||
|
|
||||||
|
for _, dir := range stats.emptyPaths {
|
||||||
|
emptyPaths = append(emptyPaths, dir+" of snapshot "+snapName)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return filesUnreadable, emptyPaths, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// skipPostBackupPrune says why the post-backup prune does not run, naming
|
||||||
|
// every cause, and returns the error the run ends with. An older snapshot
|
||||||
|
// may be the only one that still holds a file this run could not read,
|
||||||
|
// or what an empty path held. The previous snapshot is no guide: after
|
||||||
|
// one run with an empty path it is empty too.
|
||||||
|
func (v *Vaultik) skipPostBackupPrune(
|
||||||
|
filesUnreadable int, emptyPaths []string,
|
||||||
|
) error {
|
||||||
|
log.Warn("Skipping post-backup prune",
|
||||||
|
"files_unreadable", filesUnreadable, "empty_paths", emptyPaths)
|
||||||
|
|
||||||
|
if len(emptyPaths) > 0 {
|
||||||
|
v.UI.Warningf("Skipped the post-backup prune because nothing was "+
|
||||||
|
"found in %s; older snapshots are kept.",
|
||||||
|
strings.Join(emptyPaths, ", "))
|
||||||
|
}
|
||||||
|
|
||||||
|
if filesUnreadable > 0 {
|
||||||
|
v.UI.Warningf("Skipped the post-backup prune because %d file(s) "+
|
||||||
|
"could not be read; older snapshots are kept.", filesUnreadable)
|
||||||
|
|
||||||
|
return fmt.Errorf("%d %w", filesUnreadable, ErrFilesUnreadable)
|
||||||
|
}
|
||||||
|
|
||||||
|
return fmt.Errorf("%d %w", len(emptyPaths), ErrSourcePathEmpty)
|
||||||
|
}
|
||||||
|
|
||||||
// finishCreateSnapshot ends a run whose snapshots are all complete and
|
// finishCreateSnapshot ends a run whose snapshots are all complete and
|
||||||
// kept. Like finishRestore, it returns an error when --skip-errors
|
// kept. Like finishRestore, it returns an error when --skip-errors
|
||||||
// skipped anything it could not read or list; otherwise it prints the
|
// skipped anything it could not read or list; otherwise it prints the
|
||||||
@@ -274,6 +325,9 @@ type snapshotStats struct {
|
|||||||
// could not be read or listed.
|
// could not be read or listed.
|
||||||
totalFilesUnreadable int
|
totalFilesUnreadable int
|
||||||
|
|
||||||
|
// The configured paths in which the scan found nothing.
|
||||||
|
emptyPaths []string
|
||||||
|
|
||||||
// The sizes of all blobs the snapshot references, set by
|
// The sizes of all blobs the snapshot references, set by
|
||||||
// finalizeSnapshotMetadata once snapshot_blobs is populated.
|
// finalizeSnapshotMetadata once snapshot_blobs is populated.
|
||||||
blobSize int64
|
blobSize int64
|
||||||
@@ -281,11 +335,10 @@ type snapshotStats struct {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// createNamedSnapshot creates a single named snapshot and returns the
|
// createNamedSnapshot creates a single named snapshot and returns the
|
||||||
// number of entries --skip-errors skipped because it could not read or
|
// statistics of its scan.
|
||||||
// list them.
|
|
||||||
func (v *Vaultik) createNamedSnapshot(
|
func (v *Vaultik) createNamedSnapshot(
|
||||||
opts *SnapshotCreateOptions, hostname, snapName string, idx, total int,
|
opts *SnapshotCreateOptions, hostname, snapName string, idx, total int,
|
||||||
) (int, error) {
|
) (*snapshotStats, error) {
|
||||||
snapshotStartTime := time.Now()
|
snapshotStartTime := time.Now()
|
||||||
|
|
||||||
if total > 1 {
|
if total > 1 {
|
||||||
@@ -294,12 +347,12 @@ func (v *Vaultik) createNamedSnapshot(
|
|||||||
|
|
||||||
resolvedDirs, err := v.resolveSnapshotPaths(snapName)
|
resolvedDirs, err := v.resolveSnapshotPaths(snapName)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return 0, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
previousSnapshotID, err := v.previousSnapshotID(snapName)
|
previousSnapshotID, err := v.previousSnapshotID(snapName)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return 0, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
scanner := v.ScannerFactory(snapshot.ScannerParams{
|
scanner := v.ScannerFactory(snapshot.ScannerParams{
|
||||||
@@ -314,7 +367,7 @@ func (v *Vaultik) createNamedSnapshot(
|
|||||||
snapshotID, err := v.SnapshotManager.CreateSnapshotWithName(
|
snapshotID, err := v.SnapshotManager.CreateSnapshotWithName(
|
||||||
v.ctx, hostname, snapName, v.Globals.Version, v.Globals.Commit)
|
v.ctx, hostname, snapName, v.Globals.Version, v.Globals.Commit)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return 0, fmt.Errorf("creating snapshot: %w", err)
|
return nil, fmt.Errorf("creating snapshot: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
log.Info("Beginning snapshot", "snapshot_id", snapshotID, "name", snapName)
|
log.Info("Beginning snapshot", "snapshot_id", snapshotID, "name", snapName)
|
||||||
@@ -322,12 +375,12 @@ func (v *Vaultik) createNamedSnapshot(
|
|||||||
|
|
||||||
stats, err := v.scanAllDirectories(scanner, resolvedDirs, snapshotID)
|
stats, err := v.scanAllDirectories(scanner, resolvedDirs, snapshotID)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return 0, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
err = v.finalizeSnapshotMetadata(snapshotID, stats)
|
err = v.finalizeSnapshotMetadata(snapshotID, stats)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return 0, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
log.Info("Snapshot complete",
|
log.Info("Snapshot complete",
|
||||||
@@ -340,7 +393,7 @@ func (v *Vaultik) createNamedSnapshot(
|
|||||||
|
|
||||||
v.printSnapshotSummary(snapshotID, snapshotStartTime, stats)
|
v.printSnapshotSummary(snapshotID, snapshotStartTime, stats)
|
||||||
|
|
||||||
return stats.totalFilesUnreadable, nil
|
return stats, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// previousSnapshotID returns the ID of the most recent completed snapshot
|
// previousSnapshotID returns the ID of the most recent completed snapshot
|
||||||
@@ -362,7 +415,8 @@ func (v *Vaultik) previousSnapshotID(snapName string) (string, error) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// resolveSnapshotPaths resolves source directories to absolute paths
|
// resolveSnapshotPaths resolves source directories to absolute paths
|
||||||
// with symlink resolution.
|
// with symlink resolution. A path that is a symlink whose target does
|
||||||
|
// not exist is an error.
|
||||||
func (v *Vaultik) resolveSnapshotPaths(snapName string) ([]string, error) {
|
func (v *Vaultik) resolveSnapshotPaths(snapName string) ([]string, error) {
|
||||||
snapConfig := v.Config.Snapshots[snapName]
|
snapConfig := v.Config.Snapshots[snapName]
|
||||||
resolvedDirs := make([]string, 0, len(snapConfig.Paths))
|
resolvedDirs := make([]string, 0, len(snapConfig.Paths))
|
||||||
@@ -375,12 +429,18 @@ func (v *Vaultik) resolveSnapshotPaths(snapName string) ([]string, error) {
|
|||||||
|
|
||||||
resolvedPath, err := filepath.EvalSymlinks(absPath)
|
resolvedPath, err := filepath.EvalSymlinks(absPath)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
if os.IsNotExist(err) {
|
// A path that does not exist is left for the scan to report.
|
||||||
resolvedPath = absPath
|
// One that exists is a symlink whose target is gone, as one
|
||||||
} else {
|
// into a volume that is not mounted: the scan would record
|
||||||
|
// only the link, and --prune would then drop the older
|
||||||
|
// snapshots that hold the files.
|
||||||
|
_, lstatErr := os.Lstat(absPath)
|
||||||
|
if !os.IsNotExist(err) || lstatErr == nil {
|
||||||
return nil, fmt.Errorf(
|
return nil, fmt.Errorf(
|
||||||
"failed to resolve symlinks for %s: %w", absPath, err)
|
"failed to resolve symlinks for %s: %w", absPath, err)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
resolvedPath = absPath
|
||||||
}
|
}
|
||||||
|
|
||||||
resolvedDirs = append(resolvedDirs, resolvedPath)
|
resolvedDirs = append(resolvedDirs, resolvedPath)
|
||||||
@@ -434,6 +494,10 @@ func (v *Vaultik) scanAllDirectories(
|
|||||||
stats.totalBytesUploaded += result.BytesUploaded
|
stats.totalBytesUploaded += result.BytesUploaded
|
||||||
stats.uploadDuration += result.UploadDuration
|
stats.uploadDuration += result.UploadDuration
|
||||||
|
|
||||||
|
if result.EntriesFound == 0 {
|
||||||
|
stats.emptyPaths = append(stats.emptyPaths, dir)
|
||||||
|
}
|
||||||
|
|
||||||
log.Info("Directory scan complete",
|
log.Info("Directory scan complete",
|
||||||
"path", dir,
|
"path", dir,
|
||||||
"files", result.FilesScanned,
|
"files", result.FilesScanned,
|
||||||
|
|||||||
@@ -1,8 +1,15 @@
|
|||||||
package vaultik_test
|
package vaultik_test
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"context"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
|
"github.com/stretchr/testify/assert"
|
||||||
|
"github.com/stretchr/testify/require"
|
||||||
|
"sneak.berlin/go/vaultik/internal/config"
|
||||||
|
"sneak.berlin/go/vaultik/internal/log"
|
||||||
"sneak.berlin/go/vaultik/internal/vaultik"
|
"sneak.berlin/go/vaultik/internal/vaultik"
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -25,3 +32,230 @@ func TestSnapshotCreateOptions_PruneFlag(t *testing.T) {
|
|||||||
t.Error("Expected Prune to be false")
|
t.Error("Expected Prune to be false")
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// A backup under --prune that finds nothing in a configured path, as in
|
||||||
|
// the mount point of a volume that is not mounted, does not prune: run
|
||||||
|
// after run, the snapshot that holds the files is kept with everything
|
||||||
|
// it stored, and the run fails. That holds whether the empty path is the
|
||||||
|
// snapshot's only path or one of two. See
|
||||||
|
// https://git.eeqj.de/sneak/vaultik/issues/388.
|
||||||
|
func TestBackupWithEmptySourcePathDoesNotPrune(t *testing.T) {
|
||||||
|
log.Initialize(log.Config{})
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
// emptied is the path, relative to the source directory, that is
|
||||||
|
// emptied after the first backup.
|
||||||
|
const (
|
||||||
|
snapshotName = "media"
|
||||||
|
emptied = "music"
|
||||||
|
)
|
||||||
|
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
paths []string // relative to the source directory
|
||||||
|
}{
|
||||||
|
{name: "only path", paths: []string{emptied}},
|
||||||
|
{name: "one of two paths", paths: []string{emptied, "docs"}},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tt := range tests {
|
||||||
|
t.Run(tt.name, func(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
ctx := context.Background()
|
||||||
|
env := newDeletedCountEnv(t, map[string][]byte{
|
||||||
|
emptied + "/a.mp3": []byte("song a"),
|
||||||
|
emptied + "/b.mp3": []byte("song b"),
|
||||||
|
"docs/notes.txt": []byte("notes"),
|
||||||
|
})
|
||||||
|
|
||||||
|
paths := make([]string, 0, len(tt.paths))
|
||||||
|
|
||||||
|
for _, rel := range tt.paths {
|
||||||
|
paths = append(paths, filepath.Join(env.src, rel))
|
||||||
|
}
|
||||||
|
|
||||||
|
env.v.Config.Snapshots[snapshotName] = config.SnapshotConfig{
|
||||||
|
Paths: paths,
|
||||||
|
}
|
||||||
|
env.backUp(t, snapshotName)
|
||||||
|
|
||||||
|
firstID := localSnapshotID(ctx, t, env.v.Repositories, snapshotName)
|
||||||
|
stored, err := env.v.Storage.List(ctx, "")
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
// A volume that is not mounted leaves its mount point behind
|
||||||
|
// as an empty directory.
|
||||||
|
emptiedDir := filepath.Join(env.src, emptied)
|
||||||
|
require.NoError(t, os.RemoveAll(emptiedDir))
|
||||||
|
require.NoError(t, os.Mkdir(emptiedDir, 0o750))
|
||||||
|
|
||||||
|
for range 2 {
|
||||||
|
env.out.Reset()
|
||||||
|
|
||||||
|
err := env.v.CreateSnapshot(&vaultik.SnapshotCreateOptions{
|
||||||
|
Prune: true,
|
||||||
|
Snapshots: []string{snapshotName},
|
||||||
|
})
|
||||||
|
require.ErrorIs(t, err, vaultik.ErrSourcePathEmpty)
|
||||||
|
assert.Contains(t, env.out.String(),
|
||||||
|
"Skipped the post-backup prune because nothing was found in ")
|
||||||
|
assert.Contains(t, env.out.String(),
|
||||||
|
emptied+" of snapshot "+snapshotName+";")
|
||||||
|
assert.NotContains(t, env.out.String(), "Running post-backup prune.")
|
||||||
|
}
|
||||||
|
|
||||||
|
snapshots, err := env.v.Repositories.Snapshots.ListRecent(
|
||||||
|
ctx, listRecentTestLimit)
|
||||||
|
require.NoError(t, err)
|
||||||
|
require.Len(t, snapshots, 3)
|
||||||
|
assert.Equal(t, firstID, snapshots[2].ID.String(),
|
||||||
|
"the snapshot that holds the files is kept")
|
||||||
|
|
||||||
|
for _, key := range stored {
|
||||||
|
_, err := env.v.Storage.Stat(ctx, key)
|
||||||
|
require.NoError(t, err, "%s is kept", key)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A backup under --prune of a configured path that is a symlink whose
|
||||||
|
// target is gone, as one into a volume that is not mounted, fails before
|
||||||
|
// anything is pruned: the snapshot that holds the files is kept with
|
||||||
|
// everything it stored. See https://git.eeqj.de/sneak/vaultik/issues/388.
|
||||||
|
func TestBackupThroughSymlinkWhoseTargetIsGoneDoesNotPrune(t *testing.T) {
|
||||||
|
log.Initialize(log.Config{})
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
const snapshotName = "photos"
|
||||||
|
|
||||||
|
ctx := context.Background()
|
||||||
|
env := newDeletedCountEnv(t, map[string][]byte{
|
||||||
|
"volume/a.jpg": []byte("photo a"),
|
||||||
|
"volume/b.jpg": []byte("photo b"),
|
||||||
|
})
|
||||||
|
|
||||||
|
target := filepath.Join(env.src, "volume")
|
||||||
|
link := filepath.Join(env.src, "photos")
|
||||||
|
require.NoError(t, os.Symlink(target, link))
|
||||||
|
env.setSnapshot(snapshotName, "photos")
|
||||||
|
env.backUp(t, snapshotName)
|
||||||
|
|
||||||
|
firstID := localSnapshotID(ctx, t, env.v.Repositories, snapshotName)
|
||||||
|
stored, err := env.v.Storage.List(ctx, "")
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
// The volume is not mounted, so the symlink points at nothing.
|
||||||
|
require.NoError(t, os.RemoveAll(target))
|
||||||
|
|
||||||
|
env.out.Reset()
|
||||||
|
err = env.v.CreateSnapshot(&vaultik.SnapshotCreateOptions{
|
||||||
|
Prune: true,
|
||||||
|
Snapshots: []string{snapshotName},
|
||||||
|
})
|
||||||
|
require.ErrorIs(t, err, os.ErrNotExist)
|
||||||
|
assert.Contains(t, err.Error(), link)
|
||||||
|
assert.NotContains(t, env.out.String(), "Running post-backup prune.")
|
||||||
|
|
||||||
|
snapshots, err := env.v.Repositories.Snapshots.ListRecent(
|
||||||
|
ctx, listRecentTestLimit)
|
||||||
|
require.NoError(t, err)
|
||||||
|
require.Len(t, snapshots, 1)
|
||||||
|
assert.Equal(t, firstID, snapshots[0].ID.String(),
|
||||||
|
"the snapshot that holds the files is kept")
|
||||||
|
|
||||||
|
for _, key := range stored {
|
||||||
|
_, err := env.v.Storage.Stat(ctx, key)
|
||||||
|
require.NoError(t, err, "%s is kept", key)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A backup under --skip-errors --prune that both skipped a file it could
|
||||||
|
// not read and found nothing in a configured path names the empty path
|
||||||
|
// too when it skips the prune. See
|
||||||
|
// https://git.eeqj.de/sneak/vaultik/issues/388.
|
||||||
|
func TestBackupWithEmptySourcePathAndUnreadableFileNamesBoth(t *testing.T) {
|
||||||
|
log.Initialize(log.Config{})
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
const snapshotName = "media"
|
||||||
|
|
||||||
|
env := newDeletedCountEnv(t, map[string][]byte{
|
||||||
|
"docs/notes.txt": []byte("notes"),
|
||||||
|
})
|
||||||
|
require.NoError(t, os.Mkdir(filepath.Join(env.src, "music"), 0o750))
|
||||||
|
env.v.Config.Snapshots[snapshotName] = config.SnapshotConfig{
|
||||||
|
Paths: []string{
|
||||||
|
filepath.Join(env.src, "docs"),
|
||||||
|
filepath.Join(env.src, "music"),
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
// A mode does not lock out root, who runs the tests, so opening the
|
||||||
|
// file is made to fail instead.
|
||||||
|
env.v.Fs = &openFailFs{
|
||||||
|
path: filepath.Join(env.src, "docs", "notes.txt"),
|
||||||
|
err: os.ErrPermission,
|
||||||
|
}
|
||||||
|
|
||||||
|
err := env.v.CreateSnapshot(&vaultik.SnapshotCreateOptions{
|
||||||
|
Prune: true,
|
||||||
|
SkipErrors: true,
|
||||||
|
Snapshots: []string{snapshotName},
|
||||||
|
})
|
||||||
|
require.ErrorIs(t, err, vaultik.ErrFilesUnreadable)
|
||||||
|
assert.Contains(t, env.out.String(),
|
||||||
|
"Skipped the post-backup prune because 1 file(s) could not be read")
|
||||||
|
assert.Contains(t, env.out.String(),
|
||||||
|
"Skipped the post-backup prune because nothing was found in ")
|
||||||
|
assert.Contains(t, env.out.String(), "music of snapshot "+snapshotName+";")
|
||||||
|
assert.NotContains(t, env.out.String(), "Running post-backup prune.")
|
||||||
|
}
|
||||||
|
|
||||||
|
// A backup under --prune that finds something in each configured path
|
||||||
|
// prunes the previous snapshot, also when a path is a single file. See
|
||||||
|
// https://git.eeqj.de/sneak/vaultik/issues/388.
|
||||||
|
func TestBackupWithNoEmptySourcePathPrunes(t *testing.T) {
|
||||||
|
log.Initialize(log.Config{})
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
const (
|
||||||
|
snapshotName = "album"
|
||||||
|
file = "pictures/one.jpg" // relative to the source directory
|
||||||
|
)
|
||||||
|
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
path string // relative to the source directory
|
||||||
|
}{
|
||||||
|
{name: "directory", path: filepath.Dir(file)},
|
||||||
|
{name: "single file", path: file},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tt := range tests {
|
||||||
|
t.Run(tt.name, func(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
ctx := context.Background()
|
||||||
|
env := newDeletedCountEnv(t, map[string][]byte{
|
||||||
|
file: []byte("picture"),
|
||||||
|
})
|
||||||
|
env.setSnapshot(snapshotName, tt.path)
|
||||||
|
env.backUp(t, snapshotName)
|
||||||
|
|
||||||
|
firstID := localSnapshotID(ctx, t, env.v.Repositories, snapshotName)
|
||||||
|
|
||||||
|
require.NoError(t, env.v.CreateSnapshot(&vaultik.SnapshotCreateOptions{
|
||||||
|
Prune: true,
|
||||||
|
Snapshots: []string{snapshotName},
|
||||||
|
}))
|
||||||
|
|
||||||
|
snapshots, err := env.v.Repositories.Snapshots.ListRecent(
|
||||||
|
ctx, listRecentTestLimit)
|
||||||
|
require.NoError(t, err)
|
||||||
|
require.Len(t, snapshots, 1, "the previous snapshot is pruned")
|
||||||
|
assert.NotEqual(t, firstID, snapshots[0].ID.String())
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user