Compare commits

..
Author SHA1 Message Date
sneak 2aa6fc8f66 Report snapshot blobs missing from the destination in remote info (closes #425)
check / check (push) Waiting to run
remote info now counts the blobs that a manifest lists but the
destination does not hold, with their total size, and names each
snapshot that lists one, since it cannot be restored. When any blob is
missing the command exits non-zero after writing its report. --json
gives the figures in missing_blob_count, missing_blob_size and
snapshots_missing_blobs.

Model: opus-5-5
2026-11-19 08:24:27 +00:00
10 changed files with 182 additions and 345 deletions

No files matched your search

+6 -4
View File
@@ -554,6 +554,12 @@ count and blob size unknown, `null` in `--json`. A file under `blobs/` whose
name is not a blob hash is not counted as a blob, since `prune` never deletes name is not a blob hash is not counted as a blob, since `prune` never deletes
it. it.
A blob that a manifest lists but the destination does not hold is reported as
missing, with a count and size. The report names each snapshot that lists a
missing blob, since that snapshot cannot be restored. The command then exits
non-zero. `--json` gives these in `missing_blob_count`, `missing_blob_size` and
`snapshots_missing_blobs`.
- `--json`: Output as JSON - `--json`: Output as JSON
**`remote nuke`**: Delete every object under `metadata/` and `blobs/` on the **`remote nuke`**: Delete every object under `metadata/` and `blobs/` on the
@@ -618,10 +624,6 @@ being written, a killed upload can leave a truncated object under its name,
which a later backup takes for complete. A leftover `.partial` file is ignored which a later backup takes for complete. A leftover `.partial` file is ignored
and can be deleted. and can be deleted.
A blob upload that fails, such as one cut off by a lost connection, is tried
again up to five times, three seconds apart. Each new try is announced with a
warning. If the last try fails too, the backup stops with the upload's error.
A destination shared between hosts or users, each with its own local index, A destination shared between hosts or users, each with its own local index,
relies on `prune_grace_period` to keep one's prune from deleting the blobs of relies on `prune_grace_period` to keep one's prune from deleting the blobs of
another's backup still running, or its directory under `metadata/` that has no another's backup still running, or its directory under `metadata/` that has no
+3 -1
View File
@@ -75,7 +75,9 @@ func newRemoteInfoCommand() *cobra.Command {
- Size of all snapshot metadata (per snapshot and total) - Size of all snapshot metadata (per snapshot and total)
- Count and total size of all blobs - Count and total size of all blobs
- Count and size of referenced blobs (from all manifests) - Count and size of referenced blobs (from all manifests)
- Count and size of orphaned blobs (not referenced by any manifest)`, - Count and size of orphaned blobs (not referenced by any manifest)
- Count and size of referenced blobs missing from remote storage, and the
snapshots that list them; the command exits non-zero when any are missing`,
Args: cobra.NoArgs, Args: cobra.NoArgs,
RunE: func(cmd *cobra.Command, _ []string) error { RunE: func(cmd *cobra.Command, _ []string) error {
// Use unified config resolution // Use unified config resolution
-154
View File
@@ -1,154 +0,0 @@
//nolint:testpackage // exercises the unexported uploadBlobIfNeeded
package snapshot
import (
"bytes"
"context"
"io"
"path/filepath"
"testing"
"time"
"github.com/spf13/afero"
"github.com/stretchr/testify/require"
"sneak.berlin/go/vaultik/internal/blob"
"sneak.berlin/go/vaultik/internal/config"
"sneak.berlin/go/vaultik/internal/storage"
"sneak.berlin/go/vaultik/internal/storage/faultstore"
"sneak.berlin/go/vaultik/internal/ui"
)
// retryTestBlobSize is larger than what a failing faultstore Put reads,
// so a new try that did not rewind the blob would store a short object.
const retryTestBlobSize = 4096
// retryTestBlobKey is where the retry tests upload their blob.
const retryTestBlobKey = "blobs/retry"
// uploadRetryTestBlob uploads a retryTestBlobSize blob through store,
// waiting delay between tries, and returns the blob's bytes, the warnings
// the upload printed and its error.
func uploadRetryTestBlob(
ctx context.Context, store storage.Storer, delay time.Duration,
) ([]byte, int, error) {
data := bytes.Repeat([]byte("0123456789abcdef"), retryTestBlobSize/16)
output := ui.NewWithColor(io.Discard, false)
s := &Scanner{storage: store, ui: output, blobUploadRetryDelay: delay}
_, _, err := s.uploadBlobIfNeeded(ctx, retryTestBlobKey, &blob.WithReader{
FinishedBlob: &blob.FinishedBlob{
Hash: "retry",
Compressed: int64(len(data)),
},
Reader: bytes.NewReader(data),
}, time.Now(), &ScanResult{})
return data, output.WarningCount(), err
}
// A blob upload that fails is tried again from the start of the blob,
// and each failure is a warning. See
// https://git.eeqj.de/sneak/vaultik/issues/476.
func TestFailedBlobUploadIsTriedAgain(t *testing.T) {
t.Parallel()
inner, err := storage.NewFileStorer(t.TempDir())
require.NoError(t, err)
puts := 0
store := faultstore.New(inner)
store.OnPut = func(string) faultstore.PutAction {
puts++
if puts <= 2 {
return faultstore.PutFail
}
return faultstore.PutNormal
}
data, warnings, err := uploadRetryTestBlob(context.Background(), store, 0)
require.NoError(t, err)
require.Equal(t, 3, puts)
require.Equal(t, 2, warnings)
rc, err := inner.Get(context.Background(), retryTestBlobKey)
require.NoError(t, err)
defer func() { _ = rc.Close() }()
stored, err := io.ReadAll(rc)
require.NoError(t, err)
require.Equal(t, data, stored)
}
// A blob upload that keeps failing is tried blobUploadRetries more times,
// BlobUploadRetryDelay apart, and then the backup stops with its error.
func TestBlobUploadStopsAfterRetries(t *testing.T) {
t.Parallel()
inner, err := storage.NewFileStorer(t.TempDir())
require.NoError(t, err)
puts := 0
store := faultstore.New(inner)
store.OnPut = func(string) faultstore.PutAction {
puts++
return faultstore.PutFail
}
const delay = 10 * time.Millisecond
start := time.Now()
_, warnings, err := uploadRetryTestBlob(context.Background(), store, delay)
require.ErrorIs(t, err, faultstore.ErrInjectedUpload)
require.Equal(t, 1+blobUploadRetries, puts)
require.Equal(t, blobUploadRetries, warnings)
require.GreaterOrEqual(t, time.Since(start), blobUploadRetries*delay)
}
// An upload stopped by Ctrl-C or SIGTERM is not tried again.
func TestInterruptedBlobUploadIsNotTriedAgain(t *testing.T) {
t.Parallel()
inner, err := storage.NewFileStorer(t.TempDir())
require.NoError(t, err)
ctx, cancel := context.WithCancel(context.Background())
t.Cleanup(cancel)
puts := 0
store := faultstore.New(inner)
store.OnPut = func(string) faultstore.PutAction {
puts++
cancel()
return faultstore.PutFail
}
_, warnings, err := uploadRetryTestBlob(ctx, store, 0)
require.Error(t, err)
require.Equal(t, 1, puts)
require.Zero(t, warnings)
}
// Backups wait BlobUploadRetryDelay before each new try.
func TestBackupScannerWaitsBetweenBlobUploadRetries(t *testing.T) {
t.Parallel()
cfg := &config.Config{
AgeRecipients: []string{testAgeRecipient},
ChunkSize: config.Size(1024 * 1024),
BlobSizeLimit: config.Size(4 * 1024 * 1024),
IndexPath: filepath.Join(t.TempDir(), "index.sqlite"),
}
scanner := provideScannerFactory(cfg, nil, nil)(ScannerParams{
Fs: afero.NewMemMapFs(),
})
require.NotNil(t, scanner)
require.Equal(t, BlobUploadRetryDelay, scanner.blobUploadRetryDelay)
}
-35
View File
@@ -4,10 +4,8 @@ package snapshot
import ( import (
"os" "os"
"path/filepath" "path/filepath"
"syscall"
"testing" "testing"
"github.com/spf13/afero"
"github.com/stretchr/testify/assert" "github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require" "github.com/stretchr/testify/require"
) )
@@ -40,36 +38,3 @@ func TestDeviceNumberFromStat(t *testing.T) {
// /dev is a different filesystem from the temp dir on Linux, macOS and in Docker. // /dev is a different filesystem from the temp dir on Linux, macOS and in Docker.
assert.NotEqual(t, deviceOf(dir), deviceOf("/dev")) assert.NotEqual(t, deviceOf(dir), deviceOf("/dev"))
} }
// fileInfoWithStat is a file's info whose Sys returns stat.
type fileInfoWithStat struct {
os.FileInfo
stat *syscall.Stat_t
}
func (i fileInfoWithStat) Sys() any { return i.stat }
// The in-memory test filesystem has no device numbers. The pinned afero
// gives no stat; its unreleased master gives a stat holding only uid, gid
// and size. See https://git.eeqj.de/sneak/vaultik/issues/462.
func TestDeviceNumberFromStatInMemory(t *testing.T) {
t.Parallel()
fs := afero.NewMemMapFs()
require.NoError(t, afero.WriteFile(fs, "/file.txt", []byte("data"), 0o600))
info, err := fs.Stat("/file.txt")
require.NoError(t, err)
_, ok := deviceNumberFromStat(info)
assert.False(t, ok, "device number read from a file info with no stat")
zeroed := fileInfoWithStat{
FileInfo: info,
stat: &syscall.Stat_t{Uid: 1000, Gid: 1000, Size: 4},
}
_, ok = deviceNumberFromStat(zeroed)
assert.False(t, ok, "device number read from a stat with no device or inode number")
}
+14 -15
View File
@@ -54,21 +54,20 @@ func provideScannerFactory(
} }
return NewScanner(ScannerConfig{ return NewScanner(ScannerConfig{
FS: params.Fs, FS: params.Fs,
ChunkSize: cfg.ChunkSize.Int64(), ChunkSize: cfg.ChunkSize.Int64(),
Repositories: repos, Repositories: repos,
Storage: storer, Storage: storer,
MaxBlobSize: cfg.BlobSizeLimit.Int64(), MaxBlobSize: cfg.BlobSizeLimit.Int64(),
CompressionLevel: cfg.CompressionLevel, CompressionLevel: cfg.CompressionLevel,
AgeRecipients: cfg.AgeRecipients, AgeRecipients: cfg.AgeRecipients,
EnableProgress: params.EnableProgress, EnableProgress: params.EnableProgress,
UI: params.UI, UI: params.UI,
Exclude: excludes, Exclude: excludes,
IndexAndLockFiles: indexAndLockFiles(cfg.IndexPath, pidlock.Dir()), IndexAndLockFiles: indexAndLockFiles(cfg.IndexPath, pidlock.Dir()),
SkipErrors: params.SkipErrors, SkipErrors: params.SkipErrors,
PreviousSnapshotID: params.PreviousSnapshotID, PreviousSnapshotID: params.PreviousSnapshotID,
BlobUploadRetryDelay: BlobUploadRetryDelay, DeviceNumber: params.DeviceNumber,
DeviceNumber: params.DeviceNumber,
}) })
} }
} }
+24 -75
View File
@@ -75,8 +75,6 @@ type Scanner struct {
// skipErrors skips files that cannot be opened or read (logged loudly); // skipErrors skips files that cannot be opened or read (logged loudly);
// packer, database, encryption, and upload errors still abort the run. // packer, database, encryption, and upload errors still abort the run.
skipErrors bool skipErrors bool
// blobUploadRetryDelay is ScannerConfig.BlobUploadRetryDelay.
blobUploadRetryDelay time.Duration
// ui is the user-facing output; never nil (defaults to a discarding writer). // ui is the user-facing output; never nil (defaults to a discarding writer).
ui *ui.Writer ui *ui.Writer
// deviceNumber returns the number of the device that holds a file; // deviceNumber returns the number of the device that holds a file;
@@ -125,16 +123,8 @@ const (
// snapshotAssocBatchSize is how many file IDs are associated with the // snapshotAssocBatchSize is how many file IDs are associated with the
// snapshot per transaction. // snapshot per transaction.
snapshotAssocBatchSize = 1000 snapshotAssocBatchSize = 1000
// blobUploadRetries is how many more times a failed blob upload is
// tried before the backup stops with the error.
blobUploadRetries = 5
) )
// BlobUploadRetryDelay is the wait before each new try of a failed blob
// upload.
const BlobUploadRetryDelay = 3 * time.Second
// ScannerConfig contains configuration for the scanner // ScannerConfig contains configuration for the scanner
type ScannerConfig struct { type ScannerConfig struct {
FS afero.Fs FS afero.Fs
@@ -158,10 +148,6 @@ type ScannerConfig struct {
// PreviousSnapshotID is the most recent completed snapshot of the // PreviousSnapshotID is the most recent completed snapshot of the
// same name, "" for none. // same name, "" for none.
PreviousSnapshotID string PreviousSnapshotID string
// BlobUploadRetryDelay is the wait before each new try of a failed
// blob upload. Backups use snapshot.BlobUploadRetryDelay; tests leave
// it zero to try again at once.
BlobUploadRetryDelay time.Duration
// DeviceNumber returns the number of the device that holds a file, // DeviceNumber returns the number of the device that holds a file,
// and false when there is none to read. nil reads it from the file's // and false when there is none to read. nil reads it from the file's
// stat; tests replace it, since the in-memory test filesystem has none. // stat; tests replace it, since the in-memory test filesystem has none.
@@ -259,24 +245,23 @@ func NewScanner(cfg ScannerConfig) *Scanner {
} }
return &Scanner{ return &Scanner{
fs: cfg.FS, fs: cfg.FS,
chunker: chunker.NewChunker(cfg.ChunkSize), chunker: chunker.NewChunker(cfg.ChunkSize),
packer: packer, packer: packer,
repos: cfg.Repositories, repos: cfg.Repositories,
storage: cfg.Storage, storage: cfg.Storage,
maxBlobSize: cfg.MaxBlobSize, maxBlobSize: cfg.MaxBlobSize,
compressionLevel: cfg.CompressionLevel, compressionLevel: cfg.CompressionLevel,
ageRecipient: strings.Join(cfg.AgeRecipients, ","), ageRecipient: strings.Join(cfg.AgeRecipients, ","),
exclude: cfg.Exclude, exclude: cfg.Exclude,
compiledExclude: compiledExclude, compiledExclude: compiledExclude,
indexAndLockFiles: indexAndLockFiles, indexAndLockFiles: indexAndLockFiles,
progress: progress, progress: progress,
skipErrors: cfg.SkipErrors, skipErrors: cfg.SkipErrors,
previousSnapshotID: cfg.PreviousSnapshotID, previousSnapshotID: cfg.PreviousSnapshotID,
blobUploadRetryDelay: cfg.BlobUploadRetryDelay, ui: uiw,
ui: uiw, deviceNumber: deviceNumber,
deviceNumber: deviceNumber, pendingChunkHashes: make(map[string]struct{}),
pendingChunkHashes: make(map[string]struct{}),
} }
} }
@@ -1453,12 +1438,11 @@ func (s *Scanner) deviceNumberAt(path string) (uint64, bool) {
} }
// deviceNumberFromStat returns the number of the device that holds the // deviceNumberFromStat returns the number of the device that holds the
// file info describes, and false when info carries no stat from a real // file info describes, and false when info carries no stat, as from the
// filesystem. The in-memory test filesystem gives either no stat or one // in-memory test filesystem.
// whose device and inode numbers are both zero, which no real file has.
func deviceNumberFromStat(info os.FileInfo) (uint64, bool) { func deviceNumberFromStat(info os.FileInfo) (uint64, bool) {
stat, ok := info.Sys().(*syscall.Stat_t) stat, ok := info.Sys().(*syscall.Stat_t)
if !ok || (stat.Dev == 0 && stat.Ino == 0) { if !ok {
return 0, false return 0, false
} }
@@ -1891,7 +1875,10 @@ func (s *Scanner) uploadBlobIfNeeded(
s.ui.Hex(finishedBlob.Hash), s.ui.Size(finishedBlob.Compressed), s.ui.Hex(finishedBlob.Hash), s.ui.Size(finishedBlob.Compressed),
s.ui.Path(destination)) s.ui.Path(destination))
err = s.putBlob(ctx, blobPath, blobWithReader) progressCallback := s.makeUploadProgressCallback(ctx, finishedBlob, startTime)
err = s.storage.PutWithProgress(ctx, blobPath, blobWithReader.Reader,
finishedBlob.Compressed, progressCallback)
if err != nil { if err != nil {
// The record is at info level: the returned error ends the backup // The record is at info level: the returned error ends the backup
// and is printed as its ERROR: line. Only an interrupt cancels // and is printed as its ERROR: line. Only an interrupt cancels
@@ -1935,44 +1922,6 @@ func (s *Scanner) uploadBlobIfNeeded(
return finishedBlob.Compressed, false, nil return finishedBlob.Compressed, false, nil
} }
// putBlob uploads the blob to blobPath. After a failed upload it waits
// s.blobUploadRetryDelay and uploads the whole blob again, up to
// blobUploadRetries times, and then returns the last error. An interrupt
// stops it at once.
func (s *Scanner) putBlob(
ctx context.Context, blobPath string, blobWithReader *blob.WithReader,
) error {
finishedBlob := blobWithReader.FinishedBlob
for retries := 0; ; retries++ {
_, err := blobWithReader.Reader.Seek(0, io.SeekStart)
if err != nil {
return fmt.Errorf("rewinding blob for upload: %w", err)
}
progressCallback := s.makeUploadProgressCallback(
ctx, finishedBlob, time.Now().UTC())
err = s.storage.PutWithProgress(ctx, blobPath, blobWithReader.Reader,
finishedBlob.Compressed, progressCallback)
if err == nil || ctx.Err() != nil || retries == blobUploadRetries {
return err
}
log.Info("Blob upload failed, trying again",
"hash", finishedBlob.Hash, "retry", retries+1, "error", err)
s.ui.Warningf("Upload of blob %s failed: %v. Trying again in %s (retry %d of %d).",
s.ui.Hex(finishedBlob.Hash), err,
s.ui.Duration(s.blobUploadRetryDelay), retries+1, blobUploadRetries)
select {
case <-ctx.Done():
return err
case <-time.After(s.blobUploadRetryDelay):
}
}
}
// makeUploadProgressCallback creates a progress callback for blob uploads. // makeUploadProgressCallback creates a progress callback for blob uploads.
// It updates the live progress reporter ~twice/sec for ETAs and prints a // It updates the live progress reporter ~twice/sec for ETAs and prints a
// human-readable status line to s.output at most every 15 seconds. // human-readable status line to s.output at most every 15 seconds.
+74 -7
View File
@@ -9,6 +9,7 @@ import (
"github.com/dustin/go-humanize" "github.com/dustin/go-humanize"
"sneak.berlin/go/vaultik/internal/log" "sneak.berlin/go/vaultik/internal/log"
"sneak.berlin/go/vaultik/internal/storage"
) )
// ShowInfo displays system and configuration information // ShowInfo displays system and configuration information
@@ -187,6 +188,9 @@ type SnapshotMetadataInfo struct {
// backup interrupted before its manifest upload leaves a directory // backup interrupted before its manifest upload leaves a directory
// without one, which prune does not treat as a snapshot. // without one, which prune does not treat as a snapshot.
hasManifest bool hasManifest bool
// Hashes of the blobs the snapshot's manifest lists.
blobHashes []string
} }
// RemoteInfoResult contains all remote storage information // RemoteInfoResult contains all remote storage information
@@ -216,6 +220,13 @@ type RemoteInfoResult struct {
OrphanedBlobCount *int `json:"orphaned_blob_count"` OrphanedBlobCount *int `json:"orphaned_blob_count"`
OrphanedBlobSize *int64 `json:"orphaned_blob_size"` OrphanedBlobSize *int64 `json:"orphaned_blob_size"`
// Blobs that a manifest lists but the destination does not hold,
// and the remote key of each snapshot whose manifest lists one.
// Those snapshots cannot be restored.
MissingBlobCount int `json:"missing_blob_count"`
MissingBlobSize int64 `json:"missing_blob_size"`
SnapshotsMissingBlobs []string `json:"snapshots_missing_blobs,omitempty"`
// Remote key of each snapshot whose manifest could not be read // Remote key of each snapshot whose manifest could not be read
UnreadableManifests []string `json:"unreadable_manifests,omitempty"` UnreadableManifests []string `json:"unreadable_manifests,omitempty"`
@@ -283,19 +294,30 @@ func (v *Vaultik) RemoteInfo(jsonOutput bool) error {
"snapshots", result.TotalMetadataCount, "snapshots", result.TotalMetadataCount,
"total_blobs", result.TotalBlobCount, "total_blobs", result.TotalBlobCount,
"referenced_blobs", result.ReferencedBlobCount, "referenced_blobs", result.ReferencedBlobCount,
"missing_blobs", result.MissingBlobCount,
"unreadable_manifests", len(result.UnreadableManifests)) "unreadable_manifests", len(result.UnreadableManifests))
if jsonOutput { if jsonOutput {
enc := json.NewEncoder(v.Stdout) enc := json.NewEncoder(v.Stdout)
enc.SetIndent("", " ") enc.SetIndent("", " ")
return enc.Encode(result) err = enc.Encode(result)
if err != nil {
return fmt.Errorf("encoding JSON: %w", err)
}
} }
if showText { if showText {
v.printRemoteInfoTable(result) v.printRemoteInfoTable(result)
} }
// The report is written first, so the exit status adds to it.
if result.MissingBlobCount > 0 {
return fmt.Errorf("%w: %d blob(s) that snapshots list are missing; "+
"%d snapshot(s) cannot be restored", errProblemsInStorage,
result.MissingBlobCount, len(result.SnapshotsMissingBlobs))
}
return nil return nil
} }
@@ -405,6 +427,7 @@ func (v *Vaultik) collectReferencedBlobsFromManifests(
for _, blob := range manifest.Blobs { for _, blob := range manifest.Blobs {
referencedBlobs[blob.Hash] = blob.CompressedSize referencedBlobs[blob.Hash] = blob.CompressedSize
blobsSize += blob.CompressedSize blobsSize += blob.CompressedSize
info.blobHashes = append(info.blobHashes, blob.Hash)
} }
info.BlobCount = &blobCount info.BlobCount = &blobCount
@@ -462,6 +485,8 @@ func (v *Vaultik) scanRemoteBlobStorage(
result.TotalBlobSize += blob.Size result.TotalBlobSize += blob.Size
} }
countMissingBlobs(result, referencedBlobs, allBlobs)
// A blob named only by a manifest that could not be read, or by one // A blob named only by a manifest that could not be read, or by one
// under a skipped name, would be counted as orphaned, so the orphan // under a skipped name, would be counted as orphaned, so the orphan
// figures stay unknown. // figures stay unknown.
@@ -487,6 +512,33 @@ func (v *Vaultik) scanRemoteBlobStorage(
return nil return nil
} }
// countMissingBlobs records in result the referenced blobs that
// allBlobs, the listing of the destination, does not hold, and the
// snapshots whose manifests list one of them. It reads the snapshots
// from result.Snapshots, so populateRemoteInfoResult must run first.
func countMissingBlobs(
result *RemoteInfoResult, referencedBlobs map[string]int64,
allBlobs map[string]storage.ObjectInfo,
) {
for hash, size := range referencedBlobs {
if _, present := allBlobs[hash]; !present {
result.MissingBlobCount++
result.MissingBlobSize += size
}
}
for _, info := range result.Snapshots {
for _, hash := range info.blobHashes {
if _, present := allBlobs[hash]; !present {
result.SnapshotsMissingBlobs = append(
result.SnapshotsMissingBlobs, info.SnapshotID)
break
}
}
}
}
// printRemoteInfoTable renders the human-readable remote info output // printRemoteInfoTable renders the human-readable remote info output
func (v *Vaultik) printRemoteInfoTable(result *RemoteInfoResult) { func (v *Vaultik) printRemoteInfoTable(result *RemoteInfoResult) {
const ( const (
@@ -539,6 +591,12 @@ func (v *Vaultik) printRemoteInfoTable(result *RemoteInfoResult) {
"", "", ubytes(result.TotalMetadataSize)) "", "", ubytes(result.TotalMetadataSize))
} }
v.printBlobStorageSummary(result)
}
// printBlobStorageSummary renders the blob storage section that ends
// the human-readable remote info output.
func (v *Vaultik) printBlobStorageSummary(result *RemoteInfoResult) {
v.stdoutf("\n=== Blob Storage ===\n") v.stdoutf("\n=== Blob Storage ===\n")
v.stdoutf("Total blobs on remote: %s (%s)\n", v.stdoutf("Total blobs on remote: %s (%s)\n",
humanize.Comma(int64(result.TotalBlobCount)), humanize.Comma(int64(result.TotalBlobCount)),
@@ -546,22 +604,31 @@ func (v *Vaultik) printRemoteInfoTable(result *RemoteInfoResult) {
v.stdoutf("Referenced by snapshots: %s (%s)\n", v.stdoutf("Referenced by snapshots: %s (%s)\n",
humanize.Comma(int64(result.ReferencedBlobCount)), humanize.Comma(int64(result.ReferencedBlobCount)),
ubytes(result.ReferencedBlobSize)) ubytes(result.ReferencedBlobSize))
v.stdoutf("Missing from remote: %s (%s)\n",
humanize.Comma(int64(result.MissingBlobCount)),
ubytes(result.MissingBlobSize))
if result.OrphanedBlobCount == nil { if result.OrphanedBlobCount == nil {
v.stdoutf("Orphaned (unreferenced): unknown "+ v.stdoutf("Orphaned (unreferenced): unknown "+
"(%d manifest(s) could not be read, "+ "(%d manifest(s) could not be read, "+
"%d manifest(s) under a non-conforming name skipped)\n", "%d manifest(s) under a non-conforming name skipped)\n",
len(result.UnreadableManifests), result.SkippedManifestCount) len(result.UnreadableManifests), result.SkippedManifestCount)
} else {
v.stdoutf("Orphaned (unreferenced): %s (%s)\n",
humanize.Comma(int64(*result.OrphanedBlobCount)),
ubytes(*result.OrphanedBlobSize))
return if *result.OrphanedBlobCount > 0 {
v.stdoutf("\nRun 'vaultik prune' to remove orphaned blobs.\n")
}
} }
v.stdoutf("Orphaned (unreferenced): %s (%s)\n", if len(result.SnapshotsMissingBlobs) > 0 {
humanize.Comma(int64(*result.OrphanedBlobCount)), v.stdoutf("\nSnapshots that cannot be restored (blobs missing):\n")
ubytes(*result.OrphanedBlobSize))
if *result.OrphanedBlobCount > 0 { for _, id := range result.SnapshotsMissingBlobs {
v.stdoutf("\nRun 'vaultik prune' to remove orphaned blobs.\n") v.stdoutf(" %s\n", id)
}
} }
} }
+60
View File
@@ -83,6 +83,7 @@ func TestRemoteInfo_UnreadableManifestLeavesSnapshotBlobsUnknown(t *testing.T) {
env := newListEnv(t) env := newListEnv(t)
readableKey := env.addRemote(t, listRemoteID, readableKey := env.addRemote(t, listRemoteID,
time.Date(2026, 3, 2, 0, 0, 0, 0, time.UTC)) time.Date(2026, 3, 2, 0, 0, 0, 0, time.UTC))
addBlob(t, env.store.testStorer, testBlobHashA)
unreadableKey := snapshot.RemoteSnapshotKey(listLocalID) unreadableKey := snapshot.RemoteSnapshotKey(listLocalID)
require.NoError(t, env.store.Put(context.Background(), require.NoError(t, env.store.Put(context.Background(),
@@ -270,6 +271,7 @@ func TestRemoteInfo_StatusLineUsesUIMarker(t *testing.T) {
env := newListEnv(t) env := newListEnv(t)
env.addRemote(t, listRemoteID, time.Date(2026, 3, 2, 0, 0, 0, 0, time.UTC)) env.addRemote(t, listRemoteID, time.Date(2026, 3, 2, 0, 0, 0, 0, time.UTC))
addBlob(t, env.store.testStorer, testBlobHashA)
const statusLine = ui.Marker + " Scanning snapshot metadata...\n" const statusLine = ui.Marker + " Scanning snapshot metadata...\n"
@@ -281,3 +283,61 @@ func TestRemoteInfo_StatusLineUsesUIMarker(t *testing.T) {
require.NoError(t, env.v.RemoteInfo(false)) require.NoError(t, env.v.RemoteInfo(false))
assert.NotContains(t, env.stdout.String(), "Scanning snapshot metadata") assert.NotContains(t, env.stdout.String(), "Scanning snapshot metadata")
} }
// TestRemoteInfo_ReportsBlobsMissingFromDestination checks that a blob a
// manifest lists but the destination does not hold is counted as
// missing, that the snapshot listing it is named, and that remote info
// then fails after writing its report.
func TestRemoteInfo_ReportsBlobsMissingFromDestination(t *testing.T) {
log.Initialize(log.Config{})
t.Parallel()
env := newListEnv(t)
// The first snapshot lists blob A, which is not stored. The second
// lists blob B, which is.
brokenKey := env.addRemote(t, listRemoteID,
time.Date(2026, 3, 2, 0, 0, 0, 0, time.UTC))
addManifest(t, env.store.testStorer, listLocalID, []string{testBlobHashB})
addBlob(t, env.store.testStorer, testBlobHashB)
const wantError = "1 blob(s) that snapshots list are missing; " +
"1 snapshot(s) cannot be restored"
err := env.v.RemoteInfo(false)
require.ErrorContains(t, err, wantError)
text := env.stdout.String()
assert.Contains(t, text, "Missing from remote: 1 (5.0 MB)")
assert.Contains(t, text, "Snapshots that cannot be restored (blobs missing):\n"+
" "+brokenKey+"\n")
assert.NotContains(t, text, snapshot.RemoteSnapshotKey(listLocalID)+"\n")
env.stdout.Reset()
require.ErrorContains(t, env.v.RemoteInfo(true), wantError)
var doc map[string]any
require.NoError(t, json.Unmarshal(env.stdout.Bytes(), &doc))
assert.InDelta(t, 1, doc["missing_blob_count"], 0)
assert.InDelta(t, fiveMegabytes, doc["missing_blob_size"], 0)
assert.Equal(t, []any{brokenKey}, doc["snapshots_missing_blobs"])
// --quiet drops the report, not the failure.
env.stdout.Reset()
env.v.UI.SetQuiet(true)
require.ErrorContains(t, env.v.RemoteInfo(false), wantError)
assert.Empty(t, env.stdout.String())
env.v.UI.SetQuiet(false)
// With blob A stored again nothing is missing.
addBlob(t, env.store.testStorer, testBlobHashA)
env.stdout.Reset()
require.NoError(t, env.v.RemoteInfo(true))
doc = nil
require.NoError(t, json.Unmarshal(env.stdout.Bytes(), &doc))
assert.InDelta(t, 0, doc["missing_blob_count"], 0)
assert.InDelta(t, 0, doc["missing_blob_size"], 0)
assert.NotContains(t, doc, "snapshots_missing_blobs")
}
+1
View File
@@ -349,6 +349,7 @@ func TestSnapshotListAndPurgePrintSizesLikeRemoteInfo(t *testing.T) {
env.addRemote(t, listLocalID, start) env.addRemote(t, listLocalID, start)
require.NoError(t, env.v.Repositories.Snapshots.UpdateCounts( require.NoError(t, env.v.Repositories.Snapshots.UpdateCounts(
context.Background(), nil, listLocalID, 0, 0, 0, 0, fiveMegabytes)) context.Background(), nil, listLocalID, 0, 0, 0, 0, fiveMegabytes))
addBlob(t, env.store.testStorer, testBlobHashA)
require.NoError(t, env.v.RemoteInfo(false)) require.NoError(t, env.v.RemoteInfo(false))
assert.Contains(t, findTableRow(t, env.stdout.String(), remoteKey[:16]), assert.Contains(t, findTableRow(t, env.stdout.String(), remoteKey[:16]),
@@ -4,7 +4,6 @@ import (
"context" "context"
"maps" "maps"
"path/filepath" "path/filepath"
"strings"
"testing" "testing"
"github.com/spf13/afero" "github.com/spf13/afero"
@@ -13,7 +12,6 @@ import (
"sneak.berlin/go/vaultik/internal/database" "sneak.berlin/go/vaultik/internal/database"
"sneak.berlin/go/vaultik/internal/log" "sneak.berlin/go/vaultik/internal/log"
"sneak.berlin/go/vaultik/internal/storage" "sneak.berlin/go/vaultik/internal/storage"
"sneak.berlin/go/vaultik/internal/storage/faultstore"
"sneak.berlin/go/vaultik/internal/vaultik" "sneak.berlin/go/vaultik/internal/vaultik"
) )
@@ -70,55 +68,3 @@ func TestBackupWithoutCronOfTwoPathSnapshotRestoresBothPaths(t *testing.T) {
assertRestoredTree(t, fs, restoreDir, files) assertRestoredTree(t, fs, restoreDir, files)
} }
// A backup without --cron whose blob uploads keep failing stops with the
// upload's error, and stops the progress reporter once. See
// https://git.eeqj.de/sneak/vaultik/issues/476.
//
//nolint:paralleltest // installs the global logger via log.Initialize
func TestBackupWithoutCronStopsWithBlobUploadError(t *testing.T) {
log.Initialize(log.Config{})
const snapshotName = "data"
fs := afero.NewOsFs()
tempDir := t.TempDir()
firstDir := filepath.Join(tempDir, "first")
secondDir := filepath.Join(tempDir, "second")
dbPath := filepath.Join(tempDir, "index.sqlite")
ctx := context.Background()
writeFaultSourceTree(t, fs, firstDir)
writeFaultSourceTree(t, fs, secondDir)
cfg := faultTestConfig()
cfg.IndexPath = dbPath
cfg.ChunkSize = config.Size(faultChunkSize)
cfg.Snapshots = map[string]config.SnapshotConfig{
snapshotName: {Paths: []string{firstDir, secondDir}},
}
inner, err := storage.NewFileStorer(t.TempDir())
require.NoError(t, err)
store := faultstore.New(inner)
store.OnPut = func(key string) faultstore.PutAction {
if strings.HasPrefix(key, "blobs/") {
return faultstore.PutFail
}
return faultstore.PutNormal
}
db, err := database.New(ctx, dbPath)
require.NoError(t, err)
t.Cleanup(func() { _ = db.Close() })
v := newBackupVaultik(ctx, cfg, store, database.NewRepositories(db), db, fs)
err = v.CreateSnapshot(&vaultik.SnapshotCreateOptions{
Snapshots: []string{snapshotName},
})
require.ErrorIs(t, err, faultstore.ErrInjectedUpload)
}