Compare commits

...
Author SHA1 Message Date
sneak 2aa6fc8f66 Report snapshot blobs missing from the destination in remote info (closes #425)
check / check (push) Waiting to run
remote info now counts the blobs that a manifest lists but the
destination does not hold, with their total size, and names each
snapshot that lists one, since it cannot be restored. When any blob is
missing the command exits non-zero after writing its report. --json
gives the figures in missing_blob_count, missing_blob_size and
snapshots_missing_blobs.

Model: opus-5-5
2026-11-19 08:24:27 +00:00
clawbot 9fedcbeb0a Refuse a relative path on an rclone remote of type local (closes #465)
check / check (push) Waiting to run
rclone resolves a relative path on its local backend against the working
directory, so rclone://localfs/srv/backup wrote under whatever directory
vaultik ran from. Opening such a destination now fails with an error that
shows the absolute form, rclone://localfs//srv/backup. Other remotes keep
relative paths, which rclone resolves on the remote. The README says how
to write an absolute rclone path.

Model: opus-5-5
2026-11-19 09:24:16 +01:00
7 changed files with 243 additions and 7 deletions

No files matched your search

+11 -1
View File
@@ -554,6 +554,12 @@ count and blob size unknown, `null` in `--json`. A file under `blobs/` whose
name is not a blob hash is not counted as a blob, since `prune` never deletes
it.
A blob that a manifest lists but the destination does not hold is reported as
missing, with a count and size. The report names each snapshot that lists a
missing blob, since that snapshot cannot be restored. The command then exits
non-zero. `--json` gives these in `missing_blob_count`, `missing_blob_size` and
`snapshots_missing_blobs`.
- `--json`: Output as JSON
**`remote nuke`**: Delete every object under `metadata/` and `blobs/` on the
@@ -602,7 +608,11 @@ only `vaultik-destination.txt` stops such a backup, and only while the local
index holds snapshots.
**Rclone** (`rclone://remote/path`): Uses rclone's 70+ supported cloud
providers. Requires rclone to be configured separately (`rclone config`).
providers. Requires rclone to be configured separately (`rclone config`). The
path after the remote name is relative: rclone resolves it on the remote, for
example against the login directory on sftp. A second slash makes it absolute,
as in `rclone://remote//srv/backup`. A remote of type `local` needs an absolute
path; a relative one is refused.
An upload cut off part-way leaves nothing under the object's name on S3, which
shows an object only once its upload has completed, and on the local filesystem
+3 -1
View File
@@ -75,7 +75,9 @@ func newRemoteInfoCommand() *cobra.Command {
- Size of all snapshot metadata (per snapshot and total)
- Count and total size of all blobs
- Count and size of referenced blobs (from all manifests)
- Count and size of orphaned blobs (not referenced by any manifest)`,
- Count and size of orphaned blobs (not referenced by any manifest)
- Count and size of referenced blobs missing from remote storage, and the
snapshots that list them; the command exits non-zero when any are missing`,
Args: cobra.NoArgs,
RunE: func(cmd *cobra.Command, _ []string) error {
// Use unified config resolution
+23
View File
@@ -8,6 +8,7 @@ import (
"fmt"
"io"
"os"
"path/filepath"
"strings"
"time"
@@ -23,6 +24,15 @@ import (
// ErrRemoteNotFound is returned when an rclone remote is not configured.
var ErrRemoteNotFound = errors.New("rclone remote not found in config")
// ErrRcloneRelativeLocalPath is returned when the path on an rclone remote
// of type local is relative. rclone would resolve it against the working
// directory, so the destination would move with it.
var ErrRcloneRelativeLocalPath = errors.New(
"the path on an rclone remote of type local must be absolute")
// rcloneLocalBackend is the type of rclone's local disk backend.
const rcloneLocalBackend = "local"
// RcloneStorer implements Storer using rclone's filesystem abstraction.
// This allows vaultik to use any of rclone's 70+ supported storage providers.
type RcloneStorer struct {
@@ -53,6 +63,19 @@ func NewRcloneStorer(ctx context.Context, remote, path string) (*RcloneStorer, e
rclonePath += path
}
// A remote rclone cannot find is reported by fs.NewFs below. Only the
// backend of what fs.ParseRemote returns is needed here.
backend, _, _, _, err := fs.ParseRemote(rclonePath) //nolint:dogsled
if err == nil && backend.Name == rcloneLocalBackend && !filepath.IsAbs(path) {
example := path
if example == "" {
example = "path/to/backup"
}
return nil, fmt.Errorf("%w: use rclone://%s//%s",
ErrRcloneRelativeLocalPath, remote, example)
}
// On S3, Swift and some other remotes rclone keeps a time of its own in
// each object's metadata, which costs one request per object to read.
// rclone's use_server_modtime setting makes ModTime the time the
+73
View File
@@ -436,6 +436,79 @@ func TestNewRcloneStorerReadsRcloneConfigEnv(t *testing.T) {
}
}
// useRcloneConfig points rclone at a config file holding contents for the
// rest of the test.
func useRcloneConfig(t *testing.T, contents string) {
t.Helper()
previous := config.GetConfigPath()
t.Cleanup(func() { _ = config.SetConfigPath(previous) })
confFile := filepath.Join(t.TempDir(), "rclone.conf")
err := os.WriteFile(confFile, []byte(contents), 0o600)
if err != nil {
t.Fatalf("writing rclone config: %v", err)
}
t.Setenv("RCLONE_CONFIG", confFile)
}
// TestNewRcloneStorerRefusesRelativeLocalPath checks that a relative path
// on a remote of type local is refused, with the absolute form in the
// error, and that nothing is created under the working directory. rclone
// would otherwise resolve the path against the working directory.
//
//nolint:paralleltest // changes the working directory and the rclone config
func TestNewRcloneStorerRefusesRelativeLocalPath(t *testing.T) {
useRcloneConfig(t, "[localfs]\ntype = local\n")
workdir := t.TempDir()
t.Chdir(workdir)
for path, wantInError := range map[string]string{
"srv/backup": "rclone://localfs//srv/backup",
"": "rclone://localfs//path/to/backup",
} {
_, err := storage.NewRcloneStorer(context.Background(), "localfs", path)
if !errors.Is(err, storage.ErrRcloneRelativeLocalPath) {
t.Errorf("path %q: error = %v, want ErrRcloneRelativeLocalPath",
path, err)
continue
}
if !strings.Contains(err.Error(), wantInError) {
t.Errorf("path %q: error %q does not show %q",
path, err, wantInError)
}
}
entries, err := os.ReadDir(workdir)
if err != nil {
t.Fatalf("reading working directory: %v", err)
}
if len(entries) != 0 {
t.Errorf("working directory holds %d entries, want none", len(entries))
}
}
// TestNewRcloneStorerKeepsRelativePathOnOtherRemotes checks that a relative
// path is accepted on a remote not of type local, where rclone resolves it
// on the remote itself, as sftp does against the login directory.
//
//nolint:paralleltest // changes the process-global rclone config
func TestNewRcloneStorerKeepsRelativePathOnOtherRemotes(t *testing.T) {
useRcloneConfig(t, "[mem]\ntype = memory\n")
_, err := storage.NewRcloneStorer(context.Background(), "mem", "backups")
if err != nil {
t.Errorf("NewRcloneStorer: %v", err)
}
}
// TestRcloneListStreamOnS3MakesNoRequestPerObject checks that listing an
// rclone S3 remote takes each object's time from the listing itself. By
// default rclone reads the time it keeps in each object's metadata, one
+72 -5
View File
@@ -9,6 +9,7 @@ import (
"github.com/dustin/go-humanize"
"sneak.berlin/go/vaultik/internal/log"
"sneak.berlin/go/vaultik/internal/storage"
)
// ShowInfo displays system and configuration information
@@ -187,6 +188,9 @@ type SnapshotMetadataInfo struct {
// backup interrupted before its manifest upload leaves a directory
// without one, which prune does not treat as a snapshot.
hasManifest bool
// Hashes of the blobs the snapshot's manifest lists.
blobHashes []string
}
// RemoteInfoResult contains all remote storage information
@@ -216,6 +220,13 @@ type RemoteInfoResult struct {
OrphanedBlobCount *int `json:"orphaned_blob_count"`
OrphanedBlobSize *int64 `json:"orphaned_blob_size"`
// Blobs that a manifest lists but the destination does not hold,
// and the remote key of each snapshot whose manifest lists one.
// Those snapshots cannot be restored.
MissingBlobCount int `json:"missing_blob_count"`
MissingBlobSize int64 `json:"missing_blob_size"`
SnapshotsMissingBlobs []string `json:"snapshots_missing_blobs,omitempty"`
// Remote key of each snapshot whose manifest could not be read
UnreadableManifests []string `json:"unreadable_manifests,omitempty"`
@@ -283,19 +294,30 @@ func (v *Vaultik) RemoteInfo(jsonOutput bool) error {
"snapshots", result.TotalMetadataCount,
"total_blobs", result.TotalBlobCount,
"referenced_blobs", result.ReferencedBlobCount,
"missing_blobs", result.MissingBlobCount,
"unreadable_manifests", len(result.UnreadableManifests))
if jsonOutput {
enc := json.NewEncoder(v.Stdout)
enc.SetIndent("", " ")
return enc.Encode(result)
err = enc.Encode(result)
if err != nil {
return fmt.Errorf("encoding JSON: %w", err)
}
}
if showText {
v.printRemoteInfoTable(result)
}
// The report is written first, so the exit status adds to it.
if result.MissingBlobCount > 0 {
return fmt.Errorf("%w: %d blob(s) that snapshots list are missing; "+
"%d snapshot(s) cannot be restored", errProblemsInStorage,
result.MissingBlobCount, len(result.SnapshotsMissingBlobs))
}
return nil
}
@@ -405,6 +427,7 @@ func (v *Vaultik) collectReferencedBlobsFromManifests(
for _, blob := range manifest.Blobs {
referencedBlobs[blob.Hash] = blob.CompressedSize
blobsSize += blob.CompressedSize
info.blobHashes = append(info.blobHashes, blob.Hash)
}
info.BlobCount = &blobCount
@@ -462,6 +485,8 @@ func (v *Vaultik) scanRemoteBlobStorage(
result.TotalBlobSize += blob.Size
}
countMissingBlobs(result, referencedBlobs, allBlobs)
// A blob named only by a manifest that could not be read, or by one
// under a skipped name, would be counted as orphaned, so the orphan
// figures stay unknown.
@@ -487,6 +512,33 @@ func (v *Vaultik) scanRemoteBlobStorage(
return nil
}
// countMissingBlobs records in result the referenced blobs that
// allBlobs, the listing of the destination, does not hold, and the
// snapshots whose manifests list one of them. It reads the snapshots
// from result.Snapshots, so populateRemoteInfoResult must run first.
func countMissingBlobs(
result *RemoteInfoResult, referencedBlobs map[string]int64,
allBlobs map[string]storage.ObjectInfo,
) {
for hash, size := range referencedBlobs {
if _, present := allBlobs[hash]; !present {
result.MissingBlobCount++
result.MissingBlobSize += size
}
}
for _, info := range result.Snapshots {
for _, hash := range info.blobHashes {
if _, present := allBlobs[hash]; !present {
result.SnapshotsMissingBlobs = append(
result.SnapshotsMissingBlobs, info.SnapshotID)
break
}
}
}
}
// printRemoteInfoTable renders the human-readable remote info output
func (v *Vaultik) printRemoteInfoTable(result *RemoteInfoResult) {
const (
@@ -539,6 +591,12 @@ func (v *Vaultik) printRemoteInfoTable(result *RemoteInfoResult) {
"", "", ubytes(result.TotalMetadataSize))
}
v.printBlobStorageSummary(result)
}
// printBlobStorageSummary renders the blob storage section that ends
// the human-readable remote info output.
func (v *Vaultik) printBlobStorageSummary(result *RemoteInfoResult) {
v.stdoutf("\n=== Blob Storage ===\n")
v.stdoutf("Total blobs on remote: %s (%s)\n",
humanize.Comma(int64(result.TotalBlobCount)),
@@ -546,16 +604,16 @@ func (v *Vaultik) printRemoteInfoTable(result *RemoteInfoResult) {
v.stdoutf("Referenced by snapshots: %s (%s)\n",
humanize.Comma(int64(result.ReferencedBlobCount)),
ubytes(result.ReferencedBlobSize))
v.stdoutf("Missing from remote: %s (%s)\n",
humanize.Comma(int64(result.MissingBlobCount)),
ubytes(result.MissingBlobSize))
if result.OrphanedBlobCount == nil {
v.stdoutf("Orphaned (unreferenced): unknown "+
"(%d manifest(s) could not be read, "+
"%d manifest(s) under a non-conforming name skipped)\n",
len(result.UnreadableManifests), result.SkippedManifestCount)
return
}
} else {
v.stdoutf("Orphaned (unreferenced): %s (%s)\n",
humanize.Comma(int64(*result.OrphanedBlobCount)),
ubytes(*result.OrphanedBlobSize))
@@ -565,6 +623,15 @@ func (v *Vaultik) printRemoteInfoTable(result *RemoteInfoResult) {
}
}
if len(result.SnapshotsMissingBlobs) > 0 {
v.stdoutf("\nSnapshots that cannot be restored (blobs missing):\n")
for _, id := range result.SnapshotsMissingBlobs {
v.stdoutf(" %s\n", id)
}
}
}
// ellipsis is appended by truncateString when it shortens a string.
const ellipsis = "..."
+60
View File
@@ -83,6 +83,7 @@ func TestRemoteInfo_UnreadableManifestLeavesSnapshotBlobsUnknown(t *testing.T) {
env := newListEnv(t)
readableKey := env.addRemote(t, listRemoteID,
time.Date(2026, 3, 2, 0, 0, 0, 0, time.UTC))
addBlob(t, env.store.testStorer, testBlobHashA)
unreadableKey := snapshot.RemoteSnapshotKey(listLocalID)
require.NoError(t, env.store.Put(context.Background(),
@@ -270,6 +271,7 @@ func TestRemoteInfo_StatusLineUsesUIMarker(t *testing.T) {
env := newListEnv(t)
env.addRemote(t, listRemoteID, time.Date(2026, 3, 2, 0, 0, 0, 0, time.UTC))
addBlob(t, env.store.testStorer, testBlobHashA)
const statusLine = ui.Marker + " Scanning snapshot metadata...\n"
@@ -281,3 +283,61 @@ func TestRemoteInfo_StatusLineUsesUIMarker(t *testing.T) {
require.NoError(t, env.v.RemoteInfo(false))
assert.NotContains(t, env.stdout.String(), "Scanning snapshot metadata")
}
// TestRemoteInfo_ReportsBlobsMissingFromDestination checks that a blob a
// manifest lists but the destination does not hold is counted as
// missing, that the snapshot listing it is named, and that remote info
// then fails after writing its report.
func TestRemoteInfo_ReportsBlobsMissingFromDestination(t *testing.T) {
log.Initialize(log.Config{})
t.Parallel()
env := newListEnv(t)
// The first snapshot lists blob A, which is not stored. The second
// lists blob B, which is.
brokenKey := env.addRemote(t, listRemoteID,
time.Date(2026, 3, 2, 0, 0, 0, 0, time.UTC))
addManifest(t, env.store.testStorer, listLocalID, []string{testBlobHashB})
addBlob(t, env.store.testStorer, testBlobHashB)
const wantError = "1 blob(s) that snapshots list are missing; " +
"1 snapshot(s) cannot be restored"
err := env.v.RemoteInfo(false)
require.ErrorContains(t, err, wantError)
text := env.stdout.String()
assert.Contains(t, text, "Missing from remote: 1 (5.0 MB)")
assert.Contains(t, text, "Snapshots that cannot be restored (blobs missing):\n"+
" "+brokenKey+"\n")
assert.NotContains(t, text, snapshot.RemoteSnapshotKey(listLocalID)+"\n")
env.stdout.Reset()
require.ErrorContains(t, env.v.RemoteInfo(true), wantError)
var doc map[string]any
require.NoError(t, json.Unmarshal(env.stdout.Bytes(), &doc))
assert.InDelta(t, 1, doc["missing_blob_count"], 0)
assert.InDelta(t, fiveMegabytes, doc["missing_blob_size"], 0)
assert.Equal(t, []any{brokenKey}, doc["snapshots_missing_blobs"])
// --quiet drops the report, not the failure.
env.stdout.Reset()
env.v.UI.SetQuiet(true)
require.ErrorContains(t, env.v.RemoteInfo(false), wantError)
assert.Empty(t, env.stdout.String())
env.v.UI.SetQuiet(false)
// With blob A stored again nothing is missing.
addBlob(t, env.store.testStorer, testBlobHashA)
env.stdout.Reset()
require.NoError(t, env.v.RemoteInfo(true))
doc = nil
require.NoError(t, json.Unmarshal(env.stdout.Bytes(), &doc))
assert.InDelta(t, 0, doc["missing_blob_count"], 0)
assert.InDelta(t, 0, doc["missing_blob_size"], 0)
assert.NotContains(t, doc, "snapshots_missing_blobs")
}
+1
View File
@@ -349,6 +349,7 @@ func TestSnapshotListAndPurgePrintSizesLikeRemoteInfo(t *testing.T) {
env.addRemote(t, listLocalID, start)
require.NoError(t, env.v.Repositories.Snapshots.UpdateCounts(
context.Background(), nil, listLocalID, 0, 0, 0, 0, fiveMegabytes))
addBlob(t, env.store.testStorer, testBlobHashA)
require.NoError(t, env.v.RemoteInfo(false))
assert.Contains(t, findTableRow(t, env.stdout.String(), remoteKey[:16]),