Check blob sizes and the database in shallow verify (closes #169)
Shallow snapshot verify only checked that each blob object existed and then reported "All blobs verified", overstating what it did. It now compares each blob stored size against the manifest compressed_size, using the same comparison as the deep path, and checks that the snapshot encrypted database (db.zst.age) is present. A blob of the wrong size no longer counts as verified. The final line reports only what was checked: presence and size, not contents. The README verify description and the CLI short/long text are corrected to match. Removed the now-unused resolveAndDownloadManifest helper and errBlobsMissing sentinel. Model: opus-4-8
This commit was merged in pull request #193.
This commit is contained in:
@@ -20,7 +20,6 @@ import (
|
||||
var (
|
||||
errSnapshotNotInConfig = errors.New("snapshot not found in config")
|
||||
errNoSnapshotsInConfig = errors.New("no snapshots configured")
|
||||
errBlobsMissing = errors.New("blobs are missing")
|
||||
errSnapshotVerifyFailed = errors.New("verification failed")
|
||||
errRemoveAllNeedsForce = errors.New("--all requires --force")
|
||||
errInvalidTableName = errors.New("invalid table name")
|
||||
@@ -670,11 +669,24 @@ func (v *Vaultik) VerifySnapshotWithOptions(
|
||||
|
||||
v.printVerifyHeader(snapshotID, opts)
|
||||
|
||||
// Resolve the identifier to the snapshot's remote key and download the
|
||||
// manifest. A human ID is hashed; a remote key (or its abbreviation,
|
||||
// as printed for a remote-only snapshot) is used as-is, so a host with
|
||||
// no local index can verify a snapshot it can only see on the store.
|
||||
manifest, err := v.resolveAndDownloadManifest(snapshotID)
|
||||
// Resolve the identifier to the snapshot's remote key. A human ID is
|
||||
// hashed; a remote key (or its abbreviation, as printed for a
|
||||
// remote-only snapshot) is used as-is, so a host with no local index
|
||||
// can verify a snapshot it can only see on the store. The key is kept
|
||||
// so we can also check for the snapshot's encrypted database below.
|
||||
remoteKey, err := v.resolveSnapshotRemoteKey(snapshotID)
|
||||
if err != nil {
|
||||
if opts.JSON {
|
||||
result.Status = verifyStatusFailed
|
||||
result.ErrorMessage = fmt.Sprintf("resolving snapshot identifier: %v", err)
|
||||
|
||||
return v.outputVerifyJSON(result)
|
||||
}
|
||||
|
||||
return fmt.Errorf("resolving snapshot identifier: %w", err)
|
||||
}
|
||||
|
||||
manifest, err := v.downloadManifestByKey(remoteKey)
|
||||
if err != nil {
|
||||
if opts.JSON {
|
||||
result.Status = verifyStatusFailed
|
||||
@@ -704,14 +716,24 @@ func (v *Vaultik) VerifySnapshotWithOptions(
|
||||
|
||||
v.printlnStdout()
|
||||
|
||||
// Check each blob exists
|
||||
v.stdoutf("Checking blob existence...\n")
|
||||
// Check each blob is present with the size the manifest records.
|
||||
v.stdoutf("Checking blob presence and sizes...\n")
|
||||
}
|
||||
|
||||
result.Verified, result.Missing, result.MissingSize =
|
||||
v.verifyManifestBlobsExist(manifest, opts)
|
||||
// A snapshot is only restorable if its encrypted database is present
|
||||
// alongside the blobs. Shallow verify checks that the object exists; it
|
||||
// does not decrypt it (that is deep verify's job).
|
||||
dbPath := fmt.Sprintf("metadata/%s/db.zst.age", remoteKey)
|
||||
|
||||
return v.formatVerifyResult(result, manifest, opts)
|
||||
_, dbErr := v.Storage.Stat(v.ctx, dbPath)
|
||||
if dbErr != nil {
|
||||
result.DatabaseMissing = true
|
||||
}
|
||||
|
||||
result.Verified, result.Missing, result.Mismatched, result.MissingSize =
|
||||
v.verifyManifestBlobs(manifest, opts)
|
||||
|
||||
return v.formatVerifyResult(result, opts)
|
||||
}
|
||||
|
||||
// printVerifyHeader prints the snapshot ID and parsed timestamp for
|
||||
@@ -736,25 +758,27 @@ func (v *Vaultik) printVerifyHeader(snapshotID string, opts *VerifyOptions) {
|
||||
}
|
||||
}
|
||||
|
||||
// verifyManifestBlobsExist checks that each blob in the manifest exists
|
||||
// in storage, returning the verified count, missing count, and total
|
||||
// missing bytes.
|
||||
func (v *Vaultik) verifyManifestBlobsExist(
|
||||
// verifyManifestBlobs checks that each blob in the manifest is present in
|
||||
// storage with the size the manifest records, returning the counts of
|
||||
// blobs that were present with the right size, absent, and present but the
|
||||
// wrong size, plus the total bytes of the absent blobs. It does not read
|
||||
// blob contents; deep verification (RunDeepVerify) does that. The size
|
||||
// comparison matches the deep path (see verifyBlobExistenceFromDB).
|
||||
func (v *Vaultik) verifyManifestBlobs(
|
||||
manifest *snapshot.Manifest, opts *VerifyOptions,
|
||||
) (int, int, int64) {
|
||||
) (int, int, int, int64) {
|
||||
var (
|
||||
verified, missing int
|
||||
missingSize int64
|
||||
verified, missing, mismatched int
|
||||
missingSize int64
|
||||
)
|
||||
|
||||
for _, blob := range manifest.Blobs {
|
||||
blobPath := fmt.Sprintf("blobs/%s/%s/%s",
|
||||
blob.Hash[:2], blob.Hash[2:4], blob.Hash)
|
||||
|
||||
// Shallow: check existence only (deep verification is handled
|
||||
// by RunDeepVerify).
|
||||
_, err := v.Storage.Stat(v.ctx, blobPath)
|
||||
if err != nil {
|
||||
stat, err := v.Storage.Stat(v.ctx, blobPath)
|
||||
switch {
|
||||
case err != nil:
|
||||
if !opts.JSON {
|
||||
v.stdoutf(" Missing: %s (%s)\n",
|
||||
blob.Hash, ubytes(blob.CompressedSize))
|
||||
@@ -762,23 +786,32 @@ func (v *Vaultik) verifyManifestBlobsExist(
|
||||
|
||||
missing++
|
||||
missingSize += blob.CompressedSize
|
||||
} else {
|
||||
case stat.Size != blob.CompressedSize:
|
||||
if !opts.JSON {
|
||||
v.stdoutf(" Wrong size: %s (store has %s, manifest lists %s)\n",
|
||||
blob.Hash, ubytes(stat.Size), ubytes(blob.CompressedSize))
|
||||
}
|
||||
|
||||
mismatched++
|
||||
default:
|
||||
verified++
|
||||
}
|
||||
}
|
||||
|
||||
return verified, missing, missingSize
|
||||
return verified, missing, mismatched, missingSize
|
||||
}
|
||||
|
||||
// formatVerifyResult outputs the final verification results as JSON or
|
||||
// human-readable text.
|
||||
func (v *Vaultik) formatVerifyResult(
|
||||
result *VerifyResult, manifest *snapshot.Manifest, opts *VerifyOptions,
|
||||
result *VerifyResult, opts *VerifyOptions,
|
||||
) error {
|
||||
failure := shallowVerifyFailure(result)
|
||||
|
||||
if opts.JSON {
|
||||
if result.Missing > 0 {
|
||||
if failure != "" {
|
||||
result.Status = verifyStatusFailed
|
||||
result.ErrorMessage = fmt.Sprintf("%d blobs are missing", result.Missing)
|
||||
result.ErrorMessage = failure
|
||||
} else {
|
||||
result.Status = "ok"
|
||||
}
|
||||
@@ -787,29 +820,57 @@ func (v *Vaultik) formatVerifyResult(
|
||||
}
|
||||
|
||||
v.stdoutf("\nVerification complete:\n")
|
||||
v.stdoutf(" Verified: %d blobs (%s)\n", result.Verified,
|
||||
ubytes(manifest.TotalCompressedSize-result.MissingSize))
|
||||
v.stdoutf(" Present with listed size: %d blobs\n", result.Verified)
|
||||
|
||||
if result.Missing > 0 {
|
||||
v.stdoutf(" Missing: %d blobs (%s)\n",
|
||||
v.stdoutf(" Missing: %d blobs (%s)\n",
|
||||
result.Missing, ubytes(result.MissingSize))
|
||||
} else {
|
||||
v.stdoutf(" Missing: 0 blobs\n")
|
||||
}
|
||||
|
||||
if result.Mismatched > 0 {
|
||||
v.stdoutf(" Wrong size: %d blobs\n", result.Mismatched)
|
||||
}
|
||||
|
||||
if result.DatabaseMissing {
|
||||
v.stdoutf(" Encrypted database: missing\n")
|
||||
}
|
||||
|
||||
v.stdoutf(" Status: ")
|
||||
|
||||
if result.Missing > 0 {
|
||||
v.stdoutf("FAILED - %d blobs are missing\n", result.Missing)
|
||||
if failure != "" {
|
||||
v.stdoutf("FAILED - %s\n", failure)
|
||||
|
||||
return fmt.Errorf("%d %w", result.Missing, errBlobsMissing)
|
||||
return fmt.Errorf("%w: %s", errSnapshotVerifyFailed, failure)
|
||||
}
|
||||
|
||||
v.stdoutf("OK - All blobs verified\n")
|
||||
// Report only what was actually checked: presence and size, not contents.
|
||||
v.stdoutf("OK - all %d blobs listed in the manifest are present with the "+
|
||||
"listed size; contents not checked (use --deep)\n", result.Verified)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// shallowVerifyFailure returns a human-readable description of everything
|
||||
// that failed shallow verification, or the empty string if it passed.
|
||||
func shallowVerifyFailure(result *VerifyResult) string {
|
||||
var parts []string
|
||||
|
||||
if result.Missing > 0 {
|
||||
parts = append(parts, fmt.Sprintf("%d blobs are missing", result.Missing))
|
||||
}
|
||||
|
||||
if result.Mismatched > 0 {
|
||||
parts = append(parts,
|
||||
fmt.Sprintf("%d blobs have the wrong size", result.Mismatched))
|
||||
}
|
||||
|
||||
if result.DatabaseMissing {
|
||||
parts = append(parts, "the encrypted database is missing")
|
||||
}
|
||||
|
||||
return strings.Join(parts, "; ")
|
||||
}
|
||||
|
||||
// outputVerifyJSON outputs the verification result as JSON
|
||||
func (v *Vaultik) outputVerifyJSON(result *VerifyResult) error {
|
||||
encoder := json.NewEncoder(v.Stdout)
|
||||
|
||||
Reference in New Issue
Block a user