sneak
7a66235eb4
check / check (push) Waiting to run
When remote info could not read a snapshot's manifest, the orphan figures were unknown but the snapshot's row still gave 0 blobs and 0 B, in the table and in --json. The row's blob count and blob size are now unknown, and null in --json. A directory with no manifest, as an interrupted backup leaves, still shows 0: the orphan figures count its blobs as orphaned, so it references none. Model: opus-5-5
594 lines
16 KiB
Go
594 lines
16 KiB
Go
package vaultik
|
|
|
|
import (
|
|
"encoding/json"
|
|
"fmt"
|
|
"runtime"
|
|
"sort"
|
|
"strings"
|
|
|
|
"github.com/dustin/go-humanize"
|
|
"sneak.berlin/go/vaultik/internal/log"
|
|
)
|
|
|
|
// ShowInfo displays system and configuration information
|
|
func (v *Vaultik) ShowInfo() error {
|
|
// The info report is the output this command exists to produce, so it
|
|
// is written plain (markers would corrupt the aligned report) through
|
|
// the UI writer's stdout; --quiet silences it like any other
|
|
// non-error output.
|
|
if v.UI.Quiet() {
|
|
return nil
|
|
}
|
|
|
|
// System Information
|
|
v.stdoutf("=== System Information ===\n")
|
|
v.stdoutf("OS/Architecture: %s/%s\n", runtime.GOOS, runtime.GOARCH)
|
|
v.stdoutf("Version: %s\n", v.Globals.Version)
|
|
v.stdoutf("Commit: %s\n", v.Globals.Commit)
|
|
v.stdoutf("Go Version: %s\n", runtime.Version())
|
|
v.printlnStdout()
|
|
|
|
v.showStorageConfig()
|
|
v.showBackupSettings()
|
|
|
|
// Encryption Configuration
|
|
v.stdoutf("=== Encryption Configuration ===\n")
|
|
v.stdoutf("Recipients:\n")
|
|
|
|
for _, recipient := range v.Config.AgeRecipients {
|
|
v.stdoutf(" - %s\n", recipient)
|
|
}
|
|
|
|
v.printlnStdout()
|
|
v.showLocalDatabase()
|
|
|
|
return nil
|
|
}
|
|
|
|
// showStorageConfig prints the storage configuration section. The
|
|
// backend is selected by storage_url (s3://, file://, rclone://); the
|
|
// legacy s3.* fields are only printed when they're actually populated,
|
|
// since the URL scheme is the primary configuration.
|
|
func (v *Vaultik) showStorageConfig() {
|
|
v.stdoutf("=== Storage Configuration ===\n")
|
|
storageInfo := v.Storage.Info()
|
|
v.stdoutf("Type: %s\n", storageInfo.Type)
|
|
v.stdoutf("Location: %s\n", storageInfo.Location)
|
|
|
|
if v.Config.StorageURL != "" {
|
|
v.stdoutf("Storage URL: %s\n", v.Config.StorageURL)
|
|
}
|
|
|
|
if v.Config.S3.Bucket != "" {
|
|
v.stdoutf("S3 Bucket: %s\n", v.Config.S3.Bucket)
|
|
}
|
|
|
|
if v.Config.S3.Prefix != "" {
|
|
v.stdoutf("S3 Prefix: %s\n", v.Config.S3.Prefix)
|
|
}
|
|
|
|
if v.Config.S3.Endpoint != "" {
|
|
v.stdoutf("S3 Endpoint: %s\n", v.Config.S3.Endpoint)
|
|
}
|
|
|
|
if v.Config.S3.Region != "" {
|
|
v.stdoutf("S3 Region: %s\n", v.Config.S3.Region)
|
|
}
|
|
|
|
v.printlnStdout()
|
|
}
|
|
|
|
// showBackupSettings prints the configured snapshots, exclude patterns,
|
|
// and chunking/compression settings.
|
|
func (v *Vaultik) showBackupSettings() {
|
|
v.stdoutf("=== Backup Settings ===\n")
|
|
|
|
// Show configured snapshots
|
|
v.stdoutf("Snapshots:\n")
|
|
|
|
for _, name := range v.Config.SnapshotNames() {
|
|
snap := v.Config.Snapshots[name]
|
|
v.stdoutf(" %s:\n", name)
|
|
|
|
for _, path := range snap.Paths {
|
|
v.stdoutf(" - %s\n", path)
|
|
}
|
|
|
|
if len(snap.Exclude) > 0 {
|
|
v.stdoutf(" exclude: %s\n", strings.Join(snap.Exclude, ", "))
|
|
}
|
|
}
|
|
|
|
// Global exclude patterns
|
|
if len(v.Config.Exclude) > 0 {
|
|
v.stdoutf("Global Exclude: %s\n", strings.Join(v.Config.Exclude, ", "))
|
|
}
|
|
|
|
v.stdoutf("Compression: zstd level %d\n", v.Config.CompressionLevel)
|
|
v.stdoutf("Chunk Size: %s\n", ubytes(int64(v.Config.ChunkSize)))
|
|
v.stdoutf("Blob Size Limit: %s\n", ubytes(int64(v.Config.BlobSizeLimit)))
|
|
v.printlnStdout()
|
|
}
|
|
|
|
// showLocalDatabase prints the local index database section, including
|
|
// record counts when the index exists.
|
|
func (v *Vaultik) showLocalDatabase() {
|
|
v.stdoutf("=== Local Database ===\n")
|
|
v.stdoutf("Index Path: %s\n", v.Config.IndexPath)
|
|
|
|
// Check if index file exists and get its size
|
|
info, err := v.Fs.Stat(v.Config.IndexPath)
|
|
if err != nil {
|
|
v.stdoutf("Index Size: (not created)\n")
|
|
|
|
return
|
|
}
|
|
|
|
v.stdoutf("Index Size: %s\n", ubytes(info.Size()))
|
|
|
|
// Get snapshot count from database
|
|
query := `SELECT COUNT(*) FROM snapshots WHERE completed_at IS NOT NULL`
|
|
|
|
var snapshotCount int
|
|
|
|
err = v.DB.Conn().QueryRowContext(v.ctx, query).Scan(&snapshotCount)
|
|
if err == nil {
|
|
v.stdoutf("Snapshots: %d\n", snapshotCount)
|
|
}
|
|
|
|
// Get blob count from database
|
|
query = `SELECT COUNT(*) FROM blobs`
|
|
|
|
var blobCount int
|
|
|
|
err = v.DB.Conn().QueryRowContext(v.ctx, query).Scan(&blobCount)
|
|
if err == nil {
|
|
v.stdoutf("Blobs: %d\n", blobCount)
|
|
}
|
|
|
|
// Get file count from database
|
|
query = `SELECT COUNT(*) FROM files`
|
|
|
|
var fileCount int
|
|
|
|
err = v.DB.Conn().QueryRowContext(v.ctx, query).Scan(&fileCount)
|
|
if err == nil {
|
|
v.stdoutf("Files: %d\n", fileCount)
|
|
}
|
|
}
|
|
|
|
// Table layout constants for the human-readable remote info output.
|
|
const (
|
|
// snapshotIDColWidth is the SNAPSHOT column width in the remote
|
|
// info table.
|
|
snapshotIDColWidth = 45
|
|
|
|
// metadataKeyParts is the minimum "/"-separated segment count of a
|
|
// metadata object key (metadata/<snapshot-id>/<filename>).
|
|
metadataKeyParts = 3
|
|
|
|
// blobKeyParts is the minimum "/"-separated segment count of a blob
|
|
// object key (blobs/<aa>/<bb>/<hash>).
|
|
blobKeyParts = 4
|
|
)
|
|
|
|
// SnapshotMetadataInfo contains information about a single snapshot's metadata
|
|
//
|
|
//nolint:tagliatelle // snake_case is the established JSON output format
|
|
type SnapshotMetadataInfo struct {
|
|
SnapshotID string `json:"snapshot_id"`
|
|
ManifestSize int64 `json:"manifest_size"`
|
|
DatabaseSize int64 `json:"database_size"`
|
|
TotalSize int64 `json:"total_size"`
|
|
|
|
// Both stay nil (null in the JSON) when the snapshot's manifest was
|
|
// listed but could not be read.
|
|
BlobCount *int `json:"blob_count"`
|
|
BlobsSize *int64 `json:"blobs_size"`
|
|
|
|
// Set when the listing holds this snapshot's manifest.json.zst. A
|
|
// backup interrupted before its manifest upload leaves a directory
|
|
// without one, which prune does not treat as a snapshot.
|
|
hasManifest bool
|
|
}
|
|
|
|
// RemoteInfoResult contains all remote storage information
|
|
//
|
|
//nolint:tagliatelle // snake_case is the established JSON output format
|
|
type RemoteInfoResult struct {
|
|
// Storage info
|
|
StorageType string `json:"storage_type"`
|
|
StorageLocation string `json:"storage_location"`
|
|
|
|
// Snapshot metadata
|
|
Snapshots []SnapshotMetadataInfo `json:"snapshots"`
|
|
TotalMetadataSize int64 `json:"total_metadata_size"`
|
|
TotalMetadataCount int `json:"total_metadata_count"`
|
|
|
|
// All blobs on remote
|
|
TotalBlobCount int `json:"total_blob_count"`
|
|
TotalBlobSize int64 `json:"total_blob_size"`
|
|
|
|
// Referenced blobs (from manifests)
|
|
ReferencedBlobCount int `json:"referenced_blob_count"`
|
|
ReferencedBlobSize int64 `json:"referenced_blob_size"`
|
|
|
|
// Orphaned blobs. Both stay nil (null in the JSON) when a manifest
|
|
// was listed but not read, since that snapshot's blobs would be
|
|
// counted as orphaned.
|
|
OrphanedBlobCount *int `json:"orphaned_blob_count"`
|
|
OrphanedBlobSize *int64 `json:"orphaned_blob_size"`
|
|
|
|
// Remote key of each snapshot whose manifest could not be read
|
|
UnreadableManifests []string `json:"unreadable_manifests,omitempty"`
|
|
|
|
// Number of manifests not read because the name above them under
|
|
// metadata/ is not a remote key. The names themselves are not
|
|
// reported: they come from the destination store and may hold
|
|
// control characters.
|
|
SkippedManifestCount int `json:"skipped_manifest_count,omitempty"`
|
|
}
|
|
|
|
// RemoteInfo displays information about remote storage
|
|
func (v *Vaultik) RemoteInfo(jsonOutput bool) error {
|
|
log.Info("Starting remote storage info gathering")
|
|
|
|
result := &RemoteInfoResult{}
|
|
|
|
storageInfo := v.Storage.Info()
|
|
result.StorageType = storageInfo.Type
|
|
result.StorageLocation = storageInfo.Location
|
|
|
|
// The human report is written only when it is neither the --json
|
|
// document (which needs stdout to itself) nor silenced by --quiet. The
|
|
// scan still runs in both cases so --json still gets a full result.
|
|
showText := !jsonOutput && !v.UI.Quiet()
|
|
|
|
if showText {
|
|
v.stdoutf("=== Remote Storage ===\n")
|
|
v.stdoutf("Type: %s\n", storageInfo.Type)
|
|
v.stdoutf("Location: %s\n", storageInfo.Location)
|
|
v.printlnStdout()
|
|
v.stdoutf("Scanning snapshot metadata...\n")
|
|
}
|
|
|
|
snapshotMetadata, snapshotIDs, skippedManifestCount, err := v.collectSnapshotMetadata()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
result.SkippedManifestCount = skippedManifestCount
|
|
|
|
if showText {
|
|
manifestCount := 0
|
|
|
|
for _, info := range snapshotMetadata {
|
|
if info.hasManifest {
|
|
manifestCount++
|
|
}
|
|
}
|
|
|
|
v.stdoutf("Downloading %d manifest(s)...\n", manifestCount)
|
|
}
|
|
|
|
referencedBlobs, unreadableManifests := v.collectReferencedBlobsFromManifests(
|
|
snapshotIDs, snapshotMetadata)
|
|
result.UnreadableManifests = unreadableManifests
|
|
|
|
v.populateRemoteInfoResult(result, snapshotMetadata, snapshotIDs, referencedBlobs)
|
|
|
|
err = v.scanRemoteBlobStorage(result, referencedBlobs, showText)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
log.Info("Remote info complete",
|
|
"snapshots", result.TotalMetadataCount,
|
|
"total_blobs", result.TotalBlobCount,
|
|
"referenced_blobs", result.ReferencedBlobCount,
|
|
"unreadable_manifests", len(result.UnreadableManifests))
|
|
|
|
if jsonOutput {
|
|
enc := json.NewEncoder(v.Stdout)
|
|
enc.SetIndent("", " ")
|
|
|
|
return enc.Encode(result)
|
|
}
|
|
|
|
if showText {
|
|
v.printRemoteInfoTable(result)
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
// collectSnapshotMetadata scans remote metadata and returns
|
|
// per-snapshot info, sorted IDs and the number of manifests it skipped
|
|
// because the name above them is not a remote key.
|
|
func (v *Vaultik) collectSnapshotMetadata() (
|
|
map[string]*SnapshotMetadataInfo, []string, int, error,
|
|
) {
|
|
snapshotMetadata := make(map[string]*SnapshotMetadataInfo)
|
|
skippedManifestCount := 0
|
|
|
|
metadataCh := v.Storage.ListStream(v.ctx, "metadata/")
|
|
for obj := range metadataCh {
|
|
if obj.Err != nil {
|
|
return nil, nil, 0, fmt.Errorf("listing metadata: %w", obj.Err)
|
|
}
|
|
|
|
parts := strings.Split(obj.Key, "/")
|
|
if len(parts) < metadataKeyParts {
|
|
continue
|
|
}
|
|
|
|
snapshotID := parts[1]
|
|
filename := parts[2]
|
|
isManifest := filename == "manifest.json.zst"
|
|
|
|
// The name comes from the destination store, which is not
|
|
// trusted, and is printed in the report. Accept it only in the
|
|
// form of a remote key.
|
|
if !isBlobHash(snapshotID) {
|
|
log.Warn("Skipping non-conforming key under metadata/",
|
|
"key", obj.Key)
|
|
|
|
if isManifest {
|
|
skippedManifestCount++
|
|
}
|
|
|
|
continue
|
|
}
|
|
|
|
if _, exists := snapshotMetadata[snapshotID]; !exists {
|
|
snapshotMetadata[snapshotID] = &SnapshotMetadataInfo{SnapshotID: snapshotID}
|
|
}
|
|
|
|
info := snapshotMetadata[snapshotID]
|
|
|
|
if isManifest {
|
|
info.hasManifest = true
|
|
}
|
|
|
|
if strings.HasPrefix(filename, "manifest") {
|
|
info.ManifestSize = obj.Size
|
|
} else if strings.HasPrefix(filename, "db") {
|
|
info.DatabaseSize = obj.Size
|
|
}
|
|
|
|
info.TotalSize = info.ManifestSize + info.DatabaseSize
|
|
}
|
|
|
|
var snapshotIDs []string
|
|
for id := range snapshotMetadata {
|
|
snapshotIDs = append(snapshotIDs, id)
|
|
}
|
|
|
|
sort.Strings(snapshotIDs)
|
|
|
|
return snapshotMetadata, snapshotIDs, skippedManifestCount, nil
|
|
}
|
|
|
|
// collectReferencedBlobsFromManifests downloads the listed manifests
|
|
// and returns referenced blob hashes with sizes, and the remote keys
|
|
// of the manifests it could not read.
|
|
func (v *Vaultik) collectReferencedBlobsFromManifests(
|
|
snapshotIDs []string, snapshotMetadata map[string]*SnapshotMetadataInfo,
|
|
) (map[string]int64, []string) {
|
|
referencedBlobs := make(map[string]int64)
|
|
|
|
var unreadable []string
|
|
|
|
for _, snapshotID := range snapshotIDs {
|
|
info := snapshotMetadata[snapshotID]
|
|
if !info.hasManifest {
|
|
// The orphan figures count this directory's blobs as
|
|
// orphaned, so it references none.
|
|
info.BlobCount, info.BlobsSize = new(int), new(int64)
|
|
|
|
continue
|
|
}
|
|
|
|
// snapshotIDs here are remote keys, taken straight from the
|
|
// metadata/ listing. downloadManifestByKey is the single reader
|
|
// for remote manifests; see its doc comment.
|
|
manifest, err := v.downloadManifestByKey(snapshotID)
|
|
if err != nil {
|
|
log.Warn("Failed to read manifest", "snapshot", snapshotID, "error", err)
|
|
|
|
unreadable = append(unreadable, snapshotID)
|
|
|
|
continue
|
|
}
|
|
|
|
blobCount := manifest.BlobCount
|
|
|
|
var blobsSize int64
|
|
|
|
for _, blob := range manifest.Blobs {
|
|
referencedBlobs[blob.Hash] = blob.CompressedSize
|
|
blobsSize += blob.CompressedSize
|
|
}
|
|
|
|
info.BlobCount = &blobCount
|
|
info.BlobsSize = &blobsSize
|
|
}
|
|
|
|
return referencedBlobs, unreadable
|
|
}
|
|
|
|
// populateRemoteInfoResult fills in the result's snapshot and
|
|
// referenced blob stats.
|
|
func (v *Vaultik) populateRemoteInfoResult(
|
|
result *RemoteInfoResult,
|
|
snapshotMetadata map[string]*SnapshotMetadataInfo,
|
|
snapshotIDs []string,
|
|
referencedBlobs map[string]int64,
|
|
) {
|
|
var totalMetadataSize int64
|
|
|
|
for _, id := range snapshotIDs {
|
|
info := snapshotMetadata[id]
|
|
result.Snapshots = append(result.Snapshots, *info)
|
|
totalMetadataSize += info.TotalSize
|
|
}
|
|
|
|
result.TotalMetadataSize = totalMetadataSize
|
|
result.TotalMetadataCount = len(snapshotIDs)
|
|
|
|
for _, size := range referencedBlobs {
|
|
result.ReferencedBlobCount++
|
|
result.ReferencedBlobSize += size
|
|
}
|
|
}
|
|
|
|
// scanRemoteBlobStorage lists all blobs on remote and computes orphan
|
|
// stats when every listed manifest was read. showText is true only
|
|
// when the human report is being printed (not --json, not --quiet),
|
|
// gating the progress line.
|
|
func (v *Vaultik) scanRemoteBlobStorage(
|
|
result *RemoteInfoResult, referencedBlobs map[string]int64, showText bool,
|
|
) error {
|
|
if showText {
|
|
v.stdoutf("Scanning blobs...\n")
|
|
}
|
|
|
|
blobCh := v.Storage.ListStream(v.ctx, "blobs/")
|
|
allBlobs := make(map[string]int64)
|
|
|
|
for obj := range blobCh {
|
|
if obj.Err != nil {
|
|
return fmt.Errorf("listing blobs: %w", obj.Err)
|
|
}
|
|
|
|
parts := strings.Split(obj.Key, "/")
|
|
if len(parts) < blobKeyParts {
|
|
continue
|
|
}
|
|
|
|
hash := parts[3]
|
|
allBlobs[hash] = obj.Size
|
|
result.TotalBlobCount++
|
|
result.TotalBlobSize += obj.Size
|
|
}
|
|
|
|
// A blob named only by a manifest that could not be read, or by one
|
|
// under a skipped name, would be counted as orphaned, so the orphan
|
|
// figures stay unknown.
|
|
if len(result.UnreadableManifests) > 0 || result.SkippedManifestCount > 0 {
|
|
return nil
|
|
}
|
|
|
|
var (
|
|
orphanedCount int
|
|
orphanedSize int64
|
|
)
|
|
|
|
for hash, size := range allBlobs {
|
|
if _, referenced := referencedBlobs[hash]; !referenced {
|
|
orphanedCount++
|
|
orphanedSize += size
|
|
}
|
|
}
|
|
|
|
result.OrphanedBlobCount = &orphanedCount
|
|
result.OrphanedBlobSize = &orphanedSize
|
|
|
|
return nil
|
|
}
|
|
|
|
// printRemoteInfoTable renders the human-readable remote info output
|
|
func (v *Vaultik) printRemoteInfoTable(result *RemoteInfoResult) {
|
|
const (
|
|
rowFormat = "%-45s %12s %12s %12s %10s %12s\n"
|
|
sizeColWidth = 12
|
|
countColWidth = 10
|
|
)
|
|
|
|
v.stdoutf("\n=== Snapshot Metadata ===\n")
|
|
|
|
if len(result.Snapshots) == 0 {
|
|
v.stdoutf("No snapshots found\n")
|
|
} else {
|
|
separator := fmt.Sprintf(rowFormat,
|
|
strings.Repeat("-", snapshotIDColWidth),
|
|
strings.Repeat("-", sizeColWidth),
|
|
strings.Repeat("-", sizeColWidth),
|
|
strings.Repeat("-", sizeColWidth),
|
|
strings.Repeat("-", countColWidth),
|
|
strings.Repeat("-", sizeColWidth))
|
|
|
|
v.stdoutf(rowFormat,
|
|
"SNAPSHOT", "MANIFEST", "DATABASE", "TOTAL", "BLOBS", "BLOB SIZE")
|
|
v.stdoutf("%s", separator)
|
|
|
|
for _, info := range result.Snapshots {
|
|
blobCount := countUnknown
|
|
if info.BlobCount != nil {
|
|
blobCount = humanize.Comma(int64(*info.BlobCount))
|
|
}
|
|
|
|
blobsSize := countUnknown
|
|
if info.BlobsSize != nil {
|
|
blobsSize = ubytes(*info.BlobsSize)
|
|
}
|
|
|
|
v.stdoutf(rowFormat,
|
|
truncateString(info.SnapshotID, snapshotIDColWidth),
|
|
ubytes(info.ManifestSize),
|
|
ubytes(info.DatabaseSize),
|
|
ubytes(info.TotalSize),
|
|
blobCount,
|
|
blobsSize,
|
|
)
|
|
}
|
|
|
|
v.stdoutf("%s", separator)
|
|
v.stdoutf("%-45s %12s %12s %12s\n",
|
|
fmt.Sprintf("Total (%d snapshots)", result.TotalMetadataCount),
|
|
"", "", ubytes(result.TotalMetadataSize))
|
|
}
|
|
|
|
v.stdoutf("\n=== Blob Storage ===\n")
|
|
v.stdoutf("Total blobs on remote: %s (%s)\n",
|
|
humanize.Comma(int64(result.TotalBlobCount)),
|
|
ubytes(result.TotalBlobSize))
|
|
v.stdoutf("Referenced by snapshots: %s (%s)\n",
|
|
humanize.Comma(int64(result.ReferencedBlobCount)),
|
|
ubytes(result.ReferencedBlobSize))
|
|
|
|
if result.OrphanedBlobCount == nil {
|
|
v.stdoutf("Orphaned (unreferenced): unknown "+
|
|
"(%d manifest(s) could not be read, "+
|
|
"%d manifest(s) under a non-conforming name skipped)\n",
|
|
len(result.UnreadableManifests), result.SkippedManifestCount)
|
|
|
|
return
|
|
}
|
|
|
|
v.stdoutf("Orphaned (unreferenced): %s (%s)\n",
|
|
humanize.Comma(int64(*result.OrphanedBlobCount)),
|
|
ubytes(*result.OrphanedBlobSize))
|
|
|
|
if *result.OrphanedBlobCount > 0 {
|
|
v.stdoutf("\nRun 'vaultik prune' to remove orphaned blobs.\n")
|
|
}
|
|
}
|
|
|
|
// ellipsis is appended by truncateString when it shortens a string.
|
|
const ellipsis = "..."
|
|
|
|
// truncateString truncates a string to maxLen, adding "..." if truncated
|
|
func truncateString(s string, maxLen int) string {
|
|
if len(s) <= maxLen {
|
|
return s
|
|
}
|
|
|
|
if maxLen <= len(ellipsis) {
|
|
return s[:maxLen]
|
|
}
|
|
|
|
return s[:maxLen-len(ellipsis)] + ellipsis
|
|
}
|