Refactor: break up oversized methods into smaller descriptive helpers (#41)
All checks were successful
check / check (push) Successful in 4m17s
All checks were successful
check / check (push) Successful in 4m17s
Closes #40 Per sneak's feedback on PR #37: methods were too long. This PR breaks all methods over 100-150 lines into smaller, descriptively named helper methods. ## Refactored methods (8 total) | Original | Lines | Helpers extracted | |---|---|---| | `createNamedSnapshot` | 214 | `resolveSnapshotPaths`, `scanAllDirectories`, `collectUploadStats`, `finalizeSnapshotMetadata`, `printSnapshotSummary`, `getSnapshotBlobSizes`, `formatUploadSpeed` | | `ListSnapshots` | 159 | `listRemoteSnapshotIDs`, `reconcileLocalWithRemote`, `buildSnapshotInfoList`, `printSnapshotTable` | | `PruneBlobs` | 170 | `collectReferencedBlobs`, `listUniqueSnapshotIDs`, `listAllRemoteBlobs`, `findUnreferencedBlobs`, `deleteUnreferencedBlobs` | | `RunDeepVerify` | 182 | `loadVerificationData`, `runVerificationSteps`, `deepVerifyFailure` | | `RemoteInfo` | 187 | `collectSnapshotMetadata`, `collectReferencedBlobsFromManifests`, `populateRemoteInfoResult`, `scanRemoteBlobStorage`, `printRemoteInfoTable` | | `handleBlobReady` | 173 | `uploadBlobIfNeeded`, `makeUploadProgressCallback`, `recordBlobMetadata`, `cleanupBlobTempFile` | | `processFileStreaming` | 146 | `updateChunkStats`, `addChunkToPacker`, `queueFileForBatchInsert` | | `finalizeCurrentBlob` | 167 | `closeBlobWriter`, `buildChunkRefs`, `commitBlobToDatabase`, `deliverFinishedBlob` | ## Verification - `go build ./...` ✅ - `make test` ✅ (all tests pass) - `golangci-lint run` ✅ (0 issues) - No behavioral changes, pure restructuring Co-authored-by: user <user@Mac.lan guest wan> Reviewed-on: #41 Co-authored-by: clawbot <clawbot@noreply.example.org> Co-committed-by: clawbot <clawbot@noreply.example.org>
This commit was merged in pull request #41.
This commit is contained in:
@@ -111,40 +111,34 @@ func (v *Vaultik) CreateSnapshot(opts *SnapshotCreateOptions) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// snapshotStats tracks aggregate statistics across directory scans
|
||||
type snapshotStats struct {
|
||||
totalFiles int
|
||||
totalBytes int64
|
||||
totalChunks int
|
||||
totalBlobs int
|
||||
totalBytesSkipped int64
|
||||
totalFilesSkipped int
|
||||
totalFilesDeleted int
|
||||
totalBytesDeleted int64
|
||||
totalBytesUploaded int64
|
||||
totalBlobsUploaded int
|
||||
uploadDuration time.Duration
|
||||
}
|
||||
|
||||
// createNamedSnapshot creates a single named snapshot
|
||||
func (v *Vaultik) createNamedSnapshot(opts *SnapshotCreateOptions, hostname, snapName string, idx, total int) error {
|
||||
snapshotStartTime := time.Now()
|
||||
|
||||
snapConfig := v.Config.Snapshots[snapName]
|
||||
|
||||
if total > 1 {
|
||||
v.printfStdout("\n=== Snapshot %d/%d: %s ===\n", idx, total, snapName)
|
||||
}
|
||||
|
||||
// Resolve source directories to absolute paths
|
||||
resolvedDirs := make([]string, 0, len(snapConfig.Paths))
|
||||
for _, dir := range snapConfig.Paths {
|
||||
absPath, err := filepath.Abs(dir)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to resolve absolute path for %s: %w", dir, err)
|
||||
}
|
||||
|
||||
// Resolve symlinks
|
||||
resolvedPath, err := filepath.EvalSymlinks(absPath)
|
||||
if err != nil {
|
||||
// If the path doesn't exist yet, use the absolute path
|
||||
if os.IsNotExist(err) {
|
||||
resolvedPath = absPath
|
||||
} else {
|
||||
return fmt.Errorf("failed to resolve symlinks for %s: %w", absPath, err)
|
||||
}
|
||||
}
|
||||
|
||||
resolvedDirs = append(resolvedDirs, resolvedPath)
|
||||
resolvedDirs, err := v.resolveSnapshotPaths(snapName)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Create scanner with progress enabled (unless in cron mode)
|
||||
// Pass the combined excludes for this snapshot
|
||||
scanner := v.ScannerFactory(snapshot.ScannerParams{
|
||||
EnableProgress: !opts.Cron,
|
||||
Fs: v.Fs,
|
||||
@@ -152,20 +146,6 @@ func (v *Vaultik) createNamedSnapshot(opts *SnapshotCreateOptions, hostname, sna
|
||||
SkipErrors: opts.SkipErrors,
|
||||
})
|
||||
|
||||
// Statistics tracking
|
||||
totalFiles := 0
|
||||
totalBytes := int64(0)
|
||||
totalChunks := 0
|
||||
totalBlobs := 0
|
||||
totalBytesSkipped := int64(0)
|
||||
totalFilesSkipped := 0
|
||||
totalFilesDeleted := 0
|
||||
totalBytesDeleted := int64(0)
|
||||
totalBytesUploaded := int64(0)
|
||||
totalBlobsUploaded := 0
|
||||
uploadDuration := time.Duration(0)
|
||||
|
||||
// Create a new snapshot at the beginning (with snapshot name in ID)
|
||||
snapshotID, err := v.SnapshotManager.CreateSnapshotWithName(v.ctx, hostname, snapName, v.Globals.Version, v.Globals.Commit)
|
||||
if err != nil {
|
||||
return fmt.Errorf("creating snapshot: %w", err)
|
||||
@@ -173,12 +153,64 @@ func (v *Vaultik) createNamedSnapshot(opts *SnapshotCreateOptions, hostname, sna
|
||||
log.Info("Beginning snapshot", "snapshot_id", snapshotID, "name", snapName)
|
||||
v.printfStdout("Beginning snapshot: %s\n", snapshotID)
|
||||
|
||||
stats, err := v.scanAllDirectories(scanner, resolvedDirs, snapshotID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
v.collectUploadStats(scanner, stats)
|
||||
|
||||
if err := v.finalizeSnapshotMetadata(snapshotID, stats); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
log.Info("Snapshot complete",
|
||||
"snapshot_id", snapshotID,
|
||||
"name", snapName,
|
||||
"files", stats.totalFiles,
|
||||
"blobs_uploaded", stats.totalBlobsUploaded,
|
||||
"bytes_uploaded", stats.totalBytesUploaded,
|
||||
"duration", time.Since(snapshotStartTime))
|
||||
|
||||
v.printSnapshotSummary(snapshotID, snapshotStartTime, stats)
|
||||
return nil
|
||||
}
|
||||
|
||||
// resolveSnapshotPaths resolves source directories to absolute paths with symlink resolution
|
||||
func (v *Vaultik) resolveSnapshotPaths(snapName string) ([]string, error) {
|
||||
snapConfig := v.Config.Snapshots[snapName]
|
||||
resolvedDirs := make([]string, 0, len(snapConfig.Paths))
|
||||
|
||||
for _, dir := range snapConfig.Paths {
|
||||
absPath, err := filepath.Abs(dir)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to resolve absolute path for %s: %w", dir, err)
|
||||
}
|
||||
|
||||
resolvedPath, err := filepath.EvalSymlinks(absPath)
|
||||
if err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
resolvedPath = absPath
|
||||
} else {
|
||||
return nil, fmt.Errorf("failed to resolve symlinks for %s: %w", absPath, err)
|
||||
}
|
||||
}
|
||||
|
||||
resolvedDirs = append(resolvedDirs, resolvedPath)
|
||||
}
|
||||
|
||||
return resolvedDirs, nil
|
||||
}
|
||||
|
||||
// scanAllDirectories runs the scanner on each resolved directory and accumulates stats
|
||||
func (v *Vaultik) scanAllDirectories(scanner *snapshot.Scanner, resolvedDirs []string, snapshotID string) (*snapshotStats, error) {
|
||||
stats := &snapshotStats{}
|
||||
|
||||
for i, dir := range resolvedDirs {
|
||||
// Check if context is cancelled
|
||||
select {
|
||||
case <-v.ctx.Done():
|
||||
log.Info("Snapshot creation cancelled")
|
||||
return v.ctx.Err()
|
||||
return nil, v.ctx.Err()
|
||||
default:
|
||||
}
|
||||
|
||||
@@ -186,17 +218,17 @@ func (v *Vaultik) createNamedSnapshot(opts *SnapshotCreateOptions, hostname, sna
|
||||
v.printfStdout("Beginning directory scan (%d/%d): %s\n", i+1, len(resolvedDirs), dir)
|
||||
result, err := scanner.Scan(v.ctx, dir, snapshotID)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to scan %s: %w", dir, err)
|
||||
return nil, fmt.Errorf("failed to scan %s: %w", dir, err)
|
||||
}
|
||||
|
||||
totalFiles += result.FilesScanned
|
||||
totalBytes += result.BytesScanned
|
||||
totalChunks += result.ChunksCreated
|
||||
totalBlobs += result.BlobsCreated
|
||||
totalFilesSkipped += result.FilesSkipped
|
||||
totalBytesSkipped += result.BytesSkipped
|
||||
totalFilesDeleted += result.FilesDeleted
|
||||
totalBytesDeleted += result.BytesDeleted
|
||||
stats.totalFiles += result.FilesScanned
|
||||
stats.totalBytes += result.BytesScanned
|
||||
stats.totalChunks += result.ChunksCreated
|
||||
stats.totalBlobs += result.BlobsCreated
|
||||
stats.totalFilesSkipped += result.FilesSkipped
|
||||
stats.totalBytesSkipped += result.BytesSkipped
|
||||
stats.totalFilesDeleted += result.FilesDeleted
|
||||
stats.totalBytesDeleted += result.BytesDeleted
|
||||
|
||||
log.Info("Directory scan complete",
|
||||
"path", dir,
|
||||
@@ -207,85 +239,79 @@ func (v *Vaultik) createNamedSnapshot(opts *SnapshotCreateOptions, hostname, sna
|
||||
"chunks", result.ChunksCreated,
|
||||
"blobs", result.BlobsCreated,
|
||||
"duration", result.EndTime.Sub(result.StartTime))
|
||||
|
||||
// Remove per-directory summary - the scanner already prints its own summary
|
||||
}
|
||||
|
||||
// Get upload statistics from scanner progress if available
|
||||
return stats, nil
|
||||
}
|
||||
|
||||
// collectUploadStats gathers upload statistics from the scanner's progress reporter
|
||||
func (v *Vaultik) collectUploadStats(scanner *snapshot.Scanner, stats *snapshotStats) {
|
||||
if s := scanner.GetProgress(); s != nil {
|
||||
stats := s.GetStats()
|
||||
totalBytesUploaded = stats.BytesUploaded.Load()
|
||||
totalBlobsUploaded = int(stats.BlobsUploaded.Load())
|
||||
uploadDuration = time.Duration(stats.UploadDurationMs.Load()) * time.Millisecond
|
||||
progressStats := s.GetStats()
|
||||
stats.totalBytesUploaded = progressStats.BytesUploaded.Load()
|
||||
stats.totalBlobsUploaded = int(progressStats.BlobsUploaded.Load())
|
||||
stats.uploadDuration = time.Duration(progressStats.UploadDurationMs.Load()) * time.Millisecond
|
||||
}
|
||||
}
|
||||
|
||||
// Update snapshot statistics with extended fields
|
||||
// finalizeSnapshotMetadata updates stats, marks complete, and exports metadata
|
||||
func (v *Vaultik) finalizeSnapshotMetadata(snapshotID string, stats *snapshotStats) error {
|
||||
extStats := snapshot.ExtendedBackupStats{
|
||||
BackupStats: snapshot.BackupStats{
|
||||
FilesScanned: totalFiles,
|
||||
BytesScanned: totalBytes,
|
||||
ChunksCreated: totalChunks,
|
||||
BlobsCreated: totalBlobs,
|
||||
BytesUploaded: totalBytesUploaded,
|
||||
FilesScanned: stats.totalFiles,
|
||||
BytesScanned: stats.totalBytes,
|
||||
ChunksCreated: stats.totalChunks,
|
||||
BlobsCreated: stats.totalBlobs,
|
||||
BytesUploaded: stats.totalBytesUploaded,
|
||||
},
|
||||
BlobUncompressedSize: 0, // Will be set from database query below
|
||||
BlobUncompressedSize: 0,
|
||||
CompressionLevel: v.Config.CompressionLevel,
|
||||
UploadDurationMs: uploadDuration.Milliseconds(),
|
||||
UploadDurationMs: stats.uploadDuration.Milliseconds(),
|
||||
}
|
||||
|
||||
if err := v.SnapshotManager.UpdateSnapshotStatsExtended(v.ctx, snapshotID, extStats); err != nil {
|
||||
return fmt.Errorf("updating snapshot stats: %w", err)
|
||||
}
|
||||
|
||||
// Mark snapshot as complete
|
||||
if err := v.SnapshotManager.CompleteSnapshot(v.ctx, snapshotID); err != nil {
|
||||
return fmt.Errorf("completing snapshot: %w", err)
|
||||
}
|
||||
|
||||
// Export snapshot metadata
|
||||
// Export snapshot metadata without closing the database
|
||||
// The export function should handle its own database connection
|
||||
if err := v.SnapshotManager.ExportSnapshotMetadata(v.ctx, v.Config.IndexPath, snapshotID); err != nil {
|
||||
return fmt.Errorf("exporting snapshot metadata: %w", err)
|
||||
}
|
||||
|
||||
// Calculate final statistics
|
||||
snapshotDuration := time.Since(snapshotStartTime)
|
||||
totalFilesChanged := totalFiles - totalFilesSkipped
|
||||
totalBytesChanged := totalBytes
|
||||
totalBytesAll := totalBytes + totalBytesSkipped
|
||||
return nil
|
||||
}
|
||||
|
||||
// Calculate upload speed
|
||||
var avgUploadSpeed string
|
||||
if totalBytesUploaded > 0 && uploadDuration > 0 {
|
||||
bytesPerSec := float64(totalBytesUploaded) / uploadDuration.Seconds()
|
||||
bitsPerSec := bytesPerSec * 8
|
||||
if bitsPerSec >= 1e9 {
|
||||
avgUploadSpeed = fmt.Sprintf("%.1f Gbit/s", bitsPerSec/1e9)
|
||||
} else if bitsPerSec >= 1e6 {
|
||||
avgUploadSpeed = fmt.Sprintf("%.0f Mbit/s", bitsPerSec/1e6)
|
||||
} else if bitsPerSec >= 1e3 {
|
||||
avgUploadSpeed = fmt.Sprintf("%.0f Kbit/s", bitsPerSec/1e3)
|
||||
} else {
|
||||
avgUploadSpeed = fmt.Sprintf("%.0f bit/s", bitsPerSec)
|
||||
}
|
||||
} else {
|
||||
avgUploadSpeed = "N/A"
|
||||
// formatUploadSpeed formats bytes uploaded and duration into a human-readable speed string
|
||||
func formatUploadSpeed(bytesUploaded int64, duration time.Duration) string {
|
||||
if bytesUploaded <= 0 || duration <= 0 {
|
||||
return "N/A"
|
||||
}
|
||||
bytesPerSec := float64(bytesUploaded) / duration.Seconds()
|
||||
bitsPerSec := bytesPerSec * 8
|
||||
switch {
|
||||
case bitsPerSec >= 1e9:
|
||||
return fmt.Sprintf("%.1f Gbit/s", bitsPerSec/1e9)
|
||||
case bitsPerSec >= 1e6:
|
||||
return fmt.Sprintf("%.0f Mbit/s", bitsPerSec/1e6)
|
||||
case bitsPerSec >= 1e3:
|
||||
return fmt.Sprintf("%.0f Kbit/s", bitsPerSec/1e3)
|
||||
default:
|
||||
return fmt.Sprintf("%.0f bit/s", bitsPerSec)
|
||||
}
|
||||
}
|
||||
|
||||
// printSnapshotSummary prints the comprehensive snapshot completion summary
|
||||
func (v *Vaultik) printSnapshotSummary(snapshotID string, startTime time.Time, stats *snapshotStats) {
|
||||
snapshotDuration := time.Since(startTime)
|
||||
totalFilesChanged := stats.totalFiles - stats.totalFilesSkipped
|
||||
totalBytesAll := stats.totalBytes + stats.totalBytesSkipped
|
||||
|
||||
// Get total blob sizes from database
|
||||
totalBlobSizeCompressed := int64(0)
|
||||
totalBlobSizeUncompressed := int64(0)
|
||||
if blobHashes, err := v.Repositories.Snapshots.GetBlobHashes(v.ctx, snapshotID); err == nil {
|
||||
for _, hash := range blobHashes {
|
||||
if blob, err := v.Repositories.Blobs.GetByHash(v.ctx, hash); err == nil && blob != nil {
|
||||
totalBlobSizeCompressed += blob.CompressedSize
|
||||
totalBlobSizeUncompressed += blob.UncompressedSize
|
||||
}
|
||||
}
|
||||
}
|
||||
totalBlobSizeCompressed, totalBlobSizeUncompressed := v.getSnapshotBlobSizes(snapshotID)
|
||||
|
||||
// Calculate compression ratio
|
||||
var compressionRatio float64
|
||||
if totalBlobSizeUncompressed > 0 {
|
||||
compressionRatio = float64(totalBlobSizeCompressed) / float64(totalBlobSizeUncompressed)
|
||||
@@ -293,55 +319,96 @@ func (v *Vaultik) createNamedSnapshot(opts *SnapshotCreateOptions, hostname, sna
|
||||
compressionRatio = 1.0
|
||||
}
|
||||
|
||||
// Print comprehensive summary
|
||||
v.printfStdout("=== Snapshot Complete ===\n")
|
||||
v.printfStdout("ID: %s\n", snapshotID)
|
||||
v.printfStdout("Files: %s examined, %s to process, %s unchanged",
|
||||
formatNumber(totalFiles),
|
||||
formatNumber(stats.totalFiles),
|
||||
formatNumber(totalFilesChanged),
|
||||
formatNumber(totalFilesSkipped))
|
||||
if totalFilesDeleted > 0 {
|
||||
v.printfStdout(", %s deleted", formatNumber(totalFilesDeleted))
|
||||
formatNumber(stats.totalFilesSkipped))
|
||||
if stats.totalFilesDeleted > 0 {
|
||||
v.printfStdout(", %s deleted", formatNumber(stats.totalFilesDeleted))
|
||||
}
|
||||
v.printlnStdout()
|
||||
v.printfStdout("Data: %s total (%s to process)",
|
||||
humanize.Bytes(uint64(totalBytesAll)),
|
||||
humanize.Bytes(uint64(totalBytesChanged)))
|
||||
if totalBytesDeleted > 0 {
|
||||
v.printfStdout(", %s deleted", humanize.Bytes(uint64(totalBytesDeleted)))
|
||||
humanize.Bytes(uint64(stats.totalBytes)))
|
||||
if stats.totalBytesDeleted > 0 {
|
||||
v.printfStdout(", %s deleted", humanize.Bytes(uint64(stats.totalBytesDeleted)))
|
||||
}
|
||||
v.printlnStdout()
|
||||
if totalBlobsUploaded > 0 {
|
||||
if stats.totalBlobsUploaded > 0 {
|
||||
v.printfStdout("Storage: %s compressed from %s (%.2fx)\n",
|
||||
humanize.Bytes(uint64(totalBlobSizeCompressed)),
|
||||
humanize.Bytes(uint64(totalBlobSizeUncompressed)),
|
||||
compressionRatio)
|
||||
v.printfStdout("Upload: %d blobs, %s in %s (%s)\n",
|
||||
totalBlobsUploaded,
|
||||
humanize.Bytes(uint64(totalBytesUploaded)),
|
||||
formatDuration(uploadDuration),
|
||||
avgUploadSpeed)
|
||||
stats.totalBlobsUploaded,
|
||||
humanize.Bytes(uint64(stats.totalBytesUploaded)),
|
||||
formatDuration(stats.uploadDuration),
|
||||
formatUploadSpeed(stats.totalBytesUploaded, stats.uploadDuration))
|
||||
}
|
||||
v.printfStdout("Duration: %s\n", formatDuration(snapshotDuration))
|
||||
}
|
||||
|
||||
return nil
|
||||
// getSnapshotBlobSizes returns total compressed and uncompressed blob sizes for a snapshot
|
||||
func (v *Vaultik) getSnapshotBlobSizes(snapshotID string) (compressed int64, uncompressed int64) {
|
||||
blobHashes, err := v.Repositories.Snapshots.GetBlobHashes(v.ctx, snapshotID)
|
||||
if err != nil {
|
||||
return 0, 0
|
||||
}
|
||||
for _, hash := range blobHashes {
|
||||
if blob, err := v.Repositories.Blobs.GetByHash(v.ctx, hash); err == nil && blob != nil {
|
||||
compressed += blob.CompressedSize
|
||||
uncompressed += blob.UncompressedSize
|
||||
}
|
||||
}
|
||||
return compressed, uncompressed
|
||||
}
|
||||
|
||||
// ListSnapshots lists all snapshots
|
||||
func (v *Vaultik) ListSnapshots(jsonOutput bool) error {
|
||||
// Get all remote snapshots
|
||||
log.Info("Listing snapshots")
|
||||
remoteSnapshots, err := v.listRemoteSnapshotIDs()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
localSnapshotMap, err := v.reconcileLocalWithRemote(remoteSnapshots)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
snapshots, err := v.buildSnapshotInfoList(remoteSnapshots, localSnapshotMap)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Sort by timestamp (newest first)
|
||||
sort.Slice(snapshots, func(i, j int) bool {
|
||||
return snapshots[i].Timestamp.After(snapshots[j].Timestamp)
|
||||
})
|
||||
|
||||
if jsonOutput {
|
||||
encoder := json.NewEncoder(v.Stdout)
|
||||
encoder.SetIndent("", " ")
|
||||
return encoder.Encode(snapshots)
|
||||
}
|
||||
|
||||
return v.printSnapshotTable(snapshots)
|
||||
}
|
||||
|
||||
// listRemoteSnapshotIDs returns a set of snapshot IDs found in remote storage
|
||||
func (v *Vaultik) listRemoteSnapshotIDs() (map[string]bool, error) {
|
||||
remoteSnapshots := make(map[string]bool)
|
||||
objectCh := v.Storage.ListStream(v.ctx, "metadata/")
|
||||
|
||||
for object := range objectCh {
|
||||
if object.Err != nil {
|
||||
return fmt.Errorf("listing remote snapshots: %w", object.Err)
|
||||
return nil, fmt.Errorf("listing remote snapshots: %w", object.Err)
|
||||
}
|
||||
|
||||
// Extract snapshot ID from paths like metadata/hostname-20240115-143052Z/
|
||||
parts := strings.Split(object.Key, "/")
|
||||
if len(parts) >= 2 && parts[0] == "metadata" && parts[1] != "" {
|
||||
// Skip macOS resource fork files (._*) and other hidden files
|
||||
if strings.HasPrefix(parts[1], ".") {
|
||||
continue
|
||||
}
|
||||
@@ -349,56 +416,46 @@ func (v *Vaultik) ListSnapshots(jsonOutput bool) error {
|
||||
}
|
||||
}
|
||||
|
||||
// Get all local snapshots
|
||||
return remoteSnapshots, nil
|
||||
}
|
||||
|
||||
// reconcileLocalWithRemote removes local snapshots not in remote and returns the surviving local map
|
||||
func (v *Vaultik) reconcileLocalWithRemote(remoteSnapshots map[string]bool) (map[string]*database.Snapshot, error) {
|
||||
localSnapshots, err := v.Repositories.Snapshots.ListRecent(v.ctx, 10000)
|
||||
if err != nil {
|
||||
return fmt.Errorf("listing local snapshots: %w", err)
|
||||
return nil, fmt.Errorf("listing local snapshots: %w", err)
|
||||
}
|
||||
|
||||
// Build a map of local snapshots for quick lookup
|
||||
localSnapshotMap := make(map[string]*database.Snapshot)
|
||||
for _, s := range localSnapshots {
|
||||
localSnapshotMap[s.ID.String()] = s
|
||||
}
|
||||
|
||||
// Remove local snapshots that don't exist remotely
|
||||
for _, snapshot := range localSnapshots {
|
||||
snapshotIDStr := snapshot.ID.String()
|
||||
for _, snap := range localSnapshots {
|
||||
snapshotIDStr := snap.ID.String()
|
||||
if !remoteSnapshots[snapshotIDStr] {
|
||||
log.Info("Removing local snapshot not found in remote", "snapshot_id", snapshot.ID)
|
||||
|
||||
// Delete related records first to avoid foreign key constraints
|
||||
if err := v.Repositories.Snapshots.DeleteSnapshotFiles(v.ctx, snapshotIDStr); err != nil {
|
||||
log.Error("Failed to delete snapshot files", "snapshot_id", snapshot.ID, "error", err)
|
||||
}
|
||||
if err := v.Repositories.Snapshots.DeleteSnapshotBlobs(v.ctx, snapshotIDStr); err != nil {
|
||||
log.Error("Failed to delete snapshot blobs", "snapshot_id", snapshot.ID, "error", err)
|
||||
}
|
||||
if err := v.Repositories.Snapshots.DeleteSnapshotUploads(v.ctx, snapshotIDStr); err != nil {
|
||||
log.Error("Failed to delete snapshot uploads", "snapshot_id", snapshot.ID, "error", err)
|
||||
}
|
||||
|
||||
// Now delete the snapshot itself
|
||||
if err := v.Repositories.Snapshots.Delete(v.ctx, snapshotIDStr); err != nil {
|
||||
log.Error("Failed to delete local snapshot", "snapshot_id", snapshot.ID, "error", err)
|
||||
log.Info("Removing local snapshot not found in remote", "snapshot_id", snap.ID)
|
||||
if err := v.deleteSnapshotFromLocalDB(snapshotIDStr); err != nil {
|
||||
log.Error("Failed to delete local snapshot", "snapshot_id", snap.ID, "error", err)
|
||||
} else {
|
||||
log.Info("Deleted local snapshot not found in remote", "snapshot_id", snapshot.ID)
|
||||
log.Info("Deleted local snapshot not found in remote", "snapshot_id", snap.ID)
|
||||
delete(localSnapshotMap, snapshotIDStr)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Build final snapshot list
|
||||
return localSnapshotMap, nil
|
||||
}
|
||||
|
||||
// buildSnapshotInfoList constructs SnapshotInfo entries from remote IDs and local data
|
||||
func (v *Vaultik) buildSnapshotInfoList(remoteSnapshots map[string]bool, localSnapshotMap map[string]*database.Snapshot) ([]SnapshotInfo, error) {
|
||||
snapshots := make([]SnapshotInfo, 0, len(remoteSnapshots))
|
||||
|
||||
for snapshotID := range remoteSnapshots {
|
||||
// Check if we have this snapshot locally
|
||||
if localSnap, exists := localSnapshotMap[snapshotID]; exists && localSnap.CompletedAt != nil {
|
||||
// Get total compressed size of all blobs referenced by this snapshot
|
||||
totalSize, err := v.Repositories.Snapshots.GetSnapshotTotalCompressedSize(v.ctx, snapshotID)
|
||||
if err != nil {
|
||||
log.Warn("Failed to get total compressed size", "id", snapshotID, "error", err)
|
||||
// Fall back to stored blob size
|
||||
totalSize = localSnap.BlobSize
|
||||
}
|
||||
|
||||
@@ -408,17 +465,15 @@ func (v *Vaultik) ListSnapshots(jsonOutput bool) error {
|
||||
CompressedSize: totalSize,
|
||||
})
|
||||
} else {
|
||||
// Remote snapshot not in local DB - fetch manifest to get size
|
||||
timestamp, err := parseSnapshotTimestamp(snapshotID)
|
||||
if err != nil {
|
||||
log.Warn("Failed to parse snapshot timestamp", "id", snapshotID, "error", err)
|
||||
continue
|
||||
}
|
||||
|
||||
// Try to download manifest to get size
|
||||
totalSize, err := v.getManifestSize(snapshotID)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to get manifest size for %s: %w", snapshotID, err)
|
||||
return nil, fmt.Errorf("failed to get manifest size for %s: %w", snapshotID, err)
|
||||
}
|
||||
|
||||
snapshots = append(snapshots, SnapshotInfo{
|
||||
@@ -429,22 +484,13 @@ func (v *Vaultik) ListSnapshots(jsonOutput bool) error {
|
||||
}
|
||||
}
|
||||
|
||||
// Sort by timestamp (newest first)
|
||||
sort.Slice(snapshots, func(i, j int) bool {
|
||||
return snapshots[i].Timestamp.After(snapshots[j].Timestamp)
|
||||
})
|
||||
return snapshots, nil
|
||||
}
|
||||
|
||||
if jsonOutput {
|
||||
// JSON output
|
||||
encoder := json.NewEncoder(v.Stdout)
|
||||
encoder.SetIndent("", " ")
|
||||
return encoder.Encode(snapshots)
|
||||
}
|
||||
|
||||
// Table output
|
||||
// printSnapshotTable renders the snapshot list as a formatted table
|
||||
func (v *Vaultik) printSnapshotTable(snapshots []SnapshotInfo) error {
|
||||
w := tabwriter.NewWriter(v.Stdout, 0, 0, 3, ' ', 0)
|
||||
|
||||
// Show configured snapshots from config file
|
||||
if _, err := fmt.Fprintln(w, "CONFIGURED SNAPSHOTS:"); err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -465,7 +511,6 @@ func (v *Vaultik) ListSnapshots(jsonOutput bool) error {
|
||||
return err
|
||||
}
|
||||
|
||||
// Show remote snapshots
|
||||
if _, err := fmt.Fprintln(w, "REMOTE SNAPSHOTS:"); err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -518,26 +563,9 @@ func (v *Vaultik) PurgeSnapshots(keepLatest bool, olderThan string, force bool)
|
||||
return snapshots[i].Timestamp.After(snapshots[j].Timestamp)
|
||||
})
|
||||
|
||||
var toDelete []SnapshotInfo
|
||||
|
||||
if keepLatest {
|
||||
// Keep only the most recent snapshot
|
||||
if len(snapshots) > 1 {
|
||||
toDelete = snapshots[1:]
|
||||
}
|
||||
} else if olderThan != "" {
|
||||
// Parse duration
|
||||
duration, err := parseDuration(olderThan)
|
||||
if err != nil {
|
||||
return fmt.Errorf("invalid duration: %w", err)
|
||||
}
|
||||
|
||||
cutoff := time.Now().UTC().Add(-duration)
|
||||
for _, snap := range snapshots {
|
||||
if snap.Timestamp.Before(cutoff) {
|
||||
toDelete = append(toDelete, snap)
|
||||
}
|
||||
}
|
||||
toDelete, err := v.collectSnapshotsToPurge(snapshots, keepLatest, olderThan)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if len(toDelete) == 0 {
|
||||
@@ -545,6 +573,41 @@ func (v *Vaultik) PurgeSnapshots(keepLatest bool, olderThan string, force bool)
|
||||
return nil
|
||||
}
|
||||
|
||||
return v.confirmAndExecutePurge(toDelete, force)
|
||||
}
|
||||
|
||||
// collectSnapshotsToPurge determines which snapshots to delete based on retention criteria
|
||||
func (v *Vaultik) collectSnapshotsToPurge(snapshots []SnapshotInfo, keepLatest bool, olderThan string) ([]SnapshotInfo, error) {
|
||||
if keepLatest {
|
||||
// Keep only the most recent snapshot
|
||||
if len(snapshots) > 1 {
|
||||
return snapshots[1:], nil
|
||||
}
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
if olderThan != "" {
|
||||
// Parse duration
|
||||
duration, err := parseDuration(olderThan)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("invalid duration: %w", err)
|
||||
}
|
||||
|
||||
cutoff := time.Now().UTC().Add(-duration)
|
||||
var toDelete []SnapshotInfo
|
||||
for _, snap := range snapshots {
|
||||
if snap.Timestamp.Before(cutoff) {
|
||||
toDelete = append(toDelete, snap)
|
||||
}
|
||||
}
|
||||
return toDelete, nil
|
||||
}
|
||||
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
// confirmAndExecutePurge shows deletion candidates, confirms with user, and deletes snapshots
|
||||
func (v *Vaultik) confirmAndExecutePurge(toDelete []SnapshotInfo, force bool) error {
|
||||
// Show what will be deleted
|
||||
v.printfStdout("The following snapshots will be deleted:\n\n")
|
||||
for _, snap := range toDelete {
|
||||
@@ -610,29 +673,7 @@ func (v *Vaultik) VerifySnapshotWithOptions(snapshotID string, opts *VerifyOptio
|
||||
result.Mode = "deep"
|
||||
}
|
||||
|
||||
// Parse snapshot ID to extract timestamp
|
||||
parts := strings.Split(snapshotID, "-")
|
||||
var snapshotTime time.Time
|
||||
if len(parts) >= 3 {
|
||||
// Format: hostname-YYYYMMDD-HHMMSSZ
|
||||
dateStr := parts[len(parts)-2]
|
||||
timeStr := parts[len(parts)-1]
|
||||
if len(dateStr) == 8 && len(timeStr) == 7 && strings.HasSuffix(timeStr, "Z") {
|
||||
timeStr = timeStr[:6] // Remove Z
|
||||
timestamp, err := time.Parse("20060102150405", dateStr+timeStr)
|
||||
if err == nil {
|
||||
snapshotTime = timestamp
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if !opts.JSON {
|
||||
v.printfStdout("Verifying snapshot %s\n", snapshotID)
|
||||
if !snapshotTime.IsZero() {
|
||||
v.printfStdout("Snapshot time: %s\n", snapshotTime.Format("2006-01-02 15:04:05 MST"))
|
||||
}
|
||||
v.printlnStdout()
|
||||
}
|
||||
v.printVerifyHeader(snapshotID, opts)
|
||||
|
||||
// Download and parse manifest
|
||||
manifest, err := v.downloadManifest(snapshotID)
|
||||
@@ -663,10 +704,40 @@ func (v *Vaultik) VerifySnapshotWithOptions(snapshotID string, opts *VerifyOptio
|
||||
v.printfStdout("Checking blob existence...\n")
|
||||
}
|
||||
|
||||
missing := 0
|
||||
verified := 0
|
||||
missingSize := int64(0)
|
||||
result.Verified, result.Missing, result.MissingSize = v.verifyManifestBlobsExist(manifest, opts)
|
||||
|
||||
return v.formatVerifyResult(result, manifest, opts)
|
||||
}
|
||||
|
||||
// printVerifyHeader prints the snapshot ID and parsed timestamp for verification output
|
||||
func (v *Vaultik) printVerifyHeader(snapshotID string, opts *VerifyOptions) {
|
||||
// Parse snapshot ID to extract timestamp
|
||||
parts := strings.Split(snapshotID, "-")
|
||||
var snapshotTime time.Time
|
||||
if len(parts) >= 3 {
|
||||
// Format: hostname-YYYYMMDD-HHMMSSZ
|
||||
dateStr := parts[len(parts)-2]
|
||||
timeStr := parts[len(parts)-1]
|
||||
if len(dateStr) == 8 && len(timeStr) == 7 && strings.HasSuffix(timeStr, "Z") {
|
||||
timeStr = timeStr[:6] // Remove Z
|
||||
timestamp, err := time.Parse("20060102150405", dateStr+timeStr)
|
||||
if err == nil {
|
||||
snapshotTime = timestamp
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if !opts.JSON {
|
||||
v.printfStdout("Verifying snapshot %s\n", snapshotID)
|
||||
if !snapshotTime.IsZero() {
|
||||
v.printfStdout("Snapshot time: %s\n", snapshotTime.Format("2006-01-02 15:04:05 MST"))
|
||||
}
|
||||
v.printlnStdout()
|
||||
}
|
||||
}
|
||||
|
||||
// verifyManifestBlobsExist checks that each blob in the manifest exists in storage
|
||||
func (v *Vaultik) verifyManifestBlobsExist(manifest *snapshot.Manifest, opts *VerifyOptions) (verified, missing int, missingSize int64) {
|
||||
for _, blob := range manifest.Blobs {
|
||||
blobPath := fmt.Sprintf("blobs/%s/%s/%s", blob.Hash[:2], blob.Hash[2:4], blob.Hash)
|
||||
|
||||
@@ -682,15 +753,15 @@ func (v *Vaultik) VerifySnapshotWithOptions(snapshotID string, opts *VerifyOptio
|
||||
verified++
|
||||
}
|
||||
}
|
||||
return verified, missing, missingSize
|
||||
}
|
||||
|
||||
result.Verified = verified
|
||||
result.Missing = missing
|
||||
result.MissingSize = missingSize
|
||||
|
||||
// formatVerifyResult outputs the final verification results as JSON or human-readable text
|
||||
func (v *Vaultik) formatVerifyResult(result *VerifyResult, manifest *snapshot.Manifest, opts *VerifyOptions) error {
|
||||
if opts.JSON {
|
||||
if missing > 0 {
|
||||
if result.Missing > 0 {
|
||||
result.Status = "failed"
|
||||
result.ErrorMessage = fmt.Sprintf("%d blobs are missing", missing)
|
||||
result.ErrorMessage = fmt.Sprintf("%d blobs are missing", result.Missing)
|
||||
} else {
|
||||
result.Status = "ok"
|
||||
}
|
||||
@@ -698,20 +769,19 @@ func (v *Vaultik) VerifySnapshotWithOptions(snapshotID string, opts *VerifyOptio
|
||||
}
|
||||
|
||||
v.printfStdout("\nVerification complete:\n")
|
||||
v.printfStdout(" Verified: %d blobs (%s)\n", verified,
|
||||
humanize.Bytes(uint64(manifest.TotalCompressedSize-missingSize)))
|
||||
if missing > 0 {
|
||||
v.printfStdout(" Missing: %d blobs (%s)\n", missing, humanize.Bytes(uint64(missingSize)))
|
||||
v.printfStdout(" Verified: %d blobs (%s)\n", result.Verified,
|
||||
humanize.Bytes(uint64(manifest.TotalCompressedSize-result.MissingSize)))
|
||||
if result.Missing > 0 {
|
||||
v.printfStdout(" Missing: %d blobs (%s)\n", result.Missing, humanize.Bytes(uint64(result.MissingSize)))
|
||||
} else {
|
||||
v.printfStdout(" Missing: 0 blobs\n")
|
||||
}
|
||||
v.printfStdout(" Status: ")
|
||||
if missing > 0 {
|
||||
v.printfStdout("FAILED - %d blobs are missing\n", missing)
|
||||
return fmt.Errorf("%d blobs are missing", missing)
|
||||
} else {
|
||||
v.printfStdout("OK - All blobs verified\n")
|
||||
if result.Missing > 0 {
|
||||
v.printfStdout("FAILED - %d blobs are missing\n", result.Missing)
|
||||
return fmt.Errorf("%d blobs are missing", result.Missing)
|
||||
}
|
||||
v.printfStdout("OK - All blobs verified\n")
|
||||
|
||||
return nil
|
||||
}
|
||||
@@ -907,9 +977,27 @@ func (v *Vaultik) RemoveSnapshot(snapshotID string, opts *RemoveOptions) (*Remov
|
||||
|
||||
// RemoveAllSnapshots removes all snapshots from local database and optionally from remote
|
||||
func (v *Vaultik) RemoveAllSnapshots(opts *RemoveOptions) (*RemoveResult, error) {
|
||||
result := &RemoveResult{}
|
||||
snapshotIDs, err := v.listAllRemoteSnapshotIDs()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// List all snapshots
|
||||
if len(snapshotIDs) == 0 {
|
||||
if !opts.JSON {
|
||||
v.printlnStdout("No snapshots found")
|
||||
}
|
||||
return &RemoveResult{}, nil
|
||||
}
|
||||
|
||||
if opts.DryRun {
|
||||
return v.handleRemoveAllDryRun(snapshotIDs, opts)
|
||||
}
|
||||
|
||||
return v.executeRemoveAll(snapshotIDs, opts)
|
||||
}
|
||||
|
||||
// listAllRemoteSnapshotIDs collects all unique snapshot IDs from remote storage
|
||||
func (v *Vaultik) listAllRemoteSnapshotIDs() ([]string, error) {
|
||||
log.Info("Listing all snapshots")
|
||||
objectCh := v.Storage.ListStream(v.ctx, "metadata/")
|
||||
|
||||
@@ -941,32 +1029,33 @@ func (v *Vaultik) RemoveAllSnapshots(opts *RemoveOptions) (*RemoveResult, error)
|
||||
}
|
||||
}
|
||||
|
||||
if len(snapshotIDs) == 0 {
|
||||
if !opts.JSON {
|
||||
v.printlnStdout("No snapshots found")
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
return snapshotIDs, nil
|
||||
}
|
||||
|
||||
if opts.DryRun {
|
||||
result.DryRun = true
|
||||
result.SnapshotsRemoved = snapshotIDs
|
||||
if !opts.JSON {
|
||||
v.printfStdout("Would remove %d snapshot(s):\n", len(snapshotIDs))
|
||||
for _, id := range snapshotIDs {
|
||||
v.printfStdout(" %s\n", id)
|
||||
}
|
||||
if opts.Remote {
|
||||
v.printlnStdout("Would also remove from remote storage")
|
||||
}
|
||||
v.printlnStdout("[Dry run - no changes made]")
|
||||
}
|
||||
if opts.JSON {
|
||||
return result, v.outputRemoveJSON(result)
|
||||
}
|
||||
return result, nil
|
||||
// handleRemoveAllDryRun handles the dry-run mode for removing all snapshots
|
||||
func (v *Vaultik) handleRemoveAllDryRun(snapshotIDs []string, opts *RemoveOptions) (*RemoveResult, error) {
|
||||
result := &RemoveResult{
|
||||
DryRun: true,
|
||||
SnapshotsRemoved: snapshotIDs,
|
||||
}
|
||||
if !opts.JSON {
|
||||
v.printfStdout("Would remove %d snapshot(s):\n", len(snapshotIDs))
|
||||
for _, id := range snapshotIDs {
|
||||
v.printfStdout(" %s\n", id)
|
||||
}
|
||||
if opts.Remote {
|
||||
v.printlnStdout("Would also remove from remote storage")
|
||||
}
|
||||
v.printlnStdout("[Dry run - no changes made]")
|
||||
}
|
||||
if opts.JSON {
|
||||
return result, v.outputRemoveJSON(result)
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
// executeRemoveAll removes all snapshots from local database and optionally from remote storage
|
||||
func (v *Vaultik) executeRemoveAll(snapshotIDs []string, opts *RemoveOptions) (*RemoveResult, error) {
|
||||
// --all requires --force
|
||||
if !opts.Force {
|
||||
return nil, fmt.Errorf("--all requires --force")
|
||||
@@ -974,6 +1063,7 @@ func (v *Vaultik) RemoveAllSnapshots(opts *RemoveOptions) (*RemoveResult, error)
|
||||
|
||||
log.Info("Removing all snapshots", "count", len(snapshotIDs))
|
||||
|
||||
result := &RemoveResult{}
|
||||
for _, snapshotID := range snapshotIDs {
|
||||
log.Info("Removing snapshot", "snapshot_id", snapshotID)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user