From a544fa80f2222462a170fe7adc258bea4384f51e Mon Sep 17 00:00:00 2001 From: sneak Date: Sat, 26 Jul 2025 03:27:47 +0200 Subject: [PATCH] Major refactoring: Updated manifest format and renamed backup to snapshot - Created manifest.go with proper Manifest structure including blob sizes - Updated manifest generation to include compressed size for each blob - Added TotalCompressedSize field to manifest for quick access - Renamed backup package to snapshot for clarity - Updated snapshot list to show all remote snapshots - Remote snapshots not in local DB fetch manifest to get size - Local snapshots not in remote are automatically deleted - Removed backwards compatibility code (pre-1.0, no users) - Fixed prune command to use new manifest format - Updated all imports and references from backup to snapshot --- internal/cli/prune.go | 33 +-- internal/cli/snapshot.go | 203 +++++++++++------- internal/{backup => snapshot}/backup_test.go | 2 +- .../{backup => snapshot}/file_change_test.go | 8 +- internal/snapshot/manifest.go | 70 ++++++ internal/{backup => snapshot}/module.go | 2 +- internal/{backup => snapshot}/progress.go | 2 +- internal/{backup => snapshot}/scanner.go | 2 +- internal/{backup => snapshot}/scanner_test.go | 16 +- internal/{backup => snapshot}/snapshot.go | 82 ++++--- .../{backup => snapshot}/snapshot_test.go | 2 +- 11 files changed, 254 insertions(+), 168 deletions(-) rename internal/{backup => snapshot}/backup_test.go (99%) rename internal/{backup => snapshot}/file_change_test.go (97%) create mode 100644 internal/snapshot/manifest.go rename internal/{backup => snapshot}/module.go (98%) rename internal/{backup => snapshot}/progress.go (99%) rename internal/{backup => snapshot}/scanner.go (99%) rename internal/{backup => snapshot}/scanner_test.go (96%) rename internal/{backup => snapshot}/snapshot.go (95%) rename internal/{backup => snapshot}/snapshot_test.go (99%) diff --git a/internal/cli/prune.go b/internal/cli/prune.go index ef82069..a68d763 100644 --- a/internal/cli/prune.go +++ b/internal/cli/prune.go @@ -2,18 +2,16 @@ package cli import ( "context" - "encoding/json" "fmt" "strings" - "git.eeqj.de/sneak/vaultik/internal/backup" "git.eeqj.de/sneak/vaultik/internal/config" "git.eeqj.de/sneak/vaultik/internal/database" "git.eeqj.de/sneak/vaultik/internal/globals" "git.eeqj.de/sneak/vaultik/internal/log" "git.eeqj.de/sneak/vaultik/internal/s3" + "git.eeqj.de/sneak/vaultik/internal/snapshot" "github.com/dustin/go-humanize" - "github.com/klauspost/compress/zstd" "github.com/spf13/cobra" "go.uber.org/fx" ) @@ -66,7 +64,7 @@ specifying a path using --config or by setting VAULTIK_CONFIG to a path.`, Debug: rootFlags.Debug, }, Modules: []fx.Option{ - backup.Module, + snapshot.Module, s3.Module, fx.Provide(fx.Annotate( func(g *globals.Globals, cfg *config.Config, repos *database.Repositories, @@ -195,8 +193,8 @@ func (app *PruneApp) runPrune(ctx context.Context, opts *PruneOptions) error { // Step 5: Build set of referenced blobs referencedBlobs := make(map[string]bool) - for _, blobHash := range manifest.Blobs { - referencedBlobs[blobHash] = true + for _, blob := range manifest.Blobs { + referencedBlobs[blob.Hash] = true } // Step 6: List all blobs in S3 @@ -277,16 +275,8 @@ func (app *PruneApp) runPrune(ctx context.Context, opts *PruneOptions) error { return nil } -// BlobManifest represents the structure of a snapshot's blob manifest -type BlobManifest struct { - SnapshotID string `json:"snapshot_id"` - Timestamp string `json:"timestamp"` - BlobCount int `json:"blob_count"` - Blobs []string `json:"blobs"` -} - // downloadManifest downloads and decompresses a snapshot manifest -func (app *PruneApp) downloadManifest(ctx context.Context, snapshotID string) (*BlobManifest, error) { +func (app *PruneApp) downloadManifest(ctx context.Context, snapshotID string) (*snapshot.Manifest, error) { manifestPath := fmt.Sprintf("metadata/%s/manifest.json.zst", snapshotID) // Download the compressed manifest @@ -296,18 +286,11 @@ func (app *PruneApp) downloadManifest(ctx context.Context, snapshotID string) (* } defer func() { _ = reader.Close() }() - // Decompress using zstd - zr, err := zstd.NewReader(reader) + // Decode manifest + manifest, err := snapshot.DecodeManifest(reader) if err != nil { - return nil, fmt.Errorf("creating zstd reader: %w", err) - } - defer zr.Close() - - // Decode JSON manifest - var manifest BlobManifest - if err := json.NewDecoder(zr).Decode(&manifest); err != nil { return nil, fmt.Errorf("decoding manifest: %w", err) } - return &manifest, nil + return manifest, nil } diff --git a/internal/cli/snapshot.go b/internal/cli/snapshot.go index 890a3bc..78f0a00 100644 --- a/internal/cli/snapshot.go +++ b/internal/cli/snapshot.go @@ -12,14 +12,13 @@ import ( "text/tabwriter" "time" - "git.eeqj.de/sneak/vaultik/internal/backup" "git.eeqj.de/sneak/vaultik/internal/config" "git.eeqj.de/sneak/vaultik/internal/database" "git.eeqj.de/sneak/vaultik/internal/globals" "git.eeqj.de/sneak/vaultik/internal/log" "git.eeqj.de/sneak/vaultik/internal/s3" + "git.eeqj.de/sneak/vaultik/internal/snapshot" "github.com/dustin/go-humanize" - "github.com/klauspost/compress/zstd" "github.com/spf13/cobra" "go.uber.org/fx" ) @@ -36,8 +35,8 @@ type SnapshotCreateApp struct { Globals *globals.Globals Config *config.Config Repositories *database.Repositories - ScannerFactory backup.ScannerFactory - SnapshotManager *backup.SnapshotManager + ScannerFactory snapshot.ScannerFactory + SnapshotManager *snapshot.SnapshotManager S3Client *s3.Client DB *database.DB Lifecycle fx.Lifecycle @@ -106,11 +105,11 @@ specifying a path using --config or by setting VAULTIK_CONFIG to a path.`, Cron: opts.Cron, }, Modules: []fx.Option{ - backup.Module, + snapshot.Module, s3.Module, fx.Provide(fx.Annotate( func(g *globals.Globals, cfg *config.Config, repos *database.Repositories, - scannerFactory backup.ScannerFactory, snapshotManager *backup.SnapshotManager, + scannerFactory snapshot.ScannerFactory, snapshotManager *snapshot.SnapshotManager, s3Client *s3.Client, db *database.DB, lc fx.Lifecycle, shutdowner fx.Shutdowner) *SnapshotCreateApp { return &SnapshotCreateApp{ @@ -226,7 +225,7 @@ func (app *SnapshotCreateApp) runSnapshot(ctx context.Context, opts *SnapshotCre } // Create scanner with progress enabled (unless in cron mode) - scanner := app.ScannerFactory(backup.ScannerParams{ + scanner := app.ScannerFactory(snapshot.ScannerParams{ EnableProgress: !opts.Cron, }) @@ -306,8 +305,8 @@ func (app *SnapshotCreateApp) runSnapshot(ctx context.Context, opts *SnapshotCre } // Update snapshot statistics with extended fields - extStats := backup.ExtendedBackupStats{ - BackupStats: backup.BackupStats{ + extStats := snapshot.ExtendedBackupStats{ + BackupStats: snapshot.BackupStats{ FilesScanned: totalFiles, BytesScanned: totalBytes, ChunksCreated: totalChunks, @@ -487,15 +486,78 @@ func newSnapshotVerifyCommand() *cobra.Command { // List lists all snapshots func (app *SnapshotApp) List(ctx context.Context, jsonOutput bool) error { - // First, sync with remote snapshots - if err := app.syncWithRemote(ctx); err != nil { - return fmt.Errorf("syncing with remote: %w", err) + // Get all remote snapshots + remoteSnapshots := make(map[string]bool) + objectCh := app.S3Client.ListObjectsStream(ctx, "metadata/", false) + + for object := range objectCh { + if object.Err != nil { + return fmt.Errorf("listing remote snapshots: %w", object.Err) + } + + // Extract snapshot ID from paths like metadata/hostname-20240115-143052Z/ + parts := strings.Split(object.Key, "/") + if len(parts) >= 2 && parts[0] == "metadata" && parts[1] != "" { + remoteSnapshots[parts[1]] = true + } } - // Now get snapshots from S3 - snapshots, err := app.getSnapshots(ctx) + // Get all local snapshots + localSnapshots, err := app.Repositories.Snapshots.ListRecent(ctx, 10000) if err != nil { - return err + return fmt.Errorf("listing local snapshots: %w", err) + } + + // Build a map of local snapshots for quick lookup + localSnapshotMap := make(map[string]*database.Snapshot) + for _, s := range localSnapshots { + localSnapshotMap[s.ID] = s + } + + // Remove local snapshots that don't exist remotely + for _, snapshot := range localSnapshots { + if !remoteSnapshots[snapshot.ID] { + log.Info("Removing local snapshot not found in remote", "snapshot_id", snapshot.ID) + if err := app.Repositories.Snapshots.Delete(ctx, snapshot.ID); err != nil { + log.Error("Failed to delete local snapshot", "snapshot_id", snapshot.ID, "error", err) + } + delete(localSnapshotMap, snapshot.ID) + } + } + + // Build final snapshot list + snapshots := make([]SnapshotInfo, 0, len(remoteSnapshots)) + + for snapshotID := range remoteSnapshots { + // Check if we have this snapshot locally + if localSnap, exists := localSnapshotMap[snapshotID]; exists && localSnap.CompletedAt != nil { + // Use local data + snapshots = append(snapshots, SnapshotInfo{ + ID: localSnap.ID, + Timestamp: localSnap.StartedAt, + CompressedSize: localSnap.BlobSize, + }) + } else { + // Remote snapshot not in local DB - fetch manifest to get size + timestamp, err := parseSnapshotTimestamp(snapshotID) + if err != nil { + log.Warn("Failed to parse snapshot timestamp", "id", snapshotID, "error", err) + continue + } + + // Try to download manifest to get size + totalSize, err := app.getManifestSize(ctx, snapshotID) + if err != nil { + log.Warn("Failed to get manifest size", "id", snapshotID, "error", err) + totalSize = 0 + } + + snapshots = append(snapshots, SnapshotInfo{ + ID: snapshotID, + Timestamp: timestamp, + CompressedSize: totalSize, + }) + } } // Sort by timestamp (newest first) @@ -512,17 +574,18 @@ func (app *SnapshotApp) List(ctx context.Context, jsonOutput bool) error { // Table output w := tabwriter.NewWriter(os.Stdout, 0, 0, 3, ' ', 0) - if _, err := fmt.Fprintln(w, "SNAPSHOT ID\tTIMESTAMP"); err != nil { + if _, err := fmt.Fprintln(w, "SNAPSHOT ID\tTIMESTAMP\tCOMPRESSED SIZE"); err != nil { return err } - if _, err := fmt.Fprintln(w, "───────────\t─────────"); err != nil { + if _, err := fmt.Fprintln(w, "───────────\t─────────\t───────────────"); err != nil { return err } for _, snap := range snapshots { - if _, err := fmt.Fprintf(w, "%s\t%s\n", + if _, err := fmt.Fprintf(w, "%s\t%s\t%s\n", snap.ID, - snap.Timestamp.Format("2006-01-02 15:04:05")); err != nil { + snap.Timestamp.Format("2006-01-02 15:04:05"), + formatBytes(snap.CompressedSize)); err != nil { return err } } @@ -532,9 +595,27 @@ func (app *SnapshotApp) List(ctx context.Context, jsonOutput bool) error { // Purge removes old snapshots based on criteria func (app *SnapshotApp) Purge(ctx context.Context, keepLatest bool, olderThan string, force bool) error { - snapshots, err := app.getSnapshots(ctx) + // Sync with remote first + if err := app.syncWithRemote(ctx); err != nil { + return fmt.Errorf("syncing with remote: %w", err) + } + + // Get snapshots from local database + dbSnapshots, err := app.Repositories.Snapshots.ListRecent(ctx, 10000) if err != nil { - return err + return fmt.Errorf("listing snapshots: %w", err) + } + + // Convert to SnapshotInfo format, only including completed snapshots + snapshots := make([]SnapshotInfo, 0, len(dbSnapshots)) + for _, s := range dbSnapshots { + if s.CompletedAt != nil { + snapshots = append(snapshots, SnapshotInfo{ + ID: s.ID, + Timestamp: s.StartedAt, + CompressedSize: s.BlobSize, + }) + } } // Sort by timestamp (newest first) @@ -657,57 +738,25 @@ func (app *SnapshotApp) Verify(ctx context.Context, snapshotID string, deep bool return nil } -// getSnapshots retrieves all snapshots from S3 -func (app *SnapshotApp) getSnapshots(ctx context.Context) ([]SnapshotInfo, error) { - var snapshots []SnapshotInfo +// getManifestSize downloads a manifest and returns the total compressed size +func (app *SnapshotApp) getManifestSize(ctx context.Context, snapshotID string) (int64, error) { + manifestPath := fmt.Sprintf("metadata/%s/manifest.json.zst", snapshotID) - // List all objects under metadata/ - objectCh := app.S3Client.ListObjectsStream(ctx, "metadata/", true) + reader, err := app.S3Client.GetObject(ctx, manifestPath) + if err != nil { + return 0, fmt.Errorf("downloading manifest: %w", err) + } + defer func() { _ = reader.Close() }() - // Track unique snapshots - snapshotMap := make(map[string]*SnapshotInfo) - - for object := range objectCh { - if object.Err != nil { - return nil, fmt.Errorf("listing objects: %w", object.Err) - } - - // Extract snapshot ID from paths like metadata/2024-01-15-143052-hostname/manifest.json.zst - parts := strings.Split(object.Key, "/") - if len(parts) < 3 || parts[0] != "metadata" { - continue - } - - snapshotID := parts[1] - if snapshotID == "" { - continue - } - - // Initialize snapshot info if not seen - if _, exists := snapshotMap[snapshotID]; !exists { - timestamp, err := parseSnapshotTimestamp(snapshotID) - if err != nil { - log.Warn("Failed to parse snapshot timestamp", "id", snapshotID, "error", err) - continue - } - - snapshotMap[snapshotID] = &SnapshotInfo{ - ID: snapshotID, - Timestamp: timestamp, - CompressedSize: 0, - } - } + manifest, err := snapshot.DecodeManifest(reader) + if err != nil { + return 0, fmt.Errorf("decoding manifest: %w", err) } - // Convert map to slice without downloading manifests - for _, snap := range snapshotMap { - snapshots = append(snapshots, *snap) - } - - return snapshots, nil + return manifest.TotalCompressedSize, nil } -// downloadManifest downloads and parses a snapshot manifest +// downloadManifest downloads and parses a snapshot manifest (for verify command) func (app *SnapshotApp) downloadManifest(ctx context.Context, snapshotID string) ([]string, error) { manifestPath := fmt.Sprintf("metadata/%s/manifest.json.zst", snapshotID) @@ -717,25 +766,17 @@ func (app *SnapshotApp) downloadManifest(ctx context.Context, snapshotID string) } defer func() { _ = reader.Close() }() - // Decompress - zr, err := zstd.NewReader(reader) + manifest, err := snapshot.DecodeManifest(reader) if err != nil { - return nil, fmt.Errorf("creating zstd reader: %w", err) - } - defer zr.Close() - - // Decode JSON - manifest is an object with a "blobs" field - var manifest struct { - SnapshotID string `json:"snapshot_id"` - Timestamp string `json:"timestamp"` - BlobCount int `json:"blob_count"` - Blobs []string `json:"blobs"` - } - if err := json.NewDecoder(zr).Decode(&manifest); err != nil { return nil, fmt.Errorf("decoding manifest: %w", err) } - return manifest.Blobs, nil + // Extract blob hashes + hashes := make([]string, len(manifest.Blobs)) + for i, blob := range manifest.Blobs { + hashes[i] = blob.Hash + } + return hashes, nil } // deleteSnapshot removes a snapshot and its metadata diff --git a/internal/backup/backup_test.go b/internal/snapshot/backup_test.go similarity index 99% rename from internal/backup/backup_test.go rename to internal/snapshot/backup_test.go index 77c46ed..f3256be 100644 --- a/internal/backup/backup_test.go +++ b/internal/snapshot/backup_test.go @@ -1,4 +1,4 @@ -package backup +package snapshot import ( "context" diff --git a/internal/backup/file_change_test.go b/internal/snapshot/file_change_test.go similarity index 97% rename from internal/backup/file_change_test.go rename to internal/snapshot/file_change_test.go index 2c5c0fd..b422df3 100644 --- a/internal/backup/file_change_test.go +++ b/internal/snapshot/file_change_test.go @@ -1,4 +1,4 @@ -package backup_test +package snapshot_test import ( "context" @@ -6,9 +6,9 @@ import ( "testing" "time" - "git.eeqj.de/sneak/vaultik/internal/backup" "git.eeqj.de/sneak/vaultik/internal/database" "git.eeqj.de/sneak/vaultik/internal/log" + "git.eeqj.de/sneak/vaultik/internal/snapshot" "github.com/spf13/afero" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -39,7 +39,7 @@ func TestFileContentChange(t *testing.T) { repos := database.NewRepositories(db) // Create scanner - scanner := backup.NewScanner(backup.ScannerConfig{ + scanner := snapshot.NewScanner(snapshot.ScannerConfig{ FS: fs, ChunkSize: int64(1024 * 16), // 16KB chunks for testing Repositories: repos, @@ -168,7 +168,7 @@ func TestMultipleFileChanges(t *testing.T) { repos := database.NewRepositories(db) // Create scanner - scanner := backup.NewScanner(backup.ScannerConfig{ + scanner := snapshot.NewScanner(snapshot.ScannerConfig{ FS: fs, ChunkSize: int64(1024 * 16), // 16KB chunks for testing Repositories: repos, diff --git a/internal/snapshot/manifest.go b/internal/snapshot/manifest.go new file mode 100644 index 0000000..3b152cc --- /dev/null +++ b/internal/snapshot/manifest.go @@ -0,0 +1,70 @@ +package snapshot + +import ( + "bytes" + "encoding/json" + "fmt" + "io" + + "github.com/klauspost/compress/zstd" +) + +// Manifest represents the structure of a snapshot's blob manifest +type Manifest struct { + SnapshotID string `json:"snapshot_id"` + Timestamp string `json:"timestamp"` + BlobCount int `json:"blob_count"` + TotalCompressedSize int64 `json:"total_compressed_size"` + Blobs []BlobInfo `json:"blobs"` +} + +// BlobInfo represents information about a single blob in the manifest +type BlobInfo struct { + Hash string `json:"hash"` + CompressedSize int64 `json:"compressed_size"` +} + +// DecodeManifest decodes a manifest from a reader containing compressed JSON +func DecodeManifest(r io.Reader) (*Manifest, error) { + // Decompress using zstd + zr, err := zstd.NewReader(r) + if err != nil { + return nil, fmt.Errorf("creating zstd reader: %w", err) + } + defer zr.Close() + + // Decode JSON manifest + var manifest Manifest + if err := json.NewDecoder(zr).Decode(&manifest); err != nil { + return nil, fmt.Errorf("decoding manifest: %w", err) + } + + return &manifest, nil +} + +// EncodeManifest encodes a manifest to compressed JSON +func EncodeManifest(manifest *Manifest, compressionLevel int) ([]byte, error) { + // Marshal to JSON + jsonData, err := json.MarshalIndent(manifest, "", " ") + if err != nil { + return nil, fmt.Errorf("marshaling manifest: %w", err) + } + + // Compress using zstd + var compressedBuf bytes.Buffer + writer, err := zstd.NewWriter(&compressedBuf, zstd.WithEncoderLevel(zstd.EncoderLevelFromZstd(compressionLevel))) + if err != nil { + return nil, fmt.Errorf("creating zstd writer: %w", err) + } + + if _, err := writer.Write(jsonData); err != nil { + _ = writer.Close() + return nil, fmt.Errorf("writing compressed data: %w", err) + } + + if err := writer.Close(); err != nil { + return nil, fmt.Errorf("closing zstd writer: %w", err) + } + + return compressedBuf.Bytes(), nil +} diff --git a/internal/backup/module.go b/internal/snapshot/module.go similarity index 98% rename from internal/backup/module.go rename to internal/snapshot/module.go index 6e6161a..e785c96 100644 --- a/internal/backup/module.go +++ b/internal/snapshot/module.go @@ -1,4 +1,4 @@ -package backup +package snapshot import ( "git.eeqj.de/sneak/vaultik/internal/config" diff --git a/internal/backup/progress.go b/internal/snapshot/progress.go similarity index 99% rename from internal/backup/progress.go rename to internal/snapshot/progress.go index 16add71..039632f 100644 --- a/internal/backup/progress.go +++ b/internal/snapshot/progress.go @@ -1,4 +1,4 @@ -package backup +package snapshot import ( "context" diff --git a/internal/backup/scanner.go b/internal/snapshot/scanner.go similarity index 99% rename from internal/backup/scanner.go rename to internal/snapshot/scanner.go index e4b5cd3..2a59bc5 100644 --- a/internal/backup/scanner.go +++ b/internal/snapshot/scanner.go @@ -1,4 +1,4 @@ -package backup +package snapshot import ( "context" diff --git a/internal/backup/scanner_test.go b/internal/snapshot/scanner_test.go similarity index 96% rename from internal/backup/scanner_test.go rename to internal/snapshot/scanner_test.go index 908c673..fd86edc 100644 --- a/internal/backup/scanner_test.go +++ b/internal/snapshot/scanner_test.go @@ -1,4 +1,4 @@ -package backup_test +package snapshot_test import ( "context" @@ -7,9 +7,9 @@ import ( "testing" "time" - "git.eeqj.de/sneak/vaultik/internal/backup" "git.eeqj.de/sneak/vaultik/internal/database" "git.eeqj.de/sneak/vaultik/internal/log" + "git.eeqj.de/sneak/vaultik/internal/snapshot" "github.com/spf13/afero" ) @@ -60,7 +60,7 @@ func TestScannerSimpleDirectory(t *testing.T) { repos := database.NewRepositories(db) // Create scanner - scanner := backup.NewScanner(backup.ScannerConfig{ + scanner := snapshot.NewScanner(snapshot.ScannerConfig{ FS: fs, ChunkSize: int64(1024 * 16), // 16KB chunks for testing Repositories: repos, @@ -93,7 +93,7 @@ func TestScannerSimpleDirectory(t *testing.T) { } // Scan the directory - var result *backup.ScanResult + var result *snapshot.ScanResult result, err = scanner.Scan(ctx, "/source", snapshotID) if err != nil { t.Fatalf("scan failed: %v", err) @@ -207,7 +207,7 @@ func TestScannerWithSymlinks(t *testing.T) { repos := database.NewRepositories(db) // Create scanner - scanner := backup.NewScanner(backup.ScannerConfig{ + scanner := snapshot.NewScanner(snapshot.ScannerConfig{ FS: fs, ChunkSize: 1024 * 16, Repositories: repos, @@ -240,7 +240,7 @@ func TestScannerWithSymlinks(t *testing.T) { } // Scan the directory - var result *backup.ScanResult + var result *snapshot.ScanResult result, err = scanner.Scan(ctx, "/source", snapshotID) if err != nil { t.Fatalf("scan failed: %v", err) @@ -308,7 +308,7 @@ func TestScannerLargeFile(t *testing.T) { repos := database.NewRepositories(db) // Create scanner with 64KB average chunk size - scanner := backup.NewScanner(backup.ScannerConfig{ + scanner := snapshot.NewScanner(snapshot.ScannerConfig{ FS: fs, ChunkSize: int64(1024 * 64), // 64KB average chunks Repositories: repos, @@ -341,7 +341,7 @@ func TestScannerLargeFile(t *testing.T) { } // Scan the directory - var result *backup.ScanResult + var result *snapshot.ScanResult result, err = scanner.Scan(ctx, "/source", snapshotID) if err != nil { t.Fatalf("scan failed: %v", err) diff --git a/internal/backup/snapshot.go b/internal/snapshot/snapshot.go similarity index 95% rename from internal/backup/snapshot.go rename to internal/snapshot/snapshot.go index ca892f4..dff801f 100644 --- a/internal/backup/snapshot.go +++ b/internal/snapshot/snapshot.go @@ -1,9 +1,9 @@ -package backup +package snapshot // Snapshot Metadata Export Process // ================================ // -// The snapshot metadata contains all information needed to restore a backup. +// The snapshot metadata contains all information needed to restore a snapshot. // Instead of creating a custom format, we use a trimmed copy of the SQLite // database containing only data relevant to the current snapshot. // @@ -42,7 +42,6 @@ import ( "bytes" "context" "database/sql" - "encoding/json" "fmt" "io" "os" @@ -56,7 +55,6 @@ import ( "git.eeqj.de/sneak/vaultik/internal/log" "git.eeqj.de/sneak/vaultik/internal/s3" "github.com/dustin/go-humanize" - "github.com/klauspost/compress/zstd" "go.uber.org/fx" ) @@ -540,60 +538,54 @@ func (sm *SnapshotManager) generateBlobManifest(ctx context.Context, dbPath stri // Get all blobs for this snapshot log.Debug("Querying blobs for snapshot", "snapshot_id", snapshotID) - blobs, err := repos.Snapshots.GetBlobHashes(ctx, snapshotID) + blobHashes, err := repos.Snapshots.GetBlobHashes(ctx, snapshotID) if err != nil { return nil, fmt.Errorf("getting snapshot blobs: %w", err) } - log.Debug("Found blobs", "count", len(blobs)) + log.Debug("Found blobs", "count", len(blobHashes)) - // Create manifest structure - manifest := struct { - SnapshotID string `json:"snapshot_id"` - Timestamp string `json:"timestamp"` - BlobCount int `json:"blob_count"` - Blobs []string `json:"blobs"` - }{ - SnapshotID: snapshotID, - Timestamp: time.Now().UTC().Format(time.RFC3339), - BlobCount: len(blobs), - Blobs: blobs, + // Get blob details including sizes + blobs := make([]BlobInfo, 0, len(blobHashes)) + totalCompressedSize := int64(0) + + for _, hash := range blobHashes { + blob, err := repos.Blobs.GetByHash(ctx, hash) + if err != nil { + log.Warn("Failed to get blob details", "hash", hash, "error", err) + continue + } + if blob != nil { + blobs = append(blobs, BlobInfo{ + Hash: hash, + CompressedSize: blob.CompressedSize, + }) + totalCompressedSize += blob.CompressedSize + } } - // Marshal to JSON - log.Debug("Marshaling manifest to JSON") - jsonData, err := json.MarshalIndent(manifest, "", " ") + // Create manifest + manifest := &Manifest{ + SnapshotID: snapshotID, + Timestamp: time.Now().UTC().Format(time.RFC3339), + BlobCount: len(blobs), + TotalCompressedSize: totalCompressedSize, + Blobs: blobs, + } + + // Encode manifest + log.Debug("Encoding manifest") + compressedData, err := EncodeManifest(manifest, sm.config.CompressionLevel) if err != nil { - return nil, fmt.Errorf("marshaling manifest: %w", err) + return nil, fmt.Errorf("encoding manifest: %w", err) } - log.Debug("JSON manifest created", "size", len(jsonData)) - - // Compress only (no encryption) - manifests must be readable without private keys for pruning - log.Debug("Compressing manifest") - - var compressedBuf bytes.Buffer - writer, err := zstd.NewWriter(&compressedBuf, zstd.WithEncoderLevel(zstd.EncoderLevelFromZstd(sm.config.CompressionLevel))) - if err != nil { - return nil, fmt.Errorf("creating zstd writer: %w", err) - } - if _, err := writer.Write(jsonData); err != nil { - _ = writer.Close() - return nil, fmt.Errorf("writing compressed data: %w", err) - } - if err := writer.Close(); err != nil { - return nil, fmt.Errorf("closing zstd writer: %w", err) - } - - log.Debug("Manifest compressed", - "original_size", len(jsonData), - "compressed_size", compressedBuf.Len()) log.Info("Generated blob manifest", "snapshot_id", snapshotID, "blob_count", len(blobs), - "json_size", len(jsonData), - "compressed_size", compressedBuf.Len()) + "total_compressed_size", totalCompressedSize, + "manifest_size", len(compressedData)) - return compressedBuf.Bytes(), nil + return compressedData, nil } // compressData compresses data using zstd diff --git a/internal/backup/snapshot_test.go b/internal/snapshot/snapshot_test.go similarity index 99% rename from internal/backup/snapshot_test.go rename to internal/snapshot/snapshot_test.go index 6859946..7883b97 100644 --- a/internal/backup/snapshot_test.go +++ b/internal/snapshot/snapshot_test.go @@ -1,4 +1,4 @@ -package backup +package snapshot import ( "context"