Major refactoring: Updated manifest format and renamed backup to snapshot
- Created manifest.go with proper Manifest structure including blob sizes - Updated manifest generation to include compressed size for each blob - Added TotalCompressedSize field to manifest for quick access - Renamed backup package to snapshot for clarity - Updated snapshot list to show all remote snapshots - Remote snapshots not in local DB fetch manifest to get size - Local snapshots not in remote are automatically deleted - Removed backwards compatibility code (pre-1.0, no users) - Fixed prune command to use new manifest format - Updated all imports and references from backup to snapshot
This commit is contained in:
@@ -12,14 +12,13 @@ import (
|
||||
"text/tabwriter"
|
||||
"time"
|
||||
|
||||
"git.eeqj.de/sneak/vaultik/internal/backup"
|
||||
"git.eeqj.de/sneak/vaultik/internal/config"
|
||||
"git.eeqj.de/sneak/vaultik/internal/database"
|
||||
"git.eeqj.de/sneak/vaultik/internal/globals"
|
||||
"git.eeqj.de/sneak/vaultik/internal/log"
|
||||
"git.eeqj.de/sneak/vaultik/internal/s3"
|
||||
"git.eeqj.de/sneak/vaultik/internal/snapshot"
|
||||
"github.com/dustin/go-humanize"
|
||||
"github.com/klauspost/compress/zstd"
|
||||
"github.com/spf13/cobra"
|
||||
"go.uber.org/fx"
|
||||
)
|
||||
@@ -36,8 +35,8 @@ type SnapshotCreateApp struct {
|
||||
Globals *globals.Globals
|
||||
Config *config.Config
|
||||
Repositories *database.Repositories
|
||||
ScannerFactory backup.ScannerFactory
|
||||
SnapshotManager *backup.SnapshotManager
|
||||
ScannerFactory snapshot.ScannerFactory
|
||||
SnapshotManager *snapshot.SnapshotManager
|
||||
S3Client *s3.Client
|
||||
DB *database.DB
|
||||
Lifecycle fx.Lifecycle
|
||||
@@ -106,11 +105,11 @@ specifying a path using --config or by setting VAULTIK_CONFIG to a path.`,
|
||||
Cron: opts.Cron,
|
||||
},
|
||||
Modules: []fx.Option{
|
||||
backup.Module,
|
||||
snapshot.Module,
|
||||
s3.Module,
|
||||
fx.Provide(fx.Annotate(
|
||||
func(g *globals.Globals, cfg *config.Config, repos *database.Repositories,
|
||||
scannerFactory backup.ScannerFactory, snapshotManager *backup.SnapshotManager,
|
||||
scannerFactory snapshot.ScannerFactory, snapshotManager *snapshot.SnapshotManager,
|
||||
s3Client *s3.Client, db *database.DB,
|
||||
lc fx.Lifecycle, shutdowner fx.Shutdowner) *SnapshotCreateApp {
|
||||
return &SnapshotCreateApp{
|
||||
@@ -226,7 +225,7 @@ func (app *SnapshotCreateApp) runSnapshot(ctx context.Context, opts *SnapshotCre
|
||||
}
|
||||
|
||||
// Create scanner with progress enabled (unless in cron mode)
|
||||
scanner := app.ScannerFactory(backup.ScannerParams{
|
||||
scanner := app.ScannerFactory(snapshot.ScannerParams{
|
||||
EnableProgress: !opts.Cron,
|
||||
})
|
||||
|
||||
@@ -306,8 +305,8 @@ func (app *SnapshotCreateApp) runSnapshot(ctx context.Context, opts *SnapshotCre
|
||||
}
|
||||
|
||||
// Update snapshot statistics with extended fields
|
||||
extStats := backup.ExtendedBackupStats{
|
||||
BackupStats: backup.BackupStats{
|
||||
extStats := snapshot.ExtendedBackupStats{
|
||||
BackupStats: snapshot.BackupStats{
|
||||
FilesScanned: totalFiles,
|
||||
BytesScanned: totalBytes,
|
||||
ChunksCreated: totalChunks,
|
||||
@@ -487,15 +486,78 @@ func newSnapshotVerifyCommand() *cobra.Command {
|
||||
|
||||
// List lists all snapshots
|
||||
func (app *SnapshotApp) List(ctx context.Context, jsonOutput bool) error {
|
||||
// First, sync with remote snapshots
|
||||
if err := app.syncWithRemote(ctx); err != nil {
|
||||
return fmt.Errorf("syncing with remote: %w", err)
|
||||
// Get all remote snapshots
|
||||
remoteSnapshots := make(map[string]bool)
|
||||
objectCh := app.S3Client.ListObjectsStream(ctx, "metadata/", false)
|
||||
|
||||
for object := range objectCh {
|
||||
if object.Err != nil {
|
||||
return fmt.Errorf("listing remote snapshots: %w", object.Err)
|
||||
}
|
||||
|
||||
// Extract snapshot ID from paths like metadata/hostname-20240115-143052Z/
|
||||
parts := strings.Split(object.Key, "/")
|
||||
if len(parts) >= 2 && parts[0] == "metadata" && parts[1] != "" {
|
||||
remoteSnapshots[parts[1]] = true
|
||||
}
|
||||
}
|
||||
|
||||
// Now get snapshots from S3
|
||||
snapshots, err := app.getSnapshots(ctx)
|
||||
// Get all local snapshots
|
||||
localSnapshots, err := app.Repositories.Snapshots.ListRecent(ctx, 10000)
|
||||
if err != nil {
|
||||
return err
|
||||
return fmt.Errorf("listing local snapshots: %w", err)
|
||||
}
|
||||
|
||||
// Build a map of local snapshots for quick lookup
|
||||
localSnapshotMap := make(map[string]*database.Snapshot)
|
||||
for _, s := range localSnapshots {
|
||||
localSnapshotMap[s.ID] = s
|
||||
}
|
||||
|
||||
// Remove local snapshots that don't exist remotely
|
||||
for _, snapshot := range localSnapshots {
|
||||
if !remoteSnapshots[snapshot.ID] {
|
||||
log.Info("Removing local snapshot not found in remote", "snapshot_id", snapshot.ID)
|
||||
if err := app.Repositories.Snapshots.Delete(ctx, snapshot.ID); err != nil {
|
||||
log.Error("Failed to delete local snapshot", "snapshot_id", snapshot.ID, "error", err)
|
||||
}
|
||||
delete(localSnapshotMap, snapshot.ID)
|
||||
}
|
||||
}
|
||||
|
||||
// Build final snapshot list
|
||||
snapshots := make([]SnapshotInfo, 0, len(remoteSnapshots))
|
||||
|
||||
for snapshotID := range remoteSnapshots {
|
||||
// Check if we have this snapshot locally
|
||||
if localSnap, exists := localSnapshotMap[snapshotID]; exists && localSnap.CompletedAt != nil {
|
||||
// Use local data
|
||||
snapshots = append(snapshots, SnapshotInfo{
|
||||
ID: localSnap.ID,
|
||||
Timestamp: localSnap.StartedAt,
|
||||
CompressedSize: localSnap.BlobSize,
|
||||
})
|
||||
} else {
|
||||
// Remote snapshot not in local DB - fetch manifest to get size
|
||||
timestamp, err := parseSnapshotTimestamp(snapshotID)
|
||||
if err != nil {
|
||||
log.Warn("Failed to parse snapshot timestamp", "id", snapshotID, "error", err)
|
||||
continue
|
||||
}
|
||||
|
||||
// Try to download manifest to get size
|
||||
totalSize, err := app.getManifestSize(ctx, snapshotID)
|
||||
if err != nil {
|
||||
log.Warn("Failed to get manifest size", "id", snapshotID, "error", err)
|
||||
totalSize = 0
|
||||
}
|
||||
|
||||
snapshots = append(snapshots, SnapshotInfo{
|
||||
ID: snapshotID,
|
||||
Timestamp: timestamp,
|
||||
CompressedSize: totalSize,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// Sort by timestamp (newest first)
|
||||
@@ -512,17 +574,18 @@ func (app *SnapshotApp) List(ctx context.Context, jsonOutput bool) error {
|
||||
|
||||
// Table output
|
||||
w := tabwriter.NewWriter(os.Stdout, 0, 0, 3, ' ', 0)
|
||||
if _, err := fmt.Fprintln(w, "SNAPSHOT ID\tTIMESTAMP"); err != nil {
|
||||
if _, err := fmt.Fprintln(w, "SNAPSHOT ID\tTIMESTAMP\tCOMPRESSED SIZE"); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := fmt.Fprintln(w, "───────────\t─────────"); err != nil {
|
||||
if _, err := fmt.Fprintln(w, "───────────\t─────────\t───────────────"); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
for _, snap := range snapshots {
|
||||
if _, err := fmt.Fprintf(w, "%s\t%s\n",
|
||||
if _, err := fmt.Fprintf(w, "%s\t%s\t%s\n",
|
||||
snap.ID,
|
||||
snap.Timestamp.Format("2006-01-02 15:04:05")); err != nil {
|
||||
snap.Timestamp.Format("2006-01-02 15:04:05"),
|
||||
formatBytes(snap.CompressedSize)); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
@@ -532,9 +595,27 @@ func (app *SnapshotApp) List(ctx context.Context, jsonOutput bool) error {
|
||||
|
||||
// Purge removes old snapshots based on criteria
|
||||
func (app *SnapshotApp) Purge(ctx context.Context, keepLatest bool, olderThan string, force bool) error {
|
||||
snapshots, err := app.getSnapshots(ctx)
|
||||
// Sync with remote first
|
||||
if err := app.syncWithRemote(ctx); err != nil {
|
||||
return fmt.Errorf("syncing with remote: %w", err)
|
||||
}
|
||||
|
||||
// Get snapshots from local database
|
||||
dbSnapshots, err := app.Repositories.Snapshots.ListRecent(ctx, 10000)
|
||||
if err != nil {
|
||||
return err
|
||||
return fmt.Errorf("listing snapshots: %w", err)
|
||||
}
|
||||
|
||||
// Convert to SnapshotInfo format, only including completed snapshots
|
||||
snapshots := make([]SnapshotInfo, 0, len(dbSnapshots))
|
||||
for _, s := range dbSnapshots {
|
||||
if s.CompletedAt != nil {
|
||||
snapshots = append(snapshots, SnapshotInfo{
|
||||
ID: s.ID,
|
||||
Timestamp: s.StartedAt,
|
||||
CompressedSize: s.BlobSize,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// Sort by timestamp (newest first)
|
||||
@@ -657,57 +738,25 @@ func (app *SnapshotApp) Verify(ctx context.Context, snapshotID string, deep bool
|
||||
return nil
|
||||
}
|
||||
|
||||
// getSnapshots retrieves all snapshots from S3
|
||||
func (app *SnapshotApp) getSnapshots(ctx context.Context) ([]SnapshotInfo, error) {
|
||||
var snapshots []SnapshotInfo
|
||||
// getManifestSize downloads a manifest and returns the total compressed size
|
||||
func (app *SnapshotApp) getManifestSize(ctx context.Context, snapshotID string) (int64, error) {
|
||||
manifestPath := fmt.Sprintf("metadata/%s/manifest.json.zst", snapshotID)
|
||||
|
||||
// List all objects under metadata/
|
||||
objectCh := app.S3Client.ListObjectsStream(ctx, "metadata/", true)
|
||||
reader, err := app.S3Client.GetObject(ctx, manifestPath)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("downloading manifest: %w", err)
|
||||
}
|
||||
defer func() { _ = reader.Close() }()
|
||||
|
||||
// Track unique snapshots
|
||||
snapshotMap := make(map[string]*SnapshotInfo)
|
||||
|
||||
for object := range objectCh {
|
||||
if object.Err != nil {
|
||||
return nil, fmt.Errorf("listing objects: %w", object.Err)
|
||||
}
|
||||
|
||||
// Extract snapshot ID from paths like metadata/2024-01-15-143052-hostname/manifest.json.zst
|
||||
parts := strings.Split(object.Key, "/")
|
||||
if len(parts) < 3 || parts[0] != "metadata" {
|
||||
continue
|
||||
}
|
||||
|
||||
snapshotID := parts[1]
|
||||
if snapshotID == "" {
|
||||
continue
|
||||
}
|
||||
|
||||
// Initialize snapshot info if not seen
|
||||
if _, exists := snapshotMap[snapshotID]; !exists {
|
||||
timestamp, err := parseSnapshotTimestamp(snapshotID)
|
||||
if err != nil {
|
||||
log.Warn("Failed to parse snapshot timestamp", "id", snapshotID, "error", err)
|
||||
continue
|
||||
}
|
||||
|
||||
snapshotMap[snapshotID] = &SnapshotInfo{
|
||||
ID: snapshotID,
|
||||
Timestamp: timestamp,
|
||||
CompressedSize: 0,
|
||||
}
|
||||
}
|
||||
manifest, err := snapshot.DecodeManifest(reader)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("decoding manifest: %w", err)
|
||||
}
|
||||
|
||||
// Convert map to slice without downloading manifests
|
||||
for _, snap := range snapshotMap {
|
||||
snapshots = append(snapshots, *snap)
|
||||
}
|
||||
|
||||
return snapshots, nil
|
||||
return manifest.TotalCompressedSize, nil
|
||||
}
|
||||
|
||||
// downloadManifest downloads and parses a snapshot manifest
|
||||
// downloadManifest downloads and parses a snapshot manifest (for verify command)
|
||||
func (app *SnapshotApp) downloadManifest(ctx context.Context, snapshotID string) ([]string, error) {
|
||||
manifestPath := fmt.Sprintf("metadata/%s/manifest.json.zst", snapshotID)
|
||||
|
||||
@@ -717,25 +766,17 @@ func (app *SnapshotApp) downloadManifest(ctx context.Context, snapshotID string)
|
||||
}
|
||||
defer func() { _ = reader.Close() }()
|
||||
|
||||
// Decompress
|
||||
zr, err := zstd.NewReader(reader)
|
||||
manifest, err := snapshot.DecodeManifest(reader)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("creating zstd reader: %w", err)
|
||||
}
|
||||
defer zr.Close()
|
||||
|
||||
// Decode JSON - manifest is an object with a "blobs" field
|
||||
var manifest struct {
|
||||
SnapshotID string `json:"snapshot_id"`
|
||||
Timestamp string `json:"timestamp"`
|
||||
BlobCount int `json:"blob_count"`
|
||||
Blobs []string `json:"blobs"`
|
||||
}
|
||||
if err := json.NewDecoder(zr).Decode(&manifest); err != nil {
|
||||
return nil, fmt.Errorf("decoding manifest: %w", err)
|
||||
}
|
||||
|
||||
return manifest.Blobs, nil
|
||||
// Extract blob hashes
|
||||
hashes := make([]string, len(manifest.Blobs))
|
||||
for i, blob := range manifest.Blobs {
|
||||
hashes[i] = blob.Hash
|
||||
}
|
||||
return hashes, nil
|
||||
}
|
||||
|
||||
// deleteSnapshot removes a snapshot and its metadata
|
||||
|
||||
Reference in New Issue
Block a user