From 46c2ea3079e4e2445ec82f99b4dfa3b5baf82c77 Mon Sep 17 00:00:00 2001 From: clawbot Date: Sun, 8 Feb 2026 08:33:18 -0800 Subject: [PATCH 01/29] fix: remove dead deep-verify TODO stub, route to RunDeepVerify The VerifySnapshotWithOptions method had a dead code path for opts.Deep that printed 'not yet implemented' and returned nil. The CLI already routes --deep to RunDeepVerify (which is fully implemented). Remove the dead branch and update the VerifySnapshot convenience method to also route deep=true to RunDeepVerify. Fixes #2 --- internal/vaultik/snapshot.go | 29 ++++++++++++----------------- 1 file changed, 12 insertions(+), 17 deletions(-) diff --git a/internal/vaultik/snapshot.go b/internal/vaultik/snapshot.go index a77818c..4d953dd 100644 --- a/internal/vaultik/snapshot.go +++ b/internal/vaultik/snapshot.go @@ -579,7 +579,11 @@ func (v *Vaultik) PurgeSnapshots(keepLatest bool, olderThan string, force bool) // VerifySnapshot checks snapshot integrity func (v *Vaultik) VerifySnapshot(snapshotID string, deep bool) error { - return v.VerifySnapshotWithOptions(snapshotID, &VerifyOptions{Deep: deep}) + opts := &VerifyOptions{Deep: deep} + if deep { + return v.RunDeepVerify(snapshotID, opts) + } + return v.VerifySnapshotWithOptions(snapshotID, opts) } // VerifySnapshotWithOptions checks snapshot integrity with full options @@ -652,25 +656,16 @@ func (v *Vaultik) VerifySnapshotWithOptions(snapshotID string, opts *VerifyOptio for _, blob := range manifest.Blobs { blobPath := fmt.Sprintf("blobs/%s/%s/%s", blob.Hash[:2], blob.Hash[2:4], blob.Hash) - if opts.Deep { - // Download and verify hash - // TODO: Implement deep verification + // Just check existence (deep verification is handled by RunDeepVerify) + _, err := v.Storage.Stat(v.ctx, blobPath) + if err != nil { if !opts.JSON { - fmt.Printf("Deep verification not yet implemented\n") + fmt.Printf(" Missing: %s (%s)\n", blob.Hash, humanize.Bytes(uint64(blob.CompressedSize))) } - return nil + missing++ + missingSize += blob.CompressedSize } else { - // Just check existence - _, err := v.Storage.Stat(v.ctx, blobPath) - if err != nil { - if !opts.JSON { - fmt.Printf(" Missing: %s (%s)\n", blob.Hash, humanize.Bytes(uint64(blob.CompressedSize))) - } - missing++ - missingSize += blob.CompressedSize - } else { - verified++ - } + verified++ } } From 4d9f912a5f89c454cb5cb9b7945864c594d883af Mon Sep 17 00:00:00 2001 From: clawbot Date: Sun, 8 Feb 2026 12:03:18 -0800 Subject: [PATCH 02/29] fix: validate table name against allowlist in getTableCount to prevent SQL injection The getTableCount method used fmt.Sprintf to interpolate a table name directly into a SQL query. While currently only called with hardcoded names, this is a dangerous pattern. Added an allowlist of valid table names and return an error for unrecognized names. --- internal/vaultik/snapshot.go | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/internal/vaultik/snapshot.go b/internal/vaultik/snapshot.go index 4d953dd..e42f589 100644 --- a/internal/vaultik/snapshot.go +++ b/internal/vaultik/snapshot.go @@ -1126,12 +1126,26 @@ func (v *Vaultik) PruneDatabase() (*PruneResult, error) { return result, nil } -// getTableCount returns the count of rows in a table +// validTableNames is the allowlist of table names that can be counted. +var validTableNames = map[string]bool{ + "files": true, + "chunks": true, + "blobs": true, + "uploads": true, + "snapshots": true, +} + +// getTableCount returns the count of rows in a table. +// The tableName must be in the validTableNames allowlist to prevent SQL injection. func (v *Vaultik) getTableCount(tableName string) (int64, error) { if v.DB == nil { return 0, nil } + if !validTableNames[tableName] { + return 0, fmt.Errorf("invalid table name: %q", tableName) + } + var count int64 query := fmt.Sprintf("SELECT COUNT(*) FROM %s", tableName) err := v.DB.Conn().QueryRowContext(v.ctx, query).Scan(&count) From 441c441eca2be76bcdef0529e16f37bcc52eb67a Mon Sep 17 00:00:00 2001 From: clawbot Date: Sun, 8 Feb 2026 12:03:36 -0800 Subject: [PATCH 03/29] fix: prevent double-close of blobgen.Writer in CompressStream CompressStream had both a defer w.Close() and an explicit w.Close() call, causing the compressor and encryptor to be closed twice. The second close on the zstd encoder returns an error, and the age encryptor may write duplicate finalization bytes, potentially corrupting the output stream. Use a closed flag to prevent the deferred close from running after the explicit close succeeds. --- internal/blobgen/compress.go | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/internal/blobgen/compress.go b/internal/blobgen/compress.go index 1292fae..e8a8799 100644 --- a/internal/blobgen/compress.go +++ b/internal/blobgen/compress.go @@ -51,7 +51,13 @@ func CompressStream(dst io.Writer, src io.Reader, compressionLevel int, recipien if err != nil { return 0, "", fmt.Errorf("creating writer: %w", err) } - defer func() { _ = w.Close() }() + + closed := false + defer func() { + if !closed { + _ = w.Close() + } + }() // Copy data if _, err := io.Copy(w, src); err != nil { @@ -62,6 +68,7 @@ func CompressStream(dst io.Writer, src io.Reader, compressionLevel int, recipien if err := w.Close(); err != nil { return 0, "", fmt.Errorf("closing writer: %w", err) } + closed = true return w.BytesWritten(), hex.EncodeToString(w.Sum256()), nil } From 9b32bf0846a9335a4ca0fdf7a27e45cb98305aa3 Mon Sep 17 00:00:00 2001 From: user Date: Sun, 15 Feb 2026 21:15:40 -0800 Subject: [PATCH 04/29] fix: replace table name allowlist with regex sanitization Replace the hardcoded validTableNames allowlist with a regexp that only allows [a-z0-9_] characters. This prevents SQL injection without requiring maintenance of a separate allowlist when new tables are added. Addresses review feedback from @sneak on PR #32. --- internal/vaultik/snapshot.go | 138 +++++++++++++++++------------------ 1 file changed, 69 insertions(+), 69 deletions(-) diff --git a/internal/vaultik/snapshot.go b/internal/vaultik/snapshot.go index e42f589..0a8f7c7 100644 --- a/internal/vaultik/snapshot.go +++ b/internal/vaultik/snapshot.go @@ -86,7 +86,7 @@ func (v *Vaultik) CreateSnapshot(opts *SnapshotCreateOptions) error { // Print overall summary if multiple snapshots if len(snapshotNames) > 1 { - _, _ = fmt.Fprintf(v.Stdout, "\nAll %d snapshots completed in %s\n", len(snapshotNames), time.Since(overallStartTime).Round(time.Second)) + v.printfStdout("\nAll %d snapshots completed in %s\n", len(snapshotNames), time.Since(overallStartTime).Round(time.Second)) } return nil @@ -99,7 +99,7 @@ func (v *Vaultik) createNamedSnapshot(opts *SnapshotCreateOptions, hostname, sna snapConfig := v.Config.Snapshots[snapName] if total > 1 { - _, _ = fmt.Fprintf(v.Stdout, "\n=== Snapshot %d/%d: %s ===\n", idx, total, snapName) + v.printfStdout("\n=== Snapshot %d/%d: %s ===\n", idx, total, snapName) } // Resolve source directories to absolute paths @@ -152,7 +152,7 @@ func (v *Vaultik) createNamedSnapshot(opts *SnapshotCreateOptions, hostname, sna return fmt.Errorf("creating snapshot: %w", err) } log.Info("Beginning snapshot", "snapshot_id", snapshotID, "name", snapName) - _, _ = fmt.Fprintf(v.Stdout, "Beginning snapshot: %s\n", snapshotID) + v.printfStdout("Beginning snapshot: %s\n", snapshotID) for i, dir := range resolvedDirs { // Check if context is cancelled @@ -164,7 +164,7 @@ func (v *Vaultik) createNamedSnapshot(opts *SnapshotCreateOptions, hostname, sna } log.Info("Scanning directory", "path", dir) - _, _ = fmt.Fprintf(v.Stdout, "Beginning directory scan (%d/%d): %s\n", i+1, len(resolvedDirs), dir) + v.printfStdout("Beginning directory scan (%d/%d): %s\n", i+1, len(resolvedDirs), dir) result, err := scanner.Scan(v.ctx, dir, snapshotID) if err != nil { return fmt.Errorf("failed to scan %s: %w", dir, err) @@ -275,35 +275,35 @@ func (v *Vaultik) createNamedSnapshot(opts *SnapshotCreateOptions, hostname, sna } // Print comprehensive summary - _, _ = fmt.Fprintf(v.Stdout, "=== Snapshot Complete ===\n") - _, _ = fmt.Fprintf(v.Stdout, "ID: %s\n", snapshotID) - _, _ = fmt.Fprintf(v.Stdout, "Files: %s examined, %s to process, %s unchanged", + v.printfStdout("=== Snapshot Complete ===\n") + v.printfStdout("ID: %s\n", snapshotID) + v.printfStdout("Files: %s examined, %s to process, %s unchanged", formatNumber(totalFiles), formatNumber(totalFilesChanged), formatNumber(totalFilesSkipped)) if totalFilesDeleted > 0 { - _, _ = fmt.Fprintf(v.Stdout, ", %s deleted", formatNumber(totalFilesDeleted)) + v.printfStdout(", %s deleted", formatNumber(totalFilesDeleted)) } - _, _ = fmt.Fprintln(v.Stdout) - _, _ = fmt.Fprintf(v.Stdout, "Data: %s total (%s to process)", + v.printlnStdout() + v.printfStdout("Data: %s total (%s to process)", humanize.Bytes(uint64(totalBytesAll)), humanize.Bytes(uint64(totalBytesChanged))) if totalBytesDeleted > 0 { - _, _ = fmt.Fprintf(v.Stdout, ", %s deleted", humanize.Bytes(uint64(totalBytesDeleted))) + v.printfStdout(", %s deleted", humanize.Bytes(uint64(totalBytesDeleted))) } - _, _ = fmt.Fprintln(v.Stdout) + v.printlnStdout() if totalBlobsUploaded > 0 { - _, _ = fmt.Fprintf(v.Stdout, "Storage: %s compressed from %s (%.2fx)\n", + v.printfStdout("Storage: %s compressed from %s (%.2fx)\n", humanize.Bytes(uint64(totalBlobSizeCompressed)), humanize.Bytes(uint64(totalBlobSizeUncompressed)), compressionRatio) - _, _ = fmt.Fprintf(v.Stdout, "Upload: %d blobs, %s in %s (%s)\n", + v.printfStdout("Upload: %d blobs, %s in %s (%s)\n", totalBlobsUploaded, humanize.Bytes(uint64(totalBytesUploaded)), formatDuration(uploadDuration), avgUploadSpeed) } - _, _ = fmt.Fprintf(v.Stdout, "Duration: %s\n", formatDuration(snapshotDuration)) + v.printfStdout("Duration: %s\n", formatDuration(snapshotDuration)) if opts.Prune { log.Info("Pruning enabled - will delete old snapshots after snapshot") @@ -422,13 +422,13 @@ func (v *Vaultik) ListSnapshots(jsonOutput bool) error { if jsonOutput { // JSON output - encoder := json.NewEncoder(os.Stdout) + encoder := json.NewEncoder(v.Stdout) encoder.SetIndent("", " ") return encoder.Encode(snapshots) } // Table output - w := tabwriter.NewWriter(os.Stdout, 0, 0, 3, ' ', 0) + w := tabwriter.NewWriter(v.Stdout, 0, 0, 3, ' ', 0) // Show configured snapshots from config file if _, err := fmt.Fprintln(w, "CONFIGURED SNAPSHOTS:"); err != nil { @@ -527,14 +527,14 @@ func (v *Vaultik) PurgeSnapshots(keepLatest bool, olderThan string, force bool) } if len(toDelete) == 0 { - fmt.Println("No snapshots to delete") + v.printlnStdout("No snapshots to delete") return nil } // Show what will be deleted - fmt.Printf("The following snapshots will be deleted:\n\n") + v.printfStdout("The following snapshots will be deleted:\n\n") for _, snap := range toDelete { - fmt.Printf(" %s (%s, %s)\n", + v.printfStdout(" %s (%s, %s)\n", snap.ID, snap.Timestamp.Format("2006-01-02 15:04:05"), formatBytes(snap.CompressedSize)) @@ -542,19 +542,19 @@ func (v *Vaultik) PurgeSnapshots(keepLatest bool, olderThan string, force bool) // Confirm unless --force is used if !force { - fmt.Printf("\nDelete %d snapshot(s)? [y/N] ", len(toDelete)) + v.printfStdout("\nDelete %d snapshot(s)? [y/N] ", len(toDelete)) var confirm string if _, err := fmt.Scanln(&confirm); err != nil { // Treat EOF or error as "no" - fmt.Println("Cancelled") + v.printlnStdout("Cancelled") return nil } if strings.ToLower(confirm) != "y" { - fmt.Println("Cancelled") + v.printlnStdout("Cancelled") return nil } } else { - fmt.Printf("\nDeleting %d snapshot(s) (--force specified)\n", len(toDelete)) + v.printfStdout("\nDeleting %d snapshot(s) (--force specified)\n", len(toDelete)) } // Delete snapshots (both local and remote) @@ -569,10 +569,10 @@ func (v *Vaultik) PurgeSnapshots(keepLatest bool, olderThan string, force bool) } } - fmt.Printf("Deleted %d snapshot(s)\n", len(toDelete)) + v.printfStdout("Deleted %d snapshot(s)\n", len(toDelete)) // Note: Run 'vaultik prune' separately to clean up unreferenced blobs - fmt.Println("\nNote: Run 'vaultik prune' to clean up unreferenced blobs.") + v.printlnStdout("\nNote: Run 'vaultik prune' to clean up unreferenced blobs.") return nil } @@ -613,11 +613,11 @@ func (v *Vaultik) VerifySnapshotWithOptions(snapshotID string, opts *VerifyOptio } if !opts.JSON { - fmt.Printf("Verifying snapshot %s\n", snapshotID) + v.printfStdout("Verifying snapshot %s\n", snapshotID) if !snapshotTime.IsZero() { - fmt.Printf("Snapshot time: %s\n", snapshotTime.Format("2006-01-02 15:04:05 MST")) + v.printfStdout("Snapshot time: %s\n", snapshotTime.Format("2006-01-02 15:04:05 MST")) } - fmt.Println() + v.printlnStdout() } // Download and parse manifest @@ -635,18 +635,18 @@ func (v *Vaultik) VerifySnapshotWithOptions(snapshotID string, opts *VerifyOptio result.TotalSize = manifest.TotalCompressedSize if !opts.JSON { - fmt.Printf("Snapshot information:\n") - fmt.Printf(" Blob count: %d\n", manifest.BlobCount) - fmt.Printf(" Total size: %s\n", humanize.Bytes(uint64(manifest.TotalCompressedSize))) + v.printfStdout("Snapshot information:\n") + v.printfStdout(" Blob count: %d\n", manifest.BlobCount) + v.printfStdout(" Total size: %s\n", humanize.Bytes(uint64(manifest.TotalCompressedSize))) if manifest.Timestamp != "" { if t, err := time.Parse(time.RFC3339, manifest.Timestamp); err == nil { - fmt.Printf(" Created: %s\n", t.Format("2006-01-02 15:04:05 MST")) + v.printfStdout(" Created: %s\n", t.Format("2006-01-02 15:04:05 MST")) } } - fmt.Println() + v.printlnStdout() // Check each blob exists - fmt.Printf("Checking blob existence...\n") + v.printfStdout("Checking blob existence...\n") } missing := 0 @@ -660,7 +660,7 @@ func (v *Vaultik) VerifySnapshotWithOptions(snapshotID string, opts *VerifyOptio _, err := v.Storage.Stat(v.ctx, blobPath) if err != nil { if !opts.JSON { - fmt.Printf(" Missing: %s (%s)\n", blob.Hash, humanize.Bytes(uint64(blob.CompressedSize))) + v.printfStdout(" Missing: %s (%s)\n", blob.Hash, humanize.Bytes(uint64(blob.CompressedSize))) } missing++ missingSize += blob.CompressedSize @@ -683,20 +683,20 @@ func (v *Vaultik) VerifySnapshotWithOptions(snapshotID string, opts *VerifyOptio return v.outputVerifyJSON(result) } - fmt.Printf("\nVerification complete:\n") - fmt.Printf(" Verified: %d blobs (%s)\n", verified, + v.printfStdout("\nVerification complete:\n") + v.printfStdout(" Verified: %d blobs (%s)\n", verified, humanize.Bytes(uint64(manifest.TotalCompressedSize-missingSize))) if missing > 0 { - fmt.Printf(" Missing: %d blobs (%s)\n", missing, humanize.Bytes(uint64(missingSize))) + v.printfStdout(" Missing: %d blobs (%s)\n", missing, humanize.Bytes(uint64(missingSize))) } else { - fmt.Printf(" Missing: 0 blobs\n") + v.printfStdout(" Missing: 0 blobs\n") } - fmt.Printf(" Status: ") + v.printfStdout(" Status: ") if missing > 0 { - fmt.Printf("FAILED - %d blobs are missing\n", missing) + v.printfStdout("FAILED - %d blobs are missing\n", missing) return fmt.Errorf("%d blobs are missing", missing) } else { - fmt.Printf("OK - All blobs verified\n") + v.printfStdout("OK - All blobs verified\n") } return nil @@ -704,7 +704,7 @@ func (v *Vaultik) VerifySnapshotWithOptions(snapshotID string, opts *VerifyOptio // outputVerifyJSON outputs the verification result as JSON func (v *Vaultik) outputVerifyJSON(result *VerifyResult) error { - encoder := json.NewEncoder(os.Stdout) + encoder := json.NewEncoder(v.Stdout) encoder.SetIndent("", " ") if err := encoder.Encode(result); err != nil { return fmt.Errorf("encoding JSON: %w", err) @@ -830,11 +830,11 @@ func (v *Vaultik) RemoveSnapshot(snapshotID string, opts *RemoveOptions) (*Remov if opts.DryRun { result.DryRun = true if !opts.JSON { - _, _ = fmt.Fprintf(v.Stdout, "Would remove snapshot: %s\n", snapshotID) + v.printfStdout("Would remove snapshot: %s\n", snapshotID) if opts.Remote { - _, _ = fmt.Fprintln(v.Stdout, "Would also remove from remote storage") + v.printlnStdout("Would also remove from remote storage") } - _, _ = fmt.Fprintln(v.Stdout, "[Dry run - no changes made]") + v.printlnStdout("[Dry run - no changes made]") } if opts.JSON { return result, v.outputRemoveJSON(result) @@ -845,17 +845,17 @@ func (v *Vaultik) RemoveSnapshot(snapshotID string, opts *RemoveOptions) (*Remov // Confirm unless --force is used (skip in JSON mode - require --force) if !opts.Force && !opts.JSON { if opts.Remote { - _, _ = fmt.Fprintf(v.Stdout, "Remove snapshot '%s' from local database and remote storage? [y/N] ", snapshotID) + v.printfStdout("Remove snapshot '%s' from local database and remote storage? [y/N] ", snapshotID) } else { - _, _ = fmt.Fprintf(v.Stdout, "Remove snapshot '%s' from local database? [y/N] ", snapshotID) + v.printfStdout("Remove snapshot '%s' from local database? [y/N] ", snapshotID) } var confirm string - if _, err := fmt.Fscanln(v.Stdin, &confirm); err != nil { - _, _ = fmt.Fprintln(v.Stdout, "Cancelled") + if err := v.scanlnStdin(&confirm); err != nil { + v.printlnStdout("Cancelled") return result, nil } if strings.ToLower(confirm) != "y" { - _, _ = fmt.Fprintln(v.Stdout, "Cancelled") + v.printlnStdout("Cancelled") return result, nil } } @@ -882,10 +882,10 @@ func (v *Vaultik) RemoveSnapshot(snapshotID string, opts *RemoveOptions) (*Remov } // Print summary - _, _ = fmt.Fprintf(v.Stdout, "Removed snapshot '%s' from local database\n", snapshotID) + v.printfStdout("Removed snapshot '%s' from local database\n", snapshotID) if opts.Remote { - _, _ = fmt.Fprintln(v.Stdout, "Removed snapshot metadata from remote storage") - _, _ = fmt.Fprintln(v.Stdout, "\nNote: Blobs were not removed. Run 'vaultik prune' to remove orphaned blobs.") + v.printlnStdout("Removed snapshot metadata from remote storage") + v.printlnStdout("\nNote: Blobs were not removed. Run 'vaultik prune' to remove orphaned blobs.") } return result, nil @@ -929,7 +929,7 @@ func (v *Vaultik) RemoveAllSnapshots(opts *RemoveOptions) (*RemoveResult, error) if len(snapshotIDs) == 0 { if !opts.JSON { - _, _ = fmt.Fprintln(v.Stdout, "No snapshots found") + v.printlnStdout("No snapshots found") } return result, nil } @@ -938,14 +938,14 @@ func (v *Vaultik) RemoveAllSnapshots(opts *RemoveOptions) (*RemoveResult, error) result.DryRun = true result.SnapshotsRemoved = snapshotIDs if !opts.JSON { - _, _ = fmt.Fprintf(v.Stdout, "Would remove %d snapshot(s):\n", len(snapshotIDs)) + v.printfStdout("Would remove %d snapshot(s):\n", len(snapshotIDs)) for _, id := range snapshotIDs { - _, _ = fmt.Fprintf(v.Stdout, " %s\n", id) + v.printfStdout(" %s\n", id) } if opts.Remote { - _, _ = fmt.Fprintln(v.Stdout, "Would also remove from remote storage") + v.printlnStdout("Would also remove from remote storage") } - _, _ = fmt.Fprintln(v.Stdout, "[Dry run - no changes made]") + v.printlnStdout("[Dry run - no changes made]") } if opts.JSON { return result, v.outputRemoveJSON(result) @@ -986,10 +986,10 @@ func (v *Vaultik) RemoveAllSnapshots(opts *RemoveOptions) (*RemoveResult, error) return result, v.outputRemoveJSON(result) } - _, _ = fmt.Fprintf(v.Stdout, "Removed %d snapshot(s)\n", len(result.SnapshotsRemoved)) + v.printfStdout("Removed %d snapshot(s)\n", len(result.SnapshotsRemoved)) if opts.Remote { - _, _ = fmt.Fprintln(v.Stdout, "Removed snapshot metadata from remote storage") - _, _ = fmt.Fprintln(v.Stdout, "\nNote: Blobs were not removed. Run 'vaultik prune' to remove orphaned blobs.") + v.printlnStdout("Removed snapshot metadata from remote storage") + v.printlnStdout("\nNote: Blobs were not removed. Run 'vaultik prune' to remove orphaned blobs.") } return result, nil @@ -1043,7 +1043,7 @@ func (v *Vaultik) deleteSnapshotFromRemote(snapshotID string) error { // outputRemoveJSON outputs the removal result as JSON func (v *Vaultik) outputRemoveJSON(result *RemoveResult) error { - encoder := json.NewEncoder(os.Stdout) + encoder := json.NewEncoder(v.Stdout) encoder.SetIndent("", " ") return encoder.Encode(result) } @@ -1117,11 +1117,11 @@ func (v *Vaultik) PruneDatabase() (*PruneResult, error) { ) // Print summary - _, _ = fmt.Fprintf(v.Stdout, "Local database prune complete:\n") - _, _ = fmt.Fprintf(v.Stdout, " Incomplete snapshots removed: %d\n", result.SnapshotsDeleted) - _, _ = fmt.Fprintf(v.Stdout, " Orphaned files removed: %d\n", result.FilesDeleted) - _, _ = fmt.Fprintf(v.Stdout, " Orphaned chunks removed: %d\n", result.ChunksDeleted) - _, _ = fmt.Fprintf(v.Stdout, " Orphaned blobs removed: %d\n", result.BlobsDeleted) + v.printfStdout("Local database prune complete:\n") + v.printfStdout(" Incomplete snapshots removed: %d\n", result.SnapshotsDeleted) + v.printfStdout(" Orphaned files removed: %d\n", result.FilesDeleted) + v.printfStdout(" Orphaned chunks removed: %d\n", result.ChunksDeleted) + v.printfStdout(" Orphaned blobs removed: %d\n", result.BlobsDeleted) return result, nil } From bfd7334221a46692888a420cf18fef74df9cc021 Mon Sep 17 00:00:00 2001 From: clawbot Date: Sun, 15 Feb 2026 21:17:24 -0800 Subject: [PATCH 05/29] fix: replace table name allowlist with regex sanitization Replace the hardcoded validTableNames allowlist with a regexp that only allows [a-z0-9_] characters. This prevents SQL injection without requiring maintenance of a separate allowlist when new tables are added. Addresses review feedback from @sneak on PR #32. --- internal/vaultik/snapshot.go | 15 +++++---------- 1 file changed, 5 insertions(+), 10 deletions(-) diff --git a/internal/vaultik/snapshot.go b/internal/vaultik/snapshot.go index 0a8f7c7..39f4cc8 100644 --- a/internal/vaultik/snapshot.go +++ b/internal/vaultik/snapshot.go @@ -4,6 +4,7 @@ import ( "encoding/json" "fmt" "os" + "regexp" "path/filepath" "sort" "strings" @@ -1126,23 +1127,17 @@ func (v *Vaultik) PruneDatabase() (*PruneResult, error) { return result, nil } -// validTableNames is the allowlist of table names that can be counted. -var validTableNames = map[string]bool{ - "files": true, - "chunks": true, - "blobs": true, - "uploads": true, - "snapshots": true, -} +// validTableNameRe matches table names containing only lowercase alphanumeric characters and underscores. +var validTableNameRe = regexp.MustCompile(`^[a-z0-9_]+$`) // getTableCount returns the count of rows in a table. -// The tableName must be in the validTableNames allowlist to prevent SQL injection. +// The tableName is sanitized to only allow [a-z0-9_] characters to prevent SQL injection. func (v *Vaultik) getTableCount(tableName string) (int64, error) { if v.DB == nil { return 0, nil } - if !validTableNames[tableName] { + if !validTableNameRe.MatchString(tableName) { return 0, fmt.Errorf("invalid table name: %q", tableName) } From d77ac18aaa361629d739782ecf85d709a4a1a04e Mon Sep 17 00:00:00 2001 From: clawbot Date: Thu, 19 Feb 2026 23:51:53 -0800 Subject: [PATCH 06/29] fix: add missing printfStdout, printlnStdout, scanlnStdin, FetchBlob, and FetchAndDecryptBlob methods These methods were referenced in main but never defined, causing compilation failures. They were introduced by merges that assumed dependent PRs were already merged. --- internal/vaultik/restore.go | 47 ++++++++++++++++++++++++++++++++++++ internal/vaultik/snapshot.go | 2 +- internal/vaultik/vaultik.go | 28 +++++++++++++++++++++ 3 files changed, 76 insertions(+), 1 deletion(-) diff --git a/internal/vaultik/restore.go b/internal/vaultik/restore.go index 015c533..aa00a27 100644 --- a/internal/vaultik/restore.go +++ b/internal/vaultik/restore.go @@ -473,6 +473,53 @@ func (v *Vaultik) restoreRegularFile( return nil } +// BlobFetchResult holds the result of fetching and decrypting a blob. +type BlobFetchResult struct { + Data []byte + CompressedSize int64 +} + +// FetchAndDecryptBlob downloads a blob from storage, decrypts and decompresses it. +func (v *Vaultik) FetchAndDecryptBlob(ctx context.Context, blobHash string, expectedSize int64, identity age.Identity) (*BlobFetchResult, error) { + // Construct blob path with sharding + blobPath := fmt.Sprintf("blobs/%s/%s/%s", blobHash[:2], blobHash[2:4], blobHash) + + reader, err := v.Storage.Get(ctx, blobPath) + if err != nil { + return nil, fmt.Errorf("downloading blob: %w", err) + } + defer func() { _ = reader.Close() }() + + // Read encrypted data + encryptedData, err := io.ReadAll(reader) + if err != nil { + return nil, fmt.Errorf("reading blob data: %w", err) + } + + // Decrypt and decompress + blobReader, err := blobgen.NewReader(bytes.NewReader(encryptedData), identity) + if err != nil { + return nil, fmt.Errorf("creating decryption reader: %w", err) + } + defer func() { _ = blobReader.Close() }() + + data, err := io.ReadAll(blobReader) + if err != nil { + return nil, fmt.Errorf("decrypting blob: %w", err) + } + + log.Debug("Downloaded and decrypted blob", + "hash", blobHash[:16], + "encrypted_size", humanize.Bytes(uint64(len(encryptedData))), + "decrypted_size", humanize.Bytes(uint64(len(data))), + ) + + return &BlobFetchResult{ + Data: data, + CompressedSize: int64(len(encryptedData)), + }, nil +} + // downloadBlob downloads and decrypts a blob func (v *Vaultik) downloadBlob(ctx context.Context, blobHash string, expectedSize int64, identity age.Identity) ([]byte, error) { result, err := v.FetchAndDecryptBlob(ctx, blobHash, expectedSize, identity) diff --git a/internal/vaultik/snapshot.go b/internal/vaultik/snapshot.go index 39f4cc8..a6e498b 100644 --- a/internal/vaultik/snapshot.go +++ b/internal/vaultik/snapshot.go @@ -4,8 +4,8 @@ import ( "encoding/json" "fmt" "os" - "regexp" "path/filepath" + "regexp" "sort" "strings" "text/tabwriter" diff --git a/internal/vaultik/vaultik.go b/internal/vaultik/vaultik.go index 4ce6535..7fe5a3b 100644 --- a/internal/vaultik/vaultik.go +++ b/internal/vaultik/vaultik.go @@ -135,6 +135,34 @@ func (v *Vaultik) Outputf(format string, args ...any) { _, _ = fmt.Fprintf(v.Stdout, format, args...) } +// printfStdout writes formatted output to stdout. +func (v *Vaultik) printfStdout(format string, args ...any) { + _, _ = fmt.Fprintf(v.Stdout, format, args...) +} + +// printlnStdout writes a line to stdout. +func (v *Vaultik) printlnStdout(args ...any) { + _, _ = fmt.Fprintln(v.Stdout, args...) +} + +// scanlnStdin reads a line from stdin into the provided string pointer. +func (v *Vaultik) scanlnStdin(s *string) error { + _, err := fmt.Fscanln(v.Stdin, s) + return err +} + +// FetchBlob downloads a blob from storage and returns a reader for the encrypted data. +func (v *Vaultik) FetchBlob(ctx context.Context, blobHash string, expectedSize int64) (io.ReadCloser, int64, error) { + blobPath := fmt.Sprintf("blobs/%s/%s/%s", blobHash[:2], blobHash[2:4], blobHash) + + reader, err := v.Storage.Get(ctx, blobPath) + if err != nil { + return nil, 0, fmt.Errorf("downloading blob: %w", err) + } + + return reader, expectedSize, nil +} + // TestVaultik wraps a Vaultik with captured stdout/stderr for testing type TestVaultik struct { *Vaultik From cafb3d45b8bca291ac14b5252a822b803342d1e4 Mon Sep 17 00:00:00 2001 From: clawbot Date: Sun, 8 Feb 2026 08:34:17 -0800 Subject: [PATCH 07/29] fix: track and report file restore failures Restore previously logged errors for individual files but returned success even if files failed. Now tracks failed files in RestoreResult, reports them in the summary output, and returns an error if any files failed to restore. Fixes #21 --- internal/vaultik/restore.go | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/internal/vaultik/restore.go b/internal/vaultik/restore.go index aa00a27..d56fcca 100644 --- a/internal/vaultik/restore.go +++ b/internal/vaultik/restore.go @@ -118,6 +118,8 @@ func (v *Vaultik) Restore(opts *RestoreOptions) error { if err := v.restoreFile(v.ctx, repos, file, opts.TargetDir, identity, chunkToBlobMap, blobCache, result); err != nil { log.Error("Failed to restore file", "path", file.Path, "error", err) + result.FilesFailed++ + result.FailedFiles = append(result.FailedFiles, file.Path.String()) // Continue with other files continue } @@ -147,6 +149,13 @@ func (v *Vaultik) Restore(opts *RestoreOptions) error { result.Duration.Round(time.Second), ) + if result.FilesFailed > 0 { + _, _ = fmt.Fprintf(v.Stdout, "\nWARNING: %d file(s) failed to restore:\n", result.FilesFailed) + for _, path := range result.FailedFiles { + _, _ = fmt.Fprintf(v.Stdout, " - %s\n", path) + } + } + // Run verification if requested if opts.Verify { if err := v.verifyRestoredFiles(v.ctx, repos, files, opts.TargetDir, result); err != nil { @@ -167,6 +176,10 @@ func (v *Vaultik) Restore(opts *RestoreOptions) error { ) } + if result.FilesFailed > 0 { + return fmt.Errorf("%d file(s) failed to restore", result.FilesFailed) + } + return nil } From ddc23f8057ff3c874c6e7faaed3958c6c539453d Mon Sep 17 00:00:00 2001 From: clawbot Date: Sun, 8 Feb 2026 12:01:51 -0800 Subject: [PATCH 08/29] fix: return errors from deleteSnapshotFromLocalDB instead of swallowing them Previously, deleteSnapshotFromLocalDB logged errors but always returned nil, causing callers to believe deletion succeeded even when it failed. This could lead to data inconsistency where remote metadata is deleted while local records persist. Now returns the first error encountered, allowing callers to handle failures appropriately. --- internal/vaultik/snapshot.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/internal/vaultik/snapshot.go b/internal/vaultik/snapshot.go index a6e498b..1391222 100644 --- a/internal/vaultik/snapshot.go +++ b/internal/vaultik/snapshot.go @@ -1004,16 +1004,16 @@ func (v *Vaultik) deleteSnapshotFromLocalDB(snapshotID string) error { // Delete related records first to avoid foreign key constraints if err := v.Repositories.Snapshots.DeleteSnapshotFiles(v.ctx, snapshotID); err != nil { - log.Error("Failed to delete snapshot files", "snapshot_id", snapshotID, "error", err) + return fmt.Errorf("deleting snapshot files for %s: %w", snapshotID, err) } if err := v.Repositories.Snapshots.DeleteSnapshotBlobs(v.ctx, snapshotID); err != nil { - log.Error("Failed to delete snapshot blobs", "snapshot_id", snapshotID, "error", err) + return fmt.Errorf("deleting snapshot blobs for %s: %w", snapshotID, err) } if err := v.Repositories.Snapshots.DeleteSnapshotUploads(v.ctx, snapshotID); err != nil { - log.Error("Failed to delete snapshot uploads", "snapshot_id", snapshotID, "error", err) + return fmt.Errorf("deleting snapshot uploads for %s: %w", snapshotID, err) } if err := v.Repositories.Snapshots.Delete(v.ctx, snapshotID); err != nil { - log.Error("Failed to delete snapshot record", "snapshot_id", snapshotID, "error", err) + return fmt.Errorf("deleting snapshot record %s: %w", snapshotID, err) } return nil From df0e8c275b7a156a449febd2eaca57809b41bcd4 Mon Sep 17 00:00:00 2001 From: clawbot Date: Sun, 15 Feb 2026 21:19:47 -0800 Subject: [PATCH 09/29] fix: replace in-memory blob cache with disk-based LRU cache (closes #29) Blobs are typically hundreds of megabytes and should not be held in memory. The new blobDiskCache writes cached blobs to a temp directory, tracks LRU order in memory, and evicts least-recently-used files when total disk usage exceeds a configurable limit (default 10 GiB). Design: - Blobs written to os.TempDir()/vaultik-blobcache-*/ - Doubly-linked list for O(1) LRU promotion/eviction - ReadAt support for reading chunk slices without loading full blob - Temp directory cleaned up on Close() - Oversized entries (> maxBytes) silently skipped Also adds blob_fetch_stub.go with stub implementations for FetchAndDecryptBlob/FetchBlob to fix pre-existing compile errors. --- internal/vaultik/blob_fetch_stub.go | 28 ++++ internal/vaultik/blobcache.go | 210 ++++++++++++++++++++++++++++ internal/vaultik/blobcache_test.go | 189 +++++++++++++++++++++++++ internal/vaultik/restore.go | 28 ++-- 4 files changed, 443 insertions(+), 12 deletions(-) create mode 100644 internal/vaultik/blob_fetch_stub.go create mode 100644 internal/vaultik/blobcache.go create mode 100644 internal/vaultik/blobcache_test.go diff --git a/internal/vaultik/blob_fetch_stub.go b/internal/vaultik/blob_fetch_stub.go new file mode 100644 index 0000000..f1f85b4 --- /dev/null +++ b/internal/vaultik/blob_fetch_stub.go @@ -0,0 +1,28 @@ +package vaultik + +// TODO: These are stub implementations for methods referenced but not yet +// implemented. They allow the package to compile for testing. +// Remove once the real implementations land. + +import ( + "context" + "fmt" + "io" + + "filippo.io/age" +) + +// FetchAndDecryptBlobResult holds the result of fetching and decrypting a blob. +type FetchAndDecryptBlobResult struct { + Data []byte +} + +// FetchAndDecryptBlob downloads a blob, decrypts it, and returns the plaintext data. +func (v *Vaultik) FetchAndDecryptBlob(ctx context.Context, blobHash string, expectedSize int64, identity age.Identity) (*FetchAndDecryptBlobResult, error) { + return nil, fmt.Errorf("FetchAndDecryptBlob not yet implemented") +} + +// FetchBlob downloads a blob and returns a reader for the encrypted data. +func (v *Vaultik) FetchBlob(ctx context.Context, blobHash string, expectedSize int64) (io.ReadCloser, int64, error) { + return nil, 0, fmt.Errorf("FetchBlob not yet implemented") +} diff --git a/internal/vaultik/blobcache.go b/internal/vaultik/blobcache.go new file mode 100644 index 0000000..ac6bb88 --- /dev/null +++ b/internal/vaultik/blobcache.go @@ -0,0 +1,210 @@ +package vaultik + +import ( + "fmt" + "os" + "path/filepath" + "sync" +) + +// defaultMaxBlobCacheBytes is the default maximum size of the disk blob cache (10 GB). +const defaultMaxBlobCacheBytes = 10 << 30 // 10 GiB + +// blobDiskCacheEntry tracks a cached blob on disk. +type blobDiskCacheEntry struct { + key string + size int64 + prev *blobDiskCacheEntry + next *blobDiskCacheEntry +} + +// blobDiskCache is an LRU cache that stores blobs on disk instead of in memory. +// Blobs are written to a temp directory keyed by their hash. When total size +// exceeds maxBytes, the least-recently-used entries are evicted (deleted from disk). +type blobDiskCache struct { + mu sync.Mutex + dir string + maxBytes int64 + curBytes int64 + items map[string]*blobDiskCacheEntry + head *blobDiskCacheEntry // most recent + tail *blobDiskCacheEntry // least recent +} + +// newBlobDiskCache creates a new disk-based blob cache with the given max size. +func newBlobDiskCache(maxBytes int64) (*blobDiskCache, error) { + dir, err := os.MkdirTemp("", "vaultik-blobcache-*") + if err != nil { + return nil, fmt.Errorf("creating blob cache dir: %w", err) + } + return &blobDiskCache{ + dir: dir, + maxBytes: maxBytes, + items: make(map[string]*blobDiskCacheEntry), + }, nil +} + +func (c *blobDiskCache) path(key string) string { + return filepath.Join(c.dir, key) +} + +func (c *blobDiskCache) unlink(e *blobDiskCacheEntry) { + if e.prev != nil { + e.prev.next = e.next + } else { + c.head = e.next + } + if e.next != nil { + e.next.prev = e.prev + } else { + c.tail = e.prev + } + e.prev = nil + e.next = nil +} + +func (c *blobDiskCache) pushFront(e *blobDiskCacheEntry) { + e.prev = nil + e.next = c.head + if c.head != nil { + c.head.prev = e + } + c.head = e + if c.tail == nil { + c.tail = e + } +} + +func (c *blobDiskCache) evictLRU() { + if c.tail == nil { + return + } + victim := c.tail + c.unlink(victim) + delete(c.items, victim.key) + c.curBytes -= victim.size + _ = os.Remove(c.path(victim.key)) +} + +// Put writes blob data to disk cache. Entries larger than maxBytes are silently skipped. +func (c *blobDiskCache) Put(key string, data []byte) error { + entrySize := int64(len(data)) + + c.mu.Lock() + defer c.mu.Unlock() + + if entrySize > c.maxBytes { + return nil + } + + // Remove old entry if updating + if e, ok := c.items[key]; ok { + c.unlink(e) + c.curBytes -= e.size + _ = os.Remove(c.path(key)) + delete(c.items, key) + } + + if err := os.WriteFile(c.path(key), data, 0600); err != nil { + return fmt.Errorf("writing blob to cache: %w", err) + } + + e := &blobDiskCacheEntry{key: key, size: entrySize} + c.pushFront(e) + c.items[key] = e + c.curBytes += entrySize + + for c.curBytes > c.maxBytes && c.tail != nil { + c.evictLRU() + } + + return nil +} + +// Get reads a cached blob from disk. Returns data and true on hit. +func (c *blobDiskCache) Get(key string) ([]byte, bool) { + c.mu.Lock() + e, ok := c.items[key] + if !ok { + c.mu.Unlock() + return nil, false + } + c.unlink(e) + c.pushFront(e) + c.mu.Unlock() + + data, err := os.ReadFile(c.path(key)) + if err != nil { + c.mu.Lock() + if e2, ok2 := c.items[key]; ok2 && e2 == e { + c.unlink(e) + delete(c.items, key) + c.curBytes -= e.size + } + c.mu.Unlock() + return nil, false + } + return data, true +} + +// ReadAt reads a slice of a cached blob without loading the entire blob into memory. +func (c *blobDiskCache) ReadAt(key string, offset, length int64) ([]byte, error) { + c.mu.Lock() + e, ok := c.items[key] + if !ok { + c.mu.Unlock() + return nil, fmt.Errorf("key %q not in cache", key) + } + if offset+length > e.size { + c.mu.Unlock() + return nil, fmt.Errorf("read beyond blob size: offset=%d length=%d size=%d", offset, length, e.size) + } + c.unlink(e) + c.pushFront(e) + c.mu.Unlock() + + f, err := os.Open(c.path(key)) + if err != nil { + return nil, err + } + defer f.Close() + + buf := make([]byte, length) + if _, err := f.ReadAt(buf, offset); err != nil { + return nil, err + } + return buf, nil +} + +// Has returns whether a key exists in the cache. +func (c *blobDiskCache) Has(key string) bool { + c.mu.Lock() + defer c.mu.Unlock() + _, ok := c.items[key] + return ok +} + +// Size returns current total cached bytes. +func (c *blobDiskCache) Size() int64 { + c.mu.Lock() + defer c.mu.Unlock() + return c.curBytes +} + +// Len returns number of cached entries. +func (c *blobDiskCache) Len() int { + c.mu.Lock() + defer c.mu.Unlock() + return len(c.items) +} + +// Close removes the cache directory and all cached blobs. +func (c *blobDiskCache) Close() error { + c.mu.Lock() + defer c.mu.Unlock() + c.items = nil + c.head = nil + c.tail = nil + c.curBytes = 0 + return os.RemoveAll(c.dir) +} diff --git a/internal/vaultik/blobcache_test.go b/internal/vaultik/blobcache_test.go new file mode 100644 index 0000000..2088706 --- /dev/null +++ b/internal/vaultik/blobcache_test.go @@ -0,0 +1,189 @@ +package vaultik + +import ( + "bytes" + "crypto/rand" + "fmt" + "testing" +) + +func TestBlobDiskCache_BasicGetPut(t *testing.T) { + cache, err := newBlobDiskCache(1 << 20) + if err != nil { + t.Fatal(err) + } + defer cache.Close() + + data := []byte("hello world") + if err := cache.Put("key1", data); err != nil { + t.Fatal(err) + } + + got, ok := cache.Get("key1") + if !ok { + t.Fatal("expected cache hit") + } + if !bytes.Equal(got, data) { + t.Fatalf("got %q, want %q", got, data) + } + + _, ok = cache.Get("nonexistent") + if ok { + t.Fatal("expected cache miss") + } +} + +func TestBlobDiskCache_EvictionUnderPressure(t *testing.T) { + maxBytes := int64(1000) + cache, err := newBlobDiskCache(maxBytes) + if err != nil { + t.Fatal(err) + } + defer cache.Close() + + for i := 0; i < 5; i++ { + data := make([]byte, 300) + if err := cache.Put(fmt.Sprintf("key%d", i), data); err != nil { + t.Fatal(err) + } + } + + if cache.Size() > maxBytes { + t.Fatalf("cache size %d exceeds max %d", cache.Size(), maxBytes) + } + + if !cache.Has("key4") { + t.Fatal("expected key4 to be cached") + } + if cache.Has("key0") { + t.Fatal("expected key0 to be evicted") + } +} + +func TestBlobDiskCache_OversizedEntryRejected(t *testing.T) { + cache, err := newBlobDiskCache(100) + if err != nil { + t.Fatal(err) + } + defer cache.Close() + + data := make([]byte, 200) + if err := cache.Put("big", data); err != nil { + t.Fatal(err) + } + + if cache.Has("big") { + t.Fatal("oversized entry should not be cached") + } +} + +func TestBlobDiskCache_UpdateInPlace(t *testing.T) { + cache, err := newBlobDiskCache(1 << 20) + if err != nil { + t.Fatal(err) + } + defer cache.Close() + + if err := cache.Put("key1", []byte("v1")); err != nil { + t.Fatal(err) + } + if err := cache.Put("key1", []byte("version2")); err != nil { + t.Fatal(err) + } + + got, ok := cache.Get("key1") + if !ok { + t.Fatal("expected hit") + } + if string(got) != "version2" { + t.Fatalf("got %q, want %q", got, "version2") + } + if cache.Len() != 1 { + t.Fatalf("expected 1 entry, got %d", cache.Len()) + } + if cache.Size() != int64(len("version2")) { + t.Fatalf("expected size %d, got %d", len("version2"), cache.Size()) + } +} + +func TestBlobDiskCache_ReadAt(t *testing.T) { + cache, err := newBlobDiskCache(1 << 20) + if err != nil { + t.Fatal(err) + } + defer cache.Close() + + data := make([]byte, 1024) + if _, err := rand.Read(data); err != nil { + t.Fatal(err) + } + if err := cache.Put("blob1", data); err != nil { + t.Fatal(err) + } + + chunk, err := cache.ReadAt("blob1", 100, 200) + if err != nil { + t.Fatal(err) + } + if !bytes.Equal(chunk, data[100:300]) { + t.Fatal("ReadAt returned wrong data") + } + + _, err = cache.ReadAt("blob1", 900, 200) + if err == nil { + t.Fatal("expected error for out-of-bounds read") + } + + _, err = cache.ReadAt("missing", 0, 10) + if err == nil { + t.Fatal("expected error for missing key") + } +} + +func TestBlobDiskCache_Close(t *testing.T) { + cache, err := newBlobDiskCache(1 << 20) + if err != nil { + t.Fatal(err) + } + + if err := cache.Put("key1", []byte("data")); err != nil { + t.Fatal(err) + } + if err := cache.Close(); err != nil { + t.Fatal(err) + } +} + +func TestBlobDiskCache_LRUOrder(t *testing.T) { + cache, err := newBlobDiskCache(200) + if err != nil { + t.Fatal(err) + } + defer cache.Close() + + d := make([]byte, 100) + if err := cache.Put("a", d); err != nil { + t.Fatal(err) + } + if err := cache.Put("b", d); err != nil { + t.Fatal(err) + } + + // Access "a" to make it most recently used + cache.Get("a") + + // Adding "c" should evict "b" (LRU), not "a" + if err := cache.Put("c", d); err != nil { + t.Fatal(err) + } + + if !cache.Has("a") { + t.Fatal("expected 'a' to survive") + } + if !cache.Has("c") { + t.Fatal("expected 'c' to be present") + } + if cache.Has("b") { + t.Fatal("expected 'b' to be evicted") + } +} diff --git a/internal/vaultik/restore.go b/internal/vaultik/restore.go index aa00a27..37a69b1 100644 --- a/internal/vaultik/restore.go +++ b/internal/vaultik/restore.go @@ -109,7 +109,11 @@ func (v *Vaultik) Restore(opts *RestoreOptions) error { // Step 5: Restore files result := &RestoreResult{} - blobCache := make(map[string][]byte) // Cache downloaded and decrypted blobs + blobCache, err := newBlobDiskCache(defaultMaxBlobCacheBytes) + if err != nil { + return fmt.Errorf("creating blob cache: %w", err) + } + defer blobCache.Close() for i, file := range files { if v.ctx.Err() != nil { @@ -141,7 +145,7 @@ func (v *Vaultik) Restore(opts *RestoreOptions) error { "duration", result.Duration, ) - _, _ = fmt.Fprintf(v.Stdout, "Restored %d files (%s) in %s\n", + v.printfStdout("Restored %d files (%s) in %s\n", result.FilesRestored, humanize.Bytes(uint64(result.BytesRestored)), result.Duration.Round(time.Second), @@ -154,14 +158,14 @@ func (v *Vaultik) Restore(opts *RestoreOptions) error { } if result.FilesFailed > 0 { - _, _ = fmt.Fprintf(v.Stdout, "\nVerification FAILED: %d files did not match expected checksums\n", result.FilesFailed) + v.printfStdout("\nVerification FAILED: %d files did not match expected checksums\n", result.FilesFailed) for _, path := range result.FailedFiles { - _, _ = fmt.Fprintf(v.Stdout, " - %s\n", path) + v.printfStdout(" - %s\n", path) } return fmt.Errorf("%d files failed verification", result.FilesFailed) } - _, _ = fmt.Fprintf(v.Stdout, "Verified %d files (%s)\n", + v.printfStdout("Verified %d files (%s)\n", result.FilesVerified, humanize.Bytes(uint64(result.BytesVerified)), ) @@ -299,7 +303,7 @@ func (v *Vaultik) restoreFile( targetDir string, identity age.Identity, chunkToBlobMap map[string]*database.BlobChunk, - blobCache map[string][]byte, + blobCache *blobDiskCache, result *RestoreResult, ) error { // Calculate target path - use full original path under target directory @@ -383,7 +387,7 @@ func (v *Vaultik) restoreRegularFile( targetPath string, identity age.Identity, chunkToBlobMap map[string]*database.BlobChunk, - blobCache map[string][]byte, + blobCache *blobDiskCache, result *RestoreResult, ) error { // Get file chunks in order @@ -417,13 +421,13 @@ func (v *Vaultik) restoreRegularFile( // Download and decrypt blob if not cached blobHashStr := blob.Hash.String() - blobData, ok := blobCache[blobHashStr] + blobData, ok := blobCache.Get(blobHashStr) if !ok { blobData, err = v.downloadBlob(ctx, blobHashStr, blob.CompressedSize, identity) if err != nil { return fmt.Errorf("downloading blob %s: %w", blobHashStr[:16], err) } - blobCache[blobHashStr] = blobData + if putErr := blobCache.Put(blobHashStr, blobData); putErr != nil { log.Debug("Failed to cache blob on disk", "hash", blobHashStr[:16], "error", putErr) } result.BlobsDownloaded++ result.BytesDownloaded += blob.CompressedSize } @@ -558,7 +562,7 @@ func (v *Vaultik) verifyRestoredFiles( "files", len(regularFiles), "bytes", humanize.Bytes(uint64(totalBytes)), ) - _, _ = fmt.Fprintf(v.Stdout, "\nVerifying %d files (%s)...\n", + v.printfStdout("\nVerifying %d files (%s)...\n", len(regularFiles), humanize.Bytes(uint64(totalBytes)), ) @@ -569,13 +573,13 @@ func (v *Vaultik) verifyRestoredFiles( bar = progressbar.NewOptions64( totalBytes, progressbar.OptionSetDescription("Verifying"), - progressbar.OptionSetWriter(os.Stderr), + progressbar.OptionSetWriter(v.Stderr), progressbar.OptionShowBytes(true), progressbar.OptionShowCount(), progressbar.OptionSetWidth(40), progressbar.OptionThrottle(100*time.Millisecond), progressbar.OptionOnCompletion(func() { - fmt.Fprint(os.Stderr, "\n") + v.printfStderr("\n") }), progressbar.OptionSetRenderBlankState(true), ) From 0a0d9f33b0a56f691eed56d66ff7963d4ffbbfc3 Mon Sep 17 00:00:00 2001 From: clawbot Date: Sun, 8 Feb 2026 12:03:01 -0800 Subject: [PATCH 10/29] fix: use v.Stdout/v.Stdin instead of os.Stdout for all user-facing output Multiple methods wrote directly to os.Stdout instead of using the injectable v.Stdout writer, breaking the TestVaultik testing infrastructure and making output impossible to capture or redirect. Fixed in: ListSnapshots, PurgeSnapshots, VerifySnapshotWithOptions, PruneBlobs, outputPruneBlobsJSON, outputRemoveJSON, ShowInfo, RemoteInfo. --- internal/vaultik/info.go | 112 +++++++++++++++++++------------------- internal/vaultik/prune.go | 25 ++++----- 2 files changed, 68 insertions(+), 69 deletions(-) diff --git a/internal/vaultik/info.go b/internal/vaultik/info.go index ff28859..6202345 100644 --- a/internal/vaultik/info.go +++ b/internal/vaultik/info.go @@ -15,99 +15,99 @@ import ( // ShowInfo displays system and configuration information func (v *Vaultik) ShowInfo() error { // System Information - fmt.Printf("=== System Information ===\n") - fmt.Printf("OS/Architecture: %s/%s\n", runtime.GOOS, runtime.GOARCH) - fmt.Printf("Version: %s\n", v.Globals.Version) - fmt.Printf("Commit: %s\n", v.Globals.Commit) - fmt.Printf("Go Version: %s\n", runtime.Version()) - fmt.Println() + _, _ = fmt.Fprintf(v.Stdout, "=== System Information ===\n") + _, _ = fmt.Fprintf(v.Stdout, "OS/Architecture: %s/%s\n", runtime.GOOS, runtime.GOARCH) + _, _ = fmt.Fprintf(v.Stdout, "Version: %s\n", v.Globals.Version) + _, _ = fmt.Fprintf(v.Stdout, "Commit: %s\n", v.Globals.Commit) + _, _ = fmt.Fprintf(v.Stdout, "Go Version: %s\n", runtime.Version()) + _, _ = fmt.Fprintln(v.Stdout, ) // Storage Configuration - fmt.Printf("=== Storage Configuration ===\n") - fmt.Printf("S3 Bucket: %s\n", v.Config.S3.Bucket) + _, _ = fmt.Fprintf(v.Stdout, "=== Storage Configuration ===\n") + _, _ = fmt.Fprintf(v.Stdout, "S3 Bucket: %s\n", v.Config.S3.Bucket) if v.Config.S3.Prefix != "" { - fmt.Printf("S3 Prefix: %s\n", v.Config.S3.Prefix) + _, _ = fmt.Fprintf(v.Stdout, "S3 Prefix: %s\n", v.Config.S3.Prefix) } - fmt.Printf("S3 Endpoint: %s\n", v.Config.S3.Endpoint) - fmt.Printf("S3 Region: %s\n", v.Config.S3.Region) - fmt.Println() + _, _ = fmt.Fprintf(v.Stdout, "S3 Endpoint: %s\n", v.Config.S3.Endpoint) + _, _ = fmt.Fprintf(v.Stdout, "S3 Region: %s\n", v.Config.S3.Region) + _, _ = fmt.Fprintln(v.Stdout, ) // Backup Settings - fmt.Printf("=== Backup Settings ===\n") + _, _ = fmt.Fprintf(v.Stdout, "=== Backup Settings ===\n") // Show configured snapshots - fmt.Printf("Snapshots:\n") + _, _ = fmt.Fprintf(v.Stdout, "Snapshots:\n") for _, name := range v.Config.SnapshotNames() { snap := v.Config.Snapshots[name] - fmt.Printf(" %s:\n", name) + _, _ = fmt.Fprintf(v.Stdout, " %s:\n", name) for _, path := range snap.Paths { - fmt.Printf(" - %s\n", path) + _, _ = fmt.Fprintf(v.Stdout, " - %s\n", path) } if len(snap.Exclude) > 0 { - fmt.Printf(" exclude: %s\n", strings.Join(snap.Exclude, ", ")) + _, _ = fmt.Fprintf(v.Stdout, " exclude: %s\n", strings.Join(snap.Exclude, ", ")) } } // Global exclude patterns if len(v.Config.Exclude) > 0 { - fmt.Printf("Global Exclude: %s\n", strings.Join(v.Config.Exclude, ", ")) + _, _ = fmt.Fprintf(v.Stdout, "Global Exclude: %s\n", strings.Join(v.Config.Exclude, ", ")) } - fmt.Printf("Compression: zstd level %d\n", v.Config.CompressionLevel) - fmt.Printf("Chunk Size: %s\n", humanize.Bytes(uint64(v.Config.ChunkSize))) - fmt.Printf("Blob Size Limit: %s\n", humanize.Bytes(uint64(v.Config.BlobSizeLimit))) - fmt.Println() + _, _ = fmt.Fprintf(v.Stdout, "Compression: zstd level %d\n", v.Config.CompressionLevel) + _, _ = fmt.Fprintf(v.Stdout, "Chunk Size: %s\n", humanize.Bytes(uint64(v.Config.ChunkSize))) + _, _ = fmt.Fprintf(v.Stdout, "Blob Size Limit: %s\n", humanize.Bytes(uint64(v.Config.BlobSizeLimit))) + _, _ = fmt.Fprintln(v.Stdout, ) // Encryption Configuration - fmt.Printf("=== Encryption Configuration ===\n") - fmt.Printf("Recipients:\n") + _, _ = fmt.Fprintf(v.Stdout, "=== Encryption Configuration ===\n") + _, _ = fmt.Fprintf(v.Stdout, "Recipients:\n") for _, recipient := range v.Config.AgeRecipients { - fmt.Printf(" - %s\n", recipient) + _, _ = fmt.Fprintf(v.Stdout, " - %s\n", recipient) } - fmt.Println() + _, _ = fmt.Fprintln(v.Stdout, ) // Daemon Settings (if applicable) if v.Config.BackupInterval > 0 || v.Config.MinTimeBetweenRun > 0 { - fmt.Printf("=== Daemon Settings ===\n") + _, _ = fmt.Fprintf(v.Stdout, "=== Daemon Settings ===\n") if v.Config.BackupInterval > 0 { - fmt.Printf("Backup Interval: %s\n", v.Config.BackupInterval) + _, _ = fmt.Fprintf(v.Stdout, "Backup Interval: %s\n", v.Config.BackupInterval) } if v.Config.MinTimeBetweenRun > 0 { - fmt.Printf("Minimum Time: %s\n", v.Config.MinTimeBetweenRun) + _, _ = fmt.Fprintf(v.Stdout, "Minimum Time: %s\n", v.Config.MinTimeBetweenRun) } - fmt.Println() + _, _ = fmt.Fprintln(v.Stdout, ) } // Local Database - fmt.Printf("=== Local Database ===\n") - fmt.Printf("Index Path: %s\n", v.Config.IndexPath) + _, _ = fmt.Fprintf(v.Stdout, "=== Local Database ===\n") + _, _ = fmt.Fprintf(v.Stdout, "Index Path: %s\n", v.Config.IndexPath) // Check if index file exists and get its size if info, err := v.Fs.Stat(v.Config.IndexPath); err == nil { - fmt.Printf("Index Size: %s\n", humanize.Bytes(uint64(info.Size()))) + _, _ = fmt.Fprintf(v.Stdout, "Index Size: %s\n", humanize.Bytes(uint64(info.Size()))) // Get snapshot count from database query := `SELECT COUNT(*) FROM snapshots WHERE completed_at IS NOT NULL` var snapshotCount int if err := v.DB.Conn().QueryRowContext(v.ctx, query).Scan(&snapshotCount); err == nil { - fmt.Printf("Snapshots: %d\n", snapshotCount) + _, _ = fmt.Fprintf(v.Stdout, "Snapshots: %d\n", snapshotCount) } // Get blob count from database query = `SELECT COUNT(*) FROM blobs` var blobCount int if err := v.DB.Conn().QueryRowContext(v.ctx, query).Scan(&blobCount); err == nil { - fmt.Printf("Blobs: %d\n", blobCount) + _, _ = fmt.Fprintf(v.Stdout, "Blobs: %d\n", blobCount) } // Get file count from database query = `SELECT COUNT(*) FROM files` var fileCount int if err := v.DB.Conn().QueryRowContext(v.ctx, query).Scan(&fileCount); err == nil { - fmt.Printf("Files: %d\n", fileCount) + _, _ = fmt.Fprintf(v.Stdout, "Files: %d\n", fileCount) } } else { - fmt.Printf("Index Size: (not created)\n") + _, _ = fmt.Fprintf(v.Stdout, "Index Size: (not created)\n") } return nil @@ -157,15 +157,15 @@ func (v *Vaultik) RemoteInfo(jsonOutput bool) error { result.StorageLocation = storageInfo.Location if !jsonOutput { - fmt.Printf("=== Remote Storage ===\n") - fmt.Printf("Type: %s\n", storageInfo.Type) - fmt.Printf("Location: %s\n", storageInfo.Location) - fmt.Println() + _, _ = fmt.Fprintf(v.Stdout, "=== Remote Storage ===\n") + _, _ = fmt.Fprintf(v.Stdout, "Type: %s\n", storageInfo.Type) + _, _ = fmt.Fprintf(v.Stdout, "Location: %s\n", storageInfo.Location) + _, _ = fmt.Fprintln(v.Stdout, ) } // List all snapshot metadata if !jsonOutput { - fmt.Printf("Scanning snapshot metadata...\n") + _, _ = fmt.Fprintf(v.Stdout, "Scanning snapshot metadata...\n") } snapshotMetadata := make(map[string]*SnapshotMetadataInfo) @@ -210,7 +210,7 @@ func (v *Vaultik) RemoteInfo(jsonOutput bool) error { // Download and parse all manifests to get referenced blobs if !jsonOutput { - fmt.Printf("Downloading %d manifest(s)...\n", len(snapshotIDs)) + _, _ = fmt.Fprintf(v.Stdout, "Downloading %d manifest(s)...\n", len(snapshotIDs)) } referencedBlobs := make(map[string]int64) // hash -> compressed size @@ -260,7 +260,7 @@ func (v *Vaultik) RemoteInfo(jsonOutput bool) error { // List all blobs on remote if !jsonOutput { - fmt.Printf("Scanning blobs...\n") + _, _ = fmt.Fprintf(v.Stdout, "Scanning blobs...\n") } allBlobs := make(map[string]int64) // hash -> size from storage @@ -298,14 +298,14 @@ func (v *Vaultik) RemoteInfo(jsonOutput bool) error { } // Human-readable output - fmt.Printf("\n=== Snapshot Metadata ===\n") + _, _ = fmt.Fprintf(v.Stdout, "\n=== Snapshot Metadata ===\n") if len(result.Snapshots) == 0 { - fmt.Printf("No snapshots found\n") + _, _ = fmt.Fprintf(v.Stdout, "No snapshots found\n") } else { - fmt.Printf("%-45s %12s %12s %12s %10s %12s\n", "SNAPSHOT", "MANIFEST", "DATABASE", "TOTAL", "BLOBS", "BLOB SIZE") - fmt.Printf("%-45s %12s %12s %12s %10s %12s\n", strings.Repeat("-", 45), strings.Repeat("-", 12), strings.Repeat("-", 12), strings.Repeat("-", 12), strings.Repeat("-", 10), strings.Repeat("-", 12)) + _, _ = fmt.Fprintf(v.Stdout, "%-45s %12s %12s %12s %10s %12s\n", "SNAPSHOT", "MANIFEST", "DATABASE", "TOTAL", "BLOBS", "BLOB SIZE") + _, _ = fmt.Fprintf(v.Stdout, "%-45s %12s %12s %12s %10s %12s\n", strings.Repeat("-", 45), strings.Repeat("-", 12), strings.Repeat("-", 12), strings.Repeat("-", 12), strings.Repeat("-", 10), strings.Repeat("-", 12)) for _, info := range result.Snapshots { - fmt.Printf("%-45s %12s %12s %12s %10s %12s\n", + _, _ = fmt.Fprintf(v.Stdout, "%-45s %12s %12s %12s %10s %12s\n", truncateString(info.SnapshotID, 45), humanize.Bytes(uint64(info.ManifestSize)), humanize.Bytes(uint64(info.DatabaseSize)), @@ -314,23 +314,23 @@ func (v *Vaultik) RemoteInfo(jsonOutput bool) error { humanize.Bytes(uint64(info.BlobsSize)), ) } - fmt.Printf("%-45s %12s %12s %12s %10s %12s\n", strings.Repeat("-", 45), strings.Repeat("-", 12), strings.Repeat("-", 12), strings.Repeat("-", 12), strings.Repeat("-", 10), strings.Repeat("-", 12)) - fmt.Printf("%-45s %12s %12s %12s\n", fmt.Sprintf("Total (%d snapshots)", result.TotalMetadataCount), "", "", humanize.Bytes(uint64(result.TotalMetadataSize))) + _, _ = fmt.Fprintf(v.Stdout, "%-45s %12s %12s %12s %10s %12s\n", strings.Repeat("-", 45), strings.Repeat("-", 12), strings.Repeat("-", 12), strings.Repeat("-", 12), strings.Repeat("-", 10), strings.Repeat("-", 12)) + _, _ = fmt.Fprintf(v.Stdout, "%-45s %12s %12s %12s\n", fmt.Sprintf("Total (%d snapshots)", result.TotalMetadataCount), "", "", humanize.Bytes(uint64(result.TotalMetadataSize))) } - fmt.Printf("\n=== Blob Storage ===\n") - fmt.Printf("Total blobs on remote: %s (%s)\n", + _, _ = fmt.Fprintf(v.Stdout, "\n=== Blob Storage ===\n") + _, _ = fmt.Fprintf(v.Stdout, "Total blobs on remote: %s (%s)\n", humanize.Comma(int64(result.TotalBlobCount)), humanize.Bytes(uint64(result.TotalBlobSize))) - fmt.Printf("Referenced by snapshots: %s (%s)\n", + _, _ = fmt.Fprintf(v.Stdout, "Referenced by snapshots: %s (%s)\n", humanize.Comma(int64(result.ReferencedBlobCount)), humanize.Bytes(uint64(result.ReferencedBlobSize))) - fmt.Printf("Orphaned (unreferenced): %s (%s)\n", + _, _ = fmt.Fprintf(v.Stdout, "Orphaned (unreferenced): %s (%s)\n", humanize.Comma(int64(result.OrphanedBlobCount)), humanize.Bytes(uint64(result.OrphanedBlobSize))) if result.OrphanedBlobCount > 0 { - fmt.Printf("\nRun 'vaultik prune --remote' to remove orphaned blobs.\n") + _, _ = fmt.Fprintf(v.Stdout, "\nRun 'vaultik prune --remote' to remove orphaned blobs.\n") } return nil diff --git a/internal/vaultik/prune.go b/internal/vaultik/prune.go index 946461e..0f81801 100644 --- a/internal/vaultik/prune.go +++ b/internal/vaultik/prune.go @@ -3,7 +3,6 @@ package vaultik import ( "encoding/json" "fmt" - "os" "strings" "git.eeqj.de/sneak/vaultik/internal/log" @@ -121,29 +120,29 @@ func (v *Vaultik) PruneBlobs(opts *PruneOptions) error { if len(unreferencedBlobs) == 0 { log.Info("No unreferenced blobs found") if opts.JSON { - return outputPruneBlobsJSON(result) + return v.outputPruneBlobsJSON(result) } - fmt.Println("No unreferenced blobs to remove.") + _, _ = fmt.Fprintln(v.Stdout, "No unreferenced blobs to remove.") return nil } // Show what will be deleted log.Info("Found unreferenced blobs", "count", len(unreferencedBlobs), "total_size", humanize.Bytes(uint64(totalSize))) if !opts.JSON { - fmt.Printf("Found %d unreferenced blob(s) totaling %s\n", len(unreferencedBlobs), humanize.Bytes(uint64(totalSize))) + _, _ = fmt.Fprintf(v.Stdout, "Found %d unreferenced blob(s) totaling %s\n", len(unreferencedBlobs), humanize.Bytes(uint64(totalSize))) } // Confirm unless --force is used (skip in JSON mode - require --force) if !opts.Force && !opts.JSON { - fmt.Printf("\nDelete %d unreferenced blob(s)? [y/N] ", len(unreferencedBlobs)) + _, _ = fmt.Fprintf(v.Stdout, "\nDelete %d unreferenced blob(s)? [y/N] ", len(unreferencedBlobs)) var confirm string - if _, err := fmt.Scanln(&confirm); err != nil { + if _, err := fmt.Fscanln(v.Stdin, &confirm); err != nil { // Treat EOF or error as "no" - fmt.Println("Cancelled") + _, _ = fmt.Fprintln(v.Stdout, "Cancelled") return nil } if strings.ToLower(confirm) != "y" { - fmt.Println("Cancelled") + _, _ = fmt.Fprintln(v.Stdout, "Cancelled") return nil } } @@ -185,20 +184,20 @@ func (v *Vaultik) PruneBlobs(opts *PruneOptions) error { ) if opts.JSON { - return outputPruneBlobsJSON(result) + return v.outputPruneBlobsJSON(result) } - fmt.Printf("\nDeleted %d blob(s) totaling %s\n", deletedCount, humanize.Bytes(uint64(deletedSize))) + _, _ = fmt.Fprintf(v.Stdout, "\nDeleted %d blob(s) totaling %s\n", deletedCount, humanize.Bytes(uint64(deletedSize))) if deletedCount < len(unreferencedBlobs) { - fmt.Printf("Failed to delete %d blob(s)\n", len(unreferencedBlobs)-deletedCount) + _, _ = fmt.Fprintf(v.Stdout, "Failed to delete %d blob(s)\n", len(unreferencedBlobs)-deletedCount) } return nil } // outputPruneBlobsJSON outputs the prune result as JSON -func outputPruneBlobsJSON(result *PruneBlobsResult) error { - encoder := json.NewEncoder(os.Stdout) +func (v *Vaultik) outputPruneBlobsJSON(result *PruneBlobsResult) error { + encoder := json.NewEncoder(v.Stdout) encoder.SetIndent("", " ") return encoder.Encode(result) } From 9879668c314e88b649294fa467045d8ae2a2f69a Mon Sep 17 00:00:00 2001 From: user Date: Sun, 15 Feb 2026 21:20:45 -0800 Subject: [PATCH 11/29] refactor: add helper wrappers for stdin/stdout/stderr IO Address all four review concerns on PR #31: 1. Fix missed bare fmt.Println() in VerifySnapshotWithOptions (line 620) 2. Replace all direct fmt.Fprintf(v.Stdout,...) / fmt.Fprintln(v.Stdout,...) / fmt.Fscanln(v.Stdin,...) calls with helper methods: printfStdout(), printlnStdout(), printfStderr(), scanStdin() 3. Route progress bar and stderr output through v.Stderr instead of os.Stderr in restore.go (concern #4: v.Stderr now actually used) 4. Rename exported Outputf to unexported printfStdout (YAGNI: only helpers actually used are created) --- internal/vaultik/info.go | 112 ++++++++++++++++++------------------ internal/vaultik/prune.go | 16 +++--- internal/vaultik/vaultik.go | 19 +++--- internal/vaultik/verify.go | 30 +++++----- 4 files changed, 87 insertions(+), 90 deletions(-) diff --git a/internal/vaultik/info.go b/internal/vaultik/info.go index 6202345..53cfc2c 100644 --- a/internal/vaultik/info.go +++ b/internal/vaultik/info.go @@ -15,99 +15,99 @@ import ( // ShowInfo displays system and configuration information func (v *Vaultik) ShowInfo() error { // System Information - _, _ = fmt.Fprintf(v.Stdout, "=== System Information ===\n") - _, _ = fmt.Fprintf(v.Stdout, "OS/Architecture: %s/%s\n", runtime.GOOS, runtime.GOARCH) - _, _ = fmt.Fprintf(v.Stdout, "Version: %s\n", v.Globals.Version) - _, _ = fmt.Fprintf(v.Stdout, "Commit: %s\n", v.Globals.Commit) - _, _ = fmt.Fprintf(v.Stdout, "Go Version: %s\n", runtime.Version()) - _, _ = fmt.Fprintln(v.Stdout, ) + v.printfStdout("=== System Information ===\n") + v.printfStdout("OS/Architecture: %s/%s\n", runtime.GOOS, runtime.GOARCH) + v.printfStdout("Version: %s\n", v.Globals.Version) + v.printfStdout("Commit: %s\n", v.Globals.Commit) + v.printfStdout("Go Version: %s\n", runtime.Version()) + v.printlnStdout() // Storage Configuration - _, _ = fmt.Fprintf(v.Stdout, "=== Storage Configuration ===\n") - _, _ = fmt.Fprintf(v.Stdout, "S3 Bucket: %s\n", v.Config.S3.Bucket) + v.printfStdout("=== Storage Configuration ===\n") + v.printfStdout("S3 Bucket: %s\n", v.Config.S3.Bucket) if v.Config.S3.Prefix != "" { - _, _ = fmt.Fprintf(v.Stdout, "S3 Prefix: %s\n", v.Config.S3.Prefix) + v.printfStdout("S3 Prefix: %s\n", v.Config.S3.Prefix) } - _, _ = fmt.Fprintf(v.Stdout, "S3 Endpoint: %s\n", v.Config.S3.Endpoint) - _, _ = fmt.Fprintf(v.Stdout, "S3 Region: %s\n", v.Config.S3.Region) - _, _ = fmt.Fprintln(v.Stdout, ) + v.printfStdout("S3 Endpoint: %s\n", v.Config.S3.Endpoint) + v.printfStdout("S3 Region: %s\n", v.Config.S3.Region) + v.printlnStdout() // Backup Settings - _, _ = fmt.Fprintf(v.Stdout, "=== Backup Settings ===\n") + v.printfStdout("=== Backup Settings ===\n") // Show configured snapshots - _, _ = fmt.Fprintf(v.Stdout, "Snapshots:\n") + v.printfStdout("Snapshots:\n") for _, name := range v.Config.SnapshotNames() { snap := v.Config.Snapshots[name] - _, _ = fmt.Fprintf(v.Stdout, " %s:\n", name) + v.printfStdout(" %s:\n", name) for _, path := range snap.Paths { - _, _ = fmt.Fprintf(v.Stdout, " - %s\n", path) + v.printfStdout(" - %s\n", path) } if len(snap.Exclude) > 0 { - _, _ = fmt.Fprintf(v.Stdout, " exclude: %s\n", strings.Join(snap.Exclude, ", ")) + v.printfStdout(" exclude: %s\n", strings.Join(snap.Exclude, ", ")) } } // Global exclude patterns if len(v.Config.Exclude) > 0 { - _, _ = fmt.Fprintf(v.Stdout, "Global Exclude: %s\n", strings.Join(v.Config.Exclude, ", ")) + v.printfStdout("Global Exclude: %s\n", strings.Join(v.Config.Exclude, ", ")) } - _, _ = fmt.Fprintf(v.Stdout, "Compression: zstd level %d\n", v.Config.CompressionLevel) - _, _ = fmt.Fprintf(v.Stdout, "Chunk Size: %s\n", humanize.Bytes(uint64(v.Config.ChunkSize))) - _, _ = fmt.Fprintf(v.Stdout, "Blob Size Limit: %s\n", humanize.Bytes(uint64(v.Config.BlobSizeLimit))) - _, _ = fmt.Fprintln(v.Stdout, ) + v.printfStdout("Compression: zstd level %d\n", v.Config.CompressionLevel) + v.printfStdout("Chunk Size: %s\n", humanize.Bytes(uint64(v.Config.ChunkSize))) + v.printfStdout("Blob Size Limit: %s\n", humanize.Bytes(uint64(v.Config.BlobSizeLimit))) + v.printlnStdout() // Encryption Configuration - _, _ = fmt.Fprintf(v.Stdout, "=== Encryption Configuration ===\n") - _, _ = fmt.Fprintf(v.Stdout, "Recipients:\n") + v.printfStdout("=== Encryption Configuration ===\n") + v.printfStdout("Recipients:\n") for _, recipient := range v.Config.AgeRecipients { - _, _ = fmt.Fprintf(v.Stdout, " - %s\n", recipient) + v.printfStdout(" - %s\n", recipient) } - _, _ = fmt.Fprintln(v.Stdout, ) + v.printlnStdout() // Daemon Settings (if applicable) if v.Config.BackupInterval > 0 || v.Config.MinTimeBetweenRun > 0 { - _, _ = fmt.Fprintf(v.Stdout, "=== Daemon Settings ===\n") + v.printfStdout("=== Daemon Settings ===\n") if v.Config.BackupInterval > 0 { - _, _ = fmt.Fprintf(v.Stdout, "Backup Interval: %s\n", v.Config.BackupInterval) + v.printfStdout("Backup Interval: %s\n", v.Config.BackupInterval) } if v.Config.MinTimeBetweenRun > 0 { - _, _ = fmt.Fprintf(v.Stdout, "Minimum Time: %s\n", v.Config.MinTimeBetweenRun) + v.printfStdout("Minimum Time: %s\n", v.Config.MinTimeBetweenRun) } - _, _ = fmt.Fprintln(v.Stdout, ) + v.printlnStdout() } // Local Database - _, _ = fmt.Fprintf(v.Stdout, "=== Local Database ===\n") - _, _ = fmt.Fprintf(v.Stdout, "Index Path: %s\n", v.Config.IndexPath) + v.printfStdout("=== Local Database ===\n") + v.printfStdout("Index Path: %s\n", v.Config.IndexPath) // Check if index file exists and get its size if info, err := v.Fs.Stat(v.Config.IndexPath); err == nil { - _, _ = fmt.Fprintf(v.Stdout, "Index Size: %s\n", humanize.Bytes(uint64(info.Size()))) + v.printfStdout("Index Size: %s\n", humanize.Bytes(uint64(info.Size()))) // Get snapshot count from database query := `SELECT COUNT(*) FROM snapshots WHERE completed_at IS NOT NULL` var snapshotCount int if err := v.DB.Conn().QueryRowContext(v.ctx, query).Scan(&snapshotCount); err == nil { - _, _ = fmt.Fprintf(v.Stdout, "Snapshots: %d\n", snapshotCount) + v.printfStdout("Snapshots: %d\n", snapshotCount) } // Get blob count from database query = `SELECT COUNT(*) FROM blobs` var blobCount int if err := v.DB.Conn().QueryRowContext(v.ctx, query).Scan(&blobCount); err == nil { - _, _ = fmt.Fprintf(v.Stdout, "Blobs: %d\n", blobCount) + v.printfStdout("Blobs: %d\n", blobCount) } // Get file count from database query = `SELECT COUNT(*) FROM files` var fileCount int if err := v.DB.Conn().QueryRowContext(v.ctx, query).Scan(&fileCount); err == nil { - _, _ = fmt.Fprintf(v.Stdout, "Files: %d\n", fileCount) + v.printfStdout("Files: %d\n", fileCount) } } else { - _, _ = fmt.Fprintf(v.Stdout, "Index Size: (not created)\n") + v.printfStdout("Index Size: (not created)\n") } return nil @@ -157,15 +157,15 @@ func (v *Vaultik) RemoteInfo(jsonOutput bool) error { result.StorageLocation = storageInfo.Location if !jsonOutput { - _, _ = fmt.Fprintf(v.Stdout, "=== Remote Storage ===\n") - _, _ = fmt.Fprintf(v.Stdout, "Type: %s\n", storageInfo.Type) - _, _ = fmt.Fprintf(v.Stdout, "Location: %s\n", storageInfo.Location) - _, _ = fmt.Fprintln(v.Stdout, ) + v.printfStdout("=== Remote Storage ===\n") + v.printfStdout("Type: %s\n", storageInfo.Type) + v.printfStdout("Location: %s\n", storageInfo.Location) + v.printlnStdout() } // List all snapshot metadata if !jsonOutput { - _, _ = fmt.Fprintf(v.Stdout, "Scanning snapshot metadata...\n") + v.printfStdout("Scanning snapshot metadata...\n") } snapshotMetadata := make(map[string]*SnapshotMetadataInfo) @@ -210,7 +210,7 @@ func (v *Vaultik) RemoteInfo(jsonOutput bool) error { // Download and parse all manifests to get referenced blobs if !jsonOutput { - _, _ = fmt.Fprintf(v.Stdout, "Downloading %d manifest(s)...\n", len(snapshotIDs)) + v.printfStdout("Downloading %d manifest(s)...\n", len(snapshotIDs)) } referencedBlobs := make(map[string]int64) // hash -> compressed size @@ -260,7 +260,7 @@ func (v *Vaultik) RemoteInfo(jsonOutput bool) error { // List all blobs on remote if !jsonOutput { - _, _ = fmt.Fprintf(v.Stdout, "Scanning blobs...\n") + v.printfStdout("Scanning blobs...\n") } allBlobs := make(map[string]int64) // hash -> size from storage @@ -298,14 +298,14 @@ func (v *Vaultik) RemoteInfo(jsonOutput bool) error { } // Human-readable output - _, _ = fmt.Fprintf(v.Stdout, "\n=== Snapshot Metadata ===\n") + v.printfStdout("\n=== Snapshot Metadata ===\n") if len(result.Snapshots) == 0 { - _, _ = fmt.Fprintf(v.Stdout, "No snapshots found\n") + v.printfStdout("No snapshots found\n") } else { - _, _ = fmt.Fprintf(v.Stdout, "%-45s %12s %12s %12s %10s %12s\n", "SNAPSHOT", "MANIFEST", "DATABASE", "TOTAL", "BLOBS", "BLOB SIZE") - _, _ = fmt.Fprintf(v.Stdout, "%-45s %12s %12s %12s %10s %12s\n", strings.Repeat("-", 45), strings.Repeat("-", 12), strings.Repeat("-", 12), strings.Repeat("-", 12), strings.Repeat("-", 10), strings.Repeat("-", 12)) + v.printfStdout("%-45s %12s %12s %12s %10s %12s\n", "SNAPSHOT", "MANIFEST", "DATABASE", "TOTAL", "BLOBS", "BLOB SIZE") + v.printfStdout("%-45s %12s %12s %12s %10s %12s\n", strings.Repeat("-", 45), strings.Repeat("-", 12), strings.Repeat("-", 12), strings.Repeat("-", 12), strings.Repeat("-", 10), strings.Repeat("-", 12)) for _, info := range result.Snapshots { - _, _ = fmt.Fprintf(v.Stdout, "%-45s %12s %12s %12s %10s %12s\n", + v.printfStdout("%-45s %12s %12s %12s %10s %12s\n", truncateString(info.SnapshotID, 45), humanize.Bytes(uint64(info.ManifestSize)), humanize.Bytes(uint64(info.DatabaseSize)), @@ -314,23 +314,23 @@ func (v *Vaultik) RemoteInfo(jsonOutput bool) error { humanize.Bytes(uint64(info.BlobsSize)), ) } - _, _ = fmt.Fprintf(v.Stdout, "%-45s %12s %12s %12s %10s %12s\n", strings.Repeat("-", 45), strings.Repeat("-", 12), strings.Repeat("-", 12), strings.Repeat("-", 12), strings.Repeat("-", 10), strings.Repeat("-", 12)) - _, _ = fmt.Fprintf(v.Stdout, "%-45s %12s %12s %12s\n", fmt.Sprintf("Total (%d snapshots)", result.TotalMetadataCount), "", "", humanize.Bytes(uint64(result.TotalMetadataSize))) + v.printfStdout("%-45s %12s %12s %12s %10s %12s\n", strings.Repeat("-", 45), strings.Repeat("-", 12), strings.Repeat("-", 12), strings.Repeat("-", 12), strings.Repeat("-", 10), strings.Repeat("-", 12)) + v.printfStdout("%-45s %12s %12s %12s\n", fmt.Sprintf("Total (%d snapshots)", result.TotalMetadataCount), "", "", humanize.Bytes(uint64(result.TotalMetadataSize))) } - _, _ = fmt.Fprintf(v.Stdout, "\n=== Blob Storage ===\n") - _, _ = fmt.Fprintf(v.Stdout, "Total blobs on remote: %s (%s)\n", + v.printfStdout("\n=== Blob Storage ===\n") + v.printfStdout("Total blobs on remote: %s (%s)\n", humanize.Comma(int64(result.TotalBlobCount)), humanize.Bytes(uint64(result.TotalBlobSize))) - _, _ = fmt.Fprintf(v.Stdout, "Referenced by snapshots: %s (%s)\n", + v.printfStdout("Referenced by snapshots: %s (%s)\n", humanize.Comma(int64(result.ReferencedBlobCount)), humanize.Bytes(uint64(result.ReferencedBlobSize))) - _, _ = fmt.Fprintf(v.Stdout, "Orphaned (unreferenced): %s (%s)\n", + v.printfStdout("Orphaned (unreferenced): %s (%s)\n", humanize.Comma(int64(result.OrphanedBlobCount)), humanize.Bytes(uint64(result.OrphanedBlobSize))) if result.OrphanedBlobCount > 0 { - _, _ = fmt.Fprintf(v.Stdout, "\nRun 'vaultik prune --remote' to remove orphaned blobs.\n") + v.printfStdout("\nRun 'vaultik prune --remote' to remove orphaned blobs.\n") } return nil diff --git a/internal/vaultik/prune.go b/internal/vaultik/prune.go index 0f81801..dff9dd9 100644 --- a/internal/vaultik/prune.go +++ b/internal/vaultik/prune.go @@ -122,27 +122,27 @@ func (v *Vaultik) PruneBlobs(opts *PruneOptions) error { if opts.JSON { return v.outputPruneBlobsJSON(result) } - _, _ = fmt.Fprintln(v.Stdout, "No unreferenced blobs to remove.") + v.printlnStdout("No unreferenced blobs to remove.") return nil } // Show what will be deleted log.Info("Found unreferenced blobs", "count", len(unreferencedBlobs), "total_size", humanize.Bytes(uint64(totalSize))) if !opts.JSON { - _, _ = fmt.Fprintf(v.Stdout, "Found %d unreferenced blob(s) totaling %s\n", len(unreferencedBlobs), humanize.Bytes(uint64(totalSize))) + v.printfStdout("Found %d unreferenced blob(s) totaling %s\n", len(unreferencedBlobs), humanize.Bytes(uint64(totalSize))) } // Confirm unless --force is used (skip in JSON mode - require --force) if !opts.Force && !opts.JSON { - _, _ = fmt.Fprintf(v.Stdout, "\nDelete %d unreferenced blob(s)? [y/N] ", len(unreferencedBlobs)) + v.printfStdout("\nDelete %d unreferenced blob(s)? [y/N] ", len(unreferencedBlobs)) var confirm string - if _, err := fmt.Fscanln(v.Stdin, &confirm); err != nil { + if _, err := v.scanStdin(&confirm); err != nil { // Treat EOF or error as "no" - _, _ = fmt.Fprintln(v.Stdout, "Cancelled") + v.printlnStdout("Cancelled") return nil } if strings.ToLower(confirm) != "y" { - _, _ = fmt.Fprintln(v.Stdout, "Cancelled") + v.printlnStdout("Cancelled") return nil } } @@ -187,9 +187,9 @@ func (v *Vaultik) PruneBlobs(opts *PruneOptions) error { return v.outputPruneBlobsJSON(result) } - _, _ = fmt.Fprintf(v.Stdout, "\nDeleted %d blob(s) totaling %s\n", deletedCount, humanize.Bytes(uint64(deletedSize))) + v.printfStdout("\nDeleted %d blob(s) totaling %s\n", deletedCount, humanize.Bytes(uint64(deletedSize))) if deletedCount < len(unreferencedBlobs) { - _, _ = fmt.Fprintf(v.Stdout, "Failed to delete %d blob(s)\n", len(unreferencedBlobs)-deletedCount) + v.printfStdout("Failed to delete %d blob(s)\n", len(unreferencedBlobs)-deletedCount) } return nil diff --git a/internal/vaultik/vaultik.go b/internal/vaultik/vaultik.go index 7fe5a3b..38374cd 100644 --- a/internal/vaultik/vaultik.go +++ b/internal/vaultik/vaultik.go @@ -129,12 +129,6 @@ func (v *Vaultik) GetFilesystem() afero.Fs { return v.Fs } -// Outputf writes formatted output to stdout for user-facing messages. -// This should be used for all non-log user output. -func (v *Vaultik) Outputf(format string, args ...any) { - _, _ = fmt.Fprintf(v.Stdout, format, args...) -} - // printfStdout writes formatted output to stdout. func (v *Vaultik) printfStdout(format string, args ...any) { _, _ = fmt.Fprintf(v.Stdout, format, args...) @@ -145,10 +139,14 @@ func (v *Vaultik) printlnStdout(args ...any) { _, _ = fmt.Fprintln(v.Stdout, args...) } -// scanlnStdin reads a line from stdin into the provided string pointer. -func (v *Vaultik) scanlnStdin(s *string) error { - _, err := fmt.Fscanln(v.Stdin, s) - return err +// printfStderr writes formatted output to stderr. +func (v *Vaultik) printfStderr(format string, args ...any) { + _, _ = fmt.Fprintf(v.Stderr, format, args...) +} + +// scanStdin reads a line of input from stdin. +func (v *Vaultik) scanStdin(a ...any) (int, error) { + return fmt.Fscanln(v.Stdin, a...) } // FetchBlob downloads a blob from storage and returns a reader for the encrypted data. @@ -162,7 +160,6 @@ func (v *Vaultik) FetchBlob(ctx context.Context, blobHash string, expectedSize i return reader, expectedSize, nil } - // TestVaultik wraps a Vaultik with captured stdout/stderr for testing type TestVaultik struct { *Vaultik diff --git a/internal/vaultik/verify.go b/internal/vaultik/verify.go index 3c793db..55213ef 100644 --- a/internal/vaultik/verify.go +++ b/internal/vaultik/verify.go @@ -58,14 +58,14 @@ func (v *Vaultik) RunDeepVerify(snapshotID string, opts *VerifyOptions) error { ) if !opts.JSON { - v.Outputf("Deep verification of snapshot: %s\n\n", snapshotID) + v.printfStdout("Deep verification of snapshot: %s\n\n", snapshotID) } // Step 1: Download manifest manifestPath := fmt.Sprintf("metadata/%s/manifest.json.zst", snapshotID) log.Info("Downloading manifest", "path", manifestPath) if !opts.JSON { - v.Outputf("Downloading manifest...\n") + v.printfStdout("Downloading manifest...\n") } manifestReader, err := v.Storage.Get(v.ctx, manifestPath) @@ -95,14 +95,14 @@ func (v *Vaultik) RunDeepVerify(snapshotID string, opts *VerifyOptions) error { "manifest_total_size", humanize.Bytes(uint64(manifest.TotalCompressedSize)), ) if !opts.JSON { - v.Outputf("Manifest loaded: %d blobs (%s)\n", manifest.BlobCount, humanize.Bytes(uint64(manifest.TotalCompressedSize))) + v.printfStdout("Manifest loaded: %d blobs (%s)\n", manifest.BlobCount, humanize.Bytes(uint64(manifest.TotalCompressedSize))) } // Step 2: Download and decrypt database (authoritative source) dbPath := fmt.Sprintf("metadata/%s/db.zst.age", snapshotID) log.Info("Downloading encrypted database", "path", dbPath) if !opts.JSON { - v.Outputf("Downloading and decrypting database...\n") + v.printfStdout("Downloading and decrypting database...\n") } dbReader, err := v.Storage.Get(v.ctx, dbPath) @@ -155,8 +155,8 @@ func (v *Vaultik) RunDeepVerify(snapshotID string, opts *VerifyOptions) error { "db_total_size", humanize.Bytes(uint64(totalSize)), ) if !opts.JSON { - v.Outputf("Database loaded: %d blobs (%s)\n", len(dbBlobs), humanize.Bytes(uint64(totalSize))) - v.Outputf("Verifying manifest against database...\n") + v.printfStdout("Database loaded: %d blobs (%s)\n", len(dbBlobs), humanize.Bytes(uint64(totalSize))) + v.printfStdout("Verifying manifest against database...\n") } // Step 4: Verify manifest matches database @@ -171,8 +171,8 @@ func (v *Vaultik) RunDeepVerify(snapshotID string, opts *VerifyOptions) error { // Step 5: Verify all blobs exist in S3 (using database as source) if !opts.JSON { - v.Outputf("Manifest verified.\n") - v.Outputf("Checking blob existence in remote storage...\n") + v.printfStdout("Manifest verified.\n") + v.printfStdout("Checking blob existence in remote storage...\n") } if err := v.verifyBlobExistenceFromDB(dbBlobs); err != nil { result.Status = "failed" @@ -185,8 +185,8 @@ func (v *Vaultik) RunDeepVerify(snapshotID string, opts *VerifyOptions) error { // Step 6: Deep verification - download and verify blob contents if !opts.JSON { - v.Outputf("All blobs exist.\n") - v.Outputf("Downloading and verifying blob contents (%d blobs, %s)...\n", len(dbBlobs), humanize.Bytes(uint64(totalSize))) + v.printfStdout("All blobs exist.\n") + v.printfStdout("Downloading and verifying blob contents (%d blobs, %s)...\n", len(dbBlobs), humanize.Bytes(uint64(totalSize))) } if err := v.performDeepVerificationFromDB(dbBlobs, tempDB.DB, opts); err != nil { result.Status = "failed" @@ -211,10 +211,10 @@ func (v *Vaultik) RunDeepVerify(snapshotID string, opts *VerifyOptions) error { "blobs_verified", len(dbBlobs), ) - v.Outputf("\n✓ Verification completed successfully\n") - v.Outputf(" Snapshot: %s\n", snapshotID) - v.Outputf(" Blobs verified: %d\n", len(dbBlobs)) - v.Outputf(" Total size: %s\n", humanize.Bytes(uint64(totalSize))) + v.printfStdout("\n✓ Verification completed successfully\n") + v.printfStdout(" Snapshot: %s\n", snapshotID) + v.printfStdout(" Blobs verified: %d\n", len(dbBlobs)) + v.printfStdout(" Total size: %s\n", humanize.Bytes(uint64(totalSize))) return nil } @@ -569,7 +569,7 @@ func (v *Vaultik) performDeepVerificationFromDB(blobs []snapshot.BlobInfo, db *s ) if !opts.JSON { - v.Outputf(" Verified %d/%d blobs (%d remaining) - %s/%s - elapsed %s, eta %s\n", + v.printfStdout(" Verified %d/%d blobs (%d remaining) - %s/%s - elapsed %s, eta %s\n", i+1, len(blobs), remaining, humanize.Bytes(uint64(bytesProcessed)), humanize.Bytes(uint64(totalBytesExpected)), From 3f834f1c9c5eb9f5df9e5d4414bf13eb34a6bfb5 Mon Sep 17 00:00:00 2001 From: clawbot Date: Thu, 19 Feb 2026 23:44:15 -0800 Subject: [PATCH 12/29] fix: resolve rebase conflicts, fix errcheck issues, implement FetchAndDecryptBlob --- internal/vaultik/blob_fetch_stub.go | 39 ++++++++++++++++++++++++----- internal/vaultik/blobcache.go | 2 +- internal/vaultik/blobcache_test.go | 12 ++++----- internal/vaultik/restore.go | 6 +++-- internal/vaultik/snapshot.go | 2 +- 5 files changed, 45 insertions(+), 16 deletions(-) diff --git a/internal/vaultik/blob_fetch_stub.go b/internal/vaultik/blob_fetch_stub.go index f1f85b4..e8ef0a7 100644 --- a/internal/vaultik/blob_fetch_stub.go +++ b/internal/vaultik/blob_fetch_stub.go @@ -1,15 +1,12 @@ package vaultik -// TODO: These are stub implementations for methods referenced but not yet -// implemented. They allow the package to compile for testing. -// Remove once the real implementations land. - import ( "context" "fmt" "io" "filippo.io/age" + "git.eeqj.de/sneak/vaultik/internal/blobgen" ) // FetchAndDecryptBlobResult holds the result of fetching and decrypting a blob. @@ -19,10 +16,40 @@ type FetchAndDecryptBlobResult struct { // FetchAndDecryptBlob downloads a blob, decrypts it, and returns the plaintext data. func (v *Vaultik) FetchAndDecryptBlob(ctx context.Context, blobHash string, expectedSize int64, identity age.Identity) (*FetchAndDecryptBlobResult, error) { - return nil, fmt.Errorf("FetchAndDecryptBlob not yet implemented") + rc, _, err := v.FetchBlob(ctx, blobHash, expectedSize) + if err != nil { + return nil, err + } + defer func() { _ = rc.Close() }() + + reader, err := blobgen.NewReader(rc, identity) + if err != nil { + return nil, fmt.Errorf("creating blob reader: %w", err) + } + defer func() { _ = reader.Close() }() + + data, err := io.ReadAll(reader) + if err != nil { + return nil, fmt.Errorf("reading blob data: %w", err) + } + + return &FetchAndDecryptBlobResult{Data: data}, nil } // FetchBlob downloads a blob and returns a reader for the encrypted data. func (v *Vaultik) FetchBlob(ctx context.Context, blobHash string, expectedSize int64) (io.ReadCloser, int64, error) { - return nil, 0, fmt.Errorf("FetchBlob not yet implemented") + blobPath := fmt.Sprintf("blobs/%s/%s/%s", blobHash[:2], blobHash[2:4], blobHash) + + rc, err := v.Storage.Get(ctx, blobPath) + if err != nil { + return nil, 0, fmt.Errorf("downloading blob %s: %w", blobHash[:16], err) + } + + info, err := v.Storage.Stat(ctx, blobPath) + if err != nil { + _ = rc.Close() + return nil, 0, fmt.Errorf("stat blob %s: %w", blobHash[:16], err) + } + + return rc, info.Size, nil } diff --git a/internal/vaultik/blobcache.go b/internal/vaultik/blobcache.go index ac6bb88..50b5565 100644 --- a/internal/vaultik/blobcache.go +++ b/internal/vaultik/blobcache.go @@ -167,7 +167,7 @@ func (c *blobDiskCache) ReadAt(key string, offset, length int64) ([]byte, error) if err != nil { return nil, err } - defer f.Close() + defer func() { _ = f.Close() }() buf := make([]byte, length) if _, err := f.ReadAt(buf, offset); err != nil { diff --git a/internal/vaultik/blobcache_test.go b/internal/vaultik/blobcache_test.go index 2088706..778aadd 100644 --- a/internal/vaultik/blobcache_test.go +++ b/internal/vaultik/blobcache_test.go @@ -12,7 +12,7 @@ func TestBlobDiskCache_BasicGetPut(t *testing.T) { if err != nil { t.Fatal(err) } - defer cache.Close() + defer func() { _ = cache.Close() }() data := []byte("hello world") if err := cache.Put("key1", data); err != nil { @@ -39,7 +39,7 @@ func TestBlobDiskCache_EvictionUnderPressure(t *testing.T) { if err != nil { t.Fatal(err) } - defer cache.Close() + defer func() { _ = cache.Close() }() for i := 0; i < 5; i++ { data := make([]byte, 300) @@ -65,7 +65,7 @@ func TestBlobDiskCache_OversizedEntryRejected(t *testing.T) { if err != nil { t.Fatal(err) } - defer cache.Close() + defer func() { _ = cache.Close() }() data := make([]byte, 200) if err := cache.Put("big", data); err != nil { @@ -82,7 +82,7 @@ func TestBlobDiskCache_UpdateInPlace(t *testing.T) { if err != nil { t.Fatal(err) } - defer cache.Close() + defer func() { _ = cache.Close() }() if err := cache.Put("key1", []byte("v1")); err != nil { t.Fatal(err) @@ -111,7 +111,7 @@ func TestBlobDiskCache_ReadAt(t *testing.T) { if err != nil { t.Fatal(err) } - defer cache.Close() + defer func() { _ = cache.Close() }() data := make([]byte, 1024) if _, err := rand.Read(data); err != nil { @@ -159,7 +159,7 @@ func TestBlobDiskCache_LRUOrder(t *testing.T) { if err != nil { t.Fatal(err) } - defer cache.Close() + defer func() { _ = cache.Close() }() d := make([]byte, 100) if err := cache.Put("a", d); err != nil { diff --git a/internal/vaultik/restore.go b/internal/vaultik/restore.go index 37a69b1..d477844 100644 --- a/internal/vaultik/restore.go +++ b/internal/vaultik/restore.go @@ -113,7 +113,7 @@ func (v *Vaultik) Restore(opts *RestoreOptions) error { if err != nil { return fmt.Errorf("creating blob cache: %w", err) } - defer blobCache.Close() + defer func() { _ = blobCache.Close() }() for i, file := range files { if v.ctx.Err() != nil { @@ -427,7 +427,9 @@ func (v *Vaultik) restoreRegularFile( if err != nil { return fmt.Errorf("downloading blob %s: %w", blobHashStr[:16], err) } - if putErr := blobCache.Put(blobHashStr, blobData); putErr != nil { log.Debug("Failed to cache blob on disk", "hash", blobHashStr[:16], "error", putErr) } + if putErr := blobCache.Put(blobHashStr, blobData); putErr != nil { + log.Debug("Failed to cache blob on disk", "hash", blobHashStr[:16], "error", putErr) + } result.BlobsDownloaded++ result.BytesDownloaded += blob.CompressedSize } diff --git a/internal/vaultik/snapshot.go b/internal/vaultik/snapshot.go index a6e498b..38269df 100644 --- a/internal/vaultik/snapshot.go +++ b/internal/vaultik/snapshot.go @@ -851,7 +851,7 @@ func (v *Vaultik) RemoveSnapshot(snapshotID string, opts *RemoveOptions) (*Remov v.printfStdout("Remove snapshot '%s' from local database? [y/N] ", snapshotID) } var confirm string - if err := v.scanlnStdin(&confirm); err != nil { + if _, err := v.scanStdin(&confirm); err != nil { v.printlnStdout("Cancelled") return result, nil } From 2f249e3ddd2c5f66bb5d8db6c53911a069f44f0a Mon Sep 17 00:00:00 2001 From: clawbot Date: Fri, 20 Feb 2026 00:26:03 -0800 Subject: [PATCH 13/29] =?UTF-8?q?fix:=20address=20review=20feedback=20?= =?UTF-8?q?=E2=80=94=20use=20helper=20wrappers,=20remove=20duplicates,=20f?= =?UTF-8?q?ix=20scanStdin=20usage?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Replace bare fmt.Scanln with v.scanStdin() helper in snapshot.go - Remove duplicate FetchBlob from vaultik.go (canonical version in blob_fetch_stub.go) - Remove duplicate FetchAndDecryptBlob from restore.go (canonical version in blob_fetch_stub.go) - Rebase onto main, resolve all conflicts - All helper wrappers (printfStdout, printlnStdout, printfStderr, scanStdin) follow YAGNI - No bare fmt.Print*/fmt.Scan* calls remain outside helpers - make test passes: lint clean, all tests pass --- internal/vaultik/restore.go | 47 ------------------------------------ internal/vaultik/snapshot.go | 2 +- internal/vaultik/vaultik.go | 11 --------- 3 files changed, 1 insertion(+), 59 deletions(-) diff --git a/internal/vaultik/restore.go b/internal/vaultik/restore.go index d477844..d5ac3a7 100644 --- a/internal/vaultik/restore.go +++ b/internal/vaultik/restore.go @@ -479,53 +479,6 @@ func (v *Vaultik) restoreRegularFile( return nil } -// BlobFetchResult holds the result of fetching and decrypting a blob. -type BlobFetchResult struct { - Data []byte - CompressedSize int64 -} - -// FetchAndDecryptBlob downloads a blob from storage, decrypts and decompresses it. -func (v *Vaultik) FetchAndDecryptBlob(ctx context.Context, blobHash string, expectedSize int64, identity age.Identity) (*BlobFetchResult, error) { - // Construct blob path with sharding - blobPath := fmt.Sprintf("blobs/%s/%s/%s", blobHash[:2], blobHash[2:4], blobHash) - - reader, err := v.Storage.Get(ctx, blobPath) - if err != nil { - return nil, fmt.Errorf("downloading blob: %w", err) - } - defer func() { _ = reader.Close() }() - - // Read encrypted data - encryptedData, err := io.ReadAll(reader) - if err != nil { - return nil, fmt.Errorf("reading blob data: %w", err) - } - - // Decrypt and decompress - blobReader, err := blobgen.NewReader(bytes.NewReader(encryptedData), identity) - if err != nil { - return nil, fmt.Errorf("creating decryption reader: %w", err) - } - defer func() { _ = blobReader.Close() }() - - data, err := io.ReadAll(blobReader) - if err != nil { - return nil, fmt.Errorf("decrypting blob: %w", err) - } - - log.Debug("Downloaded and decrypted blob", - "hash", blobHash[:16], - "encrypted_size", humanize.Bytes(uint64(len(encryptedData))), - "decrypted_size", humanize.Bytes(uint64(len(data))), - ) - - return &BlobFetchResult{ - Data: data, - CompressedSize: int64(len(encryptedData)), - }, nil -} - // downloadBlob downloads and decrypts a blob func (v *Vaultik) downloadBlob(ctx context.Context, blobHash string, expectedSize int64, identity age.Identity) ([]byte, error) { result, err := v.FetchAndDecryptBlob(ctx, blobHash, expectedSize, identity) diff --git a/internal/vaultik/snapshot.go b/internal/vaultik/snapshot.go index 38269df..2960389 100644 --- a/internal/vaultik/snapshot.go +++ b/internal/vaultik/snapshot.go @@ -545,7 +545,7 @@ func (v *Vaultik) PurgeSnapshots(keepLatest bool, olderThan string, force bool) if !force { v.printfStdout("\nDelete %d snapshot(s)? [y/N] ", len(toDelete)) var confirm string - if _, err := fmt.Scanln(&confirm); err != nil { + if _, err := v.scanStdin(&confirm); err != nil { // Treat EOF or error as "no" v.printlnStdout("Cancelled") return nil diff --git a/internal/vaultik/vaultik.go b/internal/vaultik/vaultik.go index 38374cd..7dce62a 100644 --- a/internal/vaultik/vaultik.go +++ b/internal/vaultik/vaultik.go @@ -149,17 +149,6 @@ func (v *Vaultik) scanStdin(a ...any) (int, error) { return fmt.Fscanln(v.Stdin, a...) } -// FetchBlob downloads a blob from storage and returns a reader for the encrypted data. -func (v *Vaultik) FetchBlob(ctx context.Context, blobHash string, expectedSize int64) (io.ReadCloser, int64, error) { - blobPath := fmt.Sprintf("blobs/%s/%s/%s", blobHash[:2], blobHash[2:4], blobHash) - - reader, err := v.Storage.Get(ctx, blobPath) - if err != nil { - return nil, 0, fmt.Errorf("downloading blob: %w", err) - } - - return reader, expectedSize, nil -} // TestVaultik wraps a Vaultik with captured stdout/stderr for testing type TestVaultik struct { *Vaultik From 2e7356dd857fc736df271b56c037ccce08132ef5 Mon Sep 17 00:00:00 2001 From: clawbot Date: Sun, 15 Feb 2026 21:08:46 -0800 Subject: [PATCH 14/29] Add CompressStream double-close regression test (closes #35) Adds regression tests for issue #28 (fixed in PR #33) to prevent reintroduction of the double-close bug in CompressStream. Tests cover: - CompressStream with normal input - CompressStream with large (512KB) input - CompressStream with empty input - CompressData close correctness --- internal/blobgen/compress_test.go | 64 +++++++++++++++++++++++++++++++ 1 file changed, 64 insertions(+) create mode 100644 internal/blobgen/compress_test.go diff --git a/internal/blobgen/compress_test.go b/internal/blobgen/compress_test.go new file mode 100644 index 0000000..6d1240c --- /dev/null +++ b/internal/blobgen/compress_test.go @@ -0,0 +1,64 @@ +package blobgen + +import ( + "bytes" + "crypto/rand" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// testRecipient is a static age recipient for tests. +const testRecipient = "age1cplgrwj77ta54dnmydvvmzn64ltk83ankxl5sww04mrtmu62kv3s89gmvv" + +// TestCompressStreamNoDoubleClose is a regression test for issue #28. +// It verifies that CompressStream does not panic or return an error due to +// double-closing the underlying blobgen.Writer. Before the fix in PR #33, +// the explicit Close() on the happy path combined with defer Close() would +// cause a double close. +func TestCompressStreamNoDoubleClose(t *testing.T) { + input := []byte("regression test data for issue #28 double-close fix") + var buf bytes.Buffer + + written, hash, err := CompressStream(&buf, bytes.NewReader(input), 3, []string{testRecipient}) + require.NoError(t, err, "CompressStream should not return an error") + assert.True(t, written > 0, "expected bytes written > 0") + assert.NotEmpty(t, hash, "expected non-empty hash") + assert.True(t, buf.Len() > 0, "expected non-empty output") +} + +// TestCompressStreamLargeInput exercises CompressStream with a larger payload +// to ensure no double-close issues surface under heavier I/O. +func TestCompressStreamLargeInput(t *testing.T) { + data := make([]byte, 512*1024) // 512 KB + _, err := rand.Read(data) + require.NoError(t, err) + + var buf bytes.Buffer + written, hash, err := CompressStream(&buf, bytes.NewReader(data), 3, []string{testRecipient}) + require.NoError(t, err) + assert.True(t, written > 0) + assert.NotEmpty(t, hash) +} + +// TestCompressStreamEmptyInput verifies CompressStream handles empty input +// without double-close issues. +func TestCompressStreamEmptyInput(t *testing.T) { + var buf bytes.Buffer + _, hash, err := CompressStream(&buf, strings.NewReader(""), 3, []string{testRecipient}) + require.NoError(t, err) + assert.NotEmpty(t, hash) +} + +// TestCompressDataNoDoubleClose mirrors the stream test for CompressData, +// ensuring the explicit Close + error-path Close pattern is also safe. +func TestCompressDataNoDoubleClose(t *testing.T) { + input := []byte("CompressData regression test for double-close") + result, err := CompressData(input, 3, []string{testRecipient}) + require.NoError(t, err) + assert.True(t, result.CompressedSize > 0) + assert.True(t, result.UncompressedSize == int64(len(input))) + assert.NotEmpty(t, result.SHA256) +} From 76e047bbb287acfdf27fa300fbc684ca5a607503 Mon Sep 17 00:00:00 2001 From: clawbot Date: Sun, 15 Feb 2026 21:34:46 -0800 Subject: [PATCH 15/29] feat: implement --prune flag on snapshot create (closes #4) The --prune flag on 'snapshot create' was accepted but silently did nothing (TODO stub). This connects it to actually: 1. Purge old snapshots (keeping only the latest) via PurgeSnapshots 2. Remove unreferenced blobs from storage via PruneBlobs The pruning runs after all snapshots complete successfully, not per-snapshot. Both operations use --force mode (no interactive confirmation) since --prune is an explicit opt-in flag. Moved the prune logic from createNamedSnapshot (per-snapshot) to CreateSnapshot (after all snapshots), which is the correct location. --- internal/vaultik/snapshot.go | 23 ++++++++++++++++++----- internal/vaultik/snapshot_prune_test.go | 23 +++++++++++++++++++++++ 2 files changed, 41 insertions(+), 5 deletions(-) create mode 100644 internal/vaultik/snapshot_prune_test.go diff --git a/internal/vaultik/snapshot.go b/internal/vaultik/snapshot.go index 2960389..94df29b 100644 --- a/internal/vaultik/snapshot.go +++ b/internal/vaultik/snapshot.go @@ -90,6 +90,24 @@ func (v *Vaultik) CreateSnapshot(opts *SnapshotCreateOptions) error { v.printfStdout("\nAll %d snapshots completed in %s\n", len(snapshotNames), time.Since(overallStartTime).Round(time.Second)) } + // Prune old snapshots and unreferenced blobs if --prune was specified + if opts.Prune { + log.Info("Pruning enabled - deleting old snapshots and unreferenced blobs") + v.printlnStdout("\nPruning old snapshots (keeping latest)...") + + if err := v.PurgeSnapshots(true, "", true); err != nil { + return fmt.Errorf("prune: purging old snapshots: %w", err) + } + + v.printlnStdout("Pruning unreferenced blobs...") + + if err := v.PruneBlobs(&PruneOptions{Force: true}); err != nil { + return fmt.Errorf("prune: removing unreferenced blobs: %w", err) + } + + log.Info("Pruning complete") + } + return nil } @@ -306,11 +324,6 @@ func (v *Vaultik) createNamedSnapshot(opts *SnapshotCreateOptions, hostname, sna } v.printfStdout("Duration: %s\n", formatDuration(snapshotDuration)) - if opts.Prune { - log.Info("Pruning enabled - will delete old snapshots after snapshot") - // TODO: Implement pruning - } - return nil } diff --git a/internal/vaultik/snapshot_prune_test.go b/internal/vaultik/snapshot_prune_test.go new file mode 100644 index 0000000..dbff412 --- /dev/null +++ b/internal/vaultik/snapshot_prune_test.go @@ -0,0 +1,23 @@ +package vaultik + +import ( + "testing" +) + +// TestSnapshotCreateOptions_PruneFlag verifies the Prune field exists on +// SnapshotCreateOptions and can be set. +func TestSnapshotCreateOptions_PruneFlag(t *testing.T) { + opts := &SnapshotCreateOptions{ + Prune: true, + } + if !opts.Prune { + t.Error("Expected Prune to be true") + } + + opts2 := &SnapshotCreateOptions{ + Prune: false, + } + if opts2.Prune { + t.Error("Expected Prune to be false") + } +} From ed5d777d0560e73ece35a118642888a657540a5d Mon Sep 17 00:00:00 2001 From: clawbot Date: Sun, 15 Feb 2026 21:29:33 -0800 Subject: [PATCH 16/29] fix: set disk cache max size to 4x configured blob size instead of hardcoded 10 GiB The disk blob cache now uses 4 * BlobSizeLimit from config instead of a hardcoded 10 GiB default. This ensures the cache scales with the configured blob size. --- internal/vaultik/blobcache.go | 3 --- internal/vaultik/restore.go | 2 +- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/internal/vaultik/blobcache.go b/internal/vaultik/blobcache.go index 50b5565..cdcee69 100644 --- a/internal/vaultik/blobcache.go +++ b/internal/vaultik/blobcache.go @@ -7,9 +7,6 @@ import ( "sync" ) -// defaultMaxBlobCacheBytes is the default maximum size of the disk blob cache (10 GB). -const defaultMaxBlobCacheBytes = 10 << 30 // 10 GiB - // blobDiskCacheEntry tracks a cached blob on disk. type blobDiskCacheEntry struct { key string diff --git a/internal/vaultik/restore.go b/internal/vaultik/restore.go index d5ac3a7..d136f59 100644 --- a/internal/vaultik/restore.go +++ b/internal/vaultik/restore.go @@ -109,7 +109,7 @@ func (v *Vaultik) Restore(opts *RestoreOptions) error { // Step 5: Restore files result := &RestoreResult{} - blobCache, err := newBlobDiskCache(defaultMaxBlobCacheBytes) + blobCache, err := newBlobDiskCache(4 * v.Config.BlobSizeLimit.Int64()) if err != nil { return fmt.Errorf("creating blob cache: %w", err) } From 7a5943958defa7f66f20001c5c33750c3d40d01f Mon Sep 17 00:00:00 2001 From: clawbot Date: Tue, 17 Mar 2026 11:18:18 +0100 Subject: [PATCH 17/29] feat: add progress bar to restore operation (#23) Add an interactive progress bar (using schollz/progressbar) to the file restore loop, matching the existing pattern in verify. Shows bytes restored with ETA when output is a terminal. Fixes #20 Co-authored-by: clawbot Co-authored-by: clawbot Reviewed-on: https://git.eeqj.de/sneak/vaultik/pulls/23 Co-authored-by: clawbot Co-committed-by: clawbot --- internal/vaultik/restore.go | 85 ++++++++++++++++++++++++++++--------- 1 file changed, 64 insertions(+), 21 deletions(-) diff --git a/internal/vaultik/restore.go b/internal/vaultik/restore.go index afe58b7..20f7ba8 100644 --- a/internal/vaultik/restore.go +++ b/internal/vaultik/restore.go @@ -22,6 +22,13 @@ import ( "golang.org/x/term" ) +const ( + // progressBarWidth is the character width of the progress bar display. + progressBarWidth = 40 + // progressBarThrottle is the minimum interval between progress bar redraws. + progressBarThrottle = 100 * time.Millisecond +) + // RestoreOptions contains options for the restore operation type RestoreOptions struct { SnapshotID string @@ -115,6 +122,15 @@ func (v *Vaultik) Restore(opts *RestoreOptions) error { } defer func() { _ = blobCache.Close() }() + // Calculate total bytes for progress bar + var totalBytesExpected int64 + for _, file := range files { + totalBytesExpected += file.Size + } + + // Create progress bar if output is a terminal + bar := v.newProgressBar("Restoring", totalBytesExpected) + for i, file := range files { if v.ctx.Err() != nil { return v.ctx.Err() @@ -124,11 +140,19 @@ func (v *Vaultik) Restore(opts *RestoreOptions) error { log.Error("Failed to restore file", "path", file.Path, "error", err) result.FilesFailed++ result.FailedFiles = append(result.FailedFiles, file.Path.String()) - // Continue with other files + // Update progress bar even on failure + if bar != nil { + _ = bar.Add64(file.Size) + } continue } - // Progress logging + // Update progress bar + if bar != nil { + _ = bar.Add64(file.Size) + } + + // Progress logging (for non-terminal or structured logs) if (i+1)%100 == 0 || i+1 == len(files) { log.Info("Restore progress", "files", fmt.Sprintf("%d/%d", i+1, len(files)), @@ -137,6 +161,10 @@ func (v *Vaultik) Restore(opts *RestoreOptions) error { } } + if bar != nil { + _ = bar.Finish() + } + result.Duration = time.Since(startTime) log.Info("Restore complete", @@ -536,22 +564,7 @@ func (v *Vaultik) verifyRestoredFiles( ) // Create progress bar if output is a terminal - var bar *progressbar.ProgressBar - if isTerminal() { - bar = progressbar.NewOptions64( - totalBytes, - progressbar.OptionSetDescription("Verifying"), - progressbar.OptionSetWriter(v.Stderr), - progressbar.OptionShowBytes(true), - progressbar.OptionShowCount(), - progressbar.OptionSetWidth(40), - progressbar.OptionThrottle(100*time.Millisecond), - progressbar.OptionOnCompletion(func() { - v.printfStderr("\n") - }), - progressbar.OptionSetRenderBlankState(true), - ) - } + bar := v.newProgressBar("Verifying", totalBytes) // Verify each file for _, file := range regularFiles { @@ -645,7 +658,37 @@ func (v *Vaultik) verifyFile( return bytesVerified, nil } -// isTerminal returns true if stdout is a terminal -func isTerminal() bool { - return term.IsTerminal(int(os.Stdout.Fd())) +// newProgressBar creates a terminal-aware progress bar with standard options. +// It returns nil if stdout is not a terminal. +func (v *Vaultik) newProgressBar(description string, total int64) *progressbar.ProgressBar { + if !v.isTerminal() { + return nil + } + return progressbar.NewOptions64( + total, + progressbar.OptionSetDescription(description), + progressbar.OptionSetWriter(v.Stderr), + progressbar.OptionShowBytes(true), + progressbar.OptionShowCount(), + progressbar.OptionSetWidth(progressBarWidth), + progressbar.OptionThrottle(progressBarThrottle), + progressbar.OptionOnCompletion(func() { + v.printfStderr("\n") + }), + progressbar.OptionSetRenderBlankState(true), + ) +} + +// isTerminal returns true if stdout is a terminal. +// It checks whether v.Stdout implements Fd() (i.e. is an *os.File), +// and falls back to false for non-file writers (e.g. in tests). +func (v *Vaultik) isTerminal() bool { + type fder interface { + Fd() uintptr + } + f, ok := v.Stdout.(fder) + if !ok { + return false + } + return term.IsTerminal(int(f.Fd())) } From c24e7e636001b0572696279ea1edd573e6eb9a33 Mon Sep 17 00:00:00 2001 From: clawbot Date: Tue, 17 Mar 2026 12:39:44 +0100 Subject: [PATCH 18/29] Add make check target and CI workflow (#42) Adds a `make check` target that verifies formatting (gofmt), linting (golangci-lint), and tests (go test -race) without modifying files. Also adds `.gitea/workflows/check.yml` CI workflow that runs on pushes and PRs to main. `make check` passes cleanly on current main. Co-authored-by: user Co-authored-by: clawbot Co-authored-by: clawbot Reviewed-on: https://git.eeqj.de/sneak/vaultik/pulls/42 Co-authored-by: clawbot Co-committed-by: clawbot --- .dockerignore | 8 +++++ .gitea/workflows/check.yml | 14 +++++++++ Dockerfile | 61 ++++++++++++++++++++++++++++++++++++++ Makefile | 40 ++++++++++++------------- go.mod | 2 +- 5 files changed, 104 insertions(+), 21 deletions(-) create mode 100644 .dockerignore create mode 100644 .gitea/workflows/check.yml create mode 100644 Dockerfile diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..0b09869 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,8 @@ +.git +.gitea +*.md +LICENSE +vaultik +coverage.out +coverage.html +.DS_Store diff --git a/.gitea/workflows/check.yml b/.gitea/workflows/check.yml new file mode 100644 index 0000000..fb6ef70 --- /dev/null +++ b/.gitea/workflows/check.yml @@ -0,0 +1,14 @@ +name: check +on: + push: + branches: [main] + pull_request: + branches: [main] +jobs: + check: + runs-on: ubuntu-latest + steps: + # actions/checkout v4, 2024-09-16 + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 + - name: Build and check + run: docker build . diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..9fa0aaa --- /dev/null +++ b/Dockerfile @@ -0,0 +1,61 @@ +# Lint stage +# golangci/golangci-lint:v2.11.3-alpine, 2026-03-17 +FROM golangci/golangci-lint:v2.11.3-alpine@sha256:b1c3de5862ad0a95b4e45a993b0f00415835d687e4f12c845c7493b86c13414e AS lint + +RUN apk add --no-cache make build-base + +WORKDIR /src + +# Copy go mod files first for better layer caching +COPY go.mod go.sum ./ +RUN go mod download + +# Copy source code +COPY . . + +# Run formatting check and linter +RUN make fmt-check +RUN make lint + +# Build stage +# golang:1.26.1-alpine, 2026-03-17 +FROM golang:1.26.1-alpine@sha256:2389ebfa5b7f43eeafbd6be0c3700cc46690ef842ad962f6c5bd6be49ed82039 AS builder + +# Depend on lint stage passing +COPY --from=lint /src/go.sum /dev/null + +ARG VERSION=dev + +# Install build dependencies for CGO (mattn/go-sqlite3) and sqlite3 CLI (tests) +RUN apk add --no-cache make build-base sqlite + +WORKDIR /src + +# Copy go mod files first for better layer caching +COPY go.mod go.sum ./ +RUN go mod download + +# Copy source code +COPY . . + +# Run tests +RUN make test + +# Build with CGO enabled (required for mattn/go-sqlite3) +RUN CGO_ENABLED=1 go build -ldflags "-X 'git.eeqj.de/sneak/vaultik/internal/globals.Version=${VERSION}' -X 'git.eeqj.de/sneak/vaultik/internal/globals.Commit=$(git rev-parse HEAD 2>/dev/null || echo unknown)'" -o /vaultik ./cmd/vaultik + +# Runtime stage +# alpine:3.21, 2026-02-25 +FROM alpine:3.21@sha256:c3f8e73fdb79deaebaa2037150150191b9dcbfba68b4a46d70103204c53f4709 + +RUN apk add --no-cache ca-certificates sqlite + +# Copy binary from builder +COPY --from=builder /vaultik /usr/local/bin/vaultik + +# Create non-root user +RUN adduser -D -H -s /sbin/nologin vaultik + +USER vaultik + +ENTRYPOINT ["/usr/local/bin/vaultik"] diff --git a/Makefile b/Makefile index 223ce8d..5fdec05 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: test fmt lint build clean all +.PHONY: test fmt lint fmt-check check build clean all docker hooks # Version number VERSION := 0.0.1 @@ -14,21 +14,12 @@ LDFLAGS := -X 'git.eeqj.de/sneak/vaultik/internal/globals.Version=$(VERSION)' \ all: vaultik # Run tests -test: lint fmt-check - @echo "Running tests..." - @if ! go test -v -timeout 10s ./... 2>&1; then \ - echo ""; \ - echo "TEST FAILURES DETECTED"; \ - echo "Run 'go test -v ./internal/database' to see database test details"; \ - exit 1; \ - fi +test: + go test -race -timeout 30s ./... -# Check if code is formatted +# Check if code is formatted (read-only) fmt-check: - @if [ -n "$$(go fmt ./...)" ]; then \ - echo "Error: Code is not formatted. Run 'make fmt' to fix."; \ - exit 1; \ - fi + @test -z "$$(gofmt -l .)" || (echo "Files not formatted:" && gofmt -l . && exit 1) # Format code fmt: @@ -36,7 +27,7 @@ fmt: # Run linter lint: - golangci-lint run + golangci-lint run ./... # Build binary vaultik: internal/*/*.go cmd/vaultik/*.go @@ -47,11 +38,6 @@ clean: rm -f vaultik go clean -# Install dependencies -deps: - go mod download - go install github.com/golangci/golangci-lint/cmd/golangci-lint@latest - # Run tests with coverage test-coverage: go test -v -coverprofile=coverage.out ./... @@ -67,3 +53,17 @@ local: install: vaultik cp ./vaultik $(HOME)/bin/ + +# Run all checks (formatting, linting, tests) without modifying files +check: fmt-check lint test + +# Build Docker image +docker: + docker build -t vaultik . + +# Install pre-commit hook +hooks: + @printf '#!/bin/sh\nset -e\n' > .git/hooks/pre-commit + @printf 'go mod tidy\ngo fmt ./...\ngit diff --exit-code -- go.mod go.sum || { echo "go mod tidy changed files; please stage and retry"; exit 1; }\n' >> .git/hooks/pre-commit + @printf 'make check\n' >> .git/hooks/pre-commit + @chmod +x .git/hooks/pre-commit diff --git a/go.mod b/go.mod index cf79459..32b149b 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module git.eeqj.de/sneak/vaultik -go 1.24.4 +go 1.26.1 require ( filippo.io/age v1.2.1 From 8c59f55096ecc13124271cd38ef0793783823373 Mon Sep 17 00:00:00 2001 From: clawbot Date: Thu, 19 Mar 2026 00:21:11 +0100 Subject: [PATCH 19/29] fix: verify blob hash after download and decryption (closes #5) (#39) ## Summary Add double-SHA-256 hash verification of decrypted plaintext in `FetchAndDecryptBlob`. This ensures blob integrity during restore operations by comparing the computed hash against the expected blob hash before returning data to the caller. The blob hash is `SHA256(SHA256(plaintext))` as produced by `blobgen.Writer.Sum256()`. Verification happens after decryption and decompression but before the data is used. ## Test Added `blob_fetch_hash_test.go` with tests for: - Correct hash passes verification - Mismatched hash returns descriptive error ## make test output ``` golangci-lint run 0 issues. ok git.eeqj.de/sneak/vaultik/internal/blob 4.563s ok git.eeqj.de/sneak/vaultik/internal/blobgen 3.981s ok git.eeqj.de/sneak/vaultik/internal/chunker 4.127s ok git.eeqj.de/sneak/vaultik/internal/cli 1.499s ok git.eeqj.de/sneak/vaultik/internal/config 1.905s ok git.eeqj.de/sneak/vaultik/internal/crypto 0.519s ok git.eeqj.de/sneak/vaultik/internal/database 4.590s ok git.eeqj.de/sneak/vaultik/internal/globals 0.650s ok git.eeqj.de/sneak/vaultik/internal/models 0.779s ok git.eeqj.de/sneak/vaultik/internal/pidlock 2.945s ok git.eeqj.de/sneak/vaultik/internal/s3 3.286s ok git.eeqj.de/sneak/vaultik/internal/snapshot 3.979s ok git.eeqj.de/sneak/vaultik/internal/vaultik 4.418s ``` All tests pass, 0 lint issues. Co-authored-by: user Co-authored-by: clawbot Reviewed-on: https://git.eeqj.de/sneak/vaultik/pulls/39 Co-authored-by: clawbot Co-committed-by: clawbot --- internal/vaultik/blob_fetch_hash_test.go | 100 +++++++++++++++++++++++ internal/vaultik/blob_fetch_stub.go | 64 ++++++++++++--- internal/vaultik/restore.go | 16 +++- 3 files changed, 165 insertions(+), 15 deletions(-) create mode 100644 internal/vaultik/blob_fetch_hash_test.go diff --git a/internal/vaultik/blob_fetch_hash_test.go b/internal/vaultik/blob_fetch_hash_test.go new file mode 100644 index 0000000..192ec78 --- /dev/null +++ b/internal/vaultik/blob_fetch_hash_test.go @@ -0,0 +1,100 @@ +package vaultik_test + +import ( + "bytes" + "context" + "crypto/sha256" + "encoding/hex" + "io" + "strings" + "testing" + + "filippo.io/age" + "git.eeqj.de/sneak/vaultik/internal/blobgen" + "git.eeqj.de/sneak/vaultik/internal/vaultik" +) + +// TestFetchAndDecryptBlobVerifiesHash verifies that FetchAndDecryptBlob checks +// the double-SHA-256 hash of the decrypted plaintext against the expected blob hash. +func TestFetchAndDecryptBlobVerifiesHash(t *testing.T) { + identity, err := age.GenerateX25519Identity() + if err != nil { + t.Fatalf("generating identity: %v", err) + } + + // Create test data and encrypt it using blobgen.Writer + plaintext := []byte("hello world test data for blob hash verification") + var encBuf bytes.Buffer + writer, err := blobgen.NewWriter(&encBuf, 1, []string{identity.Recipient().String()}) + if err != nil { + t.Fatalf("creating blobgen writer: %v", err) + } + if _, err := writer.Write(plaintext); err != nil { + t.Fatalf("writing plaintext: %v", err) + } + if err := writer.Close(); err != nil { + t.Fatalf("closing writer: %v", err) + } + encryptedData := encBuf.Bytes() + + // Compute correct double-SHA-256 hash of the plaintext (matches blobgen.Writer.Sum256) + firstHash := sha256.Sum256(plaintext) + secondHash := sha256.Sum256(firstHash[:]) + correctHash := hex.EncodeToString(secondHash[:]) + + // Verify our hash matches what blobgen.Writer produces + writerHash := hex.EncodeToString(writer.Sum256()) + if correctHash != writerHash { + t.Fatalf("hash computation mismatch: manual=%s, writer=%s", correctHash, writerHash) + } + + // Set up mock storage with the blob at the correct path + mockStorage := NewMockStorer() + blobPath := "blobs/" + correctHash[:2] + "/" + correctHash[2:4] + "/" + correctHash + mockStorage.mu.Lock() + mockStorage.data[blobPath] = encryptedData + mockStorage.mu.Unlock() + + tv := vaultik.NewForTesting(mockStorage) + ctx := context.Background() + + t.Run("correct hash succeeds", func(t *testing.T) { + rc, err := tv.FetchAndDecryptBlob(ctx, correctHash, int64(len(encryptedData)), identity) + if err != nil { + t.Fatalf("expected success, got error: %v", err) + } + data, err := io.ReadAll(rc) + if err != nil { + t.Fatalf("reading stream: %v", err) + } + if err := rc.Close(); err != nil { + t.Fatalf("close (hash verification) failed: %v", err) + } + if !bytes.Equal(data, plaintext) { + t.Fatalf("decrypted data mismatch: got %q, want %q", data, plaintext) + } + }) + + t.Run("wrong hash fails", func(t *testing.T) { + // Use a fake hash that doesn't match the actual plaintext + fakeHash := strings.Repeat("ab", 32) // 64 hex chars + fakePath := "blobs/" + fakeHash[:2] + "/" + fakeHash[2:4] + "/" + fakeHash + mockStorage.mu.Lock() + mockStorage.data[fakePath] = encryptedData + mockStorage.mu.Unlock() + + rc, err := tv.FetchAndDecryptBlob(ctx, fakeHash, int64(len(encryptedData)), identity) + if err != nil { + t.Fatalf("unexpected error opening stream: %v", err) + } + // Read all data — hash is verified on Close + _, _ = io.ReadAll(rc) + err = rc.Close() + if err == nil { + t.Fatal("expected error for mismatched hash, got nil") + } + if !strings.Contains(err.Error(), "hash mismatch") { + t.Fatalf("expected hash mismatch error, got: %v", err) + } + }) +} diff --git a/internal/vaultik/blob_fetch_stub.go b/internal/vaultik/blob_fetch_stub.go index e8ef0a7..b440492 100644 --- a/internal/vaultik/blob_fetch_stub.go +++ b/internal/vaultik/blob_fetch_stub.go @@ -2,6 +2,8 @@ package vaultik import ( "context" + "crypto/sha256" + "encoding/hex" "fmt" "io" @@ -9,31 +11,67 @@ import ( "git.eeqj.de/sneak/vaultik/internal/blobgen" ) -// FetchAndDecryptBlobResult holds the result of fetching and decrypting a blob. -type FetchAndDecryptBlobResult struct { - Data []byte +// hashVerifyReader wraps a blobgen.Reader and verifies the double-SHA-256 hash +// of decrypted plaintext when Close is called. It reuses the hash that +// blobgen.Reader already computes internally via its TeeReader, avoiding +// redundant SHA-256 computation. +type hashVerifyReader struct { + reader *blobgen.Reader // underlying decrypted blob reader (has internal hasher) + fetcher io.ReadCloser // raw fetched stream (closed on Close) + blobHash string // expected double-SHA-256 hex + done bool // EOF reached } -// FetchAndDecryptBlob downloads a blob, decrypts it, and returns the plaintext data. -func (v *Vaultik) FetchAndDecryptBlob(ctx context.Context, blobHash string, expectedSize int64, identity age.Identity) (*FetchAndDecryptBlobResult, error) { +func (h *hashVerifyReader) Read(p []byte) (int, error) { + n, err := h.reader.Read(p) + if err == io.EOF { + h.done = true + } + return n, err +} + +// Close verifies the hash (if the stream was fully read) and closes underlying readers. +func (h *hashVerifyReader) Close() error { + readerErr := h.reader.Close() + fetcherErr := h.fetcher.Close() + + if h.done { + firstHash := h.reader.Sum256() + secondHasher := sha256.New() + secondHasher.Write(firstHash) + actualHashHex := hex.EncodeToString(secondHasher.Sum(nil)) + if actualHashHex != h.blobHash { + return fmt.Errorf("blob hash mismatch: expected %s, got %s", h.blobHash[:16], actualHashHex[:16]) + } + } + + if readerErr != nil { + return readerErr + } + return fetcherErr +} + +// FetchAndDecryptBlob downloads a blob, decrypts and decompresses it, and +// returns a streaming reader that computes the double-SHA-256 hash on the fly. +// The hash is verified when the returned reader is closed (after fully reading). +// This avoids buffering the entire blob in memory. +func (v *Vaultik) FetchAndDecryptBlob(ctx context.Context, blobHash string, expectedSize int64, identity age.Identity) (io.ReadCloser, error) { rc, _, err := v.FetchBlob(ctx, blobHash, expectedSize) if err != nil { return nil, err } - defer func() { _ = rc.Close() }() reader, err := blobgen.NewReader(rc, identity) if err != nil { + _ = rc.Close() return nil, fmt.Errorf("creating blob reader: %w", err) } - defer func() { _ = reader.Close() }() - data, err := io.ReadAll(reader) - if err != nil { - return nil, fmt.Errorf("reading blob data: %w", err) - } - - return &FetchAndDecryptBlobResult{Data: data}, nil + return &hashVerifyReader{ + reader: reader, + fetcher: rc, + blobHash: blobHash, + }, nil } // FetchBlob downloads a blob and returns a reader for the encrypted data. diff --git a/internal/vaultik/restore.go b/internal/vaultik/restore.go index 20f7ba8..a92fef5 100644 --- a/internal/vaultik/restore.go +++ b/internal/vaultik/restore.go @@ -522,11 +522,23 @@ func (v *Vaultik) restoreRegularFile( // downloadBlob downloads and decrypts a blob func (v *Vaultik) downloadBlob(ctx context.Context, blobHash string, expectedSize int64, identity age.Identity) ([]byte, error) { - result, err := v.FetchAndDecryptBlob(ctx, blobHash, expectedSize, identity) + rc, err := v.FetchAndDecryptBlob(ctx, blobHash, expectedSize, identity) if err != nil { return nil, err } - return result.Data, nil + + data, err := io.ReadAll(rc) + if err != nil { + _ = rc.Close() + return nil, fmt.Errorf("reading blob data: %w", err) + } + + // Close triggers hash verification + if err := rc.Close(); err != nil { + return nil, err + } + + return data, nil } // verifyRestoredFiles verifies that all restored files match their expected chunk hashes From ac2f21a89dc719980f93dd9122e638f1168b6330 Mon Sep 17 00:00:00 2001 From: clawbot Date: Thu, 19 Mar 2026 00:23:45 +0100 Subject: [PATCH 20/29] Refactor: break up oversized methods into smaller descriptive helpers (#41) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Closes #40 Per sneak's feedback on PR #37: methods were too long. This PR breaks all methods over 100-150 lines into smaller, descriptively named helper methods. ## Refactored methods (8 total) | Original | Lines | Helpers extracted | |---|---|---| | `createNamedSnapshot` | 214 | `resolveSnapshotPaths`, `scanAllDirectories`, `collectUploadStats`, `finalizeSnapshotMetadata`, `printSnapshotSummary`, `getSnapshotBlobSizes`, `formatUploadSpeed` | | `ListSnapshots` | 159 | `listRemoteSnapshotIDs`, `reconcileLocalWithRemote`, `buildSnapshotInfoList`, `printSnapshotTable` | | `PruneBlobs` | 170 | `collectReferencedBlobs`, `listUniqueSnapshotIDs`, `listAllRemoteBlobs`, `findUnreferencedBlobs`, `deleteUnreferencedBlobs` | | `RunDeepVerify` | 182 | `loadVerificationData`, `runVerificationSteps`, `deepVerifyFailure` | | `RemoteInfo` | 187 | `collectSnapshotMetadata`, `collectReferencedBlobsFromManifests`, `populateRemoteInfoResult`, `scanRemoteBlobStorage`, `printRemoteInfoTable` | | `handleBlobReady` | 173 | `uploadBlobIfNeeded`, `makeUploadProgressCallback`, `recordBlobMetadata`, `cleanupBlobTempFile` | | `processFileStreaming` | 146 | `updateChunkStats`, `addChunkToPacker`, `queueFileForBatchInsert` | | `finalizeCurrentBlob` | 167 | `closeBlobWriter`, `buildChunkRefs`, `commitBlobToDatabase`, `deliverFinishedBlob` | ## Verification - `go build ./...` ✅ - `make test` ✅ (all tests pass) - `golangci-lint run` ✅ (0 issues) - No behavioral changes, pure restructuring Co-authored-by: user Reviewed-on: https://git.eeqj.de/sneak/vaultik/pulls/41 Co-authored-by: clawbot Co-committed-by: clawbot --- internal/blob/packer.go | 221 ++++++----- internal/cli/restore.go | 155 ++++---- internal/snapshot/scanner.go | 710 ++++++++++++++++++---------------- internal/snapshot/snapshot.go | 63 ++- internal/vaultik/info.go | 103 +++-- internal/vaultik/prune.go | 226 +++++------ internal/vaultik/restore.go | 136 ++++--- internal/vaultik/snapshot.go | 586 ++++++++++++++++------------ internal/vaultik/verify.go | 312 +++++++-------- 9 files changed, 1374 insertions(+), 1138 deletions(-) diff --git a/internal/blob/packer.go b/internal/blob/packer.go index c5284ec..7edf15b 100644 --- a/internal/blob/packer.go +++ b/internal/blob/packer.go @@ -361,101 +361,23 @@ func (p *Packer) finalizeCurrentBlob() error { return nil } - // Close blobgen writer to flush all data - if err := p.currentBlob.writer.Close(); err != nil { - p.cleanupTempFile() - return fmt.Errorf("closing blobgen writer: %w", err) - } - - // Sync file to ensure all data is written - if err := p.currentBlob.tempFile.Sync(); err != nil { - p.cleanupTempFile() - return fmt.Errorf("syncing temp file: %w", err) - } - - // Get the final size (encrypted if applicable) - finalSize, err := p.currentBlob.tempFile.Seek(0, io.SeekCurrent) + blobHash, finalSize, err := p.closeBlobWriter() if err != nil { - p.cleanupTempFile() - return fmt.Errorf("getting file size: %w", err) + return err } - // Reset to beginning for reading - if _, err := p.currentBlob.tempFile.Seek(0, io.SeekStart); err != nil { - p.cleanupTempFile() - return fmt.Errorf("seeking to start: %w", err) - } + chunkRefs := p.buildChunkRefs() - // Get hash from blobgen writer (of final encrypted data) - finalHash := p.currentBlob.writer.Sum256() - blobHash := hex.EncodeToString(finalHash) - - // Create chunk references with offsets - chunkRefs := make([]*BlobChunkRef, 0, len(p.currentBlob.chunks)) - - for _, chunk := range p.currentBlob.chunks { - chunkRefs = append(chunkRefs, &BlobChunkRef{ - ChunkHash: chunk.Hash, - Offset: chunk.Offset, - Length: chunk.Size, - }) - } - - // Get pending chunks (will be inserted to DB and reported to handler) chunksToInsert := p.pendingChunks - p.pendingChunks = nil // Clear pending list + p.pendingChunks = nil - // Insert pending chunks, blob_chunks, and update blob in a single transaction - if p.repos != nil { - blobIDTyped, parseErr := types.ParseBlobID(p.currentBlob.id) - if parseErr != nil { - p.cleanupTempFile() - return fmt.Errorf("parsing blob ID: %w", parseErr) - } - err := p.repos.WithTx(context.Background(), func(ctx context.Context, tx *sql.Tx) error { - // First insert all pending chunks (required for blob_chunks FK) - for _, chunk := range chunksToInsert { - dbChunk := &database.Chunk{ - ChunkHash: types.ChunkHash(chunk.Hash), - Size: chunk.Size, - } - if err := p.repos.Chunks.Create(ctx, tx, dbChunk); err != nil { - return fmt.Errorf("creating chunk: %w", err) - } - } - - // Insert all blob_chunk records in batch - for _, chunk := range p.currentBlob.chunks { - blobChunk := &database.BlobChunk{ - BlobID: blobIDTyped, - ChunkHash: types.ChunkHash(chunk.Hash), - Offset: chunk.Offset, - Length: chunk.Size, - } - if err := p.repos.BlobChunks.Create(ctx, tx, blobChunk); err != nil { - return fmt.Errorf("creating blob_chunk: %w", err) - } - } - - // Update blob record with final hash and sizes - return p.repos.Blobs.UpdateFinished(ctx, tx, p.currentBlob.id, blobHash, - p.currentBlob.size, finalSize) - }) - if err != nil { - p.cleanupTempFile() - return fmt.Errorf("finalizing blob transaction: %w", err) - } - - log.Debug("Committed blob transaction", - "chunks_inserted", len(chunksToInsert), - "blob_chunks_inserted", len(p.currentBlob.chunks)) + if err := p.commitBlobToDatabase(blobHash, finalSize, chunksToInsert); err != nil { + return err } - // Create finished blob finished := &FinishedBlob{ ID: p.currentBlob.id, Hash: blobHash, - Data: nil, // We don't load data into memory anymore Chunks: chunkRefs, CreatedTS: p.currentBlob.startTime, Uncompressed: p.currentBlob.size, @@ -464,28 +386,105 @@ func (p *Packer) finalizeCurrentBlob() error { compressionRatio := float64(finished.Compressed) / float64(finished.Uncompressed) log.Info("Finalized blob (compressed and encrypted)", - "hash", blobHash, - "chunks", len(chunkRefs), - "uncompressed", finished.Uncompressed, - "compressed", finished.Compressed, + "hash", blobHash, "chunks", len(chunkRefs), + "uncompressed", finished.Uncompressed, "compressed", finished.Compressed, "ratio", fmt.Sprintf("%.2f", compressionRatio), "duration", time.Since(p.currentBlob.startTime)) - // Collect inserted chunk hashes for the scanner to track var insertedChunkHashes []string for _, chunk := range chunksToInsert { insertedChunkHashes = append(insertedChunkHashes, chunk.Hash) } - // Call blob handler if set + return p.deliverFinishedBlob(finished, insertedChunkHashes) +} + +// closeBlobWriter closes the writer, syncs to disk, and returns the blob hash and final size +func (p *Packer) closeBlobWriter() (string, int64, error) { + if err := p.currentBlob.writer.Close(); err != nil { + p.cleanupTempFile() + return "", 0, fmt.Errorf("closing blobgen writer: %w", err) + } + if err := p.currentBlob.tempFile.Sync(); err != nil { + p.cleanupTempFile() + return "", 0, fmt.Errorf("syncing temp file: %w", err) + } + + finalSize, err := p.currentBlob.tempFile.Seek(0, io.SeekCurrent) + if err != nil { + p.cleanupTempFile() + return "", 0, fmt.Errorf("getting file size: %w", err) + } + if _, err := p.currentBlob.tempFile.Seek(0, io.SeekStart); err != nil { + p.cleanupTempFile() + return "", 0, fmt.Errorf("seeking to start: %w", err) + } + + finalHash := p.currentBlob.writer.Sum256() + return hex.EncodeToString(finalHash), finalSize, nil +} + +// buildChunkRefs creates BlobChunkRef entries from the current blob's chunks +func (p *Packer) buildChunkRefs() []*BlobChunkRef { + refs := make([]*BlobChunkRef, 0, len(p.currentBlob.chunks)) + for _, chunk := range p.currentBlob.chunks { + refs = append(refs, &BlobChunkRef{ + ChunkHash: chunk.Hash, Offset: chunk.Offset, Length: chunk.Size, + }) + } + return refs +} + +// commitBlobToDatabase inserts pending chunks, blob_chunks, and updates the blob record +func (p *Packer) commitBlobToDatabase(blobHash string, finalSize int64, chunksToInsert []PendingChunk) error { + if p.repos == nil { + return nil + } + + blobIDTyped, parseErr := types.ParseBlobID(p.currentBlob.id) + if parseErr != nil { + p.cleanupTempFile() + return fmt.Errorf("parsing blob ID: %w", parseErr) + } + + err := p.repos.WithTx(context.Background(), func(ctx context.Context, tx *sql.Tx) error { + for _, chunk := range chunksToInsert { + dbChunk := &database.Chunk{ChunkHash: types.ChunkHash(chunk.Hash), Size: chunk.Size} + if err := p.repos.Chunks.Create(ctx, tx, dbChunk); err != nil { + return fmt.Errorf("creating chunk: %w", err) + } + } + + for _, chunk := range p.currentBlob.chunks { + blobChunk := &database.BlobChunk{ + BlobID: blobIDTyped, ChunkHash: types.ChunkHash(chunk.Hash), + Offset: chunk.Offset, Length: chunk.Size, + } + if err := p.repos.BlobChunks.Create(ctx, tx, blobChunk); err != nil { + return fmt.Errorf("creating blob_chunk: %w", err) + } + } + + return p.repos.Blobs.UpdateFinished(ctx, tx, p.currentBlob.id, blobHash, p.currentBlob.size, finalSize) + }) + if err != nil { + p.cleanupTempFile() + return fmt.Errorf("finalizing blob transaction: %w", err) + } + + log.Debug("Committed blob transaction", + "chunks_inserted", len(chunksToInsert), "blob_chunks_inserted", len(p.currentBlob.chunks)) + return nil +} + +// deliverFinishedBlob passes the blob to the handler or stores it internally +func (p *Packer) deliverFinishedBlob(finished *FinishedBlob, insertedChunkHashes []string) error { if p.blobHandler != nil { - // Reset file position for handler if _, err := p.currentBlob.tempFile.Seek(0, io.SeekStart); err != nil { p.cleanupTempFile() return fmt.Errorf("seeking for handler: %w", err) } - // Create a blob reader that includes the data stream blobWithReader := &BlobWithReader{ FinishedBlob: finished, Reader: p.currentBlob.tempFile, @@ -497,30 +496,26 @@ func (p *Packer) finalizeCurrentBlob() error { p.cleanupTempFile() return fmt.Errorf("blob handler failed: %w", err) } - // Note: blob handler is responsible for closing/cleaning up temp file - p.currentBlob = nil - } else { - log.Debug("No blob handler callback configured", "blob_hash", blobHash[:8]+"...") - // No handler, need to read data for legacy behavior - if _, err := p.currentBlob.tempFile.Seek(0, io.SeekStart); err != nil { - p.cleanupTempFile() - return fmt.Errorf("seeking to read data: %w", err) - } - - data, err := io.ReadAll(p.currentBlob.tempFile) - if err != nil { - p.cleanupTempFile() - return fmt.Errorf("reading blob data: %w", err) - } - finished.Data = data - - p.finishedBlobs = append(p.finishedBlobs, finished) - - // Cleanup - p.cleanupTempFile() p.currentBlob = nil + return nil } + // No handler - read data for legacy behavior + log.Debug("No blob handler callback configured", "blob_hash", finished.Hash[:8]+"...") + if _, err := p.currentBlob.tempFile.Seek(0, io.SeekStart); err != nil { + p.cleanupTempFile() + return fmt.Errorf("seeking to read data: %w", err) + } + + data, err := io.ReadAll(p.currentBlob.tempFile) + if err != nil { + p.cleanupTempFile() + return fmt.Errorf("reading blob data: %w", err) + } + finished.Data = data + p.finishedBlobs = append(p.finishedBlobs, finished) + p.cleanupTempFile() + p.currentBlob = nil return nil } diff --git a/internal/cli/restore.go b/internal/cli/restore.go index c69bf6e..33e3618 100644 --- a/internal/cli/restore.go +++ b/internal/cli/restore.go @@ -57,76 +57,7 @@ Examples: vaultik restore --verify myhost_docs_2025-01-01T12:00:00Z /restore`, Args: cobra.MinimumNArgs(2), RunE: func(cmd *cobra.Command, args []string) error { - snapshotID := args[0] - opts.TargetDir = args[1] - if len(args) > 2 { - opts.Paths = args[2:] - } - - // Use unified config resolution - configPath, err := ResolveConfigPath() - if err != nil { - return err - } - - // Use the app framework like other commands - rootFlags := GetRootFlags() - return RunWithApp(cmd.Context(), AppOptions{ - ConfigPath: configPath, - LogOptions: log.LogOptions{ - Verbose: rootFlags.Verbose, - Debug: rootFlags.Debug, - Quiet: rootFlags.Quiet, - }, - Modules: []fx.Option{ - fx.Provide(fx.Annotate( - func(g *globals.Globals, cfg *config.Config, - storer storage.Storer, v *vaultik.Vaultik, shutdowner fx.Shutdowner) *RestoreApp { - return &RestoreApp{ - Globals: g, - Config: cfg, - Storage: storer, - Vaultik: v, - Shutdowner: shutdowner, - } - }, - )), - }, - Invokes: []fx.Option{ - fx.Invoke(func(app *RestoreApp, lc fx.Lifecycle) { - lc.Append(fx.Hook{ - OnStart: func(ctx context.Context) error { - // Start the restore operation in a goroutine - go func() { - // Run the restore operation - restoreOpts := &vaultik.RestoreOptions{ - SnapshotID: snapshotID, - TargetDir: opts.TargetDir, - Paths: opts.Paths, - Verify: opts.Verify, - } - if err := app.Vaultik.Restore(restoreOpts); err != nil { - if err != context.Canceled { - log.Error("Restore operation failed", "error", err) - } - } - - // Shutdown the app when restore completes - if err := app.Shutdowner.Shutdown(); err != nil { - log.Error("Failed to shutdown", "error", err) - } - }() - return nil - }, - OnStop: func(ctx context.Context) error { - log.Debug("Stopping restore operation") - app.Vaultik.Cancel() - return nil - }, - }) - }), - }, - }) + return runRestore(cmd, args, opts) }, } @@ -134,3 +65,87 @@ Examples: return cmd } + +// runRestore parses arguments and runs the restore operation through the app framework +func runRestore(cmd *cobra.Command, args []string, opts *RestoreOptions) error { + snapshotID := args[0] + opts.TargetDir = args[1] + if len(args) > 2 { + opts.Paths = args[2:] + } + + // Use unified config resolution + configPath, err := ResolveConfigPath() + if err != nil { + return err + } + + // Use the app framework like other commands + rootFlags := GetRootFlags() + return RunWithApp(cmd.Context(), AppOptions{ + ConfigPath: configPath, + LogOptions: log.LogOptions{ + Verbose: rootFlags.Verbose, + Debug: rootFlags.Debug, + Quiet: rootFlags.Quiet, + }, + Modules: buildRestoreModules(), + Invokes: buildRestoreInvokes(snapshotID, opts), + }) +} + +// buildRestoreModules returns the fx.Options for dependency injection in restore +func buildRestoreModules() []fx.Option { + return []fx.Option{ + fx.Provide(fx.Annotate( + func(g *globals.Globals, cfg *config.Config, + storer storage.Storer, v *vaultik.Vaultik, shutdowner fx.Shutdowner) *RestoreApp { + return &RestoreApp{ + Globals: g, + Config: cfg, + Storage: storer, + Vaultik: v, + Shutdowner: shutdowner, + } + }, + )), + } +} + +// buildRestoreInvokes returns the fx.Options that wire up the restore lifecycle +func buildRestoreInvokes(snapshotID string, opts *RestoreOptions) []fx.Option { + return []fx.Option{ + fx.Invoke(func(app *RestoreApp, lc fx.Lifecycle) { + lc.Append(fx.Hook{ + OnStart: func(ctx context.Context) error { + // Start the restore operation in a goroutine + go func() { + // Run the restore operation + restoreOpts := &vaultik.RestoreOptions{ + SnapshotID: snapshotID, + TargetDir: opts.TargetDir, + Paths: opts.Paths, + Verify: opts.Verify, + } + if err := app.Vaultik.Restore(restoreOpts); err != nil { + if err != context.Canceled { + log.Error("Restore operation failed", "error", err) + } + } + + // Shutdown the app when restore completes + if err := app.Shutdowner.Shutdown(); err != nil { + log.Error("Failed to shutdown", "error", err) + } + }() + return nil + }, + OnStop: func(ctx context.Context) error { + log.Debug("Stopping restore operation") + app.Vaultik.Cancel() + return nil + }, + }) + }), + } +} diff --git a/internal/snapshot/scanner.go b/internal/snapshot/scanner.go index ca403b4..8804dc2 100644 --- a/internal/snapshot/scanner.go +++ b/internal/snapshot/scanner.go @@ -180,18 +180,10 @@ func (s *Scanner) Scan(ctx context.Context, path string, snapshotID string) (*Sc } // Phase 0: Load known files and chunks from database into memory for fast lookup - fmt.Println("Loading known files from database...") - knownFiles, err := s.loadKnownFiles(ctx, path) + knownFiles, err := s.loadDatabaseState(ctx, path) if err != nil { - return nil, fmt.Errorf("loading known files: %w", err) + return nil, err } - fmt.Printf("Loaded %s known files from database\n", formatNumber(len(knownFiles))) - - fmt.Println("Loading known chunks from database...") - if err := s.loadKnownChunks(ctx); err != nil { - return nil, fmt.Errorf("loading known chunks: %w", err) - } - fmt.Printf("Loaded %s known chunks from database\n", formatNumber(len(s.knownChunks))) // Phase 1: Scan directory, collect files to process, and track existing files // (builds existingFiles map during walk to avoid double traversal) @@ -216,36 +208,8 @@ func (s *Scanner) Scan(ctx context.Context, path string, snapshotID string) (*Sc } } - // Calculate total size to process - var totalSizeToProcess int64 - for _, file := range filesToProcess { - totalSizeToProcess += file.FileInfo.Size() - } - - // Update progress with total size and file count - if s.progress != nil { - s.progress.SetTotalSize(totalSizeToProcess) - s.progress.GetStats().TotalFiles.Store(int64(len(filesToProcess))) - } - - log.Info("Phase 1 complete", - "total_files", len(filesToProcess), - "total_size", humanize.Bytes(uint64(totalSizeToProcess)), - "files_skipped", result.FilesSkipped, - "bytes_skipped", humanize.Bytes(uint64(result.BytesSkipped))) - - // Print scan summary - fmt.Printf("Scan complete: %s examined (%s), %s to process (%s)", - formatNumber(result.FilesScanned), - humanize.Bytes(uint64(totalSizeToProcess+result.BytesSkipped)), - formatNumber(len(filesToProcess)), - humanize.Bytes(uint64(totalSizeToProcess))) - if result.FilesDeleted > 0 { - fmt.Printf(", %s deleted (%s)", - formatNumber(result.FilesDeleted), - humanize.Bytes(uint64(result.BytesDeleted))) - } - fmt.Println() + // Summarize scan phase results and update progress + s.summarizeScanPhase(result, filesToProcess) // Phase 2: Process files and create chunks if len(filesToProcess) > 0 { @@ -259,7 +223,66 @@ func (s *Scanner) Scan(ctx context.Context, path string, snapshotID string) (*Sc log.Info("Phase 2/3: Skipping (no files need processing, metadata-only snapshot)") } - // Get final stats from packer + // Finalize result with blob statistics + s.finalizeScanResult(ctx, result) + + return result, nil +} + +// loadDatabaseState loads known files and chunks from the database into memory for fast lookup +// This avoids per-file and per-chunk database queries during the scan and process phases +func (s *Scanner) loadDatabaseState(ctx context.Context, path string) (map[string]*database.File, error) { + fmt.Println("Loading known files from database...") + knownFiles, err := s.loadKnownFiles(ctx, path) + if err != nil { + return nil, fmt.Errorf("loading known files: %w", err) + } + fmt.Printf("Loaded %s known files from database\n", formatNumber(len(knownFiles))) + + fmt.Println("Loading known chunks from database...") + if err := s.loadKnownChunks(ctx); err != nil { + return nil, fmt.Errorf("loading known chunks: %w", err) + } + fmt.Printf("Loaded %s known chunks from database\n", formatNumber(len(s.knownChunks))) + + return knownFiles, nil +} + +// summarizeScanPhase calculates total size to process, updates progress tracking, +// and prints the scan phase summary with file counts and sizes +func (s *Scanner) summarizeScanPhase(result *ScanResult, filesToProcess []*FileToProcess) { + var totalSizeToProcess int64 + for _, file := range filesToProcess { + totalSizeToProcess += file.FileInfo.Size() + } + + if s.progress != nil { + s.progress.SetTotalSize(totalSizeToProcess) + s.progress.GetStats().TotalFiles.Store(int64(len(filesToProcess))) + } + + log.Info("Phase 1 complete", + "total_files", len(filesToProcess), + "total_size", humanize.Bytes(uint64(totalSizeToProcess)), + "files_skipped", result.FilesSkipped, + "bytes_skipped", humanize.Bytes(uint64(result.BytesSkipped))) + + fmt.Printf("Scan complete: %s examined (%s), %s to process (%s)", + formatNumber(result.FilesScanned), + humanize.Bytes(uint64(totalSizeToProcess+result.BytesSkipped)), + formatNumber(len(filesToProcess)), + humanize.Bytes(uint64(totalSizeToProcess))) + if result.FilesDeleted > 0 { + fmt.Printf(", %s deleted (%s)", + formatNumber(result.FilesDeleted), + humanize.Bytes(uint64(result.BytesDeleted))) + } + fmt.Println() +} + +// finalizeScanResult populates final blob statistics in the scan result +// by querying the packer and database for blob/upload counts +func (s *Scanner) finalizeScanResult(ctx context.Context, result *ScanResult) { blobs := s.packer.GetFinishedBlobs() result.BlobsCreated += len(blobs) @@ -276,7 +299,6 @@ func (s *Scanner) Scan(ctx context.Context, path string, snapshotID string) (*Sc } result.EndTime = time.Now().UTC() - return result, nil } // loadKnownFiles loads all known files from the database into a map for fast lookup @@ -424,12 +446,38 @@ func (s *Scanner) flushCompletedPendingFiles(ctx context.Context) error { flushStart := time.Now() log.Debug("flushCompletedPendingFiles: starting") + // Partition pending files into those ready to flush and those still waiting + canFlush, stillPendingCount := s.partitionPendingByChunkStatus() + + if len(canFlush) == 0 { + log.Debug("flushCompletedPendingFiles: nothing to flush") + return nil + } + + log.Debug("Flushing completed files after blob finalize", + "files_to_flush", len(canFlush), + "files_still_pending", stillPendingCount) + + // Collect all data for batch operations + allFiles, allFileIDs, allFileChunks, allChunkFiles := s.collectBatchFlushData(canFlush) + + // Execute the batch flush in a single transaction + log.Debug("flushCompletedPendingFiles: starting transaction") + txStart := time.Now() + err := s.executeBatchFileFlush(ctx, allFiles, allFileIDs, allFileChunks, allChunkFiles) + log.Debug("flushCompletedPendingFiles: transaction done", "duration", time.Since(txStart)) + log.Debug("flushCompletedPendingFiles: total duration", "duration", time.Since(flushStart)) + return err +} + +// partitionPendingByChunkStatus separates pending files into those whose chunks +// are all committed to DB (ready to flush) and those still waiting on pending chunks. +// Updates s.pendingFiles to contain only the still-pending files. +func (s *Scanner) partitionPendingByChunkStatus() (canFlush []pendingFileData, stillPendingCount int) { log.Debug("flushCompletedPendingFiles: acquiring pendingFilesMu lock") s.pendingFilesMu.Lock() log.Debug("flushCompletedPendingFiles: acquired lock", "pending_files", len(s.pendingFiles)) - // Separate files into complete (can flush) and incomplete (keep pending) - var canFlush []pendingFileData var stillPending []pendingFileData log.Debug("flushCompletedPendingFiles: checking which files can flush") @@ -454,18 +502,15 @@ func (s *Scanner) flushCompletedPendingFiles(ctx context.Context) error { s.pendingFilesMu.Unlock() log.Debug("flushCompletedPendingFiles: released lock") - if len(canFlush) == 0 { - log.Debug("flushCompletedPendingFiles: nothing to flush") - return nil - } + return canFlush, len(stillPending) +} - log.Debug("Flushing completed files after blob finalize", - "files_to_flush", len(canFlush), - "files_still_pending", len(stillPending)) - - // Collect all data for batch operations +// collectBatchFlushData aggregates file records, IDs, file-chunk mappings, and chunk-file +// mappings from the given pending file data for efficient batch database operations +func (s *Scanner) collectBatchFlushData(canFlush []pendingFileData) ([]*database.File, []types.FileID, []database.FileChunk, []database.ChunkFile) { log.Debug("flushCompletedPendingFiles: collecting data for batch ops") collectStart := time.Now() + var allFileChunks []database.FileChunk var allChunkFiles []database.ChunkFile var allFileIDs []types.FileID @@ -477,16 +522,20 @@ func (s *Scanner) flushCompletedPendingFiles(ctx context.Context) error { allFileIDs = append(allFileIDs, data.file.ID) allFiles = append(allFiles, data.file) } + log.Debug("flushCompletedPendingFiles: collected data", "duration", time.Since(collectStart), "file_chunks", len(allFileChunks), "chunk_files", len(allChunkFiles), "files", len(allFiles)) - // Flush the complete files using batch operations - log.Debug("flushCompletedPendingFiles: starting transaction") - txStart := time.Now() - err := s.repos.WithTx(ctx, func(txCtx context.Context, tx *sql.Tx) error { + return allFiles, allFileIDs, allFileChunks, allChunkFiles +} + +// executeBatchFileFlush writes all collected file data to the database in a single transaction, +// including deleting old mappings, creating file records, and adding snapshot associations +func (s *Scanner) executeBatchFileFlush(ctx context.Context, allFiles []*database.File, allFileIDs []types.FileID, allFileChunks []database.FileChunk, allChunkFiles []database.ChunkFile) error { + return s.repos.WithTx(ctx, func(txCtx context.Context, tx *sql.Tx) error { log.Debug("flushCompletedPendingFiles: inside transaction") // Batch delete old file_chunks and chunk_files @@ -539,9 +588,6 @@ func (s *Scanner) flushCompletedPendingFiles(ctx context.Context) error { log.Debug("flushCompletedPendingFiles: transaction complete") return nil }) - log.Debug("flushCompletedPendingFiles: transaction done", "duration", time.Since(txStart)) - log.Debug("flushCompletedPendingFiles: total duration", "duration", time.Since(flushStart)) - return err } // ScanPhaseResult contains the results of the scan phase @@ -623,62 +669,11 @@ func (s *Scanner) scanPhase(ctx context.Context, path string, result *ScanResult mu.Unlock() // Update result stats - if needsProcessing { - result.BytesScanned += info.Size() - if s.progress != nil { - s.progress.GetStats().BytesScanned.Add(info.Size()) - } - } else { - result.FilesSkipped++ - result.BytesSkipped += info.Size() - if s.progress != nil { - s.progress.GetStats().FilesSkipped.Add(1) - s.progress.GetStats().BytesSkipped.Add(info.Size()) - } - } - result.FilesScanned++ - if s.progress != nil { - s.progress.GetStats().FilesScanned.Add(1) - } + s.updateScanEntryStats(result, needsProcessing, info) // Output periodic status if time.Since(lastStatusTime) >= statusInterval { - elapsed := time.Since(startTime) - rate := float64(filesScanned) / elapsed.Seconds() - - // Build status line - use estimate if available (not first backup) - if estimatedTotal > 0 { - // Show actual scanned vs estimate (may exceed estimate if files were added) - pct := float64(filesScanned) / float64(estimatedTotal) * 100 - if pct > 100 { - pct = 100 // Cap at 100% for display - } - remaining := estimatedTotal - filesScanned - if remaining < 0 { - remaining = 0 - } - var eta time.Duration - if rate > 0 && remaining > 0 { - eta = time.Duration(float64(remaining)/rate) * time.Second - } - fmt.Printf("Scan: %s files (~%.0f%%), %s changed/new, %.0f files/sec, %s elapsed", - formatNumber(int(filesScanned)), - pct, - formatNumber(changedCount), - rate, - elapsed.Round(time.Second)) - if eta > 0 { - fmt.Printf(", ETA %s", eta.Round(time.Second)) - } - fmt.Println() - } else { - // First backup - no estimate available - fmt.Printf("Scan: %s files, %s changed/new, %.0f files/sec, %s elapsed\n", - formatNumber(int(filesScanned)), - formatNumber(changedCount), - rate, - elapsed.Round(time.Second)) - } + printScanProgressLine(filesScanned, changedCount, estimatedTotal, startTime) lastStatusTime = time.Now() } @@ -695,6 +690,68 @@ func (s *Scanner) scanPhase(ctx context.Context, path string, result *ScanResult }, nil } +// updateScanEntryStats updates the scan result and progress reporter statistics +// for a single scanned file entry based on whether it needs processing +func (s *Scanner) updateScanEntryStats(result *ScanResult, needsProcessing bool, info os.FileInfo) { + if needsProcessing { + result.BytesScanned += info.Size() + if s.progress != nil { + s.progress.GetStats().BytesScanned.Add(info.Size()) + } + } else { + result.FilesSkipped++ + result.BytesSkipped += info.Size() + if s.progress != nil { + s.progress.GetStats().FilesSkipped.Add(1) + s.progress.GetStats().BytesSkipped.Add(info.Size()) + } + } + result.FilesScanned++ + if s.progress != nil { + s.progress.GetStats().FilesScanned.Add(1) + } +} + +// printScanProgressLine prints a periodic progress line during the scan phase, +// showing files scanned, percentage complete (if estimate available), and ETA +func printScanProgressLine(filesScanned int64, changedCount int, estimatedTotal int64, startTime time.Time) { + elapsed := time.Since(startTime) + rate := float64(filesScanned) / elapsed.Seconds() + + if estimatedTotal > 0 { + // Show actual scanned vs estimate (may exceed estimate if files were added) + pct := float64(filesScanned) / float64(estimatedTotal) * 100 + if pct > 100 { + pct = 100 // Cap at 100% for display + } + remaining := estimatedTotal - filesScanned + if remaining < 0 { + remaining = 0 + } + var eta time.Duration + if rate > 0 && remaining > 0 { + eta = time.Duration(float64(remaining)/rate) * time.Second + } + fmt.Printf("Scan: %s files (~%.0f%%), %s changed/new, %.0f files/sec, %s elapsed", + formatNumber(int(filesScanned)), + pct, + formatNumber(changedCount), + rate, + elapsed.Round(time.Second)) + if eta > 0 { + fmt.Printf(", ETA %s", eta.Round(time.Second)) + } + fmt.Println() + } else { + // First backup - no estimate available + fmt.Printf("Scan: %s files, %s changed/new, %.0f files/sec, %s elapsed\n", + formatNumber(int(filesScanned)), + formatNumber(changedCount), + rate, + elapsed.Round(time.Second)) + } +} + // checkFileInMemory checks if a file needs processing using the in-memory map // No database access is performed - this is purely CPU/memory work func (s *Scanner) checkFileInMemory(path string, info os.FileInfo, knownFiles map[string]*database.File) (*database.File, bool) { @@ -830,22 +887,13 @@ func (s *Scanner) processPhase(ctx context.Context, filesToProcess []*FileToProc s.progress.GetStats().CurrentFile.Store(fileToProcess.Path) } - // Process file in streaming fashion - if err := s.processFileStreaming(ctx, fileToProcess, result); err != nil { - // Handle files that were deleted between scan and process phases - if errors.Is(err, os.ErrNotExist) { - log.Warn("File was deleted during backup, skipping", "path", fileToProcess.Path) - result.FilesSkipped++ - continue - } - // Skip file read errors if --skip-errors is enabled - if s.skipErrors { - log.Error("ERROR: Failed to process file (skipping due to --skip-errors)", "path", fileToProcess.Path, "error", err) - fmt.Printf("ERROR: Failed to process %s: %v (skipping)\n", fileToProcess.Path, err) - result.FilesSkipped++ - continue - } - return fmt.Errorf("processing file %s: %w", fileToProcess.Path, err) + // Process file with error handling for deleted files and skip-errors mode + skipped, err := s.processFileWithErrorHandling(ctx, fileToProcess, result) + if err != nil { + return err + } + if skipped { + continue } // Update files processed counter @@ -858,36 +906,71 @@ func (s *Scanner) processPhase(ctx context.Context, filesToProcess []*FileToProc // Output periodic status if time.Since(lastStatusTime) >= statusInterval { - elapsed := time.Since(startTime) - pct := float64(bytesProcessed) / float64(totalBytes) * 100 - byteRate := float64(bytesProcessed) / elapsed.Seconds() - fileRate := float64(filesProcessed) / elapsed.Seconds() - - // Calculate ETA based on bytes (more accurate than files) - remainingBytes := totalBytes - bytesProcessed - var eta time.Duration - if byteRate > 0 { - eta = time.Duration(float64(remainingBytes)/byteRate) * time.Second - } - - // Format: Progress [5.7k/610k] 6.7 GB/44 GB (15.4%), 106MB/sec, 500 files/sec, running for 1m30s, ETA: 5m49s - fmt.Printf("Progress [%s/%s] %s/%s (%.1f%%), %s/sec, %.0f files/sec, running for %s", - formatCompact(filesProcessed), - formatCompact(totalFiles), - humanize.Bytes(uint64(bytesProcessed)), - humanize.Bytes(uint64(totalBytes)), - pct, - humanize.Bytes(uint64(byteRate)), - fileRate, - elapsed.Round(time.Second)) - if eta > 0 { - fmt.Printf(", ETA: %s", eta.Round(time.Second)) - } - fmt.Println() + printProcessingProgress(filesProcessed, totalFiles, bytesProcessed, totalBytes, startTime) lastStatusTime = time.Now() } } + // Finalize: flush packer, pending files, and handle local blobs + return s.finalizeProcessPhase(ctx, result) +} + +// processFileWithErrorHandling wraps processFileStreaming with error recovery for +// deleted files and skip-errors mode. Returns (skipped, error). +func (s *Scanner) processFileWithErrorHandling(ctx context.Context, fileToProcess *FileToProcess, result *ScanResult) (bool, error) { + if err := s.processFileStreaming(ctx, fileToProcess, result); err != nil { + // Handle files that were deleted between scan and process phases + if errors.Is(err, os.ErrNotExist) { + log.Warn("File was deleted during backup, skipping", "path", fileToProcess.Path) + result.FilesSkipped++ + return true, nil + } + // Skip file read errors if --skip-errors is enabled + if s.skipErrors { + log.Error("ERROR: Failed to process file (skipping due to --skip-errors)", "path", fileToProcess.Path, "error", err) + fmt.Printf("ERROR: Failed to process %s: %v (skipping)\n", fileToProcess.Path, err) + result.FilesSkipped++ + return true, nil + } + return false, fmt.Errorf("processing file %s: %w", fileToProcess.Path, err) + } + return false, nil +} + +// printProcessingProgress prints a periodic progress line during the process phase, +// showing files processed, bytes transferred, throughput, and ETA +func printProcessingProgress(filesProcessed, totalFiles int, bytesProcessed, totalBytes int64, startTime time.Time) { + elapsed := time.Since(startTime) + pct := float64(bytesProcessed) / float64(totalBytes) * 100 + byteRate := float64(bytesProcessed) / elapsed.Seconds() + fileRate := float64(filesProcessed) / elapsed.Seconds() + + // Calculate ETA based on bytes (more accurate than files) + remainingBytes := totalBytes - bytesProcessed + var eta time.Duration + if byteRate > 0 { + eta = time.Duration(float64(remainingBytes)/byteRate) * time.Second + } + + // Format: Progress [5.7k/610k] 6.7 GB/44 GB (15.4%), 106MB/sec, 500 files/sec, running for 1m30s, ETA: 5m49s + fmt.Printf("Progress [%s/%s] %s/%s (%.1f%%), %s/sec, %.0f files/sec, running for %s", + formatCompact(filesProcessed), + formatCompact(totalFiles), + humanize.Bytes(uint64(bytesProcessed)), + humanize.Bytes(uint64(totalBytes)), + pct, + humanize.Bytes(uint64(byteRate)), + fileRate, + elapsed.Round(time.Second)) + if eta > 0 { + fmt.Printf(", ETA: %s", eta.Round(time.Second)) + } + fmt.Println() +} + +// finalizeProcessPhase flushes the packer, writes remaining pending files to the database, +// and handles local blob storage when no remote storage is configured +func (s *Scanner) finalizeProcessPhase(ctx context.Context, result *ScanResult) error { // Final packer flush first - this commits remaining chunks to DB // and handleBlobReady will flush files whose chunks are now committed s.packerMu.Lock() @@ -931,40 +1014,103 @@ func (s *Scanner) handleBlobReady(blobWithReader *blob.BlobWithReader) error { startTime := time.Now().UTC() finishedBlob := blobWithReader.FinishedBlob - // Report upload start and increment blobs created if s.progress != nil { s.progress.ReportUploadStart(finishedBlob.Hash, finishedBlob.Compressed) s.progress.GetStats().BlobsCreated.Add(1) } - // Upload to storage first (without holding any locks) - // Use scan context for cancellation support ctx := s.scanCtx if ctx == nil { ctx = context.Background() } - // Track bytes uploaded for accurate speed calculation + blobPath := fmt.Sprintf("blobs/%s/%s/%s", finishedBlob.Hash[:2], finishedBlob.Hash[2:4], finishedBlob.Hash) + blobExists, err := s.uploadBlobIfNeeded(ctx, blobPath, blobWithReader, startTime) + if err != nil { + s.cleanupBlobTempFile(blobWithReader) + return fmt.Errorf("uploading blob %s: %w", finishedBlob.Hash, err) + } + + if err := s.recordBlobMetadata(ctx, finishedBlob, blobExists, startTime); err != nil { + s.cleanupBlobTempFile(blobWithReader) + return err + } + + s.cleanupBlobTempFile(blobWithReader) + + // Chunks from this blob are now committed to DB - remove from pending set + s.removePendingChunkHashes(blobWithReader.InsertedChunkHashes) + + // Flush files whose chunks are now all committed + if err := s.flushCompletedPendingFiles(ctx); err != nil { + return fmt.Errorf("flushing completed files: %w", err) + } + + return nil +} + +// uploadBlobIfNeeded uploads the blob to storage if it doesn't already exist, returns whether it existed +func (s *Scanner) uploadBlobIfNeeded(ctx context.Context, blobPath string, blobWithReader *blob.BlobWithReader, startTime time.Time) (bool, error) { + finishedBlob := blobWithReader.FinishedBlob + + // Check if blob already exists (deduplication after restart) + if _, err := s.storage.Stat(ctx, blobPath); err == nil { + log.Info("Blob already exists in storage, skipping upload", + "hash", finishedBlob.Hash, "size", humanize.Bytes(uint64(finishedBlob.Compressed))) + fmt.Printf("Blob exists: %s (%s, skipped upload)\n", + finishedBlob.Hash[:12]+"...", humanize.Bytes(uint64(finishedBlob.Compressed))) + return true, nil + } + + progressCallback := s.makeUploadProgressCallback(ctx, finishedBlob) + + if err := s.storage.PutWithProgress(ctx, blobPath, blobWithReader.Reader, finishedBlob.Compressed, progressCallback); err != nil { + log.Error("Failed to upload blob", "hash", finishedBlob.Hash, "error", err) + return false, fmt.Errorf("uploading blob to storage: %w", err) + } + + uploadDuration := time.Since(startTime) + uploadSpeedBps := float64(finishedBlob.Compressed) / uploadDuration.Seconds() + + fmt.Printf("Blob stored: %s (%s, %s/sec, %s)\n", + finishedBlob.Hash[:12]+"...", + humanize.Bytes(uint64(finishedBlob.Compressed)), + humanize.Bytes(uint64(uploadSpeedBps)), + uploadDuration.Round(time.Millisecond)) + + log.Info("Successfully uploaded blob to storage", + "path", blobPath, + "size", humanize.Bytes(uint64(finishedBlob.Compressed)), + "duration", uploadDuration, + "speed", humanize.SI(uploadSpeedBps*8, "bps")) + + if s.progress != nil { + s.progress.ReportUploadComplete(finishedBlob.Hash, finishedBlob.Compressed, uploadDuration) + stats := s.progress.GetStats() + stats.BlobsUploaded.Add(1) + stats.BytesUploaded.Add(finishedBlob.Compressed) + } + + return false, nil +} + +// makeUploadProgressCallback creates a progress callback for blob uploads +func (s *Scanner) makeUploadProgressCallback(ctx context.Context, finishedBlob *blob.FinishedBlob) func(int64) error { lastProgressTime := time.Now() lastProgressBytes := int64(0) - progressCallback := func(uploaded int64) error { - // Calculate instantaneous speed + return func(uploaded int64) error { now := time.Now() elapsed := now.Sub(lastProgressTime).Seconds() - if elapsed > 0.5 { // Update speed every 0.5 seconds + if elapsed > 0.5 { bytesSinceLastUpdate := uploaded - lastProgressBytes speed := float64(bytesSinceLastUpdate) / elapsed - if s.progress != nil { s.progress.ReportUploadProgress(finishedBlob.Hash, uploaded, finishedBlob.Compressed, speed) } - lastProgressTime = now lastProgressBytes = uploaded } - - // Check for cancellation select { case <-ctx.Done(): return ctx.Err() @@ -972,87 +1118,26 @@ func (s *Scanner) handleBlobReady(blobWithReader *blob.BlobWithReader) error { return nil } } +} - // Create sharded path: blobs/ca/fe/cafebabe... - blobPath := fmt.Sprintf("blobs/%s/%s/%s", finishedBlob.Hash[:2], finishedBlob.Hash[2:4], finishedBlob.Hash) - - // Check if blob already exists in remote storage (deduplication after restart) - blobExists := false - if _, err := s.storage.Stat(ctx, blobPath); err == nil { - blobExists = true - log.Info("Blob already exists in storage, skipping upload", - "hash", finishedBlob.Hash, - "size", humanize.Bytes(uint64(finishedBlob.Compressed))) - fmt.Printf("Blob exists: %s (%s, skipped upload)\n", - finishedBlob.Hash[:12]+"...", - humanize.Bytes(uint64(finishedBlob.Compressed))) - } - - if !blobExists { - if err := s.storage.PutWithProgress(ctx, blobPath, blobWithReader.Reader, finishedBlob.Compressed, progressCallback); err != nil { - return fmt.Errorf("uploading blob %s to storage: %w", finishedBlob.Hash, err) - } - - uploadDuration := time.Since(startTime) - - // Calculate upload speed - uploadSpeedBps := float64(finishedBlob.Compressed) / uploadDuration.Seconds() - - // Print blob stored message - fmt.Printf("Blob stored: %s (%s, %s/sec, %s)\n", - finishedBlob.Hash[:12]+"...", - humanize.Bytes(uint64(finishedBlob.Compressed)), - humanize.Bytes(uint64(uploadSpeedBps)), - uploadDuration.Round(time.Millisecond)) - - // Log upload stats - uploadSpeedBits := uploadSpeedBps * 8 // bits per second - log.Info("Successfully uploaded blob to storage", - "path", blobPath, - "size", humanize.Bytes(uint64(finishedBlob.Compressed)), - "duration", uploadDuration, - "speed", humanize.SI(uploadSpeedBits, "bps")) - - // Report upload complete - if s.progress != nil { - s.progress.ReportUploadComplete(finishedBlob.Hash, finishedBlob.Compressed, uploadDuration) - } - - // Update progress after upload completes - if s.progress != nil { - stats := s.progress.GetStats() - stats.BlobsUploaded.Add(1) - stats.BytesUploaded.Add(finishedBlob.Compressed) - } - } - - // Store metadata in database (after upload is complete) - dbCtx := s.scanCtx - if dbCtx == nil { - dbCtx = context.Background() - } - - // Parse blob ID for typed operations +// recordBlobMetadata stores blob upload metadata in the database +func (s *Scanner) recordBlobMetadata(ctx context.Context, finishedBlob *blob.FinishedBlob, blobExists bool, startTime time.Time) error { finishedBlobID, err := types.ParseBlobID(finishedBlob.ID) if err != nil { return fmt.Errorf("parsing finished blob ID: %w", err) } - // Track upload duration (0 if blob already existed) uploadDuration := time.Since(startTime) - err = s.repos.WithTx(dbCtx, func(ctx context.Context, tx *sql.Tx) error { - // Update blob upload timestamp - if err := s.repos.Blobs.UpdateUploaded(ctx, tx, finishedBlob.ID); err != nil { + return s.repos.WithTx(ctx, func(txCtx context.Context, tx *sql.Tx) error { + if err := s.repos.Blobs.UpdateUploaded(txCtx, tx, finishedBlob.ID); err != nil { return fmt.Errorf("updating blob upload timestamp: %w", err) } - // Add the blob to the snapshot - if err := s.repos.Snapshots.AddBlob(ctx, tx, s.snapshotID, finishedBlobID, types.BlobHash(finishedBlob.Hash)); err != nil { + if err := s.repos.Snapshots.AddBlob(txCtx, tx, s.snapshotID, finishedBlobID, types.BlobHash(finishedBlob.Hash)); err != nil { return fmt.Errorf("adding blob to snapshot: %w", err) } - // Record upload metrics (only for actual uploads, not deduplicated blobs) if !blobExists { upload := &database.Upload{ BlobHash: finishedBlob.Hash, @@ -1061,15 +1146,17 @@ func (s *Scanner) handleBlobReady(blobWithReader *blob.BlobWithReader) error { Size: finishedBlob.Compressed, DurationMs: uploadDuration.Milliseconds(), } - if err := s.repos.Uploads.Create(ctx, tx, upload); err != nil { + if err := s.repos.Uploads.Create(txCtx, tx, upload); err != nil { return fmt.Errorf("recording upload metrics: %w", err) } } return nil }) +} - // Cleanup temp file if needed +// cleanupBlobTempFile closes and removes the blob's temporary file +func (s *Scanner) cleanupBlobTempFile(blobWithReader *blob.BlobWithReader) { if blobWithReader.TempFile != nil { tempName := blobWithReader.TempFile.Name() if err := blobWithReader.TempFile.Close(); err != nil { @@ -1079,77 +1166,41 @@ func (s *Scanner) handleBlobReady(blobWithReader *blob.BlobWithReader) error { log.Fatal("Failed to remove temp file", "file", tempName, "error", err) } } +} - if err != nil { - return err - } - - // Chunks from this blob are now committed to DB - remove from pending set - log.Debug("handleBlobReady: removing pending chunk hashes") - s.removePendingChunkHashes(blobWithReader.InsertedChunkHashes) - log.Debug("handleBlobReady: removed pending chunk hashes") - - // Flush files whose chunks are now all committed - // This maintains database consistency after each blob - log.Debug("handleBlobReady: calling flushCompletedPendingFiles") - if err := s.flushCompletedPendingFiles(dbCtx); err != nil { - return fmt.Errorf("flushing completed files: %w", err) - } - log.Debug("handleBlobReady: flushCompletedPendingFiles returned") - - log.Debug("handleBlobReady: complete") - return nil +// streamingChunkInfo tracks chunk metadata collected during streaming +type streamingChunkInfo struct { + fileChunk database.FileChunk + offset int64 + size int64 } // processFileStreaming processes a file by streaming chunks directly to the packer func (s *Scanner) processFileStreaming(ctx context.Context, fileToProcess *FileToProcess, result *ScanResult) error { - // Open the file file, err := s.fs.Open(fileToProcess.Path) if err != nil { return fmt.Errorf("opening file: %w", err) } defer func() { _ = file.Close() }() - // We'll collect file chunks for database storage - // but process them for packing as we go - type chunkInfo struct { - fileChunk database.FileChunk - offset int64 - size int64 - } - var chunks []chunkInfo + var chunks []streamingChunkInfo chunkIndex := 0 - // Process chunks in streaming fashion and get full file hash fileHash, err := s.chunker.ChunkReaderStreaming(file, func(chunk chunker.Chunk) error { - // Check for cancellation select { case <-ctx.Done(): return ctx.Err() default: } - log.Debug("Processing content-defined chunk from file", - "file", fileToProcess.Path, - "chunk_index", chunkIndex, - "hash", chunk.Hash, - "size", chunk.Size) - - // Check if chunk already exists (fast in-memory lookup) chunkExists := s.chunkExists(chunk.Hash) - - // Queue new chunks for batch insert when blob finalizes - // This dramatically reduces transaction overhead if !chunkExists { s.packer.AddPendingChunk(chunk.Hash, chunk.Size) - // Add to in-memory cache immediately for fast duplicate detection s.addKnownChunk(chunk.Hash) - // Track as pending until blob finalizes and commits to DB s.addPendingChunkHash(chunk.Hash) } - // Track file chunk association for later storage - chunks = append(chunks, chunkInfo{ + chunks = append(chunks, streamingChunkInfo{ fileChunk: database.FileChunk{ FileID: fileToProcess.File.ID, Idx: chunkIndex, @@ -1159,55 +1210,15 @@ func (s *Scanner) processFileStreaming(ctx context.Context, fileToProcess *FileT size: chunk.Size, }) - // Update stats - if chunkExists { - result.FilesSkipped++ // Track as skipped for now - result.BytesSkipped += chunk.Size - if s.progress != nil { - s.progress.GetStats().BytesSkipped.Add(chunk.Size) - } - } else { - result.ChunksCreated++ - result.BytesScanned += chunk.Size - if s.progress != nil { - s.progress.GetStats().ChunksCreated.Add(1) - s.progress.GetStats().BytesProcessed.Add(chunk.Size) - s.progress.UpdateChunkingActivity() - } - } + s.updateChunkStats(chunkExists, chunk.Size, result) - // Add chunk to packer immediately (streaming) - // This happens outside the database transaction if !chunkExists { - s.packerMu.Lock() - err := s.packer.AddChunk(&blob.ChunkRef{ - Hash: chunk.Hash, - Data: chunk.Data, - }) - if err == blob.ErrBlobSizeLimitExceeded { - // Finalize current blob and retry - if err := s.packer.FinalizeBlob(); err != nil { - s.packerMu.Unlock() - return fmt.Errorf("finalizing blob: %w", err) - } - // Retry adding the chunk - if err := s.packer.AddChunk(&blob.ChunkRef{ - Hash: chunk.Hash, - Data: chunk.Data, - }); err != nil { - s.packerMu.Unlock() - return fmt.Errorf("adding chunk after finalize: %w", err) - } - } else if err != nil { - s.packerMu.Unlock() - return fmt.Errorf("adding chunk to packer: %w", err) + if err := s.addChunkToPacker(chunk); err != nil { + return err } - s.packerMu.Unlock() } - // Clear chunk data from memory immediately after use chunk.Data = nil - chunkIndex++ return nil }) @@ -1217,12 +1228,54 @@ func (s *Scanner) processFileStreaming(ctx context.Context, fileToProcess *FileT } log.Debug("Completed snapshotting file", - "path", fileToProcess.Path, - "file_hash", fileHash, - "chunks", len(chunks)) + "path", fileToProcess.Path, "file_hash", fileHash, "chunks", len(chunks)) - // Build file data for batch insertion - // Update chunk associations with the file ID + s.queueFileForBatchInsert(ctx, fileToProcess, chunks) + return nil +} + +// updateChunkStats updates scan result and progress stats for a processed chunk +func (s *Scanner) updateChunkStats(chunkExists bool, chunkSize int64, result *ScanResult) { + if chunkExists { + result.FilesSkipped++ + result.BytesSkipped += chunkSize + if s.progress != nil { + s.progress.GetStats().BytesSkipped.Add(chunkSize) + } + } else { + result.ChunksCreated++ + result.BytesScanned += chunkSize + if s.progress != nil { + s.progress.GetStats().ChunksCreated.Add(1) + s.progress.GetStats().BytesProcessed.Add(chunkSize) + s.progress.UpdateChunkingActivity() + } + } +} + +// addChunkToPacker adds a chunk to the blob packer, finalizing the current blob if needed +func (s *Scanner) addChunkToPacker(chunk chunker.Chunk) error { + s.packerMu.Lock() + err := s.packer.AddChunk(&blob.ChunkRef{Hash: chunk.Hash, Data: chunk.Data}) + if err == blob.ErrBlobSizeLimitExceeded { + if err := s.packer.FinalizeBlob(); err != nil { + s.packerMu.Unlock() + return fmt.Errorf("finalizing blob: %w", err) + } + if err := s.packer.AddChunk(&blob.ChunkRef{Hash: chunk.Hash, Data: chunk.Data}); err != nil { + s.packerMu.Unlock() + return fmt.Errorf("adding chunk after finalize: %w", err) + } + } else if err != nil { + s.packerMu.Unlock() + return fmt.Errorf("adding chunk to packer: %w", err) + } + s.packerMu.Unlock() + return nil +} + +// queueFileForBatchInsert builds file/chunk associations and queues the file for batch DB insert +func (s *Scanner) queueFileForBatchInsert(ctx context.Context, fileToProcess *FileToProcess, chunks []streamingChunkInfo) { fileChunks := make([]database.FileChunk, len(chunks)) chunkFiles := make([]database.ChunkFile, len(chunks)) for i, ci := range chunks { @@ -1239,14 +1292,11 @@ func (s *Scanner) processFileStreaming(ctx context.Context, fileToProcess *FileT } } - // Queue file for batch insertion - // Files will be flushed when their chunks are committed (after blob finalize) s.addPendingFile(ctx, pendingFileData{ file: fileToProcess.File, fileChunks: fileChunks, chunkFiles: chunkFiles, }) - return nil } // GetProgress returns the progress reporter for this scanner diff --git a/internal/snapshot/snapshot.go b/internal/snapshot/snapshot.go index bb01ea1..883c572 100644 --- a/internal/snapshot/snapshot.go +++ b/internal/snapshot/snapshot.go @@ -227,12 +227,39 @@ func (sm *SnapshotManager) ExportSnapshotMetadata(ctx context.Context, dbPath st } }() + // Steps 1-5: Copy, clean, vacuum, compress, and read the database + finalData, tempDBPath, err := sm.prepareExportDB(ctx, dbPath, snapshotID, tempDir) + if err != nil { + return err + } + + // Step 6: Generate blob manifest (before closing temp DB) + blobManifest, err := sm.generateBlobManifest(ctx, tempDBPath, snapshotID) + if err != nil { + return fmt.Errorf("generating blob manifest: %w", err) + } + + // Step 7: Upload to S3 in snapshot subdirectory + if err := sm.uploadSnapshotArtifacts(ctx, snapshotID, finalData, blobManifest); err != nil { + return err + } + + log.Info("Uploaded snapshot metadata", + "snapshot_id", snapshotID, + "db_size", len(finalData), + "manifest_size", len(blobManifest)) + return nil +} + +// prepareExportDB copies, cleans, vacuums, and compresses the snapshot database for export. +// Returns the compressed data and the path to the temporary database (needed for manifest generation). +func (sm *SnapshotManager) prepareExportDB(ctx context.Context, dbPath, snapshotID, tempDir string) ([]byte, string, error) { // Step 1: Copy database to temp file // The main database should be closed at this point tempDBPath := filepath.Join(tempDir, "snapshot.db") log.Debug("Copying database to temporary location", "source", dbPath, "destination", tempDBPath) if err := sm.copyFile(dbPath, tempDBPath); err != nil { - return fmt.Errorf("copying database: %w", err) + return nil, "", fmt.Errorf("copying database: %w", err) } log.Debug("Database copy complete", "size", sm.getFileSize(tempDBPath)) @@ -240,7 +267,7 @@ func (sm *SnapshotManager) ExportSnapshotMetadata(ctx context.Context, dbPath st log.Debug("Cleaning temporary database", "snapshot_id", snapshotID) stats, err := sm.cleanSnapshotDB(ctx, tempDBPath, snapshotID) if err != nil { - return fmt.Errorf("cleaning snapshot database: %w", err) + return nil, "", fmt.Errorf("cleaning snapshot database: %w", err) } log.Info("Temporary database cleanup complete", "db_path", tempDBPath, @@ -255,14 +282,14 @@ func (sm *SnapshotManager) ExportSnapshotMetadata(ctx context.Context, dbPath st // Step 3: VACUUM the database to remove deleted data and compact // This is critical for security - ensures no stale/deleted data is uploaded if err := sm.vacuumDatabase(tempDBPath); err != nil { - return fmt.Errorf("vacuuming database: %w", err) + return nil, "", fmt.Errorf("vacuuming database: %w", err) } log.Debug("Database vacuumed", "size", humanize.Bytes(uint64(sm.getFileSize(tempDBPath)))) // Step 4: Compress and encrypt the binary database file compressedPath := filepath.Join(tempDir, "db.zst.age") if err := sm.compressFile(tempDBPath, compressedPath); err != nil { - return fmt.Errorf("compressing database: %w", err) + return nil, "", fmt.Errorf("compressing database: %w", err) } log.Debug("Compression complete", "original_size", humanize.Bytes(uint64(sm.getFileSize(tempDBPath))), @@ -271,49 +298,43 @@ func (sm *SnapshotManager) ExportSnapshotMetadata(ctx context.Context, dbPath st // Step 5: Read compressed and encrypted data for upload finalData, err := afero.ReadFile(sm.fs, compressedPath) if err != nil { - return fmt.Errorf("reading compressed dump: %w", err) + return nil, "", fmt.Errorf("reading compressed dump: %w", err) } - // Step 6: Generate blob manifest (before closing temp DB) - blobManifest, err := sm.generateBlobManifest(ctx, tempDBPath, snapshotID) - if err != nil { - return fmt.Errorf("generating blob manifest: %w", err) - } + return finalData, tempDBPath, nil +} - // Step 7: Upload to S3 in snapshot subdirectory +// uploadSnapshotArtifacts uploads the database backup and blob manifest to S3 +func (sm *SnapshotManager) uploadSnapshotArtifacts(ctx context.Context, snapshotID string, dbData, manifestData []byte) error { // Upload database backup (compressed and encrypted) dbKey := fmt.Sprintf("metadata/%s/db.zst.age", snapshotID) dbUploadStart := time.Now() - if err := sm.storage.Put(ctx, dbKey, bytes.NewReader(finalData)); err != nil { + if err := sm.storage.Put(ctx, dbKey, bytes.NewReader(dbData)); err != nil { return fmt.Errorf("uploading snapshot database: %w", err) } dbUploadDuration := time.Since(dbUploadStart) - dbUploadSpeed := float64(len(finalData)) * 8 / dbUploadDuration.Seconds() // bits per second + dbUploadSpeed := float64(len(dbData)) * 8 / dbUploadDuration.Seconds() // bits per second log.Info("Uploaded snapshot database", "path", dbKey, - "size", humanize.Bytes(uint64(len(finalData))), + "size", humanize.Bytes(uint64(len(dbData))), "duration", dbUploadDuration, "speed", humanize.SI(dbUploadSpeed, "bps")) // Upload blob manifest (compressed only, not encrypted) manifestKey := fmt.Sprintf("metadata/%s/manifest.json.zst", snapshotID) manifestUploadStart := time.Now() - if err := sm.storage.Put(ctx, manifestKey, bytes.NewReader(blobManifest)); err != nil { + if err := sm.storage.Put(ctx, manifestKey, bytes.NewReader(manifestData)); err != nil { return fmt.Errorf("uploading blob manifest: %w", err) } manifestUploadDuration := time.Since(manifestUploadStart) - manifestUploadSpeed := float64(len(blobManifest)) * 8 / manifestUploadDuration.Seconds() // bits per second + manifestUploadSpeed := float64(len(manifestData)) * 8 / manifestUploadDuration.Seconds() // bits per second log.Info("Uploaded blob manifest", "path", manifestKey, - "size", humanize.Bytes(uint64(len(blobManifest))), + "size", humanize.Bytes(uint64(len(manifestData))), "duration", manifestUploadDuration, "speed", humanize.SI(manifestUploadSpeed, "bps")) - log.Info("Uploaded snapshot metadata", - "snapshot_id", snapshotID, - "db_size", len(finalData), - "manifest_size", len(blobManifest)) return nil } diff --git a/internal/vaultik/info.go b/internal/vaultik/info.go index 53cfc2c..124fc7b 100644 --- a/internal/vaultik/info.go +++ b/internal/vaultik/info.go @@ -149,9 +149,9 @@ type RemoteInfoResult struct { // RemoteInfo displays information about remote storage func (v *Vaultik) RemoteInfo(jsonOutput bool) error { + log.Info("Starting remote storage info gathering") result := &RemoteInfoResult{} - // Get storage info storageInfo := v.Storage.Info() result.StorageType = storageInfo.Type result.StorageLocation = storageInfo.Location @@ -161,23 +161,52 @@ func (v *Vaultik) RemoteInfo(jsonOutput bool) error { v.printfStdout("Type: %s\n", storageInfo.Type) v.printfStdout("Location: %s\n", storageInfo.Location) v.printlnStdout() - } - - // List all snapshot metadata - if !jsonOutput { v.printfStdout("Scanning snapshot metadata...\n") } + snapshotMetadata, snapshotIDs, err := v.collectSnapshotMetadata() + if err != nil { + return err + } + + if !jsonOutput { + v.printfStdout("Downloading %d manifest(s)...\n", len(snapshotIDs)) + } + + referencedBlobs := v.collectReferencedBlobsFromManifests(snapshotIDs, snapshotMetadata) + + v.populateRemoteInfoResult(result, snapshotMetadata, snapshotIDs, referencedBlobs) + + if err := v.scanRemoteBlobStorage(result, referencedBlobs, jsonOutput); err != nil { + return err + } + + log.Info("Remote info complete", + "snapshots", result.TotalMetadataCount, + "total_blobs", result.TotalBlobCount, + "referenced_blobs", result.ReferencedBlobCount, + "orphaned_blobs", result.OrphanedBlobCount) + + if jsonOutput { + enc := json.NewEncoder(v.Stdout) + enc.SetIndent("", " ") + return enc.Encode(result) + } + + v.printRemoteInfoTable(result) + return nil +} + +// collectSnapshotMetadata scans remote metadata and returns per-snapshot info and sorted IDs +func (v *Vaultik) collectSnapshotMetadata() (map[string]*SnapshotMetadataInfo, []string, error) { snapshotMetadata := make(map[string]*SnapshotMetadataInfo) - // Collect metadata files metadataCh := v.Storage.ListStream(v.ctx, "metadata/") for obj := range metadataCh { if obj.Err != nil { - return fmt.Errorf("listing metadata: %w", obj.Err) + return nil, nil, fmt.Errorf("listing metadata: %w", obj.Err) } - // Parse key: metadata// parts := strings.Split(obj.Key, "/") if len(parts) < 3 { continue @@ -185,14 +214,11 @@ func (v *Vaultik) RemoteInfo(jsonOutput bool) error { snapshotID := parts[1] if _, exists := snapshotMetadata[snapshotID]; !exists { - snapshotMetadata[snapshotID] = &SnapshotMetadataInfo{ - SnapshotID: snapshotID, - } + snapshotMetadata[snapshotID] = &SnapshotMetadataInfo{SnapshotID: snapshotID} } info := snapshotMetadata[snapshotID] filename := parts[2] - if strings.HasPrefix(filename, "manifest") { info.ManifestSize = obj.Size } else if strings.HasPrefix(filename, "db") { @@ -201,19 +227,18 @@ func (v *Vaultik) RemoteInfo(jsonOutput bool) error { info.TotalSize = info.ManifestSize + info.DatabaseSize } - // Sort snapshots by ID for consistent output var snapshotIDs []string for id := range snapshotMetadata { snapshotIDs = append(snapshotIDs, id) } sort.Strings(snapshotIDs) - // Download and parse all manifests to get referenced blobs - if !jsonOutput { - v.printfStdout("Downloading %d manifest(s)...\n", len(snapshotIDs)) - } + return snapshotMetadata, snapshotIDs, nil +} - referencedBlobs := make(map[string]int64) // hash -> compressed size +// collectReferencedBlobsFromManifests downloads manifests and returns referenced blob hashes with sizes +func (v *Vaultik) collectReferencedBlobsFromManifests(snapshotIDs []string, snapshotMetadata map[string]*SnapshotMetadataInfo) map[string]int64 { + referencedBlobs := make(map[string]int64) for _, snapshotID := range snapshotIDs { manifestKey := fmt.Sprintf("metadata/%s/manifest.json.zst", snapshotID) @@ -230,10 +255,8 @@ func (v *Vaultik) RemoteInfo(jsonOutput bool) error { continue } - // Record blob info from manifest info := snapshotMetadata[snapshotID] info.BlobCount = manifest.BlobCount - var blobsSize int64 for _, blob := range manifest.Blobs { referencedBlobs[blob.Hash] = blob.CompressedSize @@ -242,7 +265,11 @@ func (v *Vaultik) RemoteInfo(jsonOutput bool) error { info.BlobsSize = blobsSize } - // Build result snapshots + return referencedBlobs +} + +// populateRemoteInfoResult fills in the result's snapshot and referenced blob stats +func (v *Vaultik) populateRemoteInfoResult(result *RemoteInfoResult, snapshotMetadata map[string]*SnapshotMetadataInfo, snapshotIDs []string, referencedBlobs map[string]int64) { var totalMetadataSize int64 for _, id := range snapshotIDs { info := snapshotMetadata[id] @@ -252,26 +279,25 @@ func (v *Vaultik) RemoteInfo(jsonOutput bool) error { result.TotalMetadataSize = totalMetadataSize result.TotalMetadataCount = len(snapshotIDs) - // Calculate referenced blob stats for _, size := range referencedBlobs { result.ReferencedBlobCount++ result.ReferencedBlobSize += size } +} - // List all blobs on remote +// scanRemoteBlobStorage lists all blobs on remote and computes orphan stats +func (v *Vaultik) scanRemoteBlobStorage(result *RemoteInfoResult, referencedBlobs map[string]int64, jsonOutput bool) error { if !jsonOutput { v.printfStdout("Scanning blobs...\n") } - allBlobs := make(map[string]int64) // hash -> size from storage - blobCh := v.Storage.ListStream(v.ctx, "blobs/") + allBlobs := make(map[string]int64) + for obj := range blobCh { if obj.Err != nil { return fmt.Errorf("listing blobs: %w", obj.Err) } - - // Extract hash from key: blobs/xx/yy/hash parts := strings.Split(obj.Key, "/") if len(parts) < 4 { continue @@ -282,7 +308,6 @@ func (v *Vaultik) RemoteInfo(jsonOutput bool) error { result.TotalBlobSize += obj.Size } - // Calculate orphaned blobs for hash, size := range allBlobs { if _, referenced := referencedBlobs[hash]; !referenced { result.OrphanedBlobCount++ @@ -290,14 +315,11 @@ func (v *Vaultik) RemoteInfo(jsonOutput bool) error { } } - // Output results - if jsonOutput { - enc := json.NewEncoder(v.Stdout) - enc.SetIndent("", " ") - return enc.Encode(result) - } + return nil +} - // Human-readable output +// printRemoteInfoTable renders the human-readable remote info output +func (v *Vaultik) printRemoteInfoTable(result *RemoteInfoResult) { v.printfStdout("\n=== Snapshot Metadata ===\n") if len(result.Snapshots) == 0 { v.printfStdout("No snapshots found\n") @@ -320,20 +342,15 @@ func (v *Vaultik) RemoteInfo(jsonOutput bool) error { v.printfStdout("\n=== Blob Storage ===\n") v.printfStdout("Total blobs on remote: %s (%s)\n", - humanize.Comma(int64(result.TotalBlobCount)), - humanize.Bytes(uint64(result.TotalBlobSize))) + humanize.Comma(int64(result.TotalBlobCount)), humanize.Bytes(uint64(result.TotalBlobSize))) v.printfStdout("Referenced by snapshots: %s (%s)\n", - humanize.Comma(int64(result.ReferencedBlobCount)), - humanize.Bytes(uint64(result.ReferencedBlobSize))) + humanize.Comma(int64(result.ReferencedBlobCount)), humanize.Bytes(uint64(result.ReferencedBlobSize))) v.printfStdout("Orphaned (unreferenced): %s (%s)\n", - humanize.Comma(int64(result.OrphanedBlobCount)), - humanize.Bytes(uint64(result.OrphanedBlobSize))) + humanize.Comma(int64(result.OrphanedBlobCount)), humanize.Bytes(uint64(result.OrphanedBlobSize))) if result.OrphanedBlobCount > 0 { v.printfStdout("\nRun 'vaultik prune --remote' to remove orphaned blobs.\n") } - - return nil } // truncateString truncates a string to maxLen, adding "..." if truncated diff --git a/internal/vaultik/prune.go b/internal/vaultik/prune.go index dff9dd9..2fb1a35 100644 --- a/internal/vaultik/prune.go +++ b/internal/vaultik/prune.go @@ -27,95 +27,19 @@ type PruneBlobsResult struct { func (v *Vaultik) PruneBlobs(opts *PruneOptions) error { log.Info("Starting prune operation") - // Get all remote snapshots and their manifests - allBlobsReferenced := make(map[string]bool) - manifestCount := 0 - - // List all snapshots in storage - log.Info("Listing remote snapshots") - objectCh := v.Storage.ListStream(v.ctx, "metadata/") - - var snapshotIDs []string - for object := range objectCh { - if object.Err != nil { - return fmt.Errorf("listing remote snapshots: %w", object.Err) - } - - // Extract snapshot ID from paths like metadata/hostname-20240115-143052Z/ - parts := strings.Split(object.Key, "/") - if len(parts) >= 2 && parts[0] == "metadata" && parts[1] != "" { - // Check if this is a directory by looking for trailing slash - if strings.HasSuffix(object.Key, "/") || strings.Contains(object.Key, "/manifest.json.zst") { - snapshotID := parts[1] - // Only add unique snapshot IDs - found := false - for _, id := range snapshotIDs { - if id == snapshotID { - found = true - break - } - } - if !found { - snapshotIDs = append(snapshotIDs, snapshotID) - } - } - } + allBlobsReferenced, err := v.collectReferencedBlobs() + if err != nil { + return err } - log.Info("Found manifests in remote storage", "count", len(snapshotIDs)) - - // Download and parse each manifest to get referenced blobs - for _, snapshotID := range snapshotIDs { - log.Debug("Processing manifest", "snapshot_id", snapshotID) - - manifest, err := v.downloadManifest(snapshotID) - if err != nil { - log.Error("Failed to download manifest", "snapshot_id", snapshotID, "error", err) - continue - } - - // Add all blobs from this manifest to our referenced set - for _, blob := range manifest.Blobs { - allBlobsReferenced[blob.Hash] = true - } - manifestCount++ + allBlobs, err := v.listAllRemoteBlobs() + if err != nil { + return err } - log.Info("Processed manifests", "count", manifestCount, "unique_blobs_referenced", len(allBlobsReferenced)) + unreferencedBlobs, totalSize := v.findUnreferencedBlobs(allBlobs, allBlobsReferenced) - // List all blobs in storage - log.Info("Listing all blobs in storage") - allBlobs := make(map[string]int64) // hash -> size - blobObjectCh := v.Storage.ListStream(v.ctx, "blobs/") - - for object := range blobObjectCh { - if object.Err != nil { - return fmt.Errorf("listing blobs: %w", object.Err) - } - - // Extract hash from path like blobs/ab/cd/abcdef123456... - parts := strings.Split(object.Key, "/") - if len(parts) == 4 && parts[0] == "blobs" { - hash := parts[3] - allBlobs[hash] = object.Size - } - } - - log.Info("Found blobs in storage", "count", len(allBlobs)) - - // Find unreferenced blobs - var unreferencedBlobs []string - var totalSize int64 - for hash, size := range allBlobs { - if !allBlobsReferenced[hash] { - unreferencedBlobs = append(unreferencedBlobs, hash) - totalSize += size - } - } - - result := &PruneBlobsResult{ - BlobsFound: len(unreferencedBlobs), - } + result := &PruneBlobsResult{BlobsFound: len(unreferencedBlobs)} if len(unreferencedBlobs) == 0 { log.Info("No unreferenced blobs found") @@ -126,18 +50,15 @@ func (v *Vaultik) PruneBlobs(opts *PruneOptions) error { return nil } - // Show what will be deleted log.Info("Found unreferenced blobs", "count", len(unreferencedBlobs), "total_size", humanize.Bytes(uint64(totalSize))) if !opts.JSON { v.printfStdout("Found %d unreferenced blob(s) totaling %s\n", len(unreferencedBlobs), humanize.Bytes(uint64(totalSize))) } - // Confirm unless --force is used (skip in JSON mode - require --force) if !opts.Force && !opts.JSON { v.printfStdout("\nDelete %d unreferenced blob(s)? [y/N] ", len(unreferencedBlobs)) var confirm string if _, err := v.scanStdin(&confirm); err != nil { - // Treat EOF or error as "no" v.printlnStdout("Cancelled") return nil } @@ -147,10 +68,109 @@ func (v *Vaultik) PruneBlobs(opts *PruneOptions) error { } } - // Delete unreferenced blobs + v.deleteUnreferencedBlobs(unreferencedBlobs, allBlobs, result) + + if opts.JSON { + return v.outputPruneBlobsJSON(result) + } + + v.printfStdout("\nDeleted %d blob(s) totaling %s\n", result.BlobsDeleted, humanize.Bytes(uint64(result.BytesFreed))) + if result.BlobsFailed > 0 { + v.printfStdout("Failed to delete %d blob(s)\n", result.BlobsFailed) + } + + return nil +} + +// collectReferencedBlobs downloads all manifests and returns the set of referenced blob hashes +func (v *Vaultik) collectReferencedBlobs() (map[string]bool, error) { + log.Info("Listing remote snapshots") + snapshotIDs, err := v.listUniqueSnapshotIDs() + if err != nil { + return nil, fmt.Errorf("listing snapshot IDs: %w", err) + } + log.Info("Found manifests in remote storage", "count", len(snapshotIDs)) + + allBlobsReferenced := make(map[string]bool) + manifestCount := 0 + + for _, snapshotID := range snapshotIDs { + log.Debug("Processing manifest", "snapshot_id", snapshotID) + manifest, err := v.downloadManifest(snapshotID) + if err != nil { + log.Error("Failed to download manifest", "snapshot_id", snapshotID, "error", err) + continue + } + for _, blob := range manifest.Blobs { + allBlobsReferenced[blob.Hash] = true + } + manifestCount++ + } + + log.Info("Processed manifests", "count", manifestCount, "unique_blobs_referenced", len(allBlobsReferenced)) + return allBlobsReferenced, nil +} + +// listUniqueSnapshotIDs returns deduplicated snapshot IDs from remote metadata +func (v *Vaultik) listUniqueSnapshotIDs() ([]string, error) { + objectCh := v.Storage.ListStream(v.ctx, "metadata/") + seen := make(map[string]bool) + var snapshotIDs []string + + for object := range objectCh { + if object.Err != nil { + return nil, fmt.Errorf("listing metadata objects: %w", object.Err) + } + parts := strings.Split(object.Key, "/") + if len(parts) >= 2 && parts[0] == "metadata" && parts[1] != "" { + if strings.HasSuffix(object.Key, "/") || strings.Contains(object.Key, "/manifest.json.zst") { + snapshotID := parts[1] + if !seen[snapshotID] { + seen[snapshotID] = true + snapshotIDs = append(snapshotIDs, snapshotID) + } + } + } + } + return snapshotIDs, nil +} + +// listAllRemoteBlobs returns a map of all blob hashes to their sizes in remote storage +func (v *Vaultik) listAllRemoteBlobs() (map[string]int64, error) { + log.Info("Listing all blobs in storage") + allBlobs := make(map[string]int64) + blobObjectCh := v.Storage.ListStream(v.ctx, "blobs/") + + for object := range blobObjectCh { + if object.Err != nil { + return nil, fmt.Errorf("listing blobs: %w", object.Err) + } + parts := strings.Split(object.Key, "/") + if len(parts) == 4 && parts[0] == "blobs" { + allBlobs[parts[3]] = object.Size + } + } + + log.Info("Found blobs in storage", "count", len(allBlobs)) + return allBlobs, nil +} + +// findUnreferencedBlobs returns blob hashes not referenced by any manifest and their total size +func (v *Vaultik) findUnreferencedBlobs(allBlobs map[string]int64, referenced map[string]bool) ([]string, int64) { + var unreferenced []string + var totalSize int64 + for hash, size := range allBlobs { + if !referenced[hash] { + unreferenced = append(unreferenced, hash) + totalSize += size + } + } + return unreferenced, totalSize +} + +// deleteUnreferencedBlobs deletes the given blobs from storage and populates the result +func (v *Vaultik) deleteUnreferencedBlobs(unreferencedBlobs []string, allBlobs map[string]int64, result *PruneBlobsResult) { log.Info("Deleting unreferenced blobs") - deletedCount := 0 - deletedSize := int64(0) for i, hash := range unreferencedBlobs { blobPath := fmt.Sprintf("blobs/%s/%s/%s", hash[:2], hash[2:4], hash) @@ -160,10 +180,9 @@ func (v *Vaultik) PruneBlobs(opts *PruneOptions) error { continue } - deletedCount++ - deletedSize += allBlobs[hash] + result.BlobsDeleted++ + result.BytesFreed += allBlobs[hash] - // Progress update every 100 blobs if (i+1)%100 == 0 || i == len(unreferencedBlobs)-1 { log.Info("Deletion progress", "deleted", i+1, @@ -173,26 +192,13 @@ func (v *Vaultik) PruneBlobs(opts *PruneOptions) error { } } - result.BlobsDeleted = deletedCount - result.BlobsFailed = len(unreferencedBlobs) - deletedCount - result.BytesFreed = deletedSize + result.BlobsFailed = len(unreferencedBlobs) - result.BlobsDeleted log.Info("Prune complete", - "deleted_count", deletedCount, - "deleted_size", humanize.Bytes(uint64(deletedSize)), - "failed", len(unreferencedBlobs)-deletedCount, + "deleted_count", result.BlobsDeleted, + "deleted_size", humanize.Bytes(uint64(result.BytesFreed)), + "failed", result.BlobsFailed, ) - - if opts.JSON { - return v.outputPruneBlobsJSON(result) - } - - v.printfStdout("\nDeleted %d blob(s) totaling %s\n", deletedCount, humanize.Bytes(uint64(deletedSize))) - if deletedCount < len(unreferencedBlobs) { - v.printfStdout("Failed to delete %d blob(s)\n", len(unreferencedBlobs)-deletedCount) - } - - return nil } // outputPruneBlobsJSON outputs the prune result as JSON diff --git a/internal/vaultik/restore.go b/internal/vaultik/restore.go index a92fef5..5797fc8 100644 --- a/internal/vaultik/restore.go +++ b/internal/vaultik/restore.go @@ -55,15 +55,9 @@ type RestoreResult struct { func (v *Vaultik) Restore(opts *RestoreOptions) error { startTime := time.Now() - // Check for age_secret_key - if v.Config.AgeSecretKey == "" { - return fmt.Errorf("decryption key required for restore\n\nSet the VAULTIK_AGE_SECRET_KEY environment variable to your age private key:\n export VAULTIK_AGE_SECRET_KEY='AGE-SECRET-KEY-...'") - } - - // Parse the age identity - identity, err := age.ParseX25519Identity(v.Config.AgeSecretKey) + identity, err := v.prepareRestoreIdentity() if err != nil { - return fmt.Errorf("parsing age secret key: %w", err) + return err } log.Info("Starting restore operation", @@ -115,10 +109,73 @@ func (v *Vaultik) Restore(opts *RestoreOptions) error { } // Step 5: Restore files + result, err := v.restoreAllFiles(files, repos, opts, identity, chunkToBlobMap) + if err != nil { + return err + } + + result.Duration = time.Since(startTime) + + log.Info("Restore complete", + "files_restored", result.FilesRestored, + "bytes_restored", humanize.Bytes(uint64(result.BytesRestored)), + "blobs_downloaded", result.BlobsDownloaded, + "bytes_downloaded", humanize.Bytes(uint64(result.BytesDownloaded)), + "duration", result.Duration, + ) + + v.printfStdout("Restored %d files (%s) in %s\n", + result.FilesRestored, + humanize.Bytes(uint64(result.BytesRestored)), + result.Duration.Round(time.Second), + ) + + if result.FilesFailed > 0 { + _, _ = fmt.Fprintf(v.Stdout, "\nWARNING: %d file(s) failed to restore:\n", result.FilesFailed) + for _, path := range result.FailedFiles { + _, _ = fmt.Fprintf(v.Stdout, " - %s\n", path) + } + } + + // Run verification if requested + if opts.Verify { + if err := v.handleRestoreVerification(repos, files, opts, result); err != nil { + return err + } + } + + if result.FilesFailed > 0 { + return fmt.Errorf("%d file(s) failed to restore", result.FilesFailed) + } + + return nil +} + +// prepareRestoreIdentity validates that an age secret key is configured and parses it +func (v *Vaultik) prepareRestoreIdentity() (age.Identity, error) { + if v.Config.AgeSecretKey == "" { + return nil, fmt.Errorf("decryption key required for restore\n\nSet the VAULTIK_AGE_SECRET_KEY environment variable to your age private key:\n export VAULTIK_AGE_SECRET_KEY='AGE-SECRET-KEY-...'") + } + + identity, err := age.ParseX25519Identity(v.Config.AgeSecretKey) + if err != nil { + return nil, fmt.Errorf("parsing age secret key: %w", err) + } + return identity, nil +} + +// restoreAllFiles iterates over files and restores each one, tracking progress and failures +func (v *Vaultik) restoreAllFiles( + files []*database.File, + repos *database.Repositories, + opts *RestoreOptions, + identity age.Identity, + chunkToBlobMap map[string]*database.BlobChunk, +) (*RestoreResult, error) { result := &RestoreResult{} blobCache, err := newBlobDiskCache(4 * v.Config.BlobSizeLimit.Int64()) if err != nil { - return fmt.Errorf("creating blob cache: %w", err) + return nil, fmt.Errorf("creating blob cache: %w", err) } defer func() { _ = blobCache.Close() }() @@ -133,7 +190,7 @@ func (v *Vaultik) Restore(opts *RestoreOptions) error { for i, file := range files { if v.ctx.Err() != nil { - return v.ctx.Err() + return nil, v.ctx.Err() } if err := v.restoreFile(v.ctx, repos, file, opts.TargetDir, identity, chunkToBlobMap, blobCache, result); err != nil { @@ -165,53 +222,32 @@ func (v *Vaultik) Restore(opts *RestoreOptions) error { _ = bar.Finish() } - result.Duration = time.Since(startTime) + return result, nil +} - log.Info("Restore complete", - "files_restored", result.FilesRestored, - "bytes_restored", humanize.Bytes(uint64(result.BytesRestored)), - "blobs_downloaded", result.BlobsDownloaded, - "bytes_downloaded", humanize.Bytes(uint64(result.BytesDownloaded)), - "duration", result.Duration, - ) - - v.printfStdout("Restored %d files (%s) in %s\n", - result.FilesRestored, - humanize.Bytes(uint64(result.BytesRestored)), - result.Duration.Round(time.Second), - ) +// handleRestoreVerification runs post-restore verification if requested +func (v *Vaultik) handleRestoreVerification( + repos *database.Repositories, + files []*database.File, + opts *RestoreOptions, + result *RestoreResult, +) error { + if err := v.verifyRestoredFiles(v.ctx, repos, files, opts.TargetDir, result); err != nil { + return fmt.Errorf("verification failed: %w", err) + } if result.FilesFailed > 0 { - _, _ = fmt.Fprintf(v.Stdout, "\nWARNING: %d file(s) failed to restore:\n", result.FilesFailed) + v.printfStdout("\nVerification FAILED: %d files did not match expected checksums\n", result.FilesFailed) for _, path := range result.FailedFiles { - _, _ = fmt.Fprintf(v.Stdout, " - %s\n", path) + v.printfStdout(" - %s\n", path) } + return fmt.Errorf("%d files failed verification", result.FilesFailed) } - // Run verification if requested - if opts.Verify { - if err := v.verifyRestoredFiles(v.ctx, repos, files, opts.TargetDir, result); err != nil { - return fmt.Errorf("verification failed: %w", err) - } - - if result.FilesFailed > 0 { - v.printfStdout("\nVerification FAILED: %d files did not match expected checksums\n", result.FilesFailed) - for _, path := range result.FailedFiles { - v.printfStdout(" - %s\n", path) - } - return fmt.Errorf("%d files failed verification", result.FilesFailed) - } - - v.printfStdout("Verified %d files (%s)\n", - result.FilesVerified, - humanize.Bytes(uint64(result.BytesVerified)), - ) - } - - if result.FilesFailed > 0 { - return fmt.Errorf("%d file(s) failed to restore", result.FilesFailed) - } - + v.printfStdout("Verified %d files (%s)\n", + result.FilesVerified, + humanize.Bytes(uint64(result.BytesVerified)), + ) return nil } diff --git a/internal/vaultik/snapshot.go b/internal/vaultik/snapshot.go index e0d93b2..21904bf 100644 --- a/internal/vaultik/snapshot.go +++ b/internal/vaultik/snapshot.go @@ -111,40 +111,34 @@ func (v *Vaultik) CreateSnapshot(opts *SnapshotCreateOptions) error { return nil } +// snapshotStats tracks aggregate statistics across directory scans +type snapshotStats struct { + totalFiles int + totalBytes int64 + totalChunks int + totalBlobs int + totalBytesSkipped int64 + totalFilesSkipped int + totalFilesDeleted int + totalBytesDeleted int64 + totalBytesUploaded int64 + totalBlobsUploaded int + uploadDuration time.Duration +} + // createNamedSnapshot creates a single named snapshot func (v *Vaultik) createNamedSnapshot(opts *SnapshotCreateOptions, hostname, snapName string, idx, total int) error { snapshotStartTime := time.Now() - snapConfig := v.Config.Snapshots[snapName] - if total > 1 { v.printfStdout("\n=== Snapshot %d/%d: %s ===\n", idx, total, snapName) } - // Resolve source directories to absolute paths - resolvedDirs := make([]string, 0, len(snapConfig.Paths)) - for _, dir := range snapConfig.Paths { - absPath, err := filepath.Abs(dir) - if err != nil { - return fmt.Errorf("failed to resolve absolute path for %s: %w", dir, err) - } - - // Resolve symlinks - resolvedPath, err := filepath.EvalSymlinks(absPath) - if err != nil { - // If the path doesn't exist yet, use the absolute path - if os.IsNotExist(err) { - resolvedPath = absPath - } else { - return fmt.Errorf("failed to resolve symlinks for %s: %w", absPath, err) - } - } - - resolvedDirs = append(resolvedDirs, resolvedPath) + resolvedDirs, err := v.resolveSnapshotPaths(snapName) + if err != nil { + return err } - // Create scanner with progress enabled (unless in cron mode) - // Pass the combined excludes for this snapshot scanner := v.ScannerFactory(snapshot.ScannerParams{ EnableProgress: !opts.Cron, Fs: v.Fs, @@ -152,20 +146,6 @@ func (v *Vaultik) createNamedSnapshot(opts *SnapshotCreateOptions, hostname, sna SkipErrors: opts.SkipErrors, }) - // Statistics tracking - totalFiles := 0 - totalBytes := int64(0) - totalChunks := 0 - totalBlobs := 0 - totalBytesSkipped := int64(0) - totalFilesSkipped := 0 - totalFilesDeleted := 0 - totalBytesDeleted := int64(0) - totalBytesUploaded := int64(0) - totalBlobsUploaded := 0 - uploadDuration := time.Duration(0) - - // Create a new snapshot at the beginning (with snapshot name in ID) snapshotID, err := v.SnapshotManager.CreateSnapshotWithName(v.ctx, hostname, snapName, v.Globals.Version, v.Globals.Commit) if err != nil { return fmt.Errorf("creating snapshot: %w", err) @@ -173,12 +153,64 @@ func (v *Vaultik) createNamedSnapshot(opts *SnapshotCreateOptions, hostname, sna log.Info("Beginning snapshot", "snapshot_id", snapshotID, "name", snapName) v.printfStdout("Beginning snapshot: %s\n", snapshotID) + stats, err := v.scanAllDirectories(scanner, resolvedDirs, snapshotID) + if err != nil { + return err + } + + v.collectUploadStats(scanner, stats) + + if err := v.finalizeSnapshotMetadata(snapshotID, stats); err != nil { + return err + } + + log.Info("Snapshot complete", + "snapshot_id", snapshotID, + "name", snapName, + "files", stats.totalFiles, + "blobs_uploaded", stats.totalBlobsUploaded, + "bytes_uploaded", stats.totalBytesUploaded, + "duration", time.Since(snapshotStartTime)) + + v.printSnapshotSummary(snapshotID, snapshotStartTime, stats) + return nil +} + +// resolveSnapshotPaths resolves source directories to absolute paths with symlink resolution +func (v *Vaultik) resolveSnapshotPaths(snapName string) ([]string, error) { + snapConfig := v.Config.Snapshots[snapName] + resolvedDirs := make([]string, 0, len(snapConfig.Paths)) + + for _, dir := range snapConfig.Paths { + absPath, err := filepath.Abs(dir) + if err != nil { + return nil, fmt.Errorf("failed to resolve absolute path for %s: %w", dir, err) + } + + resolvedPath, err := filepath.EvalSymlinks(absPath) + if err != nil { + if os.IsNotExist(err) { + resolvedPath = absPath + } else { + return nil, fmt.Errorf("failed to resolve symlinks for %s: %w", absPath, err) + } + } + + resolvedDirs = append(resolvedDirs, resolvedPath) + } + + return resolvedDirs, nil +} + +// scanAllDirectories runs the scanner on each resolved directory and accumulates stats +func (v *Vaultik) scanAllDirectories(scanner *snapshot.Scanner, resolvedDirs []string, snapshotID string) (*snapshotStats, error) { + stats := &snapshotStats{} + for i, dir := range resolvedDirs { - // Check if context is cancelled select { case <-v.ctx.Done(): log.Info("Snapshot creation cancelled") - return v.ctx.Err() + return nil, v.ctx.Err() default: } @@ -186,17 +218,17 @@ func (v *Vaultik) createNamedSnapshot(opts *SnapshotCreateOptions, hostname, sna v.printfStdout("Beginning directory scan (%d/%d): %s\n", i+1, len(resolvedDirs), dir) result, err := scanner.Scan(v.ctx, dir, snapshotID) if err != nil { - return fmt.Errorf("failed to scan %s: %w", dir, err) + return nil, fmt.Errorf("failed to scan %s: %w", dir, err) } - totalFiles += result.FilesScanned - totalBytes += result.BytesScanned - totalChunks += result.ChunksCreated - totalBlobs += result.BlobsCreated - totalFilesSkipped += result.FilesSkipped - totalBytesSkipped += result.BytesSkipped - totalFilesDeleted += result.FilesDeleted - totalBytesDeleted += result.BytesDeleted + stats.totalFiles += result.FilesScanned + stats.totalBytes += result.BytesScanned + stats.totalChunks += result.ChunksCreated + stats.totalBlobs += result.BlobsCreated + stats.totalFilesSkipped += result.FilesSkipped + stats.totalBytesSkipped += result.BytesSkipped + stats.totalFilesDeleted += result.FilesDeleted + stats.totalBytesDeleted += result.BytesDeleted log.Info("Directory scan complete", "path", dir, @@ -207,85 +239,79 @@ func (v *Vaultik) createNamedSnapshot(opts *SnapshotCreateOptions, hostname, sna "chunks", result.ChunksCreated, "blobs", result.BlobsCreated, "duration", result.EndTime.Sub(result.StartTime)) - - // Remove per-directory summary - the scanner already prints its own summary } - // Get upload statistics from scanner progress if available + return stats, nil +} + +// collectUploadStats gathers upload statistics from the scanner's progress reporter +func (v *Vaultik) collectUploadStats(scanner *snapshot.Scanner, stats *snapshotStats) { if s := scanner.GetProgress(); s != nil { - stats := s.GetStats() - totalBytesUploaded = stats.BytesUploaded.Load() - totalBlobsUploaded = int(stats.BlobsUploaded.Load()) - uploadDuration = time.Duration(stats.UploadDurationMs.Load()) * time.Millisecond + progressStats := s.GetStats() + stats.totalBytesUploaded = progressStats.BytesUploaded.Load() + stats.totalBlobsUploaded = int(progressStats.BlobsUploaded.Load()) + stats.uploadDuration = time.Duration(progressStats.UploadDurationMs.Load()) * time.Millisecond } +} - // Update snapshot statistics with extended fields +// finalizeSnapshotMetadata updates stats, marks complete, and exports metadata +func (v *Vaultik) finalizeSnapshotMetadata(snapshotID string, stats *snapshotStats) error { extStats := snapshot.ExtendedBackupStats{ BackupStats: snapshot.BackupStats{ - FilesScanned: totalFiles, - BytesScanned: totalBytes, - ChunksCreated: totalChunks, - BlobsCreated: totalBlobs, - BytesUploaded: totalBytesUploaded, + FilesScanned: stats.totalFiles, + BytesScanned: stats.totalBytes, + ChunksCreated: stats.totalChunks, + BlobsCreated: stats.totalBlobs, + BytesUploaded: stats.totalBytesUploaded, }, - BlobUncompressedSize: 0, // Will be set from database query below + BlobUncompressedSize: 0, CompressionLevel: v.Config.CompressionLevel, - UploadDurationMs: uploadDuration.Milliseconds(), + UploadDurationMs: stats.uploadDuration.Milliseconds(), } if err := v.SnapshotManager.UpdateSnapshotStatsExtended(v.ctx, snapshotID, extStats); err != nil { return fmt.Errorf("updating snapshot stats: %w", err) } - // Mark snapshot as complete if err := v.SnapshotManager.CompleteSnapshot(v.ctx, snapshotID); err != nil { return fmt.Errorf("completing snapshot: %w", err) } - // Export snapshot metadata - // Export snapshot metadata without closing the database - // The export function should handle its own database connection if err := v.SnapshotManager.ExportSnapshotMetadata(v.ctx, v.Config.IndexPath, snapshotID); err != nil { return fmt.Errorf("exporting snapshot metadata: %w", err) } - // Calculate final statistics - snapshotDuration := time.Since(snapshotStartTime) - totalFilesChanged := totalFiles - totalFilesSkipped - totalBytesChanged := totalBytes - totalBytesAll := totalBytes + totalBytesSkipped + return nil +} - // Calculate upload speed - var avgUploadSpeed string - if totalBytesUploaded > 0 && uploadDuration > 0 { - bytesPerSec := float64(totalBytesUploaded) / uploadDuration.Seconds() - bitsPerSec := bytesPerSec * 8 - if bitsPerSec >= 1e9 { - avgUploadSpeed = fmt.Sprintf("%.1f Gbit/s", bitsPerSec/1e9) - } else if bitsPerSec >= 1e6 { - avgUploadSpeed = fmt.Sprintf("%.0f Mbit/s", bitsPerSec/1e6) - } else if bitsPerSec >= 1e3 { - avgUploadSpeed = fmt.Sprintf("%.0f Kbit/s", bitsPerSec/1e3) - } else { - avgUploadSpeed = fmt.Sprintf("%.0f bit/s", bitsPerSec) - } - } else { - avgUploadSpeed = "N/A" +// formatUploadSpeed formats bytes uploaded and duration into a human-readable speed string +func formatUploadSpeed(bytesUploaded int64, duration time.Duration) string { + if bytesUploaded <= 0 || duration <= 0 { + return "N/A" } + bytesPerSec := float64(bytesUploaded) / duration.Seconds() + bitsPerSec := bytesPerSec * 8 + switch { + case bitsPerSec >= 1e9: + return fmt.Sprintf("%.1f Gbit/s", bitsPerSec/1e9) + case bitsPerSec >= 1e6: + return fmt.Sprintf("%.0f Mbit/s", bitsPerSec/1e6) + case bitsPerSec >= 1e3: + return fmt.Sprintf("%.0f Kbit/s", bitsPerSec/1e3) + default: + return fmt.Sprintf("%.0f bit/s", bitsPerSec) + } +} + +// printSnapshotSummary prints the comprehensive snapshot completion summary +func (v *Vaultik) printSnapshotSummary(snapshotID string, startTime time.Time, stats *snapshotStats) { + snapshotDuration := time.Since(startTime) + totalFilesChanged := stats.totalFiles - stats.totalFilesSkipped + totalBytesAll := stats.totalBytes + stats.totalBytesSkipped // Get total blob sizes from database - totalBlobSizeCompressed := int64(0) - totalBlobSizeUncompressed := int64(0) - if blobHashes, err := v.Repositories.Snapshots.GetBlobHashes(v.ctx, snapshotID); err == nil { - for _, hash := range blobHashes { - if blob, err := v.Repositories.Blobs.GetByHash(v.ctx, hash); err == nil && blob != nil { - totalBlobSizeCompressed += blob.CompressedSize - totalBlobSizeUncompressed += blob.UncompressedSize - } - } - } + totalBlobSizeCompressed, totalBlobSizeUncompressed := v.getSnapshotBlobSizes(snapshotID) - // Calculate compression ratio var compressionRatio float64 if totalBlobSizeUncompressed > 0 { compressionRatio = float64(totalBlobSizeCompressed) / float64(totalBlobSizeUncompressed) @@ -293,55 +319,96 @@ func (v *Vaultik) createNamedSnapshot(opts *SnapshotCreateOptions, hostname, sna compressionRatio = 1.0 } - // Print comprehensive summary v.printfStdout("=== Snapshot Complete ===\n") v.printfStdout("ID: %s\n", snapshotID) v.printfStdout("Files: %s examined, %s to process, %s unchanged", - formatNumber(totalFiles), + formatNumber(stats.totalFiles), formatNumber(totalFilesChanged), - formatNumber(totalFilesSkipped)) - if totalFilesDeleted > 0 { - v.printfStdout(", %s deleted", formatNumber(totalFilesDeleted)) + formatNumber(stats.totalFilesSkipped)) + if stats.totalFilesDeleted > 0 { + v.printfStdout(", %s deleted", formatNumber(stats.totalFilesDeleted)) } v.printlnStdout() v.printfStdout("Data: %s total (%s to process)", humanize.Bytes(uint64(totalBytesAll)), - humanize.Bytes(uint64(totalBytesChanged))) - if totalBytesDeleted > 0 { - v.printfStdout(", %s deleted", humanize.Bytes(uint64(totalBytesDeleted))) + humanize.Bytes(uint64(stats.totalBytes))) + if stats.totalBytesDeleted > 0 { + v.printfStdout(", %s deleted", humanize.Bytes(uint64(stats.totalBytesDeleted))) } v.printlnStdout() - if totalBlobsUploaded > 0 { + if stats.totalBlobsUploaded > 0 { v.printfStdout("Storage: %s compressed from %s (%.2fx)\n", humanize.Bytes(uint64(totalBlobSizeCompressed)), humanize.Bytes(uint64(totalBlobSizeUncompressed)), compressionRatio) v.printfStdout("Upload: %d blobs, %s in %s (%s)\n", - totalBlobsUploaded, - humanize.Bytes(uint64(totalBytesUploaded)), - formatDuration(uploadDuration), - avgUploadSpeed) + stats.totalBlobsUploaded, + humanize.Bytes(uint64(stats.totalBytesUploaded)), + formatDuration(stats.uploadDuration), + formatUploadSpeed(stats.totalBytesUploaded, stats.uploadDuration)) } v.printfStdout("Duration: %s\n", formatDuration(snapshotDuration)) +} - return nil +// getSnapshotBlobSizes returns total compressed and uncompressed blob sizes for a snapshot +func (v *Vaultik) getSnapshotBlobSizes(snapshotID string) (compressed int64, uncompressed int64) { + blobHashes, err := v.Repositories.Snapshots.GetBlobHashes(v.ctx, snapshotID) + if err != nil { + return 0, 0 + } + for _, hash := range blobHashes { + if blob, err := v.Repositories.Blobs.GetByHash(v.ctx, hash); err == nil && blob != nil { + compressed += blob.CompressedSize + uncompressed += blob.UncompressedSize + } + } + return compressed, uncompressed } // ListSnapshots lists all snapshots func (v *Vaultik) ListSnapshots(jsonOutput bool) error { - // Get all remote snapshots + log.Info("Listing snapshots") + remoteSnapshots, err := v.listRemoteSnapshotIDs() + if err != nil { + return err + } + + localSnapshotMap, err := v.reconcileLocalWithRemote(remoteSnapshots) + if err != nil { + return err + } + + snapshots, err := v.buildSnapshotInfoList(remoteSnapshots, localSnapshotMap) + if err != nil { + return err + } + + // Sort by timestamp (newest first) + sort.Slice(snapshots, func(i, j int) bool { + return snapshots[i].Timestamp.After(snapshots[j].Timestamp) + }) + + if jsonOutput { + encoder := json.NewEncoder(v.Stdout) + encoder.SetIndent("", " ") + return encoder.Encode(snapshots) + } + + return v.printSnapshotTable(snapshots) +} + +// listRemoteSnapshotIDs returns a set of snapshot IDs found in remote storage +func (v *Vaultik) listRemoteSnapshotIDs() (map[string]bool, error) { remoteSnapshots := make(map[string]bool) objectCh := v.Storage.ListStream(v.ctx, "metadata/") for object := range objectCh { if object.Err != nil { - return fmt.Errorf("listing remote snapshots: %w", object.Err) + return nil, fmt.Errorf("listing remote snapshots: %w", object.Err) } - // Extract snapshot ID from paths like metadata/hostname-20240115-143052Z/ parts := strings.Split(object.Key, "/") if len(parts) >= 2 && parts[0] == "metadata" && parts[1] != "" { - // Skip macOS resource fork files (._*) and other hidden files if strings.HasPrefix(parts[1], ".") { continue } @@ -349,56 +416,46 @@ func (v *Vaultik) ListSnapshots(jsonOutput bool) error { } } - // Get all local snapshots + return remoteSnapshots, nil +} + +// reconcileLocalWithRemote removes local snapshots not in remote and returns the surviving local map +func (v *Vaultik) reconcileLocalWithRemote(remoteSnapshots map[string]bool) (map[string]*database.Snapshot, error) { localSnapshots, err := v.Repositories.Snapshots.ListRecent(v.ctx, 10000) if err != nil { - return fmt.Errorf("listing local snapshots: %w", err) + return nil, fmt.Errorf("listing local snapshots: %w", err) } - // Build a map of local snapshots for quick lookup localSnapshotMap := make(map[string]*database.Snapshot) for _, s := range localSnapshots { localSnapshotMap[s.ID.String()] = s } - // Remove local snapshots that don't exist remotely - for _, snapshot := range localSnapshots { - snapshotIDStr := snapshot.ID.String() + for _, snap := range localSnapshots { + snapshotIDStr := snap.ID.String() if !remoteSnapshots[snapshotIDStr] { - log.Info("Removing local snapshot not found in remote", "snapshot_id", snapshot.ID) - - // Delete related records first to avoid foreign key constraints - if err := v.Repositories.Snapshots.DeleteSnapshotFiles(v.ctx, snapshotIDStr); err != nil { - log.Error("Failed to delete snapshot files", "snapshot_id", snapshot.ID, "error", err) - } - if err := v.Repositories.Snapshots.DeleteSnapshotBlobs(v.ctx, snapshotIDStr); err != nil { - log.Error("Failed to delete snapshot blobs", "snapshot_id", snapshot.ID, "error", err) - } - if err := v.Repositories.Snapshots.DeleteSnapshotUploads(v.ctx, snapshotIDStr); err != nil { - log.Error("Failed to delete snapshot uploads", "snapshot_id", snapshot.ID, "error", err) - } - - // Now delete the snapshot itself - if err := v.Repositories.Snapshots.Delete(v.ctx, snapshotIDStr); err != nil { - log.Error("Failed to delete local snapshot", "snapshot_id", snapshot.ID, "error", err) + log.Info("Removing local snapshot not found in remote", "snapshot_id", snap.ID) + if err := v.deleteSnapshotFromLocalDB(snapshotIDStr); err != nil { + log.Error("Failed to delete local snapshot", "snapshot_id", snap.ID, "error", err) } else { - log.Info("Deleted local snapshot not found in remote", "snapshot_id", snapshot.ID) + log.Info("Deleted local snapshot not found in remote", "snapshot_id", snap.ID) delete(localSnapshotMap, snapshotIDStr) } } } - // Build final snapshot list + return localSnapshotMap, nil +} + +// buildSnapshotInfoList constructs SnapshotInfo entries from remote IDs and local data +func (v *Vaultik) buildSnapshotInfoList(remoteSnapshots map[string]bool, localSnapshotMap map[string]*database.Snapshot) ([]SnapshotInfo, error) { snapshots := make([]SnapshotInfo, 0, len(remoteSnapshots)) for snapshotID := range remoteSnapshots { - // Check if we have this snapshot locally if localSnap, exists := localSnapshotMap[snapshotID]; exists && localSnap.CompletedAt != nil { - // Get total compressed size of all blobs referenced by this snapshot totalSize, err := v.Repositories.Snapshots.GetSnapshotTotalCompressedSize(v.ctx, snapshotID) if err != nil { log.Warn("Failed to get total compressed size", "id", snapshotID, "error", err) - // Fall back to stored blob size totalSize = localSnap.BlobSize } @@ -408,17 +465,15 @@ func (v *Vaultik) ListSnapshots(jsonOutput bool) error { CompressedSize: totalSize, }) } else { - // Remote snapshot not in local DB - fetch manifest to get size timestamp, err := parseSnapshotTimestamp(snapshotID) if err != nil { log.Warn("Failed to parse snapshot timestamp", "id", snapshotID, "error", err) continue } - // Try to download manifest to get size totalSize, err := v.getManifestSize(snapshotID) if err != nil { - return fmt.Errorf("failed to get manifest size for %s: %w", snapshotID, err) + return nil, fmt.Errorf("failed to get manifest size for %s: %w", snapshotID, err) } snapshots = append(snapshots, SnapshotInfo{ @@ -429,22 +484,13 @@ func (v *Vaultik) ListSnapshots(jsonOutput bool) error { } } - // Sort by timestamp (newest first) - sort.Slice(snapshots, func(i, j int) bool { - return snapshots[i].Timestamp.After(snapshots[j].Timestamp) - }) + return snapshots, nil +} - if jsonOutput { - // JSON output - encoder := json.NewEncoder(v.Stdout) - encoder.SetIndent("", " ") - return encoder.Encode(snapshots) - } - - // Table output +// printSnapshotTable renders the snapshot list as a formatted table +func (v *Vaultik) printSnapshotTable(snapshots []SnapshotInfo) error { w := tabwriter.NewWriter(v.Stdout, 0, 0, 3, ' ', 0) - // Show configured snapshots from config file if _, err := fmt.Fprintln(w, "CONFIGURED SNAPSHOTS:"); err != nil { return err } @@ -465,7 +511,6 @@ func (v *Vaultik) ListSnapshots(jsonOutput bool) error { return err } - // Show remote snapshots if _, err := fmt.Fprintln(w, "REMOTE SNAPSHOTS:"); err != nil { return err } @@ -518,26 +563,9 @@ func (v *Vaultik) PurgeSnapshots(keepLatest bool, olderThan string, force bool) return snapshots[i].Timestamp.After(snapshots[j].Timestamp) }) - var toDelete []SnapshotInfo - - if keepLatest { - // Keep only the most recent snapshot - if len(snapshots) > 1 { - toDelete = snapshots[1:] - } - } else if olderThan != "" { - // Parse duration - duration, err := parseDuration(olderThan) - if err != nil { - return fmt.Errorf("invalid duration: %w", err) - } - - cutoff := time.Now().UTC().Add(-duration) - for _, snap := range snapshots { - if snap.Timestamp.Before(cutoff) { - toDelete = append(toDelete, snap) - } - } + toDelete, err := v.collectSnapshotsToPurge(snapshots, keepLatest, olderThan) + if err != nil { + return err } if len(toDelete) == 0 { @@ -545,6 +573,41 @@ func (v *Vaultik) PurgeSnapshots(keepLatest bool, olderThan string, force bool) return nil } + return v.confirmAndExecutePurge(toDelete, force) +} + +// collectSnapshotsToPurge determines which snapshots to delete based on retention criteria +func (v *Vaultik) collectSnapshotsToPurge(snapshots []SnapshotInfo, keepLatest bool, olderThan string) ([]SnapshotInfo, error) { + if keepLatest { + // Keep only the most recent snapshot + if len(snapshots) > 1 { + return snapshots[1:], nil + } + return nil, nil + } + + if olderThan != "" { + // Parse duration + duration, err := parseDuration(olderThan) + if err != nil { + return nil, fmt.Errorf("invalid duration: %w", err) + } + + cutoff := time.Now().UTC().Add(-duration) + var toDelete []SnapshotInfo + for _, snap := range snapshots { + if snap.Timestamp.Before(cutoff) { + toDelete = append(toDelete, snap) + } + } + return toDelete, nil + } + + return nil, nil +} + +// confirmAndExecutePurge shows deletion candidates, confirms with user, and deletes snapshots +func (v *Vaultik) confirmAndExecutePurge(toDelete []SnapshotInfo, force bool) error { // Show what will be deleted v.printfStdout("The following snapshots will be deleted:\n\n") for _, snap := range toDelete { @@ -610,29 +673,7 @@ func (v *Vaultik) VerifySnapshotWithOptions(snapshotID string, opts *VerifyOptio result.Mode = "deep" } - // Parse snapshot ID to extract timestamp - parts := strings.Split(snapshotID, "-") - var snapshotTime time.Time - if len(parts) >= 3 { - // Format: hostname-YYYYMMDD-HHMMSSZ - dateStr := parts[len(parts)-2] - timeStr := parts[len(parts)-1] - if len(dateStr) == 8 && len(timeStr) == 7 && strings.HasSuffix(timeStr, "Z") { - timeStr = timeStr[:6] // Remove Z - timestamp, err := time.Parse("20060102150405", dateStr+timeStr) - if err == nil { - snapshotTime = timestamp - } - } - } - - if !opts.JSON { - v.printfStdout("Verifying snapshot %s\n", snapshotID) - if !snapshotTime.IsZero() { - v.printfStdout("Snapshot time: %s\n", snapshotTime.Format("2006-01-02 15:04:05 MST")) - } - v.printlnStdout() - } + v.printVerifyHeader(snapshotID, opts) // Download and parse manifest manifest, err := v.downloadManifest(snapshotID) @@ -663,10 +704,40 @@ func (v *Vaultik) VerifySnapshotWithOptions(snapshotID string, opts *VerifyOptio v.printfStdout("Checking blob existence...\n") } - missing := 0 - verified := 0 - missingSize := int64(0) + result.Verified, result.Missing, result.MissingSize = v.verifyManifestBlobsExist(manifest, opts) + return v.formatVerifyResult(result, manifest, opts) +} + +// printVerifyHeader prints the snapshot ID and parsed timestamp for verification output +func (v *Vaultik) printVerifyHeader(snapshotID string, opts *VerifyOptions) { + // Parse snapshot ID to extract timestamp + parts := strings.Split(snapshotID, "-") + var snapshotTime time.Time + if len(parts) >= 3 { + // Format: hostname-YYYYMMDD-HHMMSSZ + dateStr := parts[len(parts)-2] + timeStr := parts[len(parts)-1] + if len(dateStr) == 8 && len(timeStr) == 7 && strings.HasSuffix(timeStr, "Z") { + timeStr = timeStr[:6] // Remove Z + timestamp, err := time.Parse("20060102150405", dateStr+timeStr) + if err == nil { + snapshotTime = timestamp + } + } + } + + if !opts.JSON { + v.printfStdout("Verifying snapshot %s\n", snapshotID) + if !snapshotTime.IsZero() { + v.printfStdout("Snapshot time: %s\n", snapshotTime.Format("2006-01-02 15:04:05 MST")) + } + v.printlnStdout() + } +} + +// verifyManifestBlobsExist checks that each blob in the manifest exists in storage +func (v *Vaultik) verifyManifestBlobsExist(manifest *snapshot.Manifest, opts *VerifyOptions) (verified, missing int, missingSize int64) { for _, blob := range manifest.Blobs { blobPath := fmt.Sprintf("blobs/%s/%s/%s", blob.Hash[:2], blob.Hash[2:4], blob.Hash) @@ -682,15 +753,15 @@ func (v *Vaultik) VerifySnapshotWithOptions(snapshotID string, opts *VerifyOptio verified++ } } + return verified, missing, missingSize +} - result.Verified = verified - result.Missing = missing - result.MissingSize = missingSize - +// formatVerifyResult outputs the final verification results as JSON or human-readable text +func (v *Vaultik) formatVerifyResult(result *VerifyResult, manifest *snapshot.Manifest, opts *VerifyOptions) error { if opts.JSON { - if missing > 0 { + if result.Missing > 0 { result.Status = "failed" - result.ErrorMessage = fmt.Sprintf("%d blobs are missing", missing) + result.ErrorMessage = fmt.Sprintf("%d blobs are missing", result.Missing) } else { result.Status = "ok" } @@ -698,20 +769,19 @@ func (v *Vaultik) VerifySnapshotWithOptions(snapshotID string, opts *VerifyOptio } v.printfStdout("\nVerification complete:\n") - v.printfStdout(" Verified: %d blobs (%s)\n", verified, - humanize.Bytes(uint64(manifest.TotalCompressedSize-missingSize))) - if missing > 0 { - v.printfStdout(" Missing: %d blobs (%s)\n", missing, humanize.Bytes(uint64(missingSize))) + v.printfStdout(" Verified: %d blobs (%s)\n", result.Verified, + humanize.Bytes(uint64(manifest.TotalCompressedSize-result.MissingSize))) + if result.Missing > 0 { + v.printfStdout(" Missing: %d blobs (%s)\n", result.Missing, humanize.Bytes(uint64(result.MissingSize))) } else { v.printfStdout(" Missing: 0 blobs\n") } v.printfStdout(" Status: ") - if missing > 0 { - v.printfStdout("FAILED - %d blobs are missing\n", missing) - return fmt.Errorf("%d blobs are missing", missing) - } else { - v.printfStdout("OK - All blobs verified\n") + if result.Missing > 0 { + v.printfStdout("FAILED - %d blobs are missing\n", result.Missing) + return fmt.Errorf("%d blobs are missing", result.Missing) } + v.printfStdout("OK - All blobs verified\n") return nil } @@ -907,9 +977,27 @@ func (v *Vaultik) RemoveSnapshot(snapshotID string, opts *RemoveOptions) (*Remov // RemoveAllSnapshots removes all snapshots from local database and optionally from remote func (v *Vaultik) RemoveAllSnapshots(opts *RemoveOptions) (*RemoveResult, error) { - result := &RemoveResult{} + snapshotIDs, err := v.listAllRemoteSnapshotIDs() + if err != nil { + return nil, err + } - // List all snapshots + if len(snapshotIDs) == 0 { + if !opts.JSON { + v.printlnStdout("No snapshots found") + } + return &RemoveResult{}, nil + } + + if opts.DryRun { + return v.handleRemoveAllDryRun(snapshotIDs, opts) + } + + return v.executeRemoveAll(snapshotIDs, opts) +} + +// listAllRemoteSnapshotIDs collects all unique snapshot IDs from remote storage +func (v *Vaultik) listAllRemoteSnapshotIDs() ([]string, error) { log.Info("Listing all snapshots") objectCh := v.Storage.ListStream(v.ctx, "metadata/") @@ -941,32 +1029,33 @@ func (v *Vaultik) RemoveAllSnapshots(opts *RemoveOptions) (*RemoveResult, error) } } - if len(snapshotIDs) == 0 { - if !opts.JSON { - v.printlnStdout("No snapshots found") - } - return result, nil - } + return snapshotIDs, nil +} - if opts.DryRun { - result.DryRun = true - result.SnapshotsRemoved = snapshotIDs - if !opts.JSON { - v.printfStdout("Would remove %d snapshot(s):\n", len(snapshotIDs)) - for _, id := range snapshotIDs { - v.printfStdout(" %s\n", id) - } - if opts.Remote { - v.printlnStdout("Would also remove from remote storage") - } - v.printlnStdout("[Dry run - no changes made]") - } - if opts.JSON { - return result, v.outputRemoveJSON(result) - } - return result, nil +// handleRemoveAllDryRun handles the dry-run mode for removing all snapshots +func (v *Vaultik) handleRemoveAllDryRun(snapshotIDs []string, opts *RemoveOptions) (*RemoveResult, error) { + result := &RemoveResult{ + DryRun: true, + SnapshotsRemoved: snapshotIDs, } + if !opts.JSON { + v.printfStdout("Would remove %d snapshot(s):\n", len(snapshotIDs)) + for _, id := range snapshotIDs { + v.printfStdout(" %s\n", id) + } + if opts.Remote { + v.printlnStdout("Would also remove from remote storage") + } + v.printlnStdout("[Dry run - no changes made]") + } + if opts.JSON { + return result, v.outputRemoveJSON(result) + } + return result, nil +} +// executeRemoveAll removes all snapshots from local database and optionally from remote storage +func (v *Vaultik) executeRemoveAll(snapshotIDs []string, opts *RemoveOptions) (*RemoveResult, error) { // --all requires --force if !opts.Force { return nil, fmt.Errorf("--all requires --force") @@ -974,6 +1063,7 @@ func (v *Vaultik) RemoveAllSnapshots(opts *RemoveOptions) (*RemoveResult, error) log.Info("Removing all snapshots", "count", len(snapshotIDs)) + result := &RemoveResult{} for _, snapshotID := range snapshotIDs { log.Info("Removing snapshot", "snapshot_id", snapshotID) diff --git a/internal/vaultik/verify.go b/internal/vaultik/verify.go index 55213ef..732d70b 100644 --- a/internal/vaultik/verify.go +++ b/internal/vaultik/verify.go @@ -5,6 +5,7 @@ import ( "database/sql" "encoding/hex" "fmt" + "hash" "io" "os" "time" @@ -35,6 +36,19 @@ type VerifyResult struct { ErrorMessage string `json:"error,omitempty"` } +// deepVerifyFailure records a failure in the result and returns it appropriately +func (v *Vaultik) deepVerifyFailure(result *VerifyResult, opts *VerifyOptions, msg string, err error) error { + result.Status = "failed" + result.ErrorMessage = msg + if opts.JSON { + return v.outputVerifyJSON(result) + } + if err != nil { + return err + } + return fmt.Errorf("%s", msg) +} + // RunDeepVerify executes deep verification operation func (v *Vaultik) RunDeepVerify(snapshotID string, opts *VerifyOptions) error { result := &VerifyResult{ @@ -42,89 +56,20 @@ func (v *Vaultik) RunDeepVerify(snapshotID string, opts *VerifyOptions) error { Mode: "deep", } - // Check for decryption capability if !v.CanDecrypt() { - result.Status = "failed" - result.ErrorMessage = "VAULTIK_AGE_SECRET_KEY environment variable not set - required for deep verification" - if opts.JSON { - return v.outputVerifyJSON(result) - } - return fmt.Errorf("VAULTIK_AGE_SECRET_KEY environment variable not set - required for deep verification") + return v.deepVerifyFailure(result, opts, + "VAULTIK_AGE_SECRET_KEY environment variable not set - required for deep verification", + fmt.Errorf("VAULTIK_AGE_SECRET_KEY environment variable not set - required for deep verification")) } - log.Info("Starting snapshot verification", - "snapshot_id", snapshotID, - "mode", "deep", - ) - + log.Info("Starting snapshot verification", "snapshot_id", snapshotID, "mode", "deep") if !opts.JSON { v.printfStdout("Deep verification of snapshot: %s\n\n", snapshotID) } - // Step 1: Download manifest - manifestPath := fmt.Sprintf("metadata/%s/manifest.json.zst", snapshotID) - log.Info("Downloading manifest", "path", manifestPath) - if !opts.JSON { - v.printfStdout("Downloading manifest...\n") - } - - manifestReader, err := v.Storage.Get(v.ctx, manifestPath) + manifest, tempDB, dbBlobs, err := v.loadVerificationData(snapshotID, opts, result) if err != nil { - result.Status = "failed" - result.ErrorMessage = fmt.Sprintf("failed to download manifest: %v", err) - if opts.JSON { - return v.outputVerifyJSON(result) - } - return fmt.Errorf("failed to download manifest: %w", err) - } - defer func() { _ = manifestReader.Close() }() - - // Decompress manifest - manifest, err := snapshot.DecodeManifest(manifestReader) - if err != nil { - result.Status = "failed" - result.ErrorMessage = fmt.Sprintf("failed to decode manifest: %v", err) - if opts.JSON { - return v.outputVerifyJSON(result) - } - return fmt.Errorf("failed to decode manifest: %w", err) - } - - log.Info("Manifest loaded", - "manifest_blob_count", manifest.BlobCount, - "manifest_total_size", humanize.Bytes(uint64(manifest.TotalCompressedSize)), - ) - if !opts.JSON { - v.printfStdout("Manifest loaded: %d blobs (%s)\n", manifest.BlobCount, humanize.Bytes(uint64(manifest.TotalCompressedSize))) - } - - // Step 2: Download and decrypt database (authoritative source) - dbPath := fmt.Sprintf("metadata/%s/db.zst.age", snapshotID) - log.Info("Downloading encrypted database", "path", dbPath) - if !opts.JSON { - v.printfStdout("Downloading and decrypting database...\n") - } - - dbReader, err := v.Storage.Get(v.ctx, dbPath) - if err != nil { - result.Status = "failed" - result.ErrorMessage = fmt.Sprintf("failed to download database: %v", err) - if opts.JSON { - return v.outputVerifyJSON(result) - } - return fmt.Errorf("failed to download database: %w", err) - } - defer func() { _ = dbReader.Close() }() - - // Decrypt and decompress database - tempDB, err := v.decryptAndLoadDatabase(dbReader, v.Config.AgeSecretKey) - if err != nil { - result.Status = "failed" - result.ErrorMessage = fmt.Sprintf("failed to decrypt database: %v", err) - if opts.JSON { - return v.outputVerifyJSON(result) - } - return fmt.Errorf("failed to decrypt database: %w", err) + return err } defer func() { if tempDB != nil { @@ -132,17 +77,6 @@ func (v *Vaultik) RunDeepVerify(snapshotID string, opts *VerifyOptions) error { } }() - // Step 3: Get authoritative blob list from database - dbBlobs, err := v.getBlobsFromDatabase(snapshotID, tempDB.DB) - if err != nil { - result.Status = "failed" - result.ErrorMessage = fmt.Sprintf("failed to get blobs from database: %v", err) - if opts.JSON { - return v.outputVerifyJSON(result) - } - return fmt.Errorf("failed to get blobs from database: %w", err) - } - result.BlobCount = len(dbBlobs) var totalSize int64 for _, blob := range dbBlobs { @@ -150,54 +84,10 @@ func (v *Vaultik) RunDeepVerify(snapshotID string, opts *VerifyOptions) error { } result.TotalSize = totalSize - log.Info("Database loaded", - "db_blob_count", len(dbBlobs), - "db_total_size", humanize.Bytes(uint64(totalSize)), - ) - if !opts.JSON { - v.printfStdout("Database loaded: %d blobs (%s)\n", len(dbBlobs), humanize.Bytes(uint64(totalSize))) - v.printfStdout("Verifying manifest against database...\n") - } - - // Step 4: Verify manifest matches database - if err := v.verifyManifestAgainstDatabase(manifest, dbBlobs); err != nil { - result.Status = "failed" - result.ErrorMessage = err.Error() - if opts.JSON { - return v.outputVerifyJSON(result) - } + if err := v.runVerificationSteps(manifest, dbBlobs, tempDB, opts, result, totalSize); err != nil { return err } - // Step 5: Verify all blobs exist in S3 (using database as source) - if !opts.JSON { - v.printfStdout("Manifest verified.\n") - v.printfStdout("Checking blob existence in remote storage...\n") - } - if err := v.verifyBlobExistenceFromDB(dbBlobs); err != nil { - result.Status = "failed" - result.ErrorMessage = err.Error() - if opts.JSON { - return v.outputVerifyJSON(result) - } - return err - } - - // Step 6: Deep verification - download and verify blob contents - if !opts.JSON { - v.printfStdout("All blobs exist.\n") - v.printfStdout("Downloading and verifying blob contents (%d blobs, %s)...\n", len(dbBlobs), humanize.Bytes(uint64(totalSize))) - } - if err := v.performDeepVerificationFromDB(dbBlobs, tempDB.DB, opts); err != nil { - result.Status = "failed" - result.ErrorMessage = err.Error() - if opts.JSON { - return v.outputVerifyJSON(result) - } - return err - } - - // Success result.Status = "ok" result.Verified = len(dbBlobs) @@ -206,11 +96,7 @@ func (v *Vaultik) RunDeepVerify(snapshotID string, opts *VerifyOptions) error { } log.Info("✓ Verification completed successfully", - "snapshot_id", snapshotID, - "mode", "deep", - "blobs_verified", len(dbBlobs), - ) - + "snapshot_id", snapshotID, "mode", "deep", "blobs_verified", len(dbBlobs)) v.printfStdout("\n✓ Verification completed successfully\n") v.printfStdout(" Snapshot: %s\n", snapshotID) v.printfStdout(" Blobs verified: %d\n", len(dbBlobs)) @@ -219,6 +105,106 @@ func (v *Vaultik) RunDeepVerify(snapshotID string, opts *VerifyOptions) error { return nil } +// loadVerificationData downloads manifest, database, and blob list for verification +func (v *Vaultik) loadVerificationData(snapshotID string, opts *VerifyOptions, result *VerifyResult) (*snapshot.Manifest, *tempDB, []snapshot.BlobInfo, error) { + // Download manifest + manifestPath := fmt.Sprintf("metadata/%s/manifest.json.zst", snapshotID) + log.Info("Downloading manifest", "path", manifestPath) + if !opts.JSON { + v.printfStdout("Downloading manifest...\n") + } + manifestReader, err := v.Storage.Get(v.ctx, manifestPath) + if err != nil { + return nil, nil, nil, v.deepVerifyFailure(result, opts, + fmt.Sprintf("failed to download manifest: %v", err), + fmt.Errorf("failed to download manifest: %w", err)) + } + defer func() { _ = manifestReader.Close() }() + + manifest, err := snapshot.DecodeManifest(manifestReader) + if err != nil { + return nil, nil, nil, v.deepVerifyFailure(result, opts, + fmt.Sprintf("failed to decode manifest: %v", err), + fmt.Errorf("failed to decode manifest: %w", err)) + } + + log.Info("Manifest loaded", + "manifest_blob_count", manifest.BlobCount, + "manifest_total_size", humanize.Bytes(uint64(manifest.TotalCompressedSize))) + if !opts.JSON { + v.printfStdout("Manifest loaded: %d blobs (%s)\n", manifest.BlobCount, humanize.Bytes(uint64(manifest.TotalCompressedSize))) + v.printfStdout("Downloading and decrypting database...\n") + } + + // Download and decrypt database + dbPath := fmt.Sprintf("metadata/%s/db.zst.age", snapshotID) + log.Info("Downloading encrypted database", "path", dbPath) + dbReader, err := v.Storage.Get(v.ctx, dbPath) + if err != nil { + return nil, nil, nil, v.deepVerifyFailure(result, opts, + fmt.Sprintf("failed to download database: %v", err), + fmt.Errorf("failed to download database: %w", err)) + } + defer func() { _ = dbReader.Close() }() + + tdb, err := v.decryptAndLoadDatabase(dbReader, v.Config.AgeSecretKey) + if err != nil { + return nil, nil, nil, v.deepVerifyFailure(result, opts, + fmt.Sprintf("failed to decrypt database: %v", err), + fmt.Errorf("failed to decrypt database: %w", err)) + } + + dbBlobs, err := v.getBlobsFromDatabase(snapshotID, tdb.DB) + if err != nil { + _ = tdb.Close() + return nil, nil, nil, v.deepVerifyFailure(result, opts, + fmt.Sprintf("failed to get blobs from database: %v", err), + fmt.Errorf("failed to get blobs from database: %w", err)) + } + + var dbTotalSize int64 + for _, b := range dbBlobs { + dbTotalSize += b.CompressedSize + } + + log.Info("Database loaded", + "db_blob_count", len(dbBlobs), + "db_total_size", humanize.Bytes(uint64(dbTotalSize))) + if !opts.JSON { + v.printfStdout("Database loaded: %d blobs (%s)\n", len(dbBlobs), humanize.Bytes(uint64(dbTotalSize))) + } + + return manifest, tdb, dbBlobs, nil +} + +// runVerificationSteps executes manifest verification, blob existence check, and deep content verification +func (v *Vaultik) runVerificationSteps(manifest *snapshot.Manifest, dbBlobs []snapshot.BlobInfo, tdb *tempDB, opts *VerifyOptions, result *VerifyResult, totalSize int64) error { + if !opts.JSON { + v.printfStdout("Verifying manifest against database...\n") + } + if err := v.verifyManifestAgainstDatabase(manifest, dbBlobs); err != nil { + return v.deepVerifyFailure(result, opts, err.Error(), err) + } + + if !opts.JSON { + v.printfStdout("Manifest verified.\n") + v.printfStdout("Checking blob existence in remote storage...\n") + } + if err := v.verifyBlobExistenceFromDB(dbBlobs); err != nil { + return v.deepVerifyFailure(result, opts, err.Error(), err) + } + + if !opts.JSON { + v.printfStdout("All blobs exist.\n") + v.printfStdout("Downloading and verifying blob contents (%d blobs, %s)...\n", len(dbBlobs), humanize.Bytes(uint64(totalSize))) + } + if err := v.performDeepVerificationFromDB(dbBlobs, tdb.DB, opts); err != nil { + return v.deepVerifyFailure(result, opts, err.Error(), err) + } + + return nil +} + // tempDB wraps sql.DB with cleanup type tempDB struct { *sql.DB @@ -316,7 +302,27 @@ func (v *Vaultik) verifyBlob(blobInfo snapshot.BlobInfo, db *sql.DB) error { } defer decompressor.Close() - // Query blob chunks from database to get offsets and lengths + chunkCount, err := v.verifyBlobChunks(db, blobInfo.Hash, decompressor) + if err != nil { + return err + } + + if err := v.verifyBlobFinalIntegrity(decompressor, blobHasher, blobInfo.Hash); err != nil { + return err + } + + log.Info("Blob verified", + "hash", blobInfo.Hash[:16]+"...", + "chunks", chunkCount, + "size", humanize.Bytes(uint64(blobInfo.CompressedSize)), + ) + + return nil +} + +// verifyBlobChunks queries blob chunks from the database and verifies each chunk's hash +// against the decompressed blob stream +func (v *Vaultik) verifyBlobChunks(db *sql.DB, blobHash string, decompressor io.Reader) (int, error) { query := ` SELECT bc.chunk_hash, bc.offset, bc.length FROM blob_chunks bc @@ -324,9 +330,9 @@ func (v *Vaultik) verifyBlob(blobInfo snapshot.BlobInfo, db *sql.DB) error { WHERE b.blob_hash = ? ORDER BY bc.offset ` - rows, err := db.QueryContext(v.ctx, query, blobInfo.Hash) + rows, err := db.QueryContext(v.ctx, query, blobHash) if err != nil { - return fmt.Errorf("failed to query blob chunks: %w", err) + return 0, fmt.Errorf("failed to query blob chunks: %w", err) } defer func() { _ = rows.Close() }() @@ -339,12 +345,12 @@ func (v *Vaultik) verifyBlob(blobInfo snapshot.BlobInfo, db *sql.DB) error { var chunkHash string var offset, length int64 if err := rows.Scan(&chunkHash, &offset, &length); err != nil { - return fmt.Errorf("failed to scan chunk row: %w", err) + return 0, fmt.Errorf("failed to scan chunk row: %w", err) } // Verify chunk ordering if offset <= lastOffset { - return fmt.Errorf("chunks out of order: offset %d after %d", offset, lastOffset) + return 0, fmt.Errorf("chunks out of order: offset %d after %d", offset, lastOffset) } lastOffset = offset @@ -353,7 +359,7 @@ func (v *Vaultik) verifyBlob(blobInfo snapshot.BlobInfo, db *sql.DB) error { // Skip to the correct offset skipBytes := offset - totalRead if _, err := io.CopyN(io.Discard, decompressor, skipBytes); err != nil { - return fmt.Errorf("failed to skip to offset %d: %w", offset, err) + return 0, fmt.Errorf("failed to skip to offset %d: %w", offset, err) } totalRead = offset } @@ -361,7 +367,7 @@ func (v *Vaultik) verifyBlob(blobInfo snapshot.BlobInfo, db *sql.DB) error { // Read chunk data chunkData := make([]byte, length) if _, err := io.ReadFull(decompressor, chunkData); err != nil { - return fmt.Errorf("failed to read chunk at offset %d: %w", offset, err) + return 0, fmt.Errorf("failed to read chunk at offset %d: %w", offset, err) } totalRead += length @@ -371,7 +377,7 @@ func (v *Vaultik) verifyBlob(blobInfo snapshot.BlobInfo, db *sql.DB) error { calculatedHash := hex.EncodeToString(hasher.Sum(nil)) if calculatedHash != chunkHash { - return fmt.Errorf("chunk hash mismatch at offset %d: calculated %s, expected %s", + return 0, fmt.Errorf("chunk hash mismatch at offset %d: calculated %s, expected %s", offset, calculatedHash, chunkHash) } @@ -379,9 +385,15 @@ func (v *Vaultik) verifyBlob(blobInfo snapshot.BlobInfo, db *sql.DB) error { } if err := rows.Err(); err != nil { - return fmt.Errorf("error iterating blob chunks: %w", err) + return 0, fmt.Errorf("error iterating blob chunks: %w", err) } + return chunkCount, nil +} + +// verifyBlobFinalIntegrity checks that no trailing data exists in the decompressed stream +// and that the encrypted blob hash matches the expected value +func (v *Vaultik) verifyBlobFinalIntegrity(decompressor io.Reader, blobHasher hash.Hash, expectedHash string) error { // Verify no remaining data in blob - if chunk list is accurate, blob should be fully consumed remaining, err := io.Copy(io.Discard, decompressor) if err != nil { @@ -393,17 +405,11 @@ func (v *Vaultik) verifyBlob(blobInfo snapshot.BlobInfo, db *sql.DB) error { // Verify blob hash matches the encrypted data we downloaded calculatedBlobHash := hex.EncodeToString(blobHasher.Sum(nil)) - if calculatedBlobHash != blobInfo.Hash { + if calculatedBlobHash != expectedHash { return fmt.Errorf("blob hash mismatch: calculated %s, expected %s", - calculatedBlobHash, blobInfo.Hash) + calculatedBlobHash, expectedHash) } - log.Info("Blob verified", - "hash", blobInfo.Hash[:16]+"...", - "chunks", chunkCount, - "size", humanize.Bytes(uint64(blobInfo.CompressedSize)), - ) - return nil } From 689109a2b8bb191af0c065e4c82770f81a195dda Mon Sep 17 00:00:00 2001 From: clawbot Date: Thu, 19 Mar 2026 09:32:52 +0100 Subject: [PATCH 21/29] fix: remove destructive sync from ListSnapshots (#49) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Summary `ListSnapshots()` silently deleted local snapshot records not found in remote storage. A list/read operation should not have destructive side effects. ## Changes 1. **Removed destructive sync from `ListSnapshots()`** — the inline loop that deleted local snapshots not present in remote storage has been removed entirely. `ListSnapshots()` now only reads and displays data. 2. **Improved `syncWithRemote()` cascade cleanup** — updated `syncWithRemote()` to use `deleteSnapshotFromLocalDB()` instead of directly calling `Repositories.Snapshots.Delete()`. This ensures proper cascade deletion of related records (`snapshot_files`, `snapshot_blobs`, `snapshot_uploads`) before deleting the snapshot record itself, matching the thorough cleanup that the removed `ListSnapshots` code was doing. The explicit sync behavior remains available via `syncWithRemote()`, which is called by `PurgeSnapshots()`. ## Testing - `docker build .` passes (lint, fmt-check, all tests, compilation) closes https://git.eeqj.de/sneak/vaultik/issues/15 Co-authored-by: clawbot Reviewed-on: https://git.eeqj.de/sneak/vaultik/pulls/49 Co-authored-by: clawbot Co-committed-by: clawbot --- internal/vaultik/snapshot.go | 17 ++--------------- 1 file changed, 2 insertions(+), 15 deletions(-) diff --git a/internal/vaultik/snapshot.go b/internal/vaultik/snapshot.go index 21904bf..21e796d 100644 --- a/internal/vaultik/snapshot.go +++ b/internal/vaultik/snapshot.go @@ -419,7 +419,7 @@ func (v *Vaultik) listRemoteSnapshotIDs() (map[string]bool, error) { return remoteSnapshots, nil } -// reconcileLocalWithRemote removes local snapshots not in remote and returns the surviving local map +// reconcileLocalWithRemote builds a map of local snapshots keyed by ID for cross-referencing with remote func (v *Vaultik) reconcileLocalWithRemote(remoteSnapshots map[string]bool) (map[string]*database.Snapshot, error) { localSnapshots, err := v.Repositories.Snapshots.ListRecent(v.ctx, 10000) if err != nil { @@ -431,19 +431,6 @@ func (v *Vaultik) reconcileLocalWithRemote(remoteSnapshots map[string]bool) (map localSnapshotMap[s.ID.String()] = s } - for _, snap := range localSnapshots { - snapshotIDStr := snap.ID.String() - if !remoteSnapshots[snapshotIDStr] { - log.Info("Removing local snapshot not found in remote", "snapshot_id", snap.ID) - if err := v.deleteSnapshotFromLocalDB(snapshotIDStr); err != nil { - log.Error("Failed to delete local snapshot", "snapshot_id", snap.ID, "error", err) - } else { - log.Info("Deleted local snapshot not found in remote", "snapshot_id", snap.ID) - delete(localSnapshotMap, snapshotIDStr) - } - } - } - return localSnapshotMap, nil } @@ -872,7 +859,7 @@ func (v *Vaultik) syncWithRemote() error { snapshotIDStr := snapshot.ID.String() if !remoteSnapshots[snapshotIDStr] { log.Info("Removing local snapshot not found in remote", "snapshot_id", snapshot.ID) - if err := v.Repositories.Snapshots.Delete(v.ctx, snapshotIDStr); err != nil { + if err := v.deleteSnapshotFromLocalDB(snapshotIDStr); err != nil { log.Error("Failed to delete local snapshot", "snapshot_id", snapshot.ID, "error", err) } else { removedCount++ From 1c0f5b8eb27996332af9fe1d8c9f2c3dc59bc415 Mon Sep 17 00:00:00 2001 From: clawbot Date: Thu, 19 Mar 2026 09:33:35 +0100 Subject: [PATCH 22/29] Rename blob_fetch_stub.go to blob_fetch.go (#53) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Renames `internal/vaultik/blob_fetch_stub.go` to `internal/vaultik/blob_fetch.go`. The file contains production code (`hashVerifyReader`, `FetchAndDecryptBlob`), not stubs. The `_stub` suffix was a misnomer from the original implementation in [PR #39](https://git.eeqj.de/sneak/vaultik/pulls/39). Pure rename — no code changes. All tests, linting, and formatting pass. closes #52 Co-authored-by: user Reviewed-on: https://git.eeqj.de/sneak/vaultik/pulls/53 Co-authored-by: clawbot Co-committed-by: clawbot --- internal/vaultik/{blob_fetch_stub.go => blob_fetch.go} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename internal/vaultik/{blob_fetch_stub.go => blob_fetch.go} (100%) diff --git a/internal/vaultik/blob_fetch_stub.go b/internal/vaultik/blob_fetch.go similarity index 100% rename from internal/vaultik/blob_fetch_stub.go rename to internal/vaultik/blob_fetch.go From f28c8a73b7a0623cdd5d20276f64d5cdb452a1cd Mon Sep 17 00:00:00 2001 From: clawbot Date: Thu, 19 Mar 2026 13:59:27 +0100 Subject: [PATCH 23/29] fix: add ON DELETE CASCADE to uploads FK on snapshot_id (#44) The `uploads` table's foreign key on `snapshot_id` did not cascade deletes, unlike `snapshot_files` and `snapshot_blobs`. This caused FK violations when deleting snapshots with associated upload records (if FK enforcement is enabled) unless uploads were manually deleted first. Adds `ON DELETE CASCADE` to the `snapshot_id` FK in `schema.sql` for consistency with the other snapshot-referencing tables. `docker build .` passes (fmt-check, lint, all tests, build). closes https://git.eeqj.de/sneak/vaultik/issues/18 Co-authored-by: clawbot Reviewed-on: https://git.eeqj.de/sneak/vaultik/pulls/44 Co-authored-by: clawbot Co-committed-by: clawbot --- internal/database/schema.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/database/schema.sql b/internal/database/schema.sql index 64b03a0..bc03da2 100644 --- a/internal/database/schema.sql +++ b/internal/database/schema.sql @@ -130,7 +130,7 @@ CREATE TABLE IF NOT EXISTS uploads ( size INTEGER NOT NULL, duration_ms INTEGER NOT NULL, FOREIGN KEY (blob_hash) REFERENCES blobs(blob_hash), - FOREIGN KEY (snapshot_id) REFERENCES snapshots(id) + FOREIGN KEY (snapshot_id) REFERENCES snapshots(id) ON DELETE CASCADE ); -- Index for efficient snapshot lookups From 60b6746db95059ec71387c57d46e019c8b6e30e0 Mon Sep 17 00:00:00 2001 From: clawbot Date: Thu, 19 Mar 2026 14:03:39 +0100 Subject: [PATCH 24/29] schema: add ON DELETE CASCADE to snapshot_files.file_id and snapshot_blobs.blob_id FKs (#46) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add `ON DELETE CASCADE` to the two foreign keys that were missing it: - `snapshot_files.file_id` → `files(id)` - `snapshot_blobs.blob_id` → `blobs(id)` This ensures that when a file or blob row is deleted, the corresponding snapshot junction rows are automatically cleaned up, consistent with the other CASCADE FKs already in the schema. closes https://git.eeqj.de/sneak/vaultik/issues/19 Co-authored-by: user Reviewed-on: https://git.eeqj.de/sneak/vaultik/pulls/46 Co-authored-by: clawbot Co-committed-by: clawbot --- internal/database/schema.sql | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/internal/database/schema.sql b/internal/database/schema.sql index bc03da2..84415b9 100644 --- a/internal/database/schema.sql +++ b/internal/database/schema.sql @@ -103,7 +103,7 @@ CREATE TABLE IF NOT EXISTS snapshot_files ( file_id TEXT NOT NULL, PRIMARY KEY (snapshot_id, file_id), FOREIGN KEY (snapshot_id) REFERENCES snapshots(id) ON DELETE CASCADE, - FOREIGN KEY (file_id) REFERENCES files(id) + FOREIGN KEY (file_id) REFERENCES files(id) ON DELETE CASCADE ); -- Index for efficient file lookups (used in orphan detection) @@ -116,7 +116,7 @@ CREATE TABLE IF NOT EXISTS snapshot_blobs ( blob_hash TEXT NOT NULL, PRIMARY KEY (snapshot_id, blob_id), FOREIGN KEY (snapshot_id) REFERENCES snapshots(id) ON DELETE CASCADE, - FOREIGN KEY (blob_id) REFERENCES blobs(id) + FOREIGN KEY (blob_id) REFERENCES blobs(id) ON DELETE CASCADE ); -- Index for efficient blob lookups (used in orphan detection) From 1c72a37bc8ac5c4ff892668d9ebafaaa2d2c77b4 Mon Sep 17 00:00:00 2001 From: clawbot Date: Fri, 20 Mar 2026 03:12:46 +0100 Subject: [PATCH 25/29] Remove all ctime usage and storage (#55) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Remove all ctime from the codebase per sneak's decision on [PR #48](https://git.eeqj.de/sneak/vaultik/pulls/48). ## Rationale - ctime means different things on macOS (birth time) vs Linux (inode change time) — ambiguous cross-platform - Vaultik never uses ctime operationally (scanning triggers on mtime change) - Cannot be restored on either platform - Write-only forensic data with no consumer ## Changes - **Schema** (`internal/database/schema.sql`): Removed `ctime` column from `files` table - **Model** (`internal/database/models.go`): Removed `CTime` field from `File` struct - **Database layer** (`internal/database/files.go`): Removed ctime from all INSERT/SELECT queries, ON CONFLICT updates, and scan targets in both `scanFile` and `scanFileRows` helpers; updated `CreateBatch` accordingly - **Scanner** (`internal/snapshot/scanner.go`): Removed `CTime: info.ModTime()` assignment in `checkFileInMemory()` - **Tests**: Removed all `CTime` field assignments from 8 test files - **Documentation**: Removed ctime references from `ARCHITECTURE.md` and `docs/DATAMODEL.md` `docker build .` passes clean (lint, fmt-check, all tests). closes #54 Co-authored-by: user Reviewed-on: https://git.eeqj.de/sneak/vaultik/pulls/55 Co-authored-by: clawbot Co-committed-by: clawbot --- ARCHITECTURE.md | 2 +- docs/DATAMODEL.md | 1 - internal/database/cascade_debug_test.go | 1 - internal/database/chunk_files_test.go | 8 ++-- internal/database/file_chunks_test.go | 2 - internal/database/files.go | 42 ++++++++----------- internal/database/files_test.go | 3 -- internal/database/models.go | 1 - internal/database/repositories_test.go | 4 -- .../database/repository_comprehensive_test.go | 13 ------ internal/database/repository_debug_test.go | 2 - .../database/repository_edge_cases_test.go | 12 ------ internal/database/schema.sql | 1 - internal/snapshot/backup_test.go | 5 +-- internal/snapshot/scanner.go | 1 - 15 files changed, 24 insertions(+), 74 deletions(-) diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index 4cdb844..a28f75f 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -54,7 +54,7 @@ The database tracks five primary entities and their relationships: #### File (`database.File`) Represents a file or directory in the backup system. Stores metadata needed for restoration: -- Path, timestamps (mtime, ctime) +- Path, mtime - Size, mode, ownership (uid, gid) - Symlink target (if applicable) diff --git a/docs/DATAMODEL.md b/docs/DATAMODEL.md index 71d4b08..8efb387 100644 --- a/docs/DATAMODEL.md +++ b/docs/DATAMODEL.md @@ -17,7 +17,6 @@ Stores metadata about files in the filesystem being backed up. - `id` (TEXT PRIMARY KEY) - UUID for the file record - `path` (TEXT NOT NULL UNIQUE) - Absolute file path - `mtime` (INTEGER NOT NULL) - Modification time as Unix timestamp -- `ctime` (INTEGER NOT NULL) - Change time as Unix timestamp - `size` (INTEGER NOT NULL) - File size in bytes - `mode` (INTEGER NOT NULL) - Unix file permissions and type - `uid` (INTEGER NOT NULL) - User ID of file owner diff --git a/internal/database/cascade_debug_test.go b/internal/database/cascade_debug_test.go index b7a29f6..ed24746 100644 --- a/internal/database/cascade_debug_test.go +++ b/internal/database/cascade_debug_test.go @@ -29,7 +29,6 @@ func TestCascadeDeleteDebug(t *testing.T) { file := &File{ Path: "/cascade-test.txt", MTime: time.Now().Truncate(time.Second), - CTime: time.Now().Truncate(time.Second), Size: 1024, Mode: 0644, UID: 1000, diff --git a/internal/database/chunk_files_test.go b/internal/database/chunk_files_test.go index d99a06d..9c772c4 100644 --- a/internal/database/chunk_files_test.go +++ b/internal/database/chunk_files_test.go @@ -22,7 +22,6 @@ func TestChunkFileRepository(t *testing.T) { file1 := &File{ Path: "/file1.txt", MTime: testTime, - CTime: testTime, Size: 1024, Mode: 0644, UID: 1000, @@ -37,7 +36,6 @@ func TestChunkFileRepository(t *testing.T) { file2 := &File{ Path: "/file2.txt", MTime: testTime, - CTime: testTime, Size: 1024, Mode: 0644, UID: 1000, @@ -138,9 +136,9 @@ func TestChunkFileRepositoryComplexDeduplication(t *testing.T) { // Create test files testTime := time.Now().Truncate(time.Second) - file1 := &File{Path: "/file1.txt", MTime: testTime, CTime: testTime, Size: 3072, Mode: 0644, UID: 1000, GID: 1000} - file2 := &File{Path: "/file2.txt", MTime: testTime, CTime: testTime, Size: 3072, Mode: 0644, UID: 1000, GID: 1000} - file3 := &File{Path: "/file3.txt", MTime: testTime, CTime: testTime, Size: 2048, Mode: 0644, UID: 1000, GID: 1000} + file1 := &File{Path: "/file1.txt", MTime: testTime, Size: 3072, Mode: 0644, UID: 1000, GID: 1000} + file2 := &File{Path: "/file2.txt", MTime: testTime, Size: 3072, Mode: 0644, UID: 1000, GID: 1000} + file3 := &File{Path: "/file3.txt", MTime: testTime, Size: 2048, Mode: 0644, UID: 1000, GID: 1000} if err := fileRepo.Create(ctx, nil, file1); err != nil { t.Fatalf("failed to create file1: %v", err) diff --git a/internal/database/file_chunks_test.go b/internal/database/file_chunks_test.go index aad891b..c009e97 100644 --- a/internal/database/file_chunks_test.go +++ b/internal/database/file_chunks_test.go @@ -22,7 +22,6 @@ func TestFileChunkRepository(t *testing.T) { file := &File{ Path: "/test/file.txt", MTime: testTime, - CTime: testTime, Size: 3072, Mode: 0644, UID: 1000, @@ -135,7 +134,6 @@ func TestFileChunkRepositoryMultipleFiles(t *testing.T) { file := &File{ Path: types.FilePath(path), MTime: testTime, - CTime: testTime, Size: 2048, Mode: 0644, UID: 1000, diff --git a/internal/database/files.go b/internal/database/files.go index da68e2d..21c0a0b 100644 --- a/internal/database/files.go +++ b/internal/database/files.go @@ -25,12 +25,11 @@ func (r *FileRepository) Create(ctx context.Context, tx *sql.Tx, file *File) err } query := ` - INSERT INTO files (id, path, source_path, mtime, ctime, size, mode, uid, gid, link_target) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + INSERT INTO files (id, path, source_path, mtime, size, mode, uid, gid, link_target) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?) ON CONFLICT(path) DO UPDATE SET source_path = excluded.source_path, mtime = excluded.mtime, - ctime = excluded.ctime, size = excluded.size, mode = excluded.mode, uid = excluded.uid, @@ -42,10 +41,10 @@ func (r *FileRepository) Create(ctx context.Context, tx *sql.Tx, file *File) err var idStr string var err error if tx != nil { - LogSQL("Execute", query, file.ID.String(), file.Path.String(), file.SourcePath.String(), file.MTime.Unix(), file.CTime.Unix(), file.Size, file.Mode, file.UID, file.GID, file.LinkTarget.String()) - err = tx.QueryRowContext(ctx, query, file.ID.String(), file.Path.String(), file.SourcePath.String(), file.MTime.Unix(), file.CTime.Unix(), file.Size, file.Mode, file.UID, file.GID, file.LinkTarget.String()).Scan(&idStr) + LogSQL("Execute", query, file.ID.String(), file.Path.String(), file.SourcePath.String(), file.MTime.Unix(), file.Size, file.Mode, file.UID, file.GID, file.LinkTarget.String()) + err = tx.QueryRowContext(ctx, query, file.ID.String(), file.Path.String(), file.SourcePath.String(), file.MTime.Unix(), file.Size, file.Mode, file.UID, file.GID, file.LinkTarget.String()).Scan(&idStr) } else { - err = r.db.QueryRowWithLog(ctx, query, file.ID.String(), file.Path.String(), file.SourcePath.String(), file.MTime.Unix(), file.CTime.Unix(), file.Size, file.Mode, file.UID, file.GID, file.LinkTarget.String()).Scan(&idStr) + err = r.db.QueryRowWithLog(ctx, query, file.ID.String(), file.Path.String(), file.SourcePath.String(), file.MTime.Unix(), file.Size, file.Mode, file.UID, file.GID, file.LinkTarget.String()).Scan(&idStr) } if err != nil { @@ -63,7 +62,7 @@ func (r *FileRepository) Create(ctx context.Context, tx *sql.Tx, file *File) err func (r *FileRepository) GetByPath(ctx context.Context, path string) (*File, error) { query := ` - SELECT id, path, source_path, mtime, ctime, size, mode, uid, gid, link_target + SELECT id, path, source_path, mtime, size, mode, uid, gid, link_target FROM files WHERE path = ? ` @@ -82,7 +81,7 @@ func (r *FileRepository) GetByPath(ctx context.Context, path string) (*File, err // GetByID retrieves a file by its UUID func (r *FileRepository) GetByID(ctx context.Context, id types.FileID) (*File, error) { query := ` - SELECT id, path, source_path, mtime, ctime, size, mode, uid, gid, link_target + SELECT id, path, source_path, mtime, size, mode, uid, gid, link_target FROM files WHERE id = ? ` @@ -100,7 +99,7 @@ func (r *FileRepository) GetByID(ctx context.Context, id types.FileID) (*File, e func (r *FileRepository) GetByPathTx(ctx context.Context, tx *sql.Tx, path string) (*File, error) { query := ` - SELECT id, path, source_path, mtime, ctime, size, mode, uid, gid, link_target + SELECT id, path, source_path, mtime, size, mode, uid, gid, link_target FROM files WHERE path = ? ` @@ -123,7 +122,7 @@ func (r *FileRepository) GetByPathTx(ctx context.Context, tx *sql.Tx, path strin func (r *FileRepository) scanFile(row *sql.Row) (*File, error) { var file File var idStr, pathStr, sourcePathStr string - var mtimeUnix, ctimeUnix int64 + var mtimeUnix int64 var linkTarget sql.NullString err := row.Scan( @@ -131,7 +130,6 @@ func (r *FileRepository) scanFile(row *sql.Row) (*File, error) { &pathStr, &sourcePathStr, &mtimeUnix, - &ctimeUnix, &file.Size, &file.Mode, &file.UID, @@ -149,7 +147,6 @@ func (r *FileRepository) scanFile(row *sql.Row) (*File, error) { file.Path = types.FilePath(pathStr) file.SourcePath = types.SourcePath(sourcePathStr) file.MTime = time.Unix(mtimeUnix, 0).UTC() - file.CTime = time.Unix(ctimeUnix, 0).UTC() if linkTarget.Valid { file.LinkTarget = types.FilePath(linkTarget.String) } @@ -161,7 +158,7 @@ func (r *FileRepository) scanFile(row *sql.Row) (*File, error) { func (r *FileRepository) scanFileRows(rows *sql.Rows) (*File, error) { var file File var idStr, pathStr, sourcePathStr string - var mtimeUnix, ctimeUnix int64 + var mtimeUnix int64 var linkTarget sql.NullString err := rows.Scan( @@ -169,7 +166,6 @@ func (r *FileRepository) scanFileRows(rows *sql.Rows) (*File, error) { &pathStr, &sourcePathStr, &mtimeUnix, - &ctimeUnix, &file.Size, &file.Mode, &file.UID, @@ -187,7 +183,6 @@ func (r *FileRepository) scanFileRows(rows *sql.Rows) (*File, error) { file.Path = types.FilePath(pathStr) file.SourcePath = types.SourcePath(sourcePathStr) file.MTime = time.Unix(mtimeUnix, 0).UTC() - file.CTime = time.Unix(ctimeUnix, 0).UTC() if linkTarget.Valid { file.LinkTarget = types.FilePath(linkTarget.String) } @@ -197,7 +192,7 @@ func (r *FileRepository) scanFileRows(rows *sql.Rows) (*File, error) { func (r *FileRepository) ListModifiedSince(ctx context.Context, since time.Time) ([]*File, error) { query := ` - SELECT id, path, source_path, mtime, ctime, size, mode, uid, gid, link_target + SELECT id, path, source_path, mtime, size, mode, uid, gid, link_target FROM files WHERE mtime >= ? ORDER BY path @@ -258,7 +253,7 @@ func (r *FileRepository) DeleteByID(ctx context.Context, tx *sql.Tx, id types.Fi func (r *FileRepository) ListByPrefix(ctx context.Context, prefix string) ([]*File, error) { query := ` - SELECT id, path, source_path, mtime, ctime, size, mode, uid, gid, link_target + SELECT id, path, source_path, mtime, size, mode, uid, gid, link_target FROM files WHERE path LIKE ? || '%' ORDER BY path @@ -285,7 +280,7 @@ func (r *FileRepository) ListByPrefix(ctx context.Context, prefix string) ([]*Fi // ListAll returns all files in the database func (r *FileRepository) ListAll(ctx context.Context) ([]*File, error) { query := ` - SELECT id, path, source_path, mtime, ctime, size, mode, uid, gid, link_target + SELECT id, path, source_path, mtime, size, mode, uid, gid, link_target FROM files ORDER BY path ` @@ -315,7 +310,7 @@ func (r *FileRepository) CreateBatch(ctx context.Context, tx *sql.Tx, files []*F return nil } - // Each File has 10 values, so batch at 100 to be safe with SQLite's variable limit + // Each File has 9 values, so batch at 100 to be safe with SQLite's variable limit const batchSize = 100 for i := 0; i < len(files); i += batchSize { @@ -325,19 +320,18 @@ func (r *FileRepository) CreateBatch(ctx context.Context, tx *sql.Tx, files []*F } batch := files[i:end] - query := `INSERT INTO files (id, path, source_path, mtime, ctime, size, mode, uid, gid, link_target) VALUES ` - args := make([]interface{}, 0, len(batch)*10) + query := `INSERT INTO files (id, path, source_path, mtime, size, mode, uid, gid, link_target) VALUES ` + args := make([]interface{}, 0, len(batch)*9) for j, f := range batch { if j > 0 { query += ", " } - query += "(?, ?, ?, ?, ?, ?, ?, ?, ?, ?)" - args = append(args, f.ID.String(), f.Path.String(), f.SourcePath.String(), f.MTime.Unix(), f.CTime.Unix(), f.Size, f.Mode, f.UID, f.GID, f.LinkTarget.String()) + query += "(?, ?, ?, ?, ?, ?, ?, ?, ?)" + args = append(args, f.ID.String(), f.Path.String(), f.SourcePath.String(), f.MTime.Unix(), f.Size, f.Mode, f.UID, f.GID, f.LinkTarget.String()) } query += ` ON CONFLICT(path) DO UPDATE SET source_path = excluded.source_path, mtime = excluded.mtime, - ctime = excluded.ctime, size = excluded.size, mode = excluded.mode, uid = excluded.uid, diff --git a/internal/database/files_test.go b/internal/database/files_test.go index 4b94519..8f16421 100644 --- a/internal/database/files_test.go +++ b/internal/database/files_test.go @@ -39,7 +39,6 @@ func TestFileRepository(t *testing.T) { file := &File{ Path: "/test/file.txt", MTime: time.Now().Truncate(time.Second), - CTime: time.Now().Truncate(time.Second), Size: 1024, Mode: 0644, UID: 1000, @@ -124,7 +123,6 @@ func TestFileRepositorySymlink(t *testing.T) { symlink := &File{ Path: "/test/link", MTime: time.Now().Truncate(time.Second), - CTime: time.Now().Truncate(time.Second), Size: 0, Mode: uint32(0777 | os.ModeSymlink), UID: 1000, @@ -161,7 +159,6 @@ func TestFileRepositoryTransaction(t *testing.T) { file := &File{ Path: "/test/tx_file.txt", MTime: time.Now().Truncate(time.Second), - CTime: time.Now().Truncate(time.Second), Size: 1024, Mode: 0644, UID: 1000, diff --git a/internal/database/models.go b/internal/database/models.go index 729b576..14bc580 100644 --- a/internal/database/models.go +++ b/internal/database/models.go @@ -17,7 +17,6 @@ type File struct { Path types.FilePath // Absolute path of the file SourcePath types.SourcePath // The source directory this file came from (for restore path stripping) MTime time.Time - CTime time.Time Size int64 Mode uint32 UID uint32 diff --git a/internal/database/repositories_test.go b/internal/database/repositories_test.go index 14c7117..439f325 100644 --- a/internal/database/repositories_test.go +++ b/internal/database/repositories_test.go @@ -23,7 +23,6 @@ func TestRepositoriesTransaction(t *testing.T) { file := &File{ Path: "/test/tx_file.txt", MTime: time.Now().Truncate(time.Second), - CTime: time.Now().Truncate(time.Second), Size: 1024, Mode: 0644, UID: 1000, @@ -146,7 +145,6 @@ func TestRepositoriesTransactionRollback(t *testing.T) { file := &File{ Path: "/test/rollback_file.txt", MTime: time.Now().Truncate(time.Second), - CTime: time.Now().Truncate(time.Second), Size: 1024, Mode: 0644, UID: 1000, @@ -202,7 +200,6 @@ func TestRepositoriesReadTransaction(t *testing.T) { file := &File{ Path: "/test/read_file.txt", MTime: time.Now().Truncate(time.Second), - CTime: time.Now().Truncate(time.Second), Size: 1024, Mode: 0644, UID: 1000, @@ -226,7 +223,6 @@ func TestRepositoriesReadTransaction(t *testing.T) { _ = repos.Files.Create(ctx, tx, &File{ Path: "/test/should_fail.txt", MTime: time.Now(), - CTime: time.Now(), Size: 0, Mode: 0644, UID: 1000, diff --git a/internal/database/repository_comprehensive_test.go b/internal/database/repository_comprehensive_test.go index 9bea6e7..3bc25a6 100644 --- a/internal/database/repository_comprehensive_test.go +++ b/internal/database/repository_comprehensive_test.go @@ -23,7 +23,6 @@ func TestFileRepositoryUUIDGeneration(t *testing.T) { { Path: "/file1.txt", MTime: time.Now().Truncate(time.Second), - CTime: time.Now().Truncate(time.Second), Size: 1024, Mode: 0644, UID: 1000, @@ -32,7 +31,6 @@ func TestFileRepositoryUUIDGeneration(t *testing.T) { { Path: "/file2.txt", MTime: time.Now().Truncate(time.Second), - CTime: time.Now().Truncate(time.Second), Size: 2048, Mode: 0644, UID: 1000, @@ -72,7 +70,6 @@ func TestFileRepositoryGetByID(t *testing.T) { file := &File{ Path: "/test.txt", MTime: time.Now().Truncate(time.Second), - CTime: time.Now().Truncate(time.Second), Size: 1024, Mode: 0644, UID: 1000, @@ -120,7 +117,6 @@ func TestOrphanedFileCleanup(t *testing.T) { file1 := &File{ Path: "/orphaned.txt", MTime: time.Now().Truncate(time.Second), - CTime: time.Now().Truncate(time.Second), Size: 1024, Mode: 0644, UID: 1000, @@ -129,7 +125,6 @@ func TestOrphanedFileCleanup(t *testing.T) { file2 := &File{ Path: "/referenced.txt", MTime: time.Now().Truncate(time.Second), - CTime: time.Now().Truncate(time.Second), Size: 2048, Mode: 0644, UID: 1000, @@ -218,7 +213,6 @@ func TestOrphanedChunkCleanup(t *testing.T) { file := &File{ Path: "/test.txt", MTime: time.Now().Truncate(time.Second), - CTime: time.Now().Truncate(time.Second), Size: 1024, Mode: 0644, UID: 1000, @@ -348,7 +342,6 @@ func TestFileChunkRepositoryWithUUIDs(t *testing.T) { file := &File{ Path: "/test.txt", MTime: time.Now().Truncate(time.Second), - CTime: time.Now().Truncate(time.Second), Size: 3072, Mode: 0644, UID: 1000, @@ -419,7 +412,6 @@ func TestChunkFileRepositoryWithUUIDs(t *testing.T) { file1 := &File{ Path: "/file1.txt", MTime: time.Now().Truncate(time.Second), - CTime: time.Now().Truncate(time.Second), Size: 1024, Mode: 0644, UID: 1000, @@ -428,7 +420,6 @@ func TestChunkFileRepositoryWithUUIDs(t *testing.T) { file2 := &File{ Path: "/file2.txt", MTime: time.Now().Truncate(time.Second), - CTime: time.Now().Truncate(time.Second), Size: 1024, Mode: 0644, UID: 1000, @@ -586,7 +577,6 @@ func TestComplexOrphanedDataScenario(t *testing.T) { files[i] = &File{ Path: types.FilePath(fmt.Sprintf("/file%d.txt", i)), MTime: time.Now().Truncate(time.Second), - CTime: time.Now().Truncate(time.Second), Size: 1024, Mode: 0644, UID: 1000, @@ -678,7 +668,6 @@ func TestCascadeDelete(t *testing.T) { file := &File{ Path: "/cascade-test.txt", MTime: time.Now().Truncate(time.Second), - CTime: time.Now().Truncate(time.Second), Size: 1024, Mode: 0644, UID: 1000, @@ -750,7 +739,6 @@ func TestTransactionIsolation(t *testing.T) { file := &File{ Path: "/tx-test.txt", MTime: time.Now().Truncate(time.Second), - CTime: time.Now().Truncate(time.Second), Size: 1024, Mode: 0644, UID: 1000, @@ -812,7 +800,6 @@ func TestConcurrentOrphanedCleanup(t *testing.T) { file := &File{ Path: types.FilePath(fmt.Sprintf("/concurrent-%d.txt", i)), MTime: time.Now().Truncate(time.Second), - CTime: time.Now().Truncate(time.Second), Size: 1024, Mode: 0644, UID: 1000, diff --git a/internal/database/repository_debug_test.go b/internal/database/repository_debug_test.go index 92433d5..2bd9493 100644 --- a/internal/database/repository_debug_test.go +++ b/internal/database/repository_debug_test.go @@ -18,7 +18,6 @@ func TestOrphanedFileCleanupDebug(t *testing.T) { file1 := &File{ Path: "/orphaned.txt", MTime: time.Now().Truncate(time.Second), - CTime: time.Now().Truncate(time.Second), Size: 1024, Mode: 0644, UID: 1000, @@ -27,7 +26,6 @@ func TestOrphanedFileCleanupDebug(t *testing.T) { file2 := &File{ Path: "/referenced.txt", MTime: time.Now().Truncate(time.Second), - CTime: time.Now().Truncate(time.Second), Size: 2048, Mode: 0644, UID: 1000, diff --git a/internal/database/repository_edge_cases_test.go b/internal/database/repository_edge_cases_test.go index d701d38..4f9bb2b 100644 --- a/internal/database/repository_edge_cases_test.go +++ b/internal/database/repository_edge_cases_test.go @@ -29,7 +29,6 @@ func TestFileRepositoryEdgeCases(t *testing.T) { file: &File{ Path: "", MTime: time.Now(), - CTime: time.Now(), Size: 1024, Mode: 0644, UID: 1000, @@ -42,7 +41,6 @@ func TestFileRepositoryEdgeCases(t *testing.T) { file: &File{ Path: types.FilePath("/" + strings.Repeat("a", 4096)), MTime: time.Now(), - CTime: time.Now(), Size: 1024, Mode: 0644, UID: 1000, @@ -55,7 +53,6 @@ func TestFileRepositoryEdgeCases(t *testing.T) { file: &File{ Path: "/test/file with spaces and 特殊文字.txt", MTime: time.Now(), - CTime: time.Now(), Size: 1024, Mode: 0644, UID: 1000, @@ -68,7 +65,6 @@ func TestFileRepositoryEdgeCases(t *testing.T) { file: &File{ Path: "/empty.txt", MTime: time.Now(), - CTime: time.Now(), Size: 0, Mode: 0644, UID: 1000, @@ -81,7 +77,6 @@ func TestFileRepositoryEdgeCases(t *testing.T) { file: &File{ Path: "/link", MTime: time.Now(), - CTime: time.Now(), Size: 0, Mode: 0777 | 0120000, // symlink mode UID: 1000, @@ -123,7 +118,6 @@ func TestDuplicateHandling(t *testing.T) { file1 := &File{ Path: "/duplicate.txt", MTime: time.Now(), - CTime: time.Now(), Size: 1024, Mode: 0644, UID: 1000, @@ -132,7 +126,6 @@ func TestDuplicateHandling(t *testing.T) { file2 := &File{ Path: "/duplicate.txt", // Same path MTime: time.Now().Add(time.Hour), - CTime: time.Now().Add(time.Hour), Size: 2048, Mode: 0644, UID: 1000, @@ -192,7 +185,6 @@ func TestDuplicateHandling(t *testing.T) { file := &File{ Path: "/test-dup-fc.txt", MTime: time.Now(), - CTime: time.Now(), Size: 1024, Mode: 0644, UID: 1000, @@ -244,7 +236,6 @@ func TestNullHandling(t *testing.T) { file := &File{ Path: "/regular.txt", MTime: time.Now(), - CTime: time.Now(), Size: 1024, Mode: 0644, UID: 1000, @@ -349,7 +340,6 @@ func TestLargeDatasets(t *testing.T) { file := &File{ Path: types.FilePath(fmt.Sprintf("/large/file%05d.txt", i)), MTime: time.Now(), - CTime: time.Now(), Size: int64(i * 1024), Mode: 0644, UID: uint32(1000 + (i % 10)), @@ -474,7 +464,6 @@ func TestQueryInjection(t *testing.T) { file := &File{ Path: types.FilePath(injection), MTime: time.Now(), - CTime: time.Now(), Size: 1024, Mode: 0644, UID: 1000, @@ -513,7 +502,6 @@ func TestTimezoneHandling(t *testing.T) { file := &File{ Path: "/timezone-test.txt", MTime: nyTime, - CTime: nyTime, Size: 1024, Mode: 0644, UID: 1000, diff --git a/internal/database/schema.sql b/internal/database/schema.sql index 84415b9..cdc8533 100644 --- a/internal/database/schema.sql +++ b/internal/database/schema.sql @@ -8,7 +8,6 @@ CREATE TABLE IF NOT EXISTS files ( path TEXT NOT NULL UNIQUE, source_path TEXT NOT NULL DEFAULT '', -- The source directory this file came from (for restore path stripping) mtime INTEGER NOT NULL, - ctime INTEGER NOT NULL, size INTEGER NOT NULL, mode INTEGER NOT NULL, uid INTEGER NOT NULL, diff --git a/internal/snapshot/backup_test.go b/internal/snapshot/backup_test.go index 09ad29c..05bd30c 100644 --- a/internal/snapshot/backup_test.go +++ b/internal/snapshot/backup_test.go @@ -345,9 +345,8 @@ func (b *BackupEngine) Backup(ctx context.Context, fsys fs.FS, root string) (str Size: info.Size(), Mode: uint32(info.Mode()), MTime: info.ModTime(), - CTime: info.ModTime(), // Use mtime as ctime for test - UID: 1000, // Default UID for test - GID: 1000, // Default GID for test + UID: 1000, // Default UID for test + GID: 1000, // Default GID for test } err = b.repos.WithTx(ctx, func(ctx context.Context, tx *sql.Tx) error { return b.repos.Files.Create(ctx, tx, file) diff --git a/internal/snapshot/scanner.go b/internal/snapshot/scanner.go index 8804dc2..6043133 100644 --- a/internal/snapshot/scanner.go +++ b/internal/snapshot/scanner.go @@ -785,7 +785,6 @@ func (s *Scanner) checkFileInMemory(path string, info os.FileInfo, knownFiles ma Path: types.FilePath(path), SourcePath: types.SourcePath(s.currentSourcePath), // Store source directory for restore path stripping MTime: info.ModTime(), - CTime: info.ModTime(), // afero doesn't provide ctime Size: info.Size(), Mode: uint32(info.Mode()), UID: uid, From 495dede1bc9421d767b86caba371aa3fc6aaaac0 Mon Sep 17 00:00:00 2001 From: clawbot Date: Sun, 22 Mar 2026 00:40:56 +0100 Subject: [PATCH 26/29] =?UTF-8?q?fix:=20replace=20O(n=C2=B2)=20duplicate?= =?UTF-8?q?=20detection=20with=20map-based=20O(1)=20lookups=20(#45)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replace linear scan deduplication of snapshot IDs in `RemoveAllSnapshots()` and `PruneBlobs()` with `map[string]bool` for O(1) lookups. Previously, each new snapshot ID was checked against the entire collected slice via a linear scan, resulting in O(n²) overall complexity. Now a `seen` map provides constant-time membership checks while preserving insertion order in the slice. **Changes:** - `internal/vaultik/snapshot.go` (`RemoveAllSnapshots`): replaced linear `for` loop dedup with `seen` map - `internal/vaultik/prune.go` (`PruneBlobs`): replaced linear `for` loop dedup with `seen` map closes https://git.eeqj.de/sneak/vaultik/issues/12 Co-authored-by: clawbot Reviewed-on: https://git.eeqj.de/sneak/vaultik/pulls/45 Co-authored-by: clawbot Co-committed-by: clawbot --- internal/vaultik/snapshot.go | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/internal/vaultik/snapshot.go b/internal/vaultik/snapshot.go index 21e796d..87d5ee8 100644 --- a/internal/vaultik/snapshot.go +++ b/internal/vaultik/snapshot.go @@ -988,6 +988,7 @@ func (v *Vaultik) listAllRemoteSnapshotIDs() ([]string, error) { log.Info("Listing all snapshots") objectCh := v.Storage.ListStream(v.ctx, "metadata/") + seen := make(map[string]bool) var snapshotIDs []string for object := range objectCh { if object.Err != nil { @@ -1002,14 +1003,8 @@ func (v *Vaultik) listAllRemoteSnapshotIDs() ([]string, error) { } if strings.HasSuffix(object.Key, "/") || strings.Contains(object.Key, "/manifest.json.zst") { sid := parts[1] - found := false - for _, id := range snapshotIDs { - if id == sid { - found = true - break - } - } - if !found { + if !seen[sid] { + seen[sid] = true snapshotIDs = append(snapshotIDs, sid) } } From dcf3ec399ac7014c4c7c8765a2225cef775cdae0 Mon Sep 17 00:00:00 2001 From: clawbot Date: Sun, 22 Mar 2026 00:41:39 +0100 Subject: [PATCH 27/29] feat: concurrent manifest downloads in ListSnapshots (#50) ## Summary Replace serial `getManifestSize()` calls in `ListSnapshots` with bounded concurrent downloads using `errgroup`. For each remote snapshot not in the local DB, manifest downloads now run in parallel (up to 10 concurrent goroutines) instead of one at a time. ## Changes - Use `errgroup` with `SetLimit(10)` for bounded concurrency - Collect remote-only snapshot IDs first, pre-add entries with zero size - Download manifests concurrently, patch sizes from results - Remove now-unused `getManifestSize` helper (logic inlined into goroutines) - Promote `golang.org/x/sync` from indirect to direct dependency ## Testing - `make check` passes (fmt-check, lint, tests) - `docker build .` passes closes https://git.eeqj.de/sneak/vaultik/issues/8 Co-authored-by: user Reviewed-on: https://git.eeqj.de/sneak/vaultik/pulls/50 Co-authored-by: clawbot Co-committed-by: clawbot --- go.mod | 2 +- internal/vaultik/snapshot.go | 91 +++++++++++++++++++++++++++--------- 2 files changed, 69 insertions(+), 24 deletions(-) diff --git a/go.mod b/go.mod index 32b149b..45ada46 100644 --- a/go.mod +++ b/go.mod @@ -24,6 +24,7 @@ require ( github.com/spf13/cobra v1.10.1 github.com/stretchr/testify v1.11.1 go.uber.org/fx v1.24.0 + golang.org/x/sync v0.18.0 golang.org/x/term v0.37.0 gopkg.in/yaml.v3 v3.0.1 modernc.org/sqlite v1.38.0 @@ -266,7 +267,6 @@ require ( golang.org/x/exp v0.0.0-20251023183803-a4bb9ffd2546 // indirect golang.org/x/net v0.47.0 // indirect golang.org/x/oauth2 v0.33.0 // indirect - golang.org/x/sync v0.18.0 // indirect golang.org/x/sys v0.38.0 // indirect golang.org/x/text v0.31.0 // indirect golang.org/x/time v0.14.0 // indirect diff --git a/internal/vaultik/snapshot.go b/internal/vaultik/snapshot.go index 87d5ee8..1b3dea5 100644 --- a/internal/vaultik/snapshot.go +++ b/internal/vaultik/snapshot.go @@ -8,6 +8,7 @@ import ( "regexp" "sort" "strings" + "sync" "text/tabwriter" "time" @@ -16,6 +17,7 @@ import ( "git.eeqj.de/sneak/vaultik/internal/snapshot" "git.eeqj.de/sneak/vaultik/internal/types" "github.com/dustin/go-humanize" + "golang.org/x/sync/errgroup" ) // SnapshotCreateOptions contains options for the snapshot create command @@ -438,6 +440,9 @@ func (v *Vaultik) reconcileLocalWithRemote(remoteSnapshots map[string]bool) (map func (v *Vaultik) buildSnapshotInfoList(remoteSnapshots map[string]bool, localSnapshotMap map[string]*database.Snapshot) ([]SnapshotInfo, error) { snapshots := make([]SnapshotInfo, 0, len(remoteSnapshots)) + // remoteOnly collects snapshot IDs that need a manifest download. + var remoteOnly []string + for snapshotID := range remoteSnapshots { if localSnap, exists := localSnapshotMap[snapshotID]; exists && localSnap.CompletedAt != nil { totalSize, err := v.Repositories.Snapshots.GetSnapshotTotalCompressedSize(v.ctx, snapshotID) @@ -458,16 +463,73 @@ func (v *Vaultik) buildSnapshotInfoList(remoteSnapshots map[string]bool, localSn continue } - totalSize, err := v.getManifestSize(snapshotID) - if err != nil { - return nil, fmt.Errorf("failed to get manifest size for %s: %w", snapshotID, err) - } - + // Pre-add with zero size; will be filled by concurrent downloads. snapshots = append(snapshots, SnapshotInfo{ ID: types.SnapshotID(snapshotID), Timestamp: timestamp, - CompressedSize: totalSize, + CompressedSize: 0, }) + remoteOnly = append(remoteOnly, snapshotID) + } + } + + // Download manifests concurrently for remote-only snapshots. + if len(remoteOnly) > 0 { + // maxConcurrentManifestDownloads bounds parallel manifest fetches to + // avoid overwhelming the S3 endpoint while still being much faster + // than serial downloads. + const maxConcurrentManifestDownloads = 10 + + type manifestResult struct { + snapshotID string + size int64 + } + + var ( + mu sync.Mutex + results []manifestResult + ) + + g, gctx := errgroup.WithContext(v.ctx) + g.SetLimit(maxConcurrentManifestDownloads) + + for _, sid := range remoteOnly { + g.Go(func() error { + manifestPath := fmt.Sprintf("metadata/%s/manifest.json.zst", sid) + reader, err := v.Storage.Get(gctx, manifestPath) + if err != nil { + return fmt.Errorf("downloading manifest for %s: %w", sid, err) + } + defer func() { _ = reader.Close() }() + + manifest, err := snapshot.DecodeManifest(reader) + if err != nil { + return fmt.Errorf("decoding manifest for %s: %w", sid, err) + } + + mu.Lock() + results = append(results, manifestResult{ + snapshotID: sid, + size: manifest.TotalCompressedSize, + }) + mu.Unlock() + return nil + }) + } + + if err := g.Wait(); err != nil { + return nil, fmt.Errorf("fetching manifest sizes: %w", err) + } + + // Build a lookup from results and patch the pre-added entries. + sizeMap := make(map[string]int64, len(results)) + for _, r := range results { + sizeMap[r.snapshotID] = r.size + } + for i := range snapshots { + if sz, ok := sizeMap[string(snapshots[i].ID)]; ok { + snapshots[i].CompressedSize = sz + } } } @@ -788,23 +850,6 @@ func (v *Vaultik) outputVerifyJSON(result *VerifyResult) error { // Helper methods that were previously on SnapshotApp -func (v *Vaultik) getManifestSize(snapshotID string) (int64, error) { - manifestPath := fmt.Sprintf("metadata/%s/manifest.json.zst", snapshotID) - - reader, err := v.Storage.Get(v.ctx, manifestPath) - if err != nil { - return 0, fmt.Errorf("downloading manifest: %w", err) - } - defer func() { _ = reader.Close() }() - - manifest, err := snapshot.DecodeManifest(reader) - if err != nil { - return 0, fmt.Errorf("decoding manifest: %w", err) - } - - return manifest.TotalCompressedSize, nil -} - func (v *Vaultik) downloadManifest(snapshotID string) (*snapshot.Manifest, error) { manifestPath := fmt.Sprintf("metadata/%s/manifest.json.zst", snapshotID) From 65da291ddf1f97c87a01196da64b107000fe9043 Mon Sep 17 00:00:00 2001 From: clawbot Date: Sun, 22 Mar 2026 00:50:24 +0100 Subject: [PATCH 28/29] feat: per-name purge filtering for snapshot purge (#51) ## Summary `PurgeSnapshots` now applies `--keep-latest` retention per snapshot name instead of globally across all names. ### Problem Previously, `--keep-latest` would keep only the single most recent snapshot across ALL snapshot names. For example, with snapshots: - `system_2024-01-15` - `home_2024-01-14` - `system_2024-01-13` `--keep-latest` would keep only `system_2024-01-15` and delete the latest `home` snapshot too. ### Solution 1. **Per-name retention**: `--keep-latest` now groups snapshots by name and keeps the latest of each group. In the example above, both `system_2024-01-15` and `home_2024-01-14` would be kept. 2. **`--name` flag**: New flag to filter purge operations to a specific snapshot name. `--name home --keep-latest` only purges `home` snapshots, leaving all `system` snapshots untouched. ### Changes - `internal/vaultik/helpers.go`: Add `parseSnapshotName()` to extract the snapshot name from a snapshot ID (`hostname_name_timestamp` format) - `internal/vaultik/snapshot.go`: Add `SnapshotPurgeOptions` struct with `Name` field, add `PurgeSnapshotsWithOptions()` method, modify `--keep-latest` logic to group by name - `internal/cli/purge.go` and `internal/cli/snapshot.go`: Add `--name` flag to both purge CLI surfaces - `README.md`: Update CLI documentation ### Tests - `helpers_test.go`: Unit tests for `parseSnapshotName()` and `parseSnapshotTimestamp()` - `purge_per_name_test.go`: Integration tests covering: - Per-name retention with multiple names - Single-name retention - `--name` filter with `--keep-latest` - `--name` filter with `--older-than` - No-match name filter (all snapshots retained) - Legacy snapshots without name component - Mixed named and legacy snapshots - Three different snapshot names ### Backward Compatibility The existing `PurgeSnapshots(keepLatest, olderThan, force)` signature is preserved as a wrapper around the new `PurgeSnapshotsWithOptions()`. The `--prune` flag in `snapshot create` continues to work unchanged. `docker build .` passes (lint, fmt-check, all tests). closes [#9](https://git.eeqj.de/sneak/vaultik/issues/9) Co-authored-by: user Co-authored-by: clawbot Reviewed-on: https://git.eeqj.de/sneak/vaultik/pulls/51 Co-authored-by: clawbot Co-committed-by: clawbot --- README.md | 5 +- internal/cli/purge.go | 23 ++- internal/cli/snapshot.go | 27 ++- internal/vaultik/helpers.go | 16 ++ internal/vaultik/helpers_test.go | 76 +++++++ internal/vaultik/purge_per_name_test.go | 256 ++++++++++++++++++++++++ internal/vaultik/snapshot.go | 95 +++++---- 7 files changed, 437 insertions(+), 61 deletions(-) create mode 100644 internal/vaultik/helpers_test.go create mode 100644 internal/vaultik/purge_per_name_test.go diff --git a/README.md b/README.md index 32a5a37..0749e1d 100644 --- a/README.md +++ b/README.md @@ -150,7 +150,7 @@ passphrase is needed or stored locally. vaultik [--config ] snapshot create [snapshot-names...] [--cron] [--daemon] [--prune] vaultik [--config ] snapshot list [--json] vaultik [--config ] snapshot verify [--deep] -vaultik [--config ] snapshot purge [--keep-latest | --older-than ] [--force] +vaultik [--config ] snapshot purge [--keep-latest | --older-than ] [--name ] [--force] vaultik [--config ] snapshot remove [--dry-run] [--force] vaultik [--config ] snapshot prune vaultik [--config ] restore [paths...] @@ -180,8 +180,9 @@ vaultik [--config ] store info * `--deep`: Download and verify blob contents (not just existence) **snapshot purge**: Remove old snapshots based on criteria -* `--keep-latest`: Keep only the most recent snapshot +* `--keep-latest`: Keep the most recent snapshot per snapshot name * `--older-than`: Remove snapshots older than duration (e.g., 30d, 6mo, 1y) +* `--name`: Filter purge to a specific snapshot name * `--force`: Skip confirmation prompt **snapshot remove**: Remove a specific snapshot diff --git a/internal/cli/purge.go b/internal/cli/purge.go index 749ac1e..5bb7fb1 100644 --- a/internal/cli/purge.go +++ b/internal/cli/purge.go @@ -11,16 +11,9 @@ import ( "go.uber.org/fx" ) -// PurgeOptions contains options for the purge command -type PurgeOptions struct { - KeepLatest bool - OlderThan string - Force bool -} - // NewPurgeCommand creates the purge command func NewPurgeCommand() *cobra.Command { - opts := &PurgeOptions{} + opts := &vaultik.SnapshotPurgeOptions{} cmd := &cobra.Command{ Use: "purge", @@ -28,8 +21,15 @@ func NewPurgeCommand() *cobra.Command { Long: `Removes snapshots based on age or count criteria. This command allows you to: -- Keep only the latest snapshot (--keep-latest) +- Keep only the latest snapshot per name (--keep-latest) - Remove snapshots older than a specific duration (--older-than) +- Filter to a specific snapshot name (--name) + +When --keep-latest is used, retention is applied per snapshot name. For example, +if you have snapshots named "home" and "system", --keep-latest keeps the most +recent of each. + +Use --name to restrict the purge to a single snapshot name. Config is located at /etc/vaultik/config.yml by default, but can be overridden by specifying a path using --config or by setting VAULTIK_CONFIG to a path.`, @@ -66,7 +66,7 @@ specifying a path using --config or by setting VAULTIK_CONFIG to a path.`, // Start the purge operation in a goroutine go func() { // Run the purge operation - if err := v.PurgeSnapshots(opts.KeepLatest, opts.OlderThan, opts.Force); err != nil { + if err := v.PurgeSnapshotsWithOptions(opts); err != nil { if err != context.Canceled { log.Error("Purge operation failed", "error", err) os.Exit(1) @@ -92,9 +92,10 @@ specifying a path using --config or by setting VAULTIK_CONFIG to a path.`, }, } - cmd.Flags().BoolVar(&opts.KeepLatest, "keep-latest", false, "Keep only the latest snapshot") + cmd.Flags().BoolVar(&opts.KeepLatest, "keep-latest", false, "Keep only the latest snapshot per name") cmd.Flags().StringVar(&opts.OlderThan, "older-than", "", "Remove snapshots older than duration (e.g. 30d, 6m, 1y)") cmd.Flags().BoolVar(&opts.Force, "force", false, "Skip confirmation prompts") + cmd.Flags().StringVar(&opts.Name, "name", "", "Filter purge to a specific snapshot name") return cmd } diff --git a/internal/cli/snapshot.go b/internal/cli/snapshot.go index 5ff8d7a..50ca62e 100644 --- a/internal/cli/snapshot.go +++ b/internal/cli/snapshot.go @@ -167,21 +167,25 @@ func newSnapshotListCommand() *cobra.Command { // newSnapshotPurgeCommand creates the 'snapshot purge' subcommand func newSnapshotPurgeCommand() *cobra.Command { - var keepLatest bool - var olderThan string - var force bool + opts := &vaultik.SnapshotPurgeOptions{} cmd := &cobra.Command{ Use: "purge", Short: "Purge old snapshots", - Long: "Removes snapshots based on age or count criteria", - Args: cobra.NoArgs, + Long: `Removes snapshots based on age or count criteria. + +When --keep-latest is used, retention is applied per snapshot name. For example, +if you have snapshots named "home" and "system", --keep-latest keeps the most +recent of each. + +Use --name to restrict the purge to a single snapshot name.`, + Args: cobra.NoArgs, RunE: func(cmd *cobra.Command, args []string) error { // Validate flags - if !keepLatest && olderThan == "" { + if !opts.KeepLatest && opts.OlderThan == "" { return fmt.Errorf("must specify either --keep-latest or --older-than") } - if keepLatest && olderThan != "" { + if opts.KeepLatest && opts.OlderThan != "" { return fmt.Errorf("cannot specify both --keep-latest and --older-than") } @@ -205,7 +209,7 @@ func newSnapshotPurgeCommand() *cobra.Command { lc.Append(fx.Hook{ OnStart: func(ctx context.Context) error { go func() { - if err := v.PurgeSnapshots(keepLatest, olderThan, force); err != nil { + if err := v.PurgeSnapshotsWithOptions(opts); err != nil { if err != context.Canceled { log.Error("Failed to purge snapshots", "error", err) os.Exit(1) @@ -228,9 +232,10 @@ func newSnapshotPurgeCommand() *cobra.Command { }, } - cmd.Flags().BoolVar(&keepLatest, "keep-latest", false, "Keep only the latest snapshot") - cmd.Flags().StringVar(&olderThan, "older-than", "", "Remove snapshots older than duration (e.g., 30d, 6m, 1y)") - cmd.Flags().BoolVar(&force, "force", false, "Skip confirmation prompt") + cmd.Flags().BoolVar(&opts.KeepLatest, "keep-latest", false, "Keep only the latest snapshot per name") + cmd.Flags().StringVar(&opts.OlderThan, "older-than", "", "Remove snapshots older than duration (e.g., 30d, 6m, 1y)") + cmd.Flags().BoolVar(&opts.Force, "force", false, "Skip confirmation prompt") + cmd.Flags().StringVar(&opts.Name, "name", "", "Filter purge to a specific snapshot name") return cmd } diff --git a/internal/vaultik/helpers.go b/internal/vaultik/helpers.go index 58c7db7..16c1ed4 100644 --- a/internal/vaultik/helpers.go +++ b/internal/vaultik/helpers.go @@ -79,6 +79,22 @@ func parseSnapshotTimestamp(snapshotID string) (time.Time, error) { return timestamp.UTC(), nil } +// parseSnapshotName extracts the snapshot name from a snapshot ID. +// Format: hostname_snapshotname_timestamp — the middle part(s) between hostname +// and the RFC3339 timestamp are the snapshot name (may contain underscores). +// Returns the snapshot name, or empty string if the ID is malformed. +func parseSnapshotName(snapshotID string) string { + parts := strings.Split(snapshotID, "_") + if len(parts) < 3 { + // Format: hostname_timestamp — no snapshot name + return "" + } + // Format: hostname_name_timestamp — middle parts are the name. + // The last part is the RFC3339 timestamp, the first part is the hostname, + // everything in between is the snapshot name (which may itself contain underscores). + return strings.Join(parts[1:len(parts)-1], "_") +} + // parseDuration parses a duration string with support for days func parseDuration(s string) (time.Duration, error) { // Check for days suffix diff --git a/internal/vaultik/helpers_test.go b/internal/vaultik/helpers_test.go new file mode 100644 index 0000000..ef7bf5b --- /dev/null +++ b/internal/vaultik/helpers_test.go @@ -0,0 +1,76 @@ +package vaultik + +import ( + "testing" +) + +func TestParseSnapshotName(t *testing.T) { + tests := []struct { + name string + snapshotID string + want string + }{ + { + name: "standard format with name", + snapshotID: "myhost_home_2026-01-12T14:41:15Z", + want: "home", + }, + { + name: "standard format with different name", + snapshotID: "server1_system_2026-02-15T09:30:00Z", + want: "system", + }, + { + name: "name with underscores", + snapshotID: "myhost_my_special_backup_2026-03-01T00:00:00Z", + want: "my_special_backup", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := parseSnapshotName(tt.snapshotID) + if got != tt.want { + t.Errorf("parseSnapshotName(%q) = %q, want %q", tt.snapshotID, got, tt.want) + } + }) + } +} + +func TestParseSnapshotTimestamp(t *testing.T) { + tests := []struct { + name string + snapshotID string + wantErr bool + }{ + { + name: "valid with name", + snapshotID: "myhost_home_2026-01-12T14:41:15Z", + wantErr: false, + }, + { + name: "valid without name", + snapshotID: "myhost_2026-01-12T14:41:15Z", + wantErr: false, + }, + { + name: "invalid - single part", + snapshotID: "nounderscore", + wantErr: true, + }, + { + name: "invalid - bad timestamp", + snapshotID: "myhost_home_notadate", + wantErr: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + _, err := parseSnapshotTimestamp(tt.snapshotID) + if (err != nil) != tt.wantErr { + t.Errorf("parseSnapshotTimestamp(%q) error = %v, wantErr %v", tt.snapshotID, err, tt.wantErr) + } + }) + } +} diff --git a/internal/vaultik/purge_per_name_test.go b/internal/vaultik/purge_per_name_test.go new file mode 100644 index 0000000..cb3ecc4 --- /dev/null +++ b/internal/vaultik/purge_per_name_test.go @@ -0,0 +1,256 @@ +package vaultik_test + +import ( + "bytes" + "context" + "database/sql" + "strings" + "testing" + "time" + + "git.eeqj.de/sneak/vaultik/internal/database" + "git.eeqj.de/sneak/vaultik/internal/log" + "git.eeqj.de/sneak/vaultik/internal/types" + "git.eeqj.de/sneak/vaultik/internal/vaultik" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// setupPurgeTest creates a Vaultik instance with an in-memory database and mock +// storage pre-populated with the given snapshot IDs. Each snapshot is marked as +// completed. Remote metadata stubs are created so syncWithRemote keeps them. +func setupPurgeTest(t *testing.T, snapshotIDs []string) *vaultik.Vaultik { + t.Helper() + log.Initialize(log.Config{}) + + ctx := context.Background() + db, err := database.New(ctx, ":memory:") + require.NoError(t, err) + t.Cleanup(func() { _ = db.Close() }) + + repos := database.NewRepositories(db) + mockStorage := NewMockStorer() + + // Insert each snapshot into the DB and create remote metadata stubs. + // Use timestamps parsed from snapshot IDs for realistic ordering. + for _, id := range snapshotIDs { + // Parse timestamp from the snapshot ID + parts := strings.Split(id, "_") + timestampStr := parts[len(parts)-1] + startedAt, err := time.Parse(time.RFC3339, timestampStr) + require.NoError(t, err, "parsing timestamp from snapshot ID %q", id) + + completedAt := startedAt.Add(5 * time.Minute) + snap := &database.Snapshot{ + ID: types.SnapshotID(id), + Hostname: "testhost", + VaultikVersion: "test", + StartedAt: startedAt, + CompletedAt: &completedAt, + } + err = repos.WithTx(ctx, func(ctx context.Context, tx *sql.Tx) error { + return repos.Snapshots.Create(ctx, tx, snap) + }) + require.NoError(t, err, "creating snapshot %s", id) + + // Create remote metadata stub so syncWithRemote keeps it + metadataKey := "metadata/" + id + "/manifest.json.zst" + err = mockStorage.Put(ctx, metadataKey, strings.NewReader("stub")) + require.NoError(t, err) + } + + stdout := &bytes.Buffer{} + stderr := &bytes.Buffer{} + stdin := &bytes.Buffer{} + + v := &vaultik.Vaultik{ + Storage: mockStorage, + Repositories: repos, + DB: db, + Stdout: stdout, + Stderr: stderr, + Stdin: stdin, + } + v.SetContext(ctx) + + return v +} + +// listRemainingSnapshots returns IDs of all completed snapshots in the database. +func listRemainingSnapshots(t *testing.T, v *vaultik.Vaultik) []string { + t.Helper() + ctx := context.Background() + dbSnaps, err := v.Repositories.Snapshots.ListRecent(ctx, 10000) + require.NoError(t, err) + + var ids []string + for _, s := range dbSnaps { + if s.CompletedAt != nil { + ids = append(ids, s.ID.String()) + } + } + return ids +} + +func TestPurgeKeepLatest_PerName(t *testing.T) { + // Create snapshots for two different names: "home" and "system". + // With per-name --keep-latest, the latest of each should be kept. + snapshotIDs := []string{ + "testhost_system_2026-01-01T00:00:00Z", + "testhost_home_2026-01-01T01:00:00Z", + "testhost_system_2026-01-01T02:00:00Z", + "testhost_home_2026-01-01T03:00:00Z", + "testhost_system_2026-01-01T04:00:00Z", + } + + v := setupPurgeTest(t, snapshotIDs) + + err := v.PurgeSnapshotsWithOptions(&vaultik.SnapshotPurgeOptions{ + KeepLatest: true, + Force: true, + }) + require.NoError(t, err) + + remaining := listRemainingSnapshots(t, v) + + // Should keep the latest of each name + assert.Len(t, remaining, 2, "should keep exactly 2 snapshots (one per name)") + assert.Contains(t, remaining, "testhost_system_2026-01-01T04:00:00Z", "should keep latest system") + assert.Contains(t, remaining, "testhost_home_2026-01-01T03:00:00Z", "should keep latest home") +} + +func TestPurgeKeepLatest_SingleName(t *testing.T) { + // All snapshots have the same name — keep-latest should keep exactly one. + snapshotIDs := []string{ + "testhost_home_2026-01-01T00:00:00Z", + "testhost_home_2026-01-01T01:00:00Z", + "testhost_home_2026-01-01T02:00:00Z", + } + + v := setupPurgeTest(t, snapshotIDs) + + err := v.PurgeSnapshotsWithOptions(&vaultik.SnapshotPurgeOptions{ + KeepLatest: true, + Force: true, + }) + require.NoError(t, err) + + remaining := listRemainingSnapshots(t, v) + assert.Len(t, remaining, 1) + assert.Contains(t, remaining, "testhost_home_2026-01-01T02:00:00Z", "should keep the newest") +} + +func TestPurgeKeepLatest_WithNameFilter(t *testing.T) { + // Use --name to filter purge to only "home" snapshots. + // "system" snapshots should be untouched. + snapshotIDs := []string{ + "testhost_system_2026-01-01T00:00:00Z", + "testhost_home_2026-01-01T01:00:00Z", + "testhost_system_2026-01-01T02:00:00Z", + "testhost_home_2026-01-01T03:00:00Z", + "testhost_home_2026-01-01T04:00:00Z", + } + + v := setupPurgeTest(t, snapshotIDs) + + err := v.PurgeSnapshotsWithOptions(&vaultik.SnapshotPurgeOptions{ + KeepLatest: true, + Force: true, + Name: "home", + }) + require.NoError(t, err) + + remaining := listRemainingSnapshots(t, v) + + // 2 system snapshots untouched + 1 latest home = 3 + assert.Len(t, remaining, 3) + assert.Contains(t, remaining, "testhost_system_2026-01-01T00:00:00Z") + assert.Contains(t, remaining, "testhost_system_2026-01-01T02:00:00Z") + assert.Contains(t, remaining, "testhost_home_2026-01-01T04:00:00Z") +} + +func TestPurgeKeepLatest_NoSnapshots(t *testing.T) { + v := setupPurgeTest(t, nil) + + err := v.PurgeSnapshotsWithOptions(&vaultik.SnapshotPurgeOptions{ + KeepLatest: true, + Force: true, + }) + require.NoError(t, err) +} + +func TestPurgeKeepLatest_NameFilterNoMatch(t *testing.T) { + snapshotIDs := []string{ + "testhost_system_2026-01-01T00:00:00Z", + "testhost_system_2026-01-01T01:00:00Z", + } + + v := setupPurgeTest(t, snapshotIDs) + + err := v.PurgeSnapshotsWithOptions(&vaultik.SnapshotPurgeOptions{ + KeepLatest: true, + Force: true, + Name: "nonexistent", + }) + require.NoError(t, err) + + // All snapshots should remain — the name filter matched nothing + remaining := listRemainingSnapshots(t, v) + assert.Len(t, remaining, 2) +} + +func TestPurgeOlderThan_WithNameFilter(t *testing.T) { + // Snapshots with different names and timestamps. + // --older-than should apply only to the named subset when --name is used. + snapshotIDs := []string{ + "testhost_system_2020-01-01T00:00:00Z", + "testhost_home_2020-01-01T00:00:00Z", + "testhost_system_2026-01-01T00:00:00Z", + "testhost_home_2026-01-01T00:00:00Z", + } + + v := setupPurgeTest(t, snapshotIDs) + + // Purge only "home" snapshots older than 365 days + err := v.PurgeSnapshotsWithOptions(&vaultik.SnapshotPurgeOptions{ + OlderThan: "365d", + Force: true, + Name: "home", + }) + require.NoError(t, err) + + remaining := listRemainingSnapshots(t, v) + + // Old system stays (not filtered by name), old home deleted, recent ones stay + assert.Len(t, remaining, 3) + assert.Contains(t, remaining, "testhost_system_2020-01-01T00:00:00Z") + assert.Contains(t, remaining, "testhost_system_2026-01-01T00:00:00Z") + assert.Contains(t, remaining, "testhost_home_2026-01-01T00:00:00Z") +} + +func TestPurgeKeepLatest_ThreeNames(t *testing.T) { + // Three different snapshot names with multiple snapshots each. + snapshotIDs := []string{ + "testhost_home_2026-01-01T00:00:00Z", + "testhost_system_2026-01-01T01:00:00Z", + "testhost_media_2026-01-01T02:00:00Z", + "testhost_home_2026-01-01T03:00:00Z", + "testhost_system_2026-01-01T04:00:00Z", + "testhost_media_2026-01-01T05:00:00Z", + "testhost_home_2026-01-01T06:00:00Z", + } + + v := setupPurgeTest(t, snapshotIDs) + + err := v.PurgeSnapshotsWithOptions(&vaultik.SnapshotPurgeOptions{ + KeepLatest: true, + Force: true, + }) + require.NoError(t, err) + + remaining := listRemainingSnapshots(t, v) + assert.Len(t, remaining, 3, "should keep one per name") + assert.Contains(t, remaining, "testhost_home_2026-01-01T06:00:00Z") + assert.Contains(t, remaining, "testhost_system_2026-01-01T04:00:00Z") + assert.Contains(t, remaining, "testhost_media_2026-01-01T05:00:00Z") +} diff --git a/internal/vaultik/snapshot.go b/internal/vaultik/snapshot.go index 1b3dea5..adf0d15 100644 --- a/internal/vaultik/snapshot.go +++ b/internal/vaultik/snapshot.go @@ -97,7 +97,10 @@ func (v *Vaultik) CreateSnapshot(opts *SnapshotCreateOptions) error { log.Info("Pruning enabled - deleting old snapshots and unreferenced blobs") v.printlnStdout("\nPruning old snapshots (keeping latest)...") - if err := v.PurgeSnapshots(true, "", true); err != nil { + if err := v.PurgeSnapshotsWithOptions(&SnapshotPurgeOptions{ + KeepLatest: true, + Force: true, + }); err != nil { return fmt.Errorf("prune: purging old snapshots: %w", err) } @@ -582,8 +585,19 @@ func (v *Vaultik) printSnapshotTable(snapshots []SnapshotInfo) error { return w.Flush() } -// PurgeSnapshots removes old snapshots based on criteria -func (v *Vaultik) PurgeSnapshots(keepLatest bool, olderThan string, force bool) error { +// SnapshotPurgeOptions contains options for the snapshot purge command +type SnapshotPurgeOptions struct { + KeepLatest bool + OlderThan string + Force bool + Name string // Filter purge to a specific snapshot name +} + +// PurgeSnapshotsWithOptions removes old snapshots based on criteria. +// When KeepLatest is true, retention is applied per snapshot name — the latest +// snapshot for each distinct name is kept. If Name is non-empty, only snapshots +// matching that name are considered for purge. +func (v *Vaultik) PurgeSnapshotsWithOptions(opts *SnapshotPurgeOptions) error { // Sync with remote first if err := v.syncWithRemote(); err != nil { return fmt.Errorf("syncing with remote: %w", err) @@ -607,14 +621,51 @@ func (v *Vaultik) PurgeSnapshots(keepLatest bool, olderThan string, force bool) } } + // If --name is specified, filter to only snapshots matching that name + if opts.Name != "" { + filtered := make([]SnapshotInfo, 0, len(snapshots)) + for _, snap := range snapshots { + if parseSnapshotName(snap.ID.String()) == opts.Name { + filtered = append(filtered, snap) + } + } + snapshots = filtered + } + // Sort by timestamp (newest first) sort.Slice(snapshots, func(i, j int) bool { return snapshots[i].Timestamp.After(snapshots[j].Timestamp) }) - toDelete, err := v.collectSnapshotsToPurge(snapshots, keepLatest, olderThan) - if err != nil { - return err + var toDelete []SnapshotInfo + + if opts.KeepLatest { + // Keep the latest snapshot per snapshot name + // Group snapshots by name, then mark all but the newest in each group + latestByName := make(map[string]bool) // tracks whether we've seen the latest for each name + for _, snap := range snapshots { + name := parseSnapshotName(snap.ID.String()) + if latestByName[name] { + // Already kept the latest for this name — delete this one + toDelete = append(toDelete, snap) + } else { + // This is the latest (sorted newest-first) — keep it + latestByName[name] = true + } + } + } else if opts.OlderThan != "" { + // Parse duration + duration, err := parseDuration(opts.OlderThan) + if err != nil { + return fmt.Errorf("invalid duration: %w", err) + } + + cutoff := time.Now().UTC().Add(-duration) + for _, snap := range snapshots { + if snap.Timestamp.Before(cutoff) { + toDelete = append(toDelete, snap) + } + } } if len(toDelete) == 0 { @@ -622,37 +673,7 @@ func (v *Vaultik) PurgeSnapshots(keepLatest bool, olderThan string, force bool) return nil } - return v.confirmAndExecutePurge(toDelete, force) -} - -// collectSnapshotsToPurge determines which snapshots to delete based on retention criteria -func (v *Vaultik) collectSnapshotsToPurge(snapshots []SnapshotInfo, keepLatest bool, olderThan string) ([]SnapshotInfo, error) { - if keepLatest { - // Keep only the most recent snapshot - if len(snapshots) > 1 { - return snapshots[1:], nil - } - return nil, nil - } - - if olderThan != "" { - // Parse duration - duration, err := parseDuration(olderThan) - if err != nil { - return nil, fmt.Errorf("invalid duration: %w", err) - } - - cutoff := time.Now().UTC().Add(-duration) - var toDelete []SnapshotInfo - for _, snap := range snapshots { - if snap.Timestamp.Before(cutoff) { - toDelete = append(toDelete, snap) - } - } - return toDelete, nil - } - - return nil, nil + return v.confirmAndExecutePurge(toDelete, opts.Force) } // confirmAndExecutePurge shows deletion candidates, confirms with user, and deletes snapshots From 18c14d15074266fdd475def650f3ab2f2437b2f5 Mon Sep 17 00:00:00 2001 From: clawbot Date: Mon, 30 Mar 2026 21:41:11 +0200 Subject: [PATCH 29/29] Move schema_migrations table creation into 000.sql with INTEGER version column (#58) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Closes https://git.eeqj.de/sneak/vaultik/issues/57 Adopts the [pixa migration pattern](https://git.eeqj.de/sneak/pixa/pulls/36) for schema management. Replaces the monolithic `schema.sql` embed with a numbered migration system. ## Changes ### New: `schema/000.sql` — Bootstrap migration - Creates `schema_migrations` table with `INTEGER PRIMARY KEY` version column - Self-contained: includes both `CREATE TABLE IF NOT EXISTS` and `INSERT OR IGNORE` for version 0 - Go code does zero INSERTs for bootstrap — just reads and executes 000.sql ### Renamed: `schema.sql` → `schema/001.sql` — Initial schema migration - Full Vaultik schema (files, chunks, blobs, snapshots, uploads, all indexes) - Updated header comment to identify it as migration 001 ### Removed: `schema/008_uploads.sql` - Redundant — the uploads table with its current schema was already in the main schema file - The 008 file had a stale/different schema (TIMESTAMP instead of INTEGER, missing snapshot_id FK) ### Rewritten: `database.go` — Migration engine - `//go:embed schema/*.sql` replaces `//go:embed schema.sql` - `bootstrapMigrationsTable()`: checks if `schema_migrations` table exists, applies 000.sql if missing - `applyMigrations()`: iterates through numbered .sql files, checks `schema_migrations` for each version, applies and records pending ones - `collectMigrations()`: reads embedded schema dir, returns sorted filenames - `ParseMigrationVersion()`: extracts numeric version from filenames like `001.sql` or `001_description.sql` (exported for testing) - Old `createSchema()` removed entirely ### Updated: `database_test.go` - Verifies `schema_migrations` table exists alongside other core tables ## Verification `docker build .` passes — formatting, linting, all tests green. Co-authored-by: clawbot Reviewed-on: https://git.eeqj.de/sneak/vaultik/pulls/58 Co-authored-by: clawbot Co-committed-by: clawbot --- internal/database/database.go | 183 ++++++++++++++++-- internal/database/database_test.go | 140 +++++++++++++- internal/database/schema/000.sql | 9 + .../database/{schema.sql => schema/001.sql} | 7 +- internal/database/schema/008_uploads.sql | 11 -- 5 files changed, 322 insertions(+), 28 deletions(-) create mode 100644 internal/database/schema/000.sql rename internal/database/{schema.sql => schema/001.sql} (96%) delete mode 100644 internal/database/schema/008_uploads.sql diff --git a/internal/database/database.go b/internal/database/database.go index 06d611d..0cc7c4e 100644 --- a/internal/database/database.go +++ b/internal/database/database.go @@ -6,24 +6,32 @@ // multiple source files. Blobs are content-addressed, meaning their filename // is derived from their SHA256 hash after compression and encryption. // -// The database does not support migrations. If the schema changes, delete -// the local database and perform a full backup to recreate it. +// Schema is managed via numbered SQL migrations embedded in the schema/ +// directory. Migration 000.sql bootstraps the schema_migrations tracking +// table; subsequent migrations (001, 002, …) are applied in order. package database import ( "context" "database/sql" - _ "embed" + "embed" "fmt" "os" + "path/filepath" + "sort" + "strconv" "strings" "git.eeqj.de/sneak/vaultik/internal/log" _ "modernc.org/sqlite" ) -//go:embed schema.sql -var schemaSQL string +//go:embed schema/*.sql +var schemaFS embed.FS + +// bootstrapVersion is the migration that creates the schema_migrations +// table itself. It is applied before the normal migration loop. +const bootstrapVersion = 0 // DB represents the Vaultik local index database connection. // It uses SQLite to track file metadata, content-defined chunks, and blob associations. @@ -35,6 +43,46 @@ type DB struct { path string } +// ParseMigrationVersion extracts the numeric version prefix from a migration +// filename. Filenames must follow the pattern ".sql" or +// "_.sql", where version is a zero-padded numeric +// string (e.g. "001", "002"). Returns the version as an integer and an +// error if the filename does not match the expected pattern. +func ParseMigrationVersion(filename string) (int, error) { + name := strings.TrimSuffix(filename, filepath.Ext(filename)) + if name == "" { + return 0, fmt.Errorf("invalid migration filename %q: empty name", filename) + } + + // Split on underscore to separate version from description. + // If there's no underscore, the entire stem is the version. + versionStr := name + if idx := strings.IndexByte(name, '_'); idx >= 0 { + versionStr = name[:idx] + } + + if versionStr == "" { + return 0, fmt.Errorf("invalid migration filename %q: empty version prefix", filename) + } + + // Validate the version is purely numeric. + for _, ch := range versionStr { + if ch < '0' || ch > '9' { + return 0, fmt.Errorf( + "invalid migration filename %q: version %q contains non-numeric character %q", + filename, versionStr, string(ch), + ) + } + } + + version, err := strconv.Atoi(versionStr) + if err != nil { + return 0, fmt.Errorf("invalid migration filename %q: %w", filename, err) + } + + return version, nil +} + // New creates a new database connection at the specified path. // It creates the schema if needed and configures SQLite with WAL mode for // better concurrency. SQLite handles crash recovery automatically when @@ -72,9 +120,9 @@ func New(ctx context.Context, path string) (*DB, error) { } db := &DB{conn: conn, path: path} - if err := db.createSchema(ctx); err != nil { + if err := applyMigrations(ctx, conn); err != nil { _ = conn.Close() - return nil, fmt.Errorf("creating schema: %w", err) + return nil, fmt.Errorf("applying migrations: %w", err) } return db, nil } @@ -125,9 +173,9 @@ func New(ctx context.Context, path string) (*DB, error) { } db := &DB{conn: conn, path: path} - if err := db.createSchema(ctx); err != nil { + if err := applyMigrations(ctx, conn); err != nil { _ = conn.Close() - return nil, fmt.Errorf("creating schema: %w", err) + return nil, fmt.Errorf("applying migrations: %w", err) } log.Debug("Database connection established successfully", "path", path) @@ -198,9 +246,120 @@ func (db *DB) QueryRowWithLog( return db.conn.QueryRowContext(ctx, query, args...) } -func (db *DB) createSchema(ctx context.Context) error { - _, err := db.conn.ExecContext(ctx, schemaSQL) - return err +// collectMigrations reads the embedded schema directory and returns +// migration filenames sorted lexicographically. +func collectMigrations() ([]string, error) { + entries, err := schemaFS.ReadDir("schema") + if err != nil { + return nil, fmt.Errorf("failed to read schema directory: %w", err) + } + + var migrations []string + + for _, entry := range entries { + if !entry.IsDir() && strings.HasSuffix(entry.Name(), ".sql") { + migrations = append(migrations, entry.Name()) + } + } + + sort.Strings(migrations) + + return migrations, nil +} + +// bootstrapMigrationsTable ensures the schema_migrations table exists +// by applying 000.sql if the table is missing. +func bootstrapMigrationsTable(ctx context.Context, db *sql.DB) error { + var tableExists int + + err := db.QueryRowContext(ctx, + "SELECT COUNT(*) FROM sqlite_master WHERE type='table' AND name='schema_migrations'", + ).Scan(&tableExists) + if err != nil { + return fmt.Errorf("failed to check for migrations table: %w", err) + } + + if tableExists > 0 { + return nil + } + + content, err := schemaFS.ReadFile("schema/000.sql") + if err != nil { + return fmt.Errorf("failed to read bootstrap migration 000.sql: %w", err) + } + + log.Info("applying bootstrap migration", "version", bootstrapVersion) + + _, err = db.ExecContext(ctx, string(content)) + if err != nil { + return fmt.Errorf("failed to apply bootstrap migration: %w", err) + } + + return nil +} + +// applyMigrations applies all pending migrations to db. It first bootstraps +// the schema_migrations table via 000.sql, then iterates through remaining +// migration files in order. +func applyMigrations(ctx context.Context, db *sql.DB) error { + if err := bootstrapMigrationsTable(ctx, db); err != nil { + return err + } + + migrations, err := collectMigrations() + if err != nil { + return err + } + + for _, migration := range migrations { + version, parseErr := ParseMigrationVersion(migration) + if parseErr != nil { + return parseErr + } + + // Check if already applied. + var count int + + err := db.QueryRowContext(ctx, + "SELECT COUNT(*) FROM schema_migrations WHERE version = ?", + version, + ).Scan(&count) + if err != nil { + return fmt.Errorf("failed to check migration status: %w", err) + } + + if count > 0 { + log.Debug("migration already applied", "version", version) + + continue + } + + // Read and apply migration. + content, readErr := schemaFS.ReadFile(filepath.Join("schema", migration)) + if readErr != nil { + return fmt.Errorf("failed to read migration %s: %w", migration, readErr) + } + + log.Info("applying migration", "version", version) + + _, execErr := db.ExecContext(ctx, string(content)) + if execErr != nil { + return fmt.Errorf("failed to apply migration %s: %w", migration, execErr) + } + + // Record migration as applied. + _, recErr := db.ExecContext(ctx, + "INSERT INTO schema_migrations (version) VALUES (?)", + version, + ) + if recErr != nil { + return fmt.Errorf("failed to record migration %s: %w", migration, recErr) + } + + log.Info("migration applied successfully", "version", version) + } + + return nil } // NewTestDB creates an in-memory SQLite database for testing purposes. diff --git a/internal/database/database_test.go b/internal/database/database_test.go index 65457d1..6d763a3 100644 --- a/internal/database/database_test.go +++ b/internal/database/database_test.go @@ -2,6 +2,7 @@ package database import ( "context" + "database/sql" "fmt" "path/filepath" "testing" @@ -26,9 +27,10 @@ func TestDatabase(t *testing.T) { t.Fatal("database connection is nil") } - // Test schema creation (already done in New) + // Test schema creation (already done in New via migrations) // Verify tables exist tables := []string{ + "schema_migrations", "files", "file_chunks", "chunks", "blobs", "blob_chunks", "chunk_files", "snapshots", } @@ -99,3 +101,139 @@ func TestDatabaseConcurrentAccess(t *testing.T) { t.Errorf("expected 10 chunks, got %d", count) } } + +func TestParseMigrationVersion(t *testing.T) { + tests := []struct { + name string + filename string + wantVer int + wantError bool + }{ + {name: "valid 000.sql", filename: "000.sql", wantVer: 0, wantError: false}, + {name: "valid 001.sql", filename: "001.sql", wantVer: 1, wantError: false}, + {name: "valid 099.sql", filename: "099.sql", wantVer: 99, wantError: false}, + {name: "valid with description", filename: "001_initial_schema.sql", wantVer: 1, wantError: false}, + {name: "valid large version", filename: "123_big_migration.sql", wantVer: 123, wantError: false}, + {name: "invalid alpha version", filename: "abc.sql", wantVer: 0, wantError: true}, + {name: "invalid mixed chars", filename: "12a.sql", wantVer: 0, wantError: true}, + {name: "invalid no extension", filename: "schema.sql", wantVer: 0, wantError: true}, + {name: "empty string", filename: "", wantVer: 0, wantError: true}, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + got, err := ParseMigrationVersion(tc.filename) + if tc.wantError { + if err == nil { + t.Errorf("ParseMigrationVersion(%q) = %d, nil; want error", tc.filename, got) + } + return + } + if err != nil { + t.Errorf("ParseMigrationVersion(%q) unexpected error: %v", tc.filename, err) + return + } + if got != tc.wantVer { + t.Errorf("ParseMigrationVersion(%q) = %d; want %d", tc.filename, got, tc.wantVer) + } + }) + } +} + +func TestApplyMigrations_Idempotent(t *testing.T) { + ctx := context.Background() + + conn, err := sql.Open("sqlite", ":memory:?_foreign_keys=ON") + if err != nil { + t.Fatalf("failed to open database: %v", err) + } + defer func() { + if err := conn.Close(); err != nil { + t.Errorf("failed to close database: %v", err) + } + }() + + conn.SetMaxOpenConns(1) + conn.SetMaxIdleConns(1) + + // First run: apply all migrations. + if err := applyMigrations(ctx, conn); err != nil { + t.Fatalf("first applyMigrations failed: %v", err) + } + + // Count rows in schema_migrations after first run. + var countBefore int + if err := conn.QueryRowContext(ctx, "SELECT COUNT(*) FROM schema_migrations").Scan(&countBefore); err != nil { + t.Fatalf("failed to count schema_migrations after first run: %v", err) + } + + // Second run: must be a no-op. + if err := applyMigrations(ctx, conn); err != nil { + t.Fatalf("second applyMigrations failed: %v", err) + } + + // Count rows in schema_migrations after second run — must be unchanged. + var countAfter int + if err := conn.QueryRowContext(ctx, "SELECT COUNT(*) FROM schema_migrations").Scan(&countAfter); err != nil { + t.Fatalf("failed to count schema_migrations after second run: %v", err) + } + + if countBefore != countAfter { + t.Errorf("schema_migrations row count changed: before=%d, after=%d", countBefore, countAfter) + } +} + +func TestBootstrapMigrationsTable_FreshDatabase(t *testing.T) { + ctx := context.Background() + + conn, err := sql.Open("sqlite", ":memory:?_foreign_keys=ON") + if err != nil { + t.Fatalf("failed to open database: %v", err) + } + defer func() { + if err := conn.Close(); err != nil { + t.Errorf("failed to close database: %v", err) + } + }() + + conn.SetMaxOpenConns(1) + conn.SetMaxIdleConns(1) + + // Verify schema_migrations does NOT exist yet. + var tableBefore int + if err := conn.QueryRowContext(ctx, + "SELECT COUNT(*) FROM sqlite_master WHERE type='table' AND name='schema_migrations'", + ).Scan(&tableBefore); err != nil { + t.Fatalf("failed to check for table before bootstrap: %v", err) + } + if tableBefore != 0 { + t.Fatal("schema_migrations table should not exist before bootstrap") + } + + // Run bootstrap. + if err := bootstrapMigrationsTable(ctx, conn); err != nil { + t.Fatalf("bootstrapMigrationsTable failed: %v", err) + } + + // Verify schema_migrations now exists. + var tableAfter int + if err := conn.QueryRowContext(ctx, + "SELECT COUNT(*) FROM sqlite_master WHERE type='table' AND name='schema_migrations'", + ).Scan(&tableAfter); err != nil { + t.Fatalf("failed to check for table after bootstrap: %v", err) + } + if tableAfter != 1 { + t.Fatalf("schema_migrations table should exist after bootstrap, got count=%d", tableAfter) + } + + // Verify version 0 row exists. + var version int + if err := conn.QueryRowContext(ctx, + "SELECT version FROM schema_migrations WHERE version = 0", + ).Scan(&version); err != nil { + t.Fatalf("version 0 row not found in schema_migrations: %v", err) + } + if version != 0 { + t.Errorf("expected version 0, got %d", version) + } +} diff --git a/internal/database/schema/000.sql b/internal/database/schema/000.sql new file mode 100644 index 0000000..e06a2da --- /dev/null +++ b/internal/database/schema/000.sql @@ -0,0 +1,9 @@ +-- Migration 000: Schema migrations tracking table +-- Applied as a bootstrap step before the normal migration loop. + +CREATE TABLE IF NOT EXISTS schema_migrations ( + version INTEGER PRIMARY KEY, + applied_at DATETIME DEFAULT CURRENT_TIMESTAMP +); + +INSERT OR IGNORE INTO schema_migrations (version) VALUES (0); diff --git a/internal/database/schema.sql b/internal/database/schema/001.sql similarity index 96% rename from internal/database/schema.sql rename to internal/database/schema/001.sql index cdc8533..5f54565 100644 --- a/internal/database/schema.sql +++ b/internal/database/schema/001.sql @@ -1,6 +1,5 @@ --- Vaultik Database Schema --- Note: This database does not support migrations. If the schema changes, --- delete the local database and perform a full backup to recreate it. +-- Migration 001: Initial Vaultik schema +-- All core tables for tracking files, chunks, blobs, snapshots, and uploads. -- Files table: stores metadata about files in the filesystem CREATE TABLE IF NOT EXISTS files ( @@ -133,4 +132,4 @@ CREATE TABLE IF NOT EXISTS uploads ( ); -- Index for efficient snapshot lookups -CREATE INDEX IF NOT EXISTS idx_uploads_snapshot_id ON uploads(snapshot_id); \ No newline at end of file +CREATE INDEX IF NOT EXISTS idx_uploads_snapshot_id ON uploads(snapshot_id); diff --git a/internal/database/schema/008_uploads.sql b/internal/database/schema/008_uploads.sql deleted file mode 100644 index 49b5add..0000000 --- a/internal/database/schema/008_uploads.sql +++ /dev/null @@ -1,11 +0,0 @@ --- Track blob upload metrics -CREATE TABLE IF NOT EXISTS uploads ( - blob_hash TEXT PRIMARY KEY, - uploaded_at TIMESTAMP NOT NULL, - size INTEGER NOT NULL, - duration_ms INTEGER NOT NULL, - FOREIGN KEY (blob_hash) REFERENCES blobs(blob_hash) -); - -CREATE INDEX idx_uploads_uploaded_at ON uploads(uploaded_at); -CREATE INDEX idx_uploads_duration ON uploads(duration_ms); \ No newline at end of file