sneak/vaultik
1
0
Fork 0
Code Issues 18 Pull Requests 4 Actions Packages Projects Releases Wiki Activity

Compare commits

base: sneak:2e7356dd857fc736df271b56c037ccce08132ef5
Branches Tags
sneak:main
sneak:refactor/break-up-long-methods
sneak:add-make-check
sneak:fix/verify-blob-hash
sneak:feat/restore-progress-bar
sneak:feature/restore-progress-bar
sneak:feature/implement-prune-flag-on-snapshot-create
sneak:fix/restore-error-handling
sneak:fix/issue-25
sneak:fix/sql-injection-whitelist
sneak:fix/issue-29
sneak:add-compressstream-regression-test
sneak:fix/issue-26
sneak:fix/issue-27
sneak:fix/issue-28
sneak:feature/pluggable-storage-backend
..
compare: sneak:b13368a68a67df33bc8a4227a0f8135f528a789c
Branches Tags
sneak:refactor/break-up-long-methods
sneak:add-make-check
sneak:fix/verify-blob-hash
sneak:feat/restore-progress-bar
sneak:feature/restore-progress-bar
sneak:main
sneak:feature/implement-prune-flag-on-snapshot-create
sneak:fix/restore-error-handling
sneak:fix/issue-25
sneak:fix/sql-injection-whitelist
sneak:fix/issue-29
sneak:add-compressstream-regression-test
sneak:fix/issue-26
sneak:fix/issue-27
sneak:fix/issue-28
sneak:feature/pluggable-storage-backend

1 Commits
2e7356dd85 ... b13368a68a

Author SHA1 Message Date
clawbot
b13368a68a Add CompressStream double-close regression test (closes #35) 
Adds regression tests for issue #28 (fixed in PR #33) to prevent
reintroduction of the double-close bug in CompressStream.

Tests cover:
- CompressStream with normal input
- CompressStream with large (512KB) input
- CompressStream with empty input
- CompressData close correctness
 2026-02-15 21:08:46 -08:00
9 changed files with 170 additions and 652 deletions
Show Stats Expand all files Collapse all files

55
internal/vaultik/blob_fetch_stub.go
View File

@ -1,55 +0,0 @@
package vaultik
import (
"context"
"fmt"
"io"
"filippo.io/age"
"git.eeqj.de/sneak/vaultik/internal/blobgen"
)
// FetchAndDecryptBlobResult holds the result of fetching and decrypting a blob.
type FetchAndDecryptBlobResult struct {
Data []byte
}
// FetchAndDecryptBlob downloads a blob, decrypts it, and returns the plaintext data.
func (v *Vaultik) FetchAndDecryptBlob(ctx context.Context, blobHash string, expectedSize int64, identity age.Identity) (*FetchAndDecryptBlobResult, error) {
rc, _, err := v.FetchBlob(ctx, blobHash, expectedSize)
if err != nil {
return nil, err
}
defer func() { _ = rc.Close() }()
reader, err := blobgen.NewReader(rc, identity)
if err != nil {
return nil, fmt.Errorf("creating blob reader: %w", err)
}
defer func() { _ = reader.Close() }()
data, err := io.ReadAll(reader)
if err != nil {
return nil, fmt.Errorf("reading blob data: %w", err)
}
return &FetchAndDecryptBlobResult{Data: data}, nil
}
// FetchBlob downloads a blob and returns a reader for the encrypted data.
func (v *Vaultik) FetchBlob(ctx context.Context, blobHash string, expectedSize int64) (io.ReadCloser, int64, error) {
blobPath := fmt.Sprintf("blobs/%s/%s/%s", blobHash[:2], blobHash[2:4], blobHash)
rc, err := v.Storage.Get(ctx, blobPath)
if err != nil {
return nil, 0, fmt.Errorf("downloading blob %s: %w", blobHash[:16], err)
}
info, err := v.Storage.Stat(ctx, blobPath)
if err != nil {
_ = rc.Close()
return nil, 0, fmt.Errorf("stat blob %s: %w", blobHash[:16], err)
}
return rc, info.Size, nil
}

210
internal/vaultik/blobcache.go
View File

@ -1,210 +0,0 @@
package vaultik
import (
"fmt"
"os"
"path/filepath"
"sync"
)
// defaultMaxBlobCacheBytes is the default maximum size of the disk blob cache (10 GB).
const defaultMaxBlobCacheBytes = 10 << 30 // 10 GiB
// blobDiskCacheEntry tracks a cached blob on disk.
type blobDiskCacheEntry struct {
key string
size int64
prev *blobDiskCacheEntry
next *blobDiskCacheEntry
}
// blobDiskCache is an LRU cache that stores blobs on disk instead of in memory.
// Blobs are written to a temp directory keyed by their hash. When total size
// exceeds maxBytes, the least-recently-used entries are evicted (deleted from disk).
type blobDiskCache struct {
mu sync.Mutex
dir string
maxBytes int64
curBytes int64
items map[string]*blobDiskCacheEntry
head *blobDiskCacheEntry // most recent
tail *blobDiskCacheEntry // least recent
}
// newBlobDiskCache creates a new disk-based blob cache with the given max size.
func newBlobDiskCache(maxBytes int64) (*blobDiskCache, error) {
dir, err := os.MkdirTemp("", "vaultik-blobcache-*")
if err != nil {
return nil, fmt.Errorf("creating blob cache dir: %w", err)
}
return &blobDiskCache{
dir: dir,
maxBytes: maxBytes,
items: make(map[string]*blobDiskCacheEntry),
}, nil
}
func (c *blobDiskCache) path(key string) string {
return filepath.Join(c.dir, key)
}
func (c *blobDiskCache) unlink(e *blobDiskCacheEntry) {
if e.prev != nil {
e.prev.next = e.next
} else {
c.head = e.next
}
if e.next != nil {
e.next.prev = e.prev
} else {
c.tail = e.prev
}
e.prev = nil
e.next = nil
}
func (c *blobDiskCache) pushFront(e *blobDiskCacheEntry) {
e.prev = nil
e.next = c.head
if c.head != nil {
c.head.prev = e
}
c.head = e
if c.tail == nil {
c.tail = e
}
}
func (c *blobDiskCache) evictLRU() {
if c.tail == nil {
return
}
victim := c.tail
c.unlink(victim)
delete(c.items, victim.key)
c.curBytes -= victim.size
_ = os.Remove(c.path(victim.key))
}
// Put writes blob data to disk cache. Entries larger than maxBytes are silently skipped.
func (c *blobDiskCache) Put(key string, data []byte) error {
entrySize := int64(len(data))
c.mu.Lock()
defer c.mu.Unlock()
if entrySize > c.maxBytes {
return nil
}
// Remove old entry if updating
if e, ok := c.items[key]; ok {
c.unlink(e)
c.curBytes -= e.size
_ = os.Remove(c.path(key))
delete(c.items, key)
}
if err := os.WriteFile(c.path(key), data, 0600); err != nil {
return fmt.Errorf("writing blob to cache: %w", err)
}
e := &blobDiskCacheEntry{key: key, size: entrySize}
c.pushFront(e)
c.items[key] = e
c.curBytes += entrySize
for c.curBytes > c.maxBytes && c.tail != nil {
c.evictLRU()
}
return nil
}
// Get reads a cached blob from disk. Returns data and true on hit.
func (c *blobDiskCache) Get(key string) ([]byte, bool) {
c.mu.Lock()
e, ok := c.items[key]
if !ok {
c.mu.Unlock()
return nil, false
}
c.unlink(e)
c.pushFront(e)
c.mu.Unlock()
data, err := os.ReadFile(c.path(key))
if err != nil {
c.mu.Lock()
if e2, ok2 := c.items[key]; ok2 && e2 == e {
c.unlink(e)
delete(c.items, key)
c.curBytes -= e.size
}
c.mu.Unlock()
return nil, false
}
return data, true
}
// ReadAt reads a slice of a cached blob without loading the entire blob into memory.
func (c *blobDiskCache) ReadAt(key string, offset, length int64) ([]byte, error) {
c.mu.Lock()
e, ok := c.items[key]
if !ok {
c.mu.Unlock()
return nil, fmt.Errorf("key %q not in cache", key)
}
if offset+length > e.size {
c.mu.Unlock()
return nil, fmt.Errorf("read beyond blob size: offset=%d length=%d size=%d", offset, length, e.size)
}
c.unlink(e)
c.pushFront(e)
c.mu.Unlock()
f, err := os.Open(c.path(key))
if err != nil {
return nil, err
}
defer func() { _ = f.Close() }()
buf := make([]byte, length)
if _, err := f.ReadAt(buf, offset); err != nil {
return nil, err
}
return buf, nil
}
// Has returns whether a key exists in the cache.
func (c *blobDiskCache) Has(key string) bool {
c.mu.Lock()
defer c.mu.Unlock()
_, ok := c.items[key]
return ok
}
// Size returns current total cached bytes.
func (c *blobDiskCache) Size() int64 {
c.mu.Lock()
defer c.mu.Unlock()
return c.curBytes
}
// Len returns number of cached entries.
func (c *blobDiskCache) Len() int {
c.mu.Lock()
defer c.mu.Unlock()
return len(c.items)
}
// Close removes the cache directory and all cached blobs.
func (c *blobDiskCache) Close() error {
c.mu.Lock()
defer c.mu.Unlock()
c.items = nil
c.head = nil
c.tail = nil
c.curBytes = 0
return os.RemoveAll(c.dir)
}

189
internal/vaultik/blobcache_test.go
View File

@ -1,189 +0,0 @@
package vaultik
import (
"bytes"
"crypto/rand"
"fmt"
"testing"
)
func TestBlobDiskCache_BasicGetPut(t *testing.T) {
cache, err := newBlobDiskCache(1 << 20)
if err != nil {
t.Fatal(err)
}
defer func() { _ = cache.Close() }()
data := []byte("hello world")
if err := cache.Put("key1", data); err != nil {
t.Fatal(err)
}
got, ok := cache.Get("key1")
if !ok {
t.Fatal("expected cache hit")
}
if !bytes.Equal(got, data) {
t.Fatalf("got %q, want %q", got, data)
}
_, ok = cache.Get("nonexistent")
if ok {
t.Fatal("expected cache miss")
}
}
func TestBlobDiskCache_EvictionUnderPressure(t *testing.T) {
maxBytes := int64(1000)
cache, err := newBlobDiskCache(maxBytes)
if err != nil {
t.Fatal(err)
}
defer func() { _ = cache.Close() }()
for i := 0; i < 5; i++ {
data := make([]byte, 300)
if err := cache.Put(fmt.Sprintf("key%d", i), data); err != nil {
t.Fatal(err)
}
}
if cache.Size() > maxBytes {
t.Fatalf("cache size %d exceeds max %d", cache.Size(), maxBytes)
}
if !cache.Has("key4") {
t.Fatal("expected key4 to be cached")
}
if cache.Has("key0") {
t.Fatal("expected key0 to be evicted")
}
}
func TestBlobDiskCache_OversizedEntryRejected(t *testing.T) {
cache, err := newBlobDiskCache(100)
if err != nil {
t.Fatal(err)
}
defer func() { _ = cache.Close() }()
data := make([]byte, 200)
if err := cache.Put("big", data); err != nil {
t.Fatal(err)
}
if cache.Has("big") {
t.Fatal("oversized entry should not be cached")
}
}
func TestBlobDiskCache_UpdateInPlace(t *testing.T) {
cache, err := newBlobDiskCache(1 << 20)
if err != nil {
t.Fatal(err)
}
defer func() { _ = cache.Close() }()
if err := cache.Put("key1", []byte("v1")); err != nil {
t.Fatal(err)
}
if err := cache.Put("key1", []byte("version2")); err != nil {
t.Fatal(err)
}
got, ok := cache.Get("key1")
if !ok {
t.Fatal("expected hit")
}
if string(got) != "version2" {
t.Fatalf("got %q, want %q", got, "version2")
}
if cache.Len() != 1 {
t.Fatalf("expected 1 entry, got %d", cache.Len())
}
if cache.Size() != int64(len("version2")) {
t.Fatalf("expected size %d, got %d", len("version2"), cache.Size())
}
}
func TestBlobDiskCache_ReadAt(t *testing.T) {
cache, err := newBlobDiskCache(1 << 20)
if err != nil {
t.Fatal(err)
}
defer func() { _ = cache.Close() }()
data := make([]byte, 1024)
if _, err := rand.Read(data); err != nil {
t.Fatal(err)
}
if err := cache.Put("blob1", data); err != nil {
t.Fatal(err)
}
chunk, err := cache.ReadAt("blob1", 100, 200)
if err != nil {
t.Fatal(err)
}
if !bytes.Equal(chunk, data[100:300]) {
t.Fatal("ReadAt returned wrong data")
}
_, err = cache.ReadAt("blob1", 900, 200)
if err == nil {
t.Fatal("expected error for out-of-bounds read")
}
_, err = cache.ReadAt("missing", 0, 10)
if err == nil {
t.Fatal("expected error for missing key")
}
}
func TestBlobDiskCache_Close(t *testing.T) {
cache, err := newBlobDiskCache(1 << 20)
if err != nil {
t.Fatal(err)
}
if err := cache.Put("key1", []byte("data")); err != nil {
t.Fatal(err)
}
if err := cache.Close(); err != nil {
t.Fatal(err)
}
}
func TestBlobDiskCache_LRUOrder(t *testing.T) {
cache, err := newBlobDiskCache(200)
if err != nil {
t.Fatal(err)
}
defer func() { _ = cache.Close() }()
d := make([]byte, 100)
if err := cache.Put("a", d); err != nil {
t.Fatal(err)
}
if err := cache.Put("b", d); err != nil {
t.Fatal(err)
}
// Access "a" to make it most recently used
cache.Get("a")
// Adding "c" should evict "b" (LRU), not "a"
if err := cache.Put("c", d); err != nil {
t.Fatal(err)
}
if !cache.Has("a") {
t.Fatal("expected 'a' to survive")
}
if !cache.Has("c") {
t.Fatal("expected 'c' to be present")
}
if cache.Has("b") {
t.Fatal("expected 'b' to be evicted")
}
}

112
internal/vaultik/info.go
View File

@ -15,99 +15,99 @@ import (
// ShowInfo displays system and configuration information // ShowInfo displays system and configuration information
func (v *Vaultik) ShowInfo() error { func (v *Vaultik) ShowInfo() error {
// System Information // System Information
v.printfStdout("=== System Information ===\n") fmt.Printf("=== System Information ===\n")
v.printfStdout("OS/Architecture: %s/%s\n", runtime.GOOS, runtime.GOARCH) fmt.Printf("OS/Architecture: %s/%s\n", runtime.GOOS, runtime.GOARCH)
v.printfStdout("Version: %s\n", v.Globals.Version) fmt.Printf("Version: %s\n", v.Globals.Version)
v.printfStdout("Commit: %s\n", v.Globals.Commit) fmt.Printf("Commit: %s\n", v.Globals.Commit)
v.printfStdout("Go Version: %s\n", runtime.Version()) fmt.Printf("Go Version: %s\n", runtime.Version())
v.printlnStdout() fmt.Println()
// Storage Configuration // Storage Configuration
v.printfStdout("=== Storage Configuration ===\n") fmt.Printf("=== Storage Configuration ===\n")
v.printfStdout("S3 Bucket: %s\n", v.Config.S3.Bucket) fmt.Printf("S3 Bucket: %s\n", v.Config.S3.Bucket)
if v.Config.S3.Prefix != "" { if v.Config.S3.Prefix != "" {
v.printfStdout("S3 Prefix: %s\n", v.Config.S3.Prefix) fmt.Printf("S3 Prefix: %s\n", v.Config.S3.Prefix)
} }
v.printfStdout("S3 Endpoint: %s\n", v.Config.S3.Endpoint) fmt.Printf("S3 Endpoint: %s\n", v.Config.S3.Endpoint)
v.printfStdout("S3 Region: %s\n", v.Config.S3.Region) fmt.Printf("S3 Region: %s\n", v.Config.S3.Region)
v.printlnStdout() fmt.Println()
// Backup Settings // Backup Settings
v.printfStdout("=== Backup Settings ===\n") fmt.Printf("=== Backup Settings ===\n")
// Show configured snapshots // Show configured snapshots
v.printfStdout("Snapshots:\n") fmt.Printf("Snapshots:\n")
for _, name := range v.Config.SnapshotNames() { for _, name := range v.Config.SnapshotNames() {
snap := v.Config.Snapshots[name] snap := v.Config.Snapshots[name]
v.printfStdout(" %s:\n", name) fmt.Printf(" %s:\n", name)
for _, path := range snap.Paths { for _, path := range snap.Paths {
v.printfStdout(" - %s\n", path) fmt.Printf(" - %s\n", path)
} }
if len(snap.Exclude) > 0 { if len(snap.Exclude) > 0 {
v.printfStdout(" exclude: %s\n", strings.Join(snap.Exclude, ", ")) fmt.Printf(" exclude: %s\n", strings.Join(snap.Exclude, ", "))
} }
} }
// Global exclude patterns // Global exclude patterns
if len(v.Config.Exclude) > 0 { if len(v.Config.Exclude) > 0 {
v.printfStdout("Global Exclude: %s\n", strings.Join(v.Config.Exclude, ", ")) fmt.Printf("Global Exclude: %s\n", strings.Join(v.Config.Exclude, ", "))
} }
v.printfStdout("Compression: zstd level %d\n", v.Config.CompressionLevel) fmt.Printf("Compression: zstd level %d\n", v.Config.CompressionLevel)
v.printfStdout("Chunk Size: %s\n", humanize.Bytes(uint64(v.Config.ChunkSize))) fmt.Printf("Chunk Size: %s\n", humanize.Bytes(uint64(v.Config.ChunkSize)))
v.printfStdout("Blob Size Limit: %s\n", humanize.Bytes(uint64(v.Config.BlobSizeLimit))) fmt.Printf("Blob Size Limit: %s\n", humanize.Bytes(uint64(v.Config.BlobSizeLimit)))
v.printlnStdout() fmt.Println()
// Encryption Configuration // Encryption Configuration
v.printfStdout("=== Encryption Configuration ===\n") fmt.Printf("=== Encryption Configuration ===\n")
v.printfStdout("Recipients:\n") fmt.Printf("Recipients:\n")
for _, recipient := range v.Config.AgeRecipients { for _, recipient := range v.Config.AgeRecipients {
v.printfStdout(" - %s\n", recipient) fmt.Printf(" - %s\n", recipient)
} }
v.printlnStdout() fmt.Println()
// Daemon Settings (if applicable) // Daemon Settings (if applicable)
if v.Config.BackupInterval > 0 || v.Config.MinTimeBetweenRun > 0 { if v.Config.BackupInterval > 0 || v.Config.MinTimeBetweenRun > 0 {
v.printfStdout("=== Daemon Settings ===\n") fmt.Printf("=== Daemon Settings ===\n")
if v.Config.BackupInterval > 0 { if v.Config.BackupInterval > 0 {
v.printfStdout("Backup Interval: %s\n", v.Config.BackupInterval) fmt.Printf("Backup Interval: %s\n", v.Config.BackupInterval)
} }
if v.Config.MinTimeBetweenRun > 0 { if v.Config.MinTimeBetweenRun > 0 {
v.printfStdout("Minimum Time: %s\n", v.Config.MinTimeBetweenRun) fmt.Printf("Minimum Time: %s\n", v.Config.MinTimeBetweenRun)
} }
v.printlnStdout() fmt.Println()
} }
// Local Database // Local Database
v.printfStdout("=== Local Database ===\n") fmt.Printf("=== Local Database ===\n")
v.printfStdout("Index Path: %s\n", v.Config.IndexPath) fmt.Printf("Index Path: %s\n", v.Config.IndexPath)
// Check if index file exists and get its size // Check if index file exists and get its size
if info, err := v.Fs.Stat(v.Config.IndexPath); err == nil { if info, err := v.Fs.Stat(v.Config.IndexPath); err == nil {
v.printfStdout("Index Size: %s\n", humanize.Bytes(uint64(info.Size()))) fmt.Printf("Index Size: %s\n", humanize.Bytes(uint64(info.Size())))
// Get snapshot count from database // Get snapshot count from database
query := `SELECT COUNT(*) FROM snapshots WHERE completed_at IS NOT NULL` query := `SELECT COUNT(*) FROM snapshots WHERE completed_at IS NOT NULL`
var snapshotCount int var snapshotCount int
if err := v.DB.Conn().QueryRowContext(v.ctx, query).Scan(&snapshotCount); err == nil { if err := v.DB.Conn().QueryRowContext(v.ctx, query).Scan(&snapshotCount); err == nil {
v.printfStdout("Snapshots: %d\n", snapshotCount) fmt.Printf("Snapshots: %d\n", snapshotCount)
} }
// Get blob count from database // Get blob count from database
query = `SELECT COUNT(*) FROM blobs` query = `SELECT COUNT(*) FROM blobs`
var blobCount int var blobCount int
if err := v.DB.Conn().QueryRowContext(v.ctx, query).Scan(&blobCount); err == nil { if err := v.DB.Conn().QueryRowContext(v.ctx, query).Scan(&blobCount); err == nil {
v.printfStdout("Blobs: %d\n", blobCount) fmt.Printf("Blobs: %d\n", blobCount)
} }
// Get file count from database // Get file count from database
query = `SELECT COUNT(*) FROM files` query = `SELECT COUNT(*) FROM files`
var fileCount int var fileCount int
if err := v.DB.Conn().QueryRowContext(v.ctx, query).Scan(&fileCount); err == nil { if err := v.DB.Conn().QueryRowContext(v.ctx, query).Scan(&fileCount); err == nil {
v.printfStdout("Files: %d\n", fileCount) fmt.Printf("Files: %d\n", fileCount)
} }
} else { } else {
v.printfStdout("Index Size: (not created)\n") fmt.Printf("Index Size: (not created)\n")
} }
return nil return nil
@ -157,15 +157,15 @@ func (v *Vaultik) RemoteInfo(jsonOutput bool) error {
result.StorageLocation = storageInfo.Location result.StorageLocation = storageInfo.Location
if !jsonOutput { if !jsonOutput {
v.printfStdout("=== Remote Storage ===\n") fmt.Printf("=== Remote Storage ===\n")
v.printfStdout("Type: %s\n", storageInfo.Type) fmt.Printf("Type: %s\n", storageInfo.Type)
v.printfStdout("Location: %s\n", storageInfo.Location) fmt.Printf("Location: %s\n", storageInfo.Location)
v.printlnStdout() fmt.Println()
} }
// List all snapshot metadata // List all snapshot metadata
if !jsonOutput { if !jsonOutput {
v.printfStdout("Scanning snapshot metadata...\n") fmt.Printf("Scanning snapshot metadata...\n")
} }
snapshotMetadata := make(map[string]*SnapshotMetadataInfo) snapshotMetadata := make(map[string]*SnapshotMetadataInfo)
@ -210,7 +210,7 @@ func (v *Vaultik) RemoteInfo(jsonOutput bool) error {
// Download and parse all manifests to get referenced blobs // Download and parse all manifests to get referenced blobs
if !jsonOutput { if !jsonOutput {
v.printfStdout("Downloading %d manifest(s)...\n", len(snapshotIDs)) fmt.Printf("Downloading %d manifest(s)...\n", len(snapshotIDs))
} }
referencedBlobs := make(map[string]int64) // hash -> compressed size referencedBlobs := make(map[string]int64) // hash -> compressed size
@ -260,7 +260,7 @@ func (v *Vaultik) RemoteInfo(jsonOutput bool) error {
// List all blobs on remote // List all blobs on remote
if !jsonOutput { if !jsonOutput {
v.printfStdout("Scanning blobs...\n") fmt.Printf("Scanning blobs...\n")
} }
allBlobs := make(map[string]int64) // hash -> size from storage allBlobs := make(map[string]int64) // hash -> size from storage
@ -298,14 +298,14 @@ func (v *Vaultik) RemoteInfo(jsonOutput bool) error {
} }
// Human-readable output // Human-readable output
v.printfStdout("\n=== Snapshot Metadata ===\n") fmt.Printf("\n=== Snapshot Metadata ===\n")
if len(result.Snapshots) == 0 { if len(result.Snapshots) == 0 {
v.printfStdout("No snapshots found\n") fmt.Printf("No snapshots found\n")
} else { } else {
v.printfStdout("%-45s %12s %12s %12s %10s %12s\n", "SNAPSHOT", "MANIFEST", "DATABASE", "TOTAL", "BLOBS", "BLOB SIZE") fmt.Printf("%-45s %12s %12s %12s %10s %12s\n", "SNAPSHOT", "MANIFEST", "DATABASE", "TOTAL", "BLOBS", "BLOB SIZE")
v.printfStdout("%-45s %12s %12s %12s %10s %12s\n", strings.Repeat("-", 45), strings.Repeat("-", 12), strings.Repeat("-", 12), strings.Repeat("-", 12), strings.Repeat("-", 10), strings.Repeat("-", 12)) fmt.Printf("%-45s %12s %12s %12s %10s %12s\n", strings.Repeat("-", 45), strings.Repeat("-", 12), strings.Repeat("-", 12), strings.Repeat("-", 12), strings.Repeat("-", 10), strings.Repeat("-", 12))
for _, info := range result.Snapshots { for _, info := range result.Snapshots {
v.printfStdout("%-45s %12s %12s %12s %10s %12s\n", fmt.Printf("%-45s %12s %12s %12s %10s %12s\n",
truncateString(info.SnapshotID, 45), truncateString(info.SnapshotID, 45),
humanize.Bytes(uint64(info.ManifestSize)), humanize.Bytes(uint64(info.ManifestSize)),
humanize.Bytes(uint64(info.DatabaseSize)), humanize.Bytes(uint64(info.DatabaseSize)),
@ -314,23 +314,23 @@ func (v *Vaultik) RemoteInfo(jsonOutput bool) error {
humanize.Bytes(uint64(info.BlobsSize)), humanize.Bytes(uint64(info.BlobsSize)),
) )
} }
v.printfStdout("%-45s %12s %12s %12s %10s %12s\n", strings.Repeat("-", 45), strings.Repeat("-", 12), strings.Repeat("-", 12), strings.Repeat("-", 12), strings.Repeat("-", 10), strings.Repeat("-", 12)) fmt.Printf("%-45s %12s %12s %12s %10s %12s\n", strings.Repeat("-", 45), strings.Repeat("-", 12), strings.Repeat("-", 12), strings.Repeat("-", 12), strings.Repeat("-", 10), strings.Repeat("-", 12))
v.printfStdout("%-45s %12s %12s %12s\n", fmt.Sprintf("Total (%d snapshots)", result.TotalMetadataCount), "", "", humanize.Bytes(uint64(result.TotalMetadataSize))) fmt.Printf("%-45s %12s %12s %12s\n", fmt.Sprintf("Total (%d snapshots)", result.TotalMetadataCount), "", "", humanize.Bytes(uint64(result.TotalMetadataSize)))
} }
v.printfStdout("\n=== Blob Storage ===\n") fmt.Printf("\n=== Blob Storage ===\n")
v.printfStdout("Total blobs on remote: %s (%s)\n", fmt.Printf("Total blobs on remote: %s (%s)\n",
humanize.Comma(int64(result.TotalBlobCount)), humanize.Comma(int64(result.TotalBlobCount)),
humanize.Bytes(uint64(result.TotalBlobSize))) humanize.Bytes(uint64(result.TotalBlobSize)))
v.printfStdout("Referenced by snapshots: %s (%s)\n", fmt.Printf("Referenced by snapshots: %s (%s)\n",
humanize.Comma(int64(result.ReferencedBlobCount)), humanize.Comma(int64(result.ReferencedBlobCount)),
humanize.Bytes(uint64(result.ReferencedBlobSize))) humanize.Bytes(uint64(result.ReferencedBlobSize)))
v.printfStdout("Orphaned (unreferenced): %s (%s)\n", fmt.Printf("Orphaned (unreferenced): %s (%s)\n",
humanize.Comma(int64(result.OrphanedBlobCount)), humanize.Comma(int64(result.OrphanedBlobCount)),
humanize.Bytes(uint64(result.OrphanedBlobSize))) humanize.Bytes(uint64(result.OrphanedBlobSize)))
if result.OrphanedBlobCount > 0 { if result.OrphanedBlobCount > 0 {
v.printfStdout("\nRun 'vaultik prune --remote' to remove orphaned blobs.\n") fmt.Printf("\nRun 'vaultik prune --remote' to remove orphaned blobs.\n")
} }
return nil return nil

25
internal/vaultik/prune.go
View File

@ -3,6 +3,7 @@ package vaultik
import ( import (
"encoding/json" "encoding/json"
"fmt" "fmt"
"os"
"strings" "strings"
"git.eeqj.de/sneak/vaultik/internal/log" "git.eeqj.de/sneak/vaultik/internal/log"
@ -120,29 +121,29 @@ func (v *Vaultik) PruneBlobs(opts *PruneOptions) error {
if len(unreferencedBlobs) == 0 { if len(unreferencedBlobs) == 0 {
log.Info("No unreferenced blobs found") log.Info("No unreferenced blobs found")
if opts.JSON { if opts.JSON {
return v.outputPruneBlobsJSON(result) return outputPruneBlobsJSON(result)
} }
v.printlnStdout("No unreferenced blobs to remove.") fmt.Println("No unreferenced blobs to remove.")
return nil return nil
} }
// Show what will be deleted // Show what will be deleted
log.Info("Found unreferenced blobs", "count", len(unreferencedBlobs), "total_size", humanize.Bytes(uint64(totalSize))) log.Info("Found unreferenced blobs", "count", len(unreferencedBlobs), "total_size", humanize.Bytes(uint64(totalSize)))
if !opts.JSON { if !opts.JSON {
v.printfStdout("Found %d unreferenced blob(s) totaling %s\n", len(unreferencedBlobs), humanize.Bytes(uint64(totalSize))) fmt.Printf("Found %d unreferenced blob(s) totaling %s\n", len(unreferencedBlobs), humanize.Bytes(uint64(totalSize)))
} }
// Confirm unless --force is used (skip in JSON mode - require --force) // Confirm unless --force is used (skip in JSON mode - require --force)
if !opts.Force && !opts.JSON { if !opts.Force && !opts.JSON {
v.printfStdout("\nDelete %d unreferenced blob(s)? [y/N] ", len(unreferencedBlobs)) fmt.Printf("\nDelete %d unreferenced blob(s)? [y/N] ", len(unreferencedBlobs))
var confirm string var confirm string
if _, err := v.scanStdin(&confirm); err != nil { if _, err := fmt.Scanln(&confirm); err != nil {
// Treat EOF or error as "no" // Treat EOF or error as "no"
v.printlnStdout("Cancelled") fmt.Println("Cancelled")
return nil return nil
} }
if strings.ToLower(confirm) != "y" { if strings.ToLower(confirm) != "y" {
v.printlnStdout("Cancelled") fmt.Println("Cancelled")
return nil return nil
} }
} }
@ -184,20 +185,20 @@ func (v *Vaultik) PruneBlobs(opts *PruneOptions) error {
) )
if opts.JSON { if opts.JSON {
return v.outputPruneBlobsJSON(result) return outputPruneBlobsJSON(result)
} }
v.printfStdout("\nDeleted %d blob(s) totaling %s\n", deletedCount, humanize.Bytes(uint64(deletedSize))) fmt.Printf("\nDeleted %d blob(s) totaling %s\n", deletedCount, humanize.Bytes(uint64(deletedSize)))
if deletedCount < len(unreferencedBlobs) { if deletedCount < len(unreferencedBlobs) {
v.printfStdout("Failed to delete %d blob(s)\n", len(unreferencedBlobs)-deletedCount) fmt.Printf("Failed to delete %d blob(s)\n", len(unreferencedBlobs)-deletedCount)
} }
return nil return nil
} }
// outputPruneBlobsJSON outputs the prune result as JSON // outputPruneBlobsJSON outputs the prune result as JSON
func (v *Vaultik) outputPruneBlobsJSON(result *PruneBlobsResult) error { func outputPruneBlobsJSON(result *PruneBlobsResult) error {
encoder := json.NewEncoder(v.Stdout) encoder := json.NewEncoder(os.Stdout)
encoder.SetIndent("", " ") encoder.SetIndent("", " ")
return encoder.Encode(result) return encoder.Encode(result)
} }

30
internal/vaultik/restore.go
View File

@ -109,11 +109,7 @@ func (v *Vaultik) Restore(opts *RestoreOptions) error {
// Step 5: Restore files // Step 5: Restore files
result := &RestoreResult{} result := &RestoreResult{}
blobCache, err := newBlobDiskCache(defaultMaxBlobCacheBytes) blobCache := make(map[string][]byte) // Cache downloaded and decrypted blobs
if err != nil {
return fmt.Errorf("creating blob cache: %w", err)
}
defer func() { _ = blobCache.Close() }()
for i, file := range files { for i, file := range files {
if v.ctx.Err() != nil { if v.ctx.Err() != nil {
@ -145,7 +141,7 @@ func (v *Vaultik) Restore(opts *RestoreOptions) error {
"duration", result.Duration, "duration", result.Duration,
) )
v.printfStdout("Restored %d files (%s) in %s\n", _, _ = fmt.Fprintf(v.Stdout, "Restored %d files (%s) in %s\n",
result.FilesRestored, result.FilesRestored,
humanize.Bytes(uint64(result.BytesRestored)), humanize.Bytes(uint64(result.BytesRestored)),
result.Duration.Round(time.Second), result.Duration.Round(time.Second),
@ -158,14 +154,14 @@ func (v *Vaultik) Restore(opts *RestoreOptions) error {
} }
if result.FilesFailed > 0 { if result.FilesFailed > 0 {
v.printfStdout("\nVerification FAILED: %d files did not match expected checksums\n", result.FilesFailed) _, _ = fmt.Fprintf(v.Stdout, "\nVerification FAILED: %d files did not match expected checksums\n", result.FilesFailed)
for _, path := range result.FailedFiles { for _, path := range result.FailedFiles {
v.printfStdout(" - %s\n", path) _, _ = fmt.Fprintf(v.Stdout, " - %s\n", path)
} }
return fmt.Errorf("%d files failed verification", result.FilesFailed) return fmt.Errorf("%d files failed verification", result.FilesFailed)
} }
v.printfStdout("Verified %d files (%s)\n", _, _ = fmt.Fprintf(v.Stdout, "Verified %d files (%s)\n",
result.FilesVerified, result.FilesVerified,
humanize.Bytes(uint64(result.BytesVerified)), humanize.Bytes(uint64(result.BytesVerified)),
) )
@ -303,7 +299,7 @@ func (v *Vaultik) restoreFile(
targetDir string, targetDir string,
identity age.Identity, identity age.Identity,
chunkToBlobMap map[string]*database.BlobChunk, chunkToBlobMap map[string]*database.BlobChunk,
blobCache *blobDiskCache, blobCache map[string][]byte,
result *RestoreResult, result *RestoreResult,
) error { ) error {
// Calculate target path - use full original path under target directory // Calculate target path - use full original path under target directory
@ -387,7 +383,7 @@ func (v *Vaultik) restoreRegularFile(
targetPath string, targetPath string,
identity age.Identity, identity age.Identity,
chunkToBlobMap map[string]*database.BlobChunk, chunkToBlobMap map[string]*database.BlobChunk,
blobCache *blobDiskCache, blobCache map[string][]byte,
result *RestoreResult, result *RestoreResult,
) error { ) error {
// Get file chunks in order // Get file chunks in order
@ -421,15 +417,13 @@ func (v *Vaultik) restoreRegularFile(
// Download and decrypt blob if not cached // Download and decrypt blob if not cached
blobHashStr := blob.Hash.String() blobHashStr := blob.Hash.String()
blobData, ok := blobCache.Get(blobHashStr) blobData, ok := blobCache[blobHashStr]
if !ok { if !ok {
blobData, err = v.downloadBlob(ctx, blobHashStr, blob.CompressedSize, identity) blobData, err = v.downloadBlob(ctx, blobHashStr, blob.CompressedSize, identity)
if err != nil { if err != nil {
return fmt.Errorf("downloading blob %s: %w", blobHashStr[:16], err) return fmt.Errorf("downloading blob %s: %w", blobHashStr[:16], err)
} }
if putErr := blobCache.Put(blobHashStr, blobData); putErr != nil { blobCache[blobHashStr] = blobData
log.Debug("Failed to cache blob on disk", "hash", blobHashStr[:16], "error", putErr)
}
result.BlobsDownloaded++ result.BlobsDownloaded++
result.BytesDownloaded += blob.CompressedSize result.BytesDownloaded += blob.CompressedSize
} }
@ -517,7 +511,7 @@ func (v *Vaultik) verifyRestoredFiles(
"files", len(regularFiles), "files", len(regularFiles),
"bytes", humanize.Bytes(uint64(totalBytes)), "bytes", humanize.Bytes(uint64(totalBytes)),
) )
v.printfStdout("\nVerifying %d files (%s)...\n", _, _ = fmt.Fprintf(v.Stdout, "\nVerifying %d files (%s)...\n",
len(regularFiles), len(regularFiles),
humanize.Bytes(uint64(totalBytes)), humanize.Bytes(uint64(totalBytes)),
) )
@ -528,13 +522,13 @@ func (v *Vaultik) verifyRestoredFiles(
bar = progressbar.NewOptions64( bar = progressbar.NewOptions64(
totalBytes, totalBytes,
progressbar.OptionSetDescription("Verifying"), progressbar.OptionSetDescription("Verifying"),
progressbar.OptionSetWriter(v.Stderr), progressbar.OptionSetWriter(os.Stderr),
progressbar.OptionShowBytes(true), progressbar.OptionShowBytes(true),
progressbar.OptionShowCount(), progressbar.OptionShowCount(),
progressbar.OptionSetWidth(40), progressbar.OptionSetWidth(40),
progressbar.OptionThrottle(100*time.Millisecond), progressbar.OptionThrottle(100*time.Millisecond),
progressbar.OptionOnCompletion(func() { progressbar.OptionOnCompletion(func() {
v.printfStderr("\n") fmt.Fprint(os.Stderr, "\n")
}), }),
progressbar.OptionSetRenderBlankState(true), progressbar.OptionSetRenderBlankState(true),
) )

151
internal/vaultik/snapshot.go
View File

@ -5,7 +5,6 @@ import (
"fmt" "fmt"
"os" "os"
"path/filepath" "path/filepath"
"regexp"
"sort" "sort"
"strings" "strings"
"text/tabwriter" "text/tabwriter"
@ -87,7 +86,7 @@ func (v *Vaultik) CreateSnapshot(opts *SnapshotCreateOptions) error {
// Print overall summary if multiple snapshots // Print overall summary if multiple snapshots
if len(snapshotNames) > 1 { if len(snapshotNames) > 1 {
v.printfStdout("\nAll %d snapshots completed in %s\n", len(snapshotNames), time.Since(overallStartTime).Round(time.Second)) _, _ = fmt.Fprintf(v.Stdout, "\nAll %d snapshots completed in %s\n", len(snapshotNames), time.Since(overallStartTime).Round(time.Second))
} }
return nil return nil
@ -100,7 +99,7 @@ func (v *Vaultik) createNamedSnapshot(opts *SnapshotCreateOptions, hostname, sna
snapConfig := v.Config.Snapshots[snapName] snapConfig := v.Config.Snapshots[snapName]
if total > 1 { if total > 1 {
v.printfStdout("\n=== Snapshot %d/%d: %s ===\n", idx, total, snapName) _, _ = fmt.Fprintf(v.Stdout, "\n=== Snapshot %d/%d: %s ===\n", idx, total, snapName)
} }
// Resolve source directories to absolute paths // Resolve source directories to absolute paths
@ -153,7 +152,7 @@ func (v *Vaultik) createNamedSnapshot(opts *SnapshotCreateOptions, hostname, sna
return fmt.Errorf("creating snapshot: %w", err) return fmt.Errorf("creating snapshot: %w", err)
} }
log.Info("Beginning snapshot", "snapshot_id", snapshotID, "name", snapName) log.Info("Beginning snapshot", "snapshot_id", snapshotID, "name", snapName)
v.printfStdout("Beginning snapshot: %s\n", snapshotID) _, _ = fmt.Fprintf(v.Stdout, "Beginning snapshot: %s\n", snapshotID)
for i, dir := range resolvedDirs { for i, dir := range resolvedDirs {
// Check if context is cancelled // Check if context is cancelled
@ -165,7 +164,7 @@ func (v *Vaultik) createNamedSnapshot(opts *SnapshotCreateOptions, hostname, sna
} }
log.Info("Scanning directory", "path", dir) log.Info("Scanning directory", "path", dir)
v.printfStdout("Beginning directory scan (%d/%d): %s\n", i+1, len(resolvedDirs), dir) _, _ = fmt.Fprintf(v.Stdout, "Beginning directory scan (%d/%d): %s\n", i+1, len(resolvedDirs), dir)
result, err := scanner.Scan(v.ctx, dir, snapshotID) result, err := scanner.Scan(v.ctx, dir, snapshotID)
if err != nil { if err != nil {
return fmt.Errorf("failed to scan %s: %w", dir, err) return fmt.Errorf("failed to scan %s: %w", dir, err)
@ -276,35 +275,35 @@ func (v *Vaultik) createNamedSnapshot(opts *SnapshotCreateOptions, hostname, sna
} }
// Print comprehensive summary // Print comprehensive summary
v.printfStdout("=== Snapshot Complete ===\n") _, _ = fmt.Fprintf(v.Stdout, "=== Snapshot Complete ===\n")
v.printfStdout("ID: %s\n", snapshotID) _, _ = fmt.Fprintf(v.Stdout, "ID: %s\n", snapshotID)
v.printfStdout("Files: %s examined, %s to process, %s unchanged", _, _ = fmt.Fprintf(v.Stdout, "Files: %s examined, %s to process, %s unchanged",
formatNumber(totalFiles), formatNumber(totalFiles),
formatNumber(totalFilesChanged), formatNumber(totalFilesChanged),
formatNumber(totalFilesSkipped)) formatNumber(totalFilesSkipped))
if totalFilesDeleted > 0 { if totalFilesDeleted > 0 {
v.printfStdout(", %s deleted", formatNumber(totalFilesDeleted)) _, _ = fmt.Fprintf(v.Stdout, ", %s deleted", formatNumber(totalFilesDeleted))
} }
v.printlnStdout() _, _ = fmt.Fprintln(v.Stdout)
v.printfStdout("Data: %s total (%s to process)", _, _ = fmt.Fprintf(v.Stdout, "Data: %s total (%s to process)",
humanize.Bytes(uint64(totalBytesAll)), humanize.Bytes(uint64(totalBytesAll)),
humanize.Bytes(uint64(totalBytesChanged))) humanize.Bytes(uint64(totalBytesChanged)))
if totalBytesDeleted > 0 { if totalBytesDeleted > 0 {
v.printfStdout(", %s deleted", humanize.Bytes(uint64(totalBytesDeleted))) _, _ = fmt.Fprintf(v.Stdout, ", %s deleted", humanize.Bytes(uint64(totalBytesDeleted)))
} }
v.printlnStdout() _, _ = fmt.Fprintln(v.Stdout)
if totalBlobsUploaded > 0 { if totalBlobsUploaded > 0 {
v.printfStdout("Storage: %s compressed from %s (%.2fx)\n", _, _ = fmt.Fprintf(v.Stdout, "Storage: %s compressed from %s (%.2fx)\n",
humanize.Bytes(uint64(totalBlobSizeCompressed)), humanize.Bytes(uint64(totalBlobSizeCompressed)),
humanize.Bytes(uint64(totalBlobSizeUncompressed)), humanize.Bytes(uint64(totalBlobSizeUncompressed)),
compressionRatio) compressionRatio)
v.printfStdout("Upload: %d blobs, %s in %s (%s)\n", _, _ = fmt.Fprintf(v.Stdout, "Upload: %d blobs, %s in %s (%s)\n",
totalBlobsUploaded, totalBlobsUploaded,
humanize.Bytes(uint64(totalBytesUploaded)), humanize.Bytes(uint64(totalBytesUploaded)),
formatDuration(uploadDuration), formatDuration(uploadDuration),
avgUploadSpeed) avgUploadSpeed)
} }
v.printfStdout("Duration: %s\n", formatDuration(snapshotDuration)) _, _ = fmt.Fprintf(v.Stdout, "Duration: %s\n", formatDuration(snapshotDuration))
if opts.Prune { if opts.Prune {
log.Info("Pruning enabled - will delete old snapshots after snapshot") log.Info("Pruning enabled - will delete old snapshots after snapshot")
@ -423,13 +422,13 @@ func (v *Vaultik) ListSnapshots(jsonOutput bool) error {
if jsonOutput { if jsonOutput {
// JSON output // JSON output
encoder := json.NewEncoder(v.Stdout) encoder := json.NewEncoder(os.Stdout)
encoder.SetIndent("", " ") encoder.SetIndent("", " ")
return encoder.Encode(snapshots) return encoder.Encode(snapshots)
} }
// Table output // Table output
w := tabwriter.NewWriter(v.Stdout, 0, 0, 3, ' ', 0) w := tabwriter.NewWriter(os.Stdout, 0, 0, 3, ' ', 0)
// Show configured snapshots from config file // Show configured snapshots from config file
if _, err := fmt.Fprintln(w, "CONFIGURED SNAPSHOTS:"); err != nil { if _, err := fmt.Fprintln(w, "CONFIGURED SNAPSHOTS:"); err != nil {
@ -528,14 +527,14 @@ func (v *Vaultik) PurgeSnapshots(keepLatest bool, olderThan string, force bool)
} }
if len(toDelete) == 0 { if len(toDelete) == 0 {
v.printlnStdout("No snapshots to delete") fmt.Println("No snapshots to delete")
return nil return nil
} }
// Show what will be deleted // Show what will be deleted
v.printfStdout("The following snapshots will be deleted:\n\n") fmt.Printf("The following snapshots will be deleted:\n\n")
for _, snap := range toDelete { for _, snap := range toDelete {
v.printfStdout(" %s (%s, %s)\n", fmt.Printf(" %s (%s, %s)\n",
snap.ID, snap.ID,
snap.Timestamp.Format("2006-01-02 15:04:05"), snap.Timestamp.Format("2006-01-02 15:04:05"),
formatBytes(snap.CompressedSize)) formatBytes(snap.CompressedSize))
@ -543,19 +542,19 @@ func (v *Vaultik) PurgeSnapshots(keepLatest bool, olderThan string, force bool)
// Confirm unless --force is used // Confirm unless --force is used
if !force { if !force {
v.printfStdout("\nDelete %d snapshot(s)? [y/N] ", len(toDelete)) fmt.Printf("\nDelete %d snapshot(s)? [y/N] ", len(toDelete))
var confirm string var confirm string
if _, err := v.scanStdin(&confirm); err != nil { if _, err := fmt.Scanln(&confirm); err != nil {
// Treat EOF or error as "no" // Treat EOF or error as "no"
v.printlnStdout("Cancelled") fmt.Println("Cancelled")
return nil return nil
} }
if strings.ToLower(confirm) != "y" { if strings.ToLower(confirm) != "y" {
v.printlnStdout("Cancelled") fmt.Println("Cancelled")
return nil return nil
} }
} else { } else {
v.printfStdout("\nDeleting %d snapshot(s) (--force specified)\n", len(toDelete)) fmt.Printf("\nDeleting %d snapshot(s) (--force specified)\n", len(toDelete))
} }
// Delete snapshots (both local and remote) // Delete snapshots (both local and remote)
@ -570,10 +569,10 @@ func (v *Vaultik) PurgeSnapshots(keepLatest bool, olderThan string, force bool)
} }
} }
v.printfStdout("Deleted %d snapshot(s)\n", len(toDelete)) fmt.Printf("Deleted %d snapshot(s)\n", len(toDelete))
// Note: Run 'vaultik prune' separately to clean up unreferenced blobs // Note: Run 'vaultik prune' separately to clean up unreferenced blobs
v.printlnStdout("\nNote: Run 'vaultik prune' to clean up unreferenced blobs.") fmt.Println("\nNote: Run 'vaultik prune' to clean up unreferenced blobs.")
return nil return nil
} }
@ -614,11 +613,11 @@ func (v *Vaultik) VerifySnapshotWithOptions(snapshotID string, opts *VerifyOptio
} }
if !opts.JSON { if !opts.JSON {
v.printfStdout("Verifying snapshot %s\n", snapshotID) fmt.Printf("Verifying snapshot %s\n", snapshotID)
if !snapshotTime.IsZero() { if !snapshotTime.IsZero() {
v.printfStdout("Snapshot time: %s\n", snapshotTime.Format("2006-01-02 15:04:05 MST")) fmt.Printf("Snapshot time: %s\n", snapshotTime.Format("2006-01-02 15:04:05 MST"))
} }
v.printlnStdout() fmt.Println()
} }
// Download and parse manifest // Download and parse manifest
@ -636,18 +635,18 @@ func (v *Vaultik) VerifySnapshotWithOptions(snapshotID string, opts *VerifyOptio
result.TotalSize = manifest.TotalCompressedSize result.TotalSize = manifest.TotalCompressedSize
if !opts.JSON { if !opts.JSON {
v.printfStdout("Snapshot information:\n") fmt.Printf("Snapshot information:\n")
v.printfStdout(" Blob count: %d\n", manifest.BlobCount) fmt.Printf(" Blob count: %d\n", manifest.BlobCount)
v.printfStdout(" Total size: %s\n", humanize.Bytes(uint64(manifest.TotalCompressedSize))) fmt.Printf(" Total size: %s\n", humanize.Bytes(uint64(manifest.TotalCompressedSize)))
if manifest.Timestamp != "" { if manifest.Timestamp != "" {
if t, err := time.Parse(time.RFC3339, manifest.Timestamp); err == nil { if t, err := time.Parse(time.RFC3339, manifest.Timestamp); err == nil {
v.printfStdout(" Created: %s\n", t.Format("2006-01-02 15:04:05 MST")) fmt.Printf(" Created: %s\n", t.Format("2006-01-02 15:04:05 MST"))
} }
} }
v.printlnStdout() fmt.Println()
// Check each blob exists // Check each blob exists
v.printfStdout("Checking blob existence...\n") fmt.Printf("Checking blob existence...\n")
} }
missing := 0 missing := 0
@ -661,7 +660,7 @@ func (v *Vaultik) VerifySnapshotWithOptions(snapshotID string, opts *VerifyOptio
_, err := v.Storage.Stat(v.ctx, blobPath) _, err := v.Storage.Stat(v.ctx, blobPath)
if err != nil { if err != nil {
if !opts.JSON { if !opts.JSON {
v.printfStdout(" Missing: %s (%s)\n", blob.Hash, humanize.Bytes(uint64(blob.CompressedSize))) fmt.Printf(" Missing: %s (%s)\n", blob.Hash, humanize.Bytes(uint64(blob.CompressedSize)))
} }
missing++ missing++
missingSize += blob.CompressedSize missingSize += blob.CompressedSize
@ -684,20 +683,20 @@ func (v *Vaultik) VerifySnapshotWithOptions(snapshotID string, opts *VerifyOptio
return v.outputVerifyJSON(result) return v.outputVerifyJSON(result)
} }
v.printfStdout("\nVerification complete:\n") fmt.Printf("\nVerification complete:\n")
v.printfStdout(" Verified: %d blobs (%s)\n", verified, fmt.Printf(" Verified: %d blobs (%s)\n", verified,
humanize.Bytes(uint64(manifest.TotalCompressedSize-missingSize))) humanize.Bytes(uint64(manifest.TotalCompressedSize-missingSize)))
if missing > 0 { if missing > 0 {
v.printfStdout(" Missing: %d blobs (%s)\n", missing, humanize.Bytes(uint64(missingSize))) fmt.Printf(" Missing: %d blobs (%s)\n", missing, humanize.Bytes(uint64(missingSize)))
} else { } else {
v.printfStdout(" Missing: 0 blobs\n") fmt.Printf(" Missing: 0 blobs\n")
} }
v.printfStdout(" Status: ") fmt.Printf(" Status: ")
if missing > 0 { if missing > 0 {
v.printfStdout("FAILED - %d blobs are missing\n", missing) fmt.Printf("FAILED - %d blobs are missing\n", missing)
return fmt.Errorf("%d blobs are missing", missing) return fmt.Errorf("%d blobs are missing", missing)
} else { } else {
v.printfStdout("OK - All blobs verified\n") fmt.Printf("OK - All blobs verified\n")
} }
return nil return nil
@ -705,7 +704,7 @@ func (v *Vaultik) VerifySnapshotWithOptions(snapshotID string, opts *VerifyOptio
// outputVerifyJSON outputs the verification result as JSON // outputVerifyJSON outputs the verification result as JSON
func (v *Vaultik) outputVerifyJSON(result *VerifyResult) error { func (v *Vaultik) outputVerifyJSON(result *VerifyResult) error {
encoder := json.NewEncoder(v.Stdout) encoder := json.NewEncoder(os.Stdout)
encoder.SetIndent("", " ") encoder.SetIndent("", " ")
if err := encoder.Encode(result); err != nil { if err := encoder.Encode(result); err != nil {
return fmt.Errorf("encoding JSON: %w", err) return fmt.Errorf("encoding JSON: %w", err)
@ -831,11 +830,11 @@ func (v *Vaultik) RemoveSnapshot(snapshotID string, opts *RemoveOptions) (*Remov
if opts.DryRun { if opts.DryRun {
result.DryRun = true result.DryRun = true
if !opts.JSON { if !opts.JSON {
v.printfStdout("Would remove snapshot: %s\n", snapshotID) _, _ = fmt.Fprintf(v.Stdout, "Would remove snapshot: %s\n", snapshotID)
if opts.Remote { if opts.Remote {
v.printlnStdout("Would also remove from remote storage") _, _ = fmt.Fprintln(v.Stdout, "Would also remove from remote storage")
} }
v.printlnStdout("[Dry run - no changes made]") _, _ = fmt.Fprintln(v.Stdout, "[Dry run - no changes made]")
} }
if opts.JSON { if opts.JSON {
return result, v.outputRemoveJSON(result) return result, v.outputRemoveJSON(result)
@ -846,17 +845,17 @@ func (v *Vaultik) RemoveSnapshot(snapshotID string, opts *RemoveOptions) (*Remov
// Confirm unless --force is used (skip in JSON mode - require --force) // Confirm unless --force is used (skip in JSON mode - require --force)
if !opts.Force && !opts.JSON { if !opts.Force && !opts.JSON {
if opts.Remote { if opts.Remote {
v.printfStdout("Remove snapshot '%s' from local database and remote storage? [y/N] ", snapshotID) _, _ = fmt.Fprintf(v.Stdout, "Remove snapshot '%s' from local database and remote storage? [y/N] ", snapshotID)
} else { } else {
v.printfStdout("Remove snapshot '%s' from local database? [y/N] ", snapshotID) _, _ = fmt.Fprintf(v.Stdout, "Remove snapshot '%s' from local database? [y/N] ", snapshotID)
} }
var confirm string var confirm string
if _, err := v.scanStdin(&confirm); err != nil { if _, err := fmt.Fscanln(v.Stdin, &confirm); err != nil {
v.printlnStdout("Cancelled") _, _ = fmt.Fprintln(v.Stdout, "Cancelled")
return result, nil return result, nil
} }
if strings.ToLower(confirm) != "y" { if strings.ToLower(confirm) != "y" {
v.printlnStdout("Cancelled") _, _ = fmt.Fprintln(v.Stdout, "Cancelled")
return result, nil return result, nil
} }
} }
@ -883,10 +882,10 @@ func (v *Vaultik) RemoveSnapshot(snapshotID string, opts *RemoveOptions) (*Remov
} }
// Print summary // Print summary
v.printfStdout("Removed snapshot '%s' from local database\n", snapshotID) _, _ = fmt.Fprintf(v.Stdout, "Removed snapshot '%s' from local database\n", snapshotID)
if opts.Remote { if opts.Remote {
v.printlnStdout("Removed snapshot metadata from remote storage") _, _ = fmt.Fprintln(v.Stdout, "Removed snapshot metadata from remote storage")
v.printlnStdout("\nNote: Blobs were not removed. Run 'vaultik prune' to remove orphaned blobs.") _, _ = fmt.Fprintln(v.Stdout, "\nNote: Blobs were not removed. Run 'vaultik prune' to remove orphaned blobs.")
} }
return result, nil return result, nil
@ -930,7 +929,7 @@ func (v *Vaultik) RemoveAllSnapshots(opts *RemoveOptions) (*RemoveResult, error)
if len(snapshotIDs) == 0 { if len(snapshotIDs) == 0 {
if !opts.JSON { if !opts.JSON {
v.printlnStdout("No snapshots found") _, _ = fmt.Fprintln(v.Stdout, "No snapshots found")
} }
return result, nil return result, nil
} }
@ -939,14 +938,14 @@ func (v *Vaultik) RemoveAllSnapshots(opts *RemoveOptions) (*RemoveResult, error)
result.DryRun = true result.DryRun = true
result.SnapshotsRemoved = snapshotIDs result.SnapshotsRemoved = snapshotIDs
if !opts.JSON { if !opts.JSON {
v.printfStdout("Would remove %d snapshot(s):\n", len(snapshotIDs)) _, _ = fmt.Fprintf(v.Stdout, "Would remove %d snapshot(s):\n", len(snapshotIDs))
for _, id := range snapshotIDs { for _, id := range snapshotIDs {
v.printfStdout(" %s\n", id) _, _ = fmt.Fprintf(v.Stdout, " %s\n", id)
} }
if opts.Remote { if opts.Remote {
v.printlnStdout("Would also remove from remote storage") _, _ = fmt.Fprintln(v.Stdout, "Would also remove from remote storage")
} }
v.printlnStdout("[Dry run - no changes made]") _, _ = fmt.Fprintln(v.Stdout, "[Dry run - no changes made]")
} }
if opts.JSON { if opts.JSON {
return result, v.outputRemoveJSON(result) return result, v.outputRemoveJSON(result)
@ -987,10 +986,10 @@ func (v *Vaultik) RemoveAllSnapshots(opts *RemoveOptions) (*RemoveResult, error)
return result, v.outputRemoveJSON(result) return result, v.outputRemoveJSON(result)
} }
v.printfStdout("Removed %d snapshot(s)\n", len(result.SnapshotsRemoved)) _, _ = fmt.Fprintf(v.Stdout, "Removed %d snapshot(s)\n", len(result.SnapshotsRemoved))
if opts.Remote { if opts.Remote {
v.printlnStdout("Removed snapshot metadata from remote storage") _, _ = fmt.Fprintln(v.Stdout, "Removed snapshot metadata from remote storage")
v.printlnStdout("\nNote: Blobs were not removed. Run 'vaultik prune' to remove orphaned blobs.") _, _ = fmt.Fprintln(v.Stdout, "\nNote: Blobs were not removed. Run 'vaultik prune' to remove orphaned blobs.")
} }
return result, nil return result, nil
@ -1044,7 +1043,7 @@ func (v *Vaultik) deleteSnapshotFromRemote(snapshotID string) error {
// outputRemoveJSON outputs the removal result as JSON // outputRemoveJSON outputs the removal result as JSON
func (v *Vaultik) outputRemoveJSON(result *RemoveResult) error { func (v *Vaultik) outputRemoveJSON(result *RemoveResult) error {
encoder := json.NewEncoder(v.Stdout) encoder := json.NewEncoder(os.Stdout)
encoder.SetIndent("", " ") encoder.SetIndent("", " ")
return encoder.Encode(result) return encoder.Encode(result)
} }
@ -1118,29 +1117,21 @@ func (v *Vaultik) PruneDatabase() (*PruneResult, error) {
) )
// Print summary // Print summary
v.printfStdout("Local database prune complete:\n") _, _ = fmt.Fprintf(v.Stdout, "Local database prune complete:\n")
v.printfStdout(" Incomplete snapshots removed: %d\n", result.SnapshotsDeleted) _, _ = fmt.Fprintf(v.Stdout, " Incomplete snapshots removed: %d\n", result.SnapshotsDeleted)
v.printfStdout(" Orphaned files removed: %d\n", result.FilesDeleted) _, _ = fmt.Fprintf(v.Stdout, " Orphaned files removed: %d\n", result.FilesDeleted)
v.printfStdout(" Orphaned chunks removed: %d\n", result.ChunksDeleted) _, _ = fmt.Fprintf(v.Stdout, " Orphaned chunks removed: %d\n", result.ChunksDeleted)
v.printfStdout(" Orphaned blobs removed: %d\n", result.BlobsDeleted) _, _ = fmt.Fprintf(v.Stdout, " Orphaned blobs removed: %d\n", result.BlobsDeleted)
return result, nil return result, nil
} }
// validTableNameRe matches table names containing only lowercase alphanumeric characters and underscores. // getTableCount returns the count of rows in a table
var validTableNameRe = regexp.MustCompile(`^[a-z0-9_]+$`)
// getTableCount returns the count of rows in a table.
// The tableName is sanitized to only allow [a-z0-9_] characters to prevent SQL injection.
func (v *Vaultik) getTableCount(tableName string) (int64, error) { func (v *Vaultik) getTableCount(tableName string) (int64, error) {
if v.DB == nil { if v.DB == nil {
return 0, nil return 0, nil
} }
if !validTableNameRe.MatchString(tableName) {
return 0, fmt.Errorf("invalid table name: %q", tableName)
}
var count int64 var count int64
query := fmt.Sprintf("SELECT COUNT(*) FROM %s", tableName) query := fmt.Sprintf("SELECT COUNT(*) FROM %s", tableName)
err := v.DB.Conn().QueryRowContext(v.ctx, query).Scan(&count) err := v.DB.Conn().QueryRowContext(v.ctx, query).Scan(&count)

20
internal/vaultik/vaultik.go
View File

@ -129,26 +129,12 @@ func (v *Vaultik) GetFilesystem() afero.Fs {
return v.Fs return v.Fs
} }
// printfStdout writes formatted output to stdout. // Outputf writes formatted output to stdout for user-facing messages.
func (v *Vaultik) printfStdout(format string, args ...any) { // This should be used for all non-log user output.
func (v *Vaultik) Outputf(format string, args ...any) {
_, _ = fmt.Fprintf(v.Stdout, format, args...) _, _ = fmt.Fprintf(v.Stdout, format, args...)
} }
// printlnStdout writes a line to stdout.
func (v *Vaultik) printlnStdout(args ...any) {
_, _ = fmt.Fprintln(v.Stdout, args...)
}
// printfStderr writes formatted output to stderr.
func (v *Vaultik) printfStderr(format string, args ...any) {
_, _ = fmt.Fprintf(v.Stderr, format, args...)
}
// scanStdin reads a line of input from stdin.
func (v *Vaultik) scanStdin(a ...any) (int, error) {
return fmt.Fscanln(v.Stdin, a...)
}
// TestVaultik wraps a Vaultik with captured stdout/stderr for testing // TestVaultik wraps a Vaultik with captured stdout/stderr for testing
type TestVaultik struct { type TestVaultik struct {
*Vaultik *Vaultik

30
internal/vaultik/verify.go
View File

@ -58,14 +58,14 @@ func (v *Vaultik) RunDeepVerify(snapshotID string, opts *VerifyOptions) error {
) )
if !opts.JSON { if !opts.JSON {
v.printfStdout("Deep verification of snapshot: %s\n\n", snapshotID) v.Outputf("Deep verification of snapshot: %s\n\n", snapshotID)
} }
// Step 1: Download manifest // Step 1: Download manifest
manifestPath := fmt.Sprintf("metadata/%s/manifest.json.zst", snapshotID) manifestPath := fmt.Sprintf("metadata/%s/manifest.json.zst", snapshotID)
log.Info("Downloading manifest", "path", manifestPath) log.Info("Downloading manifest", "path", manifestPath)
if !opts.JSON { if !opts.JSON {
v.printfStdout("Downloading manifest...\n") v.Outputf("Downloading manifest...\n")
} }
manifestReader, err := v.Storage.Get(v.ctx, manifestPath) manifestReader, err := v.Storage.Get(v.ctx, manifestPath)
@ -95,14 +95,14 @@ func (v *Vaultik) RunDeepVerify(snapshotID string, opts *VerifyOptions) error {
"manifest_total_size", humanize.Bytes(uint64(manifest.TotalCompressedSize)), "manifest_total_size", humanize.Bytes(uint64(manifest.TotalCompressedSize)),
) )
if !opts.JSON { if !opts.JSON {
v.printfStdout("Manifest loaded: %d blobs (%s)\n", manifest.BlobCount, humanize.Bytes(uint64(manifest.TotalCompressedSize))) v.Outputf("Manifest loaded: %d blobs (%s)\n", manifest.BlobCount, humanize.Bytes(uint64(manifest.TotalCompressedSize)))
} }
// Step 2: Download and decrypt database (authoritative source) // Step 2: Download and decrypt database (authoritative source)
dbPath := fmt.Sprintf("metadata/%s/db.zst.age", snapshotID) dbPath := fmt.Sprintf("metadata/%s/db.zst.age", snapshotID)
log.Info("Downloading encrypted database", "path", dbPath) log.Info("Downloading encrypted database", "path", dbPath)
if !opts.JSON { if !opts.JSON {
v.printfStdout("Downloading and decrypting database...\n") v.Outputf("Downloading and decrypting database...\n")
} }
dbReader, err := v.Storage.Get(v.ctx, dbPath) dbReader, err := v.Storage.Get(v.ctx, dbPath)
@ -155,8 +155,8 @@ func (v *Vaultik) RunDeepVerify(snapshotID string, opts *VerifyOptions) error {
"db_total_size", humanize.Bytes(uint64(totalSize)), "db_total_size", humanize.Bytes(uint64(totalSize)),
) )
if !opts.JSON { if !opts.JSON {
v.printfStdout("Database loaded: %d blobs (%s)\n", len(dbBlobs), humanize.Bytes(uint64(totalSize))) v.Outputf("Database loaded: %d blobs (%s)\n", len(dbBlobs), humanize.Bytes(uint64(totalSize)))
v.printfStdout("Verifying manifest against database...\n") v.Outputf("Verifying manifest against database...\n")
} }
// Step 4: Verify manifest matches database // Step 4: Verify manifest matches database
@ -171,8 +171,8 @@ func (v *Vaultik) RunDeepVerify(snapshotID string, opts *VerifyOptions) error {
// Step 5: Verify all blobs exist in S3 (using database as source) // Step 5: Verify all blobs exist in S3 (using database as source)
if !opts.JSON { if !opts.JSON {
v.printfStdout("Manifest verified.\n") v.Outputf("Manifest verified.\n")
v.printfStdout("Checking blob existence in remote storage...\n") v.Outputf("Checking blob existence in remote storage...\n")
} }
if err := v.verifyBlobExistenceFromDB(dbBlobs); err != nil { if err := v.verifyBlobExistenceFromDB(dbBlobs); err != nil {
result.Status = "failed" result.Status = "failed"
@ -185,8 +185,8 @@ func (v *Vaultik) RunDeepVerify(snapshotID string, opts *VerifyOptions) error {
// Step 6: Deep verification - download and verify blob contents // Step 6: Deep verification - download and verify blob contents
if !opts.JSON { if !opts.JSON {
v.printfStdout("All blobs exist.\n") v.Outputf("All blobs exist.\n")
v.printfStdout("Downloading and verifying blob contents (%d blobs, %s)...\n", len(dbBlobs), humanize.Bytes(uint64(totalSize))) v.Outputf("Downloading and verifying blob contents (%d blobs, %s)...\n", len(dbBlobs), humanize.Bytes(uint64(totalSize)))
} }
if err := v.performDeepVerificationFromDB(dbBlobs, tempDB.DB, opts); err != nil { if err := v.performDeepVerificationFromDB(dbBlobs, tempDB.DB, opts); err != nil {
result.Status = "failed" result.Status = "failed"
@ -211,10 +211,10 @@ func (v *Vaultik) RunDeepVerify(snapshotID string, opts *VerifyOptions) error {
"blobs_verified", len(dbBlobs), "blobs_verified", len(dbBlobs),
) )
v.printfStdout("\n✓ Verification completed successfully\n") v.Outputf("\n✓ Verification completed successfully\n")
v.printfStdout(" Snapshot: %s\n", snapshotID) v.Outputf(" Snapshot: %s\n", snapshotID)
v.printfStdout(" Blobs verified: %d\n", len(dbBlobs)) v.Outputf(" Blobs verified: %d\n", len(dbBlobs))
v.printfStdout(" Total size: %s\n", humanize.Bytes(uint64(totalSize))) v.Outputf(" Total size: %s\n", humanize.Bytes(uint64(totalSize)))
return nil return nil
} }
@ -569,7 +569,7 @@ func (v *Vaultik) performDeepVerificationFromDB(blobs []snapshot.BlobInfo, db *s
) )
if !opts.JSON { if !opts.JSON {
v.printfStdout(" Verified %d/%d blobs (%d remaining) - %s/%s - elapsed %s, eta %s\n", v.Outputf(" Verified %d/%d blobs (%d remaining) - %s/%s - elapsed %s, eta %s\n",
i+1, len(blobs), remaining, i+1, len(blobs), remaining,
humanize.Bytes(uint64(bytesProcessed)), humanize.Bytes(uint64(bytesProcessed)),
humanize.Bytes(uint64(totalBytesExpected)), humanize.Bytes(uint64(totalBytesExpected)),