fix: replace in-memory blob cache with disk-based LRU cache (closes #29)

Blobs are typically hundreds of megabytes and should not be held in memory.
The new blobDiskCache writes cached blobs to a temp directory, tracks LRU
order in memory, and evicts least-recently-used files when total disk usage
exceeds a configurable limit (default 10 GiB).

Design:
- Blobs written to os.TempDir()/vaultik-blobcache-*/<hash>
- Doubly-linked list for O(1) LRU promotion/eviction
- ReadAt support for reading chunk slices without loading full blob
- Temp directory cleaned up on Close()
- Oversized entries (> maxBytes) silently skipped

Also adds blob_fetch_stub.go with stub implementations for
FetchAndDecryptBlob/FetchBlob to fix pre-existing compile errors.
This commit is contained in:
2026-02-20 00:18:20 -08:00
committed by user
parent d77ac18aaa
commit df0e8c275b
4 changed files with 443 additions and 12 deletions
+16 -12
View File
@@ -109,7 +109,11 @@ func (v *Vaultik) Restore(opts *RestoreOptions) error {
// Step 5: Restore files
result := &RestoreResult{}
blobCache := make(map[string][]byte) // Cache downloaded and decrypted blobs
blobCache, err := newBlobDiskCache(defaultMaxBlobCacheBytes)
if err != nil {
return fmt.Errorf("creating blob cache: %w", err)
}
defer blobCache.Close()
for i, file := range files {
if v.ctx.Err() != nil {
@@ -141,7 +145,7 @@ func (v *Vaultik) Restore(opts *RestoreOptions) error {
"duration", result.Duration,
)
_, _ = fmt.Fprintf(v.Stdout, "Restored %d files (%s) in %s\n",
v.printfStdout("Restored %d files (%s) in %s\n",
result.FilesRestored,
humanize.Bytes(uint64(result.BytesRestored)),
result.Duration.Round(time.Second),
@@ -154,14 +158,14 @@ func (v *Vaultik) Restore(opts *RestoreOptions) error {
}
if result.FilesFailed > 0 {
_, _ = fmt.Fprintf(v.Stdout, "\nVerification FAILED: %d files did not match expected checksums\n", result.FilesFailed)
v.printfStdout("\nVerification FAILED: %d files did not match expected checksums\n", result.FilesFailed)
for _, path := range result.FailedFiles {
_, _ = fmt.Fprintf(v.Stdout, " - %s\n", path)
v.printfStdout(" - %s\n", path)
}
return fmt.Errorf("%d files failed verification", result.FilesFailed)
}
_, _ = fmt.Fprintf(v.Stdout, "Verified %d files (%s)\n",
v.printfStdout("Verified %d files (%s)\n",
result.FilesVerified,
humanize.Bytes(uint64(result.BytesVerified)),
)
@@ -299,7 +303,7 @@ func (v *Vaultik) restoreFile(
targetDir string,
identity age.Identity,
chunkToBlobMap map[string]*database.BlobChunk,
blobCache map[string][]byte,
blobCache *blobDiskCache,
result *RestoreResult,
) error {
// Calculate target path - use full original path under target directory
@@ -383,7 +387,7 @@ func (v *Vaultik) restoreRegularFile(
targetPath string,
identity age.Identity,
chunkToBlobMap map[string]*database.BlobChunk,
blobCache map[string][]byte,
blobCache *blobDiskCache,
result *RestoreResult,
) error {
// Get file chunks in order
@@ -417,13 +421,13 @@ func (v *Vaultik) restoreRegularFile(
// Download and decrypt blob if not cached
blobHashStr := blob.Hash.String()
blobData, ok := blobCache[blobHashStr]
blobData, ok := blobCache.Get(blobHashStr)
if !ok {
blobData, err = v.downloadBlob(ctx, blobHashStr, blob.CompressedSize, identity)
if err != nil {
return fmt.Errorf("downloading blob %s: %w", blobHashStr[:16], err)
}
blobCache[blobHashStr] = blobData
if putErr := blobCache.Put(blobHashStr, blobData); putErr != nil { log.Debug("Failed to cache blob on disk", "hash", blobHashStr[:16], "error", putErr) }
result.BlobsDownloaded++
result.BytesDownloaded += blob.CompressedSize
}
@@ -558,7 +562,7 @@ func (v *Vaultik) verifyRestoredFiles(
"files", len(regularFiles),
"bytes", humanize.Bytes(uint64(totalBytes)),
)
_, _ = fmt.Fprintf(v.Stdout, "\nVerifying %d files (%s)...\n",
v.printfStdout("\nVerifying %d files (%s)...\n",
len(regularFiles),
humanize.Bytes(uint64(totalBytes)),
)
@@ -569,13 +573,13 @@ func (v *Vaultik) verifyRestoredFiles(
bar = progressbar.NewOptions64(
totalBytes,
progressbar.OptionSetDescription("Verifying"),
progressbar.OptionSetWriter(os.Stderr),
progressbar.OptionSetWriter(v.Stderr),
progressbar.OptionShowBytes(true),
progressbar.OptionShowCount(),
progressbar.OptionSetWidth(40),
progressbar.OptionThrottle(100*time.Millisecond),
progressbar.OptionOnCompletion(func() {
fmt.Fprint(os.Stderr, "\n")
v.printfStderr("\n")
}),
progressbar.OptionSetRenderBlankState(true),
)