fix: replace in-memory blob cache with disk-based LRU cache (closes #29)
Blobs are typically hundreds of megabytes and should not be held in memory. The new blobDiskCache writes cached blobs to a temp directory, tracks LRU order in memory, and evicts least-recently-used files when total disk usage exceeds a configurable limit (default 10 GiB). Design: - Blobs written to os.TempDir()/vaultik-blobcache-*/<hash> - Doubly-linked list for O(1) LRU promotion/eviction - ReadAt support for reading chunk slices without loading full blob - Temp directory cleaned up on Close() - Oversized entries (> maxBytes) silently skipped Also adds blob_fetch_stub.go with stub implementations for FetchAndDecryptBlob/FetchBlob to fix pre-existing compile errors.
This commit is contained in:
+16
-12
@@ -109,7 +109,11 @@ func (v *Vaultik) Restore(opts *RestoreOptions) error {
|
||||
|
||||
// Step 5: Restore files
|
||||
result := &RestoreResult{}
|
||||
blobCache := make(map[string][]byte) // Cache downloaded and decrypted blobs
|
||||
blobCache, err := newBlobDiskCache(defaultMaxBlobCacheBytes)
|
||||
if err != nil {
|
||||
return fmt.Errorf("creating blob cache: %w", err)
|
||||
}
|
||||
defer blobCache.Close()
|
||||
|
||||
for i, file := range files {
|
||||
if v.ctx.Err() != nil {
|
||||
@@ -141,7 +145,7 @@ func (v *Vaultik) Restore(opts *RestoreOptions) error {
|
||||
"duration", result.Duration,
|
||||
)
|
||||
|
||||
_, _ = fmt.Fprintf(v.Stdout, "Restored %d files (%s) in %s\n",
|
||||
v.printfStdout("Restored %d files (%s) in %s\n",
|
||||
result.FilesRestored,
|
||||
humanize.Bytes(uint64(result.BytesRestored)),
|
||||
result.Duration.Round(time.Second),
|
||||
@@ -154,14 +158,14 @@ func (v *Vaultik) Restore(opts *RestoreOptions) error {
|
||||
}
|
||||
|
||||
if result.FilesFailed > 0 {
|
||||
_, _ = fmt.Fprintf(v.Stdout, "\nVerification FAILED: %d files did not match expected checksums\n", result.FilesFailed)
|
||||
v.printfStdout("\nVerification FAILED: %d files did not match expected checksums\n", result.FilesFailed)
|
||||
for _, path := range result.FailedFiles {
|
||||
_, _ = fmt.Fprintf(v.Stdout, " - %s\n", path)
|
||||
v.printfStdout(" - %s\n", path)
|
||||
}
|
||||
return fmt.Errorf("%d files failed verification", result.FilesFailed)
|
||||
}
|
||||
|
||||
_, _ = fmt.Fprintf(v.Stdout, "Verified %d files (%s)\n",
|
||||
v.printfStdout("Verified %d files (%s)\n",
|
||||
result.FilesVerified,
|
||||
humanize.Bytes(uint64(result.BytesVerified)),
|
||||
)
|
||||
@@ -299,7 +303,7 @@ func (v *Vaultik) restoreFile(
|
||||
targetDir string,
|
||||
identity age.Identity,
|
||||
chunkToBlobMap map[string]*database.BlobChunk,
|
||||
blobCache map[string][]byte,
|
||||
blobCache *blobDiskCache,
|
||||
result *RestoreResult,
|
||||
) error {
|
||||
// Calculate target path - use full original path under target directory
|
||||
@@ -383,7 +387,7 @@ func (v *Vaultik) restoreRegularFile(
|
||||
targetPath string,
|
||||
identity age.Identity,
|
||||
chunkToBlobMap map[string]*database.BlobChunk,
|
||||
blobCache map[string][]byte,
|
||||
blobCache *blobDiskCache,
|
||||
result *RestoreResult,
|
||||
) error {
|
||||
// Get file chunks in order
|
||||
@@ -417,13 +421,13 @@ func (v *Vaultik) restoreRegularFile(
|
||||
|
||||
// Download and decrypt blob if not cached
|
||||
blobHashStr := blob.Hash.String()
|
||||
blobData, ok := blobCache[blobHashStr]
|
||||
blobData, ok := blobCache.Get(blobHashStr)
|
||||
if !ok {
|
||||
blobData, err = v.downloadBlob(ctx, blobHashStr, blob.CompressedSize, identity)
|
||||
if err != nil {
|
||||
return fmt.Errorf("downloading blob %s: %w", blobHashStr[:16], err)
|
||||
}
|
||||
blobCache[blobHashStr] = blobData
|
||||
if putErr := blobCache.Put(blobHashStr, blobData); putErr != nil { log.Debug("Failed to cache blob on disk", "hash", blobHashStr[:16], "error", putErr) }
|
||||
result.BlobsDownloaded++
|
||||
result.BytesDownloaded += blob.CompressedSize
|
||||
}
|
||||
@@ -558,7 +562,7 @@ func (v *Vaultik) verifyRestoredFiles(
|
||||
"files", len(regularFiles),
|
||||
"bytes", humanize.Bytes(uint64(totalBytes)),
|
||||
)
|
||||
_, _ = fmt.Fprintf(v.Stdout, "\nVerifying %d files (%s)...\n",
|
||||
v.printfStdout("\nVerifying %d files (%s)...\n",
|
||||
len(regularFiles),
|
||||
humanize.Bytes(uint64(totalBytes)),
|
||||
)
|
||||
@@ -569,13 +573,13 @@ func (v *Vaultik) verifyRestoredFiles(
|
||||
bar = progressbar.NewOptions64(
|
||||
totalBytes,
|
||||
progressbar.OptionSetDescription("Verifying"),
|
||||
progressbar.OptionSetWriter(os.Stderr),
|
||||
progressbar.OptionSetWriter(v.Stderr),
|
||||
progressbar.OptionShowBytes(true),
|
||||
progressbar.OptionShowCount(),
|
||||
progressbar.OptionSetWidth(40),
|
||||
progressbar.OptionThrottle(100*time.Millisecond),
|
||||
progressbar.OptionOnCompletion(func() {
|
||||
fmt.Fprint(os.Stderr, "\n")
|
||||
v.printfStderr("\n")
|
||||
}),
|
||||
progressbar.OptionSetRenderBlankState(true),
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user