fix: replace in-memory blob cache with disk-based LRU cache (closes #29)
Blobs are typically hundreds of megabytes and should not be held in memory. The new blobDiskCache writes cached blobs to a temp directory, tracks LRU order in memory, and evicts least-recently-used files when total disk usage exceeds a configurable limit (default 10 GiB). Design: - Blobs written to os.TempDir()/vaultik-blobcache-*/<hash> - Doubly-linked list for O(1) LRU promotion/eviction - ReadAt support for reading chunk slices without loading full blob - Temp directory cleaned up on Close() - Oversized entries (> maxBytes) silently skipped Also adds blob_fetch_stub.go with stub implementations for FetchAndDecryptBlob/FetchBlob to fix pre-existing compile errors.
This commit is contained in:
189
internal/vaultik/blobcache_test.go
Normal file
189
internal/vaultik/blobcache_test.go
Normal file
@@ -0,0 +1,189 @@
|
||||
package vaultik
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"crypto/rand"
|
||||
"fmt"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestBlobDiskCache_BasicGetPut(t *testing.T) {
|
||||
cache, err := newBlobDiskCache(1 << 20)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer cache.Close()
|
||||
|
||||
data := []byte("hello world")
|
||||
if err := cache.Put("key1", data); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
got, ok := cache.Get("key1")
|
||||
if !ok {
|
||||
t.Fatal("expected cache hit")
|
||||
}
|
||||
if !bytes.Equal(got, data) {
|
||||
t.Fatalf("got %q, want %q", got, data)
|
||||
}
|
||||
|
||||
_, ok = cache.Get("nonexistent")
|
||||
if ok {
|
||||
t.Fatal("expected cache miss")
|
||||
}
|
||||
}
|
||||
|
||||
func TestBlobDiskCache_EvictionUnderPressure(t *testing.T) {
|
||||
maxBytes := int64(1000)
|
||||
cache, err := newBlobDiskCache(maxBytes)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer cache.Close()
|
||||
|
||||
for i := 0; i < 5; i++ {
|
||||
data := make([]byte, 300)
|
||||
if err := cache.Put(fmt.Sprintf("key%d", i), data); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
if cache.Size() > maxBytes {
|
||||
t.Fatalf("cache size %d exceeds max %d", cache.Size(), maxBytes)
|
||||
}
|
||||
|
||||
if !cache.Has("key4") {
|
||||
t.Fatal("expected key4 to be cached")
|
||||
}
|
||||
if cache.Has("key0") {
|
||||
t.Fatal("expected key0 to be evicted")
|
||||
}
|
||||
}
|
||||
|
||||
func TestBlobDiskCache_OversizedEntryRejected(t *testing.T) {
|
||||
cache, err := newBlobDiskCache(100)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer cache.Close()
|
||||
|
||||
data := make([]byte, 200)
|
||||
if err := cache.Put("big", data); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
if cache.Has("big") {
|
||||
t.Fatal("oversized entry should not be cached")
|
||||
}
|
||||
}
|
||||
|
||||
func TestBlobDiskCache_UpdateInPlace(t *testing.T) {
|
||||
cache, err := newBlobDiskCache(1 << 20)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer cache.Close()
|
||||
|
||||
if err := cache.Put("key1", []byte("v1")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := cache.Put("key1", []byte("version2")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
got, ok := cache.Get("key1")
|
||||
if !ok {
|
||||
t.Fatal("expected hit")
|
||||
}
|
||||
if string(got) != "version2" {
|
||||
t.Fatalf("got %q, want %q", got, "version2")
|
||||
}
|
||||
if cache.Len() != 1 {
|
||||
t.Fatalf("expected 1 entry, got %d", cache.Len())
|
||||
}
|
||||
if cache.Size() != int64(len("version2")) {
|
||||
t.Fatalf("expected size %d, got %d", len("version2"), cache.Size())
|
||||
}
|
||||
}
|
||||
|
||||
func TestBlobDiskCache_ReadAt(t *testing.T) {
|
||||
cache, err := newBlobDiskCache(1 << 20)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer cache.Close()
|
||||
|
||||
data := make([]byte, 1024)
|
||||
if _, err := rand.Read(data); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := cache.Put("blob1", data); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
chunk, err := cache.ReadAt("blob1", 100, 200)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !bytes.Equal(chunk, data[100:300]) {
|
||||
t.Fatal("ReadAt returned wrong data")
|
||||
}
|
||||
|
||||
_, err = cache.ReadAt("blob1", 900, 200)
|
||||
if err == nil {
|
||||
t.Fatal("expected error for out-of-bounds read")
|
||||
}
|
||||
|
||||
_, err = cache.ReadAt("missing", 0, 10)
|
||||
if err == nil {
|
||||
t.Fatal("expected error for missing key")
|
||||
}
|
||||
}
|
||||
|
||||
func TestBlobDiskCache_Close(t *testing.T) {
|
||||
cache, err := newBlobDiskCache(1 << 20)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
if err := cache.Put("key1", []byte("data")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := cache.Close(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBlobDiskCache_LRUOrder(t *testing.T) {
|
||||
cache, err := newBlobDiskCache(200)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer cache.Close()
|
||||
|
||||
d := make([]byte, 100)
|
||||
if err := cache.Put("a", d); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := cache.Put("b", d); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// Access "a" to make it most recently used
|
||||
cache.Get("a")
|
||||
|
||||
// Adding "c" should evict "b" (LRU), not "a"
|
||||
if err := cache.Put("c", d); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
if !cache.Has("a") {
|
||||
t.Fatal("expected 'a' to survive")
|
||||
}
|
||||
if !cache.Has("c") {
|
||||
t.Fatal("expected 'c' to be present")
|
||||
}
|
||||
if cache.Has("b") {
|
||||
t.Fatal("expected 'b' to be evicted")
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user