Check cached sources with upstream after the cache TTL (closes #69)
check / check (push) Canceled after 0s
check / check (push) Canceled after 0s
Cached images were served until eviction, so upstream changes were missed. After 24 hours a request checks the source with upstream, sending its stored ETag and Last-Modified. A 304 renews it with no download or conversion. New bytes replace it; older variants are converted again. A failed check serves the cached image and puts the URL in the negative cache. Images not checked for 30 days are evicted. Variant rows keep the source's ETag and Last-Modified for checking a variant of an evicted source. A variant file's modification time is its source's check time when it was made or renewed. Hits read it without a database call. The ETag includes it, so new bytes get a new ETag. Model: opus-5-5
This commit is contained in:
1 parent
b3b8408d18
commit
445ff628fd
29 files changed
+1709
-269
No files matched your search
@@ -0,0 +1,88 @@
|
||||
package imgcache
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"os"
|
||||
"slices"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"sneak.berlin/go/pixa/internal/imagerequest"
|
||||
)
|
||||
|
||||
// TestEvictNotCheckedRecently stores a source and a variant last checked with
|
||||
// upstream 31 days ago and a source and a variant checked 29 days ago.
|
||||
// EvictNotCheckedRecently must remove the rows and files of the first two and
|
||||
// keep the other two.
|
||||
func TestEvictNotCheckedRecently(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
cache, _ := newEvictionTestCache(t, 1<<30)
|
||||
old := time.Now().Add(-31 * 24 * time.Hour)
|
||||
recent := time.Now().Add(-29 * 24 * time.Hour)
|
||||
|
||||
oldSource := imagerequest.Source{Host: testHostCDN, Path: "/old.jpg"}
|
||||
recentSource := imagerequest.Source{Host: testHostCDN, Path: "/recent.jpg"}
|
||||
|
||||
for src, checkedAt := range map[imagerequest.Source]time.Time{
|
||||
oldSource: old, recentSource: recent,
|
||||
} {
|
||||
_, err := cache.PutSource(t.Context(), src, bytes.NewReader([]byte("source")),
|
||||
SourceInfo{FetchedAt: checkedAt, CheckedAt: checkedAt})
|
||||
if err != nil {
|
||||
t.Fatalf("PutSource(%s) error = %v", src.Path, err)
|
||||
}
|
||||
}
|
||||
|
||||
oldVariant := imagerequest.VariantKey("aabbccdd0001")
|
||||
recentVariant := imagerequest.VariantKey("aabbccdd0002")
|
||||
|
||||
for key, checkedAt := range map[imagerequest.VariantKey]time.Time{
|
||||
oldVariant: old, recentVariant: recent,
|
||||
} {
|
||||
err := cache.PutVariant(t.Context(), key, SourceInfo{CheckedAt: checkedAt},
|
||||
bytes.NewReader([]byte("variant")))
|
||||
if err != nil {
|
||||
t.Fatalf("PutVariant(%s) error = %v", key, err)
|
||||
}
|
||||
}
|
||||
|
||||
err := cache.EvictNotCheckedRecently(t.Context())
|
||||
if err != nil {
|
||||
t.Fatalf("EvictNotCheckedRecently() error = %v", err)
|
||||
}
|
||||
|
||||
sourceRows, err := queryStringColumn[imagerequest.SourceKey](t.Context(),
|
||||
cache.db, `SELECT source_key FROM source_content`, "sources", "source")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
variantRows, err := queryStringColumn[imagerequest.VariantKey](t.Context(),
|
||||
cache.db, `SELECT cache_key FROM variant_content`, "variants", "variant")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
t.Logf("source rows left = %v, variant rows left = %v", sourceRows, variantRows)
|
||||
|
||||
if !slices.Equal(sourceRows, []imagerequest.SourceKey{recentSource.Key()}) {
|
||||
t.Errorf("source rows left = %v, want only %s", sourceRows, recentSource.Key())
|
||||
}
|
||||
|
||||
if !slices.Equal(variantRows, []imagerequest.VariantKey{recentVariant}) {
|
||||
t.Errorf("variant rows left = %v, want only %s", variantRows, recentVariant)
|
||||
}
|
||||
|
||||
for path, wantKept := range map[string]bool{
|
||||
cache.srcContent.keyToPath(string(oldSource.Key())): false,
|
||||
cache.srcContent.keyToPath(string(recentSource.Key())): true,
|
||||
cache.variants.keyToPath(string(oldVariant)): false,
|
||||
cache.variants.keyToPath(string(recentVariant)): true,
|
||||
} {
|
||||
_, statErr := os.Stat(path)
|
||||
if kept := statErr == nil; kept != wantKept {
|
||||
t.Errorf("file %s kept = %t, want %t (%v)", path, kept, wantKept, statErr)
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user