Check cached sources with upstream after the cache TTL (closes #69)
check / check (push) Canceled after 0s
check / check (push) Canceled after 0s
Cached images were served until eviction, so upstream changes were missed. After 24 hours a request checks the source with upstream, sending its stored ETag and Last-Modified. A 304 renews it with no download or conversion. New bytes replace it; older variants are converted again. A failed check serves the cached image and puts the URL in the negative cache. Images not checked for 30 days are evicted. Variant rows keep the source's ETag and Last-Modified for checking a variant of an evicted source. A variant file's modification time is its source's check time when it was made or renewed. Hits read it without a database call. The ETag includes it, so new bytes get a new ETag. Model: opus-5-5
This commit is contained in:
1 parent
b3b8408d18
commit
445ff628fd
29 files changed
+1709
-269
No files matched your search
@@ -3,9 +3,7 @@ package imageproxy
|
||||
import (
|
||||
"context"
|
||||
"io"
|
||||
"os"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"sneak.berlin/go/pixa/internal/imagerequest"
|
||||
"sneak.berlin/go/pixa/internal/magic"
|
||||
@@ -263,97 +261,6 @@ func TestProxy_Get_Caching(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// TestProxy_Get_ServesSourceAndVariantCachedTwoDaysAgo caches a source and
|
||||
// a variant of it and sets back two days every time pixa stored for them.
|
||||
// For the source: last_accessed_at in source_content, and the access and
|
||||
// modification times of its file. For the variant: last_accessed_at in
|
||||
// variant_content, and the access and modification times of its file. It
|
||||
// then checks that the variant is still served from the cache and that a new
|
||||
// variant is made from the cached source without fetching it again.
|
||||
//
|
||||
// Left at the present: the files' status change and birth times, which a
|
||||
// program cannot set, and the times of the directories holding the files,
|
||||
// which other entries share.
|
||||
func TestProxy_Get_ServesSourceAndVariantCachedTwoDaysAgo(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
svc, fixtures := setupTestProxy(t)
|
||||
variant := photoVariant(fixtures, 85, imagerequest.FitCover)
|
||||
|
||||
resp, err := svc.Get(t.Context(), variant)
|
||||
if err != nil {
|
||||
t.Fatalf("Get() first request error = %v", err)
|
||||
}
|
||||
|
||||
_ = resp.Content.Close()
|
||||
|
||||
twoDaysAgo := time.Now().AddDate(0, 0, -2)
|
||||
backdateSource(t, svc, variant, twoDaysAgo)
|
||||
backdateVariant(t, svc, variant, twoDaysAgo)
|
||||
|
||||
resp, err = svc.Get(t.Context(), variant)
|
||||
if err != nil {
|
||||
t.Fatalf("Get() of the cached variant error = %v", err)
|
||||
}
|
||||
|
||||
_ = resp.Content.Close()
|
||||
|
||||
if resp.CacheStatus != CacheHit {
|
||||
t.Errorf("cached variant CacheStatus = %q, want %q",
|
||||
resp.CacheStatus, CacheHit)
|
||||
}
|
||||
|
||||
// Another quality is a new variant, made from the cached source
|
||||
resp, err = svc.Get(t.Context(), photoVariant(fixtures, 40, imagerequest.FitCover))
|
||||
if err != nil {
|
||||
t.Fatalf("Get() of a new variant error = %v", err)
|
||||
}
|
||||
|
||||
_ = resp.Content.Close()
|
||||
|
||||
fetches := readCounters(t, svc).upstreamFetchCount
|
||||
if fetches != 1 {
|
||||
t.Errorf("upstream fetches = %d, want 1", fetches)
|
||||
}
|
||||
}
|
||||
|
||||
// backdateSource sets every time pixa stored for req's cached source to when,
|
||||
// as listed on TestProxy_Get_ServesSourceAndVariantCachedTwoDaysAgo.
|
||||
func backdateSource(
|
||||
t *testing.T, svc *testProxy, req *imagerequest.Request, when time.Time,
|
||||
) {
|
||||
t.Helper()
|
||||
|
||||
key := req.Source.Key()
|
||||
setSourceLastAccessed(t, svc.db, key, when)
|
||||
|
||||
path := sourceFilePath(svc, key)
|
||||
|
||||
err := os.Chtimes(path, when, when)
|
||||
if err != nil {
|
||||
t.Fatalf("setting back the times of %s: %v", path, err)
|
||||
}
|
||||
}
|
||||
|
||||
// backdateVariant sets every time pixa stored for the cached variant req asks
|
||||
// for to when, as listed on
|
||||
// TestProxy_Get_ServesSourceAndVariantCachedTwoDaysAgo.
|
||||
func backdateVariant(
|
||||
t *testing.T, svc *testProxy, req *imagerequest.Request, when time.Time,
|
||||
) {
|
||||
t.Helper()
|
||||
|
||||
key := imagerequest.CacheKey(req)
|
||||
path := variantFilePath(svc, key)
|
||||
|
||||
setVariantLastAccessed(t, svc.db, key, when)
|
||||
|
||||
err := os.Chtimes(path, when, when)
|
||||
if err != nil {
|
||||
t.Fatalf("setting back the times of %s: %v", path, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestProxy_Get_DifferentSizes(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
|
||||
Reference in new issue
Block a user