Check cached sources with upstream after the cache TTL (closes #69)

Cached images were served until eviction, so upstream changes were
missed.

After 24 hours a request checks the source with upstream, sending its
stored ETag and Last-Modified. A 304 renews it with no download or
conversion. New bytes replace it; older variants are converted again.
A failed check serves the cached image and puts the URL in the
negative cache. Images not checked for 30 days are evicted.

Variant rows keep the source's ETag and Last-Modified for checking a
variant of an evicted source.

A variant file's modification time is its source's check time when it
was made or renewed. Hits read it without a database call. The ETag
includes it, so new bytes get a new ETag.

Model: opus-5-5

internal/imageproxy/cache_ttl_internal_test.go

@@ -0,0 +1,493 @@
+package imageproxy
+
+import (
+	"bytes"
+	"context"
+	"errors"
+	"fmt"
+	"image/color"
+	"io"
+	"net/http"
+	"os"
+	"sync"
+	"testing"
+	"time"
+
+	"sneak.berlin/go/pixa/internal/httpfetcher"
+	"sneak.berlin/go/pixa/internal/imagerequest"
+)
+
+// Validators the test upstream sends with its image.
+const (
+	testETag         = `"v1"`
+	testLastModified = "Mon, 02 Nov 2026 10:00:00 GMT"
+)
+
+// errConnectionRefused stands for an upstream that cannot be reached.
+var errConnectionRefused = errors.New("connection refused")
+
+// upstreamFailure is one way a fetch from upstream fails.
+type upstreamFailure struct {
+	name string
+	err  error
+}
+
+// upstreamFailures returns the ways a check of a cached image with upstream
+// can fail: an error status, a timeout and no connection.
+func upstreamFailures() []upstreamFailure {
+	return []upstreamFailure{
+		{"error status", fmt.Errorf("%w: status 503", httpfetcher.ErrUpstreamError)},
+		{"timeout", httpfetcher.ErrUpstreamTimeout},
+		{"no connection", errConnectionRefused},
+	}
+}
+
+// conditionalUpstream serves one JPEG image with an ETag or a Last-Modified
+// header, or both. It answers 304 Not Modified to a fetch whose If-None-Match
+// is its ETag, or, when it has no ETag, whose If-Modified-Since is its
+// Last-Modified. While err is set, every fetch fails with it. It records the
+// conditional headers of every fetch.
+type conditionalUpstream struct {
+	mu           sync.Mutex
+	image        []byte
+	etag         string
+	lastModified string
+	err          error
+
+	ifNoneMatch     []string
+	ifModifiedSince []string
+}
+
+func (u *conditionalUpstream) Fetch(
+	_ context.Context, _ imagerequest.Source,
+	ifNoneMatch, ifModifiedSince string,
+) (*httpfetcher.FetchResult, error) {
+	u.mu.Lock()
+	defer u.mu.Unlock()
+
+	u.ifNoneMatch = append(u.ifNoneMatch, ifNoneMatch)
+	u.ifModifiedSince = append(u.ifModifiedSince, ifModifiedSince)
+
+	if u.err != nil {
+		return nil, u.err
+	}
+
+	unchanged := ifNoneMatch == u.etag
+	if u.etag == "" {
+		unchanged = ifModifiedSince == u.lastModified
+	}
+
+	if unchanged {
+		return &httpfetcher.FetchResult{
+			Content:    http.NoBody,
+			StatusCode: http.StatusNotModified,
+		}, nil
+	}
+
+	headers := http.Header{}
+	if u.etag != "" {
+		headers.Set("ETag", u.etag)
+	}
+
+	if u.lastModified != "" {
+		headers.Set("Last-Modified", u.lastModified)
+	}
+
+	return &httpfetcher.FetchResult{
+		Content:       io.NopCloser(bytes.NewReader(u.image)),
+		ContentLength: int64(len(u.image)),
+		ContentType:   testContentTypeJPEG,
+		Headers:       headers,
+		StatusCode:    http.StatusOK,
+	}, nil
+}
+
+// change replaces the upstream image and its ETag.
+func (u *conditionalUpstream) change(image []byte, etag string) {
+	u.mu.Lock()
+	defer u.mu.Unlock()
+
+	u.image = image
+	u.etag = etag
+}
+
+// fail makes every later fetch fail with err.
+func (u *conditionalUpstream) fail(err error) {
+	u.mu.Lock()
+	defer u.mu.Unlock()
+
+	u.err = err
+}
+
+// fetches returns how many fetches upstream was asked for.
+func (u *conditionalUpstream) fetches() int {
+	u.mu.Lock()
+	defer u.mu.Unlock()
+
+	return len(u.ifNoneMatch)
+}
+
+// lastConditionalHeaders returns the If-None-Match and If-Modified-Since of
+// the last fetch.
+func (u *conditionalUpstream) lastConditionalHeaders() (string, string) {
+	u.mu.Lock()
+	defer u.mu.Unlock()
+
+	last := len(u.ifNoneMatch) - 1
+
+	return u.ifNoneMatch[last], u.ifModifiedSince[last]
+}
+
+// setupConditionalUpstreamProxy returns a proxy whose upstream is a
+// conditionalUpstream serving a red JPEG with an ETag.
+func setupConditionalUpstreamProxy(
+	t *testing.T,
+) (*testProxy, *TestFixtures, *conditionalUpstream) {
+	t.Helper()
+
+	svc, fixtures := setupTestProxy(t)
+	upstream := &conditionalUpstream{
+		image: generateTestJPEG(t, 100, 100, color.RGBA{255, 0, 0, 255}),
+		etag:  testETag,
+	}
+	svc.fetcher = upstream
+
+	return svc, fixtures, upstream
+}
+
+// TestProxy_Get_FreshVariantAsksUpstreamNothing checks that a variant whose
+// source was checked with upstream less than the cache TTL ago is served from
+// the cache without asking upstream.
+func TestProxy_Get_FreshVariantAsksUpstreamNothing(t *testing.T) {
+	t.Parallel()
+
+	svc, fixtures, upstream := setupConditionalUpstreamProxy(t)
+	variant := photoVariant(fixtures, 85, imagerequest.FitCover)
+
+	first, _ := getImage(t, svc, variant)
+	second, status := getImage(t, svc, variant)
+
+	t.Logf("cache status = %s, upstream fetches = %d", status, upstream.fetches())
+
+	if status != CacheHit {
+		t.Errorf("CacheStatus = %q, want %q", status, CacheHit)
+	}
+
+	if !bytes.Equal(first, second) {
+		t.Error("the cached variant differs from the one first served")
+	}
+
+	if n := upstream.fetches(); n != 1 {
+		t.Errorf("upstream fetches = %d, want 1", n)
+	}
+}
+
+// TestProxy_Get_UnchangedSourceAfterTTLIsNotDownloadedAgain caches a variant,
+// sets its source's check time and the variant's file time two days back,
+// and requests it again, with the source still cached and with the source's
+// row and file removed, as eviction does. Upstream must be asked with the
+// validator it sent, answer 304, and send no bytes; the variant must be served
+// again without being made again, and then be served from the cache until the
+// TTL passes.
+func TestProxy_Get_UnchangedSourceAfterTTLIsNotDownloadedAgain(t *testing.T) {
+	t.Parallel()
+
+	tests := []struct {
+		name                string
+		sourceEvicted       bool
+		etag                string
+		lastModified        string
+		wantIfNoneMatch     string
+		wantIfModifiedSince string
+	}{
+		{"ETag", false, testETag, "", testETag, ""},
+		{"Last-Modified only", false, "", testLastModified, "", testLastModified},
+		{"both", false, testETag, testLastModified, testETag, testLastModified},
+		{"ETag, source evicted", true, testETag, "", testETag, ""},
+		{"Last-Modified only, source evicted", true, "", testLastModified, "",
+			testLastModified},
+		{"both, source evicted", true, testETag, testLastModified, testETag,
+			testLastModified},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			t.Parallel()
+
+			svc, fixtures, upstream := setupConditionalUpstreamProxy(t)
+			upstream.etag = tt.etag
+			upstream.lastModified = tt.lastModified
+			variant := photoVariant(fixtures, 85, imagerequest.FitCover)
+
+			first, _ := getImage(t, svc, variant)
+			setBackSourceAndVariant(t, svc, variant, time.Now().AddDate(0, 0, -2))
+
+			if tt.sourceEvicted {
+				evictSource(t, svc, variant.Source)
+			}
+
+			second, status := getImage(t, svc, variant)
+
+			ifNoneMatch, ifModifiedSince := upstream.lastConditionalHeaders()
+			counters := readCounters(t, svc)
+			t.Logf("cache status = %s, If-None-Match = %q, If-Modified-Since = %q, "+
+				"counters = %+v", status, ifNoneMatch, ifModifiedSince, counters)
+
+			if ifNoneMatch != tt.wantIfNoneMatch {
+				t.Errorf("If-None-Match = %q, want %q", ifNoneMatch, tt.wantIfNoneMatch)
+			}
+
+			if ifModifiedSince != tt.wantIfModifiedSince {
+				t.Errorf("If-Modified-Since = %q, want %q",
+					ifModifiedSince, tt.wantIfModifiedSince)
+			}
+
+			if !bytes.Equal(first, second) {
+				t.Error("the variant served after the check differs from the cached one")
+			}
+
+			if counters.upstreamFetchCount != 1 || counters.transformCount != 1 {
+				t.Errorf("upstream downloads = %d, transforms = %d; want 1 and 1",
+					counters.upstreamFetchCount, counters.transformCount)
+			}
+
+			_, status = getImage(t, svc, variant)
+			if status != CacheHit {
+				t.Errorf("CacheStatus after the check = %q, want %q", status, CacheHit)
+			}
+
+			if n := upstream.fetches(); n != 2 {
+				t.Errorf("upstream fetches = %d, want 2", n)
+			}
+		})
+	}
+}
+
+// TestProxy_Get_ChangedSourceAfterTTLReplacesSourceAndVariants caches two
+// variants of one source, changes the image upstream, and sets the source's
+// check time and both variants' file times two days back. Each variant must
+// then be made again from the new image, which is downloaded once.
+func TestProxy_Get_ChangedSourceAfterTTLReplacesSourceAndVariants(t *testing.T) {
+	t.Parallel()
+
+	svc, fixtures, upstream := setupConditionalUpstreamProxy(t)
+	variants := []*imagerequest.Request{
+		photoVariant(fixtures, 85, imagerequest.FitCover),
+		photoVariant(fixtures, 40, imagerequest.FitCover),
+	}
+
+	oldImages := make([][]byte, len(variants))
+	for i, variant := range variants {
+		oldImages[i], _ = getImage(t, svc, variant)
+	}
+
+	newSource := generateTestJPEG(t, 120, 90, color.RGBA{0, 0, 255, 255})
+	upstream.change(newSource, `"v2"`)
+
+	twoDaysAgo := time.Now().AddDate(0, 0, -2)
+	for _, variant := range variants {
+		setBackSourceAndVariant(t, svc, variant, twoDaysAgo)
+	}
+
+	for i, variant := range variants {
+		image, _ := getImage(t, svc, variant)
+		if bytes.Equal(image, oldImages[i]) {
+			t.Errorf("variant %d is still made from the old image", i)
+		}
+	}
+
+	counters := readCounters(t, svc)
+	t.Logf("counters = %+v, upstream fetches = %d", counters, upstream.fetches())
+
+	if counters.upstreamFetchCount != 2 {
+		t.Errorf("upstream downloads = %d, want 2", counters.upstreamFetchCount)
+	}
+
+	cached, err := os.ReadFile(sourceFilePath(svc, variants[0].Source.Key()))
+	if err != nil {
+		t.Fatalf("reading the cached source: %v", err)
+	}
+
+	if !bytes.Equal(cached, newSource) {
+		t.Error("the cached source is not the new image")
+	}
+}
+
+// TestProxy_Get_UpstreamFailureAfterTTLServesCachedImage caches a variant,
+// sets its source's check time and the variant's file time two days back, and
+// makes upstream fail. The cached variant must still be served, and a variant
+// not yet cached must be made from the cached source.
+func TestProxy_Get_UpstreamFailureAfterTTLServesCachedImage(t *testing.T) {
+	t.Parallel()
+
+	for _, failure := range upstreamFailures() {
+		t.Run(failure.name, func(t *testing.T) {
+			t.Parallel()
+
+			svc, fixtures, upstream := setupConditionalUpstreamProxy(t)
+			variant := photoVariant(fixtures, 85, imagerequest.FitCover)
+
+			first, _ := getImage(t, svc, variant)
+			setBackSourceAndVariant(t, svc, variant, time.Now().AddDate(0, 0, -2))
+			upstream.fail(failure.err)
+
+			second, _ := getImage(t, svc, variant)
+			t.Logf("upstream fetches = %d", upstream.fetches())
+
+			if upstream.fetches() != 2 {
+				t.Errorf("upstream fetches = %d, want 2", upstream.fetches())
+			}
+
+			if !bytes.Equal(first, second) {
+				t.Error("the variant served while upstream fails differs from the cached one")
+			}
+
+			getImage(t, svc, photoVariant(fixtures, 40, imagerequest.FitCover))
+		})
+	}
+}
+
+// TestProxy_Get_FailingUpstreamIsAskedOnceWhileImageIsServedOften caches a
+// variant, sets its source's check time and the variant's file time two days
+// back, and makes every fetch time out. Serving the variant ten times, and a
+// variant not yet cached, must ask upstream once: the failed check puts the
+// source URL in the negative cache.
+func TestProxy_Get_FailingUpstreamIsAskedOnceWhileImageIsServedOften(t *testing.T) {
+	t.Parallel()
+
+	svc, fixtures, upstream := setupConditionalUpstreamProxy(t)
+	variant := photoVariant(fixtures, 85, imagerequest.FitCover)
+
+	getImage(t, svc, variant)
+	setBackSourceAndVariant(t, svc, variant, time.Now().AddDate(0, 0, -2))
+	upstream.fail(httpfetcher.ErrUpstreamTimeout)
+
+	for range 10 {
+		getImage(t, svc, variant)
+	}
+
+	getImage(t, svc, photoVariant(fixtures, 40, imagerequest.FitCover))
+
+	t.Logf("upstream fetches = %d", upstream.fetches())
+
+	if n := upstream.fetches(); n != 2 {
+		t.Errorf("upstream fetches = %d, want 2: the first fetch and one check", n)
+	}
+}
+
+// TestProxy_Get_UpstreamFailureAfterTTLServesVariantOfEvictedSource caches a
+// variant, sets its file time two days back, removes its source's row and
+// file, as eviction does, and makes upstream fail. The cached variant must
+// still be served.
+func TestProxy_Get_UpstreamFailureAfterTTLServesVariantOfEvictedSource(t *testing.T) {
+	t.Parallel()
+
+	for _, failure := range upstreamFailures() {
+		t.Run(failure.name, func(t *testing.T) {
+			t.Parallel()
+
+			svc, fixtures, upstream := setupConditionalUpstreamProxy(t)
+			variant := photoVariant(fixtures, 85, imagerequest.FitCover)
+
+			first, _ := getImage(t, svc, variant)
+			setBackSourceAndVariant(t, svc, variant, time.Now().AddDate(0, 0, -2))
+			evictSource(t, svc, variant.Source)
+			upstream.fail(failure.err)
+
+			second, _ := getImage(t, svc, variant)
+			t.Logf("upstream fetches = %d", upstream.fetches())
+
+			if !bytes.Equal(first, second) {
+				t.Error("the variant served while upstream fails differs from the cached one")
+			}
+
+			if n := upstream.fetches(); n != 2 {
+				t.Errorf("upstream fetches = %d, want 2", n)
+			}
+		})
+	}
+}
+
+// TestProxy_Get_FailingUpstreamIsAskedOnceForVariantOfEvictedSource caches a
+// variant, sets its file time two days back, removes its source's row and
+// file, as eviction does, and makes every fetch time out. Serving the variant
+// ten times must ask upstream once: the failed check puts the source URL in
+// the negative cache.
+func TestProxy_Get_FailingUpstreamIsAskedOnceForVariantOfEvictedSource(t *testing.T) {
+	t.Parallel()
+
+	svc, fixtures, upstream := setupConditionalUpstreamProxy(t)
+	variant := photoVariant(fixtures, 85, imagerequest.FitCover)
+
+	first, _ := getImage(t, svc, variant)
+	setBackSourceAndVariant(t, svc, variant, time.Now().AddDate(0, 0, -2))
+	evictSource(t, svc, variant.Source)
+	upstream.fail(httpfetcher.ErrUpstreamTimeout)
+
+	for range 10 {
+		image, _ := getImage(t, svc, variant)
+		if !bytes.Equal(image, first) {
+			t.Fatal("the variant served while upstream fails differs from the cached one")
+		}
+	}
+
+	t.Logf("upstream fetches = %d", upstream.fetches())
+
+	if n := upstream.fetches(); n != 2 {
+		t.Errorf("upstream fetches = %d, want 2: the first fetch and one check", n)
+	}
+}
+
+// evictSource removes the row and then the file of src's cached source, as
+// the cache's eviction does, and leaves its variants cached.
+func evictSource(t *testing.T, svc *testProxy, src imagerequest.Source) {
+	t.Helper()
+
+	_, err := svc.db.ExecContext(t.Context(),
+		`DELETE FROM source_content WHERE source_key = ?`, string(src.Key()))
+	if err != nil {
+		t.Fatalf("deleting the source's row: %v", err)
+	}
+
+	err = os.Remove(sourceFilePath(svc, src.Key()))
+	if err != nil {
+		t.Fatalf("removing the source's file: %v", err)
+	}
+}
+
+// setBackSourceAndVariant sets back to when the times the cache keeps for
+// req's source and for the variant req asks for: the source's fetch and check
+// times, both last access times, the variant's check time, and the variant
+// file's modification time.
+func setBackSourceAndVariant(
+	t *testing.T, svc *testProxy, req *imagerequest.Request, when time.Time,
+) {
+	t.Helper()
+
+	key := req.Source.Key()
+	setSourceLastAccessed(t, svc.db, key, when)
+
+	_, err := svc.db.ExecContext(t.Context(),
+		`UPDATE source_content SET fetched_at = ?, checked_at = ?
+		WHERE source_key = ?`, when.UTC(), when.UTC(), string(key))
+	if err != nil {
+		t.Fatalf("setting back the source's fetch and check times: %v", err)
+	}
+
+	variantKey := imagerequest.CacheKey(req)
+	setVariantLastAccessed(t, svc.db, variantKey, when)
+
+	_, err = svc.db.ExecContext(t.Context(),
+		`UPDATE variant_content SET checked_at = ? WHERE cache_key = ?`,
+		when.UTC(), string(variantKey))
+	if err != nil {
+		t.Fatalf("setting back the variant's check time: %v", err)
+	}
+
+	path := variantFilePath(svc, variantKey)
+
+	err = os.Chtimes(path, when, when)
+	if err != nil {
+		t.Fatalf("setting back the times of %s: %v", path, err)
+	}
+}

internal/imageproxy/concurrent_misses_internal_test.go

@@ -37,6 +37,7 @@ type heldFetcher struct {
 
 func (f *heldFetcher) Fetch(
 	ctx context.Context, source imagerequest.Source,
+	ifNoneMatch, ifModifiedSince string,
 ) (*httpfetcher.FetchResult, error) {
 	f.fetches.Add(1)
 
@@ -48,7 +49,7 @@ func (f *heldFetcher) Fetch(
 		return nil, ctx.Err()
 	}
 
-	return f.upstream.Fetch(ctx, source)
+	return f.upstream.Fetch(ctx, source, ifNoneMatch, ifModifiedSince)
 }
 
 // releaseFetches lets every held fetch, and every later one, go on.
@@ -489,7 +490,7 @@ func TestProxy_Get_WaiterGetsProcessingTimedOut(t *testing.T) {
 type panickingFetcher struct{}
 
 func (panickingFetcher) Fetch(
-	context.Context, imagerequest.Source,
+	context.Context, imagerequest.Source, string, string,
 ) (*httpfetcher.FetchResult, error) {
 	panic("upstream fetcher panicked")
 }

internal/imageproxy/imageproxy.go

@@ -10,6 +10,7 @@ import (
 	"fmt"
 	"io"
 	"log/slog"
+	"net/http"
 	"runtime/debug"
 	"time"
 
@@ -157,7 +158,9 @@ const (
 //
 // A cache hit opens the variant file and makes no database call: the cache
 // hands its access time to the goroutine that writes access times, without
-// waiting for it, and its hit is counted in memory. A variant file that
+// waiting for it, and its hit is counted in memory. A variant whose source
+// was last checked with upstream CacheTTL or more ago is a miss, which checks
+// the source first (see processFromSourceOrFetch). A variant file that
 // cannot be served, such as one whose first bytes are not those of an output
 // format, is a miss, and the variant is made again.
 func (p *Proxy) Get(
@@ -165,20 +168,20 @@ func (p *Proxy) Get(
 ) (*ImageResponse, error) {
 	cacheKey := imagerequest.CacheKey(req)
 
-	reader, size, contentType, err := p.cache.Variant(cacheKey)
+	cached, err := p.cache.Variant(cacheKey)
 	if err == nil {
 		p.cache.IncrementStats(true)
 
 		return &ImageResponse{
-			Content:       reader,
-			ContentLength: size,
-			ContentType:   contentType,
+			Content:       cached.Content,
+			ContentLength: cached.Size,
+			ContentType:   cached.ContentType,
 			CacheStatus:   CacheHit,
-			ETag:          formatETag(cacheKey),
+			ETag:          formatETag(cacheKey, cached.CheckedAt),
 		}, nil
 	}
 
-	if !errors.Is(err, imgcache.ErrNotFound) {
+	if !errors.Is(err, imgcache.ErrNotFound) && !errors.Is(err, imgcache.ErrStale) {
 		p.log.Warn("cached variant not served, making it again",
 			"cache_key", cacheKey, "error", err)
 	}
@@ -285,7 +288,7 @@ func (p *Proxy) processOrWait(
 		ContentLength: int64(len(variant.data)),
 		ContentType:   variant.contentType,
 		FetchedBytes:  variant.fetchedBytes,
-		ETag:          formatETag(cacheKey),
+		ETag:          formatETag(cacheKey, variant.checkedAt),
 	}, nil
 }
 
@@ -337,13 +340,50 @@ func (p *Proxy) loadCachedSource(
 	return reader, size, format
 }
 
-// processFromSourceOrFetch processes an image, using cached source content
-// if available.
+// processFromSourceOrFetch makes the variant req asks for, from the cached
+// source when there is one and from a fetched source otherwise.
+//
+// A cached source last checked with upstream CacheTTL or more ago is checked
+// first: upstream is asked to send it only if it changed since. When it
+// changed, the new bytes replace the cached source and the variant is made
+// from them. When it did not change, or the check fails, the cached source
+// is used: the variant's cached file is served again if it was made from the
+// cached source's bytes, and otherwise the variant is made from them. When
+// the source is not cached but the variant is, the variant is checked with
+// upstream the same way (see fetchAndProcess).
 func (p *Proxy) processFromSourceOrFetch(
 	ctx context.Context,
 	req *imagerequest.Request,
 	cacheKey imagerequest.VariantKey,
 ) (*processedVariant, error) {
+	cached, err := p.cache.LookupSource(ctx, req.Source)
+	if err != nil {
+		if !errors.Is(err, imgcache.ErrNotFound) {
+			p.log.Warn("failed to look up cached source, fetching", "error", err)
+		}
+
+		return p.fetchAndProcess(ctx, req, cacheKey)
+	}
+
+	if p.cache.Stale(cached.CheckedAt) {
+		fetched, err := p.fetchSource(ctx, req, cached.ETag, cached.LastModified)
+
+		switch {
+		case errors.Is(err, errSourceNotModified):
+			cached.CheckedAt = time.Now().UTC()
+			p.renewSource(ctx, req.Source, cached.CheckedAt)
+		case err != nil:
+			p.recordFailedCheck(ctx, req, err)
+		default:
+			return p.processFetched(ctx, req, cacheKey, fetched)
+		}
+	}
+
+	variant := p.renewVariant(ctx, cacheKey, cached)
+	if variant != nil {
+		return variant, nil
+	}
+
 	source, sourceSize, sourceFormat := p.loadCachedSource(ctx, req.Source)
 
 	// Fetch from upstream if we don't have source data or it's empty
@@ -356,18 +396,199 @@ func (p *Proxy) processFromSourceOrFetch(
 	// Process using cached source; nothing was fetched from upstream. The
 	// image processor reads the source only once it has a processing slot,
 	// so a request waiting for one holds none of it in memory.
-	return p.processAndStore(ctx, req, cacheKey, source, sourceFormat, sourceSize)
+	return p.processAndStore(ctx, req, cacheKey, cached, source, sourceFormat,
+		sourceSize)
 }
 
-// fetchAndProcess fetches from upstream, processes, and caches the result.
-// It counts the fetch with the bytes read from upstream, including when
-// reading the response or a later step fails. It fetches nothing while the
-// source URL is in the negative cache.
+// recordFailedCheck logs a failed check of req's source with upstream, after
+// which the image is still served from the cache. It puts the source URL in
+// the negative cache, whatever the failure, a timeout included, so upstream
+// is asked at most once per NegativeTTL while it fails, however often the
+// image is served. A failure that was a negative cache hit records nothing.
+func (p *Proxy) recordFailedCheck(
+	ctx context.Context, req *imagerequest.Request, err error,
+) {
+	if errors.Is(err, ErrNegativeCached) {
+		return
+	}
+
+	p.log.Warn("checking source with upstream failed, serving from the cache",
+		"request_id", middleware.GetReqID(ctx),
+		"host", req.Source.Host, "path", req.Source.Path, "error", err)
+	p.storeNegative(ctx, req)
+}
+
+// renewSource records that upstream answered at checkedAt that src's cached
+// source had not changed. A failure to record it is only logged: the source
+// is then checked again on its next use.
+func (p *Proxy) renewSource(
+	ctx context.Context, src imagerequest.Source, checkedAt time.Time,
+) {
+	err := p.cache.RenewSource(ctx, src, checkedAt)
+	if err != nil {
+		p.log.Warn("failed to renew cached source", "source_key", src.Key(),
+			"error", err)
+	}
+}
+
+// renewVariant returns the cached variant of cacheKey when it was made from
+// the bytes of the cached source that source describes, and renews it to
+// source.CheckedAt (see imgcache.Cache.RenewVariant). It returns nil when
+// there is no such variant or it cannot be read.
+func (p *Proxy) renewVariant(
+	ctx context.Context, cacheKey imagerequest.VariantKey,
+	source imgcache.SourceInfo,
+) *processedVariant {
+	cached, err := p.cache.RenewVariant(ctx, cacheKey, source)
+	if err != nil {
+		if !errors.Is(err, imgcache.ErrNotFound) {
+			p.log.Warn("cached variant not renewed, making it again",
+				"cache_key", cacheKey, "error", err)
+		}
+
+		return nil
+	}
+
+	return p.readCachedVariant(cacheKey, cached)
+}
+
+// cachedVariantIgnoringTTL returns the cached variant of cacheKey whatever
+// its age, with what its row keeps about the source bytes it was made from
+// (see imgcache.Cache.LookupVariant). It returns nil when the variant is not
+// cached or cannot be read. The SourceInfo is empty when the variant is not
+// returned, or its row is missing or cannot be read.
+func (p *Proxy) cachedVariantIgnoringTTL(
+	ctx context.Context, cacheKey imagerequest.VariantKey,
+) (*processedVariant, imgcache.SourceInfo) {
+	cached, err := p.cache.VariantIgnoringTTL(cacheKey)
+	if err != nil {
+		if !errors.Is(err, imgcache.ErrNotFound) {
+			p.log.Warn("cached variant not served", "cache_key", cacheKey,
+				"error", err)
+		}
+
+		return nil, imgcache.SourceInfo{}
+	}
+
+	variant := p.readCachedVariant(cacheKey, cached)
+	if variant == nil {
+		return nil, imgcache.SourceInfo{}
+	}
+
+	madeFrom, err := p.cache.LookupVariant(ctx, cacheKey)
+	if err != nil && !errors.Is(err, imgcache.ErrNotFound) {
+		p.log.Warn("failed to look up cached variant", "cache_key", cacheKey,
+			"error", err)
+	}
+
+	return variant, madeFrom
+}
+
+// readCachedVariant reads the cached variant of cacheKey from its opened file,
+// and closes the file. It returns nil when the variant cannot be read.
+func (p *Proxy) readCachedVariant(
+	cacheKey imagerequest.VariantKey, file imgcache.VariantFile,
+) *processedVariant {
+	defer func() { _ = file.Content.Close() }()
+
+	data, err := io.ReadAll(file.Content)
+	if err != nil {
+		p.log.Warn("failed to read cached variant", "cache_key", cacheKey,
+			"error", err)
+
+		return nil
+	}
+
+	return &processedVariant{
+		data:        data,
+		contentType: file.ContentType,
+		checkedAt:   file.CheckedAt,
+	}
+}
+
+// fetchAndProcess fetches the source from upstream, stores it, and makes
+// and stores the variant from it.
+//
+// The variant can still be cached after eviction removed its source. The
+// fetch then sends the ETag and Last-Modified of the source bytes the variant
+// was made from, so upstream sends the source only if it changed since. When
+// it did not change, the cached variant is renewed and served, with no
+// download and no conversion. When the fetch fails, the cached variant is
+// served whatever its age, and the failure is recorded as a failed check (see
+// recordFailedCheck).
 func (p *Proxy) fetchAndProcess(
 	ctx context.Context,
 	req *imagerequest.Request,
 	cacheKey imagerequest.VariantKey,
 ) (*processedVariant, error) {
+	variant, madeFrom := p.cachedVariantIgnoringTTL(ctx, cacheKey)
+
+	fetched, err := p.fetchSource(ctx, req, madeFrom.ETag, madeFrom.LastModified)
+
+	switch {
+	case err == nil:
+		return p.processFetched(ctx, req, cacheKey, fetched)
+	case variant == nil:
+		return nil, err
+	case errors.Is(err, errSourceNotModified):
+		variant.checkedAt = time.Now().UTC()
+
+		err = p.cache.SetVariantCheckedAt(ctx, cacheKey, variant.checkedAt)
+		if err != nil {
+			p.log.Warn("failed to renew cached variant", "cache_key", cacheKey,
+				"error", err)
+		}
+	default:
+		p.recordFailedCheck(ctx, req, err)
+	}
+
+	return variant, nil
+}
+
+// processFetched makes and stores the variant from the source fetchSource
+// fetched, then closes the fetch's response.
+func (p *Proxy) processFetched(
+	ctx context.Context,
+	req *imagerequest.Request,
+	cacheKey imagerequest.VariantKey,
+	fetched *fetchedSource,
+) (*processedVariant, error) {
+	// Closing the response frees the upstream connection. It is closed only
+	// after processing, so the fetcher's connection limit also bounds the
+	// fetched sources held in memory while their requests wait for a
+	// processing slot.
+	defer func() { _ = fetched.response.Close() }()
+
+	return p.processAndStore(ctx, req, cacheKey, fetched.info,
+		bytes.NewReader(fetched.data), fetched.format, int64(len(fetched.data)))
+}
+
+// errSourceNotModified is returned by fetchSource when upstream answers 304
+// Not Modified: the cached source has not changed.
+var errSourceNotModified = errors.New("upstream source not modified")
+
+// fetchedSource is a source fetchSource fetched and stored in the cache.
+type fetchedSource struct {
+	data   []byte
+	format imagerequest.Format
+	// info is what the cache keeps about the source
+	info imgcache.SourceInfo
+	// response is the fetch's response, still open; closing it frees the
+	// upstream connection
+	response io.Closer
+}
+
+// fetchSource fetches req's source from upstream and stores it in the cache.
+// A non-empty ifNoneMatch or ifModifiedSince asks upstream to send the source
+// only if it changed since; errSourceNotModified says it did not. It counts
+// the fetch with the bytes read from upstream, including when reading the
+// response or a later step fails. It fetches nothing while the source URL is
+// in the negative cache. The caller closes the returned response.
+func (p *Proxy) fetchSource(
+	ctx context.Context,
+	req *imagerequest.Request,
+	ifNoneMatch, ifModifiedSince string,
+) (*fetchedSource, error) {
 	// Check negative cache - skip fetching for recently-failed URLs
 	negHit, err := p.cache.Failure(ctx, req.Source)
 	if err != nil {
@@ -387,7 +608,8 @@ func (p *Proxy) fetchAndProcess(
 	p.log.Debug("fetching from upstream", "host", req.Source.Host,
 		"path", req.Source.Path, "query", req.Source.Query)
 
-	fetchResult, err := p.fetcher.Fetch(ctx, req.Source)
+	fetchResult, err := p.fetcher.Fetch(ctx, req.Source,
+		ifNoneMatch, ifModifiedSince)
 	if err != nil {
 		// Store negative cache for certain errors
 		if isNegativeCacheable(err) {
@@ -397,11 +619,15 @@ func (p *Proxy) fetchAndProcess(
 		return nil, fmt.Errorf("upstream fetch failed: %w", err)
 	}
 
-	// Closing the body frees the upstream connection. It is closed only
-	// after processing, so the fetcher's connection limit also bounds the
-	// fetched sources held in memory while their requests wait for a
-	// processing slot.
-	defer func() { _ = fetchResult.Content.Close() }()
+	if fetchResult.StatusCode == http.StatusNotModified {
+		_ = fetchResult.Content.Close()
+
+		p.log.Info("upstream source not modified",
+			"request_id", middleware.GetReqID(ctx), "host", req.Source.Host,
+			"path", req.Source.Path, "fetch_ms", fetchResult.FetchDurationMs)
+
+		return nil, errSourceNotModified
+	}
 
 	// Read and validate the source content
 	sourceData, err := io.ReadAll(fetchResult.Content)
@@ -411,31 +637,12 @@ func (p *Proxy) fetchAndProcess(
 	p.incrementUpstreamFetch(fetchBytes)
 
 	if err != nil {
+		_ = fetchResult.Content.Close()
+
 		return nil, fmt.Errorf("failed to read upstream response: %w", err)
 	}
 
-	// Calculate download bitrate
-	var downloadRate string
-
-	if fetchResult.FetchDurationMs > 0 {
-		seconds := float64(fetchResult.FetchDurationMs) / 1000.0 //nolint:mnd // ms to seconds
-		bitsPerSecond := float64(fetchBytes*8) / seconds         //nolint:mnd // bytes to bits
-		downloadRate = humanize.SI(bitsPerSecond, "bps")
-	}
-
-	// Log upstream fetch details
-	p.log.Info("upstream fetched",
-		"request_id", middleware.GetReqID(ctx),
-		"host", req.Source.Host,
-		"path", req.Source.Path,
-		"bytes", fetchBytes,
-		"fetch_ms", fetchResult.FetchDurationMs,
-		"rate", downloadRate,
-		"remote_addr", fetchResult.RemoteAddr,
-		"http", fetchResult.HTTPVersion,
-		"tls", fetchResult.TLSVersion,
-		"cipher", fetchResult.TLSCipherSuite,
-	)
+	p.logUpstreamFetch(ctx, req, fetchResult, fetchBytes)
 
 	// The source's format, detected from its first bytes, which must match
 	// its content type. A body that does not is the upstream's error, so it
@@ -448,18 +655,70 @@ func (p *Proxy) fetchAndProcess(
 			"content_type", fetchResult.ContentType, "error", err)
 		p.storeNegative(ctx, req)
 
+		_ = fetchResult.Content.Close()
+
 		return nil, fmt.Errorf("content validation failed: %w", err)
 	}
 
+	info := fetchedSourceInfo(fetchResult)
+
 	// Store source content
-	_, err = p.cache.PutSource(ctx, req.Source, bytes.NewReader(sourceData))
+	_, err = p.cache.PutSource(ctx, req.Source, bytes.NewReader(sourceData), info)
 	if err != nil {
 		p.log.Warn("failed to store source content", "error", err)
 		// Continue even if caching fails
 	}
 
-	return p.processAndStore(
-		ctx, req, cacheKey, bytes.NewReader(sourceData), sourceFormat, fetchBytes,
+	return &fetchedSource{
+		data:     sourceData,
+		format:   sourceFormat,
+		info:     info,
+		response: fetchResult.Content,
+	}, nil
+}
+
+// fetchedSourceInfo returns what the cache keeps about a source upstream
+// just sent: fetched and checked now, with the ETag and Last-Modified
+// headers of fetchResult.
+func fetchedSourceInfo(fetchResult *httpfetcher.FetchResult) imgcache.SourceInfo {
+	now := time.Now().UTC()
+	upstreamHeaders := http.Header(fetchResult.Headers)
+
+	return imgcache.SourceInfo{
+		FetchedAt:    now,
+		CheckedAt:    now,
+		ETag:         upstreamHeaders.Get("ETag"),
+		LastModified: upstreamHeaders.Get("Last-Modified"),
+	}
+}
+
+// logUpstreamFetch logs a fetch of req's source that read fetchBytes bytes,
+// with its download rate and connection details.
+func (p *Proxy) logUpstreamFetch(
+	ctx context.Context,
+	req *imagerequest.Request,
+	fetchResult *httpfetcher.FetchResult,
+	fetchBytes int64,
+) {
+	var downloadRate string
+
+	if fetchResult.FetchDurationMs > 0 {
+		seconds := float64(fetchResult.FetchDurationMs) / 1000.0 //nolint:mnd // ms to seconds
+		bitsPerSecond := float64(fetchBytes*8) / seconds         //nolint:mnd // bytes to bits
+		downloadRate = humanize.SI(bitsPerSecond, "bps")
+	}
+
+	p.log.Info("upstream fetched",
+		"request_id", middleware.GetReqID(ctx),
+		"host", req.Source.Host,
+		"path", req.Source.Path,
+		"bytes", fetchBytes,
+		"fetch_ms", fetchResult.FetchDurationMs,
+		"rate", downloadRate,
+		"remote_addr", fetchResult.RemoteAddr,
+		"http", fetchResult.HTTPVersion,
+		"tls", fetchResult.TLSVersion,
+		"cipher", fetchResult.TLSCipherSuite,
 	)
 }
 
@@ -491,14 +750,19 @@ type processedVariant struct {
 	data         []byte
 	contentType  string
 	fetchedBytes int64
+	// checkedAt is its source's check time when the variant was made or last
+	// renewed, which its cached file has as its modification time
+	checkedAt time.Time
 }
 
 // processAndStore processes the image read from source, which is in
-// sourceFormat, and stores the result.
+// sourceFormat, and stores the result. sourceInfo describes the source's
+// bytes, and is stored with the variant (see imgcache.Cache.PutVariant).
 func (p *Proxy) processAndStore(
 	ctx context.Context,
 	req *imagerequest.Request,
 	cacheKey imagerequest.VariantKey,
+	sourceInfo imgcache.SourceInfo,
 	source io.Reader,
 	sourceFormat imagerequest.Format,
 	fetchBytes int64,
@@ -559,7 +823,8 @@ func (p *Proxy) processAndStore(
 	)
 
 	// Store variant to cache
-	err = p.cache.PutVariant(ctx, cacheKey, bytes.NewReader(processedData))
+	err = p.cache.PutVariant(ctx, cacheKey, sourceInfo,
+		bytes.NewReader(processedData))
 	if err != nil {
 		p.log.Warn("failed to store variant", "error", err)
 		// Continue even if caching fails
@@ -569,6 +834,7 @@ func (p *Proxy) processAndStore(
 		data:         processedData,
 		contentType:  processResult.ContentType,
 		fetchedBytes: fetchBytes,
+		checkedAt:    sourceInfo.CheckedAt,
 	}, nil
 }
 
@@ -580,13 +846,16 @@ func isNegativeCacheable(err error) bool {
 // etagHashLength is the number of hash characters to use for ETags.
 const etagHashLength = 16
 
-// formatETag formats a VariantKey as a quoted ETag value.
-func formatETag(key imagerequest.VariantKey) string {
+// formatETag returns the quoted ETag of the variant of key: the first
+// etagHashLength characters of key, then checkedAt in Unix nanoseconds.
+// checkedAt is its source's check time when the variant was made or last
+// renewed, so a variant made again from new source bytes gets another ETag.
+func formatETag(key imagerequest.VariantKey, checkedAt time.Time) string {
 	hash := string(key)
 	// Use first 16 characters of hash for a shorter but still unique ETag
 	if len(hash) > etagHashLength {
 		hash = hash[:etagHashLength]
 	}
 
-	return `"` + hash + `"`
+	return fmt.Sprintf(`"%s-%d"`, hash, checkedAt.UnixNano())
 }

internal/imageproxy/imageproxy_internal_test.go

@@ -3,9 +3,7 @@ package imageproxy
 import (
 	"context"
 	"io"
-	"os"
 	"testing"
-	"time"
 
 	"sneak.berlin/go/pixa/internal/imagerequest"
 	"sneak.berlin/go/pixa/internal/magic"
@@ -263,97 +261,6 @@ func TestProxy_Get_Caching(t *testing.T) {
 	}
 }
 
-// TestProxy_Get_ServesSourceAndVariantCachedTwoDaysAgo caches a source and
-// a variant of it and sets back two days every time pixa stored for them.
-// For the source: last_accessed_at in source_content, and the access and
-// modification times of its file. For the variant: last_accessed_at in
-// variant_content, and the access and modification times of its file. It
-// then checks that the variant is still served from the cache and that a new
-// variant is made from the cached source without fetching it again.
-//
-// Left at the present: the files' status change and birth times, which a
-// program cannot set, and the times of the directories holding the files,
-// which other entries share.
-func TestProxy_Get_ServesSourceAndVariantCachedTwoDaysAgo(t *testing.T) {
-	t.Parallel()
-
-	svc, fixtures := setupTestProxy(t)
-	variant := photoVariant(fixtures, 85, imagerequest.FitCover)
-
-	resp, err := svc.Get(t.Context(), variant)
-	if err != nil {
-		t.Fatalf("Get() first request error = %v", err)
-	}
-
-	_ = resp.Content.Close()
-
-	twoDaysAgo := time.Now().AddDate(0, 0, -2)
-	backdateSource(t, svc, variant, twoDaysAgo)
-	backdateVariant(t, svc, variant, twoDaysAgo)
-
-	resp, err = svc.Get(t.Context(), variant)
-	if err != nil {
-		t.Fatalf("Get() of the cached variant error = %v", err)
-	}
-
-	_ = resp.Content.Close()
-
-	if resp.CacheStatus != CacheHit {
-		t.Errorf("cached variant CacheStatus = %q, want %q",
-			resp.CacheStatus, CacheHit)
-	}
-
-	// Another quality is a new variant, made from the cached source
-	resp, err = svc.Get(t.Context(), photoVariant(fixtures, 40, imagerequest.FitCover))
-	if err != nil {
-		t.Fatalf("Get() of a new variant error = %v", err)
-	}
-
-	_ = resp.Content.Close()
-
-	fetches := readCounters(t, svc).upstreamFetchCount
-	if fetches != 1 {
-		t.Errorf("upstream fetches = %d, want 1", fetches)
-	}
-}
-
-// backdateSource sets every time pixa stored for req's cached source to when,
-// as listed on TestProxy_Get_ServesSourceAndVariantCachedTwoDaysAgo.
-func backdateSource(
-	t *testing.T, svc *testProxy, req *imagerequest.Request, when time.Time,
-) {
-	t.Helper()
-
-	key := req.Source.Key()
-	setSourceLastAccessed(t, svc.db, key, when)
-
-	path := sourceFilePath(svc, key)
-
-	err := os.Chtimes(path, when, when)
-	if err != nil {
-		t.Fatalf("setting back the times of %s: %v", path, err)
-	}
-}
-
-// backdateVariant sets every time pixa stored for the cached variant req asks
-// for to when, as listed on
-// TestProxy_Get_ServesSourceAndVariantCachedTwoDaysAgo.
-func backdateVariant(
-	t *testing.T, svc *testProxy, req *imagerequest.Request, when time.Time,
-) {
-	t.Helper()
-
-	key := imagerequest.CacheKey(req)
-	path := variantFilePath(svc, key)
-
-	setVariantLastAccessed(t, svc.db, key, when)
-
-	err := os.Chtimes(path, when, when)
-	if err != nil {
-		t.Fatalf("setting back the times of %s: %v", path, err)
-	}
-}
-
 func TestProxy_Get_DifferentSizes(t *testing.T) {
 	t.Parallel()
 

internal/imageproxy/stats_internal_test.go

@@ -94,7 +94,7 @@ type fakeUpstream struct {
 }
 
 func (u *fakeUpstream) Fetch(
-	context.Context, imagerequest.Source,
+	context.Context, imagerequest.Source, string, string,
 ) (*httpfetcher.FetchResult, error) {
 	return &httpfetcher.FetchResult{
 		Content:       io.NopCloser(u),