Check cached sources with upstream after the cache TTL (closes #69)

Cached images were served until eviction, so upstream changes were
missed.

After 24 hours a request checks the source with upstream, sending its
stored ETag and Last-Modified. A 304 renews it with no download or
conversion. New bytes replace it; older variants are converted again.
A failed check serves the cached image and puts the URL in the
negative cache. Images not checked for 30 days are evicted.

Variant rows keep the source's ETag and Last-Modified for checking a
variant of an evicted source.

A variant file's modification time is its source's check time when it
was made or renewed. Hits read it without a database call. The ETag
includes it, so new bytes get a new ETag.

Model: opus-5-5

internal/handlers/client_gone_internal_test.go

@@ -26,7 +26,7 @@ type heldFetcher struct {
 // Fetch sends on f.started, waits for f.release to be closed, and returns
 // httpfetcher.ErrUpstreamError.
 func (f heldFetcher) Fetch(
-	context.Context, imagerequest.Source,
+	context.Context, imagerequest.Source, string, string,
 ) (*httpfetcher.FetchResult, error) {
 	f.started <- struct{}{}
 

internal/handlers/handlers_internal_test.go

@@ -131,7 +131,7 @@ func newMockFetcher(fs fs.FS) *mockFetcher {
 }
 
 func (f *mockFetcher) Fetch(
-	_ context.Context, source imagerequest.Source,
+	_ context.Context, source imagerequest.Source, _, _ string,
 ) (*httpfetcher.FetchResult, error) {
 	path := source.Host + source.Path
 
@@ -257,6 +257,90 @@ func TestHandleImage_ConditionalRequest_IfNoneMatch_DifferentETag(t *testing.T)
 	}
 }
 
+// TestHandleImage_ConditionalRequest_EarlierETagGetsChangedImage serves an
+// image, changes it upstream, and requests it again with the ETag first
+// served, once the cache TTL has passed. The changed image must be served,
+// not 304 Not Modified.
+func TestHandleImage_ConditionalRequest_EarlierETagGetsChangedImage(t *testing.T) {
+	t.Parallel()
+
+	const host = "goodhost.example.com"
+
+	upstream := fstest.MapFS{
+		host + photoPath: &fstest.MapFile{
+			Data: generateTestJPEG(t, 100, 100, color.RGBA{255, 0, 0, 255}),
+		},
+	}
+
+	// A cache TTL of one nanosecond has passed by the second request
+	cache, err := imgcache.NewCache(setupTestDB(t), imgcache.Config{
+		StateDir:    t.TempDir(),
+		NegativeTTL: 5 * time.Minute,
+		CacheTTL:    time.Nanosecond,
+		MaxBytes:    1 << 30,
+	}, prometheus.NewRegistry())
+	if err != nil {
+		t.Fatalf("imgcache.NewCache() error = %v", err)
+	}
+
+	svc, err := imageproxy.New(&imageproxy.Config{
+		Cache:   cache,
+		Fetcher: newMockFetcher(upstream),
+		Processor: imageprocessor.New(imageprocessor.Params{
+			MaxInputBytes:           50 << 20,
+			MaxConcurrentProcessing: 2,
+		}),
+		MaxResponseSize: 50 << 20,
+		Logger:          slog.New(slog.DiscardHandler),
+	}, prometheus.NewRegistry())
+	if err != nil {
+		t.Fatalf("imageproxy.New() error = %v", err)
+	}
+
+	h := &Handlers{
+		imgSvc:    svc,
+		log:       slog.New(slog.DiscardHandler),
+		signer:    signature.New(testSigningKey),
+		allowlist: allowlist.New([]string{host}),
+	}
+
+	r := chi.NewRouter()
+	r.Get("/v1/image/*", h.HandleImage())
+
+	target := "/v1/image/" + host + photoPath + "/50x50.jpeg"
+
+	first := httptest.NewRecorder()
+	r.ServeHTTP(first,
+		httptest.NewRequestWithContext(t.Context(), http.MethodGet, target, nil))
+
+	etag := first.Header().Get("ETag")
+	if first.Code != http.StatusOK || etag == "" {
+		t.Fatalf("first request: status = %d, ETag = %q, want %d and an ETag",
+			first.Code, etag, http.StatusOK)
+	}
+
+	upstream[host+photoPath] = &fstest.MapFile{
+		Data: generateTestJPEG(t, 100, 100, color.RGBA{0, 0, 255, 255}),
+	}
+
+	req := httptest.NewRequestWithContext(t.Context(), http.MethodGet, target, nil)
+	req.Header.Set("If-None-Match", etag)
+
+	second := httptest.NewRecorder()
+	r.ServeHTTP(second, req)
+	t.Logf("first ETag %s; with it after the change: status %d, ETag %s, %d bytes",
+		etag, second.Code, second.Header().Get("ETag"), second.Body.Len())
+
+	if second.Code != http.StatusOK {
+		t.Fatalf("status = %d, want %d with the changed image",
+			second.Code, http.StatusOK)
+	}
+
+	if bytes.Equal(second.Body.Bytes(), first.Body.Bytes()) {
+		t.Error("the image served is the one made before the change")
+	}
+}
+
 func TestHandleImage_ETagHeader(t *testing.T) {
 	t.Parallel()
 

internal/handlers/image_errors_internal_test.go

@@ -428,7 +428,7 @@ type failingFetcher struct {
 
 // Fetch returns f's error.
 func (f failingFetcher) Fetch(
-	context.Context, imagerequest.Source,
+	context.Context, imagerequest.Source, string, string,
 ) (*httpfetcher.FetchResult, error) {
 	return nil, f.err
 }

internal/handlers/referer_blocklist_internal_test.go

@@ -36,10 +36,11 @@ type countingFetcher struct {
 // Fetch counts the fetch and passes it on.
 func (f *countingFetcher) Fetch(
 	ctx context.Context, source imagerequest.Source,
+	ifNoneMatch, ifModifiedSince string,
 ) (*httpfetcher.FetchResult, error) {
 	f.fetches.Add(1)
 
-	return f.Fetcher.Fetch(ctx, source)
+	return f.Fetcher.Fetch(ctx, source, ifNoneMatch, ifModifiedSince)
 }
 
 // newRefererRoutes returns both image routes of a Handlers whose