Check cached sources with upstream after the cache TTL (closes #69)
check / check (push) Canceled after 0s
check / check (push) Canceled after 0s
Cached images were served until eviction, so upstream changes were missed. After 24 hours a request checks the source with upstream, sending its stored ETag and Last-Modified. A 304 renews it with no download or conversion. New bytes replace it; older variants are converted again. A failed check serves the cached image and puts the URL in the negative cache. Images not checked for 30 days are evicted. Variant rows keep the source's ETag and Last-Modified for checking a variant of an evicted source. A variant file's modification time is its source's check time when it was made or renewed. Hits read it without a database call. The ETag includes it, so new bytes get a new ETag. Model: opus-5-5
This commit is contained in:
1 parent
b3b8408d18
commit
445ff628fd
29 files changed
+1709
-269
No files matched your search
@@ -0,0 +1,76 @@
|
||||
package httpfetcher
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"sync"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// TestFetch_SendsConditionalHeadersAndHandlesNotModified checks that Fetch
|
||||
// sends a non-empty ifNoneMatch and ifModifiedSince as request headers, that
|
||||
// upstream's 304 to such a request is a result with status 304 and no
|
||||
// content, and that a 304 to a request with neither is ErrUpstreamError.
|
||||
func TestFetch_SendsConditionalHeadersAndHandlesNotModified(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
const (
|
||||
etag = `"abc"`
|
||||
lastModified = "Mon, 02 Nov 2026 10:00:00 GMT"
|
||||
)
|
||||
|
||||
var (
|
||||
mu sync.Mutex
|
||||
received http.Header
|
||||
)
|
||||
|
||||
srv := httptest.NewServer(http.HandlerFunc(
|
||||
func(w http.ResponseWriter, r *http.Request) {
|
||||
mu.Lock()
|
||||
received = r.Header.Clone()
|
||||
mu.Unlock()
|
||||
|
||||
w.WriteHeader(http.StatusNotModified)
|
||||
}))
|
||||
t.Cleanup(srv.Close)
|
||||
|
||||
f, _ := newServerFetcher(t, srv, nil)
|
||||
|
||||
res, err := f.Fetch(testContext(t), upstreamSource("/image"), etag, lastModified)
|
||||
if err != nil {
|
||||
t.Fatalf("conditional Fetch() error = %v", err)
|
||||
}
|
||||
|
||||
body, err := io.ReadAll(res.Content)
|
||||
_ = res.Content.Close()
|
||||
|
||||
mu.Lock()
|
||||
t.Logf("status = %d, body = %q, request headers = %v", res.StatusCode, body, received)
|
||||
gotIfNoneMatch := received.Get("If-None-Match")
|
||||
gotIfModifiedSince := received.Get("If-Modified-Since")
|
||||
mu.Unlock()
|
||||
|
||||
if err != nil || len(body) != 0 {
|
||||
t.Errorf("304 content = %q, error = %v; want empty", body, err)
|
||||
}
|
||||
|
||||
if res.StatusCode != http.StatusNotModified {
|
||||
t.Errorf("StatusCode = %d, want 304", res.StatusCode)
|
||||
}
|
||||
|
||||
if gotIfNoneMatch != etag || gotIfModifiedSince != lastModified {
|
||||
t.Errorf("If-None-Match = %q, If-Modified-Since = %q; want %q and %q",
|
||||
gotIfNoneMatch, gotIfModifiedSince, etag, lastModified)
|
||||
}
|
||||
|
||||
if held := len(f.allHostsSemaphore); held != 0 {
|
||||
t.Errorf("connection still held after the 304: %d held", held)
|
||||
}
|
||||
|
||||
_, err = f.Fetch(testContext(t), upstreamSource("/image"), "", "")
|
||||
if !errors.Is(err, ErrUpstreamError) {
|
||||
t.Errorf("unconditional Fetch() answered 304: error = %v, want ErrUpstreamError", err)
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user