clawbot
8feed69143
check / check (push) Canceled after 0s
A cached source and its variants were served until eviction, so a changed image at the same URL was never picked up. Once a source was last checked 24 hours ago, the next request for one of its images asks upstream with If-None-Match and If-Modified-Since, from the ETag and Last-Modified it stored. A 304 renews the source and its variant without a download or a conversion. New bytes replace the source, and each variant made before them is converted again. A failed check serves the cached image and puts the URL in the negative cache. Sources and variants not checked successfully for 30 days are evicted. A variant file's modification time is its source's check time, so a hit still makes no database call. Existing tests change only in call signatures. The test that asserted two-day-old entries are served without asking upstream is replaced by the new ones. Model: opus-5-5
77 lines
2.0 KiB
Go
77 lines
2.0 KiB
Go
package httpfetcher
|
|
|
|
import (
|
|
"errors"
|
|
"io"
|
|
"net/http"
|
|
"net/http/httptest"
|
|
"sync"
|
|
"testing"
|
|
)
|
|
|
|
// TestFetch_SendsConditionalHeadersAndHandlesNotModified checks that Fetch
|
|
// sends a non-empty ifNoneMatch and ifModifiedSince as request headers, that
|
|
// upstream's 304 to such a request is a result with status 304 and no
|
|
// content, and that a 304 to a request with neither is ErrUpstreamError.
|
|
func TestFetch_SendsConditionalHeadersAndHandlesNotModified(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
const (
|
|
etag = `"abc"`
|
|
lastModified = "Mon, 02 Nov 2026 10:00:00 GMT"
|
|
)
|
|
|
|
var (
|
|
mu sync.Mutex
|
|
received http.Header
|
|
)
|
|
|
|
srv := httptest.NewServer(http.HandlerFunc(
|
|
func(w http.ResponseWriter, r *http.Request) {
|
|
mu.Lock()
|
|
received = r.Header.Clone()
|
|
mu.Unlock()
|
|
|
|
w.WriteHeader(http.StatusNotModified)
|
|
}))
|
|
t.Cleanup(srv.Close)
|
|
|
|
f, _ := newServerFetcher(t, srv, nil)
|
|
|
|
res, err := f.Fetch(testContext(t), upstreamSource("/image"), etag, lastModified)
|
|
if err != nil {
|
|
t.Fatalf("conditional Fetch() error = %v", err)
|
|
}
|
|
|
|
body, err := io.ReadAll(res.Content)
|
|
_ = res.Content.Close()
|
|
|
|
mu.Lock()
|
|
t.Logf("status = %d, body = %q, request headers = %v", res.StatusCode, body, received)
|
|
gotIfNoneMatch := received.Get("If-None-Match")
|
|
gotIfModifiedSince := received.Get("If-Modified-Since")
|
|
mu.Unlock()
|
|
|
|
if err != nil || len(body) != 0 {
|
|
t.Errorf("304 content = %q, error = %v; want empty", body, err)
|
|
}
|
|
|
|
if res.StatusCode != http.StatusNotModified {
|
|
t.Errorf("StatusCode = %d, want 304", res.StatusCode)
|
|
}
|
|
|
|
if gotIfNoneMatch != etag || gotIfModifiedSince != lastModified {
|
|
t.Errorf("If-None-Match = %q, If-Modified-Since = %q; want %q and %q",
|
|
gotIfNoneMatch, gotIfModifiedSince, etag, lastModified)
|
|
}
|
|
|
|
if held := len(f.allHostsSemaphore); held != 0 {
|
|
t.Errorf("connection still held after the 304: %d held", held)
|
|
}
|
|
|
|
_, err = f.Fetch(testContext(t), upstreamSource("/image"), "", "")
|
|
if !errors.Is(err, ErrUpstreamError) {
|
|
t.Errorf("unconditional Fetch() answered 304: error = %v, want ErrUpstreamError", err)
|
|
}
|
|
}
|