clawbot
bfe3861314
check / check (push) Waiting to run
Cached images were served until eviction, so upstream changes were missed. After 24 hours a request checks the source with upstream, sending its stored ETag and Last-Modified. A 304 renews it with no download or conversion. New bytes replace it; older variants are converted again. A failed check serves the cached image and puts the URL in the negative cache. Images not checked for 30 days are evicted. Variant rows keep the source's ETag and Last-Modified for checking a variant of an evicted source. A variant file's modification time is its source's check time when it was made or renewed. Hits read it without a database call. The ETag includes it, so new bytes get a new ETag. Model: opus-5-5
77 lines
2.2 KiB
Go
77 lines
2.2 KiB
Go
package httpfetcher
|
|
|
|
import (
|
|
"errors"
|
|
"net"
|
|
"testing"
|
|
"time"
|
|
)
|
|
|
|
// TestNewUsesCheckedDialerWithoutDialContext checks that a fetcher built
|
|
// without DialContext, as pixa builds it, refuses to connect to a local
|
|
// server.
|
|
func TestNewUsesCheckedDialerWithoutDialContext(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
srv := startUpstream(t)
|
|
transport := transportOf(t, New(&Config{
|
|
Timeout: 30 * time.Second,
|
|
MaxResponseSize: 50 << 20,
|
|
MaxConnectionsPerHost: 20,
|
|
MaxConnections: 64,
|
|
}))
|
|
|
|
addr := srv.Listener.Addr().String()
|
|
|
|
_, err := transport.DialContext(testContext(t), "tcp", addr)
|
|
if !errors.Is(err, ErrSSRFBlocked) {
|
|
t.Fatalf("DialContext(%s) error = %v, want ErrSSRFBlocked", addr, err)
|
|
}
|
|
}
|
|
|
|
// TestDialContextReplacesOnlyTheDialer checks that a fetcher built with
|
|
// DialContext connects through it, while the URL check still refuses a
|
|
// loopback URL and the redirect check a redirect to a link-local address.
|
|
func TestDialContextReplacesOnlyTheDialer(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
srv := startUpstream(t)
|
|
dialer := &recordingDialer{target: srv.Listener.Addr().String()}
|
|
|
|
f := New(&Config{
|
|
Timeout: 30 * time.Second,
|
|
MaxResponseSize: 50 << 20,
|
|
AllowHTTP: true,
|
|
MaxConnectionsPerHost: 20,
|
|
MaxConnections: 64,
|
|
DialContext: dialer.dialContext,
|
|
})
|
|
|
|
if body := fetchBody(t, f, "/image"); body != imagePayload {
|
|
t.Errorf("body = %q, want %q", body, imagePayload)
|
|
}
|
|
|
|
_, err := f.Fetch(testContext(t), imageSourceOn("127.0.0.1"), "", "")
|
|
if !errors.Is(err, ErrSSRFBlocked) {
|
|
t.Errorf("Fetch(loopback URL) error = %v, want ErrSSRFBlocked", err)
|
|
}
|
|
|
|
_, err = f.Fetch(testContext(t), upstreamSource("/redirect/private"), "", "")
|
|
if !errors.Is(err, ErrSSRFBlocked) {
|
|
t.Errorf("Fetch(/redirect/private) error = %v, want ErrSSRFBlocked", err)
|
|
}
|
|
|
|
// The upstream server is reached through DialContext, and nothing else
|
|
// is asked of it.
|
|
dialed := dialer.dialedAddrs()
|
|
if len(dialed) == 0 {
|
|
t.Error("DialContext was never called")
|
|
}
|
|
|
|
for _, addr := range dialed {
|
|
if addr != net.JoinHostPort(testPublicHost, "80") {
|
|
t.Errorf("DialContext was asked to connect to %s", addr)
|
|
}
|
|
}
|
|
}
|