Files
pixa/internal/httpfetcher/httpfetcher_internal_test.go
T
clawbot 880845f38c
check / check (push) Canceled after 0s
Check cached sources with upstream after the cache TTL (closes #69)
A cached image was served until eviction, so a change at the same URL
was never picked up.

After 24 hours a request checks the source with upstream, sending its
stored ETag and Last-Modified. A 304 renews it with no download or
conversion. New bytes replace it, and older variants are converted
again. A failed check serves the cached image and puts the URL in the
negative cache. Images not checked for 30 days are evicted.

Variant rows keep the source's ETag and Last-Modified, so a variant
whose source was evicted is checked the same way.

A variant file's modification time is its source's check time when it
was made or renewed, so a hit reads no database row.

Model: opus-5-5
2026-11-11 10:38:51 +00:00

353 lines
8.4 KiB
Go

package httpfetcher
import (
"context"
"errors"
"io"
"net"
"net/url"
"testing"
"testing/fstest"
"time"
"sneak.berlin/go/pixa/internal/imagerequest"
)
// testHost is the hostname used by mock fetch tests.
const testHost = "example.com"
func TestIsAllowedContentType(t *testing.T) {
t.Parallel()
f := New(&Config{
Timeout: 30 * time.Second,
MaxResponseSize: 50 << 20,
MaxConnectionsPerHost: 20,
MaxConnections: 64,
})
tests := []struct {
contentType string
want bool
}{
{imagerequest.FormatJPEG.MIMEType(), true},
{imagerequest.FormatPNG.MIMEType(), true},
{imagerequest.FormatWebP.MIMEType(), true},
{"image/jpeg; charset=utf-8", true},
{"IMAGE/JPEG", true},
{"text/html", false},
{contentTypeOctetStream, false},
{"", false},
}
for _, tc := range tests {
t.Run(tc.contentType, func(t *testing.T) {
t.Parallel()
got := f.isAllowedContentType(tc.contentType)
if got != tc.want {
t.Errorf("isAllowedContentType(%q) = %v, want %v", tc.contentType, got, tc.want)
}
})
}
}
func TestIsLocalhost(t *testing.T) {
t.Parallel()
tests := []struct {
host string
want bool
}{
{"localhost", true},
{"LOCALHOST", true},
{localhostIPv4, true},
{localhostIPv6, true},
{"[::1]", true},
{"foo.localhost", true},
{"foo.local", true},
{testHost, false},
{"127.0.0.2", false}, // Handled by isPrivateIP, not isLocalhost string match
}
for _, tc := range tests {
t.Run(tc.host, func(t *testing.T) {
t.Parallel()
got := isLocalhost(tc.host)
if got != tc.want {
t.Errorf("isLocalhost(%q) = %v, want %v", tc.host, got, tc.want)
}
})
}
}
func TestIsPrivateIP(t *testing.T) {
t.Parallel()
tests := []struct {
ip string
want bool
}{
{localhostIPv4, true}, // loopback
{"10.0.0.1", true}, // private
{"192.168.1.1", true}, // private
{"172.16.0.1", true}, // private
{"169.254.1.1", true}, // link-local
{"0.0.0.0", true}, // unspecified
{"224.0.0.1", true}, // multicast
{localhostIPv6, true}, // IPv6 loopback
{"fe80::1", true}, // IPv6 link-local
{"8.8.8.8", false}, // public
{"2001:4860:4860::8888", false}, // public IPv6
}
for _, tc := range tests {
t.Run(tc.ip, func(t *testing.T) {
t.Parallel()
ip := net.ParseIP(tc.ip)
if ip == nil {
t.Fatalf("failed to parse IP %q", tc.ip)
}
got := isPrivateIP(ip)
if got != tc.want {
t.Errorf("isPrivateIP(%q) = %v, want %v", tc.ip, got, tc.want)
}
})
}
if !isPrivateIP(nil) {
t.Error("isPrivateIP(nil) should return true")
}
}
// TestIsPrivateIPJudgesEmbeddedIPv4Address checks IPv6 addresses that hold an
// IPv4 address in their last four bytes: the IPv4-compatible form (::/96) and
// the NAT64 form (64:ff9b::/96). One holding a loopback, private or
// link-local IPv4 address is refused as that address is, and one holding a
// public IPv4 address in the IPv4-compatible form passes.
func TestIsPrivateIPJudgesEmbeddedIPv4Address(t *testing.T) {
t.Parallel()
tests := []struct {
ip string
want bool
}{
{"::127.0.0.1", true}, // loopback
{"::10.0.0.1", true}, // private
{"::169.254.169.254", true}, // link-local
{"64:ff9b::7f00:1", true}, // NAT64 form of 127.0.0.1
{"::8.8.8.8", false}, // public
}
for _, tc := range tests {
t.Run(tc.ip, func(t *testing.T) {
t.Parallel()
ip := net.ParseIP(tc.ip)
if ip == nil {
t.Fatalf("failed to parse IP %q", tc.ip)
}
got := isPrivateIP(ip)
t.Logf("isPrivateIP(%s) = %v", ip, got)
if got != tc.want {
t.Errorf("isPrivateIP(%q) = %v, want %v", tc.ip, got, tc.want)
}
})
}
}
func TestValidateURL_RejectsNonHTTPS(t *testing.T) {
t.Parallel()
err := validateURL(t.Context(), &url.URL{Scheme: "http", Host: testHost}, false)
if !errors.Is(err, ErrUnsupportedScheme) {
t.Errorf("validateURL http = %v, want ErrUnsupportedScheme", err)
}
}
func TestValidateURL_AllowsHTTPWhenConfigured(t *testing.T) {
t.Parallel()
// Use a host that won't resolve (explicit .invalid TLD) so we don't hit DNS.
err := validateURL(t.Context(),
&url.URL{Scheme: "http", Host: "nonexistent.invalid"}, true)
// We expect a host resolution error, not ErrUnsupportedScheme.
if errors.Is(err, ErrUnsupportedScheme) {
t.Error("validateURL with AllowHTTP should not return ErrUnsupportedScheme")
}
}
func TestValidateURL_RejectsLocalhost(t *testing.T) {
t.Parallel()
err := validateURL(t.Context(), &url.URL{Scheme: "https", Host: "localhost"}, false)
if !errors.Is(err, ErrSSRFBlocked) {
t.Errorf("validateURL localhost = %v, want ErrSSRFBlocked", err)
}
}
func TestValidateURL_EmptyHost(t *testing.T) {
t.Parallel()
err := validateURL(t.Context(), &url.URL{Scheme: "https"}, false)
if !errors.Is(err, ErrInvalidHost) {
t.Errorf("validateURL empty host = %v, want ErrInvalidHost", err)
}
}
func TestMockFetcher_FetchesFile(t *testing.T) {
t.Parallel()
mockFS := fstest.MapFS{
"example.com/images/photo.jpg": &fstest.MapFile{Data: []byte("fake-jpeg-data")},
}
m := NewMock(mockFS)
result, err := m.Fetch(context.Background(),
imagerequest.Source{Host: testHost, Path: "/images/photo.jpg"}, "", "")
if err != nil {
t.Fatalf("Fetch() error = %v", err)
}
defer func() { _ = result.Content.Close() }()
if result.ContentType != imagerequest.FormatJPEG.MIMEType() {
t.Errorf("ContentType = %q, want image/jpeg", result.ContentType)
}
data, err := io.ReadAll(result.Content)
if err != nil {
t.Fatalf("read content: %v", err)
}
if string(data) != "fake-jpeg-data" {
t.Errorf("Content = %q, want %q", string(data), "fake-jpeg-data")
}
if result.ContentLength != int64(len("fake-jpeg-data")) {
t.Errorf("ContentLength = %d, want %d", result.ContentLength, len("fake-jpeg-data"))
}
}
func TestMockFetcher_MissingFileReturnsUpstreamError(t *testing.T) {
t.Parallel()
mockFS := fstest.MapFS{}
m := NewMock(mockFS)
_, err := m.Fetch(context.Background(),
imagerequest.Source{Host: testHost, Path: "/missing.jpg"}, "", "")
if !errors.Is(err, ErrUpstreamError) {
t.Errorf("Fetch() error = %v, want ErrUpstreamError", err)
}
}
func TestMockFetcher_RespectsContextCancellation(t *testing.T) {
t.Parallel()
mockFS := fstest.MapFS{
"example.com/photo.jpg": &fstest.MapFile{Data: []byte("data")},
}
m := NewMock(mockFS)
ctx, cancel := context.WithCancel(context.Background())
cancel()
_, err := m.Fetch(ctx, imagerequest.Source{Host: testHost, Path: "/photo.jpg"}, "", "")
if !errors.Is(err, context.Canceled) {
t.Errorf("Fetch() error = %v, want context.Canceled", err)
}
}
func TestDetectContentTypeFromPath(t *testing.T) {
t.Parallel()
tests := []struct {
path string
want string
}{
{"foo/bar.jpg", imagerequest.FormatJPEG.MIMEType()},
{"foo/bar.JPG", imagerequest.FormatJPEG.MIMEType()},
{"foo/bar.jpeg", imagerequest.FormatJPEG.MIMEType()},
{"foo/bar.png", imagerequest.FormatPNG.MIMEType()},
{"foo/bar.gif", imagerequest.FormatGIF.MIMEType()},
{"foo/bar.webp", imagerequest.FormatWebP.MIMEType()},
{"foo/bar.avif", imagerequest.FormatAVIF.MIMEType()},
{"foo/bar.svg", contentTypeSVG},
{"foo/bar.bin", contentTypeOctetStream},
{"foo/bar", contentTypeOctetStream},
}
for _, tc := range tests {
t.Run(tc.path, func(t *testing.T) {
t.Parallel()
got := detectContentTypeFromPath(tc.path)
if got != tc.want {
t.Errorf("detectContentTypeFromPath(%q) = %q, want %q", tc.path, got, tc.want)
}
})
}
}
func TestLimitedReader_EnforcesLimit(t *testing.T) {
t.Parallel()
src := make([]byte, 100)
r := &limitedReader{
reader: &byteReader{data: src},
remaining: 50,
}
buf := make([]byte, 100)
n, err := r.Read(buf)
if err != nil {
t.Fatalf("first Read error = %v", err)
}
if n > 50 {
t.Errorf("read %d bytes, should be capped at 50", n)
}
// Drain until limit is exhausted.
total := n
for total < 50 {
nn, err := r.Read(buf)
if err != nil {
t.Fatalf("during drain: %v", err)
}
total += nn
}
// Now the limit is exhausted — next read should error.
_, err = r.Read(buf)
if !errors.Is(err, ErrResponseTooLarge) {
t.Errorf("exhausted Read error = %v, want ErrResponseTooLarge", err)
}
}
// byteReader is a minimal io.Reader over a byte slice for testing.
type byteReader struct {
data []byte
pos int
}
func (r *byteReader) Read(p []byte) (int, error) {
if r.pos >= len(r.data) {
return 0, io.EOF
}
n := copy(p, r.data[r.pos:])
r.pos += n
return n, nil
}