test: cover redirect SSRF and semaphore release in httpfetcher (closes #78)

Adds httptest-server tests for the risk-bearing paths that only had helper-level coverage: the CheckRedirect validator (a 302 to a link-local address is refused with ErrSSRFBlocked and never dialed, while a redirect chain and a redirect to a public target still succeed), per-host semaphore release on the error, full-read-close and partial-read-close paths (proven by saturating a one-slot host), the MaxResponseSize limit through Fetch, non-2xx and disallowed content-type rejection, and ssrfSafeDialer's block of a private target. The upstream host is a TEST-NET-1 literal (192.0.2.10) that isPrivateIP treats as public and that resolves with no DNS, with a recording dialer routing it to the local server so the real SSRF checks still run. Full DNS-rebinding simulation is out of scope; the dial-time private-target block that closes that window is tested directly. Model: opus-4-8
2026-09-21 17:28:55 +00:00
parent a96eba8083
commit 624639c1ec
1 changed files with 421 additions and 0 deletions
@@ -0,0 +1,421 @@
+package httpfetcher
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"io"
+	"net"
+	"net/http"
+	"net/http/httptest"
+	"slices"
+	"strings"
+	"sync"
+	"testing"
+	"time"
+)
+
+// testPublicHost is a TEST-NET-1 (RFC 5737) literal. isPrivateIP treats it as
+// public, so validateURL and the redirect check accept it with no DNS lookup,
+// while the recording dialer routes it to the local httptest server. The
+// address is reserved for documentation and is never routed on the network.
+const testPublicHost = "192.0.2.10"
+
+// imagePayload is the body served by the fake upstream's image route.
+const imagePayload = "fake-jpeg-bytes"
+
+// errUnexpectedDial reports a dial to any host other than testPublicHost, which
+// would mean SSRF protection let a forbidden target reach the transport.
+var errUnexpectedDial = errors.New("unexpected dial target")
+
+// upstreamURL builds a fetch URL on the fake public host for the given path.
+func upstreamURL(path string) string {
+	return "http://" + testPublicHost + path
+}
+
+// recordingDialer records every address the transport asks it to dial and
+// routes connections for testPublicHost to a real local server, so the SSRF
+// checks run against a public-looking host while bytes go to httptest.
+type recordingDialer struct {
+	target string
+
+	mu     sync.Mutex
+	dialed []string
+}
+
+func (d *recordingDialer) dialContext(
+	ctx context.Context,
+	network, addr string,
+) (net.Conn, error) {
+	d.mu.Lock()
+	d.dialed = append(d.dialed, addr)
+	d.mu.Unlock()
+
+	host, _, err := net.SplitHostPort(addr)
+	if err != nil {
+		return nil, err
+	}
+
+	if host != testPublicHost {
+		return nil, fmt.Errorf("%w: %s", errUnexpectedDial, addr)
+	}
+
+	var dialer net.Dialer
+
+	return dialer.DialContext(ctx, network, d.target)
+}
+
+// dialedAddrs returns a copy of the addresses the dialer was asked to reach.
+func (d *recordingDialer) dialedAddrs() []string {
+	d.mu.Lock()
+	defer d.mu.Unlock()
+
+	return slices.Clone(d.dialed)
+}
+
+// startUpstream launches a fake upstream with the routes the fetch tests
+// exercise and stops it when the test finishes.
+func startUpstream(t *testing.T) *httptest.Server {
+	t.Helper()
+
+	mux := http.NewServeMux()
+	mux.HandleFunc("/image", func(w http.ResponseWriter, _ *http.Request) {
+		w.Header().Set("Content-Type", contentTypeJPEG)
+		_, _ = io.WriteString(w, imagePayload)
+	})
+	mux.HandleFunc("/status/500", func(w http.ResponseWriter, _ *http.Request) {
+		w.WriteHeader(http.StatusInternalServerError)
+	})
+	mux.HandleFunc("/html", func(w http.ResponseWriter, _ *http.Request) {
+		w.Header().Set("Content-Type", "text/html; charset=utf-8")
+		_, _ = io.WriteString(w, "<html></html>")
+	})
+	mux.HandleFunc("/redirect/private", func(w http.ResponseWriter, r *http.Request) {
+		http.Redirect(w, r, "http://169.254.169.254/latest/meta-data/", http.StatusFound)
+	})
+	mux.HandleFunc("/redirect/public", func(w http.ResponseWriter, r *http.Request) {
+		http.Redirect(w, r, "/image", http.StatusFound)
+	})
+	mux.HandleFunc("/redirect/chain", func(w http.ResponseWriter, r *http.Request) {
+		http.Redirect(w, r, "/redirect/hop", http.StatusFound)
+	})
+	mux.HandleFunc("/redirect/hop", func(w http.ResponseWriter, r *http.Request) {
+		http.Redirect(w, r, "/image", http.StatusFound)
+	})
+
+	srv := httptest.NewServer(mux)
+	t.Cleanup(srv.Close)
+
+	return srv
+}
+
+// newServerFetcher builds a fetcher whose transport routes testPublicHost to
+// srv, leaving the real SSRF validation and redirect checks in place.
+func newServerFetcher(
+	t *testing.T,
+	srv *httptest.Server,
+	cfg *Config,
+) (*HTTPFetcher, *recordingDialer) {
+	t.Helper()
+
+	if cfg == nil {
+		cfg = DefaultConfig()
+	}
+
+	cfg.AllowHTTP = true
+
+	f := New(cfg)
+
+	transport, ok := f.client.Transport.(*http.Transport)
+	if !ok {
+		t.Fatalf("transport is %T, want *http.Transport", f.client.Transport)
+	}
+
+	dialer := &recordingDialer{target: srv.Listener.Addr().String()}
+	transport.DialContext = dialer.dialContext
+
+	return f, dialer
+}
+
+// testContext returns a context cancelled when the test ends, bounding any
+// fetch that would otherwise block on a leaked semaphore slot.
+func testContext(t *testing.T) context.Context {
+	t.Helper()
+
+	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+	t.Cleanup(cancel)
+
+	return ctx
+}
+
+// fetchImage fetches path from the fake upstream and fails on error.
+func fetchImage(t *testing.T, f *HTTPFetcher, path string) *FetchResult {
+	t.Helper()
+
+	res, err := f.Fetch(testContext(t), upstreamURL(path))
+	if err != nil {
+		t.Fatalf("Fetch(%s) error = %v", path, err)
+	}
+
+	return res
+}
+
+// fetchExpectError fetches path and fails unless Fetch returns an error.
+func fetchExpectError(t *testing.T, f *HTTPFetcher, path string) error {
+	t.Helper()
+
+	res, err := f.Fetch(testContext(t), upstreamURL(path))
+	if err == nil {
+		_ = res.Content.Close()
+
+		t.Fatalf("Fetch(%s) = nil error, want an error", path)
+	}
+
+	return err
+}
+
+// fetchBody fetches path and returns the fully read, closed response body.
+func fetchBody(t *testing.T, f *HTTPFetcher, path string) string {
+	t.Helper()
+
+	res := fetchImage(t, f, path)
+	defer func() { _ = res.Content.Close() }()
+
+	data, err := io.ReadAll(res.Content)
+	if err != nil {
+		t.Fatalf("read body: %v", err)
+	}
+
+	return string(data)
+}
+
+// semLen reports how many per-host semaphore slots are currently held.
+func semLen(f *HTTPFetcher, host string) int {
+	return len(f.getHostSemaphore(host))
+}
+
+func TestFetchRedirectToPrivateIPBlocked(t *testing.T) {
+	t.Parallel()
+
+	srv := startUpstream(t)
+	f, dialer := newServerFetcher(t, srv, nil)
+
+	_, err := f.Fetch(testContext(t), upstreamURL("/redirect/private"))
+	if !errors.Is(err, ErrSSRFBlocked) {
+		t.Fatalf("Fetch() error = %v, want ErrSSRFBlocked", err)
+	}
+
+	for _, addr := range dialer.dialedAddrs() {
+		if strings.Contains(addr, "169.254.169.254") {
+			t.Errorf("dialer connected to the private redirect target: %s", addr)
+		}
+	}
+}
+
+func TestFetchRedirectToPublicSucceeds(t *testing.T) {
+	t.Parallel()
+
+	srv := startUpstream(t)
+	f, _ := newServerFetcher(t, srv, nil)
+
+	if body := fetchBody(t, f, "/redirect/public"); body != imagePayload {
+		t.Errorf("body = %q, want %q", body, imagePayload)
+	}
+}
+
+func TestFetchRedirectChainSucceeds(t *testing.T) {
+	t.Parallel()
+
+	srv := startUpstream(t)
+	f, _ := newServerFetcher(t, srv, nil)
+
+	if body := fetchBody(t, f, "/redirect/chain"); body != imagePayload {
+		t.Errorf("body = %q, want %q", body, imagePayload)
+	}
+}
+
+func TestFetchRejectsNon2xx(t *testing.T) {
+	t.Parallel()
+
+	srv := startUpstream(t)
+	f, _ := newServerFetcher(t, srv, nil)
+
+	err := fetchExpectError(t, f, "/status/500")
+	if !errors.Is(err, ErrUpstreamError) {
+		t.Fatalf("Fetch() error = %v, want ErrUpstreamError", err)
+	}
+}
+
+func TestFetchRejectsDisallowedContentType(t *testing.T) {
+	t.Parallel()
+
+	srv := startUpstream(t)
+	f, _ := newServerFetcher(t, srv, nil)
+
+	err := fetchExpectError(t, f, "/html")
+	if !errors.Is(err, ErrInvalidContentType) {
+		t.Fatalf("Fetch() error = %v, want ErrInvalidContentType", err)
+	}
+}
+
+func TestFetchMaxResponseSizeEnforced(t *testing.T) {
+	t.Parallel()
+
+	srv := startUpstream(t)
+
+	cfg := DefaultConfig()
+	cfg.MaxResponseSize = 8
+
+	f, _ := newServerFetcher(t, srv, cfg)
+
+	res := fetchImage(t, f, "/image")
+	defer func() { _ = res.Content.Close() }()
+
+	data, err := io.ReadAll(res.Content)
+	if !errors.Is(err, ErrResponseTooLarge) {
+		t.Fatalf("read error = %v, want ErrResponseTooLarge", err)
+	}
+
+	if int64(len(data)) > cfg.MaxResponseSize {
+		t.Errorf("read %d bytes, exceeds limit %d", len(data), cfg.MaxResponseSize)
+	}
+}
+
+func TestFetchSemaphoreReleasedOnError(t *testing.T) {
+	t.Parallel()
+
+	srv := startUpstream(t)
+
+	cfg := DefaultConfig()
+	cfg.MaxConnectionsPerHost = 1
+
+	f, _ := newServerFetcher(t, srv, cfg)
+
+	err := fetchExpectError(t, f, "/status/500")
+	if !errors.Is(err, ErrUpstreamError) {
+		t.Fatalf("Fetch() error = %v, want ErrUpstreamError", err)
+	}
+
+	if held := semLen(f, testPublicHost); held != 0 {
+		t.Fatalf("semaphore slot leaked after error: %d held", held)
+	}
+
+	// One slot per host: this fetch proceeds only if the slot was released.
+	res := fetchImage(t, f, "/image")
+	_ = res.Content.Close()
+}
+
+// assertSlotReleasedByClose fetches an image over a one-slot host, hands the
+// open result to consume, and asserts the slot is held before and freed after,
+// then that a follow-up fetch can still acquire it.
+func assertSlotReleasedByClose(
+	t *testing.T,
+	consume func(*testing.T, *FetchResult),
+) {
+	t.Helper()
+
+	srv := startUpstream(t)
+
+	cfg := DefaultConfig()
+	cfg.MaxConnectionsPerHost = 1
+
+	f, _ := newServerFetcher(t, srv, cfg)
+
+	res := fetchImage(t, f, "/image")
+	if held := semLen(f, testPublicHost); held != 1 {
+		t.Fatalf("slot not held while body is open: %d held", held)
+	}
+
+	consume(t, res)
+
+	if held := semLen(f, testPublicHost); held != 0 {
+		t.Fatalf("slot not released after close: %d held", held)
+	}
+
+	next := fetchImage(t, f, "/image")
+	_ = next.Content.Close()
+}
+
+func TestFetchSemaphoreReleasedOnBodyClose(t *testing.T) {
+	t.Parallel()
+
+	assertSlotReleasedByClose(t, func(t *testing.T, res *FetchResult) {
+		t.Helper()
+
+		_, err := io.ReadAll(res.Content)
+		if err != nil {
+			t.Fatalf("read body: %v", err)
+		}
+
+		err = res.Content.Close()
+		if err != nil {
+			t.Fatalf("close body: %v", err)
+		}
+	})
+}
+
+func TestFetchSemaphoreReleasedOnPartialReadClose(t *testing.T) {
+	t.Parallel()
+
+	assertSlotReleasedByClose(t, func(t *testing.T, res *FetchResult) {
+		t.Helper()
+
+		buf := make([]byte, 1)
+
+		_, err := res.Content.Read(buf)
+		if err != nil {
+			t.Fatalf("partial read: %v", err)
+		}
+
+		err = res.Content.Close()
+		if err != nil {
+			t.Fatalf("close body: %v", err)
+		}
+	})
+}
+
+// The dial-time re-resolution in ssrfSafeDialer is what closes the DNS
+// rebinding window: even if validateURL saw a public answer earlier, the
+// dialer independently re-checks the address it is about to connect to. A full
+// rebinding simulation (a resolver returning public, then private) would mean
+// replacing the global net.DefaultResolver with a fake DNS server, which is
+// heavyweight and unsafe to mutate under parallel -race tests. The property is
+// proven directly here instead: the dialer rejects a private target outright,
+// which is exactly the check that fires when a validated host later resolves
+// to a private address.
+func TestSSRFSafeDialerBlocksPrivateTarget(t *testing.T) {
+	t.Parallel()
+
+	for _, addr := range []string{
+		"169.254.169.254:80", // link-local (cloud metadata)
+		"127.0.0.1:80",       // loopback
+		"10.0.0.5:80",        // RFC 1918 private
+	} {
+		t.Run(addr, func(t *testing.T) {
+			t.Parallel()
+
+			_, err := ssrfSafeDialer(context.Background(), "tcp", addr)
+			if !errors.Is(err, ErrSSRFBlocked) {
+				t.Errorf("ssrfSafeDialer(%q) = %v, want ErrSSRFBlocked", addr, err)
+			}
+		})
+	}
+}
+
+func TestSSRFSafeDialerAllowsPublicTarget(t *testing.T) {
+	t.Parallel()
+
+	// A cancelled context makes the dial fail immediately without touching the
+	// network; the point is only that a public literal is not SSRF-blocked.
+	ctx, cancel := context.WithCancel(context.Background())
+	cancel()
+
+	_, err := ssrfSafeDialer(ctx, "tcp", testPublicHost+":80")
+	if err == nil {
+		t.Fatal("expected a dial error for an unreachable public target")
+	}
+
+	if errors.Is(err, ErrSSRFBlocked) {
+		t.Errorf("public target was SSRF-blocked: %v", err)
+	}
+}