pixa/internal/server/source_url_internal_test.go

package server

import (
	"net/http"
	"net/http/httptest"
	"net/netip"
	"net/url"
	"regexp"
	"slices"
	"strconv"
	"strings"
	"sync"
	"testing"
	"time"

	"sneak.berlin/go/pixa/internal/encurl"
	"sneak.berlin/go/pixa/internal/imgcache"
	"sneak.berlin/go/pixa/internal/session"
	"sneak.berlin/go/pixa/internal/signature"
)

// percentSignPath is a source path holding a "%", which a URL writes %25.
const percentSignPath = "/100%25.png"

// generateFormPattern extracts the CSRF token rendered into the generator
// page's form.
var generateFormPattern = regexp.MustCompile(
	`(?s)action="/generate".*?name="gorilla\.csrf\.Token" value="([^"]+)"`)

// generatedURLPattern extracts the path of the encrypted URL the generator
// page shows.
var generatedURLPattern = regexp.MustCompile(`value="[^"]*(/v1/e/[^"]+)"`)

// recordingUpstream is a test upstream that answers every request with a PNG
// and records the request target, the path and query as sent, of each request
// it receives.
type recordingUpstream struct {
	*httptest.Server

	mu       sync.Mutex
	received []string
}

// startRecordingUpstream starts a recordingUpstream, which is closed when the
// test ends.
func startRecordingUpstream(t *testing.T) *recordingUpstream {
	t.Helper()

	source := encodeTestPNG(t, 64, 48)
	upstream := &recordingUpstream{}

	upstream.Server = httptest.NewServer(http.HandlerFunc(
		func(w http.ResponseWriter, r *http.Request) {
			upstream.mu.Lock()
			upstream.received = append(upstream.received, r.RequestURI)
			upstream.mu.Unlock()

			w.Header().Set("Content-Type", "image/png")
			_, _ = w.Write(source)
		}))
	t.Cleanup(upstream.Close)

	return upstream
}

// requestTargets returns the request targets the upstream has received, in
// the order it received them.
func (u *recordingUpstream) requestTargets() []string {
	u.mu.Lock()
	defer u.mu.Unlock()

	return slices.Clone(u.received)
}

// pageOrigin is the origin of the login and generator pages below, opened
// over HTTPS; httptest gives every request the host example.com.
const pageOrigin = "https://example.com"

// openRoot sends GET / to s over HTTPS, with cookie when it is not nil, and
// returns the page: the login page, or the generator page for a login session.
func openRoot(
	t *testing.T, s *Server, cookie *http.Cookie,
) *httptest.ResponseRecorder {
	t.Helper()

	req := httptest.NewRequestWithContext(t.Context(), http.MethodGet,
		pageOrigin+"/", nil)
	if cookie != nil {
		req.AddCookie(cookie)
	}

	page := httptest.NewRecorder()
	s.ServeHTTP(page, req)

	return page
}

// submitForm posts form to path on s as a browser submits it from page over
// HTTPS: with the CSRF token tokenPattern finds in page, the cookies page set,
// cookie when it is not nil, and a Referer naming the page, which pixa needs
// while debug is off, as it is in startImageProxy's config.
func submitForm(
	t *testing.T, s *Server, page *httptest.ResponseRecorder,
	tokenPattern *regexp.Regexp, path string, form url.Values,
	cookie *http.Cookie,
) *httptest.ResponseRecorder {
	t.Helper()

	match := tokenPattern.FindStringSubmatch(page.Body.String())
	if match == nil {
		t.Fatalf("the page has no form for %s with a CSRF token", path)
	}

	form.Set("gorilla.csrf.Token", match[1])

	req := httptest.NewRequestWithContext(t.Context(), http.MethodPost,
		pageOrigin+path, strings.NewReader(form.Encode()))
	req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
	req.Header.Set("Referer", pageOrigin+"/")

	for _, c := range page.Result().Cookies() {
		req.AddCookie(c)
	}

	if cookie != nil {
		req.AddCookie(cookie)
	}

	rec := httptest.NewRecorder()
	s.ServeHTTP(rec, req)

	return rec
}

// postGenerate logs in to s with the signing key, submits the generator page's
// form with sourceURL, and returns the response.
func postGenerate(
	t *testing.T, s *Server, sourceURL string,
) *httptest.ResponseRecorder {
	t.Helper()

	login := submitForm(t, s, openRoot(t, s, nil), csrfFieldPattern, "/",
		url.Values{"key": {testSigningKey}}, nil)

	var sessionCookie *http.Cookie

	for _, c := range login.Result().Cookies() {
		if c.Name == session.CookieName {
			sessionCookie = c
		}
	}

	if sessionCookie == nil {
		t.Fatalf("logging in answered %d and set no session cookie", login.Code)
	}

	rec := submitForm(t, s, openRoot(t, s, sessionCookie), generateFormPattern,
		"/generate", url.Values{"url": {sourceURL}}, sessionCookie)
	t.Logf("POST /generate with url %s: %d", sourceURL, rec.Code)

	return rec
}

// TestImageURLFetchesItsSourcePathAndQuery requests image URLs whose source
// query holds a space, written percent-encoded, or a "'", and whose source
// path holds a "%", written percent-encoded, and checks the one request the
// upstream receives: the source path percent-encoded again, and the source
// query with the space written %20 and the "'" written %27, as a browser
// writes them in a query.
func TestImageURLFetchesItsSourcePathAndQuery(t *testing.T) {
	t.Parallel()

	tests := []struct {
		name string
		// pathAndQuery is the source path and query as the image URL writes
		// them
		pathAndQuery string
		// wantRequest is the path and query the upstream must receive
		wantRequest string
	}{
		{"space in the query", "/photo.png%3Fname=my%20cat",
			"/photo.png?name=my%20cat"},
		{"apostrophe in the query", "/photo.png%3Fname=it's",
			"/photo.png?name=it%27s"},
		{"percent sign in the path", percentSignPath, percentSignPath},
	}

	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
			t.Parallel()

			upstream := startRecordingUpstream(t)
			s, _, _ := startImageProxy(t, upstream.Server)

			getImage(t, s, "/v1/image/"+upstreamHost+tt.pathAndQuery+"/orig.png")

			got := upstream.requestTargets()
			t.Logf("upstream received %q", got)

			if !slices.Equal(got, []string{tt.wantRequest}) {
				t.Errorf("upstream received %q, want [%q]", got, tt.wantRequest)
			}
		})
	}
}

// TestSourcePathsHoldingSpaceAndPercent20AreFetchedApart requests twice each
// of the source paths /my cat.png and /my%20cat.png, which the image URL
// writes my%20cat.png and my%2520cat.png. Each is fetched once, as a request
// of its own, and its second request is served from the cache.
func TestSourcePathsHoldingSpaceAndPercent20AreFetchedApart(t *testing.T) {
	t.Parallel()

	upstream := startRecordingUpstream(t)
	s, _, _ := startImageProxy(t, upstream.Server)

	imageURLs := []string{
		"/v1/image/" + upstreamHost + "/my%20cat.png/orig.png",
		"/v1/image/" + upstreamHost + "/my%2520cat.png/orig.png",
	}

	for _, imageURL := range imageURLs {
		getImage(t, s, imageURL)
	}

	for _, imageURL := range imageURLs {
		cacheStatus := getImage(t, s, imageURL).Header().Get("X-Pixa-Cache")
		if cacheStatus != string(imgcache.CacheHit) {
			t.Errorf("second GET %s: X-Pixa-Cache = %q, want HIT",
				imageURL, cacheStatus)
		}
	}

	got := upstream.requestTargets()
	t.Logf("upstream received %q", got)

	want := []string{"/my%20cat.png", "/my%2520cat.png"}
	if !slices.Equal(got, want) {
		t.Errorf("upstream received %q, want %q", got, want)
	}
}

// TestSourceQueryWrittenTwoWaysIsOneUpstreamRequest requests the source query
// name=my%20cat written as the README says, name=my%2520cat, and with the
// space written %20, name=my%20cat. Both are one source query: the upstream
// receives one request, and the second image URL is served from the cache.
func TestSourceQueryWrittenTwoWaysIsOneUpstreamRequest(t *testing.T) {
	t.Parallel()

	upstream := startRecordingUpstream(t)
	s, _, _ := startImageProxy(t, upstream.Server)

	getImage(t, s, "/v1/image/"+upstreamHost+"/photo.png%3Fname=my%2520cat/orig.png")

	second := getImage(t, s,
		"/v1/image/"+upstreamHost+"/photo.png%3Fname=my%20cat/orig.png")
	if got := second.Header().Get("X-Pixa-Cache"); got != string(imgcache.CacheHit) {
		t.Errorf("second X-Pixa-Cache = %q, want HIT", got)
	}

	got := upstream.requestTargets()
	t.Logf("upstream received %q", got)

	want := []string{"/photo.png?name=my%20cat"}
	if !slices.Equal(got, want) {
		t.Errorf("upstream received %q, want %q", got, want)
	}
}

// TestGeneratedURLFetchesTheSourceURLAsEntered makes an encrypted URL on the
// generator page for source URLs holding %20, %25 and %23 in the path or the
// query, requests it, and checks that the upstream receives the path and
// query exactly as entered.
func TestGeneratedURLFetchesTheSourceURLAsEntered(t *testing.T) {
	t.Parallel()

	// The names hold no "%": each test's state directory is named after it
	// and goes into the database URL, which decodes percent-escapes
	tests := []struct {
		name         string
		pathAndQuery string
	}{
		{"space in the path", "/my%20cat.png"},
		{"percent sign in the path", percentSignPath},
		{"number sign in the path", "/a%23b.png"},
		{"all three in the query", "/photo.png?name=my%20cat%25%23"},
	}

	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
			t.Parallel()

			upstream := startRecordingUpstream(t)
			s, _, _ := startImageProxy(t, upstream.Server)

			rec := postGenerate(t, s, "https://"+upstreamHost+tt.pathAndQuery)

			match := generatedURLPattern.FindStringSubmatch(rec.Body.String())
			if rec.Code != http.StatusOK || match == nil {
				t.Fatalf("POST /generate: %d with no encrypted URL; body %s",
					rec.Code, rec.Body.String())
			}

			getImage(t, s, match[1])

			got := upstream.requestTargets()
			t.Logf("upstream received %q", got)

			if !slices.Equal(got, []string{tt.pathAndQuery}) {
				t.Errorf("upstream received %q, want [%q]", got, tt.pathAndQuery)
			}
		})
	}
}

// TestGeneratorMakesOneImageURLForTwoSpellingsOfASourceURL enters two
// spellings of one source URL on the generator page, requests the encrypted
// URL made for each, and checks that the upstream receives one request and
// that the second encrypted URL is served from the cache. One pair writes the
// host in upper and in lower case, which /v1/image/ lower-cases. The host is
// an IPv6 address, whose hex digits have letter case, because the fetcher
// would look a host name up in DNS. It has a port, as the fetcher refuses an
// IPv6 address written without one. The other pair is a source URL with no
// path and with the path "/", both of which fetch "/".
func TestGeneratorMakesOneImageURLForTwoSpellingsOfASourceURL(t *testing.T) {
	t.Parallel()

	tests := []struct {
		name string
		// sourceURLs are the two spellings, after "https://"
		sourceURLs []string
		// wantRequest is the path the upstream must receive
		wantRequest string
	}{
		{"host in upper and in lower case",
			[]string{"[2001:DB8::A]:8443/a.png", "[2001:db8::a]:8443/a.png"},
			"/a.png"},
		{"no path and the path written as a slash",
			[]string{upstreamHost, upstreamHost + "/"}, "/"},
	}

	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
			t.Parallel()

			upstream := startRecordingUpstream(t)
			s, _, _ := startImageProxy(t, upstream.Server)

			var imageURLs []string

			for _, sourceURL := range tt.sourceURLs {
				rec := postGenerate(t, s, "https://"+sourceURL)

				match := generatedURLPattern.FindStringSubmatch(rec.Body.String())
				if rec.Code != http.StatusOK || match == nil {
					t.Fatalf("POST /generate: %d with no encrypted URL; body %s",
						rec.Code, rec.Body.String())
				}

				imageURLs = append(imageURLs, match[1])
			}

			getImage(t, s, imageURLs[0])

			second := getImage(t, s, imageURLs[1])
			if got := second.Header().Get("X-Pixa-Cache"); got != string(imgcache.CacheHit) {
				t.Errorf("second X-Pixa-Cache = %q, want HIT", got)
			}

			got := upstream.requestTargets()
			t.Logf("upstream received %q", got)

			if !slices.Equal(got, []string{tt.wantRequest}) {
				t.Errorf("upstream received %q, want [%q]", got, tt.wantRequest)
			}
		})
	}
}

// TestGeneratorRefusesSourceURLItCannotFetchAsWritten submits source URLs the
// generator page must make no encrypted URL for, and checks that it answers
// 400 and shows why. pixa keeps the source path decoded and fetches it
// percent-encoded again, so a path holding %2F would be fetched with a "/"
// there, whatever else the path holds. pixa sends no user name or password,
// and no "?" with no query after it. For these the page shows the URL pixa
// would fetch. A source path holding a ".." segment is refused, as
// /v1/image/ refuses it.
func TestGeneratorRefusesSourceURLItCannotFetchAsWritten(t *testing.T) {
	t.Parallel()

	// The test config sets allow_http, so pixa would fetch over plain HTTP
	const fetchedFrom = "http://" + upstreamHost

	tests := []struct {
		name string
		// sourceURL is the source URL after "https://"
		sourceURL string
		// wantShown is what the page must show: the URL pixa would fetch, or
		// why it makes no encrypted URL
		wantShown string
	}{
		{"encoded slash", upstreamHost + "/a%2Fb.png",
			fetchedFrom + "/a/b.png"},
		{"encoded slash and a space", upstreamHost + "/a%2Fb c.png",
			fetchedFrom + "/a/b%20c.png"},
		{"encoded slash and a bar", upstreamHost + "/a%2Fb|c.png",
			fetchedFrom + "/a/b%7Cc.png"},
		{"encoded slash and a letter outside ASCII", upstreamHost + "/café%2Fx.png",
			fetchedFrom + "/caf%C3%A9/x.png"},
		{"user name and password", "user:pass@" + upstreamHost + "/a.png",
			fetchedFrom + "/a.png"},
		{"question mark with no query", upstreamHost + "/a.png?",
			fetchedFrom + "/a.png"},
		{"two-dot segment", upstreamHost + "/a/../b.png",
			"cannot hold a .. segment"},
	}

	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
			t.Parallel()

			s, _, _ := startImageProxy(t, startRecordingUpstream(t).Server)

			rec := postGenerate(t, s, "https://"+tt.sourceURL)
			body := rec.Body.String()

			if rec.Code != http.StatusBadRequest {
				t.Errorf("status = %d, want %d", rec.Code, http.StatusBadRequest)
			}

			if strings.Contains(body, "/v1/e/") {
				t.Error("the page shows an encrypted URL")
			}

			if !strings.Contains(body, tt.wantShown) {
				t.Errorf("the page does not show %s; body %s", tt.wantShown, body)
			}
		})
	}
}

// ipv6SourcePath is the source path of the image URLs signedImageURL and
// encryptedImageURL make.
const ipv6SourcePath = "/ipv6.png"

// TestImageRoutesFetchIPv6SourceHostWithOrWithoutPort requests ipv6SourcePath
// on an IPv6 source host, written with and without a port, through a
// /v1/image/ URL with a signature and through an encrypted URL. The public
// address 2001:db8::a, a documentation address (RFC 3849) the fetcher accepts
// as public, answers 200, and the upstream receives the request for
// ipv6SourcePath. The private address fd00::1 answers 403, and the upstream
// receives no request: the test's fetcher connects every address to the test
// upstream, so only the fetcher's check of the source URL can refuse it.
func TestImageRoutesFetchIPv6SourceHostWithOrWithoutPort(t *testing.T) {
	t.Parallel()

	tests := []struct {
		name       string
		sourceHost string
		wantStatus int
		// wantRequests are the request targets the upstream must receive
		wantRequests []string
	}{
		{"public address without a port", "[2001:db8::a]",
			http.StatusOK, []string{ipv6SourcePath}},
		{"public address with a port", "[2001:db8::a]:8443",
			http.StatusOK, []string{ipv6SourcePath}},
		{"private address without a port", "[fd00::1]",
			http.StatusForbidden, nil},
		{"private address with a port", "[fd00::1]:8443",
			http.StatusForbidden, nil},
	}

	for _, tt := range tests {
		imageURLs := map[string]string{
			"image URL with a signature": signedImageURL(t, tt.sourceHost),
			"encrypted URL":              encryptedImageURL(t, tt.sourceHost),
		}

		for route, imageURL := range imageURLs {
			t.Run(tt.name+" "+route, func(t *testing.T) {
				t.Parallel()

				upstream := startRecordingUpstream(t)
				s, _, _ := startImageProxy(t, upstream.Server)

				rec := httptest.NewRecorder()
				s.ServeHTTP(rec, httptest.NewRequestWithContext(
					t.Context(), http.MethodGet, imageURL, nil))
				t.Logf("GET %s: %d", imageURL, rec.Code)

				if rec.Code != tt.wantStatus {
					t.Errorf("status = %d, want %d; body %s",
						rec.Code, tt.wantStatus, rec.Body.String())
				}

				got := upstream.requestTargets()
				t.Logf("upstream received %q", got)

				if !slices.Equal(got, tt.wantRequests) {
					t.Errorf("upstream received %q, want %q", got, tt.wantRequests)
				}
			})
		}
	}
}

// signedImageURL returns the /v1/image/ URL of ipv6SourcePath on sourceHost at
// its original size as PNG, with a signature made with the test signing key,
// which a host not on the allowlist needs. The URL has no q or fit, so it is
// signed with the quality 85 and the fit cover.
func signedImageURL(t *testing.T, sourceHost string) string {
	t.Helper()

	path, sig, exp := signature.New(testSigningKey).GenerateSignedURL(
		&signature.Request{
			SourceHost: sourceHost,
			SourcePath: ipv6SourcePath,
			Format:     string(imgcache.FormatPNG),
			Quality:    85,
			FitMode:    string(imgcache.FitCover),
		}, time.Hour)

	return path + "?sig=" + sig + "&exp=" + strconv.FormatInt(exp, 10)
}

// encryptedImageURL returns the encrypted URL of ipv6SourcePath on sourceHost
// at its original size as PNG, made with the test signing key.
func encryptedImageURL(t *testing.T, sourceHost string) string {
	t.Helper()

	generator, err := encurl.NewGenerator(testSigningKey)
	if err != nil {
		t.Fatalf("encurl.NewGenerator: %v", err)
	}

	token, err := generator.Generate(&encurl.Payload{
		SourceHost: sourceHost, SourcePath: ipv6SourcePath,
		Format: imgcache.FormatPNG,
	})
	if err != nil {
		t.Fatalf("encurl Generate: %v", err)
	}

	return "/v1/e/" + token + "/img.png"
}

// TestImageRoutesRefuseIPv6SourceHostHoldingInternalIPv4Address requests
// ipv6SourcePath on IPv6 source hosts that hold a loopback or private IPv4
// address in their last four bytes, in the IPv4-compatible form (::/96) or the
// NAT64 form (64:ff9b::/96), through a /v1/image/ URL with a signature and
// through an encrypted URL. Each answers 403, and the upstream receives no
// request: the test's fetcher connects every address to the test upstream, so
// only the fetcher's check of the source URL can refuse it.
func TestImageRoutesRefuseIPv6SourceHostHoldingInternalIPv4Address(t *testing.T) {
	t.Parallel()

	upstream := startRecordingUpstream(t)
	s, _, _ := startImageProxy(t, upstream.Server)

	sourceHosts := []string{
		"[::127.0.0.1]",     // IPv4-compatible form of a loopback address
		"[::10.0.0.1]",      // IPv4-compatible form of a private address
		"[64:ff9b::7f00:1]", // NAT64 form of 127.0.0.1
	}

	for _, sourceHost := range sourceHosts {
		imageURLs := []string{
			signedImageURL(t, sourceHost),
			encryptedImageURL(t, sourceHost),
		}

		for _, imageURL := range imageURLs {
			rec := httptest.NewRecorder()
			s.ServeHTTP(rec, httptest.NewRequestWithContext(
				t.Context(), http.MethodGet, imageURL, nil))
			t.Logf("GET %s (source host %s): %d", imageURL, sourceHost, rec.Code)

			if rec.Code != http.StatusForbidden {
				t.Errorf("source host %s: status = %d, want %d; body %s",
					sourceHost, rec.Code, http.StatusForbidden, rec.Body.String())
			}
		}
	}

	if got := upstream.requestTargets(); len(got) != 0 {
		t.Errorf("upstream received %q, want no request", got)
	}
}

// TestImageRoutesCheckIPv4AddressInIPv6SourceHost requests ipv6SourcePath on
// source hosts that hold an IPv4 address, through a /v1/image/ URL with a
// signature and through an encrypted URL, with the fetcher pixad uses.
// [64:ff9b:1::a00:1], in the local-use NAT64 range (64:ff9b:1::/48, RFC 8215),
// answers 403. [::203.0.113.5] holds the public address 203.0.113.5 in the
// IPv4-compatible form (::/96) and answers 403 when blocked_networks lists
// 203.0.113.0/24 or ::/96. [::ffff:203.0.113.5] holds it in the IPv4-mapped
// form (::ffff:0:0/96) and answers 403 when blocked_networks lists
// 203.0.113.0/24. Go keeps the plain address 203.0.113.5 in the mapped form
// too, and an IPv6 range must not refuse it: with ::/0 listed it is not
// refused. Neither is [::203.0.113.5] with blocked_networks empty. Where a
// host is not refused, the fetcher tries to connect to it, finds nothing
// there, and the request fails with another status.
func TestImageRoutesCheckIPv4AddressInIPv6SourceHost(t *testing.T) {
	t.Parallel()

	const (
		ipv4CompatibleHost = "[::203.0.113.5]"
		ipv4MappedHost     = "[::ffff:203.0.113.5]"
	)

	tests := []struct {
		name            string
		sourceHost      string
		blockedNetworks []netip.Prefix
		wantForbidden   bool
	}{
		{"local-use NAT64 form of 10.0.0.1", "[64:ff9b:1::a00:1]", nil, true},
		{"IPv4-compatible form of an address in blocked_networks",
			ipv4CompatibleHost,
			[]netip.Prefix{netip.MustParsePrefix("203.0.113.0/24")}, true},
		{"IPv4-compatible form with ::/96 in blocked_networks",
			ipv4CompatibleHost,
			[]netip.Prefix{netip.MustParsePrefix("::/96")}, true},
		{"IPv4-mapped form of an address in blocked_networks",
			ipv4MappedHost,
			[]netip.Prefix{netip.MustParsePrefix("203.0.113.0/24")}, true},
		{"plain IPv4 address with ::/0 in blocked_networks",
			"203.0.113.5",
			[]netip.Prefix{netip.MustParsePrefix("::/0")}, false},
		{"IPv4-compatible form of an address with blocked_networks empty",
			ipv4CompatibleHost, nil, false},
	}

	for _, tt := range tests {
		imageURLs := map[string]string{
			"image URL with a signature": signedImageURL(t, tt.sourceHost),
			"encrypted URL":              encryptedImageURL(t, tt.sourceHost),
		}

		for route, imageURL := range imageURLs {
			t.Run(tt.name+" "+route, func(t *testing.T) {
				t.Parallel()

				s := startImageProxyWithBlockedNetworks(t, tt.blockedNetworks)

				rec := httptest.NewRecorder()
				s.ServeHTTP(rec, httptest.NewRequestWithContext(
					t.Context(), http.MethodGet, imageURL, nil))
				t.Logf("GET %s: %d", imageURL, rec.Code)

				forbidden := rec.Code == http.StatusForbidden
				if tt.wantForbidden && !forbidden {
					t.Errorf("status = %d, want %d; body %s",
						rec.Code, http.StatusForbidden, rec.Body.String())
				}

				if !tt.wantForbidden && forbidden {
					t.Errorf("status = %d, want any other status; body %s",
						rec.Code, rec.Body.String())
				}
			})
		}
	}
}