Test the image proxy flow end to end (closes #80)

TestImageProxyFlow in internal/server starts the database, handlers and
middleware from the constructors pixad uses, in an fx app with a fresh
state directory, and replaces only the upstream origin with a local test
server, reached through the real fetcher by its new DialContext. For a
resize with a change to JPEG and for orig, the first request answers 200
with the right content type, decoded size and X-Pixa-Cache MISS; the
second answers HIT with the same image while the upstream has had one
request; the source and the converted image are then on disk with their
rows in SQLite. TODO.md records it and drops the item from Future Steps.

Model: opus-5-5

README.md

@@ -45,7 +45,8 @@ everything in this list without further settings. The reverse proxy must:
   Routes);
 - pass the `Host`, `Origin` and `Referer` headers on unchanged, as pixa refuses
   a form from those pages unless `Origin` or `Referer` names the host in `Host`,
-  and builds encrypted URLs from `Host`;
+  builds encrypted URLs from `Host`, and checks `Referer` against
+  `referer_blocklist`;
 - set `X-Forwarded-For` to the client's address, with `trusted_proxies` set to
   the address pixa sees the proxy's requests come from, so the login limit
   counts each client by its own address (see `trusted_proxies` under
@@ -175,11 +176,13 @@ path under `/v1/` answers 200, in maintenance mode too.
   allowlisted (see Source Hosts). Answers: 200; 304 when `If-None-Match` matches
   the image's `ETag`; 400 for a URL or parameter that is not valid; 401 for a
   missing or wrong signature, a missing `exp` or an `exp` in the past; 403 when
-  the upstream host, or a host it redirects to, is `localhost`, ends in
+  the request's `Referer` names a host in `referer_blocklist`, checked before
-  `.localhost` or `.local`, or has an address in a blocked network (see
+  the signature, the cache and the upstream fetch; 403 when the upstream host,
-  `blocked_networks`); 502 when the upstream answered with an error status, and
+  or a host it redirects to, is `localhost`, ends in `.localhost` or `.local`,
-  for 5 minutes after that for the same source URL; 503 when pixa is busy or in
+  or has an address in a blocked network (see `blocked_networks`); 502 when the
-  maintenance mode; 500 for any other failure.
+  upstream answered with an error status, and for 5 minutes after that for the
+  same source URL; 503 when pixa is busy or in maintenance mode; 500 for any
+  other failure.
 - `GET` or `HEAD` `/v1/e/<token>/<name>` — an image through an encrypted URL
   (see Encrypted URLs). Needs: nothing but the URL. Answers: 200; 304 when
   `If-None-Match` matches the image's `ETag`; 400 for a token that does not
@@ -392,6 +395,11 @@ and the URL is
 - **Suffix match**: `.example.com` — matches `cdn.example.com`,
   `images.example.com`, and `example.com`
 
+An IP address is matched exactly; write an IPv6 address without brackets. An
+entry that is neither a host name (letters, digits, hyphens, underscores and
+dots, with at most one leading dot) nor an IP address, such as one with a port
+or a `*.` wildcard, aborts startup.
+
 ### Configuration
 
 Every setting can be given as an environment variable, in a YAML config
@@ -428,6 +436,7 @@ tell. With no file, pixa uses the environment and the defaults.
 | `PIXA_DB_URL`                        | `db_url`                        | SQLite database URL; default `state.sqlite3` in the state directory          |
 | `PIXA_CACHE_MAX_BYTES`               | `cache_max_bytes`               | Disk cache limit in bytes; `0` disables it; default 75% of (free + cached)   |
 | `PIXA_ALLOWLIST_HOSTS`               | `allowlist_hosts`               | Upstream hosts served without a signature                                    |
+| `PIXA_REFERER_BLOCKLIST`             | `referer_blocklist`             | Hosts whose pages the image routes refuse with 403, by `Referer`             |
 | `PIXA_BLOCKED_NETWORKS`              | `blocked_networks`              | CIDR ranges never fetched from, on top of the built-in ones                  |
 | `PIXA_TRUSTED_PROXIES`               | `trusted_proxies`               | CIDR ranges of proxies whose `X-Forwarded-For` is believed; default RFC 1918 |
 | `PIXA_ALLOW_HTTP`                    | `allow_http`                    | Allow plain-HTTP upstreams, for testing only; default `false`                |
@@ -456,6 +465,18 @@ Key settings in more detail:
   that has no leading zero and is not the scheme's default. Any other value,
   including another scheme such as a browser extension's, aborts startup
 - `allowlist_hosts` — list of allowed upstream hosts
+- `referer_blocklist` — list of hosts whose pages may not show pixa's images, to
+  stop other sites hotlinking them. Entries are written and matched as for
+  `allowlist_hosts` (see Allowlist patterns), and an entry that is neither a
+  host name nor an IP address aborts startup. A request to `/v1/image/` or
+  `/v1/e/` whose `Referer` header names a listed host is refused with 403 before
+  its signature or token is checked and before the cache or the upstream host is
+  used, so it fetches nothing, and it is refused even when the image is cached.
+  A request with no `Referer`, or one that does not parse as a URL
+  with a host, is served, as many clients send none. So this is easily got
+  around: a site whose pages send no `Referer` (for example with
+  `Referrer-Policy: no-referrer`) is not stopped. It does not apply to the login
+  and generator pages. Default: empty
 - `blocked_networks` — list of CIDR ranges to refuse for SSRF protection,
   added to the always-enforced built-in ranges (loopback, private,
   link-local, CGNAT, benchmark, NAT64, and the like); an invalid CIDR

TODO.md

@@ -27,10 +27,44 @@ The disk cache is now size-bounded with LRU eviction
 
 # Next Step
 
-P2: security: referer blacklist
+P2: security: per-IP rate limiting on the image routes
 
 # Completed Steps
 
+- 2026-10-04 an integration test of the image proxy flow (closes #80):
+  `TestImageProxyFlow` in `internal/server` starts the database, handlers and
+  middleware from the constructors `pixad` uses, with a fresh state directory,
+  and replaces only the upstream origin with a local test server. For a resize
+  with a change to JPEG and for `orig`, the first request goes through the
+  router, the real fetcher, libvips, the disk cache and SQLite and answers 200
+  with the right content type and size and `X-Pixa-Cache: MISS`; the second
+  answers `HIT` with the same image and the upstream has had one request; the
+  source and the converted image are then in `cache/sources` and
+  `cache/variants`, with their rows in `source_content`, `source_metadata` and
+  `variant_content`. Two optional fields make this possible, which `pixad` does
+  not set and the config file and environment cannot:
+  `httpfetcher.Config.DialContext` connects in place of the dialer that refuses
+  internal addresses, the URL and redirect checks still running, and
+  `handlers.Params.Fetcher` replaces the fetcher the handlers build.
+- 2026-10-04 a URL made on the generator page with a `ttl` is tested to
+  expire (closes #199): a new test in `internal/handlers` makes a URL on the
+  generator page with a `ttl` of one second, checks that `/v1/e/` serves it at
+  once, waits two seconds and checks that it then answers 410. The test waits
+  for real, as pixa reads the clock directly when it makes and checks a URL; it
+  waits two seconds because the time a URL expires is kept in whole seconds.
+  Test only.
+- 2026-10-04 referer blocklist (closes #90): `referer_blocklist`
+  (`PIXA_REFERER_BLOCKLIST`) lists hosts, written and matched as for
+  `allowlist_hosts` with the same matcher; an entry of either list that is
+  neither a host name (letters, digits, hyphens, underscores and dots, with at
+  most one leading dot) nor an IP address, such as one with a port or a `*.`
+  wildcard, aborts startup naming the setting and the entry. Both image routes
+  refuse a request whose `Referer` names a listed host with 403 and a JSON error
+  before the signature, the cache and the upstream fetch, so it fetches nothing
+  and is refused whether or not the image is cached. A request with no
+  `Referer`, or one that does not parse as a URL with a host, is served, so the
+  list is easily got around; `README.md` and `configs/config.example.yml` say
+  so. It does not apply to the login and generator pages.
 - 2026-10-04 fewer files in the repository root (closes #97):
   `config.example.yml` moved unchanged to `configs/config.example.yml`, and
   `README.md`, the comments in `internal/config/config.go` and the startup error
@@ -561,7 +595,6 @@ P2: security: referer blacklist
 # Future Steps
 
 - P2: security
-    - per-IP rate limiting on the image routes
     - per-origin rate limiting
 - P2: HTTP response handling
     - Last-Modified headers
@@ -573,5 +606,4 @@ P2: security: referer blacklist
     - optional Sentry error reporting
     - comprehensive request logging
     - Prometheus performance metrics
-    - integration tests for the image proxy flow
     - load tests to verify the 1k to 5k req/s target

configs/config.example.yml

@@ -46,6 +46,8 @@ signing_key: "CHANGE_ME_generate_with_openssl_rand_base64_32"
 
 # Hosts that don't require signatures (default: none)
 # Use "." prefix for wildcard subdomain matching (e.g., ".example.com" matches "cdn.example.com")
+# An entry that is neither a host name nor an IP address (IPv6 without
+# brackets), such as one with a port or a "*." wildcard, aborts startup.
 allowlist_hosts:
   - s3.sneak.cloud
   - static.sneak.cloud
@@ -53,6 +55,16 @@ allowlist_hosts:
   - github.com
   - user-images.githubusercontent.com
 
+# Hosts whose pages may not show pixa's images, written as for
+# allowlist_hosts. A request to /v1/image/ or /v1/e/ whose Referer header
+# names one of them is answered 403 before anything is fetched, even when
+# the image is cached. A request with no Referer, or one that does not
+# parse, is served, so a site whose pages send no Referer is not stopped.
+# The login and generator pages are not covered. (default: none)
+# referer_blocklist:
+#   - leech.example
+#   - .hotlinker.example
+
 # Additional CIDR ranges to refuse when fetching upstream, extending the
 # SSRF protection. These are added to the always-enforced built-in ranges
 # (loopback, RFC 1918 private, link-local, CGNAT, benchmark, NAT64, and

internal/config/config.go

@@ -11,6 +11,7 @@ import (
 	"net/url"
 	"os"
 	"path/filepath"
+	"regexp"
 	"runtime"
 	"sort"
 	"strconv"
@@ -48,6 +49,7 @@ const (
 	keyMetricsPassword            = "metrics.password"
 	keySigningKey                 = "signing_key"
 	keyAllowlistHosts             = "allowlist_hosts"
+	keyRefererBlocklist           = "referer_blocklist"
 	keyAllowHTTP                  = "allow_http"
 	keyUpstreamConnectionsPerHost = "upstream_connections_per_host"
 	keyUpstreamConnections        = "upstream_connections"
@@ -97,12 +99,11 @@ var (
 		"value is null; omit the key entirely to use the default")
 	errValuesNull = errors.New(
 		"value is null; omit a key entirely to use its default")
-	errNotBareHostname = errors.New(
+	errNotAHost = errors.New("must be a host name such as " +
-		"must be a bare hostname without scheme, path, or whitespace")
+		"cdn.example.com or .example.com, or an IP address")
-	errNoHostnameLabels = errors.New("contains no hostname labels")
+	errNotADuration   = errors.New("not a duration such as 30s or 2m")
-	errNotADuration     = errors.New("not a duration such as 30s or 2m")
+	errMustBePositive = errors.New("must be positive")
-	errMustBePositive   = errors.New("must be positive")
+	errNotAnOrigin    = errors.New(
-	errNotAnOrigin      = errors.New(
 		`not "*" or an origin such as https://example.com`)
 )
 
@@ -130,6 +131,10 @@ type Config struct {
 	AllowHTTP                  bool     // Allow non-TLS upstream (testing only)
 	UpstreamConnectionsPerHost int      // Max concurrent connections per upstream host
 
+	// RefererBlocklist holds host patterns, matched as AllowlistHosts is: the
+	// image routes refuse a request whose Referer names a matching host.
+	RefererBlocklist []string
+
 	// UpstreamConnections is the most concurrent connections to all
 	// upstream hosts together, on top of the per-host limit.
 	// MaxConcurrentProcessing is the most images processed at once.
@@ -270,7 +275,6 @@ func newFromSmartConfig(sc *smartconfig.Config) (*Config, error) {
 	}
 
 	loader := &strictLoader{sc: sc}
-
 	c := &Config{
 		Debug:           loader.boolVal(keyDebug, false),
 		MaintenanceMode: loader.boolVal(keyMaintenanceMode, false),
@@ -298,9 +302,10 @@ func newFromSmartConfig(sc *smartconfig.Config) (*Config, error) {
 			keyAccessControlAllowOrigin, DefaultAccessControlAllowOrigin),
 		DownstreamTimeout: loader.durationVal(
 			keyDownstreamTimeout, DefaultDownstreamTimeout),
-		CacheMaxBytes:   loader.int64Val(keyCacheMaxBytes, 0),
+		CacheMaxBytes:    loader.int64Val(keyCacheMaxBytes, 0),
-		BlockedNetworks: blockedNetworks,
+		BlockedNetworks:  blockedNetworks,
-		TrustedProxies:  trustedProxies,
+		TrustedProxies:   trustedProxies,
+		RefererBlocklist: loader.hostListVal(keyRefererBlocklist),
 	}
 
 	// The default for an omitted cache_max_bytes is worked out when
@@ -420,7 +425,8 @@ func isKnownConfigKey(key string) bool {
 		keyUpstreamConnectionsPerHost, keyUpstreamConnections,
 		keyMaxConcurrentProcessing, keyCacheMaxBytes, keyBlockedNetworks,
 		keyTrustedProxies, keyAccessControlAllowOrigin, keyUpstreamFetchTimeout,
-		keyUpstreamMaxResponseSize, keyDownstreamTimeout, "env":
+		keyUpstreamMaxResponseSize, keyDownstreamTimeout, keyRefererBlocklist,
+		"env":
 		return true
 	}
 
@@ -443,6 +449,7 @@ func envVarNames() map[string]string {
 		keyMetricsPassword:            "PIXA_METRICS_PASSWORD",
 		keySigningKey:                 "PIXA_SIGNING_KEY",
 		keyAllowlistHosts:             "PIXA_ALLOWLIST_HOSTS",
+		keyRefererBlocklist:           "PIXA_REFERER_BLOCKLIST",
 		keyAllowHTTP:                  "PIXA_ALLOW_HTTP",
 		keyUpstreamConnectionsPerHost: "PIXA_UPSTREAM_CONNECTIONS_PER_HOST",
 		keyUpstreamConnections:        "PIXA_UPSTREAM_CONNECTIONS",
@@ -612,7 +619,7 @@ func (c *Config) validate() error {
 	}
 
 	for _, host := range c.AllowlistHosts {
-		err := validateAllowlistHost(host)
+		err := validateHostPattern(keyAllowlistHosts, host)
 		if err != nil {
 			return err
 		}
@@ -735,25 +742,24 @@ func (c *Config) validateConcurrencyLimits() error {
 	return nil
 }
 
-// validateAllowlistHost checks that an allowlist_hosts entry is a bare
+// hostNamePattern matches a host name: letters, digits, hyphens, underscores
-// hostname, optionally with a leading dot for suffix matching. URLs,
+// and dots, optionally after one leading dot.
-// paths, and whitespace indicate a misconfigured entry. An entry with
+var hostNamePattern = regexp.MustCompile(`^\.?[A-Za-z0-9_-][A-Za-z0-9_.-]*$`)
-// no hostname labels (such as ".") is rejected: the allowlist matcher
+
-// treats a leading dot as a suffix pattern, so a bare "." would match
+// validateHostPattern checks that an entry of the named key, allowlist_hosts
-// any upstream host written in FQDN trailing-dot form and effectively
+// or referer_blocklist, is an IP address or a host name, the host name
-// disable URL signing.
+// optionally with one leading dot for suffix matching. Anything else, such as
-func validateAllowlistHost(host string) error {
+// a URL, a port or a "*." wildcard, can never match a host name that resolves,
-	if strings.Contains(host, "://") || strings.ContainsAny(host, "/ \t") {
+// so it is refused.
-		return fmt.Errorf("%s: entry %q %w",
+// So is "." alone: the allowlist matcher would match it against any host
-			settingName(keyAllowlistHosts), host, errNotBareHostname)
+// written with a trailing dot, which in allowlist_hosts disables URL signing.
+func validateHostPattern(key, host string) error {
+	_, err := netip.ParseAddr(host)
+	if err == nil || hostNamePattern.MatchString(host) {
+		return nil
 	}
 
-	if strings.Trim(host, ".") == "" {
+	return fmt.Errorf("%s: entry %q %w", settingName(key), host, errNotAHost)
-		return fmt.Errorf("%s: entry %q %w",
-			settingName(keyAllowlistHosts), host, errNoHostnameLabels)
-	}
-
-	return nil
 }
 
 // loadConfigFile loads configuration from the PIXA_CONFIG_PATH env var
@@ -883,6 +889,19 @@ func (l *strictLoader) boolVal(key string, defaultVal bool) bool {
 	return val
 }
 
+func (l *strictLoader) hostListVal(key string) []string {
+	if l.err != nil {
+		return nil
+	}
+
+	val, err := parseHostList(l.sc, key)
+	if err != nil {
+		l.err = err
+	}
+
+	return val
+}
+
 // getString returns the string value for key, or defaultVal if the key
 // is omitted. A present value that is not a string, or is explicitly
 // null, is an error.
@@ -1180,7 +1199,7 @@ func parseCIDRList(sc *smartconfig.Config, key string) ([]netip.Prefix, error) {
 		return nil, errNullConfigValue(key)
 	}
 
-	entries, err := cidrListEntries(raw, key)
+	entries, err := listEntries(raw, key)
 	if err != nil {
 		return nil, err
 	}
@@ -1200,11 +1219,41 @@ func parseCIDRList(sc *smartconfig.Config, key string) ([]netip.Prefix, error) {
 	return prefixes, nil
 }
 
-// cidrListEntries extracts the raw entries of the named CIDR-list key as
+// parseHostList parses the value of the named config key into host patterns,
-// trimmed, non-empty strings, from either a YAML list of strings or a
+// or returns nil if the key is omitted. It accepts a YAML list of strings or a
-// comma-separated string; an empty string is an empty list, as for
+// comma-separated string. An explicitly null value, a wrong type, an empty
-// allowlist_hosts. Any other shape is a configuration error.
+// entry, a non-string entry, or an entry validateHostPattern rejects aborts
-func cidrListEntries(raw any, key string) ([]string, error) {
+// startup naming the key and the offending value.
+func parseHostList(sc *smartconfig.Config, key string) ([]string, error) {
+	raw, ok := lookupValue(sc, key)
+	if !ok {
+		return nil, nil
+	}
+
+	if raw == nil {
+		return nil, errNullConfigValue(key)
+	}
+
+	entries, err := listEntries(raw, key)
+	if err != nil {
+		return nil, err
+	}
+
+	for _, entry := range entries {
+		err := validateHostPattern(key, entry)
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	return entries, nil
+}
+
+// listEntries extracts the raw entries of the named list key as trimmed,
+// non-empty strings, from either a YAML list of strings or a comma-separated
+// string; an empty string is an empty list, as for allowlist_hosts. Any other
+// shape is a configuration error.
+func listEntries(raw any, key string) ([]string, error) {
 	switch val := raw.(type) {
 	case []any:
 		entries := make([]string, 0, len(val))

internal/config/config_validation_internal_test.go

@@ -5,6 +5,7 @@ import (
 	"log/slog"
 	"os"
 	"path/filepath"
+	"slices"
 	"strings"
 	"testing"
 	"time"
@@ -216,6 +217,25 @@ func TestCommaSeparatedAllowlistStillSupported(t *testing.T) {
 	}
 }
 
+// TestAllowlistHostsAcceptsUnderscore checks that an upstream host name with
+// an underscore, which pixa can fetch from, is accepted as an entry.
+func TestAllowlistHostsAcceptsUnderscore(t *testing.T) {
+	t.Parallel()
+
+	c, err := configFromYAML(t, signingKeyLine+`allowlist_hosts:
+  - my_bucket.example.com
+  - .my_bucket.example.org
+`)
+	if err != nil {
+		t.Fatalf("host names with an underscore should load, got error: %v", err)
+	}
+
+	want := []string{"my_bucket.example.com", ".my_bucket.example.org"}
+	if !slices.Equal(c.AllowlistHosts, want) {
+		t.Errorf("AllowlistHosts = %v, want %v", c.AllowlistHosts, want)
+	}
+}
+
 // runAbortCases asserts that each case's config aborts startup with an
 // error message mentioning every expected substring.
 func runAbortCases(t *testing.T, cases []abortCase) {
@@ -318,6 +338,20 @@ func invalidHostAndCredentialCases() []abortCase {
 				keyAllowlistHosts, "example.com/images",
 			},
 		},
+		{
+			name: "allowlist host with wildcard",
+			yaml: signingKeyLine + "allowlist_hosts:\n  - \"*.example.com\"\n",
+			wantErrSubstrings: []string{
+				keyAllowlistHosts, "*.example.com",
+			},
+		},
+		{
+			name: "allowlist host with port",
+			yaml: signingKeyLine + "allowlist_hosts:\n  - example.com:8443\n",
+			wantErrSubstrings: []string{
+				keyAllowlistHosts, "example.com:8443",
+			},
+		},
 		{
 			name:              "allowlist host with whitespace",
 			yaml:              signingKeyLine + "allowlist_hosts:\n  - \"exa mple.com\"\n",

internal/config/env_internal_test.go

@@ -65,6 +65,7 @@ func TestEnvironmentSetsEveryKey(t *testing.T) {
 	t.Setenv("PIXA_METRICS_PASSWORD", "metricspass")
 	t.Setenv("PIXA_SIGNING_KEY", validTestSigningKey)
 	t.Setenv("PIXA_ALLOWLIST_HOSTS", "s3.sneak.cloud,.example.com")
+	t.Setenv("PIXA_REFERER_BLOCKLIST", "hotlinker.example,.leech.example")
 	t.Setenv("PIXA_ALLOW_HTTP", "true")
 	t.Setenv("PIXA_UPSTREAM_CONNECTIONS_PER_HOST", "5")
 	t.Setenv("PIXA_UPSTREAM_CONNECTIONS", "10")
@@ -93,6 +94,7 @@ func TestEnvironmentSetsEveryKey(t *testing.T) {
 		MetricsPassword:            "metricspass",
 		SigningKey:                 validTestSigningKey,
 		AllowlistHosts:             []string{testHostS3, ".example.com"},
+		RefererBlocklist:           []string{"hotlinker.example", ".leech.example"},
 		AllowHTTP:                  true,
 		UpstreamConnectionsPerHost: 5,
 		UpstreamConnections:        10,

internal/config/referer_blocklist_internal_test.go

@@ -0,0 +1,166 @@
+package config
+
+import (
+	"slices"
+	"testing"
+)
+
+// TestRefererBlocklistParsed loads a referer_blocklist with a host and a
+// pattern starting with "." and checks both are kept in order.
+func TestRefererBlocklistParsed(t *testing.T) {
+	t.Parallel()
+
+	c, err := configFromYAML(t, signingKeyLine+`referer_blocklist:
+  - leech.example
+  - .hotlinker.example
+`)
+	if err != nil {
+		t.Fatalf("valid referer_blocklist should load, got error: %v", err)
+	}
+
+	want := []string{"leech.example", ".hotlinker.example"}
+	if !slices.Equal(c.RefererBlocklist, want) {
+		t.Errorf("RefererBlocklist = %v, want %v", c.RefererBlocklist, want)
+	}
+}
+
+// TestRefererBlocklistAcceptsIPAddresses checks that IPv4 and IPv6 addresses,
+// the IPv6 one written without brackets, are accepted as entries.
+func TestRefererBlocklistAcceptsIPAddresses(t *testing.T) {
+	t.Parallel()
+
+	c, err := configFromYAML(t, signingKeyLine+`referer_blocklist:
+  - 192.0.2.7
+  - "2001:db8::7"
+`)
+	if err != nil {
+		t.Fatalf("IP address entries should load, got error: %v", err)
+	}
+
+	want := []string{"192.0.2.7", "2001:db8::7"}
+	if !slices.Equal(c.RefererBlocklist, want) {
+		t.Errorf("RefererBlocklist = %v, want %v", c.RefererBlocklist, want)
+	}
+}
+
+// TestRefererBlocklistAcceptsUnderscore checks that a host name with an
+// underscore, which a page can be served from, is accepted as an entry.
+func TestRefererBlocklistAcceptsUnderscore(t *testing.T) {
+	t.Parallel()
+
+	c, err := configFromYAML(t, signingKeyLine+`referer_blocklist:
+  - my_site.leech.example
+  - .my_site.hotlinker.example
+`)
+	if err != nil {
+		t.Fatalf("host names with an underscore should load, got error: %v", err)
+	}
+
+	want := []string{"my_site.leech.example", ".my_site.hotlinker.example"}
+	if !slices.Equal(c.RefererBlocklist, want) {
+		t.Errorf("RefererBlocklist = %v, want %v", c.RefererBlocklist, want)
+	}
+}
+
+// TestRefererBlocklistOmittedIsEmpty checks that an omitted key blocks no
+// referer.
+func TestRefererBlocklistOmittedIsEmpty(t *testing.T) {
+	t.Parallel()
+
+	c, err := configFromYAML(t, signingKeyLine)
+	if err != nil {
+		t.Fatalf("minimal config should be valid, got error: %v", err)
+	}
+
+	if len(c.RefererBlocklist) != 0 {
+		t.Errorf("RefererBlocklist = %v, want empty", c.RefererBlocklist)
+	}
+}
+
+// TestRefererBlocklistInvalidAbortsStartup checks that an entry that is not a
+// host, or a value that is not a list of them, aborts startup with an error
+// naming the key and the entry.
+func TestRefererBlocklistInvalidAbortsStartup(t *testing.T) {
+	t.Parallel()
+
+	runAbortCases(t, []abortCase{
+		{
+			name: "entry with a scheme",
+			yaml: signingKeyLine + "referer_blocklist:\n  - https://leech.example\n",
+			wantErrSubstrings: []string{
+				keyRefererBlocklist, "https://leech.example",
+			},
+		},
+		{
+			name: "entry with a path",
+			yaml: signingKeyLine + "referer_blocklist:\n  - leech.example/page\n",
+			wantErrSubstrings: []string{
+				keyRefererBlocklist, "leech.example/page",
+			},
+		},
+		{
+			name: "wildcard entry",
+			yaml: signingKeyLine + "referer_blocklist:\n  - \"*.leech.example\"\n",
+			wantErrSubstrings: []string{
+				keyRefererBlocklist, "*.leech.example",
+			},
+		},
+		{
+			name: "entry with a port",
+			yaml: signingKeyLine + "referer_blocklist:\n  - leech.example:8080\n",
+			wantErrSubstrings: []string{
+				keyRefererBlocklist, "leech.example:8080",
+			},
+		},
+		{
+			name: "two leading dots",
+			yaml: signingKeyLine + "referer_blocklist:\n  - ..leech.example\n",
+			wantErrSubstrings: []string{
+				keyRefererBlocklist, "..leech.example",
+			},
+		},
+		{
+			name:              "dot only",
+			yaml:              signingKeyLine + "referer_blocklist:\n  - \".\"\n",
+			wantErrSubstrings: []string{keyRefererBlocklist, `"."`},
+		},
+		{
+			name:              "empty entry",
+			yaml:              signingKeyLine + "referer_blocklist:\n  - \"\"\n",
+			wantErrSubstrings: []string{keyRefererBlocklist},
+		},
+		{
+			name:              "entry not a string",
+			yaml:              signingKeyLine + "referer_blocklist:\n  - 42\n",
+			wantErrSubstrings: []string{keyRefererBlocklist, "42"},
+		},
+		{
+			name:              "null value",
+			yaml:              signingKeyLine + "referer_blocklist:\n",
+			wantErrSubstrings: []string{keyRefererBlocklist, nullValueText},
+		},
+	})
+}
+
+// TestRefererBlocklistFromEnvironment checks that PIXA_REFERER_BLOCKLIST
+// takes comma-separated entries, and that an entry in it that is not a host
+// aborts startup naming the variable and the entry.
+func TestRefererBlocklistFromEnvironment(t *testing.T) {
+	t.Setenv("PIXA_SIGNING_KEY", validTestSigningKey)
+	t.Setenv("PIXA_REFERER_BLOCKLIST", " leech.example , .hotlinker.example ")
+
+	c, err := newFromSmartConfig(nil)
+	if err != nil {
+		t.Fatalf("valid PIXA_REFERER_BLOCKLIST should load, got error: %v", err)
+	}
+
+	want := []string{"leech.example", ".hotlinker.example"}
+	if !slices.Equal(c.RefererBlocklist, want) {
+		t.Errorf("RefererBlocklist = %v, want %v", c.RefererBlocklist, want)
+	}
+
+	t.Setenv("PIXA_REFERER_BLOCKLIST", "leech.example,https://hotlinker.example")
+
+	_, err = newFromSmartConfig(nil)
+	wantStartupError(t, err, "PIXA_REFERER_BLOCKLIST", "https://hotlinker.example")
+}

internal/handlers/auth_session_internal_test.go

@@ -8,6 +8,7 @@ import (
 	"regexp"
 	"strings"
 	"testing"
+	"time"
 
 	"sneak.berlin/go/pixa/internal/imgcache"
 	"sneak.berlin/go/pixa/internal/session"
@@ -241,3 +242,51 @@ func TestGeneratePost_URLServesImage(t *testing.T) {
 
 	requireServedPhoto(t, imageRec)
 }
+
+// TestGeneratePost_URLWithTTLExpires verifies that a URL the generator page
+// makes with a ttl of one second is served by /v1/e/ at once and answers 410
+// once the ttl has passed.
+func TestGeneratePost_URLWithTTLExpires(t *testing.T) {
+	t.Parallel()
+
+	_, imageSrv := newSignedHostServer(t, slog.New(slog.DiscardHandler))
+
+	rec := generatePost(t, url.Values{
+		sourceURLField: {"https://" + signedHost + photoPath},
+		widthField:     {"50"},
+		heightField:    {"50"},
+		formatField:    {string(imgcache.FormatJPEG)},
+		ttlField:       {"1"},
+	})
+
+	if rec.Code != http.StatusOK {
+		t.Fatalf("POST /generate status = %d, want %d", rec.Code, http.StatusOK)
+	}
+
+	match := generatedURLPattern.FindStringSubmatch(rec.Body.String())
+	if match == nil {
+		t.Fatalf("generator page shows no URL: %s", rec.Body.String())
+	}
+
+	imageRec := httptest.NewRecorder()
+	imageSrv.ServeHTTP(imageRec, httptest.NewRequestWithContext(
+		t.Context(), http.MethodGet, match[1], nil))
+
+	requireServedPhoto(t, imageRec)
+
+	// The URL keeps the time it expires in whole seconds and is served
+	// through the whole of that second, so a ttl of one second has passed
+	// for certain two seconds after the URL was made.
+	time.Sleep(2 * time.Second)
+
+	imageRec = httptest.NewRecorder()
+	imageSrv.ServeHTTP(imageRec, httptest.NewRequestWithContext(
+		t.Context(), http.MethodGet, match[1], nil))
+
+	t.Logf("GET %s after the ttl: %d %q", match[1], imageRec.Code, imageRec.Body)
+
+	if imageRec.Code != http.StatusGone {
+		t.Errorf("status after the ttl = %d, want %d",
+			imageRec.Code, http.StatusGone)
+	}
+}

internal/handlers/fetcher_internal_test.go

@@ -0,0 +1,61 @@
+package handlers
+
+import (
+	"net/http"
+	"net/netip"
+	"path/filepath"
+	"testing"
+	"time"
+
+	"github.com/go-chi/chi/v5"
+	"go.uber.org/fx"
+	"go.uber.org/fx/fxtest"
+
+	"sneak.berlin/go/pixa/internal/config"
+	"sneak.berlin/go/pixa/internal/database"
+	"sneak.berlin/go/pixa/internal/globals"
+	"sneak.berlin/go/pixa/internal/healthcheck"
+	"sneak.berlin/go/pixa/internal/logger"
+)
+
+// TestHandlersBuildTheirOwnFetcherWhenNoneIsProvided builds the handlers as
+// pixad does, in an fx app that provides no fetcher, and requests an image
+// from 192.0.2.10, which is on the allowlist and in blocked_networks. The URL
+// check accepts that address; only the dialer that refuses internal
+// addresses checks blocked_networks, so the answer is 403 only if the
+// fetcher the handlers build from the config connects with that dialer. Any
+// other dialer would try to connect until the upstream fetch timeout, which
+// is short so that the test then fails quickly.
+func TestHandlersBuildTheirOwnFetcherWhenNoneIsProvided(t *testing.T) {
+	t.Parallel()
+
+	const host = "192.0.2.10"
+
+	stateDir := t.TempDir()
+	cfg := &config.Config{
+		SigningKey:           testSigningKey,
+		StateDir:             stateDir,
+		DBURL:                "file:" + filepath.Join(stateDir, "state.sqlite3"),
+		AllowlistHosts:       []string{host},
+		BlockedNetworks:      []netip.Prefix{netip.MustParsePrefix("192.0.2.0/24")},
+		UpstreamFetchTimeout: 2 * time.Second,
+		// With no connection slots, the fetch would fail before dialing.
+		UpstreamConnections: config.DefaultUpstreamConnections,
+	}
+
+	var h *Handlers
+
+	app := fxtest.New(t,
+		fx.Supply(cfg),
+		fx.Provide(globals.New, logger.New, database.New, healthcheck.New, New),
+		fx.Populate(&h),
+	)
+	app.RequireStart()
+	t.Cleanup(app.RequireStop)
+
+	r := chi.NewRouter()
+	r.Get("/v1/image/*", h.HandleImage())
+
+	rec := sendGet(t, r, photoURL(host))
+	checkErrorBody(t, rec, http.StatusForbidden, "forbidden")
+}

internal/handlers/handlers.go

@@ -9,6 +9,7 @@ import (
 	"time"
 
 	"go.uber.org/fx"
+	"sneak.berlin/go/pixa/internal/allowlist"
 	"sneak.berlin/go/pixa/internal/config"
 	"sneak.berlin/go/pixa/internal/database"
 	"sneak.berlin/go/pixa/internal/encurl"
@@ -27,6 +28,11 @@ type Params struct {
 	Healthcheck *healthcheck.Healthcheck
 	Database    *database.Database
 	Config      *config.Config
+
+	// Fetcher, when provided, fetches upstream images in place of the
+	// fetcher the handlers build from the config. Only tests provide one;
+	// pixad does not.
+	Fetcher httpfetcher.Fetcher `optional:"true"`
 }
 
 // Handlers provides HTTP request handlers.
@@ -35,11 +41,16 @@ type Handlers struct {
 	hc          *healthcheck.Healthcheck
 	db          *database.Database
 	config      *config.Config
+	fetcher     httpfetcher.Fetcher
 	imgSvc      *imgcache.Service
 	imgCache    *imgcache.Cache
 	sessMgr     *session.Manager
 	encGen      *encurl.Generator
 	csrfProtect func(http.Handler) http.Handler
+
+	// refererBlocklist matches the hosts of referer_blocklist; its IsAllowed
+	// reports whether a URL's host is on that list.
+	refererBlocklist *allowlist.HostAllowList
 }
 
 // New creates a new Handlers instance.
@@ -50,11 +61,13 @@ func New(lc fx.Lifecycle, params Params) (*Handlers, error) {
 	}
 
 	s := &Handlers{
-		log:         params.Logger.Get(),
+		log:              params.Logger.Get(),
-		hc:          params.Healthcheck,
+		hc:               params.Healthcheck,
-		db:          params.Database,
+		db:               params.Database,
-		config:      params.Config,
+		config:           params.Config,
-		csrfProtect: csrfProtect,
+		fetcher:          params.Fetcher,
+		csrfProtect:      csrfProtect,
+		refererBlocklist: allowlist.New(params.Config.RefererBlocklist),
 	}
 
 	lc.Append(fx.Hook{
@@ -122,10 +135,12 @@ func (s *Handlers) initImageService() error {
 	fetcherCfg.MaxConnections = s.config.UpstreamConnections
 	fetcherCfg.BlockedNetworks = s.config.BlockedNetworks
 
-	// Create the service
+	// Create the service. With no fetcher provided, it builds its own from
+	// fetcherCfg.
 	svc, err := imgcache.NewService(&imgcache.ServiceConfig{
 		Cache:                   cache,
 		FetcherConfig:           fetcherCfg,
+		Fetcher:                 s.fetcher,
 		SigningKey:              s.config.SigningKey,
 		Allowlist:               s.config.AllowlistHosts,
 		MaxConcurrentProcessing: s.config.MaxConcurrentProcessing,

internal/handlers/image.go

@@ -21,6 +21,10 @@ import (
 // /v1/image/<host>/<path>/<width>x<height>.<format>
 func (s *Handlers) HandleImage() http.HandlerFunc {
 	return func(w http.ResponseWriter, r *http.Request) {
+		if s.refuseBlockedReferer(w, r) {
+			return
+		}
+
 		req, ok := s.parseImageRequest(w, r)
 		if !ok {
 			return
@@ -248,6 +252,23 @@ func cacheControl(expires time.Time) string {
 	return fmt.Sprintf("public, max-age=%d, immutable", int64(maxAge/time.Second))
 }
 
+// refuseBlockedReferer answers 403 with a JSON error when the request's Referer
+// names a host on referer_blocklist, and reports whether it answered. A request
+// with no Referer, or one that does not parse as a URL with a host, is not
+// refused.
+func (s *Handlers) refuseBlockedReferer(
+	w http.ResponseWriter, r *http.Request,
+) bool {
+	referer, err := url.Parse(r.Referer())
+	if err != nil || !s.refererBlocklist.IsAllowed(referer) {
+		return false
+	}
+
+	s.respondError(w, "referer blocked", http.StatusForbidden)
+
+	return true
+}
+
 // notModified sets the ETag header to etag and, when the request's
 // If-None-Match is that ETag, answers 304 Not Modified. It reports whether it
 // answered. An empty etag sets no header and never answers.

internal/handlers/imageenc.go

@@ -22,6 +22,10 @@ import (
 // browsers identify the content type.
 func (s *Handlers) HandleImageEnc() http.HandlerFunc {
 	return func(w http.ResponseWriter, r *http.Request) {
+		if s.refuseBlockedReferer(w, r) {
+			return
+		}
+
 		ctx := r.Context()
 		start := time.Now()
 

internal/handlers/referer_blocklist_internal_test.go

@@ -0,0 +1,185 @@
+package handlers
+
+import (
+	"context"
+	"log/slog"
+	"net/http"
+	"net/http/httptest"
+	"sync/atomic"
+	"testing"
+	"time"
+
+	"github.com/go-chi/chi/v5"
+	"sneak.berlin/go/pixa/internal/allowlist"
+	"sneak.berlin/go/pixa/internal/encurl"
+	"sneak.berlin/go/pixa/internal/httpfetcher"
+	"sneak.berlin/go/pixa/internal/imgcache"
+)
+
+// blockedReferer is a page on leech.example, which newRefererRoutes puts on
+// referer_blocklist.
+const blockedReferer = "https://leech.example/page.html"
+
+// countingFetcher passes each fetch on to the fetcher it holds and counts it.
+type countingFetcher struct {
+	httpfetcher.Fetcher
+
+	fetches atomic.Int32
+}
+
+// Fetch counts the fetch and passes it on.
+func (f *countingFetcher) Fetch(
+	ctx context.Context, url string,
+) (*httpfetcher.FetchResult, error) {
+	f.fetches.Add(1)
+
+	return f.Fetcher.Fetch(ctx, url)
+}
+
+// newRefererRoutes returns both image routes of a Handlers whose
+// referer_blocklist is "leech.example" and ".hotlinker.example", the
+// Handlers, and the fetcher the routes fetch through. The JPEG at photoPath
+// exists on allowlistedHost and on signedHost.
+func newRefererRoutes(t *testing.T) (http.Handler, *Handlers, *countingFetcher) {
+	t.Helper()
+
+	fetcher := &countingFetcher{
+		Fetcher: newPhotoFetcher(t, allowlistedHost, signedHost),
+	}
+
+	cache, err := imgcache.NewCache(setupTestDB(t), imgcache.CacheConfig{
+		StateDir:    t.TempDir(),
+		CacheTTL:    time.Hour,
+		NegativeTTL: 5 * time.Minute,
+	})
+	if err != nil {
+		t.Fatalf("imgcache.NewCache() error = %v", err)
+	}
+
+	svc, err := imgcache.NewService(&imgcache.ServiceConfig{
+		Cache:      cache,
+		Fetcher:    fetcher,
+		SigningKey: testSigningKey,
+		Allowlist:  []string{allowlistedHost},
+	})
+	if err != nil {
+		t.Fatalf("imgcache.NewService() error = %v", err)
+	}
+
+	encGen, err := encurl.NewGenerator(testSigningKey)
+	if err != nil {
+		t.Fatalf("encurl.NewGenerator() error = %v", err)
+	}
+
+	h := &Handlers{
+		log:    slog.New(slog.DiscardHandler),
+		imgSvc: svc,
+		encGen: encGen,
+		refererBlocklist: allowlist.New(
+			[]string{"leech.example", ".hotlinker.example"}),
+	}
+
+	r := chi.NewRouter()
+	r.Get("/v1/image/*", h.HandleImage())
+	r.Get("/v1/e/{token}/*", h.HandleImageEnc())
+
+	return r, h, fetcher
+}
+
+// getWithReferer sends a GET for target to routes with referer as its
+// Referer header, or with none when referer is empty, and returns the
+// response.
+func getWithReferer(
+	t *testing.T, routes http.Handler, target, referer string,
+) *httptest.ResponseRecorder {
+	t.Helper()
+
+	req := httptest.NewRequestWithContext(t.Context(), http.MethodGet, target, nil)
+	if referer != "" {
+		req.Header.Set("Referer", referer)
+	}
+
+	rec := httptest.NewRecorder()
+
+	routes.ServeHTTP(rec, req)
+	t.Logf("GET %s with Referer %q: %d", target, referer, rec.Code)
+
+	return rec
+}
+
+// TestRefererBlocklist verifies that both image routes refuse a request whose
+// Referer names a host on referer_blocklist with 403 and the JSON error,
+// without fetching from the upstream host, and serve a request with no
+// Referer, one that does not parse, or one naming any other host. Hosts are
+// matched as allowlist_hosts matches them.
+func TestRefererBlocklist(t *testing.T) {
+	t.Parallel()
+
+	cases := []struct {
+		name    string
+		referer string
+		want    int
+	}{
+		{"no referer", "", http.StatusOK},
+		{"unlisted host", "https://unlisted.example/page.html", http.StatusOK},
+		{"unparseable", "%zz", http.StatusOK},
+		{"listed host", blockedReferer, http.StatusForbidden},
+		{"subdomain of listed host", "https://www.leech.example/", http.StatusOK},
+		{"subdomain of dot pattern", "https://www.hotlinker.example/a.html",
+			http.StatusForbidden},
+		{"dot pattern without its dot", "https://hotlinker.example/",
+			http.StatusForbidden},
+		{"host continuing past dot pattern",
+			"https://hotlinker.example.evil.example/", http.StatusOK},
+	}
+
+	// The photo's URL on each image route.
+	photoURLs := map[string]func(t *testing.T, h *Handlers) string{
+		"plain URL": func(t *testing.T, _ *Handlers) string {
+			t.Helper()
+
+			return photoURL(allowlistedHost)
+		},
+		"encrypted URL": encPhotoURL,
+	}
+
+	for urlName, photoURLFor := range photoURLs {
+		for _, tc := range cases {
+			t.Run(urlName+", "+tc.name, func(t *testing.T) {
+				t.Parallel()
+
+				routes, h, fetcher := newRefererRoutes(t)
+
+				rec := getWithReferer(t, routes, photoURLFor(t, h), tc.referer)
+
+				if tc.want == http.StatusOK {
+					requireServedPhoto(t, rec)
+
+					return
+				}
+
+				checkErrorBody(t, rec, http.StatusForbidden, "referer blocked")
+
+				if n := fetcher.fetches.Load(); n != 0 {
+					t.Errorf("upstream fetched %d times, want 0", n)
+				}
+			})
+		}
+	}
+}
+
+// TestBlockedRefererRefusedWhenImageIsCached verifies that a request whose
+// Referer is on referer_blocklist is refused even when the image it asks for
+// is already cached, so the answer does not depend on the cache.
+func TestBlockedRefererRefusedWhenImageIsCached(t *testing.T) {
+	t.Parallel()
+
+	routes, h, _ := newRefererRoutes(t)
+
+	for _, target := range []string{photoURL(allowlistedHost), encPhotoURL(t, h)} {
+		requireServedPhoto(t, getWithReferer(t, routes, target, ""))
+
+		rec := getWithReferer(t, routes, target, blockedReferer)
+		checkErrorBody(t, rec, http.StatusForbidden, "referer blocked")
+	}
+}

internal/httpfetcher/dial_context_internal_test.go

@@ -0,0 +1,66 @@
+package httpfetcher
+
+import (
+	"errors"
+	"net"
+	"testing"
+)
+
+// TestNewUsesCheckedDialerWithoutDialContext checks that a fetcher built
+// without DialContext, as pixa builds it, refuses to connect to a local
+// server.
+func TestNewUsesCheckedDialerWithoutDialContext(t *testing.T) {
+	t.Parallel()
+
+	srv := startUpstream(t)
+	transport := transportOf(t, New(DefaultConfig()))
+
+	addr := srv.Listener.Addr().String()
+
+	_, err := transport.DialContext(testContext(t), "tcp", addr)
+	if !errors.Is(err, ErrSSRFBlocked) {
+		t.Fatalf("DialContext(%s) error = %v, want ErrSSRFBlocked", addr, err)
+	}
+}
+
+// TestDialContextReplacesOnlyTheDialer checks that a fetcher built with
+// DialContext connects through it, while the URL check still refuses a
+// loopback URL and the redirect check a redirect to a link-local address.
+func TestDialContextReplacesOnlyTheDialer(t *testing.T) {
+	t.Parallel()
+
+	srv := startUpstream(t)
+	dialer := &recordingDialer{target: srv.Listener.Addr().String()}
+
+	cfg := DefaultConfig()
+	cfg.AllowHTTP = true
+	cfg.DialContext = dialer.dialContext
+	f := New(cfg)
+
+	if body := fetchBody(t, f, "/image"); body != imagePayload {
+		t.Errorf("body = %q, want %q", body, imagePayload)
+	}
+
+	_, err := f.Fetch(testContext(t), "http://127.0.0.1/image")
+	if !errors.Is(err, ErrSSRFBlocked) {
+		t.Errorf("Fetch(loopback URL) error = %v, want ErrSSRFBlocked", err)
+	}
+
+	_, err = f.Fetch(testContext(t), upstreamURL("/redirect/private"))
+	if !errors.Is(err, ErrSSRFBlocked) {
+		t.Errorf("Fetch(/redirect/private) error = %v, want ErrSSRFBlocked", err)
+	}
+
+	// The upstream server is reached through DialContext, and nothing else
+	// is asked of it.
+	dialed := dialer.dialedAddrs()
+	if len(dialed) == 0 {
+		t.Error("DialContext was never called")
+	}
+
+	for _, addr := range dialed {
+		if addr != net.JoinHostPort(testPublicHost, "80") {
+			t.Errorf("DialContext was asked to connect to %s", addr)
+		}
+	}
+}

internal/httpfetcher/httpfetcher.go

@@ -137,6 +137,11 @@ type Config struct {
 	// BlockedNetworks are operator-supplied CIDR ranges refused by the
 	// dialer, in addition to the always-enforced built-in ranges.
 	BlockedNetworks []netip.Prefix
+	// DialContext, when set, makes the fetcher's connections in place of
+	// the dialer that refuses internal addresses; the URL and redirect
+	// checks still run. Only tests set it, to reach a local server; the
+	// config file and the environment cannot.
+	DialContext func(ctx context.Context, network, addr string) (net.Conn, error)
 }
 
 // DefaultConfig returns a Config with sensible defaults.
@@ -190,13 +195,19 @@ func New(config *Config) *HTTPFetcher {
 		config = DefaultConfig()
 	}
 
-	// Create transport with SSRF-safe dialer. The dialer re-resolves and
+	// Unless config.DialContext replaces it, the transport connects with
-	// re-checks at connect time (closing the DNS-rebinding window) against
+	// the SSRF-safe dialer, which re-resolves and re-checks at connect time
-	// both the built-in ranges and the operator-supplied blocklist.
+	// (closing the DNS-rebinding window) against both the built-in ranges
-	transport := &http.Transport{
+	// and the operator-supplied blocklist.
-		DialContext: func(ctx context.Context, network, addr string) (net.Conn, error) {
+	dialContext := config.DialContext
+	if dialContext == nil {
+		dialContext = func(ctx context.Context, network, addr string) (net.Conn, error) {
 			return dialSSRFSafe(ctx, network, addr, config.BlockedNetworks)
-		},
+		}
+	}
+
+	transport := &http.Transport{
+		DialContext:         dialContext,
 		TLSHandshakeTimeout: DefaultTLSTimeout,
 		MaxIdleConns:        DefaultMaxIdleConns,
 		IdleConnTimeout:     DefaultIdleConnTimeout,

internal/imgcache/service.go

@@ -42,7 +42,8 @@ type Service struct {
 type ServiceConfig struct {
 	// Cache is the cache instance
 	Cache *Cache
-	// FetcherConfig configures the upstream fetcher (ignored if Fetcher is set)
+	// FetcherConfig configures the upstream fetcher built when Fetcher is
+	// not set. Its AllowHTTP and MaxResponseSize are used either way.
 	FetcherConfig *httpfetcher.Config
 	// Fetcher is an optional custom fetcher (for testing)
 	Fetcher httpfetcher.Fetcher

internal/server/image_proxy_internal_test.go

@@ -0,0 +1,302 @@
+package server
+
+import (
+	"bytes"
+	"context"
+	"crypto/sha256"
+	"database/sql"
+	"encoding/hex"
+	"image"
+	"image/color"
+	"image/jpeg"
+	"image/png"
+	"io"
+	"net"
+	"net/http"
+	"net/http/httptest"
+	"os"
+	"path/filepath"
+	"sync/atomic"
+	"testing"
+
+	"go.uber.org/fx"
+	"go.uber.org/fx/fxtest"
+
+	"sneak.berlin/go/pixa/internal/config"
+	"sneak.berlin/go/pixa/internal/database"
+	"sneak.berlin/go/pixa/internal/globals"
+	"sneak.berlin/go/pixa/internal/handlers"
+	"sneak.berlin/go/pixa/internal/healthcheck"
+	"sneak.berlin/go/pixa/internal/httpfetcher"
+	"sneak.berlin/go/pixa/internal/logger"
+	"sneak.berlin/go/pixa/internal/middleware"
+)
+
+// upstreamHost is the upstream host of the image URLs below. It is a
+// documentation address (RFC 5737), which the fetcher's URL check accepts as
+// public; the fetcher's dial function connects it to the test upstream server.
+const upstreamHost = "192.0.2.10"
+
+// TestImageProxyFlow requests images through pixa's router, handlers,
+// upstream fetcher, image processor, disk cache and database, with only the
+// upstream origin replaced by a local test server. The first request for a URL
+// is fetched and converted; the second is served from the cache without
+// another upstream request. The source and the converted image are then on
+// disk, with their rows in the database.
+func TestImageProxyFlow(t *testing.T) {
+	t.Parallel()
+
+	source := encodeTestPNG(t, 64, 48)
+
+	tests := []struct {
+		name          string
+		sizeFormat    string // the <size>.<format> part of the image URL
+		contentType   string
+		decodeConfig  func(io.Reader) (image.Config, error)
+		width, height int
+	}{
+		{"resize and convert to JPEG", "32x24.jpeg", "image/jpeg",
+			jpeg.DecodeConfig, 32, 24},
+		{"orig", "orig.orig", "image/png", png.DecodeConfig, 64, 48},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			t.Parallel()
+
+			var upstreamRequests atomic.Int32
+
+			upstream := httptest.NewServer(http.HandlerFunc(
+				func(w http.ResponseWriter, _ *http.Request) {
+					upstreamRequests.Add(1)
+					w.Header().Set("Content-Type", "image/png")
+					_, _ = w.Write(source)
+				}))
+			t.Cleanup(upstream.Close)
+
+			s, db, stateDir := startImageProxy(t, upstream)
+			target := "/v1/image/" + upstreamHost + "/photo.png/" + tt.sizeFormat
+
+			first := getImage(t, s, target)
+
+			if got := first.Header().Get("X-Pixa-Cache"); got != "MISS" {
+				t.Errorf("first X-Pixa-Cache = %q, want MISS", got)
+			}
+
+			if got := first.Header().Get("Content-Type"); got != tt.contentType {
+				t.Errorf("Content-Type = %q, want %q", got, tt.contentType)
+			}
+
+			decoded, err := tt.decodeConfig(bytes.NewReader(first.Body.Bytes()))
+			if err != nil {
+				t.Fatalf("decoding the image: %v", err)
+			}
+
+			if decoded.Width != tt.width || decoded.Height != tt.height {
+				t.Errorf("image is %dx%d, want %dx%d",
+					decoded.Width, decoded.Height, tt.width, tt.height)
+			}
+
+			second := getImage(t, s, target)
+
+			if got := second.Header().Get("X-Pixa-Cache"); got != "HIT" {
+				t.Errorf("second X-Pixa-Cache = %q, want HIT", got)
+			}
+
+			if !bytes.Equal(second.Body.Bytes(), first.Body.Bytes()) {
+				t.Error("the second response is not the image the first served")
+			}
+
+			if got := upstreamRequests.Load(); got != 1 {
+				t.Errorf("upstream received %d requests, want 1", got)
+			}
+
+			checkSourceCached(t, db, stateDir, source)
+			checkVariantCached(t, db, stateDir, first.Body.Bytes(), tt.contentType)
+		})
+	}
+}
+
+// startImageProxy starts the components pixad's fx app builds, from a config
+// with a fresh state directory and upstreamHost on the allowlist, and with an
+// upstream fetcher that connects every upstream address to upstream. It
+// returns the server with its routes, the database and the state directory.
+func startImageProxy(
+	t *testing.T, upstream *httptest.Server,
+) (*Server, *sql.DB, string) {
+	t.Helper()
+
+	stateDir := t.TempDir()
+	cfg := &config.Config{
+		SigningKey:     testSigningKey,
+		StateDir:       stateDir,
+		DBURL:          "file:" + filepath.Join(stateDir, "state.sqlite3"),
+		AllowlistHosts: []string{upstreamHost},
+		// The test upstream server has no TLS.
+		AllowHTTP: true,
+		// A limit of its own, so the cache does not size itself from the
+		// host's free disk space.
+		CacheMaxBytes:           64 << 20,
+		CacheMaxBytesExplicit:   true,
+		UpstreamMaxResponseSize: config.DefaultUpstreamMaxResponseSize,
+		DownstreamTimeout:       config.DefaultDownstreamTimeout,
+	}
+
+	fetcherCfg := httpfetcher.DefaultConfig()
+	fetcherCfg.AllowHTTP = true
+	fetcherCfg.DialContext = func(
+		ctx context.Context, network, _ string,
+	) (net.Conn, error) {
+		var dialer net.Dialer
+
+		return dialer.DialContext(ctx, network, upstream.Listener.Addr().String())
+	}
+	fetcher := httpfetcher.New(fetcherCfg)
+
+	var (
+		h  *handlers.Handlers
+		mw *middleware.Middleware
+		db *database.Database
+	)
+
+	app := fxtest.New(t,
+		fx.Supply(cfg),
+		fx.Provide(
+			globals.New,
+			logger.New,
+			database.New,
+			healthcheck.New,
+			handlers.New,
+			middleware.New,
+			func() httpfetcher.Fetcher { return fetcher },
+		),
+		fx.Populate(&h, &mw, &db),
+	)
+	app.RequireStart()
+	t.Cleanup(app.RequireStop)
+
+	// Requests go straight to the router, as in newTestServer; the server's
+	// own start hook, which listens on a port, is left out.
+	s := &Server{config: cfg, mw: mw, h: h}
+	s.SetupRoutes()
+
+	return s, db.DB(), stateDir
+}
+
+// getImage sends a GET for target to s and fails unless it answers 200.
+func getImage(t *testing.T, s *Server, target string) *httptest.ResponseRecorder {
+	t.Helper()
+
+	rec := httptest.NewRecorder()
+	s.ServeHTTP(rec, httptest.NewRequestWithContext(
+		t.Context(), http.MethodGet, target, nil))
+	t.Logf("GET %s: %d, X-Pixa-Cache %s",
+		target, rec.Code, rec.Header().Get("X-Pixa-Cache"))
+
+	if rec.Code != http.StatusOK {
+		t.Fatalf("GET %s status = %d, want %d; body %s",
+			target, rec.Code, http.StatusOK, rec.Body.String())
+	}
+
+	return rec
+}
+
+// checkSourceCached checks that source is stored under its SHA-256 in
+// cache/sources, recorded in source_content, and that the source URL's row in
+// source_metadata points at it.
+func checkSourceCached(t *testing.T, db *sql.DB, stateDir string, source []byte) {
+	t.Helper()
+
+	sum := sha256.Sum256(source)
+	hash := hex.EncodeToString(sum[:])
+
+	checkFile(t, filepath.Join(stateDir, "cache", "sources", hash[0:2], hash[2:4], hash),
+		source)
+
+	var rows int
+
+	err := db.QueryRowContext(t.Context(),
+		"SELECT COUNT(*) FROM source_content WHERE content_hash = ?", hash,
+	).Scan(&rows)
+	if err != nil || rows != 1 {
+		t.Errorf("source_content rows for the source = %d (error %v), want 1",
+			rows, err)
+	}
+
+	var metadataHash string
+
+	err = db.QueryRowContext(t.Context(),
+		`SELECT content_hash FROM source_metadata
+			WHERE source_host = ? AND source_path = ?`,
+		upstreamHost, "/photo.png",
+	).Scan(&metadataHash)
+	if err != nil || metadataHash != hash {
+		t.Errorf("source_metadata content_hash = %q (error %v), want %q",
+			metadataHash, err, hash)
+	}
+}
+
+// checkVariantCached checks that the converted image served is recorded in
+// variant_content with contentType, and stored under its cache key in
+// cache/variants.
+func checkVariantCached(
+	t *testing.T, db *sql.DB, stateDir string, served []byte, contentType string,
+) {
+	t.Helper()
+
+	var cacheKey, storedType string
+
+	err := db.QueryRowContext(t.Context(),
+		"SELECT cache_key, content_type FROM variant_content",
+	).Scan(&cacheKey, &storedType)
+	if err != nil {
+		t.Fatalf("variant_content row: %v", err)
+	}
+
+	if storedType != contentType {
+		t.Errorf("variant_content content_type = %q, want %q",
+			storedType, contentType)
+	}
+
+	checkFile(t, filepath.Join(stateDir, "cache", "variants",
+		cacheKey[0:2], cacheKey[2:4], cacheKey), served)
+}
+
+// checkFile checks that the file at path holds want.
+func checkFile(t *testing.T, path string, want []byte) {
+	t.Helper()
+
+	//nolint:gosec // G304: a path under the test's state directory
+	got, err := os.ReadFile(path)
+	if err != nil {
+		t.Errorf("reading %s: %v", path, err)
+
+		return
+	}
+
+	if !bytes.Equal(got, want) {
+		t.Errorf("%s holds %d bytes that are not the %d expected",
+			path, len(got), len(want))
+	}
+}
+
+// encodeTestPNG returns an opaque width x height PNG of one color.
+func encodeTestPNG(t *testing.T, width, height int) []byte {
+	t.Helper()
+
+	img := image.NewRGBA(image.Rect(0, 0, width, height))
+	for y := range height {
+		for x := range width {
+			img.Set(x, y, color.RGBA{R: 200, G: 40, B: 40, A: 255})
+		}
+	}
+
+	var buf bytes.Buffer
+
+	err := png.Encode(&buf, img)
+	if err != nil {
+		t.Fatalf("encoding the test PNG: %v", err)
+	}
+
+	return buf.Bytes()
+}