Refuse image requests whose Referer is on referer_blocklist (closes #90)

A new setting, referer_blocklist (PIXA_REFERER_BLOCKLIST), lists hosts
whose pages may not show pixa's images. Entries are written and matched
as allowlist_hosts are, with the same matcher. Both image routes check
the Referer before the signature, the cache and the upstream fetch, and
answer 403 with the JSON error, so a blocked request costs nothing and
is refused whether or not the image is cached. No Referer, or one that
does not parse, is served; README.md says this makes the list easy to
get around. An entry of either host list that is not a host name
(letters, digits, hyphens, underscores, dots, at most one leading dot)
or an IP address now aborts startup naming the setting and the entry.

Model: opus-5-5

internal/config/config.go

@@ -11,6 +11,7 @@ import (
 	"net/url"
 	"os"
 	"path/filepath"
+	"regexp"
 	"runtime"
 	"sort"
 	"strconv"
@@ -48,6 +49,7 @@ const (
 	keyMetricsPassword            = "metrics.password"
 	keySigningKey                 = "signing_key"
 	keyAllowlistHosts             = "allowlist_hosts"
+	keyRefererBlocklist           = "referer_blocklist"
 	keyAllowHTTP                  = "allow_http"
 	keyUpstreamConnectionsPerHost = "upstream_connections_per_host"
 	keyUpstreamConnections        = "upstream_connections"
@@ -97,12 +99,11 @@ var (
 		"value is null; omit the key entirely to use the default")
 	errValuesNull = errors.New(
 		"value is null; omit a key entirely to use its default")
-	errNotBareHostname = errors.New(
-		"must be a bare hostname without scheme, path, or whitespace")
-	errNoHostnameLabels = errors.New("contains no hostname labels")
-	errNotADuration     = errors.New("not a duration such as 30s or 2m")
-	errMustBePositive   = errors.New("must be positive")
-	errNotAnOrigin      = errors.New(
+	errNotAHost = errors.New("must be a host name such as " +
+		"cdn.example.com or .example.com, or an IP address")
+	errNotADuration   = errors.New("not a duration such as 30s or 2m")
+	errMustBePositive = errors.New("must be positive")
+	errNotAnOrigin    = errors.New(
 		`not "*" or an origin such as https://example.com`)
 )
 
@@ -130,6 +131,10 @@ type Config struct {
 	AllowHTTP                  bool     // Allow non-TLS upstream (testing only)
 	UpstreamConnectionsPerHost int      // Max concurrent connections per upstream host
 
+	// RefererBlocklist holds host patterns, matched as AllowlistHosts is: the
+	// image routes refuse a request whose Referer names a matching host.
+	RefererBlocklist []string
+
 	// UpstreamConnections is the most concurrent connections to all
 	// upstream hosts together, on top of the per-host limit.
 	// MaxConcurrentProcessing is the most images processed at once.
@@ -270,7 +275,6 @@ func newFromSmartConfig(sc *smartconfig.Config) (*Config, error) {
 	}
 
 	loader := &strictLoader{sc: sc}
-
 	c := &Config{
 		Debug:           loader.boolVal(keyDebug, false),
 		MaintenanceMode: loader.boolVal(keyMaintenanceMode, false),
@@ -298,9 +302,10 @@ func newFromSmartConfig(sc *smartconfig.Config) (*Config, error) {
 			keyAccessControlAllowOrigin, DefaultAccessControlAllowOrigin),
 		DownstreamTimeout: loader.durationVal(
 			keyDownstreamTimeout, DefaultDownstreamTimeout),
-		CacheMaxBytes:   loader.int64Val(keyCacheMaxBytes, 0),
-		BlockedNetworks: blockedNetworks,
-		TrustedProxies:  trustedProxies,
+		CacheMaxBytes:    loader.int64Val(keyCacheMaxBytes, 0),
+		BlockedNetworks:  blockedNetworks,
+		TrustedProxies:   trustedProxies,
+		RefererBlocklist: loader.hostListVal(keyRefererBlocklist),
 	}
 
 	// The default for an omitted cache_max_bytes is worked out when
@@ -420,7 +425,8 @@ func isKnownConfigKey(key string) bool {
 		keyUpstreamConnectionsPerHost, keyUpstreamConnections,
 		keyMaxConcurrentProcessing, keyCacheMaxBytes, keyBlockedNetworks,
 		keyTrustedProxies, keyAccessControlAllowOrigin, keyUpstreamFetchTimeout,
-		keyUpstreamMaxResponseSize, keyDownstreamTimeout, "env":
+		keyUpstreamMaxResponseSize, keyDownstreamTimeout, keyRefererBlocklist,
+		"env":
 		return true
 	}
 
@@ -443,6 +449,7 @@ func envVarNames() map[string]string {
 		keyMetricsPassword:            "PIXA_METRICS_PASSWORD",
 		keySigningKey:                 "PIXA_SIGNING_KEY",
 		keyAllowlistHosts:             "PIXA_ALLOWLIST_HOSTS",
+		keyRefererBlocklist:           "PIXA_REFERER_BLOCKLIST",
 		keyAllowHTTP:                  "PIXA_ALLOW_HTTP",
 		keyUpstreamConnectionsPerHost: "PIXA_UPSTREAM_CONNECTIONS_PER_HOST",
 		keyUpstreamConnections:        "PIXA_UPSTREAM_CONNECTIONS",
@@ -612,7 +619,7 @@ func (c *Config) validate() error {
 	}
 
 	for _, host := range c.AllowlistHosts {
-		err := validateAllowlistHost(host)
+		err := validateHostPattern(keyAllowlistHosts, host)
 		if err != nil {
 			return err
 		}
@@ -735,25 +742,24 @@ func (c *Config) validateConcurrencyLimits() error {
 	return nil
 }
 
-// validateAllowlistHost checks that an allowlist_hosts entry is a bare
-// hostname, optionally with a leading dot for suffix matching. URLs,
-// paths, and whitespace indicate a misconfigured entry. An entry with
-// no hostname labels (such as ".") is rejected: the allowlist matcher
-// treats a leading dot as a suffix pattern, so a bare "." would match
-// any upstream host written in FQDN trailing-dot form and effectively
-// disable URL signing.
-func validateAllowlistHost(host string) error {
-	if strings.Contains(host, "://") || strings.ContainsAny(host, "/ \t") {
-		return fmt.Errorf("%s: entry %q %w",
-			settingName(keyAllowlistHosts), host, errNotBareHostname)
+// hostNamePattern matches a host name: letters, digits, hyphens, underscores
+// and dots, optionally after one leading dot.
+var hostNamePattern = regexp.MustCompile(`^\.?[A-Za-z0-9_-][A-Za-z0-9_.-]*$`)
+
+// validateHostPattern checks that an entry of the named key, allowlist_hosts
+// or referer_blocklist, is an IP address or a host name, the host name
+// optionally with one leading dot for suffix matching. Anything else, such as
+// a URL, a port or a "*." wildcard, can never match a host name that resolves,
+// so it is refused.
+// So is "." alone: the allowlist matcher would match it against any host
+// written with a trailing dot, which in allowlist_hosts disables URL signing.
+func validateHostPattern(key, host string) error {
+	_, err := netip.ParseAddr(host)
+	if err == nil || hostNamePattern.MatchString(host) {
+		return nil
 	}
 
-	if strings.Trim(host, ".") == "" {
-		return fmt.Errorf("%s: entry %q %w",
-			settingName(keyAllowlistHosts), host, errNoHostnameLabels)
-	}
-
-	return nil
+	return fmt.Errorf("%s: entry %q %w", settingName(key), host, errNotAHost)
 }
 
 // loadConfigFile loads configuration from the PIXA_CONFIG_PATH env var
@@ -883,6 +889,19 @@ func (l *strictLoader) boolVal(key string, defaultVal bool) bool {
 	return val
 }
 
+func (l *strictLoader) hostListVal(key string) []string {
+	if l.err != nil {
+		return nil
+	}
+
+	val, err := parseHostList(l.sc, key)
+	if err != nil {
+		l.err = err
+	}
+
+	return val
+}
+
 // getString returns the string value for key, or defaultVal if the key
 // is omitted. A present value that is not a string, or is explicitly
 // null, is an error.
@@ -1180,7 +1199,7 @@ func parseCIDRList(sc *smartconfig.Config, key string) ([]netip.Prefix, error) {
 		return nil, errNullConfigValue(key)
 	}
 
-	entries, err := cidrListEntries(raw, key)
+	entries, err := listEntries(raw, key)
 	if err != nil {
 		return nil, err
 	}
@@ -1200,11 +1219,41 @@ func parseCIDRList(sc *smartconfig.Config, key string) ([]netip.Prefix, error) {
 	return prefixes, nil
 }
 
-// cidrListEntries extracts the raw entries of the named CIDR-list key as
-// trimmed, non-empty strings, from either a YAML list of strings or a
-// comma-separated string; an empty string is an empty list, as for
-// allowlist_hosts. Any other shape is a configuration error.
-func cidrListEntries(raw any, key string) ([]string, error) {
+// parseHostList parses the value of the named config key into host patterns,
+// or returns nil if the key is omitted. It accepts a YAML list of strings or a
+// comma-separated string. An explicitly null value, a wrong type, an empty
+// entry, a non-string entry, or an entry validateHostPattern rejects aborts
+// startup naming the key and the offending value.
+func parseHostList(sc *smartconfig.Config, key string) ([]string, error) {
+	raw, ok := lookupValue(sc, key)
+	if !ok {
+		return nil, nil
+	}
+
+	if raw == nil {
+		return nil, errNullConfigValue(key)
+	}
+
+	entries, err := listEntries(raw, key)
+	if err != nil {
+		return nil, err
+	}
+
+	for _, entry := range entries {
+		err := validateHostPattern(key, entry)
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	return entries, nil
+}
+
+// listEntries extracts the raw entries of the named list key as trimmed,
+// non-empty strings, from either a YAML list of strings or a comma-separated
+// string; an empty string is an empty list, as for allowlist_hosts. Any other
+// shape is a configuration error.
+func listEntries(raw any, key string) ([]string, error) {
 	switch val := raw.(type) {
 	case []any:
 		entries := make([]string, 0, len(val))

internal/config/config_validation_internal_test.go

@@ -5,6 +5,7 @@ import (
 	"log/slog"
 	"os"
 	"path/filepath"
+	"slices"
 	"strings"
 	"testing"
 	"time"
@@ -216,6 +217,25 @@ func TestCommaSeparatedAllowlistStillSupported(t *testing.T) {
 	}
 }
 
+// TestAllowlistHostsAcceptsUnderscore checks that an upstream host name with
+// an underscore, which pixa can fetch from, is accepted as an entry.
+func TestAllowlistHostsAcceptsUnderscore(t *testing.T) {
+	t.Parallel()
+
+	c, err := configFromYAML(t, signingKeyLine+`allowlist_hosts:
+  - my_bucket.example.com
+  - .my_bucket.example.org
+`)
+	if err != nil {
+		t.Fatalf("host names with an underscore should load, got error: %v", err)
+	}
+
+	want := []string{"my_bucket.example.com", ".my_bucket.example.org"}
+	if !slices.Equal(c.AllowlistHosts, want) {
+		t.Errorf("AllowlistHosts = %v, want %v", c.AllowlistHosts, want)
+	}
+}
+
 // runAbortCases asserts that each case's config aborts startup with an
 // error message mentioning every expected substring.
 func runAbortCases(t *testing.T, cases []abortCase) {
@@ -318,6 +338,20 @@ func invalidHostAndCredentialCases() []abortCase {
 				keyAllowlistHosts, "example.com/images",
 			},
 		},
+		{
+			name: "allowlist host with wildcard",
+			yaml: signingKeyLine + "allowlist_hosts:\n  - \"*.example.com\"\n",
+			wantErrSubstrings: []string{
+				keyAllowlistHosts, "*.example.com",
+			},
+		},
+		{
+			name: "allowlist host with port",
+			yaml: signingKeyLine + "allowlist_hosts:\n  - example.com:8443\n",
+			wantErrSubstrings: []string{
+				keyAllowlistHosts, "example.com:8443",
+			},
+		},
 		{
 			name:              "allowlist host with whitespace",
 			yaml:              signingKeyLine + "allowlist_hosts:\n  - \"exa mple.com\"\n",

internal/config/env_internal_test.go

@@ -65,6 +65,7 @@ func TestEnvironmentSetsEveryKey(t *testing.T) {
 	t.Setenv("PIXA_METRICS_PASSWORD", "metricspass")
 	t.Setenv("PIXA_SIGNING_KEY", validTestSigningKey)
 	t.Setenv("PIXA_ALLOWLIST_HOSTS", "s3.sneak.cloud,.example.com")
+	t.Setenv("PIXA_REFERER_BLOCKLIST", "hotlinker.example,.leech.example")
 	t.Setenv("PIXA_ALLOW_HTTP", "true")
 	t.Setenv("PIXA_UPSTREAM_CONNECTIONS_PER_HOST", "5")
 	t.Setenv("PIXA_UPSTREAM_CONNECTIONS", "10")
@@ -93,6 +94,7 @@ func TestEnvironmentSetsEveryKey(t *testing.T) {
 		MetricsPassword:            "metricspass",
 		SigningKey:                 validTestSigningKey,
 		AllowlistHosts:             []string{testHostS3, ".example.com"},
+		RefererBlocklist:           []string{"hotlinker.example", ".leech.example"},
 		AllowHTTP:                  true,
 		UpstreamConnectionsPerHost: 5,
 		UpstreamConnections:        10,

internal/config/referer_blocklist_internal_test.go

@@ -0,0 +1,166 @@
+package config
+
+import (
+	"slices"
+	"testing"
+)
+
+// TestRefererBlocklistParsed loads a referer_blocklist with a host and a
+// pattern starting with "." and checks both are kept in order.
+func TestRefererBlocklistParsed(t *testing.T) {
+	t.Parallel()
+
+	c, err := configFromYAML(t, signingKeyLine+`referer_blocklist:
+  - leech.example
+  - .hotlinker.example
+`)
+	if err != nil {
+		t.Fatalf("valid referer_blocklist should load, got error: %v", err)
+	}
+
+	want := []string{"leech.example", ".hotlinker.example"}
+	if !slices.Equal(c.RefererBlocklist, want) {
+		t.Errorf("RefererBlocklist = %v, want %v", c.RefererBlocklist, want)
+	}
+}
+
+// TestRefererBlocklistAcceptsIPAddresses checks that IPv4 and IPv6 addresses,
+// the IPv6 one written without brackets, are accepted as entries.
+func TestRefererBlocklistAcceptsIPAddresses(t *testing.T) {
+	t.Parallel()
+
+	c, err := configFromYAML(t, signingKeyLine+`referer_blocklist:
+  - 192.0.2.7
+  - "2001:db8::7"
+`)
+	if err != nil {
+		t.Fatalf("IP address entries should load, got error: %v", err)
+	}
+
+	want := []string{"192.0.2.7", "2001:db8::7"}
+	if !slices.Equal(c.RefererBlocklist, want) {
+		t.Errorf("RefererBlocklist = %v, want %v", c.RefererBlocklist, want)
+	}
+}
+
+// TestRefererBlocklistAcceptsUnderscore checks that a host name with an
+// underscore, which a page can be served from, is accepted as an entry.
+func TestRefererBlocklistAcceptsUnderscore(t *testing.T) {
+	t.Parallel()
+
+	c, err := configFromYAML(t, signingKeyLine+`referer_blocklist:
+  - my_site.leech.example
+  - .my_site.hotlinker.example
+`)
+	if err != nil {
+		t.Fatalf("host names with an underscore should load, got error: %v", err)
+	}
+
+	want := []string{"my_site.leech.example", ".my_site.hotlinker.example"}
+	if !slices.Equal(c.RefererBlocklist, want) {
+		t.Errorf("RefererBlocklist = %v, want %v", c.RefererBlocklist, want)
+	}
+}
+
+// TestRefererBlocklistOmittedIsEmpty checks that an omitted key blocks no
+// referer.
+func TestRefererBlocklistOmittedIsEmpty(t *testing.T) {
+	t.Parallel()
+
+	c, err := configFromYAML(t, signingKeyLine)
+	if err != nil {
+		t.Fatalf("minimal config should be valid, got error: %v", err)
+	}
+
+	if len(c.RefererBlocklist) != 0 {
+		t.Errorf("RefererBlocklist = %v, want empty", c.RefererBlocklist)
+	}
+}
+
+// TestRefererBlocklistInvalidAbortsStartup checks that an entry that is not a
+// host, or a value that is not a list of them, aborts startup with an error
+// naming the key and the entry.
+func TestRefererBlocklistInvalidAbortsStartup(t *testing.T) {
+	t.Parallel()
+
+	runAbortCases(t, []abortCase{
+		{
+			name: "entry with a scheme",
+			yaml: signingKeyLine + "referer_blocklist:\n  - https://leech.example\n",
+			wantErrSubstrings: []string{
+				keyRefererBlocklist, "https://leech.example",
+			},
+		},
+		{
+			name: "entry with a path",
+			yaml: signingKeyLine + "referer_blocklist:\n  - leech.example/page\n",
+			wantErrSubstrings: []string{
+				keyRefererBlocklist, "leech.example/page",
+			},
+		},
+		{
+			name: "wildcard entry",
+			yaml: signingKeyLine + "referer_blocklist:\n  - \"*.leech.example\"\n",
+			wantErrSubstrings: []string{
+				keyRefererBlocklist, "*.leech.example",
+			},
+		},
+		{
+			name: "entry with a port",
+			yaml: signingKeyLine + "referer_blocklist:\n  - leech.example:8080\n",
+			wantErrSubstrings: []string{
+				keyRefererBlocklist, "leech.example:8080",
+			},
+		},
+		{
+			name: "two leading dots",
+			yaml: signingKeyLine + "referer_blocklist:\n  - ..leech.example\n",
+			wantErrSubstrings: []string{
+				keyRefererBlocklist, "..leech.example",
+			},
+		},
+		{
+			name:              "dot only",
+			yaml:              signingKeyLine + "referer_blocklist:\n  - \".\"\n",
+			wantErrSubstrings: []string{keyRefererBlocklist, `"."`},
+		},
+		{
+			name:              "empty entry",
+			yaml:              signingKeyLine + "referer_blocklist:\n  - \"\"\n",
+			wantErrSubstrings: []string{keyRefererBlocklist},
+		},
+		{
+			name:              "entry not a string",
+			yaml:              signingKeyLine + "referer_blocklist:\n  - 42\n",
+			wantErrSubstrings: []string{keyRefererBlocklist, "42"},
+		},
+		{
+			name:              "null value",
+			yaml:              signingKeyLine + "referer_blocklist:\n",
+			wantErrSubstrings: []string{keyRefererBlocklist, nullValueText},
+		},
+	})
+}
+
+// TestRefererBlocklistFromEnvironment checks that PIXA_REFERER_BLOCKLIST
+// takes comma-separated entries, and that an entry in it that is not a host
+// aborts startup naming the variable and the entry.
+func TestRefererBlocklistFromEnvironment(t *testing.T) {
+	t.Setenv("PIXA_SIGNING_KEY", validTestSigningKey)
+	t.Setenv("PIXA_REFERER_BLOCKLIST", " leech.example , .hotlinker.example ")
+
+	c, err := newFromSmartConfig(nil)
+	if err != nil {
+		t.Fatalf("valid PIXA_REFERER_BLOCKLIST should load, got error: %v", err)
+	}
+
+	want := []string{"leech.example", ".hotlinker.example"}
+	if !slices.Equal(c.RefererBlocklist, want) {
+		t.Errorf("RefererBlocklist = %v, want %v", c.RefererBlocklist, want)
+	}
+
+	t.Setenv("PIXA_REFERER_BLOCKLIST", "leech.example,https://hotlinker.example")
+
+	_, err = newFromSmartConfig(nil)
+	wantStartupError(t, err, "PIXA_REFERER_BLOCKLIST", "https://hotlinker.example")
+}