Refuse image requests whose Referer is on referer_blocklist (closes #90)

A new setting, referer_blocklist (PIXA_REFERER_BLOCKLIST), lists hosts
whose pages may not show pixa's images. Entries are written and matched
as allowlist_hosts are, with the same matcher. Both image routes check
the Referer before the signature, the cache and the upstream fetch, and
answer 403 with the JSON error, so a blocked request costs nothing and
is refused whether or not the image is cached. No Referer, or one that
does not parse, is served; README.md says this makes the list easy to
get around. An entry of either host list that is not a host name
(letters, digits, hyphens, underscores, dots, at most one leading dot)
or an IP address now aborts startup naming the setting and the entry.

Model: opus-5-5

internal/config/config.go

@@ -11,6 +11,7 @@ import (
 	"net/url"
 	"os"
 	"path/filepath"
+	"regexp"
 	"runtime"
 	"sort"
 	"strconv"
@@ -48,6 +49,7 @@ const (
 	keyMetricsPassword            = "metrics.password"
 	keySigningKey                 = "signing_key"
 	keyAllowlistHosts             = "allowlist_hosts"
+	keyRefererBlocklist           = "referer_blocklist"
 	keyAllowHTTP                  = "allow_http"
 	keyUpstreamConnectionsPerHost = "upstream_connections_per_host"
 	keyUpstreamConnections        = "upstream_connections"
@@ -97,12 +99,11 @@ var (
 		"value is null; omit the key entirely to use the default")
 	errValuesNull = errors.New(
 		"value is null; omit a key entirely to use its default")
-	errNotBareHostname = errors.New(
-		"must be a bare hostname without scheme, path, or whitespace")
-	errNoHostnameLabels = errors.New("contains no hostname labels")
-	errNotADuration     = errors.New("not a duration such as 30s or 2m")
-	errMustBePositive   = errors.New("must be positive")
-	errNotAnOrigin      = errors.New(
+	errNotAHost = errors.New("must be a host name such as " +
+		"cdn.example.com or .example.com, or an IP address")
+	errNotADuration   = errors.New("not a duration such as 30s or 2m")
+	errMustBePositive = errors.New("must be positive")
+	errNotAnOrigin    = errors.New(
 		`not "*" or an origin such as https://example.com`)
 )
 
@@ -130,6 +131,10 @@ type Config struct {
 	AllowHTTP                  bool     // Allow non-TLS upstream (testing only)
 	UpstreamConnectionsPerHost int      // Max concurrent connections per upstream host
 
+	// RefererBlocklist holds host patterns, matched as AllowlistHosts is: the
+	// image routes refuse a request whose Referer names a matching host.
+	RefererBlocklist []string
+
 	// UpstreamConnections is the most concurrent connections to all
 	// upstream hosts together, on top of the per-host limit.
 	// MaxConcurrentProcessing is the most images processed at once.
@@ -270,7 +275,6 @@ func newFromSmartConfig(sc *smartconfig.Config) (*Config, error) {
 	}
 
 	loader := &strictLoader{sc: sc}
-
 	c := &Config{
 		Debug:           loader.boolVal(keyDebug, false),
 		MaintenanceMode: loader.boolVal(keyMaintenanceMode, false),
@@ -298,9 +302,10 @@ func newFromSmartConfig(sc *smartconfig.Config) (*Config, error) {
 			keyAccessControlAllowOrigin, DefaultAccessControlAllowOrigin),
 		DownstreamTimeout: loader.durationVal(
 			keyDownstreamTimeout, DefaultDownstreamTimeout),
-		CacheMaxBytes:   loader.int64Val(keyCacheMaxBytes, 0),
-		BlockedNetworks: blockedNetworks,
-		TrustedProxies:  trustedProxies,
+		CacheMaxBytes:    loader.int64Val(keyCacheMaxBytes, 0),
+		BlockedNetworks:  blockedNetworks,
+		TrustedProxies:   trustedProxies,
+		RefererBlocklist: loader.hostListVal(keyRefererBlocklist),
 	}
 
 	// The default for an omitted cache_max_bytes is worked out when
@@ -420,7 +425,8 @@ func isKnownConfigKey(key string) bool {
 		keyUpstreamConnectionsPerHost, keyUpstreamConnections,
 		keyMaxConcurrentProcessing, keyCacheMaxBytes, keyBlockedNetworks,
 		keyTrustedProxies, keyAccessControlAllowOrigin, keyUpstreamFetchTimeout,
-		keyUpstreamMaxResponseSize, keyDownstreamTimeout, "env":
+		keyUpstreamMaxResponseSize, keyDownstreamTimeout, keyRefererBlocklist,
+		"env":
 		return true
 	}
 
@@ -443,6 +449,7 @@ func envVarNames() map[string]string {
 		keyMetricsPassword:            "PIXA_METRICS_PASSWORD",
 		keySigningKey:                 "PIXA_SIGNING_KEY",
 		keyAllowlistHosts:             "PIXA_ALLOWLIST_HOSTS",
+		keyRefererBlocklist:           "PIXA_REFERER_BLOCKLIST",
 		keyAllowHTTP:                  "PIXA_ALLOW_HTTP",
 		keyUpstreamConnectionsPerHost: "PIXA_UPSTREAM_CONNECTIONS_PER_HOST",
 		keyUpstreamConnections:        "PIXA_UPSTREAM_CONNECTIONS",
@@ -612,7 +619,7 @@ func (c *Config) validate() error {
 	}
 
 	for _, host := range c.AllowlistHosts {
-		err := validateAllowlistHost(host)
+		err := validateHostPattern(keyAllowlistHosts, host)
 		if err != nil {
 			return err
 		}
@@ -735,25 +742,24 @@ func (c *Config) validateConcurrencyLimits() error {
 	return nil
 }
 
-// validateAllowlistHost checks that an allowlist_hosts entry is a bare
-// hostname, optionally with a leading dot for suffix matching. URLs,
-// paths, and whitespace indicate a misconfigured entry. An entry with
-// no hostname labels (such as ".") is rejected: the allowlist matcher
-// treats a leading dot as a suffix pattern, so a bare "." would match
-// any upstream host written in FQDN trailing-dot form and effectively
-// disable URL signing.
-func validateAllowlistHost(host string) error {
-	if strings.Contains(host, "://") || strings.ContainsAny(host, "/ \t") {
-		return fmt.Errorf("%s: entry %q %w",
-			settingName(keyAllowlistHosts), host, errNotBareHostname)
+// hostNamePattern matches a host name: letters, digits, hyphens, underscores
+// and dots, optionally after one leading dot.
+var hostNamePattern = regexp.MustCompile(`^\.?[A-Za-z0-9_-][A-Za-z0-9_.-]*$`)
+
+// validateHostPattern checks that an entry of the named key, allowlist_hosts
+// or referer_blocklist, is an IP address or a host name, the host name
+// optionally with one leading dot for suffix matching. Anything else, such as
+// a URL, a port or a "*." wildcard, can never match a host name that resolves,
+// so it is refused.
+// So is "." alone: the allowlist matcher would match it against any host
+// written with a trailing dot, which in allowlist_hosts disables URL signing.
+func validateHostPattern(key, host string) error {
+	_, err := netip.ParseAddr(host)
+	if err == nil || hostNamePattern.MatchString(host) {
+		return nil
 	}
 
-	if strings.Trim(host, ".") == "" {
-		return fmt.Errorf("%s: entry %q %w",
-			settingName(keyAllowlistHosts), host, errNoHostnameLabels)
-	}
-
-	return nil
+	return fmt.Errorf("%s: entry %q %w", settingName(key), host, errNotAHost)
 }
 
 // loadConfigFile loads configuration from the PIXA_CONFIG_PATH env var
@@ -883,6 +889,19 @@ func (l *strictLoader) boolVal(key string, defaultVal bool) bool {
 	return val
 }
 
+func (l *strictLoader) hostListVal(key string) []string {
+	if l.err != nil {
+		return nil
+	}
+
+	val, err := parseHostList(l.sc, key)
+	if err != nil {
+		l.err = err
+	}
+
+	return val
+}
+
 // getString returns the string value for key, or defaultVal if the key
 // is omitted. A present value that is not a string, or is explicitly
 // null, is an error.
@@ -1180,7 +1199,7 @@ func parseCIDRList(sc *smartconfig.Config, key string) ([]netip.Prefix, error) {
 		return nil, errNullConfigValue(key)
 	}
 
-	entries, err := cidrListEntries(raw, key)
+	entries, err := listEntries(raw, key)
 	if err != nil {
 		return nil, err
 	}
@@ -1200,11 +1219,41 @@ func parseCIDRList(sc *smartconfig.Config, key string) ([]netip.Prefix, error) {
 	return prefixes, nil
 }
 
-// cidrListEntries extracts the raw entries of the named CIDR-list key as
-// trimmed, non-empty strings, from either a YAML list of strings or a
-// comma-separated string; an empty string is an empty list, as for
-// allowlist_hosts. Any other shape is a configuration error.
-func cidrListEntries(raw any, key string) ([]string, error) {
+// parseHostList parses the value of the named config key into host patterns,
+// or returns nil if the key is omitted. It accepts a YAML list of strings or a
+// comma-separated string. An explicitly null value, a wrong type, an empty
+// entry, a non-string entry, or an entry validateHostPattern rejects aborts
+// startup naming the key and the offending value.
+func parseHostList(sc *smartconfig.Config, key string) ([]string, error) {
+	raw, ok := lookupValue(sc, key)
+	if !ok {
+		return nil, nil
+	}
+
+	if raw == nil {
+		return nil, errNullConfigValue(key)
+	}
+
+	entries, err := listEntries(raw, key)
+	if err != nil {
+		return nil, err
+	}
+
+	for _, entry := range entries {
+		err := validateHostPattern(key, entry)
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	return entries, nil
+}
+
+// listEntries extracts the raw entries of the named list key as trimmed,
+// non-empty strings, from either a YAML list of strings or a comma-separated
+// string; an empty string is an empty list, as for allowlist_hosts. Any other
+// shape is a configuration error.
+func listEntries(raw any, key string) ([]string, error) {
 	switch val := raw.(type) {
 	case []any:
 		entries := make([]string, 0, len(val))