Refuse image requests whose Referer is on referer_blocklist (closes #90)
check / check (push) Failing after 2s
check / check (push) Failing after 2s
A new setting, referer_blocklist (PIXA_REFERER_BLOCKLIST), lists hosts whose pages may not show pixa's images. Entries are written and matched as allowlist_hosts are, with the same matcher. Both image routes check the Referer before the signature, the cache and the upstream fetch, and answer 403 with the JSON error, so a blocked request costs nothing and is refused whether or not the image is cached. No Referer, or one that does not parse, is served; README.md says this makes the list easy to get around. An entry of either host list that is not a host name (letters, digits, hyphens, underscores, dots, at most one leading dot) or an IP address now aborts startup naming the setting and the entry. Model: opus-5-5
This commit was merged in pull request #197.
This commit is contained in:
+84
-35
@@ -11,6 +11,7 @@ import (
|
||||
"net/url"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"runtime"
|
||||
"sort"
|
||||
"strconv"
|
||||
@@ -48,6 +49,7 @@ const (
|
||||
keyMetricsPassword = "metrics.password"
|
||||
keySigningKey = "signing_key"
|
||||
keyAllowlistHosts = "allowlist_hosts"
|
||||
keyRefererBlocklist = "referer_blocklist"
|
||||
keyAllowHTTP = "allow_http"
|
||||
keyUpstreamConnectionsPerHost = "upstream_connections_per_host"
|
||||
keyUpstreamConnections = "upstream_connections"
|
||||
@@ -97,12 +99,11 @@ var (
|
||||
"value is null; omit the key entirely to use the default")
|
||||
errValuesNull = errors.New(
|
||||
"value is null; omit a key entirely to use its default")
|
||||
errNotBareHostname = errors.New(
|
||||
"must be a bare hostname without scheme, path, or whitespace")
|
||||
errNoHostnameLabels = errors.New("contains no hostname labels")
|
||||
errNotADuration = errors.New("not a duration such as 30s or 2m")
|
||||
errMustBePositive = errors.New("must be positive")
|
||||
errNotAnOrigin = errors.New(
|
||||
errNotAHost = errors.New("must be a host name such as " +
|
||||
"cdn.example.com or .example.com, or an IP address")
|
||||
errNotADuration = errors.New("not a duration such as 30s or 2m")
|
||||
errMustBePositive = errors.New("must be positive")
|
||||
errNotAnOrigin = errors.New(
|
||||
`not "*" or an origin such as https://example.com`)
|
||||
)
|
||||
|
||||
@@ -130,6 +131,10 @@ type Config struct {
|
||||
AllowHTTP bool // Allow non-TLS upstream (testing only)
|
||||
UpstreamConnectionsPerHost int // Max concurrent connections per upstream host
|
||||
|
||||
// RefererBlocklist holds host patterns, matched as AllowlistHosts is: the
|
||||
// image routes refuse a request whose Referer names a matching host.
|
||||
RefererBlocklist []string
|
||||
|
||||
// UpstreamConnections is the most concurrent connections to all
|
||||
// upstream hosts together, on top of the per-host limit.
|
||||
// MaxConcurrentProcessing is the most images processed at once.
|
||||
@@ -270,7 +275,6 @@ func newFromSmartConfig(sc *smartconfig.Config) (*Config, error) {
|
||||
}
|
||||
|
||||
loader := &strictLoader{sc: sc}
|
||||
|
||||
c := &Config{
|
||||
Debug: loader.boolVal(keyDebug, false),
|
||||
MaintenanceMode: loader.boolVal(keyMaintenanceMode, false),
|
||||
@@ -298,9 +302,10 @@ func newFromSmartConfig(sc *smartconfig.Config) (*Config, error) {
|
||||
keyAccessControlAllowOrigin, DefaultAccessControlAllowOrigin),
|
||||
DownstreamTimeout: loader.durationVal(
|
||||
keyDownstreamTimeout, DefaultDownstreamTimeout),
|
||||
CacheMaxBytes: loader.int64Val(keyCacheMaxBytes, 0),
|
||||
BlockedNetworks: blockedNetworks,
|
||||
TrustedProxies: trustedProxies,
|
||||
CacheMaxBytes: loader.int64Val(keyCacheMaxBytes, 0),
|
||||
BlockedNetworks: blockedNetworks,
|
||||
TrustedProxies: trustedProxies,
|
||||
RefererBlocklist: loader.hostListVal(keyRefererBlocklist),
|
||||
}
|
||||
|
||||
// The default for an omitted cache_max_bytes is worked out when
|
||||
@@ -420,7 +425,8 @@ func isKnownConfigKey(key string) bool {
|
||||
keyUpstreamConnectionsPerHost, keyUpstreamConnections,
|
||||
keyMaxConcurrentProcessing, keyCacheMaxBytes, keyBlockedNetworks,
|
||||
keyTrustedProxies, keyAccessControlAllowOrigin, keyUpstreamFetchTimeout,
|
||||
keyUpstreamMaxResponseSize, keyDownstreamTimeout, "env":
|
||||
keyUpstreamMaxResponseSize, keyDownstreamTimeout, keyRefererBlocklist,
|
||||
"env":
|
||||
return true
|
||||
}
|
||||
|
||||
@@ -443,6 +449,7 @@ func envVarNames() map[string]string {
|
||||
keyMetricsPassword: "PIXA_METRICS_PASSWORD",
|
||||
keySigningKey: "PIXA_SIGNING_KEY",
|
||||
keyAllowlistHosts: "PIXA_ALLOWLIST_HOSTS",
|
||||
keyRefererBlocklist: "PIXA_REFERER_BLOCKLIST",
|
||||
keyAllowHTTP: "PIXA_ALLOW_HTTP",
|
||||
keyUpstreamConnectionsPerHost: "PIXA_UPSTREAM_CONNECTIONS_PER_HOST",
|
||||
keyUpstreamConnections: "PIXA_UPSTREAM_CONNECTIONS",
|
||||
@@ -612,7 +619,7 @@ func (c *Config) validate() error {
|
||||
}
|
||||
|
||||
for _, host := range c.AllowlistHosts {
|
||||
err := validateAllowlistHost(host)
|
||||
err := validateHostPattern(keyAllowlistHosts, host)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -735,25 +742,24 @@ func (c *Config) validateConcurrencyLimits() error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// validateAllowlistHost checks that an allowlist_hosts entry is a bare
|
||||
// hostname, optionally with a leading dot for suffix matching. URLs,
|
||||
// paths, and whitespace indicate a misconfigured entry. An entry with
|
||||
// no hostname labels (such as ".") is rejected: the allowlist matcher
|
||||
// treats a leading dot as a suffix pattern, so a bare "." would match
|
||||
// any upstream host written in FQDN trailing-dot form and effectively
|
||||
// disable URL signing.
|
||||
func validateAllowlistHost(host string) error {
|
||||
if strings.Contains(host, "://") || strings.ContainsAny(host, "/ \t") {
|
||||
return fmt.Errorf("%s: entry %q %w",
|
||||
settingName(keyAllowlistHosts), host, errNotBareHostname)
|
||||
// hostNamePattern matches a host name: letters, digits, hyphens, underscores
|
||||
// and dots, optionally after one leading dot.
|
||||
var hostNamePattern = regexp.MustCompile(`^\.?[A-Za-z0-9_-][A-Za-z0-9_.-]*$`)
|
||||
|
||||
// validateHostPattern checks that an entry of the named key, allowlist_hosts
|
||||
// or referer_blocklist, is an IP address or a host name, the host name
|
||||
// optionally with one leading dot for suffix matching. Anything else, such as
|
||||
// a URL, a port or a "*." wildcard, can never match a host name that resolves,
|
||||
// so it is refused.
|
||||
// So is "." alone: the allowlist matcher would match it against any host
|
||||
// written with a trailing dot, which in allowlist_hosts disables URL signing.
|
||||
func validateHostPattern(key, host string) error {
|
||||
_, err := netip.ParseAddr(host)
|
||||
if err == nil || hostNamePattern.MatchString(host) {
|
||||
return nil
|
||||
}
|
||||
|
||||
if strings.Trim(host, ".") == "" {
|
||||
return fmt.Errorf("%s: entry %q %w",
|
||||
settingName(keyAllowlistHosts), host, errNoHostnameLabels)
|
||||
}
|
||||
|
||||
return nil
|
||||
return fmt.Errorf("%s: entry %q %w", settingName(key), host, errNotAHost)
|
||||
}
|
||||
|
||||
// loadConfigFile loads configuration from the PIXA_CONFIG_PATH env var
|
||||
@@ -883,6 +889,19 @@ func (l *strictLoader) boolVal(key string, defaultVal bool) bool {
|
||||
return val
|
||||
}
|
||||
|
||||
func (l *strictLoader) hostListVal(key string) []string {
|
||||
if l.err != nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
val, err := parseHostList(l.sc, key)
|
||||
if err != nil {
|
||||
l.err = err
|
||||
}
|
||||
|
||||
return val
|
||||
}
|
||||
|
||||
// getString returns the string value for key, or defaultVal if the key
|
||||
// is omitted. A present value that is not a string, or is explicitly
|
||||
// null, is an error.
|
||||
@@ -1180,7 +1199,7 @@ func parseCIDRList(sc *smartconfig.Config, key string) ([]netip.Prefix, error) {
|
||||
return nil, errNullConfigValue(key)
|
||||
}
|
||||
|
||||
entries, err := cidrListEntries(raw, key)
|
||||
entries, err := listEntries(raw, key)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -1200,11 +1219,41 @@ func parseCIDRList(sc *smartconfig.Config, key string) ([]netip.Prefix, error) {
|
||||
return prefixes, nil
|
||||
}
|
||||
|
||||
// cidrListEntries extracts the raw entries of the named CIDR-list key as
|
||||
// trimmed, non-empty strings, from either a YAML list of strings or a
|
||||
// comma-separated string; an empty string is an empty list, as for
|
||||
// allowlist_hosts. Any other shape is a configuration error.
|
||||
func cidrListEntries(raw any, key string) ([]string, error) {
|
||||
// parseHostList parses the value of the named config key into host patterns,
|
||||
// or returns nil if the key is omitted. It accepts a YAML list of strings or a
|
||||
// comma-separated string. An explicitly null value, a wrong type, an empty
|
||||
// entry, a non-string entry, or an entry validateHostPattern rejects aborts
|
||||
// startup naming the key and the offending value.
|
||||
func parseHostList(sc *smartconfig.Config, key string) ([]string, error) {
|
||||
raw, ok := lookupValue(sc, key)
|
||||
if !ok {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
if raw == nil {
|
||||
return nil, errNullConfigValue(key)
|
||||
}
|
||||
|
||||
entries, err := listEntries(raw, key)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
for _, entry := range entries {
|
||||
err := validateHostPattern(key, entry)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
return entries, nil
|
||||
}
|
||||
|
||||
// listEntries extracts the raw entries of the named list key as trimmed,
|
||||
// non-empty strings, from either a YAML list of strings or a comma-separated
|
||||
// string; an empty string is an empty list, as for allowlist_hosts. Any other
|
||||
// shape is a configuration error.
|
||||
func listEntries(raw any, key string) ([]string, error) {
|
||||
switch val := raw.(type) {
|
||||
case []any:
|
||||
entries := make([]string, 0, len(val))
|
||||
|
||||
@@ -5,6 +5,7 @@ import (
|
||||
"log/slog"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"slices"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
@@ -216,6 +217,25 @@ func TestCommaSeparatedAllowlistStillSupported(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// TestAllowlistHostsAcceptsUnderscore checks that an upstream host name with
|
||||
// an underscore, which pixa can fetch from, is accepted as an entry.
|
||||
func TestAllowlistHostsAcceptsUnderscore(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
c, err := configFromYAML(t, signingKeyLine+`allowlist_hosts:
|
||||
- my_bucket.example.com
|
||||
- .my_bucket.example.org
|
||||
`)
|
||||
if err != nil {
|
||||
t.Fatalf("host names with an underscore should load, got error: %v", err)
|
||||
}
|
||||
|
||||
want := []string{"my_bucket.example.com", ".my_bucket.example.org"}
|
||||
if !slices.Equal(c.AllowlistHosts, want) {
|
||||
t.Errorf("AllowlistHosts = %v, want %v", c.AllowlistHosts, want)
|
||||
}
|
||||
}
|
||||
|
||||
// runAbortCases asserts that each case's config aborts startup with an
|
||||
// error message mentioning every expected substring.
|
||||
func runAbortCases(t *testing.T, cases []abortCase) {
|
||||
@@ -318,6 +338,20 @@ func invalidHostAndCredentialCases() []abortCase {
|
||||
keyAllowlistHosts, "example.com/images",
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "allowlist host with wildcard",
|
||||
yaml: signingKeyLine + "allowlist_hosts:\n - \"*.example.com\"\n",
|
||||
wantErrSubstrings: []string{
|
||||
keyAllowlistHosts, "*.example.com",
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "allowlist host with port",
|
||||
yaml: signingKeyLine + "allowlist_hosts:\n - example.com:8443\n",
|
||||
wantErrSubstrings: []string{
|
||||
keyAllowlistHosts, "example.com:8443",
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "allowlist host with whitespace",
|
||||
yaml: signingKeyLine + "allowlist_hosts:\n - \"exa mple.com\"\n",
|
||||
|
||||
@@ -65,6 +65,7 @@ func TestEnvironmentSetsEveryKey(t *testing.T) {
|
||||
t.Setenv("PIXA_METRICS_PASSWORD", "metricspass")
|
||||
t.Setenv("PIXA_SIGNING_KEY", validTestSigningKey)
|
||||
t.Setenv("PIXA_ALLOWLIST_HOSTS", "s3.sneak.cloud,.example.com")
|
||||
t.Setenv("PIXA_REFERER_BLOCKLIST", "hotlinker.example,.leech.example")
|
||||
t.Setenv("PIXA_ALLOW_HTTP", "true")
|
||||
t.Setenv("PIXA_UPSTREAM_CONNECTIONS_PER_HOST", "5")
|
||||
t.Setenv("PIXA_UPSTREAM_CONNECTIONS", "10")
|
||||
@@ -93,6 +94,7 @@ func TestEnvironmentSetsEveryKey(t *testing.T) {
|
||||
MetricsPassword: "metricspass",
|
||||
SigningKey: validTestSigningKey,
|
||||
AllowlistHosts: []string{testHostS3, ".example.com"},
|
||||
RefererBlocklist: []string{"hotlinker.example", ".leech.example"},
|
||||
AllowHTTP: true,
|
||||
UpstreamConnectionsPerHost: 5,
|
||||
UpstreamConnections: 10,
|
||||
|
||||
@@ -0,0 +1,166 @@
|
||||
package config
|
||||
|
||||
import (
|
||||
"slices"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// TestRefererBlocklistParsed loads a referer_blocklist with a host and a
|
||||
// pattern starting with "." and checks both are kept in order.
|
||||
func TestRefererBlocklistParsed(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
c, err := configFromYAML(t, signingKeyLine+`referer_blocklist:
|
||||
- leech.example
|
||||
- .hotlinker.example
|
||||
`)
|
||||
if err != nil {
|
||||
t.Fatalf("valid referer_blocklist should load, got error: %v", err)
|
||||
}
|
||||
|
||||
want := []string{"leech.example", ".hotlinker.example"}
|
||||
if !slices.Equal(c.RefererBlocklist, want) {
|
||||
t.Errorf("RefererBlocklist = %v, want %v", c.RefererBlocklist, want)
|
||||
}
|
||||
}
|
||||
|
||||
// TestRefererBlocklistAcceptsIPAddresses checks that IPv4 and IPv6 addresses,
|
||||
// the IPv6 one written without brackets, are accepted as entries.
|
||||
func TestRefererBlocklistAcceptsIPAddresses(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
c, err := configFromYAML(t, signingKeyLine+`referer_blocklist:
|
||||
- 192.0.2.7
|
||||
- "2001:db8::7"
|
||||
`)
|
||||
if err != nil {
|
||||
t.Fatalf("IP address entries should load, got error: %v", err)
|
||||
}
|
||||
|
||||
want := []string{"192.0.2.7", "2001:db8::7"}
|
||||
if !slices.Equal(c.RefererBlocklist, want) {
|
||||
t.Errorf("RefererBlocklist = %v, want %v", c.RefererBlocklist, want)
|
||||
}
|
||||
}
|
||||
|
||||
// TestRefererBlocklistAcceptsUnderscore checks that a host name with an
|
||||
// underscore, which a page can be served from, is accepted as an entry.
|
||||
func TestRefererBlocklistAcceptsUnderscore(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
c, err := configFromYAML(t, signingKeyLine+`referer_blocklist:
|
||||
- my_site.leech.example
|
||||
- .my_site.hotlinker.example
|
||||
`)
|
||||
if err != nil {
|
||||
t.Fatalf("host names with an underscore should load, got error: %v", err)
|
||||
}
|
||||
|
||||
want := []string{"my_site.leech.example", ".my_site.hotlinker.example"}
|
||||
if !slices.Equal(c.RefererBlocklist, want) {
|
||||
t.Errorf("RefererBlocklist = %v, want %v", c.RefererBlocklist, want)
|
||||
}
|
||||
}
|
||||
|
||||
// TestRefererBlocklistOmittedIsEmpty checks that an omitted key blocks no
|
||||
// referer.
|
||||
func TestRefererBlocklistOmittedIsEmpty(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
c, err := configFromYAML(t, signingKeyLine)
|
||||
if err != nil {
|
||||
t.Fatalf("minimal config should be valid, got error: %v", err)
|
||||
}
|
||||
|
||||
if len(c.RefererBlocklist) != 0 {
|
||||
t.Errorf("RefererBlocklist = %v, want empty", c.RefererBlocklist)
|
||||
}
|
||||
}
|
||||
|
||||
// TestRefererBlocklistInvalidAbortsStartup checks that an entry that is not a
|
||||
// host, or a value that is not a list of them, aborts startup with an error
|
||||
// naming the key and the entry.
|
||||
func TestRefererBlocklistInvalidAbortsStartup(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
runAbortCases(t, []abortCase{
|
||||
{
|
||||
name: "entry with a scheme",
|
||||
yaml: signingKeyLine + "referer_blocklist:\n - https://leech.example\n",
|
||||
wantErrSubstrings: []string{
|
||||
keyRefererBlocklist, "https://leech.example",
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "entry with a path",
|
||||
yaml: signingKeyLine + "referer_blocklist:\n - leech.example/page\n",
|
||||
wantErrSubstrings: []string{
|
||||
keyRefererBlocklist, "leech.example/page",
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "wildcard entry",
|
||||
yaml: signingKeyLine + "referer_blocklist:\n - \"*.leech.example\"\n",
|
||||
wantErrSubstrings: []string{
|
||||
keyRefererBlocklist, "*.leech.example",
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "entry with a port",
|
||||
yaml: signingKeyLine + "referer_blocklist:\n - leech.example:8080\n",
|
||||
wantErrSubstrings: []string{
|
||||
keyRefererBlocklist, "leech.example:8080",
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "two leading dots",
|
||||
yaml: signingKeyLine + "referer_blocklist:\n - ..leech.example\n",
|
||||
wantErrSubstrings: []string{
|
||||
keyRefererBlocklist, "..leech.example",
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "dot only",
|
||||
yaml: signingKeyLine + "referer_blocklist:\n - \".\"\n",
|
||||
wantErrSubstrings: []string{keyRefererBlocklist, `"."`},
|
||||
},
|
||||
{
|
||||
name: "empty entry",
|
||||
yaml: signingKeyLine + "referer_blocklist:\n - \"\"\n",
|
||||
wantErrSubstrings: []string{keyRefererBlocklist},
|
||||
},
|
||||
{
|
||||
name: "entry not a string",
|
||||
yaml: signingKeyLine + "referer_blocklist:\n - 42\n",
|
||||
wantErrSubstrings: []string{keyRefererBlocklist, "42"},
|
||||
},
|
||||
{
|
||||
name: "null value",
|
||||
yaml: signingKeyLine + "referer_blocklist:\n",
|
||||
wantErrSubstrings: []string{keyRefererBlocklist, nullValueText},
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
// TestRefererBlocklistFromEnvironment checks that PIXA_REFERER_BLOCKLIST
|
||||
// takes comma-separated entries, and that an entry in it that is not a host
|
||||
// aborts startup naming the variable and the entry.
|
||||
func TestRefererBlocklistFromEnvironment(t *testing.T) {
|
||||
t.Setenv("PIXA_SIGNING_KEY", validTestSigningKey)
|
||||
t.Setenv("PIXA_REFERER_BLOCKLIST", " leech.example , .hotlinker.example ")
|
||||
|
||||
c, err := newFromSmartConfig(nil)
|
||||
if err != nil {
|
||||
t.Fatalf("valid PIXA_REFERER_BLOCKLIST should load, got error: %v", err)
|
||||
}
|
||||
|
||||
want := []string{"leech.example", ".hotlinker.example"}
|
||||
if !slices.Equal(c.RefererBlocklist, want) {
|
||||
t.Errorf("RefererBlocklist = %v, want %v", c.RefererBlocklist, want)
|
||||
}
|
||||
|
||||
t.Setenv("PIXA_REFERER_BLOCKLIST", "leech.example,https://hotlinker.example")
|
||||
|
||||
_, err = newFromSmartConfig(nil)
|
||||
wantStartupError(t, err, "PIXA_REFERER_BLOCKLIST", "https://hotlinker.example")
|
||||
}
|
||||
@@ -9,6 +9,7 @@ import (
|
||||
"time"
|
||||
|
||||
"go.uber.org/fx"
|
||||
"sneak.berlin/go/pixa/internal/allowlist"
|
||||
"sneak.berlin/go/pixa/internal/config"
|
||||
"sneak.berlin/go/pixa/internal/database"
|
||||
"sneak.berlin/go/pixa/internal/encurl"
|
||||
@@ -40,6 +41,10 @@ type Handlers struct {
|
||||
sessMgr *session.Manager
|
||||
encGen *encurl.Generator
|
||||
csrfProtect func(http.Handler) http.Handler
|
||||
|
||||
// refererBlocklist matches the hosts of referer_blocklist; its IsAllowed
|
||||
// reports whether a URL's host is on that list.
|
||||
refererBlocklist *allowlist.HostAllowList
|
||||
}
|
||||
|
||||
// New creates a new Handlers instance.
|
||||
@@ -50,11 +55,12 @@ func New(lc fx.Lifecycle, params Params) (*Handlers, error) {
|
||||
}
|
||||
|
||||
s := &Handlers{
|
||||
log: params.Logger.Get(),
|
||||
hc: params.Healthcheck,
|
||||
db: params.Database,
|
||||
config: params.Config,
|
||||
csrfProtect: csrfProtect,
|
||||
log: params.Logger.Get(),
|
||||
hc: params.Healthcheck,
|
||||
db: params.Database,
|
||||
config: params.Config,
|
||||
csrfProtect: csrfProtect,
|
||||
refererBlocklist: allowlist.New(params.Config.RefererBlocklist),
|
||||
}
|
||||
|
||||
lc.Append(fx.Hook{
|
||||
|
||||
@@ -21,6 +21,10 @@ import (
|
||||
// /v1/image/<host>/<path>/<width>x<height>.<format>
|
||||
func (s *Handlers) HandleImage() http.HandlerFunc {
|
||||
return func(w http.ResponseWriter, r *http.Request) {
|
||||
if s.refuseBlockedReferer(w, r) {
|
||||
return
|
||||
}
|
||||
|
||||
req, ok := s.parseImageRequest(w, r)
|
||||
if !ok {
|
||||
return
|
||||
@@ -248,6 +252,23 @@ func cacheControl(expires time.Time) string {
|
||||
return fmt.Sprintf("public, max-age=%d, immutable", int64(maxAge/time.Second))
|
||||
}
|
||||
|
||||
// refuseBlockedReferer answers 403 with a JSON error when the request's Referer
|
||||
// names a host on referer_blocklist, and reports whether it answered. A request
|
||||
// with no Referer, or one that does not parse as a URL with a host, is not
|
||||
// refused.
|
||||
func (s *Handlers) refuseBlockedReferer(
|
||||
w http.ResponseWriter, r *http.Request,
|
||||
) bool {
|
||||
referer, err := url.Parse(r.Referer())
|
||||
if err != nil || !s.refererBlocklist.IsAllowed(referer) {
|
||||
return false
|
||||
}
|
||||
|
||||
s.respondError(w, "referer blocked", http.StatusForbidden)
|
||||
|
||||
return true
|
||||
}
|
||||
|
||||
// notModified sets the ETag header to etag and, when the request's
|
||||
// If-None-Match is that ETag, answers 304 Not Modified. It reports whether it
|
||||
// answered. An empty etag sets no header and never answers.
|
||||
|
||||
@@ -22,6 +22,10 @@ import (
|
||||
// browsers identify the content type.
|
||||
func (s *Handlers) HandleImageEnc() http.HandlerFunc {
|
||||
return func(w http.ResponseWriter, r *http.Request) {
|
||||
if s.refuseBlockedReferer(w, r) {
|
||||
return
|
||||
}
|
||||
|
||||
ctx := r.Context()
|
||||
start := time.Now()
|
||||
|
||||
|
||||
@@ -0,0 +1,185 @@
|
||||
package handlers
|
||||
|
||||
import (
|
||||
"context"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/go-chi/chi/v5"
|
||||
"sneak.berlin/go/pixa/internal/allowlist"
|
||||
"sneak.berlin/go/pixa/internal/encurl"
|
||||
"sneak.berlin/go/pixa/internal/httpfetcher"
|
||||
"sneak.berlin/go/pixa/internal/imgcache"
|
||||
)
|
||||
|
||||
// blockedReferer is a page on leech.example, which newRefererRoutes puts on
|
||||
// referer_blocklist.
|
||||
const blockedReferer = "https://leech.example/page.html"
|
||||
|
||||
// countingFetcher passes each fetch on to the fetcher it holds and counts it.
|
||||
type countingFetcher struct {
|
||||
httpfetcher.Fetcher
|
||||
|
||||
fetches atomic.Int32
|
||||
}
|
||||
|
||||
// Fetch counts the fetch and passes it on.
|
||||
func (f *countingFetcher) Fetch(
|
||||
ctx context.Context, url string,
|
||||
) (*httpfetcher.FetchResult, error) {
|
||||
f.fetches.Add(1)
|
||||
|
||||
return f.Fetcher.Fetch(ctx, url)
|
||||
}
|
||||
|
||||
// newRefererRoutes returns both image routes of a Handlers whose
|
||||
// referer_blocklist is "leech.example" and ".hotlinker.example", the
|
||||
// Handlers, and the fetcher the routes fetch through. The JPEG at photoPath
|
||||
// exists on allowlistedHost and on signedHost.
|
||||
func newRefererRoutes(t *testing.T) (http.Handler, *Handlers, *countingFetcher) {
|
||||
t.Helper()
|
||||
|
||||
fetcher := &countingFetcher{
|
||||
Fetcher: newPhotoFetcher(t, allowlistedHost, signedHost),
|
||||
}
|
||||
|
||||
cache, err := imgcache.NewCache(setupTestDB(t), imgcache.CacheConfig{
|
||||
StateDir: t.TempDir(),
|
||||
CacheTTL: time.Hour,
|
||||
NegativeTTL: 5 * time.Minute,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("imgcache.NewCache() error = %v", err)
|
||||
}
|
||||
|
||||
svc, err := imgcache.NewService(&imgcache.ServiceConfig{
|
||||
Cache: cache,
|
||||
Fetcher: fetcher,
|
||||
SigningKey: testSigningKey,
|
||||
Allowlist: []string{allowlistedHost},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("imgcache.NewService() error = %v", err)
|
||||
}
|
||||
|
||||
encGen, err := encurl.NewGenerator(testSigningKey)
|
||||
if err != nil {
|
||||
t.Fatalf("encurl.NewGenerator() error = %v", err)
|
||||
}
|
||||
|
||||
h := &Handlers{
|
||||
log: slog.New(slog.DiscardHandler),
|
||||
imgSvc: svc,
|
||||
encGen: encGen,
|
||||
refererBlocklist: allowlist.New(
|
||||
[]string{"leech.example", ".hotlinker.example"}),
|
||||
}
|
||||
|
||||
r := chi.NewRouter()
|
||||
r.Get("/v1/image/*", h.HandleImage())
|
||||
r.Get("/v1/e/{token}/*", h.HandleImageEnc())
|
||||
|
||||
return r, h, fetcher
|
||||
}
|
||||
|
||||
// getWithReferer sends a GET for target to routes with referer as its
|
||||
// Referer header, or with none when referer is empty, and returns the
|
||||
// response.
|
||||
func getWithReferer(
|
||||
t *testing.T, routes http.Handler, target, referer string,
|
||||
) *httptest.ResponseRecorder {
|
||||
t.Helper()
|
||||
|
||||
req := httptest.NewRequestWithContext(t.Context(), http.MethodGet, target, nil)
|
||||
if referer != "" {
|
||||
req.Header.Set("Referer", referer)
|
||||
}
|
||||
|
||||
rec := httptest.NewRecorder()
|
||||
|
||||
routes.ServeHTTP(rec, req)
|
||||
t.Logf("GET %s with Referer %q: %d", target, referer, rec.Code)
|
||||
|
||||
return rec
|
||||
}
|
||||
|
||||
// TestRefererBlocklist verifies that both image routes refuse a request whose
|
||||
// Referer names a host on referer_blocklist with 403 and the JSON error,
|
||||
// without fetching from the upstream host, and serve a request with no
|
||||
// Referer, one that does not parse, or one naming any other host. Hosts are
|
||||
// matched as allowlist_hosts matches them.
|
||||
func TestRefererBlocklist(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
cases := []struct {
|
||||
name string
|
||||
referer string
|
||||
want int
|
||||
}{
|
||||
{"no referer", "", http.StatusOK},
|
||||
{"unlisted host", "https://unlisted.example/page.html", http.StatusOK},
|
||||
{"unparseable", "%zz", http.StatusOK},
|
||||
{"listed host", blockedReferer, http.StatusForbidden},
|
||||
{"subdomain of listed host", "https://www.leech.example/", http.StatusOK},
|
||||
{"subdomain of dot pattern", "https://www.hotlinker.example/a.html",
|
||||
http.StatusForbidden},
|
||||
{"dot pattern without its dot", "https://hotlinker.example/",
|
||||
http.StatusForbidden},
|
||||
{"host continuing past dot pattern",
|
||||
"https://hotlinker.example.evil.example/", http.StatusOK},
|
||||
}
|
||||
|
||||
// The photo's URL on each image route.
|
||||
photoURLs := map[string]func(t *testing.T, h *Handlers) string{
|
||||
"plain URL": func(t *testing.T, _ *Handlers) string {
|
||||
t.Helper()
|
||||
|
||||
return photoURL(allowlistedHost)
|
||||
},
|
||||
"encrypted URL": encPhotoURL,
|
||||
}
|
||||
|
||||
for urlName, photoURLFor := range photoURLs {
|
||||
for _, tc := range cases {
|
||||
t.Run(urlName+", "+tc.name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
routes, h, fetcher := newRefererRoutes(t)
|
||||
|
||||
rec := getWithReferer(t, routes, photoURLFor(t, h), tc.referer)
|
||||
|
||||
if tc.want == http.StatusOK {
|
||||
requireServedPhoto(t, rec)
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
checkErrorBody(t, rec, http.StatusForbidden, "referer blocked")
|
||||
|
||||
if n := fetcher.fetches.Load(); n != 0 {
|
||||
t.Errorf("upstream fetched %d times, want 0", n)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestBlockedRefererRefusedWhenImageIsCached verifies that a request whose
|
||||
// Referer is on referer_blocklist is refused even when the image it asks for
|
||||
// is already cached, so the answer does not depend on the cache.
|
||||
func TestBlockedRefererRefusedWhenImageIsCached(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
routes, h, _ := newRefererRoutes(t)
|
||||
|
||||
for _, target := range []string{photoURL(allowlistedHost), encPhotoURL(t, h)} {
|
||||
requireServedPhoto(t, getWithReferer(t, routes, target, ""))
|
||||
|
||||
rec := getWithReferer(t, routes, target, blockedReferer)
|
||||
checkErrorBody(t, rec, http.StatusForbidden, "referer blocked")
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user