Add failing tests for referer_blocklist

Both image routes must refuse a request whose Referer names a host on
referer_blocklist with 403 and the JSON error, without fetching from the
upstream host and whether or not the image is cached, and must serve a
request with no Referer, one that does not parse, or one naming another
host. Hosts match as allowlist_hosts matches them. The config tests check
the list is read from the file and from PIXA_REFERER_BLOCKLIST, and that an
entry that is not a host aborts startup naming the setting and the entry.
These do not compile until the setting exists.

Model: opus-5-5

internal/config/env_internal_test.go

@@ -65,6 +65,7 @@ func TestEnvironmentSetsEveryKey(t *testing.T) {
 	t.Setenv("PIXA_METRICS_PASSWORD", "metricspass")
 	t.Setenv("PIXA_SIGNING_KEY", validTestSigningKey)
 	t.Setenv("PIXA_ALLOWLIST_HOSTS", "s3.sneak.cloud,.example.com")
+	t.Setenv("PIXA_REFERER_BLOCKLIST", "leech.example,.hotlinker.example")
 	t.Setenv("PIXA_ALLOW_HTTP", "true")
 	t.Setenv("PIXA_UPSTREAM_CONNECTIONS_PER_HOST", "5")
 	t.Setenv("PIXA_UPSTREAM_CONNECTIONS", "10")
@@ -93,6 +94,7 @@ func TestEnvironmentSetsEveryKey(t *testing.T) {
 		MetricsPassword:            "metricspass",
 		SigningKey:                 validTestSigningKey,
 		AllowlistHosts:             []string{testHostS3, ".example.com"},
+		RefererBlocklist:           []string{"leech.example", ".hotlinker.example"},
 		AllowHTTP:                  true,
 		UpstreamConnectionsPerHost: 5,
 		UpstreamConnections:        10,

internal/config/referer_blocklist_internal_test.go

@@ -0,0 +1,107 @@
+package config
+
+import (
+	"slices"
+	"testing"
+)
+
+// TestRefererBlocklistParsed loads a referer_blocklist with a host and a
+// pattern starting with "." and checks both are kept in order.
+func TestRefererBlocklistParsed(t *testing.T) {
+	t.Parallel()
+
+	c, err := configFromYAML(t, signingKeyLine+`referer_blocklist:
+  - leech.example
+  - .hotlinker.example
+`)
+	if err != nil {
+		t.Fatalf("valid referer_blocklist should load, got error: %v", err)
+	}
+
+	want := []string{"leech.example", ".hotlinker.example"}
+	if !slices.Equal(c.RefererBlocklist, want) {
+		t.Errorf("RefererBlocklist = %v, want %v", c.RefererBlocklist, want)
+	}
+}
+
+// TestRefererBlocklistOmittedIsEmpty checks that an omitted key blocks no
+// referer.
+func TestRefererBlocklistOmittedIsEmpty(t *testing.T) {
+	t.Parallel()
+
+	c, err := configFromYAML(t, signingKeyLine)
+	if err != nil {
+		t.Fatalf("minimal config should be valid, got error: %v", err)
+	}
+
+	if len(c.RefererBlocklist) != 0 {
+		t.Errorf("RefererBlocklist = %v, want empty", c.RefererBlocklist)
+	}
+}
+
+// TestRefererBlocklistInvalidAbortsStartup checks that an entry that is not a
+// host, or a value that is not a list of them, aborts startup with an error
+// naming the key and the entry.
+func TestRefererBlocklistInvalidAbortsStartup(t *testing.T) {
+	t.Parallel()
+
+	runAbortCases(t, []abortCase{
+		{
+			name: "url",
+			yaml: signingKeyLine + "referer_blocklist:\n  - https://leech.example\n",
+			wantErrSubstrings: []string{
+				keyRefererBlocklist, "https://leech.example",
+			},
+		},
+		{
+			name: "path",
+			yaml: signingKeyLine + "referer_blocklist:\n  - leech.example/page\n",
+			wantErrSubstrings: []string{
+				keyRefererBlocklist, "leech.example/page",
+			},
+		},
+		{
+			name:              "dot only",
+			yaml:              signingKeyLine + "referer_blocklist:\n  - \".\"\n",
+			wantErrSubstrings: []string{keyRefererBlocklist, `"."`},
+		},
+		{
+			name:              "empty entry",
+			yaml:              signingKeyLine + "referer_blocklist:\n  - \"\"\n",
+			wantErrSubstrings: []string{keyRefererBlocklist},
+		},
+		{
+			name:              "entry not a string",
+			yaml:              signingKeyLine + "referer_blocklist:\n  - 42\n",
+			wantErrSubstrings: []string{keyRefererBlocklist, "42"},
+		},
+		{
+			name:              "null",
+			yaml:              signingKeyLine + "referer_blocklist:\n",
+			wantErrSubstrings: []string{keyRefererBlocklist, nullValueText},
+		},
+	})
+}
+
+// TestRefererBlocklistFromEnvironment checks that PIXA_REFERER_BLOCKLIST
+// takes comma-separated entries, and that an entry in it that is not a host
+// aborts startup naming the variable and the entry.
+func TestRefererBlocklistFromEnvironment(t *testing.T) {
+	t.Setenv("PIXA_SIGNING_KEY", validTestSigningKey)
+	t.Setenv("PIXA_REFERER_BLOCKLIST", " leech.example , .hotlinker.example ")
+
+	c, err := newFromSmartConfig(nil)
+	if err != nil {
+		t.Fatalf("valid PIXA_REFERER_BLOCKLIST should load, got error: %v", err)
+	}
+
+	want := []string{"leech.example", ".hotlinker.example"}
+	if !slices.Equal(c.RefererBlocklist, want) {
+		t.Errorf("RefererBlocklist = %v, want %v", c.RefererBlocklist, want)
+	}
+
+	t.Setenv("PIXA_REFERER_BLOCKLIST", "leech.example,https://hotlinker.example")
+
+	_, err = newFromSmartConfig(nil)
+	wantStartupError(t, err, "PIXA_REFERER_BLOCKLIST", "https://hotlinker.example")
+}

internal/handlers/referer_blocklist_internal_test.go

@@ -0,0 +1,180 @@
+package handlers
+
+import (
+	"context"
+	"log/slog"
+	"net/http"
+	"net/http/httptest"
+	"sync/atomic"
+	"testing"
+	"time"
+
+	"github.com/go-chi/chi/v5"
+	"sneak.berlin/go/pixa/internal/allowlist"
+	"sneak.berlin/go/pixa/internal/encurl"
+	"sneak.berlin/go/pixa/internal/httpfetcher"
+	"sneak.berlin/go/pixa/internal/imgcache"
+)
+
+// blockedReferer is a page on leech.example, which newRefererRoutes puts on
+// referer_blocklist.
+const blockedReferer = "https://leech.example/page.html"
+
+// countingFetcher passes each fetch on to the fetcher it holds and counts it.
+type countingFetcher struct {
+	httpfetcher.Fetcher
+
+	fetches atomic.Int32
+}
+
+// Fetch counts the fetch and passes it on.
+func (f *countingFetcher) Fetch(
+	ctx context.Context, url string,
+) (*httpfetcher.FetchResult, error) {
+	f.fetches.Add(1)
+
+	return f.Fetcher.Fetch(ctx, url)
+}
+
+// newRefererRoutes returns both image routes of a Handlers whose
+// referer_blocklist is "leech.example" and ".hotlinker.example", the
+// Handlers, and the fetcher the routes fetch through. The JPEG at photoPath
+// exists on allowlistedHost and on signedHost.
+func newRefererRoutes(t *testing.T) (http.Handler, *Handlers, *countingFetcher) {
+	t.Helper()
+
+	fetcher := &countingFetcher{
+		Fetcher: newPhotoFetcher(t, allowlistedHost, signedHost),
+	}
+
+	cache, err := imgcache.NewCache(setupTestDB(t), imgcache.CacheConfig{
+		StateDir:    t.TempDir(),
+		CacheTTL:    time.Hour,
+		NegativeTTL: 5 * time.Minute,
+	})
+	if err != nil {
+		t.Fatalf("imgcache.NewCache() error = %v", err)
+	}
+
+	svc, err := imgcache.NewService(&imgcache.ServiceConfig{
+		Cache:      cache,
+		Fetcher:    fetcher,
+		SigningKey: testSigningKey,
+		Allowlist:  []string{allowlistedHost},
+	})
+	if err != nil {
+		t.Fatalf("imgcache.NewService() error = %v", err)
+	}
+
+	encGen, err := encurl.NewGenerator(testSigningKey)
+	if err != nil {
+		t.Fatalf("encurl.NewGenerator() error = %v", err)
+	}
+
+	h := &Handlers{
+		log:    slog.New(slog.DiscardHandler),
+		imgSvc: svc,
+		encGen: encGen,
+		refererBlocklist: allowlist.New(
+			[]string{"leech.example", ".hotlinker.example"}),
+	}
+
+	r := chi.NewRouter()
+	r.Get("/v1/image/*", h.HandleImage())
+	r.Get("/v1/e/{token}/*", h.HandleImageEnc())
+
+	return r, h, fetcher
+}
+
+// getWithReferer sends a GET for target to routes with referer as its
+// Referer header, or with none when referer is empty, and returns the
+// response.
+func getWithReferer(
+	t *testing.T, routes http.Handler, target, referer string,
+) *httptest.ResponseRecorder {
+	t.Helper()
+
+	req := httptest.NewRequestWithContext(t.Context(), http.MethodGet, target, nil)
+	if referer != "" {
+		req.Header.Set("Referer", referer)
+	}
+
+	rec := httptest.NewRecorder()
+
+	routes.ServeHTTP(rec, req)
+	t.Logf("GET %s with Referer %q: %d", target, referer, rec.Code)
+
+	return rec
+}
+
+// TestRefererBlocklist verifies that both image routes refuse a request whose
+// Referer names a host on referer_blocklist with 403 and the JSON error,
+// without fetching from the upstream host, and serve a request with no
+// Referer, one that does not parse, or one naming any other host. Hosts are
+// matched as allowlist_hosts matches them.
+func TestRefererBlocklist(t *testing.T) {
+	t.Parallel()
+
+	cases := []struct {
+		name    string
+		referer string
+		want    int
+	}{
+		{"no referer", "", http.StatusOK},
+		{"unlisted host", "https://unlisted.example/page.html", http.StatusOK},
+		{"unparseable", "%zz", http.StatusOK},
+		{"listed host", blockedReferer, http.StatusForbidden},
+		{"subdomain of listed host", "https://www.leech.example/", http.StatusOK},
+		{"subdomain of dot pattern", "https://www.hotlinker.example/a.html",
+			http.StatusForbidden},
+		{"dot pattern without its dot", "https://hotlinker.example/",
+			http.StatusForbidden},
+		{"host continuing past dot pattern",
+			"https://hotlinker.example.evil.example/", http.StatusOK},
+	}
+
+	for _, route := range []string{"/v1/image/", "/v1/e/"} {
+		for _, tc := range cases {
+			t.Run(route+" "+tc.name, func(t *testing.T) {
+				t.Parallel()
+
+				routes, h, fetcher := newRefererRoutes(t)
+
+				target := photoURL(allowlistedHost)
+				if route == "/v1/e/" {
+					target = encPhotoURL(t, h)
+				}
+
+				rec := getWithReferer(t, routes, target, tc.referer)
+
+				if tc.want == http.StatusOK {
+					requireServedPhoto(t, rec)
+
+					return
+				}
+
+				checkErrorBody(t, rec, http.StatusForbidden, "referer blocked")
+
+				if n := fetcher.fetches.Load(); n != 0 {
+					t.Errorf("upstream fetched %d times, want 0", n)
+				}
+			})
+		}
+	}
+}
+
+// TestBlockedRefererRefusedWhenImageIsCached verifies that a request whose
+// Referer is on referer_blocklist is refused even when the image it asks for
+// is already cached, so the answer does not depend on the cache.
+func TestBlockedRefererRefusedWhenImageIsCached(t *testing.T) {
+	t.Parallel()
+
+	routes, h, _ := newRefererRoutes(t)
+
+	for _, target := range []string{photoURL(allowlistedHost), encPhotoURL(t, h)} {
+		requireServedPhoto(t, getWithReferer(t, routes, target, ""))
+
+		rec := getWithReferer(t, routes, target, blockedReferer)
+		checkErrorBody(t, rec, http.StatusForbidden, "referer blocked")
+	}
+}