routewatch/internal/server/middleware_internal_test.go

package server

import (
	"bytes"
	"context"
	"encoding/json"
	"io"
	"log/slog"
	"net"
	"net/http"
	"net/http/httptest"
	"net/netip"
	"os"
	"regexp"
	"strings"
	"testing"
	"time"

	"git.eeqj.de/sneak/routewatch/internal/config"
	"git.eeqj.de/sneak/routewatch/internal/database"
	"git.eeqj.de/sneak/routewatch/internal/logger"
	"git.eeqj.de/sneak/routewatch/internal/metrics"
	"git.eeqj.de/sneak/routewatch/internal/streamer"
)

// The addresses the forwarding tests use: the client a proxy names in
// X-Forwarded-For, the range of trusted proxies, a peer inside it, and a peer
// outside it, which lies in testPrefixV4 so that an IP lookup of it finds a
// route. reverseProxyPeer is a reverse proxy at a private address, as Docker
// gives one.
const (
	forwardedClient   = "203.0.113.9"
	trustedProxyRange = "10.0.0.0/8"
	trustedPeer       = "10.0.0.2:1234"
	untrustedPeerAddr = "192.0.2.1"
	untrustedPeer     = untrustedPeerAddr + ":1234"
	reverseProxyPeer  = "172.19.0.14:1234"
)

// TestResponsesCarrySecurityHeaders checks that a page, a JSON route and a
// path no route matches each answer with the security headers REPO_POLICIES.md
// requires, and that the Content-Security-Policy allows no inline content
// without a nonce (https://git.eeqj.de/sneak/routewatch/issues/108).
func TestResponsesCarrySecurityHeaders(t *testing.T) {
	t.Parallel()

	s := newServerWithTestRoutes(t)

	headers := []string{
		"Strict-Transport-Security",
		"Content-Security-Policy",
		"X-Frame-Options",
		"X-Content-Type-Options",
		"Referrer-Policy",
		"Permissions-Policy",
	}

	for _, path := range []string{"/", statsPath, "/no-such-page"} {
		rec := httptest.NewRecorder()
		s.router.ServeHTTP(rec, httptest.NewRequestWithContext(
			t.Context(), http.MethodGet, path, nil))

		for _, name := range headers {
			if rec.Header().Get(name) == "" {
				t.Errorf("%s: no %s header", path, name)
			}
		}

		policy := rec.Header().Get("Content-Security-Policy")
		if strings.Contains(policy, "'unsafe-") {
			t.Errorf("%s: Content-Security-Policy %q allows unsafe content",
				path, policy)
		}
	}
}

// TestJSONValidationKeeps4xxWithoutBody wraps a handler that answers 400
// without a body in JSONValidationMiddleware and checks that the JSON error
// body keeps the 400 and gives the status's own text. Before the fix it said
// "Internal server error"
// (https://git.eeqj.de/sneak/routewatch/issues/111).
func TestJSONValidationKeeps4xxWithoutBody(t *testing.T) {
	t.Parallel()

	handler := JSONValidationMiddleware(http.HandlerFunc(
		func(w http.ResponseWriter, _ *http.Request) {
			w.WriteHeader(http.StatusBadRequest)
		}))

	rec := httptest.NewRecorder()
	handler.ServeHTTP(rec, httptest.NewRequestWithContext(
		t.Context(), http.MethodGet, "/", nil))

	const name = "400 without a body"

	if rec.Code != http.StatusBadRequest {
		t.Errorf("%s: status %d, want %d; body %s",
			name, rec.Code, http.StatusBadRequest, rec.Body)
	}

	checkJSONErrorCode(t, name, rec, http.StatusBadRequest)
	checkJSONErrorMessage(t, name, rec, http.StatusText(http.StatusBadRequest))
}

// TestJSONAnswersPutNoUnitInAString sends a request to each JSON route and
// checks each answer. Its @meta.execution_time_ms is a number, and none of its
// strings is a number with a unit, such as "12 ms", "1.1 GB" or "1h18m34s"
// (https://git.eeqj.de/sneak/routewatch/issues/199). The health check answers
// 503, since the stream is not connected, and the last path matches no route.
func TestJSONAnswersPutNoUnitInAString(t *testing.T) {
	t.Parallel()

	numberWithUnit := regexp.MustCompile(
		`^([0-9]+(\.[0-9]+)? ?(ns|µs|ms|s|m|h|B|kB|MB|GB|TB))+$`)

	s := newServerWithTestRoutes(t)
	s.resolver = &fakeResolver{}
	s.SetASNFetcher(fixedIntervalASNFetcher{interval: time.Second})

	paths := []string{
		statusJSONPath,
		statsPath,
		healthCheckPath,
		"/api/v1/ip/" + untrustedPeerAddr,
		"/ip/" + untrustedPeerAddr,
		asJSONPath,
		"/api/v1/prefix/" + testPrefixV4,
		"/api/v1/search/as?q=google",
		"/api/v1/no-such-route",
	}

	for _, path := range paths {
		rec := httptest.NewRecorder()
		s.router.ServeHTTP(rec, httptest.NewRequestWithContext(
			t.Context(), http.MethodGet, path, nil))

		var answer map[string]any

		err := json.Unmarshal(rec.Body.Bytes(), &answer)
		if err != nil {
			t.Errorf("%s: decoding the answer: %v; answer %s", path, err, rec.Body)

			continue
		}

		meta, _ := answer["@meta"].(map[string]any)
		if _, ok := meta["execution_time_ms"].(float64); !ok {
			t.Errorf("%s: @meta.execution_time_ms is not a number: %s",
				path, rec.Body)
		}

		for _, value := range stringsIn(answer) {
			if numberWithUnit.MatchString(value) {
				t.Errorf("%s: %q is a number with a unit in a string", path, value)
			}
		}
	}
}

// TestJSONAnswersCarryMeta sends requests to every JSON route through the
// server's router, and checks that each answer carries @meta with its three
// fields. The answers include successes, and errors that the handlers, the
// router and the middleware write. Before the fix a panic in a handler, and a
// method /status.json or the health check does not take, answered with an
// empty body (https://git.eeqj.de/sneak/routewatch/issues/201).
func TestJSONAnswersCarryMeta(t *testing.T) {
	t.Parallel()

	// JSON routes whose handlers write nothing or panic, so that the JSON
	// check on the route writes the error answer itself.
	const (
		writesNothingPath = "/writes-nothing.json"
		panicsPath        = "/panics.json"
	)

	s := newServerWithTestRoutes(t)
	s.resolver = &fakeResolver{}
	s.streamer.GetMetricsTracker().SetConnected(true)
	s.router.Get(writesNothingPath, JSONValidationMiddleware(http.HandlerFunc(
		func(http.ResponseWriter, *http.Request) {})).ServeHTTP)
	s.router.Get(panicsPath, JSONValidationMiddleware(http.HandlerFunc(
		func(http.ResponseWriter, *http.Request) { panic(errDiskIO) })).ServeHTTP)

	tests := []struct {
		method   string
		path     string
		wantCode int
	}{
		// Answers that succeed
		{http.MethodGet, statusJSONPath, http.StatusOK},
		{http.MethodGet, healthCheckPath, http.StatusOK},
		{http.MethodGet, "/ip", http.StatusOK},
		{http.MethodGet, ownAddressPath, http.StatusOK},
		{http.MethodGet, "/ip/" + addrInTestPrefixV4, http.StatusOK},
		{http.MethodGet, statsPath, http.StatusOK},
		{http.MethodGet, "/api/v1/ip/" + addrInTestPrefixV4, http.StatusOK},
		{http.MethodGet, asJSONPath, http.StatusOK},
		{http.MethodGet, "/api/v1/prefix/" + testPrefixV4, http.StatusOK},
		{http.MethodGet, "/api/v1/search/as?q=google", http.StatusOK},
		// Errors the handlers write
		{http.MethodGet, "/ip/no-such-host.example", http.StatusBadRequest},
		{http.MethodGet, "/api/v1/ip/no-such-host.example", http.StatusBadRequest},
		{http.MethodGet, "/api/v1/as/foo", http.StatusBadRequest},
		{http.MethodGet, "/api/v1/as/0", http.StatusNotFound},
		{http.MethodGet, "/api/v1/prefix/192.0.2.0/33", http.StatusBadRequest},
		{http.MethodGet, "/api/v1/prefix/198.51.100.0/24", http.StatusNotFound},
		{http.MethodGet, "/api/v1/search/as", http.StatusBadRequest},
		// Errors the router and the middleware write
		{http.MethodGet, "/api/v1/no-such-route", http.StatusNotFound},
		{http.MethodGet, "/ip/192.0.2.1/nothing", http.StatusNotFound},
		{http.MethodPost, statsPath, http.StatusMethodNotAllowed},
		{http.MethodPost, ownAddressPath, http.StatusMethodNotAllowed},
		{http.MethodPost, statusJSONPath, http.StatusMethodNotAllowed},
		{http.MethodPut, healthCheckPath, http.StatusMethodNotAllowed},
		{http.MethodGet, writesNothingPath, http.StatusInternalServerError},
		{http.MethodGet, panicsPath, http.StatusInternalServerError},
	}

	for _, tt := range tests {
		checkAnswerCarriesMeta(t, s, httptest.NewRequestWithContext(
			t.Context(), tt.method, tt.path, nil), tt.wantCode)
	}

	// A request whose time limit has already passed answers 504. The handlers
	// write the answers of the first five routes, and the JSON check on the
	// route writes the last.
	pastTimeLimit, cancel := context.WithDeadline(t.Context(), time.Now())
	defer cancel()

	for _, path := range []string{
		statusJSONPath, statsPath, "/ip/" + addrInTestPrefixV4,
		"/api/v1/ip/" + addrInTestPrefixV4, asJSONPath, writesNothingPath,
	} {
		checkAnswerCarriesMeta(t, s, httptest.NewRequestWithContext(
			pastTimeLimit, http.MethodGet, path, nil), http.StatusGatewayTimeout)
	}

	// With a store whose reads fail, the health check answers 503, and the
	// handlers of the other routes 500.
	failing := New(failingDB{}, newTestTable(),
		streamer.New(logger.New(), metrics.New()), logger.New(), &config.Config{})
	failing.resolver = &fakeResolver{}

	checkAnswerCarriesMeta(t, failing, httptest.NewRequestWithContext(
		t.Context(), http.MethodGet, healthCheckPath, nil),
		http.StatusServiceUnavailable)

	for _, path := range []string{
		statusJSONPath, statsPath, asJSONPath,
		"/api/v1/ip/" + addrInTestPrefixV4, "/ip/" + addrInTestPrefixV4,
	} {
		checkAnswerCarriesMeta(t, failing, httptest.NewRequestWithContext(
			t.Context(), http.MethodGet, path, nil), http.StatusInternalServerError)
	}
}

// checkAnswerCarriesMeta serves req through the server's router, and checks
// that it answers wantCode with @meta and its three fields.
func checkAnswerCarriesMeta(
	t *testing.T, s *Server, req *http.Request, wantCode int,
) {
	t.Helper()

	// The values README "JSON API" gives the two fields that never change.
	const (
		wantTimeZone   = "UTC"
		wantAPIVersion = 1
	)

	name := req.Method + " " + req.URL.String()

	rec := httptest.NewRecorder()
	s.router.ServeHTTP(rec, req)

	if rec.Code != wantCode {
		t.Errorf("%s: status %d, want %d; body %s", name, rec.Code, wantCode, rec.Body)
	}

	var answer map[string]any

	err := json.Unmarshal(rec.Body.Bytes(), &answer)
	if err != nil {
		t.Errorf("%s: decoding the answer %q: %v", name, rec.Body, err)

		return
	}

	meta, _ := answer[jsonKeyMeta].(map[string]any)
	_, timeIsNumber := meta[metaKeyExecutionTimeMs].(float64)

	if meta[metaKeyTimeZone] != wantTimeZone ||
		meta[metaKeyAPIVersion] != float64(wantAPIVersion) || !timeIsNumber {
		t.Errorf("%s: @meta %v, want time_zone %q, api_version %d and a "+
			"number in execution_time_ms", name, meta, wantTimeZone, wantAPIVersion)
	}
}

// stringsIn returns every string in a decoded JSON value, at any depth.
func stringsIn(value any) []string {
	var found []string

	switch v := value.(type) {
	case string:
		found = append(found, v)
	case []any:
		for _, item := range v {
			found = append(found, stringsIn(item)...)
		}
	case map[string]any:
		for _, item := range v {
			found = append(found, stringsIn(item)...)
		}
	}

	return found
}

// TestPagesCarryTheCSPNonce checks that every script and style block of each
// page carries the nonce its Content-Security-Policy header names, so that the
// browser runs and applies them, that each answer has a nonce of its own, and
// that no page uses a style attribute, which the policy does not allow.
func TestPagesCarryTheCSPNonce(t *testing.T) {
	t.Parallel()

	s := newServerWithTestRoutes(t)

	_, err := s.db.GetOrCreateASN(64500, time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC))
	if err != nil {
		t.Fatalf("GetOrCreateASN: %v", err)
	}

	paths := []string{
		"/", "/status", asPagePath, "/prefix/" + testPrefixV4,
		prefixLengthPagePath, "/prefixlength6/32",
	}
	seen := make(map[string]bool)

	for _, path := range paths {
		rec := httptest.NewRecorder()
		s.router.ServeHTTP(rec, httptest.NewRequestWithContext(
			t.Context(), http.MethodGet, path, nil))

		if rec.Code != http.StatusOK {
			t.Errorf("%s: status %d, want %d; body %s",
				path, rec.Code, http.StatusOK, rec.Body)

			continue
		}

		policy := rec.Header().Get("Content-Security-Policy")

		_, nonce, found := strings.Cut(policy, "'nonce-")
		if !found {
			t.Errorf("%s: Content-Security-Policy %q names no nonce", path, policy)

			continue
		}

		nonce, _, _ = strings.Cut(nonce, "'")
		if seen[nonce] {
			t.Errorf("%s: nonce %q was already used for another answer", path, nonce)
		}

		seen[nonce] = true
		page := rec.Body.String()

		for _, tag := range []string{"<script", "<style"} {
			withNonce := tag + ` nonce="` + nonce + `"`
			if strings.Count(page, tag) != strings.Count(page, withNonce) {
				t.Errorf("%s: a %s tag does not carry the nonce %q", path, tag, nonce)
			}
		}

		if strings.Contains(page, "style=") {
			t.Errorf("%s: the page uses a style attribute", path)
		}
	}
}

// TestForwardedForFromUntrustedPeerIsIgnored asks /ip/ for the caller's own
// address from a peer that is not a trusted proxy, with forwarding headers
// naming another address, and checks that the lookup is for the TCP peer.
// Before the fix any client could choose the address it was looked up and
// logged as (https://git.eeqj.de/sneak/routewatch/issues/108).
func TestForwardedForFromUntrustedPeerIsIgnored(t *testing.T) {
	t.Parallel()

	s := newServerWithTestRoutes(t)

	req := httptest.NewRequestWithContext(t.Context(), http.MethodGet, "/ip/", nil)
	req.RemoteAddr = untrustedPeer
	req.Header.Set("X-Forwarded-For", forwardedClient)
	req.Header.Set("X-Real-IP", forwardedClient)
	req.Header.Set("True-Client-IP", forwardedClient)

	rec := httptest.NewRecorder()
	s.router.ServeHTTP(rec, req)

	var body struct {
		Data IPLookupResponse `json:"data"`
	}

	err := json.Unmarshal(rec.Body.Bytes(), &body)
	if err != nil {
		t.Fatalf("decoding the answer %q: %v", rec.Body, err)
	}

	if body.Data.Query != untrustedPeerAddr {
		t.Errorf("looked up %q, want the TCP peer %s; answer %s",
			body.Data.Query, untrustedPeerAddr, rec.Body)
	}
}

// TestDefaultTrustedProxiesGiveTheForwardedClient builds the server from the
// configuration with TRUSTED_PROXIES unset. A request from a reverse proxy at
// a private address must be looked up by /ip/, shown on the home page's card
// and logged as the client its X-Forwarded-For names. A direct request from a
// public address with a forged X-Forwarded-For keeps its own address. Before
// the fix no proxy was trusted unless TRUSTED_PROXIES named it, so /ip/ gave
// the proxy's address (https://git.eeqj.de/sneak/routewatch/issues/198).
func TestDefaultTrustedProxiesGiveTheForwardedClient(t *testing.T) {
	t.Setenv("PORT", "")
	t.Setenv("XDG_DATA_HOME", "")

	// t.Setenv puts the variable back when the test ends.
	t.Setenv("TRUSTED_PROXIES", "")

	err := os.Unsetenv("TRUSTED_PROXIES")
	if err != nil {
		t.Fatal(err)
	}

	cfg, err := config.New()
	if err != nil {
		t.Fatalf("config.New: %v", err)
	}

	var requestLog bytes.Buffer

	s := newServerWithConfig(t, cfg, &requestLog)

	tests := []struct {
		name string
		peer string
		want string
	}{
		{"through a reverse proxy", reverseProxyPeer, forwardedClient},
		{"direct from a public address", untrustedPeer, untrustedPeerAddr},
	}

	for _, tt := range tests {
		requestLog.Reset()

		get := func(path string) *httptest.ResponseRecorder {
			req := httptest.NewRequestWithContext(t.Context(), http.MethodGet, path, nil)
			req.RemoteAddr = tt.peer
			req.Header.Set("X-Forwarded-For", forwardedClient)

			rec := httptest.NewRecorder()
			s.router.ServeHTTP(rec, req)

			return rec
		}

		var answer struct {
			Data IPLookupResponse `json:"data"`
		}

		rec := get("/ip/")

		err := json.Unmarshal(rec.Body.Bytes(), &answer)
		if err != nil {
			t.Fatalf("%s: decoding the answer %q: %v", tt.name, rec.Body, err)
		}

		if answer.Data.Query != tt.want {
			t.Errorf("%s: /ip/ looked up %q, want %q", tt.name, answer.Data.Query, tt.want)
		}

		page := get("/").Body.String()
		if !strings.Contains(page, `data-client-ip="`+tt.want+`"`) {
			t.Errorf("%s: the home page's card does not hold %s", tt.name, tt.want)
		}

		logged := loggedAddresses(requestLog.String())
		if len(logged) == 0 {
			t.Errorf("%s: the request log names no address", tt.name)
		}

		for _, addr := range logged {
			if addr != tt.want {
				t.Errorf("%s: the request log gives %s, want %s", tt.name, addr, tt.want)
			}
		}
	}
}

// newServerWithConfig returns a server built with cfg, on a new database, that
// reads the routes of newTestTable and logs its requests as JSON to
// requestLog.
func newServerWithConfig(
	t *testing.T, cfg *config.Config, requestLog io.Writer,
) *Server {
	t.Helper()

	cfg.StateDir = t.TempDir()

	db, err := database.New(cfg, logger.New())
	if err != nil {
		t.Fatalf("database.New: %v", err)
	}

	t.Cleanup(func() { _ = db.Close() })

	s := New(db, newTestTable(), streamer.New(logger.New(), metrics.New()),
		&logger.Logger{Logger: slog.New(slog.NewJSONHandler(requestLog, nil))}, cfg)
	s.resolver = &fakeResolver{}

	return s
}

// loggedAddresses returns the address each line of the JSON request log
// names, without the port a direct request is logged with.
func loggedAddresses(requestLog string) []string {
	var addrs []string

	for line := range strings.Lines(requestLog) {
		//nolint:tagliatelle // the field name is the key the request log uses
		var entry struct {
			RemoteAddr string `json:"remote_addr"`
		}

		err := json.Unmarshal([]byte(line), &entry)
		if err != nil || entry.RemoteAddr == "" {
			continue
		}

		addr, _, err := net.SplitHostPort(entry.RemoteAddr)
		if err != nil {
			addr = entry.RemoteAddr
		}

		addrs = append(addrs, addr)
	}

	return addrs
}

// TestRealIPMiddlewareTrustsOnlyConfiguredProxies checks which address
// RealIPMiddleware gives a request: the TCP peer, unless the peer is a trusted
// proxy; then the last X-Forwarded-For entry that is not a trusted proxy,
// since the entries left of it are whatever the client sent.
func TestRealIPMiddlewareTrustsOnlyConfiguredProxies(t *testing.T) {
	t.Parallel()

	trusted := []netip.Prefix{netip.MustParsePrefix(trustedProxyRange)}

	tests := []struct {
		name         string
		peer         string
		forwardedFor string
		want         string
	}{
		{"untrusted peer", untrustedPeer, forwardedClient, untrustedPeer},
		{"trusted peer", trustedPeer, forwardedClient, forwardedClient},
		{
			"entries the client wrote", trustedPeer,
			"198.51.100.7, " + forwardedClient, forwardedClient,
		},
		{
			"two trusted proxies", trustedPeer,
			forwardedClient + ", 10.0.0.3", forwardedClient,
		},
		{"no header", trustedPeer, "", trustedPeer},
		{"entry not an address", trustedPeer, "client", trustedPeer},
	}

	for _, tt := range tests {
		var got string

		handler := RealIPMiddleware(trusted)(http.HandlerFunc(
			func(_ http.ResponseWriter, r *http.Request) { got = r.RemoteAddr }))

		req := httptest.NewRequestWithContext(t.Context(), http.MethodGet, "/", nil)
		req.RemoteAddr = tt.peer

		if tt.forwardedFor != "" {
			req.Header.Set("X-Forwarded-For", tt.forwardedFor)
		}

		handler.ServeHTTP(httptest.NewRecorder(), req)

		if got != tt.want {
			t.Errorf("%s: RemoteAddr %q, want %q", tt.name, got, tt.want)
		}
	}
}