Report handler panics through the logger and answer 500 (closes #187)

internal/middleware/accesslog_test.go

@@ -376,7 +376,7 @@ func lineSizeCases() map[string]sizeCase {
 
 	// The url field on a 5xx keeps the concrete path, so it reaches its
 	// own budget on the same line as the three header fields. That is
-	// the widest line the service can be made to write.
+	// the widest access log line the service can be made to write.
 	longPath := "/boom/" + strings.Repeat("x", oversizedSegmentBytes)
 	wantLongURL := longPath[:maxFieldBytes] + truncationSuffix
 

internal/middleware/middleware.go

@@ -133,16 +133,12 @@ const (
 	//   - The "log" delivery target, which exists to write the whole
 	//     inbound event to the log. Deliberate; see
 	//     internal/delivery/target_log.go.
-	//   - The widest line the service can write, which is neither an
-	//     access log line nor client-chosen. A handler panic arrives
-	//     through net/http's nil ErrorLog as one record carrying a
-	//     whole goroutine stack, above this figure — measured at
-	//     roughly 2,770 bytes. The exact width is not an invariant: it
-	//     moves with the goroutine number and with the source paths
-	//     baked into the stack. That it exceeds this ceiling does not
-	//     move. The value is the runtime's, not a client's; see
-	//     README.md and
-	//     https://git.eeqj.de/sneak/webhooker/issues/187.
+	//   - The record a recovered panic writes, which is not an access
+	//     log line: its client-supplied fields are charged the same
+	//     budgets, but it carries a whole goroutine stack as well and
+	//     is wider than this figure. It has its own stated ceiling,
+	//     MaxPanicLogLineBytes in recoverer.go, and is written once
+	//     per recovered panic rather than once per request.
 	MaxAccessLogLineBytes = 2560
 )
 

internal/middleware/recoverer.go

@@ -0,0 +1,209 @@
+package middleware
+
+import (
+	"errors"
+	"fmt"
+	"net/http"
+	"runtime/debug"
+
+	"github.com/go-chi/chi/middleware"
+	"sneak.berlin/go/webhooker/internal/logfield"
+)
+
+const (
+	// maxPanicValueBytes bounds the recovered panic value. The value
+	// is our own text, but a handler is free to build one out of the
+	// request — panic(fmt.Sprintf("bad %q", r.URL.Path)) — so it is
+	// charged the same budget the access log gives a field the
+	// client supplies outright.
+	maxPanicValueBytes = logfield.MaxBytes
+
+	// maxPanicStackBytes bounds the stack, in the same ENCODED bytes
+	// logfield.Truncate charges everywhere else. Nothing a client
+	// sends chooses the depth of our own call stack, so this is not
+	// a safety limit; it is what makes MaxPanicLogLineBytes an
+	// arithmetic ceiling rather than an observation. A stack is cut
+	// at its far end, which is net/http's accept frames — the panic
+	// site and the handler that reached it are at the near end and
+	// are always kept.
+	//
+	// For scale: a handler panicking under the full shipped
+	// middleware chain produces a stack of roughly 3,690 bytes in a
+	// record of roughly 3,960, so this budget holds better than
+	// twice the depth that case reaches. Neither number is an
+	// invariant — debug.Stack() embeds absolute source paths, so
+	// both move with where the tree sits, and four checkouts have
+	// reported records of 3,959, 3,961, 3,984 and 4,026 bytes.
+	// internal/server's TestPanicThroughProductionRouter asserts the
+	// ceiling and that the stack arrived uncut, not the figures.
+	maxPanicStackBytes = 8192
+
+	// MaxPanicLogLineBytes is the ceiling on the single line a
+	// recovered panic writes. It is wider than
+	// MaxAccessLogLineBytes, which bounds a line written once per
+	// request, where this one is written once per recovered panic.
+	//
+	//	panic                     512+11 =  523
+	//	stack                    8192+11 = 8203
+	//	request_id                128+11 =  139
+	//	fixed portion                    =  256
+	//	                                    ----
+	//	                                    9121
+	//
+	// The fixed portion is the JSON punctuation, the field names,
+	// the level, the message, the timestamp at its longest and the
+	// response_committed boolean.
+	//
+	// Stated at 10240 so the figure carries headroom rather than
+	// sitting on the arithmetic, exactly as MaxAccessLogLineBytes
+	// is. Both handlers internal/logger can install are covered, for
+	// the reason given there: logfield.EncodedBytes charges every
+	// rune the wider of the two.
+	//
+	// The 9121 and the 10240 are the invariants here. What follows
+	// is illustration: with all three growable fields — the stack,
+	// the panic value and a client-supplied X-Request-Id — driven
+	// past their budgets at once, TestRecovererBoundsTheStack
+	// measured 9,009 bytes on the JSON handler and 8,982 to 8,983 on
+	// the text one in this checkout. Neither is fixed: the stack's
+	// own content decides where its cut lands, so the figures move by
+	// a byte or so between runs and with the checkout. The test
+	// asserts the ceiling and that every growable field was cut,
+	// never the figures.
+	MaxPanicLogLineBytes = 10240
+)
+
+// recoverResponseWriter records whether the response has been
+// committed, which is the one thing the recoverer cannot learn from
+// the panic itself: a handler that panics after writing a status has
+// already spent the response, and a second WriteHeader would only
+// draw net/http's "superfluous response.WriteHeader" complaint
+// without changing what the client received.
+type recoverResponseWriter struct {
+	http.ResponseWriter
+
+	committed bool
+}
+
+func (w *recoverResponseWriter) WriteHeader(code int) {
+	w.committed = true
+
+	w.ResponseWriter.WriteHeader(code)
+}
+
+func (w *recoverResponseWriter) Write(b []byte) (int, error) {
+	// An unheralded Write commits the response just as surely as
+	// WriteHeader does: net/http sends 200 in front of it.
+	w.committed = true
+
+	//nolint:wrapcheck // Pass the writer's own error through unchanged.
+	return w.ResponseWriter.Write(b)
+}
+
+// Unwrap lets http.ResponseController reach the writer underneath, so
+// a handler can still flush or set a write deadline through this
+// wrapper.
+func (w *recoverResponseWriter) Unwrap() http.ResponseWriter {
+	return w.ResponseWriter
+}
+
+// Recoverer returns middleware that turns a handler panic into one
+// structured ERROR record and a 500, rather than a dropped
+// connection.
+//
+// It replaces chi's middleware.Recoverer, which does neither on a
+// current Go release. chi v1.5.5's pretty-printer scans the stack for
+// a frame beginning "panic(0x", which the runtime has not emitted
+// since it started printing "panic({0x...}"; the scan therefore never
+// terminates early, every line reaches decorateFuncCallLine, and that
+// function slices pkg[strings.Index(pkg, "."):] without checking for
+// -1. The resulting second panic escapes chi's own deferred function,
+// so its WriteHeader(500) never runs and net/http closes the
+// connection reporting its own crash instead of the original one.
+// See https://git.eeqj.de/sneak/webhooker/issues/187.
+//
+// chi v5.3.1 has since fixed both halves of that — it scans for
+// "panic(" and guards the index — so upgrading would restore the 500.
+// It would not give what this does: v5 still writes an ANSI-coloured
+// pretty stack straight to os.Stderr, outside internal/logger, outside
+// any budget, at no level the operator set.
+//
+// Where this sits in the chain is load-bearing, and routes.go states
+// it: inside everything that observes the response, so the 500 is
+// what the access log records and the metrics count, and outside the
+// sentryhttp handler, whose Repanic option depends on something
+// further out recovering what it re-raises.
+func (s *Middleware) Recoverer() func(http.Handler) http.Handler {
+	return func(next http.Handler) http.Handler {
+		return http.HandlerFunc(func(
+			w http.ResponseWriter,
+			r *http.Request,
+		) {
+			rw := &recoverResponseWriter{ResponseWriter: w}
+
+			defer func() {
+				rvr := recover()
+				if rvr == nil {
+					return
+				}
+
+				// http.ErrAbortHandler is a handler stating that it
+				// is abandoning the connection on purpose, not a
+				// fault. net/http special-cases it, suppressing both
+				// the stack trace and any response, so it is passed
+				// straight back out rather than logged and answered.
+				err, isError := rvr.(error)
+				if isError &&
+					errors.Is(err, http.ErrAbortHandler) {
+					panic(rvr)
+				}
+
+				s.logPanic(r, rvr, rw.committed)
+
+				if rw.committed {
+					return
+				}
+
+				http.Error(
+					rw,
+					http.StatusText(
+						http.StatusInternalServerError,
+					),
+					http.StatusInternalServerError,
+				)
+			}()
+
+			next.ServeHTTP(rw, r)
+		})
+	}
+}
+
+// logPanic writes the record. Every field it can grow is truncated to
+// a fixed budget, so MaxPanicLogLineBytes holds.
+//
+// The request is identified by request_id alone rather than by
+// repeating the method, URL and address: the access log line for the
+// same request carries all of those, already bounded, and — because
+// the recoverer runs inside the logging middleware — now carries the
+// 500 as its status too. Repeating them here would double those
+// budgets against a line already wider than the access log's ceiling,
+// to say a second time what one join already says.
+func (s *Middleware) logPanic(
+	r *http.Request,
+	rvr any,
+	committed bool,
+) {
+	s.log.Error("handler panic",
+		"panic", logfield.Truncate(
+			fmt.Sprint(rvr), maxPanicValueBytes,
+		),
+		"stack", logfield.Truncate(
+			string(debug.Stack()), maxPanicStackBytes,
+		),
+		"request_id", logfield.Truncate(
+			middleware.GetReqID(r.Context()),
+			maxLogRequestIDBytes,
+		),
+		"response_committed", committed,
+	)
+}

internal/middleware/recoverer_test.go

@@ -0,0 +1,645 @@
+package middleware_test
+
+import (
+	"bytes"
+	"encoding/json"
+	"io"
+	"log"
+	"net/http"
+	"net/http/httptest"
+	"strings"
+	"testing"
+
+	"github.com/go-chi/chi"
+	chimw "github.com/go-chi/chi/middleware"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"sneak.berlin/go/webhooker/internal/middleware"
+)
+
+// panicMarker is the panic value the probe handlers raise. The
+// recoverer's whole job is to put this string, and not some second
+// panic's, in front of an operator.
+const panicMarker = "QQORIGINALPANICVALUEQQ"
+
+// probeFuncName appears in the stack of every panic raised below,
+// since that is the function raising it. Its presence is how these
+// tests tell a real stack from an empty field.
+const probeFuncName = "panicProbe"
+
+// committedStatus is the status a handler sends before panicking in
+// the already-committed case. It is deliberately not 200, so a test
+// cannot pass on net/http's implicit default.
+const committedStatus = http.StatusMultiStatus
+
+// recovererProbe is a test server carrying one panicking route,
+// behind the production recoverer.
+type recovererProbe struct {
+	server *httptest.Server
+
+	// logs holds every record the middleware wrote.
+	logs *bytes.Buffer
+
+	// serverErrors holds everything net/http wrote to its own error
+	// log. A working recoverer leaves it empty: net/http only reports
+	// a request when a panic escapes the handler chain, which is the
+	// failure this issue is about.
+	serverErrors *bytes.Buffer
+}
+
+// newRecovererProbe stands up a real HTTP server — a real listener, a
+// real connection, a real client — behind the production recoverer.
+//
+// A real server rather than an httptest.ResponseRecorder, because a
+// recorder cannot express the outcome that made this a defect: chi's
+// Recoverer left net/http to close the connection, which a recorder
+// records as an ordinary unwritten response while a client sees EOF.
+// The status a client actually receives is only observable over a
+// socket.
+func newRecovererProbe(
+	t *testing.T,
+	textHandler bool,
+	handler http.HandlerFunc,
+) *recovererProbe {
+	t.Helper()
+
+	newMiddleware := capturingMiddleware
+	if textHandler {
+		newMiddleware = capturingTextMiddleware
+	}
+
+	m, logs := newMiddleware(t)
+
+	router := chi.NewRouter()
+	// The registration order the production router uses: RequestID
+	// outside so the recoverer's record can name the request,
+	// Logging outside so the recovered 500 is the status it records.
+	router.Use(chimw.RequestID)
+	router.Use(m.Logging())
+	router.Use(m.Recoverer())
+	router.Get("/probe", handler)
+
+	serverErrors := new(bytes.Buffer)
+
+	server := httptest.NewUnstartedServer(router)
+	server.Config.ErrorLog = log.New(serverErrors, "", 0)
+	server.Start()
+	t.Cleanup(server.Close)
+
+	return &recovererProbe{
+		server:       server,
+		logs:         logs,
+		serverErrors: serverErrors,
+	}
+}
+
+// get drives one request at the probe route and returns the response,
+// or the transport error if the connection was dropped instead.
+func (p *recovererProbe) get(t *testing.T) (*http.Response, error) {
+	t.Helper()
+
+	return p.getWithRequestID(t, "")
+}
+
+// getWithRequestID drives the same request carrying a client-supplied
+// X-Request-Id. chi's RequestID middleware adopts that header verbatim
+// when it is present and only generates a value when it is absent, so
+// this is the third growable field on the panic record and the only
+// one a client fills outright.
+func (p *recovererProbe) getWithRequestID(
+	t *testing.T,
+	requestID string,
+) (*http.Response, error) {
+	t.Helper()
+
+	req, err := http.NewRequestWithContext(
+		t.Context(), http.MethodGet, p.server.URL+"/probe", nil,
+	)
+	require.NoError(t, err)
+
+	if requestID != "" {
+		req.Header.Set(chimw.RequestIDHeader, requestID)
+	}
+
+	return p.server.Client().Do(req)
+}
+
+// wait shuts the server down and blocks until every in-flight request
+// has finished, which is what makes the log buffer safe to read.
+//
+// A client returns as soon as the response is complete — or, for a
+// deliberately aborted connection, as soon as it is closed — while the
+// access log line for the same request is still being written on the
+// server goroutine. It is idempotent, so a test may call it directly
+// before reading the buffer itself.
+func (p *recovererProbe) wait() {
+	p.server.Close()
+}
+
+// records decodes every JSON log line the probe captured.
+func (p *recovererProbe) records(t *testing.T) []map[string]any {
+	t.Helper()
+
+	p.wait()
+
+	var out []map[string]any
+
+	for line := range strings.SplitSeq(
+		strings.TrimSpace(p.logs.String()), "\n",
+	) {
+		if line == "" {
+			continue
+		}
+
+		record := map[string]any{}
+		require.NoError(t, json.Unmarshal([]byte(line), &record))
+
+		out = append(out, record)
+	}
+
+	return out
+}
+
+// panicRecord returns the single "handler panic" record, failing if
+// there is not exactly one.
+func (p *recovererProbe) panicRecord(t *testing.T) map[string]any {
+	t.Helper()
+
+	var found []map[string]any
+
+	for _, record := range p.records(t) {
+		if record["msg"] == "handler panic" {
+			found = append(found, record)
+		}
+	}
+
+	require.Len(
+		t, found, 1,
+		"exactly one panic record expected, log was:\n%s",
+		p.logs.String(),
+	)
+
+	return found[0]
+}
+
+// panicProbe panics with the marker. It is a named function so the
+// stack assertions have something to look for.
+func panicProbe(http.ResponseWriter, *http.Request) {
+	panic(panicMarker)
+}
+
+func TestRecovererAnswers500AndLogsTheOriginalPanic(t *testing.T) {
+	t.Parallel()
+
+	probe := newRecovererProbe(t, false, panicProbe)
+
+	resp, err := probe.get(t)
+	require.NoError(
+		t, err,
+		"a panicking handler must answer, not drop the connection",
+	)
+
+	defer func() { _ = resp.Body.Close() }()
+
+	body, err := io.ReadAll(resp.Body)
+	require.NoError(t, err)
+
+	assert.Equal(t, http.StatusInternalServerError, resp.StatusCode)
+	assert.Contains(t, string(body), "Internal Server Error")
+
+	record := probe.panicRecord(t)
+	assert.Equal(t, "ERROR", record["level"])
+	assert.Equal(t, panicMarker, record["panic"])
+	assert.Equal(t, false, record["response_committed"])
+
+	stack, ok := record["stack"].(string)
+	require.True(t, ok, "the record must carry a stack")
+	assert.Contains(
+		t, stack, probeFuncName,
+		"the stack must reach the function that panicked",
+	)
+	assert.NotContains(
+		t, stack, "slice bounds out of range",
+		"a secondary panic must not have occurred",
+	)
+
+	assert.Empty(
+		t, probe.serverErrors.String(),
+		"net/http must not have had to report anything",
+	)
+}
+
+// TestRecovererStatusReachesTheAccessLog pins the placement. The
+// recoverer runs inside the logging middleware precisely so the status
+// it writes is the one the access log records; registered outside it,
+// as chi's Recoverer was, the same request is logged as a 200 that the
+// client never received.
+func TestRecovererStatusReachesTheAccessLog(t *testing.T) {
+	t.Parallel()
+
+	probe := newRecovererProbe(t, false, panicProbe)
+
+	resp, err := probe.get(t)
+	require.NoError(t, err)
+	require.NoError(t, resp.Body.Close())
+	require.Equal(t, http.StatusInternalServerError, resp.StatusCode)
+
+	var access map[string]any
+
+	for _, record := range probe.records(t) {
+		if record["msg"] == "http request" {
+			access = record
+		}
+	}
+
+	require.NotNil(t, access, "the request must still be logged")
+	assert.EqualValues(
+		t, http.StatusInternalServerError, access["status"],
+		"the access log must record the status the client got",
+	)
+
+	// The panic record identifies its request by request_id alone,
+	// so that join has to work.
+	assert.Equal(
+		t, access["request_id"],
+		probe.panicRecord(t)["request_id"],
+	)
+	assert.NotEmpty(t, access["request_id"])
+}
+
+// TestRecovererRepanicsErrAbortHandler covers the one panic value that
+// must not be turned into a 500. net/http documents it as the way a
+// handler abandons a connection deliberately and special-cases it,
+// suppressing both the response and its own stack report.
+func TestRecovererRepanicsErrAbortHandler(t *testing.T) {
+	t.Parallel()
+
+	probe := newRecovererProbe(
+		t, false,
+		func(http.ResponseWriter, *http.Request) {
+			panic(http.ErrAbortHandler)
+		},
+	)
+
+	resp, err := probe.get(t)
+	if err == nil {
+		_ = resp.Body.Close()
+	}
+
+	require.Error(
+		t, err,
+		"an aborted handler must not answer with a status",
+	)
+
+	for _, record := range probe.records(t) {
+		assert.NotEqual(
+			t, "handler panic", record["msg"],
+			"a deliberate abort is not a fault to report",
+		)
+	}
+
+	assert.Empty(
+		t, probe.serverErrors.String(),
+		"net/http suppresses ErrAbortHandler; it must still see it",
+	)
+}
+
+// TestRecovererKeepsAnAlreadyCommittedResponse covers a handler that
+// panics after sending its status. The bytes are already on the wire,
+// so a second WriteHeader would change nothing the client sees and
+// would draw net/http's "superfluous response.WriteHeader" report.
+func TestRecovererKeepsAnAlreadyCommittedResponse(t *testing.T) {
+	t.Parallel()
+
+	probe := newRecovererProbe(
+		t, false,
+		func(w http.ResponseWriter, _ *http.Request) {
+			w.WriteHeader(committedStatus)
+			_, _ = w.Write([]byte("partial"))
+
+			panic(panicMarker)
+		},
+	)
+
+	resp, err := probe.get(t)
+	require.NoError(t, err)
+
+	defer func() { _ = resp.Body.Close() }()
+
+	body, err := io.ReadAll(resp.Body)
+	require.NoError(t, err)
+
+	assert.Equal(t, committedStatus, resp.StatusCode)
+	assert.Equal(t, "partial", string(body))
+
+	record := probe.panicRecord(t)
+	assert.Equal(t, panicMarker, record["panic"])
+	assert.Equal(
+		t, true, record["response_committed"],
+		"the record must say why no 500 was sent",
+	)
+
+	assert.NotContains(
+		t, probe.serverErrors.String(),
+		"superfluous response.WriteHeader",
+	)
+}
+
+// TestRecovererKeepsAnImplicitlyCommittedResponse is the same case
+// without an explicit WriteHeader: a bare Write commits the response
+// to 200 just as surely.
+func TestRecovererKeepsAnImplicitlyCommittedResponse(t *testing.T) {
+	t.Parallel()
+
+	probe := newRecovererProbe(
+		t, false,
+		func(w http.ResponseWriter, _ *http.Request) {
+			_, _ = w.Write([]byte("partial"))
+
+			panic(panicMarker)
+		},
+	)
+
+	resp, err := probe.get(t)
+	require.NoError(t, err)
+	require.NoError(t, resp.Body.Close())
+
+	assert.Equal(t, http.StatusOK, resp.StatusCode)
+	assert.Equal(
+		t, true, probe.panicRecord(t)["response_committed"],
+	)
+	assert.NotContains(
+		t, probe.serverErrors.String(),
+		"superfluous response.WriteHeader",
+	)
+}
+
+// panicLogHandler names one of the two handlers internal/logger can
+// install. The recoverer's probe selects between them with a bool
+// rather than by constructing one, which is why this does not reuse
+// logHandlers() the way the fills reuse escapeFills().
+type panicLogHandler struct {
+	name string
+	text bool
+}
+
+func panicLogHandlers() []panicLogHandler {
+	return []panicLogHandler{{"json", false}, {"text", true}}
+}
+
+// TestRecovererBoundsThePanicRecord holds the record to its stated
+// ceiling with a panic value the size of a request. A handler is free
+// to build a panic value out of what the client sent, so the value is
+// charged a client-sized budget even though the stack is not.
+func TestRecovererBoundsThePanicRecord(t *testing.T) {
+	t.Parallel()
+
+	for _, handler := range panicLogHandlers() {
+		// The fills are internal/middleware's own access log fills,
+		// shared rather than restated: plain text, the characters
+		// both handlers escape to two bytes, a bare C0 control, and
+		// an astral non-printable the text handler spells with a
+		// ten-byte \U escape.
+		for fillName, fillRune := range escapeFills() {
+			t.Run(handler.name+"/"+fillName, func(t *testing.T) {
+				t.Parallel()
+
+				value := strings.Repeat(
+					fillRune, oversizedSegmentBytes,
+				) + tailMarker
+
+				probe := newRecovererProbe(
+					t, handler.text,
+					func(http.ResponseWriter, *http.Request) {
+						panic(value)
+					},
+				)
+
+				resp, err := probe.get(t)
+				require.NoError(t, err)
+				require.NoError(t, resp.Body.Close())
+				require.Equal(
+					t, http.StatusInternalServerError,
+					resp.StatusCode,
+				)
+				probe.wait()
+
+				for line := range strings.SplitSeq(
+					strings.TrimSpace(probe.logs.String()), "\n",
+				) {
+					assert.LessOrEqual(
+						t, len(line),
+						middleware.MaxPanicLogLineBytes,
+						"log line exceeded its stated bound",
+					)
+					assert.NotContains(
+						t, line, tailMarker,
+						"the far end of the panic value reached "+
+							"the log, so nothing truncated it",
+					)
+				}
+			})
+		}
+	}
+}
+
+// deepPanic recurses to depth and then panics, so the stack itself
+// overruns its budget. It is the only way to exercise the stack cut:
+// the shipped middleware chain does not come close (see
+// TestPanicThroughProductionRouter in internal/server).
+func deepPanic(depth int, value string) int {
+	if depth == 0 {
+		panic(value)
+	}
+
+	return deepPanic(depth-1, value) + 1
+}
+
+// assertEveryFieldWasCut holds each of the record's three growable
+// fields to its own budget, which is what the ceiling is the sum of.
+// The stack is cut at its far end, so its near end — the panic site —
+// has to survive; the request id is the client's own bytes, so its
+// cut is the one that bounds an attacker rather than our own call
+// depth.
+func assertEveryFieldWasCut(t *testing.T, record map[string]any) {
+	t.Helper()
+
+	stack, ok := record["stack"].(string)
+	require.True(t, ok)
+	assert.True(
+		t, strings.HasSuffix(stack, truncationSuffix),
+		"an oversized stack must be marked as cut",
+	)
+	assert.Contains(
+		t, stack, "deepPanic",
+		"the near end of the stack must survive the cut",
+	)
+	assert.NotContains(
+		t, stack, "net/http.(*conn).serve",
+		"the far end is what a cut discards",
+	)
+
+	id, ok := record["request_id"].(string)
+	require.True(t, ok)
+	assert.True(
+		t, strings.HasSuffix(id, truncationSuffix),
+		"an oversized request id must be marked as cut",
+	)
+	assert.LessOrEqual(
+		t, len(id), maxRequestIDBytes+len(truncationSuffix),
+		"the request id must be held to its own budget",
+	)
+
+	value, ok := record["panic"].(string)
+	require.True(t, ok)
+	assert.True(
+		t, strings.HasSuffix(value, truncationSuffix),
+		"an oversized panic value must be marked as cut",
+	)
+}
+
+// TestRecovererBoundsTheStack drives every growable field on the
+// record past its budget at once — an oversized stack, an oversized
+// panic value and an oversized client-supplied X-Request-Id — over
+// both log handlers. It holds that line to the stated ceiling and
+// reports what it measured, and it pins that a cut stack keeps its
+// near end — the panic site — rather than its far one.
+//
+// The two fields the test picks the content of — the panic value and
+// the request id — are filled with the quotation mark. Both handlers
+// escape it to two bytes, which is exactly what logfield charges for
+// it, so each of those fields emits every byte of its budget; no fill
+// emits more, since logfield charges each rune the wider of the two
+// handlers and a field can therefore never emit more than it spent.
+// The stack is not a fill: recursion drives it past its budget and
+// the cut lands wherever its own content puts it, which is why the
+// measured widths move by a byte between runs.
+func TestRecovererBoundsTheStack(t *testing.T) {
+	t.Parallel()
+
+	for _, handler := range panicLogHandlers() {
+		t.Run(handler.name, func(t *testing.T) {
+			t.Parallel()
+
+			value := strings.Repeat(`"`, oversizedSegmentBytes) +
+				tailMarker
+			requestID := strings.Repeat(`"`, oversizedSegmentBytes) +
+				tailMarker
+
+			probe := newRecovererProbe(
+				t, handler.text,
+				func(http.ResponseWriter, *http.Request) {
+					_ = deepPanic(512, value)
+				},
+			)
+
+			resp, err := probe.getWithRequestID(t, requestID)
+			require.NoError(t, err)
+			require.NoError(t, resp.Body.Close())
+			require.Equal(
+				t, http.StatusInternalServerError, resp.StatusCode,
+			)
+			probe.wait()
+
+			// The text handler does not emit JSON, so the field-level
+			// assertions run on the JSON one; the line bound below
+			// is asserted on both, which is the point of the sweep.
+			if !handler.text {
+				assertEveryFieldWasCut(t, probe.panicRecord(t))
+			}
+
+			widest := 0
+
+			for line := range strings.SplitSeq(
+				strings.TrimSpace(probe.logs.String()), "\n",
+			) {
+				assert.LessOrEqual(
+					t, len(line), middleware.MaxPanicLogLineBytes,
+				)
+				assert.NotContains(t, line, tailMarker)
+
+				widest = max(widest, len(line))
+			}
+
+			t.Logf(
+				"widest line measured: %d bytes (ceiling %d)",
+				widest, middleware.MaxPanicLogLineBytes,
+			)
+		})
+	}
+}
+
+// TestRecovererIgnoresANonPanickingHandler is the negative control:
+// the middleware must be inert on the ordinary path.
+func TestRecovererIgnoresANonPanickingHandler(t *testing.T) {
+	t.Parallel()
+
+	probe := newRecovererProbe(
+		t, false,
+		func(w http.ResponseWriter, _ *http.Request) {
+			w.WriteHeader(http.StatusTeapot)
+		},
+	)
+
+	resp, err := probe.get(t)
+	require.NoError(t, err)
+	require.NoError(t, resp.Body.Close())
+
+	assert.Equal(t, http.StatusTeapot, resp.StatusCode)
+
+	for _, record := range probe.records(t) {
+		assert.NotEqual(t, "handler panic", record["msg"])
+	}
+}
+
+// TestRecovererKeepsResponseControllerWorking pins the Unwrap method.
+// The middleware wraps the ResponseWriter to learn whether the
+// response was committed, and a wrapper without Unwrap hides
+// net/http's own writer from http.ResponseController, so a handler
+// that flushes or sets a deadline starts failing.
+//
+// The recoverer is the only middleware in the chain here. The access
+// logger's own wrapper does not implement Unwrap, so a chain
+// containing it fails this regardless of what the recoverer does;
+// what is being pinned is that the recoverer adds no such opacity of
+// its own.
+func TestRecovererKeepsResponseControllerWorking(t *testing.T) {
+	t.Parallel()
+
+	m, _ := capturingMiddleware(t)
+
+	handler := m.Recoverer()(http.HandlerFunc(
+		func(w http.ResponseWriter, _ *http.Request) {
+			_, _ = w.Write([]byte("chunk"))
+
+			flushErr := http.NewResponseController(w).Flush()
+			if flushErr != nil {
+				http.Error(
+					w, "flush failed",
+					http.StatusInternalServerError,
+				)
+
+				return
+			}
+		},
+	))
+
+	server := httptest.NewServer(handler)
+	t.Cleanup(server.Close)
+
+	req, err := http.NewRequestWithContext(
+		t.Context(), http.MethodGet, server.URL, nil,
+	)
+	require.NoError(t, err)
+
+	resp, err := server.Client().Do(req)
+	require.NoError(t, err)
+
+	defer func() { _ = resp.Body.Close() }()
+
+	body, err := io.ReadAll(resp.Body)
+	require.NoError(t, err)
+
+	assert.Equal(t, http.StatusOK, resp.StatusCode)
+	assert.Equal(t, "chunk", string(body))
+}