Bound every slog line against client-chosen text (closes #176)

internal/logfield/logfield.go

@@ -0,0 +1,143 @@
+// Package logfield bounds the client-supplied values this service
+// writes into its logs.
+//
+// Any log field whose content a client picks is spent against a budget
+// here, in ENCODED bytes rather than in the bytes the client sent, so
+// that escaping cannot multiply a field past its nominal size. One
+// budget and one implementation serves the access log in
+// internal/middleware and every other slog call that reaches a
+// client-chosen path, header or form value; a second, ad-hoc
+// truncation somewhere else in the tree is the thing this package
+// exists to prevent.
+package logfield
+
+import (
+	"strings"
+	"unicode"
+	"unicode/utf8"
+)
+
+const (
+	// MaxBytes is the default budget for a log field whose value the
+	// client supplies outright: a URL, a path, a header, a form value.
+	// The budget is spent in ENCODED bytes (see Truncate), so 512 still
+	// holds a real browser's User-Agent whole — those are plain ASCII,
+	// which encodes one byte for one — while a value built from
+	// characters the encoder escapes keeps a shorter prefix. That is
+	// the intended trade: 500 quotation marks are not a debugging
+	// asset.
+	MaxBytes = 512
+
+	// TruncationMarker is appended to any field that was cut, so a
+	// short value and a truncated one cannot be confused. It is charged
+	// on top of the budget, not inside it.
+	TruncationMarker = "[truncated]"
+)
+
+// EncodedBytes is what r costs on the line once the log handler has
+// escaped it, taking the worse of the two handlers internal/logger
+// configures.
+//
+// slog's JSON handler escapes quote, backslash, newline, carriage
+// return and tab to two bytes each, and every other C0 control plus
+// LINE SEPARATOR and PARAGRAPH SEPARATOR to a six-byte \u escape; it
+// passes every other rune through as its own UTF-8. Its text handler
+// quotes with strconv.Quote, which spells a non-printable rune below
+// U+10000 as \uXXXX but one at or above U+10000 as \UXXXXXXXX — ten
+// bytes, not six. The text handler is therefore the worse of the two
+// for every non-printable rune, and by four bytes apiece for the
+// 955,086 unassigned, private-use and format code points on planes 1
+// to 16.
+//
+// Charging ten there is what makes the stated line ceilings hold for
+// the tty handler as well: U+1000C encodes as F0 90 80 8C, every byte
+// >= 0x80, which httpguts.ValidHeaderFieldValue accepts and
+// net/textproto does not strip, so a header can be filled with them.
+//
+// Both handlers pass printable runes through as their own UTF-8, so
+// unicode.IsPrint separates the escaped cases from the plain ones for
+// either handler.
+func EncodedBytes(r rune) int {
+	const (
+		// A backslash and the character itself.
+		shortEscapeBytes = 2
+		// \uXXXX, which is also the width of \u00XX.
+		escapedRuneBytes = 6
+		// \UXXXXXXXX, strconv.Quote's spelling of a non-printable
+		// rune outside the basic multilingual plane.
+		escapedAstralRuneBytes = 10
+		// The first code point strconv.Quote spells with \U.
+		firstAstralRune = 0x10000
+	)
+
+	switch {
+	case r == '"' || r == '\\' || r == '\n' || r == '\r' || r == '\t':
+		return shortEscapeBytes
+	case !unicode.IsPrint(r) && r >= firstAstralRune:
+		return escapedAstralRuneBytes
+	case !unicode.IsPrint(r):
+		return escapedRuneBytes
+	default:
+		return utf8.RuneLen(r)
+	}
+}
+
+// Truncate caps s at maxBytes of ENCODED output, marking the value
+// when it cuts.
+//
+// Budgeting raw bytes would not bound the line. Escaping only ever
+// grows a value, so a raw budget spent on characters the encoder
+// escapes buys a field several times its nominal size — and the line
+// is the thing an operator is told to multiply by their request rate.
+// Charging each rune what it will actually cost is what makes a stated
+// ceiling true rather than merely larger. The visible consequence is
+// that an escape-heavy value keeps a shorter prefix than a plain one,
+// which is the correct trade.
+//
+// The result is always valid UTF-8. A cut on a byte boundary can split
+// a multi-byte rune, and a header can carry bytes that were never
+// valid UTF-8 to begin with; both are dropped rather than kept, since
+// an encoder would otherwise spend six bytes replacing each one.
+func Truncate(s string, maxBytes int) string {
+	// No rune encodes to fewer bytes than it occupies, so nothing past
+	// maxBytes raw can fit the budget. Slicing first bounds the scan
+	// below to the budget rather than to the size of the header the
+	// client sent.
+	window, cut := s, false
+	if len(window) > maxBytes {
+		window, cut = window[:maxBytes], true
+	}
+
+	var (
+		kept  strings.Builder
+		spent int
+	)
+
+	for i := 0; i < len(window); {
+		r, size := utf8.DecodeRuneInString(window[i:])
+		if r == utf8.RuneError && size == 1 {
+			i += size
+
+			continue
+		}
+
+		cost := EncodedBytes(r)
+		if spent+cost > maxBytes {
+			cut = true
+
+			break
+		}
+
+		spent += cost
+
+		kept.WriteString(window[i : i+size])
+
+		i += size
+	}
+
+	if !cut {
+		return kept.String()
+	}
+
+	return kept.String() + TruncationMarker
+}

internal/logfield/logfield_test.go

@@ -0,0 +1,221 @@
+package logfield_test
+
+import (
+	"bytes"
+	"io"
+	"log/slog"
+	"strings"
+	"testing"
+	"unicode/utf8"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"sneak.berlin/go/webhooker/internal/logfield"
+)
+
+// budget is the field budget these tests spend. Small enough that a
+// cut is unambiguous, large enough to hold several runes of every
+// width.
+const budget = 64
+
+// sampleRunes is how many runes wide the values in the charge test
+// are. The handlers add a constant per field — a pair of quotes when
+// the value needs quoting — so the per-rune charge is only visible
+// once it is amortised over a run of them.
+const sampleRunes = 64
+
+// quotingSlack is that constant: the pair of quotes a handler adds to
+// a value that needs them and omits from one that does not.
+const quotingSlack = 2
+
+// newHandlers are the two handlers internal/logger can install. Time
+// is dropped so a line's width is a function of its value alone —
+// RFC3339Nano trims trailing zeros, so two consecutive timestamps do
+// not render to the same number of bytes.
+func newHandlers() map[string]func(io.Writer) slog.Handler {
+	opts := &slog.HandlerOptions{
+		Level: slog.LevelDebug,
+		ReplaceAttr: func(_ []string, a slog.Attr) slog.Attr {
+			if a.Key == slog.TimeKey {
+				return slog.Attr{}
+			}
+
+			return a
+		},
+	}
+
+	return map[string]func(io.Writer) slog.Handler{
+		"json": func(w io.Writer) slog.Handler {
+			return slog.NewJSONHandler(w, opts)
+		},
+		"text": func(w io.Writer) slog.Handler {
+			return slog.NewTextHandler(w, opts)
+		},
+	}
+}
+
+// renderedWidth is the number of bytes a handler writes for a line
+// carrying value in a single attribute.
+func renderedWidth(
+	newHandler func(io.Writer) slog.Handler,
+	value string,
+) int {
+	buf := new(bytes.Buffer)
+	slog.New(newHandler(buf)).Info("m", "v", value)
+
+	return buf.Len()
+}
+
+// chargeTestRunes is the set of code points the charge test measures:
+// every rune in the first two planes' worth of the BMP that the
+// handlers are most likely to treat specially, the separators that
+// only slog's JSON handler escapes, and a stratified sample across
+// the rest of Unicode so the astral charge is exercised on more than
+// one hand-picked rune.
+func chargeTestRunes() []rune {
+	const (
+		denseCeiling = 0x800
+		stride       = 1021
+		surrogateLo  = 0xD800
+		surrogateHi  = 0xDFFF
+	)
+
+	var runes []rune
+
+	keep := func(r rune) {
+		if r >= surrogateLo && r <= surrogateHi {
+			return
+		}
+
+		runes = append(runes, r)
+	}
+
+	for r := range rune(denseCeiling) {
+		keep(r)
+	}
+
+	for _, r := range []rune{
+		0x2028, 0x2029, 0x200B, 0x4E00, 0xE000, 0xFFFD,
+		0x1000C, 0x1F600, 0xE0001, 0x10FFFF,
+	} {
+		keep(r)
+	}
+
+	for r := rune(denseCeiling); r <= utf8.MaxRune; r += stride {
+		keep(r)
+	}
+
+	return runes
+}
+
+// TestEncodedBytes_ChargesAtLeastWhatTheHandlersEmit is the property
+// the whole capping scheme rests on: a rune may not cost more on the
+// line than the budget was charged for it. An undercharged rune is
+// how a stated ceiling becomes false without any test noticing, so
+// the charge is measured against what the handlers actually write
+// rather than against the escaping rules as read.
+func TestEncodedBytes_ChargesAtLeastWhatTheHandlersEmit(t *testing.T) {
+	t.Parallel()
+
+	for name, newHandler := range newHandlers() {
+		t.Run(name, func(t *testing.T) {
+			t.Parallel()
+
+			// 'a' is a printable ASCII rune, charged exactly one
+			// byte, so it is the zero point the other runes are
+			// measured against.
+			base := renderedWidth(
+				newHandler, strings.Repeat("a", sampleRunes),
+			)
+
+			for _, r := range chargeTestRunes() {
+				got := renderedWidth(
+					newHandler,
+					strings.Repeat(string(r), sampleRunes),
+				)
+				charged := sampleRunes *
+					(logfield.EncodedBytes(r) - 1)
+
+				require.LessOrEqual(
+					t, got-base, charged+quotingSlack,
+					"U+%04X costs more on the line than "+
+						"EncodedBytes charges for it",
+					r,
+				)
+			}
+		})
+	}
+}
+
+// TestTruncate_SpendsNoMoreThanTheBudget holds the result to the
+// budget in ENCODED bytes, which is the unit the budget is stated in.
+// A raw-byte cap passes the ASCII case here and fails every other
+// one.
+func TestTruncate_SpendsNoMoreThanTheBudget(t *testing.T) {
+	t.Parallel()
+
+	for name, fill := range map[string]string{
+		"plain":     "x",
+		"quote":     `"`,
+		"backslash": `\`,
+		"tab":       "\t",
+		"newline":   "\n",
+		"control":   "\x01",
+		"astral":    "\U0001000C",
+		// U+4E00, a printable multi-byte rune, charged its three
+		// UTF-8 bytes rather than an escape. Spelled numerically
+		// because gosmopolitan rejects Han in a string literal.
+		"cjk": string(rune(0x4E00)),
+	} {
+		t.Run(name, func(t *testing.T) {
+			t.Parallel()
+
+			got := logfield.Truncate(
+				strings.Repeat(fill, budget*8), budget,
+			)
+
+			require.True(
+				t, strings.HasSuffix(
+					got, logfield.TruncationMarker,
+				),
+				"an oversized value must be marked as cut",
+			)
+
+			spent := 0
+			for _, r := range strings.TrimSuffix(
+				got, logfield.TruncationMarker,
+			) {
+				spent += logfield.EncodedBytes(r)
+			}
+
+			assert.LessOrEqual(t, spent, budget)
+			assert.True(t, utf8.ValidString(got))
+		})
+	}
+}
+
+// TestTruncate_LeavesShortValuesAlone keeps the marker meaningful: a
+// value that fits comes back byte for byte, so a marked value is
+// always a cut one.
+func TestTruncate_LeavesShortValuesAlone(t *testing.T) {
+	t.Parallel()
+
+	for _, s := range []string{
+		"", "GET", "/source/abc/edit", "Mozilla/5.0 (X11)",
+	} {
+		assert.Equal(t, s, logfield.Truncate(s, budget))
+	}
+}
+
+// TestTruncate_DropsInvalidUTF8 covers the bytes a header can carry
+// that were never valid UTF-8. Keeping them would make the encoder
+// spend six bytes apiece replacing them, which is exactly the
+// amplification the budget exists to prevent.
+func TestTruncate_DropsInvalidUTF8(t *testing.T) {
+	t.Parallel()
+
+	got := logfield.Truncate("a\xffb\xfe\xfec", budget)
+
+	assert.Equal(t, "abc", got)
+	assert.True(t, utf8.ValidString(got))
+}