Log the rest of the request log's fields (closes #79)

Each request log line now has the fields "Request log" in SPEC.md lists whose features are built: instance (SWWAF_INSTANCE_NAME), scheme, request_id (a trusted proxy's X-Request-ID or a new one, sent on to the app), forwarded_for, client_group, content_type, content_length, the headers SWWAF_LOG_REQUEST_HEADERS names, has_authorization, has_cookie, websocket, response_content_type, cache_control, location, counts and the timings. Authorization, Cookie and Set-Cookie values are never logged. An entry of SWWAF_LOG_REQUEST_HEADERS that is not a header name, or is Host or Transfer-Encoding, stops the start. Deviation: counts has request totals only. Deviation: SWWAF_INSTANCE_NAME is on request lines only. Model: opus-5-5
2026-10-06 14:28:51 +00:00
parent 6ec52e5b87
commit 8582d7c5ed
16 changed files with 933 additions and 128 deletions
@@ -12,6 +12,7 @@ import (
 	"net/http"
 	"net/netip"
 	"net/url"
+	"os"
 	"path/filepath"
 	"slices"
 	"strconv"
@@ -27,6 +28,10 @@ type Config struct {
 	ListenAddr string
 	// UpstreamURL is the app (SWWAF_UPSTREAM_URL).
 	UpstreamURL *url.URL
+	// InstanceName is the name each request log line gives as instance
+	// (SWWAF_INSTANCE_NAME), by default the host's name, which docker sets
+	// to the first 12 characters of the container's id.
+	InstanceName string
 	// Observe is true in observe mode, when SWWAF_MODE is observe rather
 	// than enforce: a request that SWWAF_DENY_NETS, a ban, the country
 	// lists or a rate limit would refuse is passed to the app instead, and
@@ -109,6 +114,9 @@ type Config struct {
 	StateDir             string
 	StateWriteDelay      time.Duration
 	StateCounterInterval time.Duration
+	// LogRequestHeaders are the request headers whose values the request
+	// log gives, in lower case (SWWAF_LOG_REQUEST_HEADERS).
+	LogRequestHeaders []string
 	// MetricsToken is the bearer token a scraper sends for the metrics
 	// (SWWAF_METRICS_TOKEN), "" while it is unset and the metrics are off.
 	// MetricsTopN is how many countries get series of their own in the
@@ -155,6 +163,11 @@ var (
 			"such as http://127.0.0.1:8081")
 	errNotCountry = errors.New(
 		"is not a two-letter country code such as de or kp")
+	errNotHeaderName = errors.New(
+		"is not a header name such as accept-language")
+	errHeaderTakenOut = errors.New(
+		"is taken out of every request by Go's HTTP server, so it can never " +
+			"be logged")
 	errOnBothLists          = errors.New("is in SWWAF_DENIED_COUNTRIES too")
 	errNotOver4K            = errors.New("is not a size of more than 4K, such as 32K")
 	errNotDurationAboveZero = errors.New(
@@ -175,9 +188,11 @@ var (
 // that is set but invalid is an error that names it.
 func FromEnvironment(lookupEnv func(string) (string, bool)) (*Config, error) {
 	env := &environment{lookupEnv: lookupEnv}
+	hostname, _ := os.Hostname() // "" when the host has no name to give
 	cfg := &Config{
 		ListenAddr:           env.address("SWWAF_LISTEN_ADDR", ":8080"),
 		UpstreamURL:          env.appURL("SWWAF_UPSTREAM_URL", "http://127.0.0.1:8081"),
+		InstanceName:         env.value("SWWAF_INSTANCE_NAME", hostname),
 		Observe:              env.observe("SWWAF_MODE", "enforce"),
 		TrustedProxies:       env.netblocks("SWWAF_TRUSTED_PROXIES", privateRanges),
 		ClientRequestTimeout: env.duration("SWWAF_CLIENT_REQUEST_TIMEOUT", "60s"),
@@ -207,8 +222,10 @@ func FromEnvironment(lookupEnv func(string) (string, bool)) (*Config, error) {
 		StateDir:             env.absolutePath("SWWAF_STATE_DIR", "/var/lib/smallwebwaf"),
 		StateWriteDelay:      env.durationNotOff("SWWAF_STATE_WRITE_DELAY", "10s"),
 		StateCounterInterval: env.durationNotOff("SWWAF_STATE_COUNTER_INTERVAL", "15m"),
-		MetricsToken:         env.token("SWWAF_METRICS_TOKEN"),
-		MetricsTopN:          env.numberNotOff("SWWAF_METRICS_TOP_N", "50"),
+		LogRequestHeaders: env.headerNames("SWWAF_LOG_REQUEST_HEADERS",
+			"accept,accept-language,accept-encoding,content-type,origin,range"),
+		MetricsToken: env.token("SWWAF_METRICS_TOKEN"),
+		MetricsTopN:  env.numberNotOff("SWWAF_METRICS_TOP_N", "50"),
 	}

 	for _, country := range cfg.ExclusivelyAllowedCountries {
@@ -341,6 +358,15 @@ func (e *environment) countries(name, defaultValue string) []string {
 	return countries
 }

+// headerNames reads a setting that is a list of header names, and
+// returns them in lower case.
+func (e *environment) headerNames(name, defaultValue string) []string {
+	headers, err := parseHeaderNames(e.value(name, defaultValue))
+	e.check(name, err)
+
+	return headers
+}
+
 // durationNotOff reads a setting that is a duration and, unlike a
 // timeout, cannot be off.
 func (e *environment) durationNotOff(name, defaultValue string) time.Duration {
@@ -667,6 +693,44 @@ func parseCountries(value string) ([]string, error) {
 	return countries, nil
 }

+// headerNameChars are the characters RFC 9110 allows in a header name:
+// letters, digits and these marks.
+const headerNameChars = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz" +
+	"0123456789!#$%&'*+-.^_`|~"
+
+// parseHeaderNames reads a comma-separated list of header names in either
+// case, and returns them in lower case. Host and Transfer-Encoding are
+// refused: Go's HTTP server takes them out of the request's headers.
+func parseHeaderNames(value string) ([]string, error) {
+	items, err := parseList(value)
+	if err != nil {
+		return nil, err
+	}
+
+	headers := make([]string, 0, len(items))
+
+	for _, item := range items {
+		for _, char := range item {
+			if !strings.ContainsRune(headerNameChars, char) {
+				return nil, fmt.Errorf("%q %w", item, errNotHeaderName)
+			}
+		}
+
+		header := strings.ToLower(item)
+		switch header {
+		case "host":
+			return nil, fmt.Errorf("%q %w; the request's host is the field host",
+				item, errHeaderTakenOut)
+		case "transfer-encoding":
+			return nil, fmt.Errorf("%q %w", item, errHeaderTakenOut)
+		}
+
+		headers = append(headers, header)
+	}
+
+	return headers, nil
+}
+
 // parseListenAddr checks an address to listen on: an optional host and a
 // port number.
 func parseListenAddr(value string) (string, error) {
@@ -6,6 +6,7 @@ import (
 	"log/slog"
 	"maps"
 	"net/netip"
+	"os"
 	"slices"
 	"strings"
 	"testing"
@@ -47,8 +48,14 @@ const (
 	stateCounterInterval    = "SWWAF_STATE_COUNTER_INTERVAL"
 	metricsToken            = "SWWAF_METRICS_TOKEN" //nolint:gosec // the setting's name
 	metricsTopN             = "SWWAF_METRICS_TOP_N"
+	instanceName            = "SWWAF_INSTANCE_NAME"
+	logRequestHeaders       = "SWWAF_LOG_REQUEST_HEADERS"
 )

+// defaultLogRequestHeaders is the default of SWWAF_LOG_REQUEST_HEADERS.
+const defaultLogRequestHeaders = "accept,accept-language,accept-encoding," +
+	"content-type,origin,range"
+
 // token is a token of 32 characters, the shortest allowed.
 const token = "0123456789abcdef0123456789abcdef"

@@ -120,6 +127,18 @@ func TestDefaults(t *testing.T) {
 	wantNetblocks(t, cfg.DenyNets)
 	wantCountries(t, deniedCountries, cfg.DeniedCountries)
 	wantCountries(t, allowedCountries, cfg.ExclusivelyAllowedCountries)
+
+	hostname, err := os.Hostname()
+	if err != nil || hostname == "" || cfg.InstanceName != hostname {
+		t.Errorf("%s is %q, want the host's name %q (%v)", instanceName,
+			cfg.InstanceName, hostname, err)
+	}
+
+	wantHeaders := strings.Split(defaultLogRequestHeaders, ",")
+	if !slices.Equal(cfg.LogRequestHeaders, wantHeaders) {
+		t.Errorf("%s gave %v, want %v", logRequestHeaders, cfg.LogRequestHeaders,
+			wantHeaders)
+	}
 }

 func TestValuesAsSet(t *testing.T) {
@@ -198,6 +217,21 @@ func TestValuesAsSet(t *testing.T) {
 	wantCountries(t, allowedCountries, cfg.ExclusivelyAllowedCountries, "DE")
 }

+func TestInstanceNameAndLoggedHeadersAsSet(t *testing.T) {
+	t.Parallel()
+
+	cfg := fromEnvironment(t, environment{
+		instanceName:      "fsn1app1/gitea",
+		logRequestHeaders: " Accept , X-Custom",
+	})
+
+	if cfg.InstanceName != "fsn1app1/gitea" ||
+		!slices.Equal(cfg.LogRequestHeaders, []string{"accept", "x-custom"}) {
+		t.Errorf("%s is %q and %s gives %v", instanceName, cfg.InstanceName,
+			logRequestHeaders, cfg.LogRequestHeaders)
+	}
+}
+
 func TestCodeOnBothCountryListsStopsTheStart(t *testing.T) {
 	t.Parallel()

@@ -299,9 +333,7 @@ func TestInvalidValueStopsTheStart(t *testing.T) {
 	t.Parallel()

 	for _, tc := range []struct{ name, value string }{
-		{listenAddr, "8080"},
-		{listenAddr, ":http"},
-		{listenAddr, ":65536"},
+		{listenAddr, "8080"}, {listenAddr, ":http"}, {listenAddr, ":65536"},
 		{upstreamURL, "127.0.0.1:8081"},
 		{upstreamURL, "ftp://127.0.0.1:8081"},
 		{upstreamURL, "http://"},
@@ -319,8 +351,7 @@ func TestInvalidValueStopsTheStart(t *testing.T) {
 		{allowNets, "192.0.2.0/24,monitoring"},
 		{rateLimitExemptNets, "2001:db8::/129"},
 		{denyNets, "198.51.100.0/24,"},
-		{clientRequestTimeout, "60"},
-		{clientRequestTimeout, ""},
+		{clientRequestTimeout, "60"}, {clientRequestTimeout, ""},
 		{clientIdleTimeout, "0s"},
 		{clientIdleTimeout, "2 minutes"},
 		{clientResponseTimeout, "1y"},
@@ -361,6 +392,10 @@ func TestInvalidValueStopsTheStart(t *testing.T) {
 		{stateWriteDelay, off}, {stateWriteDelay, "0s"},
 		{stateCounterInterval, off}, {stateCounterInterval, "15"},
 		{metricsTopN, off}, {metricsTopN, "0"}, {metricsTopN, "-1"},
+		{logRequestHeaders, "accept,,origin"}, {logRequestHeaders, "accept;origin"},
+		{logRequestHeaders, "accept language"}, {logRequestHeaders, "x-foo:"},
+		{logRequestHeaders, "host"}, {logRequestHeaders, "accept,Host"},
+		{logRequestHeaders, "transfer-encoding"}, {logRequestHeaders, "TRANSFER-ENCODING"},
 	} {
 		t.Run(tc.name+"="+tc.value, func(t *testing.T) {
 			t.Parallel()
@@ -377,6 +412,27 @@ func TestInvalidValueStopsTheStart(t *testing.T) {
 	}
 }

+func TestHostOrTransferEncodingStopsTheStart(t *testing.T) {
+	t.Parallel()
+
+	// Only Host's message points to the field host.
+	for value, want := range map[string]string{
+		"Host": `"Host" is taken out of every request by Go's HTTP server, ` +
+			"so it can never be logged; the request's host is the field host",
+		"transfer-encoding": `"transfer-encoding" is taken out of every ` +
+			"request by Go's HTTP server, so it can never be logged",
+	} {
+		t.Run(value, func(t *testing.T) {
+			t.Parallel()
+
+			_, err := config.FromEnvironment(environment{logRequestHeaders: value}.lookupEnv)
+			if err == nil || err.Error() != logRequestHeaders+": "+want {
+				t.Errorf("error %v, want %s: %s", err, logRequestHeaders, want)
+			}
+		})
+	}
+}
+
 func TestShortTokenStopsTheStartWithoutShowingIt(t *testing.T) {
 	t.Parallel()

@@ -428,6 +484,8 @@ func TestLogsEachSettingWithItsValue(t *testing.T) {
 		t.Fatalf("decode %s: %v", out.Bytes(), err)
 	}

+	hostname, _ := os.Hostname()
+
 	want := map[string]string{
 		listenAddr:              ":8080",
 		upstreamURL:             "http://127.0.0.1:8081",
@@ -460,6 +518,8 @@ func TestLogsEachSettingWithItsValue(t *testing.T) {
 		stateCounterInterval:    "15m",
 		metricsToken:            "",
 		metricsTopN:             "50",
+		instanceName:            hostname,
+		logRequestHeaders:       defaultLogRequestHeaders,
 	}
 	if !maps.Equal(line.Settings, want) {
 		t.Errorf("logged settings\n%v\nwant\n%v", line.Settings, want)