smallwebwaf/internal/requestlog/requestlog.go

// Package requestlog writes the lines smallwebwaf prints on stdout: one
// JSON object per request, two from servewaf and a short one from rproxy, marked
// "type":"request", and the process's own messages as JSON lines marked
// "type":"process".
package requestlog

import (
	"encoding/json"
	"fmt"
	"io"
	"log/slog"
	"time"

	"sneak.berlin/go/smallwebwaf/internal/ratelimit"
)

// The action a request line names: what smallwebwaf did with the
// request.
const (
	// ActionForward is a request passed to the app.
	ActionForward = "forward"
	// ActionTooLarge is a request or response over its size limit.
	ActionTooLarge = "too_large"
	// ActionTimedOut is a request or response that ran out of time.
	ActionTimedOut = "timed_out"
	// ActionUpstreamError is a request the app could not be reached
	// for, or whose answer could not be passed on.
	ActionUpstreamError = "upstream_error"
	// ActionRateLimited is a request refused because it took its client
	// over a rate limit, which bans the client.
	ActionRateLimited = "rate_limited"
	// ActionBanned is a request refused because a ban covers its client,
	// or because it matched a ban rule, asked for a trap path or the
	// CrowdSec decision list lists its client, each of which bans the
	// client.
	ActionBanned = "banned"
	// ActionRuleBlocked is a request refused because it matched a block
	// rule.
	ActionRuleBlocked = "rule_blocked"
	// ActionWAFBlocked is a request refused because the Core Rule Set
	// scored it at or over SWWAF_WAF_ANOMALY_THRESHOLD.
	ActionWAFBlocked = "waf_blocked"
	// ActionDenied is a request refused because its client is in
	// SWWAF_DENY_NETS, in a blocklist while SWWAF_BLOCKLIST_ACTION is deny,
	// or listed by a DNSBL zone, or scored a hit by AbuseIPDB, while
	// SWWAF_REPUTATION_ACTION is deny.
	ActionDenied = "denied"
	// ActionCountryDenied is a request refused for its client's country.
	ActionCountryDenied = "country_denied"
	// ActionAdmin is a request smallwebwaf answered at one of its own
	// endpoints, under /_smallwebwaf/.
	ActionAdmin = "admin"
)

// OffenceLimit is the offence a request line names for a request that
// broke a rate limit or the error burst, or whose bytes broke a byte
// limit.
const OffenceLimit = "limit"

// LimitHitErrorBurst is the limit_hit a request line names for a request
// that broke the error burst.
const LimitHitErrorBurst = "error_burst"

// The servewaf_line a request line names, which says which of servewaf's two
// lines about a question it is.
const (
	// ServeWAFLineDecision is servewaf's line at its decision.
	ServeWAFLineDecision = "decision"
	// ServeWAFLineReport is servewaf's line when rproxy's report comes, with
	// the decision's fields and the report's together.
	ServeWAFLineReport = "report"
)

// timeLayout is RFC 3339 with milliseconds.
const timeLayout = "2006-01-02T15:04:05.000Z07:00"

// Line is one request's line in the request log. The field names, and
// their order, are those of docs/request-log.md. A field that may not
// apply to a request is left out of its line when it does not.
//
//nolint:tagliatelle // docs/request-log.md names the fields in snake_case
type Line struct {
	Type string `json:"type"`
	// ServeWAFLine is ServeWAFLineDecision or ServeWAFLineReport on servewaf's
	// two lines about a question.
	ServeWAFLine string `json:"servewaf_line,omitempty"`

	// The standard web log fields. Scheme is how the client reached
	// smallwebwaf, or the trusted proxy in front of it.
	Time          string `json:"time"`
	Instance      string `json:"instance"`
	ClientIP      string `json:"client_ip"`
	Method        string `json:"method"`
	Scheme        string `json:"scheme"`
	Host          string `json:"host"`
	Path          string `json:"path"`
	Query         string `json:"query"`
	Protocol      string `json:"protocol"`
	Status        int    `json:"status"`
	RequestBytes  int64  `json:"request_bytes"`
	ResponseBytes int64  `json:"response_bytes"`
	Referer       string `json:"referer"`
	UserAgent     string `json:"user_agent"`

	// Request detail. RequestID is the X-Request-ID a trusted proxy sent,
	// or a new one, and is sent on to the app. ForwardedFor is the
	// X-Forwarded-For header as received. ClientGroup is the netblock the
	// client is counted as. ASN, ASName and Country are the client's AS
	// number, AS name and country, as looked up.
	RequestID    string `json:"request_id"`
	PeerIP       string `json:"peer_ip"`
	ForwardedFor string `json:"forwarded_for,omitempty"`
	ClientGroup  string `json:"client_group"`
	ASN          string `json:"asn"`
	ASName       string `json:"as_name"`
	Country      string `json:"country"`
	ContentType  string `json:"content_type,omitempty"`
	// ContentLength is the length of its body the request announced.
	ContentLength int64 `json:"content_length,omitempty"`
	// RequestHeaders are the headers SWWAF_LOG_REQUEST_HEADERS names that
	// the request carried, by name in lower case.
	RequestHeaders   map[string]string `json:"request_headers,omitempty"`
	HasAuthorization bool              `json:"has_authorization,omitempty"`
	HasCookie        bool              `json:"has_cookie,omitempty"`
	// Websocket is true when the connection was upgraded, as for a
	// WebSocket.
	Websocket bool `json:"websocket,omitempty"`

	// Response detail, from the headers of the answer: the app's, as
	// passed on, or those of smallwebwaf's own. Aborted is true when the
	// client went away early.
	ResponseContentType string `json:"response_content_type,omitempty"`
	UpstreamStatus      int    `json:"upstream_status,omitempty"`
	CacheControl        string `json:"cache_control,omitempty"`
	Location            string `json:"location,omitempty"`
	Aborted             bool   `json:"aborted,omitempty"`

	// The decision.
	Action string `json:"action"`
	// WouldAction is, in observe mode, the action enforce mode would have
	// taken with a request it would have refused: ActionDenied,
	// ActionBanned, ActionCountryDenied, ActionRateLimited,
	// ActionRuleBlocked or ActionWAFBlocked.
	WouldAction string `json:"would_action,omitempty"`
	// LimitPercent and LimitPercentSetting are, for a request the rate
	// limits counted whose client a biased threshold gives a percentage of
	// the rate limits below 100, that percentage and the setting that gave
	// it. BytesPercent and BytesPercentSetting are the same for the byte
	// limits.
	LimitPercent        *int64 `json:"limit_percent,omitempty"`
	LimitPercentSetting string `json:"limit_percent_setting,omitempty"`
	BytesPercent        *int64 `json:"bytes_percent,omitempty"`
	BytesPercentSetting string `json:"bytes_percent_setting,omitempty"`
	// Counts are, for a request the rate limits counted, the client's
	// requests as they counted them with this one, and its bytes as the
	// byte limits counted them, with this request's once it has ended if
	// they count them.
	Counts ratelimit.Counts `json:"counts,omitzero"`
	// RuleIDs are the ids of the rule file rules the request matched.
	RuleIDs []string `json:"rule_ids,omitempty"`
	// WAFRuleIDs are the ids of the Core Rule Set's rules the request
	// matched, and WAFScore its anomaly score, nil for a request the Core
	// Rule Set did not inspect.
	WAFRuleIDs []int `json:"waf_rule_ids,omitempty"`
	WAFScore   *int  `json:"waf_score,omitempty"`
	// LimitHit is the window whose limit the request went over, named as
	// Counts names its count: minute, hour or day for a rate limit, and
	// minute_bytes, hour_bytes or day_bytes for a byte limit; or
	// LimitHitErrorBurst for the error burst.
	LimitHit string `json:"limit_hit,omitempty"`
	// Reputation are the URLs of the blocklists that list the client, then
	// that of the CrowdSec decision list when it does, then the DNSBL zones
	// whose verdict lists it, their keys masked, then abuseipdb when its
	// score is a hit.
	Reputation []string `json:"reputation,omitempty"`
	// Offence is the offence the request was held as, OffenceLimit.
	Offence string `json:"offence,omitempty"`
	// BanExpires is when the ban the request made, or was refused under,
	// ends: a time, or "permanent".
	BanExpires string `json:"ban_expires,omitempty"`

	// The timings, in milliseconds. DurationChecks is the time until the
	// checks were done, and DurationWAF the part of it the Core Rule Set
	// took. DurationUpstreamConnect, DurationUpstreamFirstByte and
	// DurationUpstreamTotal run from when the request was handed to the
	// app: until there was a connection to it, until the first byte of its
	// answer arrived, and until the end. Each but DurationTotal is nil for
	// a request that did not get that far.
	DurationTotal             float64  `json:"duration_total"`
	DurationChecks            *float64 `json:"duration_checks,omitempty"`
	DurationWAF               *float64 `json:"duration_waf,omitempty"`
	DurationUpstreamConnect   *float64 `json:"duration_upstream_connect,omitempty"`
	DurationUpstreamFirstByte *float64 `json:"duration_upstream_first_byte,omitempty"`
	DurationUpstreamTotal     *float64 `json:"duration_upstream_total,omitempty"`
}

// RProxyLine is rproxy's short line in the request log about one request. The
// field names, and their order, are those of docs/request-log.md. A field that
// does not apply to a request is left out of its line.
//
//nolint:tagliatelle // docs/request-log.md names the fields in snake_case
type RProxyLine struct {
	Type          string `json:"type"`
	Time          string `json:"time"`
	RequestID     string `json:"request_id"`
	ClientIP      string `json:"client_ip"`
	Host          string `json:"host"`
	Method        string `json:"method"`
	Path          string `json:"path"`
	Query         string `json:"query"`
	Status        int    `json:"status"`
	RequestBytes  int64  `json:"request_bytes"`
	ResponseBytes int64  `json:"response_bytes"`
	// DurationRequest runs in milliseconds from the request's headers being read
	// until the app has been sent the whole request, or the answer has started
	// if that is sooner. DurationResponse runs from then until the answer ends.
	DurationRequest  float64 `json:"duration_request"`
	DurationResponse float64 `json:"duration_response"`
	// Disposition is servewaf's answer or, while DefaultDisposition is true, the
	// value of SWRP_DEFAULT_DISPOSITION. It is "" when servewaf was not asked.
	Disposition        string `json:"disposition,omitempty"`
	DefaultDisposition bool   `json:"default_disposition,omitempty"`
}

// Write writes line to w as one JSON line marked "type":"request".
func Write(w io.Writer, line *Line) error {
	line.Type = "request"

	return writeJSONLine(w, line)
}

// WriteRProxyLine writes line to w as one JSON line marked "type":"request".
func WriteRProxyLine(w io.Writer, line *RProxyLine) error {
	line.Type = "request"

	return writeJSONLine(w, line)
}

// writeJSONLine writes line to w as one JSON line.
func writeJSONLine(w io.Writer, line any) error {
	encoded, err := json.Marshal(line)
	if err != nil {
		return fmt.Errorf("encode the request log line: %w", err)
	}

	_, err = w.Write(append(encoded, '\n'))
	if err != nil {
		return fmt.Errorf("write the request log line: %w", err)
	}

	return nil
}

// FormatTime formats t for a line's time field: RFC 3339 in UTC, with
// milliseconds.
func FormatTime(t time.Time) string {
	return t.UTC().Format(timeLayout)
}

// Milliseconds is d in milliseconds, to the microsecond.
func Milliseconds(d time.Duration) float64 {
	return float64(d.Microseconds()) / float64(time.Millisecond/time.Microsecond)
}

// NewProcessLogger returns the logger for the process's own messages:
// JSON lines on w, marked "type":"process", with the time in the same form
// as a request line's, and instanceName, SWWAF_INSTANCE_NAME, as instance.
// It writes only the messages at level, SWWAF_LOG_LEVEL, or more severe;
// the request lines Write writes are never held back.
func NewProcessLogger(w io.Writer, instanceName string, level slog.Level) *slog.Logger {
	handler := slog.NewJSONHandler(w, &slog.HandlerOptions{
		Level: level,
		ReplaceAttr: func(groups []string, attr slog.Attr) slog.Attr {
			if attr.Key == slog.TimeKey && len(groups) == 0 {
				return slog.String(slog.TimeKey, FormatTime(attr.Value.Time()))
			}

			return attr
		},
	})

	return slog.New(handler).With("type", "process", "instance", instanceName)
}