Name log fields, metrics and the Core Rule Set package for what they are (closes #192)

Log field, metric, action and event names now say what they count
and, for timings, the unit. Package internal/waf is internal/coreruleset.

Model: opus-5-5

internal/coreruleset/coreruleset.go

@@ -0,0 +1,473 @@
+// Package coreruleset runs the OWASP Core Rule Set 4.25.0, through Coraza, on the
+// method, the URL with its query and the headers of a request, and on its
+// body while SWW_WAF_BODY_LIMIT is set, with the eight changes sww
+// makes to it and the exclusions in SWW_CORE_RULE_SET_EXCLUSIONS_FILE, as
+// "Attack detection" under "Settings" in SPEC.md describes them. It reads no
+// response.
+//
+// sww writes only to its state directory, so Coraza is built with
+// its no_fs_access tag, as the Dockerfile and script/build build it: of a
+// file in a multipart body, Coraza then counts the bytes instead of
+// writing them to the system's temporary directory.
+package coreruleset
+
+import (
+	_ "embed"
+	"errors"
+	"fmt"
+	"io"
+	"net/http"
+	"net/netip"
+	"os"
+	"regexp"
+	"slices"
+	"strconv"
+	"strings"
+
+	coreruleset "github.com/corazawaf/coraza-coreruleset/v4"
+	"github.com/corazawaf/coraza/v3"
+	"github.com/corazawaf/coraza/v3/experimental/plugins/plugintypes"
+	"github.com/corazawaf/coraza/v3/types"
+)
+
+// directives are the Core Rule Set as sww runs it, with the
+// paranoia level for %d, bodyDirectives for the first %s while
+// SWW_WAF_BODY_LIMIT is set, the exclusions file for the second, and
+// giteaExclusions for the third.
+// Each rule sww adds has an id from 900000 to 900999, the ids the
+// Core Rule Set keeps for the rules that set it up, which
+// SWW_WAF_DISABLED_RULES and the exclusions file refuse, so that no
+// setting switches one off. Coraza joins a line ending in \ to the next,
+// without the spaces at the start of the next.
+const directives = `
+# The engine only detects. sww compares the request's anomaly
+# score with SWW_WAF_ANOMALY_THRESHOLD itself, in block and detect mode
+# alike. It reads no body, unless bodyDirectives switch that on.
+SecRuleEngine DetectionOnly
+SecRequestBodyAccess Off
+SecResponseBodyAccess Off
+
+Include @crs-setup.conf.example
+
+SecAction "id:900000,phase:1,pass,nolog,\
+    setvar:tx.blocking_paranoia_level=%d"
+
+# The first change: PUT, PATCH and DELETE are allowed besides GET, HEAD,
+# POST and OPTIONS.
+SecAction "id:900200,phase:1,pass,nolog,\
+    setvar:'tx.allowed_methods=GET HEAD POST OPTIONS PUT PATCH DELETE'"
+
+# The second: Expect and Content-Encoding are taken off the Core Rule Set's
+# list of the headers it refuses. Content-Encoding goes back on it for a
+# body the Core Rule Set reads (900260 in bodyDirectives).
+SecAction "id:900250,phase:1,pass,nolog,\
+    setvar:'tx.restricted_headers_basic=/proxy/ /lock-token/ /content-range/ \
+    /if/ /x-http-method-override/ /x-http-method/ /x-method-override/ \
+    /x-middleware-subrequest/'"
+%s
+# Coraza keeps the first 1000 query parameters of a request, and the first
+# 1000 fields of a form data or JSON body, and drops the rest, which no
+# rule then reads, so a request with more adds 5 to the score, as a rule
+# the Core Rule Set rates critical does. Coraza's recommended
+# configuration refuses such a request in its rules 200004 and 200005.
+# This rule runs once the body is read, and before the Core Rule Set adds
+# up the score in the same phase.
+SecArgumentsLimit 1000
+SecRule ARGUMENTS_LIMIT_REACHED "@eq 1" "id:900300,phase:2,pass,\
+    severity:'CRITICAL',setvar:'tx.inbound_anomaly_score_pl1=+5'"
+
+# The exclusions file comes before the rules, where the Core Rule Set puts
+# its own file of exclusions that run before the rules, so that they run
+# before the rules of phase 1 too.
+%s
+
+# The eighth change comes before the rules for the same reason.
+%s
+
+# The sixth: only the rules for requests are loaded, and no response is
+# inspected.
+Include @owasp_crs/REQUEST-*.conf
+
+# The third: redirect_uri is not checked for a URL naming an IP address or
+# localhost. Coraza matches a parameter name here, and in the fourth,
+# without regard to case. ARGS holds the fields of a form data or multipart
+# body Coraza reads as well as the query parameters, so a field of one of
+# these names is left out too.
+SecRuleUpdateTargetById 931100 "!ARGS:redirect_uri"
+SecRuleUpdateTargetById 934110 "!ARGS:redirect_uri"
+
+# The fourth: the query parameters in which gitea sends names within a
+# repository or its own records, or a page of its own site, are not
+# checked against the lists of system files, shell paths and command
+# names. Coraza takes one rule id per directive.
+SecRuleUpdateTargetById 930120 "!ARGS:path|!ARGS:files|!ARGS:skip-to|\
+    !ARGS:sub_path|!ARGS:ref|!ARGS:sha|!ARGS:branch|!ARGS:workflow|\
+    !ARGS:artifactName|!ARGS:redirect_to"
+SecRuleUpdateTargetById 932160 "!ARGS:path|!ARGS:files|!ARGS:skip-to|\
+    !ARGS:sub_path|!ARGS:ref|!ARGS:sha|!ARGS:branch|!ARGS:workflow|\
+    !ARGS:artifactName|!ARGS:redirect_to"
+SecRuleUpdateTargetById 932260 "!ARGS:path|!ARGS:files|!ARGS:skip-to|\
+    !ARGS:sub_path|!ARGS:ref|!ARGS:sha|!ARGS:branch|!ARGS:workflow|\
+    !ARGS:artifactName|!ARGS:redirect_to"
+
+# The fifth, for Referer: it is not checked for a Unix command without
+# arguments, for Java starting a process, or for a Java class name. 944130
+# refused every page opened from the page of java.lang.String.md. The
+# cookies are left out in Inspect.
+SecRuleUpdateTargetById 932340 "!REQUEST_HEADERS:Referer"
+SecRuleUpdateTargetById 944110 "!REQUEST_HEADERS:Referer"
+SecRuleUpdateTargetById 944130 "!REQUEST_HEADERS:Referer"
+
+# The seventh: a container image push that reuses a layer names the image
+# it takes the layer from in the query parameter from, which is not
+# checked for command names. 932260 refused the push of a layer from
+# owner/python3-app.
+SecRule REQUEST_FILENAME "@rx ^/v2/.+/blobs/uploads/$" \
+    "id:900500,phase:1,pass,nolog,ctl:ruleRemoveTargetById=932260;ARGS:from"
+`
+
+// bodyDirectives have the Core Rule Set read the part of a request body
+// Inspect gives it, which is at most one byte longer than the limit, up to
+// the limit, %d bytes, and read JSON and XML as Coraza's recommended
+// configuration has it in its rules 200000, 200001 and 200006, with
+// text/json, and any application or text type ending in +xml or +json,
+// besides; form data and multipart Coraza knows by itself. %% stands for
+// a % Coraza reads.
+const bodyDirectives = `
+SecRequestBodyAccess On
+SecRequestBodyLimit %d
+SecRequestBodyLimitAction ProcessPartial
+
+SecRule REQUEST_HEADERS:Content-Type \
+    "@rx ^(?:application|text)/(?:[a-z0-9.-]+[+])?xml" \
+    "id:900410,phase:1,pass,nolog,t:none,t:lowercase,ctl:requestBodyProcessor=XML"
+SecRule REQUEST_HEADERS:Content-Type \
+    "@rx ^(?:application|text)/(?:[a-z0-9.-]+[+])?json" \
+    "id:900420,phase:1,pass,nolog,t:none,t:lowercase,ctl:requestBodyProcessor=JSON"
+
+# The rest of the second change: Content-Encoding is refused again on a
+# body of a kind the Core Rule Set reads, since a compressed body cannot be
+# inspected.
+SecRule REQBODY_PROCESSOR "@rx ^(?:URLENCODED|MULTIPART|JSON|XML)$" \
+    "id:900260,phase:1,pass,nolog,\
+    setvar:'tx.restricted_headers_basic=%%{tx.restricted_headers_basic} \
+    /content-encoding/'"
+
+# A body Coraza fails to parse (900440), and a multipart body that fails
+# its strict checks (900450), each add 5 to the score, as a rule the Core
+# Rule Set rates critical does: no rule reads what comes after the fault,
+# which the app may still read. Coraza's recommended configuration refuses
+# them in its rules 200002 and 200003. A multipart body the limit cuts
+# before the colon of a part's header line, or between the carriage return
+# and the line feed that end a part's header line or the empty line after
+# its headers, adds 5 too, since Coraza takes the line the limit cuts for a
+# malformed header. Coraza parses any form data body.
+SecRule REQBODY_ERROR "!@eq 0" "id:900440,phase:2,pass,severity:'CRITICAL',\
+    setvar:'tx.inbound_anomaly_score_pl1=+5'"
+SecRule MULTIPART_STRICT_ERROR "!@eq 0" "id:900450,phase:2,pass,\
+    severity:'CRITICAL',setvar:'tx.inbound_anomaly_score_pl1=+5'"
+`
+
+// giteaExclusions are the eighth change: the fields of gitea's own
+// requests that skip the rules that refused them there, as Coraza
+// directives.
+//
+//go:embed gitea.conf
+var giteaExclusions string
+
+// cookiesNotRead are the cookies the Core Rule Set reads a request
+// without, the rest of the fifth change.
+//
+//nolint:gochecknoglobals // a constant cannot be a list
+var cookiesNotRead = []string{"gitea_flash", "redirect_to"}
+
+// Params are what New needs.
+type Params struct {
+	// ParanoiaLevel is SWW_WAF_PARANOIA_LEVEL, from 1 to 4.
+	ParanoiaLevel int
+	// DisabledRules are the ids of the rules switched off
+	// (SWW_WAF_DISABLED_RULES).
+	DisabledRules []int
+	// BodyLimit is the most of a request body the Core Rule Set reads
+	// (SWW_WAF_BODY_LIMIT), 0 while it is off and it reads none.
+	BodyLimit int64
+	// ExclusionsFile is the path of a file of exclusions loaded before the
+	// Core Rule Set's rules (SWW_CORE_RULE_SET_EXCLUSIONS_FILE), "" while
+	// there is none.
+	ExclusionsFile string
+}
+
+// The ids of sww's own rules, which an exclusions file may not
+// name.
+const (
+	firstOwnRuleID = 900000
+	lastOwnRuleID  = 900999
+)
+
+// numberOrRange matches a number, or a range of numbers such as
+// 900000-900999. Coraza reads a number with a + before it too, so the
+// end of a range may have one.
+var numberOrRange = regexp.MustCompile(`(\d+)(?:-\+?(\d+))?`)
+
+// errSwwRuleID is the error of an exclusions file that names an
+// id of sww's own rules.
+var errSwwRuleID = errors.New("an id of sww's own rules")
+
+// errChangesNoRule is the error of an exclusions file that holds a
+// directive that removes or changes rules as it loads, such as
+// SecRuleRemoveById. The Core Rule Set's rules load after the file, so
+// the directive finds none of them.
+var errChangesNoRule = errors.New("the Core Rule Set's rules load after the file; " +
+	"switch a rule off with SWW_WAF_DISABLED_RULES, or on some requests " +
+	"with a ctl exclusion")
+
+// directivesIn returns the directives in text as Coraza reads them: it
+// skips blank lines and comment lines, and joins a line ending in \ to
+// the next, without the spaces around either.
+func directivesIn(text string) []string {
+	var directives []string
+
+	joined := ""
+
+	for line := range strings.Lines(text) {
+		line = strings.TrimSpace(line)
+		if line == "" || strings.HasPrefix(line, "#") {
+			continue
+		}
+
+		if start, found := strings.CutSuffix(line, `\`); found {
+			joined += start
+
+			continue
+		}
+
+		directives = append(directives, joined+line)
+		joined = ""
+	}
+
+	return directives
+}
+
+// checkExclusions returns an error if exclusions, the text of an
+// exclusions file, names an id of sww's own rules, alone or in a
+// range, or holds a directive whose name starts with SecRuleRemove or
+// SecRuleUpdate.
+func checkExclusions(exclusions string) error {
+	for _, directive := range directivesIn(exclusions) {
+		for _, match := range numberOrRange.FindAllStringSubmatch(directive, -1) {
+			first, _ := strconv.Atoi(match[1])
+
+			last := first
+			if match[2] != "" {
+				last, _ = strconv.Atoi(match[2])
+			}
+
+			if first <= lastOwnRuleID && last >= firstOwnRuleID {
+				return fmt.Errorf("the exclusions file names %s: %w", match[0],
+					errSwwRuleID)
+			}
+		}
+
+		// Coraza takes the directive's name up to the first space, without
+		// regard to case.
+		name, _, _ := strings.Cut(directive, " ")
+		if lower := strings.ToLower(name); strings.HasPrefix(lower, "secruleremove") ||
+			strings.HasPrefix(lower, "secruleupdate") {
+			return fmt.Errorf("%s in the exclusions file changes no rule: %w", name,
+				errChangesNoRule)
+		}
+	}
+
+	return nil
+}
+
+// CoreRuleSet is the Core Rule Set, ready to inspect requests. It is safe
+// for concurrent use.
+type CoreRuleSet struct {
+	waf       coraza.WAF
+	bodyLimit int64
+}
+
+// New returns the Core Rule Set with the eight changes, at params'
+// paranoia level, without the rules it switches off, reading request
+// bodies up to params' limit, and with params' exclusions file loaded
+// before the Core Rule Set's rules. An exclusions file is an error if it
+// cannot be read or does not parse, or if checkExclusions refuses it.
+func New(params Params) (*CoreRuleSet, error) {
+	body := ""
+	if params.BodyLimit > 0 {
+		body = fmt.Sprintf(bodyDirectives, params.BodyLimit)
+	}
+
+	exclusions := ""
+
+	if params.ExclusionsFile != "" {
+		file, err := os.ReadFile(params.ExclusionsFile)
+		if err != nil {
+			return nil, fmt.Errorf("read the exclusions file: %w", err)
+		}
+
+		exclusions = string(file)
+
+		err = checkExclusions(exclusions)
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	text := fmt.Sprintf(directives, params.ParanoiaLevel, body, exclusions,
+		giteaExclusions)
+
+	if len(params.DisabledRules) > 0 {
+		ids := make([]string, len(params.DisabledRules))
+		for i, id := range params.DisabledRules {
+			ids[i] = strconv.Itoa(id)
+		}
+
+		text += "SecRuleRemoveById " + strings.Join(ids, " ") + "\n"
+	}
+
+	waf, err := coraza.NewWAF(coraza.NewWAFConfig().
+		WithRootFS(coreruleset.FS).
+		WithDirectives(text))
+	if err != nil {
+		return nil, fmt.Errorf("load the Core Rule Set: %w", err)
+	}
+
+	return &CoreRuleSet{waf: waf, bodyLimit: params.BodyLimit}, nil
+}
+
+// Result is what the Core Rule Set found in a request.
+type Result struct {
+	// RuleIDs are the ids of the rules that matched, in the order they
+	// ran.
+	RuleIDs []int
+	// Score is the request's anomaly score: what those rules add up to.
+	Score int
+}
+
+// Inspect runs the Core Rule Set on r, a request from client: on its
+// method, its URL with the query, and its headers, the Cookie header
+// without the cookies in cookiesNotRead, and on body, r's body as the
+// caller has it, as readBody reads it. It returns what it found, what it
+// read of body, and the error that ended the reading early, if one did.
+func (c *CoreRuleSet) Inspect(
+	r *http.Request, client netip.Addr, body io.Reader,
+) (Result, []byte, error) {
+	tx := c.waf.NewTransaction()
+	// Closing would remove the files Coraza wrote, and it writes none.
+	defer func() { _ = tx.Close() }()
+
+	tx.ProcessConnection(client.String(), 0, "", 0)
+	tx.ProcessURI(r.URL.String(), r.Method, r.Proto)
+
+	for name, values := range r.Header {
+		for _, value := range values {
+			if name == "Cookie" {
+				value = withoutCookiesNotRead(value)
+				if value == "" {
+					continue // it held those cookies alone
+				}
+			}
+
+			tx.AddRequestHeader(name, value)
+		}
+	}
+
+	// Go's server takes these two out of the headers.
+	tx.AddRequestHeader("Host", r.Host)
+
+	for _, encoding := range r.TransferEncoding {
+		tx.AddRequestHeader("Transfer-Encoding", encoding)
+	}
+
+	tx.ProcessRequestHeaders()
+
+	read, err := c.readBody(tx, body)
+
+	// This reads the body in memory and runs the rest of the rules, and
+	// cannot fail.
+	_, _ = tx.ProcessRequestBody()
+
+	var ids []int
+
+	for _, matched := range tx.MatchedRules() {
+		// The rules that look for attacks have a severity; the others set
+		// the Core Rule Set up and add up the score.
+		rule := matched.Rule()
+		if rule.Severity() != types.RuleSeverityUnset {
+			ids = append(ids, rule.ID())
+		}
+	}
+
+	return Result{RuleIDs: ids, Score: score(tx)}, read, err
+}
+
+// readBody reads body, the body of the request in tx, which has run on
+// the request's headers, while SWW_WAF_BODY_LIMIT is set and the body is
+// of a kind the Core Rule Set reads: form data and multipart, of which it
+// reads the first c.bodyLimit bytes, and JSON and XML, which it reads only
+// when they are no longer than that, since they cannot be read in part.
+// readBody reads one byte past the limit, to tell which they are, gives
+// the Core Rule Set what it reads, and returns what it read and the error
+// that ended the reading early, if one did.
+func (c *CoreRuleSet) readBody(tx types.Transaction, body io.Reader) ([]byte, error) {
+	if c.bodyLimit == 0 {
+		return nil, nil
+	}
+
+	inPart := false
+	// How the body is read is a variable of the transaction, which only
+	// Coraza's interface for plugins reads.
+	state := tx.(plugintypes.TransactionState) //nolint:forcetypeassert // every one is
+
+	switch state.Variables().RequestBodyProcessor().Get() {
+	case "URLENCODED", "MULTIPART":
+		inPart = true
+	case "JSON", "XML":
+	default:
+		return nil, nil
+	}
+
+	read, err := io.ReadAll(io.LimitReader(body, c.bodyLimit+1))
+
+	if inPart || (err == nil && int64(len(read)) <= c.bodyLimit) {
+		// Coraza holds what it reads of the body in memory, up to the
+		// limit, so this cannot fail.
+		_, _, _ = tx.WriteRequestBody(read)
+	}
+
+	return read, err
+}
+
+// score returns the anomaly score the Core Rule Set added up in tx, a
+// transaction it has run, or 0 if a rule that adds it up is switched off.
+func score(tx types.Transaction) int {
+	// The score is in a variable of the transaction, which only Coraza's
+	// interface for plugins reads.
+	state := tx.(plugintypes.TransactionState) //nolint:forcetypeassert // every one is
+
+	values := state.Variables().TX().Get("blocking_inbound_anomaly_score")
+	if len(values) == 0 {
+		return 0
+	}
+
+	n, _ := strconv.Atoi(values[0])
+
+	return n
+}
+
+// withoutCookiesNotRead returns value, a Cookie header's, without the
+// cookies in cookiesNotRead.
+func withoutCookiesNotRead(value string) string {
+	var kept []string
+
+	for cookie := range strings.SplitSeq(value, ";") {
+		name, _, _ := strings.Cut(strings.TrimSpace(cookie), "=")
+		if !slices.Contains(cookiesNotRead, name) {
+			kept = append(kept, cookie)
+		}
+	}
+
+	return strings.Join(kept, ";")
+}