smallwebwaf/internal/rules/rules.go

// Package rules reads the rule files: the plain text files in
// SWW_WAF_RULES_DIR, one rule to a line, that each request is checked
// against, as the "Rule files" section of SPEC.md describes. They are read
// at start, and again once the directory has had no change for a short
// time after one is edited, added or removed.
package rules

import (
	"context"
	"encoding/hex"
	"errors"
	"fmt"
	"log/slog"
	"net/http"
	"os"
	"path/filepath"
	"regexp"
	"slices"
	"strings"
	"sync/atomic"
	"time"

	"github.com/fsnotify/fsnotify"

	"sneak.berlin/go/smallwebwaf/internal/alerts"
	"sneak.berlin/go/smallwebwaf/internal/config"
)

// The actions a rule takes when it matches.
const (
	// ActionLog notes the match in the request log, and does nothing else.
	ActionLog = "log"
	// ActionBlock refuses the request with 403.
	ActionBlock = "block"
	// ActionBan refuses the request and bans the client's netblock: the
	// request is a clear sign of attack.
	ActionBan = "ban"
)

// extension ends the name of every rule file.
const extension = ".rules"

// quietTime is how long SWW_WAF_RULES_DIR must go without a change before
// the rule files are read again, so that a file still being written, such
// as one saved in place, appended to or copied in with scp, is read only
// once whole.
const quietTime = 2 * time.Second

// headerTarget starts the target that is one request header,
// header:<Name>.
const headerTarget = "header:"

// pathTarget and uriTarget are the targets matched both as received and
// once percent-decoded: the path, and the path and query together.
const (
	pathTarget = "path"
	uriTarget  = "uri"
)

// escapeLength is the length of a percent escape, such as %2e.
const escapeLength = 3

var (
	// ruleLine is a rule: four fields separated by spaces or tabs, of
	// which the fourth, the regex, runs to the end of the line.
	ruleLine = regexp.MustCompile(`^([^ \t]+)[ \t]+([^ \t]+)[ \t]+([^ \t]+)[ \t]+(.+)$`)
	// idChars are the characters of a rule's id.
	idChars = regexp.MustCompile(`^[A-Za-z0-9_-]+$`)
)

var (
	errNotRule = errors.New(
		"is not a rule: an id, a target, an action and a regex, " +
			"separated by spaces or tabs")
	errNotID     = errors.New("is not an id of letters, digits, - and _")
	errNotTarget = errors.New(
		"is not path, query, uri, method, host, user_agent, referer or header:<Name>")
	errNotHeaderName = errors.New(
		"has a character after header: that no header name can have")
	errHeaderTakenOut = errors.New(
		"names a header that Go's HTTP server takes out of every request, " +
			"so a rule never sees it")
	errNotAction = errors.New("is not log, block or ban")
	errNotRegex  = errors.New("does not compile")
	errUsedTwice = errors.New("is already the id of the rule at")
)

// Rule is one rule of a rule file.
type Rule struct {
	// ID names the rule in the request log, the metrics and ban notes.
	ID string
	// Target is what the regex is matched against, such as path or
	// header:Accept.
	Target string
	// Action is ActionLog, ActionBlock or ActionBan.
	Action string

	regex *regexp.Regexp
}

// Params are what Load needs.
type Params struct {
	// Dir is the directory of the rule files (SWW_WAF_RULES_DIR).
	Dir string
	// Enabled is SWW_WAF_RULES_ENABLED: while it is false, no file is read
	// and no rule loaded.
	Enabled bool
	// ProcessLog receives how many rules were read, and the error in a
	// rule file edited while sww runs.
	ProcessLog *slog.Logger
	// Alerts receive a file_error alert for that error.
	Alerts *alerts.Queue
}

// Files are the rule files of a running sww, and the rules read
// from them. They are safe for concurrent use.
type Files struct {
	params Params
	// rules are the rules loaded, in the order of their files' names, and
	// then of their lines.
	rules atomic.Pointer[[]Rule]
}

// Load reads the rules of every *.rules file in Dir, in the order of the
// files' names, unless Enabled is false. A Dir that cannot be read is an
// error, and so is a line that is not a rule, a header name with a
// character no header name can have, a rule for the Host or the
// Transfer-Encoding header, which Go's HTTP server takes out of every
// request, a regex that does not compile and an id used twice, each named
// with its file and line.
func Load(params Params) (*Files, error) {
	f := &Files{params: params}
	f.rules.Store(&[]Rule{})

	if !params.Enabled {
		return f, nil
	}

	rules, _, err := read(params.Dir)
	if err != nil {
		return nil, err
	}

	f.rules.Store(&rules)
	f.logRead(len(rules))

	return f, nil
}

// Match checks r against the rules, in order, and returns those it
// matches, up to the first whose action refuses it, block or ban, which
// is then the last one returned.
func (f *Files) Match(r *http.Request) []Rule {
	var matched []Rule

	for _, rule := range *f.rules.Load() {
		if !rule.matches(r) {
			continue
		}

		matched = append(matched, rule)
		if rule.Action != ActionLog {
			break
		}
	}

	return matched
}

// Len returns how many rules are loaded.
func (f *Files) Len() int {
	return len(*f.rules.Load())
}

// Watch watches Dir until ctx is done, and reads the rule files again
// once Dir has had no change for quietTime, after one is edited, added or
// removed, and after Watch starts watching. If they then hold an error,
// the rules stay as they were, the error is logged with its file and
// line, and the files are read again after the next change. If Dir cannot
// be watched, that is logged, and the rules stay as they were loaded.
// While Enabled is false, Watch returns at once.
func (f *Files) Watch(ctx context.Context) {
	if !f.params.Enabled {
		return
	}

	watcher, err := fsnotify.NewWatcher()
	if err == nil {
		defer func() {
			_ = watcher.Close()
		}()

		err = watcher.Add(f.params.Dir)
	}

	if err != nil {
		f.params.ProcessLog.Error("cannot watch the rule files for edits",
			"error", err.Error())

		return
	}

	f.params.ProcessLog.Info("watching the rule files for edits",
		"directory", f.params.Dir)

	f.readAfterChanges(ctx, watcher.Events, watcher.Errors)
}

// readAfterChanges reads the rule files again once quietTime has passed
// without a change from events, until ctx is done, and logs the errors
// from errs. The wait starts at once, as if for a change, so that an edit
// saved after Load read the files, and before Dir was watched, is taken
// in too.
func (f *Files) readAfterChanges(
	ctx context.Context, events <-chan fsnotify.Event, errs <-chan error,
) {
	quiet := time.NewTimer(quietTime)
	defer quiet.Stop()

	for {
		select {
		case <-ctx.Done():
			return
		case <-events:
			quiet.Reset(quietTime)
		case <-quiet.C:
			f.readAgain()
		case err := <-errs:
			f.params.ProcessLog.Warn("watching the rule files failed",
				"error", err.Error())
		}
	}
}

// readAgain reads the rule files again, in place of the rules loaded, or
// logs the error that keeps the rules as they were, and raises a
// file_error alert for it, for the file it is in.
func (f *Files) readAgain() {
	rules, path, err := read(f.params.Dir)
	if err != nil {
		const kept = "a rule file has an error, and the rules stay as they were"

		// Raised before it is logged, so that the alert is there once the
		// log line is.
		f.params.Alerts.Raise(alerts.Alert{
			Event:  alerts.EventFileError,
			Reason: kept,
			Detail: map[string]any{"file": path, "error": err.Error()},
		})
		f.params.ProcessLog.Error(kept, "error", err.Error())

		return
	}

	f.rules.Store(&rules)
	f.logRead(len(rules))
}

// logRead logs that the rule files were read, and how many rules they
// hold, which can be none.
func (f *Files) logRead(count int) {
	f.params.ProcessLog.Info("read the rule files",
		"directory", f.params.Dir, "rules", count)
}

// read returns the rules of every rule file in dir, in the order of the
// files' names, and then of their lines, or an error, with the path of the
// rule file it is in, or dir. A file whose name starts with a dot, such as
// an editor's lock file .#50-app.rules, is not a rule file, as a shell's
// *.rules would not match it.
func read(dir string) ([]Rule, string, error) {
	entries, err := os.ReadDir(dir)
	if err != nil {
		return nil, dir, fmt.Errorf("SWW_WAF_RULES_DIR cannot be read: %w", err)
	}

	var rules []Rule

	// places are where each id is, as "<file>, line <n>".
	places := map[string]string{}

	for _, entry := range entries {
		name := entry.Name()
		if entry.IsDir() || strings.HasPrefix(name, ".") || filepath.Ext(name) != extension {
			continue
		}

		path := filepath.Join(dir, name)

		rules, err = readFile(path, rules, places)
		if err != nil {
			return nil, path, err
		}
	}

	return rules, "", nil
}

// readFile appends the rules of the rule file at path to rules. places
// are where each id read so far is, and gain those of the file.
func readFile(path string, rules []Rule, places map[string]string) ([]Rule, error) {
	data, err := os.ReadFile(path) //nolint:gosec // a rule file, in SWW_WAF_RULES_DIR
	if err != nil {
		return nil, err
	}

	number := 0

	for line := range strings.Lines(string(data)) {
		number++
		place := fmt.Sprintf("%s, line %d", path, number)

		text := strings.TrimSuffix(strings.TrimSuffix(line, "\n"), "\r")

		rule, isRule, err := parse(text)
		if err != nil {
			return nil, fmt.Errorf("%s: %w", place, err)
		}

		if !isRule {
			continue
		}

		first, used := places[rule.ID]
		if used {
			return nil, fmt.Errorf("%s: the id %q %w %s", place, rule.ID, errUsedTwice, first)
		}

		places[rule.ID] = place
		rules = append(rules, rule)
	}

	return rules, nil
}

// parse reads a line of a rule file. It returns false for a blank line
// and for a comment, a line that starts with #. Spaces and tabs at the
// end of the line are not part of its regex, so a line with only those
// after its action has no regex, and is not a rule.
func parse(line string) (Rule, bool, error) {
	line = strings.Trim(line, " \t")
	if line == "" || strings.HasPrefix(line, "#") {
		return Rule{}, false, nil
	}

	fields := ruleLine.FindStringSubmatch(line)
	if fields == nil {
		return Rule{}, false, errNotRule
	}

	rule := Rule{ID: fields[1], Target: fields[2], Action: fields[3]}
	headerName, isHeader := strings.CutPrefix(rule.Target, headerTarget)

	switch {
	case !idChars.MatchString(rule.ID):
		return Rule{}, false, fmt.Errorf("the id %q %w", rule.ID, errNotID)
	case !isTarget(rule.Target):
		return Rule{}, false, fmt.Errorf("the target %q %w", rule.Target, errNotTarget)
	case isHeader && !config.IsHeaderName(headerName):
		return Rule{}, false, fmt.Errorf("the target %q %w", rule.Target, errNotHeaderName)
	case strings.EqualFold(rule.Target, headerTarget+"Host"):
		return Rule{}, false, fmt.Errorf(
			"the target %q %w; the request's host is the target host",
			rule.Target, errHeaderTakenOut)
	case strings.EqualFold(rule.Target, headerTarget+"Transfer-Encoding"):
		return Rule{}, false, fmt.Errorf("the target %q %w", rule.Target, errHeaderTakenOut)
	case !slices.Contains([]string{ActionLog, ActionBlock, ActionBan}, rule.Action):
		return Rule{}, false, fmt.Errorf("the action %q %w", rule.Action, errNotAction)
	}

	regex, err := regexp.Compile(fields[4])
	if err != nil {
		return Rule{}, false, fmt.Errorf("the regex %w: %w", errNotRegex, err)
	}

	rule.regex = regex

	return rule, true, nil
}

// isTarget reports whether target is one a rule may have.
func isTarget(target string) bool {
	switch target {
	case pathTarget, "query", uriTarget, "method", "host", "user_agent", "referer":
		return true
	}

	name, isHeader := strings.CutPrefix(target, headerTarget)

	return isHeader && name != ""
}

// matches reports whether the rule's regex matches its target in r. For
// path and uri it is matched against the target as received, and against
// it once percent-decoded, so that an encoded probe cannot slip past.
func (rule Rule) matches(r *http.Request) bool {
	target := value(rule.Target, r)

	if rule.Target == pathTarget || rule.Target == uriTarget {
		return rule.regex.MatchString(target) || rule.regex.MatchString(DecodeOnce(target))
	}

	return rule.regex.MatchString(target)
}

// Path returns r's path as the client sent it, before any decoding or
// re-encoding, up to the first ?: what a path rule is matched against, as
// it is and once percent-decoded.
func Path(r *http.Request) string {
	path, _, _ := strings.Cut(pathAndQuery(r), "?")

	return path
}

// value returns what a rule with target is matched against in r: the
// path, as Path gives it, the path and query, as pathAndQuery gives
// them, and the query as the client sent it, before any decoding or
// re-encoding, after the first ?, and a header's values joined by ", ",
// as HTTP joins those of a header sent more than once.
func value(target string, r *http.Request) string {
	switch target {
	case pathTarget:
		return Path(r)
	case uriTarget:
		return pathAndQuery(r)
	case "query":
		_, query, _ := strings.Cut(pathAndQuery(r), "?")

		return query
	case "method":
		return r.Method
	case "host":
		return r.Host
	case "user_agent":
		return header(r, "User-Agent")
	case "referer":
		return header(r, "Referer")
	default:
		return header(r, strings.TrimPrefix(target, headerTarget))
	}
}

// pathAndQuery returns the target of r's request line, r.RequestURI, as
// the client sent it, less any scheme and host: a target with a scheme
// gives what follows the scheme and its :, and the host when // follows.
// So http://host/path, as a client sends it to a proxy, gives /path, and
// so does http:/path, which Go reads as a target with a scheme and no
// host. r.URL is not used: when the path holds a character it escapes,
// such as \ or a non-ASCII byte, it decodes the whole path and escapes it
// again, so that \ becomes %5C and %2e a dot.
func pathAndQuery(r *http.Request) string {
	if !r.URL.IsAbs() {
		return r.RequestURI
	}

	_, afterScheme, _ := strings.Cut(r.RequestURI, ":")

	hostAndRest, hasHost := strings.CutPrefix(afterScheme, "//")
	if !hasHost {
		return afterScheme
	}

	start := strings.IndexAny(hostAndRest, "/?")
	if start < 0 {
		return ""
	}

	return hostAndRest[start:]
}

// header returns the values of r's header name joined by ", ", or "" if
// r has no such header.
func header(r *http.Request, name string) string {
	return strings.Join(r.Header.Values(name), ", ")
}

// DecodeOnce returns s with each percent escape, such as %2e, replaced by
// the byte it stands for. A % that is not followed by two hex digits is
// left as it is, so that a malformed escape cannot keep the rest of s
// from being decoded.
func DecodeOnce(s string) string {
	var decoded strings.Builder

	for i := 0; i < len(s); i++ {
		if s[i] == '%' && i+escapeLength <= len(s) {
			b, err := hex.DecodeString(s[i+1 : i+escapeLength])
			if err == nil {
				decoded.Write(b)

				i += escapeLength - 1

				continue
			}
		}

		decoded.WriteByte(s[i])
	}

	return decoded.String()
}