smallwebwaf/internal/proxy/request.go

package proxy

import (
	"context"
	"net/http"
	"net/netip"
	"slices"
	"strings"
	"time"

	"sneak.berlin/go/smallwebwaf/internal/anomaly"
	"sneak.berlin/go/smallwebwaf/internal/bans"
	"sneak.berlin/go/smallwebwaf/internal/config"
	"sneak.berlin/go/smallwebwaf/internal/lookup"
	"sneak.berlin/go/smallwebwaf/internal/ratelimit"
	"sneak.berlin/go/smallwebwaf/internal/requestlog"
	"sneak.berlin/go/smallwebwaf/internal/rules"
)

// refusal is servewaf refusing a request: the status the visitor is to get, 0
// to close its connection without an answer, and the action the log line names.
type refusal struct {
	status int
	action string
}

// decision is what the checks note of a request: its log line, and what
// counting the request needs once it has ended. servewaf keeps a copy of it
// until the request's report comes. It holds no credentials, since the log line
// never does, and neither the request's objects nor their bodies.
type decision struct {
	line   requestlog.Line
	client netip.Addr
	// requestURI is the path with the query, as a ban's notes keep it.
	requestURI string
	// site is, for a question, the host the visitor asked for, which the
	// alerts name, since one servewaf serves many sites.
	site string
	// allowed is true for a question answered with DispositionAllow.
	allowed bool
	// lookedUp is true once the client's AS number and country have been
	// looked up, whether or not an answer was there, and lookupAnswer is
	// what the lookup gave then, the zero Answer while GeoJS had given none.
	lookedUp     bool
	lookupAnswer lookup.Answer
	// skippedLookupWait is true for a request that did not wait for GeoJS and
	// found no answer kept. SWWAF_UNKNOWN_LIMIT_PERCENT does not apply to it,
	// so that not waiting never lowers its client's limits.
	skippedLookupWait bool
	// counted is true for a request the rate limits counted, whose bytes
	// the byte limits count once it has ended. limitPercent and
	// bytesPercent are then its client's limit percentages for the rate
	// limits and for the byte limits.
	counted                    bool
	limitPercent, bytesPercent percentage
	// attack is true for a request that matched a ban rule or asked for a
	// trap path, ruleBlocked for one a block rule refused, and wafBlocked for
	// one the Core Rule Set refused, each an offence its client's history
	// counts.
	attack, ruleBlocked, wafBlocked bool
	// reputation is the reputation sources that list the client, for the
	// notes of a ban the request makes.
	reputation []bans.ReputationHit
}

// request is one request to servewaf, from the moment its headers have been
// read to its log line.
type request struct {
	decision

	h   *handler
	in  *http.Request
	out *responseWriter

	// blocklisted is true once a blocklist is found to list the client,
	// dnsblListed once a DNSBL zone's verdict is, and abuseIPDBHit once
	// AbuseIPDB's score of it is a hit.
	blocklisted, dnsblListed, abuseIPDBHit bool
	start                                  time.Time
	// checked is when the checks were done.
	checked time.Time
	// refused is what the request was refused with, nil if it was not.
	refused *refusal
	// requestBytes is how many bytes of the request's body have been read.
	requestBytes int64
}

// startRequest starts handling r from client, which came through peer: it
// notes the time, counts the request as under way, and starts the log line
// with what is known of the request. isQuestion says whether r is a question,
// whose headers give the request's id and scheme.
func (h *handler) startRequest(
	w http.ResponseWriter, r *http.Request, client, peer netip.Addr, isQuestion bool,
) *request {
	h.metrics.RequestStarted()

	start := time.Now()
	forwardedFor := r.Header.Values("X-Forwarded-For")

	rq := &request{
		decision: decision{
			client:     client,
			requestURI: r.URL.RequestURI(),
			line: requestlog.Line{
				Time:             requestlog.FormatTime(start),
				Instance:         h.config.InstanceName,
				ClientIP:         client.String(),
				Method:           r.Method,
				Scheme:           scheme(r, isQuestion),
				Host:             r.Host,
				Path:             rules.Path(r),
				Query:            r.URL.RawQuery,
				Protocol:         r.Proto,
				Referer:          r.Referer(),
				UserAgent:        r.UserAgent(),
				RequestID:        requestID(r, isQuestion),
				PeerIP:           peer.String(),
				ForwardedFor:     strings.Join(forwardedFor, ", "),
				ClientGroup:      h.clientGroup(client).String(),
				ContentType:      r.Header.Get("Content-Type"),
				RequestHeaders:   requestHeaders(r, h.config.LogRequestHeaders),
				HasAuthorization: len(r.Header.Values("Authorization")) > 0,
				HasCookie:        len(r.Header.Values("Cookie")) > 0,
				Action:           requestlog.ActionForward,
			},
		},
		h:     h,
		in:    r,
		out:   &responseWriter{ResponseWriter: w},
		start: start,
	}

	// A length of -1 is a body whose length was not announced.
	if r.ContentLength > 0 {
		rq.line.ContentLength = r.ContentLength
	}

	return rq
}

// requestHeaders returns the headers of r that names lists, by name in
// lower case, each with its values joined by ", ". Authorization, Cookie
// and Set-Cookie are never among them, whatever names says.
func requestHeaders(r *http.Request, names []string) map[string]string {
	headers := map[string]string{}

	for _, name := range names {
		switch name {
		case "authorization", "cookie", "set-cookie":
			continue
		}

		values := r.Header.Values(name)
		if len(values) > 0 {
			headers[name] = strings.Join(values, ", ")
		}
	}

	return headers
}

// check is the one place where a request can be refused. It returns nil to let
// the request through. The checks of checkClient decide: a request they refuse
// is answered with SWWAF_BAN_RESPONSE, 403 for a block rule or the Core Rule Set,
// or 431 for the header limits. In observe mode nothing is refused, and the log
// line names what enforce mode would have done. ctx is the request's own context.
func (rq *request) check(ctx context.Context) *refusal {
	action := rq.checkClient(ctx)

	switch {
	case action == "":
		return nil
	case rq.h.config.Observe:
		rq.line.WouldAction = action

		return nil
	case action == requestlog.ActionRuleBlocked || action == requestlog.ActionWAFBlocked:
		return &refusal{status: http.StatusForbidden, action: action}
	case action == requestlog.ActionTooLarge:
		return &refusal{status: http.StatusRequestHeaderFieldsTooLarge, action: action}
	default:
		return rq.banResponse(action)
	}
}

// checkClient runs the checks on the request's client, and returns the
// action of the first that refuses the request, or "" when none does. A
// client in SWWAF_ALLOW_NETS skips them, and is not looked up. For any
// other client, SWWAF_DENY_NETS comes first, then a ban on its netblock,
// so that a client either refuses is not looked up, then the lookup of
// its AS number and country, then the country lists, then the blocklists,
// then the CrowdSec decision list, which bans the client it lists, then
// the DNSBL zones' verdicts, and then AbuseIPDB's score; a request any of
// them refuses is not counted for the rate limits. Then come the
// rate limits, unless the client is in SWWAF_RATE_LIMIT_EXEMPT_NETS or the
// request's path is exempt under SWWAF_RATE_LIMIT_EXEMPT_PATHS, so that
// every other request is counted, each of them by the client's limit
// percentages, then SWWAF_TRAP_PATHS, then the rule files, then the header
// limits, and last the Core Rule Set. A request exempt from the rate limits
// is exempt from the byte limits too. ctx is the request's own context.
func (rq *request) checkClient(ctx context.Context) string {
	cfg := rq.h.config
	if isInside(rq.client, cfg.AllowNets) {
		return ""
	}

	now := rq.h.now()

	if isInside(rq.client, cfg.DenyNets) {
		return requestlog.ActionDenied
	}

	if rq.banned(now) {
		return requestlog.ActionBanned
	}

	rq.lookUp(ctx, now)

	// The lookup can wait for GeoJS, so the checks after it read the time
	// again: a request counted at the time it arrived could be counted
	// after a later one of its client's, which the rate limits would take
	// for the clock set back.
	now = rq.h.now()

	if rq.countryDenied() {
		return requestlog.ActionCountryDenied
	}

	if rq.blocklistDenied() {
		return requestlog.ActionDenied
	}

	if rq.crowdSecBanned(now) {
		return requestlog.ActionBanned
	}

	if rq.dnsblDenied(ctx) || rq.abuseIPDBDenied(ctx) {
		return requestlog.ActionDenied
	}

	rq.counted = rq.countedByRateLimits()
	if rq.counted {
		rq.limitPercent, rq.bytesPercent = rq.limitPercentages()
		rq.line.LimitPercent, rq.line.LimitPercentSetting = rq.limitPercent.logged()
		rq.line.BytesPercent, rq.line.BytesPercentSetting = rq.bytesPercent.logged()
	}

	if rq.counted && rq.limitBroken(now) {
		return requestlog.ActionRateLimited
	}

	if rq.trapPath(now) {
		return requestlog.ActionBanned
	}

	action := rq.checkRules(now)
	if action != "" {
		return action
	}

	if rq.overHeaderLimits() {
		return requestlog.ActionTooLarge
	}

	return rq.checkCoreRuleSet()
}

// countedByRateLimits reports whether the rate limits count the request: its
// client is not in SWWAF_RATE_LIMIT_EXEMPT_NETS, and its path is not exempt
// under SWWAF_RATE_LIMIT_EXEMPT_PATHS.
func (rq *request) countedByRateLimits() bool {
	cfg := rq.h.config

	return !isInside(rq.client, cfg.RateLimitExemptNets) &&
		!pathExempt(rq.in, cfg.RateLimitExemptPaths)
}

// pathExempt reports whether r is exempt under prefixes,
// SWWAF_RATE_LIMIT_EXEMPT_PATHS or SWWAF_WAF_EXEMPT_PATHS: whether its
// path as sent, the path the app receives, not percent-decoded, starts
// with one of prefixes, so that /%61ssets/x is not under /assets/ for an
// app whose router matches the path as received. A request whose decoded
// path contains .. anywhere or a backslash, or whose path as sent holds an
// encoded slash (%2F or %2f), never is, since an app may act on it as a
// path outside every prefix: /assets/..%2Flogin as /login, or /assets%2Fx
// as one path segment, as Go's router does.
func pathExempt(r *http.Request, prefixes []string) bool {
	decoded := r.URL.Path
	sent := rules.Path(r)

	if strings.Contains(decoded, "..") || strings.Contains(decoded, `\`) ||
		strings.Contains(strings.ToLower(sent), "%2f") {
		return false
	}

	return slices.ContainsFunc(prefixes, func(prefix string) bool {
		return strings.HasPrefix(sent, prefix)
	})
}

// finish counts the request in the metrics and writes its log line.
func (rq *request) finish() {
	line := &rq.line
	line.Status = rq.out.status
	line.ResponseBytes = rq.out.bytes
	line.ResponseContentType = rq.out.Header().Get("Content-Type")
	line.RequestBytes = rq.requestBytes

	switch {
	case rq.refused != nil:
		line.Action = rq.refused.action
	case rq.out.err != nil || rq.in.Context().Err() != nil:
		line.Aborted = true
	}

	duration := time.Since(rq.start)
	line.DurationTotal = requestlog.Milliseconds(duration)
	line.DurationChecks = timing(rq.start, rq.checked)

	// Counted before the log line is written, so that the metrics count
	// every request whose line is out. An allowed request is the exception:
	// its report gives the status, bytes and times the metrics count.
	if rq.allowed {
		rq.h.metrics.RequestAllowed()
	} else {
		rq.h.metrics.RequestEnded(line, duration)
	}

	rq.writeLine()
}

// writeLine writes the request's log line.
func (rq *request) writeLine() {
	err := requestlog.Write(rq.h.requestLog, &rq.line)
	if err != nil {
		rq.h.processLog.Error("writing the request log failed", "error", err.Error())
	}
}

// timing is the time from start to end in milliseconds, for one of the
// log line's timings, or nil when end is zero: what it times never
// happened.
func timing(start, end time.Time) *float64 {
	if end.IsZero() {
		return nil
	}

	return new(requestlog.Milliseconds(end.Sub(start)))
}

// addToHistory adds the request, which has ended, to its client's
// history, and counts its offences in the metrics, and then the lookup's
// answer about the client, as answerAtTheEnd gives it, to that history and
// to the notes of the bans on its netblock: an answer may have come
// before either was there, and one from GeoJS that comes later is added
// when it comes.
func (rq *request) addToHistory() {
	request := ratelimit.Request{
		Forwarded:     rq.allowed,
		Refused:       rq.refused != nil,
		Status:        rq.out.status,
		RequestBytes:  rq.requestBytes,
		ResponseBytes: rq.out.bytes,
		BrokeLimit:    rq.line.Offence == requestlog.OffenceLimit,
		Attack:        rq.attack,
		RuleBlocked:   rq.ruleBlocked,
		WAFBlocked:    rq.wafBlocked,
	}

	rq.h.limiter.AddToHistory(rq.h.clientGroup(rq.client), rq.h.now(), request)
	rq.h.metrics.Offences(request)

	answer, found := rq.answerAtTheEnd()
	if found {
		rq.h.addLookup(answer)
	}
}

// countAnomalies counts the request, which has ended, and its bytes, as
// countedBytes gives them, for the anomaly thresholds, whatever was done
// with it: a request refused, one from a client in SWWAF_ALLOW_NETS or
// SWWAF_RATE_LIMIT_EXEMPT_NETS, and one for a path in
// SWWAF_RATE_LIMIT_EXEMPT_PATHS are counted too. It is counted for its
// client's AS number when answerAtTheEnd gives one. With every anomaly
// threshold off, the default, it does nothing.
func (rq *request) countAnomalies() {
	if !anomalyThresholdsSet(rq.h.config) {
		return
	}

	answer, _ := rq.answerAtTheEnd()

	rq.h.anomalies.Count(rq.h.now(), anomaly.Request{
		Host:        rq.site,
		Client:      rq.client,
		ClientGroup: rq.h.clientGroup(rq.client),
		ASN:         answer.ASN,
		ASName:      answer.ASName,
		Country:     answer.Country,
		Bytes:       rq.countedBytes(),
	})
}

// anomalyThresholdsSet reports whether any anomaly threshold is set.
func anomalyThresholdsSet(cfg *config.Config) bool {
	off := anomaly.Thresholds{}

	return cfg.AnomalyClient != off || cfg.AnomalyNet != off || cfg.AnomalyASN != off ||
		cfg.AnomalyTotal != off || cfg.AnomalyWatch != off
}

// answerAtTheEnd returns, for a client that was looked up, the lookup's
// answer about it as the request ends, and whether there is one: the
// lookup database's, which was there at once, or the one GeoJS has given
// by then, which a request does not wait for unless a setting needs it.
func (rq *request) answerAtTheEnd() (lookup.Answer, bool) {
	if !rq.lookedUp {
		return lookup.Answer{}, false
	}

	if rq.h.config.LookupSource == "file" {
		return rq.lookupAnswer, true
	}

	return rq.h.geojs.Kept(rq.h.clientGroup(rq.client))
}

// responseWriter is servewaf's answer to a request. It notes the status and
// size for the log line, and the first error writing the answer.
type responseWriter struct {
	http.ResponseWriter

	// status is the status sent, or zero before one is.
	status int
	bytes  int64
	err    error
}

// WriteHeader sends the status and headers.
func (w *responseWriter) WriteHeader(status int) {
	if w.status == 0 {
		w.status = status
	}

	w.ResponseWriter.WriteHeader(status)
}

// Write sends part of the body.
func (w *responseWriter) Write(p []byte) (int, error) {
	if w.status == 0 {
		w.status = http.StatusOK
	}

	n, err := w.ResponseWriter.Write(p)
	w.bytes += int64(n)

	if w.err == nil {
		w.err = err
	}

	return n, err
}