Rename internal/proxy to internal/servewaf (closes #173)

The package passes nothing to an app since rproxy took that over, so it is named after the command it serves. The rproxy tests call their stand-in for servewaf standIn, since a local servewaf would hide the package. No change of behaviour. Model: opus-5-5
2026-11-06 18:00:32 +00:00
parent 6deb9d7581
commit 6acde5c3bc
50 changed files with 488 additions and 484 deletions
@@ -0,0 +1,462 @@
+package servewaf
+
+import (
+	"context"
+	"net/http"
+	"net/netip"
+	"slices"
+	"strings"
+	"time"
+
+	"sneak.berlin/go/smallwebwaf/internal/anomaly"
+	"sneak.berlin/go/smallwebwaf/internal/bans"
+	"sneak.berlin/go/smallwebwaf/internal/config"
+	"sneak.berlin/go/smallwebwaf/internal/lookup"
+	"sneak.berlin/go/smallwebwaf/internal/ratelimit"
+	"sneak.berlin/go/smallwebwaf/internal/requestlog"
+	"sneak.berlin/go/smallwebwaf/internal/rules"
+)
+
+// refusal is servewaf refusing a request: the status the visitor is to get, 0
+// to close its connection without an answer, and the action the log line names.
+type refusal struct {
+	status int
+	action string
+}
+
+// decision is what the checks note of a request: its log line, and what
+// counting the request needs once it has ended. servewaf keeps a copy of it
+// until the request's report comes. It holds no credentials, since the log line
+// never does, and neither the request's objects nor their bodies.
+type decision struct {
+	line   requestlog.Line
+	client netip.Addr
+	// requestURI is the path with the query, as a ban's notes keep it.
+	requestURI string
+	// site is, for a question, the host the visitor asked for, which the
+	// alerts name, since one servewaf serves many sites.
+	site string
+	// allowed is true for a question answered with DispositionAllow.
+	allowed bool
+	// lookedUp is true once the client's AS number and country have been
+	// looked up, whether or not an answer was there, and lookupAnswer is
+	// what the lookup gave then, the zero Answer while GeoJS had given none.
+	lookedUp     bool
+	lookupAnswer lookup.Answer
+	// skippedLookupWait is true for a request that did not wait for GeoJS and
+	// found no answer kept. SWWAF_UNKNOWN_LIMIT_PERCENT does not apply to it,
+	// so that not waiting never lowers its client's limits.
+	skippedLookupWait bool
+	// counted is true for a request the rate limits counted, whose bytes
+	// the byte limits count once it has ended. limitPercent and
+	// bytesPercent are then its client's limit percentages for the rate
+	// limits and for the byte limits.
+	counted                    bool
+	limitPercent, bytesPercent percentage
+	// attack is true for a request that matched a ban rule or asked for a
+	// trap path, ruleBlocked for one a block rule refused, and wafBlocked for
+	// one the Core Rule Set refused, each an offence its client's history
+	// counts.
+	attack, ruleBlocked, wafBlocked bool
+	// reputation is the reputation sources that list the client, for the
+	// notes of a ban the request makes.
+	reputation []bans.ReputationHit
+}
+
+// request is one request to servewaf, from the moment its headers have been
+// read to its log line.
+type request struct {
+	decision
+
+	h   *handler
+	in  *http.Request
+	out *responseWriter
+
+	// blocklisted is true once a blocklist is found to list the client,
+	// dnsblListed once a DNSBL zone's verdict is, and abuseIPDBHit once
+	// AbuseIPDB's score of it is a hit.
+	blocklisted, dnsblListed, abuseIPDBHit bool
+	start                                  time.Time
+	// checked is when the checks were done.
+	checked time.Time
+	// refused is what the request was refused with, nil if it was not.
+	refused *refusal
+	// requestBytes is how many bytes of the request's body have been read.
+	requestBytes int64
+}
+
+// startRequest starts handling r from client, which came through peer: it
+// notes the time, counts the request as under way, and starts the log line
+// with what is known of the request. isQuestion says whether r is a question,
+// whose headers give the request's id and scheme.
+func (h *handler) startRequest(
+	w http.ResponseWriter, r *http.Request, client, peer netip.Addr, isQuestion bool,
+) *request {
+	h.metrics.RequestStarted()
+
+	start := time.Now()
+	forwardedFor := r.Header.Values("X-Forwarded-For")
+
+	rq := &request{
+		decision: decision{
+			client:     client,
+			requestURI: r.URL.RequestURI(),
+			line: requestlog.Line{
+				Time:             requestlog.FormatTime(start),
+				Instance:         h.config.InstanceName,
+				ClientIP:         client.String(),
+				Method:           r.Method,
+				Scheme:           scheme(r, isQuestion),
+				Host:             r.Host,
+				Path:             rules.Path(r),
+				Query:            r.URL.RawQuery,
+				Protocol:         r.Proto,
+				Referer:          r.Referer(),
+				UserAgent:        r.UserAgent(),
+				RequestID:        requestID(r, isQuestion),
+				PeerIP:           peer.String(),
+				ForwardedFor:     strings.Join(forwardedFor, ", "),
+				ClientGroup:      h.clientGroup(client).String(),
+				ContentType:      r.Header.Get("Content-Type"),
+				RequestHeaders:   requestHeaders(r, h.config.LogRequestHeaders),
+				HasAuthorization: len(r.Header.Values("Authorization")) > 0,
+				HasCookie:        len(r.Header.Values("Cookie")) > 0,
+				Action:           requestlog.ActionForward,
+			},
+		},
+		h:     h,
+		in:    r,
+		out:   &responseWriter{ResponseWriter: w},
+		start: start,
+	}
+
+	// A length of -1 is a body whose length was not announced.
+	if r.ContentLength > 0 {
+		rq.line.ContentLength = r.ContentLength
+	}
+
+	return rq
+}
+
+// requestHeaders returns the headers of r that names lists, by name in
+// lower case, each with its values joined by ", ". Authorization, Cookie
+// and Set-Cookie are never among them, whatever names says.
+func requestHeaders(r *http.Request, names []string) map[string]string {
+	headers := map[string]string{}
+
+	for _, name := range names {
+		switch name {
+		case "authorization", "cookie", "set-cookie":
+			continue
+		}
+
+		values := r.Header.Values(name)
+		if len(values) > 0 {
+			headers[name] = strings.Join(values, ", ")
+		}
+	}
+
+	return headers
+}
+
+// check is the one place where a request can be refused. It returns nil to let
+// the request through. The checks of checkClient decide: a request they refuse
+// is answered with SWWAF_BAN_RESPONSE, 403 for a block rule or the Core Rule Set,
+// or 431 for the header limits. In observe mode nothing is refused, and the log
+// line names what enforce mode would have done. ctx is the request's own context.
+func (rq *request) check(ctx context.Context) *refusal {
+	action := rq.checkClient(ctx)
+
+	switch {
+	case action == "":
+		return nil
+	case rq.h.config.Observe:
+		rq.line.WouldAction = action
+
+		return nil
+	case action == requestlog.ActionRuleBlocked || action == requestlog.ActionWAFBlocked:
+		return &refusal{status: http.StatusForbidden, action: action}
+	case action == requestlog.ActionTooLarge:
+		return &refusal{status: http.StatusRequestHeaderFieldsTooLarge, action: action}
+	default:
+		return rq.banResponse(action)
+	}
+}
+
+// checkClient runs the checks on the request's client, and returns the
+// action of the first that refuses the request, or "" when none does. A
+// client in SWWAF_ALLOW_NETS skips them, and is not looked up. For any
+// other client, SWWAF_DENY_NETS comes first, then a ban on its netblock,
+// so that a client either refuses is not looked up, then the lookup of
+// its AS number and country, then the country lists, then the blocklists,
+// then the CrowdSec decision list, which bans the client it lists, then
+// the DNSBL zones' verdicts, and then AbuseIPDB's score; a request any of
+// them refuses is not counted for the rate limits. Then come the
+// rate limits, unless the client is in SWWAF_RATE_LIMIT_EXEMPT_NETS or the
+// request's path is exempt under SWWAF_RATE_LIMIT_EXEMPT_PATHS, so that
+// every other request is counted, each of them by the client's limit
+// percentages, then SWWAF_TRAP_PATHS, then the rule files, then the header
+// limits, and last the Core Rule Set. A request exempt from the rate limits
+// is exempt from the byte limits too. ctx is the request's own context.
+func (rq *request) checkClient(ctx context.Context) string {
+	cfg := rq.h.config
+	if isInside(rq.client, cfg.AllowNets) {
+		return ""
+	}
+
+	now := rq.h.now()
+
+	if isInside(rq.client, cfg.DenyNets) {
+		return requestlog.ActionDenied
+	}
+
+	if rq.banned(now) {
+		return requestlog.ActionBanned
+	}
+
+	rq.lookUp(ctx, now)
+
+	// The lookup can wait for GeoJS, so the checks after it read the time
+	// again: a request counted at the time it arrived could be counted
+	// after a later one of its client's, which the rate limits would take
+	// for the clock set back.
+	now = rq.h.now()
+
+	if rq.countryDenied() {
+		return requestlog.ActionCountryDenied
+	}
+
+	if rq.blocklistDenied() {
+		return requestlog.ActionDenied
+	}
+
+	if rq.crowdSecBanned(now) {
+		return requestlog.ActionBanned
+	}
+
+	if rq.dnsblDenied(ctx) || rq.abuseIPDBDenied(ctx) {
+		return requestlog.ActionDenied
+	}
+
+	rq.counted = rq.countedByRateLimits()
+	if rq.counted {
+		rq.limitPercent, rq.bytesPercent = rq.limitPercentages()
+		rq.line.LimitPercent, rq.line.LimitPercentSetting = rq.limitPercent.logged()
+		rq.line.BytesPercent, rq.line.BytesPercentSetting = rq.bytesPercent.logged()
+	}
+
+	if rq.counted && rq.limitBroken(now) {
+		return requestlog.ActionRateLimited
+	}
+
+	if rq.trapPath(now) {
+		return requestlog.ActionBanned
+	}
+
+	action := rq.checkRules(now)
+	if action != "" {
+		return action
+	}
+
+	if rq.overHeaderLimits() {
+		return requestlog.ActionTooLarge
+	}
+
+	return rq.checkCoreRuleSet()
+}
+
+// countedByRateLimits reports whether the rate limits count the request: its
+// client is not in SWWAF_RATE_LIMIT_EXEMPT_NETS, and its path is not exempt
+// under SWWAF_RATE_LIMIT_EXEMPT_PATHS.
+func (rq *request) countedByRateLimits() bool {
+	cfg := rq.h.config
+
+	return !isInside(rq.client, cfg.RateLimitExemptNets) &&
+		!pathExempt(rq.in, cfg.RateLimitExemptPaths)
+}
+
+// pathExempt reports whether r is exempt under prefixes,
+// SWWAF_RATE_LIMIT_EXEMPT_PATHS or SWWAF_WAF_EXEMPT_PATHS: whether its
+// path as sent, the path the app receives, not percent-decoded, starts
+// with one of prefixes, so that /%61ssets/x is not under /assets/ for an
+// app whose router matches the path as received. A request whose decoded
+// path contains .. anywhere or a backslash, or whose path as sent holds an
+// encoded slash (%2F or %2f), never is, since an app may act on it as a
+// path outside every prefix: /assets/..%2Flogin as /login, or /assets%2Fx
+// as one path segment, as Go's router does.
+func pathExempt(r *http.Request, prefixes []string) bool {
+	decoded := r.URL.Path
+	sent := rules.Path(r)
+
+	if strings.Contains(decoded, "..") || strings.Contains(decoded, `\`) ||
+		strings.Contains(strings.ToLower(sent), "%2f") {
+		return false
+	}
+
+	return slices.ContainsFunc(prefixes, func(prefix string) bool {
+		return strings.HasPrefix(sent, prefix)
+	})
+}
+
+// finish counts the request in the metrics and writes its log line.
+func (rq *request) finish() {
+	line := &rq.line
+	line.Status = rq.out.status
+	line.ResponseBytes = rq.out.bytes
+	line.ResponseContentType = rq.out.Header().Get("Content-Type")
+	line.RequestBytes = rq.requestBytes
+
+	switch {
+	case rq.refused != nil:
+		line.Action = rq.refused.action
+	case rq.out.err != nil || rq.in.Context().Err() != nil:
+		line.Aborted = true
+	}
+
+	duration := time.Since(rq.start)
+	line.DurationTotal = requestlog.Milliseconds(duration)
+	line.DurationChecks = timing(rq.start, rq.checked)
+
+	// Counted before the log line is written, so that the metrics count
+	// every request whose line is out. An allowed request is the exception:
+	// its report gives the status, bytes and times the metrics count.
+	if rq.allowed {
+		rq.h.metrics.RequestAllowed()
+	} else {
+		rq.h.metrics.RequestEnded(line, duration)
+	}
+
+	rq.writeLine()
+}
+
+// writeLine writes the request's log line.
+func (rq *request) writeLine() {
+	err := requestlog.Write(rq.h.requestLog, &rq.line)
+	if err != nil {
+		rq.h.processLog.Error("writing the request log failed", "error", err.Error())
+	}
+}
+
+// timing is the time from start to end in milliseconds, for one of the
+// log line's timings, or nil when end is zero: what it times never
+// happened.
+func timing(start, end time.Time) *float64 {
+	if end.IsZero() {
+		return nil
+	}
+
+	return new(requestlog.Milliseconds(end.Sub(start)))
+}
+
+// addToHistory adds the request, which has ended, to its client's
+// history, and counts its offences in the metrics, and then the lookup's
+// answer about the client, as answerAtTheEnd gives it, to that history and
+// to the notes of the bans on its netblock: an answer may have come
+// before either was there, and one from GeoJS that comes later is added
+// when it comes.
+func (rq *request) addToHistory() {
+	request := ratelimit.Request{
+		Forwarded:     rq.allowed,
+		Refused:       rq.refused != nil,
+		Status:        rq.out.status,
+		RequestBytes:  rq.requestBytes,
+		ResponseBytes: rq.out.bytes,
+		BrokeLimit:    rq.line.Offence == requestlog.OffenceLimit,
+		Attack:        rq.attack,
+		RuleBlocked:   rq.ruleBlocked,
+		WAFBlocked:    rq.wafBlocked,
+	}
+
+	rq.h.limiter.AddToHistory(rq.h.clientGroup(rq.client), rq.h.now(), request)
+	rq.h.metrics.Offences(request)
+
+	answer, found := rq.answerAtTheEnd()
+	if found {
+		rq.h.addLookup(answer)
+	}
+}
+
+// countAnomalies counts the request, which has ended, and its bytes, as
+// countedBytes gives them, for the anomaly thresholds, whatever was done
+// with it: a request refused, one from a client in SWWAF_ALLOW_NETS or
+// SWWAF_RATE_LIMIT_EXEMPT_NETS, and one for a path in
+// SWWAF_RATE_LIMIT_EXEMPT_PATHS are counted too. It is counted for its
+// client's AS number when answerAtTheEnd gives one. With every anomaly
+// threshold off, the default, it does nothing.
+func (rq *request) countAnomalies() {
+	if !anomalyThresholdsSet(rq.h.config) {
+		return
+	}
+
+	answer, _ := rq.answerAtTheEnd()
+
+	rq.h.anomalies.Count(rq.h.now(), anomaly.Request{
+		Host:        rq.site,
+		Client:      rq.client,
+		ClientGroup: rq.h.clientGroup(rq.client),
+		ASN:         answer.ASN,
+		ASName:      answer.ASName,
+		Country:     answer.Country,
+		Bytes:       rq.countedBytes(),
+	})
+}
+
+// anomalyThresholdsSet reports whether any anomaly threshold is set.
+func anomalyThresholdsSet(cfg *config.Config) bool {
+	off := anomaly.Thresholds{}
+
+	return cfg.AnomalyClient != off || cfg.AnomalyNet != off || cfg.AnomalyASN != off ||
+		cfg.AnomalyTotal != off || cfg.AnomalyWatch != off
+}
+
+// answerAtTheEnd returns, for a client that was looked up, the lookup's
+// answer about it as the request ends, and whether there is one: the
+// lookup database's, which was there at once, or the one GeoJS has given
+// by then, which a request does not wait for unless a setting needs it.
+func (rq *request) answerAtTheEnd() (lookup.Answer, bool) {
+	if !rq.lookedUp {
+		return lookup.Answer{}, false
+	}
+
+	if rq.h.config.LookupSource == "file" {
+		return rq.lookupAnswer, true
+	}
+
+	return rq.h.geojs.Kept(rq.h.clientGroup(rq.client))
+}
+
+// responseWriter is servewaf's answer to a request. It notes the status and
+// size for the log line, and the first error writing the answer.
+type responseWriter struct {
+	http.ResponseWriter
+
+	// status is the status sent, or zero before one is.
+	status int
+	bytes  int64
+	err    error
+}
+
+// WriteHeader sends the status and headers.
+func (w *responseWriter) WriteHeader(status int) {
+	if w.status == 0 {
+		w.status = status
+	}
+
+	w.ResponseWriter.WriteHeader(status)
+}
+
+// Write sends part of the body.
+func (w *responseWriter) Write(p []byte) (int, error) {
+	if w.status == 0 {
+		w.status = http.StatusOK
+	}
+
+	n, err := w.ResponseWriter.Write(p)
+	w.bytes += int64(n)
+
+	if w.err == nil {
+		w.err = err
+	}
+
+	return n, err
+}