package proxy import ( "context" "net/http" "net/netip" "slices" "strings" "time" "sneak.berlin/go/smallwebwaf/internal/anomaly" "sneak.berlin/go/smallwebwaf/internal/bans" "sneak.berlin/go/smallwebwaf/internal/config" "sneak.berlin/go/smallwebwaf/internal/lookup" "sneak.berlin/go/smallwebwaf/internal/ratelimit" "sneak.berlin/go/smallwebwaf/internal/requestlog" "sneak.berlin/go/smallwebwaf/internal/rules" ) // refusal is servewaf refusing a request: the status the visitor is to get, 0 // to close its connection without an answer, and the action the log line names. type refusal struct { status int action string } // decision is what the checks note of a request: its log line, and what // counting the request needs once it has ended. servewaf keeps a copy of it // until the request's report comes. It holds no credentials, since the log line // never does, and neither the request's objects nor their bodies. type decision struct { line requestlog.Line client netip.Addr // requestURI is the path with the query, as a ban's notes keep it. requestURI string // site is, for a question, the host the visitor asked for, which the // alerts name, since one servewaf serves many sites. site string // allowed is true for a question answered with DispositionAllow. allowed bool // lookedUp is true once the client's AS number and country have been // looked up, whether or not an answer was there, and lookupAnswer is // what the lookup gave then, the zero Answer while GeoJS had given none. lookedUp bool lookupAnswer lookup.Answer // skippedLookupWait is true for a request that did not wait for GeoJS and // found no answer kept. SWWAF_UNKNOWN_LIMIT_PERCENT does not apply to it, // so that not waiting never lowers its client's limits. skippedLookupWait bool // counted is true for a request the rate limits counted, whose bytes // the byte limits count once it has ended. limitPercent and // bytesPercent are then its client's limit percentages for the rate // limits and for the byte limits. counted bool limitPercent, bytesPercent percentage // attack is true for a request that matched a ban rule or asked for a // trap path, ruleBlocked for one a block rule refused, and wafBlocked for // one the Core Rule Set refused, each an offence its client's history // counts. attack, ruleBlocked, wafBlocked bool // reputation is the reputation sources that list the client, for the // notes of a ban the request makes. reputation []bans.ReputationHit } // request is one request to servewaf, from the moment its headers have been // read to its log line. type request struct { decision h *handler in *http.Request out *responseWriter // blocklisted is true once a blocklist is found to list the client, // dnsblListed once a DNSBL zone's verdict is, and abuseIPDBHit once // AbuseIPDB's score of it is a hit. blocklisted, dnsblListed, abuseIPDBHit bool start time.Time // checked is when the checks were done. checked time.Time // refused is what the request was refused with, nil if it was not. refused *refusal // requestBytes is how many bytes of the request's body have been read. requestBytes int64 } // startRequest starts handling r from client, which came through peer: it // notes the time, counts the request as under way, and starts the log line // with what is known of the request. isQuestion says whether r is a question, // whose headers give the request's id and scheme. func (h *handler) startRequest( w http.ResponseWriter, r *http.Request, client, peer netip.Addr, isQuestion bool, ) *request { h.metrics.RequestStarted() start := time.Now() forwardedFor := r.Header.Values("X-Forwarded-For") rq := &request{ decision: decision{ client: client, requestURI: r.URL.RequestURI(), line: requestlog.Line{ Time: requestlog.FormatTime(start), Instance: h.config.InstanceName, ClientIP: client.String(), Method: r.Method, Scheme: scheme(r, isQuestion), Host: r.Host, Path: rules.Path(r), Query: r.URL.RawQuery, Protocol: r.Proto, Referer: r.Referer(), UserAgent: r.UserAgent(), RequestID: requestID(r, isQuestion), PeerIP: peer.String(), ForwardedFor: strings.Join(forwardedFor, ", "), ClientGroup: h.clientGroup(client).String(), ContentType: r.Header.Get("Content-Type"), RequestHeaders: requestHeaders(r, h.config.LogRequestHeaders), HasAuthorization: len(r.Header.Values("Authorization")) > 0, HasCookie: len(r.Header.Values("Cookie")) > 0, Action: requestlog.ActionForward, }, }, h: h, in: r, out: &responseWriter{ResponseWriter: w}, start: start, } // A length of -1 is a body whose length was not announced. if r.ContentLength > 0 { rq.line.ContentLength = r.ContentLength } return rq } // requestHeaders returns the headers of r that names lists, by name in // lower case, each with its values joined by ", ". Authorization, Cookie // and Set-Cookie are never among them, whatever names says. func requestHeaders(r *http.Request, names []string) map[string]string { headers := map[string]string{} for _, name := range names { switch name { case "authorization", "cookie", "set-cookie": continue } values := r.Header.Values(name) if len(values) > 0 { headers[name] = strings.Join(values, ", ") } } return headers } // check is the one place where a request can be refused. It returns nil to let // the request through. The checks of checkClient decide: a request they refuse // is answered with SWWAF_BAN_RESPONSE, 403 for a block rule or the Core Rule Set, // or 431 for the header limits. In observe mode nothing is refused, and the log // line names what enforce mode would have done. ctx is the request's own context. func (rq *request) check(ctx context.Context) *refusal { action := rq.checkClient(ctx) switch { case action == "": return nil case rq.h.config.Observe: rq.line.WouldAction = action return nil case action == requestlog.ActionRuleBlocked || action == requestlog.ActionWAFBlocked: return &refusal{status: http.StatusForbidden, action: action} case action == requestlog.ActionTooLarge: return &refusal{status: http.StatusRequestHeaderFieldsTooLarge, action: action} default: return rq.banResponse(action) } } // checkClient runs the checks on the request's client, and returns the // action of the first that refuses the request, or "" when none does. A // client in SWWAF_ALLOW_NETS skips them, and is not looked up. For any // other client, SWWAF_DENY_NETS comes first, then a ban on its netblock, // so that a client either refuses is not looked up, then the lookup of // its AS number and country, then the country lists, then the blocklists, // then the CrowdSec decision list, which bans the client it lists, then // the DNSBL zones' verdicts, and then AbuseIPDB's score; a request any of // them refuses is not counted for the rate limits. Then come the // rate limits, unless the client is in SWWAF_RATE_LIMIT_EXEMPT_NETS or the // request's path is exempt under SWWAF_RATE_LIMIT_EXEMPT_PATHS, so that // every other request is counted, each of them by the client's limit // percentages, then SWWAF_TRAP_PATHS, then the rule files, then the header // limits, and last the Core Rule Set. A request exempt from the rate limits // is exempt from the byte limits too. ctx is the request's own context. func (rq *request) checkClient(ctx context.Context) string { cfg := rq.h.config if isInside(rq.client, cfg.AllowNets) { return "" } now := rq.h.now() if isInside(rq.client, cfg.DenyNets) { return requestlog.ActionDenied } if rq.banned(now) { return requestlog.ActionBanned } rq.lookUp(ctx, now) // The lookup can wait for GeoJS, so the checks after it read the time // again: a request counted at the time it arrived could be counted // after a later one of its client's, which the rate limits would take // for the clock set back. now = rq.h.now() if rq.countryDenied() { return requestlog.ActionCountryDenied } if rq.blocklistDenied() { return requestlog.ActionDenied } if rq.crowdSecBanned(now) { return requestlog.ActionBanned } if rq.dnsblDenied(ctx) || rq.abuseIPDBDenied(ctx) { return requestlog.ActionDenied } rq.counted = rq.countedByRateLimits() if rq.counted { rq.limitPercent, rq.bytesPercent = rq.limitPercentages() rq.line.LimitPercent, rq.line.LimitPercentSetting = rq.limitPercent.logged() rq.line.BytesPercent, rq.line.BytesPercentSetting = rq.bytesPercent.logged() } if rq.counted && rq.limitBroken(now) { return requestlog.ActionRateLimited } if rq.trapPath(now) { return requestlog.ActionBanned } action := rq.checkRules(now) if action != "" { return action } if rq.overHeaderLimits() { return requestlog.ActionTooLarge } return rq.checkCoreRuleSet() } // countedByRateLimits reports whether the rate limits count the request: its // client is not in SWWAF_RATE_LIMIT_EXEMPT_NETS, and its path is not exempt // under SWWAF_RATE_LIMIT_EXEMPT_PATHS. func (rq *request) countedByRateLimits() bool { cfg := rq.h.config return !isInside(rq.client, cfg.RateLimitExemptNets) && !pathExempt(rq.in, cfg.RateLimitExemptPaths) } // pathExempt reports whether r is exempt under prefixes, // SWWAF_RATE_LIMIT_EXEMPT_PATHS or SWWAF_WAF_EXEMPT_PATHS: whether its // path as sent, the path the app receives, not percent-decoded, starts // with one of prefixes, so that /%61ssets/x is not under /assets/ for an // app whose router matches the path as received. A request whose decoded // path contains .. anywhere or a backslash, or whose path as sent holds an // encoded slash (%2F or %2f), never is, since an app may act on it as a // path outside every prefix: /assets/..%2Flogin as /login, or /assets%2Fx // as one path segment, as Go's router does. func pathExempt(r *http.Request, prefixes []string) bool { decoded := r.URL.Path sent := rules.Path(r) if strings.Contains(decoded, "..") || strings.Contains(decoded, `\`) || strings.Contains(strings.ToLower(sent), "%2f") { return false } return slices.ContainsFunc(prefixes, func(prefix string) bool { return strings.HasPrefix(sent, prefix) }) } // finish counts the request in the metrics and writes its log line. func (rq *request) finish() { line := &rq.line line.Status = rq.out.status line.ResponseBytes = rq.out.bytes line.ResponseContentType = rq.out.Header().Get("Content-Type") line.RequestBytes = rq.requestBytes switch { case rq.refused != nil: line.Action = rq.refused.action case rq.out.err != nil || rq.in.Context().Err() != nil: line.Aborted = true } duration := time.Since(rq.start) line.DurationTotal = requestlog.Milliseconds(duration) line.DurationChecks = timing(rq.start, rq.checked) // Counted before the log line is written, so that the metrics count // every request whose line is out. An allowed request is the exception: // its report gives the status, bytes and times the metrics count. if rq.allowed { rq.h.metrics.RequestAllowed() } else { rq.h.metrics.RequestEnded(line, duration) } rq.writeLine() } // writeLine writes the request's log line. func (rq *request) writeLine() { err := requestlog.Write(rq.h.requestLog, &rq.line) if err != nil { rq.h.processLog.Error("writing the request log failed", "error", err.Error()) } } // timing is the time from start to end in milliseconds, for one of the // log line's timings, or nil when end is zero: what it times never // happened. func timing(start, end time.Time) *float64 { if end.IsZero() { return nil } return new(requestlog.Milliseconds(end.Sub(start))) } // addToHistory adds the request, which has ended, to its client's // history, and counts its offences in the metrics, and then the lookup's // answer about the client, as answerAtTheEnd gives it, to that history and // to the notes of the bans on its netblock: an answer may have come // before either was there, and one from GeoJS that comes later is added // when it comes. func (rq *request) addToHistory() { request := ratelimit.Request{ Forwarded: rq.allowed, Refused: rq.refused != nil, Status: rq.out.status, RequestBytes: rq.requestBytes, ResponseBytes: rq.out.bytes, BrokeLimit: rq.line.Offence == requestlog.OffenceLimit, Attack: rq.attack, RuleBlocked: rq.ruleBlocked, WAFBlocked: rq.wafBlocked, } rq.h.limiter.AddToHistory(rq.h.clientGroup(rq.client), rq.h.now(), request) rq.h.metrics.Offences(request) answer, found := rq.answerAtTheEnd() if found { rq.h.addLookup(answer) } } // countAnomalies counts the request, which has ended, and its bytes, as // countedBytes gives them, for the anomaly thresholds, whatever was done // with it: a request refused, one from a client in SWWAF_ALLOW_NETS or // SWWAF_RATE_LIMIT_EXEMPT_NETS, and one for a path in // SWWAF_RATE_LIMIT_EXEMPT_PATHS are counted too. It is counted for its // client's AS number when answerAtTheEnd gives one. With every anomaly // threshold off, the default, it does nothing. func (rq *request) countAnomalies() { if !anomalyThresholdsSet(rq.h.config) { return } answer, _ := rq.answerAtTheEnd() rq.h.anomalies.Count(rq.h.now(), anomaly.Request{ Host: rq.site, Client: rq.client, ClientGroup: rq.h.clientGroup(rq.client), ASN: answer.ASN, ASName: answer.ASName, Country: answer.Country, Bytes: rq.countedBytes(), }) } // anomalyThresholdsSet reports whether any anomaly threshold is set. func anomalyThresholdsSet(cfg *config.Config) bool { off := anomaly.Thresholds{} return cfg.AnomalyClient != off || cfg.AnomalyNet != off || cfg.AnomalyASN != off || cfg.AnomalyTotal != off || cfg.AnomalyWatch != off } // answerAtTheEnd returns, for a client that was looked up, the lookup's // answer about it as the request ends, and whether there is one: the // lookup database's, which was there at once, or the one GeoJS has given // by then, which a request does not wait for unless a setting needs it. func (rq *request) answerAtTheEnd() (lookup.Answer, bool) { if !rq.lookedUp { return lookup.Answer{}, false } if rq.h.config.LookupSource == "file" { return rq.lookupAnswer, true } return rq.h.geojs.Kept(rq.h.clientGroup(rq.client)) } // responseWriter is servewaf's answer to a request. It notes the status and // size for the log line, and the first error writing the answer. type responseWriter struct { http.ResponseWriter // status is the status sent, or zero before one is. status int bytes int64 err error } // WriteHeader sends the status and headers. func (w *responseWriter) WriteHeader(status int) { if w.status == 0 { w.status = status } w.ResponseWriter.WriteHeader(status) } // Write sends part of the body. func (w *responseWriter) Write(p []byte) (int, error) { if w.status == 0 { w.status = http.StatusOK } n, err := w.ResponseWriter.Write(p) w.bytes += int64(n) if w.err == nil { w.err = err } return n, err }